Author Topic: console streams  (Read 1090 times)

bomz

  • Member
  • **
  • Posts: 115
console streams
« on: June 30, 2021, 11:05:20 PM »
Hi for all!!!!
Can anybody help me. I try to make console stream application,
but something is wrong. Can't add posibility to work with streams more than 64 kbyte,
and sometimes incorrect. sometimes mistakes occur. attemp to write in un-existing stream
Code: [Select]
.386
.model flat,stdcall
option casemap:none

include \masm32\include\windows.inc
include \masm32\include\kernel32.inc
include \MASM32\INCLUDE\user32.inc
includelib \masm32\lib\kernel32.lib
includelib \MASM32\LIB\user32.lib

.data
;_cb db 10, 13
CONIN db "CONIN$",0
CONOUT db "CONOUT$",0
.data?
Buffer db 10000h dup (?)
Buffer1 db 10000h dup (?)
Buffer2 db 10000h dup (?)
_rb dd ?
_wb dd ?
_bs dd ?
.code
start:

invoke GetStdHandle,STD_INPUT_HANDLE
;invoke CreateFile,addr CONIN,GENERIC_READ,FILE_SHARE_READ OR FILE_SHARE_WRITE,0,OPEN_EXISTING,0,0
mov esi,eax

invoke GetStdHandle,STD_OUTPUT_HANDLE
;invoke CreateFile,addr CONOUT,GENERIC_WRITE,FILE_SHARE_WRITE OR FILE_SHARE_READ,0,OPEN_EXISTING,0,0
mov edi,eax

;invoke SetStdHandle,esi,edi
invoke SetStdHandle,STD_INPUT_HANDLE,edi

;invoke ReadConsole,esi,addr Buffer,10000h,addr _rb,0
invoke ReadFile,esi,addr Buffer,10000h,addr _rb, 0

invoke MultiByteToWideChar,CP_UTF8,0,addr Buffer,_rb,0,0
mov _bs, eax
invoke MultiByteToWideChar,CP_UTF8,0,addr Buffer,_rb,addr Buffer1,sizeof Buffer1
invoke WideCharToMultiByte,CP_OEMCP,0,addr Buffer1,_bs,addr Buffer2,sizeof Buffer2,0,0
mov _bs, eax

invoke GetStdHandle, STD_OUTPUT_HANDLE
;invoke WriteConsole,eax,addr Buffer2,_rb, addr _wb,NULL
invoke WriteFile,eax,addr Buffer2,_bs,addr _wb,NULL

invoke ExitProcess, _wb

end start
the most surprising non-reproducibility of results
« Last Edit: July 01, 2021, 03:53:29 AM by hutch-- »

hutch--

  • Administrator
  • Member
  • ******
  • Posts: 8497
  • Mnemonic Driven API Grinder
    • The MASM32 SDK
Re: console streams
« Reply #1 on: July 01, 2021, 03:55:58 AM »
I removed the flashing image as it is a distraction to reading the page.

To address your question, you need to stream in smaller amounts as from memory, it is a buffer limitation in the OS. Just break up your data so that each one is small enough and run one stream after another.
hutch at movsd dot com
http://www.masm32.com    :biggrin:  :skrewy:

bomz

  • Member
  • **
  • Posts: 115
Re: console streams
« Reply #2 on: July 01, 2021, 09:41:09 AM »
I removed the flashing image as it is a distraction to reading the page.

To address your question, you need to stream in smaller amounts as from memory, it is a buffer limitation in the OS. Just break up your data so that each one is small enough and run one stream after another.
Hi Hutch! I try to do so. change buffer. make many different experiments, but don't catch problem
I can't find any working examples. May say now that input need only GetStdHandle,
and read only using ReadFile
cmd working with buffer overflow streams
for /f "delims=*" %%a in ('type 100mb.txt') do

now I think to create PIPE and add to it stdout and stdin, PIPE have property to control buffer fill
« Last Edit: July 01, 2021, 04:33:14 PM by bomz »

Vortex

  • Member
  • *****
  • Posts: 2583
Re: console streams
« Reply #3 on: July 01, 2021, 07:12:28 PM »
Hi bomz,

You can try the stream definitions provided by the C run-time library :

Code: [Select]
.386
.model flat, stdcall
option casemap:none

include     \masm32\include\windows.inc
include     \masm32\include\kernel32.inc
include     \masm32\include\msvcrt.inc
includelib  \masm32\lib\kernel32.lib
includelib  \masm32\lib\msvcrt.lib

_iobuf STRUCT

    _ptr        DWORD ?
    _cnt        DWORD ?
    _base       DWORD ?
    _flag       DWORD ?
    _file       DWORD ?
    _charbuf    DWORD ?
    _bufsiz     DWORD ?
    _tmpfname   DWORD ?

_iobuf ENDS

FILE TYPEDEF _iobuf

.data

msg     db 'Message output to STDOUT',0

.data?

stdout  dd ?
stdin   dd ?
stderr  dd ?

.code

start:

    call    crt___p__iob
    mov     stdin,eax           ; #define stdin  (&__iob_func()[0])

    mov     ecx,SIZEOF(FILE)

    add     eax,ecx
    mov     stdout,eax          ; #define stdout (&__iob_func()[1])

    add     eax,ecx
    mov     stderr,eax          ; #define stderr (&__iob_func()[2])

    invoke  crt_fputs,ADDR msg,stdout
           
    invoke  ExitProcess,0

END start

bomz

  • Member
  • **
  • Posts: 115
Re: console streams
« Reply #4 on: July 01, 2021, 10:30:23 PM »
use C library is too easy way for such little project
little but very usefull. such pipe between cmd and winapi
allow use find and findstr in batch files to work with utf-8 text.

Vortex

  • Member
  • *****
  • Posts: 2583
Re: console streams
« Reply #5 on: July 01, 2021, 10:50:28 PM »
Hi bomz,

You could use also the search functions provided by the static library masm32.lib.

Have a look at the BusyBox tool, you can use it to investigate your text files.

https://frippery.org/busybox/

bomz

  • Member
  • **
  • Posts: 115
Re: console streams
« Reply #6 on: July 01, 2021, 10:54:17 PM »
I investigates with utf-8 text, which changes very often, iptv playlists,
and batch files need every day correction. I decide my problem I can convert utf-8 value to 866 value,
to compare sub-strings through their 866 mirroring
but now I want make full correct application and know how it working in windows

Quote
   invoke CreatePipe,ReadPipeHandle,WritePipeHandle,0,0
   invoke GetStdHandle,STD_INPUT_HANDLE
   mov esi,eax
   invoke CreateFile,addr CONOUT,GENERIC_WRITE,FILE_SHARE_WRITE OR FILE_SHARE_READ,0,OPEN_EXISTING,0,0
   mov edi,eax
   invoke SetStdHandle,STD_INPUT_HANDLE,ReadPipeHandle
   invoke SetStdHandle,STD_OUTPUT_HANDLE,WritePipeHandle
false way. not working
« Last Edit: July 02, 2021, 12:34:49 AM by bomz »

mineiro

  • Member
  • ****
  • Posts: 725
Re: console streams
« Reply #7 on: July 02, 2021, 12:34:17 AM »
I try to make console stream application,
but something is wrong. Can't add posibility to work with streams more than 64 kbyte,

now I think to create PIPE and add to it stdout and stdin, PIPE have property to control buffer fill

Hello sir bomz, welcome back.
http://masm32.com/board/index.php?topic=5536.0
I'd rather be this ambulant metamorphosis than to have that old opinion about everything

bomz

  • Member
  • **
  • Posts: 115
Re: console streams
« Reply #8 on: July 02, 2021, 12:50:28 AM »
Hi!

the most strange sometimes read 80 symbols sometimes 4176. looks like buffer overflow
with one two little string it working

bomz

  • Member
  • **
  • Posts: 115
Re: console streams
« Reply #9 on: July 02, 2021, 06:03:16 PM »
now I think that problem have not decision, because TYPE may work only with internal CMD command and files.
all other command work correctly. or try to find PIPE handle inside CMD process, and use it instead of CONOUT$

make little application which send file to stream (or pipe) in English I think it call pipe
identically equal to command type
Code: [Select]
filetostream.exe utf-8.txt|utf8toutf8.exeand all begin working correctly

p.s. try to change 64 kb buffer string to 32 mb allocated hmemory,
application begin read in most cases 4127 symbols but sometimes 80
craziness
« Last Edit: July 02, 2021, 09:05:22 PM by bomz »

Vortex

  • Member
  • *****
  • Posts: 2583
Re: console streams
« Reply #10 on: July 02, 2021, 09:09:03 PM »
Hi mineiro,

Nice example :thumbsup:

bomz

  • Member
  • **
  • Posts: 115
Re: console streams
« Reply #11 on: July 02, 2021, 09:20:15 PM »

codes need corrections, but they work
command TYPE need investigation

mineiro

  • Member
  • ****
  • Posts: 725
Re: console streams
« Reply #12 on: July 02, 2021, 10:45:52 PM »
Thanks sir Vortex, credits are not mine, I translate that program from a tasm source found in internet.
That program suffer problems with programs that need hit a key to exit. So, sometimes it's necessary kill that process.

I suppose that with powershell things can get nice.
I'd rather be this ambulant metamorphosis than to have that old opinion about everything

bomz

  • Member
  • **
  • Posts: 115
Re: console streams
« Reply #13 on: July 03, 2021, 05:13:26 AM »
any hypothesis why
Code: [Select]
for f "delims=" %%a in ('type myfile.txt') do echo %%a
Code: [Select]
type myfile.txt>newfile.txt
Code: [Select]
type myfile.txt|find "word"working, and
Code: [Select]
type myfile.txt|application.execause buffer overflow?
find and findstr not inside CMD but separate files

little progress. it looks that CMD create PIPE for called application.
no need SetStdHandle, CMD do it
always read 4127 bytes or more such way
Code: [Select]
.386
.model flat,stdcall
option casemap:none

include \masm32\include\windows.inc
include \masm32\include\kernel32.inc
include \MASM32\include\user32.inc
includelib \masm32\lib\kernel32.lib
includelib \MASM32\lib\user32.lib

.data
;_cb db 10, 13
CONIN db "CONIN$",0
CONOUT db "CONOUT$",0
.data?
Buffer db 10000h dup (?)
hMemory HANDLE ?
_rb dd ?
_wb dd ?
.code
start:

invoke LocalAlloc,LMEM_FIXED,33554432 ; зарезервируем блок памяти 32 мбайта
mov hMemory,eax

;invoke CreateFile,addr CONIN,GENERIC_READ,FILE_SHARE_READ OR FILE_SHARE_WRITE,0,OPEN_EXISTING,0,0
invoke GetStdHandle,STD_INPUT_HANDLE
mov esi,eax
mov ebx,hMemory

;invoke SetNamedPipeHandleState,esi,PIPE_READMODE_BYTE OR PIPE_NOWAIT,0,0

next:
add ebx,_rb
invoke ReadFile,esi,ebx,33554432,addr _rb, 0
cmp _rb, 0
jne next
sub ebx,hMemory

;invoke CreateFile,addr CONOUT,GENERIC_WRITE,FILE_SHARE_WRITE OR FILE_SHARE_READ,0,OPEN_EXISTING,0,0
invoke GetStdHandle, STD_OUTPUT_HANDLE
invoke WriteConsole,eax,hMemory,ebx,addr _wb,NULL
;invoke WriteFile,eax,hMemory,ebx,addr _wb,NULL

invoke LocalFree,hMemory
invoke ExitProcess, _wb

end start
« Last Edit: July 03, 2021, 01:52:35 PM by bomz »

bomz

  • Member
  • **
  • Posts: 115
Re: console streams
« Reply #14 on: July 03, 2021, 02:04:57 PM »
it works!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

Code: [Select]
.386
.model flat,stdcall
option casemap:none

include \masm32\include\windows.inc
include \masm32\include\kernel32.inc
include \MASM32\include\user32.inc
includelib \masm32\lib\kernel32.lib
includelib \MASM32\lib\user32.lib

.data
;_cb db 10, 13
CONIN db "CONIN$",0
CONOUT db "CONOUT$",0
.data?
Buffer db 10000h dup (?)
hMemory HANDLE ?
_rb dd ?
_wb dd ?
.code
start:

invoke LocalAlloc,LMEM_FIXED,33554432 ; зарезервируем блок памяти 32 мбайта
mov hMemory,eax

;invoke CreateFile,addr CONIN,GENERIC_READ,FILE_SHARE_READ OR FILE_SHARE_WRITE,0,OPEN_EXISTING,0,0
invoke GetStdHandle,STD_INPUT_HANDLE
mov esi,eax
mov ebx,hMemory

invoke GetStdHandle, STD_OUTPUT_HANDLE
mov edi,eax


;invoke SetNamedPipeHandleState,esi,PIPE_READMODE_BYTE OR PIPE_NOWAIT,0,0

next:
add ebx,_rb
invoke ReadFile,esi,hMemory,33554432,addr _rb, 0
invoke WriteConsole,edi,hMemory,_rb,addr _wb,NULL
cmp _rb, 0
jne next
;sub ebx,hMemory

;invoke CreateFile,addr CONOUT,GENERIC_WRITE,FILE_SHARE_WRITE OR FILE_SHARE_READ,0,OPEN_EXISTING,0,0
;invoke GetStdHandle, STD_OUTPUT_HANDLE
;invoke WriteConsole,eax,hMemory,ebx,addr _wb,NULL
;invoke WriteFile,eax,hMemory,ebx,addr _wb,NULL

invoke LocalFree,hMemory
invoke ExitProcess, _wb

end start