This is one of the apps I needed the word tokeniser for. As a console app most of the code is checking the command line arguments, changing the subsystem is in fact very simple.
; ¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤
include \masm32\include64\masm64rt.inc
.code
; ¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤
entry_point proc
LOCAL pcmd :QWORD
LOCAL parr :QWORD ; array pointer
LOCAL arrm[256] :QWORD ; array memory
LOCAL acnt :QWORD
LOCAL pfile :QWORD
LOCAL pOpt :QWORD
LOCAL .r15 :QWORD
LOCAL ssys :QWORD
LOCAL peImage :QWORD
LOCAL flen :QWORD
LOCAL bwrt :QWORD
mov .r15, r15
mov pcmd, rv(cmd_tail) ; get the command tail
mov parr, ptr$(arrm) ; get array pointer
.if len(pcmd) == 0 ; test length for missing command line
conout "Missing command line",lf
rcall help
waitkey ; pause to show error
jmp bye
.endif
rcall wordtok,pcmd,parr,44 ; tokenise text using ascii 44 ( , ) as delimiter
mov acnt, rax ; get the arg count
cmp acnt, 2 ; exit on incorrect arg count
je @F
conout "Incorrect argument count",lf
rcall help
waitkey ; pause to show error
jmp bye
@@:
mov r15, parr
mrm pfile, QWORD PTR [r15] ; load file name address into pointer
rcall szTrim,pfile ; trim any junk
test rv(exist,pfile), rax ; test if the file exists
jnz @F
conout "Cannot find that file",lf
rcall help
waitkey ; pause to show error
jmp bye
@@:
mrm pOpt, QWORD PTR [r15+8] ; load the subsys option text
rcall szUpper,pOpt ; set as upper case
rcall szTrim,pOpt ; trim any junk
test rv(szCmp,pOpt,"CONSOLE"), rax
jz @F
mov ssys, 3
jmp setimage
@@:
test rv(szCmp,pOpt,"WINDOWS"), rax
jz invalid
mov ssys, 2
jmp setimage
invalid:
conout "Invalid subsystem option",lf
rcall help
waitkey ; pause to show error
jmp bye
setimage:
mov peImage, loadfile(pfile) ; read file from disk
mov flen, rcx
rcall mod_pe,peImage,ssys ; set the sub system
mov bwrt, savefile(pfile,peImage,flen) ; write file back to disk
mfree peImage ; release the file memory
.if ssys == 2
conout pfile," set to subsys WINDOWS",lf
.else
conout pfile," set to subsys CONSOLE",lf
.endif
bye:
mov r15, .r15
.exit
entry_point endp
; ¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤
mod_pe proc pe_image:QWORD,subsys:QWORD
mov r11, pe_image ; load PE image into r11
mov edx, (IMAGE_DOS_HEADER PTR [r11]).e_lfanew ; get the PE header offset
add r11, rdx ; add it to r11
; ------------------------------------------
; change the optional header Subsystem value
; ------------------------------------------
.if subsys == 2
mov (IMAGE_NT_HEADERS PTR [r11]).OptionalHeader.Subsystem, 2 ; WINDOWS
.else
mov (IMAGE_NT_HEADERS PTR [r11]).OptionalHeader.Subsystem, 3 ; CONSOLE
.endif
ret
mod_pe endp
; ¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤
help proc
.data
hlpmsg db 13,10,"subsys.exe - set PE subsystem",13,10
db " OPTIONS : 1. executable file name, file must exist",13,10
db " 2. subsystem - either WINDOWS or CONSOLE",13,10,13,10
db " EXAMPLE : subsys yourfile.exe,CONSOLE",13,10,13,10
db " NOTE : the comma delimiter is required between",13,10
db " the file name and the subsystem option",13,10,0
phelp dq hlpmsg
.code
conout phelp,lf
ret
help endp
; ¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤
end