Install hook problem

[vogelsang]

Hi assembly coders,

I'm trying convert Iczelion's sample to x64. What it does? It installs hook on mouse event. App should display in dialog editboxes infos about window where mouse cursor pointing. It seems to work but when i compare result of working 32-bit version to my 64-bit it looks something is wrong.

Main app uses DLL which contains hook procedures. DLL is linked with shared .bss section(link.exe /SECTION:.bss, S ...) maybe there is something messed.
I can't figure out what is wrong. Code is quite big. Could someone help?

main app file:

Code Select Expand


;--------COMMENT---------------------------------------------------------------------
;
; This code is based on Iczelion's example form chapter 24.
;
;--------MAKE------------------------------------------------------------------------
;
; resource compiling:
; rc MouseHook.rc
; assembling:
; jwasm -win64 -Zp8 MouseHook.asm
; linking:
; link /SUBSYSTEM:WINDOWS MouseHook.obj
;
;--------INCLUDES--------------------------------------------------------------------

option casemap:none

;include MouseHook.inc

includelib HookDLL.lib
includelib      /JWASM/wininc208/lib64/kernel32.lib
includelib      /JWASM/wininc208/lib64/user32.lib


EXTERN InstallHook :PROC ;externals from HookDLL
EXTERN UninstallHook :PROC
EXTERN MouseProc :PROC

EXTERN GetModuleHandleA :PROC ;externals from Windows dlls
EXTERN DialogBoxParamA :PROC
EXTERN ExitProcess :PROC
EXTERN EndDialog :PROC
EXTERN GetWindowRect :PROC
EXTERN SetWindowPos :PROC
EXTERN wsprintfA :PROC
EXTERN lstrcmpi :PROC
EXTERN SetDlgItemTextA :PROC
EXTERN GetDlgItemTextA :PROC
EXTERN GetClassNameA :PROC
EXTERN GetClassLongPtrA :PROC
EXTERN SendMessageA :PROC

RECT STRUCT

left DD ?
top DD ?
right DD ?
bottom DD ?

RECT ENDS

;--------DATA------------------------------------------------------------------------

.CONST
IDD_MAINDLG EQU 101
IDC_CLASSNAME EQU 1000
IDC_HANDLE EQU 1001
IDC_WNDPROC EQU 1002
IDC_HOOK EQU 1004
IDC_EXIT EQU 1005
WM_MOUSEHOOK EQU WM_USER+6

SWP_SHOWWINDOW EQU 40h
TRUE EQU 1
FALSE EQU 0
WH_MOUSE EQU 7
WM_USER EQU 400h

WM_INITDIALOG EQU 110h
WM_COMMAND EQU 111h
WM_CLOSE EQU 10h
HWND_TOPMOST EQU -1
GCL_WNDPROC EQU -24
BN_CLICKED EQU 0

.DATA
HookFlag DD FALSE
HookText DB "&Hook", 0
UnhookText DB "&Unhook", 0
template DB "%lx", 0

.DATA?
hInstance DQ ?
hHook DQ ?

;--------CODE------------------------------------------------------------------------

.CODE
WinMainCRTStartup PROC FRAME

push rbp
.PUSHREG RBP
mov rbp, rsp
.SETFRAME RBP, 0
.ENDPROLOG

sub rsp, 8*6

xor ecx, ecx ;RCX = NULL
call GetModuleHandleA
mov hInstance, rax

xor r8d, r8d
mov qword ptr [rsp+8*4], r8
lea r9, DlgFunc
mov rdx, IDD_MAINDLG
mov rcx, hInstance
call DialogBoxParamA

mov rcx, rax
call ExitProcess
ALIGN 4

WinMainCRTStartup ENDP

;--------DLGFUNC---------------------------------------------------------------------

DlgFunc PROC FRAME hDlg:QWORD, uMsg:QWORD, wParam:QWORD, lParam:QWORD

push rbp
.PUSHREG RBP
mov rbp, rsp
.SETFRAME RBP, 0
sub rsp, 8 + 128 + 128 + SIZEOF RECT + 8*7
.ALLOCSTACK 8 + 128 + 128 + SIZEOF RECT + 8*7
.ENDPROLOG

hLib EQU <[rbp - 8]>
buffer EQU <[rbp - 8 - 128]> ;allocate space for two 128 bytes buffers
buffer1 EQU <[rbp - 8 - 128 - 128]>
rect EQU <[rbp - 8 - 128 - 128 + SIZEOF RECT].RECT>

mov hDlg, rcx
mov wParam, r8
dlg_msg_close:
cmp edx, WM_CLOSE
jne dlg_msg_initdialog

cmp HookFlag, TRUE ;before quit we must uninstall hook. so lets check is it installed
jne @F ;if not jump to EndDialog

call UninstallHook
@@:
xor edx, edx
mov rcx, hDlg
call EndDialog

jmp exit_DlgFunc_true
dlg_msg_initdialog:
cmp edx, WM_INITDIALOG
jne dlg_msg_mousehook

lea rdx, rect
mov rcx, hDlg
call GetWindowRect

xor r9d, r9d
mov qword ptr [rsp+8*6], SWP_SHOWWINDOW
mov r9d, rect.bottom
mov qword ptr [rsp+8*5], r9
mov r9d, rect.right
mov qword ptr [rsp+8*4], r9
mov r9d, rect.top
mov r8d, rect.left
mov r8d, r8d
mov rdx, HWND_TOPMOST
mov rcx, hDlg
call SetWindowPos

jmp exit_DlgFunc_true
dlg_msg_mousehook: ;whole dlg_msg_mousehook label is about displaying infos about wnd handle delivered with WM_MOUSEHOOK in wParam
cmp edx, WM_MOUSEHOOK
jne dlg_msg_command
;DISPLAY HANDLE VALUE IN THE DLGEDITBOX
mov r9, 128 ;first we're checking: does edit box already contains string we want to display?
lea r8, buffer1
mov rdx, IDC_HANDLE
mov rcx, hDlg
call GetDlgItemTextA ;so we must get its content

mov r8, wParam
lea rdx, template
lea rcx, buffer
call wsprintfA ;convert to ASCII hex handle from wParam and store in buffer

lea rdx, buffer1 ;to avoid text blinking compare handle from editbox to handle from wParam
lea rcx, buffer
call lstrcmpi

test rax, rax ;if they are equal - don't display it agian
jz @F

lea r8, buffer
mov rdx, IDC_HANDLE
mov rcx, hDlg
call SetDlgItemTextA ;otherwise display new handle value
@@: ;DISPLAY CLASS NAME IN THE DLGEDITBOX
mov r9, 128 ;similiar as above
lea r8, buffer1
mov rdx, IDC_CLASSNAME
mov rcx, hDlg
call GetDlgItemTextA

mov r8, 128
lea rdx, buffer
mov rcx, wParam
call GetClassNameA

lea rdx, buffer1
lea rcx, buffer
call lstrcmpi

test rax, rax
jz @F

lea r8, buffer
mov rdx, IDC_CLASSNAME
mov rcx, hDlg
call SetDlgItemTextA
@@: ;DISPLAY WNDPROC ADDRESS IN THE DLGEDITBOX
mov r9, 128 ;similiar as handle display
lea r8, buffer1
mov rdx, IDC_WNDPROC
mov rcx, hDlg
call GetDlgItemTextA

mov rdx, GCL_WNDPROC
mov rcx, wParam
call GetClassLongPtrA

mov r8, rax
lea rdx, template
lea rcx, buffer
call wsprintfA

lea rdx, buffer1
lea rcx, buffer
call lstrcmpi

test rax, rax
jz exit_DlgFunc_true

lea r8, buffer
mov rdx, IDC_WNDPROC
mov rcx, hDlg
call SetDlgItemTextA

jmp exit_DlgFunc_true
dlg_msg_command:
cmp edx, WM_COMMAND
jne exit_DlgFunc_false

test r9, r9
jz exit_DlgFunc_true

mov rax, r8
shr rax, 16

cmp ax, BN_CLICKED
jne exit_DlgFunc_true
idc_exit:
cmp r8w, IDC_EXIT
jne idc_hook

xor r9d, r9d
xor r8d, r8d
mov rdx, WM_CLOSE
mov rcx, hDlg
call SendMessageA

jmp exit_DlgFunc_true
idc_hook:
cmp HookFlag, FALSE ;if hook is not installed execute code below
jne @F ;otherwise jump to next label

mov rcx, hDlg
call InstallHook ;to install hook call HookDLL proc

test rax, rax
jz exit_DlgFunc_true

mov HookFlag, TRUE ;mark that hook is installed
lea r8, UnhookText
mov rdx, IDC_HOOK
mov rcx, hDlg
call SetDlgItemTextA ;change hook button text to "Unhook"

jmp exit_DlgFunc_true
@@:
call UninstallHook ;to uninstall hook call HookDLL proc

lea r8, HookText
mov rdx, IDC_HOOK
mov rcx, hDlg
call SetDlgItemTextA ;change hook button text to "Hook"
mov HookFlag, FALSE ;and mark that hook is uninstalled

xor r8d, r8d ;clear editboxes
mov rdx, IDC_CLASSNAME
mov rcx, hDlg
call SetDlgItemTextA

xor r8d, r8d
mov rdx, IDC_HANDLE
mov rcx, hDlg
call SetDlgItemTextA

xor r8d, r8d
mov rdx, IDC_WNDPROC
mov rcx, hDlg
call SetDlgItemTextA

jmp exit_DlgFunc_true
exit_DlgFunc_false:
xor eax, eax ;return FALSE
add rsp, 8 + 128 + 128 + SIZEOF RECT + 8*7
pop rbp
ret
exit_DlgFunc_true:
mov rax, TRUE ;return TRUE
add rsp, 8 + 128 + 128 + SIZEOF RECT + 8*7
pop rbp
ret

DlgFunc ENDP

END
;--------EOF--------------------------------------------------------------------------


rc for main app:

Code Select Expand


#define IDD_MAINDLG           101
#define DS_MODALFRAME        0x80
#define ES_AUTOHSCROLL       0x80
#define IDC_CLASSNAME        1000
#define IDC_HANDLE           1001
#define IDC_WNDPROC          1002
#define IDC_HOOK             1004
#define IDC_EXIT             1005
#define ES_READONLY         0x800
#define IDC_STATIC             -1
#define DS_MODALFRAME        0x80
#define WS_POPUP       0x80000000
#define WS_CAPTION       0xC00000
#define WS_SYSMENU        0x80000
#define ES_AUTOHSCROLL       0x80
#define ES_READONLY         0x800

IDD_MAINDLG DIALOG 0, 0, 226, 79
STYLE DS_MODALFRAME | WS_POPUP | WS_CAPTION | WS_SYSMENU
CAPTION "Hook to mouse"
FONT 8,  "MS Sans Serif"
{
    GROUPBOX      "Informations about window", IDC_STATIC, 7, 7, 213, 67
    LTEXT         "Class name:", IDC_STATIC, 13, 22, 57, 8, 0
    EDITTEXT      IDC_CLASSNAME, 75, 21, 139, 12, ES_AUTOHSCROLL | ES_READONLY
    LTEXT         "Handle:", IDC_STATIC, 13, 36, 58, 8, 0
    EDITTEXT      IDC_HANDLE, 75, 37, 77, 12, ES_AUTOHSCROLL | ES_READONLY
    LTEXT         "WndProc addr:", IDC_STATIC, 13, 52, 57, 8, 0
    EDITTEXT      IDC_WNDPROC, 75, 52, 77, 12, ES_AUTOHSCROLL | ES_READONLY
    DEFPUSHBUTTON "&Hook", IDC_HOOK, 163, 35, 50, 14
    PUSHBUTTON    "E&xit", IDC_EXIT, 163, 50, 50, 14
}


dll file:

Code Select Expand


;--------COMMENT---------------------------------------------------------------------
;
; It's based on Iczelion's example form chapter 24.
;
;--------MAKE------------------------------------------------------------------------
;
; assembling:
; jwasm -win64 -Zp8 HookDLL.asm
; linking:
; link /SECTION:.bss,S /DLL /SUBSYSTEM:WINDOWS HookDLL.obj
;
;--------INCLUDES--------------------------------------------------------------------

option casemap:none ;set case sensitive on - important for the .INC file

;include HookDLL.inc

includelib      /JWASM/wininc208/lib64/kernel32.lib
includelib      /JWASM/wininc208/lib64/user32.lib

EXTERN SetWindowsHookExA :PROC
EXTERN UnhookWindowsHookEx :PROC
EXTERN CallNextHookEx :PROC
EXTERN WindowFromPoint :PROC
EXTERN PostMessageA :PROC
EXTERN GetLastError :PROC

POINT STRUCT

x DWORD ?
y DWORD ?

POINT ENDS

MOUSEHOOKSTRUCT STRUCT

pt POINT <>
hwnd DWORD ?
wHitTestCode DWORD ?
dwExtraInfo DWORD ?

MOUSEHOOKSTRUCT ENDS

;--------DATA------------------------------------------------------------------------

.CONST
TRUE EQU 1
FALSE EQU 0
WH_MOUSE EQU 7
WM_USER EQU 400h

WM_MOUSEHOOK EQU WM_USER+6

.DATA
hInstance DQ 0

.DATA?
hHook DQ ?
hWnd DQ ?

.CODE

DllEntry PROC hModule:ptr, dwReason:dword, dwReserved:dword

mov hInstance, rcx
mov rax, TRUE

ret

DllEntry ENDP

MouseProc PROC EXPORT FRAME nCode:QWORD, wParam:QWORD, lParam:QWORD

push rbp
.PUSHREG RBP
mov rbp, rsp
.SETFRAME RBP, 0
push rdi
.PUSHREG RDI
sub rsp, 8*5
.ALLOCSTACK 8*5
.ENDPROLOG

mov lParam, r8
mov r9, r8
mov r8, rdx
mov rdx, rcx
mov rcx, hHook
call CallNextHookEx

mov rdi, lParam

ASSUME rdi:PTR MOUSEHOOKSTRUCT

xor edx, edx ;remove garbage from upper halfs
xor ecx, ecx
mov edx, [rdi].pt.y
mov ecx, [rdi].pt.x
call WindowFromPoint ;non-zero result is correct

xor r9d, r9d
mov r8, rax
mov rdx, WM_MOUSEHOOK
mov rcx, hWnd
call PostMessageA

ASSUME rdi:NOTHING

xor eax, eax
add rsp, 8*5
pop rdi
pop rbp
ret

MouseProc ENDP

InstallHook PROC EXPORT FRAME hwnd:QWORD

sub rsp, 8*5
.ALLOCSTACK 8*5
.ENDPROLOG

mov hWnd, rcx

xor r9d, r9d
mov r8, hInstance
lea rdx, MouseProc
mov rcx, WH_MOUSE
call SetWindowsHookExA
mov hHook, rax

add rsp, 8*5
ret

InstallHook ENDP

UninstallHook PROC EXPORT FRAME

sub rsp, 8*5
.ALLOCSTACK 8*5
.ENDPROLOG

mov rcx, hHook
call UnhookWindowsHookEx

add rsp, 8*5
ret

UninstallHook ENDP

END DllEntry


Thanks for any help.

[cdn4753]

I used JWasm converted, he will simply many