I have inherited a VC++ project that contains a function written in MASM. The purpose of the function is to invoke an external DLL function on the fly. The code works well, but fails when it is called recursively, that Is, the DLL function it invokes calls this function in turn to invoke another DLL function.
I kind of know the cause of the problem, namely, the code uses variables to store some key information such as RSP, return type, return value, etc. before calling the target function and restore them the call returns. As these variables are global, they become invalid when the function is called recursively. I would think that these values need to be store locally, but I do not know how as I have little experience with MSAM.
I post the assembly code there. Any suggestions or help will be greatly appreciated.
jwang
save_rsp dq 0
return_type dq 0
ptr_ret_val dq 0
check_rsp dq 0
.CODE
;---------------------------------------------------------
; Constants for call_dll_function argument displacement on stack
arg_return_type = 8
arg_func_addr = 16
arg_rcx = 24
arg_rdx = 32
arg_r8 = 40
arg_r9 = 48
arg_xmm0 = 56
arg_xmm1 = 64
arg_xmm2 = 72
arg_xmm3 = 80
arg_stack_args = 88
arg_num_stack_args = 96
arg_ret_val = 104
;---------------------------------------------------------
; Constants for toolkit return types
type_dtinteger = 1
type_dtfloat = 2
type_dtstring = 3
type_dtbool = 4
type_dtchar = 5
type_dtlong = 7
type_dtdouble = 10
type_dtvoid = 12
type_dtstr = 13
type_dttool = 14
type_dtshort = 15
type_dtcurrency = 16
;---------------------------------------------------------
call_dll_function PROC
; save RSP for restore at end of function
mov save_rsp, rsp
;;; Save registers in stack shadow space
mov qword ptr [rsp+32], r9
mov qword ptr [rsp+24], r8
mov qword ptr [rsp+16], rdx
mov qword ptr [rsp+8], rcx
mov rcx, arg_return_type[rsp] ; save return type
mov return_type, rcx
mov r10, rsp ; save rsp before it is changed by push loop
mov rcx, arg_ret_val[rsp] ; save return variable pointer
mov ptr_ret_val, rcx
push rdi
mov rdi, rsp
;;; Push arguments on stack
mov rcx, arg_num_stack_args[rsp] ; load loop count
cmp rcx, 0
je no_stack_args
mov rax, arg_stack_args[r10] ; address of pointer in rax (RAX = pStack)
mov rdx, rcx ; set RDX to number of array items in pStack
dec rdx ; subtract 1 to get array index of last item
shl rdx, 3 ; multiply by 8 to calculate address of last item
add rax, rdx ; set RAX to address of last item in pStack array
push_args:
mov rdx, qword ptr [rax]
push rdx
;push qword ptr [rax] ; push the value pointed to by RAX
sub rax, 8 ; next array index
loop push_args
no_stack_args:
;;; Put arguments in registers
mov rcx, arg_rcx[r10]
mov rdx, arg_rdx[r10]
mov r8, arg_r8[r10]
mov r9, arg_r9[r10]
movq xmm0, qword ptr arg_xmm0[r10]
movq xmm1, qword ptr arg_xmm1[r10]
movq xmm2, qword ptr arg_xmm2[r10]
movq xmm3, qword ptr arg_xmm3[r10]
;;; Call function
sub rsp, 32 ; allocate shadow space
mov check_rsp, rsp ; save RSP to check against after call
call qword ptr arg_func_addr[r10]
cmp rsp, check_rsp ; rsp must match old value
jne error_rsp
add rsp,32 ;JMW
;;; Return value from function
mov rdx, ptr_ret_val ; restore pointer to RDX
mov rcx, return_type ; return type to compare against
cmp rcx, type_dtshort
je ret_ax
;
cmp rcx, type_dtfloat
je ret_xmm0
cmp rcx, type_dtcurrency
je ret_xmm0
cmp rcx, type_dtdouble
je ret_xmm0
;
cmp rcx, type_dtchar
je ret_al
;
cmp rcx, type_dtstr
je ret_rax
;
cmp rcx, type_dtvoid
je ret_void
;
jmp ret_eax ; default
ret_ax:
movzx rcx, ax
mov [rdx], rcx
jmp finish
ret_xmm0:
movq qword ptr [rdx], xmm0
jmp finish
ret_al:
movzx rcx, al
mov [rdx], rcx
jmp finish
ret_eax:
xor rcx, rcx
mov ecx, eax
mov [rdx], rcx
jmp finish
ret_rax:
mov [rdx], rax
jmp finish
ret_void:
jmp finish
;;; Finish with success
finish:
xor rax, rax ; return 0 (no error) in AL
mov rsp, save_rsp ; restore RSP (pop stack)
ret
;;; Finish with error
error_rsp:
xor rax, rax
mov al, 1 ; return error code 1 in AL
mov rsp, save_rsp ; restore RSP (pop stack)
ret
call_dll_function ENDP
;---------------------------------------------------------
END
you can set up a stack frame and store the variables on the stack
each recursion will have it's own set of variables
unfortunately, it's 64-bit code, which - i have no experience
best left to some of the other members that are familiar with the 64-bit stack-alignment issues
Hi jwang,
try this:
push rbp
mov rbp, rsp
sub rsp, n ; the number of bytes you'll need
; other stuff here
mov rsp, rbp
pop rbp
ret
Gunther
Quote from: jwang on December 01, 2013, 01:51:50 AMthe code uses variables to store some key information such as RSP, return type, return value, etc.
This parameters are already saved on the stack - it seems like that the author of that function did not have much experience in writing x64 assembler. For example the function misalign the stack, if arg_num_stack_args is odd. Also, when RSP is corrupted, there are seriously problems that should not be hidden as it is currently done (IMO).
The following code should do it, but it is not tested:
.CODE
;---------------------------------------------------------
; Constants for call_dll_function argument displacement on stack
arg_return_type = 8+8
arg_func_addr = 16+8
arg_rcx = 24+8
arg_rdx = 32+8
arg_r8 = 40+8
arg_r9 = 48+8
arg_xmm0 = 56+8
arg_xmm1 = 64+8
arg_xmm2 = 72+8
arg_xmm3 = 80+8
arg_stack_args = 88+8
arg_num_stack_args = 96+8
arg_ret_val = 104+8
;---------------------------------------------------------
; Constants for toolkit return types
type_dtinteger = 1
type_dtfloat = 2
type_dtstring = 3
type_dtbool = 4
type_dtchar = 5
type_dtlong = 7
type_dtdouble = 10
type_dtvoid = 12
type_dtstr = 13
type_dttool = 14
type_dtshort = 15
type_dtcurrency = 16
;---------------------------------------------------------
call_dll_function PROC
mov [rsp+8],ecx ; save arg_return_type
mov [rsp+16],rdx ; save arg_func_addr
push rbp ; save rbp + stack align 16
mov rbp,rsp
mov ecx,arg_num_stack_args[rbp]
lea eax,[ecx+1] ; num_stack_args must be even to keep the stack alignment of 16
and eax,-2 ; (--> fastcall )
lea eax,[eax*8+4*32]
sub rsp,rax ; alloc. shadow space + stack arguments
; copy stack arguments
mov rdx,arg_stack_args[rbp]
test ecx,ecx
jmp @2
@1: mov rax,[rdx+rcx*8-8]
mov [rsp+4*8+rcx*8-8],rax
sub ecx,1
@2: jnz @1
; XMM args
movq xmm0, qword ptr arg_xmm0[rbp]
movq xmm1, qword ptr arg_xmm1[rbp]
movq xmm2, qword ptr arg_xmm2[rbp]
movq xmm3, qword ptr arg_xmm3[rbp]
; GPR args
mov rcx,r8
mov rdx,r9
mov r8,arg_r8[rbp]
mov r9,arg_r9[rbp]
call QWORD ptr arg_func_addr[rbp]
mov rdx,arg_ret_val[rbp]
mov ecx,arg_return_type[rbp]
cmp ecx, type_dtshort
je ret_ax
cmp ecx, type_dtfloat
je ret_xmm0
cmp ecx, type_dtcurrency
je ret_xmm0
cmp ecx, type_dtdouble
je ret_xmm0
cmp ecx, type_dtchar
je ret_al
cmp ecx, type_dtstr
je ret_rax
cmp ecx, type_dtvoid
je finish
jne ret_eax ; default = 32bit -> sure?
ret_ax:
movzx rcx, ax
mov [rdx], rcx
jmp finish
ret_xmm0:
movq qword ptr [rdx], xmm0
jmp finish
ret_al:
movzx rcx, al
mov [rdx], rcx
jmp finish
ret_eax:
mov ecx, eax
mov [rdx], rcx
jmp finish
ret_rax:
mov [rdx], rax
jmp finish
;;; Finish with success
finish:
xor eax, eax ; return 0 (no error) in AL
mov rsp,rbp
pop rbp
ret
call_dll_function ENDP
;---------------------------------------------------------
END
Thank you all very much for your prompt response to my question.
Hi, qWord,
I just plugged in your code. It works wonderfully. I really appreciate your help. I will do more testing.
Best regards,
jwang