How to store values local to the function

[jwang]

I have inherited a VC++ project that contains a function written in MASM. The purpose of the function is to invoke an external DLL function on the fly. The code works well, but fails when it is called recursively, that Is, the DLL function it invokes calls this function in turn to invoke another DLL function.

I kind of know the cause of the problem, namely, the code uses variables to store some key information such as RSP, return type, return value, etc. before calling the target function and restore them the call returns. As these variables are global, they become invalid when the function is called recursively. I would think that these values need to be store locally, but I do not know how as I have little experience with MSAM.

I post the assembly code there. Any suggestions or help will be greatly appreciated.

jwang

Code Select Expand

save_rsp    dq 0
return_type dq 0
ptr_ret_val dq 0
check_rsp dq 0

.CODE   

;---------------------------------------------------------
; Constants for call_dll_function argument displacement on stack
arg_return_type = 8
arg_func_addr = 16
arg_rcx = 24
arg_rdx = 32
arg_r8 = 40
arg_r9 = 48
arg_xmm0 = 56
arg_xmm1 = 64
arg_xmm2 = 72
arg_xmm3 = 80
arg_stack_args = 88
arg_num_stack_args = 96
arg_ret_val = 104
;---------------------------------------------------------
; Constants for toolkit return types
type_dtinteger = 1
type_dtfloat = 2
type_dtstring = 3
type_dtbool = 4
type_dtchar = 5
type_dtlong = 7
type_dtdouble = 10
type_dtvoid = 12
type_dtstr = 13
type_dttool = 14
type_dtshort = 15
type_dtcurrency = 16
;---------------------------------------------------------
call_dll_function PROC

; save RSP for restore at end of function
mov save_rsp, rsp
;;; Save registers in stack shadow space
mov qword ptr [rsp+32], r9
mov qword ptr [rsp+24], r8
mov qword ptr [rsp+16], rdx
mov qword ptr [rsp+8], rcx

mov rcx, arg_return_type[rsp]       ; save return type
mov return_type, rcx
mov r10, rsp                     ; save rsp before it is changed by push loop
mov rcx, arg_ret_val[rsp]     ; save return variable pointer
mov ptr_ret_val, rcx

push rdi
mov rdi, rsp

;;; Push arguments on stack
mov rcx, arg_num_stack_args[rsp]    ; load loop count
cmp rcx, 0
je no_stack_args
mov rax, arg_stack_args[r10]     ; address of pointer in rax (RAX = pStack)
mov rdx, rcx                   ; set RDX to number of array items in pStack
dec rdx                   ; subtract 1 to get array index of last item
shl rdx, 3                                  ; multiply by 8 to calculate address of last item
add rax, rdx                   ; set RAX to address of last item in pStack array
push_args:
mov rdx, qword ptr [rax]
push rdx
;push qword ptr [rax]     ; push the value pointed to by RAX
sub rax, 8     ; next array index
loop push_args
no_stack_args:

;;; Put arguments in registers
mov rcx, arg_rcx[r10]
mov rdx, arg_rdx[r10]
mov r8, arg_r8[r10]
mov r9, arg_r9[r10]
movq xmm0, qword ptr arg_xmm0[r10]
movq xmm1, qword ptr arg_xmm1[r10]
movq xmm2, qword ptr arg_xmm2[r10]
movq xmm3, qword ptr arg_xmm3[r10]

;;; Call function
sub rsp, 32              ; allocate shadow space
mov check_rsp, rsp                             ; save RSP to check against after call
call qword ptr arg_func_addr[r10]

cmp rsp, check_rsp                             ; rsp must match old value
jne error_rsp

add rsp,32  ;JMW

;;; Return value from function
mov rdx, ptr_ret_val ; restore pointer to RDX
mov rcx, return_type ; return type to compare against
cmp rcx, type_dtshort
je ret_ax
;
cmp rcx, type_dtfloat
je ret_xmm0
cmp rcx, type_dtcurrency
je ret_xmm0
cmp rcx, type_dtdouble
je ret_xmm0
;
cmp rcx, type_dtchar
je ret_al
;
cmp rcx, type_dtstr
je ret_rax
;
cmp rcx, type_dtvoid
je ret_void
;
jmp ret_eax ; default

ret_ax:
movzx rcx, ax
mov [rdx], rcx
jmp finish

ret_xmm0:
movq qword ptr [rdx], xmm0
jmp finish

ret_al:
movzx rcx, al
mov [rdx], rcx
jmp finish

ret_eax:
xor rcx, rcx
mov ecx, eax
mov [rdx], rcx
jmp finish

ret_rax:
mov [rdx], rax
jmp finish

ret_void:
jmp finish

;;; Finish with success
finish:
xor rax, rax ; return 0 (no error) in AL
mov rsp, save_rsp ; restore RSP (pop stack)
ret

;;; Finish with error
error_rsp:
xor rax, rax
mov al, 1 ; return error code 1 in AL
mov rsp, save_rsp ; restore RSP (pop stack)
ret

call_dll_function ENDP
;---------------------------------------------------------

END

[dedndave]

you can set up a stack frame and store the variables on the stack
each recursion will have it's own set of variables
unfortunately, it's 64-bit code, which - i have no experience
best left to some of the other members that are familiar with the 64-bit stack-alignment issues

[Gunther]

Hi jwang,

try this:

Code Select Expand


        push       rbp
        mov        rbp, rsp
        sub        rsp, n                     ; the number of bytes you'll need

        ; other stuff here

        mov        rsp, rbp
        pop        rbp
        ret


Gunther

[qWord]

the code uses variables to store some key information such as RSP, return type, return value, etc.

This parameters are already saved on the stack - it seems like that the author of that function did not have much experience in writing x64 assembler. For example the function misalign the stack, if arg_num_stack_args is odd. Also, when RSP is corrupted, there are seriously problems that should not be hidden as it is currently done (IMO).
The following code should do it, but it is not tested:

Code Select Expand

.CODE   

;---------------------------------------------------------
; Constants for call_dll_function argument displacement on stack
    arg_return_type     = 8+8
    arg_func_addr       = 16+8
    arg_rcx             = 24+8
    arg_rdx             = 32+8
    arg_r8              = 40+8
    arg_r9              = 48+8
    arg_xmm0            = 56+8
    arg_xmm1            = 64+8
    arg_xmm2            = 72+8
    arg_xmm3            = 80+8
    arg_stack_args      = 88+8
    arg_num_stack_args  = 96+8
    arg_ret_val         = 104+8
;---------------------------------------------------------
; Constants for toolkit return types
    type_dtinteger  = 1
    type_dtfloat    = 2
    type_dtstring   = 3
    type_dtbool     = 4
    type_dtchar     = 5
    type_dtlong     = 7
    type_dtdouble   = 10
    type_dtvoid     = 12
    type_dtstr      = 13
    type_dttool     = 14
    type_dtshort    = 15
    type_dtcurrency = 16
;---------------------------------------------------------
call_dll_function PROC

    mov [rsp+8],ecx  ; save arg_return_type
    mov [rsp+16],rdx ; save arg_func_addr
   
    push rbp        ; save rbp + stack align 16
    mov rbp,rsp
   
    mov ecx,arg_num_stack_args[rbp]

    lea eax,[ecx+1] ; num_stack_args must be even to keep the stack alignment of 16
    and eax,-2      ; (--> fastcall )
   
    lea eax,[eax*8+4*32]
    sub rsp,rax         ; alloc. shadow space + stack arguments
   
    ; copy stack arguments
    mov rdx,arg_stack_args[rbp]
    test ecx,ecx
    jmp @2
@1: mov rax,[rdx+rcx*8-8]
    mov [rsp+4*8+rcx*8-8],rax
    sub ecx,1   
@2: jnz @1
   
    ; XMM args
    movq xmm0, qword ptr arg_xmm0[rbp]
    movq xmm1, qword ptr arg_xmm1[rbp]
    movq xmm2, qword ptr arg_xmm2[rbp]
    movq xmm3, qword ptr arg_xmm3[rbp]
   
    ; GPR args
    mov rcx,r8
    mov rdx,r9
    mov r8,arg_r8[rbp]
    mov r9,arg_r9[rbp]
    call QWORD ptr arg_func_addr[rbp]
   
    mov rdx,arg_ret_val[rbp]
    mov ecx,arg_return_type[rbp]
   
    cmp ecx, type_dtshort
    je ret_ax
    cmp ecx, type_dtfloat
    je ret_xmm0
    cmp ecx, type_dtcurrency
    je ret_xmm0
    cmp ecx, type_dtdouble
    je ret_xmm0
    cmp ecx, type_dtchar
    je ret_al
    cmp ecx, type_dtstr
    je ret_rax
    cmp ecx, type_dtvoid
    je finish
    jne ret_eax     ; default = 32bit -> sure?
   
ret_ax:
    movzx rcx, ax
    mov [rdx], rcx
    jmp finish
   
ret_xmm0:
    movq qword ptr [rdx], xmm0
    jmp finish
   
ret_al:
    movzx rcx, al
    mov [rdx], rcx
    jmp finish
   
ret_eax:
    mov ecx, eax
    mov [rdx], rcx
    jmp finish
   
ret_rax:
    mov [rdx], rax
    jmp finish

    ;;; Finish with success
finish:
    xor eax, eax                    ; return 0 (no error) in AL
    mov rsp,rbp
    pop rbp
    ret
   
call_dll_function ENDP
;---------------------------------------------------------

END

[jwang]

Thank you all very much for your prompt response to my question. 

Hi, qWord,
I just plugged in your code. It works wonderfully.  I really appreciate your help. I will do more testing.
Best regards,

jwang