I've been working for days on an assembler subroutine to call Windows API function FormatMessageA, and I think I must have some systematic misunderstanding. My routine is shown below. I have traced it in debug mode, and I know that at the time the function is called: rcx has hex 1B00, which is the flag values I specified (dwFlags); rdx has hex 33, which is the bogus initial handle value that I plugged in to provoke the error (lpSource); r8 has 6, which is the error message number that was returned by GetLastError, which equates to ERROR_INVALID_HANDLE (dwMessageId); r9 has 0, for default language (dwLanguageId); rsp+32 has the address of msgptrp, which is my area to receive the address of the error message as allocated by the called function (lpBuffer); rsp+40 has hex 28, or decimal 40, which is the minimum number of characters I arbitrarily specified for the error message (nSize); and rsp+48 has the address of argmntsp, which according to the documentation should be ignored with the flags I specified (*Arguments).
If there's a problem, and obviously there is since rax is returned as zero, I suspect that it has to do with lpBuffer, which the documentation says has data type LPTSTR, whatever that is. Sometimes I want to shout, "Okay, that's what it is in terms of C++, but what is it really? I hope someone can easily spot where I ran off the rails, because I'm at my wits' end with this, and it's something I need to get working in order to do effective error checking for my future endeavors.
goterr PROC
;
.data
; flag values used:
; hex 00000100 FORMAT_MESSAGE_ALLOCATE_BUFFER
; hex 00000200 FORMAT_MESSAGE_IGNORE_INSERTS
; hex 00000800 FORMAT_MESSAGE_FROM_HMODULE
; hex 00001000 FORMAT_MESSAGE_FROM_SYSTEM
flagsp dd 00001B00h ; those flags combined
saveinitp dq ?
bmaskp dq 0fffffffffffffff0h
savshadp dq ?
msgptrp dq ?
handlep dd ?
argmntsp dd ?
;
.code
mov saveinitp, rsp ; save initial contents of stack pointer in this proc
sub rsp, 56 ; shadow space (max of 7 parameters * 8)
and rsp, bmaskp ; make sure it's 16 byte aligned
mov savshadp, rsp ; save address of aligned shadow area for this proc
;
mov handlep, ecx ; save passed I/O handle value locally
;
call GetLastError ; get the specific error
;
mov rsp, savshadp ; shadow area for this proc
mov ecx, flagsp ; flags for Windows error routine
mov edx, handlep ; handle passed from caller
mov r8d, eax ; msg id from GetLastError in low order dword
mov r9, 0 ; default language id
lea rax, msgptrp ; pointer to receive address of msg buffer
mov [rsp+32], rax ; put it on the stack
mov rax, 40 ; set lower doubleword to minimum size for msg buffer
mov [rsp+40], rax ; put it on the stack
lea rax, argmntsp ; variable arguments parameter (not used)
mov [rsp+48], rax ; put it on the stack
call FormatMessageA ; if rax eq 0 the called failed, otherwise rax is no chars.
mov rsp, saveinitp ; restore initial SP for this proc
ret
goterr ENDP