I'm trying to rewrite the following C code in x86_64 assembly as a learning exercise. It calls the Win32 API functions.
#include <windows.h>
__attribute__((section(".rdata"))) volatile LPCSTR lpClassName = "WindowClass";
__attribute__((section(".rdata"))) volatile LPCSTR lpText_RegisterClassAError = "Failed to register window class.";
int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdShow) {
int uExitCode;
LPCSTR lpText;
WNDCLASSA wc = {0};
wc.lpfnWndProc = DefWindowProcA;
wc.hInstance = hInstance;
wc.lpszClassName = lpClassName;
wc.hbrBackground = (HBRUSH)(COLOR_WINDOW + 1); // 0x6
if (!RegisterClassA(&wc)) {
lpText = lpText_RegisterClassAError;
goto exit_error_msgbox;
}
uExitCode = 0;
exit_error_msgbox:
MessageBoxA(NULL, lpText, NULL, MB_ICONERROR);
uExitCode = 1;
exit:
ExitProcess(uExitCode);
return uExitCode;
}
Here is my assembly code in NASM syntax:
default rel
global WinMain
extern DefWindowProcA, RegisterClassA, MessageBoxA, ExitProcess
; https://learn.microsoft.com/en-us/windows/win32/api/winuser/ns-winuser-wndclassa
struc WNDCLASSA
.style resd 1 ; UINT style; sizeof(UINT) = 4
.padding resd 1 ; C ABI padding
.lpfnWndProc resq 1 ; WNDPROC lpfnWndProc; sizeof(WNDPROC) = 8
.cbClsExtra resd 1 ; int cbClsExtra; sizeof(int) = 4
.cbWndExtra resd 1 ; int cbWndExtra; sizeof(int) = 4
.hInstance resq 1 ; HINSTANCE hInstance; sizeof(HINSTANCE) = 8
.hIcon resq 1 ; HICON hIcon; sizeof(HICON) = 8
.hCursor resq 1 ; HCURSOR hCursor; sizeof(HCURSOR) = 8
.hbrBackground resq 1 ; HBRUSH hbrBackground; sizeof(HBRUSH) = 8
.lpszMenuName resq 1 ; LPCSTR lpszMenuName; sizeof(LPCSTR) = 8
.lpszClassName resq 1 ; LPCSTR lpszClassName; sizeof(LPCSTR) = 8
endstruc
section .rdata
lpClassName: db "WindowClass", 0
lpText_RegisterClassAError: db "Failed to register window class.", 0
section .text
; int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdShow)
WinMain:
push rbp
mov rbp, rsp
sub rsp, ((WNDCLASSA_size + 15) & -16) + 32
mov r12, rcx ; hInstance
; https://learn.microsoft.com/en-us/windows/win32/api/winuser/ns-winuser-wndclassa
lea rdi, [rsp + 32]
xor rax, rax
mov rcx, WNDCLASSA_size
cld
rep stosb
mov qword [rsp + 32 + WNDCLASSA.lpfnWndProc], DefWindowProcA
mov qword [rsp + 32 + WNDCLASSA.hInstance], r12
mov qword [rsp + 32 + WNDCLASSA.hbrBackground], 0x6 ; COLOR_WINDOW + 1 = 0x6
mov qword [rsp + 32 + WNDCLASSA.lpszClassName], lpClassName
; https://learn.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-registerclassa
lea rcx, [rsp + 32] ; lpWndClass (for RegisterClassA)
call RegisterClassA
lea rdx, lpText_RegisterClassAError ; lpText (for MessageBoxA)
test rax, rax ; https://learn.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-registerclassa#return-value
jz exit_error_msgbox
xor rcx, rcx ; uExitCode = 0 (for ExitProcess)
jmp exit
exit_error_msgbox:
xor rcx, rcx ; hWnd = NULL (for MessageBoxA)
xor r8, r8 ; lpCaption = NULL (default = "Error") (for MessageBoxA)
mov r9, 0x10 ; uType = MB_ICONERROR = 0x10 (for MessageBoxA)
call MessageBoxA
mov rcx, 1 ; uExitCode = 1 (for ExitProcess)
exit:
add rsp, ((WNDCLASSA_size + 15) & -16) + 32
mov rsp, rbp
pop rbp
call ExitProcess
mov rax, rcx
ret
But when I assemble, link and run the program, it crashes with exit code -1073741819 (0xC0000005) which means access violation. The program crashes when RegisterClassA
is called and the rest of the code isn't reached.
This is the script I use for assembling and linking, but it shouldn't matter if a different linker is used.
nasm -f win64 .\main.s -o .\main.o
link /LARGEADDRESSAWARE:NO /subsystem:windows /entry:WinMain .\main.o kernel32.lib user32.lib
The WNDCLASSA struct definition is correctly padded according to the C ABI and all its members have correct sizes and offsets.
I'm reserving enough space on the stack for the WNDCLASSA instance and padding its size to 16 bytes ((WNDCLASSA_size + 15) & -16
) and reserving additional 32 bytes as shadow space for all the Win32 API function calls.
I'm out of ideas what could cause this issue.
The code as shown does not assemble due to the lea rdx, lpText_RegisterClassAError
. Make sure you post the exact code you are using.
Anyway, the problem seems to be the
mov qword [rsp + 32 + WNDCLASSA.lpfnWndProc], DefWindowProcA
You should be aware that even though you write qword
that only sets the destination size, the immediate is still only 32 bits so you are only passing the low 32 bits of the address as evidenced by this disassembly:
1b: 48 c7 44 24 28 00 00 mov QWORD PTR [rsp+0x28],0x0
22: 00 00
20: R_X86_64_32 DefWindowProcA
As per the instruction set reference this is the MOV r/m64, imm32
instruction which moves "imm32 sign extended to 64-bits to r/m64". Notice there are only 4 zero bytes for the immediate and the relocation entry is 32 bit as well. You can instead do:
lea rax, [DefWindowProcA]
mov [rsp + 32 + WNDCLASSA.lpfnWndProc], rax
Also the call ExitProcess
should be a jmp
since you have cleaned up the stack by that point.