windowsassemblywinapix86-64access-violation

Using Win32 API in assembly crashes with exit code -1073741819 (0xC0000005) - access violation


I'm trying to rewrite the following C code in x86_64 assembly as a learning exercise. It calls the Win32 API functions.

#include <windows.h>

__attribute__((section(".rdata"))) volatile LPCSTR lpClassName = "WindowClass";
__attribute__((section(".rdata"))) volatile LPCSTR lpText_RegisterClassAError = "Failed to register window class.";

int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdShow) {
    int uExitCode;
    LPCSTR lpText;
    WNDCLASSA wc = {0};
    wc.lpfnWndProc   = DefWindowProcA;
    wc.hInstance     = hInstance;
    wc.lpszClassName = lpClassName;
    wc.hbrBackground = (HBRUSH)(COLOR_WINDOW + 1); // 0x6

    if (!RegisterClassA(&wc)) {
        lpText = lpText_RegisterClassAError;
        goto exit_error_msgbox;
    }
    
    uExitCode = 0;
exit_error_msgbox:
    MessageBoxA(NULL, lpText, NULL, MB_ICONERROR);
    uExitCode = 1;
exit:
    ExitProcess(uExitCode);
    return uExitCode;
}

Here is my assembly code in NASM syntax:

default rel
global WinMain
extern DefWindowProcA, RegisterClassA, MessageBoxA, ExitProcess

; https://learn.microsoft.com/en-us/windows/win32/api/winuser/ns-winuser-wndclassa
struc WNDCLASSA
    .style         resd 1 ; UINT style;             sizeof(UINT)      = 4
    .padding       resd 1 ; C ABI padding
    .lpfnWndProc   resq 1 ; WNDPROC lpfnWndProc;    sizeof(WNDPROC)   = 8
    .cbClsExtra    resd 1 ; int cbClsExtra;         sizeof(int)       = 4
    .cbWndExtra    resd 1 ; int cbWndExtra;         sizeof(int)       = 4
    .hInstance     resq 1 ; HINSTANCE hInstance;    sizeof(HINSTANCE) = 8
    .hIcon         resq 1 ; HICON hIcon;            sizeof(HICON)     = 8
    .hCursor       resq 1 ; HCURSOR hCursor;        sizeof(HCURSOR)   = 8
    .hbrBackground resq 1 ; HBRUSH hbrBackground;   sizeof(HBRUSH)    = 8
    .lpszMenuName  resq 1 ; LPCSTR lpszMenuName;    sizeof(LPCSTR)    = 8
    .lpszClassName resq 1 ; LPCSTR lpszClassName;   sizeof(LPCSTR)    = 8
endstruc

section .rdata
    lpClassName: db "WindowClass", 0
    lpText_RegisterClassAError: db "Failed to register window class.", 0

section .text
; int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdShow)
WinMain:
    push rbp
    mov rbp, rsp
    sub rsp, ((WNDCLASSA_size + 15) & -16) + 32

    mov r12, rcx ; hInstance

    ; https://learn.microsoft.com/en-us/windows/win32/api/winuser/ns-winuser-wndclassa
    lea rdi, [rsp + 32]
    xor rax, rax
    mov rcx, WNDCLASSA_size
    cld
    rep stosb
    mov qword [rsp + 32 + WNDCLASSA.lpfnWndProc],   DefWindowProcA
    mov qword [rsp + 32 + WNDCLASSA.hInstance],     r12
    mov qword [rsp + 32 + WNDCLASSA.hbrBackground], 0x6 ; COLOR_WINDOW + 1 = 0x6
    mov qword [rsp + 32 + WNDCLASSA.lpszClassName], lpClassName

    ; https://learn.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-registerclassa
    lea rcx, [rsp + 32] ; lpWndClass (for RegisterClassA)
    call RegisterClassA
    lea rdx, lpText_RegisterClassAError ; lpText (for MessageBoxA)
    test rax, rax                       ; https://learn.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-registerclassa#return-value
    jz exit_error_msgbox

    xor rcx, rcx ; uExitCode = 0 (for ExitProcess)
    jmp exit

exit_error_msgbox:
    xor rcx, rcx ; hWnd = NULL (for MessageBoxA)
    xor r8, r8   ; lpCaption = NULL (default = "Error") (for MessageBoxA)
    mov r9, 0x10 ; uType = MB_ICONERROR = 0x10 (for MessageBoxA)
    call MessageBoxA
    mov rcx, 1 ; uExitCode = 1 (for ExitProcess)
exit:
    add rsp, ((WNDCLASSA_size + 15) & -16) + 32
    mov rsp, rbp
    pop rbp
    call ExitProcess
    mov rax, rcx
    ret

But when I assemble, link and run the program, it crashes with exit code -1073741819 (0xC0000005) which means access violation. The program crashes when RegisterClassA is called and the rest of the code isn't reached.

This is the script I use for assembling and linking, but it shouldn't matter if a different linker is used.

nasm -f win64 .\main.s -o .\main.o
link /LARGEADDRESSAWARE:NO /subsystem:windows /entry:WinMain .\main.o kernel32.lib user32.lib

The WNDCLASSA struct definition is correctly padded according to the C ABI and all its members have correct sizes and offsets. I'm reserving enough space on the stack for the WNDCLASSA instance and padding its size to 16 bytes ((WNDCLASSA_size + 15) & -16) and reserving additional 32 bytes as shadow space for all the Win32 API function calls.

I'm out of ideas what could cause this issue.


Solution

  • The code as shown does not assemble due to the lea rdx, lpText_RegisterClassAError. Make sure you post the exact code you are using.

    Anyway, the problem seems to be the

    mov qword [rsp + 32 + WNDCLASSA.lpfnWndProc], DefWindowProcA
    

    You should be aware that even though you write qword that only sets the destination size, the immediate is still only 32 bits so you are only passing the low 32 bits of the address as evidenced by this disassembly:

    1b: 48 c7 44 24 28 00 00    mov    QWORD PTR [rsp+0x28],0x0
    22: 00 00 
            20: R_X86_64_32 DefWindowProcA
    

    As per the instruction set reference this is the MOV r/m64, imm32 instruction which moves "imm32 sign extended to 64-bits to r/m64". Notice there are only 4 zero bytes for the immediate and the relocation entry is 32 bit as well. You can instead do:

    lea rax, [DefWindowProcA]
    mov [rsp + 32 + WNDCLASSA.lpfnWndProc], rax
    

    Also the call ExitProcess should be a jmp since you have cleaned up the stack by that point.