windowsassemblymasm64

What is my error in calling CreateWindowExA in MASM x64


I am beginner in assembly. The following code shows the entire ASM file that I am currently working with.

RegisterClassA does not return NULL because rax is not zero, so I suppose that my WNDCLASSA struct setup is just fine.

My problem occurs when calling CreateWindowExA which returns a NULL HWND. Referring to the CreateWindowExA docs on more error information, I called GetLastError which returned error code 87, which means I input an invalid parameter on CreateWindowExA.

How can I determine what parameter I input wrong? Or perhaps, there is another error that I am not aware of?

ExitProcess proto
MessageBoxA proto
RegisterClassA proto
GetModuleHandleA proto
LoadIconA proto
LoadCursorA proto
GetStockObject proto
CreateWindowExA proto
DefWindowProcA proto
DestroyWindow proto
PostQuitMessage proto
GetLastError proto

.const
w32_class_name byte "snake_asm_window_class", 0
w32_window_name byte "snake", 0
w32_window_width dword 720
w32_window_height dword 1280

.code
WinMainCRTStartup proc
    ; WNDCLASS Struct: 72 bytes
    ; 4 Local Vars: 32 bytes
    ; 8 more params for CreateWindowExA: 64 bytes
    ; 4 registers: 32 bytes
    ; 72 + 32 + 64 + 32 = 200 bytes
    sub rsp, 200
    
    mov ecx, 4
    call GetStockObject
    mov qword ptr [rsp + 120], rax
    
    xor rcx, rcx
    mov edx, 32512
    call LoadIconA
    mov qword ptr [rsp + 112], rax
    
    xor rcx, rcx
    mov edx, 32512
    call LoadCursorA
    mov qword ptr [rsp + 104], rax

    xor rcx, rcx
    call GetModuleHandleA
    mov qword ptr [rsp + 96], rax
    
    lea rcx, [rsp + 128]                      ; WNDCLASS into rcx
    
    mov dword ptr [rcx + 0], 23h              ; style = CS_HREDRAW | CS_OWNDC | CS_VREDRAW;

    lea rax, [WindowProc]
    mov qword ptr [rcx + 8], rax              ; lpfnWndProc = WindowProc;
    
    mov dword ptr [rcx + 16], 0               ; WindowClass.cbClsExtra = 0;
    mov dword ptr [rcx + 20], 0               ; WindowClass.cbClsExtra = 0;
    
    mov rax, qword ptr [rsp + 96]
    mov qword ptr [rcx + 24], rax             ; WindowClass.hInstance = ModuleInstance;
    
    mov rax, qword ptr [rsp + 112]
    mov qword ptr [rcx + 32], rax             ; WindowClass.hIcon = LoadIcon(0, MAKEINTRESOURCE(32512));
    
    mov rax, qword ptr [rsp + 104]
    mov qword ptr [rcx + 40], rax             ; WindowClass.hCursor = LoadCursor(0, MAKEINTRESOURCE(32512));

    mov rax, qword ptr [rsp + 120]
    mov qword ptr [rcx + 48], rax             ; WindowClass.hbrBackground = GetStockObject(BLACK_BRUSH);
    
    mov qword ptr [rcx + 56], 0               ; WindowClass.lpszMenuName = 0;
    
    lea rax, w32_class_name
    mov qword ptr [rcx + 64], rax             ; WindowClass.lpszClassName = WindowClassName;
    
    call RegisterClassA
    
    xor ecx, ecx
    lea rdx, w32_class_name
    lea r8, w32_window_name
    mov r9d, 0cf0000h
    mov dword ptr [rsp + 32], 0
    mov dword ptr [rsp + 40], 0
    mov dword ptr [rsp + 48], 1280
    mov dword ptr [rsp + 56], 720
    mov qword ptr [rsp + 64], 0
    mov qword ptr [rsp + 72], 0
    mov rax, qword ptr [rsp + 96]
    mov qword ptr [rsp + 80], rax
    mov qword ptr [rsp + 88], 0
    call CreateWindowExA
    
    call GetLastError ; returns error code 87: ERROR_INVALID_PARAMETER
    
    xor rcx, rcx
    call ExitProcess
WinMainCRTStartup endp

WindowProc proc 
    sub rsp, 40

    xor rax, rax
    test edx, 010h
    je WM_CLOSE

    call DefWindowProcA
    jmp done

WM_CLOSE:
    call DestroyWindow
    jmp done

done:
    add rsp, 40
    ret
WindowProc endp

end

Solution

  • what is just visible

    test edx, 010h
    

    you want compare message value with WM_CLOSE ( 10h) but instead cmp use test, as result you call DestroyWindow when you really not want and not need . and really not need call DestroyWindow on WM_CLOSE - the DefWindowProcA do this for you. probably you need call PostQuitMessage on WM_NCDESTROY.

    then even if CreateWindowExA is ok, you need some message loop after this, but you not do this and call ExitProcess so in any case your program just exit, even if window will be created.

    you not use WS_VISIBLE style and not call ShowWindow as result, even if window will be created and messgeloop exist - it will be invisible anyway.

    you not check return values of RegisterClassA and CreateWindowExA . and call GetLastError after CreateWindowExA exist sense only if CreateWindowExA return 0.

    use WinMainCRTStartup as entry point is ok, but really you can in concrete case use any name, and set this name in /ENTRY:name linker option (now you use default /ENTRY:WinMainCRTStartup )

    in general probably no sense write such code complete on asm, use it exist sense only for special task, but minimal visible example can be next

    EXTERN __imp_GetStockObject : QWORD
    EXTERN __imp_LoadIconA : QWORD
    EXTERN __imp_LoadCursorA : QWORD
    EXTERN __ImageBase : QWORD
    EXTERN __imp_RegisterClassA : QWORD
    EXTERN __imp_CreateWindowExA : QWORD
    EXTERN __imp_PostQuitMessage : QWORD
    EXTERN __imp_ExitProcess : QWORD
    EXTERN __imp_GetLastError : QWORD
    EXTERN __imp_DefWindowProcA : QWORD
    EXTERN __imp_GetMessageA : QWORD
    EXTERN __imp_DispatchMessageA : QWORD
    
    .code
    
    Startup proc
        ; WNDCLASS Struct: 72 bytes
        ; 4 Local Vars: 32 bytes
        ; 8 more params for CreateWindowExA: 64 bytes
        ; 4 registers: 32 bytes
        ; 72 + 32 + 64 + 32 = 200 bytes
        sub rsp, 200
        
        mov ecx, 4
        call __imp_GetStockObject
        mov qword ptr [rsp + 120], rax
        
        xor rcx, rcx
        mov edx, 32512
        call __imp_LoadIconA
        mov qword ptr [rsp + 112], rax
        
        xor rcx, rcx
        mov edx, 32512
        call __imp_LoadCursorA
        mov qword ptr [rsp + 104], rax
    
        lea rax, __ImageBase
        mov qword ptr [rsp + 96], rax
        
        lea rbp, [rsp + 128]                      ; WNDCLASS into rcx
        mov rcx,rbp
        
        mov dword ptr [rcx + 0], 23h              ; style = CS_HREDRAW | CS_OWNDC | CS_VREDRAW;
    
        lea rax, [WindowProc]
        mov qword ptr [rcx + 8], rax              ; lpfnWndProc = WindowProc;
        
        mov dword ptr [rcx + 16], 0               ; WindowClass.cbClsExtra = 0;
        mov dword ptr [rcx + 20], 0               ; WindowClass.cbClsExtra = 0;
        
        mov rax, qword ptr [rsp + 96]
        mov qword ptr [rcx + 24], rax             ; WindowClass.hInstance = ModuleInstance;
        
        mov rax, qword ptr [rsp + 112]
        mov qword ptr [rcx + 32], rax             ; WindowClass.hIcon = LoadIcon(0, MAKEINTRESOURCE(32512));
        
        mov rax, qword ptr [rsp + 104]
        mov qword ptr [rcx + 40], rax             ; WindowClass.hCursor = LoadCursor(0, MAKEINTRESOURCE(32512));
    
        mov rax, qword ptr [rsp + 120]
        mov qword ptr [rcx + 48], rax             ; WindowClass.hbrBackground = GetStockObject(BLACK_BRUSH);
        
        mov qword ptr [rcx + 56], 0               ; WindowClass.lpszMenuName = 0;
        
        lea rax, w32_class_name
        mov qword ptr [rcx + 64], rax             ; WindowClass.lpszClassName = WindowClassName;
        
        call __imp_RegisterClassA
        test ax,ax
        je @@2
        
        xor ecx, ecx
        lea rdx, w32_class_name
        lea r8, w32_window_name
        mov r9d, 10cf0000h
        mov dword ptr [rsp + 32], 0
        mov dword ptr [rsp + 40], 0
        mov dword ptr [rsp + 48], 1280
        mov dword ptr [rsp + 56], 720
        mov qword ptr [rsp + 64], 0
        mov qword ptr [rsp + 72], 0
        mov rax, qword ptr [rsp + 96]
        mov qword ptr [rsp + 80], rax
        mov qword ptr [rsp + 88], 0
        call __imp_CreateWindowExA
        
        test rax, rax
        
        je @@1
        
    @@0:
        mov rcx, rbp
        xor rdx,rdx
        mov r8,rdx
        mov r9,rdx
        call __imp_GetMessageA
        test eax, eax
        je @@2
        mov rcx, rbp
        call __imp_DispatchMessageA
        jmp @@0
        
    @@1:
        call __imp_GetLastError ; 
        
    @@2:
        xor rcx, rcx
        call __imp_ExitProcess
    Startup endp
    
    WindowProc proc 
        sub rsp, 28h
    
        cmp edx, 82h ; WM_NCDESTROY
        jne @@0
        mov [rsp + 30h], rcx
        mov [rsp + 38h], rdx
        mov [rsp + 40h], r8
        mov [rsp + 48h], r9
        
        xor ecx, ecx
        call __imp_PostQuitMessage
        
        mov rcx, [rsp + 30h]
        mov rdx, [rsp + 38h]
        mov r8, [rsp + 40h]
        mov r9, [rsp + 48h]
    @@0:
        call __imp_DefWindowProcA
        add rsp, 28h
        ret
    WindowProc endp
    
    
    .const
    w32_class_name byte "snake_asm_window_class", 0
    w32_window_name byte "snake", 0
    
    end
    

    (use /ENTRY:Startup )