assemblywinapix86-64calling-conventionmasm64

RIP corrupt after I call WinApi function


I have a wrong RIP after I call CreateWindowExA, but it's corrupt after NtUserCreateWindowEx which I find out via x64dbg, maybe I do something wrong with the stack allocation.

There are my code:

include win64a.inc

.code
WinMain proc
push rbp                        ; save old rbp
mov rbp, rsp                    ; set-up rbp
sub rsp, 20h                    ; allocate space for local vars
mov rcx, 0                      ; 1 param
mov rdx, IDC_ARROW              ; 2 param 
sub rsp, 20h                    ; allocate home-space
call LoadCursorA                
mov [rbp - 8h], rax             ; 1 local param, hCursor 
mov rcx, 0                      ; 1 param
mov rdx, IDI_APPLICATION        ; 2 param
call LoadIconA                  ; home-space allocated at 11 
mov [rbp - 10h], rax            ; 2 local param, hIcon
add rsp, 20h                    ; deallocate home-space
; Develope WNDCLASSEXA structure
push [rbp-10h]                  
pushaddr wndName
push 0
push COLOR_WINDOWFRAME
push [rbp-8h]
push [rbp-10h]
push IMAGE_BASE
push 0
pushaddr WndProc
push sizeof WNDCLASSEX

mov rcx, rsp                    ; 1 param - ptr to the structure
sub rsp, 20h                    ; allocate home-space
call RegisterClassExA
sub rsp, 40h                                ; Allocate stack for params
xor rcx, rcx                                ; dwExStyle
lea rdx, wndName                            ; lpClassName
lea r8, wndName                             ; lpWindowName
mov r9, WS_OVERLAPPEDWINDOW or WS_VISIBLE   ; dwStyle
mov qword ptr [rsp + 20h], 0                ; x
mov qword ptr [rsp + 28h], 0                ; y
mov qword ptr [rsp + 30h], 600              ; w
mov qword ptr [rsp + 38h], 800              ; h
mov qword ptr [rsp + 40h], 0                ; hWndParent
mov qword ptr [rsp + 48h], 0                ; hMenu
mov qword ptr [rsp + 50h], IMAGE_BASE       ; hInstance
mov qword ptr [rsp + 58h], 0                ; lpParam
call CreateWindowExA
add rsp, 40h                                ; Deallocate stack
mov [rbp-18h], rax                          ; 3 local param, hWnd
leave
WinMain endp

WndProc proc hWnd:HWND, uMsg:UINT, wParam:WPARAM, lParam:LPARAM
WndProc endp

.data
wndName db "test", 0
end

In win64a.inc :

OPTION DOTNAME
OPTION PROLOGUE:rbpFramePrologue
OPTION EPILOGUE:rbpFrameEpilogue
include win64.inc
include temphls.inc
include ntdll.inc
includelib ntdll.lib
include user32.inc
includelib user32.lib
include gdi32.inc
includelib gdi32.lib
include kernel32.inc
includelib kernel32.lib
include shell32.inc
includelib shell32.lib
;---------------------------
pushaddr macro x
    db 68h
    dd x
endm
IMAGE_BASE  equ 400000h

So I researched and drew graphic of stack allocation and it is completely ok for me, do I something wrong?

enter image description here

And after it calls NtUserCreateWindowEx RIP corrupt and I have something like this: enter image description here

enter image description here

And also I create version with Invoke macro and it cause the same error: (I use MASM64 SDK)

include win64a.inc  

.code 

WinMain proc <13>
local hWnd:qword
local msg:MSG
local wnd:WNDCLASSEX
mov rdi, offset wndName                 ;WndName
invoke LoadCursorA, 0, IDC_APPSTARTING
mov wnd.hCursor, rax
invoke LoadIconA, 0, IDI_APPLICATION
mov wnd.hIcon, rax
mov wnd.hIconSm, rax
mov wnd.lpszClassName, rdi
mov wnd.hInstance, IMAGE_BASE
mov rax, offset Wndproc
mov wnd.lpfnWndProc, rax
mov wnd.cbSize, sizeof WNDCLASSEX
mov wnd.hbrBackground, COLOR_WINDOWFRAME
lea rax, wnd
invoke RegisterClassExA, rax
invoke CreateWindowExA, 0, rdi, rdi, WS_OVERLAPPEDWINDOW or WS_VISIBLE, 0, 0, 600, 800, 0, 0, IMAGE_BASE, 0
mov hWnd, rax
ret
WinMain endp

Wndproc proc <12,8,4,8,8> hWnd:HWND, uMsg:UINT, wParam:WPARAM, lParam:LPARAM
jmp NtdllDefWindowProc_
ret
Wndproc endp
.data
wndName db "test", 0
end

Is generated follow code:

enter image description here

And params before CreateWindowExA call: enter image description here

And ml64.exe / link.exe options:

  1. ml64.exe /I"%masm64_path%/include" /Zp16 /c /Cp %filename%.asm >>errors.txt
    
  2. link /LIBPATH:"%masm64_path%\lib" /LARGEADDRESSAWARE:NO /SUBSYSTEM:WINDOWS ^
     /STUB:%masm64_path%\bin\stubby.exe ^
     /entry:WinMain /fixed %filename%.obj
    

Solution

  • Michael Petch & bitRAKE says that problem cause because NtUserCreateWindowEx calls WndProc function before it returns, so I need to create full WndProc with the stack handling. So I add this leave and it works now.

    Wndproc proc <12,8,4,8,8> hWnd:HWND, uMsg:UINT, wParam:WPARAM, lParam:LPARAM
    leave                   ;<----------- Added to resolve issue
    jmp NtdllDefWindowProc_
    ret
    Wndproc endp