winapimasm64

Using MASM64, I can't get the correct mouse coordinates using ReadConsoleInputW


I use MASM64 with the latest Visual Studio 2022 Preview on the latest Windows 11. No 3rd party assembly libraries are used. Only pure WinAPI.

I want to get the mouse cursor position, when the left mouse button is clicked, in screen buffer coordinates. GetMousePos returns pixels, so it's not useful.

I have setup a Console application with the correct std input handle. I have set the console mode correctly (I hope). I have set the X64 calling convention correctly.

When I call ReadConsoleInputW, I am able to make it work. It returns a correct Y coordinate, but the X coordinate is stuck at zero.

I kindly ask for help finding out why the X coordinate is always zero.

Here is the relevant setup for the input handle and console mode. Error checking exists, but is removed here for bravity.

; WinAPI libraries.
INCLUDELIB kernel32.lib
INCLUDELIB user32.lib

.CONST
    ALIGN 2
        ; From WinAPI.
        COORD STRUCT
            X WORD -1
            Y WORD -1
        COORD ENDS

.CODE
ALIGN 16
        ; Sets the input handle since its needed for the mouse functions.  
        mov ecx, -10  ;  STD_INPUT_HANDLE nStdHandle.
    
        ; https://learn.microsoft.com/en-us/windows/console/getstdhandle
        ; Retrieves a handle to the specified standard output.
        EXTERNDEF GetStdHandle:PROC ; 2000.
        call GetStdHandle
    
        ; MyStdInputHandleQ is defined as QWORD elsewere.
        mov [MyStdInputHandleQ], rax ; Saves the returned value.
    
        ; Sets the console mode to enable the mouse and disable the keyboard.
        mov rcx, [MyStdInputHandleQ] ; hConsoleHandle.
        mov edx, 90h ; ENABLE_MOUSE_INPUT + ENABLE_EXTENDED_FLAGS dwMode.
    
        ; https://learn.microsoft.com/en-us/windows/console/setconsolemode
        ; Sets the input mode of a console's input buffer.
        EXTERNDEF SetConsoleMode:PROC ; 2000.
        call SetConsoleMode

Here is the procedure:

; Gets the screen buffer mouse click coordinates.
MyReadConsoleInputWProc PROC
LOCAL MyResultD:DWORD

        push rbp
        mov rbp, rsp
        sub rsp, 4 * 8
        sub rsp, 8 ; MyResultD.
        and rsp, -16
    
    MyLoopLabel:
        mov rcx, [MyStdInputHandleQ] ; hConsoleInput.
    
    .CONST
        ALIGN 4
            
            ; From WinAPI and used only by it.
            KEY_EVENT_RECORD STRUCT
                bKeyDown DWORD -1
                wRepeatCount WORD -1
                wVirtualKeyCode WORD -1
                wVirtualScanCode WORD -1
                UNION uChar
                    UnicodeChar WORD -1
                    AsciiChar BYTE -1
                ENDS
                dwControlKeyState DWORD -1
            KEY_EVENT_RECORD ENDS
    
            ; From WinAPI and used only by it.
            MOUSE_EVENT_RECORD STRUCT
                dwMousePosition COORD <-1, -1>
                dwButtonState DWORD -1
                dwControlKeyState DWORD -1
                dwEventFlags DWORD -1
            MOUSE_EVENT_RECORD ENDS
    
            ; From WinAPI and used only by it.
            WINDOW_BUFFER_SIZE_RECORD STRUCT
                dwSize COORD <-1, -1>
            WINDOW_BUFFER_SIZE_RECORD ENDS
    
            ; From WinAPI and used only by it.
            MENU_EVENT_RECORD STRUCT
                dwCommandId DWORD -1
            MENU_EVENT_RECORD ENDS
    
            ; From WinAPI and used only by it.
            FOCUS_EVENT_RECORD STRUCT
                bSetFocus DWORD -1
            FOCUS_EVENT_RECORD ENDS
    
        ALIGN 8
    
            ; From WinAPI.
            INPUT_RECORD STRUCT
                EventType WORD -1
                UNION Event
                    KeyEvent KEY_EVENT_RECORD <>
                    MouseEvent MOUSE_EVENT_RECORD <>
                    WindowBufferSizeEvent WINDOW_BUFFER_SIZE_RECORD <>
                    MenuEvent MENU_EVENT_RECORD <>
                    FocusEvent FOCUS_EVENT_RECORD <>
                ENDS
            INPUT_RECORD ENDS
    
    .DATA
        ALIGN 8
            
            MyInputRecordS INPUT_RECORD <>
    
    .CODE
    ALIGN 16
    
        lea rdx, [MyInputRecordS] ; lpBuffer.
        mov r8d, 1 ; nLength.
        lea r9, MyResultD ; lpNumberOfEventsRead.
    
        ; https://learn.microsoft.com/en-us/windows/console/readconsoleinput
        ; Reads data from a console input buffer and removes it from the buffer.
        EXTERNDEF ReadConsoleInputW:PROC ; 2000.
        call ReadConsoleInputW
    
        ; Checks if the mouse event happend.
        mov ax, [MyInputRecordS.EventType]
        cmp ax, 2 ; MOUSE_EVENT.
        jne MyLoopLabel
    
        ; Checks if the left mouse button is pressed.
        mov eax, [MyInputRecordS.Event.MouseEvent.dwButtonState]
            test eax, 1 ; FROM_LEFT_1ST_BUTTON_PRESSED.
        jne MyLoopLabel
    
        ; Checks if the mouse button was pressed or released.
        mov eax, [MyInputRecordS.Event.MouseEvent.dwEventFlags]
        test eax, eax ; 0 means a mouse button was pressed or released.
        jne MyLoopLabel
    
        ; Gets the mouse cursor coordinates.
        mov ax, [MyInputRecordS.Event.MouseEvent.dwMousePosition.X]
        mov [MyCursorPositionCOORD.X], ax ; MyCursorPositionCOORD is defined as COORD elsewere.
        mov ax, [MyInputRecordS.Event.MouseEvent.dwMousePosition.Y]
        mov [MyCursorPositionCOORD.Y], ax
    
        leave
        ret
    MyReadConsoleInputWProc ENDP

I tried many solutions from StackOverflow, Copilot, other AI tools, C++ and assembly code samples. I used the debugger to view the internals of the structs. I tried different data alignment of structs.

I expect the X coordinate to reflect the mouse coordinate when left clicked.


Solution

  • It was an alignment problem. Once I put ALIGN 2 on the COORD struct in the MOUSE_EVENT_RECORD struct, it started to work.