assemblywinapistructmasmcalling-convention

Why doesn't my assembly struct's size match the struct size the GetOpenFileNameA function expects?


I'm trying to display an open file dialog using the Win32 API and x64 assembly. To do so, I'm matching the expected struct's memory allocations to that of the OPENFILENAMEA structure (in commdlg.h), but I seem to be off by 4 bytes, since the size of my struct is 140 bytes. The CommDlgExtendedError() function doesn't return CDERR_STRUCTSIZE (0x0001) if I manually set the lStructSize parameter to 136 or 152, but an "Access violation reading location 0x0000010000007FF7" error occurs instead.

I'm using MASM (ml64.exe) in Visual Studio 2022 Community and assembling under x64.

Here's the struct I'm trying to match (lpEditInfo and lpstrPrompt are omitted) from: https://learn.microsoft.com/en-us/windows/win32/api/commdlg/ns-commdlg-openfilenamea

typedef struct tagOFNA {
  DWORD         lStructSize;
  HWND          hwndOwner;
  HINSTANCE     hInstance;
  LPCSTR        lpstrFilter;
  LPSTR         lpstrCustomFilter;
  DWORD         nMaxCustFilter;
  DWORD         nFilterIndex;
  LPSTR         lpstrFile;
  DWORD         nMaxFile;
  LPSTR         lpstrFileTitle;
  DWORD         nMaxFileTitle;
  LPCSTR        lpstrInitialDir;
  LPCSTR        lpstrTitle;
  DWORD         Flags;
  WORD          nFileOffset;
  WORD          nFileExtension;
  LPCSTR        lpstrDefExt;
  LPARAM        lCustData;
  LPOFNHOOKPROC lpfnHook;
  LPCSTR        lpTemplateName;
  LPEDITMENU    lpEditInfo;
  LPCSTR        lpstrPrompt;
  void          *pvReserved;
  DWORD         dwReserved;
  DWORD         FlagsEx;
} OPENFILENAMEA, *LPOPENFILENAMEA;

Here's my code:

extrn ExitProcess: PROC
extrn GetOpenFileNameA: PROC
extrn CommDlgExtendedError: PROC

.data
filename db 256 dup(0)
tagOFNA STRUCT
  lStructSize       dd 0    ; 0 - dword
  hwndOwner         dq 0    ; 4 - qword
  hInstance         dq 0    ; 12    - qword
  lpstrFilter       dq 0    ; 20    - qword
  lpstrCustomFilter dq 0    ; 28    - qword
  nMaxCustFilter    dd 0    ; 36    - dword
  nFilterIndex      dd 0    ; 40    - dword
  lpstrFile         dq offset filename  ; 44    - qword
  nMaxFile          dd 256  ; 52    - dword
  lpstrFileTitle    dq 0    ; 56    - qword
  nMaxFileTitle     dd 0    ; 64    - dword
  lpstrInitialDir   dq 0    ; 68    - qword
  lpstrTitle        dq 0    ; 76    - qword
  Flags             dd 0    ; 84    - dword
  nFileOffset       dw 0    ; 88    - word
  nFileExtension    dw 0    ; 90    - word
  lpstrDefExt       dq 0    ; 92    - qword
  lCustData         dq 0    ; 100   - qword
  lpfnHook          dq 0    ; 108   - qword
  lpTemplateName    dq 0    ; 116   - qword
  pvReserved        dq 0    ; 124   - qword
  dwReserved        dd 0    ; 132   - dword
  FlagsEx           dd 0    ; 136   - dword
tagOFNA ENDS

myofn tagOFNA <sizeof tagOFNA>

.code
main PROC
    sub rsp, 28h

    lea  rcx, myofn

    call GetOpenFileNameA

    call CommDlgExtendedError
    mov rcx, rax
    call ExitProcess
main ENDP
end

I've tried setting parameters for the struct, but that didn't make any difference, so I tried to get the minimal amount of code to just open the dialog, by setting every parameter to null, except the lStructSize and lpstrFile struct parameters.


Solution

  • As Peter commented, the C struct has padding for alignment. You are missing 3 dwords which bring your size from 140 to 152 as expected. You have to insert padding after the lStructSize, nMaxFile and nMaxFileTitle to align the following qword members to multiple of 8:

      lStructSize       dd 0    ; 0 - dword
      pad1              dd 0    ; 4 - dword
      hwndOwner         dq 0    ; 8 - qword
      hInstance         dq 0    ; 16    - qword
      lpstrFilter       dq 0    ; 24    - qword
      lpstrCustomFilter dq 0    ; 32    - qword
      nMaxCustFilter    dd 0    ; 40    - dword
      nFilterIndex      dd 0    ; 44    - dword
      lpstrFile         dq offset filename  ; 48    - qword
      nMaxFile          dd 256  ; 56    - dword
      pad2              dd 0    ; 60    - dword
      lpstrFileTitle    dq 0    ; 64    - qword
      nMaxFileTitle     dd 0    ; 72    - dword
      pad3              dd 0    ; 76    - dword
      lpstrInitialDir   dq 0    ; 80    - qword
      lpstrTitle        dq 0    ; 88    - qword
      Flags             dd 0    ; 96    - dword
      nFileOffset       dw 0    ; 100   - word
      nFileExtension    dw 0    ; 102   - word
      lpstrDefExt       dq 0    ; 104   - qword
      lCustData         dq 0    ; 112   - qword
      lpfnHook          dq 0    ; 120   - qword
      lpTemplateName    dq 0    ; 128   - qword
      pvReserved        dq 0    ; 136   - qword
      dwReserved        dd 0    ; 144   - dword
      FlagsEx           dd 0    ; 148   - dword