I'm trying to display an open file dialog using the Win32 API and x64 assembly. To do so, I'm matching the expected struct's memory allocations to that of the OPENFILENAMEA
structure (in commdlg.h
), but I seem to be off by 4 bytes, since the size of my struct is 140 bytes. The CommDlgExtendedError()
function doesn't return CDERR_STRUCTSIZE
(0x0001) if I manually set the lStructSize
parameter to 136 or 152, but an "Access violation reading location 0x0000010000007FF7" error occurs instead.
I'm using MASM (ml64.exe
) in Visual Studio 2022 Community and assembling under x64.
Here's the struct I'm trying to match (lpEditInfo
and lpstrPrompt
are omitted) from: https://learn.microsoft.com/en-us/windows/win32/api/commdlg/ns-commdlg-openfilenamea
typedef struct tagOFNA {
DWORD lStructSize;
HWND hwndOwner;
HINSTANCE hInstance;
LPCSTR lpstrFilter;
LPSTR lpstrCustomFilter;
DWORD nMaxCustFilter;
DWORD nFilterIndex;
LPSTR lpstrFile;
DWORD nMaxFile;
LPSTR lpstrFileTitle;
DWORD nMaxFileTitle;
LPCSTR lpstrInitialDir;
LPCSTR lpstrTitle;
DWORD Flags;
WORD nFileOffset;
WORD nFileExtension;
LPCSTR lpstrDefExt;
LPARAM lCustData;
LPOFNHOOKPROC lpfnHook;
LPCSTR lpTemplateName;
LPEDITMENU lpEditInfo;
LPCSTR lpstrPrompt;
void *pvReserved;
DWORD dwReserved;
DWORD FlagsEx;
} OPENFILENAMEA, *LPOPENFILENAMEA;
Here's my code:
extrn ExitProcess: PROC
extrn GetOpenFileNameA: PROC
extrn CommDlgExtendedError: PROC
.data
filename db 256 dup(0)
tagOFNA STRUCT
lStructSize dd 0 ; 0 - dword
hwndOwner dq 0 ; 4 - qword
hInstance dq 0 ; 12 - qword
lpstrFilter dq 0 ; 20 - qword
lpstrCustomFilter dq 0 ; 28 - qword
nMaxCustFilter dd 0 ; 36 - dword
nFilterIndex dd 0 ; 40 - dword
lpstrFile dq offset filename ; 44 - qword
nMaxFile dd 256 ; 52 - dword
lpstrFileTitle dq 0 ; 56 - qword
nMaxFileTitle dd 0 ; 64 - dword
lpstrInitialDir dq 0 ; 68 - qword
lpstrTitle dq 0 ; 76 - qword
Flags dd 0 ; 84 - dword
nFileOffset dw 0 ; 88 - word
nFileExtension dw 0 ; 90 - word
lpstrDefExt dq 0 ; 92 - qword
lCustData dq 0 ; 100 - qword
lpfnHook dq 0 ; 108 - qword
lpTemplateName dq 0 ; 116 - qword
pvReserved dq 0 ; 124 - qword
dwReserved dd 0 ; 132 - dword
FlagsEx dd 0 ; 136 - dword
tagOFNA ENDS
myofn tagOFNA <sizeof tagOFNA>
.code
main PROC
sub rsp, 28h
lea rcx, myofn
call GetOpenFileNameA
call CommDlgExtendedError
mov rcx, rax
call ExitProcess
main ENDP
end
I've tried setting parameters for the struct, but that didn't make any difference, so I tried to get the minimal amount of code to just open the dialog, by setting every parameter to null, except the lStructSize
and lpstrFile
struct parameters.
As Peter commented, the C struct has padding for alignment. You are missing 3 dwords which bring your size from 140 to 152 as expected. You have to insert padding after the lStructSize
, nMaxFile
and nMaxFileTitle
to align the following qword members to multiple of 8:
lStructSize dd 0 ; 0 - dword
pad1 dd 0 ; 4 - dword
hwndOwner dq 0 ; 8 - qword
hInstance dq 0 ; 16 - qword
lpstrFilter dq 0 ; 24 - qword
lpstrCustomFilter dq 0 ; 32 - qword
nMaxCustFilter dd 0 ; 40 - dword
nFilterIndex dd 0 ; 44 - dword
lpstrFile dq offset filename ; 48 - qword
nMaxFile dd 256 ; 56 - dword
pad2 dd 0 ; 60 - dword
lpstrFileTitle dq 0 ; 64 - qword
nMaxFileTitle dd 0 ; 72 - dword
pad3 dd 0 ; 76 - dword
lpstrInitialDir dq 0 ; 80 - qword
lpstrTitle dq 0 ; 88 - qword
Flags dd 0 ; 96 - dword
nFileOffset dw 0 ; 100 - word
nFileExtension dw 0 ; 102 - word
lpstrDefExt dq 0 ; 104 - qword
lCustData dq 0 ; 112 - qword
lpfnHook dq 0 ; 120 - qword
lpTemplateName dq 0 ; 128 - qword
pvReserved dq 0 ; 136 - qword
dwReserved dd 0 ; 144 - dword
FlagsEx dd 0 ; 148 - dword