c++winapistack-tracedetoursdbghelp

Why does SymInitialize() invoke CreateFile()?

Firstly, I want to hook CreateFile() and rewrite it. Then I want to recode the callstack of my new CreateFile() function. But when I use SymInitialize() to Initialize a handle, it falls into an endless loop. Through my debug, the reason is SymInitialize() invokes CreateFile(). So why does SymInitialize() involve CreateFile()? How to avoid this loop? Is there any alternative method to record callstack information to avoid this loop?

#include <windows.h>
#include <stdio.h>
#include "detours.h"
#include <fstream>
#include <io.h>   

#pragma comment(lib, "detours.lib")
#include <DbgHelp.h>                    //SymInitialize
#pragma comment(lib,"dbghelp.lib")
#define STACK_INFO_LEN  200

struct stackInfo {
    PDWORD hashValue; // hash value to identify same stack
    char* szBriefInfo; // callstack info
};

stackInfo ShowTraceStack(char* szBriefInfo)
{
    static const int MAX_STACK_FRAMES = 12;
    void* pStack[MAX_STACK_FRAMES];
    static char szStackInfo[STACK_INFO_LEN * MAX_STACK_FRAMES];
    static char szFrameInfo[STACK_INFO_LEN];

    HANDLE process = GetCurrentProcess(); // The handle used must be unique to avoid sharing a session with another component,
    SymInitialize(process, NULL, TRUE);
    PDWORD hashValue = (PDWORD)malloc(sizeof(DWORD)); // allow memory for hashVavlue, it will be rewrited in function CaptureStackBackTrace
    WORD frames = CaptureStackBackTrace(0, MAX_STACK_FRAMES, pStack, hashValue);
    //printf("hash value is: %ud \n", &hashValue);
    if (szBriefInfo == NULL) {
        strcpy_s(szStackInfo, "stack traceback:\n");
    }
    else {
        strcpy_s(szStackInfo, szBriefInfo);
    }

    for (WORD i = 0; i < frames; ++i) {
        DWORD64 address = (DWORD64)(pStack[i]);

        DWORD64 displacementSym = 0;
        char buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(TCHAR)];
        PSYMBOL_INFO pSymbol = (PSYMBOL_INFO)buffer;
        pSymbol->SizeOfStruct = sizeof(SYMBOL_INFO);
        pSymbol->MaxNameLen = MAX_SYM_NAME;

        DWORD displacementLine = 0;
        IMAGEHLP_LINE64 line;
        line.SizeOfStruct = sizeof(IMAGEHLP_LINE64);

        if (SymFromAddr(process, address, &displacementSym, pSymbol) &&
            SymGetLineFromAddr64(process, address, &displacementLine, &line))
        {
            _snprintf_s(szFrameInfo, sizeof(szFrameInfo), "\t%s() at %s:%d(0x%x)\n",
                pSymbol->Name, line.FileName, line.LineNumber, pSymbol->Address);
        }
        else
        {
            _snprintf_s(szFrameInfo, sizeof(szFrameInfo), "\terror: %d\n", GetLastError());
        }
        strcat_s(szStackInfo, szFrameInfo);
    }
    stackInfo traceStackInfo;
    traceStackInfo.hashValue = hashValue;
    traceStackInfo.szBriefInfo = szStackInfo;
    return traceStackInfo;
    //printf("%s", szStackInfo); 
}

HANDLE (*__stdcall oldCreateFile)(LPCWSTR,
    DWORD,
    DWORD,
    LPSECURITY_ATTRIBUTES,
    DWORD,
    DWORD,
    HANDLE) = CreateFileW;

HANDLE WINAPI newCreateFile(
    _In_ LPCWSTR lpFileName,
    _In_ DWORD dwDesiredAccess,
    _In_ DWORD dwShareMode,
    _In_opt_ LPSECURITY_ATTRIBUTES lpSecurityAttributes,
    _In_ DWORD dwCreationDisposition,
    _In_ DWORD dwFlagsAndAttributes,
    _In_opt_ HANDLE hTemplateFile
) {
    ShowTraceStack((char*)"trace information.\n");
    return oldCreateFile(
        L".\\newFiles.txt", // L".\\NewFile.txt",     // Filename
        //lpFileName,
        dwDesiredAccess,          // Desired access
        dwShareMode,        // Share mode
        lpSecurityAttributes,                   // Security attributes
        dwCreationDisposition,             // Creates a new file, only if it doesn't already exist
        dwFlagsAndAttributes,  // Flags and attributes
        NULL);
}

void hook() {
    DetourRestoreAfterWith();
    DetourTransactionBegin();
    DetourUpdateThread(GetCurrentThread());
    DetourAttach(&(PVOID&)oldCreateFile, newCreateFile);
    DetourTransactionCommit();
}

void unhook()
{
    DetourTransactionBegin();
    DetourUpdateThread(GetCurrentThread());
    DetourDetach(&(PVOID&)oldCreateFile, newCreateFile);
    DetourTransactionCommit();
}

void myProcess() {
    HANDLE hFile = CreateFile(TEXT(".\\CreateFileDemo.txt"),     
        GENERIC_WRITE | GENERIC_READ,          
        0,                    
        NULL,                  
        CREATE_ALWAYS,          
        FILE_ATTRIBUTE_NORMAL,         NULL);                 
    if (hFile == INVALID_HANDLE_VALUE)
    {
        OutputDebugString(TEXT("CreateFile fail!\r\n"));
    }
    // write to file 
    const int BUFSIZE = 4096;
    char chBuffer[BUFSIZE];
    memcpy(chBuffer, "Test", 4);
    DWORD dwWritenSize = 0;
    BOOL bRet = WriteFile(hFile, chBuffer, 4, &dwWritenSize, NULL);
    if (bRet) {
        OutputDebugString(TEXT("WriteFile success!\r\n"));
    }
}

int main(){
    hook();
    myProcess();
    unhook();
}
Solution

  • The main problem is the call to SymInitialize where you pass through "TRUE" for fInvadeProcess parameter. This is causing it to SymLoadModuleEx to be called for each loaded module. This will cause a lot of file access to download / create / open PDB files for each loaded module. This is the reason for your infinite loop.

    The "quick" fix for this sample is to move the call to SymInitialize into your main before the hook call as it only needs to be called once. This means all the PDB modules are loaded before the hooking / call to ShowTraceStack.

    The other problems are:

    If you are trying to make someone more useful I would: