c++windowsasynchronouswritefileoverlapped-io

Windows WriteFile blocks even with FILE_FLAG_OVERLAPPED


I have the following code that creates a file using CreateFile with the FILE_FLAG_OVERLAPPED flag, and then calls WriteFile 100 times in a loop, passing in an OVERLAPPED structure

uint64_t GetPreciseTickCount()
{
    FILETIME fileTime;
    GetSystemTimePreciseAsFileTime(&fileTime);
    ULARGE_INTEGER large;
    large.LowPart = fileTime.dwLowDateTime;
    large.HighPart = fileTime.dwHighDateTime;
    return large.QuadPart;
}

uint64_t g_blockedTime = 0, g_waitTime = 0;
int main()
{
    auto hFile = CreateFile(
        L"test.dat",
        GENERIC_WRITE,
        0,
        NULL,
        CREATE_ALWAYS,
        FILE_FLAG_OVERLAPPED | FILE_FLAG_NO_BUFFERING | FILE_FLAG_WRITE_THROUGH,
        NULL);

    if (hFile == INVALID_HANDLE_VALUE)
    {
        std::cout << "CreateFile failed with err " << GetLastError() << std::endl;
        return 1;
    }

    uint32_t bufferSize = 4*1024*1024;
    char* buffer = (char*)_aligned_malloc(bufferSize, 4096);
    const int loop = 100;
    LARGE_INTEGER endPosition;
    endPosition.QuadPart = bufferSize * loop;

    auto sfpRet = SetFilePointerEx(hFile, endPosition, nullptr, FILE_BEGIN);
    if (sfpRet == INVALID_SET_FILE_POINTER)
    {
        std::cout << "SetFilePointer failed with err " << GetLastError() << std::endl;
        return 1;
    }

    if (0 == SetEndOfFile(hFile))
    {
        std::cout << "SetEndOfFile failed with err " << GetLastError() << std::endl;
        return 1;
    }

    auto start = GetPreciseTickCount();
    OVERLAPPED overlapped;
    auto completionEvent = CreateEvent(nullptr, TRUE, FALSE, nullptr);
    for (int i = 0; i < loop; ++i)
    {
        overlapped.hEvent = completionEvent;
        overlapped.Offset = i * bufferSize;
        overlapped.OffsetHigh = 0;
        overlapped.Internal = 0;
        overlapped.InternalHigh = 0;

        auto writeFileStart = GetPreciseTickCount();
        auto err = WriteFile(
            hFile,
            buffer,
            bufferSize,
            nullptr,
            &overlapped);

        auto writeFileEnd = GetPreciseTickCount();
        g_blockedTime += (writeFileEnd - writeFileStart) / 10;
        if (err == FALSE)
        {
            auto lastErr = GetLastError();
            if (lastErr != ERROR_IO_PENDING)
            {
                std::cout << "WriteFile failed with err " << lastErr << std::endl;
                return 1;
            }

            auto waitErr = WaitForSingleObject(overlapped.hEvent, INFINITE);
            g_waitTime += (GetPreciseTickCount() - writeFileEnd) / 10;
            if (waitErr != 0)
            {
                std::cout << "WaitForSingleObject failed with err " << waitErr << std::endl;
                return 1;
            }
        }
    }

    auto end = GetPreciseTickCount();
    CloseHandle(hFile);
    std::cout << "Took " << (end - start) / 10 << " micros" << std::endl;
    std::cout << "Blocked time " << g_blockedTime << " micros" << std::endl;
    std::cout << "Wait time " << g_waitTime << " micros" << std::endl;
}

The prints the following output

Took 1749086 micros
Blocked time 1700085 micros
Wait time 48896 micros

Why does WriteFile block? (as is evidenced by g_blockedTime being significantly higher than g_waitTime). Is there any way I can force it to be non-blocking?

Update: I updated the code to use SetFilePointerEx and SetEndOfFile before the loop. Still seeing the same blocking problem.


Solution

  • The solution is to call SetFilePointerEx, SetEndOfFile, and SetFileValidData before the loop. Then subsequent calls to WriteFile within the loop become non-blocking.

    uint64_t GetPreciseTickCount()
    {
        FILETIME fileTime;
        GetSystemTimePreciseAsFileTime(&fileTime);
        ULARGE_INTEGER large;
        large.LowPart = fileTime.dwLowDateTime;
        large.HighPart = fileTime.dwHighDateTime;
        return large.QuadPart;
    }
    
    uint64_t g_blockedTime = 0, g_waitTime = 0;
    int main()
    {
        HANDLE hToken;
        auto openResult = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hToken);
        if (!openResult)
        {
            std::cout << "OpenProcessToken failed with err " << GetLastError() << std::endl;
            return 1;
        }
    
        TOKEN_PRIVILEGES tp;
        tp.PrivilegeCount = 1;
        tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
        auto lookupResult = LookupPrivilegeValue(NULL, SE_MANAGE_VOLUME_NAME, &tp.Privileges[0].Luid);
        if (!lookupResult)
        {
            std::cout << "LookupPrivilegeValue failed with err " << GetLastError() << std::endl;
            return 1;
        }
    
        auto adjustResult = AdjustTokenPrivileges(hToken, FALSE, &tp, 0, NULL, NULL);
        if (!adjustResult || GetLastError() != ERROR_SUCCESS)
        {
            std::cout << "AdjustTokenPrivileges failed with err " << GetLastError() << std::endl;
            return 1;
        }
    
        auto hFile = CreateFile(
            L"test.dat",
            GENERIC_WRITE,
            0,
            NULL,
            CREATE_ALWAYS,
            FILE_FLAG_OVERLAPPED | FILE_FLAG_NO_BUFFERING | FILE_FLAG_WRITE_THROUGH,
            NULL);
    
        if (hFile == INVALID_HANDLE_VALUE)
        {
            std::cout << "CreateFile failed with err " << GetLastError() << std::endl;
            return 1;
        }
    
        uint32_t bufferSize = 4*1024*1024;
        char* buffer = (char*)_aligned_malloc(bufferSize, 4096);
        const int loop = 100;
    
        auto start = GetPreciseTickCount();
    
        LARGE_INTEGER endPosition;
        endPosition.QuadPart = bufferSize * loop;
        auto setFileErr = SetFilePointerEx(hFile, endPosition, nullptr, FILE_BEGIN);
        if (setFileErr == INVALID_SET_FILE_POINTER)
        {
            std::cout << "SetFilePointer failed with err " << GetLastError() << std::endl;
            return 1;
        }
    
        if (!SetEndOfFile(hFile))
        {
            std::cout << "SetEndOfFile failed with err " << GetLastError() << std::endl;
            return 1;
        }
    
        if (!SetFileValidData(hFile, bufferSize * loop))
        {
            std::cout << "SetFileValidData failed with err " << GetLastError() << std::endl;
            return 1;
        }
    
        OVERLAPPED overlapped;
        auto completionEvent = CreateEvent(nullptr, TRUE, FALSE, nullptr);
    
        for (int i = 0; i < loop; ++i)
        {
            overlapped.hEvent = completionEvent;
            overlapped.Offset = i * bufferSize;
            overlapped.OffsetHigh = 0;
            overlapped.Internal = 0;
            overlapped.InternalHigh = 0;
    
            auto writeFileStart = GetPreciseTickCount();
            auto err = WriteFile(
                hFile,
                buffer,
                bufferSize,
                nullptr,
                &overlapped);
    
            auto writeFileEnd = GetPreciseTickCount();
            g_blockedTime += (writeFileEnd - writeFileStart) / 10;
            if (err == FALSE)
            {
                auto lastErr = GetLastError();
                if (lastErr != ERROR_IO_PENDING)
                {
                    std::cout << "WriteFile failed with err " << lastErr << std::endl;
                    return 1;
                }
    
                auto waitErr = WaitForSingleObject(overlapped.hEvent, INFINITE);
                g_waitTime += (GetPreciseTickCount() - writeFileEnd) / 10;
                if (waitErr != 0)
                {
                    std::cout << "WaitForSingleObject failed with err " << waitErr << std::endl;
                    return 1;
                }
            }
        }
    
        auto end = GetPreciseTickCount();
        CloseHandle(hFile);
        std::cout << "Took " << (end - start) / 10 << " micros" << std::endl;
        std::cout << "Blocked time " << g_blockedTime << " micros" << std::endl;
        std::cout << "Wait time " << g_waitTime << " micros" << std::endl;
    }
    

    This produces the following output

    Took 1508131 micros
    Blocked time 19719 micros
    Wait time 1481362 micros
    

    Also, check out this article.