c++.netwinapihook.net-8.0

Trying to hook MessageBox in C++ DLL and use them in C# application

I'm running on Windows 11 x64 23H2 with Visual Studio Enterprise 2022.

I am trying to hook MessageBox in C++ to change its Background color to #457B9D and change its foreground color to White. Also, I want to change button text and button border to White and background color to transparent.

After that, I will use this in my C# .NET 8.0 Windows Forms application, because I don't want to create a new form for MessageBox, because our project has grown so complex that fixing the code in each Forms is time consuming (I have a total of 90 forms and 75 user controls).

Here is my code:

framework.h

#pragma once

#define WIN32_LEAN_AND_MEAN             // Exclude rarely-used stuff from Windows headers
// Windows Header Files
#include <windows.h>
#include <stdlib.h>

pch.h

// pch.h: This is a precompiled header file.
// Files listed below are compiled only once, improving build performance for future builds.
// This also affects IntelliSense performance, including code completion and many code browsing features.
// However, files listed here are ALL re-compiled if any one of them is updated between builds.
// Do not add files here that you will be updating frequently as this negates the performance advantage.

#ifndef PCH_H
#define PCH_H

// add headers that you want to pre-compile here
#include "framework.h"

#endif //PCH_H

pch.cpp

// pch.cpp: source file corresponding to the pre-compiled header

#include "pch.h"

// When you are using pre-compiled headers, this source file is necessary for compilation to succeed.

dllmain.cpp

// dllmain.cpp : Defines the entry point for the DLL application.
#include "pch.h"

extern "C" __declspec(dllexport) void HookMessageBoxW();
extern "C" __declspec(dllexport) void UnhookMessageBoxW();

const COLORREF bgColor = RGB(69, 123, 157); // #457B9D
const COLORREF textColor = RGB(255, 255, 255); // White

HHOOK hHook = NULL;
WNDPROC oldButtonProc = NULL;

// Button subclass procedure
LRESULT CALLBACK ButtonSubclassProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam) {
    try {
        switch (uMsg) {
        case WM_PAINT: {
            PAINTSTRUCT ps;
            HDC hdc = BeginPaint(hWnd, &ps);

            RECT rect;
            GetClientRect(hWnd, &rect);

            // Create and use a brush for background color
            HBRUSH hBrush = CreateSolidBrush(bgColor);
            FillRect(hdc, &rect, hBrush);
            DeleteObject(hBrush); // Delete the brush to avoid resource leaks

            SetTextColor(hdc, textColor);
            SetBkMode(hdc, TRANSPARENT);

            // Draw the text on the button
            WCHAR text[512];
            GetWindowText(hWnd, text, 512);
            DrawText(hdc, text, -1, &rect, DT_CENTER | DT_VCENTER | DT_SINGLELINE);

            // Draw a white border around the button
            HPEN hPen = CreatePen(PS_SOLID, 2, textColor);
            HGDIOBJ oldPen = SelectObject(hdc, hPen);
            Rectangle(hdc, rect.left, rect.top, rect.right, rect.bottom);
            SelectObject(hdc, oldPen);
            DeleteObject(hPen); // Delete the pen to avoid resource leaks

            EndPaint(hWnd, &ps);
            return 0;
        }
        default:
            break;
        }
    }
    catch (...) {
        // Log the exception or handle it accordingly
        return CallWindowProc(oldButtonProc, hWnd, uMsg, wParam, lParam);
    }

    // Default processing for other messages
    return CallWindowProc(oldButtonProc, hWnd, uMsg, wParam, lParam);
}

// MessageBox subclass procedure
LRESULT CALLBACK MessageBoxSubclassProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam) {
    try {
        HDC hdcStatic;
        HBRUSH hBrush = CreateSolidBrush(bgColor);

        switch (uMsg) {
        case WM_CTLCOLORDLG:
        case WM_CTLCOLORSTATIC:
        case WM_CTLCOLORBTN:
            hdcStatic = (HDC)wParam;
            SetTextColor(hdcStatic, textColor);
            SetBkColor(hdcStatic, bgColor);
            DeleteObject(hBrush); // Make sure to delete the brush after use
            return (LRESULT)hBrush;
        case WM_INITDIALOG: {
            HWND hButton = GetDlgItem(hWnd, IDOK);
            if (hButton) {
                oldButtonProc = (WNDPROC)SetWindowLongPtr(hButton, GWLP_WNDPROC, (LONG_PTR)ButtonSubclassProc);
            }
            hButton = GetDlgItem(hWnd, IDCANCEL);
            if (hButton) {
                SetWindowLongPtr(hButton, GWLP_WNDPROC, (LONG_PTR)ButtonSubclassProc);
            }
            break;
        }
        case WM_DESTROY:
            SetWindowLongPtr(hWnd, GWLP_WNDPROC, (LONG_PTR)GetWindowLongPtr(hWnd, GWLP_USERDATA));
            SetWindowLongPtr(hWnd, GWLP_USERDATA, 0);
            break;
        default:
            break;
        }

        DeleteObject(hBrush); // Delete the brush before returning
    }
    catch (...) {
        // Handle any exceptions to prevent crashes
        return DefWindowProc(hWnd, uMsg, wParam, lParam);
    }

    // Default processing
    return CallWindowProc((WNDPROC)GetWindowLongPtr(hWnd, GWLP_USERDATA), hWnd, uMsg, wParam, lParam);
}

// Hook procedure to capture MessageBox creation
LRESULT CALLBACK CBTProc(int nCode, WPARAM wParam, LPARAM lParam) {
    try {
        if (nCode == HCBT_CREATEWND) {
            LPCREATESTRUCT lpcs = ((LPCBT_CREATEWND)lParam)->lpcs;

            // Check if it's a MessageBox
            if (lpcs->lpszClass && wcscmp(lpcs->lpszClass, L"#32770") == 0) {
                HWND hWnd = (HWND)wParam;
                SetWindowLongPtr(hWnd, GWLP_USERDATA, SetWindowLongPtr(hWnd, GWLP_WNDPROC, (LONG_PTR)MessageBoxSubclassProc));
            }
        }
    }
    catch (...) {
        // Handle the exception gracefully
        return CallNextHookEx(NULL, nCode, wParam, lParam);
    }

    return CallNextHookEx(hHook, nCode, wParam, lParam);
}

// Exported function to hook MessageBoxW
extern "C" __declspec(dllexport) void HookMessageBoxW() {
    hHook = SetWindowsHookEx(WH_CBT, CBTProc, nullptr, GetCurrentThreadId());
}

// Exported function to unhook MessageBoxW
extern "C" __declspec(dllexport) void UnhookMessageBoxW() {
    if (hHook) {
        UnhookWindowsHookEx(hHook);
        hHook = nullptr;
    }
}

Program.cs

using OSVersionExtension;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading.Tasks;
using System.Windows.Forms;

namespace Education
{
    internal static class Program
    {
        [DllImport("Win32.dll", CallingConvention = CallingConvention.Cdecl)]
        public static extern void HookMessageBoxW();

        [DllImport("Win32.dll", CallingConvention = CallingConvention.Cdecl)]
        public static extern void UnhookMessageBoxW();

        /// <summary>
        ///  The main entry point for the application.
        /// </summary>
        [STAThread]
        [System.Runtime.InteropServices.DllImport("user32.dll")]
        private static extern bool SetProcessDPIAware();
        static void Main()
        {
            HookMessageBoxW();
            
            ApplicationConfiguration.Initialize();
            //SetProcessDPIAware();

            // Check Operating System section
            // 
            // We only have rounded corners (actually is uDWM hack) on Windows 11
            // On earlier OS version, we need to apply our custom rounded corner
            // that defined in EllipseControl.cs
            //
            // To check Windows version, we use OSCheckExt from NuGet package
            // manager
            // 
            // So, we have these cases:
            //
            // Case 1: If users have Windows 11: Let's use native uDWM hack (inside
            //         dwmapi.dll) and opt in system rounded corners
            // 
            // Case 2: If users doesn't have Windows 11: We need to create 
            //         custom interface to enable rounded corners that defined
            //         in EllipseControl.cs then enable them in Form1.cs
            // 
            // Note that on Windows Server 2022, we still doesn't have uDWM hack,
            // actually uDWM hack exists only on Windows 11. So if we detected
            // Windows Server Edition, we have to use our custom rounded corners
            // defined in EllipseControl.cs to enable rounded corners effect
            //
            // 9/3/2024

            OSVersionExtension.OperatingSystem osFetchData = OSVersion.GetOperatingSystem();

            // Windows 11 detected
            if (osFetchData == OSVersionExtension.OperatingSystem.Windows11)
            {
                Application.Run(new Education_MainForm(true));
            }
            else
            {
                Application.Run(new Education_MainForm(false));
            }
        }
    }
}

When I tried to run in Debugging mode, none of useful:

Debug output in Visual Studio Enterprise 2022

I tried to figure out what is happening using System Informer, then I can see WerFault.exe is running because my application crashed:

System Informer check

When I tried to inspect what is happening, I get this:

System Informer inspect

It says:

An unhandled exception was encountered during a user callback.

But I can't find anything useful when debugging, that make me very confusing.

UPDATE:

After enabling debug native code, I can see the error is:

Exception thrown at 0x00007FFD2766F7FA (ucrtbased.dll) in Education.exe: 0xC0000005: Access violation reading location 0x0000000000008002.

at line:

if (lpcs->lpszClass && wcscmp(lpcs->lpszClass, L"#32770") == 0)
Solution

  • After 3 days of debugging (including trying all the methods I came up with: Bounds checking, null pointer checking, reference checking,...) but still stuck at Access Violation, I found a solution that seems more perfect than 1 hook. However, it will require a significant amount of manipulation to get it working, but I think it's worth it, especially when it works perfectly.

    The steps will be quite long, but I will try to be as detailed as possible. But in short, I used XMessageBox, a reverse-engineered implementation of the Windows MessageBox API that dates back to the Windows XP and Windows Vista era and still works today, luckily, Microsoft probably kept the MessageBox API intact to this day.

    I hope that help for the people who have the similar problem with me. If you still ask for the solution to hook MessageBox API: Sorry, you can't do it anymore

    Here are the details:

    1. First, we need XMessageBox. A quick Google search will bring up the results. Then, go to the official project page on CodeProject and download it. After downloading, unzip the file. After unzipping, we will get:

    enter image description here

    1. Go to the vs6\Release folder and launch XMsgBoxTest.exe. This will be where we start

    enter image description here

    1. Here, you can try different types of MessageBox. For my requirement, I need to change the Background Color and Text Color. Let's click on Gallery...

    enter image description here

    Click on Custom Icon + Custom Colors, this is what we need

    enter image description here

    This is what we need. Now, we need to figure out how to apply it.... Let's look at this box, this is the code that we need to be able to display the MessageBox as shown in the above sample

    enter image description here

    Now, we need to SAVE this code, for future use. Now, go back to Visual Studio, create a new C++ DLL MFC project:

    enter image description here

    Give it any name you like, but make sure it is in the same solution. After creation, we will get this:

    enter image description here

    Now, come back with XMessageBox folder, navigate to src directory. You should see this:

    enter image description here

    Now we need to grab neccessary files. There are these file is required:

    HEADER FILES

    SOURCE FILES

    Here are the files we need:

    enter image description here

    Copy them to our C++ MFC DLL Project source code directory, after that, we will get this:

    enter image description here

    Come back to Visual Studio, right click on Header Files in C++ MFC DLL Project, then select Add > Existing Item...

    enter image description here

    then select all of our HEADER FILES

    enter image description here

    After that, do the same thing with Source Files. After that, you will get this project structure like:

    enter image description here

    Now, just a thing that not important, go to Win32MFC.cpp (note that name file will change, depend on your project name), then comment all of the code inside them, then add #include "pch.h" on this file. This is very important:

    enter image description here

    Do the samething with header file:

    enter image description here

    Now come back to src directory of XMessageBox, open file StdAfx.h with your favourite editor, I will use notepad:

    enter image description here

    Copy all of codes to replace current contents of your framework.h:

    enter image description here

    Now, open your pch.h files. Then add these code after #include "framework.h":

    #include "XHyperLink.h"
#include "XMessageBox.h"
#include "XTrace.h"

    Your file should like this:

    enter image description here

    Now, open XMessageBox.h files, then find this:

    int XMessageBox(HWND hwnd,
    LPCTSTR lpszMessage,
    LPCTSTR lpszCaption = NULL,
    UINT nStyle = MB_OK | MB_ICONEXCLAMATION,
    XMSGBOXPARAMS* pXMB = NULL);

    This is the main function we need to call to display our custom MessageBox. We need to export them too. Just replace the function to:

    extern "C" __declspec(dllexport) int __cdecl XMessageBox(HWND hwnd,
    LPCTSTR lpszMessage,
    LPCTSTR lpszCaption = NULL,
    UINT nStyle = MB_OK | MB_ICONEXCLAMATION,
    XMSGBOXPARAMS* pXMB = NULL);

    Now go to XMessageBox.cpp, find XMessageBox(). Here is the thing you should see:

    ///////////////////////////////////////////////////////////////////////////////
//
// XMessageBox()
//
// The XMessageBox function creates, displays, and operates a message box.
// The message box contains an application-defined message and title, plus
// any combination of predefined icons, push buttons, and checkboxes.
//
// For more information see
//     http://www.codeproject.com/KB/dialog/xmessagebox.aspx
//
// int XMessageBox(HWND hwnd,                 // handle of owner window
//                 LPCTSTR lpszMessage,       // address of text in message box
//                 LPCTSTR lpszCaption,       // address of title of message box
//                 UINT nStyle,               // style of message box
//                 XMSGBOXPARAMS * pXMB)      // optional parameters
//
// PARAMETERS
//
//     hwnd              - Identifies the owner window of the message box to be
//                         created. If this parameter is NULL, the message box
//                         has no owner window.
//
//     lpszMessage       - Pointer to a null-terminated string containing the
//                         message to be displayed.
//
//     lpszCaption       - Pointer to a null-terminated string used for the
//                         dialog box title. If this parameter is NULL, the
//                         default title Error is used.
//
//     nStyle            - Specifies a set of bit flags that determine the
//                         contents and  behavior of the dialog box. This
//                         parameter can be a combination of flags from the
//                         following groups of flags.
//
//     pXMB              - Pointer to optional parameters.  The parameters
//                         struct XMSGBOXPARAMS is defined in XMessageBox.h.
//
///////////////////////////////////////////////////////////////////////////////

int XMessageBox(HWND hwnd,
    LPCTSTR lpszMessage,
    LPCTSTR lpszCaption /*= NULL*/,
    UINT nStyle /*= MB_OK | MB_ICONEXCLAMATION*/,
    XMSGBOXPARAMS* pXMB /*= NULL*/)
{
...
}

    Replace this function to:

    extern "C" __declspec(dllexport) int __cdecl XMessageBox(HWND hwnd,
    LPCTSTR lpszMessage,
    LPCTSTR lpszCaption /*= NULL*/,
    UINT nStyle /*= MB_OK | MB_ICONEXCLAMATION*/,
    XMSGBOXPARAMS* pXMB /*= NULL*/)
{
...
}

    Then, replace XHyperLink.cpp, XMessageBox.cpp line #include "StdAfx.h" or #include "stdafx.h" to #include "pch.h":

    enter image description here

    enter image description here

    Now press Ctrl + Shift + B to build project. After that, you will get this in <Your Solution Directory>\x64\Release (Note that x64 and Release will depend on your build configuration):

    enter image description here

    You will get your DLL like:

    enter image description here

    Now let's inspect to the DLL to see the function was exported. I will use PE Viewer bundled with System Informer, you can use another tool, such as CFF Explorer. Open your DLL inside your inspector, then you will get this:

    enter image description here

    Click on Exports tab:

    enter image description here

    and you will see something like:

    enter image description here

    If not, please check if you have __declspec(dllexport) and extern "C". Note that extern "C" is required to tell to the computer that it should not be decorated the function. If not, you will see the Undecorated name have value. In that case, make sure you have already put extern "C" before __declspec(dllexport). Rebuild your project again after made changes.

    After you can sure that your DLL function was exported correctly, back to Visual Studio, then in the same solution, create a new C++/CLI DLL Project.

    If you don't know what C++/CLI DLL Project is:

    wikipedia

    C++/CLI is a variant of the C++ programming language, modified for Common Language Infrastructure. It has been part of Visual Studio 2005 and later, and provides interoperability with other .NET languages such as C#. Microsoft created C++/CLI to supersede Managed Extensions for C++. In December 2005, Ecma International published C++/CLI specifications as the ECMA-372 standard.

    Now, in Visual Studio, create an C++/CLI DLL Project, give them with any name you like, then press Create:

    enter image description here

    Depend on your project was written on .NET or .NET Framework, you must choice it correct. My application written on .NET, so I will choice .NET

    Now, look back to XMessageBox() declaration:

    extern "C" __declspec(dllexport) int __cdecl XMessageBox(HWND hwnd,
    LPCTSTR lpszMessage,
    LPCTSTR lpszCaption = NULL,
    UINT nStyle = MB_OK | MB_ICONEXCLAMATION,
    XMSGBOXPARAMS* pXMB = NULL);

    We can see parameters required XMSGBOXPARAMS. So where is XMSGBOXPARAMS? They defined in the same file XMessageBox.h:

    struct XMSGBOXPARAMS
{
    XMSGBOXPARAMS()
    {
        nTimeoutSeconds = 0;
        nDisabledSeconds = 0;
        hInstanceIcon = NULL;
        hInstanceStrings = NULL;
        lpReportFunc = NULL;
        dwReportUserData = 0;
        nIdHelp = 0;
        nIdIcon = 0;
        nIdCustomButtons = 0;
        nIdReportButtonCaption = 0;
        x = 0;
        y = 0;
        dwOptions = 0;
        lpszModule = NULL;
        nLine = 0;
        bUseUserDefinedButtonCaptions = FALSE;          //+++1.5
        crText = CLR_INVALID;   //+++1.8
        crBackground = CLR_INVALID; //+++1.8

        memset(szIcon, 0, sizeof(szIcon));
        memset(szCustomButtons, 0, sizeof(szCustomButtons));
        memset(szReportButtonCaption, 0, sizeof(szReportButtonCaption));
        memset(szCompanyName, 0, sizeof(szCompanyName));
        memset(&UserDefinedButtonCaptions, 0, sizeof(UserDefinedButtonCaptions));   //+++1.5
    }

    UINT        nIdHelp;                        // help context ID for message;
    // 0 indicates the application’s 
    // default Help context will 
    // be used
    int         nTimeoutSeconds;                // number of seconds before the
    // default button will be selected
    int         nDisabledSeconds;               // number of seconds that all the 
    // buttons will be disabled - after
    // nDisabledSeconds, all buttons
    // will be enabled
    int         x, y;                           // initial x,y screen coordinates
    enum                                        // these are bit flags for dwOptions
    {
        None = 0x0000,
        RightJustifyButtons = 0x0001,           // causes buttons to be right-justified
        VistaStyle = 0x0002,            // setting this option bit will cause the 
        // message background to be painted with 
        // the current window color (typically 
        // white), and the buttons to be 
        // right-justified.    +++1.8
        Narrow = 0x0004         // uses a narrow width for message box -
        // SM_CXSCREEN / 3

    };
    DWORD       dwOptions;                      // options flags
    HINSTANCE   hInstanceStrings;               // if specified, will be used to
    // load strings
    HINSTANCE   hInstanceIcon;                  // if specified, will be used to
    // load custom icon
    UINT        nIdIcon;                        // custom icon resource id
    TCHAR       szIcon[MAX_PATH];               // custom icon name
    UINT        nIdCustomButtons;               // custom buttons resource id
    TCHAR       szCustomButtons[MAX_PATH];      // custom buttons string
    UINT        nIdReportButtonCaption;         // report button resource id
    TCHAR       szReportButtonCaption[MAX_PATH];// report button string
    TCHAR       szCompanyName[MAX_PATH];        // used when saving checkbox state in registry
    LPCTSTR     lpszModule;                     // module name (for saving DoNotAsk state)
    int         nLine;                          // line number (for saving DoNotAsk state)
    DWORD       dwReportUserData;               // data sent to report callback function
    XMESSAGEBOX_REPORT_FUNCTION lpReportFunc;   // report function
    COLORREF    crText;                         // message text color       +++1.8
    COLORREF    crBackground;                   // message background color +++1.8

    //-[UK
    // For not loading from resource but passing directly,
    // Use the following code.
    struct CUserDefinedButtonCaptions
    {
        TCHAR   szAbort[MAX_PATH];
        TCHAR   szCancel[MAX_PATH];
        TCHAR   szContinue[MAX_PATH];
        TCHAR   szDoNotAskAgain[MAX_PATH];
        TCHAR   szDoNotTellAgain[MAX_PATH];
        TCHAR   szDoNotShowAgain[MAX_PATH];
        TCHAR   szHelp[MAX_PATH];
        TCHAR   szIgnore[MAX_PATH];
        TCHAR   szIgnoreAll[MAX_PATH];
        TCHAR   szNo[MAX_PATH];
        TCHAR   szNoToAll[MAX_PATH];
        TCHAR   szOK[MAX_PATH];
        TCHAR   szReport[MAX_PATH];
        TCHAR   szRetry[MAX_PATH];
        TCHAR   szSkip[MAX_PATH];
        TCHAR   szSkipAll[MAX_PATH];
        TCHAR   szTryAgain[MAX_PATH];
        TCHAR   szYes[MAX_PATH];
        TCHAR   szYesToAll[MAX_PATH];
    };
    BOOL                        bUseUserDefinedButtonCaptions;  //+++1.5
    CUserDefinedButtonCaptions  UserDefinedButtonCaptions;      //+++1.5
    //-]UK
};

    So we must need XMSGBOXPARAMS structure to make our DLL Wrapper working. The easy way is just copy XMessageBox.h to our project directory, add to Header Files section and comment out the function we don't need to use in our wrapper and also prevent the errors.

    Do the same thing to copy XMessageBox.h to project and add to Header files, here is the final result:

    enter image description here

    Now go to XMessageBox.h in your wrapper project, then comment this section:

    int XMessageBox(HWND hwnd,
    LPCTSTR lpszMessage,
    LPCTSTR lpszCaption = NULL,
    UINT nStyle = MB_OK | MB_ICONEXCLAMATION,
    XMSGBOXPARAMS* pXMB = NULL);


DWORD XMessageBoxGetCheckBox(LPCTSTR lpszCompanyName, LPCTSTR lpszModule, int nLine);

DWORD XMessageBoxGetCheckBox(XMSGBOXPARAMS& xmb);

    Your file now should like:

    enter image description here

    Now create new C++ file in Source Files section. Given it any name you like. Then use this code:

    #pragma once

#include "XMessageBox.h"
#using <System.dll>
#using <System.Drawing.dll>

using namespace System;
using namespace System::Runtime::InteropServices;
using namespace System::Drawing;

namespace XMessageBoxWrapper {
    public ref class XMessageBoxParams {
    private:
        XMSGBOXPARAMS* m_pNativeParams;
    public:
        XMessageBoxParams() {
            m_pNativeParams = new XMSGBOXPARAMS();
        }

        ~XMessageBoxParams() {
            this->!XMessageBoxParams();
        }

        !XMessageBoxParams() {
            if (m_pNativeParams != nullptr) {
                delete m_pNativeParams;
                m_pNativeParams = NULL;
            }
        }

        // Properties
        property UInt32 IdHelp {
            UInt32 get() {
                return m_pNativeParams->nIdHelp;
            }
            void set(UInt32 value) {
                m_pNativeParams->nIdHelp = value;
            }
        }

        property Int32 TimeoutSeconds {
            Int32 get() {
                return m_pNativeParams->nTimeoutSeconds;
            }
            void set(Int32 value) {
                m_pNativeParams->nTimeoutSeconds = value;
            }
        }

        property Int32 DisableSeconds {
            Int32 get() {
                return m_pNativeParams->nDisabledSeconds;
            }
            void set(Int32 value) {
                m_pNativeParams->nDisabledSeconds = value;
            }
        }

        property System::Drawing::Color TextColor {
            System::Drawing::Color get() {
                return System::Drawing::Color::FromArgb(
                    GetRValue(m_pNativeParams->crText),
                    GetGValue(m_pNativeParams->crText),
                    GetBValue(m_pNativeParams->crText)
                );
            }
            void set(System::Drawing::Color value) {
                m_pNativeParams->crText = RGB(value.R, value.G, value.B);
            }
        }

        property System::Drawing::Color BackgroundColor {
            System::Drawing::Color get() {
                return System::Drawing::Color::FromArgb(
                    GetRValue(m_pNativeParams->crBackground),
                    GetGValue(m_pNativeParams->crBackground),
                    GetBValue(m_pNativeParams->crBackground)
                );
            }
            void set(System::Drawing::Color value) {
                m_pNativeParams->crBackground = RGB(value.R, value.G, value.B);
            }
        }

        XMSGBOXPARAMS* GetNativePointer() {
            return m_pNativeParams;
        }
    };
    
    public ref class XMessageBoxWrapper {
    public:
        [DllImport("Win32MFC.dll", CharSet = CharSet::Unicode, CallingConvention = CallingConvention::Cdecl, ExactSpelling = true)]
        static int XMessageBox(IntPtr hWnd, 
            [MarshalAs(UnmanagedType::LPWStr)] String^ lpszMessage, 
            [MarshalAs(UnmanagedType::LPWStr)] String^ lpszCaption, 
            unsigned int nStyle, 
            IntPtr pXMB);

        // Overloads
        static int XMessageBox(IntPtr hWnd, String^ lpszMessage) {
            return XMessageBox(hWnd, lpszMessage, nullptr, 0x00000030, IntPtr::Zero);
        }

        static int XMessageBox(IntPtr hWnd, String^ lpszMessage, String^ lpszCaption) {
            return XMessageBox(hWnd, lpszMessage, lpszCaption, 0x00000030, IntPtr::Zero);
        }

        static int XMessageBox(IntPtr hWnd, String^ lpszMessage, String^ lpszCaption, unsigned int nStyle) {
            return XMessageBox(hWnd, lpszMessage, lpszCaption, nStyle, IntPtr::Zero);
        }

        static int XMessageBox(IntPtr hWnd, String^ lpszMessage, String^ lpszCaption, unsigned int nStyle, XMessageBoxParams^ params) {
            if (params == nullptr) {
                return XMessageBox(hWnd, lpszMessage, lpszCaption, nStyle, IntPtr::Zero);
            }
            else {
                return XMessageBox(hWnd, lpszMessage, lpszCaption, nStyle, IntPtr(params->GetNativePointer()));
            }
        }
    };
}

    Now, build your project. Now you should see two DLLs, like this:

    enter image description here

    If you wonder what ijwhost.dll is:

    github issue

    To support C++/CLI libraries in .NET Core, ijwhost was created as a shim for finding and loading the runtime. All C++/CLI libraries are linked to this shim, such that ijwhost.dll is found/loaded when the C++/CLI library is loaded.

    Now we are almost done! Come back to your C# project, right click to your project (It has C# logo), then select Add > Project reference...

    enter image description here

    and you will see your C++/CLI DLL Wrapper:

    enter image description here

    Check it, then select OK to add reference.

    Now, we can test our program:

    Program.cs

    using OSVersionExtension;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading.Tasks;
using System.Windows.Forms;
using XMessageBoxWrapper;

namespace Education
{
    internal static class Program
    {
        /// <summary>
        ///  The main entry point for the application.
        /// </summary>
        [STAThread]
        [System.Runtime.InteropServices.DllImport("user32.dll")]
        private static extern bool SetProcessDPIAware();
        static void Main()
        {
            // With custom parameters
            XMessageBoxParams xParams = new XMessageBoxParams();
            xParams.IdHelp = 12345;
            xParams.TimeoutSeconds = 10;
            xParams.DisableSeconds = 5;
            xParams.TextColor = Color.Red;
            xParams.BackgroundColor = Color.Blue;

            XMessageBoxWrapper.XMessageBoxWrapper.XMessageBox(IntPtr.Zero, "Hello, world!", "Custom Caption", 0x00000030, xParams);

            ApplicationConfiguration.Initialize();
            //HookMessageBoxW();
            //SetProcessDPIAware();

            // Check Operating System section
            // 
            // We only have rounded corners (actually is uDWM hack) on Windows 11
            // On earlier OS version, we need to apply our custom rounded corner
            // that defined in EllipseControl.cs
            //
            // To check Windows version, we use OSCheckExt from NuGet package
            // manager
            // 
            // So, we have these cases:
            //
            // Case 1: If users have Windows 11: Let's use native uDWM hack (inside
            //         dwmapi.dll) and opt in system rounded corners
            // 
            // Case 2: If users doesn't have Windows 11: We need to create 
            //         custom interface to enable rounded corners that defined
            //         in EllipseControl.cs then enable them in Form1.cs
            // 
            // Note that on Windows Server 2022, we still doesn't have uDWM hack,
            // actually uDWM hack exists only on Windows 11. So if we detected
            // Windows Server Edition, we have to use our custom rounded corners
            // defined in EllipseControl.cs to enable rounded corners effect
            //
            // 9/3/2024

            OSVersionExtension.OperatingSystem osFetchData = OSVersion.GetOperatingSystem();

            // Windows 11 detected
            if (osFetchData == OSVersionExtension.OperatingSystem.Windows11)
            {
                Application.Run(new Education_MainForm(true));
            }
            else
            {
                Application.Run(new Education_MainForm(false));
            }
        }
    }
}

    Build your project, but don't run your application

    Copy Win32MFC.dll to your directory that contains your C# application in EXE file. After that, your directory will look like:

    enter image description here

    Here is the result you should see:

    enter image description here

    Good luck!