C++ Inject into Java application

I'm having a really hard time figuring out why I cannot get my hooks to work in a java application..

I create a C++ WINAPI window. I load my "hook" dll and every time I send fake input to the application, it raises a flag perfectly fine.

However, if I do the same with a Java application, it does NOT work at all!

My dll looks like this:

#include <windows.h>

HINSTANCE hInstance = nullptr;
HHOOK MouseHook = {0};   /** Structure for hooking the mouse **/
HHOOK KeyboardHook = {0};  /** Structure for hooking the keyboard **/

LRESULT __stdcall MouseHookProc(int Code, WPARAM wParam, LPARAM lParam)
{
    if (Code == HC_ACTION)
    {
        MSLLHOOKSTRUCT* Info = reinterpret_cast<MSLLHOOKSTRUCT*>(lParam);
        if ((Info->flags & LLMHF_INJECTED) == LLMHF_INJECTED)
        {
            MessageBox(NULL, "DETECTED Mouse", "", 0);
        }
    }
    return CallNextHookEx(MouseHook, Code, wParam, lParam);
}

LRESULT __stdcall KeyboardHookProc(int Code, WPARAM wParam, LPARAM lParam)
{
    if (Code == HC_ACTION)
    {
        KBDLLHOOKSTRUCT* Info = reinterpret_cast<KBDLLHOOKSTRUCT*>(lParam);
        if ((Info->flags & LLKHF_INJECTED) == LLKHF_INJECTED)
        {
            MessageBox(NULL, "DETECTED Keyboard", "", 0);
        }
    }
    return CallNextHookEx(KeyboardHook, Code, wParam, lParam);
}

extern "C" __declspec(dllexport) bool __stdcall DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved)
{
    switch (fdwReason)
    {
        case DLL_PROCESS_ATTACH:
        {
            hInstance = hinstDLL;
            SetWindowsHookEx(WH_KEYBOARD_LL, KeyboardHookProc, hInstance, 0);
            SetWindowsHookEx(WH_MOUSE_LL, MouseHookProc, hInstance, 0);
        }
        break;

        case DLL_PROCESS_DETACH:
        {
            UnhookWindowsHookEx(KeyboardHook);
            UnhookWindowsHookEx(MouseHook);
        }
        break;
    }
    return true;
}

Then I created a C++ WINAPI application with the following WM_CREATE block:

case WM_CREATE:
    LoadLibrary("JavaHook.dll");
    mouse_event(MOUSEEVENTF_LEFTDOWN, 100, 100, 0, 0);
    mouse_event(MOUSEEVENTF_LEFTUP,100, 100,0,0);
break;

The MessageBox of the MouseHook pops up. This is expected behaviour.

However with Java I did:

public class HookTest {

    public static void main(String[] args) {
        JFrame frame = new JFrame("Test Window");
        frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
        frame.setVisible(true);

        System.loadLibrary("JavaHook");
    } 
}

Then I used SendInput on the application like so:

INPUT input;
input.type = INPUT_MOUSE;
input.mi.mouseData = 0;
input.mi.dx =  100 * (65536 / GetSystemMetrics(SM_CXSCREEN)); //x being coord in pixels
input.mi.dy =  100 * (65536 / GetSystemMetrics(SM_CYSCREEN)); //y being coord in pixels
input.mi.dwFlags = MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE;
SendInput(1, &input, sizeof(input));

and NOTHING pops up. I tried using mouse_event and keybd_event, still nothing. I tried using a Java mouse listener and keyboard listener and faking events.. Still NOTHING..

The hook loads for sure because I can see my mouse lag for a good 3 seconds before I can move it. Also made a popup to confirm it upon load.

However, I still cannot figure out why it doesn't work on the Java application. Any ideas how I can get them to work. Or is there another way to do it for Java preferably from C/C++? If not, is there a way to do it in Java?

You might want to read remarks to SetWindowsHookEx documentation.

Specifically it says that the thread that installed a global hook should continue to process Windows messages in order to allow both 32-bit and 64-bit applications to run the hook code in the context of the hooking application.

So, I suggest you create a separate thread from DllMain. The thread will install a hook and then run into GetMessage/ProcessMessage loop: http://pastebin.com/JmMvMX78

#include <windows.h>
#include <stdio.h>
 
HHOOK MouseHook = {0};
HHOOK KeyboardHook = {0};
 
LRESULT WINAPI MouseHookProc(int Code, WPARAM wParam, LPARAM lParam) {
    if (Code == HC_ACTION) {
        printf("Mouse\n");
    }
    return CallNextHookEx(MouseHook, Code, wParam, lParam);
}
 
LRESULT WINAPI KeyboardHookProc(int Code, WPARAM wParam, LPARAM lParam) {
    if (Code == HC_ACTION) {
        printf("Keyboard\n");
    }
    return CallNextHookEx(KeyboardHook, Code, wParam, lParam);
}
 
DWORD WINAPI ThreadProc(LPVOID param) {
    HINSTANCE hInstance = (HINSTANCE)param;
    MouseHook = SetWindowsHookEx(WH_MOUSE_LL, MouseHookProc, hInstance, 0);
    KeyboardHook = SetWindowsHookEx(WH_KEYBOARD_LL, KeyboardHookProc, hInstance, 0);
 
    MSG message;
    while (GetMessage(&message, (HWND)-1, 0, 0) > 0) {
        TranslateMessage(&message);
        DispatchMessage(&message);
    }
 
    UnhookWindowsHookEx(KeyboardHook);
    UnhookWindowsHookEx(MouseHook);
 
    return 0;
}
 
extern "C" __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) {
    if (fdwReason == DLL_PROCESS_ATTACH) {
        CloseHandle(CreateThread(NULL, 0, ThreadProc, hinstDLL, 0, NULL));
    }
    return TRUE;
}