I managed to manipulate a JavaFX window using Java FFM and MacOS native NSWindow object. However, when setting the titlebarAppearsTransparent to true and adding the NSWindowStyleMaskFullSizeContentView the window is only movable when you manage to drag it at the very upper border. Even though the title bar at this point is not visible, I'd like to be able to drag the window using an "imaginary" toolbar...
So how do I get the window to recognize the title bar as a movable rectangle even though it is transparent?
So far this is what I've done to reach this. Code is very redundant, but currently I'm just fooling around trying to proofing a concept...
private long getNSWindowHandle() throws Throwable {
// Get class NSApplication
try (Arena arena = Arena.ofConfined()) {
long nsApplication = (long) objc_getClass.invoke(arena.allocateFrom("NSApplication"));
System.out.println("nsApplication: " + Long.toHexString(nsApplication));
// Get selector for sharedApplication
long sharedAppSelector = (long) sel_registerName.invoke(arena.allocateFrom("sharedApplication"));
System.out.println("sharedAppSelector: " + Long.toHexString(sharedAppSelector));
// Get NSApplication shared instance
long sharedApplication = (long) LINKER.downcallHandle(
symbolLookup.find("objc_msgSend").orElseThrow(),
FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG))
.invoke(nsApplication, sharedAppSelector);
System.out.println("sharedApplication: " + Long.toHexString(sharedApplication));
// Get selector for windows array
long windowsSelector = (long) sel_registerName.invoke(arena.allocateFrom("windows"));
System.out.println("windowsSelector: " + Long.toHexString(windowsSelector));
// Get array of windows (NSArray of NSWindow objects)
long windowsArray = (long) LINKER.downcallHandle(
symbolLookup.find("objc_msgSend").orElseThrow(),
FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG))
.invoke(sharedApplication, windowsSelector);
System.out.println("windowsArray: " + Long.toHexString(windowsArray));
// Get selector for objectAtIndex:
long objectAtIndexSelector = (long) sel_registerName.invoke(arena.allocateFrom("objectAtIndex:"));
System.out.println("objectAtIndexSelector: " + Long.toHexString(objectAtIndexSelector));
// Get first window (NSWindow) in the array (index 0)
long nsWindow = (long) LINKER.downcallHandle(
symbolLookup.find("objc_msgSend").orElseThrow(),
FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG))
.invoke(windowsArray, objectAtIndexSelector, (long) 0);
System.out.println("nsWindow: " + Long.toHexString(nsWindow));
return nsWindow;
}
}
private void setCustomWindow(long nsWindow) throws Throwable {
try (Arena arena = Arena.ofConfined()) {
// TRANSPARENT TITLE BAR
var titleBarTransparentSelector = (long) sel_registerName.invoke(arena.allocateFrom("setTitlebarAppearsTransparent:"));
LINKER.downcallHandle(
symbolLookup.find("objc_msgSend").orElseThrow(),
FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_BOOLEAN))
.invoke(nsWindow, titleBarTransparentSelector, true);
// UPDATE STYLE MASK
long styleMaskSelector = (long) sel_registerName.invoke(arena.allocateFrom("styleMask"));
long setStyleMaskSelector = (long) sel_registerName.invoke(arena.allocateFrom("setStyleMask:"));
long styleMask = (long) LINKER.downcallHandle(
symbolLookup.find("objc_msgSend").orElseThrow(),
FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG))
.invoke(nsWindow, styleMaskSelector);
styleMask &= ~NSWindowStyleMaskUnifiedTitleAndToolbar;
styleMask |= NSWindowStyleMaskFullSizeContentView;
styleMask |= NSWindowStyleMaskTitled;
LINKER.downcallHandle(symbolLookup.find("objc_msgSend")
.orElseThrow(), FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG))
.invoke(nsWindow, setStyleMaskSelector, styleMask);
// TITLE SEPARATOR
int titleSeparator = 1;
long setTitlebarSeparator = (long) sel_registerName.invoke(arena.allocateFrom("setTitlebarSeparatorStyle:"));
LINKER.downcallHandle(symbolLookup.find("objc_msgSend")
.orElseThrow(), FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_INT))
.invoke(nsWindow, setTitlebarSeparator, titleSeparator);
// TITLE VISIBILITY (0 = visible, 1 = hidden)
int titleVisible = 0;
long setTitleVisibility = (long) sel_registerName.invoke(arena.allocateFrom("setTitleVisibility:"));
LINKER.downcallHandle(symbolLookup.find("objc_msgSend")
.orElseThrow(), FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_INT))
.invoke(nsWindow, setTitleVisibility, titleVisible);
// MOVABLE
var setMovableSelector = (long) sel_registerName.invoke(arena.allocateFrom("setMovable:"));
LINKER.downcallHandle(symbolLookup.find("objc_msgSend")
.orElseThrow(), FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_BOOLEAN))
.invoke(nsWindow, setMovableSelector, true);
// MOVABLE BY BACKGROUND
var isMovableByBackgroundSelector = (long) sel_registerName.invoke(arena.allocateFrom("setMovableByWindowBackground:"));
LINKER.downcallHandle(symbolLookup.find("objc_msgSend")
.orElseThrow(), FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_BOOLEAN))
.invoke(nsWindow, isMovableByBackgroundSelector, true);
}
}
I did it!
After digging around in memory dumps and everything I was confident, that the problem does not lie within my configuration nor within Apples Objective-C libraries (duh), I decided to take a look at the JFX library to see how they initially create a window. After fuzzing around with a lot of stuff I discovered the is section in GlassView3D.m:
- (BOOL)mouseDownCanMoveWindow
{
return NO;
}
Changing that to a YES did the trick. Window is now draggable at the "imaginary" titlebar 🥳
As of yet I need to figure out the exact way this GlassView3D is implemented but I am happy for now that I got it to work.. Maybe this helps someone...
Goes without saying, that you should do this at your own risk in a production environment.