OSX/CALayer: Revise CALayer 'RunOnMainThread' utilization, avoiding deadlocks

RunOnMainThread(waitUntilDone:=true,..) can deadlock the main-thread if called from AWT-EDT,
since the main-thread may call back to AWT-EDT while injecting a new main-thread task.

This patch revises all RunOnMainThread CALayer usage, resulting in only one required left:
  - OSXUtil.AddCASublayer() w/ waitUntilDone:=false

Hence the CALayer code has no more potential to deadlock main-thread/AWT-EDT.

OSXUtil.AddCASublayer() must be performed on main-thread, otherwise the
CALayer attachment will fail - no visible rendering result.

+++

Note: A good trigger to test this deadlock is to magnify/zoom
      the OSX desktop (click background + ctrl-mouse_wheel)
      before running some unit tests.

      TestGLCanvasAWTActionDeadlock01AWT and TestAddRemove02GLWindowNewtCanvasAWT
      also have the potential to trigger the mentioned deadlock.

1 files changed, 7 insertions, 9 deletions

diff --git a/src/nativewindow/native/macosx/OSXmisc.m b/src/nativewindow/native/macosx/OSXmisc.m
index c74d6cc58..14a9781f7 100644
--- a/src/nativewindow/native/macosx/OSXmisc.m
+++ b/src/nativewindow/native/macosx/OSXmisc.m
@@ -584,16 +584,16 @@ JNIEXPORT void JNICALL Java_jogamp_nativewindow_macosx_OSXUtil_DestroyCALayer0
 /*
  * Class:     Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow
  * Method:    SetJAWTRootSurfaceLayer0
- * Signature: (JJ)Z
+ * Signature: (JJ)V
  */
-JNIEXPORT jboolean JNICALL Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow_SetJAWTRootSurfaceLayer0
+JNIEXPORT void JNICALL Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow_SetJAWTRootSurfaceLayer0
   (JNIEnv *env, jclass unused, jobject jawtDrawingSurfaceInfoBuffer, jlong caLayer)
 {
     NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init];
     JAWT_DrawingSurfaceInfo* dsi = (JAWT_DrawingSurfaceInfo*) (*env)->GetDirectBufferAddress(env, jawtDrawingSurfaceInfoBuffer);
     if (NULL == dsi) {
         NativewindowCommon_throwNewRuntimeException(env, "Argument \"jawtDrawingSurfaceInfoBuffer\" was not a direct buffer");
-        return JNI_FALSE;
+        return;
     }
     MyCALayer* layer = (MyCALayer*) (intptr_t) caLayer;
     id <JAWT_SurfaceLayers> surfaceLayers = (id <JAWT_SurfaceLayers>)dsi->platformInfo;
@@ -601,35 +601,33 @@ JNIEXPORT jboolean JNICALL Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow
     surfaceLayers.layer = [layer retain]; // Pairs w/ Unset
     [pool release];
     DBG_PRINT("CALayer::SetJAWTRootSurfaceLayer.X: root %p (refcnt %d)\n", layer, (int)[layer retainCount]);
-    return JNI_TRUE;
 }
 
 /*
  * Class:     Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow
  * Method:    UnsetJAWTRootSurfaceLayer0
- * Signature: (JJ)Z
+ * Signature: (JJ)V
  */
-JNIEXPORT jboolean JNICALL Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow_UnsetJAWTRootSurfaceLayer0
+JNIEXPORT void JNICALL Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow_UnsetJAWTRootSurfaceLayer0
   (JNIEnv *env, jclass unused, jobject jawtDrawingSurfaceInfoBuffer, jlong caLayer)
 {
     NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init];
     JAWT_DrawingSurfaceInfo* dsi = (JAWT_DrawingSurfaceInfo*) (*env)->GetDirectBufferAddress(env, jawtDrawingSurfaceInfoBuffer);
     if (NULL == dsi) {
         NativewindowCommon_throwNewRuntimeException(env, "Argument \"jawtDrawingSurfaceInfoBuffer\" was not a direct buffer");
-        return JNI_FALSE;
+        return;
     }
     MyCALayer* layer = (MyCALayer*) (intptr_t) caLayer;
     id <JAWT_SurfaceLayers> surfaceLayers = (id <JAWT_SurfaceLayers>)dsi->platformInfo;
     if(layer != surfaceLayers.layer) {
         NativewindowCommon_throwNewRuntimeException(env, "Attached layer %p doesn't match given layer %p\n", surfaceLayers.layer, layer);
-        return JNI_FALSE;
+        return;
     }
     DBG_PRINT("CALayer::UnsetJAWTRootSurfaceLayer.0: root %p (refcnt %d) -> nil\n", layer, (int)[layer retainCount]);
     [layer release]; // Pairs w/ Set
     surfaceLayers.layer = NULL;
     [pool release];
     DBG_PRINT("CALayer::UnsetJAWTRootSurfaceLayer.X: root %p (refcnt %d) -> nil\n", layer, (int)[layer retainCount]);
-    return JNI_TRUE;
 }
 
 @interface MainRunnable : NSObject