OSX/CALayer: Revise CALayer 'RunOnMainThread' utilization, avoiding deadlocks

RunOnMainThread(waitUntilDone:=true,..) can deadlock the main-thread if called from AWT-EDT,
since the main-thread may call back to AWT-EDT while injecting a new main-thread task.

This patch revises all RunOnMainThread CALayer usage, resulting in only one required left:
  - OSXUtil.AddCASublayer() w/ waitUntilDone:=false

Hence the CALayer code has no more potential to deadlock main-thread/AWT-EDT.

OSXUtil.AddCASublayer() must be performed on main-thread, otherwise the
CALayer attachment will fail - no visible rendering result.

+++

Note: A good trigger to test this deadlock is to magnify/zoom
      the OSX desktop (click background + ctrl-mouse_wheel)
      before running some unit tests.

      TestGLCanvasAWTActionDeadlock01AWT and TestAddRemove02GLWindowNewtCanvasAWT
      also have the potential to trigger the mentioned deadlock.

4 files changed, 41 insertions, 54 deletions

diff --git a/src/nativewindow/classes/javax/media/nativewindow/OffscreenLayerSurface.java b/src/nativewindow/classes/javax/media/nativewindow/OffscreenLayerSurface.java
index 4885d5a4c..df3f04f7f 100644
--- a/src/nativewindow/classes/javax/media/nativewindow/OffscreenLayerSurface.java
+++ b/src/nativewindow/classes/javax/media/nativewindow/OffscreenLayerSurface.java
@@ -36,10 +36,6 @@ public interface OffscreenLayerSurface {
    * <p>
    * Implementation may realize all required resources at this point.
    * </p>
-   * <p>
-   * It is mandatory that any related resources, e.g. a shared context,
-   * are not locked while calling this method. 
-   * </p>
    * 
    * @see #isOffscreenLayerSurfaceEnabled()
    * @throws NativeWindowException if {@link #isOffscreenLayerSurfaceEnabled()} == false
diff --git a/src/nativewindow/classes/jogamp/nativewindow/jawt/macosx/MacOSXJAWTWindow.java b/src/nativewindow/classes/jogamp/nativewindow/jawt/macosx/MacOSXJAWTWindow.java
index 3ec54ca78..c8f758165 100644
--- a/src/nativewindow/classes/jogamp/nativewindow/jawt/macosx/MacOSXJAWTWindow.java
+++ b/src/nativewindow/classes/jogamp/nativewindow/jawt/macosx/MacOSXJAWTWindow.java
@@ -51,7 +51,6 @@ import javax.media.nativewindow.NativeWindowException;
 import javax.media.nativewindow.MutableSurface;
 import javax.media.nativewindow.util.Point;
 
-import com.jogamp.common.util.Function;
 import com.jogamp.nativewindow.awt.JAWTWindow;
 
 import jogamp.nativewindow.jawt.JAWT;
@@ -90,8 +89,11 @@ public class MacOSXJAWTWindow extends JAWTWindow implements MutableSurface {
                           JAWT_DrawingSurfaceInfo dsi = null;
                           try {
                               dsi = ds.GetDrawingSurfaceInfo();
-                              if(! UnsetJAWTRootSurfaceLayer(dsi.getBuffer(), rootSurfaceLayerHandle)) {
+                              try {
+                                  UnsetJAWTRootSurfaceLayer(dsi.getBuffer(), rootSurfaceLayerHandle);
+                              } catch (Exception e) {                                  
                                   System.err.println("Error clearing JAWT rootSurfaceLayerHandle "+toHexString(rootSurfaceLayerHandle));
+                                  e.printStackTrace();
                               }
                           } finally {
                               if ( null != dsi ) {
@@ -245,8 +247,12 @@ public class MacOSXJAWTWindow extends JAWTWindow implements MutableSurface {
                 rootSurfaceLayerHandle = OSXUtil.CreateCALayer(bounds.getX(), bounds.getY(), bounds.getWidth(), bounds.getHeight());
                 if(0 == rootSurfaceLayerHandle) {
                   errMsg = "Could not create root CALayer";                
-                } else if(!SetJAWTRootSurfaceLayer(dsi.getBuffer(), rootSurfaceLayerHandle)) {
-                  errMsg = "Could not set JAWT rootSurfaceLayerHandle "+toHexString(rootSurfaceLayerHandle);
+                } else {
+                    try {
+                        SetJAWTRootSurfaceLayer(dsi.getBuffer(), rootSurfaceLayerHandle);
+                    } catch(Exception e) {
+                        errMsg = "Could not set JAWT rootSurfaceLayerHandle "+toHexString(rootSurfaceLayerHandle)+", cause: "+e.getMessage();
+                    }
                 }
             }
         }
@@ -307,22 +313,22 @@ public class MacOSXJAWTWindow extends JAWTWindow implements MutableSurface {
   }  
   protected Point getLocationOnScreenNativeImpl(final int x0, final int y0) { return null; }
   
-  private static boolean SetJAWTRootSurfaceLayer(final Buffer jawtDrawingSurfaceInfoBuffer, final long caLayer) {
-      return OSXUtil.RunOnMainThread(true, new Function<Boolean, Object>() {
-       public Boolean eval(Object... args) {
-           return Boolean.valueOf( SetJAWTRootSurfaceLayer0(jawtDrawingSurfaceInfoBuffer, caLayer) );
-       } } ).booleanValue();
+  /** 
+   * Set the given root CALayer in the JAWT surface
+   */  
+  private static void SetJAWTRootSurfaceLayer(final Buffer jawtDrawingSurfaceInfoBuffer, final long caLayer) {
+      SetJAWTRootSurfaceLayer0(jawtDrawingSurfaceInfoBuffer, caLayer);
   }
 
-  private static boolean UnsetJAWTRootSurfaceLayer(final Buffer jawtDrawingSurfaceInfoBuffer, final long caLayer) {
-      return OSXUtil.RunOnMainThread(true, new Function<Boolean, Object>() {
-       public Boolean eval(Object... args) {
-           return Boolean.valueOf( UnsetJAWTRootSurfaceLayer0(jawtDrawingSurfaceInfoBuffer, caLayer) );
-       } } ).booleanValue();      
+  /** 
+   * Unset the given root CALayer in the JAWT surface
+   */  
+  private static void UnsetJAWTRootSurfaceLayer(final Buffer jawtDrawingSurfaceInfoBuffer, final long caLayer) {
+      UnsetJAWTRootSurfaceLayer0(jawtDrawingSurfaceInfoBuffer, caLayer);
   }
 
-  private static native boolean SetJAWTRootSurfaceLayer0(Buffer jawtDrawingSurfaceInfoBuffer, long caLayer);
-  private static native boolean UnsetJAWTRootSurfaceLayer0(Buffer jawtDrawingSurfaceInfoBuffer, long caLayer);
+  private static native void SetJAWTRootSurfaceLayer0(Buffer jawtDrawingSurfaceInfoBuffer, long caLayer);
+  private static native void UnsetJAWTRootSurfaceLayer0(Buffer jawtDrawingSurfaceInfoBuffer, long caLayer);
   
   // Variables for lockSurface/unlockSurface
   private JAWT_DrawingSurface ds;
diff --git a/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java b/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java
index aa44e2d64..59b42c249 100644
--- a/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java
+++ b/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java
@@ -137,26 +137,23 @@ public class OSXUtil implements ToolkitProperties {
     }
     
     /** 
-     * Create a CALayer suitable to act as a root CALayer on the main-thread.
+     * Create a CALayer suitable to act as a root CALayer.
      * @see #DestroyCALayer(long)
      * @see #AddCASublayer(long, long) 
      */
     public static long CreateCALayer(final int x, final int y, final int width, final int height) {
-      return OSXUtil.RunOnMainThread(true, new Function<Long, Object>() {
-       public Long eval(Object... args) {
-           return Long.valueOf( CreateCALayer0(x, y, width, height) );
-       } } ).longValue();
+      return CreateCALayer0(x, y, width, height);
     }
     
     /** 
-     * Attach a sub CALayer to the root CALayer on the main-thread w/ blocking.
+     * Attach a sub CALayer to the root CALayer on the main-thread w/o blocking.
      * <p>
      * Method will trigger a <code>display</code>
      * call to the CALayer hierarchy to enforce resource creation if required, e.g. an NSOpenGLContext.
      * </p>
      * <p>
-     * It is mandatory that any related resources, e.g. a shared NSOpenGLContext,
-     * are not locked while calling this method. 
+     * Hence it is important that related resources are not locked <i>if</i>
+     * they will be used for creation.  
      * </p>
      * @see #CreateCALayer(int, int, int, int)
      * @see #RemoveCASublayer(long, long)
@@ -165,7 +162,7 @@ public class OSXUtil implements ToolkitProperties {
         if(0==rootCALayer || 0==subCALayer) {
             throw new IllegalArgumentException("rootCALayer 0x"+Long.toHexString(rootCALayer)+", subCALayer 0x"+Long.toHexString(subCALayer));
         }
-        RunOnMainThread(true, new Runnable() {
+        RunOnMainThread(false, new Runnable() {
            public void run() {
                AddCASublayer0(rootCALayer, subCALayer);
            }
@@ -173,7 +170,7 @@ public class OSXUtil implements ToolkitProperties {
     }
     
     /** 
-     * Fix root and sub CALayer position to 0/0 and size on the main-thread w/o blocking.
+     * Fix root and sub CALayer position to 0/0 and size
      * <p>
      * If the sub CALayer implements the Objective-C NativeWindow protocol NWDedicatedSize (e.g. JOGL's MyNSOpenGLLayer),
      * the dedicated size is passed to the layer, which propagates it appropriately. 
@@ -192,38 +189,28 @@ public class OSXUtil implements ToolkitProperties {
         if( 0==rootCALayer && 0==subCALayer ) {
             return;
         }
-        RunOnMainThread(false, new Runnable() {
-           public void run() {
-               FixCALayerLayout0(rootCALayer, subCALayer, width, height);
-           }
-        });
+        FixCALayerLayout0(rootCALayer, subCALayer, width, height);
     }
     
     /** 
-     * Detach a sub CALayer from the root CALayer on the main-thread w/ blocking.
+     * Detach a sub CALayer from the root CALayer
      */
     public static void RemoveCASublayer(final long rootCALayer, final long subCALayer) {
         if(0==rootCALayer || 0==subCALayer) {
             throw new IllegalArgumentException("rootCALayer 0x"+Long.toHexString(rootCALayer)+", subCALayer 0x"+Long.toHexString(subCALayer));
         }
-        RunOnMainThread(true, new Runnable() {
-           public void run() {
-               RemoveCASublayer0(rootCALayer, subCALayer);
-           } } );
+        RemoveCASublayer0(rootCALayer, subCALayer);
     }
     
     /** 
-     * Destroy a CALayer on the main-thread w/ blocking.
+     * Destroy a CALayer
      * @see #CreateCALayer(int, int, int, int)
      */    
     public static void DestroyCALayer(final long caLayer) {
         if(0==caLayer) {
             throw new IllegalArgumentException("caLayer 0x"+Long.toHexString(caLayer));
         }
-        RunOnMainThread(true, new Runnable() {
-           public void run() {        
-               DestroyCALayer0(caLayer);
-           } } );
+        DestroyCALayer0(caLayer);
     }
     
     /**
diff --git a/src/nativewindow/native/macosx/OSXmisc.m b/src/nativewindow/native/macosx/OSXmisc.m
index c74d6cc58..14a9781f7 100644
--- a/src/nativewindow/native/macosx/OSXmisc.m
+++ b/src/nativewindow/native/macosx/OSXmisc.m
@@ -584,16 +584,16 @@ JNIEXPORT void JNICALL Java_jogamp_nativewindow_macosx_OSXUtil_DestroyCALayer0
 /*
  * Class:     Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow
  * Method:    SetJAWTRootSurfaceLayer0
- * Signature: (JJ)Z
+ * Signature: (JJ)V
  */
-JNIEXPORT jboolean JNICALL Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow_SetJAWTRootSurfaceLayer0
+JNIEXPORT void JNICALL Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow_SetJAWTRootSurfaceLayer0
   (JNIEnv *env, jclass unused, jobject jawtDrawingSurfaceInfoBuffer, jlong caLayer)
 {
     NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init];
     JAWT_DrawingSurfaceInfo* dsi = (JAWT_DrawingSurfaceInfo*) (*env)->GetDirectBufferAddress(env, jawtDrawingSurfaceInfoBuffer);
     if (NULL == dsi) {
         NativewindowCommon_throwNewRuntimeException(env, "Argument \"jawtDrawingSurfaceInfoBuffer\" was not a direct buffer");
-        return JNI_FALSE;
+        return;
     }
     MyCALayer* layer = (MyCALayer*) (intptr_t) caLayer;
     id <JAWT_SurfaceLayers> surfaceLayers = (id <JAWT_SurfaceLayers>)dsi->platformInfo;
@@ -601,35 +601,33 @@ JNIEXPORT jboolean JNICALL Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow
     surfaceLayers.layer = [layer retain]; // Pairs w/ Unset
     [pool release];
     DBG_PRINT("CALayer::SetJAWTRootSurfaceLayer.X: root %p (refcnt %d)\n", layer, (int)[layer retainCount]);
-    return JNI_TRUE;
 }
 
 /*
  * Class:     Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow
  * Method:    UnsetJAWTRootSurfaceLayer0
- * Signature: (JJ)Z
+ * Signature: (JJ)V
  */
-JNIEXPORT jboolean JNICALL Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow_UnsetJAWTRootSurfaceLayer0
+JNIEXPORT void JNICALL Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow_UnsetJAWTRootSurfaceLayer0
   (JNIEnv *env, jclass unused, jobject jawtDrawingSurfaceInfoBuffer, jlong caLayer)
 {
     NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init];
     JAWT_DrawingSurfaceInfo* dsi = (JAWT_DrawingSurfaceInfo*) (*env)->GetDirectBufferAddress(env, jawtDrawingSurfaceInfoBuffer);
     if (NULL == dsi) {
         NativewindowCommon_throwNewRuntimeException(env, "Argument \"jawtDrawingSurfaceInfoBuffer\" was not a direct buffer");
-        return JNI_FALSE;
+        return;
     }
     MyCALayer* layer = (MyCALayer*) (intptr_t) caLayer;
     id <JAWT_SurfaceLayers> surfaceLayers = (id <JAWT_SurfaceLayers>)dsi->platformInfo;
     if(layer != surfaceLayers.layer) {
         NativewindowCommon_throwNewRuntimeException(env, "Attached layer %p doesn't match given layer %p\n", surfaceLayers.layer, layer);
-        return JNI_FALSE;
+        return;
     }
     DBG_PRINT("CALayer::UnsetJAWTRootSurfaceLayer.0: root %p (refcnt %d) -> nil\n", layer, (int)[layer retainCount]);
     [layer release]; // Pairs w/ Set
     surfaceLayers.layer = NULL;
     [pool release];
     DBG_PRINT("CALayer::UnsetJAWTRootSurfaceLayer.X: root %p (refcnt %d) -> nil\n", layer, (int)[layer retainCount]);
-    return JNI_TRUE;
 }
 
 @interface MainRunnable : NSObject