Remodel OSX/CALayer Threading (commit 896e8b021b39e9415040a57a1d540d7d24b02db1): Run on main-thread w/o blocking ; Misc Changes

Commit 896e8b021b39e9415040a57a1d540d7d24b02db1 moved all native CALayer calls to the current thread
to avoid deadlocks.

Even though this seemed to be fine at least resource GC (release/dealloc calls) were issued
very late in time, probably due to multithreading synchronization of JAWT and/or OSX API.

Example: Our 'TestAddRemove01GLCanvasSwingAWT' test didn't freed CALayer resources incl. GL ctx
         when destroying the objects (AWT Frame, GLCanvas, ..), leading to resource starvation .. eventually.

Remedy is a compromise of behavior before commit 896e8b021b39e9415040a57a1d540d7d24b02db1
and that commit, i.e. to run CALayer lifecycle methods on main-thread, but do not block!

The careful part within MacOSXCGLContext.associateDrawable(..) performs the following block on main-thread:
  - lock the context
  - create NSOpenGLLayer (incl. it's own shared GL context and the DisplayLink)
  - attach NSOpenGLLayer to root CALayer
  - unlock the context

Due to the GL ctx locking, this async offthread operation is safe within our course of operations.

Details:
  - NSOpenGLContext
    - Context and CVDisplayLink creation at init
    - Call [ctx update] if texture/frame size changed
    - 'waitUntilRenderSignal' uses default TO value if given TO is 0 to avoid deadlocks

+++

Misc Changes:
  - Fix object type detection: isMemberOfClass -> isKindOfClass
    - OSXUtil_isNSView0
      OSXUtil_isNSWindow0,
      CGL_isNSOpenGLPixelBuffer

  - MacOSXCGLDrawable/MacOSXPbufferCGLDrawable: remove getNSViewHandle() method.
    MacOSXCGLContext uses common code to detect nature of the drawable handle.

  - MacOSXCGLContext/CALayer: Use safe screenVSyncTimeout values, never 0 to avoid deadlock!

  - JAWTWindow.invalidate: Call detachSurfaceLayer() if not done yet

3 files changed, 49 insertions, 43 deletions

diff --git a/src/nativewindow/classes/com/jogamp/nativewindow/awt/JAWTWindow.java b/src/nativewindow/classes/com/jogamp/nativewindow/awt/JAWTWindow.java
index 3e5e629b6..15a0f550f 100644
--- a/src/nativewindow/classes/com/jogamp/nativewindow/awt/JAWTWindow.java
+++ b/src/nativewindow/classes/com/jogamp/nativewindow/awt/JAWTWindow.java
@@ -134,6 +134,13 @@ public abstract class JAWTWindow implements NativeWindow, OffscreenLayerSurface,
   }
 
   protected synchronized void invalidate() {
+    if(DEBUG) {
+        System.err.println("JAWTWindow.invalidate()");
+        // Thread.dumpStack();
+    }
+    if( isSurfaceLayerAttached() ) {
+        detachSurfaceLayer();
+    }
     invalidateNative();
     jawt = null;
     isOffscreenLayerSurface = false;
diff --git a/src/nativewindow/classes/jogamp/nativewindow/jawt/macosx/MacOSXJAWTWindow.java b/src/nativewindow/classes/jogamp/nativewindow/jawt/macosx/MacOSXJAWTWindow.java
index 758105713..d11d24664 100644
--- a/src/nativewindow/classes/jogamp/nativewindow/jawt/macosx/MacOSXJAWTWindow.java
+++ b/src/nativewindow/classes/jogamp/nativewindow/jawt/macosx/MacOSXJAWTWindow.java
@@ -74,13 +74,16 @@ public class MacOSXJAWTWindow extends JAWTWindow implements MutableSurface {
           System.err.println("MacOSXJAWTWindow.invalidateNative(): osh-enabled "+isOffscreenLayerSurfaceEnabled()+
                              ", osh-set "+offscreenSurfaceHandleSet+
                              ", osh "+toHexString(offscreenSurfaceHandle)+
-                             ", rsh "+toHexString(rootSurfaceLayerHandle)+
-                             ", wh "+toHexString(windowHandle));
+                             ", rsh "+toHexString(rootSurfaceLayer)+
+                             ", wh "+toHexString(windowHandle)+" - "+Thread.currentThread().getName());
       }
       offscreenSurfaceHandle=0;
       offscreenSurfaceHandleSet=false;
       if( isOffscreenLayerSurfaceEnabled() ) {
-          if(0 != rootSurfaceLayerHandle) {
+          if(0 != windowHandle) {
+              OSXUtil.DestroyNSWindow(windowHandle);
+          }
+          if(0 != rootSurfaceLayer) {
               final JAWT jawt = getJAWT();
               if( null != jawt ) {
                   final JAWT_DrawingSurface ds = jawt.GetDrawingSurface(component);
@@ -90,9 +93,9 @@ public class MacOSXJAWTWindow extends JAWTWindow implements MutableSurface {
                           try {
                               dsi = ds.GetDrawingSurfaceInfo();
                               try {
-                                  UnsetJAWTRootSurfaceLayer(dsi.getBuffer(), rootSurfaceLayerHandle);
+                                  UnsetJAWTRootSurfaceLayer0(dsi.getBuffer(), rootSurfaceLayer);
                               } catch (Exception e) {                                  
-                                  System.err.println("Error clearing JAWT rootSurfaceLayerHandle "+toHexString(rootSurfaceLayerHandle));
+                                  System.err.println("Error clearing JAWT rootSurfaceLayerHandle "+toHexString(rootSurfaceLayer));
                                   e.printStackTrace();
                               }
                           } finally {
@@ -105,19 +108,15 @@ public class MacOSXJAWTWindow extends JAWTWindow implements MutableSurface {
                       jawt.FreeDrawingSurface(ds);
                   }
               }
-              OSXUtil.DestroyCALayer(rootSurfaceLayerHandle);
-              
-              rootSurfaceLayerHandle = 0;
-          }
-          if(0 != windowHandle) {
-              OSXUtil.DestroyNSWindow(windowHandle);
+              OSXUtil.DestroyCALayer(rootSurfaceLayer);
+              rootSurfaceLayer = 0;
           }
       }
       windowHandle=0;
   }
-
+  
   protected void attachSurfaceLayerImpl(final long layerHandle) {
-      OSXUtil.AddCASublayer(rootSurfaceLayerHandle, layerHandle, getWidth(), getHeight());
+      OSXUtil.AddCASublayer(rootSurfaceLayer, layerHandle, getWidth(), getHeight());
   }
   
   protected void layoutSurfaceLayerImpl() {
@@ -127,11 +126,11 @@ public class MacOSXJAWTWindow extends JAWTWindow implements MutableSurface {
       if(DEBUG) {
         System.err.println("JAWTWindow.fixSurfaceLayerLayout: "+toHexString(osl) + ", bounds "+bounds+", "+w+"x"+h);
       }
-      OSXUtil.FixCALayerLayout(rootSurfaceLayerHandle, osl, w, h);
+      OSXUtil.FixCALayerLayout(rootSurfaceLayer, osl, w, h);
   }
   
   protected void detachSurfaceLayerImpl(final long layerHandle) {
-      OSXUtil.RemoveCASublayer(rootSurfaceLayerHandle, layerHandle);
+      OSXUtil.RemoveCASublayer(rootSurfaceLayer, layerHandle);
   }
   
   @Override
@@ -243,23 +242,23 @@ public class MacOSXJAWTWindow extends JAWTWindow implements MutableSurface {
             }
         }
         if(null == errMsg) {
-            if(0 == rootSurfaceLayerHandle) {        
-                rootSurfaceLayerHandle = OSXUtil.CreateCALayer(bounds.getX(), bounds.getY(), bounds.getWidth(), bounds.getHeight());
-                if(0 == rootSurfaceLayerHandle) {
+            if(0 == rootSurfaceLayer) {        
+                rootSurfaceLayer = OSXUtil.CreateCALayer(bounds.getX(), bounds.getY(), bounds.getWidth(), bounds.getHeight());
+                if(0 == rootSurfaceLayer) {
                   errMsg = "Could not create root CALayer";                
                 } else {
                     try {
-                        SetJAWTRootSurfaceLayer(dsi.getBuffer(), rootSurfaceLayerHandle);
+                        SetJAWTRootSurfaceLayer0(dsi.getBuffer(), rootSurfaceLayer);
                     } catch(Exception e) {
-                        errMsg = "Could not set JAWT rootSurfaceLayerHandle "+toHexString(rootSurfaceLayerHandle)+", cause: "+e.getMessage();
+                        errMsg = "Could not set JAWT rootSurfaceLayerHandle "+toHexString(rootSurfaceLayer)+", cause: "+e.getMessage();
                     }
                 }
             }
         }
         if(null != errMsg) {
-            if(0 != rootSurfaceLayerHandle) {
-              OSXUtil.DestroyCALayer(rootSurfaceLayerHandle);
-              rootSurfaceLayerHandle = 0;
+            if(0 != rootSurfaceLayer) {
+              OSXUtil.DestroyCALayer(rootSurfaceLayer);
+              rootSurfaceLayer = 0;
             }
             if(0 != windowHandle) {
               OSXUtil.DestroyNSWindow(windowHandle);
@@ -316,18 +315,11 @@ public class MacOSXJAWTWindow extends JAWTWindow implements MutableSurface {
   /** 
    * Set the given root CALayer in the JAWT surface
    */  
-  private static void SetJAWTRootSurfaceLayer(final Buffer jawtDrawingSurfaceInfoBuffer, final long caLayer) {
-      SetJAWTRootSurfaceLayer0(jawtDrawingSurfaceInfoBuffer, caLayer);
-  }
-
+  private static native void SetJAWTRootSurfaceLayer0(Buffer jawtDrawingSurfaceInfoBuffer, long caLayer);
+  
   /** 
-   * Unset the given root CALayer in the JAWT surface
+   * Unset the given root CALayer in the JAWT surface, passing the NIO DrawingSurfaceInfo buffer
    */  
-  private static void UnsetJAWTRootSurfaceLayer(final Buffer jawtDrawingSurfaceInfoBuffer, final long caLayer) {
-      UnsetJAWTRootSurfaceLayer0(jawtDrawingSurfaceInfoBuffer, caLayer);
-  }
-
-  private static native void SetJAWTRootSurfaceLayer0(Buffer jawtDrawingSurfaceInfoBuffer, long caLayer);
   private static native void UnsetJAWTRootSurfaceLayer0(Buffer jawtDrawingSurfaceInfoBuffer, long caLayer);
   
   // Variables for lockSurface/unlockSurface
@@ -337,7 +329,7 @@ public class MacOSXJAWTWindow extends JAWTWindow implements MutableSurface {
   
   private JAWT_MacOSXDrawingSurfaceInfo macosxdsi;
   
-  private long rootSurfaceLayerHandle = 0; // attached to the JAWT_SurfaceLayer
+  private long rootSurfaceLayer = 0; // attached to the JAWT_SurfaceLayer
   
   private long windowHandle = 0;
   private long offscreenSurfaceHandle = 0;
diff --git a/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java b/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java
index 1563a5a8b..703c3d972 100644
--- a/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java
+++ b/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java
@@ -83,11 +83,11 @@ public class OSXUtil implements ToolkitProperties {
     public static final boolean hasThreadingIssues() { return false; }
     
     public static boolean isNSView(long object) {
-        return isNSView0(object);
+        return 0 != object ? isNSView0(object) : false;
     }
     
     public static boolean isNSWindow(long object) {
-        return isNSWindow0(object);
+        return 0 != object ? isNSWindow0(object) : false;
     }
     
     /**
@@ -164,8 +164,7 @@ public class OSXUtil implements ToolkitProperties {
         }
         RunOnMainThread(false, new Runnable() {
            public void run() {
-               AddCASublayer0(rootCALayer, subCALayer);
-               FixCALayerLayout0(rootCALayer, subCALayer, width, height);
+               AddCASublayer0(rootCALayer, subCALayer, width, height);
            }
         });
     }
@@ -194,24 +193,32 @@ public class OSXUtil implements ToolkitProperties {
     }
     
     /** 
-     * Detach a sub CALayer from the root CALayer
+     * Detach a sub CALayer from the root CALayer on the main-thread w/o blocking.
      */
     public static void RemoveCASublayer(final long rootCALayer, final long subCALayer) {
         if(0==rootCALayer || 0==subCALayer) {
             throw new IllegalArgumentException("rootCALayer 0x"+Long.toHexString(rootCALayer)+", subCALayer 0x"+Long.toHexString(subCALayer));
         }
-        RemoveCASublayer0(rootCALayer, subCALayer);
+        RunOnMainThread(false, new Runnable() {
+           public void run() {
+               RemoveCASublayer0(rootCALayer, subCALayer);
+           }
+        });
     }
     
     /** 
-     * Destroy a CALayer
+     * Destroy a CALayer on main-thread w/o blocking.
      * @see #CreateCALayer(int, int, int, int)
      */    
     public static void DestroyCALayer(final long caLayer) {
         if(0==caLayer) {
             throw new IllegalArgumentException("caLayer 0x"+Long.toHexString(caLayer));
         }
-        DestroyCALayer0(caLayer);
+        RunOnMainThread(false, new Runnable() {
+           public void run() {
+               DestroyCALayer0(caLayer);
+           }
+        });
     }
     
     /**
@@ -337,7 +344,7 @@ public class OSXUtil implements ToolkitProperties {
     private static native long GetNSView0(long nsWindow);
     private static native long GetNSWindow0(long nsView);
     private static native long CreateCALayer0(int x, int y, int width, int height);
-    private static native void AddCASublayer0(long rootCALayer, long subCALayer);
+    private static native void AddCASublayer0(long rootCALayer, long subCALayer, int width, int height);
     private static native void FixCALayerLayout0(long rootCALayer, long subCALayer, int width, int height);
     private static native void RemoveCASublayer0(long rootCALayer, long subCALayer);
     private static native void DestroyCALayer0(long caLayer);