OSX/CALayer: Revise CALayer 'RunOnMainThread' utilization, avoiding deadlocks

RunOnMainThread(waitUntilDone:=true,..) can deadlock the main-thread if called from AWT-EDT,
since the main-thread may call back to AWT-EDT while injecting a new main-thread task.

This patch revises all RunOnMainThread CALayer usage, resulting in only one required left:
  - OSXUtil.AddCASublayer() w/ waitUntilDone:=false

Hence the CALayer code has no more potential to deadlock main-thread/AWT-EDT.

OSXUtil.AddCASublayer() must be performed on main-thread, otherwise the
CALayer attachment will fail - no visible rendering result.

+++

Note: A good trigger to test this deadlock is to magnify/zoom
      the OSX desktop (click background + ctrl-mouse_wheel)
      before running some unit tests.

      TestGLCanvasAWTActionDeadlock01AWT and TestAddRemove02GLWindowNewtCanvasAWT
      also have the potential to trigger the mentioned deadlock.

2 files changed, 27 insertions, 39 deletions

diff --git a/src/jogl/classes/jogamp/opengl/macosx/cgl/MacOSXCGLContext.java b/src/jogl/classes/jogamp/opengl/macosx/cgl/MacOSXCGLContext.java
index a03850043..9e6085030 100644
--- a/src/jogl/classes/jogamp/opengl/macosx/cgl/MacOSXCGLContext.java
+++ b/src/jogl/classes/jogamp/opengl/macosx/cgl/MacOSXCGLContext.java
@@ -676,30 +676,24 @@ public abstract class MacOSXCGLContext extends GLContextImpl
                   }
                    
                   /**
-                   * Perform NSOpenGLLayer creation and attaching on main-thread,
-                   * hence release the lock on our context - which will be used to 
-                   * create a shared context within NSOpenGLLayer.
+                   * NSOpenGLLayer creation is performed on the current thread,
+                   * which immediately creates it's own GL ctx sharing this ctx
+                   * not causing any locking issues.
+                   * 
+                   * Subsequent attaching is performed on main-thread w/o blocking.
+                   * 
+                   * This is a lock free operation.
                    */
                   final long cglCtx = CGL.getCGLContext(ctx);
                   if(0 == cglCtx) {
                       throw new InternalError("Null CGLContext for: "+this);
                   }
-                  final boolean ctxUnlocked = CGL.kCGLNoError == CGL.CGLUnlockContext(cglCtx);
-                  try {                  
-                      nsOpenGLLayer = CGL.createNSOpenGLLayer(ctx, gl3ShaderProgramName, pixelFormat, pbufferHandle, texID, chosenCaps.isBackgroundOpaque(), lastWidth, lastHeight);
-                      if (DEBUG) {
-                          System.err.println("NS create nsOpenGLLayer "+toHexString(nsOpenGLLayer)+" w/ pbuffer "+toHexString(pbufferHandle)+", texID "+texID+", texSize "+lastWidth+"x"+lastHeight+", "+drawable);
-                      }
-                      backingLayerHost.attachSurfaceLayer(nsOpenGLLayer);
-                      setSwapInterval(1); // enabled per default in layered surface
-                  } finally {
-                      if( ctxUnlocked ) {
-                          if( CGL.kCGLNoError != CGL.CGLLockContext(cglCtx) ) {
-                              throw new InternalError("Could not re-lock CGLContext for: "+this);
-                          }
-                      }
+                  nsOpenGLLayer = CGL.createNSOpenGLLayer(ctx, gl3ShaderProgramName, pixelFormat, pbufferHandle, texID, chosenCaps.isBackgroundOpaque(), lastWidth, lastHeight);
+                  if (DEBUG) {
+                      System.err.println("NS create nsOpenGLLayer "+toHexString(nsOpenGLLayer)+" w/ pbuffer "+toHexString(pbufferHandle)+", texID "+texID+", texSize "+lastWidth+"x"+lastHeight+", "+drawable);
                   }
-                  backingLayerHost.layoutSurfaceLayer();
+                  backingLayerHost.attachSurfaceLayer(nsOpenGLLayer);
+                  setSwapInterval(1); // enabled per default in layered surface
               } else {
                   lastWidth = drawable.getWidth();
                   lastHeight = drawable.getHeight();                  
diff --git a/src/jogl/native/macosx/MacOSXWindowSystemInterface-calayer.m b/src/jogl/native/macosx/MacOSXWindowSystemInterface-calayer.m
index abc9d7958..f93f15241 100644
--- a/src/jogl/native/macosx/MacOSXWindowSystemInterface-calayer.m
+++ b/src/jogl/native/macosx/MacOSXWindowSystemInterface-calayer.m
@@ -131,11 +131,10 @@ extern GLboolean glIsVertexArray (GLuint array);
 {
 @private
     GLfloat gl_texCoords[8];
-    NSOpenGLContext* myCtx;
+    NSOpenGLContext* glContext;
     Bool isGLEnabled;
 
 @protected
-    NSOpenGLContext* parentCtx;
     GLuint gl3ShaderProgramName;
     GLuint vboBufVert;
     GLuint vboBufTexCoord;
@@ -247,7 +246,7 @@ static const GLfloat gl_verts[] = {
 
 @implementation MyNSOpenGLLayer
 
-- (id) setupWithContext: (NSOpenGLContext*) _parentCtx
+- (id) setupWithContext: (NSOpenGLContext*) parentCtx
        gl3ShaderProgramName: (GLuint) _gl3ShaderProgramName
        pixelFormat: (NSOpenGLPixelFormat*) _parentPixelFmt
        pbuffer: (NSOpenGLPixelBuffer*) p
@@ -262,20 +261,19 @@ static const GLfloat gl_verts[] = {
     pthread_mutex_init(&renderLock, &renderLockAttr); // recursive
     pthread_cond_init(&renderSignal, NULL); // no attribute
 
-    myCtx = NULL;
     {
         int i;
         for(i=0; i<8; i++) {
             gl_texCoords[i] = 0.0f;
         }
     }
-    parentCtx = _parentCtx;
+    parentPixelFmt = [_parentPixelFmt retain]; // until destruction
+    glContext = [[MyNSOpenGLContext alloc] initWithFormat:parentPixelFmt shareContext:parentCtx];
     gl3ShaderProgramName = _gl3ShaderProgramName;
     vboBufVert = 0;
     vboBufTexCoord = 0;
     vertAttrLoc = 0;
     texCoordAttrLoc = 0;
-    parentPixelFmt = [_parentPixelFmt retain]; // until destruction
     swapInterval = 1; // defaults to on (as w/ new GL profiles)
     swapIntervalCounter = 0;
     timespec_now(&lastWaitTime);
@@ -338,12 +336,12 @@ static const GLfloat gl_verts[] = {
 #ifdef VERBOSE_ON
     CGRect lRect = [self bounds];
     if(NULL != pbuffer) {
-        DBG_PRINT("MyNSOpenGLLayer::init (pbuffer) %p, ctx %p, pfmt %p, pbuffer %p, opaque %d, pbuffer %dx%d -> tex %dx%d, bounds: %lf/%lf %lfx%lf, displayLink %p (refcnt %d)\n", 
-            self, parentCtx, parentPixelFmt, pbuffer, opaque, [pbuffer pixelsWide], [pbuffer pixelsHigh], texWidth, texHeight,
+        DBG_PRINT("MyNSOpenGLLayer::init (pbuffer) %p, pctx %p, pfmt %p, pbuffer %p, ctx %p, opaque %d, pbuffer %dx%d -> tex %dx%d, bounds: %lf/%lf %lfx%lf, displayLink %p (refcnt %d)\n", 
+            self, parentCtx, parentPixelFmt, pbuffer, glContext, opaque, [pbuffer pixelsWide], [pbuffer pixelsHigh], texWidth, texHeight,
             lRect.origin.x, lRect.origin.y, lRect.size.width, lRect.size.height, displayLink, (int)[self retainCount]);
     } else {
-        DBG_PRINT("MyNSOpenGLLayer::init (texture) %p, ctx %p, pfmt %p, opaque %d, tex[id %d, %dx%d], bounds: %lf/%lf %lfx%lf, displayLink %p (refcnt %d)\n", 
-            self, parentCtx, parentPixelFmt, opaque, (int)textureID, texWidth, texHeight,
+        DBG_PRINT("MyNSOpenGLLayer::init (texture) %p, pctx %p, pfmt %p, ctx %p, opaque %d, tex[id %d, %dx%d], bounds: %lf/%lf %lfx%lf, displayLink %p (refcnt %d)\n", 
+            self, parentCtx, parentPixelFmt, glContext, opaque, (int)textureID, texWidth, texHeight,
             lRect.origin.x, lRect.origin.y, lRect.size.width, lRect.size.height, displayLink, (int)[self retainCount]);
     }
 #endif
@@ -493,11 +491,10 @@ static const GLfloat gl_verts[] = {
     pthread_mutex_lock(&renderLock);
     [self deallocPBuffer];
     // [[self openGLContext] release];
-    if( NULL != myCtx ) {
-        [myCtx release];
-        myCtx = NULL;
+    if( NULL != glContext ) {
+        [glContext release];
+        glContext = NULL;
     }
-    parentCtx = NULL;
     if( NULL != parentPixelFmt ) {
         [parentPixelFmt release];
         parentPixelFmt = NULL;
@@ -632,22 +629,19 @@ static const GLfloat gl_verts[] = {
 
 - (NSOpenGLContext *)openGLContextForPixelFormat:(NSOpenGLPixelFormat *)pixelFormat
 {
-    DBG_PRINT("MyNSOpenGLLayer::openGLContextForPixelFormat.0: %p (refcnt %d) - pfmt %p, parent %p, DisplayLink %p\n",
-        self, (int)[self retainCount], pixelFormat, parentCtx, displayLink);
-    // NSLog(@"MyNSOpenGLLayer::openGLContextForPixelFormat: %@",[NSThread callStackSymbols]);
-    myCtx = [[MyNSOpenGLContext alloc] initWithFormat:pixelFormat shareContext:parentCtx];
+    DBG_PRINT("MyNSOpenGLLayer::openGLContextForPixelFormat.0: %p (refcnt %d) - pfmt %p, ctx %p, DisplayLink %p\n",
+        self, (int)[self retainCount], pixelFormat, glContext, displayLink);
 #ifndef HAS_CADisplayLink
     if(NULL != displayLink) {
         CVReturn cvres;
         DBG_PRINT("MyNSOpenGLLayer::openGLContextForPixelFormat.1: setup DisplayLink %p\n", displayLink);
-        cvres = CVDisplayLinkSetCurrentCGDisplayFromOpenGLContext(displayLink, [myCtx CGLContextObj], [pixelFormat CGLPixelFormatObj]);
+        cvres = CVDisplayLinkSetCurrentCGDisplayFromOpenGLContext(displayLink, [glContext CGLContextObj], [pixelFormat CGLPixelFormatObj]);
         if(kCVReturnSuccess != cvres) {
             DBG_PRINT("MyNSOpenGLLayer::init %p, CVDisplayLinkSetCurrentCGDisplayFromOpenGLContext failed: %d\n", self, cvres);
         }
     }
 #endif
-    DBG_PRINT("MyNSOpenGLLayer::openGLContextForPixelFormat.X: new-ctx %p\n", myCtx);
-    return myCtx;
+    return glContext;
 }
 
 - (BOOL)canDrawInOpenGLContext:(NSOpenGLContext *)context pixelFormat:(NSOpenGLPixelFormat *)pixelFormat