Fix Bug 691 (part-3): NSOpenGLLayer::openGLContextForPixelFormat(..) on main-thread deadlock'ed due to locked shared context

NSOpenGLLayer::openGLContextForPixelFormat(..) is performed on main-thread at 1st NSOpenGLLayer display method.
This happened irregulary, i.e. sometimes (T0) right after NSOpenGLLayer creation and attachSurfaceLayer()/AddCASublayer(..),
sometimes later (T1).

NSOpenGLLayer::openGLContextForPixelFormat(..) uses the passed shared user context.
The shared user context is locked at NSOpenGLLayer's creation (T0) and if performed at this early time
the call deadlocks due to pthread_mutex wait for the shared user context.

This fix performs NSOpenGLLayer creation and layer attachment while the shared user context
is kept unlocked and enforces NSOpenGLLayer display and hence NSOpenGLLayer::openGLContextForPixelFormat(..).

Added CGL.setNSOpenGLLayerEnabled(..) to enable/disable NSOpenGLLayer - currently not used.

- Passed AddRemove tests for GLCanvas/Swing and GLWindow/NewtCanvasAWT w/ 100 loops on Java6 and Java7 on OSX.
- Passed Instruments Leaks test w/ 10 loops on Java6 and Java7

2 files changed, 47 insertions, 12 deletions

diff --git a/src/jogl/classes/jogamp/opengl/macosx/cgl/MacOSXCGLContext.java b/src/jogl/classes/jogamp/opengl/macosx/cgl/MacOSXCGLContext.java
index 838a0387d..37aca0cd7 100644
--- a/src/jogl/classes/jogamp/opengl/macosx/cgl/MacOSXCGLContext.java
+++ b/src/jogl/classes/jogamp/opengl/macosx/cgl/MacOSXCGLContext.java
@@ -687,13 +687,32 @@ public abstract class MacOSXCGLContext extends GLContextImpl
                   } else {
                       gl3ShaderProgramName = 0;
                   }
-                  nsOpenGLLayer = CGL.createNSOpenGLLayer(ctx, gl3ShaderProgramName, nsOpenGLLayerPFmt, pbufferHandle, texID, chosenCaps.isBackgroundOpaque(), lastWidth, lastHeight);
-                  nsOpenGLLayerPFmt = 0; // NSOpenGLLayer will release pfmt
-                  if (DEBUG) {
-                      System.err.println("NS create nsOpenGLLayer "+toHexString(nsOpenGLLayer)+" w/ pbuffer "+toHexString(pbufferHandle)+", texID "+texID+", texSize "+lastWidth+"x"+lastHeight+", "+drawable);
+                   
+                  /**
+                   * Perform NSOpenGLLayer creation and attaching on main-thread,
+                   * hence release the lock on our context - which will be used to 
+                   * create a shared context within NSOpenGLLayer.
+                   */
+                  final long cglCtx = CGL.getCGLContext(ctx);
+                  if(0 == cglCtx) {
+                      throw new InternalError("Null CGLContext for: "+this);
+                  }
+                  final boolean ctxUnlocked = CGL.kCGLNoError == CGL.CGLUnlockContext(cglCtx);
+                  try {                  
+                      nsOpenGLLayer = CGL.createNSOpenGLLayer(ctx, gl3ShaderProgramName, nsOpenGLLayerPFmt, pbufferHandle, texID, chosenCaps.isBackgroundOpaque(), lastWidth, lastHeight);
+                      nsOpenGLLayerPFmt = 0; // NSOpenGLLayer will release pfmt
+                      if (DEBUG) {
+                          System.err.println("NS create nsOpenGLLayer "+toHexString(nsOpenGLLayer)+" w/ pbuffer "+toHexString(pbufferHandle)+", texID "+texID+", texSize "+lastWidth+"x"+lastHeight+", "+drawable);
+                      }
+                      backingLayerHost.attachSurfaceLayer(nsOpenGLLayer);
+                      setSwapInterval(1); // enabled per default in layered surface
+                  } finally {
+                      if( ctxUnlocked ) {
+                          if( CGL.kCGLNoError != CGL.CGLLockContext(cglCtx) ) {
+                              throw new InternalError("Could not re-lock CGLContext for: "+this);
+                          }
+                      }
                   }
-                  backingLayerHost.attachSurfaceLayer(nsOpenGLLayer);
-                  setSwapInterval(1); // enabled per default in layered surface                
               } else {
                   lastWidth = drawable.getWidth();
                   lastHeight = drawable.getHeight();                  
diff --git a/src/jogl/native/macosx/MacOSXWindowSystemInterface-calayer.m b/src/jogl/native/macosx/MacOSXWindowSystemInterface-calayer.m
index 8d1286169..55c4ad053 100644
--- a/src/jogl/native/macosx/MacOSXWindowSystemInterface-calayer.m
+++ b/src/jogl/native/macosx/MacOSXWindowSystemInterface-calayer.m
@@ -130,6 +130,7 @@ extern GLboolean glIsVertexArray (GLuint array);
 @private
     GLfloat gl_texCoords[8];
     NSOpenGLContext* myCtx;
+    Bool isGLEnabled;
 
 @protected
     NSOpenGLContext* parentCtx;
@@ -171,6 +172,7 @@ extern GLboolean glIsVertexArray (GLuint array);
        texWidth: (int) texWidth 
        texHeight: (int) texHeight;
 
+- (void) setGLEnabled: (Bool) enable;
 - (Bool) validateTexSizeWithNewSize;
 - (Bool) validateTexSize: (int) _texWidth texHeight: (int) _texHeight;
 - (void) setTextureID: (int) _texID;
@@ -266,6 +268,7 @@ static const GLfloat gl_verts[] = {
     swapIntervalCounter = 0;
     timespec_now(&lastWaitTime);
     shallDraw = NO;
+    isGLEnabled = YES;
     newTexWidth = _texWidth;
     newTexHeight = _texHeight;
     [self validateTexSizeWithNewSize];
@@ -335,6 +338,12 @@ static const GLfloat gl_verts[] = {
     return self;
 }
 
+- (void) setGLEnabled: (Bool) enable
+{
+    DBG_PRINT("MyNSOpenGLLayer::setGLEnabled: %p, %d -> %d\n", self, (int)isGLEnabled, (int)enable);
+    isGLEnabled = enable;
+}
+
 - (Bool) validateTexSizeWithNewSize
 {
     return [self validateTexSize: newTexWidth texHeight: newTexHeight];
@@ -444,6 +453,7 @@ static const GLfloat gl_verts[] = {
 {
     DBG_PRINT("MyNSOpenGLLayer::openGLPixelFormatForDisplayMask: %p (refcnt %d) - parent-pfmt %p -> new-pfmt %p\n", 
         self, (int)[self retainCount], parentPixelFmt, parentPixelFmt);
+    // We simply take over ownership of parent PixelFormat ..
     return parentPixelFmt;
 }
 
@@ -531,7 +541,6 @@ static const GLfloat gl_verts[] = {
     [self disableAnimation];
     pthread_mutex_lock(&renderLock);
     [self deallocPBuffer];
-    // [[self openGLContext] dealloc];
     pthread_mutex_unlock(&renderLock);
     pthread_cond_destroy(&renderSignal);
     pthread_mutex_destroy(&renderLock);
@@ -548,8 +557,8 @@ static const GLfloat gl_verts[] = {
  {
     CGRect lRectS = [[self superlayer] bounds];
 
-    DBG_PRINT("MyNSOpenGLLayer::resizeWithOldSuperlayerSize: %p, texSize %dx%d, bounds: %lfx%lf -> %lfx%lf (refcnt %d)\n", 
-        self, texWidth, texHeight, size.width, size.height, lRectS.size.width, lRectS.size.height, (int)[self retainCount]);
+    DBG_PRINT("MyNSOpenGLLayer::resizeWithOldSuperlayerSize: %p, texSize %dx%d, bounds: %lfx%lf -> %lf/%lf %lfx%lf (refcnt %d)\n", 
+        self, texWidth, texHeight, size.width, size.height, lRectS.origin.x, lRectS.origin.y, lRectS.size.width, lRectS.size.height, (int)[self retainCount]);
 
     newTexWidth = lRectS.size.width;
     newTexHeight = lRectS.size.height;
@@ -562,8 +571,8 @@ static const GLfloat gl_verts[] = {
 - (BOOL)canDrawInOpenGLContext:(NSOpenGLContext *)context pixelFormat:(NSOpenGLPixelFormat *)pixelFormat 
         forLayerTime:(CFTimeInterval)timeInterval displayTime:(const CVTimeStamp *)timeStamp
 {
-    SYNC_PRINT("<? %d>", (int)shallDraw);
-    return shallDraw;
+    SYNC_PRINT("<? %d, %d>", (int)shallDraw, (int)isGLEnabled);
+    return shallDraw && isGLEnabled;
 }
 
 - (void)drawInOpenGLContext:(NSOpenGLContext *)context pixelFormat:(NSOpenGLPixelFormat *)pixelFormat 
@@ -573,7 +582,7 @@ static const GLfloat gl_verts[] = {
     SYNC_PRINT("<* ");
     // NSLog(@"MyNSOpenGLLayer::DRAW: %@",[NSThread callStackSymbols]);
 
-    if( shallDraw && ( NULL != pbuffer || NULL != newPBuffer || 0 != textureID ) ) {
+    if( isGLEnabled && shallDraw && ( NULL != pbuffer || NULL != newPBuffer || 0 != textureID ) ) {
         [context makeCurrentContext];
 
         if( NULL != newPBuffer ) { // volatile OK
@@ -825,6 +834,13 @@ NSOpenGLLayer* createNSOpenGLLayer(NSOpenGLContext* ctx, int gl3ShaderProgramNam
   return [[[MyNSOpenGLLayer alloc] init] setupWithContext:ctx gl3ShaderProgramName: (GLuint)gl3ShaderProgramName pixelFormat: fmt pbuffer: p texIDArg: (GLuint)texID
                                                               opaque: opaque texWidth: texWidth texHeight: texHeight];
 }
+ 
+void setNSOpenGLLayerEnabled(NSOpenGLLayer* layer, Bool enable) {
+    NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init];
+    MyNSOpenGLLayer* l = (MyNSOpenGLLayer*) layer;
+    [l setGLEnabled: enable];
+    [pool release];
+}
 
 void setNSOpenGLLayerSwapInterval(NSOpenGLLayer* layer, int interval) {
     NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init];