Fix Bug 691 (part-3): NSOpenGLLayer::openGLContextForPixelFormat(..) on main-thread deadlock'ed due to locked shared context

NSOpenGLLayer::openGLContextForPixelFormat(..) is performed on main-thread at 1st NSOpenGLLayer display method.
This happened irregulary, i.e. sometimes (T0) right after NSOpenGLLayer creation and attachSurfaceLayer()/AddCASublayer(..),
sometimes later (T1).

NSOpenGLLayer::openGLContextForPixelFormat(..) uses the passed shared user context.
The shared user context is locked at NSOpenGLLayer's creation (T0) and if performed at this early time
the call deadlocks due to pthread_mutex wait for the shared user context.

This fix performs NSOpenGLLayer creation and layer attachment while the shared user context
is kept unlocked and enforces NSOpenGLLayer display and hence NSOpenGLLayer::openGLContextForPixelFormat(..).

Added CGL.setNSOpenGLLayerEnabled(..) to enable/disable NSOpenGLLayer - currently not used.

- Passed AddRemove tests for GLCanvas/Swing and GLWindow/NewtCanvasAWT w/ 100 loops on Java6 and Java7 on OSX.
- Passed Instruments Leaks test w/ 10 loops on Java6 and Java7

1 files changed, 25 insertions, 6 deletions

diff --git a/src/jogl/classes/jogamp/opengl/macosx/cgl/MacOSXCGLContext.java b/src/jogl/classes/jogamp/opengl/macosx/cgl/MacOSXCGLContext.java
index 838a0387d..37aca0cd7 100644
--- a/src/jogl/classes/jogamp/opengl/macosx/cgl/MacOSXCGLContext.java
+++ b/src/jogl/classes/jogamp/opengl/macosx/cgl/MacOSXCGLContext.java
@@ -687,13 +687,32 @@ public abstract class MacOSXCGLContext extends GLContextImpl
                   } else {
                       gl3ShaderProgramName = 0;
                   }
-                  nsOpenGLLayer = CGL.createNSOpenGLLayer(ctx, gl3ShaderProgramName, nsOpenGLLayerPFmt, pbufferHandle, texID, chosenCaps.isBackgroundOpaque(), lastWidth, lastHeight);
-                  nsOpenGLLayerPFmt = 0; // NSOpenGLLayer will release pfmt
-                  if (DEBUG) {
-                      System.err.println("NS create nsOpenGLLayer "+toHexString(nsOpenGLLayer)+" w/ pbuffer "+toHexString(pbufferHandle)+", texID "+texID+", texSize "+lastWidth+"x"+lastHeight+", "+drawable);
+                   
+                  /**
+                   * Perform NSOpenGLLayer creation and attaching on main-thread,
+                   * hence release the lock on our context - which will be used to 
+                   * create a shared context within NSOpenGLLayer.
+                   */
+                  final long cglCtx = CGL.getCGLContext(ctx);
+                  if(0 == cglCtx) {
+                      throw new InternalError("Null CGLContext for: "+this);
+                  }
+                  final boolean ctxUnlocked = CGL.kCGLNoError == CGL.CGLUnlockContext(cglCtx);
+                  try {                  
+                      nsOpenGLLayer = CGL.createNSOpenGLLayer(ctx, gl3ShaderProgramName, nsOpenGLLayerPFmt, pbufferHandle, texID, chosenCaps.isBackgroundOpaque(), lastWidth, lastHeight);
+                      nsOpenGLLayerPFmt = 0; // NSOpenGLLayer will release pfmt
+                      if (DEBUG) {
+                          System.err.println("NS create nsOpenGLLayer "+toHexString(nsOpenGLLayer)+" w/ pbuffer "+toHexString(pbufferHandle)+", texID "+texID+", texSize "+lastWidth+"x"+lastHeight+", "+drawable);
+                      }
+                      backingLayerHost.attachSurfaceLayer(nsOpenGLLayer);
+                      setSwapInterval(1); // enabled per default in layered surface
+                  } finally {
+                      if( ctxUnlocked ) {
+                          if( CGL.kCGLNoError != CGL.CGLLockContext(cglCtx) ) {
+                              throw new InternalError("Could not re-lock CGLContext for: "+this);
+                          }
+                      }
                   }
-                  backingLayerHost.attachSurfaceLayer(nsOpenGLLayer);
-                  setSwapInterval(1); // enabled per default in layered surface                
               } else {
                   lastWidth = drawable.getWidth();
                   lastHeight = drawable.getHeight();