Fix Bug 691 (part-3): NSOpenGLLayer::openGLContextForPixelFormat(..) on main-thread deadlock'ed due to locked shared context

NSOpenGLLayer::openGLContextForPixelFormat(..) is performed on main-thread at 1st NSOpenGLLayer display method.
This happened irregulary, i.e. sometimes (T0) right after NSOpenGLLayer creation and attachSurfaceLayer()/AddCASublayer(..),
sometimes later (T1).

NSOpenGLLayer::openGLContextForPixelFormat(..) uses the passed shared user context.
The shared user context is locked at NSOpenGLLayer's creation (T0) and if performed at this early time
the call deadlocks due to pthread_mutex wait for the shared user context.

This fix performs NSOpenGLLayer creation and layer attachment while the shared user context
is kept unlocked and enforces NSOpenGLLayer display and hence NSOpenGLLayer::openGLContextForPixelFormat(..).

Added CGL.setNSOpenGLLayerEnabled(..) to enable/disable NSOpenGLLayer - currently not used.

- Passed AddRemove tests for GLCanvas/Swing and GLWindow/NewtCanvasAWT w/ 100 loops on Java6 and Java7 on OSX.
- Passed Instruments Leaks test w/ 10 loops on Java6 and Java7

3 files changed, 47 insertions, 5 deletions

diff --git a/src/nativewindow/classes/javax/media/nativewindow/OffscreenLayerSurface.java b/src/nativewindow/classes/javax/media/nativewindow/OffscreenLayerSurface.java
index f6bc5822b..ba60a7f38 100644
--- a/src/nativewindow/classes/javax/media/nativewindow/OffscreenLayerSurface.java
+++ b/src/nativewindow/classes/javax/media/nativewindow/OffscreenLayerSurface.java
@@ -33,6 +33,14 @@ package javax.media.nativewindow;
 public interface OffscreenLayerSurface {
   /** 
    * Attach the offscreen layer to this offscreen layer surface.
+   * <p>
+   * Implementation may realize all required resources at this point.
+   * </p>
+   * <p>
+   * It is mandatory that any related resources, e.g. a shared context,
+   * are not locked while calling this method. 
+   * </p>
+   * 
    * @see #isOffscreenLayerSurfaceEnabled()
    * @throws NativeWindowException if {@link #isOffscreenLayerSurfaceEnabled()} == false
    */
diff --git a/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java b/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java
index b765a68c3..d85d1a84b 100644
--- a/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java
+++ b/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java
@@ -136,12 +136,31 @@ public class OSXUtil implements ToolkitProperties {
       return GetNSWindow0(nsView);
     }
     
+    /** 
+     * Create a CALayer suitable to act as a root CALayer on the main-thread.
+     * @see #DestroyCALayer(long)
+     * @see #AddCASublayer(long, long) 
+     */
     public static long CreateCALayer(final int x, final int y, final int width, final int height) {
       return OSXUtil.RunOnMainThread(true, new Function<Long, Object>() {
        public Long eval(Object... args) {
            return Long.valueOf( CreateCALayer0(x, y, width, height) );
        } } ).longValue();
     }
+    
+    /** 
+     * Attach a sub CALayer to the root CALayer on the main-thread.
+     * <p>
+     * Method will trigger a <code>display</code>
+     * call to the CALayer hierarchy to enforce resource creation if required, e.g. an NSOpenGLContext.
+     * </p>
+     * <p>
+     * It is mandatory that any related resources, e.g. a shared NSOpenGLContext,
+     * are not locked while calling this method. 
+     * </p>
+     * @see #CreateCALayer(int, int, int, int)
+     * @see #RemoveCASublayer(long, long)
+     */
     public static void AddCASublayer(final long rootCALayer, final long subCALayer) {
         if(0==rootCALayer || 0==subCALayer) {
             throw new IllegalArgumentException("rootCALayer 0x"+Long.toHexString(rootCALayer)+", subCALayer 0x"+Long.toHexString(subCALayer));
@@ -152,6 +171,10 @@ public class OSXUtil implements ToolkitProperties {
            }
         });
     }
+    
+    /** 
+     * Detach a sub CALayer from the root CALayer on the main-thread.
+     */
     public static void RemoveCASublayer(final long rootCALayer, final long subCALayer) {
         if(0==rootCALayer || 0==subCALayer) {
             throw new IllegalArgumentException("rootCALayer 0x"+Long.toHexString(rootCALayer)+", subCALayer 0x"+Long.toHexString(subCALayer));
@@ -161,6 +184,11 @@ public class OSXUtil implements ToolkitProperties {
                RemoveCASublayer0(rootCALayer, subCALayer);
            } } );
     }
+    
+    /** 
+     * Destroy a CALayer on the main-thread.
+     * @see #CreateCALayer(int, int, int, int)
+     */    
     public static void DestroyCALayer(final long caLayer) {
         if(0==caLayer) {
             throw new IllegalArgumentException("caLayer 0x"+Long.toHexString(caLayer));
diff --git a/src/nativewindow/native/macosx/OSXmisc.m b/src/nativewindow/native/macosx/OSXmisc.m
index 83f3c821f..8d876c175 100644
--- a/src/nativewindow/native/macosx/OSXmisc.m
+++ b/src/nativewindow/native/macosx/OSXmisc.m
@@ -430,6 +430,9 @@ JNIEXPORT void JNICALL Java_jogamp_nativewindow_macosx_OSXUtil_AddCASublayer0
     MyCALayer* rootLayer = (MyCALayer*) ((intptr_t) rootCALayer);
     CALayer* subLayer = (CALayer*) ((intptr_t) subCALayer);
 
+    [CATransaction begin];
+    [CATransaction setValue:(id)kCFBooleanTrue forKey:kCATransactionDisableActions];
+
     CGRect lRectRoot = [rootLayer frame];
     DBG_PRINT("CALayer::AddCASublayer0.0: Origin %p frame0: %lf/%lf %lfx%lf\n", 
         rootLayer, lRectRoot.origin.x, lRectRoot.origin.y, lRectRoot.size.width, lRectRoot.size.height);
@@ -444,9 +447,6 @@ JNIEXPORT void JNICALL Java_jogamp_nativewindow_macosx_OSXUtil_AddCASublayer0
         rootLayer, (int)[rootLayer retainCount],
         subLayer, lRectRoot.origin.x, lRectRoot.origin.y, lRectRoot.size.width, lRectRoot.size.height, (int)[subLayer retainCount]);
 
-    [CATransaction begin];
-    [CATransaction setValue:(id)kCFBooleanTrue forKey:kCATransactionDisableActions];
-
     // simple 1:1 layout !
     [subLayer setFrame:lRectRoot];
     [rootLayer addSublayer:subLayer];
@@ -454,14 +454,20 @@ JNIEXPORT void JNICALL Java_jogamp_nativewindow_macosx_OSXUtil_AddCASublayer0
     // no animations for add/remove/swap sublayers etc 
     // doesn't work: [layer removeAnimationForKey: kCAOnOrderIn, kCAOnOrderOut, kCATransition]
     [rootLayer removeAllAnimations];
-    // [rootLayer addAnimation:nil forKey:kCATransition];
+    // [rootLayer addAnimation:nil forKey:kCATransition]; // JAU
     [rootLayer setAutoresizingMask: (kCALayerWidthSizable|kCALayerHeightSizable)];
     [rootLayer setNeedsDisplayOnBoundsChange: YES];
     [subLayer removeAllAnimations];
-    // [sublayer addAnimation:nil forKey:kCATransition];
+    // [subLayer addAnimation:nil forKey:kCATransition]; // JAU
     [subLayer setAutoresizingMask: (kCALayerWidthSizable|kCALayerHeightSizable)];
     [subLayer setNeedsDisplayOnBoundsChange: YES];
 
+    // Trigger display and hence ctx creation.
+    // The latter is essential since since the parent-context lock is cleared
+    // only for this window of time (method call).
+    [rootLayer setNeedsDisplay];
+    [rootLayer displayIfNeeded];
+
     [CATransaction commit];
 
     [pool release];