Fix Bug 691 (part-3): NSOpenGLLayer::openGLContextForPixelFormat(..) on main-thread deadlock'ed due to locked shared context

NSOpenGLLayer::openGLContextForPixelFormat(..) is performed on main-thread at 1st NSOpenGLLayer display method.
This happened irregulary, i.e. sometimes (T0) right after NSOpenGLLayer creation and attachSurfaceLayer()/AddCASublayer(..),
sometimes later (T1).

NSOpenGLLayer::openGLContextForPixelFormat(..) uses the passed shared user context.
The shared user context is locked at NSOpenGLLayer's creation (T0) and if performed at this early time
the call deadlocks due to pthread_mutex wait for the shared user context.

This fix performs NSOpenGLLayer creation and layer attachment while the shared user context
is kept unlocked and enforces NSOpenGLLayer display and hence NSOpenGLLayer::openGLContextForPixelFormat(..).

Added CGL.setNSOpenGLLayerEnabled(..) to enable/disable NSOpenGLLayer - currently not used.

- Passed AddRemove tests for GLCanvas/Swing and GLWindow/NewtCanvasAWT w/ 100 loops on Java6 and Java7 on OSX.
- Passed Instruments Leaks test w/ 10 loops on Java6 and Java7

1 files changed, 28 insertions, 0 deletions

diff --git a/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java b/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java
index b765a68c3..d85d1a84b 100644
--- a/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java
+++ b/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java
@@ -136,12 +136,31 @@ public class OSXUtil implements ToolkitProperties {
       return GetNSWindow0(nsView);
     }
     
+    /** 
+     * Create a CALayer suitable to act as a root CALayer on the main-thread.
+     * @see #DestroyCALayer(long)
+     * @see #AddCASublayer(long, long) 
+     */
     public static long CreateCALayer(final int x, final int y, final int width, final int height) {
       return OSXUtil.RunOnMainThread(true, new Function<Long, Object>() {
        public Long eval(Object... args) {
            return Long.valueOf( CreateCALayer0(x, y, width, height) );
        } } ).longValue();
     }
+    
+    /** 
+     * Attach a sub CALayer to the root CALayer on the main-thread.
+     * <p>
+     * Method will trigger a <code>display</code>
+     * call to the CALayer hierarchy to enforce resource creation if required, e.g. an NSOpenGLContext.
+     * </p>
+     * <p>
+     * It is mandatory that any related resources, e.g. a shared NSOpenGLContext,
+     * are not locked while calling this method. 
+     * </p>
+     * @see #CreateCALayer(int, int, int, int)
+     * @see #RemoveCASublayer(long, long)
+     */
     public static void AddCASublayer(final long rootCALayer, final long subCALayer) {
         if(0==rootCALayer || 0==subCALayer) {
             throw new IllegalArgumentException("rootCALayer 0x"+Long.toHexString(rootCALayer)+", subCALayer 0x"+Long.toHexString(subCALayer));
@@ -152,6 +171,10 @@ public class OSXUtil implements ToolkitProperties {
            }
         });
     }
+    
+    /** 
+     * Detach a sub CALayer from the root CALayer on the main-thread.
+     */
     public static void RemoveCASublayer(final long rootCALayer, final long subCALayer) {
         if(0==rootCALayer || 0==subCALayer) {
             throw new IllegalArgumentException("rootCALayer 0x"+Long.toHexString(rootCALayer)+", subCALayer 0x"+Long.toHexString(subCALayer));
@@ -161,6 +184,11 @@ public class OSXUtil implements ToolkitProperties {
                RemoveCASublayer0(rootCALayer, subCALayer);
            } } );
     }
+    
+    /** 
+     * Destroy a CALayer on the main-thread.
+     * @see #CreateCALayer(int, int, int, int)
+     */    
     public static void DestroyCALayer(final long caLayer) {
         if(0==caLayer) {
             throw new IllegalArgumentException("caLayer 0x"+Long.toHexString(caLayer));