Remodel OSX/CALayer Threading (commit 896e8b021b39e9415040a57a1d540d7d24b02db1): Run on main-thread w/o blocking ; Misc Changes

Commit 896e8b021b39e9415040a57a1d540d7d24b02db1 moved all native CALayer calls to the current thread
to avoid deadlocks.

Even though this seemed to be fine at least resource GC (release/dealloc calls) were issued
very late in time, probably due to multithreading synchronization of JAWT and/or OSX API.

Example: Our 'TestAddRemove01GLCanvasSwingAWT' test didn't freed CALayer resources incl. GL ctx
         when destroying the objects (AWT Frame, GLCanvas, ..), leading to resource starvation .. eventually.

Remedy is a compromise of behavior before commit 896e8b021b39e9415040a57a1d540d7d24b02db1
and that commit, i.e. to run CALayer lifecycle methods on main-thread, but do not block!

The careful part within MacOSXCGLContext.associateDrawable(..) performs the following block on main-thread:
  - lock the context
  - create NSOpenGLLayer (incl. it's own shared GL context and the DisplayLink)
  - attach NSOpenGLLayer to root CALayer
  - unlock the context

Due to the GL ctx locking, this async offthread operation is safe within our course of operations.

Details:
  - NSOpenGLContext
    - Context and CVDisplayLink creation at init
    - Call [ctx update] if texture/frame size changed
    - 'waitUntilRenderSignal' uses default TO value if given TO is 0 to avoid deadlocks

+++

Misc Changes:
  - Fix object type detection: isMemberOfClass -> isKindOfClass
    - OSXUtil_isNSView0
      OSXUtil_isNSWindow0,
      CGL_isNSOpenGLPixelBuffer

  - MacOSXCGLDrawable/MacOSXPbufferCGLDrawable: remove getNSViewHandle() method.
    MacOSXCGLContext uses common code to detect nature of the drawable handle.

  - MacOSXCGLContext/CALayer: Use safe screenVSyncTimeout values, never 0 to avoid deadlock!

  - JAWTWindow.invalidate: Call detachSurfaceLayer() if not done yet

2 files changed, 87 insertions, 81 deletions

diff --git a/src/jogl/native/macosx/MacOSXWindowSystemInterface-calayer.m b/src/jogl/native/macosx/MacOSXWindowSystemInterface-calayer.m
index 125ca8af8..96783c75d 100644
--- a/src/jogl/native/macosx/MacOSXWindowSystemInterface-calayer.m
+++ b/src/jogl/native/macosx/MacOSXWindowSystemInterface-calayer.m
@@ -117,6 +117,7 @@ extern GLboolean glIsVertexArray (GLuint array);
     CGLContextObj cglCtx = [self CGLContextObj];
 
     DBG_PRINT("MyNSOpenGLContext::dealloc.0 %p (refcnt %d) - CGL-Ctx %p\n", self, (int)[self retainCount], cglCtx);
+    // NSLog(@"MyNSOpenGLContext::dealloc: %@",[NSThread callStackSymbols]);
     [self clearDrawable];
     if( NULL != cglCtx ) {
         CGLDestroyContext( cglCtx );
@@ -143,8 +144,8 @@ extern GLboolean glIsVertexArray (GLuint array);
     NSOpenGLPixelFormat* parentPixelFmt;
     int texWidth;
     int texHeight;
-    int dedicatedWidth;
-    int dedicatedHeight;
+    volatile int dedicatedWidth;
+    volatile int dedicatedHeight;
     volatile NSOpenGLPixelBuffer* pbuffer;
     volatile GLuint textureID;
     volatile NSOpenGLPixelBuffer* newPBuffer;
@@ -184,7 +185,6 @@ extern GLboolean glIsVertexArray (GLuint array);
 
 - (void) setGLEnabled: (Bool) enable;
 - (Bool) validateTexSizeWithDedicatedSize;
-- (Bool) validateTexSize: (int) _texWidth texHeight: (int) _texHeight;
 - (void) setTextureID: (int) _texID;
 
 - (Bool) isSamePBuffer: (NSOpenGLPixelBuffer*) p;
@@ -316,6 +316,14 @@ static const GLfloat gl_verts[] = {
         }
     }
     if(NULL != displayLink) {
+        CVReturn cvres;
+        DBG_PRINT("MyNSOpenGLLayer::openGLContextForPixelFormat.1: setup DisplayLink %p\n", displayLink);
+        cvres = CVDisplayLinkSetCurrentCGDisplayFromOpenGLContext(displayLink, [glContext CGLContextObj], [parentPixelFmt CGLPixelFormatObj]);
+        if(kCVReturnSuccess != cvres) {
+            DBG_PRINT("MyNSOpenGLLayer::init %p, CVDisplayLinkSetCurrentCGDisplayFromOpenGLContext failed: %d\n", self, cvres);
+        }
+    }
+    if(NULL != displayLink) {
         cvres = CVDisplayLinkSetOutputCallback(displayLink, renderMyNSOpenGLLayer, self);
         if(kCVReturnSuccess != cvres) {
             DBG_PRINT("MyNSOpenGLLayer::init %p, CVDisplayLinkSetOutputCallback failed: %d\n", self, cvres);
@@ -354,14 +362,9 @@ static const GLfloat gl_verts[] = {
 
 - (Bool) validateTexSizeWithDedicatedSize
 {
-    return [self validateTexSize: dedicatedWidth texHeight: dedicatedHeight];
-}
-
-- (Bool) validateTexSize: (int) _texWidth texHeight: (int) _texHeight
-{
-    if(_texHeight != texHeight || _texWidth != texWidth) {
-        texWidth = _texWidth;
-        texHeight = _texHeight;
+    if( dedicatedHeight != texHeight || dedicatedWidth != texWidth ) {
+        texWidth = dedicatedWidth;
+        texHeight = dedicatedHeight;
 
         GLfloat texCoordWidth, texCoordHeight;
         if(NULL != pbuffer) {
@@ -477,10 +480,10 @@ static const GLfloat gl_verts[] = {
 - (void)releaseLayer
 {
     DBG_PRINT("MyNSOpenGLLayer::releaseLayer.0: %p (refcnt %d)\n", self, (int)[self retainCount]);
+    [self setGLEnabled: NO];
     [self disableAnimation];
     pthread_mutex_lock(&renderLock);
     [self deallocPBuffer];
-    // [[self openGLContext] release];
     if( NULL != glContext ) {
         [glContext release];
         glContext = NULL;
@@ -570,16 +573,6 @@ static const GLfloat gl_verts[] = {
 {
     DBG_PRINT("MyNSOpenGLLayer::openGLContextForPixelFormat.0: %p (refcnt %d) - pfmt %p, ctx %p, DisplayLink %p\n",
         self, (int)[self retainCount], pixelFormat, glContext, displayLink);
-#ifndef HAS_CADisplayLink
-    if(NULL != displayLink) {
-        CVReturn cvres;
-        DBG_PRINT("MyNSOpenGLLayer::openGLContextForPixelFormat.1: setup DisplayLink %p\n", displayLink);
-        cvres = CVDisplayLinkSetCurrentCGDisplayFromOpenGLContext(displayLink, [glContext CGLContextObj], [pixelFormat CGLPixelFormatObj]);
-        if(kCVReturnSuccess != cvres) {
-            DBG_PRINT("MyNSOpenGLLayer::init %p, CVDisplayLinkSetCurrentCGDisplayFromOpenGLContext failed: %d\n", self, cvres);
-        }
-    }
-#endif
     return glContext;
 }
 
@@ -607,6 +600,9 @@ static const GLfloat gl_verts[] = {
         GLenum textureTarget;
 
         Bool texSizeChanged = [self validateTexSizeWithDedicatedSize];
+        if( texSizeChanged ) {
+            [context update];
+        }
 
         if( NULL != pbuffer ) {
             if( texSizeChanged && 0 != textureID ) {
@@ -812,29 +808,23 @@ static const GLfloat gl_verts[] = {
                 struct timespec t0, t1, td, td2;
                 timespec_now(&t0);
             #endif
-            if(0 < to_micros) {
-                struct timespec to_abs = lastWaitTime;
-                timespec_addmicros(&to_abs, to_micros);
-                #ifdef DBG_SYNC
-                    timespec_subtract(&td, &to_abs, &t0);
-                    fprintf(stderr, ", (%ld) / ", timespec_milliseconds(&td));
-                #endif
-                wr = pthread_cond_timedwait(&renderSignal, &renderLock, &to_abs);
-                #ifdef DBG_SYNC
-                    timespec_now(&t1);
-                    timespec_subtract(&td, &t1, &t0);
-                    timespec_subtract(&td2, &t1, &lastWaitTime);
-                    fprintf(stderr, "(%ld) / (%ld) ms", timespec_milliseconds(&td), timespec_milliseconds(&td2));
-                #endif
-            } else {
-                pthread_cond_wait (&renderSignal, &renderLock);
-                #ifdef DBG_SYNC
-                    timespec_now(&t1);
-                    timespec_subtract(&td, &t1, &t0);
-                    timespec_subtract(&td2, &t1, &lastWaitTime);
-                    fprintf(stderr, "(%ld) / (%ld) ms", timespec_milliseconds(&td), timespec_milliseconds(&td2));
-                #endif
+            if( 0 >= to_micros ) {
+                to_micros = 16666 + 1000; // defaults to 1/60s + 1ms
+                NSLog(@"MyNSOpenGLContext::waitUntilRenderSignal: to_micros was zero, using defaults");
             }
+            struct timespec to_abs = lastWaitTime;
+            timespec_addmicros(&to_abs, to_micros);
+            #ifdef DBG_SYNC
+                timespec_subtract(&td, &to_abs, &t0);
+                fprintf(stderr, ", (%ld) / ", timespec_milliseconds(&td));
+            #endif
+            wr = pthread_cond_timedwait(&renderSignal, &renderLock, &to_abs);
+            #ifdef DBG_SYNC
+                timespec_now(&t1);
+                timespec_subtract(&td, &t1, &t0);
+                timespec_subtract(&td2, &t1, &lastWaitTime);
+                fprintf(stderr, "(%ld) / (%ld) ms", timespec_milliseconds(&td), timespec_milliseconds(&td2));
+            #endif
             ready = YES;
         }
     } while (NO == ready && 0 == wr) ;
@@ -923,9 +913,16 @@ void setNSOpenGLLayerNeedsDisplayPBuffer(NSOpenGLLayer* layer, NSOpenGLPixelBuff
 void releaseNSOpenGLLayer(NSOpenGLLayer* layer) {
     NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init];
     MyNSOpenGLLayer* l = (MyNSOpenGLLayer*) layer;
+
+    [CATransaction begin];
+    [CATransaction setValue:(id)kCFBooleanTrue forKey:kCATransactionDisableActions];
+
     DBG_PRINT("MyNSOpenGLLayer::releaseNSOpenGLLayer.0: %p\n", l);
     [l releaseLayer];
     DBG_PRINT("MyNSOpenGLLayer::releaseNSOpenGLLayer.X: %p\n", l);
+
+    [CATransaction commit];
+
     [pool release];
 }
 
diff --git a/src/jogl/native/macosx/MacOSXWindowSystemInterface.m b/src/jogl/native/macosx/MacOSXWindowSystemInterface.m
index f8faeb8d0..38f1789e3 100644
--- a/src/jogl/native/macosx/MacOSXWindowSystemInterface.m
+++ b/src/jogl/native/macosx/MacOSXWindowSystemInterface.m
@@ -501,6 +501,28 @@ NSView* getNSView(NSOpenGLContext* ctx) {
   return view;
 }
 
+static Bool lockViewIfReady(NSView *view) {
+    Bool viewReady = false;
+
+    if (view != nil) {
+        if ([view lockFocusIfCanDraw] == NO) {
+            DBG_PRINT("lockViewIfReady.1 [view lockFocusIfCanDraw] failed\n");
+        } else {
+            NSRect frame = [view frame];
+            if ((frame.size.width == 0) || (frame.size.height == 0)) {
+                [view unlockFocus];
+                DBG_PRINT("lockViewIfReady.2 view.frame size %dx%d\n", (int)frame.size.width, (int)frame.size.height);
+            } else {
+                DBG_PRINT("lockViewIfReady.X ready and locked\n");
+                viewReady = true;
+            }
+        }
+    } else {
+        DBG_PRINT("lockViewIfReady.3 nil view\n");
+    }
+    return viewReady;
+}
+
 NSOpenGLContext* createContext(NSOpenGLContext* share,
                     NSView* view,
                     Bool incompleteView,
@@ -515,40 +537,18 @@ NSOpenGLContext* createContext(NSOpenGLContext* share,
     DBG_PRINT("createContext.0: share %p, view %p, incompleteView %d, pixfmt %p, opaque %d\n",
         share, view, (int)incompleteView, fmt, opaque);
 
-    if (view != nil) {
-        Bool viewReady = true;
-
-        if(!incompleteView) {
-            if ([view lockFocusIfCanDraw] == NO) {
-                DBG_PRINT("createContext.1 [view lockFocusIfCanDraw] failed\n");
-                viewReady = false;
-            }
-        }
-        if(viewReady) {
-            NSRect frame = [view frame];
-            if ((frame.size.width == 0) || (frame.size.height == 0)) {
-                if(!incompleteView) {
-                    [view unlockFocus];
-                }
-                DBG_PRINT("createContext.2 view.frame size %dx%d\n", (int)frame.size.width, (int)frame.size.height);
-                viewReady = false;
-            }
-        }
+    Bool viewReadyAndLocked = incompleteView ? false : lockViewIfReady(view);
 
-        if (!viewReady)
-        {
-            if (viewNotReady != NULL)
-            {
-                *viewNotReady = 1;
-            }
+    if (nil != viewNotReady) {
+        *viewNotReady = 1;
+    }
 
-            // the view is not ready yet
-            DBG_PRINT("createContext.X: view not ready yet\n");
-            [pool release];
-            return NULL;
-        }
+    if (nil != view && !incompleteView && !viewReadyAndLocked) {
+        DBG_PRINT("createContext.X: Assumed complete view not ready yet\n");
+        [pool release];
+        return NULL;
     }
-    
+
     NSOpenGLContext* ctx = [[NSOpenGLContext alloc] initWithFormat:fmt shareContext:share];
         
     if ( nil != ctx && nil != view ) {
@@ -556,10 +556,8 @@ NSOpenGLContext* createContext(NSOpenGLContext* share,
             GLint zeroOpacity = 0;
             [ctx setValues:&zeroOpacity forParameter:NSOpenGLCPSurfaceOpacity];
         }
-        if(!incompleteView) {
-            DBG_PRINT("createContext.3.0: setView\n");
+        if( viewReadyAndLocked ) {
             [ctx setView:view];
-            DBG_PRINT("createContext.3.X: setView\n");
             [view unlockFocus];        
         }
     }
@@ -571,8 +569,19 @@ NSOpenGLContext* createContext(NSOpenGLContext* share,
 
 void setContextView(NSOpenGLContext* ctx, NSView* view) {
     NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init];
-    if ( nil != ctx && nil != view ) {
-        [ctx setView:view];
+    if ( nil != ctx ) {
+        if ( nil != view ) {
+            Bool viewReadyAndLocked = lockViewIfReady(view);
+            DBG_PRINT("setContextView.0: ctx %p, view %p: setView: %d\n", ctx, view, viewReadyAndLocked);
+            if( viewReadyAndLocked ) {
+                [ctx setView:view];
+                [view unlockFocus];        
+            }
+        } else {
+            DBG_PRINT("setContextView.1: ctx %p, view nil: clearDrawable\n", ctx);
+            [ctx clearDrawable];
+        }
+        DBG_PRINT("setContextView.X\n");
     }
     [pool release];
 }
@@ -725,7 +734,7 @@ void setContextTextureImageToPBuffer(NSOpenGLContext* ctx, NSOpenGLPixelBuffer*
 Bool isNSOpenGLPixelBuffer(uint64_t object) {
   NSObject *nsObj = (NSObject*) (intptr_t) object;
   DBG_PRINT("isNSOpenGLPixelBuffer.0: obj %p\n", object);
-  Bool res = [nsObj isMemberOfClass:[NSOpenGLPixelBuffer class]];
+  Bool res = [nsObj isKindOfClass:[NSOpenGLPixelBuffer class]];
   DBG_PRINT("isNSOpenGLPixelBuffer.X: res %d\n", (int)res);
   return res;
 }