OSX CALayer / MacOSXCGLContext: Remove pbuffer swap garbage; Split pbuffer/FBO revalidation/redraw; Don't sync @ needsDisplay

Simplified pbuffer replacement:
  Simply keep holding the pbuffer reference in native code
  until receiving newPBuffer @ swap.
  The newPBuffer will be established @draw method.
  This allows removing interaction at destroy pbuffer,
  which caused garbage and crash.

Remove pbuffer swap garbage
  See above.

Split pbuffer/FBO revalidation/redraw

Don't sync @ needsDisplay
  - No need to sync @ repaint command setNSOpenGLLayerNeedsDisplay*
    since we use volatiles and all lifecycle action is
    done @draw -> faster

3 files changed, 106 insertions, 73 deletions

diff --git a/make/stub_includes/opengl/macosx-window-system.h b/make/stub_includes/opengl/macosx-window-system.h
index 828e3c01e..64d3d736b 100644
--- a/make/stub_includes/opengl/macosx-window-system.h
+++ b/make/stub_includes/opengl/macosx-window-system.h
@@ -57,8 +57,8 @@ Bool isNSOpenGLPixelBuffer(uint64_t object);
 NSOpenGLLayer* createNSOpenGLLayer(NSOpenGLContext* ctx, NSOpenGLPixelFormat* fmt, NSOpenGLPixelBuffer* p, uint32_t texID, Bool opaque, int texWidth, int texHeight);
 void setNSOpenGLLayerSwapInterval(NSOpenGLLayer* layer, int interval);
 void waitUntilNSOpenGLLayerIsReady(NSOpenGLLayer* layer, long to_micros);
-void flushNSOpenGLLayerPBuffer(NSOpenGLLayer* layer);
-void setNSOpenGLLayerNeedsDisplay(NSOpenGLLayer* layer, NSOpenGLPixelBuffer* p, uint32_t texID, int texWidth, int texHeight);
+void setNSOpenGLLayerNeedsDisplayFBO(NSOpenGLLayer* layer, uint32_t texID, int texWidth, int texHeight);
+void setNSOpenGLLayerNeedsDisplayPBuffer(NSOpenGLLayer* layer, NSOpenGLPixelBuffer* p, int texWidth, int texHeight);
 void releaseNSOpenGLLayer(NSOpenGLLayer *glLayer);
 
 void* getProcAddress(const char *procName);
diff --git a/src/jogl/classes/jogamp/opengl/macosx/cgl/MacOSXCGLContext.java b/src/jogl/classes/jogamp/opengl/macosx/cgl/MacOSXCGLContext.java
index e04955d87..929ac00c2 100644
--- a/src/jogl/classes/jogamp/opengl/macosx/cgl/MacOSXCGLContext.java
+++ b/src/jogl/classes/jogamp/opengl/macosx/cgl/MacOSXCGLContext.java
@@ -438,7 +438,7 @@ public abstract class MacOSXCGLContext extends GLContextImpl
       private float screenVSyncTimeout; // microSec
       private int vsyncTimeout;    // microSec - for nsOpenGLLayer mode
       private int lastWidth=0, lastHeight=0; // allowing to detect size change
-      private long lastPBufferHandle = 0; // allowing to detect pbuffer recreation
+      private boolean needsSetContextPBuffer = false;
 
       @Override
       public boolean isNSContext() { return true; }
@@ -577,7 +577,6 @@ public abstract class MacOSXCGLContext extends GLContextImpl
 
       @Override
       public boolean destroy(long ctx) {
-          lastPBufferHandle = 0;
           return CGL.deleteContext(ctx, true);
       }
 
@@ -592,19 +591,20 @@ public abstract class MacOSXCGLContext extends GLContextImpl
                   final long ctx = MacOSXCGLContext.this.getHandle();
                   final int texID;
                   final long drawableHandle = drawable.getHandle();
+                  final long pbufferHandle;
                   if(drawable instanceof GLFBODrawableImpl) {
                       final GLFBODrawableImpl fbod = (GLFBODrawableImpl)drawable;
-                      texID = fbod.getTextureBuffer(GL.GL_FRONT).getName();                    
+                      texID = fbod.getTextureBuffer(GL.GL_FRONT).getName();
+                      pbufferHandle = 0;
                       fbod.setSwapBufferContext(new GLFBODrawableImpl.SwapBufferContext() {
                           public void swapBuffers(boolean doubleBuffered) {
                               MacOSXCGLContext.NSOpenGLImpl.this.swapBuffers();                            
                           } } ) ;                    
-                      lastPBufferHandle = 0;
                   } else if( chosenCaps.isPBuffer() && CGL.isNSOpenGLPixelBuffer(drawableHandle) ) {
                       texID = 0;
-                      lastPBufferHandle = drawableHandle;
+                      pbufferHandle = drawableHandle;
                       if(0 != drawableHandle) { // complete 'validatePBufferConfig(..)' procedure
-                          CGL.setContextPBuffer(ctx, drawableHandle);
+                          CGL.setContextPBuffer(ctx, pbufferHandle);
                       }
                   } else {
                       throw new GLException("BackingLayerHost w/ unknown handle (!FBO, !PBuffer): "+drawable);
@@ -614,9 +614,9 @@ public abstract class MacOSXCGLContext extends GLContextImpl
                   if(0>=lastWidth || 0>=lastHeight || !drawable.isRealized()) {
                       throw new GLException("Drawable not realized yet or invalid texture size, texSize "+lastWidth+"x"+lastHeight+", "+drawable);
                   }
-                  nsOpenGLLayer = CGL.createNSOpenGLLayer(ctx, nsOpenGLLayerPFmt, lastPBufferHandle, texID, chosenCaps.isBackgroundOpaque(), lastWidth, lastHeight);
+                  nsOpenGLLayer = CGL.createNSOpenGLLayer(ctx, nsOpenGLLayerPFmt, pbufferHandle, texID, chosenCaps.isBackgroundOpaque(), lastWidth, lastHeight);
                   if (DEBUG) {
-                      System.err.println("NS create nsOpenGLLayer "+toHexString(nsOpenGLLayer)+" w/ pbuffer "+toHexString(lastPBufferHandle)+", texID "+texID+", texSize "+lastWidth+"x"+lastHeight+", "+drawable);
+                      System.err.println("NS create nsOpenGLLayer "+toHexString(nsOpenGLLayer)+" w/ pbuffer "+toHexString(pbufferHandle)+", texID "+texID+", texSize "+lastWidth+"x"+lastHeight+", "+drawable);
                   }
                   backingLayerHost.attachSurfaceLayer(nsOpenGLLayer);
                   setSwapInterval(1); // enabled per default in layered surface                
@@ -642,18 +642,16 @@ public abstract class MacOSXCGLContext extends GLContextImpl
                   CGL.deletePixelFormat(nsOpenGLLayerPFmt);
                   nsOpenGLLayerPFmt = 0;
               }
-              lastPBufferHandle = 0;
           }
           backingLayerHost = null;
           return true;
       }
 
       private final void validatePBufferConfig(long ctx) {
-          final GLCapabilitiesImmutable chosenCaps = drawable.getChosenGLCapabilities();
           final long drawableHandle = drawable.getHandle();
-          if( chosenCaps.isPBuffer() && lastPBufferHandle != drawableHandle && CGL.isNSOpenGLPixelBuffer(drawableHandle) ) {
+          if( needsSetContextPBuffer && 0 != drawableHandle && CGL.isNSOpenGLPixelBuffer(drawableHandle) ) {
               // Must associate the pbuffer with our newly-created context
-              lastPBufferHandle = drawableHandle;
+              needsSetContextPBuffer = false;
               if(0 != drawableHandle) {
                   CGL.setContextPBuffer(ctx, drawableHandle);
               }
@@ -724,13 +722,8 @@ public abstract class MacOSXCGLContext extends GLContextImpl
 
       @Override
       public boolean detachPBuffer() {
-          if(0 != lastPBufferHandle) {
-              lastPBufferHandle = 0;
-              if(0 != nsOpenGLLayer) {
-                  CGL.flushNSOpenGLLayerPBuffer(nsOpenGLLayer); // notify invalid pbuffer
-              }
-              // CGL.setContextPBuffer(contextHandle, 0); // doesn't work, i.e. not taking nil
-          }
+          needsSetContextPBuffer = true;
+          // CGL.setContextPBuffer(contextHandle, 0); // doesn't work, i.e. not taking nil
           return true;
       }
       
@@ -759,12 +752,13 @@ public abstract class MacOSXCGLContext extends GLContextImpl
               
               final int texID;
               final boolean valid;
-              if(drawable instanceof GLFBODrawableImpl) {
+              final boolean isFBO = drawable instanceof GLFBODrawableImpl;
+              if( isFBO ){
                   texID = ((GLFBODrawableImpl)drawable).getTextureBuffer(GL.GL_FRONT).getName();
                   valid = 0 != texID;
               } else {
                   texID = 0;
-                  valid = 0 != lastPBufferHandle;
+                  valid = 0 != drawable.getHandle();
               }
               if(valid) {
                   if(0 == skipSync) {
@@ -776,8 +770,13 @@ public abstract class MacOSXCGLContext extends GLContextImpl
                   }
                   res = CGL.flushBuffer(contextHandle);
                   if(res) {
-                      // trigger CALayer to update incl. possible surface change
-                      CGL.setNSOpenGLLayerNeedsDisplay(nsOpenGLLayer, lastPBufferHandle, texID, lastWidth, lastHeight);
+                      if(isFBO) {
+                          // trigger CALayer to update incl. possible surface change (texture)
+                          CGL.setNSOpenGLLayerNeedsDisplayFBO(nsOpenGLLayer, texID, lastWidth, lastHeight);                          
+                      } else {
+                          // trigger CALayer to update incl. possible surface change (new pbuffer handle)
+                          CGL.setNSOpenGLLayerNeedsDisplayPBuffer(nsOpenGLLayer, drawable.getHandle(), lastWidth, lastHeight);                          
+                      }
                   }
               } else {
                   res = true;
diff --git a/src/jogl/native/macosx/MacOSXWindowSystemInterface-calayer.m b/src/jogl/native/macosx/MacOSXWindowSystemInterface-calayer.m
index f3495efff..e83d0adcb 100644
--- a/src/jogl/native/macosx/MacOSXWindowSystemInterface-calayer.m
+++ b/src/jogl/native/macosx/MacOSXWindowSystemInterface-calayer.m
@@ -87,6 +87,7 @@
     volatile GLuint textureID;
     volatile int texWidth;
     volatile int texHeight;
+    volatile NSOpenGLPixelBuffer* newPBuffer;
 #ifdef HAS_CADisplayLink
     CADisplayLink* displayLink;
 #else
@@ -117,7 +118,9 @@
 - (Bool) validateTexSize: (int) _texWidth texHeight: (int) _texHeight;
 - (void) setTextureID: (int) _texID;
 
-- (void) validatePBuffer: (NSOpenGLPixelBuffer*) p;
+- (Bool) isSamePBuffer: (NSOpenGLPixelBuffer*) p;
+- (void) setNewPBuffer: (NSOpenGLPixelBuffer*)p;
+- (void) applyNewPBuffer;
 
 - (NSOpenGLContext *)openGLContextForPixelFormat:(NSOpenGLPixelFormat *)pixelFormat;
 - (void)disableAnimation;
@@ -199,6 +202,7 @@ static const GLfloat gl_verts[] = {
     [self validateTexSizeWithNewSize];
     [self setTextureID: texID];
 
+    newPBuffer = NULL;
     pbuffer = p;
     if(NULL != pbuffer) {
         [pbuffer retain];
@@ -317,22 +321,60 @@ static const GLfloat gl_verts[] = {
     textureID = _texID;
 }
 
-- (void) validatePBuffer: (NSOpenGLPixelBuffer*) p
+- (Bool) isSamePBuffer: (NSOpenGLPixelBuffer*) p
 {
-    if( pbuffer != p ) {
-        DBG_PRINT("MyNSOpenGLLayer::validatePBuffer.0 %p, pbuffer %p, (refcnt %d)\n", self, p, (int)[self retainCount]);
+    return pbuffer == p || newPBuffer == p;
+}
 
-        SYNC_PRINT("{PB-nil}");
+- (void)setNewPBuffer: (NSOpenGLPixelBuffer*)p
+{
+    SYNC_PRINT("<NP-S %p -> %p>", pbuffer, p);
+    newPBuffer = p;
+    [newPBuffer retain];
+}
 
-        [self deallocPBuffer];
+- (void) applyNewPBuffer
+{
+    if( NULL != newPBuffer ) { // volatile OK
+        SYNC_PRINT("<NP-A %p -> %p>", pbuffer, newPBuffer);
 
-        pbuffer = p;
-        if(NULL != pbuffer) {
-            [pbuffer retain];
+        if( 0 != textureID ) {
+            glDeleteTextures(1, &textureID);
+            [self setTextureID: 0];
         }
-        [self setTextureID: 0];
+        [pbuffer release];
+
+        pbuffer = newPBuffer;
+        newPBuffer = NULL;
+    }
+}
+
+- (void)deallocPBuffer
+{
+    if(NULL != pbuffer) {
+        NSOpenGLContext* context = [self openGLContext];
+        if(NULL!=context) {
+            [context makeCurrentContext];
+
+            DBG_PRINT("MyNSOpenGLLayer::deallocPBuffer (with ctx) %p (refcnt %d) - context %p, pbuffer %p, texID %d\n", self, (int)[self retainCount], context, pbuffer, (int)textureID);
+
+            if( 0 != textureID ) {
+                glDeleteTextures(1, &textureID);
+                [self setTextureID: 0];
+            }
+            if(NULL != pbuffer) {
+                [pbuffer release];
+                pbuffer = NULL;
+            }
+            if(NULL != newPBuffer) {
+                [newPBuffer release];
+                newPBuffer = NULL;
+            }
 
-        shallDraw = NO;
+            [context clearDrawable];
+        } else {
+            DBG_PRINT("MyNSOpenGLLayer::deallocPBuffer (w/o ctx) %p (refcnt %d) - context %p, pbuffer %p, texID %d\n", self, (int)[self retainCount], context, pbuffer, (int)textureID);
+        }
     }
 }
 
@@ -371,29 +413,6 @@ static const GLfloat gl_verts[] = {
     pthread_mutex_unlock(&renderLock);
 }
 
-- (void)deallocPBuffer
-{
-    if(NULL != pbuffer) {
-        NSOpenGLContext* context = [self openGLContext];
-        if(NULL!=context) {
-            [context makeCurrentContext];
-
-            DBG_PRINT("MyNSOpenGLLayer::deallocPBuffer (with ctx) %p (refcnt %d) - context %p, pbuffer %p, texID %d\n", self, (int)[self retainCount], context, pbuffer, (int)textureID);
-
-            if( 0 != textureID ) {
-                glDeleteTextures(1, &textureID);
-            }
-            [pbuffer release];
-
-            [context clearDrawable];
-        } else {
-            DBG_PRINT("MyNSOpenGLLayer::deallocPBuffer (w/o ctx) %p (refcnt %d) - context %p, pbuffer %p, texID %d\n", self, (int)[self retainCount], context, pbuffer, (int)textureID);
-        }
-        pbuffer = NULL;
-        [self setTextureID: 0];
-    }
-}
-
 - (void)releaseLayer
 {
     DBG_PRINT("MyNSOpenGLLayer::releaseLayer.0: %p (refcnt %d)\n", self, (int)[self retainCount]);
@@ -423,7 +442,7 @@ static const GLfloat gl_verts[] = {
 
 - (Bool)isGLSourceValid
 {
-    return NULL != pbuffer || 0 != textureID ;
+    return NULL != pbuffer || NULL != newPBuffer || 0 != textureID ;
 }
 
 - (void)resizeWithOldSuperlayerSize:(CGSize)size
@@ -435,7 +454,7 @@ static const GLfloat gl_verts[] = {
 
     newTexWidth = lRectS.size.width;
     newTexHeight = lRectS.size.height;
-    shallDraw = YES;
+    shallDraw = [self isGLSourceValid];
     SYNC_PRINT("<SZ %dx%d>", newTexWidth, newTexHeight);
 
     [super resizeWithOldSuperlayerSize: size];
@@ -455,9 +474,13 @@ static const GLfloat gl_verts[] = {
     SYNC_PRINT("<* ");
     // NSLog(@"MyNSOpenGLLayer::DRAW: %@",[NSThread callStackSymbols]);
 
-    if( shallDraw && ( NULL != pbuffer || 0 != textureID ) ) {
+    if( shallDraw && ( NULL != pbuffer || NULL != newPBuffer || 0 != textureID ) ) {
         [context makeCurrentContext];
 
+        if( NULL != newPBuffer ) { // volatile OK
+            [self applyNewPBuffer];
+        }
+
         GLenum textureTarget;
 
         Bool texSizeChanged = [self validateTexSizeWithNewSize];
@@ -665,30 +688,41 @@ void waitUntilNSOpenGLLayerIsReady(NSOpenGLLayer* layer, long to_micros) {
     [pool release];
 }
 
-void flushNSOpenGLLayerPBuffer(NSOpenGLLayer* layer) {
+void setNSOpenGLLayerNeedsDisplayFBO(NSOpenGLLayer* layer, uint32_t texID, int texWidth, int texHeight) {
     MyNSOpenGLLayer* l = (MyNSOpenGLLayer*) layer;
     NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init];
+    Bool shallDraw;
 
-    pthread_mutex_lock(&l->renderLock);
-    [l validatePBuffer:0];
-    pthread_mutex_unlock(&l->renderLock);
+    // volatile OK
+    [l setTextureID: texID];
+    shallDraw = [l isGLSourceValid];
+    l->shallDraw = shallDraw;
 
+    SYNC_PRINT("<! T%dx%d O%dx%d %d>", texWidth, texHeight, l->newTexWidth, l->newTexHeight, (int)shallDraw);
+    if(shallDraw) {
+        if ( [NSThread isMainThread] == YES ) {
+          [l setNeedsDisplay];
+        } else {
+          // don't wait - using doublebuffering
+          [l performSelectorOnMainThread:@selector(setNeedsDisplay) withObject:nil waitUntilDone:NO];
+        }
+    }
+    // DBG_PRINT("MyNSOpenGLLayer::setNSOpenGLLayerNeedsDisplay %p\n", l);
     [pool release];
 }
 
-void setNSOpenGLLayerNeedsDisplay(NSOpenGLLayer* layer, NSOpenGLPixelBuffer* p, uint32_t texID, int texWidth, int texHeight) {
+void setNSOpenGLLayerNeedsDisplayPBuffer(NSOpenGLLayer* layer, NSOpenGLPixelBuffer* p, int texWidth, int texHeight) {
     MyNSOpenGLLayer* l = (MyNSOpenGLLayer*) layer;
     NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init];
     Bool shallDraw;
 
-    pthread_mutex_lock(&l->renderLock);
-    [l validatePBuffer:p];
-    // l->newTexWidth = texWidth;
-    // l->newTexHeight = texHeight;
-    [l setTextureID: texID];
+    if( NO == [l isSamePBuffer: p] ) {
+        [l setNewPBuffer: p];
+    }
+
+    // volatile OK
     shallDraw = [l isGLSourceValid];
     l->shallDraw = shallDraw;
-    pthread_mutex_unlock(&l->renderLock);
 
     SYNC_PRINT("<! T%dx%d O%dx%d %d>", texWidth, texHeight, l->newTexWidth, l->newTexHeight, (int)shallDraw);
     if(shallDraw) {