Fix Bug #589 (JAWT Offscreen-Layer resize) and Offscreen-Layer setSwapInterval() deadlock; Reuse JAWT instance; Cleanup

- Fixes
  - OSXUtil.CreateCALayer*(..): Pass layer target size (if known).
    This fixes Bug #589

  - MacOSXWindowSystemInterface-pbuffer.m:
     - ALL: displayLink NULL check

     - setSwapInterval(..): lock only for variable setting,
       could deadlock when start/stop CVDisplayLink

  - JAWTWindow.destroy(): use 'surfaceLock' instead of 'synchronized'

- Cleanup / Performance
  - JAWTWindow.lockSurface(): Reuse JAWT instance

  - MacOSXJAWTWindow: AttachJAWTSurfaceLayer0(..) -> SetJAWTRootSurfaceLayer0(..)
                      Reflects semantic better.

- DEBUG
  - JAWTWindow.updateBounds(..) notify of bounds change

6 files changed, 110 insertions, 71 deletions

diff --git a/src/jogl/native/macosx/MacOSXWindowSystemInterface-pbuffer.m b/src/jogl/native/macosx/MacOSXWindowSystemInterface-pbuffer.m
index 3fc6ea0f6..776284cfc 100644
--- a/src/jogl/native/macosx/MacOSXWindowSystemInterface-pbuffer.m
+++ b/src/jogl/native/macosx/MacOSXWindowSystemInterface-pbuffer.m
@@ -171,7 +171,7 @@ static CVReturn renderMyNSOpenGLLayer(CVDisplayLinkRef displayLink,
 #endif
     [self setAsynchronous: YES];
 
-    [self setNeedsDisplayOnBoundsChange: YES]; // FIXME: learn how to recreate on size change!
+    [self setNeedsDisplayOnBoundsChange: YES];
 
     [self setOpaque: opaque ? YES : NO];
 
@@ -192,10 +192,8 @@ static CVReturn renderMyNSOpenGLLayer(CVDisplayLinkRef displayLink,
         [displayLink setPaused: YES];
         [displayLink release];
 #else
-        if(NULL!=displayLink) {
-            CVDisplayLinkStop(displayLink);
-            CVDisplayLinkRelease(displayLink);
-        }
+        CVDisplayLinkStop(displayLink);
+        CVDisplayLinkRelease(displayLink);
 #endif
         displayLink = NULL;
     }
@@ -267,18 +265,6 @@ static CVReturn renderMyNSOpenGLLayer(CVDisplayLinkRef displayLink,
 {
     [context makeCurrentContext];
    
-    /**
-     * v-sync doesn't works w/ NSOpenGLLayer's context .. well :(
-     * Using CVDisplayLink .. see setSwapInterval() below.
-     *
-    if(0 <= swapInterval) {
-        GLint si;
-        [context getValues: &si forParameter: NSOpenGLCPSwapInterval];
-        if(si != swapInterval) {
-            DBG_PRINT("MyNSOpenGLLayer::drawInOpenGLContext %p setSwapInterval: %d -> %d\n", self, si, swapInterval);
-            [context setValues: &swapInterval forParameter: NSOpenGLCPSwapInterval];
-        }
-    } */
     GLenum textureTarget = [pbuffer textureTarget];
     GLfloat texCoordWidth, texCoordHeight;
     {
@@ -359,33 +345,50 @@ static CVReturn renderMyNSOpenGLLayer(CVDisplayLinkRef displayLink,
 
 - (void)setSwapInterval:(int)interval
 {
-    DBG_PRINT("MyNSOpenGLLayer::setSwapInterval: %d\n", interval);
+    /**
+     * v-sync doesn't works w/ NSOpenGLLayer's context .. well :(
+     * Using CVDisplayLink .. see setSwapInterval() below.
+     *
+        GLint si;
+        [context getValues: &si forParameter: NSOpenGLCPSwapInterval];
+        if(si != swapInterval) {
+            DBG_PRINT("MyNSOpenGLLayer::drawInOpenGLContext %p setSwapInterval: %d -> %d\n", self, si, swapInterval);
+            [context setValues: &swapInterval forParameter: NSOpenGLCPSwapInterval];
+        }
+    } */
+
+    pthread_mutex_lock(&renderLock);
+    DBG_PRINT("MyNSOpenGLLayer::setSwapInterval.0: %d - displayLink %p\n", interval, displayLink);
     swapInterval = interval;
     swapIntervalCounter = 0;
-    if(0 < swapInterval) {
-        tc = 0;
-        timespec_now(&t0);
-
-        [self setAsynchronous: NO];
-        #ifdef HAS_CADisplayLink
-            [displayLink setPaused: NO];
-            [displayLink setFrameInterval: interval];
-        #else
-            if(NULL!=displayLink) {
+    pthread_mutex_unlock(&renderLock);
+
+    if(NULL!=displayLink) {
+        if(0 < swapInterval) {
+            tc = 0;
+            timespec_now(&t0);
+
+            [self setAsynchronous: NO];
+            #ifdef HAS_CADisplayLink
+                [displayLink setPaused: NO];
+                [displayLink setFrameInterval: interval];
+            #else
+                DBG_PRINT("MyNSOpenGLLayer::setSwapInterval.1.b.1\n");
                 CVDisplayLinkStart(displayLink);
-                // FIXME: doesn't support interval ..
-            }
-        #endif
-    } else {
-        #ifdef HAS_CADisplayLink
-            [displayLink setPaused: YES];
-        #else
-            if(NULL!=displayLink) {
+                DBG_PRINT("MyNSOpenGLLayer::setSwapInterval.1.b.X\n");
+            #endif
+        } else {
+            #ifdef HAS_CADisplayLink
+                [displayLink setPaused: YES];
+            #else
+                DBG_PRINT("MyNSOpenGLLayer::setSwapInterval.0.b.1\n");
                 CVDisplayLinkStop(displayLink);
-            }
-        #endif
-        [self setAsynchronous: YES];
+                DBG_PRINT("MyNSOpenGLLayer::setSwapInterval.0.b.X\n");
+            #endif
+            [self setAsynchronous: YES];
+        }
     }
+    DBG_PRINT("MyNSOpenGLLayer::setSwapInterval.X: %d\n", interval);
 }
 
 -(void)tick
@@ -414,9 +417,7 @@ NSOpenGLLayer* createNSOpenGLLayer(NSOpenGLContext* ctx, NSOpenGLPixelFormat* fm
 
 void setNSOpenGLLayerSwapInterval(NSOpenGLLayer* layer, int interval) {
   MyNSOpenGLLayer* l = (MyNSOpenGLLayer*) layer;
-  pthread_mutex_lock(&l->renderLock);
   [l setSwapInterval: interval];
-  pthread_mutex_unlock(&l->renderLock);
 }
 
 void waitUntilNSOpenGLLayerIsReady(NSOpenGLLayer* layer, long to_micros) {
diff --git a/src/nativewindow/classes/com/jogamp/nativewindow/awt/JAWTWindow.java b/src/nativewindow/classes/com/jogamp/nativewindow/awt/JAWTWindow.java
index 3815189ef..cffe495f7 100644
--- a/src/nativewindow/classes/com/jogamp/nativewindow/awt/JAWTWindow.java
+++ b/src/nativewindow/classes/com/jogamp/nativewindow/awt/JAWTWindow.java
@@ -135,6 +135,13 @@ public abstract class JAWTWindow implements NativeWindow, OffscreenLayerSurface,
   protected abstract void invalidateNative();
 
   protected final void updateBounds(JAWT_Rectangle jawtBounds) {
+    if(DEBUG) {
+        final Rectangle jb = new Rectangle(jawtBounds.getX(), jawtBounds.getY(), jawtBounds.getWidth(), jawtBounds.getHeight());
+        if(!bounds.equals(jb)) {
+            System.err.println("JAWTWindow.updateBounds: "+bounds+" -> "+jb);
+            Thread.dumpStack();
+        }
+    }
     bounds.setX(jawtBounds.getX());
     bounds.setY(jawtBounds.getY());
     bounds.setWidth(jawtBounds.getWidth());
@@ -186,7 +193,7 @@ public abstract class JAWTWindow implements NativeWindow, OffscreenLayerSurface,
       }
       try {
           if(DEBUG) {
-            System.err.println("JAWTWindow.attachSurfaceHandle(): 0x"+Long.toHexString(layerHandle));
+            System.err.println("JAWTWindow.attachSurfaceHandle(): 0x"+Long.toHexString(layerHandle) + ", bounds "+bounds);
           }
           attachSurfaceLayerImpl(layerHandle);
       } finally {
@@ -288,8 +295,10 @@ public abstract class JAWTWindow implements NativeWindow, OffscreenLayerSurface,
             final AbstractGraphicsDevice adevice = getGraphicsConfiguration().getScreen().getDevice();
             adevice.lock();
             try {
-                jawt = fetchJAWTImpl();
-                isOffscreenLayerSurface = JAWTUtil.isJAWTUsingOffscreenLayer(jawt);
+                if(null == jawt) { // no need to re-fetch for each frame
+                    jawt = fetchJAWTImpl();
+                    isOffscreenLayerSurface = JAWTUtil.isJAWTUsingOffscreenLayer(jawt);
+                }
                 res = lockSurfaceImpl();
                 if(LOCK_SUCCESS == res && drawable_old != drawable) {
                     res = LOCK_SURFACE_CHANGED;
@@ -386,9 +395,14 @@ public abstract class JAWTWindow implements NativeWindow, OffscreenLayerSurface,
   //
 
   @Override
-  public synchronized void destroy() {
-    invalidate();
-    component = null; // don't dispose the AWT component, since we are merely an immutable uplink
+  public void destroy() {
+    surfaceLock.lock();
+    try {
+        invalidate();
+        component = null; // don't dispose the AWT component, since we are merely an immutable uplink
+    } finally {
+        surfaceLock.unlock();
+    }
   }
 
   @Override
diff --git a/src/nativewindow/classes/jogamp/nativewindow/jawt/macosx/MacOSXJAWTWindow.java b/src/nativewindow/classes/jogamp/nativewindow/jawt/macosx/MacOSXJAWTWindow.java
index 0ca5cd297..42fd08df1 100644
--- a/src/nativewindow/classes/jogamp/nativewindow/jawt/macosx/MacOSXJAWTWindow.java
+++ b/src/nativewindow/classes/jogamp/nativewindow/jawt/macosx/MacOSXJAWTWindow.java
@@ -207,20 +207,20 @@ public class MacOSXJAWTWindow extends JAWTWindow implements SurfaceChangeable {
             ((MutableGraphicsConfiguration)getGraphicsConfiguration()).setChosenCapabilities(caps);
         }
         if(0 == rootSurfaceLayerHandle) {
-            rootSurfaceLayerHandle = OSXUtil.CreateCALayer();
+            rootSurfaceLayerHandle = OSXUtil.CreateCALayer(bounds.getX(), bounds.getY(), bounds.getWidth(), bounds.getHeight());
             if(0 == rootSurfaceLayerHandle) {
               OSXUtil.DestroyNSWindow(drawable);
               drawable = 0;
               unlockSurfaceImpl();
               throw new NativeWindowException("Could not create root CALayer: "+this);                
             }
-            if(!AttachJAWTSurfaceLayer0(dsi.getBuffer(), rootSurfaceLayerHandle)) {
+            if(!SetJAWTRootSurfaceLayer0(dsi.getBuffer(), rootSurfaceLayerHandle)) {
               OSXUtil.DestroyCALayer(rootSurfaceLayerHandle);
               rootSurfaceLayerHandle = 0;
               OSXUtil.DestroyNSWindow(drawable);
               drawable = 0;
               unlockSurfaceImpl();
-              throw new NativeWindowException("Could not attach JAWT surfaceLayerHandle: "+this);
+              throw new NativeWindowException("Could not set JAWT rootSurfaceLayerHandle: "+this);
             }
         }
         ret = NativeWindow.LOCK_SUCCESS;
@@ -267,8 +267,8 @@ public class MacOSXJAWTWindow extends JAWTWindow implements SurfaceChangeable {
   }  
   protected Point getLocationOnScreenNativeImpl(final int x0, final int y0) { return null; }
 
-  private static native boolean AttachJAWTSurfaceLayer0(Buffer jawtDrawingSurfaceInfoBuffer, long caLayer);
-  // private static native boolean DetachJAWTSurfaceLayer0(Buffer jawtDrawingSurfaceInfoBuffer, long caLayer);
+  private static native boolean SetJAWTRootSurfaceLayer0(Buffer jawtDrawingSurfaceInfoBuffer, long caLayer);
+  // private static native boolean UnsetJAWTRootSurfaceLayer0(Buffer jawtDrawingSurfaceInfoBuffer, long caLayer);
   
   // Variables for lockSurface/unlockSurface
   private JAWT_DrawingSurface ds;
diff --git a/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java b/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java
index 3ca76a84a..94f949ea3 100644
--- a/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java
+++ b/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java
@@ -77,8 +77,8 @@ public class OSXUtil {
         DestroyNSWindow0(nsWindow);
     }
     
-    public static long CreateCALayer() {
-        return CreateCALayer0();
+    public static long CreateCALayer(int x, int y, int width, int height) {
+        return CreateCALayer0(x, y, width, height);
     }
     public static void AddCASublayer(long rootCALayer, long subCALayer) {
         if(0==rootCALayer || 0==subCALayer) {
@@ -117,7 +117,7 @@ public class OSXUtil {
     private static native void DestroyNSView0(long nsView);
     private static native long CreateNSWindow0(int x, int y, int width, int height);
     private static native void DestroyNSWindow0(long nsWindow);
-    private static native long CreateCALayer0();
+    private static native long CreateCALayer0(int x, int y, int width, int height);
     private static native void AddCASublayer0(long rootCALayer, long subCALayer);
     private static native void RemoveCASublayer0(long rootCALayer, long subCALayer);
     private static native void DestroyCALayer0(long caLayer);
diff --git a/src/nativewindow/native/macosx/OSXmisc.m b/src/nativewindow/native/macosx/OSXmisc.m
index 38ffde98b..e010fc440 100644
--- a/src/nativewindow/native/macosx/OSXmisc.m
+++ b/src/nativewindow/native/macosx/OSXmisc.m
@@ -225,22 +225,25 @@ JNIEXPORT void JNICALL Java_jogamp_nativewindow_macosx_OSXUtil_DestroyNSWindow0
 /*
  * Class:     Java_jogamp_nativewindow_macosx_OSXUtil
  * Method:    CreateCALayer0
- * Signature: (V)J
+ * Signature: (IIII)J
  */
 JNIEXPORT jlong JNICALL Java_jogamp_nativewindow_macosx_OSXUtil_CreateCALayer0
-  (JNIEnv *env, jclass unused)
+  (JNIEnv *env, jclass unused, jint x, jint y, jint width, jint height)
 {
     NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init];
 
     CALayer* layer = [[CALayer alloc] init];
-    DBG_PRINT("CALayer::CreateCALayer.0: %p (refcnt %d)\n", layer, (int)[layer retainCount]);
+    DBG_PRINT("CALayer::CreateCALayer.0: %p %d/%d %dx%d (refcnt %d)\n", layer, x, y, width, height, (int)[layer retainCount]);
+    // avoid zero size
+    if(0 == width) { width = 32; }
+    if(0 == height) { height = 32; }
 
     // initial dummy size !
     CGRect lRect = [layer frame];
-    lRect.origin.x = 0;
-    lRect.origin.y = 0;
-    lRect.size.width = 32;
-    lRect.size.height = 32;
+    lRect.origin.x = x;
+    lRect.origin.y = y;
+    lRect.size.width = width;
+    lRect.size.height = height;
     [layer setFrame: lRect];
     // no animations for add/remove/swap sublayers etc 
     // doesn't work: [layer removeAnimationForKey: kCAOnOrderIn, kCAOnOrderOut, kCATransition]
@@ -422,10 +425,10 @@ JNIEXPORT jboolean JNICALL Java_jogamp_nativewindow_macosx_OSXUtil_IsMainThread0
 
 /*
  * Class:     Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow
- * Method:    AttachJAWTSurfaceLayer
+ * Method:    SetJAWTRootSurfaceLayer0
  * Signature: (JJ)Z
  */
-JNIEXPORT jboolean JNICALL Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow_AttachJAWTSurfaceLayer0
+JNIEXPORT jboolean JNICALL Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow_SetJAWTRootSurfaceLayer0
   (JNIEnv *env, jclass unused, jobject jawtDrawingSurfaceInfoBuffer, jlong caLayer)
 {
     JNF_COCOA_ENTER(env);
@@ -437,9 +440,9 @@ JNIEXPORT jboolean JNICALL Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow
     CALayer* layer = (CALayer*) (intptr_t) caLayer;
     [JNFRunLoop performOnMainThreadWaiting:YES withBlock:^(){
         id <JAWT_SurfaceLayers> surfaceLayers = (id <JAWT_SurfaceLayers>)dsi->platformInfo;
-        DBG_PRINT("CALayer::attachJAWTSurfaceLayer: %p -> %p (refcnt %d)\n", surfaceLayers.layer, layer, (int)[layer retainCount]);
+        DBG_PRINT("CALayer::SetJAWTRootSurfaceLayer.0: %p -> %p (refcnt %d)\n", surfaceLayers.layer, layer, (int)[layer retainCount]);
         surfaceLayers.layer = layer; // already incr. retain count
-        DBG_PRINT("CALayer::attachJAWTSurfaceLayer.X: %p (refcnt %d)\n", layer, (int)[layer retainCount]);
+        DBG_PRINT("CALayer::SetJAWTRootSurfaceLayer.X: %p (refcnt %d)\n", layer, (int)[layer retainCount]);
     }];
     JNF_COCOA_EXIT(env);
     return JNI_TRUE;
@@ -447,9 +450,9 @@ JNIEXPORT jboolean JNICALL Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow
 
 /*
  * Class:     Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow
- * Method:    DetachJAWTSurfaceLayer
+ * Method:    UnsetJAWTRootSurfaceLayer0
  * Signature: (JJ)Z
-JNIEXPORT jboolean JNICALL Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow_DetachJAWTSurfaceLayer0
+JNIEXPORT jboolean JNICALL Java_jogamp_nativewindow_jawt_macosx_MacOSXJAWTWindow_UnsetJAWTRootSurfaceLayer0
   (JNIEnv *env, jclass unused, jobject jawtDrawingSurfaceInfoBuffer, jlong caLayer)
 {
     JNF_COCOA_ENTER(env);
diff --git a/src/test/com/jogamp/opengl/test/junit/jogl/demos/es2/awt/TestGearsES2AWT.java b/src/test/com/jogamp/opengl/test/junit/jogl/demos/es2/awt/TestGearsES2AWT.java
index c6e224548..ba9113af5 100644
--- a/src/test/com/jogamp/opengl/test/junit/jogl/demos/es2/awt/TestGearsES2AWT.java
+++ b/src/test/com/jogamp/opengl/test/junit/jogl/demos/es2/awt/TestGearsES2AWT.java
@@ -32,6 +32,8 @@ import javax.media.opengl.*;
 
 import com.jogamp.opengl.util.Animator;
 import javax.media.opengl.awt.GLCanvas;
+
+import com.jogamp.common.os.Platform;
 import com.jogamp.newt.event.awt.AWTKeyAdapter;
 import com.jogamp.newt.event.awt.AWTWindowAdapter;
 import com.jogamp.newt.event.TraceKeyAdapter;
@@ -41,7 +43,11 @@ import com.jogamp.opengl.test.junit.jogl.demos.es2.GearsES2;
 import com.jogamp.opengl.test.junit.util.MiscUtils;
 import com.jogamp.opengl.test.junit.util.UITestCase;
 import com.jogamp.opengl.test.junit.util.QuitAdapter;
+
+import java.awt.BorderLayout;
+import java.awt.Dimension;
 import java.awt.Frame;
+import java.awt.TextArea;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
@@ -57,6 +63,7 @@ public class TestGearsES2AWT extends UITestCase {
     static boolean firstUIActionOnProcess = false;
     static boolean forceES2 = false;
     static boolean shallUseOffscreenLayer = false;
+    static boolean addComp = true;
     static int swapInterval = 1;
     static boolean showFPS = false;    
 
@@ -77,8 +84,19 @@ public class TestGearsES2AWT extends UITestCase {
         final GLCanvas glCanvas = new GLCanvas(caps);
         Assert.assertNotNull(glCanvas);
         glCanvas.setShallUseOffscreenLayer(shallUseOffscreenLayer);
-        frame.add(glCanvas);
-        frame.setSize(512, 512);
+        Dimension glc_sz = new Dimension(width, height);
+        glCanvas.setMinimumSize(glc_sz);
+        glCanvas.setPreferredSize(glc_sz);
+        glCanvas.setSize(glc_sz);
+        if(addComp) {
+            final TextArea ta = new TextArea(2, 20);
+            ta.append("0123456789");
+            ta.append(Platform.getNewline());
+            ta.append("Some Text");
+            ta.append(Platform.getNewline());
+            frame.add(ta, BorderLayout.SOUTH);
+        }
+        frame.add(glCanvas, BorderLayout.CENTER);
         frame.setTitle("Gears AWT Test (translucent "+!caps.isBackgroundOpaque()+"), swapInterval "+swapInterval);
 
         glCanvas.addGLEventListener(new GearsES2(swapInterval));
@@ -91,12 +109,13 @@ public class TestGearsES2AWT extends UITestCase {
 
         javax.swing.SwingUtilities.invokeAndWait(new Runnable() {
             public void run() {
+                frame.pack();
                 frame.setVisible(true);
             }});
         animator.setUpdateFPSFrames(60, System.err);
         animator.start();
 
-        while(!quitAdapter.shouldQuit() && animator.isAnimating() && animator.getTotalFPSDuration()<duration) {
+        while(!quitAdapter.shouldQuit() /* && animator.isAnimating() */ && animator.getTotalFPSDuration()<duration) {
             Thread.sleep(100);
         }
 
@@ -145,6 +164,8 @@ public class TestGearsES2AWT extends UITestCase {
                 firstUIActionOnProcess = true;
             } else if(args[i].equals("-wait")) {
                 waitForKey = true;
+            } else if(args[i].equals("-justGears")) {
+                addComp = false;
             }
         }
         System.err.println("forceES2 "+forceES2);