Buig 1389: Fix SIGSEGV on OpenJDK11 on [NSApplicationAWT sendEvent:]

Culprit of the crash and the non propagated action on NSApp main-thread
was _simply_ our OSXUtil_KickNSApp() 'kick alive'
NSApplicationDefined NSEvent sent to the NSApp.

Java11's NSApp code overrides sendEvent and handles
  NSApplicationDefined + subtype=ExecuteBlockEvent
using the given data1 as a function pointer. 8-O

ExecuteBlockEvent defined as 0, which we have sent.

Simply passing subtype=8888 avoids this side-effect.
Whether it is still required to KickNSApp() is another question.

+++

Further, make code a bit more robuts regarding the offscreenSurfaceLayer
at JAWTWindow invalidate. I.e. if still not detached, do the late cleanup there.
This just in case the OSX Context callback to disassociate the drawable
has been missed.

4 files changed, 18 insertions, 8 deletions

diff --git a/src/nativewindow/classes/com/jogamp/nativewindow/awt/JAWTWindow.java b/src/nativewindow/classes/com/jogamp/nativewindow/awt/JAWTWindow.java
index 71a7602e6..5b8171e96 100644
--- a/src/nativewindow/classes/com/jogamp/nativewindow/awt/JAWTWindow.java
+++ b/src/nativewindow/classes/com/jogamp/nativewindow/awt/JAWTWindow.java
@@ -270,6 +270,7 @@ public abstract class JAWTWindow implements NativeWindow, OffscreenLayerSurface,
     invalidateNative();
     jawt = null;
     awtConfig = null;
+    offscreenSurfaceLayer = 0; // Bug 1389
     isOffscreenLayerSurface = false;
     drawable= 0;
     drawable_old = 0;
@@ -534,7 +535,7 @@ public abstract class JAWTWindow implements NativeWindow, OffscreenLayerSurface,
           throw new NativeWindowException("No offscreen layer attached: "+this);
       }
       if(DEBUG) {
-        System.err.println("JAWTWindow.detachSurfaceHandle(): osh "+toHexString(offscreenSurfaceLayer));
+        System.err.println("JAWTWindow.detachSurfaceHandle(): osh "+toHexString(offscreenSurfaceLayer)+" - "+Thread.currentThread().getName());
       }
       detachSurfaceLayerImpl(offscreenSurfaceLayer, detachSurfaceLayerNotify);
   }
diff --git a/src/nativewindow/classes/jogamp/nativewindow/jawt/macosx/MacOSXJAWTWindow.java b/src/nativewindow/classes/jogamp/nativewindow/jawt/macosx/MacOSXJAWTWindow.java
index 5a0c8a79a..f67cc7189 100644
--- a/src/nativewindow/classes/jogamp/nativewindow/jawt/macosx/MacOSXJAWTWindow.java
+++ b/src/nativewindow/classes/jogamp/nativewindow/jawt/macosx/MacOSXJAWTWindow.java
@@ -89,6 +89,7 @@ public class MacOSXJAWTWindow extends JAWTWindow implements MutableSurface {
                              ", osd-set "+offscreenSurfaceDrawableSet+
                              ", osd "+toHexString(offscreenSurfaceDrawable)+
                              ", osl "+toHexString(getAttachedSurfaceLayer())+
+                             ", jsl "+toHexString(jawtSurfaceLayersHandle)+
                              ", rsl "+toHexString(rootSurfaceLayer)+
                              ", wh "+toHexString(windowHandle)+" - "+Thread.currentThread().getName());
       }
@@ -98,9 +99,13 @@ public class MacOSXJAWTWindow extends JAWTWindow implements MutableSurface {
           if(0 != windowHandle) {
               OSXUtil.DestroyNSWindow(windowHandle);
           }
+          final long _offscreenSurfaceLayer = getAttachedSurfaceLayer(); // Bug 1389
           OSXUtil.RunOnMainThread(false, true /* kickNSApp */, new Runnable() {
               @Override
               public void run() {
+                  if( 0 != _offscreenSurfaceLayer ) { // Bug 1389
+                      OSXUtil.RemoveCASublayer(rootSurfaceLayer, _offscreenSurfaceLayer, true);
+                  }
                   if( 0 != jawtSurfaceLayersHandle) {
                       // null rootSurfaceLayer OK
                       UnsetJAWTRootSurfaceLayer0(jawtSurfaceLayersHandle, rootSurfaceLayer);
@@ -211,7 +216,7 @@ public class MacOSXJAWTWindow extends JAWTWindow implements MutableSurface {
               @Override
               public void run() {
                   detachNotify.run();
-                  OSXUtil.RemoveCASublayer(rootSurfaceLayer, layerHandle);
+                  OSXUtil.RemoveCASublayer(rootSurfaceLayer, layerHandle, false);
               } });
   }
 
diff --git a/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java b/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java
index e93c9574f..623880dd5 100644
--- a/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java
+++ b/src/nativewindow/classes/jogamp/nativewindow/macosx/OSXUtil.java
@@ -232,15 +232,16 @@ public class OSXUtil implements ToolkitProperties {
 
     /**
      * Detach a sub CALayer from the root CALayer.
+     * @param subCALayerRelease if true, native call will issue a final {@code [subCALayerRelease release]}.
      */
-    public static void RemoveCASublayer(final long rootCALayer, final long subCALayer) {
+    public static void RemoveCASublayer(final long rootCALayer, final long subCALayer, final boolean subCALayerRelease) {
         if(0==rootCALayer || 0==subCALayer) {
             throw new IllegalArgumentException("rootCALayer 0x"+Long.toHexString(rootCALayer)+", subCALayer 0x"+Long.toHexString(subCALayer));
         }
         if(DEBUG) {
             System.err.println("OSXUtil.DetachCALayer: 0x"+Long.toHexString(subCALayer)+" - "+Thread.currentThread().getName());
         }
-        RemoveCASublayer0(rootCALayer, subCALayer);
+        RemoveCASublayer0(rootCALayer, subCALayer, subCALayerRelease);
     }
 
     /**
@@ -414,7 +415,7 @@ public class OSXUtil implements ToolkitProperties {
     private static native void AddCASublayer0(long rootCALayer, long subCALayer, int x, int y, int width, int height, float contentsScale, int caLayerQuirks);
     private static native void FixCALayerLayout0(long rootCALayer, long subCALayer, boolean visible, int x, int y, int width, int height, int caLayerQuirks);
     private static native void SetCALayerPixelScale0(long rootCALayer, long subCALayer, float contentsScale);
-    private static native void RemoveCASublayer0(long rootCALayer, long subCALayer);
+    private static native void RemoveCASublayer0(long rootCALayer, long subCALayer, boolean subCALayerRelease);
     private static native void DestroyCALayer0(long caLayer);
     private static native void RunOnMainThread0(boolean kickNSApp, Runnable runnable);
     private static native void RunLater0(boolean onMain, boolean kickNSApp, Runnable runnable, int delay);
diff --git a/src/nativewindow/native/macosx/OSXmisc.m b/src/nativewindow/native/macosx/OSXmisc.m
index 1135c5fb2..a24f9291e 100644
--- a/src/nativewindow/native/macosx/OSXmisc.m
+++ b/src/nativewindow/native/macosx/OSXmisc.m
@@ -807,10 +807,10 @@ NS_ENDHANDLER
 /*
  * Class:     Java_jogamp_nativewindow_macosx_OSXUtil
  * Method:    RemoveCASublayer0
- * Signature: (JJ)V
+ * Signature: (JJZ)V
  */
 JNIEXPORT void JNICALL Java_jogamp_nativewindow_macosx_OSXUtil_RemoveCASublayer0
-  (JNIEnv *env, jclass unused, jlong rootCALayer, jlong subCALayer)
+  (JNIEnv *env, jclass unused, jlong rootCALayer, jlong subCALayer, jboolean subCALayerRelease)
 {
     NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init];
     MyCALayer* rootLayer = (MyCALayer*) ((intptr_t) rootCALayer);
@@ -826,6 +826,9 @@ JNIEXPORT void JNICALL Java_jogamp_nativewindow_macosx_OSXUtil_RemoveCASublayer0
 
     [subLayer removeFromSuperlayer];
     [subLayer release]; // Pairs w/ AddCASublayer
+    if( subCALayerRelease ) {
+        [subLayer release];
+    }
     [rootLayer release]; // Pairs w/ AddCASublayer
 
     [CATransaction commit];
@@ -1051,7 +1054,7 @@ static void OSXUtil_KickNSApp() {
                                        timestamp: 0.0
                                     windowNumber: 0
                                          context: nil
-                                         subtype: 0
+                                         subtype: 8888 /* Bug 1389: Avoid JDK impl sendEvent types, utilizing data1 or data2 */
                                            data1: 0
                                            data2: 0];
     [NSApp postEvent: event atStart: true];