X11Util/Xmisc setX11ErrorHandler: Force setting X11 error handler if invoked from Java ensuring to overwrite other TKs handler

SWT sets it's error handler, which exits application in case of an X11 error (e.g. n/a GL3 impl.).
We need to allow overwriting it, hence allowing multiple calls (NEWT, GLContext).

This patch also makes native NativewindowCommon_x11ErrorHandlerEnable(..) function static.

2 files changed, 25 insertions, 23 deletions

diff --git a/src/nativewindow/native/x11/Xmisc.c b/src/nativewindow/native/x11/Xmisc.c
index 129baac62..c73952693 100644
--- a/src/nativewindow/native/x11/Xmisc.c
+++ b/src/nativewindow/native/x11/Xmisc.c
@@ -220,12 +220,16 @@ static int x11ErrorHandler(Display *dpy, XErrorEvent *e)
     return 0;
 }
 
-void NativewindowCommon_x11ErrorHandlerEnable(JNIEnv * env, Display *dpy, int onoff, int quiet, int sync) {
+static void NativewindowCommon_x11ErrorHandlerEnable(JNIEnv * env, Display *dpy, int force, int onoff, int quiet, int sync) {
     errorHandlerQuiet = quiet;
     if(onoff) {
-        if(NULL==origErrorHandler) {
+        if(force || NULL==origErrorHandler) {
+            XErrorHandler prevErrorHandler;
             setupJVMVars(env);
-            origErrorHandler = XSetErrorHandler(x11ErrorHandler);
+            prevErrorHandler = XSetErrorHandler(x11ErrorHandler);
+            if(x11ErrorHandler != prevErrorHandler) { // if forced don't overwrite w/ orig w/ our handler
+                origErrorHandler = prevErrorHandler;
+            }
             if(sync && NULL!=dpy) {
                 XSync(dpy, False);
             }
@@ -292,7 +296,7 @@ Java_jogamp_nativewindow_x11_X11Util_initialize0(JNIEnv *env, jclass clazz, jboo
 
         _initClazzAccess(env);
         x11IOErrorHandlerEnable(1, env);
-        NativewindowCommon_x11ErrorHandlerEnable(env, NULL, 1, 0, 0 /* no dpy, no sync */);
+        NativewindowCommon_x11ErrorHandlerEnable(env, NULL, 1, 1, 0, 0 /* no dpy, force, no sync */);
         _initialized=1;
         if(JNI_TRUE == debug) {
             fprintf(stderr, "Info: NativeWindow native init passed\n");
@@ -303,13 +307,13 @@ Java_jogamp_nativewindow_x11_X11Util_initialize0(JNIEnv *env, jclass clazz, jboo
 
 JNIEXPORT void JNICALL 
 Java_jogamp_nativewindow_x11_X11Util_shutdown0(JNIEnv *env, jclass _unused) {
-    NativewindowCommon_x11ErrorHandlerEnable(env, NULL, 0, 0, 0 /* no dpy, no sync */);
+    NativewindowCommon_x11ErrorHandlerEnable(env, NULL, 0, 0, 0, 0 /* no dpy, no sync */);
     x11IOErrorHandlerEnable(0, env);
 }
 
 JNIEXPORT void JNICALL 
 Java_jogamp_nativewindow_x11_X11Util_setX11ErrorHandler0(JNIEnv *env, jclass _unused, jboolean onoff, jboolean quiet) {
-    NativewindowCommon_x11ErrorHandlerEnable(env, NULL, onoff ? 1 : 0, quiet ? 1 : 0, 0 /* no dpy, no sync */);
+    NativewindowCommon_x11ErrorHandlerEnable(env, NULL, 1, onoff ? 1 : 0, quiet ? 1 : 0, 0 /* no dpy, force, no sync */);
 }
 
 /*   Java->C glue code:
@@ -334,9 +338,9 @@ Java_jogamp_nativewindow_x11_X11Lib_XGetVisualInfo1__JJLjava_nio_ByteBuffer_2Lja
   if (arg3 != NULL) {
     _ptr3 = (int *) (((char*) (*env)->GetPrimitiveArrayCritical(env, arg3, NULL)) + arg3_byte_offset);
   }
-  NativewindowCommon_x11ErrorHandlerEnable(env, (Display *) (intptr_t) arg0, 1, 0, 0);
+  NativewindowCommon_x11ErrorHandlerEnable(env, (Display *) (intptr_t) arg0, 0, 1, 0, 0);
   _res = XGetVisualInfo((Display *) (intptr_t) arg0, (long) arg1, (XVisualInfo *) _ptr2, (int *) _ptr3);
-  // NativewindowCommon_x11ErrorHandlerEnable(env, (Display *) (intptr_t) arg0, 0, 0, 0);
+  // NativewindowCommon_x11ErrorHandlerEnable(env, (Display *) (intptr_t) arg0, 0, 0, 0, 0);
   count = _ptr3[0];
   if (arg3 != NULL) {
     (*env)->ReleasePrimitiveArrayCritical(env, arg3, _ptr3, 0);
@@ -363,7 +367,7 @@ Java_jogamp_nativewindow_x11_X11Lib_GetVisualIDFromWindow(JNIEnv *env, jclass _u
         return;
     }
 
-    NativewindowCommon_x11ErrorHandlerEnable(env, dpy, 1, 0, 1);
+    NativewindowCommon_x11ErrorHandlerEnable(env, dpy, 0, 1, 0, 1);
     memset(&xwa, 0, sizeof(XWindowAttributes));
     XGetWindowAttributes(dpy, w, &xwa);
     if(NULL != xwa.visual) {
@@ -371,7 +375,7 @@ Java_jogamp_nativewindow_x11_X11Lib_GetVisualIDFromWindow(JNIEnv *env, jclass _u
     } else {
         r = 0;
     }
-    // NativewindowCommon_x11ErrorHandlerEnable(env, dpy, 0, 0, 1);
+    // NativewindowCommon_x11ErrorHandlerEnable(env, dpy, 0, 0, 0, 1);
 
     return r;
 }
@@ -383,9 +387,9 @@ Java_jogamp_nativewindow_x11_X11Lib_DefaultVisualID(JNIEnv *env, jclass _unused,
     if(0==display) {
         NativewindowCommon_FatalError(env, "invalid display connection..");
     }
-  NativewindowCommon_x11ErrorHandlerEnable(env, (Display *) (intptr_t) display, 1, 0, 0);
+  NativewindowCommon_x11ErrorHandlerEnable(env, (Display *) (intptr_t) display, 0, 1, 0, 0);
   r = (jint) XVisualIDFromVisual( DefaultVisual( (Display*) (intptr_t) display, screen ) );
-  // NativewindowCommon_x11ErrorHandlerEnable(env, (Display *) (intptr_t) display, 0, 0, 0);
+  // NativewindowCommon_x11ErrorHandlerEnable(env, (Display *) (intptr_t) display, 0, 0, 0, 0);
   return r;
 }
 
@@ -426,9 +430,9 @@ Java_jogamp_nativewindow_x11_X11Lib_XCloseDisplay__J(JNIEnv *env, jclass _unused
   if(0==display) {
       NativewindowCommon_FatalError(env, "invalid display connection..");
   }
-  NativewindowCommon_x11ErrorHandlerEnable(env, NULL, 1, 0, 0);
+  NativewindowCommon_x11ErrorHandlerEnable(env, NULL, 0, 1, 0, 0);
   _res = XCloseDisplay((Display *) (intptr_t) display);
-  // NativewindowCommon_x11ErrorHandlerEnable(env, NULL, 0, 0, 0);
+  // NativewindowCommon_x11ErrorHandlerEnable(env, NULL, 0, 0, 0, 0);
   return _res;
 }
 
@@ -466,7 +470,7 @@ JNIEXPORT jlong JNICALL Java_jogamp_nativewindow_x11_X11Lib_CreateDummyWindow
         return 0;
     }
 
-    NativewindowCommon_x11ErrorHandlerEnable(env, dpy, 1, 0, 0);
+    NativewindowCommon_x11ErrorHandlerEnable(env, dpy, 0, 1, 0, 0);
 
     scrn = ScreenOfDisplay(dpy, scrn_idx);
 
@@ -486,7 +490,7 @@ JNIEXPORT jlong JNICALL Java_jogamp_nativewindow_x11_X11Lib_CreateDummyWindow
 
     if (visual==NULL)
     { 
-        // NativewindowCommon_x11ErrorHandlerEnable(env, dpy, 0, 0, 1);
+        // NativewindowCommon_x11ErrorHandlerEnable(env, dpy, 0, 0, 0, 1);
         NativewindowCommon_throwNewRuntimeException(env, "could not query Visual by given VisualID, bail out!");
         return 0;
     } 
@@ -530,7 +534,7 @@ JNIEXPORT jlong JNICALL Java_jogamp_nativewindow_x11_X11Lib_CreateDummyWindow
 
     XSelectInput(dpy, window, 0); // no events
 
-    // NativewindowCommon_x11ErrorHandlerEnable(env, dpy, 0, 0, 1);
+    // NativewindowCommon_x11ErrorHandlerEnable(env, dpy, 0, 0, 0, 1);
 
     DBG_PRINT( "X11: [CreateWindow] created window %p on display %p\n", window, dpy);
 
@@ -554,11 +558,11 @@ JNIEXPORT void JNICALL Java_jogamp_nativewindow_x11_X11Lib_DestroyDummyWindow
         return;
     }
 
-    NativewindowCommon_x11ErrorHandlerEnable(env, dpy, 1, 0, 0);
+    NativewindowCommon_x11ErrorHandlerEnable(env, dpy, 0, 1, 0, 0);
     XUnmapWindow(dpy, w);
     XSync(dpy, False);
     XDestroyWindow(dpy, w);
-    // NativewindowCommon_x11ErrorHandlerEnable(env, dpy, 0, 0, 1);
+    // NativewindowCommon_x11ErrorHandlerEnable(env, dpy, 0, 0, 0, 1);
 }
 
 /*
@@ -582,11 +586,11 @@ JNIEXPORT jobject JNICALL Java_jogamp_nativewindow_x11_X11Lib_GetRelativeLocatio
     if( 0 == jdest_win ) { dest_win = root; }
     if( 0 == jsrc_win ) { src_win = root; }
 
-    NativewindowCommon_x11ErrorHandlerEnable(env, dpy, 1, 0, 0);
+    NativewindowCommon_x11ErrorHandlerEnable(env, dpy, 0, 1, 0, 0);
 
     res = XTranslateCoordinates(dpy, src_win, dest_win, src_x, src_y, &dest_x, &dest_y, &child);
 
-    // NativewindowCommon_x11ErrorHandlerEnable(env, dpy, 0, 0, 0);
+    // NativewindowCommon_x11ErrorHandlerEnable(env, dpy, 0, 0, 0, 0);
 
     DBG_PRINT( "X11: GetRelativeLocation0: %p %d/%d -> %p %d/%d - ok: %d\n",
         (void*)src_win, src_x, src_y, (void*)dest_win, dest_x, dest_y, (int)res);
diff --git a/src/nativewindow/native/x11/Xmisc.h b/src/nativewindow/native/x11/Xmisc.h
index a44da950d..91f2ac5d9 100644
--- a/src/nativewindow/native/x11/Xmisc.h
+++ b/src/nativewindow/native/x11/Xmisc.h
@@ -39,6 +39,4 @@
 #include <X11/Xlib.h>
 #include <X11/Xutil.h>
 
-void NativewindowCommon_x11ErrorHandlerEnable(JNIEnv * env, Display *dpy, int onoff, int quiet, int sync);
-
 #endif /* Xmisc_h */