Bug 1348: Fix X11 XI Multitouch

I got access to a touchscreen laptop w/ Debian 9, hence I could fix and test the implementation.

X11 DisplayDriver.java:
- Store and pass through xi_opcode of XI extension, queried at initialization stage

X11Window.c Fixes:
- Initialize JavaWindow's xiTouchCoords[].id w/ -1, as required to track the pointer

- Pass through xi_opcode as stored in X11 DisplayDriver

X11Display.c Fixes:
- sendTouchScreenEvent: Throw RuntimeException if 0 > actionId (Internal Error: based on xiTouchCoords[].id tracking)

- DispatchMessages's windowPointer determination:
-- Query potenial XI Event first: IF XI Event, must use XIDeviceEvent's event Window
-- Only IF not an XI Event, we can use evt.xany.window as the event window

- DispatchMessages's XI Event Handling:
-- Always break deviceid search loop if id found, preserving index and time spend

Works on my Debian 9 device, tested w/ com.jogamp.opengl.test.junit.jogl.demos.es2.newt.TestGearsES2NEWT:
- One pointer (finger) press, drag and release (click)
- PinchToZoomGesture works
- DoubleTabScrollGesture works

+++

Potential Issues:

JavaWindow's xiTouchCoords[].id accuracy is crucial to pointer tracking
during XI_TouchBegin -> XI_TouchUpdate -> XI_TouchEnd.

In the normal course of action:
- XI_TouchBegin sets the id, assuming it is yet set
- XI_TouchUpdate assumes it is set
- XI_TouchEnd clears the id, assuming it is set

This field in the JavaWindow array only gets reset to -1 once at native window
creation. We may need to figure out when to reset this field to -1.

If the XI_TouchEnd events would get lost for whatever reason,
the above tracking state would be broken.

3 files changed, 112 insertions, 95 deletions

diff --git a/src/newt/native/X11Common.h b/src/newt/native/X11Common.h
index 99795b6e8..978cfffed 100644
--- a/src/newt/native/X11Common.h
+++ b/src/newt/native/X11Common.h
@@ -94,7 +94,6 @@ typedef struct {
     Bool maxVert;
     /** flag whether window is mapped */
     Bool isMapped;
-    int xiOpcode;
     int xiTouchDeviceId;
     XITouchPosition xiTouchCoords[XI_TOUCHCOORD_COUNT];
 } JavaWindow;
diff --git a/src/newt/native/X11Display.c b/src/newt/native/X11Display.c
index 981e3197f..7c6741839 100644
--- a/src/newt/native/X11Display.c
+++ b/src/newt/native/X11Display.c
@@ -248,8 +248,8 @@ JNIEXPORT jboolean JNICALL Java_jogamp_newt_driver_x11_DisplayDriver_initIDs0
         }
     }
 
-    // displayCompletedID = (*env)->GetMethodID(env, clazz, "displayCompleted", "(JJJII)V"); // Variant using XKB
-    displayCompletedID = (*env)->GetMethodID(env, clazz, "displayCompleted", "(JJII)V");
+    // displayCompletedID = (*env)->GetMethodID(env, clazz, "displayCompleted", "(JJJIII)V"); // Variant using XKB
+    displayCompletedID = (*env)->GetMethodID(env, clazz, "displayCompleted", "(JJIII)V");
     sendRRScreenChangeNotifyID = (*env)->GetMethodID(env, clazz, "sendRRScreenChangeNotify", "(J)V");
     getCurrentThreadNameID = (*env)->GetStaticMethodID(env, X11NewtWindowClazz, "getCurrentThreadName", "()Ljava/lang/String;");
     dumpStackID = (*env)->GetStaticMethodID(env, X11NewtWindowClazz, "dumpStack", "()V");
@@ -331,10 +331,13 @@ JNIEXPORT void JNICALL Java_jogamp_newt_driver_x11_DisplayDriver_CompleteDisplay
     int randr_event_base, randr_error_base;
     XRRQueryExtension(dpy, &randr_event_base, &randr_error_base);
 
+    int xi_opcode = -1, event, error;
+    XQueryExtension(dpy, "XInputExtension", &xi_opcode, &event, &error);
+
     DBG_PRINT("X11: X11Display_completeDisplay dpy %p\n", dpy);
 
     (*env)->CallVoidMethod(env, obj, displayCompletedID, javaObjectAtom, windowDeleteAtom /*, kbdHandle*/, // XKB disabled for now
-                                     randr_event_base, randr_error_base);
+                                     randr_event_base, randr_error_base, xi_opcode);
 }
 
 /*
@@ -410,6 +413,12 @@ static void sendTouchScreenEvent(JNIEnv *env, JavaWindow *jw,
             cnt++;
         }
     }
+    if( 0 > actionIdx ) {
+        NewtCommon_throwNewRuntimeException(env, "Internal Error: XI event (window %p) actionId %d not found in %d xiTouchCoords", 
+            (void*)jw->window, actionId, cnt);
+    }
+    DBG_PRINT( "X11: XI event - sendTouchScreenEvent: Window %p, action-touchid[%d] %d of %d ptr: %d/%d\n",
+        (void*)jw->window, actionIdx, actionId, cnt, x[actionIdx], y[actionIdx]);
 
     jintArray jNames = (*env)->NewIntArray(env, cnt);
     if (jNames == NULL) {
@@ -443,11 +452,11 @@ static void sendTouchScreenEvent(JNIEnv *env, JavaWindow *jw,
 /*
  * Class:     jogamp_newt_driver_x11_DisplayDriver
  * Method:    DispatchMessages0
- * Signature: (JJJII)V
+ * Signature: (JJJIII)V
  */
 JNIEXPORT void JNICALL Java_jogamp_newt_driver_x11_DisplayDriver_DispatchMessages0
   (JNIEnv *env, jobject obj, jlong display, jlong javaObjectAtom, jlong windowDeleteAtom /*, jlong kbdHandle*/,
-                             jint randr_event_base, jint randr_error_base)
+                             jint randr_event_base, jint randr_error_base, jint xi_opcode)
 {
     Display * dpy = (Display *) (intptr_t) display;
     Atom wm_delete_atom = (Atom)windowDeleteAtom;
@@ -498,16 +507,28 @@ JNIEXPORT void JNICALL Java_jogamp_newt_driver_x11_DisplayDriver_DispatchMessage
         if( randr_event_base > 0 && RRScreenChangeNotify == ( evt.type - randr_event_base ) ) {
             DBG_PRINT( "X11: DispatchMessages dpy %p, Event RRScreenChangeNotify %p\n", (void*)dpy, (void*)&evt);
             (*env)->CallVoidMethod(env, obj, sendRRScreenChangeNotifyID, (jlong)(intptr_t)&evt);
-            continue;
+            continue; // next event
         }
 
         if( 0==evt.xany.window ) {
             DBG_PRINT( "X11: DispatchMessages dpy %p, Event %d - Window NULL, ignoring\n", (void*)dpy, (int)evt.type);
-            continue;
+            continue; // next event
         }
 
-        Window windowPointer = evt.xany.window;
-        // DBG_PRINT( "X11: DispatchMessages dpy %p, win %p, Event %d\n", (void*)dpy, (void*)windowPointer, (int)evt.type);
+        // Valid registered XI Event w/ cookie data (incl. the event Window name)?
+        // Here: https://www.x.org/wiki/Development/Documentation/Multitouch/
+        XGenericEventCookie *evtCookie = &evt.xcookie; // hacks: https://keithp.com/blogs/Cursor_tracking/
+        int isXiEvent = GenericEvent == evtCookie->type && xi_opcode == evtCookie->extension && XGetEventData(dpy, evtCookie);
+        XIDeviceEvent *xiDevEv;
+        Window windowPointer;
+        if( !isXiEvent ) {
+            xiDevEv = NULL;
+            windowPointer = evt.xany.window;
+            DBG_PRINT( "X11: DispatchMessages dpy %p, win %p, Event %d\n", (void*)dpy, (void*)windowPointer, (int)evt.type);
+        } else {
+            xiDevEv = evtCookie->data;
+            windowPointer = xiDevEv->event;
+        }
 
         jw = getJavaWindowProperty(env, dpy, windowPointer, javaObjectAtom,
         #ifdef VERBOSE_ON
@@ -519,59 +540,59 @@ JNIEXPORT void JNICALL Java_jogamp_newt_driver_x11_DisplayDriver_DispatchMessage
 
         if(NULL==jw) {
             fprintf(stderr, "Warning: NEWT X11 DisplayDispatch %p, Couldn't handle event %d for X11 window %p\n", (void*)dpy, evt.type, (void*)windowPointer);
-            continue;
+            continue; // next event
         }
 
-        XGenericEventCookie *cookie = &evt.xcookie; // hacks: https://keithp.com/blogs/Cursor_tracking/
-
-        if ( GenericEvent == cookie->type && jw->xiOpcode == cookie->extension && XGetEventData(dpy, cookie) ) {
-            // Valid registered XI Event w/ cookie data
-            // Here: https://www.x.org/wiki/Development/Documentation/Multitouch/
-            XIDeviceEvent *devev = cookie->data;
-            if( devev->event != windowPointer ) {
-                DBG_PRINT( "X11: XI event - dpy %p, win %p, Event %d: Event Window %p not matching\n", (void*)dpy, (void*)windowPointer, (int)evt.type, (void*)devev->event);
-            } else if( devev->deviceid != jw->xiTouchDeviceId) {
-                DBG_PRINT( "X11: XI event - dpy %p, win %p, Event %d: DeviceID not matching: Window %d, this %d\n", (void*)dpy, (void*)windowPointer, (int)evt.type, devev->deviceid, jw->xiTouchDeviceId);
+        if ( isXiEvent ) {
+            if( xiDevEv->deviceid != jw->xiTouchDeviceId) {
+                DBG_PRINT( "X11: XI event - dpy %p, win %p, Event %d: DeviceID not matching: Window %d, this %d\n", (void*)dpy, (void*)windowPointer, (int)evt.type, xiDevEv->deviceid, jw->xiTouchDeviceId);
             } else {
                 int i;
-                switch (devev->evtype) {
+                switch (xiDevEv->evtype) {
                     case XI_TouchBegin:
                         for (i = 0; i < XI_TOUCHCOORD_COUNT; i++) {
                             if (jw->xiTouchCoords[i].id == -1) {
-                                jw->xiTouchCoords[i].id = devev->detail % 32767;
-                                jw->xiTouchCoords[i].x = devev->event_x;
-                                jw->xiTouchCoords[i].y = devev->event_y;
+                                jw->xiTouchCoords[i].id = xiDevEv->detail % 32767;
+                                jw->xiTouchCoords[i].x = xiDevEv->event_x;
+                                jw->xiTouchCoords[i].y = xiDevEv->event_y;
                                 break;
                             }
                         }
-                        DBG_PRINT( "X11: XI event - XI_TouchBegin Window %p, devid %d, touchid %d @ %d/%d\n", (void*)windowPointer, devev->deviceid, jw->xiTouchCoords[i].id, jw->xiTouchCoords[i].x, jw->xiTouchCoords[i].y);
-                        sendTouchScreenEvent(env, jw, EVENT_MOUSE_PRESSED, 0, devev->detail % 32767);
+                        DBG_PRINT( "X11: XI event - XI_TouchBegin Window %p, devid %d, touchid[%d] %d @ %d/%d\n", (void*)windowPointer, xiDevEv->deviceid, 
+                            i, jw->xiTouchCoords[i].id, jw->xiTouchCoords[i].x, jw->xiTouchCoords[i].y);
+                        sendTouchScreenEvent(env, jw, EVENT_MOUSE_PRESSED, 0, xiDevEv->detail % 32767);
                         break;
 
                     case XI_TouchUpdate:
                         for (i = 0; i < XI_TOUCHCOORD_COUNT; i++) {
-                            if (jw->xiTouchCoords[i].id == devev->detail  % 32767) {
-                                jw->xiTouchCoords[i].x = devev->event_x;
-                                jw->xiTouchCoords[i].y = devev->event_y;
+                            if (jw->xiTouchCoords[i].id == xiDevEv->detail  % 32767) {
+                                jw->xiTouchCoords[i].x = xiDevEv->event_x;
+                                jw->xiTouchCoords[i].y = xiDevEv->event_y;
+                                break;
                             }
                         }
-                        DBG_PRINT( "X11: XI event - XI_TouchUpdate: Window %p, devid %d, touchid %d @ %d/%d\n", (void*)windowPointer, devev->deviceid, jw->xiTouchCoords[i].id, jw->xiTouchCoords[i].x, jw->xiTouchCoords[i].y);
-                        sendTouchScreenEvent(env, jw, EVENT_MOUSE_MOVED, 0, devev->detail % 32767);
+                        DBG_PRINT( "X11: XI event - XI_TouchUpdate: Window %p, devid %d, touchid[%d] %d @ %d/%d\n", (void*)windowPointer, xiDevEv->deviceid, 
+                            i, jw->xiTouchCoords[i].id, jw->xiTouchCoords[i].x, jw->xiTouchCoords[i].y);
+                        sendTouchScreenEvent(env, jw, EVENT_MOUSE_MOVED, 0, xiDevEv->detail % 32767);
                         break;
 
                     case XI_TouchEnd:
-                        DBG_PRINT( "X11: XI event - XI_TouchEnd: Window %p, devid %d, touchid %d\n", (void*)windowPointer, devev->deviceid, jw->xiTouchCoords[i].id);
-                        sendTouchScreenEvent(env, jw, EVENT_MOUSE_RELEASED, 0, devev->detail % 32767);
                         for (i = 0; i < XI_TOUCHCOORD_COUNT; i++) {
-                            if (jw->xiTouchCoords[i].id == devev->detail % 32767) {
-                                jw->xiTouchCoords[i].id = -1;
+                            if (jw->xiTouchCoords[i].id == xiDevEv->detail % 32767) {
+                                break;
                             }
                         }
+                        DBG_PRINT( "X11: XI event - XI_TouchEnd: Window %p, devid %d, touchid[%d] %d @ %d/%d\n", (void*)windowPointer, xiDevEv->deviceid, 
+                            i, jw->xiTouchCoords[i].id, jw->xiTouchCoords[i].x, jw->xiTouchCoords[i].y);
+                        sendTouchScreenEvent(env, jw, EVENT_MOUSE_RELEASED, 0, xiDevEv->detail % 32767);
+                        if ( i < XI_TOUCHCOORD_COUNT ) {
+                            jw->xiTouchCoords[i].id = -1;
+                        }
                         break;
                 }
             }
-            XFreeEventData(dpy, cookie);
-            continue; // next event, skip evt.type below
+            XFreeEventData(dpy, evtCookie);
+            continue; // next event, skip evt.type handling below
         }
         
         switch(evt.type) {
diff --git a/src/newt/native/X11Window.c b/src/newt/native/X11Window.c
index f815ca41b..1c2c8e80f 100644
--- a/src/newt/native/X11Window.c
+++ b/src/newt/native/X11Window.c
@@ -212,6 +212,11 @@ static JavaWindow* createJavaWindowProperty(JNIEnv *env, Display *dpy, Window ro
         res->maxHorz = False;
         res->maxVert = False;
         res->isMapped = False;
+        int i;
+        for (i = 0; i < XI_TOUCHCOORD_COUNT; i++) {
+            res->xiTouchCoords[i].id = -1;
+        }
+        res->xiTouchDeviceId = -1;
     }
     unsigned long jogl_java_object_data[2]; // X11 is based on 'unsigned long'
     int nitems_32 = putPtrIn32Long( jogl_java_object_data, (uintptr_t) res);
@@ -255,7 +260,7 @@ JavaWindow * getJavaWindowProperty(JNIEnv *env, Display *dpy, Window window, jlo
 
         if ( Success != res ) {
             if(True==showWarning) {
-                fprintf(stderr, "Warning: NEWT X11Window: Could not fetch Atom NEWT_JAVA_OBJECT window property (res %d) nitems %ld, bytes_after %ld, result 0!\n", res, nitems, bytes_after);
+                fprintf(stderr, "Warning: NEWT X11Window: Could not fetch Atom NEWT_JAVA_OBJECT window %p property (res %d) nitems %ld, bytes_after %ld, result 0!\n", (void*)window, res, nitems, bytes_after);
             }
             return NULL;
         }
@@ -265,8 +270,8 @@ JavaWindow * getJavaWindowProperty(JNIEnv *env, Display *dpy, Window window, jlo
                 XFree(jogl_java_object_data_pp);
             }
             if(True==showWarning) {
-                fprintf(stderr, "Warning: NEWT X11Window: Fetched invalid Atom NEWT_JAVA_OBJECT window property (res %d) nitems %ld, bytes_after %ld, actual_type %ld, NEWT_JAVA_OBJECT %ld, result 0!\n", 
-                res, nitems, bytes_after, (long)actual_type, (long)javaObjectAtom);
+                fprintf(stderr, "Warning: NEWT X11Window: Fetched invalid Atom NEWT_JAVA_OBJECT window %p property (res %d) nitems %ld, bytes_after %ld, actual_type %ld, NEWT_JAVA_OBJECT %ld, result 0!\n", 
+                (void *)window, res, nitems, bytes_after, (long)actual_type, (long)javaObjectAtom);
             }
             return NULL;
         }
@@ -761,7 +766,7 @@ static void NewtWindows_setIcon(Display *dpy, Window w, int data_size, const uns
 JNIEXPORT jlongArray JNICALL Java_jogamp_newt_driver_x11_WindowDriver_CreateWindow0
   (JNIEnv *env, jobject obj, jlong parent, jlong display, jint screen_index, 
                              jint visualID, 
-                             jlong javaObjectAtom, jlong windowDeleteAtom, 
+                             jlong javaObjectAtom, jlong windowDeleteAtom, jint xi_opcode,
                              jint x, jint y, jint width, jint height, int flags,
                              jint pixelDataSize, jobject pixels, jint pixels_byte_offset, jboolean pixels_is_direct, 
                              jboolean verbose)
@@ -955,59 +960,51 @@ JNIEXPORT jlongArray JNICALL Java_jogamp_newt_driver_x11_WindowDriver_CreateWind
     
     // Register X11 Multitouch Events for new Window
     // https://www.x.org/wiki/Development/Documentation/Multitouch/
-    {
-        int xi_opcode, event, error;
-
-        javaWindow->xiOpcode = -1;
-        javaWindow->xiTouchDeviceId = -1;
-
-        if( XQueryExtension(dpy, "XInputExtension", &xi_opcode, &event, &error) ) {
-            XIDeviceInfo *di;
-            int cnt = 0;
-
-            javaWindow->xiOpcode = xi_opcode;
-            DBG_PRINT( "X11: [CreateWindow]: XI: Window %p, Extension %d\n", (void*)window, xi_opcode);
-            di = XIQueryDevice(dpy, XIAllDevices, &cnt);
-
-            if( NULL != di && 0 < cnt ) {
-                int devid = -1;
-                int i, j;
-      
-                // find the 1st XITouchClass device available
-                for (i = 0; i < cnt && -1 == devid; i ++) {
-                    XIDeviceInfo *dev = &di[i];
-                    for (j = 0; j < dev->num_classes; j ++) {
-                        XITouchClassInfo *class = (XITouchClassInfo*)(dev->classes[j]);
-                        DBG_PRINT( "X11: [CreateWindow]: XI: Scan Window %p, device[%d/%d].class[%d/%d]: type %d (is XITouchClass %d)\n", 
-                            (void*)window, (i+1), cnt, (j+1), dev->num_classes, class->type, (XITouchClass == class->type));
-                        if ( XITouchClass == class->type ) {
-                          devid = dev->deviceid;
-                          break;
-                        }
+    if( 0 <= xi_opcode ) {
+        XIDeviceInfo *di;
+        int cnt = 0;
+
+        DBG_PRINT( "X11: [CreateWindow]: XI: Window %p, Extension %d\n", (void*)window, xi_opcode);
+        di = XIQueryDevice(dpy, XIAllDevices, &cnt);
+
+        if( NULL != di && 0 < cnt ) {
+            int devid = -1;
+            int i, j;
+  
+            // find the 1st XITouchClass device available
+            for (i = 0; i < cnt && -1 == devid; i ++) {
+                XIDeviceInfo *dev = &di[i];
+                for (j = 0; j < dev->num_classes; j ++) {
+                    XITouchClassInfo *class = (XITouchClassInfo*)(dev->classes[j]);
+                    DBG_PRINT( "X11: [CreateWindow]: XI: Scan Window %p, device[%d/%d].class[%d/%d]: devid %d, type %d (is XITouchClass %d)\n", 
+                        (void*)window, (i+1), cnt, (j+1), dev->num_classes, dev->deviceid, class->type, (XITouchClass == class->type));
+                    if ( XITouchClass == class->type ) {
+                      devid = dev->deviceid;
+                      break;
                     }
                 }
-                XIFreeDeviceInfo(di);
-                di = NULL;
+            }
+            XIFreeDeviceInfo(di);
+            di = NULL;
+            
+            if( -1 != devid ) {
+                // register 1st XITouchClass device if available
+                XIEventMask mask = {
+                  .deviceid = devid,
+                  .mask_len = XIMaskLen(XI_TouchEnd) // in bytes
+                };
                 
-                if( -1 != devid ) {
-                    // register 1st XITouchClass device if available
-                    XIEventMask mask = {
-                      .deviceid = devid,
-                      .mask_len = XIMaskLen(XI_TouchEnd) // in bytes
-                    };
-                    
-                    mask.mask = (unsigned char*)calloc(mask.mask_len, sizeof(unsigned char));
-                    XISetMask(mask.mask, XI_TouchBegin);
-                    XISetMask(mask.mask, XI_TouchUpdate);
-                    XISetMask(mask.mask, XI_TouchEnd);
-      
-                    XISelectEvents(dpy, window, &mask, 1);
-      
-                    free(mask.mask);      
-      
-                    javaWindow->xiTouchDeviceId = devid;
-                    DBG_PRINT( "X11: [CreateWindow]: XI: Window %p, XITouchClass devid %d\n", (void*)window, devid);
-                }
+                mask.mask = (unsigned char*)calloc(mask.mask_len, sizeof(unsigned char));
+                XISetMask(mask.mask, XI_TouchBegin);
+                XISetMask(mask.mask, XI_TouchUpdate);
+                XISetMask(mask.mask, XI_TouchEnd);
+  
+                XISelectEvents(dpy, window, &mask, 1);
+  
+                free(mask.mask);      
+  
+                javaWindow->xiTouchDeviceId = devid;
+                DBG_PRINT( "X11: [CreateWindow]: XI: Window %p, XITouchClass devid %d\n", (void*)window, devid);
             }
         }
     }
@@ -1059,7 +1056,7 @@ JNIEXPORT jint JNICALL Java_jogamp_newt_driver_x11_WindowDriver_GetSupportedReco
  */
 JNIEXPORT void JNICALL Java_jogamp_newt_driver_x11_WindowDriver_CloseWindow0
   (JNIEnv *env, jobject obj, jlong display, jlong javaWindow /*, jlong kbdHandle*/, // XKB disabled for now
-                             jint randr_event_base, jint randr_error_base)
+                             jint randr_event_base, jint randr_error_base, jint xi_opcode)
 {
     Display * dpy = (Display *) (intptr_t) display;
     JavaWindow * jw, * jw0;
@@ -1074,7 +1071,7 @@ JNIEXPORT void JNICALL Java_jogamp_newt_driver_x11_WindowDriver_CloseWindow0
         NewtCommon_FatalError(env, "invalid JavaWindow connection..");
     }
     jw0 = getJavaWindowProperty(env, dpy, jw->window, jw->javaObjectAtom, True);
-    if(NULL==jw) {
+    if(NULL==jw0) {
         NewtCommon_throwNewRuntimeException(env, "could not fetch Java Window object, bail out!");
         return;
     }
@@ -1097,7 +1094,7 @@ JNIEXPORT void JNICALL Java_jogamp_newt_driver_x11_WindowDriver_CloseWindow0
     // Drain all events related to this window ..
     Java_jogamp_newt_driver_x11_DisplayDriver_DispatchMessages0(env, obj, display, 
                                      (jlong)(intptr_t)jw->javaObjectAtom, (jlong)(intptr_t)jw->windowDeleteAtom /*, kbdHandle */, // XKB disabled for now
-                                     randr_event_base, randr_error_base);
+                                     randr_event_base, randr_error_base, xi_opcode);
 
     XDestroyWindow(dpy, jw->window);
     if( None != xwa.colormap ) {