Bug 961: Stabilize glXGetFBConfigAttrib(..) and glXChooseFBConfig(..) Usage Against OpenGL Bugs ; Fix glXGetFBConfigs

- glXChooseFBConfig(..)
  - Remove NULL FBConfig pointer from result in native code,
    which has been observed in Mesa 8.0.5-4 libgl1-mesa-swx11 (Debian-7).

- glXGetFBConfigs
  - Add manual implementation similar to glXChooseFBConfig

- glXGetFBConfigAttrib(..)
  - glXGetFBConfig(..)
     - Returns boolean reflecting success, don't throw exception - caller handles error
     - Caller ignore failure if not essential (i.e. already chosen config)

2 files changed, 38 insertions, 9 deletions

diff --git a/src/jogl/classes/jogamp/opengl/x11/glx/X11GLXGraphicsConfiguration.java b/src/jogl/classes/jogamp/opengl/x11/glx/X11GLXGraphicsConfiguration.java
index ee3e1a3d7..c0a3b46df 100644
--- a/src/jogl/classes/jogamp/opengl/x11/glx/X11GLXGraphicsConfiguration.java
+++ b/src/jogl/classes/jogamp/opengl/x11/glx/X11GLXGraphicsConfiguration.java
@@ -254,7 +254,13 @@ public class X11GLXGraphicsConfiguration extends X11GraphicsConfiguration implem
     int val = 0;
 
     final IntBuffer tmp = Buffers.newDirectIntBuffer(1);
-    int fbtype = glXGetFBConfig(device.getHandle(), fbcfg, GLX.GLX_DRAWABLE_TYPE, tmp);
+    if( !glXGetFBConfig(device.getHandle(), fbcfg, GLX.GLX_DRAWABLE_TYPE, tmp) ) {
+        if(DEBUG) {
+            System.err.println("X11GLXGraphicsConfiguration.FBCfgDrawableTypeBits: FBConfig invalid: fbcfg: "+toHexString(fbcfg));
+        }
+        return 0;
+    }
+    final int fbtype = tmp.get(0);
 
     if ( 0 != ( fbtype & GLX.GLX_WINDOW_BIT ) ) {
         val |= GLGraphicsConfigurationUtil.WINDOW_BIT |
@@ -314,8 +320,19 @@ public class X11GLXGraphicsConfiguration extends X11GraphicsConfiguration implem
     final long display = device.getHandle();
     final int allDrawableTypeBits = FBCfgDrawableTypeBits(device, fbcfg);
     int drawableTypeBits = winattrmask & allDrawableTypeBits;
-
+    if( 0 == allDrawableTypeBits || 0 == drawableTypeBits ) {
+      if(DEBUG) {
+          System.err.println("X11GLXGraphicsConfiguration.GLXFBConfig2GLCapabilities: zero drawablebits: allDrawableTypeBits: "+toHexString(allDrawableTypeBits)+", drawableTypeBits "+toHexString(drawableTypeBits));
+      }
+      return null;
+    }
     final int fbcfgid = X11GLXGraphicsConfiguration.glXFBConfig2FBConfigID(display, fbcfg);
+    if( VisualIDHolder.VID_UNDEFINED == fbcfgid ) {
+      if(DEBUG) {
+          System.err.println("X11GLXGraphicsConfiguration.GLXFBConfig2GLCapabilities: FBConfig invalid (0): fbcfg: "+toHexString(fbcfg));
+      }
+      return null;
+    }
     final XVisualInfo visualInfo = GLX.glXGetVisualFromFBConfig(display, fbcfg);
     if(null == visualInfo) {
         if(DEBUG) {
@@ -415,20 +432,26 @@ public class X11GLXGraphicsConfiguration extends X11GraphicsConfiguration implem
     }
   }
 
-  static int glXGetFBConfig(long display, long cfg, int attrib, IntBuffer tmp) {
+  static boolean glXGetFBConfig(long display, long cfg, int attrib, IntBuffer tmp) {
     if (display == 0) {
       throw new GLException("No display connection");
     }
-    int res = GLX.glXGetFBConfigAttrib(display, cfg, attrib, tmp);
-    if (res != 0) {
-      throw new GLException("glXGetFBConfig("+toHexString(attrib)+") failed: error code " + glXGetFBConfigErrorCode(res));
+    final boolean res = GLX.GLX_BAD_ATTRIBUTE != GLX.glXGetFBConfigAttrib(display, cfg, attrib, tmp);
+    if( !res ) {
+        if(DEBUG) {
+            System.err.println("X11GLXGraphicsConfiguration.glXGetFBConfig: FBConfig invalid: fbcfg: "+toHexString(cfg));
+        }
     }
-    return tmp.get(tmp.position());
+    return res;
   }
 
   static int glXFBConfig2FBConfigID(long display, long cfg) {
       final IntBuffer tmpID = Buffers.newDirectIntBuffer(1);
-      return glXGetFBConfig(display, cfg, GLX.GLX_FBCONFIG_ID, tmpID);
+      if( glXGetFBConfig(display, cfg, GLX.GLX_FBCONFIG_ID, tmpID) ) {
+          return tmpID.get(0);
+      } else {
+          return VisualIDHolder.VID_UNDEFINED; // error
+      }
   }
 
   static long glXFBConfigID2FBConfig(long display, int screen, int id) {
diff --git a/src/jogl/classes/jogamp/opengl/x11/glx/X11GLXGraphicsConfigurationFactory.java b/src/jogl/classes/jogamp/opengl/x11/glx/X11GLXGraphicsConfigurationFactory.java
index 1f92960bc..75771f884 100644
--- a/src/jogl/classes/jogamp/opengl/x11/glx/X11GLXGraphicsConfigurationFactory.java
+++ b/src/jogl/classes/jogamp/opengl/x11/glx/X11GLXGraphicsConfigurationFactory.java
@@ -258,13 +258,19 @@ public class X11GLXGraphicsConfigurationFactory extends GLGraphicsConfigurationF
         final long fbcfg = X11GLXGraphicsConfiguration.glXFBConfigID2FBConfig(display, screen, fbID);
         if( 0 == fbcfg || !X11GLXGraphicsConfiguration.GLXFBConfigValid( display, fbcfg ) ) {
             if(DEBUG) {
-                System.err.println("X11GLXGraphicsConfiguration.chooseGraphicsConfigurationFBConfig: Failed - GLX FBConfig invalid: ("+x11Screen+","+toHexString(fbID)+"): fbcfg: "+toHexString(fbcfg));
+                System.err.println("X11GLXGraphicsConfiguration.chooseGraphicsConfigurationFBConfig: Failed.0 - GLX FBConfig invalid: ("+x11Screen+","+toHexString(fbID)+"): fbcfg: "+toHexString(fbcfg));
             }
             return null;
         }
         final X11GLXDrawableFactory factory = (X11GLXDrawableFactory) GLDrawableFactory.getDesktopFactory();
 
         final X11GLCapabilities caps = X11GLXGraphicsConfiguration.GLXFBConfig2GLCapabilities(x11Device, glp, fbcfg, GLGraphicsConfigurationUtil.ALL_BITS, factory.isGLXMultisampleAvailable(x11Device));
+        if(null==caps) {
+            if(DEBUG) {
+                System.err.println("X11GLXGraphicsConfiguration.chooseGraphicsConfigurationFBConfig: Failed.1 - GLX FBConfig invalid: ("+x11Screen+","+toHexString(fbID)+"): fbcfg: "+toHexString(fbcfg));
+            }
+            return null;
+        }
         return new X11GLXGraphicsConfiguration(x11Screen, caps, caps, new DefaultGLCapabilitiesChooser());
     }