Fix X11/Translucency (NV): Don't enable GLX.GLX_TRANSPARENT_TYPE; Always chooseConfig if translucent

Don't enable GLX.GLX_TRANSPARENT_TYPE (X11)
  - if enabled, NV driver results in odd transparency effect (very high transparency)

Always chooseConfig if translucent (X11)
  - recommended idx shall stay -1 if translucent,
    allowing our default chooser to pick up the proper config.

    this was previous reached due to the unavailable GLX_TRANSPARENT_TYPE
    request, which is now removed (see above)

Allow translucency for non FBConfig, XVisual's only

3 files changed, 36 insertions, 31 deletions

diff --git a/src/jogl/classes/jogamp/opengl/x11/glx/X11GLCapabilities.java b/src/jogl/classes/jogamp/opengl/x11/glx/X11GLCapabilities.java
index 0f496b4a2..f4f01195f 100644
--- a/src/jogl/classes/jogamp/opengl/x11/glx/X11GLCapabilities.java
+++ b/src/jogl/classes/jogamp/opengl/x11/glx/X11GLCapabilities.java
@@ -44,11 +44,11 @@ public class X11GLCapabilities extends GLCapabilities {
 
       public int compare(Object o1, Object o2) {
         if ( ! ( o1 instanceof X11GLCapabilities ) ) {
-            Class c = (null != o1) ? o1.getClass() : null ;
+            Class<?> c = (null != o1) ? o1.getClass() : null ;
             throw new ClassCastException("arg1 not a X11GLCapabilities object: " + c);
         }
         if ( ! ( o2 instanceof X11GLCapabilities ) ) {
-            Class c = (null != o2) ? o2.getClass() : null ;
+            Class<?> c = (null != o2) ? o2.getClass() : null ;
             throw new ClassCastException("arg2 not a X11GLCapabilities object: " + c);
         }
 
diff --git a/src/jogl/classes/jogamp/opengl/x11/glx/X11GLXGraphicsConfiguration.java b/src/jogl/classes/jogamp/opengl/x11/glx/X11GLXGraphicsConfiguration.java
index fd2048552..68b8135d0 100644
--- a/src/jogl/classes/jogamp/opengl/x11/glx/X11GLXGraphicsConfiguration.java
+++ b/src/jogl/classes/jogamp/opengl/x11/glx/X11GLXGraphicsConfiguration.java
@@ -138,6 +138,8 @@ public class X11GLXGraphicsConfiguration extends X11GraphicsConfiguration implem
             res[idx++] = GLX.GLX_STEREO;
             res[idx++] = caps.getStereo()?GL.GL_TRUE:GL.GL_FALSE;
             res[idx++] = GLX.GLX_TRANSPARENT_TYPE;
+            res[idx++] = GLX.GLX_NONE;
+            /**
             res[idx++] = caps.isBackgroundOpaque()?GLX.GLX_NONE:GLX.GLX_TRANSPARENT_RGB;
             if(!caps.isBackgroundOpaque()) {
                 res[idx++] = GLX.GLX_TRANSPARENT_RED_VALUE;
@@ -148,7 +150,7 @@ public class X11GLXGraphicsConfiguration extends X11GraphicsConfiguration implem
                 res[idx++] = caps.getTransparentBlueValue()>=0?caps.getTransparentBlueValue():(int)GLX.GLX_DONT_CARE;
                 res[idx++] = GLX.GLX_TRANSPARENT_ALPHA_VALUE;
                 res[idx++] = caps.getTransparentAlphaValue()>=0?caps.getTransparentAlphaValue():(int)GLX.GLX_DONT_CARE;
-            }
+            } */
         } else {
             if (caps.getDoubleBuffered()) {
               res[idx++] = GLX.GLX_DOUBLEBUFFER;
@@ -252,12 +254,12 @@ public class X11GLXGraphicsConfiguration extends X11GraphicsConfiguration implem
     return null;
   }
 
-  static boolean GLXFBConfigHasAlphaMask(long dpy, long fbcfg, long visual) {
+  static XRenderDirectFormat XVisual2XRenderMask(long dpy, long visual) {
     XRenderPictFormat renderPictFmt = X11Lib.XRenderFindVisualFormat(dpy, visual);
     if(null == renderPictFmt) {
-        return false;
+        return null;
     }
-    return renderPictFmt.getDirect().getAlphaMask() > 0 ;
+    return renderPictFmt.getDirect();
   }
 
   static boolean GLXFBConfig2GLCapabilities(ArrayList capsBucket,
@@ -306,23 +308,18 @@ public class X11GLXGraphicsConfiguration extends X11GraphicsConfiguration implem
       res.setSampleBuffers(glXGetFBConfig(display, fbcfg, GLX.GLX_SAMPLE_BUFFERS, tmp, 0) != 0);
       res.setNumSamples   (glXGetFBConfig(display, fbcfg, GLX.GLX_SAMPLES,        tmp, 0));
     }
-
-    res.setBackgroundOpaque( (null == visualInfo ) ? true : !GLXFBConfigHasAlphaMask(display, fbcfg, visualInfo.getVisual()) );
-    /***
-    res.setBackgroundOpaque(glXGetFBConfig(display, fbcfg, GLX.GLX_TRANSPARENT_TYPE, tmp, 0) == GLX.GLX_NONE);
-    if(!res.isBackgroundOpaque()) {
-        glXGetFBConfig(display, fbcfg, GLX.GLX_TRANSPARENT_RED_VALUE,  tmp, 0);
-        res.setTransparentRedValue(tmp[0]==GLX.GLX_DONT_CARE?-1:tmp[0]);
-
-        glXGetFBConfig(display, fbcfg, GLX.GLX_TRANSPARENT_GREEN_VALUE,  tmp, 0);
-        res.setTransparentGreenValue(tmp[0]==GLX.GLX_DONT_CARE?-1:tmp[0]);
-
-        glXGetFBConfig(display, fbcfg, GLX.GLX_TRANSPARENT_BLUE_VALUE,  tmp, 0);
-        res.setTransparentBlueValue(tmp[0]==GLX.GLX_DONT_CARE?-1:tmp[0]);
-
-        glXGetFBConfig(display, fbcfg, GLX.GLX_TRANSPARENT_ALPHA_VALUE,  tmp, 0);
-        res.setTransparentAlphaValue(tmp[0]==GLX.GLX_DONT_CARE?-1:tmp[0]);
-    } */
+    final XRenderDirectFormat xrmask = ( null != visualInfo ) ? 
+                                         XVisual2XRenderMask( display, visualInfo.getVisual() ) : 
+                                         null ;
+    final int alphaMask = ( null != xrmask ) ? xrmask.getAlphaMask() : 0;
+    res.setBackgroundOpaque( 0 >= alphaMask );
+    if( !res.isBackgroundOpaque() ) {
+        res.setTransparentRedValue(xrmask.getRedMask());
+        res.setTransparentGreenValue(xrmask.getGreenMask());
+        res.setTransparentBlueValue(xrmask.getBlueMask());
+        res.setTransparentAlphaValue(alphaMask);
+    }
+    
     try { 
         res.setPbufferFloatingPointBuffers(glXGetFBConfig(display, fbcfg, GLXExt.GLX_FLOAT_COMPONENTS_NV, tmp, 0) != GL.GL_FALSE);
     } catch (Exception e) {}
@@ -431,6 +428,17 @@ public class X11GLXGraphicsConfiguration extends X11GraphicsConfiguration implem
       res.setSampleBuffers(glXGetConfig(display, info, GLX.GLX_SAMPLE_BUFFERS, tmp, 0) != 0);
       res.setNumSamples   (glXGetConfig(display, info, GLX.GLX_SAMPLES,        tmp, 0));
     }
+    final XRenderDirectFormat xrmask = ( null != info ) ? 
+                                         XVisual2XRenderMask( display, info.getVisual() ) : 
+                                         null ;
+    final int alphaMask = ( null != xrmask ) ? xrmask.getAlphaMask() : 0;
+    res.setBackgroundOpaque( 0 >= alphaMask );
+    if( !res.isBackgroundOpaque() ) {
+        res.setTransparentRedValue(xrmask.getRedMask());
+        res.setTransparentGreenValue(xrmask.getGreenMask());
+        res.setTransparentBlueValue(xrmask.getBlueMask());
+        res.setTransparentAlphaValue(alphaMask);
+    }
 
     return GLGraphicsConfigurationUtil.addGLCapabilitiesPermutations(capsBucket, res, drawableTypeBits);
   }
diff --git a/src/jogl/classes/jogamp/opengl/x11/glx/X11GLXGraphicsConfigurationFactory.java b/src/jogl/classes/jogamp/opengl/x11/glx/X11GLXGraphicsConfigurationFactory.java
index 03d676ec9..0f8b94825 100644
--- a/src/jogl/classes/jogamp/opengl/x11/glx/X11GLXGraphicsConfigurationFactory.java
+++ b/src/jogl/classes/jogamp/opengl/x11/glx/X11GLXGraphicsConfigurationFactory.java
@@ -232,7 +232,6 @@ public class X11GLXGraphicsConfigurationFactory extends GLGraphicsConfigurationF
                                                                                    GLCapabilitiesImmutable capsReq,
                                                                                    GLCapabilitiesChooser chooser,
                                                                                    X11GraphicsScreen x11Screen) {
-        long recommendedFBConfig = -1;
         int recommendedIndex = -1;
         PointerBuffer fbcfgsL = null;
         GLProfile glProfile = capsChosen.getGLProfile();
@@ -262,12 +261,11 @@ public class X11GLXGraphicsConfigurationFactory extends GLGraphicsConfigurationF
                 }
             }
             if(availableCaps.size() > 0) {
-                recommendedFBConfig = fbcfgsL.get(0);
-                recommendedIndex=0;
+                recommendedIndex = capsChosen.isBackgroundOpaque() ? 0 : -1; // only use recommended idx if not translucent
                 if (DEBUG) {
-                    System.err.println("!!! glXChooseFBConfig recommended fbcfg " + toHexString(recommendedFBConfig) + ", idx " + recommendedIndex);
+                    System.err.println("!!! glXChooseFBConfig recommended fbcfg " + toHexString(fbcfgsL.get(0)) + ", idx " + recommendedIndex);
                     System.err.println("!!! user  caps " + capsChosen);
-                    System.err.println("!!! fbcfg caps " + availableCaps.get(recommendedIndex));
+                    System.err.println("!!! fbcfg caps " + availableCaps.get(0));
                 }
             } else if (DEBUG) {
                 System.err.println("!!! glXChooseFBConfig no caps for recommended fbcfg " + toHexString(fbcfgsL.get(0)));
@@ -278,7 +276,6 @@ public class X11GLXGraphicsConfigurationFactory extends GLGraphicsConfigurationF
         // 2nd choice: get all GLCapabilities available, no preferred recommendedIndex available
         if( 0 == availableCaps.size() ) {
             // reset ..
-            recommendedFBConfig = -1;
             recommendedIndex = -1;
 
             fbcfgsL = GLX.glXChooseFBConfig(display, screen, null, 0, count, 0);
@@ -355,8 +352,8 @@ public class X11GLXGraphicsConfigurationFactory extends GLGraphicsConfigurationF
                     System.err.println("X11GLXGraphicsConfiguration.getAvailableGLCapabilitiesXVisual: XVisual invalid: ("+x11Screen+"): fbcfg: "+toHexString(infos[i].getVisualid()));
                 }
             } else {
-                // Attempt to find the visual chosenIndex by glXChooseVisual
-                if (recommendedVis != null && recommendedVis.getVisualid() == infos[i].getVisualid()) {
+                // Attempt to find the visual chosenIndex by glXChooseVisual, if not translucent
+                if (capsChosen.isBackgroundOpaque() && recommendedVis != null && recommendedVis.getVisualid() == infos[i].getVisualid()) {
                     recommendedIndex = availableCaps.size() - 1;
                 }
             }