OSX CALayer Stencil/.. Fix: In case of FBO CALayer usage, use default caps/pixelformat w/ chosen GLProfile only

Using a pixelformat w/ chosen stencil for CALayer does corrupt rendering for an unknown reason,
probably due to incompatible pixelformat w/ CALayer composition.

This patch simply discards any special chosen caps, while only recognizing the desired GLProfile
for the FBO CALayer pixelformat.

2 files changed, 27 insertions, 17 deletions

diff --git a/src/jogl/classes/jogamp/opengl/GLDrawableFactoryImpl.java b/src/jogl/classes/jogamp/opengl/GLDrawableFactoryImpl.java
index 2bb22f7b0..5c5db6d31 100644
--- a/src/jogl/classes/jogamp/opengl/GLDrawableFactoryImpl.java
+++ b/src/jogl/classes/jogamp/opengl/GLDrawableFactoryImpl.java
@@ -147,6 +147,7 @@ public abstract class GLDrawableFactoryImpl extends GLDrawableFactory {
         final OffscreenLayerSurface ols = NativeWindowFactory.getOffscreenLayerSurface(target, true);
         if(null != ols) {
             final GLCapabilitiesImmutable chosenCapsMod = GLGraphicsConfigurationUtil.fixOffscreenGLCapabilities(chosenCaps, this, adevice);
+            
             // layered surface -> Offscreen/[FBO|PBuffer]
             if( !chosenCapsMod.isFBO() && !chosenCapsMod.isPBuffer() ) {
                 throw new GLException("Neither FBO nor Pbuffer is available for "+chosenCapsMod+", "+target);
@@ -155,7 +156,7 @@ public abstract class GLDrawableFactoryImpl extends GLDrawableFactory {
             ols.setChosenCapabilities(chosenCapsMod);
             if(DEBUG) {
                 System.err.println("GLDrawableFactoryImpl.createGLDrawable -> OnscreenDrawable -> Offscreen-Layer");
-                System.err.println("chosenCaps: "+chosenCaps);
+                System.err.println("chosenCaps:    "+chosenCaps);
                 System.err.println("chosenCapsMod: "+chosenCapsMod);
                 System.err.println("OffscreenLayerSurface: **** "+ols);
                 System.err.println("Target: **** "+target);
@@ -165,7 +166,6 @@ public abstract class GLDrawableFactoryImpl extends GLDrawableFactory {
                 throw new IllegalArgumentException("Passed NativeSurface must implement SurfaceChangeable for offscreen layered surface: "+target);
             }            
             if( chosenCapsMod.isFBO() ) {
-                // target surface is already a native one
                 result = createFBODrawableImpl(target, chosenCapsMod, 0);
             } else {            
                 result = createOffscreenDrawableImpl(target);
diff --git a/src/jogl/classes/jogamp/opengl/macosx/cgl/MacOSXCGLContext.java b/src/jogl/classes/jogamp/opengl/macosx/cgl/MacOSXCGLContext.java
index 1c59b8d97..cde9841b8 100644
--- a/src/jogl/classes/jogamp/opengl/macosx/cgl/MacOSXCGLContext.java
+++ b/src/jogl/classes/jogamp/opengl/macosx/cgl/MacOSXCGLContext.java
@@ -568,24 +568,38 @@ public abstract class MacOSXCGLContext extends GLContextImpl
           if( !incompleteView && surface instanceof ProxySurface ) {
               incompleteView = ((ProxySurface)surface).containsUpstreamOptionBits( ProxySurface.OPT_UPSTREAM_WINDOW_INVISIBLE );
           }
-          long pixelFormat = MacOSXCGLGraphicsConfiguration.GLCapabilities2NSPixelFormat(chosenCaps, ctp, major, minor);
+          long pixelFormat;
+          {
+              final GLCapabilitiesImmutable targetCaps;
+              if( isFBO ) {
+                  // Use minimum GLCapabilities for the target surface w/ same profile
+                  targetCaps = new GLCapabilities( chosenCaps.getGLProfile() );
+              } else {
+                  targetCaps = chosenCaps;
+              }
+              pixelFormat = MacOSXCGLGraphicsConfiguration.GLCapabilities2NSPixelFormat(targetCaps, ctp, major, minor);
+          }
           if (pixelFormat == 0) {
               if(DEBUG) {
                   System.err.println("Unable to allocate pixel format with requested GLCapabilities: "+chosenCaps);
               }
               return 0;
           }
-          GLCapabilities fixedCaps = MacOSXCGLGraphicsConfiguration.NSPixelFormat2GLCapabilities(chosenCaps.getGLProfile(), pixelFormat);
-          if( !fixedCaps.isPBuffer() && isPBuffer ) {
-              throw new InternalError("handle is PBuffer, fixedCaps not: "+drawable);
-          }
-          { // determine on-/offscreen caps, since pformat is ambiguous 
-              fixedCaps.setFBO( isFBO );         // exclusive 
-              fixedCaps.setPBuffer( isPBuffer ); // exclusive
-              fixedCaps.setBitmap( false );      // n/a in our OSX impl.
-              fixedCaps.setOnscreen( !isFBO && !isPBuffer );
+          final GLCapabilitiesImmutable fixedCaps;
+          if( isFBO ) {
+              // pixelformat of target doesn't affect caps w/ FBO
+              fixedCaps = chosenCaps;
+          } else {
+              final GLCapabilities _fixedCaps = MacOSXCGLGraphicsConfiguration.NSPixelFormat2GLCapabilities(chosenCaps.getGLProfile(), pixelFormat);
+              if( !_fixedCaps.isPBuffer() && isPBuffer ) {
+                  throw new InternalError("handle is PBuffer, fixedCaps not: "+drawable);
+              }
+              // determine on-/offscreen caps, since pformat is ambiguous 
+              _fixedCaps.setPBuffer( isPBuffer ); // exclusive
+              _fixedCaps.setBitmap( false );      // n/a in our OSX impl.
+              _fixedCaps.setOnscreen( !isFBO && !isPBuffer );
+              fixedCaps = GLGraphicsConfigurationUtil.fixOpaqueGLCapabilities(_fixedCaps, chosenCaps.isBackgroundOpaque());
           }
-          fixedCaps = GLGraphicsConfigurationUtil.fixOpaqueGLCapabilities(fixedCaps, chosenCaps.isBackgroundOpaque());
           int sRefreshRate = OSXUtil.GetScreenRefreshRate(drawable.getNativeSurface().getGraphicsConfiguration().getScreen().getIndex());
           screenVSyncTimeout = 1000000f / sRefreshRate;
           if(DEBUG) {
@@ -604,10 +618,6 @@ public abstract class MacOSXCGLContext extends GLContextImpl
               // Thread.dumpStack();
           }
           config.setChosenCapabilities(fixedCaps);
-          /**
-          if(null != backingLayerHost) {
-              backingLayerHost.setChosenCapabilities(fixedCaps);
-          }  */                  
           
           try {
               final IntBuffer viewNotReady = Buffers.newDirectIntBuffer(1);