Bug 801: Fix 183e1bc1868699b99eb9f9c8bf18d646d1120a48 'window box' Calculation

Commit 183e1bc1868699b99eb9f9c8bf18d646d1120a48 only mapped object's bbox max/min points
to window space, which is wrong due to possible rotation in 3d space.

This commit adds AABBox.mapToWindow(..) method,
which correctly either uses 4 points of the bbox in 3d space (using center-z)
or all 8-points and creating a new bounding box.

The resulting width and height of this window bbox gives the
maximum amount of rectangular pixels for AA.

3 files changed, 72 insertions, 30 deletions

diff --git a/src/jogl/classes/com/jogamp/opengl/math/geom/AABBox.java b/src/jogl/classes/com/jogamp/opengl/math/geom/AABBox.java
index 81928888c..d0566b54e 100644
--- a/src/jogl/classes/com/jogamp/opengl/math/geom/AABBox.java
+++ b/src/jogl/classes/com/jogamp/opengl/math/geom/AABBox.java
@@ -28,6 +28,7 @@
 package com.jogamp.opengl.math.geom;
 
 import com.jogamp.opengl.math.VectorUtil;
+import com.jogamp.opengl.util.PMVMatrix;
 
 
 /**
@@ -350,6 +351,53 @@ public class AABBox implements Cloneable {
                VectorUtil.checkEquality(high, other.high) ;
     }
 
+    /**
+     * Assume this bounding box as being in object space and
+     * compute the window bounding box.
+     * <p>
+     * If <code>useCenterZ</code> is <code>true</code>,
+     * only 4 {@link PMVMatrix#gluProject(float, float, float, int[], int, float[], int) gluProject}
+     * operations are made on points [1..4] using {@link #getCenter()}'s z-value.
+     * Otherwise 8 {@link PMVMatrix#gluProject(float, float, float, int[], int, float[], int) gluProject}
+     * operation on all 8 points are made.
+     * </p>
+     * <pre>
+     *  [2] ------ [4]
+     *   |          |
+     *   |          |
+     *  [1] ------ [3]
+     * </pre>
+     * @param pmv
+     * @param view
+     * @param useCenterZ
+     * @param tmpV3 TODO
+     * @return
+     */
+    public AABBox mapToWindow(final AABBox result, final PMVMatrix pmv, final int[] view, final boolean useCenterZ, float[] tmpV3) {
+        float objZ = useCenterZ ? center[2] : getMinZ();
+        pmv.gluProject(getMinX(), getMinY(), objZ, view, 0, tmpV3, 0);
+        result.reset();
+        result.resize(tmpV3, 0);
+        pmv.gluProject(getMinX(), getMaxY(), objZ, view, 0, tmpV3, 0);
+        result.resize(tmpV3, 0);
+        pmv.gluProject(getMaxX(), getMinY(), objZ, view, 0, tmpV3, 0);
+        result.resize(tmpV3, 0);
+        pmv.gluProject(getMaxX(), getMaxY(), objZ, view, 0, tmpV3, 0);
+        result.resize(tmpV3, 0);
+        if( !useCenterZ ) {
+            objZ = getMaxZ();
+            pmv.gluProject(getMinX(), getMinY(), objZ, view, 0, tmpV3, 0);
+            result.resize(tmpV3, 0);
+            pmv.gluProject(getMinX(), getMaxY(), objZ, view, 0, tmpV3, 0);
+            result.resize(tmpV3, 0);
+            pmv.gluProject(getMaxX(), getMinY(), objZ, view, 0, tmpV3, 0);
+            result.resize(tmpV3, 0);
+            pmv.gluProject(getMaxX(), getMaxY(), objZ, view, 0, tmpV3, 0);
+            result.resize(tmpV3, 0);
+        }
+        return result;
+    }
+
     @Override
     public final String toString() {
         return "[ dim "+getWidth()+" x "+getHeight()+" x "+getDepth()+
diff --git a/src/jogl/classes/jogamp/graph/curve/opengl/VBORegion2PMSAAES2.java b/src/jogl/classes/jogamp/graph/curve/opengl/VBORegion2PMSAAES2.java
index 4d653af12..176b39c25 100644
--- a/src/jogl/classes/jogamp/graph/curve/opengl/VBORegion2PMSAAES2.java
+++ b/src/jogl/classes/jogamp/graph/curve/opengl/VBORegion2PMSAAES2.java
@@ -45,6 +45,7 @@ import com.jogamp.graph.curve.opengl.RegionRenderer;
 import com.jogamp.graph.curve.opengl.RenderState;
 import com.jogamp.opengl.FBObject;
 import com.jogamp.opengl.FBObject.Attachment;
+import com.jogamp.opengl.math.geom.AABBox;
 import com.jogamp.opengl.util.GLArrayDataServer;
 import com.jogamp.opengl.util.PMVMatrix;
 import com.jogamp.opengl.util.glsl.ShaderState;
@@ -171,6 +172,10 @@ public class VBORegion2PMSAAES2  extends GLRegion {
         // the buffers were disabled, since due to real/fbo switching and other vbo usage
     }
 
+    private final AABBox drawWinBox = new AABBox();
+    private final int[] drawView = new int[] { 0, 0, 0, 0 };
+    private final float[] drawTmpV3 = new float[3];
+
     @Override
     protected void drawImpl(final GL2ES2 gl, final RegionRenderer renderer, final int[/*1*/] sampleCount) {
         final int width = renderer.getWidth();
@@ -188,26 +193,18 @@ public class VBORegion2PMSAAES2  extends GLRegion {
             {
                 // Calculate perspective pixel width/height for FBO,
                 // considering the sampleCount.
-                final int[] view = new int[] { 0, 0, width, height };
-                final float objZ = 0f;
-
-                final float[] winPosSzMin = new float[3];
-                final float[] winPosSzMax = new float[3];
-                final PMVMatrix pmv = renderer.getMatrix();
-                pmv.gluProject(box.getMinX(), box.getMinY(), objZ, view, 0, winPosSzMin, 0);
-                pmv.gluProject(box.getMaxX(), box.getMaxY(), objZ, view, 0, winPosSzMax, 0);
-                renderFboWidth = Math.abs(winPosSzMax[0] - winPosSzMin[0]);
-                renderFboHeight = Math.abs(winPosSzMax[1] - winPosSzMin[1]);
+                drawView[2] = width;
+                drawView[3] = height;
+                box.mapToWindow(drawWinBox, renderer.getMatrix(), drawView, true /* useCenterZ */, drawTmpV3);
+                renderFboWidth = drawWinBox.getWidth();
+                renderFboHeight = drawWinBox.getHeight();
                 targetFboWidth = (int)Math.ceil(renderFboWidth);
                 targetFboHeight = (int)Math.ceil(renderFboHeight);
                 diffWidth = targetFboWidth-renderFboWidth;
                 diffHeight = targetFboHeight-renderFboHeight;
                 if( DEBUG_FBO_2 ) {
-                    System.err.printf("XXX.MinMax1 min [%.1f, %.1f -> %.3f, %.3f, %.3f]"+
-                                      ", max [%.1f, %.1f -> %.3f, %.3f, %.3f], view[%d, %d]: FBO f[%.3f, %.3f], i[%d x %d], d[%.3f, %.3f], msaa %d%n",
-                            box.getMinX(), box.getMinY(), winPosSzMin[0], winPosSzMin[1], winPosSzMin[2],
-                            box.getMaxX(), box.getMaxY(), winPosSzMax[0], winPosSzMax[1], winPosSzMax[2],
-                            view[2], view[3],
+                    System.err.printf("XXX.MinMax view[%d, %d]: FBO f[%.3f, %.3f], i[%d x %d], d[%.3f, %.3f], msaa %d%n",
+                            drawView[2], drawView[3],
                             renderFboWidth, renderFboHeight, targetFboWidth, targetFboHeight,
                             diffWidth, diffHeight, sampleCount[0]);
                 }
diff --git a/src/jogl/classes/jogamp/graph/curve/opengl/VBORegion2PVBAAES2.java b/src/jogl/classes/jogamp/graph/curve/opengl/VBORegion2PVBAAES2.java
index 07a774d51..3f3709d6e 100644
--- a/src/jogl/classes/jogamp/graph/curve/opengl/VBORegion2PVBAAES2.java
+++ b/src/jogl/classes/jogamp/graph/curve/opengl/VBORegion2PVBAAES2.java
@@ -45,6 +45,7 @@ import com.jogamp.graph.curve.opengl.RenderState;
 import com.jogamp.opengl.FBObject;
 import com.jogamp.opengl.FBObject.Attachment;
 import com.jogamp.opengl.FBObject.TextureAttachment;
+import com.jogamp.opengl.math.geom.AABBox;
 import com.jogamp.opengl.util.GLArrayDataServer;
 import com.jogamp.opengl.util.PMVMatrix;
 import com.jogamp.opengl.util.glsl.ShaderState;
@@ -172,6 +173,10 @@ public class VBORegion2PVBAAES2  extends GLRegion {
         // the buffers were disabled, since due to real/fbo switching and other vbo usage
     }
 
+    private final AABBox drawWinBox = new AABBox();
+    private final int[] drawView = new int[] { 0, 0, 0, 0 };
+    private final float[] drawTmpV3 = new float[3];
+
     @Override
     protected void drawImpl(final GL2ES2 gl, final RegionRenderer renderer, final int[/*1*/] sampleCount) {
         final int width = renderer.getWidth();
@@ -190,16 +195,11 @@ public class VBORegion2PVBAAES2  extends GLRegion {
             {
                 // Calculate perspective pixel width/height for FBO,
                 // considering the sampleCount.
-                final int[] view = new int[] { 0, 0, width, height };
-                final float objZ = 0f;
-
-                final float[] winPosSzMin = new float[3];
-                final float[] winPosSzMax = new float[3];
-                final PMVMatrix pmv = renderer.getMatrix();
-                pmv.gluProject(box.getMinX(), box.getMinY(), objZ, view, 0, winPosSzMin, 0);
-                pmv.gluProject(box.getMaxX(), box.getMaxY(), objZ, view, 0, winPosSzMax, 0);
-                winWidth = Math.abs(winPosSzMax[0] - winPosSzMin[0]);
-                winHeight = Math.abs(winPosSzMax[1] - winPosSzMin[1]);
+                drawView[2] = width;
+                drawView[3] = height;
+                box.mapToWindow(drawWinBox, renderer.getMatrix(), drawView, true /* useCenterZ */, drawTmpV3);
+                winWidth = drawWinBox.getWidth();
+                winHeight = drawWinBox.getHeight();
                 diffWidth = (float)Math.ceil(winWidth)-winWidth;
                 diffHeight = (float)Math.ceil(winHeight)-winHeight;
                 renderFboWidth = winWidth*sampleCount[0];
@@ -207,11 +207,8 @@ public class VBORegion2PVBAAES2  extends GLRegion {
                 targetFboWidth = (int)Math.ceil(renderFboWidth);
                 targetFboHeight = (int)Math.ceil(renderFboHeight);
                 if( DEBUG_FBO_2 ) {
-                    System.err.printf("XXX.MinMax1 min [%.1f, %.1f -> %.3f, %.3f, %.3f]"+
-                                      ", max [%.1f, %.1f -> %.3f, %.3f, %.3f], view[%d, %d] -> win[%.3f, %.3f]: FBO f[%.3f, %.3f], i[%d x %d], d[%.3f, %.3f], msaa %d%n",
-                            box.getMinX(), box.getMinY(), winPosSzMin[0], winPosSzMin[1], winPosSzMin[2],
-                            box.getMaxX(), box.getMaxY(), winPosSzMax[0], winPosSzMax[1], winPosSzMax[2],
-                            view[2], view[3],
+                    System.err.printf("XXX.MinMax1 view[%d, %d] -> win[%.3f, %.3f]: FBO f[%.3f, %.3f], i[%d x %d], d[%.3f, %.3f], msaa %d%n",
+                            drawView[2], drawView[3],
                             winWidth, winHeight,
                             renderFboWidth, renderFboHeight, targetFboWidth, targetFboHeight,
                             diffWidth, diffHeight, sampleCount[0]);