Bug 1232 - NEWT Translucent Decorated Windows Not Working On Windows >= 8 (Lack of Aero / Blur )

Adopting new undocumented user32.dll Windows >= 8 API:
  - SetWindowCompositionAttribute / AccentState

See:
  - <https://github.com/riverar/sample-win10-aeroglass/blob/master/MainWindow.xaml.cs>
  - <http://withinrafael.com/adding-the-aero-glass-blur-to-your-windows-10-apps/>
  - <http://undoc.airesoft.co.uk/user32.dll/SetWindowCompositionAttribute.php>
  - <http://undoc.airesoft.co.uk/user32.dll/GetWindowCompositionAttribute.php>

+++

Cleaning up WindowsDWM.h, use on header file (in stub_includes)
for GlueGen and implementation.

+++

Merge java implementation within GDIUtil.DwmSetupTranslucency(..),
to be utilized by NEWT and JOGL.

NEWT issues GDIUtil.DwmSetupTranslucency(..) at creation
and when toggling decoration.

Toggling decoration on Win >= 8 leads to lost of translucency
when returning to decorated window.
On Win 7, this may work .. but is also buggy.

+++

Followup patch is needed for NEWT to _not_ clear the background!

3 files changed, 175 insertions, 53 deletions

diff --git a/src/nativewindow/classes/jogamp/nativewindow/windows/GDIUtil.java b/src/nativewindow/classes/jogamp/nativewindow/windows/GDIUtil.java
index 7c01d7b2e..c0f277f09 100644
--- a/src/nativewindow/classes/jogamp/nativewindow/windows/GDIUtil.java
+++ b/src/nativewindow/classes/jogamp/nativewindow/windows/GDIUtil.java
@@ -128,7 +128,13 @@ public class GDIUtil implements ToolkitProperties {
      * Windows >= 8, even if not manifested
      * @see https://msdn.microsoft.com/en-us/library/windows/desktop/ms724832%28v=vs.85%29.aspx
      */
-    private static final VersionNumber Win8Version = new VersionNumber(6, 2, 0);
+    public static final VersionNumber Win8Version = new VersionNumber(6, 2, 0);
+
+    /**
+     * Windows >= 10, manifested
+     * @see https://msdn.microsoft.com/en-us/library/windows/desktop/ms724832%28v=vs.85%29.aspx
+     */
+    public static final VersionNumber Win10Version = new VersionNumber(10, 0, 0);
 
     /**
      * Wrapper for {@link GDI#DwmIsCompositionEnabled()}
@@ -148,6 +154,77 @@ public class GDIUtil implements ToolkitProperties {
         }
     }
 
+    public static boolean DwmSetupTranslucency(final long hwnd, final boolean enable) {
+        if( !GDI.DwmIsExtensionAvailable() ) {
+            if(DEBUG) {
+                System.err.println("GDIUtil.DwmSetupTranslucency on wnd 0x"+Long.toHexString(hwnd)+": enable "+enable+" -> failed, extension not available");
+            }
+            return !enable;
+        }
+        final VersionNumber winVer = Platform.getOSVersionNumber();
+        final boolean isWin8 = winVer.compareTo(Win8Version) >= 0;
+        if( !isWin8 && !GDI.DwmIsCompositionEnabled() ) {
+            if(DEBUG) {
+                System.err.println("GDIUtil.DwmSetupTranslucency on wnd 0x"+Long.toHexString(hwnd)+": enable "+enable+" -> failed, composition disabled");
+            }
+            return !enable;
+        }
+        final boolean hasWinCompEXT = GDI.IsWindowCompositionExtensionAvailable();
+        final boolean useWinCompEXT = isWin8 && hasWinCompEXT;
+        final boolean isUndecorated = IsUndecorated(hwnd);
+        boolean ok;
+        if( useWinCompEXT && !isUndecorated ) {
+            final AccentPolicy accentPolicy = AccentPolicy.create();
+            if( enable ) {
+                // For undecorated windows, this would also enable the Glass effect!
+                accentPolicy.setAccentState(GDI.ACCENT_ENABLE_BLURBEHIND);
+            } else {
+                accentPolicy.setAccentState(GDI.ACCENT_DISABLED);
+            }
+            ok = GDI.SetWindowCompositionAccentPolicy(hwnd, accentPolicy);
+        } else {
+            // Works even for >= Win8, if undecorated
+            final DWM_BLURBEHIND bb = DWM_BLURBEHIND.create();
+            final int dwFlags = enable ? GDI.DWM_BB_ENABLE | GDI.DWM_BB_BLURREGION | GDI.DWM_BB_TRANSITIONONMAXIMIZED : GDI.DWM_BB_ENABLE;
+            // final int dwFlags = GDI.DWM_BB_ENABLE;
+            bb.setDwFlags( dwFlags );
+            bb.setFEnable( enable ? 1 : 0 );
+            bb.setHRgnBlur(0);
+            bb.setFTransitionOnMaximized(1);
+            ok = GDI.DwmEnableBlurBehindWindow(hwnd, bb);
+            if( ok ) {
+                final MARGINS m = MARGINS.create();
+                m.setCxLeftWidth(-1);
+                m.setCxRightWidth(-1);
+                m.setCyBottomHeight(-1);
+                m.setCyTopHeight(-1);
+                ok = GDI.DwmExtendFrameIntoClientArea(hwnd, m);
+            }
+        }
+        /***
+         * Not required ..
+         *
+        if( ok && isWin8 && !isUndecorated ) {
+            final IntBuffer pvAttribute = Buffers.newDirectIntBuffer(1);
+            if( enable ) {
+                // Glass Effect even if undecorated, hence not truly 100% translucent!
+                pvAttribute.put(0, GDI.DWMNCRP_ENABLED);
+            } else {
+                pvAttribute.put(0, GDI.DWMNCRP_DISABLED);
+            }
+            final int err = GDI.DwmSetWindowAttribute(hwnd, GDI.DWMWA_NCRENDERING_POLICY,
+                                                      pvAttribute,
+                                                      Buffers.sizeOfBufferElem(pvAttribute)*pvAttribute.capacity());
+            ok = 0 == err; // S_OK
+        } */
+        if(DEBUG) {
+            final boolean isChild = IsChild(hwnd);
+            System.err.println("GDIUtil.DwmSetupTranslucency on wnd 0x"+Long.toHexString(hwnd)+": enable "+enable+", isUndecorated "+isUndecorated+", isChild "+isChild+
+                               ", version "+winVer+", isWin8 "+isWin8+", hasWinCompEXT "+hasWinCompEXT+", useWinCompEXT "+useWinCompEXT+" -> ok: "+ok);
+        }
+        return ok;
+    }
+
     public static boolean IsUndecorated(final long win) {
         return IsUndecorated0(win);
     }
diff --git a/src/nativewindow/native/win32/WindowsDWM.c b/src/nativewindow/native/win32/WindowsDWM.c
index cc9ed6d8c..925ddd1d1 100644
--- a/src/nativewindow/native/win32/WindowsDWM.c
+++ b/src/nativewindow/native/win32/WindowsDWM.c
@@ -1,4 +1,5 @@
 
+#include <windows.h>
 #include "WindowsDWM.h"
 
 #include <stdlib.h>
@@ -21,39 +22,68 @@ typedef HRESULT (WINAPI *DwmEnableCompositionPROCADDR)(UINT uCompositionAction);
 typedef HRESULT (WINAPI *DwmIsCompositionEnabledPROCADDR)(BOOL * pfEnabled);
 typedef HRESULT (WINAPI *DwmEnableBlurBehindWindowPROCADDR)(HWND hWnd, const DWM_BLURBEHIND* pBlurBehind);  
 typedef HRESULT (WINAPI *DwmExtendFrameIntoClientAreaPROCADDR)(HWND hwnd, const MARGINS *pMarInset);  
+typedef HRESULT (WINAPI *DwmGetWindowAttributePROCADDR)(HWND hwnd, DWORD dwAttribute, PVOID pvAttribute, DWORD cbAttribute);
+typedef HRESULT (WINAPI *DwmSetWindowAttributePROCADDR)(HWND hwnd, DWORD dwAttribute, LPCVOID pvAttribute, DWORD cbAttribute);
+typedef BOOL (WINAPI *GetWindowCompositionAttributePROCADDR)(HWND hwnd, WINCOMPATTRDATA* pAttrData);
+typedef BOOL (WINAPI *SetWindowCompositionAttributePROCADDR)(HWND hwnd, WINCOMPATTRDATA* pAttrData);
 
-static int _init = 0; // 1: init, 2: has DWM extension
+#define INIT_CALLED_MASK          1 << 0
+#define INIT_HAS_DWM_EXT_MASK     1 << 1
+#define INIT_HAS_WINCOMP_EXT_MASK 1 << 2
+
+#define HAS_INIT(a)        ( 0 != ( INIT_CALLED_MASK & (a)  ) )
+#define HAS_DWM_EXT(a)     ( 0 != ( INIT_HAS_DWM_EXT_MASK & (a)  ) )
+#define HAS_WINCOMP_EXT(a) ( 0 != ( INIT_HAS_WINCOMP_EXT_MASK & (a)  ) )
+
+static int _init = 0; // INIT_ bits, see above
 static DwmEnableCompositionPROCADDR _DwmEnableComposition = NULL;
 static DwmIsCompositionEnabledPROCADDR _DwmIsCompositionEnabled = NULL;
 static DwmEnableBlurBehindWindowPROCADDR _DwmEnableBlurBehindWindow = NULL;
 static DwmExtendFrameIntoClientAreaPROCADDR _DwmExtendFrameIntoClientArea = NULL;
+static DwmGetWindowAttributePROCADDR _DwmGetWindowAttribute = NULL;
+static DwmSetWindowAttributePROCADDR _DwmSetWindowAttribute = NULL;
+static GetWindowCompositionAttributePROCADDR _GetWindowCompositionAttribute = NULL;
+static SetWindowCompositionAttributePROCADDR _SetWindowCompositionAttribute = NULL;
 
 static int initWindowsDWM() {
-    if(0 == _init) {
-        _init = 1;
-        HANDLE shell = LoadLibrary(TEXT("dwmapi.dll"));
-        if (shell) {
-            _DwmEnableComposition = (DwmEnableCompositionPROCADDR) GetProcAddressA (shell, "DwmEnableComposition");
-            _DwmIsCompositionEnabled = (DwmIsCompositionEnabledPROCADDR) GetProcAddressA (shell, "DwmIsCompositionEnabled");
-            _DwmEnableBlurBehindWindow = (DwmEnableBlurBehindWindowPROCADDR) GetProcAddressA (shell, "DwmEnableBlurBehindWindow");
-            _DwmExtendFrameIntoClientArea = (DwmExtendFrameIntoClientAreaPROCADDR) GetProcAddressA (shell, "DwmExtendFrameIntoClientArea");
+    if( !HAS_INIT(_init) ) {
+        _init |= INIT_CALLED_MASK;
+        HANDLE hDwmAPI = LoadLibrary(TEXT("dwmapi.dll"));
+        if (hDwmAPI) {
+            _DwmEnableComposition = (DwmEnableCompositionPROCADDR) GetProcAddressA (hDwmAPI, "DwmEnableComposition");
+            _DwmIsCompositionEnabled = (DwmIsCompositionEnabledPROCADDR) GetProcAddressA (hDwmAPI, "DwmIsCompositionEnabled");
+            _DwmEnableBlurBehindWindow = (DwmEnableBlurBehindWindowPROCADDR) GetProcAddressA (hDwmAPI, "DwmEnableBlurBehindWindow");
+            _DwmExtendFrameIntoClientArea = (DwmExtendFrameIntoClientAreaPROCADDR) GetProcAddressA (hDwmAPI, "DwmExtendFrameIntoClientArea");
+            _DwmGetWindowAttribute = (DwmGetWindowAttributePROCADDR) GetProcAddressA (hDwmAPI, "DwmGetWindowAttribute");
+            _DwmSetWindowAttribute = (DwmSetWindowAttributePROCADDR) GetProcAddressA (hDwmAPI, "DwmSetWindowAttribute");
             if(NULL != _DwmEnableComposition && NULL != _DwmIsCompositionEnabled && 
-               NULL != _DwmEnableBlurBehindWindow && NULL != _DwmExtendFrameIntoClientArea) {
-                _init = 2;
+               NULL != _DwmEnableBlurBehindWindow && NULL != _DwmExtendFrameIntoClientArea &&
+               NULL != _DwmGetWindowAttribute && NULL != _DwmSetWindowAttribute) {
+                _init |= INIT_HAS_DWM_EXT_MASK;
+            }
+        }
+        // FreeLibrary (hDwmAPI);  
+        HANDLE hUser32 = LoadLibrary(TEXT("user32.dll"));
+        if (hUser32) {
+            _GetWindowCompositionAttribute = (GetWindowCompositionAttributePROCADDR) GetProcAddressA (hUser32, "GetWindowCompositionAttribute");
+            _SetWindowCompositionAttribute = (SetWindowCompositionAttributePROCADDR) GetProcAddressA (hUser32, "SetWindowCompositionAttribute");
+            if( NULL != _GetWindowCompositionAttribute &&
+                NULL != _SetWindowCompositionAttribute ) {
+                _init |= INIT_HAS_WINCOMP_EXT_MASK;
             }
         }
-        // FreeLibrary (shell);  
-        DBG_PRINT("DWM - initWindowsDWM: %d - s %p, e %p, c %p\n", _init, shell, _DwmEnableBlurBehindWindow, _DwmExtendFrameIntoClientArea);
+        // FreeLibrary (hUser32);  
+        DBG_PRINT("DWM - initWindowsDWM: hasDWM %d, hasWinComp %d\n", HAS_DWM_EXT(_init), HAS_WINCOMP_EXT(_init));
     }
     return _init;
 }
 
 BOOL DwmIsExtensionAvailable() {
-    return (2 == initWindowsDWM()) ? TRUE : FALSE;
+    return HAS_DWM_EXT( initWindowsDWM() ) ? TRUE : FALSE;
 }
 
 BOOL DwmIsCompositionEnabled( ) {
-    if(2 == initWindowsDWM()) {
+    if( HAS_DWM_EXT( initWindowsDWM() ) ) {
         BOOL fEnabled = FALSE;
         if( 0 == _DwmIsCompositionEnabled(&fEnabled) ) {
             DBG_PRINT("DWM - DwmIsCompositionEnabled: %d\n", fEnabled);
@@ -65,14 +95,14 @@ BOOL DwmIsCompositionEnabled( ) {
 }
 
 BOOL DwmEnableComposition( UINT uCompositionAction ) {
-    if(2 == initWindowsDWM()) {
+    if( HAS_DWM_EXT( initWindowsDWM() ) ) {
         return 0 == _DwmEnableComposition(uCompositionAction) ? TRUE : FALSE;
     }
     return FALSE;
 }
 
 BOOL DwmEnableBlurBehindWindow(HWND hwnd, const DWM_BLURBEHIND* pBlurBehind) {  
-    if(2 == initWindowsDWM()) {
+    if( HAS_DWM_EXT( initWindowsDWM() ) ) {
         _DwmEnableBlurBehindWindow(hwnd, pBlurBehind);
         DBG_PRINT("DWM - DwmEnableBlurBehindWindow: hwnd %p, f %d, on %d, %p\n", 
             (void *)hwnd, 
@@ -86,10 +116,59 @@ BOOL DwmEnableBlurBehindWindow(HWND hwnd, const DWM_BLURBEHIND* pBlurBehind) {
 }  
 
 BOOL DwmExtendFrameIntoClientArea(HWND hwnd, const MARGINS *pMarInset) {
-    if(2 == initWindowsDWM()) {
+    if( HAS_DWM_EXT( initWindowsDWM() ) ) {
         _DwmExtendFrameIntoClientArea(hwnd, pMarInset);
         return TRUE;
     }
     return FALSE;
 }  
   
+HRESULT DwmGetWindowAttribute(HWND hwnd, DWORD dwAttribute, PVOID pvAttribute, DWORD cbAttribute) {
+    if( HAS_DWM_EXT( initWindowsDWM() ) ) {
+        return _DwmGetWindowAttribute(hwnd, dwAttribute, pvAttribute, cbAttribute);
+    }
+    return E_NOINTERFACE;
+}
+
+HRESULT DwmSetWindowAttribute(HWND hwnd, DWORD dwAttribute, LPCVOID pvAttribute, DWORD cbAttribute) {
+    if( HAS_DWM_EXT( initWindowsDWM() ) ) {
+        return _DwmSetWindowAttribute(hwnd, dwAttribute, pvAttribute, cbAttribute);
+    }
+    return E_NOINTERFACE;
+}
+
+BOOL IsWindowCompositionExtensionAvailable() {
+    return HAS_WINCOMP_EXT( initWindowsDWM() ) ? TRUE : FALSE;
+}
+
+BOOL GetWindowCompositionAccentPolicy(HWND hwnd, AccentPolicy* pAccentPolicy) {
+    if( HAS_WINCOMP_EXT( initWindowsDWM() ) ) {
+        WINCOMPATTRDATA attrData = { WCA_ACCENT_POLICY, pAccentPolicy, sizeof(AccentPolicy) };
+        return _GetWindowCompositionAttribute(hwnd, &attrData);
+    }
+    return FALSE;
+}
+BOOL SetWindowCompositionAccentPolicy(HWND hwnd, const AccentPolicy* pAccentPolicy) {
+    if( HAS_WINCOMP_EXT( initWindowsDWM() ) ) {
+        WINCOMPATTRDATA attrData = { WCA_ACCENT_POLICY, (AccentPolicy*)pAccentPolicy, sizeof(AccentPolicy) };
+        return _SetWindowCompositionAttribute(hwnd, &attrData);
+    }
+    return FALSE;
+}
+
+#if 0
+BOOL GetWindowCompositionAttribute(HWND hwnd, WINCOMPATTRDATA* pAttrData) {
+    if( HAS_WINCOMP_EXT( initWindowsDWM() ) ) {
+        return _GetWindowCompositionAttribute(hwnd, pAttrData);
+    }
+    return FALSE;
+}
+
+BOOL SetWindowCompositionAttribute(HWND hwnd, WINCOMPATTRDATA* pAttrData) {
+    if( HAS_WINCOMP_EXT( initWindowsDWM() ) ) {
+        return _SetWindowCompositionAttribute(hwnd, pAttrData);
+    }
+    return FALSE;
+}
+#endif
+
diff --git a/src/nativewindow/native/win32/WindowsDWM.h b/src/nativewindow/native/win32/WindowsDWM.h
deleted file mode 100644
index 6e5160fa4..000000000
--- a/src/nativewindow/native/win32/WindowsDWM.h
+++ /dev/null
@@ -1,34 +0,0 @@
-#ifndef _WINDOWS_DWM_H_
-#define _WINDOWS_DWM_H_
-
-    #include <windows.h> 
-
-    #define DWM_BB_ENABLE                 0x00000001  // fEnable has been specified  
-    #define DWM_BB_BLURREGION             0x00000002
-    #define DWM_BB_TRANSITIONONMAXIMIZED  0x00000004
-    #define DWM_EC_DISABLECOMPOSITION     0
-    #define DWM_EC_ENABLECOMPOSITION      1
-      
-    typedef struct _DWM_BLURBEHIND  
-    {  
-        DWORD dwFlags;  
-        BOOL fEnable;  
-        HRGN hRgnBlur;  
-        BOOL fTransitionOnMaximized;  
-    } DWM_BLURBEHIND, *PDWM_BLURBEHIND;  
-      
-    typedef struct _MARGINS  
-    {  
-        int cxLeftWidth;      // width of left border that retains its size  
-        int cxRightWidth;     // width of right border that retains its size  
-        int cyTopHeight;      // height of top border that retains its size  
-        int cyBottomHeight;   // height of bottom border that retains its size  
-    } MARGINS, *PMARGINS;  
-      
-    BOOL DwmIsExtensionAvailable();
-    BOOL DwmIsCompositionEnabled();
-    BOOL DwmEnableComposition( UINT uCompositionAction );
-    BOOL DwmEnableBlurBehindWindow(HWND hwnd, const DWM_BLURBEHIND* pBlurBehind);
-    BOOL DwmExtendFrameIntoClientArea(HWND hwnd, const MARGINS *pMarInset);
-      
-#endif /* _WINDOWS_DWM_H_ */