Bug 1249 - NEWT X11: setVisible(false) IconicState not listening to _NET_WM_STATE_HIDDEN; setVisible(true) not restoring from _NET_WM_STATE_HIDDEN

Using Gnome Shell 3.14.4-1~deb8u1 disclosed an issue w/ our newly utilized IconicState/_NET_WM_STATE_HIDDEN,
i.e. visibleChanged(false) was never received.

This is a regression of commit 2d837a7a7130702ad36b694875613fae77c7ef06,
which utilizes WM_CHANGE_STATE_IDX + IconicState for visibility
on top-level windows.

This bug consist out of _two_ isssue:

1) setVisible(false) IconicState not listening to _NET_WM_STATE_HIDDEN
  Here, we 'listen' to _NET_WM_STATE_HIDDEN when receiving ConfigureNotify
  if supported _and_ XMapWindow has been issued.
  In such case existence/non-existence of _NET_WM_STATE_HIDDEN determines visibility.
  Otherwise, we have wait for MapNotify/UnmapNotify.

  The 'XMapWindow has been issued' criteria is tracked by new field 'JavaWindow.isMapped'
  and set/cleared when we actually issue XMapWindow/XUnmapWindow!

2) setVisible(true) not restoring from _NET_WM_STATE_HIDDEN
  It has been observed that restoring IconicState/_NET_WM_STATE_HIDDEN
  via XMapWindow or even NormalState may not work reliably on WMs.
  See <https://stackoverflow.com/questions/30192347/how-to-restore-a-window-with-xlib>

  Hence we restore from this WM state via NormalState _and_ _NET_ACTIVE_WINDOW.

Both strategies seem to work well on KDE as well as on Gnome.

1 files changed, 55 insertions, 2 deletions

diff --git a/src/newt/native/X11Common.h b/src/newt/native/X11Common.h
index cdfb65d50..e46996441 100644
--- a/src/newt/native/X11Common.h
+++ b/src/newt/native/X11Common.h
@@ -71,7 +71,7 @@
 extern jclass X11NewtWindowClazz;
 extern jmethodID insetsChangedID;
 extern jmethodID visibleChangedID;
-extern jmethodID sizePosMaxInsetsChanged;
+extern jmethodID insetsVisibleChangedID;
 
 typedef struct {
     Window window;
@@ -83,13 +83,66 @@ typedef struct {
     uint32_t lastDesktop;
     Bool maxHorz;
     Bool maxVert;
+    Bool isMapped;
 } JavaWindow;
 
 JavaWindow * getJavaWindowProperty(JNIEnv *env, Display *dpy, Window window, jlong javaObjectAtom, Bool showWarning);
 
 Status NewtWindows_getRootAndParent (Display *dpy, Window w, Window * root_return, Window * parent_return);
 Bool NewtWindows_updateInsets(Display *dpy, JavaWindow * w, int *left, int *right, int *top, int *bottom);
-Bool NewtWindows_updateMaximized(Display *dpy, JavaWindow * w);
+Bool NewtWindows_updateMaximized(Display *dpy, JavaWindow * w, uint32_t netWMState);
+
+#define _MASK_NET_WM_STATE                   ( 1 <<  0 )
+#define _MASK_NET_WM_STATE_MODAL             ( 1 <<  1 )
+#define _MASK_NET_WM_STATE_STICKY            ( 1 <<  2 )
+#define _MASK_NET_WM_STATE_MAXIMIZED_VERT    ( 1 <<  3 )
+#define _MASK_NET_WM_STATE_MAXIMIZED_HORZ    ( 1 <<  4 )
+#define _MASK_NET_WM_STATE_SHADED            ( 1 <<  5 )
+#define _MASK_NET_WM_STATE_HIDDEN            ( 1 <<  8 )
+#define _MASK_NET_WM_STATE_FULLSCREEN        ( 1 <<  9 )
+#define _MASK_NET_WM_STATE_ABOVE             ( 1 << 10 )
+#define _MASK_NET_WM_STATE_BELOW             ( 1 << 11 )
+#define _MASK_NET_WM_STATE_DEMANDS_ATTENTION ( 1 << 12 )
+#define _MASK_NET_WM_STATE_FOCUSED           ( 1 << 13 )
+#define _MASK_NET_WM_BYPASS_COMPOSITOR       ( 1 << 14 )
+#define _MASK_NET_WM_DESKTOP                 ( 1 << 15 )
+#define _MASK_NET_CURRENT_DESKTOP            ( 1 << 16 )
+#define _MASK_NET_WM_WINDOW_TYPE             ( 1 << 17 )
+#define _MASK_NET_WM_WINDOW_TYPE_NORMAL      ( 1 << 18 )
+#define _MASK_NET_WM_WINDOW_TYPE_POPUP_MENU  ( 1 << 19 )
+#define _MASK_NET_FRAME_EXTENTS              ( 1 << 20 )
+#define _MASK_NET_SUPPORTED                  ( 1 << 21 )
+#define _MASK_NET_ACTIVE_WINDOW              ( 1 << 22 )
+#define _MASK_WM_CHANGE_STATE                ( 1 << 23 )
+#define _MASK_MOTIF_WM_HINTS                 ( 1 << 24 )
+
+#define _NET_WM_STATE_IDX 0
+#define _NET_WM_STATE_MODAL_IDX 1
+#define _NET_WM_STATE_STICKY_IDX 2
+#define _NET_WM_STATE_MAXIMIZED_VERT_IDX 3
+#define _NET_WM_STATE_MAXIMIZED_HORZ_IDX 4
+#define _NET_WM_STATE_SHADED_IDX 5
+#define _NET_WM_STATE_SKIP_TASKBAR_IDX 6
+#define _NET_WM_STATE_SKIP_PAGER_IDX 7
+#define _NET_WM_STATE_HIDDEN_IDX 8
+#define _NET_WM_STATE_FULLSCREEN_IDX 9
+#define _NET_WM_STATE_ABOVE_IDX 10
+#define _NET_WM_STATE_BELOW_IDX 11
+#define _NET_WM_STATE_DEMANDS_ATTENTION_IDX 12
+#define _NET_WM_STATE_FOCUSED_IDX 13
+#define _NET_WM_BYPASS_COMPOSITOR_IDX 14
+#define _NET_WM_DESKTOP_IDX 15
+#define _NET_CURRENT_DESKTOP_IDX 16
+#define _NET_WM_WINDOW_TYPE_IDX 17
+#define _NET_WM_WINDOW_TYPE_NORMAL_IDX 18
+#define _NET_WM_WINDOW_TYPE_POPUP_MENU_IDX 19
+#define _NET_FRAME_EXTENTS_IDX 20
+#define _NET_SUPPORTED_IDX 21
+#define _NET_ACTIVE_WINDOW_IDX 22
+#define _WM_CHANGE_STATE_IDX 23
+#define _MOTIF_WM_HINTS_IDX 24
+
+uint32_t NewtWindows_getNET_WM_STATE(Display *dpy, JavaWindow *w);
 
 #endif /* _X11COMMON_H_ */