Fix T98793: Wayland clamps cursor movement fails with gnome-shell

The current gnome-shell (v42.2) has a bug where grabbing the cursor
doesn't scale the region when confining it to the window.

For Hi-DPI displays this means the cursor may be confined to a quarter
of the window, making grab unusable.

Even though this has been fixed up-stream the issue remains in the
latest release - so workaround the problem by implementing window
confined grab using a software cursor.

This is only used gnome-shell for displays that use Hi-DPI scaling.

10 files changed, 160 insertions, 17 deletions

diff --git a/intern/ghost/GHOST_C-api.h b/intern/ghost/GHOST_C-api.h
index d27be40af0c..08ca9603985 100644
--- a/intern/ghost/GHOST_C-api.h
+++ b/intern/ghost/GHOST_C-api.h
@@ -404,8 +404,9 @@ extern GHOST_TSuccess GHOST_SetCursorPosition(GHOST_SystemHandle systemhandle,
 
 void GHOST_GetCursorGrabState(GHOST_WindowHandle windowhandle,
                               GHOST_TGrabCursorMode *r_mode,
-                              GHOST_TAxisFlag *r_wrap_axis,
-                              int r_bounds[4]);
+                              GHOST_TAxisFlag *r_axis_flag,
+                              int r_bounds[4],
+                              bool *r_use_software_cursor);
 
 /**
  * Grabs the cursor for a modal operation, to keep receiving
diff --git a/intern/ghost/GHOST_IWindow.h b/intern/ghost/GHOST_IWindow.h
index 7927190de04..f9552246e89 100644
--- a/intern/ghost/GHOST_IWindow.h
+++ b/intern/ghost/GHOST_IWindow.h
@@ -258,7 +258,10 @@ class GHOST_IWindow {
 
   virtual void getCursorGrabState(GHOST_TGrabCursorMode &mode,
                                   GHOST_TAxisFlag &axis_flag,
-                                  GHOST_Rect &bounds) = 0;
+                                  GHOST_Rect &bounds,
+                                  bool &use_software_cursor) = 0;
+
+  virtual bool getCursorGrabUseSoftwareDisplay() = 0;
 
   /**
    * Test if the standard cursor shape is supported by current platform.
diff --git a/intern/ghost/intern/GHOST_C-api.cpp b/intern/ghost/intern/GHOST_C-api.cpp
index c8127f59941..032ecd6aab5 100644
--- a/intern/ghost/intern/GHOST_C-api.cpp
+++ b/intern/ghost/intern/GHOST_C-api.cpp
@@ -379,15 +379,18 @@ GHOST_TSuccess GHOST_SetCursorGrab(GHOST_WindowHandle windowhandle,
 void GHOST_GetCursorGrabState(GHOST_WindowHandle windowhandle,
                               GHOST_TGrabCursorMode *r_mode,
                               GHOST_TAxisFlag *r_axis_flag,
-                              int r_bounds[4])
+                              int r_bounds[4],
+                              bool *r_use_software_cursor)
 {
   GHOST_IWindow *window = (GHOST_IWindow *)windowhandle;
   GHOST_Rect bounds_rect;
-  window->getCursorGrabState(*r_mode, *r_axis_flag, bounds_rect);
+  bool use_software_cursor;
+  window->getCursorGrabState(*r_mode, *r_axis_flag, bounds_rect, use_software_cursor);
   r_bounds[0] = bounds_rect.m_l;
   r_bounds[1] = bounds_rect.m_t;
   r_bounds[2] = bounds_rect.m_r;
   r_bounds[3] = bounds_rect.m_b;
+  *r_use_software_cursor = use_software_cursor;
 }
 
 GHOST_TSuccess GHOST_GetModifierKeyState(GHOST_SystemHandle systemhandle,
diff --git a/intern/ghost/intern/GHOST_SystemWayland.cpp b/intern/ghost/intern/GHOST_SystemWayland.cpp
index 2b0abd2cc41..2d1337df1de 100644
--- a/intern/ghost/intern/GHOST_SystemWayland.cpp
+++ b/intern/ghost/intern/GHOST_SystemWayland.cpp
@@ -44,6 +44,24 @@
 
 static GHOST_WindowWayland *window_from_surface(struct wl_surface *surface);
 
+/**
+ * GNOME (mutter 42.2 had a bug with confine not respecting scale - Hi-DPI), See: T98793.
+ * Even though this has been fixed, at time of writing it's not yet in a release.
+ * Workaround the problem by implementing confine with a software cursor.
+ * While this isn't ideal, it's not adding a lot of overhead as software
+ * cursors are already used for warping (which WAYLAND doesn't support).
+ */
+#define USE_GNOME_CONFINE_HACK
+/**
+ * Always use software confine (not just in GNOME).
+ * Useful for developing with compositors that don't need this workaround.
+ */
+// #define USE_GNOME_CONFINE_HACK_ALWAYS_ON
+
+#ifdef USE_GNOME_CONFINE_HACK
+static bool use_gnome_confine_hack = false;
+#endif
+
 /* -------------------------------------------------------------------- */
 /** \name Private Types & Defines
  * \{ */
@@ -156,6 +174,11 @@ struct input_t {
    * \endcode
    */
   wl_fixed_t xy[2] = {0, 0};
+
+#ifdef USE_GNOME_CONFINE_HACK
+  bool xy_software_confine = false;
+#endif
+
   GHOST_Buttons buttons = GHOST_Buttons();
   struct cursor_t cursor;
 
@@ -590,6 +613,21 @@ static void relative_pointer_handle_relative_motion(
   input->xy[0] += dx / scale;
   input->xy[1] += dy / scale;
 
+#ifdef USE_GNOME_CONFINE_HACK
+  if (input->xy_software_confine) {
+    GHOST_Rect bounds;
+    win->getClientBounds(bounds);
+    /* Needed or the cursor is considered outside the window and doesn't restore the location. */
+    bounds.m_r -= 1;
+    bounds.m_b -= 1;
+
+    bounds.m_l = wl_fixed_from_int(bounds.m_l) / scale;
+    bounds.m_t = wl_fixed_from_int(bounds.m_t) / scale;
+    bounds.m_r = wl_fixed_from_int(bounds.m_r) / scale;
+    bounds.m_b = wl_fixed_from_int(bounds.m_b) / scale;
+    bounds.clampPoint(input->xy[0], input->xy[1]);
+  }
+#endif
   input->system->pushEvent(new GHOST_EventCursor(input->system->getMilliSeconds(),
                                                  GHOST_kEventCursorMove,
                                                  win,
@@ -1868,6 +1906,13 @@ static void xdg_output_handle_logical_size(void *data,
      * detected otherwise), then override if necessary. */
     if ((output->size_logical[0] == width) && (output->scale_fractional == wl_fixed_from_int(1))) {
       GHOST_PRINT("xdg_output scale did not match, overriding with wl_output scale");
+
+#ifdef USE_GNOME_CONFINE_HACK
+      /* Use a bug in GNOME to check GNOME is in use. If the bug is fixed this won't cause an issue
+       * as T98793 has been fixed up-stream too, but not in a release at time of writing. */
+      use_gnome_confine_hack = true;
+#endif
+
       return;
     }
   }
@@ -2695,6 +2740,49 @@ bool GHOST_SystemWayland::supportsWindowPosition()
   return false;
 }
 
+bool GHOST_SystemWayland::getCursorGrabUseSoftwareDisplay(const GHOST_TGrabCursorMode mode)
+{
+  if (d->inputs.empty()) {
+    return false;
+  }
+  if (mode == GHOST_kGrabWrap) {
+    return true;
+  }
+#ifdef USE_GNOME_CONFINE_HACK
+  if (mode == GHOST_kGrabNormal) {
+    const input_t *input = d->inputs[0];
+    return input->xy_software_confine;
+  }
+#endif
+  return false;
+}
+
+#ifdef USE_GNOME_CONFINE_HACK
+static bool setCursorGrab_use_software_confine(const GHOST_TGrabCursorMode mode,
+                                               wl_surface *surface)
+{
+#  ifndef USE_GNOME_CONFINE_HACK_ALWAYS_ON
+  if (use_gnome_confine_hack == false) {
+    return false;
+  }
+#  endif
+  if (mode != GHOST_kGrabNormal) {
+    return false;
+  }
+  GHOST_WindowWayland *win = window_from_surface(surface);
+  if (!win) {
+    return false;
+  }
+
+#  ifndef USE_GNOME_CONFINE_HACK_ALWAYS_ON
+  if (win->scale() <= 1) {
+    return false;
+  }
+#  endif
+  return true;
+}
+#endif
+
 GHOST_TSuccess GHOST_SystemWayland::setCursorGrab(const GHOST_TGrabCursorMode mode,
                                                   const GHOST_TGrabCursorMode mode_current,
                                                   wl_surface *surface)
@@ -2715,19 +2803,28 @@ GHOST_TSuccess GHOST_SystemWayland::setCursorGrab(const GHOST_TGrabCursorMode mo
 
   input_t *input = d->inputs[0];
 
+#ifdef USE_GNOME_CONFINE_HACK
+  const bool was_software_confine = input->xy_software_confine;
+  const bool use_software_confine = setCursorGrab_use_software_confine(mode, surface);
+#else
+  const bool was_software_confine = false;
+  const bool use_software_confine = false;
+#endif
+
+  /* Warping happens to require software cursor which also hides. */
 #define MODE_NEEDS_LOCK(m) ((m) == GHOST_kGrabWrap || (m) == GHOST_kGrabHide)
-#define MODE_NEEDS_HIDE(m) ((m) == GHOST_kGrabHide)
+#define MODE_NEEDS_HIDE(m) (((m) == GHOST_kGrabHide) || ((m) == GHOST_kGrabWrap))
 #define MODE_NEEDS_CONFINE(m) ((m) == GHOST_kGrabNormal)
 
-  const bool was_lock = MODE_NEEDS_LOCK(mode_current);
-  const bool use_lock = MODE_NEEDS_LOCK(mode);
+  const bool was_lock = MODE_NEEDS_LOCK(mode_current) || was_software_confine;
+  const bool use_lock = MODE_NEEDS_LOCK(mode) || use_software_confine;
 
   /* Check for wrap as #supportsCursorWarp isn't supported. */
-  const bool was_hide = MODE_NEEDS_HIDE(mode_current) || (mode_current == GHOST_kGrabWrap);
-  const bool use_hide = MODE_NEEDS_HIDE(mode) || (mode == GHOST_kGrabWrap);
+  const bool was_hide = MODE_NEEDS_HIDE(mode_current) || was_software_confine;
+  const bool use_hide = MODE_NEEDS_HIDE(mode) || use_software_confine;
 
-  const bool was_confine = MODE_NEEDS_CONFINE(mode_current);
-  const bool use_confine = MODE_NEEDS_CONFINE(mode);
+  const bool was_confine = MODE_NEEDS_CONFINE(mode_current) && (was_software_confine == false);
+  const bool use_confine = MODE_NEEDS_CONFINE(mode) && (use_software_confine == false);
 
 #undef MODE_NEEDS_LOCK
 #undef MODE_NEEDS_HIDE
@@ -2788,6 +2885,15 @@ GHOST_TSuccess GHOST_SystemWayland::setCursorGrab(const GHOST_TGrabCursorMode mo
           wl_surface_commit(surface);
         }
       }
+#ifdef USE_GNOME_CONFINE_HACK
+      else if (mode_current == GHOST_kGrabNormal) {
+        if (input->xy_software_confine) {
+          zwp_locked_pointer_v1_set_cursor_position_hint(
+              input->locked_pointer, input->xy[0], input->xy[1]);
+          wl_surface_commit(surface);
+        }
+      }
+#endif
 
       zwp_locked_pointer_v1_destroy(input->locked_pointer);
       input->locked_pointer = nullptr;
@@ -2836,6 +2942,10 @@ GHOST_TSuccess GHOST_SystemWayland::setCursorGrab(const GHOST_TGrabCursorMode mo
     }
   }
 
+#ifdef USE_GNOME_CONFINE_HACK
+  input->xy_software_confine = use_software_confine;
+#endif
+
   return GHOST_kSuccess;
 }
 
diff --git a/intern/ghost/intern/GHOST_SystemWayland.h b/intern/ghost/intern/GHOST_SystemWayland.h
index 762ceb80e38..6faff8d57c1 100644
--- a/intern/ghost/intern/GHOST_SystemWayland.h
+++ b/intern/ghost/intern/GHOST_SystemWayland.h
@@ -127,6 +127,8 @@ class GHOST_SystemWayland : public GHOST_System {
   bool supportsCursorWarp();
   bool supportsWindowPosition();
 
+  bool getCursorGrabUseSoftwareDisplay(const GHOST_TGrabCursorMode mode);
+
   GHOST_TSuccess setCursorGrab(const GHOST_TGrabCursorMode mode,
                                const GHOST_TGrabCursorMode mode_current,
                                wl_surface *surface);
diff --git a/intern/ghost/intern/GHOST_Window.cpp b/intern/ghost/intern/GHOST_Window.cpp
index de7c5422d3f..d6dcb269c8f 100644
--- a/intern/ghost/intern/GHOST_Window.cpp
+++ b/intern/ghost/intern/GHOST_Window.cpp
@@ -164,7 +164,8 @@ GHOST_TSuccess GHOST_Window::getCursorGrabBounds(GHOST_Rect &bounds)
 
 void GHOST_Window::getCursorGrabState(GHOST_TGrabCursorMode &mode,
                                       GHOST_TAxisFlag &wrap_axis,
-                                      GHOST_Rect &bounds)
+                                      GHOST_Rect &bounds,
+                                      bool &use_software_cursor)
 {
   mode = m_cursorGrab;
   if (m_cursorGrab == GHOST_kGrabWrap) {
@@ -178,6 +179,14 @@ void GHOST_Window::getCursorGrabState(GHOST_TGrabCursorMode &mode,
     bounds.m_b = -1;
     wrap_axis = GHOST_kGrabAxisNone;
   }
+  use_software_cursor = (m_cursorGrab != GHOST_kGrabDisable) ? getCursorGrabUseSoftwareDisplay() :
+                                                               false;
+}
+
+bool GHOST_Window::getCursorGrabUseSoftwareDisplay()
+{
+  /* Sub-classes may override, by default don't use software cursor. */
+  return false;
 }
 
 GHOST_TSuccess GHOST_Window::setCursorShape(GHOST_TStandardCursor cursorShape)
diff --git a/intern/ghost/intern/GHOST_Window.h b/intern/ghost/intern/GHOST_Window.h
index adbc29eb84e..6ca2651fad0 100644
--- a/intern/ghost/intern/GHOST_Window.h
+++ b/intern/ghost/intern/GHOST_Window.h
@@ -154,7 +154,12 @@ class GHOST_Window : public GHOST_IWindow {
 
   void getCursorGrabState(GHOST_TGrabCursorMode &mode,
                           GHOST_TAxisFlag &axis_flag,
-                          GHOST_Rect &bounds);
+                          GHOST_Rect &bounds,
+                          bool &use_software_cursor);
+  /**
+   * Return true when a software cursor should be used.
+   */
+  bool getCursorGrabUseSoftwareDisplay();
 
   /**
    * Sets the progress bar value displayed in the window/application icon
diff --git a/intern/ghost/intern/GHOST_WindowWayland.cpp b/intern/ghost/intern/GHOST_WindowWayland.cpp
index 21e3793d3b1..8c7183971ab 100644
--- a/intern/ghost/intern/GHOST_WindowWayland.cpp
+++ b/intern/ghost/intern/GHOST_WindowWayland.cpp
@@ -492,6 +492,11 @@ GHOST_TSuccess GHOST_WindowWayland::setWindowCursorShape(GHOST_TStandardCursor s
   return ok;
 }
 
+bool GHOST_WindowWayland::getCursorGrabUseSoftwareDisplay()
+{
+  return m_system->getCursorGrabUseSoftwareDisplay(m_cursorGrab);
+}
+
 GHOST_TSuccess GHOST_WindowWayland::setWindowCustomCursorShape(
     uint8_t *bitmap, uint8_t *mask, int sizex, int sizey, int hotX, int hotY, bool canInvertColor)
 {
diff --git a/intern/ghost/intern/GHOST_WindowWayland.h b/intern/ghost/intern/GHOST_WindowWayland.h
index b6d9fa04079..f9effc636a1 100644
--- a/intern/ghost/intern/GHOST_WindowWayland.h
+++ b/intern/ghost/intern/GHOST_WindowWayland.h
@@ -52,6 +52,7 @@ class GHOST_WindowWayland : public GHOST_Window {
                                             int hotX,
                                             int hotY,
                                             bool canInvertColor) override;
+  bool getCursorGrabUseSoftwareDisplay() override;
 
   void setTitle(const char *title) override;
 
diff --git a/source/blender/windowmanager/intern/wm_draw.c b/source/blender/windowmanager/intern/wm_draw.c
index 63a7fb5ddaa..abf7aa65da7 100644
--- a/source/blender/windowmanager/intern/wm_draw.c
+++ b/source/blender/windowmanager/intern/wm_draw.c
@@ -158,9 +158,13 @@ static bool wm_software_cursor_needed_for_window(const wmWindow *win, struct Gra
   if (GHOST_GetCursorVisibility(win->ghostwin)) {
     /* NOTE: The value in `win->grabcursor` can't be used as it
      * doesn't always match GHOST's value in the case of tablet events. */
-    GHOST_GetCursorGrabState(
-        win->ghostwin, &grab_state->mode, &grab_state->wrap_axis, grab_state->bounds);
-    if (grab_state->mode == GHOST_kGrabWrap) {
+    bool use_software_cursor;
+    GHOST_GetCursorGrabState(win->ghostwin,
+                             &grab_state->mode,
+                             &grab_state->wrap_axis,
+                             grab_state->bounds,
+                             &use_software_cursor);
+    if (use_software_cursor) {
       return true;
     }
   }