Fix T78688: Crash changing workspace with specific fullscreen setup

When switching workspaces we need to have an unused screen layout that
we can activate. The other window now showed the only available screen
layout in fullscreen though.
Usually when there's no unused screen layout we duplicate an existing
one, but that code didn't respect the fullscreen case properly.

This also tries to clean up the logic a bit, but things are still rather
complicated to follow.

Changes in this code are always risky. Of course things worked fine in
my tests, but I wouldn't be surprised if something breaks.

1 files changed, 60 insertions, 0 deletions

diff --git a/source/blender/editors/screen/workspace_layout_edit.c b/source/blender/editors/screen/workspace_layout_edit.c
index 0af81e0db21..8a36cffa1f1 100644
--- a/source/blender/editors/screen/workspace_layout_edit.c
+++ b/source/blender/editors/screen/workspace_layout_edit.c
@@ -160,6 +160,66 @@ bool ED_workspace_layout_delete(WorkSpace *workspace, WorkSpaceLayout *layout_ol
   return false;
 }
 
+static bool workspace_change_find_new_layout_cb(const WorkSpaceLayout *layout, void *UNUSED(arg))
+{
+  /* return false to stop the iterator if we've found a layout that can be activated */
+  return workspace_layout_set_poll(layout) ? false : true;
+}
+
+static bScreen *screen_fullscreen_find_associated_normal_screen(const Main *bmain, bScreen *screen)
+{
+  for (bScreen *screen_iter = bmain->screens.first; screen_iter;
+       screen_iter = screen_iter->id.next) {
+    if ((screen_iter != screen) && ELEM(screen_iter->state, SCREENMAXIMIZED, SCREENFULL)) {
+      ScrArea *area = screen_iter->areabase.first;
+      if (area && area->full == screen) {
+        return screen_iter;
+      }
+    }
+  }
+
+  return screen;
+}
+
+static bool screen_is_used_by_other_window(const wmWindow *win, const bScreen *screen)
+{
+  return BKE_screen_is_used(screen) && (screen->winid != win->winid);
+}
+
+/**
+ * Make sure there is a non-fullscreen layout to switch to that is not used yet by an other window.
+ * Needed for workspace or screen switching to ensure valid screens.
+ *
+ * \param layout_fallback_base: As last resort, this layout is duplicated and returned.
+ */
+WorkSpaceLayout *ED_workspace_screen_change_ensure_unused_layout(
+    Main *bmain,
+    WorkSpace *workspace,
+    WorkSpaceLayout *layout_new,
+    const WorkSpaceLayout *layout_fallback_base,
+    wmWindow *win)
+{
+  WorkSpaceLayout *layout_temp = layout_new;
+  bScreen *screen_temp = BKE_workspace_layout_screen_get(layout_new);
+
+  screen_temp = screen_fullscreen_find_associated_normal_screen(bmain, screen_temp);
+  layout_temp = BKE_workspace_layout_find(workspace, screen_temp);
+
+  if (screen_is_used_by_other_window(win, screen_temp)) {
+    /* Screen is already used, try to find a free one. */
+    layout_temp = BKE_workspace_layout_iter_circular(
+        workspace, layout_new, workspace_change_find_new_layout_cb, NULL, false);
+    screen_temp = layout_temp ? BKE_workspace_layout_screen_get(layout_temp) : NULL;
+
+    if (!layout_temp || screen_is_used_by_other_window(win, screen_temp)) {
+      /* Fallback solution: duplicate layout. */
+      layout_temp = ED_workspace_layout_duplicate(bmain, workspace, layout_fallback_base, win);
+    }
+  }
+
+  return layout_temp;
+}
+
 static bool workspace_layout_cycle_iter_cb(const WorkSpaceLayout *layout, void *UNUSED(arg))
 {
   /* return false to stop iterator when we have found a layout to activate */