UI: Register File Browser as Child/Dialog-Window for the OS

For many users, this will make the File Browser window behave more like
what they would expect. It addresses the issue of the File Browser
becoming hidden behind the main window by clicking anywhere in the
latter. It communicates the interruptive, but temporary nature of the
operation a bit better.
Further, on tiling window managers the File Browser now opens as
floating by default, like in other applications.

Note that this also makes sure the File Browser is always opened as
separate window, so it doesn't re-use the Preferences, or any other
temporary window anymore. This seems to have been a common annoyance.

More concretely, this makes the File Browser window behave as follows:
* Stays on top of its parent Blender window, but not on top of
  non-Blender windows.
* Minimizes with its parent window
* Can be moved independently
* Doesn't add an own item in task bars
* Doesn't block other Blender windows (we may want to have this though)
* Opens as floating window for tiling window managers (e.g. i3wm/Sway)

Further notes:
* When opening a file browser from the Preference window (or any
  temporary window), the main window, as the file browsers parent is
  moved on top of the Preferences, which makes it seem like the
  Preferences were closed. This is the general issue of bad secondary
  window handling as window activation changes. I made it so that the
  window is moved back once the file browser is closed.
  This behavior is confusing and would be nice to avoid. It's a separate
  issue though.
* On most window managers on Linux the temporary window can not be
  minimized and maximized, they disable that for dialog windows.
* On Windows and macOS, only minimizing is disabled, as there is no
  decent way yet to restore a window if it's not shown in the taskbar.

Reviewed By: Brecht van Lommel, Campbell Barton, William Reynish
Edits and macOS implementation by Brecht.

Differential Revision: https://developer.blender.org/D5810

Part of T69652.

7 files changed, 165 insertions, 104 deletions

diff --git a/source/blender/editors/include/ED_screen.h b/source/blender/editors/include/ED_screen.h
index 7c3aac6c688..c3e61f5f2b2 100644
--- a/source/blender/editors/include/ED_screen.h
+++ b/source/blender/editors/include/ED_screen.h
@@ -246,7 +246,8 @@ ScrArea *ED_screen_temp_space_open(struct bContext *C,
                                    int sizex,
                                    int sizey,
                                    eSpace_Type space_type,
-                                   int display_type);
+                                   int display_type,
+                                   bool dialog);
 void ED_screens_header_tools_menu_create(struct bContext *C, struct uiLayout *layout, void *arg);
 void ED_screens_footer_tools_menu_create(struct bContext *C, struct uiLayout *layout, void *arg);
 void ED_screens_navigation_bar_tools_menu_create(struct bContext *C,
diff --git a/source/blender/editors/render/render_view.c b/source/blender/editors/render/render_view.c
index 6873495e962..a54701f8725 100644
--- a/source/blender/editors/render/render_view.c
+++ b/source/blender/editors/render/render_view.c
@@ -156,8 +156,8 @@ ScrArea *render_view_open(bContext *C, int mx, int my, ReportList *reports)
     }
 
     /* changes context! */
-    if (WM_window_open_temp(C, IFACE_("Blender Render"), mx, my, sizex, sizey, SPACE_IMAGE) ==
-        NULL) {
+    if (WM_window_open_temp(
+            C, IFACE_("Blender Render"), mx, my, sizex, sizey, SPACE_IMAGE, false) == NULL) {
       BKE_report(reports, RPT_ERROR, "Failed to open window!");
       return NULL;
     }
diff --git a/source/blender/editors/screen/screen_edit.c b/source/blender/editors/screen/screen_edit.c
index c8008fe3cc7..bbdddfadc30 100644
--- a/source/blender/editors/screen/screen_edit.c
+++ b/source/blender/editors/screen/screen_edit.c
@@ -1375,13 +1375,14 @@ ScrArea *ED_screen_temp_space_open(bContext *C,
                                    int sizex,
                                    int sizey,
                                    eSpace_Type space_type,
-                                   int display_type)
+                                   int display_type,
+                                   bool dialog)
 {
   ScrArea *sa = NULL;
 
   switch (display_type) {
     case USER_TEMP_SPACE_DISPLAY_WINDOW:
-      if (WM_window_open_temp(C, title, x, y, sizex, sizey, (int)space_type)) {
+      if (WM_window_open_temp(C, title, x, y, sizex, sizey, (int)space_type, dialog)) {
         sa = CTX_wm_area(C);
       }
       break;
diff --git a/source/blender/editors/screen/screen_ops.c b/source/blender/editors/screen/screen_ops.c
index 0b374617cce..866144c272d 100644
--- a/source/blender/editors/screen/screen_ops.c
+++ b/source/blender/editors/screen/screen_ops.c
@@ -4828,9 +4828,14 @@ static int userpref_show_invoke(bContext *C, wmOperator *op, const wmEvent *even
   int sizey = 520 * UI_DPI_FAC;
 
   /* changes context! */
-  if (WM_window_open_temp(
-          C, IFACE_("Blender Preferences"), event->x, event->y, sizex, sizey, SPACE_USERPREF) !=
-      NULL) {
+  if (WM_window_open_temp(C,
+                          IFACE_("Blender Preferences"),
+                          event->x,
+                          event->y,
+                          sizex,
+                          sizey,
+                          SPACE_USERPREF,
+                          false) != NULL) {
     /* The header only contains the editor switcher and looks empty.
      * So hiding in the temp window makes sense. */
     ScrArea *area = CTX_wm_area(C);
@@ -4879,9 +4884,14 @@ static int drivers_editor_show_invoke(bContext *C, wmOperator *op, const wmEvent
   but = UI_context_active_but_prop_get(C, &ptr, &prop, &index);
 
   /* changes context! */
-  if (WM_window_open_temp(
-          C, IFACE_("Blender Drivers Editor"), event->x, event->y, sizex, sizey, SPACE_GRAPH) !=
-      NULL) {
+  if (WM_window_open_temp(C,
+                          IFACE_("Blender Drivers Editor"),
+                          event->x,
+                          event->y,
+                          sizex,
+                          sizey,
+                          SPACE_GRAPH,
+                          false) != NULL) {
     ED_drivers_editor_init(C, CTX_wm_area(C));
 
     /* activate driver F-Curve for the property under the cursor */
@@ -4939,9 +4949,14 @@ static int info_log_show_invoke(bContext *C, wmOperator *op, const wmEvent *even
   int shift_y = 480;
 
   /* changes context! */
-  if (WM_window_open_temp(
-          C, IFACE_("Blender Info Log"), event->x, event->y + shift_y, sizex, sizey, SPACE_INFO) !=
-      NULL) {
+  if (WM_window_open_temp(C,
+                          IFACE_("Blender Info Log"),
+                          event->x,
+                          event->y + shift_y,
+                          sizex,
+                          sizey,
+                          SPACE_INFO,
+                          false) != NULL) {
     return OPERATOR_FINISHED;
   }
   else {
diff --git a/source/blender/windowmanager/WM_api.h b/source/blender/windowmanager/WM_api.h
index 2d9fa9e5ab6..df253d619c9 100644
--- a/source/blender/windowmanager/WM_api.h
+++ b/source/blender/windowmanager/WM_api.h
@@ -156,8 +156,14 @@ void WM_opengl_context_activate(void *context);
 void WM_opengl_context_release(void *context);
 
 struct wmWindow *WM_window_open(struct bContext *C, const struct rcti *rect);
-struct wmWindow *WM_window_open_temp(
-    struct bContext *C, const char *title, int x, int y, int sizex, int sizey, int space_type);
+struct wmWindow *WM_window_open_temp(struct bContext *C,
+                                     const char *title,
+                                     int x,
+                                     int y,
+                                     int sizex,
+                                     int sizey,
+                                     int space_type,
+                                     bool dialog);
 void WM_window_set_dpi(wmWindow *win);
 
 bool WM_stereo3d_enabled(struct wmWindow *win, bool only_fullscreen_test);
diff --git a/source/blender/windowmanager/intern/wm_event_system.c b/source/blender/windowmanager/intern/wm_event_system.c
index 6991dd2f9a6..1eec08c2e7a 100644
--- a/source/blender/windowmanager/intern/wm_event_system.c
+++ b/source/blender/windowmanager/intern/wm_event_system.c
@@ -2352,7 +2352,8 @@ static int wm_handler_fileselect_do(bContext *C,
                                             U.file_space_data.temp_win_sizex * UI_DPI_FAC,
                                             U.file_space_data.temp_win_sizey * UI_DPI_FAC,
                                             SPACE_FILE,
-                                            U.filebrowser_display_type))) {
+                                            U.filebrowser_display_type,
+                                            true))) {
         ARegion *region_header = BKE_area_find_region_type(area, RGN_TYPE_HEADER);
 
         BLI_assert(area->spacetype == SPACE_FILE);
diff --git a/source/blender/windowmanager/intern/wm_window.c b/source/blender/windowmanager/intern/wm_window.c
index 2c26a15dce0..1ae95917c99 100644
--- a/source/blender/windowmanager/intern/wm_window.c
+++ b/source/blender/windowmanager/intern/wm_window.c
@@ -409,8 +409,10 @@ void wm_quit_with_optional_confirmation_prompt(bContext *C, wmWindow *win)
 /* this is event from ghost, or exit-blender op */
 void wm_window_close(bContext *C, wmWindowManager *wm, wmWindow *win)
 {
-  /* First check if there is another main window remaining. */
   wmWindow *win_other;
+  const bool is_dialog = GHOST_IsDialogWindow(win->ghostwin);
+
+  /* First check if there is another main window remaining. */
   for (win_other = wm->windows.first; win_other; win_other = win_other->next) {
     if (win_other != win && win_other->parent == NULL && !WM_window_is_temp_screen(win_other)) {
       break;
@@ -422,10 +424,15 @@ void wm_window_close(bContext *C, wmWindowManager *wm, wmWindow *win)
     return;
   }
 
-  /* close child windows */
-  for (wmWindow *win_child = wm->windows.first; win_child; win_child = win_child->next) {
-    if (win_child->parent == win) {
-      wm_window_close(C, wm, win_child);
+  /* Close child windows and bring windows back to front that dialogs have pushed behind the main
+   * window. */
+  for (wmWindow *iter_win = wm->windows.first; iter_win; iter_win = iter_win->next) {
+    if (iter_win->parent == win) {
+      wm_window_close(C, wm, iter_win);
+    }
+    else if (is_dialog && iter_win != win && iter_win->parent &&
+             (GHOST_GetWindowState(iter_win->ghostwin) != GHOST_kWindowStateMinimized)) {
+      wm_window_raise(iter_win);
     }
   }
 
@@ -547,7 +554,10 @@ static void wm_window_ensure_eventstate(wmWindow *win)
 }
 
 /* belongs to below */
-static void wm_window_ghostwindow_add(wmWindowManager *wm, const char *title, wmWindow *win)
+static void wm_window_ghostwindow_add(wmWindowManager *wm,
+                                      const char *title,
+                                      wmWindow *win,
+                                      bool is_dialog)
 {
   GHOST_WindowHandle ghostwin;
   GHOST_GLSettings glSettings = {0};
@@ -569,15 +579,29 @@ static void wm_window_ghostwindow_add(wmWindowManager *wm, const char *title, wm
   wmWindow *prev_windrawable = wm->windrawable;
   wm_window_clear_drawable(wm);
 
-  ghostwin = GHOST_CreateWindow(g_system,
-                                title,
-                                win->posx,
-                                posy,
-                                win->sizex,
-                                win->sizey,
-                                (GHOST_TWindowState)win->windowstate,
-                                GHOST_kDrawingContextTypeOpenGL,
-                                glSettings);
+  if (is_dialog && win->parent) {
+    ghostwin = GHOST_CreateDialogWindow(g_system,
+                                        win->parent->ghostwin,
+                                        title,
+                                        win->posx,
+                                        posy,
+                                        win->sizex,
+                                        win->sizey,
+                                        (GHOST_TWindowState)win->windowstate,
+                                        GHOST_kDrawingContextTypeOpenGL,
+                                        glSettings);
+  }
+  else {
+    ghostwin = GHOST_CreateWindow(g_system,
+                                  title,
+                                  win->posx,
+                                  posy,
+                                  win->sizex,
+                                  win->sizey,
+                                  (GHOST_TWindowState)win->windowstate,
+                                  GHOST_kDrawingContextTypeOpenGL,
+                                  glSettings);
+  }
 
   if (ghostwin) {
     GHOST_RectangleHandle bounds;
@@ -635,6 +659,68 @@ static void wm_window_ghostwindow_add(wmWindowManager *wm, const char *title, wm
   }
 }
 
+static void wm_window_ghostwindow_ensure(wmWindowManager *wm, wmWindow *win, bool is_dialog)
+{
+  wmKeyMap *keymap;
+
+  if (win->ghostwin == NULL) {
+    if ((win->sizex == 0) || (wm_init_state.override_flag & WIN_OVERRIDE_GEOM)) {
+      win->posx = wm_init_state.start_x;
+      win->posy = wm_init_state.start_y;
+      win->sizex = wm_init_state.size_x;
+      win->sizey = wm_init_state.size_y;
+
+      if (wm_init_state.override_flag & WIN_OVERRIDE_GEOM) {
+        win->windowstate = GHOST_kWindowStateNormal;
+        wm_init_state.override_flag &= ~WIN_OVERRIDE_GEOM;
+      }
+      else {
+        win->windowstate = GHOST_WINDOW_STATE_DEFAULT;
+      }
+    }
+
+    if (wm_init_state.override_flag & WIN_OVERRIDE_WINSTATE) {
+      win->windowstate = wm_init_state.windowstate;
+      wm_init_state.override_flag &= ~WIN_OVERRIDE_WINSTATE;
+    }
+
+    /* without this, cursor restore may fail, T45456 */
+    if (win->cursor == 0) {
+      win->cursor = WM_CURSOR_DEFAULT;
+    }
+
+    wm_window_ghostwindow_add(wm, "Blender", win, is_dialog);
+  }
+
+  if (win->ghostwin != NULL) {
+    /* If we have no ghostwin this is a buggy window that should be removed.
+     * However we still need to initialize it correctly so the screen doesn't hang. */
+
+    /* happens after fileread */
+    wm_window_ensure_eventstate(win);
+  }
+
+  /* add keymap handlers (1 handler for all keys in map!) */
+  keymap = WM_keymap_ensure(wm->defaultconf, "Window", 0, 0);
+  WM_event_add_keymap_handler(&win->handlers, keymap);
+
+  keymap = WM_keymap_ensure(wm->defaultconf, "Screen", 0, 0);
+  WM_event_add_keymap_handler(&win->handlers, keymap);
+
+  keymap = WM_keymap_ensure(wm->defaultconf, "Screen Editing", 0, 0);
+  WM_event_add_keymap_handler(&win->modalhandlers, keymap);
+
+  /* add drop boxes */
+  {
+    ListBase *lb = WM_dropboxmap_find("Window", 0, 0);
+    WM_event_add_dropbox_handler(&win->handlers, lb);
+  }
+  wm_window_title(wm, win);
+
+  /* add topbar */
+  ED_screen_global_areas_refresh(win);
+}
+
 /**
  * Initialize #wmWindow without ghostwin, open these and clear.
  *
@@ -650,9 +736,6 @@ static void wm_window_ghostwindow_add(wmWindowManager *wm, const char *title, wm
  */
 void wm_window_ghostwindows_ensure(wmWindowManager *wm)
 {
-  wmKeyMap *keymap;
-  wmWindow *win;
-
   BLI_assert(G.background == false);
 
   /* No command-line prefsize? then we set this.
@@ -682,63 +765,8 @@ void wm_window_ghostwindows_ensure(wmWindowManager *wm)
 #endif
   }
 
-  for (win = wm->windows.first; win; win = win->next) {
-    if (win->ghostwin == NULL) {
-      if ((win->sizex == 0) || (wm_init_state.override_flag & WIN_OVERRIDE_GEOM)) {
-        win->posx = wm_init_state.start_x;
-        win->posy = wm_init_state.start_y;
-        win->sizex = wm_init_state.size_x;
-        win->sizey = wm_init_state.size_y;
-
-        if (wm_init_state.override_flag & WIN_OVERRIDE_GEOM) {
-          win->windowstate = GHOST_kWindowStateNormal;
-          wm_init_state.override_flag &= ~WIN_OVERRIDE_GEOM;
-        }
-        else {
-          win->windowstate = GHOST_WINDOW_STATE_DEFAULT;
-        }
-      }
-
-      if (wm_init_state.override_flag & WIN_OVERRIDE_WINSTATE) {
-        win->windowstate = wm_init_state.windowstate;
-        wm_init_state.override_flag &= ~WIN_OVERRIDE_WINSTATE;
-      }
-
-      /* without this, cursor restore may fail, T45456 */
-      if (win->cursor == 0) {
-        win->cursor = WM_CURSOR_DEFAULT;
-      }
-
-      wm_window_ghostwindow_add(wm, "Blender", win);
-    }
-
-    if (win->ghostwin != NULL) {
-      /* If we have no ghostwin this is a buggy window that should be removed.
-       * However we still need to initialize it correctly so the screen doesn't hang. */
-
-      /* happens after fileread */
-      wm_window_ensure_eventstate(win);
-    }
-
-    /* add keymap handlers (1 handler for all keys in map!) */
-    keymap = WM_keymap_ensure(wm->defaultconf, "Window", 0, 0);
-    WM_event_add_keymap_handler(&win->handlers, keymap);
-
-    keymap = WM_keymap_ensure(wm->defaultconf, "Screen", 0, 0);
-    WM_event_add_keymap_handler(&win->handlers, keymap);
-
-    keymap = WM_keymap_ensure(wm->defaultconf, "Screen Editing", 0, 0);
-    WM_event_add_keymap_handler(&win->modalhandlers, keymap);
-
-    /* add drop boxes */
-    {
-      ListBase *lb = WM_dropboxmap_find("Window", 0, 0);
-      WM_event_add_dropbox_handler(&win->handlers, lb);
-    }
-    wm_window_title(wm, win);
-
-    /* add topbar */
-    ED_screen_global_areas_refresh(win);
+  for (wmWindow *win = wm->windows.first; win; win = win->next) {
+    wm_window_ghostwindow_ensure(wm, win, false);
   }
 }
 
@@ -795,10 +823,17 @@ wmWindow *WM_window_open(bContext *C, const rcti *rect)
  * \param space_type: SPACE_VIEW3D, SPACE_INFO, ... (eSpace_Type)
  * \return the window or NULL in case of failure.
  */
-wmWindow *WM_window_open_temp(
-    bContext *C, const char *title, int x, int y, int sizex, int sizey, int space_type)
+wmWindow *WM_window_open_temp(bContext *C,
+                              const char *title,
+                              int x,
+                              int y,
+                              int sizex,
+                              int sizey,
+                              int space_type,
+                              bool dialog)
 {
   Main *bmain = CTX_data_main(C);
+  wmWindowManager *wm = CTX_wm_manager(C);
   wmWindow *win_prev = CTX_wm_window(C);
   wmWindow *win;
   bScreen *screen;
@@ -823,9 +858,10 @@ wmWindow *WM_window_open_temp(
   /* changes rect to fit within desktop */
   wm_window_check_position(&rect);
 
-  /* test if we have a temp screen already */
-  for (win = CTX_wm_manager(C)->windows.first; win; win = win->next) {
-    if (WM_window_is_temp_screen(win)) {
+  /* Reuse temporary or dialog window if one is open (but don't use a dialog for a regular
+   * temporary window, or vice versa). */
+  for (win = wm->windows.first; win; win = win->next) {
+    if (WM_window_is_temp_screen(win) && (dialog == GHOST_IsDialogWindow(win->ghostwin))) {
       break;
     }
   }
@@ -843,11 +879,6 @@ wmWindow *WM_window_open_temp(
   win->sizex = BLI_rcti_size_x(&rect);
   win->sizey = BLI_rcti_size_y(&rect);
 
-  if (win->ghostwin) {
-    wm_window_set_size(win, win->sizex, win->sizey);
-    wm_window_raise(win);
-  }
-
   if (WM_window_get_active_workspace(win) == NULL) {
     WorkSpace *workspace = WM_window_get_active_workspace(win_prev);
     BKE_workspace_active_set(win->workspace_hook, workspace);
@@ -872,6 +903,9 @@ wmWindow *WM_window_open_temp(
 
   /* make window active, and validate/resize */
   CTX_wm_window_set(C, win);
+  if (!win->ghostwin) {
+    wm_window_ghostwindow_ensure(wm, win, dialog);
+  }
   WM_check(C);
 
   /* It's possible `win->ghostwin == NULL`.
@@ -887,15 +921,18 @@ wmWindow *WM_window_open_temp(
   ED_area_newspace(C, sa, space_type, false);
 
   ED_screen_change(C, screen);
-  ED_screen_refresh(CTX_wm_manager(C), win); /* test scale */
+  ED_screen_refresh(wm, win); /* test scale */
 
   if (win->ghostwin) {
+    wm_window_set_size(win, win->sizex, win->sizey);
+    wm_window_raise(win);
+
     GHOST_SetTitle(win->ghostwin, title);
     return win;
   }
   else {
     /* very unlikely! but opening a new window can fail */
-    wm_window_close(C, CTX_wm_manager(C), win);
+    wm_window_close(C, wm, win);
     CTX_wm_window_set(C, win_prev);
 
     return NULL;