Fix T84013: Crash closing maximized File Browser opened from Preferences

After 1e799dd26ec1, the logic to recognize a temporary File Browser window
didn't work correctly anymore. It would recognize a maximized File Browser
inside a temporary window as a temporary File Browser window, and attempt to
close this (rather than returning to the previous layout).
The logic there was pretty weak, and still is I think. A more stable solution
would need bigger refactoring.

With this, it's also not possible to maximize or fullscreen an area within a
temporary window (Preferences, File Browser, render window) anymore. Think that
won't make a noticable difference, since you couldn't open multiple areas there
anyway, and the area seems to be maximized already.

Cleaned up the code a bit to not become more confusing with the changes.

2 files changed, 45 insertions, 31 deletions

diff --git a/source/blender/editors/screen/screen_ops.c b/source/blender/editors/screen/screen_ops.c
index 8768404d74f..87255bf8b5c 100644
--- a/source/blender/editors/screen/screen_ops.c
+++ b/source/blender/editors/screen/screen_ops.c
@@ -3138,6 +3138,8 @@ static int screen_maximize_area_exec(bContext *C, wmOperator *op)
   ScrArea *area = NULL;
   const bool hide_panels = RNA_boolean_get(op->ptr, "use_hide_panels");
 
+  BLI_assert(!screen->temp);
+
   /* search current screen for 'fullscreen' areas */
   /* prevents restoring info header, when mouse is over it */
   LISTBASE_FOREACH (ScrArea *, area_iter, &screen->areabase) {
@@ -3169,11 +3171,14 @@ static int screen_maximize_area_exec(bContext *C, wmOperator *op)
 
 static bool screen_maximize_area_poll(bContext *C)
 {
+  const wmWindow *win = CTX_wm_window(C);
   const bScreen *screen = CTX_wm_screen(C);
   const ScrArea *area = CTX_wm_area(C);
   return ED_operator_areaactive(C) &&
          /* Don't allow maximizing global areas but allow minimizing from them. */
-         ((screen->state != SCREENNORMAL) || !ED_area_is_global(area));
+         ((screen->state != SCREENNORMAL) || !ED_area_is_global(area)) &&
+         /* Don't change temporary screens. */
+         !WM_window_is_temp_screen(win);
 }
 
 static void SCREEN_OT_screen_full_area(wmOperatorType *ot)
diff --git a/source/blender/windowmanager/intern/wm_event_system.c b/source/blender/windowmanager/intern/wm_event_system.c
index 20fa500f0b3..90aa6a92d3c 100644
--- a/source/blender/windowmanager/intern/wm_event_system.c
+++ b/source/blender/windowmanager/intern/wm_event_system.c
@@ -2259,38 +2259,47 @@ static int wm_handler_fileselect_do(bContext *C,
           bScreen *screen = WM_window_get_active_screen(win);
           ScrArea *file_area = screen->areabase.first;
 
-          if (screen->temp && (file_area->spacetype == SPACE_FILE)) {
-            int win_size[2];
-            bool is_maximized;
-            ED_fileselect_window_params_get(win, win_size, &is_maximized);
-            ED_fileselect_params_to_userdef(file_area->spacedata.first, win_size, is_maximized);
-
-            if (BLI_listbase_is_single(&file_area->spacedata)) {
-              BLI_assert(ctx_win != win);
-
-              wm_window_close(C, wm, win);
-
-              CTX_wm_window_set(C, ctx_win); /* #wm_window_close() NULLs. */
-              /* Some operators expect a drawable context (for EVT_FILESELECT_EXEC). */
-              wm_window_make_drawable(wm, ctx_win);
-              /* Ensure correct cursor position, otherwise, popups may close immediately after
-               * opening (UI_BLOCK_MOVEMOUSE_QUIT). */
-              wm_get_cursor_position(ctx_win, &ctx_win->eventstate->x, &ctx_win->eventstate->y);
-              wm->winactive = ctx_win; /* Reports use this... */
-              if (handler->context.win == win) {
-                handler->context.win = NULL;
-              }
-            }
-            else if (file_area->full) {
-              ED_screen_full_prevspace(C, file_area);
-            }
-            else {
-              ED_area_prevspace(C, file_area);
-            }
+          if ((file_area->spacetype != SPACE_FILE) || !WM_window_is_temp_screen(win)) {
+            continue;
+          }
 
-            temp_win = win;
-            break;
+          if (ctx_area->full) {
+            /* Users should not be able to maximize/fullscreen an area in a temporary screen. So if
+             * there's a maximized file browser in a temporary screen, it was likely opened by
+             * #EVT_FILESELECT_FULL_OPEN. */
+            continue;
           }
+
+          int win_size[2];
+          bool is_maximized;
+          ED_fileselect_window_params_get(win, win_size, &is_maximized);
+          ED_fileselect_params_to_userdef(file_area->spacedata.first, win_size, is_maximized);
+
+          if (BLI_listbase_is_single(&file_area->spacedata)) {
+            BLI_assert(ctx_win != win);
+
+            wm_window_close(C, wm, win);
+
+            CTX_wm_window_set(C, ctx_win); /* #wm_window_close() NULLs. */
+            /* Some operators expect a drawable context (for EVT_FILESELECT_EXEC). */
+            wm_window_make_drawable(wm, ctx_win);
+            /* Ensure correct cursor position, otherwise, popups may close immediately after
+             * opening (UI_BLOCK_MOVEMOUSE_QUIT). */
+            wm_get_cursor_position(ctx_win, &ctx_win->eventstate->x, &ctx_win->eventstate->y);
+            wm->winactive = ctx_win; /* Reports use this... */
+            if (handler->context.win == win) {
+              handler->context.win = NULL;
+            }
+          }
+          else if (file_area->full) {
+            ED_screen_full_prevspace(C, file_area);
+          }
+          else {
+            ED_area_prevspace(C, file_area);
+          }
+
+          temp_win = win;
+          break;
         }
 
         if (!temp_win && ctx_area->full) {