UI: Rewrite stacked full-screen logic, fixing issues

To recreate the main issue:
* Set render and file browser to show in full-screen in the preferences
* Default scene, press F12 in 3D View
* Press Alt+S to save the image
* Escape the file browser
* Escape the image editor
The former 3D View would now show the image editor. This is a common
use-case that should work.

Full-screen code is a hassle to get to work as expected. There are
reports from 2.5, I did lots of work years ago to get these kind of
use-cases to work fine. But apparently I broke this one with a fix for
another common use-case in March (0a28bb14222c).
This now stores hints in the space, rather than the area, which should
make things much more controlable and hopefully help us fix issues like
this.
Here are a few references describing further common issues (all should
work fine now): 0a28bb14222c, e61588c5a544, T19296

Checked over this with Bastien, we agreed that at some point we should
do a big rewrite of all of this, for now this is acceptable.

1 files changed, 14 insertions, 13 deletions

diff --git a/source/blender/editors/screen/screen_edit.c b/source/blender/editors/screen/screen_edit.c
index bbdddfadc30..5b8fd33a4e9 100644
--- a/source/blender/editors/screen/screen_edit.c
+++ b/source/blender/editors/screen/screen_edit.c
@@ -1115,6 +1115,7 @@ ScrArea *ED_screen_full_newspace(bContext *C, ScrArea *sa, int type)
 {
   wmWindow *win = CTX_wm_window(C);
   ScrArea *newsa = NULL;
+  SpaceLink *newsl;
 
   if (!sa || sa->full == NULL) {
     newsa = ED_screen_state_toggle(C, win, sa, SCREENMAXIMIZED);
@@ -1125,15 +1126,14 @@ ScrArea *ED_screen_full_newspace(bContext *C, ScrArea *sa, int type)
   }
 
   BLI_assert(newsa);
+  newsl = newsa->spacedata.first;
 
-  if (sa && (sa->spacetype != type)) {
-    newsa->flag |= AREA_FLAG_TEMP_TYPE;
-  }
-  else {
-    newsa->flag &= ~AREA_FLAG_TEMP_TYPE;
+  /* Tag the active space before changing, so we can identify it when user wants to go back. */
+  if ((newsl->link_flag & SPACE_FLAG_TYPE_TEMPORARY) == 0) {
+    newsl->link_flag |= SPACE_FLAG_TYPE_WAS_ACTIVE;
   }
 
-  ED_area_newspace(C, newsa, type, (newsa->flag & AREA_FLAG_TEMP_TYPE));
+  ED_area_newspace(C, newsa, type, newsl->link_flag & SPACE_FLAG_TYPE_TEMPORARY);
 
   return newsa;
 }
@@ -1146,7 +1146,7 @@ void ED_screen_full_prevspace(bContext *C, ScrArea *sa)
   BLI_assert(sa->full);
 
   if (sa->flag & AREA_FLAG_STACKED_FULLSCREEN) {
-    /* stacked fullscreen -> only go back to previous screen and don't toggle out of fullscreen */
+    /* stacked fullscreen -> only go back to previous area and don't toggle out of fullscreen */
     ED_area_prevspace(C, sa);
   }
   else {
@@ -1156,13 +1156,13 @@ void ED_screen_full_prevspace(bContext *C, ScrArea *sa)
 
 void ED_screen_restore_temp_type(bContext *C, ScrArea *sa)
 {
+  SpaceLink *sl = sa->spacedata.first;
+
   /* In case nether functions below run. */
   ED_area_tag_redraw(sa);
 
-  if (sa->flag & AREA_FLAG_TEMP_TYPE) {
+  if (sl->link_flag & SPACE_FLAG_TYPE_TEMPORARY) {
     ED_area_prevspace(C, sa);
-    /* Flag should be cleared now. */
-    BLI_assert((sa->flag & AREA_FLAG_TEMP_TYPE) == 0);
   }
 
   if (sa->full) {
@@ -1182,7 +1182,7 @@ void ED_screen_full_restore(bContext *C, ScrArea *sa)
    * overlaid on top of an existing setup) then return to the previous space */
 
   if (sl->next) {
-    if (sa->flag & AREA_FLAG_TEMP_TYPE) {
+    if (sl->link_flag & SPACE_FLAG_TYPE_TEMPORARY) {
       ED_screen_full_prevspace(C, sa);
     }
     else {
@@ -1392,14 +1392,15 @@ ScrArea *ED_screen_temp_space_open(bContext *C,
       if (ctx_sa->full) {
         sa = ctx_sa;
         ED_area_newspace(C, ctx_sa, space_type, true);
-        /* we already had a fullscreen here -> mark new space as a stacked fullscreen */
-        sa->flag |= (AREA_FLAG_STACKED_FULLSCREEN | AREA_FLAG_TEMP_TYPE);
+        sa->flag |= AREA_FLAG_STACKED_FULLSCREEN;
+        ((SpaceLink *)sa->spacedata.first)->link_flag |= SPACE_FLAG_TYPE_TEMPORARY;
       }
       else if (ctx_sa->spacetype == space_type) {
         sa = ED_screen_state_toggle(C, CTX_wm_window(C), ctx_sa, SCREENMAXIMIZED);
       }
       else {
         sa = ED_screen_full_newspace(C, ctx_sa, (int)space_type);
+        ((SpaceLink *)sa->spacedata.first)->link_flag |= SPACE_FLAG_TYPE_TEMPORARY;
       }
       break;
     }