Stacked fullscreen area support (proper implemenation)

Adds support for stacked fullscreens. This basically means, if a user opens a
temporary fullscreen mode, such as the File Browser or the Image Editor render
view, from a different fullscreen, the "Back to Previous" function or the other
ways to escape those temporary fullscreens don't return to the split screen
layout but to the previous fullscreen he has been in.

I already committed something similar (f7e844570fea862) but that was only
supposed as a fix, it didn't work for the "Back to Previous" operator and the
implementation wasn't really reusable. This one looks a bit nicer + makes some
older hacks unnecessary :)

8 files changed, 37 insertions, 48 deletions

diff --git a/source/blender/editors/include/ED_screen.h b/source/blender/editors/include/ED_screen.h
index 46da0160c5e..8c33395cc49 100644
--- a/source/blender/editors/include/ED_screen.h
+++ b/source/blender/editors/include/ED_screen.h
@@ -107,7 +107,7 @@ void    ED_screen_set_subwinactive(struct bContext *C, struct wmEvent *event);
 void    ED_screen_exit(struct bContext *C, struct wmWindow *window, struct bScreen *screen);
 void    ED_screen_animation_timer(struct bContext *C, int redraws, int refresh, int sync, int enable);
 void    ED_screen_animation_timer_update(struct bScreen *screen, int redraws, int refresh);
-void    ED_screen_restore_temp_type(struct bContext *C, ScrArea *sa, bool is_screen_change);
+void    ED_screen_restore_temp_type(struct bContext *C, ScrArea *sa);
 ScrArea *ED_screen_full_newspace(struct bContext *C, ScrArea *sa, int type);
 void    ED_screen_full_prevspace(struct bContext *C, ScrArea *sa);
 void    ED_screen_full_restore(struct bContext *C, ScrArea *sa);
diff --git a/source/blender/editors/render/render_view.c b/source/blender/editors/render/render_view.c
index 6c3572f4db9..ed78f8e0eaa 100644
--- a/source/blender/editors/render/render_view.c
+++ b/source/blender/editors/render/render_view.c
@@ -194,6 +194,11 @@ ScrArea *render_view_open(bContext *C, int mx, int my)
 
 				/* makes ESC go back to prev space */
 				sima->flag |= SI_PREVSPACE;
+
+				/* we already had a fullscreen here -> mark new space as a stacked fullscreen */
+				if (sa->full) {
+					sa->flag |= AREA_FLAG_STACKED_FULLSCREEN;
+				}
 			}
 			else {
 				/* use any area of decent size */
diff --git a/source/blender/editors/screen/area.c b/source/blender/editors/screen/area.c
index 698d4a4df2f..83b22bb1a8a 100644
--- a/source/blender/editors/screen/area.c
+++ b/source/blender/editors/screen/area.c
@@ -1645,6 +1645,8 @@ void ED_area_prevspace(bContext *C, ScrArea *sa)
 		/* no change */
 		return;
 	}
+	sa->flag &= ~AREA_FLAG_STACKED_FULLSCREEN;
+
 	ED_area_tag_redraw(sa);
 
 	/* send space change notifier */
diff --git a/source/blender/editors/screen/screen_edit.c b/source/blender/editors/screen/screen_edit.c
index edd64c0da56..f338fa160f5 100644
--- a/source/blender/editors/screen/screen_edit.c
+++ b/source/blender/editors/screen/screen_edit.c
@@ -1762,18 +1762,16 @@ ScrArea *ED_screen_full_newspace(bContext *C, ScrArea *sa, int type)
 
 void ED_screen_full_prevspace(bContext *C, ScrArea *sa)
 {
-	wmWindow *win = CTX_wm_window(C);
-
-	ED_area_prevspace(C, sa);
-
-	if (sa->full) {
-		/* only toggle out of fullscreen if it wasn't set by the user (for stacked fullscreens) */
-		if (sa->flag & AREA_FLAG_TEMP_TYPE)
-			ED_screen_state_toggle(C, win, sa, SCREENMAXIMIZED);
+	if (sa->flag & AREA_FLAG_STACKED_FULLSCREEN) {
+		/* stacked fullscreen -> only go back to previous screen and don't toggle out of fullscreen */
+		ED_area_prevspace(C, sa);
+	}
+	else {
+		ED_screen_restore_temp_type(C, sa);
 	}
 }
 
-void ED_screen_restore_temp_type(bContext *C, ScrArea *sa, bool is_screen_change)
+void ED_screen_restore_temp_type(bContext *C, ScrArea *sa)
 {
 	/* incase nether functions below run */
 	ED_area_tag_redraw(sa);
@@ -1783,7 +1781,7 @@ void ED_screen_restore_temp_type(bContext *C, ScrArea *sa, bool is_screen_change
 		sa->flag &= ~AREA_FLAG_TEMP_TYPE;
 	}
 
-	if (is_screen_change && sa->full) {
+	if (sa->full) {
 		ED_screen_state_toggle(C, CTX_wm_window(C), sa, SCREENMAXIMIZED);
 	}
 }
@@ -1796,25 +1794,11 @@ void ED_screen_full_restore(bContext *C, ScrArea *sa)
 	bScreen *screen = CTX_wm_screen(C);
 	short state = (screen ? screen->state : SCREENMAXIMIZED);
 	
-	/* if fullscreen area has a secondary space (such as a file browser or fullscreen render 
-	 * overlaid on top of a existing setup) then return to the previous space */
+	/* if fullscreen area has a temporary space (such as a file browser or fullscreen render
+	 * overlaid on top of an existing setup) then return to the previous space */
 	
 	if (sl->next) {
-		/* specific checks for space types */
-
-		/* Special check added for non-render image window (back from fullscreen through "Back to Previous" button) */
-		if (sl->spacetype == SPACE_IMAGE) {
-			SpaceImage *sima = sa->spacedata.first;
-
-			if (sima->flag & (SI_PREVSPACE | SI_FULLWINDOW)) {
-				sima->flag &= ~SI_PREVSPACE;
-				sima->flag &= ~SI_FULLWINDOW;
-				ED_screen_full_prevspace(C, sa);
-			}
-			else
-				ED_screen_state_toggle(C, win, sa, state);
-		}
-		else if (sa->flag & AREA_FLAG_TEMP_TYPE) {
+		if (sa->flag & AREA_FLAG_TEMP_TYPE) {
 			ED_screen_full_prevspace(C, sa);
 		}
 		else {
diff --git a/source/blender/editors/screen/screen_ops.c b/source/blender/editors/screen/screen_ops.c
index 8976224d368..1990216aa30 100644
--- a/source/blender/editors/screen/screen_ops.c
+++ b/source/blender/editors/screen/screen_ops.c
@@ -3697,9 +3697,9 @@ static int fullscreen_back_exec(bContext *C, wmOperator *op)
 		BKE_report(op->reports, RPT_ERROR, "No fullscreen areas were found");
 		return OPERATOR_CANCELLED;
 	}
-	
-	ED_screen_full_restore(C, sa);
-	
+
+	ED_screen_full_prevspace(C, sa);
+
 	return OPERATOR_FINISHED;
 }
 
diff --git a/source/blender/makesdna/DNA_screen_types.h b/source/blender/makesdna/DNA_screen_types.h
index 7372f55c3de..3e02071c3dd 100644
--- a/source/blender/makesdna/DNA_screen_types.h
+++ b/source/blender/makesdna/DNA_screen_types.h
@@ -272,14 +272,16 @@ typedef struct ARegion {
 
 /* area->flag */
 enum {
-	HEADER_NO_PULLDOWN      = (1 << 0),
-	AREA_FLAG_DRAWJOINTO    = (1 << 1),
-	AREA_FLAG_DRAWJOINFROM  = (1 << 2),
-	AREA_TEMP_INFO          = (1 << 3),
-	AREA_FLAG_DRAWSPLIT_H   = (1 << 4),
-	AREA_FLAG_DRAWSPLIT_V   = (1 << 5),
+	HEADER_NO_PULLDOWN           = (1 << 0),
+	AREA_FLAG_DRAWJOINTO         = (1 << 1),
+	AREA_FLAG_DRAWJOINFROM       = (1 << 2),
+	AREA_TEMP_INFO               = (1 << 3),
+	AREA_FLAG_DRAWSPLIT_H        = (1 << 4),
+	AREA_FLAG_DRAWSPLIT_V        = (1 << 5),
 	/* used to check if we should switch back to prevspace (of a different type) */
-	AREA_FLAG_TEMP_TYPE	    = (1 << 6),
+	AREA_FLAG_TEMP_TYPE          = (1 << 6),
+	/* for temporary fullscreens (file browser, image editor render) that are opened above user set fullscreens */
+	AREA_FLAG_STACKED_FULLSCREEN = (1 << 7),
 };
 
 #define EDGEWIDTH	1
diff --git a/source/blender/windowmanager/intern/wm_event_system.c b/source/blender/windowmanager/intern/wm_event_system.c
index 885aa69b179..4e5f0d3798d 100644
--- a/source/blender/windowmanager/intern/wm_event_system.c
+++ b/source/blender/windowmanager/intern/wm_event_system.c
@@ -1677,8 +1677,12 @@ static int wm_handler_fileselect_do(bContext *C, ListBase *handlers, wmEventHand
 				sa = handler->op_area;
 			}
 					
-			if (val == EVT_FILESELECT_OPEN) {
+			if (val == EVT_FILESELECT_OPEN || sa->full) {
 				ED_area_newspace(C, sa, SPACE_FILE);     /* 'sa' is modified in-place */
+				/* we already had a fullscreen here -> mark new space as a stacked fullscreen */
+				if (sa->full) {
+					sa->flag |= AREA_FLAG_STACKED_FULLSCREEN;
+				}
 			}
 			else {
 				sa = ED_screen_full_newspace(C, sa, SPACE_FILE);    /* sets context */
@@ -1702,15 +1706,11 @@ static int wm_handler_fileselect_do(bContext *C, ListBase *handlers, wmEventHand
 		case EVT_FILESELECT_CANCEL:
 		case EVT_FILESELECT_EXTERNAL_CANCEL:
 		{
-			/* XXX validate area and region? */
-			bScreen *screen = CTX_wm_screen(C);
-
 			/* remlink now, for load file case before removing*/
 			BLI_remlink(handlers, handler);
 
 			if (val != EVT_FILESELECT_EXTERNAL_CANCEL) {
-				ScrArea *sa = CTX_wm_area(C);
-				ED_screen_restore_temp_type(C, sa, screen != handler->filescreen);
+				ED_screen_full_prevspace(C, CTX_wm_area(C));
 			}
 
 			wm_handler_op_context(C, handler);
@@ -2494,7 +2494,6 @@ void WM_event_add_fileselect(bContext *C, wmOperator *op)
 	handler->op = op;
 	handler->op_area = CTX_wm_area(C);
 	handler->op_region = CTX_wm_region(C);
-	handler->filescreen = CTX_wm_screen(C);
 	
 	BLI_addhead(&win->modalhandlers, handler);
 	
diff --git a/source/blender/windowmanager/wm_event_system.h b/source/blender/windowmanager/wm_event_system.h
index d1a94194108..f89177a82ea 100644
--- a/source/blender/windowmanager/wm_event_system.h
+++ b/source/blender/windowmanager/wm_event_system.h
@@ -65,9 +65,6 @@ typedef struct wmEventHandler {
 	struct ARegion *ui_region;          /* for derived/modal handlers */
 	struct ARegion *ui_menu;            /* for derived/modal handlers */
 
-	/* fileselect handler re-uses modal operator data */
-	struct bScreen *filescreen;         /* screen it started in, to validate exec */
-
 	/* drop box handler */
 	ListBase *dropboxes;