WM: Fix crash when a new window can't be created

Report an error instead of crashing if a new window can't be created
(typically caused by bad drivers).

11 files changed, 151 insertions, 52 deletions

diff --git a/source/blender/editors/render/render_intern.h b/source/blender/editors/render/render_intern.h
index 54429f9f066..26f313cb3fe 100644
--- a/source/blender/editors/render/render_intern.h
+++ b/source/blender/editors/render/render_intern.h
@@ -93,7 +93,7 @@ void render_view3d_update(struct RenderEngine *engine, const struct bContext *C)
 void render_view3d_draw(struct RenderEngine *engine, const struct bContext *C);
 
 /* render_view.c */
-struct ScrArea *render_view_open(struct bContext *C, int mx, int my);
+struct ScrArea *render_view_open(struct bContext *C, int mx, int my, struct ReportList *reports);
 
 void RENDER_OT_view_show(struct wmOperatorType *ot);
 void RENDER_OT_view_cancel(struct wmOperatorType *ot);
diff --git a/source/blender/editors/render/render_internal.c b/source/blender/editors/render/render_internal.c
index 10de2d667ea..4c10c07e189 100644
--- a/source/blender/editors/render/render_internal.c
+++ b/source/blender/editors/render/render_internal.c
@@ -877,7 +877,7 @@ static int screen_render_invoke(bContext *C, wmOperator *op, const wmEvent *even
 	// store spare
 
 	/* ensure at least 1 area shows result */
-	sa = render_view_open(C, event->x, event->y);
+	sa = render_view_open(C, event->x, event->y, op->reports);
 
 	jobflag = WM_JOB_EXCL_RENDER | WM_JOB_PRIORITY | WM_JOB_PROGRESS;
 	
diff --git a/source/blender/editors/render/render_opengl.c b/source/blender/editors/render/render_opengl.c
index fe4022709de..954fcdef63e 100644
--- a/source/blender/editors/render/render_opengl.c
+++ b/source/blender/editors/render/render_opengl.c
@@ -911,7 +911,7 @@ static int screen_opengl_render_invoke(bContext *C, wmOperator *op, const wmEven
 	}
 	
 	oglrender = op->customdata;
-	render_view_open(C, event->x, event->y);
+	render_view_open(C, event->x, event->y, op->reports);
 	
 	/* view may be changed above (R_OUTPUT_WINDOW) */
 	oglrender->win = CTX_wm_window(C);
diff --git a/source/blender/editors/render/render_view.c b/source/blender/editors/render/render_view.c
index f6690296890..4e36303d829 100644
--- a/source/blender/editors/render/render_view.c
+++ b/source/blender/editors/render/render_view.c
@@ -38,6 +38,7 @@
 #include "BKE_image.h"
 #include "BKE_global.h"
 #include "BKE_screen.h"
+#include "BKE_report.h"
 
 #include "WM_api.h"
 #include "WM_types.h"
@@ -125,7 +126,7 @@ static ScrArea *find_area_image_empty(bContext *C)
 /********************** open image editor for render *************************/
 
 /* new window uses x,y to set position */
-ScrArea *render_view_open(bContext *C, int mx, int my)
+ScrArea *render_view_open(bContext *C, int mx, int my, ReportList *reports)
 {
 	wmWindow *win = CTX_wm_window(C);
 	Scene *scene = CTX_data_scene(C);
@@ -155,7 +156,10 @@ ScrArea *render_view_open(bContext *C, int mx, int my)
 		rect.ymax = rect.ymin + sizey;
 
 		/* changes context! */
-		WM_window_open_temp(C, &rect, WM_WINDOW_RENDER);
+		if (WM_window_open_temp(C, &rect, WM_WINDOW_RENDER) == NULL) {
+			BKE_report(reports, RPT_ERROR, "Failed to open window!");
+			return NULL;
+		}
 
 		sa = CTX_wm_area(C);
 	}
@@ -292,7 +296,7 @@ void RENDER_OT_view_cancel(struct wmOperatorType *ot)
 
 /************************* show render viewer *****************/
 
-static int render_view_show_invoke(bContext *C, wmOperator *UNUSED(op), const wmEvent *event)
+static int render_view_show_invoke(bContext *C, wmOperator *op, const wmEvent *event)
 {
 	wmWindow *wincur = CTX_wm_window(C);
 	
@@ -341,7 +345,7 @@ static int render_view_show_invoke(bContext *C, wmOperator *UNUSED(op), const wm
 			}
 		}
 		else {
-			render_view_open(C, event->x, event->y);
+			render_view_open(C, event->x, event->y, op->reports);
 		}
 	}
 
diff --git a/source/blender/editors/screen/screen_ops.c b/source/blender/editors/screen/screen_ops.c
index 4a1c1e34414..24653dc556c 100644
--- a/source/blender/editors/screen/screen_ops.c
+++ b/source/blender/editors/screen/screen_ops.c
@@ -992,6 +992,11 @@ static int area_dupli_invoke(bContext *C, wmOperator *op, const wmEvent *event)
 	rect.ymax = rect.ymin + BLI_rcti_size_y(&rect) / U.pixelsize;
 
 	newwin = WM_window_open(C, &rect);
+	if (newwin == NULL) {
+		BKE_report(op->reports, RPT_ERROR, "Failed to open window!");
+		goto finally;
+	}
+
 	*newwin->stereo3d_format = *win->stereo3d_format;
 	
 	/* allocs new screen and adds to newly created window, using window size */
@@ -1005,11 +1010,18 @@ static int area_dupli_invoke(bContext *C, wmOperator *op, const wmEvent *event)
 
 	/* screen, areas init */
 	WM_event_add_notifier(C, NC_SCREEN | NA_EDITED, NULL);
-	
+
+
+finally:
 	if (event->type == EVT_ACTIONZONE_AREA)
 		actionzone_exit(op);
 	
-	return OPERATOR_FINISHED;
+	if (newwin) {
+		return OPERATOR_FINISHED;
+	}
+	else {
+		return OPERATOR_CANCELLED;
+	}
 }
 
 static void SCREEN_OT_area_dupli(wmOperatorType *ot)
@@ -3845,7 +3857,7 @@ static void SCREEN_OT_back_to_previous(struct wmOperatorType *ot)
 
 /* *********** show user pref window ****** */
 
-static int userpref_show_invoke(bContext *C, wmOperator *UNUSED(op), const wmEvent *event)
+static int userpref_show_invoke(bContext *C, wmOperator *op, const wmEvent *event)
 {
 	wmWindow *win = CTX_wm_window(C);
 	rcti rect;
@@ -3862,9 +3874,13 @@ static int userpref_show_invoke(bContext *C, wmOperator *UNUSED(op), const wmEve
 	rect.ymax = rect.ymin + sizey;
 	
 	/* changes context! */
-	WM_window_open_temp(C, &rect, WM_WINDOW_USERPREFS);
-	
-	return OPERATOR_FINISHED;
+	if (WM_window_open_temp(C, &rect, WM_WINDOW_USERPREFS) != NULL) {
+		return OPERATOR_FINISHED;
+	}
+	else {
+		BKE_report(op->reports, RPT_ERROR, "Failed to open window!");
+		return OPERATOR_CANCELLED;
+	}
 }
 
 
diff --git a/source/blender/gpu/intern/gpu_init_exit.c b/source/blender/gpu/intern/gpu_init_exit.c
index 3a8a6fca23b..da4dd65d2e1 100644
--- a/source/blender/gpu/intern/gpu_init_exit.c
+++ b/source/blender/gpu/intern/gpu_init_exit.c
@@ -49,7 +49,7 @@ static bool initialized = false;
 
 void GPU_init(void)
 {
-	/* can't avoid calling this multiple times, see wm_window_add_ghostwindow */
+	/* can't avoid calling this multiple times, see wm_window_ghostwindow_add */
 	if (initialized)
 		return;
 
diff --git a/source/blender/windowmanager/WM_api.h b/source/blender/windowmanager/WM_api.h
index abbbe759858..8aea5816b7f 100644
--- a/source/blender/windowmanager/WM_api.h
+++ b/source/blender/windowmanager/WM_api.h
@@ -87,18 +87,19 @@ void		WM_init_splash		(struct bContext *C);
 
 void		WM_check			(struct bContext *C);
 
-struct wmWindow	*WM_window_open	(struct bContext *C, const struct rcti *rect);
-
 int			WM_window_pixels_x		(struct wmWindow *win);
 int			WM_window_pixels_y		(struct wmWindow *win);
 bool		WM_window_is_fullscreen	(struct wmWindow *win);
 
 /* defines for 'type' WM_window_open_temp */
-#define WM_WINDOW_RENDER		0
-#define WM_WINDOW_USERPREFS		1
-// #define WM_WINDOW_FILESEL		2  // UNUSED
+enum {
+	WM_WINDOW_RENDER = 1,
+	WM_WINDOW_USERPREFS,
+	// WM_WINDOW_FILESEL // UNUSED
+};
 
-void		WM_window_open_temp	(struct bContext *C, struct rcti *position, int type);
+struct wmWindow	*WM_window_open(struct bContext *C, const struct rcti *rect);
+struct wmWindow *WM_window_open_temp(struct bContext *C, const struct rcti *rect_init, int type);
 			
 			/* returns true if draw method is triple buffer */
 bool		WM_is_draw_triple(struct wmWindow *win);
diff --git a/source/blender/windowmanager/intern/wm.c b/source/blender/windowmanager/intern/wm.c
index fe92d80c1bd..b76a1f1d422 100644
--- a/source/blender/windowmanager/intern/wm.c
+++ b/source/blender/windowmanager/intern/wm.c
@@ -376,7 +376,7 @@ void WM_check(bContext *C)
 		}
 
 		/* case: no open windows at all, for old file reads */
-		wm_window_add_ghostwindows(wm);
+		wm_window_ghostwindows_ensure(wm);
 	}
 
 	/* case: fileread */
diff --git a/source/blender/windowmanager/intern/wm_files.c b/source/blender/windowmanager/intern/wm_files.c
index d08c6c59a7f..fc5cad6d057 100644
--- a/source/blender/windowmanager/intern/wm_files.c
+++ b/source/blender/windowmanager/intern/wm_files.c
@@ -438,7 +438,14 @@ void wm_file_read_report(bContext *C)
 static void wm_file_read_post(bContext *C, bool is_startup_file)
 {
 	bool addons_loaded = false;
-	CTX_wm_window_set(C, CTX_wm_manager(C)->windows.first);
+	wmWindowManager *wm = CTX_wm_manager(C);
+
+	if (!G.background) {
+		/* remove windows which failed to be added via WM_check */
+		wm_window_ghostwindows_remove_invalid(C, wm);
+	}
+
+	CTX_wm_window_set(C, wm->windows.first);
 
 	ED_editors_init(C);
 	DAG_on_visible_update(CTX_data_main(C), true);
diff --git a/source/blender/windowmanager/intern/wm_window.c b/source/blender/windowmanager/intern/wm_window.c
index 030399a9c7d..51feea55654 100644
--- a/source/blender/windowmanager/intern/wm_window.c
+++ b/source/blender/windowmanager/intern/wm_window.c
@@ -329,12 +329,17 @@ void wm_window_close(bContext *C, wmWindowManager *wm, wmWindow *win)
 		CTX_wm_window_set(C, win);  /* needed by handlers */
 		WM_event_remove_handlers(C, &win->handlers);
 		WM_event_remove_handlers(C, &win->modalhandlers);
-		ED_screen_exit(C, win, win->screen); 
+
+		/* for regular use this will _never_ be NULL,
+		 * however we may be freeing an improperly initialized window. */
+		if (win->screen) {
+			ED_screen_exit(C, win, win->screen);
+		}
 		
 		wm_window_free(C, wm, win);
 	
 		/* if temp screen, delete it after window free (it stops jobs that can access it) */
-		if (screen->temp) {
+		if (screen && screen->temp) {
 			Main *bmain = CTX_data_main(C);
 			BKE_libblock_free(bmain, screen);
 		}
@@ -380,7 +385,7 @@ float wm_window_pixelsize(wmWindow *win)
 }
 
 /* belongs to below */
-static void wm_window_add_ghostwindow(wmWindowManager *wm, const char *title, wmWindow *win)
+static void wm_window_ghostwindow_add(wmWindowManager *wm, const char *title, wmWindow *win)
 {
 	GHOST_WindowHandle ghostwin;
 	GHOST_GLSettings glSettings = {0};
@@ -466,14 +471,26 @@ static void wm_window_add_ghostwindow(wmWindowManager *wm, const char *title, wm
 	}
 }
 
-/* for wmWindows without ghostwin, open these and clear */
-/* window size is read from window, if 0 it uses prefsize */
-/* called in WM_check, also inits stuff after file read */
-void wm_window_add_ghostwindows(wmWindowManager *wm)
+/**
+ * Initialize #wmWindows without ghostwin, open these and clear.
+ *
+ * window size is read from window, if 0 it uses prefsize
+ * called in #WM_check, also inits stuff after file read.
+ *
+ * \warning
+ * After running, 'win->ghostwin' can be NULL in rare cases
+ * (where OpenGL driver fails to create a context for eg).
+ * We could remove them with #wm_window_ghostwindows_remove_invalid
+ * but better not since caller may continue to use.
+ * Instead, caller needs to handle the error case and cleanup.
+ */
+void wm_window_ghostwindows_ensure(wmWindowManager *wm)
 {
 	wmKeyMap *keymap;
 	wmWindow *win;
 	
+	BLI_assert(G.background == false);
+
 	/* no commandline prefsize? then we set this.
 	 * Note that these values will be used only
 	 * when there is no startup.blend yet.
@@ -521,7 +538,7 @@ void wm_window_add_ghostwindows(wmWindowManager *wm)
 				win->cursor = CURSOR_STD;
 			}
 
-			wm_window_add_ghostwindow(wm, "Blender", win);
+			wm_window_ghostwindow_add(wm, "Blender", win);
 		}
 		/* happens after fileread */
 		if (win->eventstate == NULL)
@@ -546,11 +563,33 @@ void wm_window_add_ghostwindows(wmWindowManager *wm)
 	}
 }
 
-/* new window, no screen yet, but we open ghostwindow for it */
-/* also gets the window level handlers */
-/* area-rip calls this */
+/**
+ * Call after #wm_window_ghostwindows_ensure or #WM_check
+ * (after loading a new file) in the unlikely event a window couldn't be created.
+ */
+void wm_window_ghostwindows_remove_invalid(bContext *C, wmWindowManager *wm)
+{
+	wmWindow *win, *win_next;
+
+	BLI_assert(G.background == false);
+
+	for (win = wm->windows.first; win; win = win_next) {
+		win_next = win->next;
+		if (win->ghostwin == NULL) {
+			wm_window_close(C, wm, win);
+		}
+	}
+}
+
+/**
+ * new window, no screen yet, but we open ghostwindow for it,
+ * also gets the window level handlers
+ * \note area-rip calls this.
+ * \return the window or NULL.
+ */
 wmWindow *WM_window_open(bContext *C, const rcti *rect)
 {
+	wmWindow *win_prev = CTX_wm_window(C);
 	wmWindow *win = wm_window_new(C);
 	
 	win->posx = rect->xmin;
@@ -561,22 +600,35 @@ wmWindow *WM_window_open(bContext *C, const rcti *rect)
 	win->drawmethod = U.wmdrawmethod;
 
 	WM_check(C);
-	
-	return win;
-}
 
-/* uses screen->temp tag to define what to do, currently it limits
- * to only one "temp" window for render out, preferences, filewindow, etc */
-/* type is defined in WM_api.h */
+	if (win->ghostwin) {
+		return win;
+	}
+	else {
+		wm_window_close(C, CTX_wm_manager(C), win);
+		CTX_wm_window_set(C, win_prev);
+		return NULL;
+	}
+}
 
-void WM_window_open_temp(bContext *C, rcti *position, int type)
+/**
+ * Uses `screen->temp` tag to define what to do, currently it limits
+ * to only one "temp" window for render out, preferences, filewindow, etc...
+ *
+ * \param type: WM_WINDOW_RENDER, WM_WINDOW_USERPREFS...
+ * \return the window or NULL.
+ */
+wmWindow *WM_window_open_temp(bContext *C, const rcti *rect_init, int type)
 {
+	wmWindow *win_prev = CTX_wm_window(C);
 	wmWindow *win;
 	ScrArea *sa;
 	Scene *scene = CTX_data_scene(C);
-	
+	const char *title;
+	rcti rect = *rect_init;
+
 	/* changes rect to fit within desktop */
-	wm_window_check_position(position);
+	wm_window_check_position(&rect);
 	
 	/* test if we have a temp screen already */
 	for (win = CTX_wm_manager(C)->windows.first; win; win = win->next)
@@ -587,12 +639,12 @@ void WM_window_open_temp(bContext *C, rcti *position, int type)
 	if (win == NULL) {
 		win = wm_window_new(C);
 		
-		win->posx = position->xmin;
-		win->posy = position->ymin;
+		win->posx = rect.xmin;
+		win->posy = rect.ymin;
 	}
 	
-	win->sizex = BLI_rcti_size_x(position);
-	win->sizey = BLI_rcti_size_y(position);
+	win->sizex = BLI_rcti_size_x(&rect);
+	win->sizey = BLI_rcti_size_y(&rect);
 	
 	if (win->ghostwin) {
 		wm_window_set_size(win, win->sizex, win->sizey);
@@ -614,7 +666,13 @@ void WM_window_open_temp(bContext *C, rcti *position, int type)
 	/* make window active, and validate/resize */
 	CTX_wm_window_set(C, win);
 	WM_check(C);
-	
+
+	/* It's possible `win->ghostwin == NULL`.
+	 * instead of attempting to cleanup here (in a half finished state),
+	 * finish setting up the screen, then free it at the end of the function,
+	 * to avoid having to take into account a partially-created window.
+	 */
+
 	/* ensure it shows the right spacetype editor */
 	sa = win->screen->areabase.first;
 	CTX_wm_area_set(C, sa);
@@ -630,13 +688,25 @@ void WM_window_open_temp(bContext *C, rcti *position, int type)
 	ED_screen_refresh(CTX_wm_manager(C), win); /* test scale */
 	
 	if (sa->spacetype == SPACE_IMAGE)
-		GHOST_SetTitle(win->ghostwin, IFACE_("Blender Render"));
+		title = IFACE_("Blender Render");
 	else if (ELEM(sa->spacetype, SPACE_OUTLINER, SPACE_USERPREF))
-		GHOST_SetTitle(win->ghostwin, IFACE_("Blender User Preferences"));
+		title = IFACE_("Blender User Preferences");
 	else if (sa->spacetype == SPACE_FILE)
-		GHOST_SetTitle(win->ghostwin, IFACE_("Blender File View"));
+		title = IFACE_("Blender File View");
 	else
-		GHOST_SetTitle(win->ghostwin, "Blender");
+		title = "Blender";
+
+	if (win->ghostwin) {
+		GHOST_SetTitle(win->ghostwin, title);
+		return win;
+	}
+	else {
+		/* very unlikely! but opening a new window can fail */
+		wm_window_close(C, CTX_wm_manager(C), win);
+		CTX_wm_window_set(C, win_prev);
+
+		return NULL;
+	}
 }
 
 
diff --git a/source/blender/windowmanager/wm_window.h b/source/blender/windowmanager/wm_window.h
index 8633f297dda..75ccca1a5ec 100644
--- a/source/blender/windowmanager/wm_window.h
+++ b/source/blender/windowmanager/wm_window.h
@@ -48,7 +48,8 @@ void		wm_window_free			(bContext *C, wmWindowManager *wm, wmWindow *win);
 void		wm_window_close			(bContext *C, wmWindowManager *wm, wmWindow *win);
 
 void		wm_window_title				(wmWindowManager *wm, wmWindow *win);
-void		wm_window_add_ghostwindows	(wmWindowManager *wm);
+void		wm_window_ghostwindows_ensure(wmWindowManager *wm);
+void		wm_window_ghostwindows_remove_invalid(bContext *C, wmWindowManager *wm);
 void		wm_window_process_events	(const bContext *C);
 void		wm_window_process_events_nosleep(void);