Fix T49199: Combination of dialog + wm.open_mainfile causes crash

Issue was that the wm.open_mainfile OP caused all handlers to be removed and since rB45592291 cancelled (which is correct in general), the menu that triggered the OP should not be cancelled though.
Not sure if this is a nice fix or not, it's however the safest fix I found. A different fix would be to call UI_popup_block_close before WM_operator_call_ex (in dialog_exec_cb), but not sure how safe this is and want to further investigate if it makes other hacks/fixes redundant.

There's still a crash with --debug-memory that confused the heck out of me (since I always have --debug-memory enabled), but I'll commit fix for that separately.

3 files changed, 22 insertions, 1 deletions

diff --git a/source/blender/editors/include/UI_interface.h b/source/blender/editors/include/UI_interface.h
index 1f053c806b0..49e5845e3ca 100644
--- a/source/blender/editors/include/UI_interface.h
+++ b/source/blender/editors/include/UI_interface.h
@@ -393,6 +393,8 @@ struct uiLayout *UI_popup_menu_layout(uiPopupMenu *head);
 void UI_popup_menu_reports(struct bContext *C, struct ReportList *reports) ATTR_NONNULL();
 int UI_popup_menu_invoke(struct bContext *C, const char *idname, struct ReportList *reports) ATTR_NONNULL(1, 2);
 
+void UI_popup_menu_retval_set(const uiBlock *block, const int retval, const bool enable);
+
 /* Pie menus */
 typedef struct uiPieMenu uiPieMenu;
 
diff --git a/source/blender/editors/interface/interface_handlers.c b/source/blender/editors/interface/interface_handlers.c
index d05a80bed62..4972e16bf2e 100644
--- a/source/blender/editors/interface/interface_handlers.c
+++ b/source/blender/editors/interface/interface_handlers.c
@@ -9921,6 +9921,17 @@ static int ui_handle_menus_recursive(
 	return retval;
 }
 
+/**
+ * Allow setting menu return value from externals. E.g. WM might need to do this for exiting files correctly.
+ */
+void UI_popup_menu_retval_set(const uiBlock *block, const int retval, const bool enable)
+{
+	uiPopupBlockHandle *menu = block->handle;
+	if (menu) {
+		menu->menuretval = enable ? (menu->menuretval | retval) : (menu->menuretval & retval);
+	}
+}
+
 /* *************** UI event handlers **************** */
 
 static int ui_region_handler(bContext *C, const wmEvent *event, void *UNUSED(userdata))
@@ -10163,7 +10174,11 @@ static void ui_popup_handler_remove(bContext *C, void *userdata)
 {
 	uiPopupBlockHandle *menu = userdata;
 
-	if (menu->cancel_func) {
+	/* More correct would be to expect UI_RETURN_CANCEL here, but not wanting to
+	 * cancel when removing handlers because of file exit is a rare exception.
+	 * So instead of setting cancel flag for all menus before removing handlers,
+	 * just explicitly flag menu with UI_RETURN_OK to avoid cancelling it. */
+	if ((menu->menuretval & UI_RETURN_OK) == 0 && menu->cancel_func) {
 		menu->cancel_func(C, menu->popup_arg);
 	}
 
diff --git a/source/blender/windowmanager/intern/wm_operators.c b/source/blender/windowmanager/intern/wm_operators.c
index fcdab746d57..8918b6bb1b8 100644
--- a/source/blender/windowmanager/intern/wm_operators.c
+++ b/source/blender/windowmanager/intern/wm_operators.c
@@ -1375,6 +1375,10 @@ static void dialog_exec_cb(bContext *C, void *arg1, void *arg2)
 	wmOpPopUp *data = arg1;
 	uiBlock *block = arg2;
 
+	/* Explicitly set UI_RETURN_OK flag, otherwise the menu might be cancelled
+	 * in case WM_operator_call_ex exits/reloads the current file (T49199). */
+	UI_popup_menu_retval_set(block, UI_RETURN_OK, true);
+
 	WM_operator_call_ex(C, data->op, true);
 
 	/* let execute handle freeing it */