fix [#35154] Python: UI issues with invoke_props_dialog()

- highlighed buttons that were never pressed could be activated by accident when canceling search popup. 
- canceling a search popup on a popup would close both.

1 files changed, 47 insertions, 27 deletions

diff --git a/source/blender/editors/interface/interface_handlers.c b/source/blender/editors/interface/interface_handlers.c
index 1af8e9467f2..95aaa4c81c1 100644
--- a/source/blender/editors/interface/interface_handlers.c
+++ b/source/blender/editors/interface/interface_handlers.c
@@ -2541,6 +2541,8 @@ static int ui_do_but_BUT(bContext *C, uiBut *but, uiHandleButtonData *data, cons
 			return WM_UI_HANDLER_BREAK;
 		}
 		else if (event->type == LEFTMOUSE && but->block->handle) {
+			if (!(but->flag & UI_SELECT))
+				data->cancel = true;
 			button_activate_state(C, but, BUTTON_STATE_EXIT);
 			return WM_UI_HANDLER_BREAK;
 		}
@@ -6915,17 +6917,42 @@ static bool ui_menu_pass_event_to_parent_if_nonactive(uiPopupBlockHandle *menu,
 	}
 }
 
+static int ui_handle_menu_button(bContext *C, const wmEvent *event, uiPopupBlockHandle *menu)
+{
+	ARegion *ar = menu->region;
+	uiBut *but = ui_but_find_activated(ar);
+	int retval;
+
+	if (but) {
+		ScrArea *ctx_area = CTX_wm_area(C);
+		ARegion *ctx_region = CTX_wm_region(C);
+
+		if (menu->ctx_area) CTX_wm_area_set(C, menu->ctx_area);
+		if (menu->ctx_region) CTX_wm_region_set(C, menu->ctx_region);
+
+		retval = ui_handle_button_event(C, event, but);
+
+		if (menu->ctx_area) CTX_wm_area_set(C, ctx_area);
+		if (menu->ctx_region) CTX_wm_region_set(C, ctx_region);
+	}
+	else {
+		retval = ui_handle_button_over(C, event, ar);
+	}
+
+	return retval;
+}
+
 static int ui_handle_menu_event(bContext *C, const wmEvent *event, uiPopupBlockHandle *menu, int level)
 {
 	ARegion *ar;
 	uiBlock *block;
 	uiBut *but, *bt;
-	int inside, act, count, mx, my, retval;
+	int mx, my, retval;
+	bool inside;
 
 	ar = menu->region;
 	block = ar->uiblocks.first;
-	
-	act = 0;
+
 	retval = WM_UI_HANDLER_CONTINUE;
 
 	mx = event->x;
@@ -6937,7 +6964,8 @@ static int ui_handle_menu_event(bContext *C, const wmEvent *event, uiPopupBlockH
 
 	/* if there's an active modal button, don't check events or outside, except for search menu */
 	but = ui_but_find_activated(ar);
-	if (but && button_modal_state(but->active->state) && but->type != SEARCH_MENU && but->type != SEARCH_MENU_UNLINK) {
+
+	if (but && button_modal_state(but->active->state)) {
 		/* if a button is activated modal, always reset the start mouse
 		 * position of the towards mechanism to avoid loosing focus,
 		 * and don't handle events */
@@ -6963,7 +6991,9 @@ static int ui_handle_menu_event(bContext *C, const wmEvent *event, uiPopupBlockH
 		if (block->block_event_func && block->block_event_func(C, block, event)) {
 			/* pass */
 		}   /* events not for active search menu button */
-		else if (but == NULL || (but->type != SEARCH_MENU && but->type != SEARCH_MENU_UNLINK)) {
+		else {
+			int act = 0;
+
 			switch (event->type) {
 
 				/* closing sublevels of pulldowns */
@@ -7106,6 +7136,7 @@ static int ui_handle_menu_event(bContext *C, const wmEvent *event, uiPopupBlockH
 					if (act == 0) act = 10;
 
 					if ((block->flag & UI_BLOCK_NUMSELECT) && event->val == KM_PRESS) {
+						int count;
 
 						if (ui_menu_pass_event_to_parent_if_nonactive(menu, but, level, retval))
 							break;
@@ -7289,8 +7320,6 @@ static int ui_handle_menu_event(bContext *C, const wmEvent *event, uiPopupBlockH
 			}
 
 			/* end switch */
-#undef PASS_EVENT_TO_PARENT_IF_NONACTIVE
-
 		}
 	}
 
@@ -7301,22 +7330,7 @@ static int ui_handle_menu_event(bContext *C, const wmEvent *event, uiPopupBlockH
 	if ((event->type == TIMER) ||
 	    (/*inside &&*/ (!menu->menuretval || (menu->menuretval & UI_RETURN_UPDATE)) && retval == WM_UI_HANDLER_CONTINUE))
 	{
-		but = ui_but_find_activated(ar);
-
-		if (but) {
-			ScrArea *ctx_area = CTX_wm_area(C);
-			ARegion *ctx_region = CTX_wm_region(C);
-			
-			if (menu->ctx_area) CTX_wm_area_set(C, menu->ctx_area);
-			if (menu->ctx_region) CTX_wm_region_set(C, menu->ctx_region);
-			
-			retval = ui_handle_button_event(C, event, but);
-			
-			if (menu->ctx_area) CTX_wm_area_set(C, ctx_area);
-			if (menu->ctx_region) CTX_wm_region_set(C, ctx_region);
-		}
-		else
-			retval = ui_handle_button_over(C, event, ar);
+		ui_handle_menu_button(C, event, menu);
 	}
 
 	/* if we set a menu return value, ensure we continue passing this on to
@@ -7396,19 +7410,25 @@ static int ui_handle_menus_recursive(bContext *C, const wmEvent *event, uiPopupB
 
 	/* now handle events for our own menu */
 	if (retval == WM_UI_HANDLER_CONTINUE || event->type == TIMER) {
+		const bool do_but_search = (but && ELEM(but->type, SEARCH_MENU, SEARCH_MENU_UNLINK));
 		if (submenu && submenu->menuretval) {
-			int do_ret_out_parent = (submenu->menuretval & UI_RETURN_OUT_PARENT);
+			const bool do_ret_out_parent = (submenu->menuretval & UI_RETURN_OUT_PARENT) != 0;
 			retval = ui_handle_menu_return_submenu(C, event, menu);
 			submenu = NULL;  /* hint not to use this, it may be freed by call above */
 			(void)submenu;
 			/* we may want to quit the submenu and handle the even in this menu,
 			 * if its important to use it, check 'data->menu' first */
-			if ((retval == WM_UI_HANDLER_BREAK) && do_ret_out_parent) {
-				retval = ui_handle_menu_event(C, event, menu, level);
+			if (((retval == WM_UI_HANDLER_BREAK) && do_ret_out_parent) == 0) {
+				/* skip applying the event */
+				return retval;
 			}
 		}
+
+		if (do_but_search) {
+			retval = ui_handle_menu_button(C, event, menu);
+		}
 		else {
-			retval = ui_handle_menu_event(C, event, menu, level);  /* same as above */
+			retval = ui_handle_menu_event(C, event, menu, level);
 		}
 	}