Fix T61472: Hide Unselected fails w/ no selection

Also skip mesh recalculation when no hide/reveal is performed.

5 files changed, 63 insertions, 41 deletions

diff --git a/source/blender/editors/include/ED_mesh.h b/source/blender/editors/include/ED_mesh.h
index 6974c3b6699..dc43d31ff59 100644
--- a/source/blender/editors/include/ED_mesh.h
+++ b/source/blender/editors/include/ED_mesh.h
@@ -94,8 +94,8 @@ void EDBM_select_flush(struct BMEditMesh *em);
 
 bool EDBM_vert_color_check(struct BMEditMesh *em);
 
-void EDBM_mesh_hide(struct BMEditMesh *em, bool swap);
-void EDBM_mesh_reveal(struct BMEditMesh *em, bool select);
+bool EDBM_mesh_hide(struct BMEditMesh *em, bool swap);
+bool EDBM_mesh_reveal(struct BMEditMesh *em, bool select);
 
 void EDBM_update_generic(struct BMEditMesh *em, const bool do_tessface, const bool is_destructive);
 
diff --git a/source/blender/editors/mesh/editmesh_tools.c b/source/blender/editors/mesh/editmesh_tools.c
index 41921b49792..46f5d4e952e 100644
--- a/source/blender/editors/mesh/editmesh_tools.c
+++ b/source/blender/editors/mesh/editmesh_tools.c
@@ -1983,6 +1983,7 @@ static int edbm_hide_exec(bContext *C, wmOperator *op)
 {
 	const bool unselected = RNA_boolean_get(op->ptr, "unselected");
 	ViewLayer *view_layer = CTX_data_view_layer(C);
+	bool changed = false;
 
 	uint objects_len = 0;
 	Object **objects = BKE_view_layer_array_from_objects_in_edit_mode_unique_data(view_layer, CTX_wm_view3d(C), &objects_len);
@@ -1991,18 +1992,28 @@ static int edbm_hide_exec(bContext *C, wmOperator *op)
 		BMEditMesh *em = BKE_editmesh_from_object(obedit);
 		BMesh *bm = em->bm;
 
-		if ((bm->totvertsel == 0) &&
-		    (bm->totedgesel == 0) &&
-		    (bm->totfacesel == 0))
-		{
-			continue;
+		if (unselected) {
+			if (bm->totvertsel == bm->totvert) {
+				continue;
+			}
+		}
+		else {
+			if (bm->totvertsel == 0) {
+				continue;
+			}
 		}
 
-		EDBM_mesh_hide(em, unselected);
-		EDBM_update_generic(em, true, false);
+		if (EDBM_mesh_hide(em, unselected)) {
+			EDBM_update_generic(em, true, false);
+			changed = true;
+		}
 	}
-
 	MEM_freeN(objects);
+
+	if (!changed) {
+		return OPERATOR_CANCELLED;
+	}
+
 	return OPERATOR_FINISHED;
 }
 
@@ -2041,8 +2052,9 @@ static int edbm_reveal_exec(bContext *C, wmOperator *op)
 		Object *obedit = objects[ob_index];
 		BMEditMesh *em = BKE_editmesh_from_object(obedit);
 
-		EDBM_mesh_reveal(em, select);
-		EDBM_update_generic(em, true, false);
+		if (EDBM_mesh_reveal(em, select)) {
+			EDBM_update_generic(em, true, false);
+		}
 	}
 	MEM_freeN(objects);
 
diff --git a/source/blender/editors/mesh/editmesh_utils.c b/source/blender/editors/mesh/editmesh_utils.c
index e57c1aacbdb..a873331f2eb 100644
--- a/source/blender/editors/mesh/editmesh_utils.c
+++ b/source/blender/editors/mesh/editmesh_utils.c
@@ -1197,14 +1197,13 @@ void EDBM_verts_mirror_apply(BMEditMesh *em, const int sel_from, const int sel_t
  * \{ */
 
 /* swap is 0 or 1, if 1 it hides not selected */
-void EDBM_mesh_hide(BMEditMesh *em, bool swap)
+bool EDBM_mesh_hide(BMEditMesh *em, bool swap)
 {
 	BMIter iter;
 	BMElem *ele;
 	int itermode;
 	char hflag_swap = swap ? BM_ELEM_SELECT : 0;
-
-	if (em == NULL) return;
+	bool changed = true;
 
 	if (em->selectmode & SCE_SELECT_VERTEX)
 		itermode = BM_VERTS_OF_MESH;
@@ -1214,11 +1213,18 @@ void EDBM_mesh_hide(BMEditMesh *em, bool swap)
 		itermode = BM_FACES_OF_MESH;
 
 	BM_ITER_MESH (ele, &iter, em->bm, itermode) {
-		if (BM_elem_flag_test(ele, BM_ELEM_SELECT) ^ hflag_swap)
-			BM_elem_hide_set(em->bm, ele, true);
+		if (!BM_elem_flag_test(ele, BM_ELEM_HIDDEN)) {
+			if (BM_elem_flag_test(ele, BM_ELEM_SELECT) ^ hflag_swap) {
+				BM_elem_hide_set(em->bm, ele, true);
+				changed = true;
+			}
+		}
 	}
 
-	EDBM_selectmode_flush(em);
+	if (changed) {
+		EDBM_selectmode_flush(em);
+	}
+	return changed;
 
 	/* original hide flushing comment (OUTDATED):
 	 * hide happens on least dominant select mode, and flushes up, not down!
@@ -1230,7 +1236,7 @@ void EDBM_mesh_hide(BMEditMesh *em, bool swap)
 	 */
 }
 
-void EDBM_mesh_reveal(BMEditMesh *em, bool select)
+bool EDBM_mesh_reveal(BMEditMesh *em, bool select)
 {
 	const char iter_types[3] = {
 		BM_VERTS_OF_MESH,
@@ -1244,6 +1250,7 @@ void EDBM_mesh_reveal(BMEditMesh *em, bool select)
 		(em->selectmode & SCE_SELECT_FACE) != 0,
 	};
 	int i;
+	bool changed = false;
 
 	/* Use tag flag to remember what was hidden before all is revealed.
 	 * BM_ELEM_HIDDEN --> BM_ELEM_TAG */
@@ -1252,10 +1259,20 @@ void EDBM_mesh_reveal(BMEditMesh *em, bool select)
 		BMElem *ele;
 
 		BM_ITER_MESH (ele, &iter, em->bm, iter_types[i]) {
-			BM_elem_flag_set(ele, BM_ELEM_TAG, BM_elem_flag_test(ele, BM_ELEM_HIDDEN));
+			if (BM_elem_flag_test(ele, BM_ELEM_HIDDEN)) {
+				BM_elem_flag_enable(ele, BM_ELEM_TAG);
+				changed = true;
+			}
+			else {
+				BM_elem_flag_disable(ele, BM_ELEM_TAG);
+			}
 		}
 	}
 
+	if (!changed) {
+		return false;
+	}
+
 	/* Reveal everything */
 	EDBM_flag_disable_all(em, BM_ELEM_HIDDEN);
 
@@ -1279,6 +1296,8 @@ void EDBM_mesh_reveal(BMEditMesh *em, bool select)
 
 	/* hidden faces can have invalid normals */
 	EDBM_mesh_normals_update(em);
+
+	return true;
 }
 
 /** \} */
diff --git a/source/blender/editors/object/object_edit.c b/source/blender/editors/object/object_edit.c
index 89f043966db..0115682c2c0 100644
--- a/source/blender/editors/object/object_edit.c
+++ b/source/blender/editors/object/object_edit.c
@@ -200,21 +200,7 @@ static int object_hide_view_set_exec(bContext *C, wmOperator *op)
 	Scene *scene = CTX_data_scene(C);
 	ViewLayer *view_layer = CTX_data_view_layer(C);
 	const bool unselected = RNA_boolean_get(op->ptr, "unselected");
-
-	/* Do nothing if no objects was selected. */
-	bool have_selected = false;
-	for (Base *base = view_layer->object_bases.first; base; base = base->next) {
-		if (base->flag & BASE_VISIBLE) {
-			if (base->flag & BASE_SELECTED) {
-				have_selected = true;
-				break;
-			}
-		}
-	}
-
-	if (!have_selected) {
-		return OPERATOR_CANCELLED;
-	}
+	bool changed = false;
 
 	/* Hide selected or unselected objects. */
 	for (Base *base = view_layer->object_bases.first; base; base = base->next) {
@@ -226,15 +212,20 @@ static int object_hide_view_set_exec(bContext *C, wmOperator *op)
 			if (base->flag & BASE_SELECTED) {
 				ED_object_base_select(base, BA_DESELECT);
 				base->flag |= BASE_HIDDEN;
+				changed = true;
 			}
 		}
 		else {
 			if (!(base->flag & BASE_SELECTED)) {
 				ED_object_base_select(base, BA_DESELECT);
 				base->flag |= BASE_HIDDEN;
+				changed = true;
 			}
 		}
 	}
+	if (!changed) {
+		return OPERATOR_CANCELLED;
+	}
 
 	BKE_layer_collection_sync(scene, view_layer);
 	DEG_id_tag_update(&scene->id, ID_RECALC_BASE_FLAGS);
diff --git a/source/blender/editors/uvedit/uvedit_ops.c b/source/blender/editors/uvedit/uvedit_ops.c
index 9f24bd78297..8ec5fa5e57a 100644
--- a/source/blender/editors/uvedit/uvedit_ops.c
+++ b/source/blender/editors/uvedit/uvedit_ops.c
@@ -4113,9 +4113,9 @@ static int uv_hide_exec(bContext *C, wmOperator *op)
 	const int cd_loop_uv_offset  = CustomData_get_offset(&em->bm->ldata, CD_MLOOPUV);
 
 	if (ts->uv_flag & UV_SYNC_SELECTION) {
-		EDBM_mesh_hide(em, swap);
-		EDBM_update_generic(em, true, false);
-
+		if (EDBM_mesh_hide(em, swap)) {
+			EDBM_update_generic(em, true, false);
+		}
 		return OPERATOR_FINISHED;
 	}
 
@@ -4236,9 +4236,9 @@ static int uv_reveal_exec(bContext *C, wmOperator *op)
 
 	/* call the mesh function if we are in mesh sync sel */
 	if (ts->uv_flag & UV_SYNC_SELECTION) {
-		EDBM_mesh_reveal(em, select);
-		EDBM_update_generic(em, true, false);
-
+		if (EDBM_mesh_reveal(em, select)) {
+			EDBM_update_generic(em, true, false);
+		}
 		return OPERATOR_FINISHED;
 	}
 	if (use_face_center) {