3D View: move deselect all logic into an option

This removes `VIEW3D_OT_select_or_deselect_all`, adding a
deselect_all option to the `VIEW3D_OT_select` operator.

- Add utility functions to simplify de-selecting all.

- Return true from selection functions when they change the selection
  to avoid redundant updates.

- Use arrays of bases when passing objects between selection utility
  functions since some users require bases.

- Fix logical error in box selection that updated all objects after
  the first hit.

2 files changed, 89 insertions, 32 deletions

diff --git a/source/blender/editors/mesh/editface.c b/source/blender/editors/mesh/editface.c
index 12a96d41711..842394f1107 100644
--- a/source/blender/editors/mesh/editface.c
+++ b/source/blender/editors/mesh/editface.c
@@ -282,14 +282,16 @@ void paintface_select_linked(bContext *C, Object *ob, const int mval[2], const b
 	paintface_flush_flags(C, ob, SELECT);
 }
 
-void paintface_deselect_all_visible(bContext *C, Object *ob, int action, bool flush_flags)
+bool paintface_deselect_all_visible(bContext *C, Object *ob, int action, bool flush_flags)
 {
 	Mesh *me;
 	MPoly *mpoly;
 	int a;
 
 	me = BKE_mesh_from_object(ob);
-	if (me == NULL) return;
+	if (me == NULL) {
+		return false;
+	}
 
 	if (action == SEL_TOGGLE) {
 		action = SEL_SELECT;
@@ -305,28 +307,40 @@ void paintface_deselect_all_visible(bContext *C, Object *ob, int action, bool fl
 		}
 	}
 
+	bool changed = false;
+
 	mpoly = me->mpoly;
 	a = me->totpoly;
 	while (a--) {
 		if ((mpoly->flag & ME_HIDE) == 0) {
 			switch (action) {
 				case SEL_SELECT:
-					mpoly->flag |= ME_FACE_SEL;
+					if ((mpoly->flag & ME_FACE_SEL) == 0) {
+						mpoly->flag |= ME_FACE_SEL;
+						changed = true;
+					}
 					break;
 				case SEL_DESELECT:
-					mpoly->flag &= ~ME_FACE_SEL;
+					if ((mpoly->flag & ME_FACE_SEL) != 0) {
+						mpoly->flag &= ~ME_FACE_SEL;
+						changed = true;
+					}
 					break;
 				case SEL_INVERT:
 					mpoly->flag ^= ME_FACE_SEL;
+					changed = true;
 					break;
 			}
 		}
 		mpoly++;
 	}
 
-	if (flush_flags) {
-		paintface_flush_flags(C, ob, SELECT);
+	if (changed) {
+		if (flush_flags) {
+			paintface_flush_flags(C, ob, SELECT);
+		}
 	}
+	return changed;
 }
 
 bool paintface_minmax(Object *ob, float r_min[3], float r_max[3])
@@ -389,10 +403,7 @@ bool paintface_mouse_select(struct bContext *C, Object *ob, const int mval[2], b
 	mpoly = me->mpoly;
 	a = me->totpoly;
 	if (!extend && !deselect && !toggle) {
-		while (a--) {
-			mpoly->flag &= ~ME_FACE_SEL;
-			mpoly++;
-		}
+		paintface_deselect_all_visible(C, ob, SEL_DESELECT, false);
 	}
 
 	me->act_face = (int)index;
@@ -420,7 +431,7 @@ bool paintface_mouse_select(struct bContext *C, Object *ob, const int mval[2], b
 	return true;
 }
 
-int do_paintface_box_select(ViewContext *vc, const rcti *rect, int sel_op)
+bool do_paintface_box_select(ViewContext *vc, const rcti *rect, int sel_op)
 {
 	Object *ob = vc->obact;
 	Mesh *me;
@@ -431,13 +442,14 @@ int do_paintface_box_select(ViewContext *vc, const rcti *rect, int sel_op)
 
 	me = BKE_mesh_from_object(ob);
 	if ((me == NULL) || (me->totpoly == 0) || BLI_rcti_is_empty(rect)) {
-		return OPERATOR_CANCELLED;
+		return false;
 	}
 
 	selar = MEM_callocN(me->totpoly + 1, "selar");
 
+	bool changed = false;
 	if (SEL_OP_USE_PRE_DESELECT(sel_op)) {
-		paintface_deselect_all_visible(vc->C, vc->obact, SEL_DESELECT, false);
+		changed |= paintface_deselect_all_visible(vc->C, vc->obact, SEL_DESELECT, false);
 	}
 
 	uint buf_len;
@@ -464,6 +476,7 @@ int do_paintface_box_select(ViewContext *vc, const rcti *rect, int sel_op)
 			const int sel_op_result = ED_select_op_action_deselected(sel_op, is_select, is_inside);
 			if (sel_op_result != -1) {
 				SET_FLAG_FROM_TEST(mpoly->flag, sel_op_result, ME_FACE_SEL);
+				changed = true;
 			}
 		}
 	}
@@ -475,9 +488,10 @@ int do_paintface_box_select(ViewContext *vc, const rcti *rect, int sel_op)
 	glReadBuffer(GL_BACK);
 #endif
 
-	paintface_flush_flags(vc->C, vc->obact, SELECT);
-
-	return OPERATOR_FINISHED;
+	if (changed) {
+		paintface_flush_flags(vc->C, vc->obact, SELECT);
+	}
+	return changed;
 }
 
 
@@ -536,14 +550,16 @@ void paintvert_tag_select_update(struct bContext *C, struct Object *ob)
 
 /*  note: if the caller passes false to flush_flags,
  *  then they will need to run paintvert_flush_flags(ob) themselves */
-void paintvert_deselect_all_visible(Object *ob, int action, bool flush_flags)
+bool paintvert_deselect_all_visible(Object *ob, int action, bool flush_flags)
 {
 	Mesh *me;
 	MVert *mvert;
 	int a;
 
 	me = BKE_mesh_from_object(ob);
-	if (me == NULL) return;
+	if (me == NULL) {
+		return false;
+	}
 
 	if (action == SEL_TOGGLE) {
 		action = SEL_SELECT;
@@ -559,39 +575,50 @@ void paintvert_deselect_all_visible(Object *ob, int action, bool flush_flags)
 		}
 	}
 
+	bool changed = false;
 	mvert = me->mvert;
 	a = me->totvert;
 	while (a--) {
 		if ((mvert->flag & ME_HIDE) == 0) {
 			switch (action) {
 				case SEL_SELECT:
-					mvert->flag |= SELECT;
+					if ((mvert->flag & SELECT) == 0) {
+						mvert->flag |= SELECT;
+						changed = true;
+					}
 					break;
 				case SEL_DESELECT:
-					mvert->flag &= ~SELECT;
+					if ((mvert->flag & SELECT) != 0) {
+						mvert->flag &= ~SELECT;
+						changed = true;
+					}
 					break;
 				case SEL_INVERT:
 					mvert->flag ^= SELECT;
+					changed = true;
 					break;
 			}
 		}
 		mvert++;
 	}
 
-	/* handle mselect */
-	if (action == SEL_SELECT) {
-		/* pass */
-	}
-	else if (ELEM(action, SEL_DESELECT, SEL_INVERT)) {
-		BKE_mesh_mselect_clear(me);
-	}
-	else {
-		BKE_mesh_mselect_validate(me);
-	}
+	if (changed) {
+		/* handle mselect */
+		if (action == SEL_SELECT) {
+			/* pass */
+		}
+		else if (ELEM(action, SEL_DESELECT, SEL_INVERT)) {
+			BKE_mesh_mselect_clear(me);
+		}
+		else {
+			BKE_mesh_mselect_validate(me);
+		}
 
-	if (flush_flags) {
-		paintvert_flush_flags(ob);
+		if (flush_flags) {
+			paintvert_flush_flags(ob);
+		}
 	}
+	return changed;
 }
 
 void paintvert_select_ungrouped(Object *ob, bool extend, bool flush_flags)
diff --git a/source/blender/editors/mesh/editmesh_select.c b/source/blender/editors/mesh/editmesh_select.c
index 991e0f94383..1855eeab944 100644
--- a/source/blender/editors/mesh/editmesh_select.c
+++ b/source/blender/editors/mesh/editmesh_select.c
@@ -2597,6 +2597,36 @@ void EDBM_select_swap(BMEditMesh *em) /* exported for UV */
 	}
 }
 
+bool EDBM_mesh_deselect_all_multi_ex(struct Base **bases, const uint bases_len)
+{
+	bool changed_multi = false;
+	for (uint base_index = 0; base_index < bases_len; base_index++) {
+		Base *base_iter = bases[base_index];
+		Object *ob_iter = base_iter->object;
+		BMEditMesh *em_iter = BKE_editmesh_from_object(ob_iter);
+
+		if (em_iter->bm->totvertsel == 0) {
+			continue;
+		}
+
+		EDBM_flag_disable_all(em_iter, BM_ELEM_SELECT);
+		DEG_id_tag_update(ob_iter->data, ID_RECALC_SELECT);
+		changed_multi = true;
+	}
+	return changed_multi;
+}
+
+bool EDBM_mesh_deselect_all_multi(struct bContext *C)
+{
+	ViewContext vc;
+	ED_view3d_viewcontext_init(C, &vc);
+	uint bases_len = 0;
+	Base **bases = BKE_view_layer_array_from_bases_in_edit_mode_unique_data(vc.view_layer, vc.v3d, &bases_len);
+	bool changed_multi = EDBM_mesh_deselect_all_multi_ex(bases, bases_len);
+	MEM_freeN(bases);
+	return changed_multi;
+}
+
 /** \} */
 
 /* -------------------------------------------------------------------- */