Fix T63646: Box/Lasso select fails to de-select

Select 'New' wasn't de-selecting when there was nothing selected
in some cases.

3 files changed, 64 insertions, 47 deletions

diff --git a/source/blender/editors/mesh/editface.c b/source/blender/editors/mesh/editface.c
index ef2e01e05fb..7a6144d8e0f 100644
--- a/source/blender/editors/mesh/editface.c
+++ b/source/blender/editors/mesh/editface.c
@@ -433,58 +433,63 @@ bool do_paintface_box_select(ViewContext *vc, const rcti *rect, int sel_op)
 {
 	Object *ob = vc->obact;
 	Mesh *me;
-	MPoly *mpoly;
-	uint *rt;
-	char *selar;
-	int a, index;
 
 	me = BKE_mesh_from_object(ob);
-	if ((me == NULL) || (me->totpoly == 0) || BLI_rcti_is_empty(rect)) {
+	if ((me == NULL) || (me->totpoly == 0)) {
 		return false;
 	}
 
-	selar = MEM_callocN(me->totpoly + 1, "selar");
-
 	bool changed = false;
 	if (SEL_OP_USE_PRE_DESELECT(sel_op)) {
 		changed |= paintface_deselect_all_visible(vc->C, vc->obact, SEL_DESELECT, false);
 	}
 
-	uint buf_len;
-	uint *buf = ED_view3d_select_id_read_rect(vc, rect, &buf_len);
+	if (BLI_rcti_is_empty(rect)) {
+		/* pass */
+	}
+	else {
+		MPoly *mpoly;
+		uint *rt;
+		int a, index;
 
-	rt = buf;
+		char *selar = MEM_callocN(me->totpoly + 1, "selar");
 
-	a = buf_len;
-	while (a--) {
-		if (*rt) {
-			index = *rt;
-			if (index <= me->totpoly) {
-				selar[index] = 1;
+		uint buf_len;
+		uint *buf = ED_view3d_select_id_read_rect(vc, rect, &buf_len);
+
+		rt = buf;
+
+		a = buf_len;
+		while (a--) {
+			if (*rt) {
+				index = *rt;
+				if (index <= me->totpoly) {
+					selar[index] = 1;
+				}
 			}
+			rt++;
 		}
-		rt++;
-	}
 
-	mpoly = me->mpoly;
-	for (a = 1; a <= me->totpoly; a++, mpoly++) {
-		if ((mpoly->flag & ME_HIDE) == 0) {
-			const bool is_select = mpoly->flag & ME_FACE_SEL;
-			const bool is_inside = (selar[a] != 0);
-			const int sel_op_result = ED_select_op_action_deselected(sel_op, is_select, is_inside);
-			if (sel_op_result != -1) {
-				SET_FLAG_FROM_TEST(mpoly->flag, sel_op_result, ME_FACE_SEL);
-				changed = true;
+		mpoly = me->mpoly;
+		for (a = 1; a <= me->totpoly; a++, mpoly++) {
+			if ((mpoly->flag & ME_HIDE) == 0) {
+				const bool is_select = mpoly->flag & ME_FACE_SEL;
+				const bool is_inside = (selar[a] != 0);
+				const int sel_op_result = ED_select_op_action_deselected(sel_op, is_select, is_inside);
+				if (sel_op_result != -1) {
+					SET_FLAG_FROM_TEST(mpoly->flag, sel_op_result, ME_FACE_SEL);
+					changed = true;
+				}
 			}
 		}
-	}
 
-	MEM_freeN(buf);
-	MEM_freeN(selar);
+		MEM_freeN(buf);
+		MEM_freeN(selar);
 
 #ifdef __APPLE__
-	glReadBuffer(GL_BACK);
+		glReadBuffer(GL_BACK);
 #endif
+	}
 
 	if (changed) {
 		paintface_flush_flags(vc->C, vc->obact, SELECT);
diff --git a/source/blender/editors/physics/particle_edit.c b/source/blender/editors/physics/particle_edit.c
index 149dadedfb4..dcd8f7a93da 100644
--- a/source/blender/editors/physics/particle_edit.c
+++ b/source/blender/editors/physics/particle_edit.c
@@ -38,6 +38,7 @@
 #include "BLI_math.h"
 #include "BLI_lasso_2d.h"
 #include "BLI_listbase.h"
+#include "BLI_rect.h"
 #include "BLI_kdtree.h"
 #include "BLI_rand.h"
 #include "BLI_task.h"
@@ -2066,7 +2067,14 @@ bool PE_box_select(bContext *C, const rcti *rect, const int sel_op)
 	if (SEL_OP_USE_PRE_DESELECT(sel_op)) {
 		data.is_changed = PE_deselect_all_visible_ex(edit);
 	}
-	for_mouse_hit_keys(&data, select_key_op, PSEL_ALL_KEYS);
+
+	if (BLI_rcti_is_empty(rect)) {
+		/* pass */
+	}
+	else {
+		for_mouse_hit_keys(&data, select_key_op, PSEL_ALL_KEYS);
+	}
+
 	if (data.is_changed) {
 		PE_update_selection(data.depsgraph, scene, ob, 1);
 		WM_event_add_notifier(C, NC_OBJECT | ND_PARTICLE | NA_SELECTED, ob);
diff --git a/source/blender/editors/space_view3d/view3d_select.c b/source/blender/editors/space_view3d/view3d_select.c
index 3ff6a7146c6..f57b31b2ed2 100644
--- a/source/blender/editors/space_view3d/view3d_select.c
+++ b/source/blender/editors/space_view3d/view3d_select.c
@@ -700,7 +700,7 @@ static bool do_lasso_select_mesh(
 
 	if (ts->selectmode & SCE_SELECT_VERTEX) {
 		if (bbsel) {
-			data.is_changed = edbm_backbuf_check_and_select_verts(vc->em, sel_op);
+			data.is_changed |= edbm_backbuf_check_and_select_verts(vc->em, sel_op);
 		}
 		else {
 			mesh_foreachScreenVert(vc, do_lasso_select_mesh__doSelectVert, &data, V3D_PROJ_TEST_CLIP_DEFAULT);
@@ -716,7 +716,7 @@ static bool do_lasso_select_mesh(
 
 	if (ts->selectmode & SCE_SELECT_FACE) {
 		if (bbsel) {
-			data.is_changed = edbm_backbuf_check_and_select_faces(vc->em, sel_op);
+			data.is_changed |= edbm_backbuf_check_and_select_faces(vc->em, sel_op);
 		}
 		else {
 			mesh_foreachScreenFace(vc, do_lasso_select_mesh__doSelectFace, &data, V3D_PROJ_TEST_CLIP_DEFAULT);
@@ -1065,22 +1065,22 @@ static bool view3d_lasso_select(
 
 	if (vc->obedit == NULL) { /* Object Mode */
 		if (BKE_paint_select_face_test(ob)) {
-			changed_multi = do_lasso_select_paintface(vc, mcords, moves, sel_op);
+			changed_multi |= do_lasso_select_paintface(vc, mcords, moves, sel_op);
 		}
 		else if (BKE_paint_select_vert_test(ob)) {
-			changed_multi = do_lasso_select_paintvert(vc, mcords, moves, sel_op);
+			changed_multi |= do_lasso_select_paintvert(vc, mcords, moves, sel_op);
 		}
 		else if (ob && (ob->mode & (OB_MODE_VERTEX_PAINT | OB_MODE_WEIGHT_PAINT | OB_MODE_TEXTURE_PAINT))) {
 			/* pass */
 		}
 		else if (ob && (ob->mode & OB_MODE_PARTICLE_EDIT)) {
-			changed_multi = PE_lasso_select(C, mcords, moves, sel_op);
+			changed_multi |= PE_lasso_select(C, mcords, moves, sel_op);
 		}
 		else if (ob && (ob->mode & OB_MODE_POSE)) {
-			changed_multi = do_lasso_select_pose(vc, mcords, moves, sel_op);
+			changed_multi |= do_lasso_select_pose(vc, mcords, moves, sel_op);
 		}
 		else {
-			changed_multi = do_lasso_select_objects(vc, mcords, moves, sel_op);
+			changed_multi |= do_lasso_select_objects(vc, mcords, moves, sel_op);
 		}
 	}
 	else { /* Edit Mode */
@@ -2248,13 +2248,9 @@ static bool do_paintvert_box_select(
 {
 	const bool use_zbuf = !XRAY_ENABLED(vc->v3d);
 	Mesh *me;
-	MVert *mvert;
-	unsigned int *rt;
-	int a, index;
-	char *selar;
 
 	me = vc->obact->data;
-	if ((me == NULL) || (me->totvert == 0) || BLI_rcti_is_empty(rect)) {
+	if ((me == NULL) || (me->totvert == 0)) {
 		return OPERATOR_CANCELLED;
 	}
 
@@ -2263,7 +2259,15 @@ static bool do_paintvert_box_select(
 		changed |= paintvert_deselect_all_visible(vc->obact, SEL_DESELECT, false);
 	}
 
-	if (use_zbuf) {
+	if (BLI_rcti_is_empty(rect)) {
+		/* pass */
+	}
+	else if (use_zbuf) {
+		MVert *mvert;
+		unsigned int *rt;
+		int a, index;
+		char *selar;
+
 		selar = MEM_callocN(me->totvert + 1, "selar");
 
 		uint buf_len;
@@ -2479,7 +2483,7 @@ static bool do_mesh_box_select(
 
 	if (ts->selectmode & SCE_SELECT_VERTEX) {
 		if (bbsel) {
-			data.is_changed = edbm_backbuf_check_and_select_verts(vc->em, sel_op);
+			data.is_changed |= edbm_backbuf_check_and_select_verts(vc->em, sel_op);
 		}
 		else {
 			mesh_foreachScreenVert(vc, do_mesh_box_select__doSelectVert, &data, V3D_PROJ_TEST_CLIP_DEFAULT);
@@ -2495,7 +2499,7 @@ static bool do_mesh_box_select(
 
 	if (ts->selectmode & SCE_SELECT_FACE) {
 		if (bbsel) {
-			data.is_changed = edbm_backbuf_check_and_select_faces(vc->em, sel_op);
+			data.is_changed |= edbm_backbuf_check_and_select_faces(vc->em, sel_op);
 		}
 		else {
 			mesh_foreachScreenFace(vc, do_mesh_box_select__doSelectFace, &data, V3D_PROJ_TEST_CLIP_DEFAULT);