Fix BMesh selection flushing w/ mixed modes

Fix for T46494 wasn't working properly when de-selecting faces,
adjacent faces would remain selected but have unselected edges.

Logic here is admittedly rather fragile since it relies on both
selection functions and flushing afterwards.

1 files changed, 34 insertions, 25 deletions

diff --git a/source/blender/bmesh/intern/bmesh_marking.c b/source/blender/bmesh/intern/bmesh_marking.c
index cd3c8325831..3fe888736f0 100644
--- a/source/blender/bmesh/intern/bmesh_marking.c
+++ b/source/blender/bmesh/intern/bmesh_marking.c
@@ -102,7 +102,6 @@ static bool bm_vert_is_edge_select_any(const BMVert *v)
 }
 #endif
 
-#if 0
 static bool bm_edge_is_face_select_any_other(BMLoop *l_first)
 {
 	const BMLoop *l_iter = l_first;
@@ -115,7 +114,6 @@ static bool bm_edge_is_face_select_any_other(BMLoop *l_first)
 	}
 	return false;
 }
-#endif
 
 #if 0
 static bool bm_edge_is_face_select_any(const BMEdge *e)
@@ -505,33 +503,44 @@ void BM_face_select_set(BMesh *bm, BMFace *f, const bool select)
 		 * an edge bay be de-selected, but an adjacent face remains selected.
 		 *
 		 * Rely on #BM_mesh_select_mode_flush to correct these cases.
+		 *
+		 * \note flushing based on mode, see T46494
 		 */
-#if 1
-		l_iter = l_first = BM_FACE_FIRST_LOOP(f);
-		do {
-			BM_vert_select_set(bm, l_iter->v, false);
-			BM_edge_select_set(bm, l_iter->e, false);
-		} while ((l_iter = l_iter->next) != l_first);
-#else
-		/* disabled, see T46494 */
-
-		/* flush down to edges */
-		l_iter = l_first = BM_FACE_FIRST_LOOP(f);
-		do {
-			/* vertex flushing is handled below */
-			if (bm_edge_is_face_select_any_other(l_iter) == false) {
+		if (bm->selectmode & SCE_SELECT_VERTEX) {
+			l_iter = l_first = BM_FACE_FIRST_LOOP(f);
+			do {
+				BM_vert_select_set(bm, l_iter->v, false);
 				BM_edge_select_set_noflush(bm, l_iter->e, false);
+			} while ((l_iter = l_iter->next) != l_first);
+		}
+		else {
+			/**
+			 * \note use #BM_edge_select_set_noflush,
+			 * vertex flushing is handled last.
+			 */
+			if (bm->selectmode & SCE_SELECT_EDGE) {
+				l_iter = l_first = BM_FACE_FIRST_LOOP(f);
+				do {
+					BM_edge_select_set_noflush(bm, l_iter->e, false);
+				} while ((l_iter = l_iter->next) != l_first);
 			}
-		} while ((l_iter = l_iter->next) != l_first);
-
-		/* flush down to verts */
-		l_iter = l_first = BM_FACE_FIRST_LOOP(f);
-		do {
-			if (bm_vert_is_edge_select_any_other(l_iter->v, l_iter->e) == false) {
-				BM_vert_select_set(bm, l_iter->v, false);
+			else {
+				l_iter = l_first = BM_FACE_FIRST_LOOP(f);
+				do {
+					if (bm_edge_is_face_select_any_other(l_iter) == false) {
+						BM_edge_select_set_noflush(bm, l_iter->e, false);
+					}
+				} while ((l_iter = l_iter->next) != l_first);
 			}
-		} while ((l_iter = l_iter->next) != l_first);
-#endif
+
+			/* flush down to verts */
+			l_iter = l_first = BM_FACE_FIRST_LOOP(f);
+			do {
+				if (bm_vert_is_edge_select_any_other(l_iter->v, l_iter->e) == false) {
+					BM_vert_select_set(bm, l_iter->v, false);
+				}
+			} while ((l_iter = l_iter->next) != l_first);
+		}
 	}
 }