fix [#30715] bmesh: select linked not ignoring hidden verts/edges/faces

add optional flag to ignore hidden elements.
also remove loop mask flag - since it wasnt used and vert/edge/face is enough.

10 files changed, 114 insertions, 34 deletions

diff --git a/source/blender/blenkernel/BKE_deform.h b/source/blender/blenkernel/BKE_deform.h
index 96bad493a3e..17275b6ea7d 100644
--- a/source/blender/blenkernel/BKE_deform.h
+++ b/source/blender/blenkernel/BKE_deform.h
@@ -53,6 +53,7 @@ struct MDeformWeight	*defvert_find_index(const struct MDeformVert *dv, const int
 struct MDeformWeight	*defvert_verify_index(struct MDeformVert *dv, const int defgroup);
 void                     defvert_add_index_notest(struct MDeformVert *dv, int defgroup, const float weight);
 void                     defvert_remove_group(struct MDeformVert *dvert, struct MDeformWeight *dw);
+void                     defvert_clear(struct MDeformVert *dvert);
 
 float  defvert_find_weight(const struct MDeformVert *dvert, const int defgroup);
 float  defvert_array_find_weight_safe(const struct MDeformVert *dvert, const int index, const int defgroup);
diff --git a/source/blender/blenkernel/intern/deform.c b/source/blender/blenkernel/intern/deform.c
index 186a4d41403..c41f5b6f86f 100644
--- a/source/blender/blenkernel/intern/deform.c
+++ b/source/blender/blenkernel/intern/deform.c
@@ -690,3 +690,13 @@ void defvert_remove_group(MDeformVert *dvert, MDeformWeight *dw)
 		}
 	}
 }
+
+void defvert_clear(MDeformVert *dvert)
+{
+	if (dvert->dw) {
+		MEM_freeN(dvert->dw);
+		dvert->dw = NULL;
+	}
+
+	dvert->totweight = 0;
+}
diff --git a/source/blender/bmesh/intern/bmesh_walkers.c b/source/blender/bmesh/intern/bmesh_walkers.c
index 69ab7a91232..a3b5d94ba8e 100644
--- a/source/blender/bmesh/intern/bmesh_walkers.c
+++ b/source/blender/bmesh/intern/bmesh_walkers.c
@@ -73,17 +73,18 @@ void *BMW_begin(BMWalker *walker, void *start)
  * by the bitmask 'searchmask'.
  */
 void BMW_init(BMWalker *walker, BMesh *bm, int type,
-              short mask_vert, short mask_edge, short mask_loop, short mask_face,
+              short mask_vert, short mask_edge, short mask_face,
+              BMWFlag flag,
               int layer)
 {
 	memset(walker, 0, sizeof(BMWalker));
 
 	walker->layer = layer;
+	walker->flag = flag;
 	walker->bm = bm;
 
 	walker->mask_vert = mask_vert;
 	walker->mask_edge = mask_edge;
-	walker->mask_loop = mask_loop;
 	walker->mask_face = mask_face;
 
 	walker->visithash = BLI_ghash_new(BLI_ghashutil_ptrhash, BLI_ghashutil_ptrcmp, "bmesh walkers 1");
@@ -92,8 +93,8 @@ void BMW_init(BMWalker *walker, BMesh *bm, int type,
 	if (UNLIKELY(type >= BMW_MAXWALKERS || type < 0)) {
 		fprintf(stderr,
 		        "Invalid walker type in BMW_init; type: %d, "
-		        "searchmask: (v:%d, e:%d, l:%d, f:%d), flag: %d\n",
-		        type, mask_vert, mask_edge, mask_loop, mask_face, layer);
+		        "searchmask: (v:%d, e:%d, f:%d), flag: %d, layer: %d\n",
+		        type, mask_vert, mask_edge, mask_face, flag, layer);
 		BMESH_ASSERT(0);
 	}
 	
@@ -110,7 +111,6 @@ void BMW_init(BMWalker *walker, BMesh *bm, int type,
 		 * 'bm_walker_types' needs updating */
 		BLI_assert(mask_vert == 0 || (walker->valid_mask & BM_VERT));
 		BLI_assert(mask_edge == 0 || (walker->valid_mask & BM_EDGE));
-		BLI_assert(mask_loop == 0 || (walker->valid_mask & BM_LOOP));
 		BLI_assert(mask_face == 0 || (walker->valid_mask & BM_FACE));
 	}
 	
diff --git a/source/blender/bmesh/intern/bmesh_walkers.h b/source/blender/bmesh/intern/bmesh_walkers.h
index 1d548a0d4c7..4f81f38aeb3 100644
--- a/source/blender/bmesh/intern/bmesh_walkers.h
+++ b/source/blender/bmesh/intern/bmesh_walkers.h
@@ -38,6 +38,11 @@ typedef enum {
 	BMW_BREADTH_FIRST
 } BMWOrder;
 
+typedef enum {
+	BMW_FLAG_NOP = 0,
+	BMW_FLAG_TEST_HIDDEN = (1 << 0)
+} BMWFlag;
+
 /*Walkers*/
 typedef struct BMWalker {
 	void  (*begin) (struct BMWalker *walker, void *start);
@@ -54,12 +59,14 @@ typedef struct BMWalker {
 	BLI_mempool *worklist;
 	ListBase states;
 
-	/* these masks are to be tested against elements BMO_elem_flag_test() */
+	/* these masks are to be tested against elements BMO_elem_flag_test(),
+	 * should never be accessed directly only through BMW_init() and bmw_mask_check_*() functions */
 	short mask_vert;
 	short mask_edge;
-	short mask_loop;
 	short mask_face;
 
+	BMWFlag flag;
+
 	GHash *visithash;
 	GHash *secvisithash;
 	int depth;
@@ -71,7 +78,8 @@ typedef struct BMWalker {
 /* initialize a walker.  searchmask restricts some (not all) walkers to
  * elements with a specific tool flag set.  flags is specific to each walker.*/
 void BMW_init(struct BMWalker *walker, BMesh *bm, int type,
-              short mask_vert, short mask_edge, short mask_loop, short mask_face,
+              short mask_vert, short mask_edge, short mask_face,
+              BMWFlag flag,
               int layer);
 void *BMW_begin(BMWalker *walker, void *start);
 void *BMW_step(struct BMWalker *walker);
diff --git a/source/blender/bmesh/intern/bmesh_walkers_impl.c b/source/blender/bmesh/intern/bmesh_walkers_impl.c
index 493be46c976..009b5e786c4 100644
--- a/source/blender/bmesh/intern/bmesh_walkers_impl.c
+++ b/source/blender/bmesh/intern/bmesh_walkers_impl.c
@@ -34,6 +34,45 @@
 #include "intern/bmesh_private.h"
 #include "intern/bmesh_walkers_private.h"
 
+static int bmw_mask_check_vert(BMWalker *walker, BMVert *v)
+{
+	if ((walker->flag & BMW_FLAG_TEST_HIDDEN) && BM_elem_flag_test(v, BM_ELEM_HIDDEN)) {
+		return FALSE;
+	}
+	else if (walker->mask_vert && !BMO_elem_flag_test(walker->bm, v, walker->mask_vert)) {
+		return FALSE;
+	}
+	else {
+		return TRUE;
+	}
+}
+
+static int bmw_mask_check_edge(BMWalker *walker, BMEdge *e)
+{
+	if ((walker->flag & BMW_FLAG_TEST_HIDDEN) && BM_elem_flag_test(e, BM_ELEM_HIDDEN)) {
+		return FALSE;
+	}
+	else if (walker->mask_edge && !BMO_elem_flag_test(walker->bm, e, walker->mask_edge)) {
+		return FALSE;
+	}
+	else {
+		return TRUE;
+	}
+}
+
+static int bmw_mask_check_face(BMWalker *walker, BMFace *f)
+{
+	if ((walker->flag & BMW_FLAG_TEST_HIDDEN) && BM_elem_flag_test(f, BM_ELEM_HIDDEN)) {
+		return FALSE;
+	}
+	else if (walker->mask_face && !BMO_elem_flag_test(walker->bm, f, walker->mask_face)) {
+		return FALSE;
+	}
+	else {
+		return TRUE;
+	}
+}
+
 /**
  * Shell Walker:
  *
@@ -50,7 +89,7 @@ static void bmw_ShellWalker_visitEdge(BMWalker *walker, BMEdge *e)
 		return;
 	}
 
-	if (walker->mask_edge && !BMO_elem_flag_test(walker->bm, e, walker->mask_edge)) {
+	if (!bmw_mask_check_edge(walker, e)) {
 		return;
 	}
 
@@ -176,7 +215,8 @@ static void bmw_ConnectedVertexWalker_visitVertex(BMWalker *walker, BMVert *v)
 		/* already visited */
 		return;
 	}
-	if (walker->mask_vert && !BMO_elem_flag_test(walker->bm, v, walker->mask_vert)) {
+
+	if (!bmw_mask_check_vert(walker, v)) {
 		/* not flagged for walk */
 		return;
 	}
@@ -279,7 +319,8 @@ static void *bmw_IslandboundWalker_step(BMWalker *walker)
 			l = l->radial_next;
 			f = l->f;
 			e = l->e;
-			if (walker->mask_face && !BMO_elem_flag_test(walker->bm, f, walker->mask_face)) {
+
+			if (!bmw_mask_check_face(walker, f)) {
 				l = l->radial_next;
 				break;
 			}
@@ -322,7 +363,7 @@ static void bmw_IslandWalker_begin(BMWalker *walker, void *data)
 {
 	BMwIslandWalker *iwalk = NULL;
 
-	if (walker->mask_face && !BMO_elem_flag_test(walker->bm, (BMElemF *)data, walker->mask_face)) {
+	if (!bmw_mask_check_face(walker, data)) {
 		return;
 	}
 
@@ -352,13 +393,14 @@ static void *bmw_IslandWalker_step(BMWalker *walker)
 	l = BM_iter_new(&liter, walker->bm, BM_LOOPS_OF_FACE, iwalk->cur);
 	for ( ; l; l = BM_iter_step(&liter)) {
 		/* could skip loop here too, but don't add unless we need it */
-		if (walker->mask_edge && !BMO_elem_flag_test(walker->bm, l->e, walker->mask_edge)) {
+		if (!bmw_mask_check_edge(walker, l->e)) {
 			continue;
 		}
 
 		f = BM_iter_new(&iter, walker->bm, BM_FACES_OF_EDGE, l->e);
 		for ( ; f; f = BM_iter_step(&iter)) {
-			if (walker->mask_face && !BMO_elem_flag_test(walker->bm, f, walker->mask_face)) {
+
+			if (!bmw_mask_check_face(walker, f)) {
 				continue;
 			}
 
@@ -884,9 +926,10 @@ static void *bmw_UVEdgeWalker_step(BMWalker *walker)
 	type = walker->bm->ldata.layers[walker->layer].type;
 
 	BMW_state_remove(walker);
-	
-	if (walker->mask_edge && !BMO_elem_flag_test(walker->bm, l->e, walker->mask_edge))
+
+	if (!bmw_mask_check_edge(walker, l->e)) {
 		return l;
+	}
 
 	/* go over loops around l->v and nl->v and see which ones share l and nl's
 	 * mloopuv's coordinates. in addition, push on l->next if necessary */
@@ -898,13 +941,16 @@ static void *bmw_UVEdgeWalker_step(BMWalker *walker)
 			
 			rlen = BM_edge_face_count(l2->e);
 			for (j = 0; j < rlen; j++) {
-				if (BLI_ghash_haskey(walker->visithash, l2))
+				if (BLI_ghash_haskey(walker->visithash, l2)) {
 					continue;
-				if (walker->mask_edge && !(BMO_elem_flag_test(walker->bm, l2->e, walker->mask_edge))) {
-					if (l2->v != cl->v)
+				}
+
+				if (!bmw_mask_check_edge(walker, l2->e)) {
+					if (l2->v != cl->v) {
 						continue;
+					}
 				}
-				
+
 				l3 = l2->v != cl->v ? l2->next : l2;
 				d2 = CustomData_bmesh_get_layer_n(&walker->bm->ldata,
 				                                  l3->head.data, walker->layer);
diff --git a/source/blender/bmesh/operators/bmo_dissolve.c b/source/blender/bmesh/operators/bmo_dissolve.c
index 20bc01dfa22..fe5c8cd048e 100644
--- a/source/blender/bmesh/operators/bmo_dissolve.c
+++ b/source/blender/bmesh/operators/bmo_dissolve.c
@@ -47,8 +47,10 @@ static int UNUSED_FUNCTION(check_hole_in_region)(BMesh *bm, BMFace *f)
 	/* checks if there are any unmarked boundary edges in the face regio */
 
 	BMW_init(&regwalker, bm, BMW_ISLAND,
-	         BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP, FACE_MARK,
+	         BMW_MASK_NOP, BMW_MASK_NOP, FACE_MARK,
+	         BMW_FLAG_NOP, /* BMESH_TODO - should be BMW_FLAG_TEST_HIDDEN ? */
 	         BMW_NIL_LAY);
+
 	f2 = BMW_begin(&regwalker, f);
 	for ( ; f2; f2 = BMW_step(&regwalker)) {
 		l2 = BM_iter_new(&liter2, bm, BM_LOOPS_OF_FACE, f2);
@@ -106,7 +108,8 @@ void bmo_dissolve_faces_exec(BMesh *bm, BMOperator *op)
 
 		/* yay, walk */
 		BMW_init(&regwalker, bm, BMW_ISLAND,
-		         BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP, FACE_MARK,
+		         BMW_MASK_NOP, BMW_MASK_NOP, FACE_MARK,
+		         BMW_FLAG_NOP, /* BMESH_TODO - should be BMW_FLAG_TEST_HIDDEN ? */
 		         BMW_NIL_LAY);
 
 		f2 = BMW_begin(&regwalker, f);
diff --git a/source/blender/bmesh/operators/bmo_removedoubles.c b/source/blender/bmesh/operators/bmo_removedoubles.c
index 5bf40e6fd38..79b2e0773d4 100644
--- a/source/blender/bmesh/operators/bmo_removedoubles.c
+++ b/source/blender/bmesh/operators/bmo_removedoubles.c
@@ -376,7 +376,8 @@ void bmo_collapse_exec(BMesh *bm, BMOperator *op)
 	BMO_slot_buffer_flag_enable(bm, op, "edges", BM_EDGE, EDGE_MARK);
 
 	BMW_init(&walker, bm, BMW_SHELL,
-	         BMW_MASK_NOP, EDGE_MARK, BMW_MASK_NOP, BMW_MASK_NOP,
+	         BMW_MASK_NOP, EDGE_MARK, BMW_MASK_NOP,
+	         BMW_FLAG_NOP, /* BMESH_TODO - should be BMW_FLAG_TEST_HIDDEN ? */
 	         BMW_NIL_LAY);
 
 	BM_ITER(e, &iter, bm, BM_EDGES_OF_MESH, NULL) {
@@ -435,7 +436,8 @@ static void bmo_collapsecon_do_layer(BMesh *bm, BMOperator *op, int layer)
 	BMO_slot_buffer_flag_enable(bm, op, "edges", BM_EDGE, EDGE_MARK);
 
 	BMW_init(&walker, bm, BMW_LOOPDATA_ISLAND,
-	         BMW_MASK_NOP, EDGE_MARK, BMW_MASK_NOP, BMW_MASK_NOP,
+	         BMW_MASK_NOP, EDGE_MARK, BMW_MASK_NOP,
+	         BMW_FLAG_NOP, /* BMESH_TODO - should be BMW_FLAG_TEST_HIDDEN ? */
 	         layer);
 
 	BM_ITER(f, &iter, bm, BM_FACES_OF_MESH, NULL) {
diff --git a/source/blender/editors/mesh/editmesh_loopcut.c b/source/blender/editors/mesh/editmesh_loopcut.c
index b151978c1f1..638eaabbfd3 100644
--- a/source/blender/editors/mesh/editmesh_loopcut.c
+++ b/source/blender/editors/mesh/editmesh_loopcut.c
@@ -205,7 +205,8 @@ static void edgering_sel(tringselOpData *lcd, int previewlines, int select)
 
 	if (select) {
 		BMW_init(&walker, em->bm, BMW_EDGERING,
-		         BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP,
+		         BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP,
+		         BMW_FLAG_NOP, /* BMESH_TODO - should be BMW_FLAG_TEST_HIDDEN ? */
 		         BMW_NIL_LAY);
 
 		eed = BMW_begin(&walker, startedge);
@@ -218,7 +219,8 @@ static void edgering_sel(tringselOpData *lcd, int previewlines, int select)
 	}
 
 	BMW_init(&walker, em->bm, BMW_EDGERING,
-	         BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP,
+	         BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP,
+	         BMW_FLAG_NOP, /* BMESH_TODO - should be BMW_FLAG_TEST_HIDDEN ? */
 	         BMW_NIL_LAY);
 
 	eed = startedge = BMW_begin(&walker, startedge);
diff --git a/source/blender/editors/mesh/editmesh_select.c b/source/blender/editors/mesh/editmesh_select.c
index af1beba7841..c2e47ef0a09 100644
--- a/source/blender/editors/mesh/editmesh_select.c
+++ b/source/blender/editors/mesh/editmesh_select.c
@@ -899,7 +899,8 @@ static void walker_select(BMEditMesh *em, int walkercode, void *start, int selec
 	BMWalker walker;
 
 	BMW_init(&walker, bm, walkercode,
-	         BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP,
+	         BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP,
+	         BMW_FLAG_NOP, /* BMESH_TODO - should be BMW_FLAG_TEST_HIDDEN ? */
 	         BMW_NIL_LAY);
 	ele = BMW_begin(&walker, start);
 	for (; ele; ele = BMW_step(&walker)) {
@@ -1801,7 +1802,8 @@ static int edbm_select_linked_pick_invoke(bContext *C, wmOperator *op, wmEvent *
 
 		/* walk */
 		BMW_init(&walker, bm, BMW_ISLAND,
-		         BMW_MASK_NOP, limit ? BM_ELEM_SELECT : BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP,
+		         BMW_MASK_NOP, limit ? BM_ELEM_SELECT : BMW_MASK_NOP, BMW_MASK_NOP,
+		         BMW_FLAG_TEST_HIDDEN,
 		         BMW_NIL_LAY);
 
 		e = BMW_begin(&walker, efa);
@@ -1822,7 +1824,8 @@ static int edbm_select_linked_pick_invoke(bContext *C, wmOperator *op, wmEvent *
 		}
 
 		BMW_init(&walker, bm, BMW_SHELL,
-		         BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP,
+		         BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP,
+		         BMW_FLAG_TEST_HIDDEN,
 		         BMW_NIL_LAY);
 
 		e = BMW_begin(&walker, eed->v1);
@@ -1887,7 +1890,8 @@ static int edbm_select_linked_exec(bContext *C, wmOperator *op)
 		}
 
 		BMW_init(&walker, bm, BMW_ISLAND,
-		         BMW_MASK_NOP, limit ? BM_ELEM_SELECT : BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP,
+		         BMW_MASK_NOP, limit ? BM_ELEM_SELECT : BMW_MASK_NOP, BMW_MASK_NOP,
+		         BMW_FLAG_NOP, /* BMESH_TODO - should be BMW_FLAG_TEST_HIDDEN ? */
 		         BMW_NIL_LAY);
 
 		BM_ITER(efa, &iter, em->bm, BM_FACES_OF_MESH, NULL) {
@@ -1911,8 +1915,10 @@ static int edbm_select_linked_exec(bContext *C, wmOperator *op)
 		}
 
 		BMW_init(&walker, em->bm, BMW_SHELL,
-		         BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP,
+		         BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP,
+		         BMW_FLAG_NOP, /* BMESH_TODO - should be BMW_FLAG_TEST_HIDDEN ? */
 		         BMW_NIL_LAY);
+
 		BM_ITER(v, &iter, em->bm, BM_VERTS_OF_MESH, NULL) {
 			if (BM_elem_flag_test(v, BM_ELEM_TAG)) {
 				e = BMW_begin(&walker, v);
@@ -2012,7 +2018,7 @@ static void walker_deselect_nth(BMEditMesh *em, int nth, int offset, BMHeader *h
 	BMWalker walker;
 	BMIter iter;
 	int walktype = 0, itertype = 0, flushtype = 0;
-	short mask_vert = 0, mask_edge = 0, mask_loop = 0, mask_face = 0;
+	short mask_vert = 0, mask_edge = 0, mask_face = 0;
 
 	/* No active element from which to start - nothing to do */
 	if (h_act == NULL) {
@@ -2055,7 +2061,8 @@ static void walker_deselect_nth(BMEditMesh *em, int nth, int offset, BMHeader *h
 
 	/* Walk over selected elements starting at active */
 	BMW_init(&walker, bm, walktype,
-	         mask_vert, mask_edge, mask_loop, mask_face,
+	         mask_vert, mask_edge, mask_face,
+	         BMW_FLAG_NOP, /* BMESH_TODO - should be BMW_FLAG_TEST_HIDDEN ? */
 	         BMW_NIL_LAY);
 
 	BLI_assert(walker.order == BMW_BREADTH_FIRST);
diff --git a/source/blender/editors/mesh/editmesh_tools.c b/source/blender/editors/mesh/editmesh_tools.c
index 29c1496b042..824166798e9 100644
--- a/source/blender/editors/mesh/editmesh_tools.c
+++ b/source/blender/editors/mesh/editmesh_tools.c
@@ -3146,7 +3146,8 @@ static int mesh_separate_loose(Main *bmain, Scene *scene, Base *editbase, wmOper
 		/* Walk from the single vertex, selecting everything connected
 		 * to it */
 		BMW_init(&walker, bm, BMW_SHELL,
-		         BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP,
+		         BMW_MASK_NOP, BMW_MASK_NOP, BMW_MASK_NOP,
+		         BMW_FLAG_NOP, /* BMESH_TODO - should be BMW_FLAG_TEST_HIDDEN ? */
 		         BMW_NIL_LAY);
 
 		e = BMW_begin(&walker, v_seed);