Fix editmesh-connect with hidden geometry

- ignore hidden faces & verts
- when cutting a pair, select edges co-linear to the cut.

Also support creating a buffer from hidden elem's even if BMO_FLAG_RESPECT_HIDE is enabled.
(if the hflag used includes BM_ELEM_HIDDEN).

5 files changed, 90 insertions, 26 deletions

diff --git a/source/blender/bmesh/intern/bmesh_opdefines.c b/source/blender/bmesh/intern/bmesh_opdefines.c
index 886181cd204..74bd9110d8c 100644
--- a/source/blender/bmesh/intern/bmesh_opdefines.c
+++ b/source/blender/bmesh/intern/bmesh_opdefines.c
@@ -916,6 +916,7 @@ static BMOpDefine bmo_connect_verts_def = {
 	"connect_verts",
 	/* slots_in */
 	{{"verts", BMO_OP_SLOT_ELEMENT_BUF, {BM_VERT}},
+	 {"faces_exclude", BMO_OP_SLOT_ELEMENT_BUF, {BM_FACE}},
 	 {"check_degenerate", BMO_OP_SLOT_BOOL},  /* prevent splits with overlaps & intersections */
 	 {{'\0'}},
 	},
@@ -961,6 +962,8 @@ static BMOpDefine bmo_connect_vert_pair_def = {
 	"connect_vert_pair",
 	/* slots_in */
 	{{"verts", BMO_OP_SLOT_ELEMENT_BUF, {BM_VERT}},
+	 {"verts_exclude", BMO_OP_SLOT_ELEMENT_BUF, {BM_VERT}},
+	 {"faces_exclude", BMO_OP_SLOT_ELEMENT_BUF, {BM_FACE}},
 	 {{'\0'}},
 	},
 	/* slots_out */
diff --git a/source/blender/bmesh/intern/bmesh_operators.c b/source/blender/bmesh/intern/bmesh_operators.c
index a4e03b1984a..b041c010c22 100644
--- a/source/blender/bmesh/intern/bmesh_operators.c
+++ b/source/blender/bmesh/intern/bmesh_operators.c
@@ -815,7 +815,7 @@ static void bmo_slot_buffer_from_hflag(BMesh *bm, BMOperator *op, BMOpSlot slot_
 {
 	BMOpSlot *output = BMO_slot_get(slot_args, slot_name);
 	int totelement = 0, i = 0;
-	const bool respecthide = (op->flag & BMO_FLAG_RESPECT_HIDE) != 0;
+	const bool respecthide = ((op->flag & BMO_FLAG_RESPECT_HIDE) != 0) && ((hflag & BM_ELEM_HIDDEN) == 0);
 
 	BLI_assert(output->slot_type == BMO_OP_SLOT_ELEMENT_BUF);
 	BLI_assert(((output->slot_subtype.elem & BM_ALL_NOLOOP) & htype) == htype);
diff --git a/source/blender/bmesh/operators/bmo_connect.c b/source/blender/bmesh/operators/bmo_connect.c
index 4cf6e82fc8e..d4dabda1d99 100644
--- a/source/blender/bmesh/operators/bmo_connect.c
+++ b/source/blender/bmesh/operators/bmo_connect.c
@@ -36,8 +36,14 @@
 #include "intern/bmesh_operators_private.h" /* own include */
 
 #define VERT_INPUT	1
+
 #define EDGE_OUT	1
-#define FACE_TAG	2
+/* Edge spans 2 VERT_INPUT's, its a nop,
+ * but include in "edges.out" */
+#define EDGE_OUT_ADJ	2
+
+#define FACE_TAG		2
+#define FACE_EXCLUDE	4
 
 static int bm_face_connect_verts(BMesh *bm, BMFace *f, const bool check_degenerate)
 {
@@ -134,7 +140,6 @@ static int bm_face_connect_verts(BMesh *bm, BMFace *f, const bool check_degenera
 void bmo_connect_verts_exec(BMesh *bm, BMOperator *op)
 {
 	BMOIter siter;
-	BMIter iter;
 	BMVert *v;
 	BMFace *f;
 	const bool check_degenerate = BMO_slot_bool_get(op->slots_in,  "check_degenerate");
@@ -142,16 +147,35 @@ void bmo_connect_verts_exec(BMesh *bm, BMOperator *op)
 
 	BLI_LINKSTACK_INIT(faces);
 
+	/* tag so we won't touch ever (typically hidden faces) */
+	BMO_slot_buffer_flag_enable(bm, op->slots_in, "faces_exclude", BM_FACE, FACE_EXCLUDE);
+
 	/* add all faces connected to verts */
 	BMO_ITER (v, &siter, op->slots_in, "verts", BM_VERT) {
+		BMIter iter;
+		BMLoop *l_iter;
+
 		BMO_elem_flag_enable(bm, v, VERT_INPUT);
-		BM_ITER_ELEM (f, &iter, v, BM_FACES_OF_VERT) {
-			if (!BMO_elem_flag_test(bm, f, FACE_TAG)) {
-				BMO_elem_flag_enable(bm, f, FACE_TAG);
-				if (f->len > 3) {
-					BLI_LINKSTACK_PUSH(faces, f);
+		BM_ITER_ELEM (l_iter, &iter, v, BM_LOOPS_OF_VERT) {
+			f = l_iter->f;
+			if (!BMO_elem_flag_test(bm, f, FACE_EXCLUDE)) {
+				if (!BMO_elem_flag_test(bm, f, FACE_TAG)) {
+					BMO_elem_flag_enable(bm, f, FACE_TAG);
+					if (f->len > 3) {
+						BLI_LINKSTACK_PUSH(faces, f);
+					}
 				}
 			}
+
+			/* flag edges even if these are not newly created
+			 * this way cut-pairs that include co-linear edges will get
+			 * predictable output. */
+			if (BMO_elem_flag_test(bm, l_iter->prev->v, VERT_INPUT)) {
+				BMO_elem_flag_enable(bm, l_iter->prev->e, EDGE_OUT_ADJ);
+			}
+			if (BMO_elem_flag_test(bm, l_iter->next->v, VERT_INPUT)) {
+				BMO_elem_flag_enable(bm, l_iter->e, EDGE_OUT_ADJ);
+			}
 		}
 	}
 
@@ -164,5 +188,5 @@ void bmo_connect_verts_exec(BMesh *bm, BMOperator *op)
 
 	BLI_LINKSTACK_FREE(faces);
 
-	BMO_slot_buffer_from_enabled_flag(bm, op, op->slots_out, "edges.out", BM_EDGE, EDGE_OUT);
+	BMO_slot_buffer_from_enabled_flag(bm, op, op->slots_out, "edges.out", BM_EDGE, EDGE_OUT | EDGE_OUT_ADJ);
 }
diff --git a/source/blender/bmesh/operators/bmo_connect_pair.c b/source/blender/bmesh/operators/bmo_connect_pair.c
index fe4f5040a91..a0acf6ed2c5 100644
--- a/source/blender/bmesh/operators/bmo_connect_pair.c
+++ b/source/blender/bmesh/operators/bmo_connect_pair.c
@@ -51,6 +51,15 @@
 
 #define CONNECT_EPS 0.0001f
 #define VERT_OUT 1
+#define VERT_EXCLUDE 2
+
+/* typically hidden faces */
+#define FACE_EXCLUDE 2
+
+#define FACE_WALK_TEST(f)  (CHECK_TYPE_INLINE(f, BMFace *), \
+	BMO_elem_flag_test(pc->bm_bmoflag, f, FACE_EXCLUDE) == 0)
+#define VERT_WALK_TEST(v)  (CHECK_TYPE_INLINE(v, BMVert *), \
+	BMO_elem_flag_test(pc->bm_bmoflag, v, VERT_EXCLUDE) == 0)
 
 // #define DEBUG_PRINT
 
@@ -59,6 +68,9 @@ typedef struct PathContext {
 	float matrix[3][3];
 	float axis_sep;
 
+	/* only to access BMO flags */
+	BMesh *bm_bmoflag;
+
 	BMVert *v_a, *v_b;
 
 	BLI_mempool *link_pool;
@@ -204,8 +216,9 @@ static void state_link_add(PathContext *pc, PathLinkState *state,
 	state->link_last = step_new;
 }
 
-static PathLinkState *state_dupe_add(PathContext *pc,
-                                     PathLinkState *state, const PathLinkState *state_orig)
+static PathLinkState *state_dupe_add(
+        PathContext *pc,
+        PathLinkState *state, const PathLinkState *state_orig)
 {
 	state = MEM_mallocN(sizeof(*state), __func__);
 	*state = *state_orig;
@@ -214,16 +227,19 @@ static PathLinkState *state_dupe_add(PathContext *pc,
 }
 
 /* walk around the face edges */
-static PathLinkState *state_step__face_edges(PathContext *pc,
-                                             PathLinkState *state, const PathLinkState *state_orig,
-                                             BMLoop *l_iter, BMLoop *l_last)
+static PathLinkState *state_step__face_edges(
+        PathContext *pc,
+        PathLinkState *state, const PathLinkState *state_orig,
+        BMLoop *l_iter, BMLoop *l_last)
 {
 	do {
 		if (state_isect_co_pair(pc, l_iter->v->co, l_iter->next->v->co)) {
 			BMElem *ele_next      = (BMElem *)l_iter->e;
 			BMElem *ele_next_from = (BMElem *)l_iter->f;
 
-			if (state_link_find(state, ele_next) == false) {
+			if (FACE_WALK_TEST((BMFace *)ele_next_from) &&
+			    (state_link_find(state, ele_next) == false))
+			{
 				if (state_orig->link_last != state->link_last) {
 					state = state_dupe_add(pc, state, state_orig);
 				}
@@ -235,16 +251,19 @@ static PathLinkState *state_step__face_edges(PathContext *pc,
 }
 
 /* walk around the face verts */
-static PathLinkState *state_step__face_verts(PathContext *pc,
-                                             PathLinkState *state, const PathLinkState *state_orig,
-                                             BMLoop *l_iter, BMLoop *l_last)
+static PathLinkState *state_step__face_verts(
+        PathContext *pc,
+        PathLinkState *state, const PathLinkState *state_orig,
+        BMLoop *l_iter, BMLoop *l_last)
 {
 	do {
 		if (state_isect_co_exact(pc, l_iter->v->co)) {
 			BMElem *ele_next      = (BMElem *)l_iter->v;
 			BMElem *ele_next_from = (BMElem *)l_iter->f;
 
-			if (state_link_find(state, ele_next) == false) {
+			if (FACE_WALK_TEST((BMFace *)ele_next_from) &&
+			    state_link_find(state, ele_next) == false)
+			{
 				if (state_orig->link_last != state->link_last) {
 					state = state_dupe_add(pc, state, state_orig);
 				}
@@ -268,7 +287,9 @@ static bool state_step(PathContext *pc, PathLinkState *state)
 		BMLoop *l_start;
 
 		BM_ITER_ELEM (l_start, &liter, e, BM_LOOPS_OF_EDGE) {
-			if (l_start->f != ele_from) {
+			if ((l_start->f != ele_from) &&
+			    FACE_WALK_TEST(l_start->f))
+			{
 				/* very similar to block below */
 				if (BM_vert_in_face(l_start->f, pc->v_b)) {
 					if (state_orig.link_last != state->link_last) {
@@ -295,7 +316,9 @@ static bool state_step(PathContext *pc, PathLinkState *state)
 			BMLoop *l_start;
 
 			BM_ITER_ELEM (l_start, &liter, v, BM_LOOPS_OF_VERT) {
-				if (l_start->f != ele_from) {
+				if ((l_start->f != ele_from) &&
+				    FACE_WALK_TEST(l_start->f))
+				{
 					/* very similar to block above */
 					if (BM_vert_in_face(l_start->f, pc->v_b)) {
 						BMElem *ele_next      = (BMElem *)pc->v_b;
@@ -324,8 +347,10 @@ static bool state_step(PathContext *pc, PathLinkState *state)
 			BMIter eiter;
 			BMEdge *e;
 			BM_ITER_ELEM (e, &eiter, v, BM_EDGES_OF_VERT) {
-				if ((BMElem *)e != ele_from) {
-					BMVert *v_other = BM_edge_other_vert(e, v);
+				BMVert *v_other = BM_edge_other_vert(e, v);
+				if (((BMElem *)e != ele_from) &&
+				    VERT_WALK_TEST(v_other))
+				{
 					if (v_other == pc->v_b) {
 						BMElem *ele_next      = (BMElem *)pc->v_b;
 						BMElem *ele_next_from = (BMElem *)e;
@@ -371,6 +396,7 @@ void bmo_connect_vert_pair_exec(BMesh *bm, BMOperator *op)
 		return;
 	}
 
+	pc.bm_bmoflag = bm;
 	pc.v_a = ((BMVert **)op_verts_slot->data.p)[0];
 	pc.v_b = ((BMVert **)op_verts_slot->data.p)[1];
 
@@ -384,6 +410,10 @@ void bmo_connect_vert_pair_exec(BMesh *bm, BMOperator *op)
 	printf("%s: v_b: %d\n", __func__, BM_elem_index_get(pc.v_b));
 #endif
 
+	/* tag so we won't touch ever (typically hidden faces) */
+	BMO_slot_buffer_flag_enable(bm, op->slots_in, "faces_exclude", BM_FACE, FACE_EXCLUDE);
+	BMO_slot_buffer_flag_enable(bm, op->slots_in, "verts_exclude", BM_VERT, VERT_EXCLUDE);
+
 	/* setup context */
 	{
 		BLI_listbase_clear(&pc.state_lb);
@@ -536,7 +566,8 @@ void bmo_connect_vert_pair_exec(BMesh *bm, BMOperator *op)
 		 * always connect even when resulting faces are degenerate [#39418] */
 		BMOperator op_sub;
 		BMO_op_initf(bm, &op_sub, 0,
-		             "connect_verts verts=%fv", VERT_OUT);
+		             "connect_verts verts=%fv faces_exclude=%s",
+		             VERT_OUT, op, "faces_exclude");
 		BMO_op_exec(bm, &op_sub);
 		BMO_slot_copy(&op_sub, slots_out, "edges.out",
 		              op,      slots_out, "edges.out");
diff --git a/source/blender/editors/mesh/editmesh_tools.c b/source/blender/editors/mesh/editmesh_tools.c
index 12b6428f0fe..3f1023b7fb4 100644
--- a/source/blender/editors/mesh/editmesh_tools.c
+++ b/source/blender/editors/mesh/editmesh_tools.c
@@ -881,12 +881,18 @@ static int edbm_vert_connect_exec(bContext *C, wmOperator *op)
 	int len;
 	
 	if (is_pair) {
-		if (!EDBM_op_init(em, &bmop, op, "connect_vert_pair verts=%hv", BM_ELEM_SELECT)) {
+		if (!EDBM_op_init(em, &bmop, op,
+		                  "connect_vert_pair verts=%hv verts_exclude=%hv faces_exclude=%hf",
+		                  BM_ELEM_SELECT, BM_ELEM_HIDDEN, BM_ELEM_HIDDEN))
+		{
 			return OPERATOR_CANCELLED;
 		}
 	}
 	else {
-		if (!EDBM_op_init(em, &bmop, op, "connect_verts verts=%hv check_degenerate=%b", BM_ELEM_SELECT, true)) {
+		if (!EDBM_op_init(em, &bmop, op,
+		                  "connect_verts verts=%hv faces_exclude=%hf check_degenerate=%b",
+		                  BM_ELEM_SELECT, BM_ELEM_HIDDEN, true))
+		{
 			return OPERATOR_CANCELLED;
 		}
 	}