fix [#30613] B-mesh - inset created invalid mesh

inset could make a quad that used the same vertex multiple times.

1 files changed, 40 insertions, 20 deletions

diff --git a/source/blender/bmesh/operators/bmo_inset.c b/source/blender/bmesh/operators/bmo_inset.c
index 4b219b53243..d64edada06d 100644
--- a/source/blender/bmesh/operators/bmo_inset.c
+++ b/source/blender/bmesh/operators/bmo_inset.c
@@ -35,6 +35,7 @@ typedef struct SplitEdgeInfo {
 	float   length;
 	BMEdge *e_old;
 	BMEdge *e_new;
+	BMLoop *l;
 } SplitEdgeInfo;
 
 static void edge_loop_tangent(BMEdge *e, BMLoop *e_loop, float r_no[3])
@@ -170,23 +171,20 @@ void bmo_inset_exec(BMesh *bm, BMOperator *op)
 	}
 
 	for (i = 0, es = edge_info; i < edge_info_len; i++, es++) {
-		BMLoop *l;
-
-		if ((l = bm_edge_is_mixed_face_tag(es->e_old->l))) {
+		if ((es->l = bm_edge_is_mixed_face_tag(es->e_old->l))) {
 			/* do nothing */
 		}
 		else {
-			l = es->e_old->l; /* must be a boundary */
+			es->l = es->e_old->l; /* must be a boundary */
 		}
 
 
 		/* run the separate arg */
-		bmesh_edge_separate(bm, es->e_old, l);
+		bmesh_edge_separate(bm, es->e_old, es->l);
 
 		/* calc edge-split info */
-		es->e_new = l->e;
-		edge_loop_tangent(es->e_new, l, es->no);
-
+		es->e_new = es->l->e;
+		edge_loop_tangent(es->e_new, es->l, es->no);
 
 		if (es->e_new == es->e_old) { /* happens on boundary edges */
 			es->e_old = BM_edge_create(bm, es->e_new->v1, es->e_new->v2, es->e_new, FALSE);
@@ -223,8 +221,8 @@ void bmo_inset_exec(BMesh *bm, BMOperator *op)
 	for (i = 0, es = edge_info; i < edge_info_len; i++, es++) {
 		for (j = 0; j < 2; j++) {
 			v = (j == 0) ? es->e_new->v1 : es->e_new->v2;
-			/* end confusing part - just pretend this is a typical loop on verts */
 
+			/* end confusing part - just pretend this is a typical loop on verts */
 
 			/* only split of tagged verts - used by separated edges */
 
@@ -240,6 +238,12 @@ void bmo_inset_exec(BMesh *bm, BMOperator *op)
 				bmesh_vert_separate(bm, v, &vout, &r_vout_len);
 				v = NULL; /* don't use again */
 
+				/* in some cases the edge doesnt split off */
+				if (r_vout_len == 1) {
+					MEM_freeN(vout);
+					continue;
+				}
+
 				for (k = 0; k < r_vout_len; k++) {
 					BMVert *v_split = vout[k]; /* only to avoid vout[k] all over */
 
@@ -249,8 +253,7 @@ void bmo_inset_exec(BMesh *bm, BMOperator *op)
 
 					/* find adjacent */
 					BM_ITER(e, &iter, bm, BM_EDGES_OF_VERT, v_split) {
-						if (BM_edge_is_boundary(e) && /* this will be true because bmesh_edge_separate() has run */
-						    BM_elem_flag_test(e, BM_ELEM_TAG) &&
+						if (BM_elem_flag_test(e, BM_ELEM_TAG) &&
 						    BM_elem_flag_test(e->l->f, BM_ELEM_TAG))
 						{
 							if (vert_edge_tag_tot < 2) {
@@ -398,22 +401,39 @@ void bmo_inset_exec(BMesh *bm, BMOperator *op)
 
 	/* create faces */
 	for (i = 0, es = edge_info; i < edge_info_len; i++, es++) {
-		BMVert *v1, *v2, *v3, *v4;
-
+		BMVert *varr[4] = {NULL};
 		/* get the verts in the correct order */
-		BM_edge_ordered_verts(es->e_new, &v1, &v2);
-		if (v1 == es->e_new->v1) {
-			v3 = es->e_old->v2;
-			v4 = es->e_old->v1;
+		BM_edge_ordered_verts_ex(es->e_new, &varr[1], &varr[0], es->l);
+#if 0
+		if (varr[0] == es->e_new->v1) {
+			varr[2] = es->e_old->v2;
+			varr[3] = es->e_old->v1;
+		}
+		else {
+			varr[2] = es->e_old->v1;
+			varr[3] = es->e_old->v2;
+		}
+		j = 4;
+#else
+		/* slightly trickier check - since we can't assume the verts are split */
+		j = 2; /* 2 edges are set */
+		if (varr[0] == es->e_new->v1) {
+			if (es->e_old->v2 != es->e_new->v2) { varr[j++] = es->e_old->v2; }
+			if (es->e_old->v1 != es->e_new->v1) { varr[j++] = es->e_old->v1; }
 		}
 		else {
-			v3 = es->e_old->v1;
-			v4 = es->e_old->v2;
+			if (es->e_old->v1 != es->e_new->v1) { varr[j++] = es->e_old->v1; }
+			if (es->e_old->v2 != es->e_new->v2) { varr[j++] = es->e_old->v2; }
 		}
 
+		if (j == 2) {
+			/* can't make face! */
+			continue;
+		}
+#endif
 		/* no need to check doubles, we KNOW there won't be any */
 		/* yes - reverse face is correct in this case */
-		f = BM_face_create_quad_tri(bm, v4, v3, v2, v1, es->e_new->l->f, FALSE);
+		f = BM_face_create_quad_tri_v(bm, varr, j, es->l->f, FALSE);
 		BMO_elem_flag_enable(bm, f, ELE_NEW);
 	}