fix for crash when using BM_face_calc_tessellation(), its not ensured that all tris will be filled in.

(effected knife project and laplacian smooth).

4 files changed, 28 insertions, 10 deletions

diff --git a/source/blender/bmesh/intern/bmesh_polygon.c b/source/blender/bmesh/intern/bmesh_polygon.c
index e2d2fa67f31..4041fc2c755 100644
--- a/source/blender/bmesh/intern/bmesh_polygon.c
+++ b/source/blender/bmesh/intern/bmesh_polygon.c
@@ -42,6 +42,7 @@
 #include "BLI_math.h"
 #include "BLI_array.h"
 #include "BLI_scanfill.h"
+#include "BLI_listbase.h"
 
 #include "bmesh.h"
 
@@ -160,11 +161,12 @@ static void bm_face_calc_poly_normal_vertex_cos(BMFace *f, float n[3],
  * \param r_loops  Store face loop pointers, (f->len)
  * \param r_index  Store triangle triples, indicies into \a r_loops,  ((f->len - 2) * 3)
  */
-void BM_face_calc_tessellation(BMFace *f, BMLoop **r_loops, int (*_r_index)[3])
+int BM_face_calc_tessellation(BMFace *f, BMLoop **r_loops, int (*_r_index)[3])
 {
 	int *r_index = (int *)_r_index;
 	BMLoop *l_first = BM_FACE_FIRST_LOOP(f);
 	BMLoop *l_iter;
+	int totfilltri;
 
 	if (f->len == 3) {
 		*r_loops++ = (l_iter = l_first);
@@ -174,9 +176,9 @@ void BM_face_calc_tessellation(BMFace *f, BMLoop **r_loops, int (*_r_index)[3])
 		r_index[0] = 0;
 		r_index[1] = 1;
 		r_index[2] = 2;
+		totfilltri = 1;
 	}
 	else if (f->len == 4) {
-		BMLoop *l_iter;
 		*r_loops++ = (l_iter = l_first);
 		*r_loops++ = (l_iter = l_iter->next);
 		*r_loops++ = (l_iter = l_iter->next);
@@ -189,6 +191,7 @@ void BM_face_calc_tessellation(BMFace *f, BMLoop **r_loops, int (*_r_index)[3])
 		r_index[3] = 0;
 		r_index[4] = 2;
 		r_index[5] = 3;
+		totfilltri = 2;
 	}
 	else {
 		int j;
@@ -197,7 +200,6 @@ void BM_face_calc_tessellation(BMFace *f, BMLoop **r_loops, int (*_r_index)[3])
 		ScanFillVert *sf_vert, *sf_vert_last = NULL, *sf_vert_first = NULL;
 		/* ScanFillEdge *e; */ /* UNUSED */
 		ScanFillFace *sf_tri;
-		int totfilltri;
 
 		BLI_scanfill_begin(&sf_ctx);
 
@@ -228,7 +230,7 @@ void BM_face_calc_tessellation(BMFace *f, BMLoop **r_loops, int (*_r_index)[3])
 
 		totfilltri = BLI_scanfill_calc_ex(&sf_ctx, 0, f->no);
 		BLI_assert(totfilltri <= f->len - 2);
-		(void)totfilltri;
+		BLI_assert(totfilltri == BLI_countlist(&sf_ctx.fillfacebase));
 
 		for (sf_tri = sf_ctx.fillfacebase.first; sf_tri; sf_tri = sf_tri->next) {
 			int i1 = BM_elem_index_get((BMLoop *)sf_tri->v1->tmp.p);
@@ -246,6 +248,8 @@ void BM_face_calc_tessellation(BMFace *f, BMLoop **r_loops, int (*_r_index)[3])
 
 		BLI_scanfill_end(&sf_ctx);
 	}
+
+	return totfilltri;
 }
 
 /**
diff --git a/source/blender/bmesh/intern/bmesh_polygon.h b/source/blender/bmesh/intern/bmesh_polygon.h
index ccb85449808..c439a41f672 100644
--- a/source/blender/bmesh/intern/bmesh_polygon.h
+++ b/source/blender/bmesh/intern/bmesh_polygon.h
@@ -27,7 +27,12 @@
  *  \ingroup bmesh
  */
 
-void  BM_face_calc_tessellation(BMFace *f, BMLoop **r_loops, int (*r_index)[3]);
+int   BM_face_calc_tessellation(BMFace *f, BMLoop **r_loops, int (*r_index)[3])
+#ifdef __GNUC__
+	__attribute__((warn_unused_result))
+	__attribute__((nonnull))
+#endif
+;
 float BM_face_calc_area(BMFace *f);
 float BM_face_calc_perimeter(BMFace *f);
 void  BM_face_calc_center_bounds(BMFace *f, float center[3]);
diff --git a/source/blender/bmesh/intern/bmesh_queries.c b/source/blender/bmesh/intern/bmesh_queries.c
index 0525337120f..26b0e42a1c1 100644
--- a/source/blender/bmesh/intern/bmesh_queries.c
+++ b/source/blender/bmesh/intern/bmesh_queries.c
@@ -1657,12 +1657,13 @@ bool BM_face_is_any_edge_flag_test(BMFace *f, const char hflag)
 
 static void bm_mesh_calc_volume_face(BMFace *f, float *r_vol)
 {
-	const int tottri = f->len - 2;
+	int tottri = f->len - 2;
 	BMLoop **loops     = BLI_array_alloca(loops, f->len);
 	int    (*index)[3] = BLI_array_alloca(index, tottri);
 	int j;
 
-	BM_face_calc_tessellation(f, loops, index);
+	tottri = BM_face_calc_tessellation(f, loops, index);
+	BLI_assert(tottri <= f->len - 2);
 
 	for (j = 0; j < tottri; j++) {
 		const float *p1 = loops[index[j][0]]->v->co;
diff --git a/source/blender/editors/mesh/editmesh_knife.c b/source/blender/editors/mesh/editmesh_knife.c
index c10e03100f3..b5b6a92cbf5 100644
--- a/source/blender/editors/mesh/editmesh_knife.c
+++ b/source/blender/editors/mesh/editmesh_knife.c
@@ -3312,15 +3312,17 @@ void MESH_OT_knife_tool(wmOperatorType *ot)
  */
 static void edvm_mesh_knife_face_point(BMFace *f, float r_cent[3])
 {
-	const int tottri = f->len - 2;
+	int tottri = f->len - 2;
 	BMLoop **loops     = BLI_array_alloca(loops, f->len);
 	int    (*index)[3] = BLI_array_alloca(index, tottri);
 	int j;
 
 	float const *best_co[3] = {NULL};
 	float  best_area  = -1.0f;
+	bool ok = false;
 
-	BM_face_calc_tessellation(f, loops, index);
+	tottri = BM_face_calc_tessellation(f, loops, index);
+	BLI_assert(tottri <= f->len - 2);
 
 	for (j = 0; j < tottri; j++) {
 		const float *p1 = loops[index[j][0]]->v->co;
@@ -3336,10 +3338,16 @@ static void edvm_mesh_knife_face_point(BMFace *f, float r_cent[3])
 			best_co[1] = p2;
 			best_co[2] = p3;
 			best_area = area;
+			ok = true;
 		}
 	}
 
-	mid_v3_v3v3v3(r_cent, best_co[0], best_co[1], best_co[2]);
+	if (ok) {
+		mid_v3_v3v3v3(r_cent, best_co[0], best_co[1], best_co[2]);
+	}
+	else {
+		mid_v3_v3v3v3(r_cent, loops[0]->v->co, loops[1]->v->co, loops[2]->v->co);
+	}
 }
 
 static bool edbm_mesh_knife_face_isect(ARegion *ar, LinkNode *polys, BMFace *f, float projmat[4][4])