BMesh: make BM_face_calc_normal_subset apart of the bmesh api

also make face normal calculation functions return normal length

3 files changed, 41 insertions, 55 deletions

diff --git a/source/blender/bmesh/intern/bmesh_polygon.c b/source/blender/bmesh/intern/bmesh_polygon.c
index 3422656b50c..08619cd9699 100644
--- a/source/blender/bmesh/intern/bmesh_polygon.c
+++ b/source/blender/bmesh/intern/bmesh_polygon.c
@@ -73,7 +73,7 @@ static bool testedgesidef(const float v1[2], const float v2[2], const float v3[2
  *
  * Same as #normal_poly_v3 but operates directly on a bmesh face.
  */
-static void bm_face_calc_poly_normal(const BMFace *f, float n[3])
+static float bm_face_calc_poly_normal(const BMFace *f, float n[3])
 {
 	BMLoop *l_first = BM_FACE_FIRST_LOOP(f);
 	BMLoop *l_iter  = l_first;
@@ -92,9 +92,7 @@ static void bm_face_calc_poly_normal(const BMFace *f, float n[3])
 
 	} while (l_iter != l_first);
 
-	if (UNLIKELY(normalize_v3(n) == 0.0f)) {
-		n[2] = 1.0f;
-	}
+	return normalize_v3(n);
 }
 
 /**
@@ -103,7 +101,7 @@ static void bm_face_calc_poly_normal(const BMFace *f, float n[3])
  * Same as #calc_poly_normal and #bm_face_calc_poly_normal
  * but takes an array of vertex locations.
  */
-static void bm_face_calc_poly_normal_vertex_cos(BMFace *f, float r_no[3],
+static float bm_face_calc_poly_normal_vertex_cos(BMFace *f, float r_no[3],
                                                 float const (*vertexCos)[3])
 {
 	BMLoop *l_first = BM_FACE_FIRST_LOOP(f);
@@ -122,9 +120,7 @@ static void bm_face_calc_poly_normal_vertex_cos(BMFace *f, float r_no[3],
 		v_curr = vertexCos[BM_elem_index_get(l_iter->v)];
 	} while (l_iter != l_first);
 
-	if (UNLIKELY(normalize_v3(r_no) == 0.0f)) {
-		r_no[2] = 1.0f; /* other axis set to 0.0 */
-	}
+	return normalize_v3(r_no);
 }
 
 /**
@@ -475,7 +471,7 @@ void BM_vert_normal_update_all(BMVert *v)
  * is passed in as well.
  */
 
-void BM_face_calc_normal(const BMFace *f, float r_no[3])
+float BM_face_calc_normal(const BMFace *f, float r_no[3])
 {
 	BMLoop *l;
 
@@ -488,8 +484,7 @@ void BM_face_calc_normal(const BMFace *f, float r_no[3])
 			const float *co3 = (l = l->next)->v->co;
 			const float *co4 = (l->next)->v->co;
 
-			normal_quad_v3(r_no, co1, co2, co3, co4);
-			break;
+			return normal_quad_v3(r_no, co1, co2, co3, co4);
 		}
 		case 3:
 		{
@@ -497,13 +492,11 @@ void BM_face_calc_normal(const BMFace *f, float r_no[3])
 			const float *co2 = (l = l->next)->v->co;
 			const float *co3 = (l->next)->v->co;
 
-			normal_tri_v3(r_no, co1, co2, co3);
-			break;
+			return normal_tri_v3(r_no, co1, co2, co3);
 		}
 		default:
 		{
-			bm_face_calc_poly_normal(f, r_no);
-			break;
+			return bm_face_calc_poly_normal(f, r_no);
 		}
 	}
 }
@@ -513,8 +506,8 @@ void BM_face_normal_update(BMFace *f)
 }
 
 /* exact same as 'BM_face_calc_normal' but accepts vertex coords */
-void BM_face_calc_normal_vcos(BMesh *bm, BMFace *f, float r_no[3],
-                              float const (*vertexCos)[3])
+float BM_face_calc_normal_vcos(BMesh *bm, BMFace *f, float r_no[3],
+                               float const (*vertexCos)[3])
 {
 	BMLoop *l;
 
@@ -531,8 +524,7 @@ void BM_face_calc_normal_vcos(BMesh *bm, BMFace *f, float r_no[3],
 			const float *co3 = vertexCos[BM_elem_index_get((l = l->next)->v)];
 			const float *co4 = vertexCos[BM_elem_index_get((l->next)->v)];
 
-			normal_quad_v3(r_no, co1, co2, co3, co4);
-			break;
+			return normal_quad_v3(r_no, co1, co2, co3, co4);
 		}
 		case 3:
 		{
@@ -540,22 +532,38 @@ void BM_face_calc_normal_vcos(BMesh *bm, BMFace *f, float r_no[3],
 			const float *co2 = vertexCos[BM_elem_index_get((l = l->next)->v)];
 			const float *co3 = vertexCos[BM_elem_index_get((l->next)->v)];
 
-			normal_tri_v3(r_no, co1, co2, co3);
-			break;
-		}
-		case 0:
-		{
-			zero_v3(r_no);
-			break;
+			return normal_tri_v3(r_no, co1, co2, co3);
 		}
 		default:
 		{
-			bm_face_calc_poly_normal_vertex_cos(f, r_no, vertexCos);
-			break;
+			return bm_face_calc_poly_normal_vertex_cos(f, r_no, vertexCos);
 		}
 	}
 }
 
+/**
+ * Calculates the face subset normal.
+ */
+float BM_face_calc_normal_subset(BMLoop *l_first, BMLoop *l_last, float r_no[3])
+{
+	const float *v_prev, *v_curr;
+
+	/* Newell's Method */
+	BMLoop *l_iter = l_first;
+	BMLoop *l_term = l_last->next;
+
+	zero_v3(r_no);
+
+	v_prev = l_last->v->co;
+	do {
+		v_curr = l_iter->v->co;
+		add_newell_cross_v3_v3v3(r_no, v_prev, v_curr);
+		v_prev = v_curr;
+	} while ((l_iter = l_iter->next) != l_term);
+
+	return normalize_v3(r_no);
+}
+
 /* exact same as 'BM_face_calc_normal' but accepts vertex coords */
 void BM_face_calc_center_mean_vcos(BMesh *bm, BMFace *f, float r_cent[3],
                                    float const (*vertexCos)[3])
diff --git a/source/blender/bmesh/intern/bmesh_polygon.h b/source/blender/bmesh/intern/bmesh_polygon.h
index f408947f467..28fc314b329 100644
--- a/source/blender/bmesh/intern/bmesh_polygon.h
+++ b/source/blender/bmesh/intern/bmesh_polygon.h
@@ -32,9 +32,10 @@
 void  BM_bmesh_calc_tessellation(BMesh *bm, BMLoop *(*looptris)[3], int *r_looptris_tot);
 
 void  BM_face_calc_tessellation(const BMFace *f, BMLoop **r_loops, unsigned int (*r_index)[3]);
-void  BM_face_calc_normal(const BMFace *f, float r_no[3]) ATTR_NONNULL();
-void  BM_face_calc_normal_vcos(BMesh *bm, BMFace *f, float r_no[3],
+float BM_face_calc_normal(const BMFace *f, float r_no[3]) ATTR_NONNULL();
+float BM_face_calc_normal_vcos(BMesh *bm, BMFace *f, float r_no[3],
                                float const (*vertexCos)[3]) ATTR_NONNULL();
+float BM_face_calc_normal_subset(BMLoop *l_first, BMLoop *l_last, float r_no[3]) ATTR_NONNULL();
 float BM_face_calc_area(BMFace *f) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
 float BM_face_calc_perimeter(BMFace *f) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
 void  BM_face_calc_plane(BMFace *f, float r_plane[3]) ATTR_NONNULL();
diff --git a/source/blender/bmesh/operators/bmo_connect_nonplanar.c b/source/blender/bmesh/operators/bmo_connect_nonplanar.c
index 15447c97b5f..9e9f1cbf0d4 100644
--- a/source/blender/bmesh/operators/bmo_connect_nonplanar.c
+++ b/source/blender/bmesh/operators/bmo_connect_nonplanar.c
@@ -39,29 +39,6 @@
 #define FACE_OUT	(1 << 1)
 
 /**
- * Calculates the face subset normal.
- */
-static bool bm_face_subset_calc_normal(BMLoop *l_first, BMLoop *l_last, float r_no[3])
-{
-	const float *v_prev, *v_curr;
-
-	/* Newell's Method */
-	BMLoop *l_iter = l_first;
-	BMLoop *l_term = l_last->next;
-
-	zero_v3(r_no);
-
-	v_prev = l_last->v->co;
-	do {
-		v_curr = l_iter->v->co;
-		add_newell_cross_v3_v3v3(r_no, v_prev, v_curr);
-		v_prev = v_curr;
-	} while ((l_iter = l_iter->next) != l_term);
-
-	return (normalize_v3(r_no) != 0.0f);
-}
-
-/**
  * Calculates how non-planar the face subset is.
  */
 static float bm_face_subset_calc_planar(BMLoop *l_first, BMLoop *l_last, const float no[3])
@@ -115,8 +92,8 @@ static bool bm_face_split_find(BMFace *f, BMLoop *l_pair[2], float *r_angle)
 				/* first calculate normals */
 				float no_a[3], no_b[3];
 
-				if (bm_face_subset_calc_normal(l_a, l_b, no_a) &&
-				    bm_face_subset_calc_normal(l_b, l_a, no_b))
+				if (BM_face_calc_normal_subset(l_a, l_b, no_a) != 0.0f &&
+				    BM_face_calc_normal_subset(l_b, l_a, no_b) != 0.0f)
 				{
 					const float err_a = bm_face_subset_calc_planar(l_a, l_b, no_a);
 					const float err_b = bm_face_subset_calc_planar(l_b, l_a, no_b);