7 files changed, 46 insertions, 20 deletions
diff --git a/source/blender/blenlib/BLI_math_base.h b/source/blender/blenlib/BLI_math_base.h
index 13718e83621..2bc23c8a72d 100644
--- a/source/blender/blenlib/BLI_math_base.h
+++ b/source/blender/blenlib/BLI_math_base.h
@@ -194,6 +194,8 @@ MINLINE int min_iiii(int a, int b, int c, int d);
 MINLINE int max_iiii(int a, int b, int c, int d);
 
 MINLINE float signf(float f);
+MINLINE int signum_i_ex(float a, float eps);
+MINLINE int signum_i(float a);
 
 MINLINE float power_of_2(float f);
 
diff --git a/source/blender/blenlib/BLI_math_geom.h b/source/blender/blenlib/BLI_math_geom.h
index 32678bdf680..95ab3185d1f 100644
--- a/source/blender/blenlib/BLI_math_geom.h
+++ b/source/blender/blenlib/BLI_math_geom.h
@@ -60,6 +60,7 @@ float area_poly_v3(const float verts[][3], unsigned int nr);
 float area_poly_v2(const float verts[][2], unsigned int nr);
 float cotangent_tri_weight_v3(const float v1[3], const float v2[3], const float v3[3]);
 
+void          cross_tri_v3(float n[3], const float v1[3], const float v2[3], const float v3[3]);
 MINLINE float cross_tri_v2(const float v1[2], const float v2[2], const float v3[2]);
 float cross_poly_v2(const float verts[][2], unsigned int nr);
 
diff --git a/source/blender/blenlib/BLI_polyfill2d_beautify.h b/source/blender/blenlib/BLI_polyfill2d_beautify.h
index c3bb29af21e..20e53b080fe 100644
--- a/source/blender/blenlib/BLI_polyfill2d_beautify.h
+++ b/source/blender/blenlib/BLI_polyfill2d_beautify.h
@@ -33,6 +33,9 @@ void BLI_polyfill_beautify(
         /* structs for reuse */
         struct MemArena *arena, struct Heap *eheap, struct EdgeHash *eh);
 
+float BLI_polyfill_beautify_quad_rotate_calc(
+        const float v1[2], const float v2[2], const float v3[2], const float v4[2]);
+
 /* avoid realloc's when creating new structures for polyfill ngons */
 #define BLI_POLYFILL_ALLOC_NGON_RESERVE 64
 
diff --git a/source/blender/blenlib/intern/math_base_inline.c b/source/blender/blenlib/intern/math_base_inline.c
index 39116d6f30f..f5713824296 100644
--- a/source/blender/blenlib/intern/math_base_inline.c
+++ b/source/blender/blenlib/intern/math_base_inline.c
@@ -276,5 +276,18 @@ MINLINE float signf(float f)
 	return (f < 0.f) ? -1.f : 1.f;
 }
 
+MINLINE int signum_i_ex(float a, float eps)
+{
+	if (a >  eps) return  1;
+	if (a < -eps) return -1;
+	else          return  0;
+}
+
+MINLINE int signum_i(float a)
+{
+	if (a > 0.0f) return  1;
+	if (a < 0.0f) return -1;
+	else          return  0;
+}
 
 #endif /* __MATH_BASE_INLINE_C__ */
diff --git a/source/blender/blenlib/intern/math_geom.c b/source/blender/blenlib/intern/math_geom.c
index 3a0187584ff..a360148182d 100644
--- a/source/blender/blenlib/intern/math_geom.c
+++ b/source/blender/blenlib/intern/math_geom.c
@@ -50,6 +50,21 @@ void cent_quad_v3(float cent[3], const float v1[3], const float v2[3], const flo
 	cent[2] = 0.25f * (v1[2] + v2[2] + v3[2] + v4[2]);
 }
 
+void cross_tri_v3(float n[3], const float v1[3], const float v2[3], const float v3[3])
+{
+	float n1[3], n2[3];
+
+	n1[0] = v1[0] - v2[0];
+	n2[0] = v2[0] - v3[0];
+	n1[1] = v1[1] - v2[1];
+	n2[1] = v2[1] - v3[1];
+	n1[2] = v1[2] - v2[2];
+	n2[2] = v2[2] - v3[2];
+	n[0] = n1[1] * n2[2] - n1[2] * n2[1];
+	n[1] = n1[2] * n2[0] - n1[0] * n2[2];
+	n[2] = n1[0] * n2[1] - n1[1] * n2[0];
+}
+
 float normal_tri_v3(float n[3], const float v1[3], const float v2[3], const float v3[3])
 {
 	float n1[3], n2[3];
diff --git a/source/blender/blenlib/intern/polyfill2d.c b/source/blender/blenlib/intern/polyfill2d.c
index 16cf7ff960b..3aadbe5f1df 100644
--- a/source/blender/blenlib/intern/polyfill2d.c
+++ b/source/blender/blenlib/intern/polyfill2d.c
@@ -165,7 +165,7 @@ static bool       pf_ear_tip_check(PolyFill *pf, PolyIndex *pi_ear_tip);
 static void       pf_ear_tip_cut(PolyFill *pf, PolyIndex *pi_ear_tip);
 
 
-BLI_INLINE eSign signum_i(float a)
+BLI_INLINE eSign signum_enum(float a)
 {
 	if (UNLIKELY(a == 0.0f))
 		return  0;
@@ -191,7 +191,7 @@ BLI_INLINE float area_tri_signed_v2_alt_2x(const float v1[2], const float v2[2],
 
 static eSign span_tri_v2_sign(const float v1[2], const float v2[2], const float v3[2])
 {
-	return signum_i(area_tri_signed_v2_alt_2x(v3, v2, v1));
+	return signum_enum(area_tri_signed_v2_alt_2x(v3, v2, v1));
 }
 
 
diff --git a/source/blender/blenlib/intern/polyfill2d_beautify.c b/source/blender/blenlib/intern/polyfill2d_beautify.c
index b8922ef6cb2..7914b7cb39b 100644
--- a/source/blender/blenlib/intern/polyfill2d_beautify.c
+++ b/source/blender/blenlib/intern/polyfill2d_beautify.c
@@ -123,7 +123,7 @@ BLI_INLINE bool is_boundary_edge(unsigned int i_a, unsigned int i_b, const unsig
  *
  * \return (negative number means the edge can be rotated, lager == better).
  */
-static float quad_v2_rotate_beauty_calc(
+float BLI_polyfill_beautify_quad_rotate_calc(
         const float v1[2], const float v2[2], const float v3[2], const float v4[2])
 {
 	/* not a loop (only to be able to break out) */
@@ -150,24 +150,16 @@ static float quad_v2_rotate_beauty_calc(
 			}
 		}
 
-		if (is_zero_a == false && is_zero_b == false) {
-			/* both tri's are valid, check we make a concave quad */
-			if (!is_quad_convex_v2(v1, v2, v3, v4)) {
-				break;
-			}
+		/* one of the tri's was degenerate, check we're not rotating
+		 * into a different degenerate shape or flipping the face */
+		if ((fabsf(area_2x_123) <= FLT_EPSILON) || (fabsf(area_2x_134) <= FLT_EPSILON)) {
+			/* one of the new rotations is degenerate */
+			break;
 		}
-		else {
-			/* one of the tri's was degenerate, chech we're not rotating
-			 * into a different degenerate shape or flipping the face */
-			if ((fabsf(area_2x_123) <= FLT_EPSILON) || (fabsf(area_2x_134) <= FLT_EPSILON)) {
-				/* one of the new rotations is degenerate */
-				break;
-			}
 
-			if ((area_2x_123 >= 0.0f) != (area_2x_134 >= 0.0f)) {
-				/* rotation would cause flipping */
-				break;
-			}
+		if ((area_2x_123 >= 0.0f) != (area_2x_134 >= 0.0f)) {
+			/* rotation would cause flipping */
+			break;
 		}
 
 		{
@@ -225,7 +217,7 @@ static float polyedge_rotate_beauty_calc(
 	v2 = coords[e->verts[0]];
 	v4 = coords[e->verts[1]];
 
-	return quad_v2_rotate_beauty_calc(v1, v2, v3, v4);
+	return BLI_polyfill_beautify_quad_rotate_calc(v1, v2, v3, v4);
 }
 
 static void polyedge_beauty_cost_update_single(