Polyfill Beautify: option to rotate out of degenerate state

Needed for 3D iterative edge-rotation to avoid flipping when projected
from different angles,
but could keep zero area faces in 2D polygons.

3 files changed, 20 insertions, 10 deletions

diff --git a/source/blender/blenlib/BLI_polyfill2d_beautify.h b/source/blender/blenlib/BLI_polyfill2d_beautify.h
index 20e53b080fe..29a900200bb 100644
--- a/source/blender/blenlib/BLI_polyfill2d_beautify.h
+++ b/source/blender/blenlib/BLI_polyfill2d_beautify.h
@@ -33,8 +33,12 @@ void BLI_polyfill_beautify(
         /* structs for reuse */
         struct MemArena *arena, struct Heap *eheap, struct EdgeHash *eh);
 
-float BLI_polyfill_beautify_quad_rotate_calc(
-        const float v1[2], const float v2[2], const float v3[2], const float v4[2]);
+float BLI_polyfill_beautify_quad_rotate_calc_ex(
+        const float v1[2], const float v2[2], const float v3[2], const float v4[2],
+        const bool lock_degenerate);
+#define BLI_polyfill_beautify_quad_rotate_calc(v1, v2, v3, v4) \
+	BLI_polyfill_beautify_quad_rotate_calc_ex(v1, v2, v3, v4, false)
+
 
 /* avoid realloc's when creating new structures for polyfill ngons */
 #define BLI_POLYFILL_ALLOC_NGON_RESERVE 64
diff --git a/source/blender/blenlib/intern/polyfill2d_beautify.c b/source/blender/blenlib/intern/polyfill2d_beautify.c
index 896177f436c..de5f7f86bee 100644
--- a/source/blender/blenlib/intern/polyfill2d_beautify.c
+++ b/source/blender/blenlib/intern/polyfill2d_beautify.c
@@ -123,8 +123,9 @@ BLI_INLINE bool is_boundary_edge(unsigned int i_a, unsigned int i_b, const unsig
  *
  * \return (negative number means the edge can be rotated, lager == better).
  */
-float BLI_polyfill_beautify_quad_rotate_calc(
-        const float v1[2], const float v2[2], const float v3[2], const float v4[2])
+float BLI_polyfill_beautify_quad_rotate_calc_ex(
+        const float v1[2], const float v2[2], const float v3[2], const float v4[2],
+        const bool lock_degenerate)
 {
 	/* not a loop (only to be able to break out) */
 	do {
@@ -136,12 +137,12 @@ float BLI_polyfill_beautify_quad_rotate_calc(
 		const float area_2x_123 = cross_tri_v2(v1, v2, v3);
 		const float area_2x_134 = cross_tri_v2(v1, v3, v4);
 
-		{
-			BLI_assert((ELEM(v1, v2, v3, v4) == false) &&
-			           (ELEM(v2, v1, v3, v4) == false) &&
-			           (ELEM(v3, v1, v2, v4) == false) &&
-			           (ELEM(v4, v1, v2, v3) == false));
+		BLI_assert((ELEM(v1, v2, v3, v4) == false) &&
+		           (ELEM(v2, v1, v3, v4) == false) &&
+		           (ELEM(v3, v1, v2, v4) == false) &&
+		           (ELEM(v4, v1, v2, v3) == false));
 
+		if (lock_degenerate) {
 			is_zero_a = (fabsf(area_2x_234) <= FLT_EPSILON);
 			is_zero_b = (fabsf(area_2x_241) <= FLT_EPSILON);
 
diff --git a/source/blender/bmesh/tools/bmesh_beautify.c b/source/blender/bmesh/tools/bmesh_beautify.c
index f08f21a2c88..78e3e66b70a 100644
--- a/source/blender/bmesh/tools/bmesh_beautify.c
+++ b/source/blender/bmesh/tools/bmesh_beautify.c
@@ -182,7 +182,12 @@ static float bm_edge_calc_rotate_beauty__area(
 			}
 		}
 
-		return BLI_polyfill_beautify_quad_rotate_calc(v1_xy, v2_xy, v3_xy, v4_xy);
+		/**
+		 * Important to lock degenerate here,
+		 * since the triangle pars will be projected into different 2D spaces.
+		 * Allowing to rotate out of a degenerate state can flip the faces (when performed iteratively).
+		 */
+		return BLI_polyfill_beautify_quad_rotate_calc_ex(v1_xy, v2_xy, v3_xy, v4_xy, true);
 	} while (false);
 
 	return FLT_MAX;