Fix T79482: Triangulate quads with 'Beauty' can make zero area faces

3 files changed, 13 insertions, 4 deletions

diff --git a/source/blender/bmesh/operators/bmo_beautify.c b/source/blender/bmesh/operators/bmo_beautify.c
index 36122e06e9b..de26ca5ebd2 100644
--- a/source/blender/bmesh/operators/bmo_beautify.c
+++ b/source/blender/bmesh/operators/bmo_beautify.c
@@ -39,7 +39,10 @@ void bmo_beautify_fill_exec(BMesh *bm, BMOperator *op)
   BMFace *f;
   BMEdge *e;
   const bool use_restrict_tag = BMO_slot_bool_get(op->slots_in, "use_restrict_tag");
-  const short flag = (use_restrict_tag ? VERT_RESTRICT_TAG : 0);
+  const short flag =
+      ((use_restrict_tag ? VERT_RESTRICT_TAG : 0) |
+       /* Enable to avoid iterative edge rotation to cause the direction of faces to flip. */
+       EDGE_RESTRICT_DEGENERATE);
   const short method = (short)BMO_slot_int_get(op->slots_in, "method");
 
   BMEdge **edge_array;
diff --git a/source/blender/bmesh/tools/bmesh_beautify.c b/source/blender/bmesh/tools/bmesh_beautify.c
index c877c534376..a25e4666a22 100644
--- a/source/blender/bmesh/tools/bmesh_beautify.c
+++ b/source/blender/bmesh/tools/bmesh_beautify.c
@@ -141,7 +141,8 @@ static void erot_state_alternate(const BMEdge *e, EdRotState *e_state)
 static float bm_edge_calc_rotate_beauty__area(const float v1[3],
                                               const float v2[3],
                                               const float v3[3],
-                                              const float v4[3])
+                                              const float v4[3],
+                                              const bool lock_degenerate)
 {
   /* not a loop (only to be able to break out) */
   do {
@@ -199,7 +200,8 @@ static float bm_edge_calc_rotate_beauty__area(const float v1[3],
      * Allowing to rotate out of a degenerate state can flip the faces
      * (when performed iteratively).
      */
-    return BLI_polyfill_beautify_quad_rotate_calc_ex(v1_xy, v2_xy, v3_xy, v4_xy, true, NULL);
+    return BLI_polyfill_beautify_quad_rotate_calc_ex(
+        v1_xy, v2_xy, v3_xy, v4_xy, lock_degenerate, NULL);
   } while (false);
 
   return FLT_MAX;
@@ -262,7 +264,8 @@ float BM_verts_calc_rotate_beauty(const BMVert *v1,
 
     switch (method) {
       case 0:
-        return bm_edge_calc_rotate_beauty__area(v1->co, v2->co, v3->co, v4->co);
+        return bm_edge_calc_rotate_beauty__area(
+            v1->co, v2->co, v3->co, v4->co, flag & EDGE_RESTRICT_DEGENERATE);
       default:
         return bm_edge_calc_rotate_beauty__angle(v1->co, v2->co, v3->co, v4->co);
     }
diff --git a/source/blender/bmesh/tools/bmesh_beautify.h b/source/blender/bmesh/tools/bmesh_beautify.h
index f957f0d3560..d2c67693f73 100644
--- a/source/blender/bmesh/tools/bmesh_beautify.h
+++ b/source/blender/bmesh/tools/bmesh_beautify.h
@@ -22,7 +22,10 @@
  */
 
 enum {
+  /** Vertices tags must match (special case). */
   VERT_RESTRICT_TAG = (1 << 0),
+  /** Don't rotate out of degenerate state (needed for iterative rotation). */
+  EDGE_RESTRICT_DEGENERATE = (1 << 1),
 };
 
 void BM_mesh_beautify_fill(BMesh *bm,