Mesh: concave quad support

Previously quads always split along first-third vertices.
This is still the default, to avoid flickering with animated deformation
however concave quads that would create two opposing triangles now use
second-fourth split.

Reported as T53999 although this issue has been known limitation
for a long time.

4 files changed, 37 insertions, 0 deletions

diff --git a/source/blender/blenkernel/intern/mesh_evaluate.c b/source/blender/blenkernel/intern/mesh_evaluate.c
index 425f28eb123..b0c09979de9 100644
--- a/source/blender/blenkernel/intern/mesh_evaluate.c
+++ b/source/blender/blenkernel/intern/mesh_evaluate.c
@@ -2945,9 +2945,22 @@ void BKE_mesh_recalc_looptri(
 		}
 		else if (mp_totloop == 4) {
 			ML_TO_MLT(0, 1, 2);
+			MLoopTri *mlt_a = mlt;
 			mlooptri_index++;
 			ML_TO_MLT(0, 2, 3);
+			MLoopTri *mlt_b = mlt;
 			mlooptri_index++;
+
+			if (UNLIKELY(is_quad_flip_v3_first_third_fast(
+			                     mvert[mloop[mlt_a->tri[0]].v].co,
+			                     mvert[mloop[mlt_a->tri[1]].v].co,
+			                     mvert[mloop[mlt_a->tri[2]].v].co,
+			                     mvert[mloop[mlt_b->tri[2]].v].co)))
+			{
+				/* flip out of degenerate 0-2 state. */
+				mlt_a->tri[2] = mlt_b->tri[2];
+				mlt_b->tri[0] = mlt_a->tri[1];
+			}
 		}
 #endif /* USE_TESSFACE_SPEEDUP */
 		else {
diff --git a/source/blender/blenlib/BLI_math_geom.h b/source/blender/blenlib/BLI_math_geom.h
index 933e31ba84b..ffe0ce11cef 100644
--- a/source/blender/blenlib/BLI_math_geom.h
+++ b/source/blender/blenlib/BLI_math_geom.h
@@ -85,6 +85,7 @@ bool is_quad_convex_v3(const float v1[3], const float v2[3], const float v3[3],
 bool is_quad_convex_v2(const float v1[2], const float v2[2], const float v3[2], const float v4[2]);
 bool is_poly_convex_v2(const float verts[][2], unsigned int nr);
 int  is_quad_flip_v3(const float v1[3], const float v2[3], const float v3[3], const float v4[3]);
+bool is_quad_flip_v3_first_third_fast(const float v0[3], const float v1[3], const float v2[3], const float v3[3]);
 
 /********************************* Distance **********************************/
 
diff --git a/source/blender/blenlib/intern/math_geom.c b/source/blender/blenlib/intern/math_geom.c
index d3080e5530f..a3d850f9551 100644
--- a/source/blender/blenlib/intern/math_geom.c
+++ b/source/blender/blenlib/intern/math_geom.c
@@ -4903,6 +4903,18 @@ int is_quad_flip_v3(const float v1[3], const float v2[3], const float v3[3], con
 	return ret;
 }
 
+bool is_quad_flip_v3_first_third_fast(const float v1[3], const float v2[3], const float v3[3], const float v4[3])
+{
+	float d_12[3], d_13[3], d_14[3];
+	float cross_a[3], cross_b[3];
+	sub_v3_v3v3(d_12, v2, v1);
+	sub_v3_v3v3(d_13, v3, v1);
+	sub_v3_v3v3(d_14, v4, v1);
+	cross_v3_v3v3(cross_a, d_12, d_13);
+	cross_v3_v3v3(cross_b, d_14, d_13);
+	return dot_v3v3(cross_a, cross_b) > 0.0f;
+}
+
 /**
  * Return the value which the distance between points will need to be scaled by,
  * to define a handle, given both points are on a perfect circle.
diff --git a/source/blender/bmesh/intern/bmesh_polygon.c b/source/blender/bmesh/intern/bmesh_polygon.c
index 66fcd739839..66d0a83eb9f 100644
--- a/source/blender/bmesh/intern/bmesh_polygon.c
+++ b/source/blender/bmesh/intern/bmesh_polygon.c
@@ -1426,6 +1426,17 @@ void BM_mesh_calc_tessellation(BMesh *bm, BMLoop *(*looptris)[3], int *r_looptri
 			(l_ptr_a[2] = l_ptr_b[1] = l = l->next);
 			(             l_ptr_b[2] = l->next);
 #endif
+
+			if (UNLIKELY(is_quad_flip_v3_first_third_fast(
+			                     l_ptr_a[0]->v->co,
+			                     l_ptr_a[1]->v->co,
+			                     l_ptr_a[2]->v->co,
+			                     l_ptr_b[2]->v->co)))
+			{
+				/* flip out of degenerate 0-2 state. */
+				l_ptr_a[2] = l_ptr_b[2];
+				l_ptr_b[0] = l_ptr_a[1];
+			}
 		}
 
 #endif /* USE_TESSFACE_SPEEDUP */