Fix T38858: Crazy spaces was unpredictable with co-linear edges

3 files changed, 24 insertions, 12 deletions

diff --git a/source/blender/blenlib/BLI_math_rotation.h b/source/blender/blenlib/BLI_math_rotation.h
index c9f553c6fa5..54bfef4bd14 100644
--- a/source/blender/blenlib/BLI_math_rotation.h
+++ b/source/blender/blenlib/BLI_math_rotation.h
@@ -80,8 +80,8 @@ void mat3_to_quat(float q[4], float mat[3][3]);
 void mat4_to_quat(float q[4], float mat[4][4]);
 void tri_to_quat_ex(float quat[4], const float v1[3], const float v2[3], const float v3[3],
                     const float no_orig[3]);
-void tri_to_quat(float q[4], const float a[3], const float b[3], const float c[3]);
-void vec_to_quat(float q[4], const float vec[3], short axis, const short upflag);
+float tri_to_quat(float q[4], const float a[3], const float b[3], const float c[3]);
+void  vec_to_quat(float q[4], const float vec[3], short axis, const short upflag);
 /* note: v1 and v2 must be normalized */
 void rotation_between_vecs_to_quat(float q[4], const float v1[3], const float v2[3]);
 void rotation_between_quats_to_quat(float q[4], const float q1[4], const float q2[4]);
diff --git a/source/blender/blenlib/intern/math_rotation.c b/source/blender/blenlib/intern/math_rotation.c
index 0392598769f..c7c0626019f 100644
--- a/source/blender/blenlib/intern/math_rotation.c
+++ b/source/blender/blenlib/intern/math_rotation.c
@@ -663,11 +663,17 @@ void tri_to_quat_ex(float quat[4], const float v1[3], const float v2[3], const f
 	mul_qt_qtqt(quat, q1, q2);
 }
 
-void tri_to_quat(float quat[4], const float v1[3], const float v2[3], const float v3[3])
+/**
+ * \return the length of the normal, use to test for degenerate triangles.
+ */
+float tri_to_quat(float quat[4], const float v1[3], const float v2[3], const float v3[3])
 {
 	float vec[3];
-	normal_tri_v3(vec, v1, v2, v3);
+	float len;
+
+	len = normal_tri_v3(vec, v1, v2, v3);
 	tri_to_quat_ex(quat, v1, v2, v3, vec);
+	return len;
 }
 
 void print_qt(const char *str, const float q[4])
diff --git a/source/blender/editors/util/crazyspace.c b/source/blender/editors/util/crazyspace.c
index 399b0f86d5f..67b805943bf 100644
--- a/source/blender/editors/util/crazyspace.c
+++ b/source/blender/editors/util/crazyspace.c
@@ -62,21 +62,27 @@ BLI_INLINE void tan_calc_v3(float a[3], const float b[3], const float c[3])
 	a[2] = b[2] + 0.2f * (b[2] - c[2]);
 }
 
+BLI_INLINE void tan_calc_quat_v3(
+        float r_quat[4],
+        const float co_1[3], const float co_2[3], const float co_3[3])
+{
+	float vec_u[3], vec_v[3];
+	tan_calc_v3(vec_u, co_1, co_2);
+	tan_calc_v3(vec_v, co_1, co_3);
+	if (tri_to_quat(r_quat, co_1, vec_u, vec_v) < FLT_EPSILON) {
+		unit_qt(r_quat);
+	}
+}
+
 static void set_crazy_vertex_quat(
         float r_quat[4],
         const float co_1[3], const float co_2[3], const float co_3[3],
         const float vd_1[3], const float vd_2[3], const float vd_3[3])
 {
-	float vec_u[3], vec_v[3];
 	float q1[4], q2[4];
 
-	tan_calc_v3(vec_u, co_1, co_2);
-	tan_calc_v3(vec_v, co_1, co_3);
-	tri_to_quat(q1, co_1, vec_u, vec_v);
-
-	tan_calc_v3(vec_u, vd_1, vd_2);
-	tan_calc_v3(vec_v, vd_1, vd_3);
-	tri_to_quat(q2, vd_1, vec_u, vec_v);
+	tan_calc_quat_v3(q1, co_1, co_2, co_3);
+	tan_calc_quat_v3(q2, vd_1, vd_2, vd_3);
 
 	sub_qt_qtqt(r_quat, q2, q1);
 }