Fix T43711: dual quaternion deform bug with shearing in deform matrix.

This also increases the tolerances in is_orthogonal / is_orthonormal functions,
which were much too low for practical purposes.

2 files changed, 8 insertions, 6 deletions

diff --git a/source/blender/blenlib/intern/math_matrix.c b/source/blender/blenlib/intern/math_matrix.c
index 773bb419c9b..081c555ed6f 100644
--- a/source/blender/blenlib/intern/math_matrix.c
+++ b/source/blender/blenlib/intern/math_matrix.c
@@ -1159,7 +1159,7 @@ bool is_orthogonal_m3(float m[3][3])
 
 	for (i = 0; i < 3; i++) {
 		for (j = 0; j < i; j++) {
-			if (fabsf(dot_v3v3(m[i], m[j])) > 1.5f * FLT_EPSILON)
+			if (fabsf(dot_v3v3(m[i], m[j])) > 1e-5f)
 				return false;
 		}
 	}
@@ -1173,7 +1173,7 @@ bool is_orthogonal_m4(float m[4][4])
 
 	for (i = 0; i < 4; i++) {
 		for (j = 0; j < i; j++) {
-			if (fabsf(dot_v4v4(m[i], m[j])) > 1.5f * FLT_EPSILON)
+			if (fabsf(dot_v4v4(m[i], m[j])) > 1e-5f)
 				return false;
 		}
 
@@ -1188,7 +1188,7 @@ bool is_orthonormal_m3(float m[3][3])
 		int i;
 
 		for (i = 0; i < 3; i++)
-			if (fabsf(dot_v3v3(m[i], m[i]) - 1) > 1.5f * FLT_EPSILON)
+			if (fabsf(dot_v3v3(m[i], m[i]) - 1) > 1e-5f)
 				return false;
 
 		return true;
@@ -1203,7 +1203,7 @@ bool is_orthonormal_m4(float m[4][4])
 		int i;
 
 		for (i = 0; i < 4; i++)
-			if (fabsf(dot_v4v4(m[i], m[i]) - 1) > 1.5f * FLT_EPSILON)
+			if (fabsf(dot_v4v4(m[i], m[i]) - 1) > 1e-5f)
 				return false;
 
 		return true;
diff --git a/source/blender/blenlib/intern/math_rotation.c b/source/blender/blenlib/intern/math_rotation.c
index 1f0d68b09b2..5f039e89e89 100644
--- a/source/blender/blenlib/intern/math_rotation.c
+++ b/source/blender/blenlib/intern/math_rotation.c
@@ -1616,7 +1616,7 @@ void eulO_to_gimbal_axis(float gmat[3][3], const float eul[3], const short order
 
 void mat4_to_dquat(DualQuat *dq, float basemat[4][4], float mat[4][4])
 {
-	float *t, *q, dscale[3], scale[3], basequat[4];
+	float *t, *q, dscale[3], scale[3], basequat[4], mat3[3][3];
 	float baseRS[4][4], baseinv[4][4], baseR[4][4], baseRinv[4][4];
 	float R[4][4], S[4][4];
 
@@ -1629,7 +1629,9 @@ void mat4_to_dquat(DualQuat *dq, float basemat[4][4], float mat[4][4])
 	dscale[1] = scale[1] - 1.0f;
 	dscale[2] = scale[2] - 1.0f;
 
-	if ((determinant_m4(mat) < 0.0f) || len_v3(dscale) > 1e-4f) {
+	copy_m3_m4(mat3, mat);
+
+	if (!is_orthonormal_m3(mat3) || (determinant_m4(mat) < 0.0f) || len_v3(dscale) > 1e-4f) {
 		/* extract R and S  */
 		float tmp[4][4];