BLI_math: ensure non-negative matrices for mat3_to_quat calculations

Making the callers responsible for this isn't practical as matrices are
often passed indirectly to a functions such as mat3_to_axis_angle,
BKE_object_mat3_to_rot & BKE_pchan_mat3_to_rot.
Or the matrix is combined from other matrices which could be negative.

Given quaternions calculated from negative matrices are completely
invalid and checking only needs to negate matrices with a negative
determinant, move the check into mat3_to_quat and related functions.

Add mat3_normalized_to_quat_fast for cases no error checking on the
input matrix is needed such as blending rotations.

2 files changed, 4 insertions, 17 deletions

diff --git a/source/blender/python/mathutils/mathutils_Matrix.c b/source/blender/python/mathutils/mathutils_Matrix.c
index de42b11c70b..8405b966a4e 100644
--- a/source/blender/python/mathutils/mathutils_Matrix.c
+++ b/source/blender/python/mathutils/mathutils_Matrix.c
@@ -1243,19 +1243,12 @@ static PyObject *Matrix_to_quaternion(MatrixObject *self)
                     "inappropriate matrix size - expects 3x3 or 4x4 matrix");
     return NULL;
   }
-  float mat3[3][3];
   if (self->row_num == 3) {
-    copy_m3_m3(mat3, (const float(*)[3])self->matrix);
+    mat3_to_quat(quat, (const float(*)[3])self->matrix);
   }
   else {
-    copy_m3_m4(mat3, (const float(*)[4])self->matrix);
+    mat4_to_quat(quat, (const float(*)[4])self->matrix);
   }
-  normalize_m3(mat3);
-  if (is_negative_m3(mat3)) {
-    /* Without this, the results are invalid, see: T94231. */
-    negate_m3(mat3);
-  }
-  mat3_normalized_to_quat(quat, mat3);
   return Quaternion_CreatePyObject(quat, NULL);
 }
 
@@ -1894,7 +1887,7 @@ static PyObject *Matrix_decompose(MatrixObject *self)
   }
 
   mat4_to_loc_rot_size(loc, rot, size, (const float(*)[4])self->matrix);
-  mat3_to_quat(quat, rot);
+  mat3_normalized_to_quat_fast(quat, rot);
 
   ret = PyTuple_New(3);
   PyTuple_SET_ITEMS(ret,
diff --git a/source/blender/python/mathutils/mathutils_Quaternion.c b/source/blender/python/mathutils/mathutils_Quaternion.c
index 4972381d29e..a5ea09bef48 100644
--- a/source/blender/python/mathutils/mathutils_Quaternion.c
+++ b/source/blender/python/mathutils/mathutils_Quaternion.c
@@ -543,13 +543,7 @@ static PyObject *Quaternion_rotate(QuaternionObject *self, PyObject *value)
   length = normalize_qt_qt(tquat, self->quat);
   quat_to_mat3(self_rmat, tquat);
   mul_m3_m3m3(rmat, other_rmat, self_rmat);
-  normalize_m3(rmat);
-  /* This check could also be performed on `other_rmat`, use the final result instead to ensure
-   * float imprecision doesn't allow the multiplication to make `rmat` negative. */
-  if (is_negative_m3(rmat)) {
-    negate_m3(rmat);
-  }
-  mat3_normalized_to_quat(self->quat, rmat);
+  mat3_to_quat(self->quat, rmat);
   mul_qt_fl(self->quat, length); /* maintain length after rotating */
 
   (void)BaseMath_WriteCallback(self);