Fix T94231: Matrix.to_quaternion() returns invalid rotation

The result of mat3_normalized_to_quat isn't valid for negative matrices.

Isolate the fix to the Matrix.to_quaternion() instead of changing
mat3_normalized_to_quat to prevent unintended side effects elsewhere.

1 files changed, 9 insertions, 3 deletions

diff --git a/source/blender/python/mathutils/mathutils_Matrix.c b/source/blender/python/mathutils/mathutils_Matrix.c
index 1e85ece124d..de42b11c70b 100644
--- a/source/blender/python/mathutils/mathutils_Matrix.c
+++ b/source/blender/python/mathutils/mathutils_Matrix.c
@@ -1243,13 +1243,19 @@ static PyObject *Matrix_to_quaternion(MatrixObject *self)
                     "inappropriate matrix size - expects 3x3 or 4x4 matrix");
     return NULL;
   }
+  float mat3[3][3];
   if (self->row_num == 3) {
-    mat3_to_quat(quat, (float(*)[3])self->matrix);
+    copy_m3_m3(mat3, (const float(*)[3])self->matrix);
   }
   else {
-    mat4_to_quat(quat, (const float(*)[4])self->matrix);
+    copy_m3_m4(mat3, (const float(*)[4])self->matrix);
   }
-
+  normalize_m3(mat3);
+  if (is_negative_m3(mat3)) {
+    /* Without this, the results are invalid, see: T94231. */
+    negate_m3(mat3);
+  }
+  mat3_normalized_to_quat(quat, mat3);
   return Quaternion_CreatePyObject(quat, NULL);
 }