Fix Quaternion.rotate(matrix) with negative matrices

Rotating a quaternion by a negative matrix gave an invalid result.

Follow up fix for T94231 which negated negative matrices too.

1 files changed, 7 insertions, 2 deletions

diff --git a/source/blender/python/mathutils/mathutils_Quaternion.c b/source/blender/python/mathutils/mathutils_Quaternion.c
index 6994a313237..4972381d29e 100644
--- a/source/blender/python/mathutils/mathutils_Quaternion.c
+++ b/source/blender/python/mathutils/mathutils_Quaternion.c
@@ -543,8 +543,13 @@ static PyObject *Quaternion_rotate(QuaternionObject *self, PyObject *value)
   length = normalize_qt_qt(tquat, self->quat);
   quat_to_mat3(self_rmat, tquat);
   mul_m3_m3m3(rmat, other_rmat, self_rmat);
-
-  mat3_to_quat(self->quat, rmat);
+  normalize_m3(rmat);
+  /* This check could also be performed on `other_rmat`, use the final result instead to ensure
+   * float imprecision doesn't allow the multiplication to make `rmat` negative. */
+  if (is_negative_m3(rmat)) {
+    negate_m3(rmat);
+  }
+  mat3_normalized_to_quat(self->quat, rmat);
   mul_qt_fl(self->quat, length); /* maintain length after rotating */
 
   (void)BaseMath_WriteCallback(self);