From 10a4726a5b770a6c916c0ee444808d86c63802e2 Mon Sep 17 00:00:00 2001 From: Campbell Barton Date: Wed, 24 Aug 2022 15:53:17 +1000 Subject: Fix T94231: Matrix.to_quaternion() returns invalid rotation The result of mat3_normalized_to_quat isn't valid for negative matrices. Isolate the fix to the Matrix.to_quaternion() instead of changing mat3_normalized_to_quat to prevent unintended side effects elsewhere. --- source/blender/python/mathutils/mathutils_Matrix.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/source/blender/python/mathutils/mathutils_Matrix.c b/source/blender/python/mathutils/mathutils_Matrix.c index 1e85ece124d..de42b11c70b 100644 --- a/source/blender/python/mathutils/mathutils_Matrix.c +++ b/source/blender/python/mathutils/mathutils_Matrix.c @@ -1243,13 +1243,19 @@ static PyObject *Matrix_to_quaternion(MatrixObject *self) "inappropriate matrix size - expects 3x3 or 4x4 matrix"); return NULL; } + float mat3[3][3]; if (self->row_num == 3) { - mat3_to_quat(quat, (float(*)[3])self->matrix); + copy_m3_m3(mat3, (const float(*)[3])self->matrix); } else { - mat4_to_quat(quat, (const float(*)[4])self->matrix); + copy_m3_m4(mat3, (const float(*)[4])self->matrix); } - + normalize_m3(mat3); + if (is_negative_m3(mat3)) { + /* Without this, the results are invalid, see: T94231. */ + negate_m3(mat3); + } + mat3_normalized_to_quat(quat, mat3); return Quaternion_CreatePyObject(quat, NULL); } -- cgit v1.2.3