Fix T92867: Gimbal rotation broken when used for multiple objects

Support gimbal orientation for objects & bones.

7 files changed, 68 insertions, 19 deletions

diff --git a/source/blender/editors/transform/transform.h b/source/blender/editors/transform/transform.h
index 8fe3b51620d..e13e7c3f93a 100644
--- a/source/blender/editors/transform/transform.h
+++ b/source/blender/editors/transform/transform.h
@@ -621,6 +621,12 @@ typedef struct TransInfo {
     O_SET,
   } orient_curr;
 
+  /**
+   * All values from `TransInfo.orient[].type` converted into a flag
+   * to allow quickly checking which orientation types are used.
+   */
+  int orient_type_mask;
+
   short prop_mode;
 
   /** Value taken as input, either through mouse coordinates or entered as a parameter. */
diff --git a/source/blender/editors/transform/transform_constraints.c b/source/blender/editors/transform/transform_constraints.c
index 23ba335476c..a24491119c6 100644
--- a/source/blender/editors/transform/transform_constraints.c
+++ b/source/blender/editors/transform/transform_constraints.c
@@ -384,6 +384,29 @@ static void planeProjection(const TransInfo *t, const float in[3], float out[3])
   add_v3_v3v3(out, in, vec);
 }
 
+static short transform_orientation_or_default(const TransInfo *t)
+{
+  short orientation = t->orient[t->orient_curr].type;
+  if (orientation == V3D_ORIENT_CUSTOM_MATRIX) {
+    /* Use the real value of the "orient_type". */
+    orientation = t->orient[O_DEFAULT].type;
+  }
+  return orientation;
+}
+
+static const float (*transform_object_axismtx_get(const TransInfo *t,
+                                                  const TransDataContainer *UNUSED(tc),
+                                                  const TransData *td))[3]
+{
+  if (transform_orientation_or_default(t) == V3D_ORIENT_GIMBAL) {
+    BLI_assert(t->orient_type_mask & (1 << V3D_ORIENT_GIMBAL));
+    if (t->options & (CTX_POSE_BONE | CTX_OBJECT)) {
+      return td->ext->axismtx_gimbal;
+    }
+  }
+  return td->axismtx;
+}
+
 /**
  * Generic callback for constant spatial constraints applied to linear motion
  *
@@ -489,7 +512,8 @@ static void applyObjectConstraintVec(const TransInfo *t,
     copy_v3_v3(out, in);
     if (t->con.mode & CON_APPLY) {
       mul_m3_v3(t->spacemtx_inv, out);
-      mul_m3_v3(td->axismtx, out);
+      const float(*axismtx)[3] = transform_object_axismtx_get(t, tc, td);
+      mul_m3_v3(axismtx, out);
       if (t->flag & T_EDIT) {
         mul_m3_v3(tc->mat3_unit, out);
       }
@@ -535,7 +559,8 @@ static void applyObjectConstraintSize(const TransInfo *t,
     float tmat[3][3];
     float imat[3][3];
 
-    invert_m3_m3(imat, td->axismtx);
+    const float(*axismtx)[3] = transform_object_axismtx_get(t, tc, td);
+    invert_m3_m3(imat, axismtx);
 
     if (!(t->con.mode & CON_AXIS0)) {
       r_smat[0][0] = 1.0f;
@@ -551,7 +576,7 @@ static void applyObjectConstraintSize(const TransInfo *t,
     if (t->flag & T_EDIT) {
       mul_m3_m3m3(r_smat, tc->mat3_unit, r_smat);
     }
-    mul_m3_m3m3(r_smat, td->axismtx, tmat);
+    mul_m3_m3m3(r_smat, axismtx, tmat);
   }
 }
 
@@ -647,7 +672,7 @@ static void applyObjectConstraintRot(const TransInfo *t,
       axismtx = tmp_axismtx;
     }
     else {
-      axismtx = td->axismtx;
+      axismtx = transform_object_axismtx_get(t, tc, td);
     }
 
     constraints_rotation_impl(t, axismtx, r_axis, r_angle);
@@ -712,17 +737,13 @@ void setLocalConstraint(TransInfo *t, int mode, const char text[])
 void setUserConstraint(TransInfo *t, int mode, const char ftext[])
 {
   char text[256];
-  short orientation = t->orient[t->orient_curr].type;
-  if (orientation == V3D_ORIENT_CUSTOM_MATRIX) {
-    /* Use the real value of the "orient_type". */
-    orientation = t->orient[0].type;
-  }
-
+  const short orientation = transform_orientation_or_default(t);
   const char *spacename = transform_orientations_spacename_get(t, orientation);
   BLI_snprintf(text, sizeof(text), ftext, spacename);
 
   switch (orientation) {
     case V3D_ORIENT_LOCAL:
+    case V3D_ORIENT_GIMBAL:
       setLocalConstraint(t, mode, text);
       break;
     case V3D_ORIENT_NORMAL:
@@ -734,7 +755,6 @@ void setUserConstraint(TransInfo *t, int mode, const char ftext[])
     case V3D_ORIENT_GLOBAL:
     case V3D_ORIENT_VIEW:
     case V3D_ORIENT_CURSOR:
-    case V3D_ORIENT_GIMBAL:
     case V3D_ORIENT_CUSTOM_MATRIX:
     case V3D_ORIENT_CUSTOM:
     default: {
@@ -905,7 +925,7 @@ static void drawObjectConstraint(TransInfo *t)
     TransData *td = tc->data;
     for (int i = 0; i < tc->data_len; i++, td++) {
       float co[3];
-      float(*axismtx)[3];
+      const float(*axismtx)[3];
 
       if (t->flag & T_PROP_EDIT) {
         /* we're sorted, so skip the rest */
@@ -937,13 +957,14 @@ static void drawObjectConstraint(TransInfo *t)
         mul_m3_m3m3(tmp_axismtx, tc->mat3_unit, td->axismtx);
         axismtx = tmp_axismtx;
       }
-      else if (t->options & CTX_POSE_BONE) {
-        mul_v3_m4v3(co, tc->mat, td->center);
-        axismtx = td->axismtx;
-      }
       else {
-        copy_v3_v3(co, td->center);
-        axismtx = td->axismtx;
+        if (t->options & CTX_POSE_BONE) {
+          mul_v3_m4v3(co, tc->mat, td->center);
+        }
+        else {
+          copy_v3_v3(co, td->center);
+        }
+        axismtx = transform_object_axismtx_get(t, tc, td);
       }
 
       if (t->con.mode & CON_AXIS0) {
diff --git a/source/blender/editors/transform/transform_convert_armature.c b/source/blender/editors/transform/transform_convert_armature.c
index 1bbbbc6294e..88790e9645c 100644
--- a/source/blender/editors/transform/transform_convert_armature.c
+++ b/source/blender/editors/transform/transform_convert_armature.c
@@ -662,6 +662,12 @@ static void add_pose_transdata(TransInfo *t, bPoseChannel *pchan, Object *ob, Tr
   mul_m3_m3m3(td->axismtx, omat, pmat);
   normalize_m3(td->axismtx);
 
+  if (t->orient_type_mask & (1 << V3D_ORIENT_GIMBAL)) {
+    if (!gimbal_axis_pose(ob, pchan, td->ext->axismtx_gimbal)) {
+      copy_m3_m3(td->ext->axismtx_gimbal, td->axismtx);
+    }
+  }
+
   if (t->mode == TFM_BONE_ENVELOPE_DIST) {
     td->loc = NULL;
     td->val = &bone->dist;
diff --git a/source/blender/editors/transform/transform_convert_object.c b/source/blender/editors/transform/transform_convert_object.c
index 7e4e0892420..4a8ebf3fc6e 100644
--- a/source/blender/editors/transform/transform_convert_object.c
+++ b/source/blender/editors/transform/transform_convert_object.c
@@ -181,6 +181,11 @@ static void ObjectToTransData(TransInfo *t, TransData *td, Object *ob)
 
   /* axismtx has the real orientation */
   transform_orientations_create_from_axis(td->axismtx, UNPACK3(ob->obmat));
+  if (t->orient_type_mask & (1 << V3D_ORIENT_GIMBAL)) {
+    if (!gimbal_axis_object(ob, td->ext->axismtx_gimbal)) {
+      copy_m3_m3(td->ext->axismtx_gimbal, td->axismtx);
+    }
+  }
 
   td->con = ob->constraints.first;
 
diff --git a/source/blender/editors/transform/transform_data.h b/source/blender/editors/transform/transform_data.h
index 15e40ec466b..6cfceedb389 100644
--- a/source/blender/editors/transform/transform_data.h
+++ b/source/blender/editors/transform/transform_data.h
@@ -85,6 +85,8 @@ typedef struct TransDataExtension {
   float isize[3];
   /** Object matrix. */
   float obmat[4][4];
+  /** Use for #V3D_ORIENT_GIMBAL orientation. */
+  float axismtx_gimbal[3][3];
   /** Use instead of #TransData.smtx,
    * It is the same but without the #Bone.bone_mat, see #TD_PBONE_LOCAL_MTX_C. */
   float l_smtx[3][3];
diff --git a/source/blender/editors/transform/transform_generics.c b/source/blender/editors/transform/transform_generics.c
index 8effb82173b..7ff95ebeeae 100644
--- a/source/blender/editors/transform/transform_generics.c
+++ b/source/blender/editors/transform/transform_generics.c
@@ -512,6 +512,15 @@ void initTransInfo(bContext *C, TransInfo *t, wmOperator *op, const wmEvent *eve
       }
     }
 
+    t->orient_type_mask = 0;
+    for (int i = 0; i < 3; i++) {
+      const int type = t->orient[i].type;
+      if (type < V3D_ORIENT_CUSTOM_MATRIX) {
+        BLI_assert(type < 32);
+        t->orient_type_mask |= (1 << type);
+      }
+    }
+
     transform_orientations_current_set(t, (t->con.mode & CON_APPLY) ? 2 : 0);
   }
 
diff --git a/source/blender/editors/transform/transform_gizmo_3d.c b/source/blender/editors/transform/transform_gizmo_3d.c
index b40412a0845..c96852cd094 100644
--- a/source/blender/editors/transform/transform_gizmo_3d.c
+++ b/source/blender/editors/transform/transform_gizmo_3d.c
@@ -515,7 +515,7 @@ static void protectflag_to_drawflags_pchan(RegionView3D *rv3d,
 {
   /* Protect-flags apply to local space in pose mode, so only let them influence axis
    * visibility if we show the global orientation, otherwise it's confusing. */
-  if (orientation_index == V3D_ORIENT_LOCAL) {
+  if (ELEM(orientation_index, V3D_ORIENT_LOCAL, V3D_ORIENT_GIMBAL)) {
     protectflag_to_drawflags(pchan->protectflag, &rv3d->twdrawflag);
   }
 }