Fix T38843: Bone parallel to world z axis flicking when scale in edit mode.

This commit hopefully fixes all glitches we had when bone was Z-aligned. Note that when you init a transform
with a Z-aligned bone and change it to be non-Z-aligned, you will still get some brutal roll change,
there is not much things we can do here afaik...

3 files changed, 37 insertions, 9 deletions

diff --git a/source/blender/editors/transform/transform.h b/source/blender/editors/transform/transform.h
index 5cac49aff34..f262bac9762 100644
--- a/source/blender/editors/transform/transform.h
+++ b/source/blender/editors/transform/transform.h
@@ -257,7 +257,8 @@ typedef struct TransData {
 	float *loc;          /* Location of the data to transform                                              */
 	float  iloc[3];      /* Initial location                                                               */
 	float *val;          /* Value pointer for special transforms */
-	float  ival;         /* Old value*/
+	float  ival;         /* Old value */
+	float  ival2;        /* Another old value (for bone roll we need two different "old values" :/ ). */
 	float  center[3];	 /* Individual data center                                                         */
 	float  mtx[3][3];    /* Transformation matrix from data space to global space                          */
 	float  smtx[3][3];   /* Transformation matrix from global space to data space                          */
diff --git a/source/blender/editors/transform/transform_conversions.c b/source/blender/editors/transform/transform_conversions.c
index 14f8fa0ad71..0e2bdc11e47 100644
--- a/source/blender/editors/transform/transform_conversions.c
+++ b/source/blender/editors/transform/transform_conversions.c
@@ -1061,6 +1061,27 @@ static void createTransPose(TransInfo *t, Object *ob)
 
 /* ********************* armature ************** */
 
+static void createTransArmatureVerts_init_roll_fix(TransData *td, EditBone *ebo)
+{
+	/* To fix roll, see comments in transform_generic.c::recalcData_objects() */
+	const float z_axis[3] = {0.0f, 0.0f, 1.0f};
+	float vec[3];
+
+	sub_v3_v3v3(vec, ebo->tail, ebo->head);
+	normalize_v3(vec);
+
+	td->extra = ebo;
+
+	if (fabsf(dot_v3v3(vec, z_axis)) > 0.999999f) {
+		/* If nearly aligned with Z axis, do not alter roll. See T38843. */
+		td->ival = ebo->roll;
+	}
+	else {
+		td->ival = ebo->roll - ED_rollBoneToVector(ebo, z_axis, false);
+	}
+	td->ival2 = ebo->roll;
+}
+
 static void createTransArmatureVerts(TransInfo *t)
 {
 	EditBone *ebo;
@@ -1182,7 +1203,6 @@ static void createTransArmatureVerts(TransInfo *t)
 				}
 			}
 			else {
-				const float z_axis[3] = {0.0f, 0.0f, 1.0f};
 				if (ebo->flag & BONE_TIPSEL) {
 					copy_v3_v3(td->iloc, ebo->tail);
 					copy_v3_v3(td->center, (t->around == V3D_LOCAL) ? ebo->head : td->iloc);
@@ -1197,9 +1217,7 @@ static void createTransArmatureVerts(TransInfo *t)
 					ED_armature_ebone_to_mat3(ebo, td->axismtx);
 
 					if ((ebo->flag & BONE_ROOTSEL) == 0) {
-						/* To fix roll, see comments in transform_generic.c::recalcData_objects() */
-						td->extra = ebo;
-						td->ival = ebo->roll - ED_rollBoneToVector(ebo, z_axis, false);
+						createTransArmatureVerts_init_roll_fix(td, ebo);
 					}
 
 					td->ext = NULL;
@@ -1221,9 +1239,7 @@ static void createTransArmatureVerts(TransInfo *t)
 
 					ED_armature_ebone_to_mat3(ebo, td->axismtx);
 
-					/* To fix roll, see comments in transform_generic.c::recalcData_objects() */
-					td->extra = ebo;
-					td->ival = ebo->roll - ED_rollBoneToVector(ebo, z_axis, false);
+					createTransArmatureVerts_init_roll_fix(td, ebo);
 
 					td->ext = NULL;
 					td->val = NULL;
diff --git a/source/blender/editors/transform/transform_generics.c b/source/blender/editors/transform/transform_generics.c
index 5128ee3c9b6..7b4f0b8eca3 100644
--- a/source/blender/editors/transform/transform_generics.c
+++ b/source/blender/editors/transform/transform_generics.c
@@ -810,13 +810,24 @@ static void recalcData_objects(TransInfo *t)
 				 * armature's Z axis), and do the reverse to get final roll.
 				 * This method at least gives predictable, consistent results (the bone basically keeps "facing"
 				 * the armature's Z axis).
+				 * Note we need some special handling when bone is Z-aligned... sigh.
 				 */
 				for (i = 0; i < t->total; i++, td++) {
 					if (td->extra) {
 						const float z_axis[3] = {0.0f, 0.0f, 1.0f};
+						float vec[3];
 
 						ebo = td->extra;
-						ebo->roll = td->ival + ED_rollBoneToVector(ebo, z_axis, false);
+						sub_v3_v3v3(vec, ebo->tail, ebo->head);
+						normalize_v3(vec);
+
+						if (fabsf(dot_v3v3(vec, z_axis)) > 0.999999f) {
+							/* If our bone is Z-aligned, do not alter roll. See T38843. */
+							ebo->roll = td->ival2;
+						}
+						else {
+							ebo->roll = td->ival + ED_rollBoneToVector(ebo, z_axis, false);
+						}
 					}
 				}
 			}