bugfix [#24907] bone roll z up broken and python script showing correct method to roll bones

from Josh Wedlake (joshwedlake), who provided a reference script used to apply changes in ED_rollBoneToVector().

- Obvious bug fixed where Z-Up didnt work right.
- More align axis options to Recalculate Roll operator: X/Y/Z/View Axis & Negate.
- Axis Only option, ignore the axis direction, use shortest rotation to align bones.


ED_rollBoneToVector() changes:
- would give bad roll when the axis wasn't normalized or perpendicular to the bone.
  some callers accounted for this but not all.
- option to align to the axis but not the direction.

3 files changed, 94 insertions, 125 deletions

diff --git a/source/blender/editors/armature/editarmature.c b/source/blender/editors/armature/editarmature.c
index 3e40ae41b02..61d16f99f2f 100644
--- a/source/blender/editors/armature/editarmature.c
+++ b/source/blender/editors/armature/editarmature.c
@@ -31,6 +31,7 @@
 #include <string.h>
 #include <math.h> 
 #include <float.h> 
+#include <assert.h> 
 
 
 #include "DNA_anim_types.h"
@@ -2066,136 +2067,110 @@ void ED_armature_to_edit(Object *ob)
 /* adjust bone roll to align Z axis with vector
  * vec is in local space and is normalized
  */
-float ED_rollBoneToVector(EditBone *bone, float new_up_axis[3])
+
+float ED_rollBoneToVector(EditBone *bone, const float align_axis[3], const short axis_only)
 {
-	float mat[3][3], nor[3], up_axis[3], vec[3];
-	float roll;
+	float mat[3][3], nor[3];
 
 	sub_v3_v3v3(nor, bone->tail, bone->head);
-	
-	vec_roll_to_mat3(nor, 0, mat);
-	copy_v3_v3(up_axis, mat[2]);
-	
-	roll = angle_normalized_v3v3(new_up_axis, up_axis);
-	
-	cross_v3_v3v3(vec, up_axis, new_up_axis);
-	
-	if (dot_v3v3(vec, nor) < 0)
-	{
-		roll = -roll;
-	}
-	
-	return roll;
-}
+	vec_roll_to_mat3(nor, 0.0f, mat);
 
+	/* check the bone isnt aligned with the axis */
+	if(!is_zero_v3(align_axis) && angle_v3v3(align_axis, mat[2]) > FLT_EPSILON) {
+		float vec[3], align_axis_proj[3], roll;
 
-/* Set roll value for given bone -> Z-Axis Point up (original method) */
-static void auto_align_ebone_zaxisup(Scene *UNUSED(scene), View3D *UNUSED(v3d), EditBone *ebone)
-{
-	float	delta[3], curmat[3][3];
-	float	xaxis[3]={1.0f, 0.0f, 0.0f}, yaxis[3], zaxis[3]={0.0f, 0.0f, 1.0f};
-	float	targetmat[3][3], imat[3][3], diffmat[3][3];
-	
-	/* Find the current bone matrix */
-	sub_v3_v3v3(delta, ebone->tail, ebone->head);
-	vec_roll_to_mat3(delta, 0.0f, curmat);
-	
-	/* Make new matrix based on y axis & z-up */
-	copy_v3_v3(yaxis, curmat[1]);
-	
-	unit_m3(targetmat);
-	copy_v3_v3(targetmat[0], xaxis);
-	copy_v3_v3(targetmat[1], yaxis);
-	copy_v3_v3(targetmat[2], zaxis);
-	normalize_m3(targetmat);
-	
-	/* Find the difference between the two matrices */
-	invert_m3_m3(imat, targetmat);
-	mul_m3_m3m3(diffmat, imat, curmat);
-	
-	// old-method... let's see if using mat3_to_vec_roll is more accurate
-	//ebone->roll = atan2(diffmat[2][0], diffmat[2][2]);  
-	mat3_to_vec_roll(diffmat, delta, &ebone->roll);
-}
-
-void auto_align_ebone_topoint(EditBone *ebone, float *cursor)
-{
-	float	delta[3], curmat[3][3];
-	float	mat[4][4], tmat[4][4], imat[4][4];
-	float 	rmat[4][4], rot[3];
-	float	vec[3];
-
-	/* find the current bone matrix as a 4x4 matrix (in Armature Space) */
-	sub_v3_v3v3(delta, ebone->tail, ebone->head);
-	vec_roll_to_mat3(delta, ebone->roll, curmat);
-	copy_m4_m3(mat, curmat);
-	copy_v3_v3(mat[3], ebone->head);
-	
-	/* multiply bone-matrix by object matrix (so that bone-matrix is in WorldSpace) */
-	invert_m4_m4(imat, mat);
-	
-	/* find position of cursor relative to bone */
-	mul_v3_m4v3(vec, imat, cursor);
-	
-	/* check that cursor is in usable position */
-	if ((IS_EQ(vec[0], 0)==0) && (IS_EQ(vec[2], 0)==0)) {
-		/* Compute a rotation matrix around y */
-		rot[1] = (float)atan2(vec[0], vec[2]);
-		rot[0] = rot[2] = 0.0f;
-		eul_to_mat4( rmat,rot);
+		/* project the new_up_axis along the normal */
+		project_v3_v3v3(vec, align_axis, nor);
+		sub_v3_v3v3(align_axis_proj, align_axis, vec);
 		
-		/* Multiply the bone matrix by rotation matrix. This should be new bone-matrix */
-		mul_m4_m4m4(tmat, rmat, mat);
-		copy_m3_m4(curmat, tmat);
+		if(axis_only) {
+			if(angle_v3v3(align_axis_proj, mat[2]) > M_PI/2) {
+				negate_v3(align_axis_proj);
+			}
+		}
 		
-		/* Now convert from new bone-matrix, back to a roll value (in radians) */
-		mat3_to_vec_roll(curmat, delta, &ebone->roll);
-	}
-}
+		roll = angle_v3v3(align_axis_proj, mat[2]);
+		
+		cross_v3_v3v3(vec, mat[2], align_axis_proj);
+		
+		if (dot_v3v3(vec, nor) < 0) {
+			roll = -roll;
+		}
 
-static void auto_align_ebone_tocursor(Scene *scene, View3D *v3d, EditBone *ebone)
-{
-	float cursor_local[3];
-	float  	*cursor= give_cursor(scene, v3d);
-	float imat[3][3];
+		return roll;
+	}
 
-	copy_m3_m4(imat, scene->obedit->obmat);
-	invert_m3(imat);
-	copy_v3_v3(cursor_local, cursor);
-	mul_m3_v3(imat, cursor_local);
-	auto_align_ebone_topoint(ebone, cursor_local);
+	return 0.0f;
 }
 
+
 static EnumPropertyItem prop_calc_roll_types[] = {
-	{0, "GLOBALUP", 0, "Z-Axis Up", ""},
-	{1, "CURSOR", 0, "Z-Axis to Cursor", ""},
+	{0, "X", 0, "X Axis", ""},
+	{1, "Y", 0, "Y Axis", ""},
+	{2, "Z", 0, "Z Axis", ""},
+	{6, "VIEW", 0, "View Axis", ""},
+	{7, "CURSOR", 0, "Cursor", ""},
 	{0, NULL, 0, NULL, NULL}
 };
 
+
 static int armature_calc_roll_exec(bContext *C, wmOperator *op) 
 {
-	Scene *scene= CTX_data_scene(C);
-	View3D *v3d= CTX_wm_view3d(C);
 	Object *ob= CTX_data_edit_object(C);
-	void (*roll_func)(Scene *, View3D *, EditBone *) = NULL;
+	const short type= RNA_enum_get(op->ptr, "type");
+	const short axis_only= RNA_boolean_get(op->ptr, "axis_only");
+	const short axis_flip= RNA_boolean_get(op->ptr, "axis_flip");
+
+	float imat[3][3];
+
+	copy_m3_m4(imat, ob->obmat);
+	invert_m3(imat);
+
+	if(type==7) { /* Cursor */
+		Scene *scene= CTX_data_scene(C);
+		View3D *v3d= CTX_wm_view3d(C); /* can be NULL */
+		float cursor_local[3];
+		float  	*cursor= give_cursor(scene, v3d);
 	
-	/* specific method used to calculate roll depends on mode */
-	switch (RNA_enum_get(op->ptr, "type")) {
-		case 1:  /* Z-Axis point towards cursor */
-			roll_func= auto_align_ebone_tocursor;
-			break;
-		default: /* Z-Axis Point Up */
-			roll_func= auto_align_ebone_zaxisup;
-			break;
+
+		copy_v3_v3(cursor_local, cursor);
+		mul_m3_v3(imat, cursor_local);
+
+		/* cursor */
+		CTX_DATA_BEGIN(C, EditBone *, ebone, selected_editable_bones) {
+			float cursor_rel[3];
+			sub_v3_v3v3(cursor_rel, cursor_local, ebone->head);
+			if(axis_flip) negate_v3(cursor_rel);
+			ebone->roll= ED_rollBoneToVector(ebone, cursor_rel, axis_only);
+		}
+		CTX_DATA_END;
 	}
-	
-	/* recalculate roll on selected bones */
-	CTX_DATA_BEGIN(C, EditBone *, ebone, selected_editable_bones) {
-		/* roll func is a callback which assumes that all is well */
-		roll_func(scene, v3d, ebone);
+	else {
+		float vec[3]= {0.0f, 0.0f, 0.0f};
+		if(type==6) { /* View */
+			RegionView3D *rv3d= CTX_wm_region_view3d(C);
+			if(rv3d==NULL) {
+				BKE_report(op->reports, RPT_ERROR, "No region view3d available");
+				return OPERATOR_CANCELLED;
+			}
+
+			copy_v3_v3(vec, rv3d->viewinv[2]);
+		}
+		else { /* Axis */
+			assert(type >= 0 && type <= 5);
+			if(type<3)	vec[type]= 1.0f; 
+			else		vec[type-2]= -1.0f; 
+		}
+
+		mul_m3_v3(imat, vec);
+		if(axis_flip) negate_v3(vec);
+
+		CTX_DATA_BEGIN(C, EditBone *, ebone, selected_editable_bones) {
+			/* roll func is a callback which assumes that all is well */
+			ebone->roll= ED_rollBoneToVector(ebone, vec, axis_only);
+		}
+		CTX_DATA_END;
 	}
-	CTX_DATA_END;
-	
 
 	/* note, notifier might evolve */
 	WM_event_add_notifier(C, NC_OBJECT|ND_POSE, ob);
@@ -2216,9 +2191,11 @@ void ARMATURE_OT_calculate_roll(wmOperatorType *ot)
 	
 	/* flags */
 	ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
-	
+
 	/* properties */
 	ot->prop= RNA_def_enum(ot->srna, "type", prop_calc_roll_types, 0, "Type", "");
+	RNA_def_boolean(ot->srna, "axis_flip", 0, "Flip Axis", "Negate the alignment axis.");
+	RNA_def_boolean(ot->srna, "axis_only", 0, "Shortest Rotation", "Ignore the axis direction, use the shortest rotation to align.");
 }
 
 /* **************** undo for armatures ************** */
diff --git a/source/blender/editors/armature/editarmature_generate.c b/source/blender/editors/armature/editarmature_generate.c
index 6eda622ead8..6abdd85bf71 100644
--- a/source/blender/editors/armature/editarmature_generate.c
+++ b/source/blender/editors/armature/editarmature_generate.c
@@ -50,18 +50,12 @@ void setBoneRollFromNormal(EditBone *bone, float *no, float UNUSED(invmat[][4]),
 {
 	if (no != NULL && !is_zero_v3(no))
 	{
-		float tangent[3], vec[3], normal[3];
+		float normal[3];
 
-		VECCOPY(normal, no);	
+		copy_v3_v3(normal, no);	
 		mul_m3_v3(tmat, normal);
-
-		sub_v3_v3v3(tangent, bone->tail, bone->head);
-		project_v3_v3v3(vec, tangent, normal);
-		sub_v3_v3(normal, vec);
-		
-		normalize_v3(normal);
 		
-		bone->roll = ED_rollBoneToVector(bone, normal);
+		bone->roll = ED_rollBoneToVector(bone, normal, FALSE);
 	}
 }
 
diff --git a/source/blender/editors/armature/editarmature_retarget.c b/source/blender/editors/armature/editarmature_retarget.c
index d50f2d00ebb..4ddc73df469 100644
--- a/source/blender/editors/armature/editarmature_retarget.c
+++ b/source/blender/editors/armature/editarmature_retarget.c
@@ -185,12 +185,12 @@ float rollBoneByQuatAligned(EditBone *bone, float old_up_axis[3], float qrot[4],
 	if (angle_normalized_v3v3(x_axis, new_up_axis) < angle_normalized_v3v3(z_axis, new_up_axis))
 	{
 		rotation_between_vecs_to_quat(qroll, new_up_axis, x_axis); /* set roll rotation quat */
-		return ED_rollBoneToVector(bone, x_axis);
+		return ED_rollBoneToVector(bone, x_axis, FALSE);
 	}
 	else
 	{
 		rotation_between_vecs_to_quat(qroll, new_up_axis, z_axis); /* set roll rotation quat */
-		return ED_rollBoneToVector(bone, z_axis);
+		return ED_rollBoneToVector(bone, z_axis, FALSE);
 	}
 }
 
@@ -240,7 +240,7 @@ float rollBoneByQuatJoint(RigEdge *edge, RigEdge *previous, float qrot[4], float
 		/* real qroll between normal and up_axis */
 		rotation_between_vecs_to_quat(qroll, new_up_axis, normal);
 
-		return ED_rollBoneToVector(edge->bone, normal);
+		return ED_rollBoneToVector(edge->bone, normal, FALSE);
 	}
 }
 
@@ -251,9 +251,7 @@ float rollBoneByQuat(EditBone *bone, float old_up_axis[3], float qrot[4])
 	VECCOPY(new_up_axis, old_up_axis);
 	mul_qt_v3(qrot, new_up_axis);
 	
-	normalize_v3(new_up_axis);
-	
-	return ED_rollBoneToVector(bone, new_up_axis);
+	return ED_rollBoneToVector(bone, new_up_axis, FALSE);
 }
 
 /************************************ DESTRUCTORS ******************************************************/