Transform: fix rotate redo behavior

- Add XYZ option.
- Orientation now works as expected.

Now a redo for rotation works logically,
setting the axis to Z & the orientation to view.

Resolves T57205

5 files changed, 103 insertions, 75 deletions

diff --git a/source/blender/editors/transform/transform.c b/source/blender/editors/transform/transform.c
index e10d32716ac..9e86c4cb686 100644
--- a/source/blender/editors/transform/transform.c
+++ b/source/blender/editors/transform/transform.c
@@ -2156,12 +2156,15 @@ void saveTransform(bContext *C, TransInfo *t, wmOperator *op)
 		RNA_float_set(op->ptr, "proportional_size", t->prop_size);
 	}
 
-	if ((prop = RNA_struct_find_property(op->ptr, "axis"))) {
-		RNA_property_float_set_array(op->ptr, prop, t->axis);
-	}
-
-	if ((prop = RNA_struct_find_property(op->ptr, "axis_ortho"))) {
-		RNA_property_float_set_array(op->ptr, prop, t->axis_ortho);
+	if ((prop = RNA_struct_find_property(op->ptr, "orient_axis"))) {
+		if (t->flag & T_MODAL) {
+			if (t->con.mode & CON_APPLY) {
+				int orient_axis = constraintModeToIndex(t);
+				if (orient_axis != -1) {
+					RNA_property_enum_set(op->ptr, prop, orient_axis);
+				}
+			}
+		}
 	}
 
 	if ((prop = RNA_struct_find_property(op->ptr, "mirror"))) {
@@ -2194,9 +2197,12 @@ void saveTransform(bContext *C, TransInfo *t, wmOperator *op)
 					RNA_enum_set(op->ptr, "constraint_matrix_orientation", orientation);
 				}
 			}
-			if (t->con.mode & CON_APPLY || (t->orientation.unset != V3D_ORIENT_GLOBAL)) {
+			if (t->con.mode & CON_APPLY) {
 				RNA_float_set_array(op->ptr, "constraint_matrix", &t->con.mtx[0][0]);
 			}
+			else if (t->orient_matrix_is_set) {
+				RNA_float_set_array(op->ptr, "constraint_matrix", &t->orient_matrix[0][0]);
+			}
 			else {
 				RNA_float_set_array(op->ptr, "constraint_matrix", &t->spacemtx[0][0]);
 			}
@@ -2577,15 +2583,15 @@ bool initTransform(bContext *C, TransInfo *t, wmOperator *op, const wmEvent *eve
 	}
 
 	/* Transformation axis from operator */
-	if ((prop = RNA_struct_find_property(op->ptr, "axis")) && RNA_property_is_set(op->ptr, prop)) {
-		RNA_property_float_get_array(op->ptr, prop, t->axis);
-		normalize_v3(t->axis);
-		copy_v3_v3(t->axis_orig, t->axis);
+	if ((prop = RNA_struct_find_property(op->ptr, "orient_axis")) &&
+	    RNA_property_is_set(op->ptr, prop))
+	{
+		t->orient_axis = RNA_property_enum_get(op->ptr, prop);
 	}
-
-	if ((prop = RNA_struct_find_property(op->ptr, "axis_ortho")) && RNA_property_is_set(op->ptr, prop)) {
-		RNA_property_float_get_array(op->ptr, prop, t->axis_ortho);
-		normalize_v3(t->axis_ortho);
+	if ((prop = RNA_struct_find_property(op->ptr, "orient_axis_ortho")) &&
+	    RNA_property_is_set(op->ptr, prop))
+	{
+		t->orient_axis_ortho = RNA_property_enum_get(op->ptr, prop);
 	}
 
 	/* Constraint init from operator */
@@ -3396,10 +3402,10 @@ static void initShear_mouseInputMode(TransInfo *t)
 	float dir[3];
 
 	if (t->custom.mode.data == NULL) {
-		copy_v3_v3(dir, t->axis_ortho);
+		copy_v3_v3(dir, t->orient_matrix[t->orient_axis_ortho]);
 	}
 	else {
-		cross_v3_v3v3(dir, t->axis_ortho, t->axis);
+		cross_v3_v3v3(dir, t->orient_matrix[t->orient_axis_ortho], t->orient_matrix[t->orient_axis]);
 	}
 
 	mul_mat3_m4_v3(t->viewmat, dir);
@@ -3417,13 +3423,20 @@ static void initShear(TransInfo *t)
 	t->transform = applyShear;
 	t->handleEvent = handleEventShear;
 
-	if (is_zero_v3(t->axis)) {
-		negate_v3_v3(t->axis, t->viewinv[2]);
-		normalize_v3(t->axis);
-	}
-	if (is_zero_v3(t->axis_ortho)) {
-		copy_v3_v3(t->axis_ortho, t->viewinv[0]);
-		normalize_v3(t->axis_ortho);
+	t->orient_axis = 2;
+	t->orient_axis_ortho = 1;
+
+	if (t->orient_matrix_is_set == false) {
+		float *axis = t->orient_matrix[t->orient_axis];
+		float *axis_ortho = t->orient_matrix[t->orient_axis_ortho];
+		if (is_zero_v3(axis)) {
+			negate_v3_v3(axis, t->viewinv[2]);
+			normalize_v3(axis);
+		}
+		if (is_zero_v3(axis_ortho)) {
+			copy_v3_v3(axis_ortho, t->viewinv[0]);
+			normalize_v3(axis_ortho);
+		}
 	}
 
 	initShear_mouseInputMode(t);
@@ -3512,8 +3525,8 @@ static void applyShear(TransInfo *t, const int UNUSED(mval[2]))
 	else
 		smat[0][1] = value;
 
-	copy_v3_v3(axismat_inv[0], t->axis_ortho);
-	copy_v3_v3(axismat_inv[2], t->axis);
+	copy_v3_v3(axismat_inv[0], t->orient_matrix[t->orient_axis_ortho]);
+	copy_v3_v3(axismat_inv[2], t->orient_matrix[t->orient_axis]);
 	cross_v3_v3v3(axismat_inv[1], axismat_inv[0], axismat_inv[2]);
 	invert_m3_m3(axismat, axismat_inv);
 
@@ -4100,8 +4113,10 @@ static void applyToSphere(TransInfo *t, const int UNUSED(mval[2]))
 
 static void postInputRotation(TransInfo *t, float values[3])
 {
+	float axis_final[3];
+	copy_v3_v3(axis_final, t->orient_matrix[t->orient_axis]);
 	if ((t->con.mode & CON_APPLY) && t->con.applyRot) {
-		t->con.applyRot(t, NULL, NULL, t->axis, values);
+		t->con.applyRot(t, NULL, NULL, axis_final, values);
 	}
 }
 
@@ -4127,17 +4142,16 @@ static void initRotation(TransInfo *t)
 	if (t->flag & T_2D_EDIT)
 		t->flag |= T_NO_CONSTRAINT;
 
-	if (t->options & CTX_PAINT_CURVE) {
-		t->axis[0] = 0.0;
-		t->axis[1] = 0.0;
-		t->axis[2] = -1.0;
-	}
-	else {
-		negate_v3_v3(t->axis, t->viewinv[2]);
-		normalize_v3(t->axis);
+	if (t->orient_matrix_is_set == false) {
+		if ((t->options & CTX_PAINT_CURVE) == 0) {
+			t->orientation.unset = V3D_ORIENT_VIEW;
+			copy_m3_m4(t->orient_matrix, t->viewinv);
+			normalize_m3(t->orient_matrix);
+			negate_m3(t->orient_matrix);
+		}
 	}
 
-	copy_v3_v3(t->axis_orig, t->axis);
+	t->orient_axis = 2;
 }
 
 /* Used by Transform Rotation and Transform Normal Rotation */
@@ -4446,12 +4460,11 @@ static void applyRotation(TransInfo *t, const int UNUSED(mval[2]))
 
 	snapGridIncrement(t, &final);
 
+	float axis_final[3];
+	copy_v3_v3(axis_final, t->orient_matrix[t->orient_axis]);
+
 	if ((t->con.mode & CON_APPLY) && t->con.applyRot) {
-		t->con.applyRot(t, NULL, NULL, t->axis, NULL);
-	}
-	else {
-		/* reset axis if constraint is not set */
-		copy_v3_v3(t->axis, t->axis_orig);
+		t->con.applyRot(t, NULL, NULL, axis_final, NULL);
 	}
 
 	applySnapping(t, &final);
@@ -4463,7 +4476,7 @@ static void applyRotation(TransInfo *t, const int UNUSED(mval[2]))
 
 	headerRotation(t, str, final);
 
-	applyRotationValue(t, final, t->axis);
+	applyRotationValue(t, final, axis_final);
 
 	recalcData(t);
 
@@ -4654,10 +4667,14 @@ static void initNormalRotation(TransInfo *t)
 		storeCustomLNorValue(tc, bm);
 	}
 
-	negate_v3_v3(t->axis, t->viewinv[2]);
-	normalize_v3(t->axis);
+	if (t->orient_matrix_is_set == false) {
+		t->orientation.unset = V3D_ORIENT_VIEW;
+		copy_m3_m4(t->orient_matrix, t->viewinv);
+		normalize_m3(t->orient_matrix);
+		negate_m3(t->orient_matrix);
+	}
 
-	copy_v3_v3(t->axis_orig, t->axis);
+	t->orient_axis = 2;
 }
 
 /* Works by getting custom normal from clnor_data, transform, then store */
@@ -4665,12 +4682,11 @@ static void applyNormalRotation(TransInfo *t, const int UNUSED(mval[2]))
 {
 	char str[UI_MAX_DRAW_STR];
 
+	float axis_final[3];
+	copy_v3_v3(axis_final, t->orient_matrix[t->orient_axis]);
+
 	if ((t->con.mode & CON_APPLY) && t->con.applyRot) {
-		t->con.applyRot(t, NULL, NULL, t->axis, NULL);
-	}
-	else {
-		/* reset axis if constraint is not set */
-		copy_v3_v3(t->axis, t->axis_orig);
+		t->con.applyRot(t, NULL, NULL, axis_final, NULL);
 	}
 
 	FOREACH_TRANS_DATA_CONTAINER(t, tc) {
@@ -4683,7 +4699,7 @@ static void applyNormalRotation(TransInfo *t, const int UNUSED(mval[2]))
 		float axis[3];
 		float mat[3][3];
 		float angle = t->values[0];
-		copy_v3_v3(axis, t->axis);
+		copy_v3_v3(axis, axis_final);
 
 		snapGridIncrement(t, &angle);
 
diff --git a/source/blender/editors/transform/transform.h b/source/blender/editors/transform/transform.h
index b234dabdc0b..d6121948b90 100644
--- a/source/blender/editors/transform/transform.h
+++ b/source/blender/editors/transform/transform.h
@@ -628,10 +628,11 @@ typedef struct TransInfo {
 	/** Offset applied ontop of modal input. */
 	float		values_modal_offset[4];
 	float		auto_values[4];
-	float		axis[3];
-	/** TransCon can change 'axis', store the original value here. */
-	float		axis_orig[3];
-	float		axis_ortho[3];
+
+	int orient_axis;
+	int orient_axis_ortho;
+	float orient_matrix[3][3];
+	bool  orient_matrix_is_set;
 
 	/** remove elements if operator is canceled. */
 	bool		remove_on_cancel;
diff --git a/source/blender/editors/transform/transform_generics.c b/source/blender/editors/transform/transform_generics.c
index 9dc64d9fd49..0e848227378 100644
--- a/source/blender/editors/transform/transform_generics.c
+++ b/source/blender/editors/transform/transform_generics.c
@@ -1337,6 +1337,11 @@ void initTransInfo(bContext *C, TransInfo *t, wmOperator *op, const wmEvent *eve
 
 	unit_m3(t->mat);
 
+	unit_m3(t->orient_matrix);
+	negate_m3(t->orient_matrix);
+	/* Leave 't->orient_matrix_is_set' to false,
+	 * so we overwrite it when we have a useful value. */
+
 	/* if there's an event, we're modal */
 	if (event) {
 		t->flag |= T_MODAL;
@@ -1522,6 +1527,9 @@ void initTransInfo(bContext *C, TransInfo *t, wmOperator *op, const wmEvent *eve
 	      RNA_enum_get(op->ptr, "constraint_matrix_orientation"))))
 	{
 		RNA_property_float_get_array(op->ptr, prop, &t->spacemtx[0][0]);
+		/* Some transform modes use this to operate on an axis. */
+		copy_m3_m3(t->orient_matrix, t->spacemtx);
+		t->orient_matrix_is_set = true;
 		t->orientation.user = V3D_ORIENT_CUSTOM_MATRIX;
 		t->orientation.custom = 0;
 		if (t->flag & T_MODAL) {
diff --git a/source/blender/editors/transform/transform_gizmo_3d.c b/source/blender/editors/transform/transform_gizmo_3d.c
index bba91ec3b00..efc3bfdb5c2 100644
--- a/source/blender/editors/transform/transform_gizmo_3d.c
+++ b/source/blender/editors/transform/transform_gizmo_3d.c
@@ -1325,7 +1325,7 @@ void drawDial3d(const TransInfo *t)
 		}
 		else {
 			axis_idx = MAN_AXIS_ROT_C;
-			negate_v3_v3(mat_basis[2], t->axis);
+			negate_v3_v3(mat_basis[2], t->orient_matrix[t->orient_axis]);
 			scale *= 1.2f;
 			line_with -= 1.0f;
 		}
@@ -2210,8 +2210,11 @@ static void WIDGETGROUP_xform_shear_refresh(const bContext *C, wmGizmoGroup *gzg
 				else {
 					negate_v3_v3(axis, tbounds.axis[i_ortho_b]);
 				}
-				RNA_float_set_array(&gzop->ptr, "axis", axis);
-				RNA_float_set_array(&gzop->ptr, "axis_ortho", tbounds.axis[i_ortho_a]);
+				float orient_matrix[3][3];
+				cross_v3_v3v3(orient_matrix[0], tbounds.axis[i_ortho_a], axis);
+				copy_v3_v3(orient_matrix[1], tbounds.axis[i_ortho_a]);
+				copy_v3_v3(orient_matrix[2], axis);
+				RNA_float_set_array(&gzop->ptr, "orient_matrix", &orient_matrix[0][0]);
 				mul_v3_fl(gz->matrix_basis[0], 0.5f);
 				mul_v3_fl(gz->matrix_basis[1], 6.0f);
 			}
diff --git a/source/blender/editors/transform/transform_ops.c b/source/blender/editors/transform/transform_ops.c
index 31a8c8559b6..d2b59eac4f1 100644
--- a/source/blender/editors/transform/transform_ops.c
+++ b/source/blender/editors/transform/transform_ops.c
@@ -542,20 +542,20 @@ void Transform_Properties(struct wmOperatorType *ot, int flags)
 {
 	PropertyRNA *prop;
 
-	if (flags & P_AXIS) {
-		prop = RNA_def_property(ot->srna, "axis", PROP_FLOAT, PROP_DIRECTION);
-		RNA_def_property_array(prop, 3);
-		/* Make this not hidden when there's a nice axis selection widget */
-		RNA_def_property_flag(prop, PROP_HIDDEN);
-		RNA_def_property_ui_text(prop, "Axis", "The axis around which the transformation occurs");
-	}
+	if (flags & P_ORIENT_AXIS) {
+		prop = RNA_def_property(ot->srna, "orient_axis", PROP_ENUM, PROP_NONE);
+		RNA_def_property_ui_text(prop, "Axis", "");
+		RNA_def_property_enum_default(prop, 2);
+		RNA_def_property_enum_items(prop, rna_enum_axis_xyz_items);
 
-	if (flags & P_AXIS_ORTHO) {
-		prop = RNA_def_property(ot->srna, "axis_ortho", PROP_FLOAT, PROP_DIRECTION);
-		RNA_def_property_array(prop, 3);
-		/* Make this not hidden when there's a nice axis selection widget */
-		RNA_def_property_flag(prop, PROP_HIDDEN);
-		RNA_def_property_ui_text(prop, "Axis", "The orthogonal axis around which the transformation occurs");
+		prop = RNA_def_float_matrix(ot->srna, "orient_matrix", 3, 3, NULL, 0.0f, 0.0f, "Matrix", "", 0.0f, 0.0f);
+		RNA_def_property_flag(prop, PROP_HIDDEN | PROP_SKIP_SAVE);
+	}
+	if (flags & P_ORIENT_AXIS_ORTHO) {
+		prop = RNA_def_property(ot->srna, "orient_axis_ortho", PROP_ENUM, PROP_NONE);
+		RNA_def_property_ui_text(prop, "Axis Ortho", "");
+		RNA_def_property_enum_default(prop, 1);
+		RNA_def_property_enum_items(prop, rna_enum_axis_xyz_items);
 	}
 
 	if (flags & P_CONSTRAINT) {
@@ -783,7 +783,7 @@ static void TRANSFORM_OT_rotate(struct wmOperatorType *ot)
 	WM_operatortype_props_advanced_begin(ot);
 
 	Transform_Properties(
-	        ot, P_AXIS | P_CONSTRAINT | P_PROPORTIONAL | P_MIRROR | P_GEO_SNAP | P_GPENCIL_EDIT | P_CENTER);
+	        ot, P_ORIENT_AXIS | P_CONSTRAINT | P_PROPORTIONAL | P_MIRROR | P_GEO_SNAP | P_GPENCIL_EDIT | P_CENTER);
 }
 
 static void TRANSFORM_OT_tilt(struct wmOperatorType *ot)
@@ -856,7 +856,7 @@ static void TRANSFORM_OT_shear(struct wmOperatorType *ot)
 
 	WM_operatortype_props_advanced_begin(ot);
 
-	Transform_Properties(ot, P_PROPORTIONAL | P_MIRROR | P_SNAP | P_GPENCIL_EDIT | P_AXIS | P_AXIS_ORTHO);
+	Transform_Properties(ot, P_PROPORTIONAL | P_MIRROR | P_SNAP | P_GPENCIL_EDIT | P_ORIENT_AXIS | P_ORIENT_AXIS_ORTHO);
 }
 
 static void TRANSFORM_OT_push_pull(struct wmOperatorType *ot)
@@ -1099,7 +1099,7 @@ static void TRANSFORM_OT_rotate_normal(struct wmOperatorType *ot)
 
 	RNA_def_float_rotation(ot->srna, "value", 0, NULL, -FLT_MAX, FLT_MAX, "Angle", "", -M_PI * 2, M_PI * 2);
 
-	Transform_Properties(ot, P_AXIS | P_CONSTRAINT | P_MIRROR);
+	Transform_Properties(ot, P_ORIENT_AXIS | P_CONSTRAINT | P_MIRROR);
 }
 
 
@@ -1129,7 +1129,7 @@ static void TRANSFORM_OT_transform(struct wmOperatorType *ot)
 	WM_operatortype_props_advanced_begin(ot);
 
 	Transform_Properties(
-	        ot, P_AXIS | P_CONSTRAINT | P_PROPORTIONAL | P_MIRROR | P_ALIGN_SNAP | P_GPENCIL_EDIT | P_CENTER);
+	        ot, P_ORIENT_AXIS | P_CONSTRAINT | P_PROPORTIONAL | P_MIRROR | P_ALIGN_SNAP | P_GPENCIL_EDIT | P_CENTER);
 }
 
 static int transform_from_gizmo_invoke(bContext *C, wmOperator *UNUSED(op), const wmEvent *UNUSED(event))