Transform: remove constraints from the redo panel

Constraint options had confusing behavior:

- When non were pressed, the orientation was ignored.
- When any were pressed, the orientation was used,
  but only unconstrained axed could be adjusted.

Now constraining is only used for modal execution
so there is no need to show these in the interface.

When an orientation is selected, the XYZ values always transform
using that space.

Note, transform system should be refactored to support different
orientations w/o having to use constraints.

Addresses T57204

5 files changed, 88 insertions, 38 deletions

diff --git a/source/blender/editors/transform/transform.c b/source/blender/editors/transform/transform.c
index 9a3bc10c40f..cddd25252a4 100644
--- a/source/blender/editors/transform/transform.c
+++ b/source/blender/editors/transform/transform.c
@@ -2183,35 +2183,40 @@ void saveTransform(bContext *C, TransInfo *t, wmOperator *op)
 			BLI_assert(orientation >= V3D_ORIENT_CUSTOM);
 		}
 
-
-		if (t->con.mode & CON_APPLY) {
-			RNA_float_set_array(op->ptr, "constraint_matrix", &t->con.mtx[0][0]);
-		}
-		else {
-			RNA_float_set_array(op->ptr, "constraint_matrix", &t->spacemtx[0][0]);
-		}
-
 		/* Use 'constraint_matrix' instead. */
 		if (orientation != V3D_ORIENT_CUSTOM_MATRIX) {
 			RNA_enum_set(op->ptr, "constraint_orientation", orientation);
 		}
 
-		if (t->con.mode & CON_APPLY) {
-			if (t->con.mode & CON_AXIS0) {
-				constraint_axis[0] = true;
+		if (t->flag & T_MODAL) {
+			if (orientation != V3D_ORIENT_CUSTOM_MATRIX) {
+				if (t->flag & T_MODAL) {
+					RNA_enum_set(op->ptr, "constraint_matrix_orientation", orientation);
+				}
 			}
-			if (t->con.mode & CON_AXIS1) {
-				constraint_axis[1] = true;
+			if (t->con.mode & CON_APPLY) {
+				RNA_float_set_array(op->ptr, "constraint_matrix", &t->con.mtx[0][0]);
 			}
-			if (t->con.mode & CON_AXIS2) {
-				constraint_axis[2] = true;
+			else {
+				RNA_float_set_array(op->ptr, "constraint_matrix", &t->spacemtx[0][0]);
+			}
+			if (t->con.mode & CON_APPLY) {
+				if (t->con.mode & CON_AXIS0) {
+					constraint_axis[0] = true;
+				}
+				if (t->con.mode & CON_AXIS1) {
+					constraint_axis[1] = true;
+				}
+				if (t->con.mode & CON_AXIS2) {
+					constraint_axis[2] = true;
+				}
 			}
-		}
 
-		/* Only set if needed, so we can hide in the UI when nothing is set.
-		 * See 'transform_poll_property'. */
-		if (ELEM(true, UNPACK3(constraint_axis))) {
-			RNA_property_boolean_set_array(op->ptr, prop, constraint_axis);
+			/* Only set if needed, so we can hide in the UI when nothing is set.
+			 * See 'transform_poll_property'. */
+			if (ELEM(true, UNPACK3(constraint_axis))) {
+				RNA_property_boolean_set_array(op->ptr, prop, constraint_axis);
+			}
 		}
 	}
 
@@ -2584,24 +2589,42 @@ bool initTransform(bContext *C, TransInfo *t, wmOperator *op, const wmEvent *eve
 	}
 
 	/* Constraint init from operator */
-	if ((prop = RNA_struct_find_property(op->ptr, "constraint_axis")) && RNA_property_is_set(op->ptr, prop)) {
-		bool constraint_axis[3];
+	if (t->flag & T_MODAL) {
+		if ((prop = RNA_struct_find_property(op->ptr, "constraint_axis")) &&
+		    RNA_property_is_set(op->ptr, prop))
+		{
+			bool constraint_axis[3];
 
-		RNA_property_boolean_get_array(op->ptr, prop, constraint_axis);
+			RNA_property_boolean_get_array(op->ptr, prop, constraint_axis);
 
-		if (constraint_axis[0] || constraint_axis[1] || constraint_axis[2]) {
-			t->con.mode |= CON_APPLY;
+			if (constraint_axis[0] || constraint_axis[1] || constraint_axis[2]) {
+				t->con.mode |= CON_APPLY;
 
-			if (constraint_axis[0]) {
-				t->con.mode |= CON_AXIS0;
-			}
-			if (constraint_axis[1]) {
-				t->con.mode |= CON_AXIS1;
-			}
-			if (constraint_axis[2]) {
-				t->con.mode |= CON_AXIS2;
-			}
+				/* Only for interactive operation, when redoing, ignore these values since the numbers
+				 * will be constrainted already. */
+				if (t->flag & T_MODAL) {
+					if (constraint_axis[0]) {
+						t->con.mode |= CON_AXIS0;
+					}
+					if (constraint_axis[1]) {
+						t->con.mode |= CON_AXIS1;
+					}
+					if (constraint_axis[2]) {
+						t->con.mode |= CON_AXIS2;
+					}
+				}
+				else {
+					t->con.mode |= CON_AXIS0 | CON_AXIS1 | CON_AXIS2;
+				}
 
+				setUserConstraint(t, t->orientation.user, t->con.mode, "%s");
+			}
+		}
+	}
+	else {
+		/* So we can adjust in non global orientation. */
+		if (t->orientation.user != V3D_ORIENT_GLOBAL) {
+			t->con.mode |= CON_APPLY | CON_AXIS0 | CON_AXIS1 | CON_AXIS2;
 			setUserConstraint(t, t->orientation.user, t->con.mode, "%s");
 		}
 	}
diff --git a/source/blender/editors/transform/transform.h b/source/blender/editors/transform/transform.h
index 2c97974721e..881de792391 100644
--- a/source/blender/editors/transform/transform.h
+++ b/source/blender/editors/transform/transform.h
@@ -764,7 +764,16 @@ enum {
 
 /* transinfo->con->mode */
 enum {
+	/**
+	 * TODO(campbell): this has two meanings:
+	 * - Constraint axes.
+	 * - Transform values are evaluated in different orientation.
+	 *
+	 * We should split out this second meaning into another flag
+	 * because transform logic becomes hard to follow when we're
+	 * only want to support an alternate orientation. */
 	CON_APPLY =       1 << 0,
+	/** These are only used for modal execution. */
 	CON_AXIS0 =       1 << 1,
 	CON_AXIS1 =       1 << 2,
 	CON_AXIS2 =       1 << 3,
diff --git a/source/blender/editors/transform/transform_generics.c b/source/blender/editors/transform/transform_generics.c
index d74cd72f649..0bdd18793f4 100644
--- a/source/blender/editors/transform/transform_generics.c
+++ b/source/blender/editors/transform/transform_generics.c
@@ -1513,11 +1513,19 @@ void initTransInfo(bContext *C, TransInfo *t, wmOperator *op, const wmEvent *eve
 	}
 
 	if (op && ((prop = RNA_struct_find_property(op->ptr, "constraint_matrix")) &&
-	           RNA_property_is_set(op->ptr, prop)))
+	           RNA_property_is_set(op->ptr, prop)) &&
+	    ((t->flag & T_MODAL) ||
+	     /* When using redo, don't use the the custom constraint matrix
+	      * if the user selects a different orientation. */
+	     (RNA_enum_get(op->ptr, "constraint_orientation") ==
+	      RNA_enum_get(op->ptr, "constraint_matrix_orientation"))))
 	{
 		RNA_property_float_get_array(op->ptr, prop, &t->spacemtx[0][0]);
 		t->orientation.user = V3D_ORIENT_CUSTOM_MATRIX;
 		t->orientation.custom = 0;
+		if (t->flag & T_MODAL) {
+			RNA_enum_set(op->ptr, "constraint_matrix_orientation", RNA_enum_get(op->ptr, "constraint_orientation"));
+		}
 	}
 	else if (op && ((prop = RNA_struct_find_property(op->ptr, "constraint_orientation")) &&
 	                RNA_property_is_set(op->ptr, prop)))
diff --git a/source/blender/editors/transform/transform_gizmo_extrude_3d.c b/source/blender/editors/transform/transform_gizmo_extrude_3d.c
index d6c8f01edf1..f6d1b91c284 100644
--- a/source/blender/editors/transform/transform_gizmo_extrude_3d.c
+++ b/source/blender/editors/transform/transform_gizmo_extrude_3d.c
@@ -389,8 +389,9 @@ static void gizmo_mesh_extrude_invoke_prepare(const bContext *UNUSED(C), wmGizmo
 		wmGizmoOpElem *gzop = WM_gizmo_operator_get(gz, 0);
 		PointerRNA macroptr = RNA_pointer_get(&gzop->ptr, "TRANSFORM_OT_translate");
 		if (gz == ggd->adjust[0]) {
-			RNA_float_set_array(&macroptr, "constraint_matrix", &ggd->redo_xform.constraint_matrix[0][0]);
 			RNA_boolean_set_array(&macroptr, "constraint_axis", ggd->redo_xform.constraint_axis);
+			RNA_float_set_array(&macroptr, "constraint_matrix", &ggd->redo_xform.constraint_matrix[0][0]);
+			RNA_enum_set(&macroptr, "constraint_orientation", V3D_ORIENT_NORMAL);
 		}
 		RNA_float_set_array(&macroptr, "value", ggd->redo_xform.value);
 	}
@@ -409,6 +410,7 @@ static void gizmo_mesh_extrude_invoke_prepare(const bContext *UNUSED(C), wmGizmo
 			wmGizmoOpElem *gzop = WM_gizmo_operator_get(gz, 0);
 			PointerRNA macroptr = RNA_pointer_get(&gzop->ptr, "TRANSFORM_OT_translate");
 			RNA_float_set_array(&macroptr, "constraint_matrix", &ggd->data.normal_mat3[0][0]);
+			RNA_enum_set(&macroptr, "constraint_orientation", V3D_ORIENT_NORMAL);
 		}
 	}
 }
diff --git a/source/blender/editors/transform/transform_ops.c b/source/blender/editors/transform/transform_ops.c
index 46ab11cee7b..31a8c8559b6 100644
--- a/source/blender/editors/transform/transform_ops.c
+++ b/source/blender/editors/transform/transform_ops.c
@@ -515,8 +515,8 @@ static bool transform_poll_property(const bContext *UNUSED(C), wmOperator *op, c
 	/* Orientation/Constraints. */
 	{
 		/* Hide orientation axis if no constraints are set, since it wont be used. */
-		PropertyRNA *prop_con = RNA_struct_find_property(op->ptr, "constraint_axis");
-		if (prop_con && !RNA_property_is_set(op->ptr, prop_con)) {
+		PropertyRNA *prop_con = RNA_struct_find_property(op->ptr, "constraint_orientation");
+		if (prop_con != NULL && (prop_con != prop)) {
 			if (STRPREFIX(prop_id, "constraint")) {
 				return false;
 			}
@@ -565,6 +565,14 @@ void Transform_Properties(struct wmOperatorType *ot, int flags)
 		prop = RNA_def_float_matrix(ot->srna, "constraint_matrix", 3, 3, NULL, 0.0f, 0.0f, "Matrix", "", 0.0f, 0.0f);
 		RNA_def_property_flag(prop, PROP_HIDDEN | PROP_SKIP_SAVE);
 
+		/* Only use 'constraint_matrix' when 'constraint_matrix_orientation == constraint_orientation',
+		 * this allows us to reuse the orientation set by a gizmo for eg, without disabling the ability
+		 * to switch over to other orientations. */
+		prop = RNA_def_property(ot->srna, "constraint_matrix_orientation", PROP_ENUM, PROP_NONE);
+		RNA_def_property_ui_text(prop, "Matrix Orientation", "");
+		RNA_def_enum_funcs(prop, rna_TransformOrientation_itemf);
+		RNA_def_property_flag(prop, PROP_HIDDEN);
+
 		prop = RNA_def_property(ot->srna, "constraint_orientation", PROP_ENUM, PROP_NONE);
 		RNA_def_property_ui_text(prop, "Orientation", "Transformation orientation");
 		RNA_def_enum_funcs(prop, rna_TransformOrientation_itemf);