Fix gizmo extrude adjust & normal axis w/o faces

- Extruding edges/vertices on normals gave bad/strange results.
- The adjust gizmo used the orientation axis instead of the
  last-orientation (which could get out of sync).

1 files changed, 69 insertions, 78 deletions

diff --git a/source/blender/editors/mesh/editmesh_extrude_gizmo.c b/source/blender/editors/mesh/editmesh_extrude_gizmo.c
index 83c88b94ace..5de8e896ddf 100644
--- a/source/blender/editors/mesh/editmesh_extrude_gizmo.c
+++ b/source/blender/editors/mesh/editmesh_extrude_gizmo.c
@@ -83,7 +83,8 @@ typedef struct GizmoExtrudeGroup {
 
 	/* XYZ & normal. */
 	struct wmGizmo *invoke_xyz_no[4];
-	struct wmGizmo *adjust_xyz_no[5];
+	struct wmGizmo *adjust;
+	int             adjust_axis;
 
 	struct {
 		float normal_mat3[3][3];  /* use Z axis for normal. */
@@ -98,25 +99,24 @@ static void gizmo_mesh_extrude_orientation_matrix_set(
         struct GizmoExtrudeGroup *ggd, const float mat[3][3])
 {
 	for (int i = 0; i < 3; i++) {
-		/* Set orientation without location. */
-		for (int j = 0; j < 3; j++) {
-			copy_v3_v3(ggd->adjust_xyz_no[i]->matrix_basis[j], mat[j]);
-		}
-		/* nop when (i == 2). */
-		swap_v3_v3(ggd->adjust_xyz_no[i]->matrix_basis[i], ggd->adjust_xyz_no[i]->matrix_basis[2]);
-		/* Orient to normal gives generally less awkward results. */
-		if (ggd->data.orientation_type != V3D_MANIP_NORMAL) {
-			if (dot_v3v3(ggd->adjust_xyz_no[i]->matrix_basis[2], ggd->data.normal_mat3[2]) < 0.0f) {
-				negate_v3(ggd->adjust_xyz_no[i]->matrix_basis[2]);
-			}
-		}
 		mul_v3_v3fl(
 		        ggd->invoke_xyz_no[i]->matrix_offset[3],
-		        ggd->adjust_xyz_no[i]->matrix_basis[2],
+		        mat[i],
 		        (extrude_arrow_xyz_axis_scale * extrude_button_offset_scale) / extrude_button_scale);
 	}
 }
 
+static void gizmo_mesh_extrude_orientation_matrix_set_for_adjust(
+        struct GizmoExtrudeGroup *ggd, const float mat[3][3])
+{
+	/* Set orientation without location. */
+	for (int j = 0; j < 3; j++) {
+		copy_v3_v3(ggd->adjust->matrix_basis[j], mat[j]);
+	}
+	/* nop when (i == 2). */
+	swap_v3_v3(ggd->adjust->matrix_basis[ggd->adjust_axis], ggd->adjust->matrix_basis[2]);
+}
+
 static void gizmo_mesh_extrude_setup(const bContext *UNUSED(C), wmGizmoGroup *gzgroup)
 {
 	struct GizmoExtrudeGroup *ggd = MEM_callocN(sizeof(GizmoExtrudeGroup), __func__);
@@ -125,8 +125,8 @@ static void gizmo_mesh_extrude_setup(const bContext *UNUSED(C), wmGizmoGroup *gz
 	const wmGizmoType *gzt_arrow = WM_gizmotype_find("GIZMO_GT_arrow_3d", true);
 	const wmGizmoType *gzt_move = WM_gizmotype_find("GIZMO_GT_button_2d", true);
 
+	ggd->adjust = WM_gizmo_new_ptr(gzt_arrow, gzgroup, NULL);
 	for (int i = 0; i < 4; i++) {
-		ggd->adjust_xyz_no[i] = WM_gizmo_new_ptr(gzt_arrow, gzgroup, NULL);
 		ggd->invoke_xyz_no[i] = WM_gizmo_new_ptr(gzt_move, gzgroup, NULL);
 		ggd->invoke_xyz_no[i]->flag |= WM_GIZMO_DRAW_OFFSET_SCALE;
 	}
@@ -147,23 +147,16 @@ static void gizmo_mesh_extrude_setup(const bContext *UNUSED(C), wmGizmoGroup *gz
 
 	for (int i = 0; i < 3; i++) {
 		UI_GetThemeColor3fv(TH_AXIS_X + i, ggd->invoke_xyz_no[i]->color);
-		UI_GetThemeColor3fv(TH_AXIS_X + i, ggd->adjust_xyz_no[i]->color);
 	}
 	UI_GetThemeColor3fv(TH_GIZMO_PRIMARY, ggd->invoke_xyz_no[3]->color);
-	UI_GetThemeColor3fv(TH_GIZMO_PRIMARY, ggd->adjust_xyz_no[3]->color);
+	UI_GetThemeColor3fv(TH_GIZMO_PRIMARY, ggd->adjust->color);
 
 	for (int i = 0; i < 4; i++) {
 		WM_gizmo_set_scale(ggd->invoke_xyz_no[i], extrude_button_scale);
-		WM_gizmo_set_scale(ggd->adjust_xyz_no[i], extrude_arrow_scale);
 	}
-	WM_gizmo_set_scale(ggd->adjust_xyz_no[3], extrude_arrow_normal_axis_scale);
+	WM_gizmo_set_scale(ggd->adjust, extrude_arrow_scale);
 
-	for (int i = 0; i < 4; i++) {
-	}
-
-	for (int i = 0; i < 4; i++) {
-		WM_gizmo_set_flag(ggd->adjust_xyz_no[i], WM_GIZMO_DRAW_VALUE, true);
-	}
+	WM_gizmo_set_flag(ggd->adjust, WM_GIZMO_DRAW_VALUE, true);
 
 	/* XYZ & normal axis extrude. */
 	for (int i = 0; i < 4; i++) {
@@ -178,16 +171,11 @@ static void gizmo_mesh_extrude_setup(const bContext *UNUSED(C), wmGizmoGroup *gz
 	}
 
 	/* Adjust extrude. */
-	for (int i = 0; i < 4; i++) {
-		PointerRNA *ptr = WM_gizmo_operator_set(ggd->adjust_xyz_no[i], 0, ggd->ot_extrude, NULL);
-		{
-			bool constraint[3] = {0, 0, 0};
-			constraint[MIN2(i, 2)] = 1;
-			PointerRNA macroptr = RNA_pointer_get(ptr, "TRANSFORM_OT_translate");
-			RNA_boolean_set(&macroptr, "release_confirm", true);
-			RNA_boolean_set_array(&macroptr, "constraint_axis", constraint);
-		}
-		wmGizmoOpElem *gzop = WM_gizmo_operator_get(ggd->adjust_xyz_no[i], 0);
+	{
+		PointerRNA *ptr = WM_gizmo_operator_set(ggd->adjust, 0, ggd->ot_extrude, NULL);
+		PointerRNA macroptr = RNA_pointer_get(ptr, "TRANSFORM_OT_translate");
+		RNA_boolean_set(&macroptr, "release_confirm", true);
+		wmGizmoOpElem *gzop = WM_gizmo_operator_get(ggd->adjust, 0);
 		gzop->is_redo = true;
 	}
 }
@@ -198,8 +186,8 @@ static void gizmo_mesh_extrude_refresh(const bContext *C, wmGizmoGroup *gzgroup)
 
 	for (int i = 0; i < 4; i++) {
 		WM_gizmo_set_flag(ggd->invoke_xyz_no[i], WM_GIZMO_HIDDEN, true);
-		WM_gizmo_set_flag(ggd->adjust_xyz_no[i], WM_GIZMO_HIDDEN, true);
 	}
+	WM_gizmo_set_flag(ggd->adjust, WM_GIZMO_HIDDEN, true);
 
 	if (G.moving) {
 		return;
@@ -255,73 +243,76 @@ static void gizmo_mesh_extrude_refresh(const bContext *C, wmGizmoGroup *gzgroup)
 	        ggd->data.normal_mat3[2],
 	        (extrude_arrow_normal_axis_scale * extrude_button_offset_scale) / extrude_button_scale);
 
+	/* Adjust current operator. */
+	/* Don't use 'WM_operator_last_redo' because selection actions will be ignored. */
+	wmOperator *op = CTX_wm_manager(C)->operators.last;
+	bool has_redo = (op && op->type == ggd->ot_extrude);
+	wmOperator *op_transform = has_redo ? op->macro.last : NULL;
+
+	struct {
+		float constraint_matrix[3][3];
+		bool  constraint_axis[3];
+		float value[4];
+		bool  is_flip;
+	} redo;
+
+	if (has_redo) {
+		/* We can't access this from 'ot->last_properties'
+		 * because some properties use skip-save. */
+		RNA_float_get_array(op_transform->ptr, "constraint_matrix", &redo.constraint_matrix[0][0]);
+		RNA_boolean_get_array(op_transform->ptr, "constraint_axis", redo.constraint_axis);
+		RNA_float_get_array(op_transform->ptr, "value", redo.value);
+		for (int i = 0; i < 3; i++) {
+			if (redo.constraint_axis[i]) {
+				redo.is_flip = redo.value[i] < 0.0f;
+				ggd->adjust_axis = i;
+				break;
+			}
+		}
+	}
+
 	/* Needed for normal orientation. */
 	gizmo_mesh_extrude_orientation_matrix_set(ggd, tbounds.axis);
-	if (use_normal) {
-		copy_m4_m3(ggd->adjust_xyz_no[3]->matrix_basis, ggd->data.normal_mat3);
+	if (has_redo) {
+		gizmo_mesh_extrude_orientation_matrix_set_for_adjust(ggd, redo.constraint_matrix);
+		if (redo.is_flip) {
+			negate_v3(ggd->adjust->matrix_basis[2]);
+		}
 	}
 
 	/* Location. */
 	for (int i = 0; i < axis_len_used; i++) {
 		WM_gizmo_set_matrix_location(ggd->invoke_xyz_no[i], tbounds.center);
-		WM_gizmo_set_matrix_location(ggd->adjust_xyz_no[i], tbounds.center);
 	}
-
-	/* Adjust current operator. */
-	/* Don't use 'WM_operator_last_redo' because selection actions will be ignored. */
-	wmOperator *op = CTX_wm_manager(C)->operators.last;
-	bool has_redo = (op && op->type == ggd->ot_extrude);
+	WM_gizmo_set_matrix_location(ggd->adjust, tbounds.center);
 
 	/* Un-hide. */
 	for (int i = 0; i < axis_len_used; i++) {
 		WM_gizmo_set_flag(ggd->invoke_xyz_no[i], WM_GIZMO_HIDDEN, false);
-		WM_gizmo_set_flag(ggd->adjust_xyz_no[i], WM_GIZMO_HIDDEN, !has_redo);
 	}
+	WM_gizmo_set_flag(ggd->adjust, WM_GIZMO_HIDDEN, !has_redo);
 
 	/* Operator properties. */
 	if (use_normal) {
 		wmGizmoOpElem *gzop = WM_gizmo_operator_get(ggd->invoke_xyz_no[3], 0);
 		PointerRNA macroptr = RNA_pointer_get(&gzop->ptr, "TRANSFORM_OT_translate");
-		RNA_enum_set(&macroptr, "constraint_orientation", V3D_MANIP_NORMAL);
+		RNA_float_set_array(&macroptr, "constraint_matrix", &ggd->data.normal_mat3[0][0]);
 	}
 
 	/* Redo with current settings. */
 	if (has_redo) {
-		wmOperator *op_transform = op->macro.last;
-		float value[4];
-		RNA_float_get_array(op_transform->ptr, "value", value);
-		bool constraint_axis[3];
-		RNA_boolean_get_array(op_transform->ptr, "constraint_axis", constraint_axis);
-		int orientation_type = RNA_enum_get(op_transform->ptr, "constraint_orientation");
-
-		/* We could also access this from 'ot->last_properties' */
 		for (int i = 0; i < 4; i++) {
-			if ((i != 3) ?
-			    (orientation_type == ggd->data.orientation_type && constraint_axis[i]) :
-			    (orientation_type == V3D_MANIP_NORMAL && constraint_axis[2]))
-			{
-				wmGizmoOpElem *gzop = WM_gizmo_operator_get(ggd->adjust_xyz_no[i], 0);
-
-				PointerRNA macroptr = RNA_pointer_get(&gzop->ptr, "TRANSFORM_OT_translate");
-
-				RNA_float_set_array(&macroptr, "value", value);
-				RNA_boolean_set_array(&macroptr, "constraint_axis", constraint_axis);
-				RNA_enum_set(&macroptr, "constraint_orientation", orientation_type);
-			}
-			else {
-				/* TODO(campbell): ideally we could adjust all,
-				 * this is complicated by how operator redo and the transform macro works. */
-				WM_gizmo_set_flag(ggd->adjust_xyz_no[i], WM_GIZMO_HIDDEN, true);
-			}
+			RNA_enum_set(
+			        ggd->invoke_xyz_no[i]->ptr,
+			        "draw_options",
+			        (dot_v3v3(ggd->adjust->matrix_basis[2],
+			                  ggd->invoke_xyz_no[i]->matrix_offset[3]) > 0.98f) ? 0 : ED_GIZMO_BUTTON_SHOW_HELPLINE);
 		}
 	}
-
-	for (int i = 0; i < 4; i++) {
-		RNA_enum_set(
-		        ggd->invoke_xyz_no[i]->ptr,
-		        "draw_options",
-		        (ggd->adjust_xyz_no[i]->flag & WM_GIZMO_HIDDEN) ?
-		        ED_GIZMO_BUTTON_SHOW_HELPLINE : 0);
+	else {
+		for (int i = 0; i < 4; i++) {
+			RNA_enum_set(ggd->invoke_xyz_no[i]->ptr, "draw_options", ED_GIZMO_BUTTON_SHOW_HELPLINE);
+		}
 	}
 
 	/* TODO: skip calculating axis which wont be used (above). */