Manipulator: use matrix to convert view coords

Was doing this with property get/set but this made view operations
require refreshing manipulator properties.
Simplify by operating on properties in their own space.

Also disable clamping for now since it assumes pixel-space.

2 files changed, 20 insertions, 16 deletions

diff --git a/source/blender/editors/manipulator_library/manipulator_library_utils.c b/source/blender/editors/manipulator_library/manipulator_library_utils.c
index e0e326a2b2f..f521bd9dbf4 100644
--- a/source/blender/editors/manipulator_library/manipulator_library_utils.c
+++ b/source/blender/editors/manipulator_library/manipulator_library_utils.c
@@ -172,19 +172,20 @@ bool manipulator_window_project_2d(
         bContext *C, const struct wmManipulator *mpr, const float mval[2], int axis, bool use_offset,
         float r_co[2])
 {
+	float mat[4][4];
+	if (use_offset) {
+		mul_m4_m4m4(mat, mpr->matrix_basis, mpr->matrix_offset);
+	}
+	else {
+		copy_m4_m4(mat, mpr->matrix_basis);
+	}
+
 	/* rotate mouse in relation to the center and relocate it */
 	if (mpr->parent_mgroup->type->flag & WM_MANIPULATORGROUPTYPE_3D) {
 		/* For 3d views, transform 2D mouse pos onto plane. */
 		View3D *v3d = CTX_wm_view3d(C);
 		ARegion *ar = CTX_wm_region(C);
 
-		float mat[4][4];
-		if (use_offset) {
-			mul_m4_m4m4(mat, mpr->matrix_basis, mpr->matrix_offset);
-		}
-		else {
-			copy_m4_m4(mat, mpr->matrix_basis);
-		}
 		float plane[4];
 
 		plane_from_point_normal_v3(plane, mat[3], mat[2]);
@@ -207,11 +208,11 @@ bool manipulator_window_project_2d(
 		return false;
 	}
 	else {
-		sub_v2_v2v2(r_co, mval, mpr->matrix_basis[3]);
-		if (use_offset) {
-			r_co[0] -= mpr->matrix_offset[3][0];
-			r_co[1] -= mpr->matrix_offset[3][1];
-		}
+		float co[3] = {mval[0], mval[1], 0.0f};
+		float imat[4][4];
+		invert_m4_m4(imat, mat);
+		mul_m4_v3(imat, co);
+		copy_v2_v2(r_co, co);
 		return true;
 	}
 }
diff --git a/source/blender/editors/manipulator_library/manipulator_types/cage2d_manipulator.c b/source/blender/editors/manipulator_library/manipulator_types/cage2d_manipulator.c
index dbfac595fc1..5a0c45d5953 100644
--- a/source/blender/editors/manipulator_library/manipulator_types/cage2d_manipulator.c
+++ b/source/blender/editors/manipulator_library/manipulator_types/cage2d_manipulator.c
@@ -483,12 +483,14 @@ static void manipulator_rect_transform_modal(
         eWM_ManipulatorTweak UNUSED(tweak_flag))
 {
 	const int transform_flag = RNA_enum_get(mpr->ptr, "transform");
-	const bool use_clamp = (mpr->parent_mgroup->type->flag & WM_MANIPULATORGROUPTYPE_3D) == 0;
 	const bool pivot_center = (transform_flag & ED_MANIPULATOR_RECT_TRANSFORM_FLAG_TRANSLATE) == 0;
 	RectTransformInteraction *data = mpr->interaction_data;
+#if 0
 	/* needed here as well in case clamping occurs */
+	const bool use_clamp = (mpr->parent_mgroup->type->flag & WM_MANIPULATORGROUPTYPE_3D) == 0;
 	const float orig_ofx = mpr->matrix_offset[3][0];
 	const float orig_ofy = mpr->matrix_offset[3][1];
+#endif
 
 	float point_local[2];
 
@@ -551,14 +553,14 @@ static void manipulator_rect_transform_modal(
 		BLI_assert(0);
 	}
 
+	/* TODO(campbell): Complicates things too much since not all scales are in the same space. */
+#if 0
 	/* clamping - make sure manipulator is at least 5 pixels wide */
 	if (use_clamp == false) {
 		/* pass */
 	}
 	else if (transform_flag & ED_MANIPULATOR_RECT_TRANSFORM_FLAG_SCALE_UNIFORM) {
-		if (scale[0] < MANIPULATOR_RECT_MIN_WIDTH / dims[1] ||
-		    scale[0] < MANIPULATOR_RECT_MIN_WIDTH / dims[0])
-		{
+		if (scale[0] < MANIPULATOR_RECT_MIN_WIDTH / max_ff(dims[0], dims[1])) {
 			scale[0] = max_ff(MANIPULATOR_RECT_MIN_WIDTH / dims[1], MANIPULATOR_RECT_MIN_WIDTH / dims[0]);
 			mpr->matrix_offset[3][0] = orig_ofx;
 			mpr->matrix_offset[3][1] = orig_ofy;
@@ -574,6 +576,7 @@ static void manipulator_rect_transform_modal(
 			mpr->matrix_offset[3][1] = orig_ofy;
 		}
 	}
+#endif
 
 	{
 		wmManipulatorProperty *mpr_prop = WM_manipulator_target_property_find(mpr, "scale");