Manipulator: use matrix to convert view coords

Was doing this with property get/set but this made view operations
require refreshing manipulator properties.
Simplify by operating on properties in their own space.

Also disable clamping for now since it assumes pixel-space.

3 files changed, 34 insertions, 33 deletions

diff --git a/source/blender/editors/manipulator_library/manipulator_library_utils.c b/source/blender/editors/manipulator_library/manipulator_library_utils.c
index e0e326a2b2f..f521bd9dbf4 100644
--- a/source/blender/editors/manipulator_library/manipulator_library_utils.c
+++ b/source/blender/editors/manipulator_library/manipulator_library_utils.c
@@ -172,19 +172,20 @@ bool manipulator_window_project_2d(
         bContext *C, const struct wmManipulator *mpr, const float mval[2], int axis, bool use_offset,
         float r_co[2])
 {
+	float mat[4][4];
+	if (use_offset) {
+		mul_m4_m4m4(mat, mpr->matrix_basis, mpr->matrix_offset);
+	}
+	else {
+		copy_m4_m4(mat, mpr->matrix_basis);
+	}
+
 	/* rotate mouse in relation to the center and relocate it */
 	if (mpr->parent_mgroup->type->flag & WM_MANIPULATORGROUPTYPE_3D) {
 		/* For 3d views, transform 2D mouse pos onto plane. */
 		View3D *v3d = CTX_wm_view3d(C);
 		ARegion *ar = CTX_wm_region(C);
 
-		float mat[4][4];
-		if (use_offset) {
-			mul_m4_m4m4(mat, mpr->matrix_basis, mpr->matrix_offset);
-		}
-		else {
-			copy_m4_m4(mat, mpr->matrix_basis);
-		}
 		float plane[4];
 
 		plane_from_point_normal_v3(plane, mat[3], mat[2]);
@@ -207,11 +208,11 @@ bool manipulator_window_project_2d(
 		return false;
 	}
 	else {
-		sub_v2_v2v2(r_co, mval, mpr->matrix_basis[3]);
-		if (use_offset) {
-			r_co[0] -= mpr->matrix_offset[3][0];
-			r_co[1] -= mpr->matrix_offset[3][1];
-		}
+		float co[3] = {mval[0], mval[1], 0.0f};
+		float imat[4][4];
+		invert_m4_m4(imat, mat);
+		mul_m4_v3(imat, co);
+		copy_v2_v2(r_co, co);
 		return true;
 	}
 }
diff --git a/source/blender/editors/manipulator_library/manipulator_types/cage2d_manipulator.c b/source/blender/editors/manipulator_library/manipulator_types/cage2d_manipulator.c
index dbfac595fc1..5a0c45d5953 100644
--- a/source/blender/editors/manipulator_library/manipulator_types/cage2d_manipulator.c
+++ b/source/blender/editors/manipulator_library/manipulator_types/cage2d_manipulator.c
@@ -483,12 +483,14 @@ static void manipulator_rect_transform_modal(
         eWM_ManipulatorTweak UNUSED(tweak_flag))
 {
 	const int transform_flag = RNA_enum_get(mpr->ptr, "transform");
-	const bool use_clamp = (mpr->parent_mgroup->type->flag & WM_MANIPULATORGROUPTYPE_3D) == 0;
 	const bool pivot_center = (transform_flag & ED_MANIPULATOR_RECT_TRANSFORM_FLAG_TRANSLATE) == 0;
 	RectTransformInteraction *data = mpr->interaction_data;
+#if 0
 	/* needed here as well in case clamping occurs */
+	const bool use_clamp = (mpr->parent_mgroup->type->flag & WM_MANIPULATORGROUPTYPE_3D) == 0;
 	const float orig_ofx = mpr->matrix_offset[3][0];
 	const float orig_ofy = mpr->matrix_offset[3][1];
+#endif
 
 	float point_local[2];
 
@@ -551,14 +553,14 @@ static void manipulator_rect_transform_modal(
 		BLI_assert(0);
 	}
 
+	/* TODO(campbell): Complicates things too much since not all scales are in the same space. */
+#if 0
 	/* clamping - make sure manipulator is at least 5 pixels wide */
 	if (use_clamp == false) {
 		/* pass */
 	}
 	else if (transform_flag & ED_MANIPULATOR_RECT_TRANSFORM_FLAG_SCALE_UNIFORM) {
-		if (scale[0] < MANIPULATOR_RECT_MIN_WIDTH / dims[1] ||
-		    scale[0] < MANIPULATOR_RECT_MIN_WIDTH / dims[0])
-		{
+		if (scale[0] < MANIPULATOR_RECT_MIN_WIDTH / max_ff(dims[0], dims[1])) {
 			scale[0] = max_ff(MANIPULATOR_RECT_MIN_WIDTH / dims[1], MANIPULATOR_RECT_MIN_WIDTH / dims[0]);
 			mpr->matrix_offset[3][0] = orig_ofx;
 			mpr->matrix_offset[3][1] = orig_ofy;
@@ -574,6 +576,7 @@ static void manipulator_rect_transform_modal(
 			mpr->matrix_offset[3][1] = orig_ofy;
 		}
 	}
+#endif
 
 	{
 		wmManipulatorProperty *mpr_prop = WM_manipulator_target_property_find(mpr, "scale");
diff --git a/source/blender/editors/space_view3d/view3d_manipulator_camera.c b/source/blender/editors/space_view3d/view3d_manipulator_camera.c
index 856a697cf45..fe5ca5d2df9 100644
--- a/source/blender/editors/space_view3d/view3d_manipulator_camera.c
+++ b/source/blender/editors/space_view3d/view3d_manipulator_camera.c
@@ -250,11 +250,10 @@ static void manipulator_render_border_prop_size_get(
 	float *value = value_p;
 	BLI_assert(mpr_prop->type->array_length == 2);
 	struct CameraViewWidgetGroup *viewgroup = mpr_prop->custom_func.user_data;
-	const rctf *view_border = &viewgroup->state.view_border;
 	const rctf *border = viewgroup->state.edit_border;
 
-	value[0] = BLI_rctf_size_x(border) * BLI_rctf_size_x(view_border);
-	value[1] = BLI_rctf_size_y(border) * BLI_rctf_size_y(view_border);
+	value[0] = BLI_rctf_size_x(border);
+	value[1] = BLI_rctf_size_y(border);
 }
 
 static void manipulator_render_border_prop_size_set(
@@ -263,14 +262,10 @@ static void manipulator_render_border_prop_size_set(
 {
 	const float *value = value_p;
 	struct CameraViewWidgetGroup *viewgroup = mpr_prop->custom_func.user_data;
-	const rctf *view_border = &viewgroup->state.view_border;
 	rctf *border = viewgroup->state.edit_border;
 	BLI_assert(mpr_prop->type->array_length == 2);
 
-	BLI_rctf_resize(
-	        border,
-	        value[0] / BLI_rctf_size_x(view_border),
-	        value[1] / BLI_rctf_size_y(view_border));
+	BLI_rctf_resize(border, value[0], value[1]);
 	BLI_rctf_isect(&(rctf){.xmin = 0, .ymin = 0, .xmax = 1, .ymax = 1}, border, border);
 }
 
@@ -282,11 +277,10 @@ static void manipulator_render_border_prop_offset_get(
 	float *value = value_p;
 	BLI_assert(mpr_prop->type->array_length == 2);
 	struct CameraViewWidgetGroup *viewgroup = mpr_prop->custom_func.user_data;
-	const rctf *view_border = &viewgroup->state.view_border;
 	const rctf *border = viewgroup->state.edit_border;
 
-	value[0] = (BLI_rctf_cent_x(border) * BLI_rctf_size_x(view_border)) + view_border->xmin;
-	value[1] = (BLI_rctf_cent_y(border) * BLI_rctf_size_y(view_border)) + view_border->ymin;
+	value[0] = BLI_rctf_cent_x(border);
+	value[1] = BLI_rctf_cent_y(border);
 }
 
 static void manipulator_render_border_prop_offset_set(
@@ -295,15 +289,11 @@ static void manipulator_render_border_prop_offset_set(
 {
 	const float *value = value_p;
 	struct CameraViewWidgetGroup *viewgroup = mpr_prop->custom_func.user_data;
-	const rctf *view_border = &viewgroup->state.view_border;
 	rctf *border = viewgroup->state.edit_border;
 
 	BLI_assert(mpr_prop->type->array_length == 2);
 
-	BLI_rctf_recenter(
-	        border,
-	        (value[0] - view_border->xmin) / BLI_rctf_size_x(view_border),
-	        (value[1] - view_border->ymin) / BLI_rctf_size_y(view_border));
+	BLI_rctf_recenter(border, value[0], value[1]);
 	BLI_rctf_isect(&(rctf){.xmin = 0, .ymin = 0, .xmax = 1, .ymax = 1}, border, border);
 }
 
@@ -343,14 +333,21 @@ static void WIDGETGROUP_camera_view_draw_prepare(const bContext *C, wmManipulato
 
 	ARegion *ar = CTX_wm_region(C);
 	RegionView3D *rv3d = ar->regiondata;
-	View3D *v3d = CTX_wm_view3d(C);
 	if (rv3d->persp == RV3D_CAMOB) {
 		Scene *scene = CTX_data_scene(C);
+		View3D *v3d = CTX_wm_view3d(C);
 		ED_view3d_calc_camera_border(scene, ar, v3d, rv3d, &viewgroup->state.view_border, false);
 	}
 	else {
 		viewgroup->state.view_border = (rctf){.xmin = 0, .ymin = 0, .xmax = ar->winx, .ymax = ar->winy};
 	}
+
+	wmManipulator *mpr = viewgroup->border;
+	unit_m4(mpr->matrix_basis);
+	mul_v3_fl(mpr->matrix_basis[0], BLI_rctf_size_x(&viewgroup->state.view_border));
+	mul_v3_fl(mpr->matrix_basis[1], BLI_rctf_size_y(&viewgroup->state.view_border));
+	mpr->matrix_basis[3][0] = viewgroup->state.view_border.xmin;
+	mpr->matrix_basis[3][1] = viewgroup->state.view_border.ymin;
 }
 
 static void WIDGETGROUP_camera_view_refresh(const bContext *C, wmManipulatorGroup *mgroup)