fixes for using scaled cameras

- ED_view3d_from_m4() got incorrect rotation from scaled cameras, was noticable with smoothview transitions.
- when you lock the camera to the view, any view edits would reset the cameras scale to 1.
- another problem with view locking if the camera was scaled and had a parent, the parent would be transformed incorrectly.
- fly mode was chaning object scale a little over time, now restore after applying scale so it never changes.

2 files changed, 17 insertions, 4 deletions

diff --git a/source/blender/editors/space_view3d/view3d_edit.c b/source/blender/editors/space_view3d/view3d_edit.c
index 378bb181b35..e4aa44d7d78 100644
--- a/source/blender/editors/space_view3d/view3d_edit.c
+++ b/source/blender/editors/space_view3d/view3d_edit.c
@@ -134,6 +134,8 @@ bool ED_view3d_camera_lock_sync(View3D *v3d, RegionView3D *rv3d)
 
 		if ((U.uiflag & USER_CAM_LOCK_NO_PARENT) == 0 && (root_parent = v3d->camera->parent)) {
 			Object *ob_update;
+			float tmat[4][4];
+			float imat[4][4];
 			float view_mat[4][4];
 			float diff_mat[4][4];
 			float parent_mat[4][4];
@@ -144,8 +146,10 @@ bool ED_view3d_camera_lock_sync(View3D *v3d, RegionView3D *rv3d)
 
 			ED_view3d_to_m4(view_mat, rv3d->ofs, rv3d->viewquat, rv3d->dist);
 
-			invert_m4_m4(v3d->camera->imat, v3d->camera->obmat);
-			mul_m4_m4m4(diff_mat, view_mat, v3d->camera->imat);
+			normalize_m4_m4(tmat, v3d->camera->obmat);
+
+			invert_m4_m4(imat, tmat);
+			mul_m4_m4m4(diff_mat, view_mat, imat);
 
 			mul_m4_m4m4(parent_mat, diff_mat, root_parent->obmat);
 
@@ -161,9 +165,11 @@ bool ED_view3d_camera_lock_sync(View3D *v3d, RegionView3D *rv3d)
 			}
 		}
 		else {
+			/* always maintain the same scale */
+			const short protect_scale_all = (OB_LOCK_SCALEX | OB_LOCK_SCALEY | OB_LOCK_SCALEZ);
 			BKE_object_tfm_protected_backup(v3d->camera, &obtfm);
 			ED_view3d_to_object(v3d->camera, rv3d->ofs, rv3d->viewquat, rv3d->dist);
-			BKE_object_tfm_protected_restore(v3d->camera, &obtfm, v3d->camera->protectflag);
+			BKE_object_tfm_protected_restore(v3d->camera, &obtfm, v3d->camera->protectflag | protect_scale_all);
 
 			DAG_id_tag_update(&v3d->camera->id, OB_RECALC_OB);
 			WM_main_add_notifier(NC_OBJECT | ND_TRANSFORM, v3d->camera);
@@ -4167,7 +4173,8 @@ void ED_view3d_from_m4(float mat[4][4], float ofs[3], float quat[4], float *dist
 	/* Quat */
 	if (quat) {
 		float imat[4][4];
-		invert_m4_m4(imat, mat);
+		normalize_m4_m4(imat, mat);
+		invert_m4(imat);
 		mat4_to_quat(quat, imat);
 	}
 
diff --git a/source/blender/editors/space_view3d/view3d_fly.c b/source/blender/editors/space_view3d/view3d_fly.c
index 64dc0bef689..6bab677fb61 100644
--- a/source/blender/editors/space_view3d/view3d_fly.c
+++ b/source/blender/editors/space_view3d/view3d_fly.c
@@ -762,6 +762,10 @@ static void flyMoveCamera(bContext *C, RegionView3D *rv3d, FlyInfo *fly,
 	else {
 		float view_mat[4][4];
 		float size_mat[4][4];
+		float size_back[3];
+
+		/* even though we handle the size matrix, this still changes over time */
+		copy_v3_v3(size_back, v3d->camera->size);
 
 		ED_view3d_to_m4(view_mat, rv3d->ofs, rv3d->viewquat, rv3d->dist);
 		size_to_mat4(size_mat, v3d->camera->size);
@@ -769,6 +773,8 @@ static void flyMoveCamera(bContext *C, RegionView3D *rv3d, FlyInfo *fly,
 
 		BKE_object_apply_mat4(v3d->camera, view_mat, true, true);
 
+		copy_v3_v3(v3d->camera->size, size_back);
+
 		id_key = &v3d->camera->id;
 	}