Proper fix T40156 Cycles Baking and applyRotation issues

This should be the final fix for the applyrotation issue. It baffles me
that the fix involves discarding the scale transformations for the
normals but it works so I'm happy with it.

Reviewers: campbellbarton

Differential Revision: https://developer.blender.org/D554

2 files changed, 14 insertions, 7 deletions

diff --git a/source/blender/render/extern/include/RE_bake.h b/source/blender/render/extern/include/RE_bake.h
index 6ae8300b9d1..b7cfe47c01b 100644
--- a/source/blender/render/extern/include/RE_bake.h
+++ b/source/blender/render/extern/include/RE_bake.h
@@ -61,9 +61,10 @@ typedef struct BakeHighPolyData {
 	struct ModifierData *tri_mod;
 	struct Mesh *me;
 	char restrict_flag;
-	float mat_high[4][4];
-	float imat_high[4][4];
-	float scale;
+
+	float obmat[4][4];
+	float imat[4][4];
+	float rotmat[4][4];
 } BakeHighPolyData;
 
 /* external_engine.c */
diff --git a/source/blender/render/intern/source/bake_api.c b/source/blender/render/intern/source/bake_api.c
index 375ab30aa35..43cf6878a81 100644
--- a/source/blender/render/intern/source/bake_api.c
+++ b/source/blender/render/intern/source/bake_api.c
@@ -235,10 +235,10 @@ static bool cast_ray_highpoly(
 		hits[i].dist = 10000.0f;
 
 		/* transform the ray from the world space to the highpoly space */
-		mul_v3_m4v3(co_high, highpoly[i].imat_high, co);
+		mul_v3_m4v3(co_high, highpoly[i].imat, co);
 
-		copy_v3_v3(dir_high, dir);
-		mul_transposed_mat3_m4_v3(highpoly[i].mat_high, dir_high);
+		/* rotates */
+		mul_v3_m4v3(dir_high, highpoly[i].rotmat, dir);
 		normalize_v3(dir_high);
 
 		/* cast ray */
@@ -248,7 +248,13 @@ static bool cast_ray_highpoly(
 			/* cull backface */
 			const float dot = dot_v3v3(dir_high, hits[i].no);
 			if (dot < 0.0f) {
-				float distance = hits[i].dist * highpoly[i].scale;
+				float distance;
+				float hit_world[3];
+
+				/* distance comparison in world space */
+				mul_v3_m4v3(hit_world, highpoly[i].obmat, hits[i].co);
+				distance = len_squared_v3v3(hit_world, co);
+
 				if (distance < hit_distance) {
 					hit_mesh = i;
 					hit_distance = distance;