Cycles: Replace object index hack with actual checks for SD_TRANSFORM_APPLIED

Using ones complement for detecting if transform has been applied was confusing
and led to several bugs. With this proper checks are made.

Also added a few transforms where they were missing, mostly affecting baking
and displacement when `P` is used in the shader (previously `P` was in the
wrong space for these shaders)

Also removed `TIME_INVALID` as this may have resulted in incorrect
transforms in some cases.

Reviewed By: brecht

Differential Revision: https://developer.blender.org/D2192

1 files changed, 18 insertions, 20 deletions

diff --git a/intern/cycles/kernel/kernel_shader.h b/intern/cycles/kernel/kernel_shader.h
index 079bea30bdd..e59f9a6fadc 100644
--- a/intern/cycles/kernel/kernel_shader.h
+++ b/intern/cycles/kernel/kernel_shader.h
@@ -241,7 +241,8 @@ ccl_device_inline void shader_setup_from_sample(KernelGlobals *kg,
                                                 const float3 I,
                                                 int shader, int object, int prim,
                                                 float u, float v, float t,
-                                                float time)
+                                                float time,
+                                                bool object_space)
 {
 	/* vectors */
 	ccl_fetch(sd, P) = P;
@@ -263,20 +264,6 @@ ccl_device_inline void shader_setup_from_sample(KernelGlobals *kg,
 #endif
 	ccl_fetch(sd, ray_length) = t;
 
-	/* detect instancing, for non-instanced the object index is -object-1 */
-#ifdef __INSTANCING__
-	bool instanced = false;
-
-	if(ccl_fetch(sd, prim) != PRIM_NONE) {
-		if(ccl_fetch(sd, object) >= 0)
-			instanced = true;
-		else
-#endif
-			ccl_fetch(sd, object) = ~ccl_fetch(sd, object);
-#ifdef __INSTANCING__
-	}
-#endif
-
 	ccl_fetch(sd, flag) = kernel_tex_fetch(__shader_flag, (ccl_fetch(sd, shader) & SHADER_MASK)*2);
 	if(ccl_fetch(sd, object) != OBJECT_NONE) {
 		ccl_fetch(sd, flag) |= kernel_tex_fetch(__object_flag, ccl_fetch(sd, object));
@@ -290,14 +277,23 @@ ccl_device_inline void shader_setup_from_sample(KernelGlobals *kg,
 	}
 #endif
 
+	/* transform into world space */
+	if(object_space) {
+		object_position_transform_auto(kg, sd, &ccl_fetch(sd, P));
+		object_normal_transform_auto(kg, sd, &ccl_fetch(sd, Ng));
+		ccl_fetch(sd, N) = ccl_fetch(sd, Ng);
+		object_dir_transform_auto(kg, sd, &ccl_fetch(sd, I));
+	}
+
 	if(ccl_fetch(sd, type) & PRIMITIVE_TRIANGLE) {
 		/* smooth normal */
 		if(ccl_fetch(sd, shader) & SHADER_SMOOTH_NORMAL) {
 			ccl_fetch(sd, N) = triangle_smooth_normal(kg, ccl_fetch(sd, prim), ccl_fetch(sd, u), ccl_fetch(sd, v));
 
 #ifdef __INSTANCING__
-			if(instanced)
+			if(!(ccl_fetch(sd, flag) & SD_TRANSFORM_APPLIED)) {
 				object_normal_transform_auto(kg, sd, &ccl_fetch(sd, N));
+			}
 #endif
 		}
 
@@ -306,7 +302,7 @@ ccl_device_inline void shader_setup_from_sample(KernelGlobals *kg,
 		triangle_dPdudv(kg, ccl_fetch(sd, prim), &ccl_fetch(sd, dPdu), &ccl_fetch(sd, dPdv));
 
 #  ifdef __INSTANCING__
-		if(instanced) {
+		if(!(ccl_fetch(sd, flag) & SD_TRANSFORM_APPLIED)) {
 			object_dir_transform_auto(kg, sd, &ccl_fetch(sd, dPdu));
 			object_dir_transform_auto(kg, sd, &ccl_fetch(sd, dPdv));
 		}
@@ -357,9 +353,11 @@ ccl_device void shader_setup_from_displace(KernelGlobals *kg, ShaderData *sd,
 	/* force smooth shading for displacement */
 	shader |= SHADER_SMOOTH_NORMAL;
 
-	/* watch out: no instance transform currently */
-
-	shader_setup_from_sample(kg, sd, P, Ng, I, shader, object, prim, u, v, 0.0f, TIME_INVALID);
+	shader_setup_from_sample(kg, sd,
+	                         P, Ng, I,
+	                         shader, object, prim,
+	                         u, v, 0.0f, 0.5f,
+	                         !(kernel_tex_fetch(__object_flag, object) & SD_TRANSFORM_APPLIED));
 }
 
 /* ShaderData setup from ray into background */