Cycles: Replace object index hack with actual checks for SD_TRANSFORM_APPLIED

Using ones complement for detecting if transform has been applied was confusing
and led to several bugs. With this proper checks are made.

Also added a few transforms where they were missing, mostly affecting baking
and displacement when `P` is used in the shader (previously `P` was in the
wrong space for these shaders)

Also removed `TIME_INVALID` as this may have resulted in incorrect
transforms in some cases.

Reviewed By: brecht

Differential Revision: https://developer.blender.org/D2192

14 files changed, 61 insertions, 44 deletions

diff --git a/intern/cycles/blender/blender_session.cpp b/intern/cycles/blender/blender_session.cpp
index 80a16e0ef27..c250a54f259 100644
--- a/intern/cycles/blender/blender_session.cpp
+++ b/intern/cycles/blender/blender_session.cpp
@@ -681,8 +681,7 @@ void BlenderSession::bake(BL::Object& b_object,
 		}
 	}
 
-	/* when used, non-instanced convention: object = ~object */
-	int object = ~object_index;
+	int object = object_index;
 
 	BakeData *bake_data = scene->bake_manager->init(object, tri_offset, num_pixels);
 
diff --git a/intern/cycles/kernel/geom/geom_object.h b/intern/cycles/kernel/geom/geom_object.h
index 883c5dc100d..4f72c5b7b93 100644
--- a/intern/cycles/kernel/geom/geom_object.h
+++ b/intern/cycles/kernel/geom/geom_object.h
@@ -564,6 +564,15 @@ ccl_device_inline void bvh_instance_motion_pop_factor(KernelGlobals *kg,
  */
 
 #ifdef __KERNEL_OPENCL__
+ccl_device_inline void object_position_transform_addrspace(KernelGlobals *kg,
+                                                         const ShaderData *sd,
+                                                         ccl_addr_space float3 *P)
+{
+	float3 private_P = *P;
+	object_position_transform(kg, sd, &private_P);
+	*P = private_P;
+}
+
 ccl_device_inline void object_dir_transform_addrspace(KernelGlobals *kg,
                                                       const ShaderData *sd,
                                                       ccl_addr_space float3 *D)
@@ -584,9 +593,11 @@ ccl_device_inline void object_normal_transform_addrspace(KernelGlobals *kg,
 #endif
 
 #ifndef __KERNEL_OPENCL__
+#  define object_position_transform_auto object_position_transform
 #  define object_dir_transform_auto object_dir_transform
 #  define object_normal_transform_auto object_normal_transform
 #else
+#  define object_position_transform_auto object_position_transform_addrspace
 #  define object_dir_transform_auto object_dir_transform_addrspace
 #  define object_normal_transform_auto object_normal_transform_addrspace
 #endif
diff --git a/intern/cycles/kernel/geom/geom_triangle.h b/intern/cycles/kernel/geom/geom_triangle.h
index d3289d6572c..8bd01e1008d 100644
--- a/intern/cycles/kernel/geom/geom_triangle.h
+++ b/intern/cycles/kernel/geom/geom_triangle.h
@@ -52,8 +52,8 @@ ccl_device_inline void triangle_point_normal(KernelGlobals *kg, int object, int
 	float t = 1.0f - u - v;
 	*P = (u*v0 + v*v1 + t*v2);
 
-	/* get object flags, instance-aware */
-	int object_flag = kernel_tex_fetch(__object_flag, object >= 0 ? object : ~object);
+	/* get object flags */
+	int object_flag = kernel_tex_fetch(__object_flag, object);
 
 	/* compute normal */
 	if(object_flag & SD_NEGATIVE_SCALE_APPLIED)
diff --git a/intern/cycles/kernel/kernel_bake.h b/intern/cycles/kernel/kernel_bake.h
index bfbf73df54f..fd9207fd69f 100644
--- a/intern/cycles/kernel/kernel_bake.h
+++ b/intern/cycles/kernel/kernel_bake.h
@@ -41,7 +41,7 @@ ccl_device_inline void compute_light_pass(KernelGlobals *kg,
 	ray.D = -sd->Ng;
 	ray.t = FLT_MAX;
 #ifdef __CAMERA_MOTION__
-	ray.time = TIME_INVALID;
+	ray.time = 0.5f;
 #endif
 
 	/* init radiance */
@@ -313,15 +313,14 @@ ccl_device void kernel_bake_evaluate(KernelGlobals *kg, ccl_global uint4 *input,
 
 	triangle_point_normal(kg, object, prim, u, v, &P, &Ng, &shader);
 
-	/* dummy initilizations copied from SHADER_EVAL_DISPLACE */
-	float3 I = Ng;
-	float t = 1.0f;
-	float time = TIME_INVALID;
-
 	/* light passes */
 	PathRadiance L;
 
-	shader_setup_from_sample(kg, &sd, P, Ng, I, shader, object, prim, u, v, t, time);
+	shader_setup_from_sample(kg, &sd,
+	                         P, Ng, Ng,
+	                         shader, object, prim,
+	                         u, v, 1.0f, 0.5f,
+	                         !(kernel_tex_fetch(__object_flag, object) & SD_TRANSFORM_APPLIED));
 	sd.I = sd.N;
 
 	/* update differentials */
@@ -525,6 +524,8 @@ ccl_device void kernel_shader_evaluate(KernelGlobals *kg,
 		float3 P = sd.P;
 		shader_eval_displacement(kg, &sd, &state, SHADER_CONTEXT_MAIN);
 		out = sd.P - P;
+
+		object_inverse_dir_transform(kg, &sd, &out);
 	}
 	else { // SHADER_EVAL_BACKGROUND
 		/* setup ray */
diff --git a/intern/cycles/kernel/kernel_camera.h b/intern/cycles/kernel/kernel_camera.h
index 88514de514c..8e0aa678cac 100644
--- a/intern/cycles/kernel/kernel_camera.h
+++ b/intern/cycles/kernel/kernel_camera.h
@@ -321,7 +321,7 @@ ccl_device_inline void camera_sample(KernelGlobals *kg,
 #ifdef __CAMERA_MOTION__
 	/* motion blur */
 	if(kernel_data.cam.shuttertime == -1.0f) {
-		ray->time = TIME_INVALID;
+		ray->time = 0.5f;
 	}
 	else {
 		/* TODO(sergey): Such lookup is unneeded when there's rolling shutter
diff --git a/intern/cycles/kernel/kernel_emission.h b/intern/cycles/kernel/kernel_emission.h
index 149ac3ed4f9..457887f9c78 100644
--- a/intern/cycles/kernel/kernel_emission.h
+++ b/intern/cycles/kernel/kernel_emission.h
@@ -50,7 +50,10 @@ ccl_device_noinline float3 direct_emissive_eval(KernelGlobals *kg,
 	else
 #endif
 	{
-		shader_setup_from_sample(kg, emission_sd, ls->P, ls->Ng, I, ls->shader, ls->object, ls->prim, ls->u, ls->v, t, time);
+		shader_setup_from_sample(kg, emission_sd,
+		                         ls->P, ls->Ng, I,
+		                         ls->shader, ls->object, ls->prim,
+		                         ls->u, ls->v, t, time, false);
 
 		ls->Ng = ccl_fetch(emission_sd, Ng);
 
diff --git a/intern/cycles/kernel/kernel_light.h b/intern/cycles/kernel/kernel_light.h
index 1e829eaa1fa..1e99f2cb99f 100644
--- a/intern/cycles/kernel/kernel_light.h
+++ b/intern/cycles/kernel/kernel_light.h
@@ -747,7 +747,7 @@ ccl_device void object_transform_light_sample(KernelGlobals *kg, LightSample *ls
 {
 #ifdef __INSTANCING__
 	/* instance transform */
-	if(object >= 0) {
+	if(!(kernel_tex_fetch(__object_flag, object) & SD_TRANSFORM_APPLIED)) {
 #  ifdef __OBJECT_MOTION__
 		Transform itfm;
 		Transform tfm = object_fetch_transform_motion_test(kg, object, time, &itfm);
diff --git a/intern/cycles/kernel/kernel_shader.h b/intern/cycles/kernel/kernel_shader.h
index 079bea30bdd..e59f9a6fadc 100644
--- a/intern/cycles/kernel/kernel_shader.h
+++ b/intern/cycles/kernel/kernel_shader.h
@@ -241,7 +241,8 @@ ccl_device_inline void shader_setup_from_sample(KernelGlobals *kg,
                                                 const float3 I,
                                                 int shader, int object, int prim,
                                                 float u, float v, float t,
-                                                float time)
+                                                float time,
+                                                bool object_space)
 {
 	/* vectors */
 	ccl_fetch(sd, P) = P;
@@ -263,20 +264,6 @@ ccl_device_inline void shader_setup_from_sample(KernelGlobals *kg,
 #endif
 	ccl_fetch(sd, ray_length) = t;
 
-	/* detect instancing, for non-instanced the object index is -object-1 */
-#ifdef __INSTANCING__
-	bool instanced = false;
-
-	if(ccl_fetch(sd, prim) != PRIM_NONE) {
-		if(ccl_fetch(sd, object) >= 0)
-			instanced = true;
-		else
-#endif
-			ccl_fetch(sd, object) = ~ccl_fetch(sd, object);
-#ifdef __INSTANCING__
-	}
-#endif
-
 	ccl_fetch(sd, flag) = kernel_tex_fetch(__shader_flag, (ccl_fetch(sd, shader) & SHADER_MASK)*2);
 	if(ccl_fetch(sd, object) != OBJECT_NONE) {
 		ccl_fetch(sd, flag) |= kernel_tex_fetch(__object_flag, ccl_fetch(sd, object));
@@ -290,14 +277,23 @@ ccl_device_inline void shader_setup_from_sample(KernelGlobals *kg,
 	}
 #endif
 
+	/* transform into world space */
+	if(object_space) {
+		object_position_transform_auto(kg, sd, &ccl_fetch(sd, P));
+		object_normal_transform_auto(kg, sd, &ccl_fetch(sd, Ng));
+		ccl_fetch(sd, N) = ccl_fetch(sd, Ng);
+		object_dir_transform_auto(kg, sd, &ccl_fetch(sd, I));
+	}
+
 	if(ccl_fetch(sd, type) & PRIMITIVE_TRIANGLE) {
 		/* smooth normal */
 		if(ccl_fetch(sd, shader) & SHADER_SMOOTH_NORMAL) {
 			ccl_fetch(sd, N) = triangle_smooth_normal(kg, ccl_fetch(sd, prim), ccl_fetch(sd, u), ccl_fetch(sd, v));
 
 #ifdef __INSTANCING__
-			if(instanced)
+			if(!(ccl_fetch(sd, flag) & SD_TRANSFORM_APPLIED)) {
 				object_normal_transform_auto(kg, sd, &ccl_fetch(sd, N));
+			}
 #endif
 		}
 
@@ -306,7 +302,7 @@ ccl_device_inline void shader_setup_from_sample(KernelGlobals *kg,
 		triangle_dPdudv(kg, ccl_fetch(sd, prim), &ccl_fetch(sd, dPdu), &ccl_fetch(sd, dPdv));
 
 #  ifdef __INSTANCING__
-		if(instanced) {
+		if(!(ccl_fetch(sd, flag) & SD_TRANSFORM_APPLIED)) {
 			object_dir_transform_auto(kg, sd, &ccl_fetch(sd, dPdu));
 			object_dir_transform_auto(kg, sd, &ccl_fetch(sd, dPdv));
 		}
@@ -357,9 +353,11 @@ ccl_device void shader_setup_from_displace(KernelGlobals *kg, ShaderData *sd,
 	/* force smooth shading for displacement */
 	shader |= SHADER_SMOOTH_NORMAL;
 
-	/* watch out: no instance transform currently */
-
-	shader_setup_from_sample(kg, sd, P, Ng, I, shader, object, prim, u, v, 0.0f, TIME_INVALID);
+	shader_setup_from_sample(kg, sd,
+	                         P, Ng, I,
+	                         shader, object, prim,
+	                         u, v, 0.0f, 0.5f,
+	                         !(kernel_tex_fetch(__object_flag, object) & SD_TRANSFORM_APPLIED));
 }
 
 /* ShaderData setup from ray into background */
diff --git a/intern/cycles/kernel/kernel_types.h b/intern/cycles/kernel/kernel_types.h
index 0646148f6a0..1156b41fce8 100644
--- a/intern/cycles/kernel/kernel_types.h
+++ b/intern/cycles/kernel/kernel_types.h
@@ -42,7 +42,6 @@ CCL_NAMESPACE_BEGIN
 #define RAMP_TABLE_SIZE		256
 #define SHUTTER_TABLE_SIZE		256
 #define PARTICLE_SIZE 		5
-#define TIME_INVALID		FLT_MAX
 
 #define BSSRDF_MIN_RADIUS			1e-8f
 #define BSSRDF_MAX_HITS				4
diff --git a/intern/cycles/kernel/shaders/node_output_displacement.osl b/intern/cycles/kernel/shaders/node_output_displacement.osl
index d0688cfda8d..294b8dd6bf2 100644
--- a/intern/cycles/kernel/shaders/node_output_displacement.osl
+++ b/intern/cycles/kernel/shaders/node_output_displacement.osl
@@ -18,6 +18,8 @@
 
 displacement node_output_displacement(float Displacement = 0.0)
 {
-	P += N * Displacement * 0.1;  /* todo: get rid of this factor */
+	vector dP = normalize(transform("object", N));
+	dP *= Displacement * 0.1; /* todo: get rid of this factor */
+	P += transform("object", "world", dP);
 }
 
diff --git a/intern/cycles/kernel/svm/svm.h b/intern/cycles/kernel/svm/svm.h
index 9ca8917d327..88ec7fe6fcc 100644
--- a/intern/cycles/kernel/svm/svm.h
+++ b/intern/cycles/kernel/svm/svm.h
@@ -261,7 +261,7 @@ ccl_device_noinline void svm_eval_nodes(KernelGlobals *kg, ShaderData *sd, ccl_a
 				svm_node_geometry_bump_dy(kg, sd, stack, node.y, node.z);
 				break;
 			case NODE_SET_DISPLACEMENT:
-				svm_node_set_displacement(sd, stack, node.y);
+				svm_node_set_displacement(kg, sd, stack, node.y);
 				break;
 #  endif  /* NODES_FEATURE(NODE_FEATURE_BUMP) */
 #  ifdef __TEXTURES__
diff --git a/intern/cycles/kernel/svm/svm_displace.h b/intern/cycles/kernel/svm/svm_displace.h
index 12a6f9a7d18..890ab41aaaa 100644
--- a/intern/cycles/kernel/svm/svm_displace.h
+++ b/intern/cycles/kernel/svm/svm_displace.h
@@ -76,10 +76,18 @@ ccl_device void svm_node_set_bump(KernelGlobals *kg, ShaderData *sd, float *stac
 
 /* Displacement Node */
 
-ccl_device void svm_node_set_displacement(ShaderData *sd, float *stack, uint fac_offset)
+ccl_device void svm_node_set_displacement(KernelGlobals *kg, ShaderData *sd, float *stack, uint fac_offset)
 {
 	float d = stack_load_float(stack, fac_offset);
-	ccl_fetch(sd, P) += ccl_fetch(sd, N)*d*0.1f; /* todo: get rid of this factor */
+
+	float3 dP = ccl_fetch(sd, N);
+	object_inverse_normal_transform(kg, sd, &dP);
+
+	dP *= d*0.1f; /* todo: get rid of this factor */
+
+	object_dir_transform(kg, sd, &dP);
+
+	ccl_fetch(sd, P) += dP;
 }
 
 CCL_NAMESPACE_END
diff --git a/intern/cycles/render/light.cpp b/intern/cycles/render/light.cpp
index 787e5cf07b2..b6c45dda231 100644
--- a/intern/cycles/render/light.cpp
+++ b/intern/cycles/render/light.cpp
@@ -309,9 +309,6 @@ void LightManager::device_update_distribution(Device *device, DeviceScene *dscen
 		int object_id = j;
 		int shader_flag = 0;
 
-		if(transform_applied)
-			object_id = ~object_id;
-
 		if(!(object->visibility & PATH_RAY_DIFFUSE)) {
 			shader_flag |= SHADER_EXCLUDE_DIFFUSE;
 			use_light_visibility = true;
diff --git a/intern/cycles/render/mesh_displace.cpp b/intern/cycles/render/mesh_displace.cpp
index ef9cfedd412..adc5b820298 100644
--- a/intern/cycles/render/mesh_displace.cpp
+++ b/intern/cycles/render/mesh_displace.cpp
@@ -86,8 +86,7 @@ bool MeshManager::displace(Device *device, DeviceScene *dscene, Scene *scene, Me
 			done[t.v[j]] = true;
 
 			/* set up object, primitive and barycentric coordinates */
-			/* when used, non-instanced convention: object = ~object */
-			int object = ~object_index;
+			int object = object_index;
 			int prim = mesh->tri_offset + i;
 			float u, v;