Cycles: Replace object index hack with actual checks for SD_TRANSFORM_APPLIEDtemp-cycles-microdisplacement

Using ones complement for detecting if transform has been applied was confusing
and led to several bugs. With this proper checks are made. Also added a few
transforms where they were missing, mostly affecting baking and displacement
when `P` is used in the shader (previously `P` was in the wrong space for
these shaders)

10 files changed, 41 insertions, 32 deletions

diff --git a/intern/cycles/blender/blender_session.cpp b/intern/cycles/blender/blender_session.cpp
index 80a16e0ef27..c250a54f259 100644
--- a/intern/cycles/blender/blender_session.cpp
+++ b/intern/cycles/blender/blender_session.cpp
@@ -681,8 +681,7 @@ void BlenderSession::bake(BL::Object& b_object,
 		}
 	}
 
-	/* when used, non-instanced convention: object = ~object */
-	int object = ~object_index;
+	int object = object_index;
 
 	BakeData *bake_data = scene->bake_manager->init(object, tri_offset, num_pixels);
 
diff --git a/intern/cycles/kernel/geom/geom_triangle.h b/intern/cycles/kernel/geom/geom_triangle.h
index d3289d6572c..8bd01e1008d 100644
--- a/intern/cycles/kernel/geom/geom_triangle.h
+++ b/intern/cycles/kernel/geom/geom_triangle.h
@@ -52,8 +52,8 @@ ccl_device_inline void triangle_point_normal(KernelGlobals *kg, int object, int
 	float t = 1.0f - u - v;
 	*P = (u*v0 + v*v1 + t*v2);
 
-	/* get object flags, instance-aware */
-	int object_flag = kernel_tex_fetch(__object_flag, object >= 0 ? object : ~object);
+	/* get object flags */
+	int object_flag = kernel_tex_fetch(__object_flag, object);
 
 	/* compute normal */
 	if(object_flag & SD_NEGATIVE_SCALE_APPLIED)
diff --git a/intern/cycles/kernel/kernel_bake.h b/intern/cycles/kernel/kernel_bake.h
index bfbf73df54f..6202808d488 100644
--- a/intern/cycles/kernel/kernel_bake.h
+++ b/intern/cycles/kernel/kernel_bake.h
@@ -313,6 +313,14 @@ ccl_device void kernel_bake_evaluate(KernelGlobals *kg, ccl_global uint4 *input,
 
 	triangle_point_normal(kg, object, prim, u, v, &P, &Ng, &shader);
 
+	if(!(kernel_tex_fetch(__object_flag, object) & SD_TRANSFORM_APPLIED)) {
+		Transform tfm = object_fetch_transform(kg, object, OBJECT_TRANSFORM);
+		P = transform_point(&tfm, P);
+
+		tfm = object_fetch_transform(kg, object, OBJECT_INVERSE_TRANSFORM);
+		Ng = normalize(transform_direction_transposed(&tfm, Ng));
+	}
+
 	/* dummy initilizations copied from SHADER_EVAL_DISPLACE */
 	float3 I = Ng;
 	float t = 1.0f;
@@ -525,6 +533,9 @@ ccl_device void kernel_shader_evaluate(KernelGlobals *kg,
 		float3 P = sd.P;
 		shader_eval_displacement(kg, &sd, &state, SHADER_CONTEXT_MAIN);
 		out = sd.P - P;
+
+		Transform itfm = object_fetch_transform(kg, object, OBJECT_INVERSE_TRANSFORM);
+		out = transform_direction(&itfm, out);
 	}
 	else { // SHADER_EVAL_BACKGROUND
 		/* setup ray */
diff --git a/intern/cycles/kernel/kernel_light.h b/intern/cycles/kernel/kernel_light.h
index 1e829eaa1fa..1e99f2cb99f 100644
--- a/intern/cycles/kernel/kernel_light.h
+++ b/intern/cycles/kernel/kernel_light.h
@@ -747,7 +747,7 @@ ccl_device void object_transform_light_sample(KernelGlobals *kg, LightSample *ls
 {
 #ifdef __INSTANCING__
 	/* instance transform */
-	if(object >= 0) {
+	if(!(kernel_tex_fetch(__object_flag, object) & SD_TRANSFORM_APPLIED)) {
 #  ifdef __OBJECT_MOTION__
 		Transform itfm;
 		Transform tfm = object_fetch_transform_motion_test(kg, object, time, &itfm);
diff --git a/intern/cycles/kernel/kernel_shader.h b/intern/cycles/kernel/kernel_shader.h
index 079bea30bdd..0f026efa79a 100644
--- a/intern/cycles/kernel/kernel_shader.h
+++ b/intern/cycles/kernel/kernel_shader.h
@@ -263,20 +263,6 @@ ccl_device_inline void shader_setup_from_sample(KernelGlobals *kg,
 #endif
 	ccl_fetch(sd, ray_length) = t;
 
-	/* detect instancing, for non-instanced the object index is -object-1 */
-#ifdef __INSTANCING__
-	bool instanced = false;
-
-	if(ccl_fetch(sd, prim) != PRIM_NONE) {
-		if(ccl_fetch(sd, object) >= 0)
-			instanced = true;
-		else
-#endif
-			ccl_fetch(sd, object) = ~ccl_fetch(sd, object);
-#ifdef __INSTANCING__
-	}
-#endif
-
 	ccl_fetch(sd, flag) = kernel_tex_fetch(__shader_flag, (ccl_fetch(sd, shader) & SHADER_MASK)*2);
 	if(ccl_fetch(sd, object) != OBJECT_NONE) {
 		ccl_fetch(sd, flag) |= kernel_tex_fetch(__object_flag, ccl_fetch(sd, object));
@@ -296,8 +282,9 @@ ccl_device_inline void shader_setup_from_sample(KernelGlobals *kg,
 			ccl_fetch(sd, N) = triangle_smooth_normal(kg, ccl_fetch(sd, prim), ccl_fetch(sd, u), ccl_fetch(sd, v));
 
 #ifdef __INSTANCING__
-			if(instanced)
+			if(!(ccl_fetch(sd, flag) & SD_TRANSFORM_APPLIED)) {
 				object_normal_transform_auto(kg, sd, &ccl_fetch(sd, N));
+			}
 #endif
 		}
 
@@ -306,7 +293,7 @@ ccl_device_inline void shader_setup_from_sample(KernelGlobals *kg,
 		triangle_dPdudv(kg, ccl_fetch(sd, prim), &ccl_fetch(sd, dPdu), &ccl_fetch(sd, dPdv));
 
 #  ifdef __INSTANCING__
-		if(instanced) {
+		if(!(ccl_fetch(sd, flag) & SD_TRANSFORM_APPLIED)) {
 			object_dir_transform_auto(kg, sd, &ccl_fetch(sd, dPdu));
 			object_dir_transform_auto(kg, sd, &ccl_fetch(sd, dPdv));
 		}
@@ -354,11 +341,17 @@ ccl_device void shader_setup_from_displace(KernelGlobals *kg, ShaderData *sd,
 
 	triangle_point_normal(kg, object, prim, u, v, &P, &Ng, &shader);
 
+	if(!(kernel_tex_fetch(__object_flag, object) & SD_TRANSFORM_APPLIED)) {
+		Transform tfm = object_fetch_transform(kg, object, OBJECT_TRANSFORM);
+		P = transform_point(&tfm, P);
+
+		tfm = object_fetch_transform(kg, object, OBJECT_INVERSE_TRANSFORM);
+		Ng = normalize(transform_direction_transposed(&tfm, Ng));
+	}
+
 	/* force smooth shading for displacement */
 	shader |= SHADER_SMOOTH_NORMAL;
 
-	/* watch out: no instance transform currently */
-
 	shader_setup_from_sample(kg, sd, P, Ng, I, shader, object, prim, u, v, 0.0f, TIME_INVALID);
 }
 
diff --git a/intern/cycles/kernel/shaders/node_output_displacement.osl b/intern/cycles/kernel/shaders/node_output_displacement.osl
index d0688cfda8d..294b8dd6bf2 100644
--- a/intern/cycles/kernel/shaders/node_output_displacement.osl
+++ b/intern/cycles/kernel/shaders/node_output_displacement.osl
@@ -18,6 +18,8 @@
 
 displacement node_output_displacement(float Displacement = 0.0)
 {
-	P += N * Displacement * 0.1;  /* todo: get rid of this factor */
+	vector dP = normalize(transform("object", N));
+	dP *= Displacement * 0.1; /* todo: get rid of this factor */
+	P += transform("object", "world", dP);
 }
 
diff --git a/intern/cycles/kernel/svm/svm.h b/intern/cycles/kernel/svm/svm.h
index 998a32e3181..adee98f35c0 100644
--- a/intern/cycles/kernel/svm/svm.h
+++ b/intern/cycles/kernel/svm/svm.h
@@ -261,7 +261,7 @@ ccl_device_noinline void svm_eval_nodes(KernelGlobals *kg, ShaderData *sd, ccl_a
 				svm_node_geometry_bump_dy(kg, sd, stack, node.y, node.z);
 				break;
 			case NODE_SET_DISPLACEMENT:
-				svm_node_set_displacement(sd, stack, node.y);
+				svm_node_set_displacement(kg, sd, stack, node.y);
 				break;
 #  endif  /* NODES_FEATURE(NODE_FEATURE_BUMP) */
 #  ifdef __TEXTURES__
diff --git a/intern/cycles/kernel/svm/svm_displace.h b/intern/cycles/kernel/svm/svm_displace.h
index 12a6f9a7d18..890ab41aaaa 100644
--- a/intern/cycles/kernel/svm/svm_displace.h
+++ b/intern/cycles/kernel/svm/svm_displace.h
@@ -76,10 +76,18 @@ ccl_device void svm_node_set_bump(KernelGlobals *kg, ShaderData *sd, float *stac
 
 /* Displacement Node */
 
-ccl_device void svm_node_set_displacement(ShaderData *sd, float *stack, uint fac_offset)
+ccl_device void svm_node_set_displacement(KernelGlobals *kg, ShaderData *sd, float *stack, uint fac_offset)
 {
 	float d = stack_load_float(stack, fac_offset);
-	ccl_fetch(sd, P) += ccl_fetch(sd, N)*d*0.1f; /* todo: get rid of this factor */
+
+	float3 dP = ccl_fetch(sd, N);
+	object_inverse_normal_transform(kg, sd, &dP);
+
+	dP *= d*0.1f; /* todo: get rid of this factor */
+
+	object_dir_transform(kg, sd, &dP);
+
+	ccl_fetch(sd, P) += dP;
 }
 
 CCL_NAMESPACE_END
diff --git a/intern/cycles/render/light.cpp b/intern/cycles/render/light.cpp
index 787e5cf07b2..b6c45dda231 100644
--- a/intern/cycles/render/light.cpp
+++ b/intern/cycles/render/light.cpp
@@ -309,9 +309,6 @@ void LightManager::device_update_distribution(Device *device, DeviceScene *dscen
 		int object_id = j;
 		int shader_flag = 0;
 
-		if(transform_applied)
-			object_id = ~object_id;
-
 		if(!(object->visibility & PATH_RAY_DIFFUSE)) {
 			shader_flag |= SHADER_EXCLUDE_DIFFUSE;
 			use_light_visibility = true;
diff --git a/intern/cycles/render/mesh_displace.cpp b/intern/cycles/render/mesh_displace.cpp
index ef9cfedd412..adc5b820298 100644
--- a/intern/cycles/render/mesh_displace.cpp
+++ b/intern/cycles/render/mesh_displace.cpp
@@ -86,8 +86,7 @@ bool MeshManager::displace(Device *device, DeviceScene *dscene, Scene *scene, Me
 			done[t.v[j]] = true;
 
 			/* set up object, primitive and barycentric coordinates */
-			/* when used, non-instanced convention: object = ~object */
-			int object = ~object_index;
+			int object = object_index;
 			int prim = mesh->tri_offset + i;
 			float u, v;