Fix T41219: Cycles backface detection doesn't work properly

Root of the issue goes back to the on-fly normals commit and the
latest fix for it wasn't actually correct. I've mixed two fixes
in there.

So the idea here goes back to storing negative scaled object flag
and flip runtime-calculated normal if this flag is set, which is
pretty much the same as the original fix for the issue from me.

The issue with motion blur wasn't caused by the rumtime normals
patch and it had issues before, because it already did runtime
normals calculation. Now made it so motion triangles takes the
negative scale flag into account.

This actually makes code more clean imo and avoids rather confusing
flipping code in mesh.cpp.

8 files changed, 21 insertions, 22 deletions

diff --git a/intern/cycles/kernel/geom/geom_motion_triangle.h b/intern/cycles/kernel/geom/geom_motion_triangle.h
index 5ab0b731bdd..7409aa0d014 100644
--- a/intern/cycles/kernel/geom/geom_motion_triangle.h
+++ b/intern/cycles/kernel/geom/geom_motion_triangle.h
@@ -272,7 +272,11 @@ ccl_device_noinline void motion_triangle_shader_setup(KernelGlobals *kg, ShaderD
 #endif
 
 	/* compute face normal */
-	float3 Ng = normalize(cross(verts[1] - verts[0], verts[2] - verts[0]));
+	float3 Ng;
+	if(sd->flag & SD_NEGATIVE_SCALE_APPLIED)
+		Ng = normalize(cross(verts[2] - verts[0], verts[1] - verts[0]));
+	else
+		Ng = normalize(cross(verts[1] - verts[0], verts[2] - verts[0]));
 
 	sd->Ng = Ng;
 	sd->N = Ng;
diff --git a/intern/cycles/kernel/geom/geom_triangle.h b/intern/cycles/kernel/geom/geom_triangle.h
index 27d1351568a..35dd5b2324e 100644
--- a/intern/cycles/kernel/geom/geom_triangle.h
+++ b/intern/cycles/kernel/geom/geom_triangle.h
@@ -127,11 +127,14 @@ ccl_device_inline float3 triangle_normal(KernelGlobals *kg, ShaderData *sd)
 	float3 v2 = float4_to_float3(kernel_tex_fetch(__tri_verts, __float_as_int(tri_vindex.z)));
 	
 	/* return normal */
-	return normalize(cross(v1 - v0, v2 - v0));
+	if(sd->flag & SD_NEGATIVE_SCALE_APPLIED)
+		return normalize(cross(v2 - v0, v1 - v0));
+	else
+		return normalize(cross(v1 - v0, v2 - v0));
 }
 
 /* point and normal on triangle  */
-ccl_device_inline void triangle_point_normal(KernelGlobals *kg, int prim, float u, float v, float3 *P, float3 *Ng, int *shader)
+ccl_device_inline void triangle_point_normal(KernelGlobals *kg, int object, int prim, float u, float v, float3 *P, float3 *Ng, int *shader)
 {
 	/* load triangle vertices */
 	float3 tri_vindex = float4_to_float3(kernel_tex_fetch(__tri_vindex, prim));
@@ -145,7 +148,11 @@ ccl_device_inline void triangle_point_normal(KernelGlobals *kg, int prim, float
 	*P = (u*v0 + v*v1 + t*v2);
 
 	/* compute normal */
-	*Ng = normalize(cross(v1 - v0, v2 - v0));
+	int object_flag = kernel_tex_fetch(__object_flag, object);
+	if(object_flag & SD_NEGATIVE_SCALE_APPLIED)
+		*Ng = normalize(cross(v2 - v0, v1 - v0));
+	else
+		*Ng = normalize(cross(v1 - v0, v2 - v0));
 
 	/* shader`*/
 	*shader = __float_as_int(kernel_tex_fetch(__tri_shader, prim));
diff --git a/intern/cycles/kernel/kernel_bake.h b/intern/cycles/kernel/kernel_bake.h
index c2d14b7f835..0df1725edbe 100644
--- a/intern/cycles/kernel/kernel_bake.h
+++ b/intern/cycles/kernel/kernel_bake.h
@@ -215,7 +215,7 @@ ccl_device void kernel_bake_evaluate(KernelGlobals *kg, ccl_global uint4 *input,
 	int shader;
 	float3 P, Ng;
 
-	triangle_point_normal(kg, prim, u, v, &P, &Ng, &shader);
+	triangle_point_normal(kg, object, prim, u, v, &P, &Ng, &shader);
 
 	/* dummy initilizations copied from SHADER_EVAL_DISPLACE */
 	float3 I = Ng;
diff --git a/intern/cycles/kernel/kernel_light.h b/intern/cycles/kernel/kernel_light.h
index ac432d3fe04..3832631e02d 100644
--- a/intern/cycles/kernel/kernel_light.h
+++ b/intern/cycles/kernel/kernel_light.h
@@ -457,7 +457,7 @@ ccl_device void triangle_light_sample(KernelGlobals *kg, int prim, int object,
 	v = randv*randu;
 
 	/* triangle, so get position, normal, shader */
-	triangle_point_normal(kg, prim, u, v, &ls->P, &ls->Ng, &ls->shader);
+	triangle_point_normal(kg, object, prim, u, v, &ls->P, &ls->Ng, &ls->shader);
 	ls->object = object;
 	ls->prim = prim;
 	ls->lamp = LAMP_NONE;
diff --git a/intern/cycles/kernel/kernel_shader.h b/intern/cycles/kernel/kernel_shader.h
index 3b95d7055ac..fc61f1a9c2c 100644
--- a/intern/cycles/kernel/kernel_shader.h
+++ b/intern/cycles/kernel/kernel_shader.h
@@ -340,7 +340,7 @@ ccl_device void shader_setup_from_displace(KernelGlobals *kg, ShaderData *sd,
 	float3 P, Ng, I = make_float3(0.0f, 0.0f, 0.0f);
 	int shader;
 
-	triangle_point_normal(kg, prim, u, v, &P, &Ng, &shader);
+	triangle_point_normal(kg, object, prim, u, v, &P, &Ng, &shader);
 
 	/* force smooth shading for displacement */
 	shader |= SHADER_SMOOTH_NORMAL;
diff --git a/intern/cycles/kernel/kernel_types.h b/intern/cycles/kernel/kernel_types.h
index f60d813edad..0ee7e60f4f6 100644
--- a/intern/cycles/kernel/kernel_types.h
+++ b/intern/cycles/kernel/kernel_types.h
@@ -592,6 +592,7 @@ enum ShaderDataFlag {
 	SD_HOLDOUT_MASK = 524288,			/* holdout for camera rays */
 	SD_OBJECT_MOTION = 1048576,			/* has object motion blur */
 	SD_TRANSFORM_APPLIED = 2097152,		/* vertices have transform applied */
+	SD_NEGATIVE_SCALE_APPLIED = 4194304,	/* vertices have negative scale applied */
 
 	SD_OBJECT_FLAGS = (SD_HOLDOUT_MASK|SD_OBJECT_MOTION|SD_TRANSFORM_APPLIED)
 };
diff --git a/intern/cycles/render/mesh.cpp b/intern/cycles/render/mesh.cpp
index 295c934537a..273443034c5 100644
--- a/intern/cycles/render/mesh.cpp
+++ b/intern/cycles/render/mesh.cpp
@@ -341,13 +341,6 @@ void Mesh::add_vertex_normals()
 				vN[i] = -vN[i];
 		}
 	}
-	else if(flip) {
-		Attribute *attr_vN = attributes.find(ATTR_STD_VERTEX_NORMAL);
-		float3 *vN = attr_vN->data_float3();
-		for(size_t i = 0; i < verts_size; i++) {
-			vN[i] = -vN[i];
-		}
-	}
 
 	/* motion vertex normals */
 	Attribute *attr_mP = attributes.find(ATTR_STD_MOTION_VERTEX_POSITION);
@@ -382,14 +375,6 @@ void Mesh::add_vertex_normals()
 			}
 		}
 	}
-	else if(has_motion_blur() && attr_mN && flip) {
-		for(int step = 0; step < motion_steps - 1; step++) {
-			float3 *mN = attr_mN->data_float3() + step*verts.size();
-			for(size_t i = 0; i < verts_size; i++) {
-				mN[i] = -mN[i];
-			}
-		}
-	}
 }
 
 void Mesh::pack_normals(Scene *scene, float *tri_shader, float4 *vnormal)
diff --git a/intern/cycles/render/object.cpp b/intern/cycles/render/object.cpp
index 027bfd71931..1f148d34ea6 100644
--- a/intern/cycles/render/object.cpp
+++ b/intern/cycles/render/object.cpp
@@ -449,6 +449,8 @@ void ObjectManager::apply_static_transforms(DeviceScene *dscene, Scene *scene, u
 				}
 
 				object_flag[i] |= SD_TRANSFORM_APPLIED;
+				if(object->mesh->transform_negative_scaled)
+					object_flag[i] |= SD_NEGATIVE_SCALE_APPLIED;
 			}
 			else
 				have_instancing = true;