Fix T39735: New auto smooth creates artifacts with flat shaded faces(BI)

This actually had nothing specific to new split normals, it was an internal limitation
of BI raytracer, which would check against neighbor face shadowing only when they shared
a common vertex, now it also performs checks when both faces have a vertex with a common
"ancestor" (org index).

Note this allows to also fix same issue when using SplitEdges modifier (and potentially
others?), but only when AutoSmooth is enabled (due to some compute/mem overhead, we
do not want to enable this code systematically).

Thanks to Brecht for advices and review!

3 files changed, 38 insertions, 13 deletions

diff --git a/source/blender/render/intern/include/renderdatabase.h b/source/blender/render/intern/include/renderdatabase.h
index 9b14560b5f5..da45a2bfead 100644
--- a/source/blender/render/intern/include/renderdatabase.h
+++ b/source/blender/render/intern/include/renderdatabase.h
@@ -33,6 +33,10 @@
 #ifndef __RENDERDATABASE_H__
 #define __RENDERDATABASE_H__
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 struct Object;
 struct VlakRen;
 struct VertRen;
@@ -159,6 +163,9 @@ void area_lamp_vectors(struct LampRen *lar);
 void init_render_world(Render *re);
 void RE_Database_FromScene_Vectors(Render *re, struct Main *bmain, struct Scene *sce, unsigned int lay);
 
+#ifdef __cplusplus
+}
+#endif
 
 #endif /* __RENDERDATABASE_H__ */
 
diff --git a/source/blender/render/intern/raytrace/rayobject.cpp b/source/blender/render/intern/raytrace/rayobject.cpp
index 9e639501fdd..1ef44bbdd17 100644
--- a/source/blender/render/intern/raytrace/rayobject.cpp
+++ b/source/blender/render/intern/raytrace/rayobject.cpp
@@ -43,6 +43,7 @@
 #include "rayobject.h"
 #include "raycounter.h"
 #include "render_types.h"
+#include "renderdatabase.h"
 
 /* RayFace
  *
@@ -326,15 +327,33 @@ MALWAYS_INLINE int intersect_rayface(RayObject *hit_obj, RayFace *face, Isect *i
 			if (dist < 0.1f && is->orig.ob == face->ob) {
 				VlakRen *a = (VlakRen *)is->orig.face;
 				VlakRen *b = (VlakRen *)face->face;
+				ObjectRen *obr = ((ObjectInstanceRen *)face->ob)->obr;
+
+				VertRen **va, **vb;
+				int *org_idx_a, *org_idx_b;
+				int i, j;
+				bool is_neighbor = false;
+
+				/* "same" vertex means either the actual same VertRen, or the same 'final org index', if available
+				 * (autosmooth only, currently). */
+				for (i = 0, va = &a->v1; !is_neighbor && i < 4 && *va; ++i, ++va) {
+					org_idx_a = RE_vertren_get_origindex(obr, *va, false);
+					for (j = 0, vb = &b->v1; !is_neighbor && j < 4 && *vb; ++j, ++vb) {
+						if (*va == *vb) {
+							is_neighbor = true;
+						}
+						else if (org_idx_a) {
+							org_idx_b = RE_vertren_get_origindex(obr, *vb, 0);
+							if (org_idx_b && *org_idx_a == *org_idx_b) {
+								is_neighbor = true;
+							}
+						}
+					}
+				}
 
-				/* so there's a shared edge or vertex, let's intersect ray with
-				 * face itself, if that's true we can safely return 1, otherwise
-				 * we assume the intersection is invalid, 0 */
-				if (a->v1 == b->v1 || a->v2 == b->v1 || a->v3 == b->v1 || a->v4 == b->v1 ||
-				    a->v1 == b->v2 || a->v2 == b->v2 || a->v3 == b->v2 || a->v4 == b->v2 ||
-				    a->v1 == b->v3 || a->v2 == b->v3 || a->v3 == b->v3 || a->v4 == b->v3 ||
-				    (b->v4 && (a->v1 == b->v4 || a->v2 == b->v4 || a->v3 == b->v4 || a->v4 == b->v4)))
-				{
+				/* So there's a shared edge or vertex, let's intersect ray with self, if that's true
+				 * we can safely return 1, otherwise we assume the intersection is invalid, 0 */
+				if (is_neighbor) {
 					/* create RayFace from original face, transformed if necessary */
 					RayFace origface;
 					ObjectInstanceRen *ob = (ObjectInstanceRen *)is->orig.ob;
diff --git a/source/blender/render/intern/source/convertblender.c b/source/blender/render/intern/source/convertblender.c
index a9e2e1c2daf..908215f4215 100644
--- a/source/blender/render/intern/source/convertblender.c
+++ b/source/blender/render/intern/source/convertblender.c
@@ -3188,10 +3188,6 @@ static void init_render_mesh(Render *re, ObjectRen *obr, int timeoffset)
 		}
 		need_nmap_tangent= 1;
 	}
-	
-	/* origindex currently only used when baking to vertex colors */
-	if (re->flag & R_BAKING && re->r.bake_flag & R_BAKE_VCOL)
-		need_origindex= 1;
 
 	/* check autosmooth and displacement, we then have to skip only-verts optimize
 	 * Note: not sure what we want to give higher priority, currently do_displace
@@ -3201,7 +3197,10 @@ static void init_render_mesh(Render *re, ObjectRen *obr, int timeoffset)
 	do_autosmooth = ((me->flag & ME_AUTOSMOOTH) != 0) && !do_displace;
 	if (do_autosmooth || do_displace)
 		timeoffset = 0;
-	
+
+	/* origindex currently used when using autosmooth, or baking to vertex colors. */
+	need_origindex = (do_autosmooth || ((re->flag & R_BAKING) && (re->r.bake_flag & R_BAKE_VCOL)));
+
 	mask= CD_MASK_BAREMESH|CD_MASK_MTFACE|CD_MASK_MCOL;
 	if (!timeoffset)
 		if (need_orco)