Fix T43786: Cycles bake disregards Auto Smooth (vertex per-face normals)

Added an utility function which performs vertex split based on the loop
normal so now backing API matches to what's happening in Cycles and BI
in terms of autosplit.

Reviewers: dfelinto, campbellbarton

Reviewed By: campbellbarton

Differential Revision: https://developer.blender.org/D1174

4 files changed, 160 insertions, 48 deletions

diff --git a/source/blender/blenkernel/BKE_mesh.h b/source/blender/blenkernel/BKE_mesh.h
index 224be0f3685..1f3458c06c2 100644
--- a/source/blender/blenkernel/BKE_mesh.h
+++ b/source/blender/blenkernel/BKE_mesh.h
@@ -130,6 +130,9 @@ bool BKE_mesh_uv_cdlayer_rename(struct Mesh *me, const char *old_name, const cha
 
 float (*BKE_mesh_vertexCos_get(struct Mesh *me, int *r_numVerts))[3];
 
+void BKE_mesh_calc_normals_split(struct Mesh *mesh);
+void BKE_mesh_split_faces(struct Mesh *mesh);
+
 struct Mesh *BKE_mesh_new_from_object(struct Main *bmain, struct Scene *sce, struct Object *ob,
                                       int apply_modifiers, int settings, int calc_tessface, int calc_undeformed);
 
diff --git a/source/blender/blenkernel/intern/mesh.c b/source/blender/blenkernel/intern/mesh.c
index 809d2137187..8ebc318ff9e 100644
--- a/source/blender/blenkernel/intern/mesh.c
+++ b/source/blender/blenkernel/intern/mesh.c
@@ -2124,6 +2124,146 @@ void BKE_mesh_mselect_active_set(Mesh *me, int index, int type)
 	           (me->mselect[me->totselect - 1].type  == type));
 }
 
+void BKE_mesh_calc_normals_split(Mesh *mesh)
+{
+	float (*r_loopnors)[3];
+	float (*polynors)[3];
+	short (*clnors)[2] = NULL;
+	bool free_polynors = false;
+
+	if (CustomData_has_layer(&mesh->ldata, CD_NORMAL)) {
+		r_loopnors = CustomData_get_layer(&mesh->ldata, CD_NORMAL);
+		memset(r_loopnors, 0, sizeof(float[3]) * mesh->totloop);
+	}
+	else {
+		r_loopnors = CustomData_add_layer(&mesh->ldata, CD_NORMAL, CD_CALLOC, NULL, mesh->totloop);
+		CustomData_set_layer_flag(&mesh->ldata, CD_NORMAL, CD_FLAG_TEMPORARY);
+	}
+
+	/* may be NULL */
+	clnors = CustomData_get_layer(&mesh->ldata, CD_CUSTOMLOOPNORMAL);
+
+	if (CustomData_has_layer(&mesh->pdata, CD_NORMAL)) {
+		/* This assume that layer is always up to date, not sure this is the case (esp. in Edit mode?)... */
+		polynors = CustomData_get_layer(&mesh->pdata, CD_NORMAL);
+		free_polynors = false;
+	}
+	else {
+		polynors = MEM_mallocN(sizeof(float[3]) * mesh->totpoly, __func__);
+		BKE_mesh_calc_normals_poly(mesh->mvert, mesh->totvert, mesh->mloop, mesh->mpoly, mesh->totloop, mesh->totpoly,
+		                           polynors, false);
+		free_polynors = true;
+	}
+
+	BKE_mesh_normals_loop_split(
+	        mesh->mvert, mesh->totvert, mesh->medge, mesh->totedge,
+	        mesh->mloop, r_loopnors, mesh->totloop, mesh->mpoly, (const float (*)[3])polynors, mesh->totpoly,
+	        (mesh->flag & ME_AUTOSMOOTH) != 0, mesh->smoothresh, NULL, clnors, NULL);
+
+	if (free_polynors) {
+		MEM_freeN(polynors);
+	}
+}
+
+/* Spli faces based on the edge angle.
+ * Matches behavior of face splitting in render engines.
+ */
+void BKE_mesh_split_faces(Mesh *mesh)
+{
+	const int num_verts = mesh->totvert;
+	const int num_edges = mesh->totedge;
+	const int num_polys = mesh->totpoly;
+	MVert *mvert = mesh->mvert;
+	MEdge *medge = mesh->medge;
+	MLoop *mloop = mesh->mloop;
+	MPoly *mpoly = mesh->mpoly;
+	float (*lnors)[3];
+	int poly, num_new_verts = 0;
+	if ((mesh->flag & ME_AUTOSMOOTH) == 0) {
+		return;
+	}
+	BKE_mesh_tessface_clear(mesh);
+	/* Compute loop normals if needed. */
+	if (!CustomData_has_layer(&mesh->ldata, CD_NORMAL)) {
+		BKE_mesh_calc_normals_split(mesh);
+	}
+	lnors = CustomData_get_layer(&mesh->ldata, CD_NORMAL);
+	/* Count. */
+	for (poly = 0; poly < num_polys; poly++) {
+		MPoly *mp = &mpoly[poly];
+		int loop;
+		for (loop = 0; loop < mp->totloop; loop++) {
+			MLoop *ml = &mloop[mp->loopstart + loop];
+			MVert *mv = &mvert[ml->v];
+			float vn[3];
+			normal_short_to_float_v3(vn, mv->no);
+			if (!equals_v3v3(vn, lnors[mp->loopstart + loop])) {
+				num_new_verts++;
+			}
+		}
+	}
+	if (num_new_verts == 0) {
+		/* No new vertices are to be added, can do early exit. */
+		return;
+	}
+	/* Actual split. */
+	mesh->totvert += num_new_verts;
+	mesh->totedge += 2 * num_new_verts;
+	mvert = mesh->mvert = MEM_reallocN(mesh->mvert,
+	                                   sizeof(MVert) * mesh->totvert);
+	medge = mesh->medge = MEM_reallocN(mesh->medge,
+	                                   sizeof(MEdge) * mesh->totedge);
+	CustomData_set_layer(&mesh->vdata, CD_MVERT, mesh->mvert);
+	CustomData_set_layer(&mesh->edata, CD_MEDGE, mesh->medge);
+	num_new_verts = 0;
+	for (poly = 0; poly < num_polys; poly++) {
+		MPoly *mp = &mpoly[poly];
+		int loop;
+		for (loop = 0; loop < mp->totloop; loop++) {
+			int poly_loop = mp->loopstart + loop;
+			MLoop *ml = &mloop[poly_loop];
+			MVert *mv = &mvert[ml->v];
+			float vn[3];
+			normal_short_to_float_v3(vn, mv->no);
+			if (!equals_v3v3(vn, lnors[mp->loopstart + loop])) {
+				int poly_loop_prev = mp->loopstart + (loop + mp->totloop - 1) % mp->totloop;
+				MLoop *ml_prev = &mloop[poly_loop_prev];
+				int new_edge_prev, new_edge;
+				/* Cretae new vertex. */
+				int new_vert = num_verts + num_new_verts;
+				CustomData_copy_data(&mesh->vdata, &mesh->vdata,
+				                     ml->v, new_vert, 1);
+				normal_float_to_short_v3(mvert[new_vert].no,
+				                         lnors[poly_loop]);
+				/* Create new edges. */
+				new_edge_prev = num_edges + 2 * num_new_verts;
+				new_edge = num_edges + 2 * num_new_verts + 1;
+				CustomData_copy_data(&mesh->edata, &mesh->edata,
+				                     ml_prev->e, new_edge_prev, 1);
+				CustomData_copy_data(&mesh->edata, &mesh->edata,
+				                     ml->e, new_edge, 1);
+				if (medge[new_edge_prev].v1 == ml->v) {
+					medge[new_edge_prev].v1 = new_vert;
+				}
+				else {
+					medge[new_edge_prev].v2 = new_vert;
+				}
+				if (medge[new_edge].v1 == ml->v) {
+					medge[new_edge].v1 = new_vert;
+				}
+				else {
+					medge[new_edge].v2 = new_vert;
+				}
+
+				ml->v = new_vert;
+				ml_prev->e = new_edge_prev;
+				ml->e = new_edge;
+				num_new_verts++;
+			}
+		}
+	}
+}
+
 /* settings: 1 - preview, 2 - render */
 Mesh *BKE_mesh_new_from_object(
         Main *bmain, Scene *sce, Object *ob,
diff --git a/source/blender/editors/object/object_bake_api.c b/source/blender/editors/object/object_bake_api.c
index 97cb45daae3..fe9ee71aa8e 100644
--- a/source/blender/editors/object/object_bake_api.c
+++ b/source/blender/editors/object/object_bake_api.c
@@ -685,7 +685,9 @@ static int bake(
 	result = MEM_callocN(sizeof(float) * depth * num_pixels, "bake return pixels");
 
 	/* get the mesh as it arrives in the renderer */
-	me_low = BKE_mesh_new_from_object(bmain, scene, ob_low, 1, 2, 1, 0);
+	me_low = BKE_mesh_new_from_object(bmain, scene, ob_low, 1, 2, 0, 0);
+	BKE_mesh_split_faces(me_low);
+	BKE_mesh_tessface_ensure(me_low);
 
 	/* populate the pixel array with the face data */
 	if ((is_selected_to_active && (ob_cage == NULL) && is_cage) == false)
@@ -700,7 +702,9 @@ static int bake(
 
 		/* prepare cage mesh */
 		if (ob_cage) {
-			me_cage = BKE_mesh_new_from_object(bmain, scene, ob_cage, 1, 2, 1, 0);
+			me_cage = BKE_mesh_new_from_object(bmain, scene, ob_cage, 1, 2, 0, 0);
+			BKE_mesh_split_faces(me_cage);
+			BKE_mesh_tessface_ensure(me_cage);
 			if (me_low->totface != me_cage->totface) {
 				BKE_report(reports, RPT_ERROR,
 				           "Invalid cage object, the cage mesh must have the same number "
@@ -732,7 +736,9 @@ static int bake(
 			ob_low->modifiers = modifiers_tmp;
 
 			/* get the cage mesh as it arrives in the renderer */
-			me_cage = BKE_mesh_new_from_object(bmain, scene, ob_low, 1, 2, 1, 0);
+			me_cage = BKE_mesh_new_from_object(bmain, scene, ob_low, 1, 2, 0, 0);
+			BKE_mesh_split_faces(me_cage);
+			BKE_mesh_tessface_ensure(me_cage);
 			RE_bake_pixels_populate(me_cage, pixel_array_low, num_pixels, &bake_images, uv_layer);
 		}
 
@@ -760,8 +766,10 @@ static int bake(
 			tmd->quad_method = MOD_TRIANGULATE_QUAD_FIXED;
 			tmd->ngon_method = MOD_TRIANGULATE_NGON_EARCLIP;
 
-			highpoly[i].me = BKE_mesh_new_from_object(bmain, scene, highpoly[i].ob, 1, 2, 1, 0);
+			highpoly[i].me = BKE_mesh_new_from_object(bmain, scene, highpoly[i].ob, 1, 2, 0, 0);
 			highpoly[i].ob->restrictflag &= ~OB_RESTRICT_RENDER;
+			BKE_mesh_split_faces(highpoly[i].me);
+			BKE_mesh_tessface_ensure(highpoly[i].me);
 
 			/* lowpoly to highpoly transformation matrix */
 			copy_m4_m4(highpoly[i].obmat, highpoly[i].ob->obmat);
@@ -867,7 +875,9 @@ cage_cleanup:
 						md->mode &= ~eModifierMode_Render;
 					}
 
-					me_nores = BKE_mesh_new_from_object(bmain, scene, ob_low, 1, 2, 1, 0);
+					me_nores = BKE_mesh_new_from_object(bmain, scene, ob_low, 1, 2, 0, 0);
+					BKE_mesh_split_faces(me_nores);
+					BKE_mesh_tessface_ensure(me_nores);
 					RE_bake_pixels_populate(me_nores, pixel_array_low, num_pixels, &bake_images, uv_layer);
 
 					RE_bake_normal_world_to_tangent(pixel_array_low, num_pixels, depth, result, me_nores, normal_swizzle, ob_low->obmat);
diff --git a/source/blender/makesrna/intern/rna_mesh_api.c b/source/blender/makesrna/intern/rna_mesh_api.c
index abf29ef82a3..48a5f09fda0 100644
--- a/source/blender/makesrna/intern/rna_mesh_api.c
+++ b/source/blender/makesrna/intern/rna_mesh_api.c
@@ -69,47 +69,6 @@ static void rna_Mesh_create_normals_split(Mesh *mesh)
 	}
 }
 
-static void rna_Mesh_calc_normals_split(Mesh *mesh)
-{
-	float (*r_loopnors)[3];
-	float (*polynors)[3];
-	short (*clnors)[2] = NULL;
-	bool free_polynors = false;
-
-	if (CustomData_has_layer(&mesh->ldata, CD_NORMAL)) {
-		r_loopnors = CustomData_get_layer(&mesh->ldata, CD_NORMAL);
-		memset(r_loopnors, 0, sizeof(float[3]) * mesh->totloop);
-	}
-	else {
-		r_loopnors = CustomData_add_layer(&mesh->ldata, CD_NORMAL, CD_CALLOC, NULL, mesh->totloop);
-		CustomData_set_layer_flag(&mesh->ldata, CD_NORMAL, CD_FLAG_TEMPORARY);
-	}
-
-	/* may be NULL */
-	clnors = CustomData_get_layer(&mesh->ldata, CD_CUSTOMLOOPNORMAL);
-
-	if (CustomData_has_layer(&mesh->pdata, CD_NORMAL)) {
-		/* This assume that layer is always up to date, not sure this is the case (esp. in Edit mode?)... */
-		polynors = CustomData_get_layer(&mesh->pdata, CD_NORMAL);
-		free_polynors = false;
-	}
-	else {
-		polynors = MEM_mallocN(sizeof(float[3]) * mesh->totpoly, __func__);
-		BKE_mesh_calc_normals_poly(mesh->mvert, mesh->totvert, mesh->mloop, mesh->mpoly, mesh->totloop, mesh->totpoly,
-		                           polynors, false);
-		free_polynors = true;
-	}
-
-	BKE_mesh_normals_loop_split(
-	        mesh->mvert, mesh->totvert, mesh->medge, mesh->totedge,
-	        mesh->mloop, r_loopnors, mesh->totloop, mesh->mpoly, (const float (*)[3])polynors, mesh->totpoly,
-	        (mesh->flag & ME_AUTOSMOOTH) != 0, mesh->smoothresh, NULL, clnors, NULL);
-
-	if (free_polynors) {
-		MEM_freeN(polynors);
-	}
-}
-
 static void rna_Mesh_free_normals_split(Mesh *mesh)
 {
 	CustomData_free_layers(&mesh->ldata, CD_NORMAL, mesh->totloop);
@@ -130,7 +89,7 @@ static void rna_Mesh_calc_tangents(Mesh *mesh, ReportList *reports, const char *
 
 	/* Compute loop normals if needed. */
 	if (!CustomData_has_layer(&mesh->ldata, CD_NORMAL)) {
-		rna_Mesh_calc_normals_split(mesh);
+		BKE_mesh_calc_normals_split(mesh);
 	}
 
 	BKE_mesh_loop_tangents(mesh, uvmap, r_looptangents, reports);
@@ -256,7 +215,7 @@ void RNA_api_mesh(StructRNA *srna)
 	func = RNA_def_function(srna, "create_normals_split", "rna_Mesh_create_normals_split");
 	RNA_def_function_ui_description(func, "Empty split vertex normals");
 
-	func = RNA_def_function(srna, "calc_normals_split", "rna_Mesh_calc_normals_split");
+	func = RNA_def_function(srna, "calc_normals_split", "BKE_mesh_calc_normals_split");
 	RNA_def_function_ui_description(func, "Calculate split vertex normals, which preserve sharp edges");
 
 	func = RNA_def_function(srna, "free_normals_split", "rna_Mesh_free_normals_split");