Bake-API: new approach for cage

There is a new option to select whether you want to use cage or not.
When not using cage the results will be more similar with Blender
Internal, where the inwards rays (trying to hit the highpoly objects)
don't always come from smooth normals. So if the active object has sharp
edges and an EdgeSplit modifier you get bad corners.

This is useful, however, to bake to planes without the need of adding
extra loops around the edges.

When cage is "on" the user can decide on setting a cage extrusion or to
pick a Custom Cage object. The cage extrusion option works in a
duplicated copy of the active object with EdgeSplit modifiers removed to
inforce smooth normals. The custom cage option takes an object with the
same number of faces as the active object (and the same face ordering).

The custom cage now controls the direction and the origin of the
rays casted to the highpoly objects. The direction is a ray from the
point in the cage mesh to the equivalent point to the base mesh. That
means the face normals are entirely ignored when using a cage object.

For developers:
When using an object cage the ray is calculated from the cage mesh to
the base mesh. It uses the barycentric coordinate from the base mesh UV,
so we expect both meshes to have the same primitive ids (which won't be
the case if the cage gets edited in a destructive way).

That fixes T40023 (giving the expected result when 'use_cage' is false).

Thanks for Andy Davies (metalliandy) for the consulting with normal
baking workflow and extensive testing. His 'stress-test' file will be
added later to our svn tests folder. (The file itself is not public yet
since he still has to add testing notes to it).

Many thanks for the reviewers.

More on cages:
http://wiki.polycount.com/NormalMap/#Working_with_Cages

Reviewers: campbellbarton, sergey

CC: adriano, metalliandy, brecht, malkavian

Differential Revision: https://developer.blender.org/D547

2 files changed, 110 insertions, 24 deletions

diff --git a/source/blender/render/extern/include/RE_bake.h b/source/blender/render/extern/include/RE_bake.h
index 0f82082911a..4727aef460e 100644
--- a/source/blender/render/extern/include/RE_bake.h
+++ b/source/blender/render/extern/include/RE_bake.h
@@ -82,8 +82,8 @@ bool RE_bake_internal(
 
 void RE_bake_pixels_populate_from_objects(
         struct Mesh *me_low, BakePixel pixel_array_from[],
-        BakeHighPolyData highpoly[], const int tot_highpoly, const int num_pixels,
-        const float cage_extrusion, float mat_low[4][4]);
+        BakeHighPolyData highpoly[], const int tot_highpoly, const int num_pixels, const bool is_custom_cage,
+        const float cage_extrusion, float mat_low[4][4], float mat_cage[4][4], struct Mesh *me_cage);
 
 void RE_bake_pixels_populate(
         struct Mesh *me, struct BakePixel *pixel_array,
diff --git a/source/blender/render/intern/source/bake_api.c b/source/blender/render/intern/source/bake_api.c
index 48516ea895c..3de39378fad 100644
--- a/source/blender/render/intern/source/bake_api.c
+++ b/source/blender/render/intern/source/bake_api.c
@@ -158,19 +158,66 @@ void RE_bake_margin(ImBuf *ibuf, char *mask, const int margin)
 		IMB_rectfill_alpha(ibuf, 1.0f);
 }
 
+
 /**
  * This function returns the coordinate and normal of a barycentric u,v for a face defined by the primitive_id index.
+ * The returned normal is actually the direction from the same barycentric coordinate in the cage to the base mesh
+ * The returned coordinate is the point in the cage mesh
  */
-static void calc_point_from_barycentric(
-        TriTessFace *triangles, int primitive_id, float u, float v, float cage_extrusion,
+static void calc_point_from_barycentric_cage(
+        TriTessFace *triangles_low, TriTessFace *triangles_cage,
+        float mat_low[4][4], float mat_cage[4][4],
+        int primitive_id, float u, float v,
         float r_co[3], float r_dir[3])
 {
+	float data[2][3][3];
+	float coord[2][3];
+	float dir[3];
+	int i;
+
+	TriTessFace *triangle[2];
+
+	triangle[0] = &triangles_low[primitive_id];
+	triangle[1] = &triangles_cage[primitive_id];
+
+	for (i = 0; i < 2; i++) {
+		copy_v3_v3(data[i][0], triangle[i]->mverts[0]->co);
+		copy_v3_v3(data[i][1], triangle[i]->mverts[1]->co);
+		copy_v3_v3(data[i][2], triangle[i]->mverts[2]->co);
+		interp_barycentric_tri_v3(data[i], u, v, coord[i]);
+	}
+
+	/* convert from local to world space */
+	mul_m4_v3(mat_low, coord[0]);
+	mul_m4_v3(mat_cage, coord[1]);
+
+	sub_v3_v3v3(dir, coord[0], coord[1]);
+	normalize_v3(dir);
+
+	copy_v3_v3(r_co, coord[1]);
+	copy_v3_v3(r_dir, dir);
+}
+
+/**
+ * This function returns the coordinate and normal of a barycentric u,v for a face defined by the primitive_id index.
+ * The returned coordinate is extruded along the normal by cage_extrusion
+ */
+static void calc_point_from_barycentric_extrusion(
+        TriTessFace *triangles,
+        float mat[4][4], float imat[4][4],
+        int primitive_id, float u, float v,
+        float cage_extrusion,
+        float r_co[3], float r_dir[3],
+        const bool is_cage)
+{
 	float data[3][3];
 	float coord[3];
 	float dir[3];
 	float cage[3];
+	bool is_smooth;
 
 	TriTessFace *triangle = &triangles[primitive_id];
+	is_smooth = triangle->is_smooth || is_cage;
 
 	copy_v3_v3(data[0], triangle->mverts[0]->co);
 	copy_v3_v3(data[1], triangle->mverts[1]->co);
@@ -178,19 +225,29 @@ static void calc_point_from_barycentric(
 
 	interp_barycentric_tri_v3(data, u, v, coord);
 
-	normal_short_to_float_v3(data[0], triangle->mverts[0]->no);
-	normal_short_to_float_v3(data[1], triangle->mverts[1]->no);
-	normal_short_to_float_v3(data[2], triangle->mverts[2]->no);
+	if (is_smooth) {
+		normal_short_to_float_v3(data[0], triangle->mverts[0]->no);
+		normal_short_to_float_v3(data[1], triangle->mverts[1]->no);
+		normal_short_to_float_v3(data[2], triangle->mverts[2]->no);
 
-	interp_barycentric_tri_v3(data, u, v, dir);
-	normalize_v3_v3(cage, dir);
-	mul_v3_fl(cage, cage_extrusion);
+		interp_barycentric_tri_v3(data, u, v, dir);
+		normalize_v3(dir);
+	}
+	else {
+		copy_v3_v3(dir, triangle->normal);
+	}
 
+	mul_v3_v3fl(cage, dir, cage_extrusion);
 	add_v3_v3(coord, cage);
 
-	normalize_v3_v3(dir, dir);
+	normalize_v3(dir);
 	negate_v3(dir);
 
+	/* convert from local to world space */
+	mul_m4_v3(mat, coord);
+	mul_transposed_mat3_m4_v3(imat, dir);
+	normalize_v3(dir);
+
 	copy_v3_v3(r_co, coord);
 	copy_v3_v3(r_dir, dir);
 }
@@ -379,30 +436,47 @@ static void mesh_calc_tri_tessface(
 
 void RE_bake_pixels_populate_from_objects(
         struct Mesh *me_low, BakePixel pixel_array_from[],
-        BakeHighPolyData highpoly[], const int tot_highpoly, const int num_pixels,
-        const float cage_extrusion, float mat_low[4][4])
+        BakeHighPolyData highpoly[], const int tot_highpoly, const int num_pixels, const bool is_custom_cage,
+        const float cage_extrusion, float mat_low[4][4], float mat_cage[4][4], struct Mesh *me_cage)
 {
 	int i;
 	int primitive_id;
 	float u, v;
 	float imat_low [4][4];
+	bool is_cage = me_cage != NULL;
 
+	DerivedMesh *dm_low = NULL;
 	DerivedMesh **dm_highpoly;
 	BVHTreeFromMesh *treeData;
 
 	/* Note: all coordinates are in local space */
-	TriTessFace *tris_low;
+	TriTessFace *tris_low = NULL;
+	TriTessFace *tris_cage = NULL;
 	TriTessFace **tris_high;
 
 	/* assume all lowpoly tessfaces can be quads */
-	tris_low = MEM_mallocN(sizeof(TriTessFace) * (me_low->totface * 2), "MVerts Lowpoly Mesh");
 	tris_high = MEM_mallocN(sizeof(TriTessFace *) * tot_highpoly, "MVerts Highpoly Mesh Array");
 
 	/* assume all highpoly tessfaces are triangles */
-	dm_highpoly = MEM_callocN(sizeof(DerivedMesh *) * tot_highpoly, "Highpoly Derived Meshes");
+	dm_highpoly = MEM_mallocN(sizeof(DerivedMesh *) * tot_highpoly, "Highpoly Derived Meshes");
 	treeData = MEM_callocN(sizeof(BVHTreeFromMesh) * tot_highpoly, "Highpoly BVH Trees");
 
-	mesh_calc_tri_tessface(tris_low, me_low, false, NULL);
+	if (!is_cage) {
+		dm_low = CDDM_from_mesh(me_low);
+		tris_low = MEM_mallocN(sizeof(TriTessFace) * (me_low->totface * 2), "MVerts Lowpoly Mesh");
+		mesh_calc_tri_tessface(tris_low, me_low, true, dm_low);
+	}
+	else if (is_custom_cage) {
+		tris_low = MEM_mallocN(sizeof(TriTessFace) * (me_low->totface * 2), "MVerts Lowpoly Mesh");
+		mesh_calc_tri_tessface(tris_low, me_low, false, NULL);
+
+		tris_cage = MEM_mallocN(sizeof(TriTessFace) * (me_low->totface * 2), "MVerts Cage Mesh");
+		mesh_calc_tri_tessface(tris_cage, me_cage, false, NULL);
+	}
+	else {
+		tris_cage = MEM_mallocN(sizeof(TriTessFace) * (me_low->totface * 2), "MVerts Cage Mesh");
+		mesh_calc_tri_tessface(tris_cage, me_cage, false, NULL);
+	}
 
 	invert_m4_m4(imat_low, mat_low);
 
@@ -439,12 +513,15 @@ void RE_bake_pixels_populate_from_objects(
 		v = pixel_array_from[i].uv[1];
 
 		/* calculate from low poly mesh cage */
-		calc_point_from_barycentric(tris_low, primitive_id, u, v, cage_extrusion, co, dir);
-
-		/* convert from local to world space */
-		mul_m4_v3(mat_low, co);
-		mul_transposed_mat3_m4_v3(imat_low, dir);
-		normalize_v3(dir);
+		if (is_custom_cage) {
+			calc_point_from_barycentric_cage(tris_low, tris_cage, mat_low, mat_cage, primitive_id, u, v, co, dir);
+		}
+		else if (is_cage) {
+			calc_point_from_barycentric_extrusion(tris_cage, mat_low, imat_low, primitive_id, u, v, cage_extrusion, co, dir, true);
+		}
+		else {
+			calc_point_from_barycentric_extrusion(tris_low, mat_low, imat_low, primitive_id, u, v, cage_extrusion, co, dir, false);
+		}
 
 		/* cast ray */
 		if (!cast_ray_highpoly(treeData, tris_high, highpoly, co, dir, i, tot_highpoly,
@@ -464,10 +541,19 @@ cleanup:
 		MEM_freeN(tris_high[i]);
 	}
 
-	MEM_freeN(tris_low);
 	MEM_freeN(tris_high);
 	MEM_freeN(treeData);
 	MEM_freeN(dm_highpoly);
+
+	if (dm_low) {
+		dm_low->release(dm_low);
+	}
+	if (tris_low) {
+		MEM_freeN(tris_low);
+	}
+	if (tris_cage) {
+		MEM_freeN(tris_cage);
+	}
 }
 
 static void bake_differentials(BakeDataZSpan *bd, const float *uv1, const float *uv2, const float *uv3)