Bake-API: new approach for cage

There is a new option to select whether you want to use cage or not.
When not using cage the results will be more similar with Blender
Internal, where the inwards rays (trying to hit the highpoly objects)
don't always come from smooth normals. So if the active object has sharp
edges and an EdgeSplit modifier you get bad corners.

This is useful, however, to bake to planes without the need of adding
extra loops around the edges.

When cage is "on" the user can decide on setting a cage extrusion or to
pick a Custom Cage object. The cage extrusion option works in a
duplicated copy of the active object with EdgeSplit modifiers removed to
inforce smooth normals. The custom cage option takes an object with the
same number of faces as the active object (and the same face ordering).

The custom cage now controls the direction and the origin of the
rays casted to the highpoly objects. The direction is a ray from the
point in the cage mesh to the equivalent point to the base mesh. That
means the face normals are entirely ignored when using a cage object.

For developers:
When using an object cage the ray is calculated from the cage mesh to
the base mesh. It uses the barycentric coordinate from the base mesh UV,
so we expect both meshes to have the same primitive ids (which won't be
the case if the cage gets edited in a destructive way).

That fixes T40023 (giving the expected result when 'use_cage' is false).

Thanks for Andy Davies (metalliandy) for the consulting with normal
baking workflow and extensive testing. His 'stress-test' file will be
added later to our svn tests folder. (The file itself is not public yet
since he still has to add testing notes to it).

Many thanks for the reviewers.

More on cages:
http://wiki.polycount.com/NormalMap/#Working_with_Cages

Reviewers: campbellbarton, sergey

CC: adriano, metalliandy, brecht, malkavian

Differential Revision: https://developer.blender.org/D547

6 files changed, 174 insertions, 60 deletions

diff --git a/intern/cycles/blender/addon/ui.py b/intern/cycles/blender/addon/ui.py
index 5c8115b6612..1797f29365e 100644
--- a/intern/cycles/blender/addon/ui.py
+++ b/intern/cycles/blender/addon/ui.py
@@ -1259,8 +1259,13 @@ class CyclesRender_PT_bake(CyclesButtonsPanel, Panel):
         sub = sub.column()
 
         sub.active = cbk.use_selected_to_active
-        sub.prop(cbk, "cage_extrusion", text="Distance")
-        sub.prop_search(cbk, "cage", scene, "objects")
+
+        sub.prop(cbk, "use_cage", text="Cage")
+        if cbk.use_cage:
+            sub.prop(cbk, "cage_extrusion", text="Cage Extrusion")
+            sub.prop_search(cbk, "custom_cage", scene, "objects")
+        else:
+            sub.prop(cbk, "cage_extrusion", text="Ray Distance")
 
         if cscene.bake_type == 'NORMAL':
             col.separator()
diff --git a/source/blender/editors/object/object_bake_api.c b/source/blender/editors/object/object_bake_api.c
index 534ecd5a109..b69de3d0b81 100644
--- a/source/blender/editors/object/object_bake_api.c
+++ b/source/blender/editors/object/object_bake_api.c
@@ -489,6 +489,7 @@ typedef struct BakeAPIRender {
 	bool is_split_materials;
 	bool is_automatic_name;
 	bool is_selected_to_active;
+	bool is_cage;
 
 	float cage_extrusion;
 	int normal_space;
@@ -512,7 +513,7 @@ static int bake(
         Main *bmain, Scene *scene, Object *ob_low, ListBase *selected_objects, ReportList *reports,
         const ScenePassType pass_type, const int margin,
         const BakeSaveMode save_mode, const bool is_clear, const bool is_split_materials,
-        const bool is_automatic_name, const bool is_selected_to_active,
+        const bool is_automatic_name, const bool is_selected_to_active, const bool is_cage,
         const float cage_extrusion, const int normal_space, const BakeNormalSwizzle normal_swizzle[],
         const char *custom_cage, const char *filepath, const int width, const int height,
         const char *identifier, ScrArea *sa)
@@ -529,6 +530,7 @@ static int bake(
 	char restrict_flag_cage;
 
 	Mesh *me_low = NULL;
+	Mesh *me_cage = NULL;
 	Render *re;
 
 	float *result = NULL;
@@ -614,14 +616,11 @@ static int bake(
 			tot_highpoly ++;
 		}
 
-		if (custom_cage[0] != '\0') {
+		if (is_cage && custom_cage[0] != '\0') {
 			ob_cage = BLI_findstring(&bmain->object, custom_cage, offsetof(ID, name) + 2);
 
-			/* TODO check if cage object has the same topology (num of triangles and a valid UV) */
 			if (ob_cage == NULL || ob_cage->type != OB_MESH) {
 				BKE_report(reports, RPT_ERROR, "No valid cage object");
-				op_result = OPERATOR_CANCELLED;
-
 				goto cleanup;
 			}
 			else {
@@ -640,20 +639,31 @@ static int bake(
 	pixel_array_low = MEM_callocN(sizeof(BakePixel) * num_pixels, "bake pixels low poly");
 	result = MEM_callocN(sizeof(float) * depth * num_pixels, "bake return pixels");
 
+	/* get the mesh as it arrives in the renderer */
+	me_low = BKE_mesh_new_from_object(bmain, scene, ob_low, 1, 2, 1, 0);
+
+	/* populate the pixel array with the face data */
+	if ((is_selected_to_active && (ob_cage == NULL) && is_cage) == false)
+		RE_bake_pixels_populate(me_low, pixel_array_low, num_pixels, &bake_images);
+	/* else populate the pixel array with the 'cage' mesh (the smooth version of the mesh)  */
+
 	if (is_selected_to_active) {
 		CollectionPointerLink *link;
 		ModifierData *md, *nmd;
 		ListBase modifiers_tmp, modifiers_original;
-		float mat_low[4][4];
 		int i = 0;
-		highpoly = MEM_callocN(sizeof(BakeHighPolyData) * tot_highpoly, "bake high poly objects");
 
 		/* prepare cage mesh */
 		if (ob_cage) {
-			me_low = BKE_mesh_new_from_object(bmain, scene, ob_cage, 1, 2, 1, 0);
-			copy_m4_m4(mat_low, ob_cage->obmat);
+			me_cage = BKE_mesh_new_from_object(bmain, scene, ob_cage, 1, 2, 1, 0);
+			if (me_low->totface != me_cage->totface) {
+				BKE_report(reports, RPT_ERROR,
+				           "Invalid cage object, the cage mesh must have the same number "
+				           "of faces as the active object");
+				goto cleanup;
+			}
 		}
-		else {
+		else if (is_cage) {
 			modifiers_original = ob_low->modifiers;
 			BLI_listbase_clear(&modifiers_tmp);
 
@@ -677,10 +687,12 @@ static int bake(
 			ob_low->modifiers = modifiers_tmp;
 
 			/* get the cage mesh as it arrives in the renderer */
-			me_low = BKE_mesh_new_from_object(bmain, scene, ob_low, 1, 2, 1, 0);
-			copy_m4_m4(mat_low, ob_low->obmat);
+			me_cage = BKE_mesh_new_from_object(bmain, scene, ob_low, 1, 2, 1, 0);
+			RE_bake_pixels_populate(me_cage, pixel_array_low, num_pixels, &bake_images);
 		}
 
+		highpoly = MEM_callocN(sizeof(BakeHighPolyData) * tot_highpoly, "bake high poly objects");
+
 		/* populate highpoly array */
 		for (link = selected_objects->first; link; link = link->next) {
 			TriangulateModifierData *tmd;
@@ -723,15 +735,12 @@ static int bake(
 
 		BLI_assert(i == tot_highpoly);
 
-		/* populate the pixel array with the face data */
-		RE_bake_pixels_populate(me_low, pixel_array_low, num_pixels, &bake_images);
-
 		ob_low->restrictflag |= OB_RESTRICT_RENDER;
 
 		/* populate the pixel arrays with the corresponding face data for each high poly object */
 		RE_bake_pixels_populate_from_objects(
-		        me_low, pixel_array_low, highpoly, tot_highpoly,
-		        num_pixels, cage_extrusion, mat_low);
+		        me_low, pixel_array_low, highpoly, tot_highpoly, num_pixels, ob_cage != NULL,
+		        cage_extrusion, ob_low->obmat, (ob_cage ? ob_cage->obmat : ob_low->obmat), me_cage);
 
 		/* the baking itself */
 		for (i = 0; i < tot_highpoly; i++) {
@@ -752,7 +761,7 @@ static int bake(
 		if (ob_cage) {
 			ob_cage->restrictflag |= OB_RESTRICT_RENDER;
 		}
-		else {
+		else if (is_cage) {
 			ob_low->modifiers = modifiers_original;
 
 			while ((md = BLI_pophead(&modifiers_tmp))) {
@@ -761,12 +770,6 @@ static int bake(
 		}
 	}
 	else {
-		/* get the mesh as it arrives in the renderer */
-		me_low = BKE_mesh_new_from_object(bmain, scene, ob_low, 1, 2, 1, 0);
-
-		/* populate the pixel array with the face data */
-		RE_bake_pixels_populate(me_low, pixel_array_low, num_pixels, &bake_images);
-
 		/* make sure low poly renders */
 		ob_low->restrictflag &= ~OB_RESTRICT_RENDER;
 
@@ -965,6 +968,9 @@ cleanup:
 	if (me_low)
 		BKE_libblock_free(bmain, me_low);
 
+	if (me_cage)
+		BKE_libblock_free(bmain, me_cage);
+
 	return op_result;
 }
 
@@ -988,6 +994,7 @@ static void bake_init_api_data(wmOperator *op, bContext *C, BakeAPIRender *bkr)
 	bkr->is_split_materials = (!is_save_internal) && RNA_boolean_get(op->ptr, "use_split_materials");
 	bkr->is_automatic_name = RNA_boolean_get(op->ptr, "use_automatic_name");
 	bkr->is_selected_to_active = RNA_boolean_get(op->ptr, "use_selected_to_active");
+	bkr->is_cage = RNA_boolean_get(op->ptr, "use_cage");
 	bkr->cage_extrusion = RNA_float_get(op->ptr, "cage_extrusion");
 
 	bkr->normal_space = RNA_enum_get(op->ptr, "normal_space");
@@ -1036,7 +1043,7 @@ static int bake_exec(bContext *C, wmOperator *op)
 		result = bake(
 		        bkr.main, bkr.scene, bkr.ob, &bkr.selected_objects, bkr.reports,
 		        bkr.pass_type, bkr.margin, bkr.save_mode,
-		        bkr.is_clear, bkr.is_split_materials, bkr.is_automatic_name, true,
+		        bkr.is_clear, bkr.is_split_materials, bkr.is_automatic_name, true, bkr.is_cage,
 		        bkr.cage_extrusion, bkr.normal_space, bkr.normal_swizzle,
 		        bkr.custom_cage, bkr.filepath, bkr.width, bkr.height, bkr.identifier, bkr.sa);
 	}
@@ -1048,7 +1055,7 @@ static int bake_exec(bContext *C, wmOperator *op)
 			result = bake(
 			        bkr.main, bkr.scene, ob_iter, NULL, bkr.reports,
 			        bkr.pass_type, bkr.margin, bkr.save_mode,
-			        is_clear, bkr.is_split_materials, bkr.is_automatic_name, false,
+			        is_clear, bkr.is_split_materials, bkr.is_automatic_name, false, bkr.is_cage,
 			        bkr.cage_extrusion, bkr.normal_space, bkr.normal_swizzle,
 			        bkr.custom_cage, bkr.filepath, bkr.width, bkr.height, bkr.identifier, bkr.sa);
 		}
@@ -1076,7 +1083,7 @@ static void bake_startjob(void *bkv, short *UNUSED(stop), short *UNUSED(do_updat
 		bkr->result = bake(
 		        bkr->main, bkr->scene, bkr->ob, &bkr->selected_objects, bkr->reports,
 		        bkr->pass_type, bkr->margin, bkr->save_mode,
-		        bkr->is_clear, bkr->is_split_materials, bkr->is_automatic_name, true,
+		        bkr->is_clear, bkr->is_split_materials, bkr->is_automatic_name, true, bkr->is_cage,
 		        bkr->cage_extrusion, bkr->normal_space, bkr->normal_swizzle,
 		        bkr->custom_cage, bkr->filepath, bkr->width, bkr->height, bkr->identifier, bkr->sa);
 	}
@@ -1088,7 +1095,7 @@ static void bake_startjob(void *bkv, short *UNUSED(stop), short *UNUSED(do_updat
 			bkr->result = bake(
 			        bkr->main, bkr->scene, ob_iter, NULL, bkr->reports,
 			        bkr->pass_type, bkr->margin, bkr->save_mode,
-			        is_clear, bkr->is_split_materials, bkr->is_automatic_name, false,
+			        is_clear, bkr->is_split_materials, bkr->is_automatic_name, false, bkr->is_cage,
 			        bkr->cage_extrusion, bkr->normal_space, bkr->normal_swizzle,
 			        bkr->custom_cage, bkr->filepath, bkr->width, bkr->height, bkr->identifier, bkr->sa);
 
@@ -1178,6 +1185,11 @@ static void bake_set_props(wmOperator *op, Scene *scene)
 		RNA_property_boolean_set(op->ptr, prop, (bake->flag & R_BAKE_CLEAR));
 	}
 
+	prop = RNA_struct_find_property(op->ptr, "use_cage");
+	if (!RNA_property_is_set(op->ptr, prop)) {
+		RNA_property_boolean_set(op->ptr, prop, (bake->flag & R_BAKE_CAGE));
+	}
+
 	prop = RNA_struct_find_property(op->ptr, "use_split_materials");
 	if (!RNA_property_is_set(op->ptr, prop)) {
 		RNA_property_boolean_set(op->ptr, prop, (bake->flag & R_BAKE_SPLIT_MAT));
@@ -1254,8 +1266,8 @@ void OBJECT_OT_bake(wmOperatorType *ot)
 	                "Bake shading on the surface of selected objects to the active object");
 	RNA_def_float(ot->srna, "cage_extrusion", 0.0, 0.0, 1.0, "Cage Extrusion",
 	              "Distance to use for the inward ray cast when using selected to active", 0.0, 1.0);
-	RNA_def_string(ot->srna, "cage", NULL, MAX_NAME, "Cage",
-	               "Object to use as cage");
+	RNA_def_string(ot->srna, "cage", NULL, MAX_NAME, "Custom Cage",
+	               "Object to use as cage, instead of calculating the cage from the active object with cage extrusion");
 	RNA_def_enum(ot->srna, "normal_space", normal_space_items, R_BAKE_SPACE_TANGENT, "Normal Space",
 	             "Choose normal space for baking");
 	RNA_def_enum(ot->srna, "normal_r", normal_swizzle_items, R_BAKE_POSX, "R", "Axis to bake in red channel");
@@ -1265,6 +1277,8 @@ void OBJECT_OT_bake(wmOperatorType *ot)
 	             "Choose how to save the baking map");
 	RNA_def_boolean(ot->srna, "use_clear", false, "Clear",
 	                "Clear Images before baking (only for internal saving)");
+	RNA_def_boolean(ot->srna, "use_cage", false, "Cage",
+	                "Cast rays to active object from a cage");
 	RNA_def_boolean(ot->srna, "use_split_materials", false, "Split Materials",
 	                "Split baked maps per material, using material name in output file (external only)");
 	RNA_def_boolean(ot->srna, "use_automatic_name", false, "Automatic Name",
diff --git a/source/blender/makesdna/DNA_scene_types.h b/source/blender/makesdna/DNA_scene_types.h
index ceb938c3af2..c95cdb5c1f8 100644
--- a/source/blender/makesdna/DNA_scene_types.h
+++ b/source/blender/makesdna/DNA_scene_types.h
@@ -1442,8 +1442,9 @@ enum {
 #define R_BAKE_LORES_MESH	32
 #define R_BAKE_VCOL			64
 #define R_BAKE_USERSCALE	128
-#define R_BAKE_SPLIT_MAT	256
-#define R_BAKE_AUTO_NAME	512
+#define R_BAKE_CAGE			256
+#define R_BAKE_SPLIT_MAT	512
+#define R_BAKE_AUTO_NAME	1024
 
 /* bake_normal_space */
 #define R_BAKE_SPACE_CAMERA	 0
diff --git a/source/blender/makesrna/intern/rna_scene.c b/source/blender/makesrna/intern/rna_scene.c
index e272d93a1e7..92c4d3dd204 100644
--- a/source/blender/makesrna/intern/rna_scene.c
+++ b/source/blender/makesrna/intern/rna_scene.c
@@ -3218,8 +3218,10 @@ static void rna_def_bake_data(BlenderRNA *brna)
 	RNA_def_struct_nested(brna, srna, "RenderSettings");
 	RNA_def_struct_ui_text(srna, "Bake Data", "Bake data for a Scene datablock");
 
-	prop = RNA_def_property(srna, "cage", PROP_STRING, PROP_NONE);
-	RNA_def_property_ui_text(prop, "Cage", "Object to use as cage");
+	prop = RNA_def_property(srna, "custom_cage", PROP_STRING, PROP_NONE);
+	RNA_def_property_string_sdna(prop, NULL, "cage");
+	RNA_def_property_ui_text(prop, "Custom Cage", "Object to use as cage "
+	                         "instead of calculating the cage from the active object with cage extrusion");
 	RNA_def_property_update(prop, NC_SCENE | ND_RENDER_OPTIONS, NULL);
 
 	prop = RNA_def_property(srna, "filepath", PROP_STRING, PROP_FILEPATH);
@@ -3309,6 +3311,12 @@ static void rna_def_bake_data(BlenderRNA *brna)
 	RNA_def_property_ui_text(prop, "Automatic Name",
 	                         "Automatically name the output file with the pass type (external only)");
 	RNA_def_property_update(prop, NC_SCENE | ND_RENDER_OPTIONS, NULL);
+
+	prop = RNA_def_property(srna, "use_cage", PROP_BOOLEAN, PROP_NONE);
+	RNA_def_property_boolean_sdna(prop, NULL, "flag", R_BAKE_CAGE);
+	RNA_def_property_ui_text(prop, "Cage",
+	                         "Cast rays to active object from a cage");
+	RNA_def_property_update(prop, NC_SCENE | ND_RENDER_OPTIONS, NULL);
 }
 
 static void rna_def_scene_game_data(BlenderRNA *brna)
diff --git a/source/blender/render/extern/include/RE_bake.h b/source/blender/render/extern/include/RE_bake.h
index 0f82082911a..4727aef460e 100644
--- a/source/blender/render/extern/include/RE_bake.h
+++ b/source/blender/render/extern/include/RE_bake.h
@@ -82,8 +82,8 @@ bool RE_bake_internal(
 
 void RE_bake_pixels_populate_from_objects(
         struct Mesh *me_low, BakePixel pixel_array_from[],
-        BakeHighPolyData highpoly[], const int tot_highpoly, const int num_pixels,
-        const float cage_extrusion, float mat_low[4][4]);
+        BakeHighPolyData highpoly[], const int tot_highpoly, const int num_pixels, const bool is_custom_cage,
+        const float cage_extrusion, float mat_low[4][4], float mat_cage[4][4], struct Mesh *me_cage);
 
 void RE_bake_pixels_populate(
         struct Mesh *me, struct BakePixel *pixel_array,
diff --git a/source/blender/render/intern/source/bake_api.c b/source/blender/render/intern/source/bake_api.c
index 48516ea895c..3de39378fad 100644
--- a/source/blender/render/intern/source/bake_api.c
+++ b/source/blender/render/intern/source/bake_api.c
@@ -158,19 +158,66 @@ void RE_bake_margin(ImBuf *ibuf, char *mask, const int margin)
 		IMB_rectfill_alpha(ibuf, 1.0f);
 }
 
+
 /**
  * This function returns the coordinate and normal of a barycentric u,v for a face defined by the primitive_id index.
+ * The returned normal is actually the direction from the same barycentric coordinate in the cage to the base mesh
+ * The returned coordinate is the point in the cage mesh
  */
-static void calc_point_from_barycentric(
-        TriTessFace *triangles, int primitive_id, float u, float v, float cage_extrusion,
+static void calc_point_from_barycentric_cage(
+        TriTessFace *triangles_low, TriTessFace *triangles_cage,
+        float mat_low[4][4], float mat_cage[4][4],
+        int primitive_id, float u, float v,
         float r_co[3], float r_dir[3])
 {
+	float data[2][3][3];
+	float coord[2][3];
+	float dir[3];
+	int i;
+
+	TriTessFace *triangle[2];
+
+	triangle[0] = &triangles_low[primitive_id];
+	triangle[1] = &triangles_cage[primitive_id];
+
+	for (i = 0; i < 2; i++) {
+		copy_v3_v3(data[i][0], triangle[i]->mverts[0]->co);
+		copy_v3_v3(data[i][1], triangle[i]->mverts[1]->co);
+		copy_v3_v3(data[i][2], triangle[i]->mverts[2]->co);
+		interp_barycentric_tri_v3(data[i], u, v, coord[i]);
+	}
+
+	/* convert from local to world space */
+	mul_m4_v3(mat_low, coord[0]);
+	mul_m4_v3(mat_cage, coord[1]);
+
+	sub_v3_v3v3(dir, coord[0], coord[1]);
+	normalize_v3(dir);
+
+	copy_v3_v3(r_co, coord[1]);
+	copy_v3_v3(r_dir, dir);
+}
+
+/**
+ * This function returns the coordinate and normal of a barycentric u,v for a face defined by the primitive_id index.
+ * The returned coordinate is extruded along the normal by cage_extrusion
+ */
+static void calc_point_from_barycentric_extrusion(
+        TriTessFace *triangles,
+        float mat[4][4], float imat[4][4],
+        int primitive_id, float u, float v,
+        float cage_extrusion,
+        float r_co[3], float r_dir[3],
+        const bool is_cage)
+{
 	float data[3][3];
 	float coord[3];
 	float dir[3];
 	float cage[3];
+	bool is_smooth;
 
 	TriTessFace *triangle = &triangles[primitive_id];
+	is_smooth = triangle->is_smooth || is_cage;
 
 	copy_v3_v3(data[0], triangle->mverts[0]->co);
 	copy_v3_v3(data[1], triangle->mverts[1]->co);
@@ -178,19 +225,29 @@ static void calc_point_from_barycentric(
 
 	interp_barycentric_tri_v3(data, u, v, coord);
 
-	normal_short_to_float_v3(data[0], triangle->mverts[0]->no);
-	normal_short_to_float_v3(data[1], triangle->mverts[1]->no);
-	normal_short_to_float_v3(data[2], triangle->mverts[2]->no);
+	if (is_smooth) {
+		normal_short_to_float_v3(data[0], triangle->mverts[0]->no);
+		normal_short_to_float_v3(data[1], triangle->mverts[1]->no);
+		normal_short_to_float_v3(data[2], triangle->mverts[2]->no);
 
-	interp_barycentric_tri_v3(data, u, v, dir);
-	normalize_v3_v3(cage, dir);
-	mul_v3_fl(cage, cage_extrusion);
+		interp_barycentric_tri_v3(data, u, v, dir);
+		normalize_v3(dir);
+	}
+	else {
+		copy_v3_v3(dir, triangle->normal);
+	}
 
+	mul_v3_v3fl(cage, dir, cage_extrusion);
 	add_v3_v3(coord, cage);
 
-	normalize_v3_v3(dir, dir);
+	normalize_v3(dir);
 	negate_v3(dir);
 
+	/* convert from local to world space */
+	mul_m4_v3(mat, coord);
+	mul_transposed_mat3_m4_v3(imat, dir);
+	normalize_v3(dir);
+
 	copy_v3_v3(r_co, coord);
 	copy_v3_v3(r_dir, dir);
 }
@@ -379,30 +436,47 @@ static void mesh_calc_tri_tessface(
 
 void RE_bake_pixels_populate_from_objects(
         struct Mesh *me_low, BakePixel pixel_array_from[],
-        BakeHighPolyData highpoly[], const int tot_highpoly, const int num_pixels,
-        const float cage_extrusion, float mat_low[4][4])
+        BakeHighPolyData highpoly[], const int tot_highpoly, const int num_pixels, const bool is_custom_cage,
+        const float cage_extrusion, float mat_low[4][4], float mat_cage[4][4], struct Mesh *me_cage)
 {
 	int i;
 	int primitive_id;
 	float u, v;
 	float imat_low [4][4];
+	bool is_cage = me_cage != NULL;
 
+	DerivedMesh *dm_low = NULL;
 	DerivedMesh **dm_highpoly;
 	BVHTreeFromMesh *treeData;
 
 	/* Note: all coordinates are in local space */
-	TriTessFace *tris_low;
+	TriTessFace *tris_low = NULL;
+	TriTessFace *tris_cage = NULL;
 	TriTessFace **tris_high;
 
 	/* assume all lowpoly tessfaces can be quads */
-	tris_low = MEM_mallocN(sizeof(TriTessFace) * (me_low->totface * 2), "MVerts Lowpoly Mesh");
 	tris_high = MEM_mallocN(sizeof(TriTessFace *) * tot_highpoly, "MVerts Highpoly Mesh Array");
 
 	/* assume all highpoly tessfaces are triangles */
-	dm_highpoly = MEM_callocN(sizeof(DerivedMesh *) * tot_highpoly, "Highpoly Derived Meshes");
+	dm_highpoly = MEM_mallocN(sizeof(DerivedMesh *) * tot_highpoly, "Highpoly Derived Meshes");
 	treeData = MEM_callocN(sizeof(BVHTreeFromMesh) * tot_highpoly, "Highpoly BVH Trees");
 
-	mesh_calc_tri_tessface(tris_low, me_low, false, NULL);
+	if (!is_cage) {
+		dm_low = CDDM_from_mesh(me_low);
+		tris_low = MEM_mallocN(sizeof(TriTessFace) * (me_low->totface * 2), "MVerts Lowpoly Mesh");
+		mesh_calc_tri_tessface(tris_low, me_low, true, dm_low);
+	}
+	else if (is_custom_cage) {
+		tris_low = MEM_mallocN(sizeof(TriTessFace) * (me_low->totface * 2), "MVerts Lowpoly Mesh");
+		mesh_calc_tri_tessface(tris_low, me_low, false, NULL);
+
+		tris_cage = MEM_mallocN(sizeof(TriTessFace) * (me_low->totface * 2), "MVerts Cage Mesh");
+		mesh_calc_tri_tessface(tris_cage, me_cage, false, NULL);
+	}
+	else {
+		tris_cage = MEM_mallocN(sizeof(TriTessFace) * (me_low->totface * 2), "MVerts Cage Mesh");
+		mesh_calc_tri_tessface(tris_cage, me_cage, false, NULL);
+	}
 
 	invert_m4_m4(imat_low, mat_low);
 
@@ -439,12 +513,15 @@ void RE_bake_pixels_populate_from_objects(
 		v = pixel_array_from[i].uv[1];
 
 		/* calculate from low poly mesh cage */
-		calc_point_from_barycentric(tris_low, primitive_id, u, v, cage_extrusion, co, dir);
-
-		/* convert from local to world space */
-		mul_m4_v3(mat_low, co);
-		mul_transposed_mat3_m4_v3(imat_low, dir);
-		normalize_v3(dir);
+		if (is_custom_cage) {
+			calc_point_from_barycentric_cage(tris_low, tris_cage, mat_low, mat_cage, primitive_id, u, v, co, dir);
+		}
+		else if (is_cage) {
+			calc_point_from_barycentric_extrusion(tris_cage, mat_low, imat_low, primitive_id, u, v, cage_extrusion, co, dir, true);
+		}
+		else {
+			calc_point_from_barycentric_extrusion(tris_low, mat_low, imat_low, primitive_id, u, v, cage_extrusion, co, dir, false);
+		}
 
 		/* cast ray */
 		if (!cast_ray_highpoly(treeData, tris_high, highpoly, co, dir, i, tot_highpoly,
@@ -464,10 +541,19 @@ cleanup:
 		MEM_freeN(tris_high[i]);
 	}
 
-	MEM_freeN(tris_low);
 	MEM_freeN(tris_high);
 	MEM_freeN(treeData);
 	MEM_freeN(dm_highpoly);
+
+	if (dm_low) {
+		dm_low->release(dm_low);
+	}
+	if (tris_low) {
+		MEM_freeN(tris_low);
+	}
+	if (tris_cage) {
+		MEM_freeN(tris_cage);
+	}
 }
 
 static void bake_differentials(BakeDataZSpan *bd, const float *uv1, const float *uv2, const float *uv3)