Fix T60615: Cycles baking not working with some modifiers.

Refactors Cycles mesh export a bit to avoid unnecessary copies and to be in
sync with the Blender baker.

5 files changed, 71 insertions, 94 deletions

diff --git a/intern/cycles/blender/blender_mesh.cpp b/intern/cycles/blender/blender_mesh.cpp
index 7a85ff27e53..8e81e3ac121 100644
--- a/intern/cycles/blender/blender_mesh.cpp
+++ b/intern/cycles/blender/blender_mesh.cpp
@@ -756,13 +756,11 @@ static void create_mesh(Scene *scene,
 		return;
 	}
 
-	BL::Mesh::vertices_iterator v;
-	BL::Mesh::polygons_iterator p;
-
 	if(!subdivision) {
 		numtris = numfaces;
 	}
 	else {
+		BL::Mesh::polygons_iterator p;
 		for(b_mesh.polygons.begin(p); p != b_mesh.polygons.end(); ++p) {
 			numngons += (p->loop_total() == 4)? 0: 1;
 			numcorners += p->loop_total();
@@ -774,6 +772,7 @@ static void create_mesh(Scene *scene,
 	mesh->reserve_subd_faces(numfaces, numngons, numcorners);
 
 	/* create vertex coordinates and normals */
+	BL::Mesh::vertices_iterator v;
 	for(b_mesh.vertices.begin(v); v != b_mesh.vertices.end(); ++v)
 		mesh->add_vertex(get_float3(v->co()));
 
@@ -808,13 +807,10 @@ static void create_mesh(Scene *scene,
 	}
 
 	/* create faces */
-	vector<int> nverts(numfaces);
-
 	if(!subdivision) {
 		BL::Mesh::loop_triangles_iterator t;
-		int ti = 0;
 
-		for(b_mesh.loop_triangles.begin(t); t != b_mesh.loop_triangles.end(); ++t, ++ti) {
+		for(b_mesh.loop_triangles.begin(t); t != b_mesh.loop_triangles.end(); ++t) {
 			BL::MeshPolygon p = b_mesh.polygons[t->polygon_index()];
 			int3 vi = get_int3(t->vertices());
 
@@ -835,10 +831,10 @@ static void create_mesh(Scene *scene,
 			 * NOTE: Autosmooth is already taken care about.
 			 */
 			mesh->add_triangle(vi[0], vi[1], vi[2], shader, smooth);
-			nverts[ti] = 3;
 		}
 	}
 	else {
+		BL::Mesh::polygons_iterator p;
 		vector<int> vi;
 
 		for(b_mesh.polygons.begin(p); p != b_mesh.polygons.end(); ++p) {
@@ -1065,37 +1061,26 @@ Mesh *BlenderSync::sync_mesh(BL::Depsgraph& b_depsgraph,
 	mesh->name = ustring(b_ob_data.name().c_str());
 
 	if(requested_geometry_flags != Mesh::GEOMETRY_NONE) {
-		/* mesh objects does have special handle in the dependency graph,
-		 * they're ensured to have properly updated.
-		 *
-		 * updating meshes here will end up having derived mesh referencing
-		 * freed data from the blender side.
-		 */
-		if(preview && b_ob.type() != BL::Object::type_MESH) {
-			b_ob.update_from_editmode(b_data);
+		/* Adaptive subdivision setup. Not for baking since that requires
+		 * exact mapping to the Blender mesh. */
+		if(scene->bake_manager->get_baking()) {
+			mesh->subdivision_type = Mesh::SUBDIVISION_NONE;
+		}
+		else {
+			mesh->subdivision_type = object_subdivision_type(b_ob, preview, experimental);
 		}
 
 		/* For some reason, meshes do not need this... */
-		bool apply_modifiers = (b_ob.type() != BL::Object::type_MESH);
 		bool need_undeformed = mesh->need_attribute(scene, ATTR_STD_GENERATED);
 
-		mesh->subdivision_type = object_subdivision_type(b_ob, preview, experimental);
-
-		/* Disable adaptive subdivision while baking as the baking system
-		 * currently doesnt support the topology and will crash.
-		 */
-		if(scene->bake_manager->get_baking()) {
-			mesh->subdivision_type = Mesh::SUBDIVISION_NONE;
-		}
-
 		BL::Mesh b_mesh = object_to_mesh(b_data,
 		                                 b_ob,
 		                                 b_depsgraph,
-		                                 apply_modifiers,
 		                                 need_undeformed,
 		                                 mesh->subdivision_type);
 
 		if(b_mesh) {
+			/* Sync mesh itself. */
 			if(view_layer.use_surfaces && show_self) {
 				if(mesh->subdivision_type != Mesh::SUBDIVISION_NONE)
 					create_subd_mesh(scene, mesh, b_ob, b_mesh, used_shaders,
@@ -1106,11 +1091,12 @@ Mesh *BlenderSync::sync_mesh(BL::Depsgraph& b_depsgraph,
 				create_mesh_volume_attributes(scene, b_ob, mesh, b_scene.frame_current());
 			}
 
-			if(view_layer.use_hair && show_particles && mesh->subdivision_type == Mesh::SUBDIVISION_NONE)
+			/* Sync hair curves. */
+			if(view_layer.use_hair && show_particles && mesh->subdivision_type == Mesh::SUBDIVISION_NONE) {
 				sync_curves(mesh, b_mesh, b_ob, false);
+			}
 
-			/* free derived mesh */
-			b_data.meshes.remove(b_mesh, false, true, false);
+			free_object_to_mesh(b_data, b_ob, b_mesh);
 		}
 	}
 	mesh->geometry_flags = requested_geometry_flags;
@@ -1176,7 +1162,6 @@ void BlenderSync::sync_mesh_motion(BL::Depsgraph& b_depsgraph,
 		                        b_ob,
 		                        b_depsgraph,
 		                        false,
-		                        false,
 		                        Mesh::SUBDIVISION_NONE);
 	}
 
@@ -1287,7 +1272,7 @@ void BlenderSync::sync_mesh_motion(BL::Depsgraph& b_depsgraph,
 		sync_curves(mesh, b_mesh, b_ob, true, motion_step);
 
 	/* free derived mesh */
-	b_data.meshes.remove(b_mesh, false, true, false);
+	free_object_to_mesh(b_data, b_ob, b_mesh);
 }
 
 CCL_NAMESPACE_END
diff --git a/intern/cycles/blender/blender_util.h b/intern/cycles/blender/blender_util.h
index 53800cab90d..d3a8b935a6c 100644
--- a/intern/cycles/blender/blender_util.h
+++ b/intern/cycles/blender/blender_util.h
@@ -47,14 +47,14 @@ void python_thread_state_restore(void **python_thread_state);
 static inline BL::Mesh object_to_mesh(BL::BlendData& data,
                                       BL::Object& object,
                                       BL::Depsgraph& depsgraph,
-                                      bool apply_modifiers,
                                       bool calc_undeformed,
                                       Mesh::SubdivisionType subdivision_type)
 {
+	/* TODO: make this work with copy-on-write, modifiers are already evaluated. */
+#if 0
 	bool subsurf_mod_show_render = false;
 	bool subsurf_mod_show_viewport = false;
 
-	/* TODO: make this work with copy-on-write, modifiers are already evaluated. */
 	if(subdivision_type != Mesh::SUBDIVISION_NONE) {
 		BL::Modifier subsurf_mod = object.modifiers[object.modifiers.length()-1];
 
@@ -64,30 +64,50 @@ static inline BL::Mesh object_to_mesh(BL::BlendData& data,
 		subsurf_mod.show_render(false);
 		subsurf_mod.show_viewport(false);
 	}
+#endif
 
-	BL::Mesh me = data.meshes.new_from_object(depsgraph, object, apply_modifiers, calc_undeformed);
+	BL::Mesh mesh(PointerRNA_NULL);
+	if(object.type() == BL::Object::type_MESH) {
+		/* TODO: calc_undeformed is not used. */
+		mesh = BL::Mesh(object.data());
 
+		/* Make a copy to split faces if we use autosmooth, otherwise not needed. */
+		if (mesh.use_auto_smooth() && subdivision_type == Mesh::SUBDIVISION_NONE) {
+			mesh = data.meshes.new_from_object(depsgraph, object, false, false);
+		}
+	}
+	else {
+		mesh = data.meshes.new_from_object(depsgraph, object, true, calc_undeformed);
+	}
+
+#if 0
 	if(subdivision_type != Mesh::SUBDIVISION_NONE) {
 		BL::Modifier subsurf_mod = object.modifiers[object.modifiers.length()-1];
 
 		subsurf_mod.show_render(subsurf_mod_show_render);
 		subsurf_mod.show_viewport(subsurf_mod_show_viewport);
 	}
+#endif
 
-	if((bool)me) {
-		if(me.use_auto_smooth()) {
-			if(subdivision_type == Mesh::SUBDIVISION_CATMULL_CLARK) {
-				me.calc_normals_split();
-			}
-			else {
-				me.split_faces(false);
-			}
-		}
-		if(subdivision_type == Mesh::SUBDIVISION_NONE) {
-			me.calc_loop_triangles();
+	if((bool)mesh && subdivision_type == Mesh::SUBDIVISION_NONE) {
+		if(mesh.use_auto_smooth()) {
+			mesh.split_faces(false);
 		}
+
+		mesh.calc_loop_triangles();
+	}
+
+	return mesh;
+}
+
+static inline void free_object_to_mesh(BL::BlendData& data,
+                                       BL::Object& object,
+                                       BL::Mesh& mesh)
+{
+	/* Free mesh if we didn't just use the existing one. */
+	if(object.data().ptr.data != mesh.ptr.data) {
+		data.meshes.remove(mesh, false, true, false);
 	}
-	return me;
 }
 
 static inline void colorramp_to_array(BL::ColorRamp& ramp,
diff --git a/intern/cycles/render/mesh.cpp b/intern/cycles/render/mesh.cpp
index 5f884a3f871..955fa390e06 100644
--- a/intern/cycles/render/mesh.cpp
+++ b/intern/cycles/render/mesh.cpp
@@ -580,30 +580,6 @@ void Mesh::clear(bool preserve_voxel_data)
 	patch_table = NULL;
 }
 
-int Mesh::split_vertex(int vertex)
-{
-	/* copy vertex location and vertex attributes */
-	add_vertex_slow(verts[vertex]);
-
-	foreach(Attribute& attr, attributes.attributes) {
-		if(attr.element == ATTR_ELEMENT_VERTEX) {
-			array<char> tmp(attr.data_sizeof());
-			memcpy(tmp.data(), attr.data() + tmp.size()*vertex, tmp.size());
-			attr.add(tmp.data());
-		}
-	}
-
-	foreach(Attribute& attr, subd_attributes.attributes) {
-		if(attr.element == ATTR_ELEMENT_VERTEX) {
-			array<char> tmp(attr.data_sizeof());
-			memcpy(tmp.data(), attr.data() + tmp.size()*vertex, tmp.size());
-			attr.add(tmp.data());
-		}
-	}
-
-	return verts.size() - 1;
-}
-
 void Mesh::add_vertex(float3 P)
 {
 	verts.push_back_reserved(P);
diff --git a/intern/cycles/render/mesh.h b/intern/cycles/render/mesh.h
index 7d36b2cd7ca..789d1cc2b54 100644
--- a/intern/cycles/render/mesh.h
+++ b/intern/cycles/render/mesh.h
@@ -274,7 +274,6 @@ public:
 	void add_curve_key(float3 loc, float radius);
 	void add_curve(int first_key, int shader);
 	void add_subd_face(int* corners, int num_corners, int shader_, bool smooth_);
-	int split_vertex(int vertex);
 
 	void compute_bounds();
 	void add_face_normals();
diff --git a/source/blender/editors/object/object_bake_api.c b/source/blender/editors/object/object_bake_api.c
index 81715890f0a..5f403dad54a 100644
--- a/source/blender/editors/object/object_bake_api.c
+++ b/source/blender/editors/object/object_bake_api.c
@@ -635,15 +635,9 @@ static size_t initialize_internal_images(BakeImages *bake_images, ReportList *re
 /* create new mesh with edit mode changes and modifiers applied */
 static Mesh *bake_mesh_new_from_object(Depsgraph *depsgraph, Main *bmain, Scene *scene, Object *ob)
 {
-	bool apply_modifiers = false;
+	bool apply_modifiers = (ob->type != OB_MESH);
+	Mesh *me = BKE_mesh_new_from_object(depsgraph, bmain, scene, ob, apply_modifiers, false);
 
-	/* Mesh is already updated and has modifiers applied. */
-	if (ob->type != OB_MESH) {
-		ED_object_editmode_load(bmain, ob);
-		apply_modifiers = true;
-	}
-
-	Mesh *me = BKE_mesh_new_from_object(depsgraph, bmain, scene, ob, apply_modifiers, 0);
 	if (me->flag & ME_AUTOSMOOTH) {
 		BKE_mesh_split_faces(me, true);
 	}
@@ -812,7 +806,7 @@ static int bake(
 	ob_low_eval = DEG_get_evaluated_object(depsgraph, ob_low);
 
 	/* get the mesh as it arrives in the renderer */
-	me_low = bake_mesh_new_from_object(depsgraph, bmain, scene, ob_low);
+	me_low = bake_mesh_new_from_object(depsgraph, bmain, scene, ob_low_eval);
 
 	/* populate the pixel array with the face data */
 	if ((is_selected_to_active && (ob_cage == NULL) && is_cage) == false)
@@ -825,7 +819,7 @@ static int bake(
 
 		/* prepare cage mesh */
 		if (ob_cage) {
-			me_cage = bake_mesh_new_from_object(depsgraph, bmain, scene, ob_cage);
+			me_cage = bake_mesh_new_from_object(depsgraph, bmain, scene, ob_cage_eval);
 			if ((me_low->totpoly != me_cage->totpoly) || (me_low->totloop != me_cage->totloop)) {
 				BKE_report(reports, RPT_ERROR,
 				           "Invalid cage object, the cage mesh must have the same number "
@@ -834,6 +828,8 @@ static int bake(
 			}
 		}
 		else if (is_cage) {
+			BKE_object_eval_reset(ob_low_eval);
+
 			ModifierData *md = ob_low_eval->modifiers.first;
 			while (md) {
 				ModifierData *md_next = md->next;
@@ -851,7 +847,6 @@ static int bake(
 				md = md_next;
 			}
 
-			BKE_object_eval_reset(ob_low_eval);
 			me_cage = bake_mesh_new_from_object(depsgraph, bmain, scene, ob_low_eval);
 			RE_bake_pixels_populate(me_cage, pixel_array_low, num_pixels, &bake_images, uv_layer);
 		}
@@ -867,10 +862,10 @@ static int bake(
 
 			/* initialize highpoly_data */
 			highpoly[i].ob = ob_iter;
-			highpoly[i].me = bake_mesh_new_from_object(depsgraph, bmain, scene, highpoly[i].ob);
 			highpoly[i].ob_eval = DEG_get_evaluated_object(depsgraph, ob_iter);
 			highpoly[i].ob_eval->restrictflag &= ~OB_RESTRICT_RENDER;
 			highpoly[i].ob_eval->base_flag |= (BASE_VISIBLE | BASE_ENABLED_RENDER);
+			highpoly[i].me = bake_mesh_new_from_object(depsgraph, bmain, scene, highpoly[i].ob_eval);
 
 			/* lowpoly to highpoly transformation matrix */
 			copy_m4_m4(highpoly[i].obmat, highpoly[i].ob->obmat);
@@ -894,7 +889,7 @@ static int bake(
 		/* populate the pixel arrays with the corresponding face data for each high poly object */
 		if (!RE_bake_pixels_populate_from_objects(
 		            me_low, pixel_array_low, pixel_array_high, highpoly, tot_highpoly, num_pixels, ob_cage != NULL,
-		            cage_extrusion, ob_low->obmat, (ob_cage ? ob_cage->obmat : ob_low->obmat), me_cage))
+		            cage_extrusion, ob_low_eval->obmat, (ob_cage ? ob_cage->obmat : ob_low_eval->obmat), me_cage))
 		{
 			BKE_report(reports, RPT_ERROR, "Error handling selected objects");
 			goto cleanup;
@@ -912,10 +907,10 @@ static int bake(
 	}
 	else {
 		/* If low poly is not renderable it should have failed long ago. */
-		BLI_assert((ob_low->restrictflag & OB_RESTRICT_RENDER) == 0);
+		BLI_assert((ob_low_eval->restrictflag & OB_RESTRICT_RENDER) == 0);
 
 		if (RE_bake_has_engine(re)) {
-			ok = RE_bake_engine(re, depsgraph, ob_low, 0, pixel_array_low, num_pixels, depth, pass_type, pass_filter, result);
+			ok = RE_bake_engine(re, depsgraph, ob_low_eval, 0, pixel_array_low, num_pixels, depth, pass_type, pass_filter, result);
 		}
 		else {
 			BKE_report(reports, RPT_ERROR, "Current render engine does not support baking");
@@ -943,13 +938,13 @@ static int bake(
 			}
 			case R_BAKE_SPACE_OBJECT:
 			{
-				RE_bake_normal_world_to_object(pixel_array_low, num_pixels, depth, result, ob_low, normal_swizzle);
+				RE_bake_normal_world_to_object(pixel_array_low, num_pixels, depth, result, ob_low_eval, normal_swizzle);
 				break;
 			}
 			case R_BAKE_SPACE_TANGENT:
 			{
 				if (is_selected_to_active) {
-					RE_bake_normal_world_to_tangent(pixel_array_low, num_pixels, depth, result, me_low, normal_swizzle, ob_low->obmat);
+					RE_bake_normal_world_to_tangent(pixel_array_low, num_pixels, depth, result, me_low, normal_swizzle, ob_low_eval->obmat);
 				}
 				else {
 					/* from multiresolution */
@@ -957,17 +952,19 @@ static int bake(
 					ModifierData *md = NULL;
 					int mode;
 
-					md = modifiers_findByType(ob_low, eModifierType_Multires);
+					BKE_object_eval_reset(ob_low_eval);
+					md = modifiers_findByType(ob_low_eval, eModifierType_Multires);
 
 					if (md) {
 						mode = md->mode;
 						md->mode &= ~eModifierMode_Render;
 					}
 
-					me_nores = bake_mesh_new_from_object(depsgraph, bmain, scene, ob_low);
+					/* Evaluate modifiers again. */
+					me_nores = BKE_mesh_new_from_object(depsgraph, bmain, scene, ob_low_eval, true, false);
 					RE_bake_pixels_populate(me_nores, pixel_array_low, num_pixels, &bake_images, uv_layer);
 
-					RE_bake_normal_world_to_tangent(pixel_array_low, num_pixels, depth, result, me_nores, normal_swizzle, ob_low->obmat);
+					RE_bake_normal_world_to_tangent(pixel_array_low, num_pixels, depth, result, me_nores, normal_swizzle, ob_low_eval->obmat);
 					BKE_id_free(bmain, me_nores);
 
 					if (md)
@@ -1029,8 +1026,8 @@ static int bake(
 						BLI_path_suffix(name, FILE_MAX, bk_image->image->id.name + 2, "_");
 					}
 					else {
-						if (ob_low->mat[i]) {
-							BLI_path_suffix(name, FILE_MAX, ob_low->mat[i]->id.name + 2, "_");
+						if (ob_low_eval->mat[i]) {
+							BLI_path_suffix(name, FILE_MAX, ob_low_eval->mat[i]->id.name + 2, "_");
 						}
 						else if (me_low->mat[i]) {
 							BLI_path_suffix(name, FILE_MAX, me_low->mat[i]->id.name + 2, "_");