Fix threading crash due to conflict in mesh wrapper type

A mesh wrapper might be being accessed for read-only from one thread
while another thread converts the wrapper type to something else.

The proposes solution is to defer assignment of the mesh wrapper
type until the wrapper is fully converted. The good thing about this
approach is that it does not introduce extra synchronization (and,
potentially, evaluation pipeline stalls). The downside is that it
might not work with all possible wrapper types in the future. If a
wrapper type which does not clear data separation is ever added in
the future we will re-consider the threading safety then.

Unfortunately, some changes outside of the mesh wrapper file are
to be made to allow "incremental" construction of the mesh prior
changing its wrapper type.

Unfortunately, there is no simplified file which demonstrates the
issue. It was investigated using Heist production file checked at
the revision 1228: `pro/lib/char/einar/einar.shading.blend`. The
repro case is simple: tab into edit mode, possibly few times.

The gist is that there several surface deform and shrinkwrap
modifiers which uses the same target. While one of them is building
BVH tree (which changes wrapper type) the other one accesses it for
read-only via `BKE_mesh_wrapper_vert_coords_copy_with_mat4()`.

Differential Revision: https://developer.blender.org/D15424

1 files changed, 18 insertions, 10 deletions

diff --git a/source/blender/blenkernel/intern/DerivedMesh.cc b/source/blender/blenkernel/intern/DerivedMesh.cc
index c2ea01bcadf..a29d8726f21 100644
--- a/source/blender/blenkernel/intern/DerivedMesh.cc
+++ b/source/blender/blenkernel/intern/DerivedMesh.cc
@@ -84,6 +84,8 @@ static ThreadRWMutex loops_cache_lock = PTHREAD_RWLOCK_INITIALIZER;
 static void mesh_init_origspace(Mesh *mesh);
 static void editbmesh_calc_modifier_final_normals(Mesh *mesh_final,
                                                   const CustomData_MeshMasks *final_datamask);
+static void editbmesh_calc_modifier_final_normals_or_defer(
+    Mesh *mesh_final, const CustomData_MeshMasks *final_datamask);
 
 /* -------------------------------------------------------------------- */
 
@@ -663,8 +665,8 @@ static void mesh_calc_finalize(const Mesh *mesh_input, Mesh *mesh_eval)
   mesh_eval->edit_mesh = mesh_input->edit_mesh;
 }
 
-void BKE_mesh_wrapper_deferred_finalize(Mesh *me_eval,
-                                        const CustomData_MeshMasks *cd_mask_finalize)
+void BKE_mesh_wrapper_deferred_finalize_mdata(Mesh *me_eval,
+                                              const CustomData_MeshMasks *cd_mask_finalize)
 {
   if (me_eval->runtime.wrapper_type_finalize & (1 << ME_WRAPPER_TYPE_BMESH)) {
     editbmesh_calc_modifier_final_normals(me_eval, cd_mask_finalize);
@@ -1286,12 +1288,6 @@ bool editbmesh_modifier_is_enabled(const Scene *scene,
 static void editbmesh_calc_modifier_final_normals(Mesh *mesh_final,
                                                   const CustomData_MeshMasks *final_datamask)
 {
-  if (mesh_final->runtime.wrapper_type != ME_WRAPPER_TYPE_MDATA) {
-    /* Generated at draw time. */
-    mesh_final->runtime.wrapper_type_finalize = (1 << mesh_final->runtime.wrapper_type);
-    return;
-  }
-
   const bool calc_loop_normals = ((mesh_final->flag & ME_AUTOSMOOTH) != 0 ||
                                   (final_datamask->lmask & CD_MASK_NORMAL) != 0);
 
@@ -1319,6 +1315,18 @@ static void editbmesh_calc_modifier_final_normals(Mesh *mesh_final,
   }
 }
 
+static void editbmesh_calc_modifier_final_normals_or_defer(
+    Mesh *mesh_final, const CustomData_MeshMasks *final_datamask)
+{
+  if (mesh_final->runtime.wrapper_type != ME_WRAPPER_TYPE_MDATA) {
+    /* Generated at draw time. */
+    mesh_final->runtime.wrapper_type_finalize = (1 << mesh_final->runtime.wrapper_type);
+    return;
+  }
+
+  editbmesh_calc_modifier_final_normals(mesh_final, final_datamask);
+}
+
 static void editbmesh_calc_modifiers(struct Depsgraph *depsgraph,
                                      const Scene *scene,
                                      Object *ob,
@@ -1598,9 +1606,9 @@ static void editbmesh_calc_modifiers(struct Depsgraph *depsgraph,
   }
 
   /* Compute normals. */
-  editbmesh_calc_modifier_final_normals(mesh_final, &final_datamask);
+  editbmesh_calc_modifier_final_normals_or_defer(mesh_final, &final_datamask);
   if (mesh_cage && (mesh_cage != mesh_final)) {
-    editbmesh_calc_modifier_final_normals(mesh_cage, &final_datamask);
+    editbmesh_calc_modifier_final_normals_or_defer(mesh_cage, &final_datamask);
   }
 
   /* Return final mesh. */