Fix deadlock with shrinkwrap and other modifiers

More code that needs task isolation. Encountered in sprite fright production
file.

Ref D11603

2 files changed, 26 insertions, 9 deletions

diff --git a/source/blender/blenkernel/intern/mesh_runtime.c b/source/blender/blenkernel/intern/mesh_runtime.c
index fae4c87626a..011dd7e25ee 100644
--- a/source/blender/blenkernel/intern/mesh_runtime.c
+++ b/source/blender/blenkernel/intern/mesh_runtime.c
@@ -30,6 +30,7 @@
 #include "DNA_object_types.h"
 
 #include "BLI_math_geom.h"
+#include "BLI_task.h"
 #include "BLI_threads.h"
 
 #include "BKE_bvhutils.h"
@@ -151,6 +152,12 @@ int BKE_mesh_runtime_looptri_len(const Mesh *mesh)
   return looptri_len;
 }
 
+static void mesh_runtime_looptri_recalc_isolated(void *userdata)
+{
+  Mesh *mesh = userdata;
+  BKE_mesh_runtime_looptri_recalc(mesh);
+}
+
 /* This is a ported copy of dm_getLoopTriArray(dm). */
 const MLoopTri *BKE_mesh_runtime_looptri_ensure(Mesh *mesh)
 {
@@ -163,7 +170,8 @@ const MLoopTri *BKE_mesh_runtime_looptri_ensure(Mesh *mesh)
     BLI_assert(BKE_mesh_runtime_looptri_len(mesh) == mesh->runtime.looptris.len);
   }
   else {
-    BKE_mesh_runtime_looptri_recalc(mesh);
+    /* Must isolate multithreaded tasks while holding a mutex lock. */
+    BLI_task_isolate(mesh_runtime_looptri_recalc_isolated, mesh);
     looptri = mesh->runtime.looptris.array;
   }
 
diff --git a/source/blender/blenkernel/intern/mesh_wrapper.c b/source/blender/blenkernel/intern/mesh_wrapper.c
index cf1dba3325c..fe6af432314 100644
--- a/source/blender/blenkernel/intern/mesh_wrapper.c
+++ b/source/blender/blenkernel/intern/mesh_wrapper.c
@@ -40,6 +40,7 @@
 
 #include "BLI_ghash.h"
 #include "BLI_math.h"
+#include "BLI_task.h"
 #include "BLI_threads.h"
 #include "BLI_utildefines.h"
 
@@ -95,15 +96,9 @@ Mesh *BKE_mesh_wrapper_from_editmesh(BMEditMesh *em,
   return BKE_mesh_wrapper_from_editmesh_with_coords(em, cd_mask_extra, NULL, me_settings);
 }
 
-void BKE_mesh_wrapper_ensure_mdata(Mesh *me)
+static void mesh_wrapper_ensure_mdata_isolated(void *userdata)
 {
-  ThreadMutex *mesh_eval_mutex = (ThreadMutex *)me->runtime.eval_mutex;
-  BLI_mutex_lock(mesh_eval_mutex);
-
-  if (me->runtime.wrapper_type == ME_WRAPPER_TYPE_MDATA) {
-    BLI_mutex_unlock(mesh_eval_mutex);
-    return;
-  }
+  Mesh *me = userdata;
 
   const eMeshWrapperType geom_type_orig = me->runtime.wrapper_type;
   me->runtime.wrapper_type = ME_WRAPPER_TYPE_MDATA;
@@ -136,6 +131,20 @@ void BKE_mesh_wrapper_ensure_mdata(Mesh *me)
   if (me->runtime.wrapper_type_finalize) {
     BKE_mesh_wrapper_deferred_finalize(me, &me->runtime.cd_mask_extra);
   }
+}
+
+void BKE_mesh_wrapper_ensure_mdata(Mesh *me)
+{
+  ThreadMutex *mesh_eval_mutex = (ThreadMutex *)me->runtime.eval_mutex;
+  BLI_mutex_lock(mesh_eval_mutex);
+
+  if (me->runtime.wrapper_type == ME_WRAPPER_TYPE_MDATA) {
+    BLI_mutex_unlock(mesh_eval_mutex);
+    return;
+  }
+
+  /* Must isolate multithreaded tasks while holding a mutex lock. */
+  BLI_task_isolate(mesh_wrapper_ensure_mdata_isolated, me);
 
   BLI_mutex_unlock(mesh_eval_mutex);
 }