Fix T39997: Multiple boolean modifiers sharing the same right operand crashes

The issue was caused by the temporary CD layers being allocated for subsurf
meshes, same as we've got back in 881fb43.

In the long run this temporary storage is to be re-considered, but it'll also
imply re-considering of the Derivedmesh interaction as well. For now let's
use a simpler solution which is forbidding modifiers to call getArray for other
objects' derivedMeshes but use an API calls which would allocate local copy of
the data preventing race condition of shared data in DM.

5 files changed, 181 insertions, 79 deletions

diff --git a/source/blender/blenkernel/BKE_DerivedMesh.h b/source/blender/blenkernel/BKE_DerivedMesh.h
index 60920aa1bb5..1ab5ec51de8 100644
--- a/source/blender/blenkernel/BKE_DerivedMesh.h
+++ b/source/blender/blenkernel/BKE_DerivedMesh.h
@@ -762,4 +762,10 @@ BLI_INLINE int DM_origindex_mface_mpoly(const int *index_mf_to_mpoly, const int
 	return (j != ORIGINDEX_NONE) ? (index_mp_to_orig ? index_mp_to_orig[j] : j) : ORIGINDEX_NONE;
 }
 
+struct MVert *DM_get_vert_array(struct DerivedMesh *dm, bool *allocated);
+struct MEdge *DM_get_edge_array(struct DerivedMesh *dm, bool *allocated);
+struct MLoop *DM_get_loop_array(struct DerivedMesh *dm, bool *allocated);
+struct MPoly *DM_get_poly_array(struct DerivedMesh *dm, bool *allocated);
+struct MFace *DM_get_tessface_array(struct DerivedMesh *dm, bool *allocated);
+
 #endif  /* __BKE_DERIVEDMESH_H__ */
diff --git a/source/blender/blenkernel/intern/DerivedMesh.c b/source/blender/blenkernel/intern/DerivedMesh.c
index c42289f2004..26cc70e1089 100644
--- a/source/blender/blenkernel/intern/DerivedMesh.c
+++ b/source/blender/blenkernel/intern/DerivedMesh.c
@@ -3342,3 +3342,84 @@ bool DM_is_valid(DerivedMesh *dm)
 }
 
 #endif /* NDEBUG */
+
+/* -------------------------------------------------------------------- */
+
+MVert *DM_get_vert_array(DerivedMesh *dm, bool *allocated)
+{
+	CustomData *vert_data = dm->getVertDataLayout(dm);
+	MVert *mvert = CustomData_get_layer(vert_data, CD_MVERT);
+	*allocated = false;
+
+	if (mvert == NULL) {
+		mvert = MEM_mallocN(sizeof(MVert) * dm->getNumVerts(dm), "dmvh vert data array");
+		dm->copyVertArray(dm, mvert);
+		*allocated = true;
+	}
+
+	return mvert;
+}
+
+MEdge *DM_get_edge_array(DerivedMesh *dm, bool *allocated)
+{
+	CustomData *edge_data = dm->getEdgeDataLayout(dm);
+	MEdge *medge = CustomData_get_layer(edge_data, CD_MEDGE);
+	*allocated = false;
+
+	if (medge == NULL) {
+		medge = MEM_mallocN(sizeof(MEdge) * dm->getNumEdges(dm), "dm medge data array");
+		dm->copyEdgeArray(dm, medge);
+		*allocated = true;
+	}
+
+	return medge;
+}
+
+MLoop *DM_get_loop_array(DerivedMesh *dm, bool *allocated)
+{
+	CustomData *loop_data = dm->getEdgeDataLayout(dm);
+	MLoop *mloop = CustomData_get_layer(loop_data, CD_MLOOP);
+	*allocated = false;
+
+	if (mloop == NULL) {
+		mloop = MEM_mallocN(sizeof(MLoop) * dm->getNumLoops(dm), "dm loop data array");
+		dm->copyLoopArray(dm, mloop);
+		*allocated = true;
+	}
+
+	return mloop;
+}
+
+MPoly *DM_get_poly_array(DerivedMesh *dm, bool *allocated)
+{
+	CustomData *poly_data = dm->getPolyDataLayout(dm);
+	MPoly *mpoly = CustomData_get_layer(poly_data, CD_MPOLY);
+	*allocated = false;
+
+	if (mpoly == NULL) {
+		mpoly = MEM_mallocN(sizeof(MPoly) * dm->getNumPolys(dm), "dm poly data array");
+		dm->copyPolyArray(dm, mpoly);
+		*allocated = true;
+	}
+
+	return mpoly;
+}
+
+MFace *DM_get_tessface_array(DerivedMesh *dm, bool *allocated)
+{
+	CustomData *tessface_data = dm->getTessFaceDataLayout(dm);
+	MFace *mface = CustomData_get_layer(tessface_data, CD_MFACE);
+	*allocated = false;
+
+	if (mface == NULL) {
+		int numTessFaces = dm->getNumTessFaces(dm);
+
+		if (numTessFaces > 0) {
+			mface = MEM_mallocN(sizeof(MFace) * numTessFaces, "bvh mface data array");
+			dm->copyTessFaceArray(dm, mface);
+			*allocated = true;
+		}
+	}
+
+	return mface;
+}
diff --git a/source/blender/blenkernel/intern/bvhutils.c b/source/blender/blenkernel/intern/bvhutils.c
index 0ceee78874f..f85a91b66cb 100644
--- a/source/blender/blenkernel/intern/bvhutils.c
+++ b/source/blender/blenkernel/intern/bvhutils.c
@@ -519,55 +519,6 @@ static void mesh_edges_nearest_point(void *userdata, int index, const float co[3
 	}
 }
 
-static MVert *get_dm_vert_data_array(DerivedMesh *dm, bool *allocated)
-{
-	CustomData *vert_data = dm->getVertDataLayout(dm);
-	MVert *mvert = CustomData_get_layer(vert_data, CD_MVERT);
-	*allocated = false;
-
-	if (mvert == NULL) {
-		mvert = MEM_mallocN(sizeof(MVert) * dm->getNumVerts(dm), "bvh vert data array");
-		dm->copyVertArray(dm, mvert);
-		*allocated = true;
-	}
-
-	return mvert;
-}
-
-static MEdge *get_dm_edge_data_array(DerivedMesh *dm, bool *allocated)
-{
-	CustomData *edge_data = dm->getEdgeDataLayout(dm);
-	MEdge *medge = CustomData_get_layer(edge_data, CD_MEDGE);
-	*allocated = false;
-
-	if (medge == NULL) {
-		medge = MEM_mallocN(sizeof(MEdge) * dm->getNumEdges(dm), "bvh medge data array");
-		dm->copyEdgeArray(dm, medge);
-		*allocated = true;
-	}
-
-	return medge;
-}
-
-static MFace *get_dm_tessface_data_array(DerivedMesh *dm, bool *allocated)
-{
-	CustomData *tessface_data = dm->getTessFaceDataLayout(dm);
-	MFace *mface = CustomData_get_layer(tessface_data, CD_MFACE);
-	*allocated = false;
-
-	if (mface == NULL) {
-		int numTessFaces = dm->getNumTessFaces(dm);
-
-		if (numTessFaces > 0) {
-			mface = MEM_mallocN(sizeof(MFace) * numTessFaces, "bvh mface data array");
-			dm->copyTessFaceArray(dm, mface);
-			*allocated = true;
-		}
-	}
-
-	return mface;
-}
-
 /*
  * BVH builders
  */
@@ -582,7 +533,7 @@ BVHTree *bvhtree_from_mesh_verts(BVHTreeFromMesh *data, DerivedMesh *dm, float e
 	tree = bvhcache_find(&dm->bvhCache, BVHTREE_FROM_VERTICES);
 	BLI_rw_mutex_unlock(&cache_rwlock);
 
-	vert = get_dm_vert_data_array(dm, &vert_allocated);
+	vert = DM_get_vert_array(dm, &vert_allocated);
 
 	/* Not in cache */
 	if (tree == NULL) {
@@ -629,7 +580,7 @@ BVHTree *bvhtree_from_mesh_verts(BVHTreeFromMesh *data, DerivedMesh *dm, float e
 
 		data->vert = vert;
 		data->vert_allocated = vert_allocated;
-		data->face = get_dm_tessface_data_array(dm, &data->face_allocated);
+		data->face = DM_get_tessface_array(dm, &data->face_allocated);
 
 		data->sphere_radius = epsilon;
 	}
@@ -657,8 +608,8 @@ BVHTree *bvhtree_from_mesh_faces(BVHTreeFromMesh *data, DerivedMesh *dm, float e
 	BLI_rw_mutex_unlock(&cache_rwlock);
 
 	if (em == NULL) {
-		vert = get_dm_vert_data_array(dm, &vert_allocated);
-		face = get_dm_tessface_data_array(dm, &face_allocated);
+		vert = DM_get_vert_array(dm, &vert_allocated);
+		face = DM_get_tessface_array(dm, &face_allocated);
 	}
 
 	/* Not in cache */
@@ -827,8 +778,8 @@ BVHTree *bvhtree_from_mesh_edges(BVHTreeFromMesh *data, DerivedMesh *dm, float e
 	tree = bvhcache_find(&dm->bvhCache, BVHTREE_FROM_EDGES);
 	BLI_rw_mutex_unlock(&cache_rwlock);
 
-	vert = get_dm_vert_data_array(dm, &vert_allocated);
-	edge = get_dm_edge_data_array(dm, &edge_allocated);
+	vert = DM_get_vert_array(dm, &vert_allocated);
+	edge = DM_get_edge_array(dm, &edge_allocated);
 
 	/* Not in cache */
 	if (tree == NULL) {
diff --git a/source/blender/blenkernel/intern/subsurf_ccg.c b/source/blender/blenkernel/intern/subsurf_ccg.c
index 16d21827f73..a9eba54128a 100644
--- a/source/blender/blenkernel/intern/subsurf_ccg.c
+++ b/source/blender/blenkernel/intern/subsurf_ccg.c
@@ -53,6 +53,7 @@
 #include "BLI_edgehash.h"
 #include "BLI_math.h"
 #include "BLI_memarena.h"
+#include "BLI_threads.h"
 
 #include "BKE_pbvh.h"
 #include "BKE_ccg.h"
@@ -80,6 +81,9 @@
 
 extern GLubyte stipple_quarttone[128]; /* glutil.c, bad level data */
 
+static ThreadRWMutex loops_cache_rwlock = BLI_RWLOCK_INITIALIZER;
+static ThreadRWMutex origindex_cache_rwlock = BLI_RWLOCK_INITIALIZER;
+
 static CCGDerivedMesh *getCCGDerivedMesh(CCGSubSurf *ss,
                                          int drawInteriorEdges,
                                          int useSubsurfUv,
@@ -1326,16 +1330,21 @@ static void ccgDM_copyFinalLoopArray(DerivedMesh *dm, MLoop *mloop)
 	/* DMFlagMat *faceFlags = ccgdm->faceFlags; */ /* UNUSED */
 
 	if (!ccgdm->ehash) {
-		MEdge *medge;
+		BLI_rw_mutex_lock(&loops_cache_rwlock, THREAD_LOCK_WRITE);
+		if (!ccgdm->ehash) {
+			MEdge *medge;
 
-		ccgdm->ehash = BLI_edgehash_new_ex(__func__, ccgdm->dm.numEdgeData);
-		medge = ccgdm->dm.getEdgeArray((DerivedMesh *)ccgdm);
+			ccgdm->ehash = BLI_edgehash_new_ex(__func__, ccgdm->dm.numEdgeData);
+			medge = ccgdm->dm.getEdgeArray((DerivedMesh *)ccgdm);
 
-		for (i = 0; i < ccgdm->dm.numEdgeData; i++) {
-			BLI_edgehash_insert(ccgdm->ehash, medge[i].v1, medge[i].v2, SET_INT_IN_POINTER(i));
+			for (i = 0; i < ccgdm->dm.numEdgeData; i++) {
+				BLI_edgehash_insert(ccgdm->ehash, medge[i].v1, medge[i].v2, SET_INT_IN_POINTER(i));
+			}
 		}
+		BLI_rw_mutex_unlock(&loops_cache_rwlock);
 	}
 
+	BLI_rw_mutex_lock(&loops_cache_rwlock, THREAD_LOCK_READ);
 	totface = ccgSubSurf_getNumFaces(ss);
 	mv = mloop;
 	for (index = 0; index < totface; index++) {
@@ -1378,6 +1387,7 @@ static void ccgDM_copyFinalLoopArray(DerivedMesh *dm, MLoop *mloop)
 			}
 		}
 	}
+	BLI_rw_mutex_unlock(&loops_cache_rwlock);
 }
 
 static void ccgDM_copyFinalPolyArray(DerivedMesh *dm, MPoly *mpoly)
@@ -2903,11 +2913,14 @@ static void *ccgDM_get_vert_data_layer(DerivedMesh *dm, int type)
 		int a, index, totnone, totorig;
 
 		/* Avoid re-creation if the layer exists already */
+		BLI_rw_mutex_lock(&origindex_cache_rwlock, THREAD_LOCK_READ);
 		origindex = DM_get_vert_data_layer(dm, CD_ORIGINDEX);
+		BLI_rw_mutex_unlock(&origindex_cache_rwlock);
 		if (origindex) {
 			return origindex;
 		}
 
+		BLI_rw_mutex_lock(&origindex_cache_rwlock, THREAD_LOCK_WRITE);
 		DM_add_vert_layer(dm, CD_ORIGINDEX, CD_CALLOC, NULL);
 		origindex = DM_get_vert_data_layer(dm, CD_ORIGINDEX);
 
@@ -2922,6 +2935,7 @@ static void *ccgDM_get_vert_data_layer(DerivedMesh *dm, int type)
 			CCGVert *v = ccgdm->vertMap[index].vert;
 			origindex[a] = ccgDM_getVertMapIndex(ccgdm->ss, v);
 		}
+		BLI_rw_mutex_unlock(&origindex_cache_rwlock);
 
 		return origindex;
 	}
diff --git a/source/blender/modifiers/intern/MOD_boolean_util.c b/source/blender/modifiers/intern/MOD_boolean_util.c
index 251c5a8e311..99016a0a41b 100644
--- a/source/blender/modifiers/intern/MOD_boolean_util.c
+++ b/source/blender/modifiers/intern/MOD_boolean_util.c
@@ -27,11 +27,12 @@
  *  \ingroup modifiers
  */
 
+#include "MEM_guardedalloc.h"
+
 #include "DNA_material_types.h"
 #include "DNA_meshdata_types.h"
 #include "DNA_object_types.h"
 
-
 #include "BLI_utildefines.h"
 #include "BLI_alloca.h"
 #include "BLI_ghash.h"
@@ -88,6 +89,41 @@ static void DM_loop_interp_from_poly(DerivedMesh *source_dm,
 	                    source_poly->totloop, target_loop_index);
 }
 
+typedef struct DMArrays {
+	MVert *mvert;
+	MEdge *medge;
+	MLoop *mloop;
+	MPoly *mpoly;
+	bool mvert_allocated;
+	bool medge_allocated;
+	bool mloop_allocated;
+	bool mpoly_allocated;
+} DMArrays;
+
+static void dm_arrays_get(DerivedMesh *dm, DMArrays *arrays)
+{
+	arrays->mvert = DM_get_vert_array(dm, &arrays->mvert_allocated);
+	arrays->medge = DM_get_edge_array(dm, &arrays->medge_allocated);
+	arrays->mloop = DM_get_loop_array(dm, &arrays->mloop_allocated);
+	arrays->mpoly = DM_get_poly_array(dm, &arrays->mpoly_allocated);
+}
+
+static void dm_arrays_free(DMArrays *arrays)
+{
+	if (arrays->mvert_allocated) {
+		MEM_freeN(arrays->mvert);
+	}
+	if (arrays->medge_allocated) {
+		MEM_freeN(arrays->medge);
+	}
+	if (arrays->mloop_allocated) {
+		MEM_freeN(arrays->mloop);
+	}
+	if (arrays->mpoly_allocated) {
+		MEM_freeN(arrays->mpoly);
+	}
+}
+
 /* **** Importer from derived mesh to Carve ****  */
 
 typedef struct ImportMeshData {
@@ -633,25 +669,30 @@ static int operation_from_optype(int int_op_type)
 	return operation;
 }
 
-static void prepare_import_data(Object *object, DerivedMesh *dm, ImportMeshData *import_data)
+static void prepare_import_data(Object *object,
+                                DerivedMesh *dm,
+                                const DMArrays *dm_arrays,
+                                ImportMeshData *import_data)
 {
 	import_data->dm = dm;
 	copy_m4_m4(import_data->obmat, object->obmat);
-	import_data->mvert = dm->getVertArray(dm);
-	import_data->medge = dm->getEdgeArray(dm);
-	import_data->mloop = dm->getLoopArray(dm);
-	import_data->mpoly = dm->getPolyArray(dm);
+	import_data->mvert = dm_arrays->mvert;
+	import_data->medge = dm_arrays->medge;
+	import_data->mloop = dm_arrays->mloop;
+	import_data->mpoly = dm_arrays->mpoly;
 }
 
-static struct CarveMeshDescr *carve_mesh_from_dm(Object *object, DerivedMesh *dm)
+static struct CarveMeshDescr *carve_mesh_from_dm(Object *object,
+                                                 DerivedMesh *dm,
+                                                 const DMArrays *dm_arrays)
 {
 	ImportMeshData import_data;
-	prepare_import_data(object, dm, &import_data);
+	prepare_import_data(object, dm, dm_arrays, &import_data);
 	return carve_addMesh(&import_data, &MeshImporter);
 }
 
-static void prepare_export_data(Object *object_left, DerivedMesh *dm_left,
-                                Object *object_right, DerivedMesh *dm_right,
+static void prepare_export_data(Object *object_left, DerivedMesh *dm_left, const DMArrays *dm_left_arrays,
+                                Object *object_right, DerivedMesh *dm_right, const DMArrays *dm_right_arrays,
                                 ExportMeshData *export_data)
 {
 	float object_right_imat[4][4];
@@ -664,12 +705,12 @@ static void prepare_export_data(Object *object_left, DerivedMesh *dm_left,
 	export_data->dm_left = dm_left;
 	export_data->dm_right = dm_right;
 
-	export_data->mvert_left = dm_left->getVertArray(dm_left);
-	export_data->mloop_left = dm_left->getLoopArray(dm_left);
-	export_data->mpoly_left = dm_left->getPolyArray(dm_left);
-	export_data->mvert_right = dm_right->getVertArray(dm_right);
-	export_data->mloop_right = dm_right->getLoopArray(dm_right);
-	export_data->mpoly_right = dm_right->getPolyArray(dm_right);
+	export_data->mvert_left = dm_left_arrays->mvert;
+	export_data->mloop_left = dm_left_arrays->mloop;
+	export_data->mpoly_left = dm_left_arrays->mpoly;
+	export_data->mvert_right = dm_right_arrays->mvert;
+	export_data->mloop_right = dm_right_arrays->mloop;
+	export_data->mpoly_right = dm_right_arrays->mpoly;
 
 	export_data->material_hash = BLI_ghash_ptr_new("CSG_mat gh");
 
@@ -690,6 +731,7 @@ DerivedMesh *NewBooleanDerivedMesh(DerivedMesh *dm, struct Object *ob,
 	DerivedMesh *output_dm = NULL;
 	int operation;
 	bool result;
+	DMArrays dm_left_arrays, dm_right_arrays;
 
 	if (dm == NULL || dm_select == NULL) {
 		return NULL;
@@ -700,8 +742,11 @@ DerivedMesh *NewBooleanDerivedMesh(DerivedMesh *dm, struct Object *ob,
 		return NULL;
 	}
 
-	left = carve_mesh_from_dm(ob_select, dm_select);
-	right = carve_mesh_from_dm(ob, dm);
+	dm_arrays_get(dm_select, &dm_left_arrays);
+	dm_arrays_get(dm, &dm_right_arrays);
+
+	left = carve_mesh_from_dm(ob_select, dm_select, &dm_left_arrays);
+	right = carve_mesh_from_dm(ob, dm, &dm_right_arrays);
 
 	result = carve_performBooleanOperation(left, right, operation, &output);
 
@@ -711,7 +756,9 @@ DerivedMesh *NewBooleanDerivedMesh(DerivedMesh *dm, struct Object *ob,
 	if (result) {
 		ExportMeshData export_data;
 
-		prepare_export_data(ob_select, dm_select, ob, dm, &export_data);
+		prepare_export_data(ob_select, dm_select, &dm_left_arrays,
+		                    ob, dm, &dm_right_arrays,
+		                    &export_data);
 
 		carve_exportMesh(output, &MeshExporter, &export_data);
 		output_dm = export_data.dm;
@@ -723,5 +770,8 @@ DerivedMesh *NewBooleanDerivedMesh(DerivedMesh *dm, struct Object *ob,
 		carve_deleteMesh(output);
 	}
 
+	dm_arrays_free(&dm_left_arrays);
+	dm_arrays_free(&dm_right_arrays);
+
 	return output_dm;
 }