Fix T39608: Blender 2.70 crashes when performing union

This was a nasty bug which was caused by specific of how face-edge
attributes are stored in Carve.

Face pointer is used in the map key which works just fine in all
cases except for the cases when some face is getting freed after
it was stored in the map.

This might give real issues when new face is allocating because
it's possible new face would have the same address as the freed
one.

Such cases used to happen when union of separate manifolds is
needed for the operands AND jemalloc is enabled.

Solved by dropping attributes for the freed faces from the map.
Maybe not the fastest ever approach, but not sure how to make
it faster actually. Should work just fine. It only happens for
complex setups with intersecting manifolds in the operands.

3 files changed, 88 insertions, 8 deletions

diff --git a/extern/carve/carve-capi.cc b/extern/carve/carve-capi.cc
index a65d1a8d766..af9ecad685d 100644
--- a/extern/carve/carve-capi.cc
+++ b/extern/carve/carve-capi.cc
@@ -38,7 +38,7 @@ typedef std::pair<int, int> OrigIndex;
 typedef std::pair<MeshSet<3>::vertex_t *, MeshSet<3>::vertex_t *> VertexPair;
 typedef carve::interpolate::VertexAttr<OrigIndex> OrigVertMapping;
 typedef carve::interpolate::FaceAttr<OrigIndex> OrigFaceMapping;
-typedef carve::interpolate::FaceEdgeAttr<OrigIndex> OrigFaceEdgeMapping;
+typedef carve::interpolate::SwapableFaceEdgeAttr<OrigIndex> OrigFaceEdgeMapping;
 typedef carve::interpolate::SimpleFaceEdgeAttr<bool> FaceEdgeTriangulatedFlag;
 
 typedef struct CarveMeshDescr {
@@ -522,6 +522,39 @@ public:
 	}
 };
 
+template <typename Interpolator>
+void copyFaceEdgeAttrs(const MeshSet<3> *poly,
+                       Interpolator *old_interpolator,
+                       Interpolator *new_interpolator)
+{
+	for (MeshSet<3>::const_face_iter face_iter = poly->faceBegin();
+	     face_iter != poly->faceEnd();
+	     ++face_iter)
+	{
+		const MeshSet<3>::face_t *face = *face_iter;
+
+		for (int edge_index = 0;
+		     edge_index < face->nEdges();
+		     ++edge_index)
+		{
+			new_interpolator->copyAttribute(face,
+			                                edge_index,
+			                                old_interpolator);
+		}
+	}
+}
+
+template <typename Interpolator>
+void cleanupFaceEdgeAttrs(const MeshSet<3> *left,
+                          const MeshSet<3> *right,
+                          Interpolator *interpolator)
+{
+	Interpolator new_interpolator;
+	copyFaceEdgeAttrs(left, interpolator, &new_interpolator);
+	copyFaceEdgeAttrs(right, interpolator, &new_interpolator);
+	interpolator->swapAttributes(&new_interpolator);
+}
+
 }  // namespace
 
 CarveMeshDescr *carve_addMesh(struct ImportMeshData *import_data,
@@ -698,7 +731,15 @@ bool carve_performBooleanOperation(CarveMeshDescr *left_mesh,
 		// intersecting that meshes tessellation of operation result can't be
 		// done properly. The only way to make such situations working is to
 		// union intersecting meshes of the same operand.
-		carve_unionIntersections(&csg, &left, &right);
+		if (carve_unionIntersections(&csg, &left, &right)) {
+			cleanupFaceEdgeAttrs(left,
+			                     right,
+			                     &output_descr->face_edge_triangulated_flag);
+			cleanupFaceEdgeAttrs(left,
+			                     right,
+			                     &output_descr->orig_face_edge_mapping);
+		}
+
 		left_mesh->poly = left;
 		right_mesh->poly = right;
 
diff --git a/extern/carve/carve-util.cc b/extern/carve/carve-util.cc
index ac6dcbc1a94..b268dae9dd6 100644
--- a/extern/carve/carve-util.cc
+++ b/extern/carve/carve-util.cc
@@ -486,14 +486,15 @@ MeshSet<3> *unionIntersectingMeshes(carve::csg::CSG *csg,
 
 // TODO(sergey): This function is to be totally re-implemented to make it
 // more clear what's going on and hopefully optimize it as well.
-void carve_unionIntersections(carve::csg::CSG *csg,
+bool carve_unionIntersections(carve::csg::CSG *csg,
                               MeshSet<3> **left_r,
                               MeshSet<3> **right_r)
 {
 	MeshSet<3> *left = *left_r, *right = *right_r;
+	bool changed = false;
 
 	if (left->meshes.size() == 1 && right->meshes.size() == 0) {
-		return;
+		return false;
 	}
 
 	MeshSet<3>::aabb_t leftAABB = left->getAABB();
@@ -503,14 +504,19 @@ void carve_unionIntersections(carve::csg::CSG *csg,
 	right = unionIntersectingMeshes(csg, right, leftAABB);
 
 	if (left != *left_r) {
+		changed = true;
 		delete *left_r;
 	}
 
-	if (right != *right_r)
+	if (right != *right_r) {
+		changed = true;
 		delete *right_r;
+	}
 
 	*left_r = left;
 	*right_r = right;
+
+	return changed;
 }
 
 static inline void add_newell_cross_v3_v3v3(const Vector &v_prev,
diff --git a/extern/carve/carve-util.h b/extern/carve/carve-util.h
index a658b2fea96..f650810e9e3 100644
--- a/extern/carve/carve-util.h
+++ b/extern/carve/carve-util.h
@@ -70,7 +70,7 @@ void carve_getRescaleMinMax(const carve::mesh::MeshSet<3> *left,
                             carve::geom3d::Vector *min,
                             carve::geom3d::Vector *max);
 
-void carve_unionIntersections(carve::csg::CSG *csg,
+bool carve_unionIntersections(carve::csg::CSG *csg,
                               carve::mesh::MeshSet<3> **left_r,
                               carve::mesh::MeshSet<3> **right_r);
 
@@ -115,8 +115,8 @@ namespace carve {
 						attrs.find(new_edge_iter->vert);
 					if (found == attrs.end()) {
 						for (const_edge_iter_t orig_edge_iter = orig_face->begin();
-							 orig_edge_iter != orig_face->end();
-							 ++orig_edge_iter)
+						     orig_edge_iter != orig_face->end();
+						     ++orig_edge_iter)
 						{
 							if ((orig_edge_iter->vert->v - new_edge_iter->vert->v).length2() < 1e-5) {
 								attrs[new_edge_iter->vert] = attrs[orig_edge_iter->vert];
@@ -236,6 +236,20 @@ namespace carve {
 				attrs[std::make_pair(f, e)] = attr;
 			}
 
+			void copyAttribute(const meshset_t::face_t *face,
+			                   unsigned edge,
+			                   SimpleFaceEdgeAttr<attr_t> *interpolator) {
+				key_t key(face, edge);
+				typename attrmap_t::const_iterator fv = interpolator->attrs.find(key);
+				if (fv != interpolator->attrs.end()) {
+					attrs[key] = (*fv).second;
+				}
+			}
+
+			void swapAttributes(SimpleFaceEdgeAttr<attr_t> *interpolator) {
+				attrs.swap(interpolator->attrs);
+			}
+
 			SimpleFaceEdgeAttr() : Interpolator() {
 			}
 
@@ -243,6 +257,25 @@ namespace carve {
 			}
 		};
 
+		template<typename attr_t>
+		class SwapableFaceEdgeAttr : public FaceEdgeAttr<attr_t> {
+		public:
+			typedef carve::mesh::MeshSet<3> meshset_t;
+
+			void copyAttribute(const meshset_t::face_t *face,
+			                   unsigned edge,
+			                   SwapableFaceEdgeAttr<attr_t> *interpolator) {
+				typename FaceEdgeAttr<attr_t>::key_t key(face, edge);
+				typename FaceEdgeAttr<attr_t>::attrmap_t::const_iterator fv = interpolator->attrs.find(key);
+				if (fv != interpolator->attrs.end()) {
+					this->attrs[key] = (*fv).second;
+				}
+			}
+
+			void swapAttributes(SwapableFaceEdgeAttr<attr_t> *interpolator) {
+				this->attrs.swap(interpolator->attrs);
+			}
+		};
 	}  // namespace interpolate
 }  // namespace carve