use modified vertex coords for calculating display thickness and intersections.

internal improvement to editmesh_bvh.c

- optionally pass cage-coords as an arg, rather then calculating the coords in BKE_bmbvh_new(),
  since all callers already have coords calculated.
- de-duplicate coords creation function from knife and bmbvhm, move into own generic function: BKE_editmesh_vertexCos_get()

7 files changed, 82 insertions, 90 deletions

diff --git a/source/blender/blenkernel/BKE_editmesh.h b/source/blender/blenkernel/BKE_editmesh.h
index 7052cb6012c..b18354627ab 100644
--- a/source/blender/blenkernel/BKE_editmesh.h
+++ b/source/blender/blenkernel/BKE_editmesh.h
@@ -30,6 +30,7 @@ struct BMesh;
 struct BMLoop;
 struct BMFace;
 struct Mesh;
+struct Scene;
 struct DerivedMesh;
 struct MeshStatVis;
 
@@ -87,14 +88,20 @@ typedef struct BMEditMesh {
 	int mirror_cdlayer; /* -1 is invalid */
 } BMEditMesh;
 
+/* editmesh.c */
 void        BKE_editmesh_tessface_calc(BMEditMesh *em);
 BMEditMesh *BKE_editmesh_create(BMesh *bm, const bool do_tessellate);
 BMEditMesh *BKE_editmesh_copy(BMEditMesh *em);
 BMEditMesh *BKE_editmesh_from_object(struct Object *ob);
 void        BKE_editmesh_free(BMEditMesh *em);
 void        BKE_editmesh_update_linked_customdata(BMEditMesh *em);
+
+/* editderivedmesh.c */
+/* should really be defined in editmesh.c, but they use 'EditDerivedBMesh' */
 void        BKE_editmesh_statvis_calc(BMEditMesh *em, struct DerivedMesh *dm,
                                       struct MeshStatVis *statvis,
                                       unsigned char (*r_face_colors)[4]);
 
+float (*BKE_editmesh_vertexCos_get(struct BMEditMesh *em, struct Scene *scene, int *r_numVerts))[3];
+
 #endif /* __BKE_EDITMESH_H__ */
diff --git a/source/blender/blenkernel/BKE_editmesh_bvh.h b/source/blender/blenkernel/BKE_editmesh_bvh.h
index 3ae6c906a87..90520c49983 100644
--- a/source/blender/blenkernel/BKE_editmesh_bvh.h
+++ b/source/blender/blenkernel/BKE_editmesh_bvh.h
@@ -41,7 +41,7 @@ struct Scene;
 
 typedef struct BMBVHTree BMBVHTree;
 
-BMBVHTree      *BKE_bmbvh_new(struct BMEditMesh *em, int flag, struct Scene *scene);
+BMBVHTree      *BKE_bmbvh_new(struct BMEditMesh *em, int flag, float (*cos_cage)[3], const bool cos_cage_free);
 void            BKE_bmbvh_free(BMBVHTree *tree);
 struct BVHTree *BKE_bmbvh_tree_get(BMBVHTree *tree);
 struct BMFace  *BKE_bmbvh_ray_cast(BMBVHTree *tree, const float co[3], const float dir[3],
@@ -54,10 +54,9 @@ struct BMVert  *BKE_bmbvh_find_vert_closest(BMBVHTree *tree, const float co[3],
 
 /* BKE_bmbvh_new flag parameter */
 enum {
-	BMBVH_USE_CAGE        = 1, /* project geometry onto modifier cage */
-	BMBVH_RETURN_ORIG     = 2, /* use with BMBVH_USE_CAGE, returns hits in relation to original geometry */
-	BMBVH_RESPECT_SELECT  = 4, /* restrict to hidden geometry (overrides BMBVH_RESPECT_HIDDEN) */
-	BMBVH_RESPECT_HIDDEN  = 8  /* omit hidden geometry */
+	BMBVH_RETURN_ORIG     = (1 << 0), /* use with 'cos_cage', returns hits in relation to original geometry */
+	BMBVH_RESPECT_SELECT  = (1 << 1), /* restrict to hidden geometry (overrides BMBVH_RESPECT_HIDDEN) */
+	BMBVH_RESPECT_HIDDEN  = (1 << 2)  /* omit hidden geometry */
 };
 
 #endif  /* __BKE_EDITMESH_BVH_H__ */
diff --git a/source/blender/blenkernel/intern/editderivedmesh.c b/source/blender/blenkernel/intern/editderivedmesh.c
index 1f6c99bfaaa..eda8957bb1b 100644
--- a/source/blender/blenkernel/intern/editderivedmesh.c
+++ b/source/blender/blenkernel/intern/editderivedmesh.c
@@ -1652,14 +1652,12 @@ static void statvis_calc_thickness(
 		}
 	}
 
-	(void)vertexCos;
-
 	BM_mesh_elem_index_ensure(bm, BM_FACE);
 	if (vertexCos) {
 		BM_mesh_elem_index_ensure(bm, BM_VERT);
 	}
 
-	bmtree = BKE_bmbvh_new(em, 0, NULL);
+	bmtree = BKE_bmbvh_new(em, 0, vertexCos, false);
 
 	for (i = 0; i < tottri; i++) {
 		BMFace *f_hit;
@@ -1750,14 +1748,12 @@ static void statvis_calc_intersect(
 
 	memset(r_face_colors, 64, sizeof(int) * em->bm->totface);
 
-	(void)vertexCos;
-
 	BM_mesh_elem_index_ensure(bm, BM_FACE);
 	if (vertexCos) {
 		BM_mesh_elem_index_ensure(bm, BM_VERT);
 	}
 
-	bmtree = BKE_bmbvh_new(em, 0, NULL);
+	bmtree = BKE_bmbvh_new(em, 0, vertexCos, false);
 
 	BM_ITER_MESH (e, &iter, bm, BM_EDGES_OF_MESH) {
 		BMFace *f_hit;
@@ -1839,3 +1835,55 @@ void BKE_editmesh_statvis_calc(BMEditMesh *em, DerivedMesh *dm,
 		}
 	}
 }
+
+
+
+/* -------------------------------------------------------------------- */
+/* Editmesh Vert Coords */
+
+#include "BLI_bitmap.h"
+struct CageUserData {
+	int totvert;
+	float (*cos_cage)[3];
+	BLI_bitmap visit_bitmap;
+};
+
+static void cage_mapped_verts_callback(void *userData, int index, const float co[3],
+                                       const float UNUSED(no_f[3]), const short UNUSED(no_s[3]))
+{
+	struct CageUserData *data = userData;
+
+	if ((index >= 0 && index < data->totvert) && (!BLI_BITMAP_GET(data->visit_bitmap, index))) {
+		BLI_BITMAP_SET(data->visit_bitmap, index);
+		copy_v3_v3(data->cos_cage[index], co);
+	}
+}
+
+float (*BKE_editmesh_vertexCos_get(BMEditMesh *em, Scene *scene, int *r_numVerts))[3]
+{
+	DerivedMesh *cage, *final;
+	BLI_bitmap visit_bitmap;
+	struct CageUserData data;
+	float (*cos_cage)[3];
+
+	cage = editbmesh_get_derived_cage_and_final(scene, em->ob, em, &final, CD_MASK_BAREMESH);
+	cos_cage = MEM_callocN(sizeof(*cos_cage) * em->bm->totvert, "bmbvh cos_cage");
+
+	/* when initializing cage verts, we only want the first cage coordinate for each vertex,
+	 * so that e.g. mirror or array use original vertex coordinates and not mirrored or duplicate */
+	visit_bitmap = BLI_BITMAP_NEW(em->bm->totvert, __func__);
+
+	data.totvert = em->bm->totvert;
+	data.cos_cage = cos_cage;
+	data.visit_bitmap = visit_bitmap;
+
+	cage->foreachMappedVert(cage, cage_mapped_verts_callback, &data);
+
+	MEM_freeN(visit_bitmap);
+
+	if (r_numVerts) {
+		*r_numVerts = em->bm->totvert;
+	}
+
+	return cos_cage;
+}
diff --git a/source/blender/blenkernel/intern/editmesh_bvh.c b/source/blender/blenkernel/intern/editmesh_bvh.c
index ab5e5373a95..2d04ff2071f 100644
--- a/source/blender/blenkernel/intern/editmesh_bvh.c
+++ b/source/blender/blenkernel/intern/editmesh_bvh.c
@@ -34,9 +34,8 @@
 #include "DNA_object_types.h"
 
 #include "BLI_math.h"
-#include "BLI_bitmap.h"
+#include "BLI_kdopbvh.h"
 
-#include "BKE_DerivedMesh.h"
 #include "BKE_editmesh.h"
 
 #include "BKE_editmesh_bvh.h"  /* own include */
@@ -49,46 +48,29 @@ struct BMBVHTree {
 	BMesh *bm;
 
 	float (*cos_cage)[3];
-	int flag;
-};
+	bool cos_cage_free;
 
-struct CageUserData {
-	int totvert;
-	float (*cos_cage)[3];
-	BLI_bitmap vert_bitmap;
+	int flag;
 };
 
-static void cage_mapped_verts_callback(void *userData, int index, const float co[3],
-                                       const float UNUSED(no_f[3]), const short UNUSED(no_s[3]))
-{
-	struct CageUserData *data = userData;
-
-	if ((index >= 0 && index < data->totvert) && (!BLI_BITMAP_GET(data->vert_bitmap, index))) {
-		BLI_BITMAP_SET(data->vert_bitmap, index);
-		copy_v3_v3(data->cos_cage[index], co);
-	}
-}
-
-BMBVHTree *BKE_bmbvh_new(BMEditMesh *em, int flag, struct Scene *scene)
+BMBVHTree *BKE_bmbvh_new(BMEditMesh *em, int flag, float (*cos_cage)[3], const bool cos_cage_free)
 {
 	/* could become argument */
 	const float epsilon = FLT_EPSILON * 2.0f;
 
 	struct BMLoop *(*looptris)[3] = em->looptris;
 	BMBVHTree *bmtree = MEM_callocN(sizeof(*bmtree), "BMBVHTree");
-	DerivedMesh *cage, *final;
-	float cos[3][3], (*cos_cage)[3] = NULL;
+	float cos[3][3];
 	int i;
 	int tottri;
 
 	/* BKE_editmesh_tessface_calc() must be called already */
 	BLI_assert(em->tottri != 0 || em->bm->totface == 0);
 
-	/* cage-flag needs scene */
-	BLI_assert(scene || !(flag & BMBVH_USE_CAGE));
-
 	bmtree->em = em;
 	bmtree->bm = em->bm;
+	bmtree->cos_cage = cos_cage;
+	bmtree->cos_cage_free = cos_cage_free;
 	bmtree->flag = flag;
 
 	if (flag & (BMBVH_RESPECT_SELECT)) {
@@ -112,29 +94,7 @@ BMBVHTree *BKE_bmbvh_new(BMEditMesh *em, int flag, struct Scene *scene)
 	}
 
 	bmtree->tree = BLI_bvhtree_new(tottri, epsilon, 8, 8);
-	
-	if (flag & BMBVH_USE_CAGE) {
-		BLI_bitmap vert_bitmap;
-		struct CageUserData data;
-
-		cage = editbmesh_get_derived_cage_and_final(scene, em->ob, em, &final, CD_MASK_DERIVEDMESH);
-		cos_cage = MEM_callocN(sizeof(float) * 3 * em->bm->totvert, "bmbvh cos_cage");
-		
-		/* when initializing cage verts, we only want the first cage coordinate for each vertex,
-		 * so that e.g. mirror or array use original vertex coordinates and not mirrored or duplicate */
-		vert_bitmap = BLI_BITMAP_NEW(em->bm->totvert, __func__);
-
-		data.totvert = em->bm->totvert;
-		data.cos_cage = cos_cage;
-		data.vert_bitmap = vert_bitmap;
-		
-		cage->foreachMappedVert(cage, cage_mapped_verts_callback, &data);
 
-		MEM_freeN(vert_bitmap);
-	}
-	
-	bmtree->cos_cage = cos_cage;
-	
 	for (i = 0; i < em->tottri; i++) {
 
 		if (flag & BMBVH_RESPECT_SELECT) {
@@ -150,7 +110,7 @@ BMBVHTree *BKE_bmbvh_new(BMEditMesh *em, int flag, struct Scene *scene)
 			}
 		}
 
-		if (flag & BMBVH_USE_CAGE) {
+		if (cos_cage) {
 			copy_v3_v3(cos[0], cos_cage[BM_elem_index_get(looptris[i][0]->v)]);
 			copy_v3_v3(cos[1], cos_cage[BM_elem_index_get(looptris[i][1]->v)]);
 			copy_v3_v3(cos[2], cos_cage[BM_elem_index_get(looptris[i][2]->v)]);
@@ -173,8 +133,9 @@ void BKE_bmbvh_free(BMBVHTree *bmtree)
 {
 	BLI_bvhtree_free(bmtree->tree);
 	
-	if (bmtree->cos_cage)
+	if (bmtree->cos_cage && bmtree->cos_cage_free) {
 		MEM_freeN(bmtree->cos_cage);
+	}
 	
 	MEM_freeN(bmtree);
 }
diff --git a/source/blender/editors/mesh/editmesh_knife.c b/source/blender/editors/mesh/editmesh_knife.c
index 022e753dd40..040a40c470a 100644
--- a/source/blender/editors/mesh/editmesh_knife.c
+++ b/source/blender/editors/mesh/editmesh_knife.c
@@ -2954,20 +2954,6 @@ static void knifetool_exit(bContext *C, wmOperator *op)
 	op->customdata = NULL;
 }
 
-static void cage_mapped_verts_callback(void *userData, int index, const float co[3],
-                                       const float UNUSED(no_f[3]), const short UNUSED(no_s[3]))
-{
-	void **data = userData;
-	BMEditMesh *em = data[0];
-	float (*cagecos)[3] = data[1];
-	SmallHash *hash = data[2];
-
-	if (index >= 0 && index < em->bm->totvert && !BLI_smallhash_haskey(hash, index)) {
-		BLI_smallhash_insert(hash, index, NULL);
-		copy_v3_v3(cagecos[index], co);
-	}
-}
-
 static void knifetool_update_mval(KnifeTool_OpData *kcd, const float mval[2])
 {
 	knife_recalc_projmat(kcd);
@@ -2990,9 +2976,6 @@ static void knifetool_init(bContext *C, KnifeTool_OpData *kcd,
 {
 	Scene *scene = CTX_data_scene(C);
 	Object *obedit = CTX_data_edit_object(C);
-	DerivedMesh *cage, *final;
-	SmallHash shash;
-	void *data[3];
 
 	/* assign the drawing handle for drawing preview line... */
 	kcd->ob = obedit;
@@ -3004,20 +2987,12 @@ static void knifetool_init(bContext *C, KnifeTool_OpData *kcd,
 
 	BM_mesh_elem_index_ensure(kcd->em->bm, BM_VERT);
 
-	cage = editbmesh_get_derived_cage_and_final(scene, obedit, kcd->em, &final, CD_MASK_DERIVEDMESH);
-	kcd->cagecos = MEM_callocN(sizeof(float) * 3 * kcd->em->bm->totvert, "knife cagecos");
-	data[0] = kcd->em;
-	data[1] = kcd->cagecos;
-	data[2] = &shash;
-
-	BLI_smallhash_init(&shash);
-	cage->foreachMappedVert(cage, cage_mapped_verts_callback, data);
-	BLI_smallhash_release(&shash);
+	kcd->cagecos = BKE_editmesh_vertexCos_get(kcd->em, scene, NULL);
 
 	kcd->bmbvh = BKE_bmbvh_new(kcd->em,
-	                          (BMBVH_USE_CAGE | BMBVH_RETURN_ORIG) |
+	                          BMBVH_RETURN_ORIG |
 	                          (only_select ? BMBVH_RESPECT_SELECT : BMBVH_RESPECT_HIDDEN),
-	                          scene);
+	                          kcd->cagecos, false);
 
 	kcd->arena = BLI_memarena_new(1 << 15, "knife");
 	kcd->vthresh = KMAXDIST - 1;
diff --git a/source/blender/editors/mesh/editmesh_utils.c b/source/blender/editors/mesh/editmesh_utils.c
index 01e3ed4f4cb..819a8f78e6c 100644
--- a/source/blender/editors/mesh/editmesh_utils.c
+++ b/source/blender/editors/mesh/editmesh_utils.c
@@ -1166,7 +1166,7 @@ void EDBM_verts_mirror_cache_begin(BMEditMesh *em, const bool use_select)
 		ED_mesh_mirrtopo_init(me, -1, &mesh_topo_store, true);
 	}
 	else {
-		tree = BKE_bmbvh_new(em, 0, NULL);
+		tree = BKE_bmbvh_new(em, 0, NULL, false);
 	}
 
 	BM_ITER_MESH (v, &iter, bm, BM_VERTS_OF_MESH) {
@@ -1379,6 +1379,8 @@ int EDBM_view3d_poll(bContext *C)
 	return 0;
 }
 
+
+
 /* -------------------------------------------------------------------- */
 /* BMBVH functions */
 // XXX
diff --git a/source/blender/editors/transform/transform.c b/source/blender/editors/transform/transform.c
index 487c26aaf54..ca73405fb16 100644
--- a/source/blender/editors/transform/transform.c
+++ b/source/blender/editors/transform/transform.c
@@ -5176,7 +5176,7 @@ static int createEdgeSlideVerts(TransInfo *t)
 	use_btree_disp = (v3d && t->obedit->dt > OB_WIRE && v3d->drawtype > OB_WIRE);
 
 	if (use_btree_disp) {
-		btree = BKE_bmbvh_new(em, BMBVH_RESPECT_HIDDEN, NULL);
+		btree = BKE_bmbvh_new(em, BMBVH_RESPECT_HIDDEN, NULL, false);
 	}
 	else {
 		btree = NULL;