fix for problem with edge slide where it would stop shapekey modifier from being applied (because of added vertices),

now, instead of making hidden copies of faces, the faces are copied into a temp bmesh.

also remove a hash that was being created and not used (old code).

2 files changed, 43 insertions, 99 deletions

diff --git a/source/blender/editors/transform/transform.c b/source/blender/editors/transform/transform.c
index 4ff1c22a4ed..8818ad1d421 100644
--- a/source/blender/editors/transform/transform.c
+++ b/source/blender/editors/transform/transform.c
@@ -59,7 +59,6 @@
 #include "BLI_string.h"
 #include "BLI_ghash.h"
 #include "BLI_linklist.h"
-#include "BLI_smallhash.h"
 
 #include "BKE_nla.h"
 #include "BKE_bmesh.h"
@@ -4995,8 +4994,7 @@ static bool createEdgeSlideVerts(TransInfo *t)
 	TransDataEdgeSlideVert *sv_array;
 	int sv_tot;
 	BMBVHTree *btree;
-	/* BMVert -> sv_array index */
-	SmallHash table;
+	int *sv_table;  /* BMVert -> sv_array index */
 	EdgeSlideData *sld = MEM_callocN(sizeof(*sld), "sld");
 	View3D *v3d = NULL;
 	RegionView3D *rv3d = NULL;
@@ -5036,14 +5034,7 @@ static bool createEdgeSlideVerts(TransInfo *t)
 	else {
 		ED_view3d_ob_project_mat_get(rv3d, t->obedit, projectMat);
 	}
-	
-	BLI_smallhash_init(&sld->vhash);
-	BLI_smallhash_init(&table);
 
-	if (sld->use_origfaces) {
-		BLI_smallhash_init(&sld->origfaces);
-	}
-	
 	/*ensure valid selection*/
 	BM_ITER_MESH (v, &iter, bm, BM_VERTS_OF_MESH) {
 		if (BM_elem_flag_test(v, BM_ELEM_SELECT)) {
@@ -5077,20 +5068,26 @@ static bool createEdgeSlideVerts(TransInfo *t)
 		}
 	}
 
+	sv_table = MEM_mallocN(sizeof(*sv_table) * bm->totvert, __func__);
+
 	j = 0;
-	BM_ITER_MESH (v, &iter, bm, BM_VERTS_OF_MESH) {
+	BM_ITER_MESH_INDEX (v, &iter, bm, BM_VERTS_OF_MESH, i) {
 		if (BM_elem_flag_test(v, BM_ELEM_SELECT)) {
 			BM_elem_flag_enable(v, BM_ELEM_TAG);
-			BLI_smallhash_insert(&table, (uintptr_t)v, SET_INT_IN_POINTER(j));
+			sv_table[i] = j;
 			j += 1;
 		}
 		else {
 			BM_elem_flag_disable(v, BM_ELEM_TAG);
+			sv_table[i] = -1;
 		}
+		BM_elem_index_set(v, i); /* set_inline */
 	}
+	bm->elem_index_dirty &= ~BM_VERT;
 
 	if (!j) {
 		MEM_freeN(sld);
+		MEM_freeN(sv_table);
 		return false;
 	}
 
@@ -5187,9 +5184,9 @@ static bool createEdgeSlideVerts(TransInfo *t)
 			BMEdge *e_prev;
 
 			/* XXX, 'sv' will initialize multiple times, this is suspicious. see [#34024] */
-			BLI_assert(BLI_smallhash_haskey(&table, (uintptr_t)v) != false);
 			BLI_assert(v != NULL);
-			sv = sv_array + GET_INT_FROM_POINTER(BLI_smallhash_lookup(&table, (uintptr_t)v));
+			BLI_assert(sv_table[BM_elem_index_get(v)] != -1);
+			sv = &sv_array[sv_table[BM_elem_index_get(v)]];
 			sv->v = v;
 			copy_v3_v3(sv->v_co_orig, v->co);
 			sv->loop_nr = loop_nr;
@@ -5213,9 +5210,9 @@ static bool createEdgeSlideVerts(TransInfo *t)
 			e = get_other_edge(v, e);
 
 			if (!e) {
-				BLI_assert(BLI_smallhash_haskey(&table, (uintptr_t)v) != false);
 				BLI_assert(v != NULL);
-				sv = sv_array + GET_INT_FROM_POINTER(BLI_smallhash_lookup(&table, (uintptr_t)v));
+				BLI_assert(sv_table[BM_elem_index_get(v)] != -1);
+				sv = &sv_array[sv_table[BM_elem_index_get(v)]];
 				sv->v = v;
 				copy_v3_v3(sv->v_co_orig, v->co);
 				sv->loop_nr = loop_nr;
@@ -5282,7 +5279,6 @@ static bool createEdgeSlideVerts(TransInfo *t)
 		loop_nr++;
 	}
 
-
 	/* use for visibility checks */
 	use_btree_disp = (v3d && t->obedit->dt > OB_WIRE && v3d->drawtype > OB_WIRE);
 
@@ -5330,8 +5326,8 @@ static bool createEdgeSlideVerts(TransInfo *t)
 						continue;
 					}
 
-					BLI_assert(BLI_smallhash_haskey(&table, (uintptr_t)v) != false);
-					j = GET_INT_FROM_POINTER(BLI_smallhash_lookup(&table, (uintptr_t)v));
+					BLI_assert(sv_table[BM_elem_index_get(v)] != -1);
+					j = sv_table[BM_elem_index_get(v)];
 
 					if (sv_array[j].v_b) {
 						ED_view3d_project_float_v3_m4(ar, sv_array[j].v_b->co, sco_b, projectMat);
@@ -5378,36 +5374,29 @@ static bool createEdgeSlideVerts(TransInfo *t)
 
 	bmesh_edit_begin(bm, BMO_OPTYPE_FLAG_UNTAN_MULTIRES);
 
+	if (sld->use_origfaces) {
+		sld->origfaces = BLI_ghash_ptr_new(__func__);
+		sld->bm_origfaces = BM_mesh_create(&bm_mesh_allocsize_default);
+		/* we need to have matching customdata */
+		BM_mesh_copy_init_customdata(sld->bm_origfaces, bm, NULL);
+	}
+
 	/*create copies of faces for customdata projection*/
 	sv_array = sld->sv;
 	for (i = 0; i < sld->totsv; i++, sv_array++) {
-		BMIter fiter, liter;
+		BMIter fiter;
 		BMFace *f;
-		BMLoop *l;
 		
 
 		if (sld->use_origfaces) {
 			BM_ITER_ELEM (f, &fiter, sv_array->v, BM_FACES_OF_VERT) {
-
-				if (!BLI_smallhash_haskey(&sld->origfaces, (uintptr_t)f)) {
-					BMFace *copyf = BM_face_copy(bm, f, true, true);
-
-					BM_face_select_set(bm, copyf, false);
-					BM_elem_flag_enable(copyf, BM_ELEM_HIDDEN);
-					BM_ITER_ELEM (l, &liter, copyf, BM_LOOPS_OF_FACE) {
-						BM_vert_select_set(bm, l->v, false);
-						BM_elem_flag_enable(l->v, BM_ELEM_HIDDEN);
-						BM_edge_select_set(bm, l->e, false);
-						BM_elem_flag_enable(l->e, BM_ELEM_HIDDEN);
-					}
-
-					BLI_smallhash_insert(&sld->origfaces, (uintptr_t)f, copyf);
+				if (!BLI_ghash_haskey(sld->origfaces, f)) {
+					BMFace *f_copy = BM_face_copy(sld->bm_origfaces, bm, f, true, true);
+					BLI_ghash_insert(sld->origfaces, f, f_copy);
 				}
 			}
 		}
 
-		BLI_smallhash_insert(&sld->vhash, (uintptr_t)sv_array->v, sv_array);
-
 		/* switch a/b if loop direction is different from global direction */
 		l_nr = sv_array->loop_nr;
 		if (dot_v3v3(loop_dir[l_nr], mval_dir) < 0.0f) {
@@ -5419,9 +5408,8 @@ static bool createEdgeSlideVerts(TransInfo *t)
 	if (rv3d)
 		calcNonProportionalEdgeSlide(t, sld, mval);
 
-	sld->origfaces_init = true;
 	sld->em = em;
-	
+
 	/*zero out start*/
 	zero_v2(mval_start);
 
@@ -5439,18 +5427,13 @@ static bool createEdgeSlideVerts(TransInfo *t)
 	
 	t->customData = sld;
 	
-	BLI_smallhash_release(&table);
+	MEM_freeN(sv_table);
 	if (btree) {
 		BKE_bmbvh_free(btree);
 	}
 	MEM_freeN(loop_dir);
 	MEM_freeN(loop_maxdist);
 
-	if (sld->use_origfaces) {
-		/* arrays are dirty from copying faces: EDBM_index_arrays_free */
-		EDBM_update_generic(em, false, true);
-	}
-
 	return true;
 }
 
@@ -5472,15 +5455,8 @@ void projectEdgeSlideData(TransInfo *t, bool is_final)
 		BM_ITER_ELEM (l, &fiter, sv->v, BM_LOOPS_OF_VERT) {
 			BMFace *f_copy;      /* the copy of 'f' */
 			BMFace *f_copy_flip; /* the copy of 'f' or detect if we need to flip to the shorter side. */
-			bool is_sel, is_hide;
-			
-			/* the face attributes of the copied face will get
-			 * copied over, so its necessary to save the selection
-			 * and hidden state*/
-			is_sel = BM_elem_flag_test(l->f, BM_ELEM_SELECT) != 0;
-			is_hide = BM_elem_flag_test(l->f, BM_ELEM_HIDDEN) != 0;
 			
-			f_copy = BLI_smallhash_lookup(&sld->origfaces, (uintptr_t)l->f);
+			f_copy = BLI_ghash_lookup(sld->origfaces, l->f);
 			
 			/* project onto copied projection face */
 			f_copy_flip = f_copy;
@@ -5494,12 +5470,12 @@ void projectEdgeSlideData(TransInfo *t, bool is_final)
 
 				if (sld->perc < 0.0f) {
 					if (BM_vert_in_face(l_ed_sel->radial_next->f, sv->v_b)) {
-						f_copy_flip = BLI_smallhash_lookup(&sld->origfaces, (uintptr_t)l_ed_sel->radial_next->f);
+						f_copy_flip = BLI_ghash_lookup(sld->origfaces, l_ed_sel->radial_next->f);
 					}
 				}
 				else if (sld->perc > 0.0f) {
 					if (BM_vert_in_face(l_ed_sel->radial_next->f, sv->v_a)) {
-						f_copy_flip = BLI_smallhash_lookup(&sld->origfaces, (uintptr_t)l_ed_sel->radial_next->f);
+						f_copy_flip = BLI_ghash_lookup(sld->origfaces, l_ed_sel->radial_next->f);
 					}
 				}
 
@@ -5585,7 +5561,7 @@ void projectEdgeSlideData(TransInfo *t, bool is_final)
 							l_adj = l;
 						}
 
-						f_copy_flip = BLI_smallhash_lookup(&sld->origfaces, (uintptr_t)l_adj->f);
+						f_copy_flip = BLI_ghash_lookup(sld->origfaces, l_adj->f);
 					}
 				}
 			}
@@ -5602,37 +5578,24 @@ void projectEdgeSlideData(TransInfo *t, bool is_final)
 			}
 			
 			/* make sure face-attributes are correct (e.g. MTexPoly) */
-			BM_elem_attrs_copy(em->bm, em->bm, f_copy, l->f);
-			
-			/* restore selection and hidden flags */
-			BM_face_select_set(em->bm, l->f, is_sel);
-			if (!is_hide) {
-				/* this check is a workaround for bug, see note - [#30735],
-				 * without this edge can be hidden and selected */
-				BM_elem_hide_set(em->bm, l->f, is_hide);
-			}
+			BM_elem_attrs_copy(sld->bm_origfaces, em->bm, f_copy, l->f);
 		}
 	}
 }
 
 void freeEdgeSlideTempFaces(EdgeSlideData *sld)
 {
-	if (sld->origfaces_init && sld->use_origfaces) {
-		SmallHashIter hiter;
-		BMFace *copyf;
-
-		copyf = BLI_smallhash_iternew(&sld->origfaces, &hiter, NULL);
-		for (; copyf; copyf = BLI_smallhash_iternext(&hiter, NULL)) {
-			BM_face_verts_kill(sld->em->bm, copyf);
+	if (sld->use_origfaces) {
+		if (sld->bm_origfaces) {
+			BM_mesh_free(sld->bm_origfaces);
+			sld->bm_origfaces = NULL;
 		}
 
-		BLI_smallhash_release(&sld->origfaces);
-
-		/* arrays are dirty from removing faces: EDBM_index_arrays_free */
-		EDBM_update_generic(sld->em, FALSE, TRUE);
+		if (sld->origfaces) {
+			BLI_ghash_free(sld->origfaces, NULL, NULL);
+			sld->origfaces = NULL;
+		}
 	}
-
-	sld->origfaces_init = false;
 }
 
 
@@ -5640,30 +5603,12 @@ void freeEdgeSlideVerts(TransInfo *t)
 {
 	EdgeSlideData *sld = t->customData;
 	
-#if 0 /*BMESH_TODO*/
-	if (me->drawflag & ME_DRAWEXTRA_EDGELEN) {
-		TransDataEdgeSlideVert *sv;
-		LinkNode *look = sld->vertlist;
-		GHash *vertgh = sld->vhash;
-		while (look) {
-			sv  = BLI_ghash_lookup(vertgh, (EditVert *)look->link);
-			if (sv != NULL) {
-				sv->v_a->f &= !SELECT;
-				sv->v_b->f &= !SELECT;
-			}
-			look = look->next;
-		}
-	}
-#endif
-	
 	if (!sld)
 		return;
 	
 	freeEdgeSlideTempFaces(sld);
 
 	bmesh_edit_end(sld->em->bm, BMO_OPTYPE_FLAG_UNTAN_MULTIRES);
-
-	BLI_smallhash_release(&sld->vhash);
 	
 	MEM_freeN(sld->sv);
 	MEM_freeN(sld);
diff --git a/source/blender/editors/transform/transform.h b/source/blender/editors/transform/transform.h
index ea20721b5ae..911d4b0a623 100644
--- a/source/blender/editors/transform/transform.h
+++ b/source/blender/editors/transform/transform.h
@@ -201,15 +201,14 @@ typedef struct EdgeSlideData {
 	TransDataEdgeSlideVert *sv;
 	int totsv;
 	
-	struct SmallHash vhash;
-	struct SmallHash origfaces;
+	struct GHash *origfaces;
 
 	int mval_start[2], mval_end[2];
 	struct BMEditMesh *em;
 
 	/* flag that is set when origfaces is initialized */
 	bool use_origfaces;
-	bool origfaces_init;
+	struct BMesh *bm_origfaces;
 
 	float perc;