[#22262] Sculpting shape keys using the Smooth brush switches the shape to the Basis

PBVH used the same verts array as mesh data and shape key/reference key coords
were applying on the mesh data, so on some refreshing undeformed mesh was
displayed.
Added utility functions to get vert coords from key block, apply new vert coords
on keyblock and function to apply coords on bpvh, so now pbvh uses it's ovn
vertex array and no changes are making to the mesh data.

Additional change:
Store key block name in SculptUndoNode, so now shape wouldn't be copied to
wrong keyblock on undo

2 files changed, 89 insertions, 51 deletions

diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index cf760f345b5..91769c92efe 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -275,6 +275,9 @@ typedef struct SculptUndoNode {
 
 	/* layer brush */
 	float *layer_disp;
+
+	/* shape keys */
+	char *shapeName[32]; /* keep size in sync with keyblock dna */
 } SculptUndoNode;
 
 static void update_cb(PBVHNode *node, void *data)
@@ -326,16 +329,53 @@ static void sculpt_undo_restore(bContext *C, ListBase *lb)
 			continue;
 
 		if(unode->maxvert) {
+			char *shapeName= (char*)unode->shapeName;
+
 			/* regular mesh restore */
 			if(ss->totvert != unode->maxvert)
 				continue;
 
+			if (ss->kb && strcmp(ss->kb->name, shapeName)) {
+				/* shape key has been changed before calling undo operator */
+
+				Key *key= ob_get_key(ob);
+				KeyBlock *kb= key_get_named_keyblock(key, shapeName);
+
+				if (kb) {
+					ob->shapenr= BLI_findindex(&key->block, kb) + 1;
+					ob->shapeflag|= OB_SHAPE_LOCK;
+
+					sculpt_update_mesh_elements(scene, ob, 0);
+					WM_event_add_notifier(C, NC_OBJECT|ND_DATA, ob);
+				} else {
+					/* key has been removed -- skip this undo node */
+					continue;
+				}
+			}
+
 			index= unode->index;
 			mvert= ss->mvert;
 
-			for(i=0; i<unode->totvert; i++) {
-				swap_v3_v3(mvert[index[i]].co, unode->co[i]);
-				mvert[index[i]].flag |= ME_VERT_PBVH_UPDATE;
+			if (ss->kb) {
+				float (*vertCos)[3];
+				vertCos= key_to_vertcos(ob, ss->kb);
+
+				for(i=0; i<unode->totvert; i++)
+					swap_v3_v3(vertCos[index[i]], unode->co[i]);
+
+				/* propagate new coords to keyblock */
+				vertcos_to_key(ob, ss->kb, vertCos);
+
+				/* pbvh uses it's own mvert array, so coords should be */
+				/* propagated to pbvh here */
+				BLI_pbvh_apply_vertCos(ss->pbvh, vertCos);
+
+				MEM_freeN(vertCos);
+			} else {
+				for(i=0; i<unode->totvert; i++) {
+					swap_v3_v3(mvert[index[i]].co, unode->co[i]);
+					mvert[index[i]].flag |= ME_VERT_PBVH_UPDATE;
+				}
 			}
 		}
 		else if(unode->maxgrid && dm->getGridData) {
@@ -365,9 +405,6 @@ static void sculpt_undo_restore(bContext *C, ListBase *lb)
 	}
 
 	if(update) {
-		if(ss->kb) sculpt_mesh_to_key(ss->ob, ss->kb);
-		if(ss->refkb) sculpt_key_to_mesh(ss->refkb, ob);
-
 		/* we update all nodes still, should be more clever, but also
 		   needs to work correct when exiting/entering sculpt mode and
 		   the nodes get recreated, though in that case it could do all */
@@ -377,7 +414,7 @@ static void sculpt_undo_restore(bContext *C, ListBase *lb)
 		if((mmd=sculpt_multires_active(scene, ob)))
 			multires_mark_as_modified(ob);
 
-		if(sculpt_modifiers_active(scene, ob))
+		if(ss->modifiers_active || ((Mesh*)ob->data)->id.us > 1)
 			DAG_id_flush_update(&ob->id, OB_RECALC_DATA);
 	}
 }
@@ -475,7 +512,11 @@ static SculptUndoNode *sculpt_undo_push_node(SculptSession *ss, PBVHNode *node)
 
 	if(unode->grids)
 		memcpy(unode->grids, grids, sizeof(int)*totgrid);
-	
+
+	/* store active shape key */
+	if(ss->kb) BLI_strncpy((char*)unode->shapeName, ss->kb->name, sizeof(ss->kb->name));
+	else unode->shapeName[0]= '\0';
+
 	return unode;
 }
 
@@ -1367,6 +1408,17 @@ static void do_flatten_clay_brush(Sculpt *sd, SculptSession *ss, PBVHNode **node
 	}
 }
 
+/* copy the modified vertices from bvh to the active key */
+static void sculpt_update_keyblock(SculptSession *ss)
+{
+	float (*vertCos)[3]= BLI_pbvh_get_vertCos(ss->pbvh);
+
+	if (vertCos) {
+		vertcos_to_key(ss->ob, ss->kb, vertCos);
+		MEM_freeN(vertCos);
+	}
+}
+
 static void do_brush_action(Sculpt *sd, SculptSession *ss, StrokeCache *cache)
 {
 	SculptSearchSphereData data;
@@ -1424,13 +1476,15 @@ static void do_brush_action(Sculpt *sd, SculptSession *ss, StrokeCache *cache)
 			do_flatten_clay_brush(sd, ss, nodes, totnode, 1);
 			break;
 		}
-	
-		/* copy the modified vertices from mesh to the active key */
-		if(ss->kb) mesh_to_key(ss->ob->data, ss->kb);
-		
+
+		/* optimization: we could avoid copying new coords to keyblock at each */
+		/* stroke step if there are no modifiers due to pbvh is used for displaying */
+		/* so to increase speed we'll copy new coords to keyblock when stroke is done */
+		if(ss->kb && ss->modifiers_active) sculpt_update_keyblock(ss);
+
 		if(nodes)
 			MEM_freeN(nodes);
-	}	
+	}
 }
 
 /* Flip all the editdata across the axis/axes specified by symm. Used to
@@ -1506,40 +1560,18 @@ struct MultiresModifierData *sculpt_multires_active(Scene *scene, Object *ob)
 	return NULL;
 }
 
-void sculpt_key_to_mesh(KeyBlock *kb, Object *ob)
-{
-	Mesh *me= ob->data;
-
-	key_to_mesh(kb, me);
-	mesh_calc_normals(me->mvert, me->totvert, me->mface, me->totface, NULL);
-}
-
-void sculpt_mesh_to_key(Object *ob, KeyBlock *kb)
-{
-	Mesh *me= ob->data;
-
-	mesh_to_key(me, kb);
-}
-
 void sculpt_update_mesh_elements(Scene *scene, Object *ob, int need_fmap)
 {
-	DerivedMesh *dm = mesh_get_derived_final(scene, ob, 0);
+	DerivedMesh *dm = mesh_get_derived_final(scene, ob, CD_MASK_BAREMESH);
 	SculptSession *ss = ob->sculpt;
 	MultiresModifierData *mmd= sculpt_multires_active(scene, ob);
 
 	ss->ob= ob;
 
-	if((ob->shapeflag & OB_SHAPE_LOCK) && !mmd) {
-		ss->kb= ob_get_keyblock(ob);
-		ss->refkb= ob_get_reference_keyblock(ob);
-	}
-	else {
-		ss->kb= NULL;
-		ss->refkb= NULL;
-	}
+	ss->modifiers_active= sculpt_modifiers_active(scene, ob);
 
-	/* need to make PBVH with shape key coordinates */
-	if(ss->kb) sculpt_key_to_mesh(ss->kb, ss->ob);
+	if((ob->shapeflag & OB_SHAPE_LOCK) && !mmd) ss->kb= ob_get_keyblock(ob);
+	else ss->kb= NULL;
 
 	if(mmd) {
 		ss->multires = mmd;
@@ -1561,6 +1593,17 @@ void sculpt_update_mesh_elements(Scene *scene, Object *ob, int need_fmap)
 
 	ss->pbvh = dm->getPBVH(ob, dm);
 	ss->fmap = (need_fmap && dm->getFaceMap)? dm->getFaceMap(ob, dm): NULL;
+
+	/* if pbvh is deformed, key block is already applied to it */
+	if (ss->kb && !BLI_pbvh_isDeformed(ss->pbvh)) {
+		float (*vertCos)[3]= key_to_vertcos(ob, ss->kb);
+
+		if (vertCos) {
+			/* apply shape keys coordinates to PBVH */
+			BLI_pbvh_apply_vertCos(ss->pbvh, vertCos);
+			MEM_freeN(vertCos);
+		}
+	}
 }
 
 static int sculpt_mode_poll(bContext *C)
@@ -1815,9 +1858,7 @@ static void sculpt_update_cache_variants(Sculpt *sd, SculptSession *ss, struct P
 
 static void sculpt_stroke_modifiers_check(bContext *C, SculptSession *ss)
 {
-	Scene *scene= CTX_data_scene(C);
-
-	if(sculpt_modifiers_active(scene, ss->ob)) {
+	if(ss->modifiers_active) {
 		Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
 		Brush *brush = paint_brush(&sd->paint);
 
@@ -1995,7 +2036,6 @@ static void sculpt_restore_mesh(Sculpt *sd, SculptSession *ss)
 
 static void sculpt_flush_update(bContext *C)
 {
-	Scene *scene = CTX_data_scene(C);
 	Object *ob = CTX_data_active_object(C);
 	SculptSession *ss = ob->sculpt;
 	ARegion *ar = CTX_wm_region(C);
@@ -2005,7 +2045,7 @@ static void sculpt_flush_update(bContext *C)
 	if(mmd)
 		multires_mark_as_modified(ob);
 
-	if(sculpt_modifiers_active(scene, ob)) {
+	if(ss->modifiers_active) {
 		DAG_id_flush_update(&ob->id, OB_RECALC_DATA);
 		ED_region_tag_redraw(ar);
 	}
@@ -2089,10 +2129,13 @@ static void sculpt_stroke_done(bContext *C, struct PaintStroke *stroke)
 
 		BLI_pbvh_update(ss->pbvh, PBVH_UpdateOriginalBB, NULL);
 
-		if(ss->refkb) sculpt_key_to_mesh(ss->refkb, ob);
+		/* optimization: if there is locked key and active modifiers present in */
+		/* the stack, keyblock is updating at each step. otherwise we could update */
+		/* keyblock only when stroke is finished */
+		if(ss->kb && !ss->modifiers_active) sculpt_update_keyblock(ss);
 
 		ss->partial_redraw = 0;
-		
+
 		/* try to avoid calling this, only for e.g. linked duplicates now */
 		if(((Mesh*)ob->data)->id.us > 1)
 			DAG_id_flush_update(&ob->id, OB_RECALC_DATA);
@@ -2225,9 +2268,6 @@ static void sculpt_init_session(Scene *scene, Object *ob)
 	ob->sculpt = MEM_callocN(sizeof(SculptSession), "sculpt session");
 	
 	sculpt_update_mesh_elements(scene, ob, 0);
-
-	if(ob->sculpt->refkb)
-		sculpt_key_to_mesh(ob->sculpt->refkb, ob);
 }
 
 static int sculpt_toggle_mode(bContext *C, wmOperator *op)
diff --git a/source/blender/editors/sculpt_paint/sculpt_intern.h b/source/blender/editors/sculpt_paint/sculpt_intern.h
index d8043d2c988..c7c6833b695 100644
--- a/source/blender/editors/sculpt_paint/sculpt_intern.h
+++ b/source/blender/editors/sculpt_paint/sculpt_intern.h
@@ -58,8 +58,6 @@ void sculpt(Sculpt *sd);
 
 int sculpt_poll(struct bContext *C);
 void sculpt_update_mesh_elements(struct Scene *scene, struct Object *ob, int need_fmap);
-void sculpt_key_to_mesh(struct KeyBlock *kb, struct Object *ob);
-void sculpt_mesh_to_key(struct Object *ob, struct KeyBlock *kb);
 
 /* Stroke */
 struct SculptStroke *sculpt_stroke_new(const int max);