Sculpt & modifiers: patch by Sergey Sharybin, with modifications by me.

Fixes various crashes and redraw problems, most noticeable new feature
is that you can now sculpt on a multires mesh with deforming modifiers
preceding it.

I've left out support for sculpting on multires with enabled modifiers
following it, in this case only the base mesh can be sculpted now. The
code changes needed to do this are just too ugly in my opinion, would
need a more torough redesign which I don't think we should try now. In
my opinion this is also not really an important case, since it's going
to be incredibly slow anyway to run a modifier on a high res mesh while
sculpting.


So, to summarize current state:

* Fastest sculpting: base mesh with no modifiers or multires with only
  modifiers preceding it.
* Slower sculpting: base mesh with modifiers, depends on the speed of
  the modifiers.
* Not supported: multires mesh with modifiers following it.

10 files changed, 108 insertions, 30 deletions

diff --git a/source/blender/blenkernel/BKE_subsurf.h b/source/blender/blenkernel/BKE_subsurf.h
index ef3ad3ad2e9..940a0027d0b 100644
--- a/source/blender/blenkernel/BKE_subsurf.h
+++ b/source/blender/blenkernel/BKE_subsurf.h
@@ -72,6 +72,7 @@ typedef struct CCGDerivedMesh {
 	char *faceFlags;
 
 	struct PBVH *pbvh;
+	int pbvh_draw;
 	struct ListBase *fmap;
 	struct IndexNode *fmap_mem;
 
diff --git a/source/blender/blenkernel/intern/DerivedMesh.c b/source/blender/blenkernel/intern/DerivedMesh.c
index 85791d5024d..9d649edd81a 100644
--- a/source/blender/blenkernel/intern/DerivedMesh.c
+++ b/source/blender/blenkernel/intern/DerivedMesh.c
@@ -1870,10 +1870,6 @@ static void mesh_calc_modifiers(Scene *scene, Object *ob, float (*inputVertexCos
 		/* grab modifiers until index i */
 		if((index >= 0) && (modifiers_indexInObject(ob, md) >= index))
 			break;
-
-		/*don't allow other modifiers past multires if in sculpt mode*/
-		if (!useRenderParams && ((ob->mode & OB_MODE_SCULPT) && ob->sculpt))
-			break;
 	}
 
 	for(md=firstmd; md; md=md->next)
diff --git a/source/blender/blenkernel/intern/cdderivedmesh.c b/source/blender/blenkernel/intern/cdderivedmesh.c
index 204ff2a0369..e15e5bc9b45 100644
--- a/source/blender/blenkernel/intern/cdderivedmesh.c
+++ b/source/blender/blenkernel/intern/cdderivedmesh.c
@@ -74,6 +74,7 @@ typedef struct {
 
 	/* Cached */
 	struct PBVH *pbvh;
+	int pbvh_draw;
 	/* Mesh connectivity */
 	struct ListBase *fmap;
 	struct IndexNode *fmap_mem;
@@ -188,6 +189,7 @@ static ListBase *cdDM_getFaceMap(Object *ob, DerivedMesh *dm)
 static struct PBVH *cdDM_getPBVH(Object *ob, DerivedMesh *dm)
 {
 	CDDerivedMesh *cddm = (CDDerivedMesh*) dm;
+	Mesh *me= (ob)? ob->data: NULL;
 
 	if(!ob) {
 		cddm->pbvh= NULL;
@@ -196,13 +198,17 @@ static struct PBVH *cdDM_getPBVH(Object *ob, DerivedMesh *dm)
 
 	if(!ob->sculpt)
 		return NULL;
-	if(ob->sculpt->pbvh)
+	if(ob->sculpt->pbvh) {
 		cddm->pbvh= ob->sculpt->pbvh;
+		cddm->pbvh_draw = (cddm->mvert == me->mvert);
+	}
 
+	/* always build pbvh from original mesh, and only use it for drawing if
+	   this derivedmesh is just original mesh. it's the multires subsurf dm
+	   that this is actually for, to support a pbvh on a modified mesh */
 	if(!cddm->pbvh && ob->type == OB_MESH) {
-		Mesh *me= ob->data;
-
 		cddm->pbvh = BLI_pbvh_new();
+		cddm->pbvh_draw = (cddm->mvert == me->mvert);
 		BLI_pbvh_build_mesh(cddm->pbvh, me->mface, me->mvert,
 				   me->totface, me->totvert);
 	}
@@ -417,7 +423,7 @@ static void cdDM_drawFacesSolid(DerivedMesh *dm,
 	glVertex3fv(mvert[index].co);	\
 }
 
-	if(cddm->pbvh) {
+	if(cddm->pbvh && cddm->pbvh_draw) {
 		if(dm->numFaceData) {
 			float (*face_nors)[3] = CustomData_get_layer(&dm->faceData, CD_NORMAL);
 
diff --git a/source/blender/blenkernel/intern/multires.c b/source/blender/blenkernel/intern/multires.c
index ad069be50f5..ee8a74d6fbb 100644
--- a/source/blender/blenkernel/intern/multires.c
+++ b/source/blender/blenkernel/intern/multires.c
@@ -654,7 +654,9 @@ static void multiresModifier_update(DerivedMesh *dm)
 			int i, j, numGrids, highGridSize, lowGridSize;
 
 			/* create subsurf DM from original mesh at high level */
-			cddm = CDDM_from_mesh(me, NULL);
+			if (ob->derivedDeform) cddm = CDDM_copy(ob->derivedDeform);
+			else cddm = CDDM_from_mesh(me, NULL);
+
 			highdm = subsurf_dm_create_local(ob, cddm, totlvl, mmd->simple, 0);
 
 			/* create multires DM from original mesh and displacements */
@@ -705,7 +707,9 @@ static void multiresModifier_update(DerivedMesh *dm)
 		else {
 			DerivedMesh *cddm, *subdm;
 
-			cddm = CDDM_from_mesh(me, NULL);
+			if (ob->derivedDeform) cddm = CDDM_copy(ob->derivedDeform);
+			else cddm = CDDM_from_mesh(me, NULL);
+
 			subdm = subsurf_dm_create_local(ob, cddm, mmd->totlvl, mmd->simple, 0);
 			cddm->release(cddm);
 
diff --git a/source/blender/blenkernel/intern/subsurf_ccg.c b/source/blender/blenkernel/intern/subsurf_ccg.c
index 72236a76032..53206bb3970 100644
--- a/source/blender/blenkernel/intern/subsurf_ccg.c
+++ b/source/blender/blenkernel/intern/subsurf_ccg.c
@@ -44,6 +44,7 @@
 #include "BKE_cdderivedmesh.h"
 #include "BKE_global.h"
 #include "BKE_mesh.h"
+#include "BKE_modifier.h"
 #include "BKE_paint.h"
 #include "BKE_scene.h"
 #include "BKE_subsurf.h"
@@ -2229,10 +2230,28 @@ static ListBase *ccgDM_getFaceMap(Object *ob, DerivedMesh *dm)
 	return ccgdm->fmap;
 }
 
+static int ccgDM_use_grid_pbvh(CCGDerivedMesh *ccgdm)
+{
+	ModifierData *md;
+	MultiresModifierData *mmd= ccgdm->multires.mmd;
+
+	/* in sync with sculpt mode, only use multires grid pbvh if we are
+	   the last enabled modifier in the stack, otherwise we use the base
+	   mesh */
+	if(!mmd)
+		return 0;
+	
+	for(md=mmd->modifier.next; md; md= md->next)
+		if(modifier_isEnabled(mmd->modifier.scene, md, eModifierMode_Realtime))
+			return 0;
+	
+	return 1;
+}
+
 static struct PBVH *ccgDM_getPBVH(Object *ob, DerivedMesh *dm)
 {
 	CCGDerivedMesh *ccgdm= (CCGDerivedMesh*)dm;
-	int gridSize, numGrids;
+	int gridSize, numGrids, grid_pbvh;
 
 	if(!ob) {
 		ccgdm->pbvh= NULL;
@@ -2241,13 +2260,30 @@ static struct PBVH *ccgDM_getPBVH(Object *ob, DerivedMesh *dm)
 
 	if(!ob->sculpt)
 		return NULL;
-	if(ob->sculpt->pbvh)
-		ccgdm->pbvh= ob->sculpt->pbvh;
+
+	grid_pbvh = ccgDM_use_grid_pbvh(ccgdm);
+
+	if(ob->sculpt->pbvh) {
+		if(grid_pbvh) {
+			/* pbvh's grids, gridadj and gridfaces points to data inside ccgdm
+			   but this can be freed on ccgdm release, this updates the pointers
+			   when the ccgdm gets remade, the assumption is that the topology
+			   does not change. */
+			ccgdm_create_grids(dm);
+			BLI_pbvh_grids_update(ob->sculpt->pbvh, ccgdm->gridData, ccgdm->gridAdjacency, (void**)ccgdm->gridFaces);
+		}
+
+		ccgdm->pbvh = ob->sculpt->pbvh;
+		ccgdm->pbvh_draw = grid_pbvh;
+	}
 
 	if(ccgdm->pbvh)
 		return ccgdm->pbvh;
 
-	if(ccgdm->multires.mmd) {
+	/* no pbvh exists yet, we need to create one. only in case of multires
+	   we build a pbvh over the modified mesh, in other cases the base mesh
+	   is being sculpted, so we build a pbvh from that. */
+	if(grid_pbvh) {
 		ccgdm_create_grids(dm);
 
 		gridSize = ccgDM_getGridSize(dm);
@@ -2256,6 +2292,7 @@ static struct PBVH *ccgDM_getPBVH(Object *ob, DerivedMesh *dm)
 		ob->sculpt->pbvh= ccgdm->pbvh = BLI_pbvh_new();
 		BLI_pbvh_build_grids(ccgdm->pbvh, ccgdm->gridData, ccgdm->gridAdjacency,
 			numGrids, gridSize, (void**)ccgdm->gridFaces);
+		ccgdm->pbvh_draw = 1;
 	}
 	else if(ob->type == OB_MESH) {
 		Mesh *me= ob->data;
@@ -2263,6 +2300,7 @@ static struct PBVH *ccgDM_getPBVH(Object *ob, DerivedMesh *dm)
 		ob->sculpt->pbvh= ccgdm->pbvh = BLI_pbvh_new();
 		BLI_pbvh_build_mesh(ccgdm->pbvh, me->mface, me->mvert,
 				   me->totface, me->totvert);
+		ccgdm->pbvh_draw = 0;
 	}
 
 	return ccgdm->pbvh;
diff --git a/source/blender/blenlib/BLI_pbvh.h b/source/blender/blenlib/BLI_pbvh.h
index 0da5b8529bb..e32e85e70ec 100644
--- a/source/blender/blenlib/BLI_pbvh.h
+++ b/source/blender/blenlib/BLI_pbvh.h
@@ -111,6 +111,8 @@ void BLI_pbvh_node_get_original_BB(PBVHNode *node, float bb_min[3], float bb_max
 void BLI_pbvh_update(PBVH *bvh, int flags, float (*face_nors)[3]);
 void BLI_pbvh_redraw_BB(PBVH *bvh, float bb_min[3], float bb_max[3]);
 void BLI_pbvh_get_grid_updates(PBVH *bvh, int clear, void ***gridfaces, int *totface);
+void BLI_pbvh_grids_update(PBVH *bvh, struct DMGridData **grids,
+	struct DMGridAdjacency *gridadj, void **gridfaces);
 
 /* Vertex Iterator */
 
diff --git a/source/blender/blenlib/intern/pbvh.c b/source/blender/blenlib/intern/pbvh.c
index 8aff062c1d4..1fd18e2967c 100644
--- a/source/blender/blenlib/intern/pbvh.c
+++ b/source/blender/blenlib/intern/pbvh.c
@@ -1323,3 +1323,10 @@ void BLI_pbvh_draw(PBVH *bvh, float (*planes)[4], float (*face_nors)[3], int smo
 	}
 }
 
+void BLI_pbvh_grids_update(PBVH *bvh, DMGridData **grids, DMGridAdjacency *gridadj, void **gridfaces)
+{
+	bvh->grids= grids;
+	bvh->gridadj= gridadj;
+	bvh->gridfaces= gridfaces;
+}
+
diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index 245ce9f5e0d..cf760f345b5 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -286,10 +286,21 @@ static void update_cb(PBVHNode *node, void *data)
 static int sculpt_modifiers_active(Scene *scene, Object *ob)
 {
 	ModifierData *md;
+	MultiresModifierData *mmd = sculpt_multires_active(scene, ob);
+
+	/* check if there are any modifiers after what we are sculpting,
+	   for a multires modifier with a deform modifier in front, we
+	   do no need to recalculate the modifier stack. note that this
+	   needs to be in sync with ccgDM_use_grid_pbvh! */
+	if(mmd)
+		md= mmd->modifier.next;
+	else
+		md= modifiers_getVirtualModifierList(ob);
 	
-	for(md= modifiers_getVirtualModifierList(ob); md; md= md->next) {
+	/* exception for shape keys because we can edit those */
+	for(; md; md= md->next) {
 		if(modifier_isEnabled(scene, md, eModifierMode_Realtime))
-			if(!ELEM(md->type, eModifierType_Multires, eModifierType_ShapeKey))
+			if(md->type != eModifierType_ShapeKey)
 				return 1;
 	}
 	
@@ -363,7 +374,7 @@ static void sculpt_undo_restore(bContext *C, ListBase *lb)
 		BLI_pbvh_search_callback(ss->pbvh, NULL, NULL, update_cb, NULL);
 		BLI_pbvh_update(ss->pbvh, PBVH_UpdateBB|PBVH_UpdateOriginalBB|PBVH_UpdateRedraw, NULL);
 
-		if((mmd=sculpt_multires_active(ob)))
+		if((mmd=sculpt_multires_active(scene, ob)))
 			multires_mark_as_modified(ob);
 
 		if(sculpt_modifiers_active(scene, ob))
@@ -1473,7 +1484,7 @@ static void sculpt_update_tex(Sculpt *sd, SculptSession *ss)
 
 /* Sculpt mode handles multires differently from regular meshes, but only if
    it's the last modifier on the stack and it is not on the first level */
-struct MultiresModifierData *sculpt_multires_active(Object *ob)
+struct MultiresModifierData *sculpt_multires_active(Scene *scene, Object *ob)
 {
 	ModifierData *md, *nmd;
 	
@@ -1483,8 +1494,8 @@ struct MultiresModifierData *sculpt_multires_active(Object *ob)
 
 			/* Check if any of the modifiers after multires are active
 			 * if not it can use the multires struct */
-			for (nmd= md->next; nmd; nmd= nmd->next)
-				if(nmd->mode & eModifierMode_Realtime)
+			for(nmd= md->next; nmd; nmd= nmd->next)
+				if(modifier_isEnabled(scene, nmd, eModifierMode_Realtime))
 					break;
 
 			if(!nmd && mmd->sculptlvl > 0)
@@ -1514,10 +1525,11 @@ void sculpt_update_mesh_elements(Scene *scene, Object *ob, int need_fmap)
 {
 	DerivedMesh *dm = mesh_get_derived_final(scene, ob, 0);
 	SculptSession *ss = ob->sculpt;
-	
+	MultiresModifierData *mmd= sculpt_multires_active(scene, ob);
+
 	ss->ob= ob;
 
-	if((ob->shapeflag & OB_SHAPE_LOCK) && !sculpt_multires_active(ob)) {
+	if((ob->shapeflag & OB_SHAPE_LOCK) && !mmd) {
 		ss->kb= ob_get_keyblock(ob);
 		ss->refkb= ob_get_reference_keyblock(ob);
 	}
@@ -1529,7 +1541,8 @@ void sculpt_update_mesh_elements(Scene *scene, Object *ob, int need_fmap)
 	/* need to make PBVH with shape key coordinates */
 	if(ss->kb) sculpt_key_to_mesh(ss->kb, ss->ob);
 
-	if((ss->multires = sculpt_multires_active(ob))) {
+	if(mmd) {
+		ss->multires = mmd;
 		ss->totvert = dm->getNumVerts(dm);
 		ss->totface = dm->getNumFaces(dm);
 		ss->mvert= NULL;
@@ -1543,6 +1556,7 @@ void sculpt_update_mesh_elements(Scene *scene, Object *ob, int need_fmap)
 		ss->mvert = me->mvert;
 		ss->mface = me->mface;
 		ss->face_normals = NULL;
+		ss->multires = NULL;
 	}
 
 	ss->pbvh = dm->getPBVH(ob, dm);
@@ -2221,13 +2235,19 @@ static int sculpt_toggle_mode(bContext *C, wmOperator *op)
 	Scene *scene = CTX_data_scene(C);
 	ToolSettings *ts = CTX_data_tool_settings(C);
 	Object *ob = CTX_data_active_object(C);
-	MultiresModifierData *mmd = sculpt_multires_active(ob);
+	MultiresModifierData *mmd = sculpt_multires_active(scene, ob);
+	int flush_recalc= 0;
+
+	/* multires in sculpt mode could have different from object mode subdivision level */
+	flush_recalc |= mmd && mmd->sculptlvl != mmd->lvl;
+	/* if object has got active modifiers, it's dm could be different in sculpt mode  */
+	//flush_recalc |= sculpt_modifiers_active(scene, ob);
 
 	if(ob->mode & OB_MODE_SCULPT) {
-		if(sculpt_multires_active(ob))
+		if(mmd)
 			multires_force_update(ob);
 
-		if(mmd && mmd->sculptlvl != mmd->lvl)
+		if(flush_recalc)
 			DAG_id_flush_update(&ob->id, OB_RECALC_DATA);
 
 		/* Leave sculptmode */
@@ -2239,7 +2259,7 @@ static int sculpt_toggle_mode(bContext *C, wmOperator *op)
 		/* Enter sculptmode */
 		ob->mode |= OB_MODE_SCULPT;
 
-		if(mmd && mmd->sculptlvl != mmd->lvl)
+		if(flush_recalc)
 			DAG_id_flush_update(&ob->id, OB_RECALC_DATA);
 		
 		/* Create persistent sculpt mode data */
diff --git a/source/blender/editors/sculpt_paint/sculpt_intern.h b/source/blender/editors/sculpt_paint/sculpt_intern.h
index d3553c008b2..d8043d2c988 100644
--- a/source/blender/editors/sculpt_paint/sculpt_intern.h
+++ b/source/blender/editors/sculpt_paint/sculpt_intern.h
@@ -49,7 +49,7 @@ void sculptmode_draw_mesh(int);
 void sculpt_paint_brush(char clear);
 void sculpt_stroke_draw(struct SculptStroke *);
 void sculpt_radialcontrol_start(int mode);
-struct MultiresModifierData *sculpt_multires_active(struct Object *ob);
+struct MultiresModifierData *sculpt_multires_active(struct Scene *scene, struct Object *ob);
 
 struct Brush *sculptmode_brush(void);
 //void do_symmetrical_brush_actions(struct Sculpt *sd, struct wmOperator *wm, struct BrushAction *a, short *, short *);
diff --git a/source/blender/modifiers/intern/MOD_multires.c b/source/blender/modifiers/intern/MOD_multires.c
index 945f17494f0..71e31656799 100644
--- a/source/blender/modifiers/intern/MOD_multires.c
+++ b/source/blender/modifiers/intern/MOD_multires.c
@@ -30,6 +30,8 @@
 *
 */
 
+#include <stddef.h>
+
 #include "BKE_cdderivedmesh.h"
 #include "BKE_multires.h"
 #include "BKE_modifier.h"
@@ -60,6 +62,8 @@ static void copyData(ModifierData *md, ModifierData *target)
 static DerivedMesh *applyModifier(ModifierData *md, Object *ob, DerivedMesh *dm,
 						   int useRenderParams, int isFinalCalc)
 {
+	SculptSession *ss= ob->sculpt;
+	int sculpting= (ob->mode & OB_MODE_SCULPT) && ss;
 	MultiresModifierData *mmd = (MultiresModifierData*)md;
 	DerivedMesh *result;
 
@@ -73,10 +77,10 @@ static DerivedMesh *applyModifier(ModifierData *md, Object *ob, DerivedMesh *dm,
 		result->release(result);
 		result= cddm;
 	}
-	else if((ob->mode & OB_MODE_SCULPT) && ob->sculpt) {
+	else if(sculpting) {
 		/* would be created on the fly too, just nicer this
 		   way on first stroke after e.g. switching levels */
-		ob->sculpt->pbvh= result->getPBVH(ob, result);
+		ss->pbvh= result->getPBVH(ob, result);
 	}
 
 	return result;