Sculpt & modifiers: patch by Sergey Sharybin, with modifications by me.

Fixes various crashes and redraw problems, most noticeable new feature
is that you can now sculpt on a multires mesh with deforming modifiers
preceding it.

I've left out support for sculpting on multires with enabled modifiers
following it, in this case only the base mesh can be sculpted now. The
code changes needed to do this are just too ugly in my opinion, would
need a more torough redesign which I don't think we should try now. In
my opinion this is also not really an important case, since it's going
to be incredibly slow anyway to run a modifier on a high res mesh while
sculpting.


So, to summarize current state:

* Fastest sculpting: base mesh with no modifiers or multires with only
  modifiers preceding it.
* Slower sculpting: base mesh with modifiers, depends on the speed of
  the modifiers.
* Not supported: multires mesh with modifiers following it.

4 files changed, 58 insertions, 14 deletions

diff --git a/source/blender/blenkernel/intern/DerivedMesh.c b/source/blender/blenkernel/intern/DerivedMesh.c
index 85791d5024d..9d649edd81a 100644
--- a/source/blender/blenkernel/intern/DerivedMesh.c
+++ b/source/blender/blenkernel/intern/DerivedMesh.c
@@ -1870,10 +1870,6 @@ static void mesh_calc_modifiers(Scene *scene, Object *ob, float (*inputVertexCos
 		/* grab modifiers until index i */
 		if((index >= 0) && (modifiers_indexInObject(ob, md) >= index))
 			break;
-
-		/*don't allow other modifiers past multires if in sculpt mode*/
-		if (!useRenderParams && ((ob->mode & OB_MODE_SCULPT) && ob->sculpt))
-			break;
 	}
 
 	for(md=firstmd; md; md=md->next)
diff --git a/source/blender/blenkernel/intern/cdderivedmesh.c b/source/blender/blenkernel/intern/cdderivedmesh.c
index 204ff2a0369..e15e5bc9b45 100644
--- a/source/blender/blenkernel/intern/cdderivedmesh.c
+++ b/source/blender/blenkernel/intern/cdderivedmesh.c
@@ -74,6 +74,7 @@ typedef struct {
 
 	/* Cached */
 	struct PBVH *pbvh;
+	int pbvh_draw;
 	/* Mesh connectivity */
 	struct ListBase *fmap;
 	struct IndexNode *fmap_mem;
@@ -188,6 +189,7 @@ static ListBase *cdDM_getFaceMap(Object *ob, DerivedMesh *dm)
 static struct PBVH *cdDM_getPBVH(Object *ob, DerivedMesh *dm)
 {
 	CDDerivedMesh *cddm = (CDDerivedMesh*) dm;
+	Mesh *me= (ob)? ob->data: NULL;
 
 	if(!ob) {
 		cddm->pbvh= NULL;
@@ -196,13 +198,17 @@ static struct PBVH *cdDM_getPBVH(Object *ob, DerivedMesh *dm)
 
 	if(!ob->sculpt)
 		return NULL;
-	if(ob->sculpt->pbvh)
+	if(ob->sculpt->pbvh) {
 		cddm->pbvh= ob->sculpt->pbvh;
+		cddm->pbvh_draw = (cddm->mvert == me->mvert);
+	}
 
+	/* always build pbvh from original mesh, and only use it for drawing if
+	   this derivedmesh is just original mesh. it's the multires subsurf dm
+	   that this is actually for, to support a pbvh on a modified mesh */
 	if(!cddm->pbvh && ob->type == OB_MESH) {
-		Mesh *me= ob->data;
-
 		cddm->pbvh = BLI_pbvh_new();
+		cddm->pbvh_draw = (cddm->mvert == me->mvert);
 		BLI_pbvh_build_mesh(cddm->pbvh, me->mface, me->mvert,
 				   me->totface, me->totvert);
 	}
@@ -417,7 +423,7 @@ static void cdDM_drawFacesSolid(DerivedMesh *dm,
 	glVertex3fv(mvert[index].co);	\
 }
 
-	if(cddm->pbvh) {
+	if(cddm->pbvh && cddm->pbvh_draw) {
 		if(dm->numFaceData) {
 			float (*face_nors)[3] = CustomData_get_layer(&dm->faceData, CD_NORMAL);
 
diff --git a/source/blender/blenkernel/intern/multires.c b/source/blender/blenkernel/intern/multires.c
index ad069be50f5..ee8a74d6fbb 100644
--- a/source/blender/blenkernel/intern/multires.c
+++ b/source/blender/blenkernel/intern/multires.c
@@ -654,7 +654,9 @@ static void multiresModifier_update(DerivedMesh *dm)
 			int i, j, numGrids, highGridSize, lowGridSize;
 
 			/* create subsurf DM from original mesh at high level */
-			cddm = CDDM_from_mesh(me, NULL);
+			if (ob->derivedDeform) cddm = CDDM_copy(ob->derivedDeform);
+			else cddm = CDDM_from_mesh(me, NULL);
+
 			highdm = subsurf_dm_create_local(ob, cddm, totlvl, mmd->simple, 0);
 
 			/* create multires DM from original mesh and displacements */
@@ -705,7 +707,9 @@ static void multiresModifier_update(DerivedMesh *dm)
 		else {
 			DerivedMesh *cddm, *subdm;
 
-			cddm = CDDM_from_mesh(me, NULL);
+			if (ob->derivedDeform) cddm = CDDM_copy(ob->derivedDeform);
+			else cddm = CDDM_from_mesh(me, NULL);
+
 			subdm = subsurf_dm_create_local(ob, cddm, mmd->totlvl, mmd->simple, 0);
 			cddm->release(cddm);
 
diff --git a/source/blender/blenkernel/intern/subsurf_ccg.c b/source/blender/blenkernel/intern/subsurf_ccg.c
index 72236a76032..53206bb3970 100644
--- a/source/blender/blenkernel/intern/subsurf_ccg.c
+++ b/source/blender/blenkernel/intern/subsurf_ccg.c
@@ -44,6 +44,7 @@
 #include "BKE_cdderivedmesh.h"
 #include "BKE_global.h"
 #include "BKE_mesh.h"
+#include "BKE_modifier.h"
 #include "BKE_paint.h"
 #include "BKE_scene.h"
 #include "BKE_subsurf.h"
@@ -2229,10 +2230,28 @@ static ListBase *ccgDM_getFaceMap(Object *ob, DerivedMesh *dm)
 	return ccgdm->fmap;
 }
 
+static int ccgDM_use_grid_pbvh(CCGDerivedMesh *ccgdm)
+{
+	ModifierData *md;
+	MultiresModifierData *mmd= ccgdm->multires.mmd;
+
+	/* in sync with sculpt mode, only use multires grid pbvh if we are
+	   the last enabled modifier in the stack, otherwise we use the base
+	   mesh */
+	if(!mmd)
+		return 0;
+	
+	for(md=mmd->modifier.next; md; md= md->next)
+		if(modifier_isEnabled(mmd->modifier.scene, md, eModifierMode_Realtime))
+			return 0;
+	
+	return 1;
+}
+
 static struct PBVH *ccgDM_getPBVH(Object *ob, DerivedMesh *dm)
 {
 	CCGDerivedMesh *ccgdm= (CCGDerivedMesh*)dm;
-	int gridSize, numGrids;
+	int gridSize, numGrids, grid_pbvh;
 
 	if(!ob) {
 		ccgdm->pbvh= NULL;
@@ -2241,13 +2260,30 @@ static struct PBVH *ccgDM_getPBVH(Object *ob, DerivedMesh *dm)
 
 	if(!ob->sculpt)
 		return NULL;
-	if(ob->sculpt->pbvh)
-		ccgdm->pbvh= ob->sculpt->pbvh;
+
+	grid_pbvh = ccgDM_use_grid_pbvh(ccgdm);
+
+	if(ob->sculpt->pbvh) {
+		if(grid_pbvh) {
+			/* pbvh's grids, gridadj and gridfaces points to data inside ccgdm
+			   but this can be freed on ccgdm release, this updates the pointers
+			   when the ccgdm gets remade, the assumption is that the topology
+			   does not change. */
+			ccgdm_create_grids(dm);
+			BLI_pbvh_grids_update(ob->sculpt->pbvh, ccgdm->gridData, ccgdm->gridAdjacency, (void**)ccgdm->gridFaces);
+		}
+
+		ccgdm->pbvh = ob->sculpt->pbvh;
+		ccgdm->pbvh_draw = grid_pbvh;
+	}
 
 	if(ccgdm->pbvh)
 		return ccgdm->pbvh;
 
-	if(ccgdm->multires.mmd) {
+	/* no pbvh exists yet, we need to create one. only in case of multires
+	   we build a pbvh over the modified mesh, in other cases the base mesh
+	   is being sculpted, so we build a pbvh from that. */
+	if(grid_pbvh) {
 		ccgdm_create_grids(dm);
 
 		gridSize = ccgDM_getGridSize(dm);
@@ -2256,6 +2292,7 @@ static struct PBVH *ccgDM_getPBVH(Object *ob, DerivedMesh *dm)
 		ob->sculpt->pbvh= ccgdm->pbvh = BLI_pbvh_new();
 		BLI_pbvh_build_grids(ccgdm->pbvh, ccgdm->gridData, ccgdm->gridAdjacency,
 			numGrids, gridSize, (void**)ccgdm->gridFaces);
+		ccgdm->pbvh_draw = 1;
 	}
 	else if(ob->type == OB_MESH) {
 		Mesh *me= ob->data;
@@ -2263,6 +2300,7 @@ static struct PBVH *ccgDM_getPBVH(Object *ob, DerivedMesh *dm)
 		ob->sculpt->pbvh= ccgdm->pbvh = BLI_pbvh_new();
 		BLI_pbvh_build_mesh(ccgdm->pbvh, me->mface, me->mvert,
 				   me->totface, me->totvert);
+		ccgdm->pbvh_draw = 0;
 	}
 
 	return ccgdm->pbvh;