Todo issue: sculpting on deformed mesh

Used a crazyspace approach (like in edit mode), but only modifiers with
deformMatricies are allowed atm (currently shapekeys and armature modifiers only).
All the rest modifiers had an warning message that they aren't applied because
of sculpt mode. Deformation of multires is also unsupported.

With all this restictions users will always see the actual "layer" (or maybe
mesh state would be more correct word) they are sculpting on.

Internal changes:
- All modifiers could have deformMatricies callback (the same as deformMatriciesEM but
  for non-edit mode usage)
- Added function to build crazyspace for sculpting (sculpt_get_deform_matrices), but it
  could be generalized for usage in other painting modes (particle edit mode, i.e)

Todo:
- Implement crazyspace correction to support all kinds of deformation modifiers
- Maybe deformation of multires isn't so difficult?
- And maybe we could avoid extra bad-level-stub for ED_sculpt_modifiers_changed
  without code duplicating?

10 files changed, 125 insertions, 74 deletions

diff --git a/source/blender/blenkernel/BKE_DerivedMesh.h b/source/blender/blenkernel/BKE_DerivedMesh.h
index 5ce6c522e90..48127d3cf4c 100644
--- a/source/blender/blenkernel/BKE_DerivedMesh.h
+++ b/source/blender/blenkernel/BKE_DerivedMesh.h
@@ -508,6 +508,10 @@ void makeDerivedMesh(struct Scene *scene, struct Object *ob, struct EditMesh *em
 int editmesh_get_first_deform_matrices(struct Scene *, struct Object *, struct EditMesh *em,
 									   float (**deformmats)[3][3], float (**deformcos)[3]);
 
+/* returns an array of deform matrices for crazyspace correction when sculpting */
+void sculpt_get_deform_matrices(struct Scene *scene, struct Object *ob,
+								float (**deformmats)[3][3], float (**deformcos)[3]);
+
 void weight_to_rgb(float input, float *fr, float *fg, float *fb);
 
 /* convert layers requested by a GLSL material to actually available layers in
diff --git a/source/blender/blenkernel/BKE_modifier.h b/source/blender/blenkernel/BKE_modifier.h
index 97bef6ab565..5bef7527c87 100644
--- a/source/blender/blenkernel/BKE_modifier.h
+++ b/source/blender/blenkernel/BKE_modifier.h
@@ -134,6 +134,12 @@ typedef struct ModifierTypeInfo {
 						float (*vertexCos)[3], int numVerts,
 						int useRenderParams, int isFinalCalc);
 
+	/* Like deformMatricesEM but called from object mode (for supporting modifiers in sculpt mode) */
+	void (*deformMatrices)(
+				struct ModifierData *md, struct Object *ob,
+				struct DerivedMesh *derivedData,
+				float (*vertexCos)[3], float (*defMats)[3][3], int numVerts);
+
 	/* Like deformVerts but called during editmode (for supporting modifiers)
 	 */
 	void (*deformVertsEM)(
diff --git a/source/blender/blenkernel/BKE_multires.h b/source/blender/blenkernel/BKE_multires.h
index 5cafd24c1fd..23fabce68ba 100644
--- a/source/blender/blenkernel/BKE_multires.h
+++ b/source/blender/blenkernel/BKE_multires.h
@@ -53,6 +53,7 @@ struct DerivedMesh *multires_dm_create_from_derived(struct MultiresModifierData*
 
 struct MultiresModifierData *find_multires_modifier_before(struct Scene *scene,
 	struct ModifierData *lastmd);
+struct MultiresModifierData *get_multires_modifier(struct Scene *scene, struct Object *ob, int use_first);
 struct DerivedMesh *get_multires_dm(struct Scene *scene, struct MultiresModifierData *mmd,
 				struct Object *ob);
 void multiresModifier_del_levels(struct MultiresModifierData *, struct Object *, int direction);
diff --git a/source/blender/blenkernel/BKE_paint.h b/source/blender/blenkernel/BKE_paint.h
index 20742033a2e..2203277b301 100644
--- a/source/blender/blenkernel/BKE_paint.h
+++ b/source/blender/blenkernel/BKE_paint.h
@@ -78,8 +78,11 @@ typedef struct SculptSession {
 	/* PBVH acceleration structure */
 	struct PBVH *pbvh;
 
-	/* Used temporarily per-stroke */
-	float *vertexcosnos;
+	/* Paiting on deformed mesh */
+	int modifiers_active; /* object is deformed with some modifiers */
+	float (*orig_cos)[3]; /* coords of undeformed mesh */
+	float (*deform_cos)[3]; /* coords of deformed mesh but without stroke displacement */
+	float (*deform_imats)[3][3]; /* crazyspace deformation matricies */
 
 	/* Partial redraw */
 	int partial_redraw;
@@ -95,8 +98,6 @@ typedef struct SculptSession {
 
 	struct GPUDrawObject *drawobject;
 
-	int modifiers_active;
-
 	rcti previous_r;
 } SculptSession;
 
diff --git a/source/blender/blenkernel/BKE_subsurf.h b/source/blender/blenkernel/BKE_subsurf.h
index 940a0027d0b..ef3ad3ad2e9 100644
--- a/source/blender/blenkernel/BKE_subsurf.h
+++ b/source/blender/blenkernel/BKE_subsurf.h
@@ -72,7 +72,6 @@ typedef struct CCGDerivedMesh {
 	char *faceFlags;
 
 	struct PBVH *pbvh;
-	int pbvh_draw;
 	struct ListBase *fmap;
 	struct IndexNode *fmap_mem;
 
diff --git a/source/blender/blenkernel/intern/DerivedMesh.c b/source/blender/blenkernel/intern/DerivedMesh.c
index a2ef625c6e6..658d29f0046 100644
--- a/source/blender/blenkernel/intern/DerivedMesh.c
+++ b/source/blender/blenkernel/intern/DerivedMesh.c
@@ -53,6 +53,7 @@
 #include "BKE_object.h"
 #include "BKE_paint.h"
 #include "BKE_texture.h"
+#include "BKE_multires.h"
 
 
 #include "BLO_sys_types.h" // for intptr_t support
@@ -65,6 +66,8 @@
 #include "GPU_extensions.h"
 #include "GPU_material.h"
 
+#include "ED_sculpt.h" /* for ED_sculpt_modifiers_changed */
+
 ///////////////////////////////////
 ///////////////////////////////////
 
@@ -1676,6 +1679,12 @@ static void mesh_calc_modifiers(Scene *scene, Object *ob, float (*inputVertexCos
 	int required_mode;
 	int isPrevDeform= FALSE;
 	int skipVirtualArmature = (useDeform < 0);
+	MultiresModifierData *mmd= get_multires_modifier(scene, ob, 0);
+	int has_multires = mmd != NULL, multires_applied = 0;
+	int sculpt_mode = ob->mode & OB_MODE_SCULPT && ob->sculpt;
+
+	if(mmd && !mmd->sculptlvl)
+		has_multires = 0;
 
 	if(!skipVirtualArmature) {
 		firstmd = modifiers_getVirtualModifierList(ob);
@@ -1714,6 +1723,12 @@ static void mesh_calc_modifiers(Scene *scene, Object *ob, float (*inputVertexCos
 			if(useDeform < 0 && mti->dependsOnTime && mti->dependsOnTime(md)) continue;
 
 			if(mti->type == eModifierTypeType_OnlyDeform) {
+				if(sculpt_mode && !has_multires)
+					if(!ELEM(md->type, eModifierType_Armature, eModifierType_ShapeKey)) {
+						modifier_setError(md, "Not supported in sculpt mode.");
+						continue;
+					}
+
 				if(!deformedVerts)
 					deformedVerts = mesh_getVertexCos(me, &numVerts);
 
@@ -1759,13 +1774,18 @@ static void mesh_calc_modifiers(Scene *scene, Object *ob, float (*inputVertexCos
 		ModifierTypeInfo *mti = modifierType_getInfo(md->type);
 
 		md->scene= scene;
-		
+
 		if(!modifier_isEnabled(scene, md, required_mode)) continue;
 		if(mti->type == eModifierTypeType_OnlyDeform && !useDeform) continue;
 		if((mti->flags & eModifierTypeFlag_RequiresOriginalData) && dm) {
 			modifier_setError(md, "Modifier requires original data, bad stack position.");
 			continue;
 		}
+		if(sculpt_mode && (!has_multires || multires_applied))
+			if(md->type != eModifierType_Armature || multires_applied) {
+				modifier_setError(md, "Not supported in sculpt mode.");
+				continue;
+			}
 		if(needMapping && !modifier_supportsMapping(md)) continue;
 		if(useDeform < 0 && mti->dependsOnTime && mti->dependsOnTime(md)) continue;
 
@@ -1928,6 +1948,9 @@ static void mesh_calc_modifiers(Scene *scene, Object *ob, float (*inputVertexCos
 		/* grab modifiers until index i */
 		if((index >= 0) && (modifiers_indexInObject(ob, md) >= index))
 			break;
+
+		if(sculpt_mode && md->type == eModifierType_Multires)
+			multires_applied = 1;
 	}
 
 	for(md=firstmd; md; md=md->next)
@@ -2222,14 +2245,9 @@ static void clear_mesh_caches(Object *ob)
 		ob->derivedDeform->release(ob->derivedDeform);
 		ob->derivedDeform= NULL;
 	}
-	/* we free pbvh on changes, except during sculpt since it can't deal with
-	   changing PVBH node organization, we hope topology does not change in
-	   the meantime .. weak */
-	if(ob->sculpt && ob->sculpt->pbvh) {
-		if(!ob->sculpt->cache) {
-			BLI_pbvh_free(ob->sculpt->pbvh);
-			ob->sculpt->pbvh= NULL;
-		}
+
+	if(ob->sculpt) {
+		ED_sculpt_modifiers_changed(ob);
 	}
 }
 
@@ -2523,6 +2541,51 @@ int editmesh_get_first_deform_matrices(Scene *scene, Object *ob, EditMesh *em, f
 	return numleft;
 }
 
+void sculpt_get_deform_matrices(Scene *scene, Object *ob, float (**deformmats)[3][3], float (**deformcos)[3])
+{
+	ModifierData *md;
+	DerivedMesh *dm;
+	int a, numVerts= 0;
+	float (*defmats)[3][3]= NULL, (*deformedVerts)[3]= NULL;
+	MultiresModifierData *mmd= get_multires_modifier(scene, ob, 0);
+	int has_multires = mmd != NULL && mmd->sculptlvl > 0;
+
+	if(has_multires) {
+		*deformmats= NULL;
+		*deformcos= NULL;
+		return;
+	}
+
+	dm= NULL;
+	md= modifiers_getVirtualModifierList(ob);
+
+	for(; md; md= md->next) {
+		ModifierTypeInfo *mti= modifierType_getInfo(md->type);
+
+		if(!modifier_isEnabled(scene, md, eModifierMode_Realtime)) continue;
+
+		if(mti->type==eModifierTypeType_OnlyDeform && mti->deformMatrices) {
+			if(!defmats) {
+				Mesh *me= (Mesh*)ob->data;
+				dm= getMeshDerivedMesh(me, ob, NULL);
+				deformedVerts= mesh_getVertexCos(me, &numVerts);
+				defmats= MEM_callocN(sizeof(*defmats)*numVerts, "defmats");
+
+				for(a=0; a<numVerts; a++)
+					unit_m3(defmats[a]);
+			}
+
+			mti->deformMatrices(md, ob, dm, deformedVerts, defmats, numVerts);
+		}
+	}
+
+	if(dm)
+		dm->release(dm);
+
+	*deformmats= defmats;
+	*deformcos= deformedVerts;
+}
+
 /* ******************* GLSL ******************** */
 
 void DM_add_tangent_layer(DerivedMesh *dm)
diff --git a/source/blender/blenkernel/intern/cdderivedmesh.c b/source/blender/blenkernel/intern/cdderivedmesh.c
index 71e704fe6c8..932be711938 100644
--- a/source/blender/blenkernel/intern/cdderivedmesh.c
+++ b/source/blender/blenkernel/intern/cdderivedmesh.c
@@ -76,6 +76,7 @@ typedef struct {
 	/* Cached */
 	struct PBVH *pbvh;
 	int pbvh_draw;
+
 	/* Mesh connectivity */
 	struct ListBase *fmap;
 	struct IndexNode *fmap_mem;
@@ -222,6 +223,17 @@ static struct PBVH *cdDM_getPBVH(Object *ob, DerivedMesh *dm)
 		cddm->pbvh_draw = can_pbvh_draw(ob, dm);
 		BLI_pbvh_build_mesh(cddm->pbvh, me->mface, me->mvert,
 				   me->totface, me->totvert);
+
+		if(ob->sculpt->modifiers_active) {
+			float (*vertCos)[3];
+			int totvert;
+
+			totvert= dm->getNumVerts(dm);
+			vertCos= MEM_callocN(3*totvert*sizeof(float), "cdDM_getPBVH vertCos");
+			dm->getVertCos(dm, vertCos);
+			BLI_pbvh_apply_vertCos(cddm->pbvh, vertCos);
+			MEM_freeN(vertCos);
+		}
 	}
 
 	return cddm->pbvh;
diff --git a/source/blender/blenkernel/intern/multires.c b/source/blender/blenkernel/intern/multires.c
index c67abc47e6f..2a3052d10c9 100644
--- a/source/blender/blenkernel/intern/multires.c
+++ b/source/blender/blenkernel/intern/multires.c
@@ -92,8 +92,10 @@ MultiresModifierData *find_multires_modifier_before(Scene *scene, ModifierData *
 	return NULL;
 }
 
-/* used for applying scale on mdisps layer and syncing subdivide levels when joining objects */
-static MultiresModifierData *get_multires_modifier(Scene *scene, Object *ob)
+/* used for applying scale on mdisps layer and syncing subdivide levels when joining objects
+   use_first - return first multires modifier if all multires'es are disabled
+*/
+MultiresModifierData *get_multires_modifier(Scene *scene, Object *ob, int use_first)
 {
 	ModifierData *md;
 	MultiresModifierData *mmd= NULL, *firstmmd= NULL;
@@ -111,7 +113,7 @@ static MultiresModifierData *get_multires_modifier(Scene *scene, Object *ob)
 		}
 	}
 
-	if(!mmd) {
+	if(!mmd && use_first) {
 		/* active multires have not been found
 		   try to use first one */
 		return firstmmd;
@@ -1568,8 +1570,8 @@ void multires_load_old(Object *ob, Mesh *me)
 
 static void multires_sync_levels(Scene *scene, Object *ob, Object *to_ob)
 {
-	MultiresModifierData *mmd= get_multires_modifier(scene, ob);
-	MultiresModifierData *to_mmd= get_multires_modifier(scene, to_ob);
+	MultiresModifierData *mmd= get_multires_modifier(scene, ob, 1);
+	MultiresModifierData *to_mmd= get_multires_modifier(scene, to_ob, 1);
 
 	if(!mmd) {
 		/* object could have MDISP even when there is no multires modifier
@@ -1599,7 +1601,7 @@ void multires_apply_smat(Scene *scene, Object *ob, float smat[3][3])
 	int *gridOffset;
 	int i, /*numGrids,*/ gridSize, dGridSize, dSkip, totvert;
 	float (*vertCos)[3] = NULL;
-	MultiresModifierData *mmd= get_multires_modifier(scene, ob);
+	MultiresModifierData *mmd= get_multires_modifier(scene, ob, 1);
 	MultiresModifierData high_mmd;
 
 	CustomData_external_read(&me->fdata, &me->id, CD_MASK_MDISPS, me->totface);
@@ -1725,7 +1727,7 @@ void multires_topology_changed(Scene *scene, Object *ob)
 	Mesh *me= (Mesh*)ob->data;
 	MDisps *mdisp= NULL, *cur= NULL;
 	int i, grid= 0, corners;
-	MultiresModifierData *mmd= get_multires_modifier(scene, ob);
+	MultiresModifierData *mmd= get_multires_modifier(scene, ob, 1);
 
 	if(mmd)
 		multires_set_tot_mdisps(me, mmd->totlvl);
diff --git a/source/blender/blenkernel/intern/object.c b/source/blender/blenkernel/intern/object.c
index 9df55b825b7..0d3fb7107f0 100644
--- a/source/blender/blenkernel/intern/object.c
+++ b/source/blender/blenkernel/intern/object.c
@@ -246,6 +246,13 @@ void free_sculptsession(Object *ob)
 		if(ss->layer_co)
 			MEM_freeN(ss->layer_co);
 
+		if(ss->orig_cos)
+			MEM_freeN(ss->orig_cos);
+		if(ss->deform_cos)
+			MEM_freeN(ss->deform_cos);
+		if(ss->deform_imats)
+			MEM_freeN(ss->deform_imats);
+
 		MEM_freeN(ss);
 
 		ob->sculpt = NULL;
diff --git a/source/blender/blenkernel/intern/subsurf_ccg.c b/source/blender/blenkernel/intern/subsurf_ccg.c
index e75dc27c7b5..a2d3016099d 100644
--- a/source/blender/blenkernel/intern/subsurf_ccg.c
+++ b/source/blender/blenkernel/intern/subsurf_ccg.c
@@ -70,8 +70,6 @@ static int ccgDM_getVertMapIndex(CCGSubSurf *ss, CCGVert *v);
 static int ccgDM_getEdgeMapIndex(CCGSubSurf *ss, CCGEdge *e);
 static int ccgDM_getFaceMapIndex(CCGSubSurf *ss, CCGFace *f);
 
-static int ccgDM_use_grid_pbvh(CCGDerivedMesh *ccgdm);
-
 ///
 
 static void *arena_alloc(CCGAllocatorHDL a, int numBytes) {
@@ -1150,7 +1148,7 @@ static void ccgDM_drawVerts(DerivedMesh *dm) {
 
 static void ccgdm_pbvh_update(CCGDerivedMesh *ccgdm)
 {
-	if(ccgdm->pbvh && ccgDM_use_grid_pbvh(ccgdm)) {
+	if(ccgdm->pbvh) {
 		CCGFace **faces;
 		int totface;
 
@@ -2240,28 +2238,10 @@ static ListBase *ccgDM_getFaceMap(Object *ob, DerivedMesh *dm)
 	return ccgdm->fmap;
 }
 
-static int ccgDM_use_grid_pbvh(CCGDerivedMesh *ccgdm)
-{
-	ModifierData *md;
-	MultiresModifierData *mmd= ccgdm->multires.mmd;
-
-	/* in sync with sculpt mode, only use multires grid pbvh if we are
-	   the last enabled modifier in the stack, otherwise we use the base
-	   mesh */
-	if(!mmd)
-		return 0;
-	
-	for(md=mmd->modifier.next; md; md= md->next)
-		if(modifier_isEnabled(mmd->modifier.scene, md, eModifierMode_Realtime))
-			return 0;
-	
-	return 1;
-}
-
 static struct PBVH *ccgDM_getPBVH(Object *ob, DerivedMesh *dm)
 {
 	CCGDerivedMesh *ccgdm= (CCGDerivedMesh*)dm;
-	int gridSize, numGrids, grid_pbvh;
+	int gridSize, numGrids;
 
 	if(!ob) {
 		ccgdm->pbvh= NULL;
@@ -2271,21 +2251,8 @@ static struct PBVH *ccgDM_getPBVH(Object *ob, DerivedMesh *dm)
 	if(!ob->sculpt)
 		return NULL;
 
-	grid_pbvh = ccgDM_use_grid_pbvh(ccgdm);
-
-	if(ob->sculpt->pbvh) {
-		if(grid_pbvh) {
-			/* pbvh's grids, gridadj and gridfaces points to data inside ccgdm
-			   but this can be freed on ccgdm release, this updates the pointers
-			   when the ccgdm gets remade, the assumption is that the topology
-			   does not change. */
-			ccgdm_create_grids(dm);
-			BLI_pbvh_grids_update(ob->sculpt->pbvh, ccgdm->gridData, ccgdm->gridAdjacency, (void**)ccgdm->gridFaces);
-		}
-
+	if(ob->sculpt->pbvh)
 		ccgdm->pbvh = ob->sculpt->pbvh;
-		ccgdm->pbvh_draw = grid_pbvh;
-	}
 
 	if(ccgdm->pbvh)
 		return ccgdm->pbvh;
@@ -2293,25 +2260,14 @@ static struct PBVH *ccgDM_getPBVH(Object *ob, DerivedMesh *dm)
 	/* no pbvh exists yet, we need to create one. only in case of multires
 	   we build a pbvh over the modified mesh, in other cases the base mesh
 	   is being sculpted, so we build a pbvh from that. */
-	if(grid_pbvh) {
-		ccgdm_create_grids(dm);
-
-		gridSize = ccgDM_getGridSize(dm);
-		numGrids = ccgDM_getNumGrids(dm);
+	ccgdm_create_grids(dm);
 
-		ob->sculpt->pbvh= ccgdm->pbvh = BLI_pbvh_new();
-		BLI_pbvh_build_grids(ccgdm->pbvh, ccgdm->gridData, ccgdm->gridAdjacency,
-			numGrids, gridSize, (void**)ccgdm->gridFaces);
-		ccgdm->pbvh_draw = 1;
-	}
-	else if(ob->type == OB_MESH) {
-		Mesh *me= ob->data;
+	gridSize = ccgDM_getGridSize(dm);
+	numGrids = ccgDM_getNumGrids(dm);
 
-		ob->sculpt->pbvh= ccgdm->pbvh = BLI_pbvh_new();
-		BLI_pbvh_build_mesh(ccgdm->pbvh, me->mface, me->mvert,
-				   me->totface, me->totvert);
-		ccgdm->pbvh_draw = 0;
-	}
+	ob->sculpt->pbvh= ccgdm->pbvh = BLI_pbvh_new();
+	BLI_pbvh_build_grids(ccgdm->pbvh, ccgdm->gridData, ccgdm->gridAdjacency,
+		numGrids, gridSize, (void**)ccgdm->gridFaces);
 
 	return ccgdm->pbvh;
 }