Sculpt Mode Bugfixes:

* #20833: layer brush doesn't work with multires.
* #20946: sculpt mode partially removes parts of the mesh in the viewport.
* #20420: grab brush stops after moving some distance.
* #20906: sculpt grab tool moves in wrong direction.
* #21132 and #21272: undo on object with subdivision surface modifier crashes.
* #21115: subsurf + multires + sculpting + undo causes crash.
* #20683: sculpt + multires apply + undo crash.
* #19094: wrong outline in solid mode.

9 files changed, 104 insertions, 41 deletions

diff --git a/source/blender/blenkernel/BKE_DerivedMesh.h b/source/blender/blenkernel/BKE_DerivedMesh.h
index cb87638af44..965bad31991 100644
--- a/source/blender/blenkernel/BKE_DerivedMesh.h
+++ b/source/blender/blenkernel/BKE_DerivedMesh.h
@@ -227,7 +227,7 @@ struct DerivedMesh {
 	 *
 	 * Also called for *final* editmode DerivedMeshes
 	 */
-	void (*drawEdges)(DerivedMesh *dm, int drawLooseEdges);
+	void (*drawEdges)(DerivedMesh *dm, int drawLooseEdges, int drawAllEdges);
 	
 	/* Draw all loose edges (edges w/ no adjoining faces) */
 	void (*drawLooseEdges)(DerivedMesh *dm);
diff --git a/source/blender/blenkernel/BKE_paint.h b/source/blender/blenkernel/BKE_paint.h
index 8fd4f079ebf..81fb724b3a5 100644
--- a/source/blender/blenkernel/BKE_paint.h
+++ b/source/blender/blenkernel/BKE_paint.h
@@ -69,13 +69,15 @@ typedef struct SculptSession {
 	struct MFace *mface;
 	int totvert, totface;
 	float *face_normals;
-	struct PBVH *tree;
 	struct Object *ob;
 	struct KeyBlock *kb, *refkb;
 	
 	/* Mesh connectivity */
 	struct ListBase *fmap;
 
+	/* PBVH acceleration structure */
+	struct PBVH *pbvh;
+
 	/* Used temporarily per-stroke */
 	float *vertexcosnos;
 
@@ -87,7 +89,6 @@ typedef struct SculptSession {
 
 	/* Layer brush persistence between strokes */
 	 float (*layer_co)[3]; /* Copy of the mesh vertices' locations */
-	float *layer_disps; /* Displacements for each vertex */
 
 	struct SculptStroke *stroke;
 	struct StrokeCache *cache;
@@ -95,6 +96,6 @@ typedef struct SculptSession {
 	struct GPUDrawObject *drawobject;
 } SculptSession;
 
-void free_sculptsession(SculptSession **);
+void free_sculptsession(struct Object *ob);
 
 #endif
diff --git a/source/blender/blenkernel/BKE_subsurf.h b/source/blender/blenkernel/BKE_subsurf.h
index fca7c79d96b..853fd99aa86 100644
--- a/source/blender/blenkernel/BKE_subsurf.h
+++ b/source/blender/blenkernel/BKE_subsurf.h
@@ -28,19 +28,21 @@
 #ifndef BKE_SUBSURF_H
 #define BKE_SUBSURF_H
 
-struct Mesh;
-struct Object;
+struct DMGridAdjacency;
+struct DMGridData;
 struct DerivedMesh;
 struct EditMesh;
+struct IndexNode;
+struct ListBase;
+struct Mesh;
 struct MultiresSubsurf;
+struct Object;
+struct PBVH;
 struct SubsurfModifierData;
-struct _CCGSubsurf;
-struct _CCGVert;
 struct _CCGEdge;
 struct _CCGFace;
-struct PBVH;
-struct DMGridData;
-struct DMGridAdjacency;
+struct _CCGSubsurf;
+struct _CCGVert;
 
 /**************************** External *****************************/
 
@@ -70,6 +72,8 @@ typedef struct CCGDerivedMesh {
 	char *faceFlags;
 
 	struct PBVH *pbvh;
+	struct ListBase *fmap;
+	struct IndexNode *fmap_mem;
 
 	struct DMGridData **gridData;
 	struct DMGridAdjacency *gridAdjacency;
diff --git a/source/blender/blenkernel/intern/DerivedMesh.c b/source/blender/blenkernel/intern/DerivedMesh.c
index 6908d987a36..9d15ac3f348 100644
--- a/source/blender/blenkernel/intern/DerivedMesh.c
+++ b/source/blender/blenkernel/intern/DerivedMesh.c
@@ -41,10 +41,11 @@
 #include "DNA_object_types.h"
 #include "DNA_scene_types.h" // N_T
 
-#include "BLI_math.h"
 #include "BLI_blenlib.h"
 #include "BLI_editVert.h"
+#include "BLI_math.h"
 #include "BLI_memarena.h"
+#include "BLI_pbvh.h"
 
 #include "BKE_cdderivedmesh.h"
 #include "BKE_displist.h"
@@ -509,7 +510,7 @@ static void emDM_drawMappedEdges(DerivedMesh *dm, int (*setDrawOptions)(void *us
 		glEnd();
 	}
 }
-static void emDM_drawEdges(DerivedMesh *dm, int drawLooseEdges)
+static void emDM_drawEdges(DerivedMesh *dm, int drawLooseEdges, int drawAllEdges)
 {
 	emDM_drawMappedEdges(dm, NULL, NULL);
 }
@@ -2094,6 +2095,15 @@ static void clear_mesh_caches(Object *ob)
 		ob->derivedDeform->release(ob->derivedDeform);
 		ob->derivedDeform= NULL;
 	}
+	/* we free pbvh on changes, except during sculpt since it can't deal with
+	   changing PVBH node organization, we hope topology does not change in
+	   the meantime .. weak */
+	if(ob->sculpt && ob->sculpt->pbvh) {
+		if(!ob->sculpt->cache) {
+			BLI_pbvh_free(ob->sculpt->pbvh);
+			ob->sculpt->pbvh= NULL;
+		}
+	}
 }
 
 static void mesh_build_data(Scene *scene, Object *ob, CustomDataMask dataMask)
diff --git a/source/blender/blenkernel/intern/cdderivedmesh.c b/source/blender/blenkernel/intern/cdderivedmesh.c
index cca554b5880..6399d55150a 100644
--- a/source/blender/blenkernel/intern/cdderivedmesh.c
+++ b/source/blender/blenkernel/intern/cdderivedmesh.c
@@ -39,12 +39,13 @@
 #include "BKE_cdderivedmesh.h"
 #include "BKE_global.h"
 #include "BKE_mesh.h"
+#include "BKE_paint.h"
 #include "BKE_utildefines.h"
 
-#include "BLI_math.h"
 #include "BLI_blenlib.h"
 #include "BLI_edgehash.h"
 #include "BLI_editVert.h"
+#include "BLI_math.h"
 #include "BLI_pbvh.h"
 
 #include "DNA_meshdata_types.h"
@@ -188,6 +189,16 @@ static struct PBVH *cdDM_getPBVH(Object *ob, DerivedMesh *dm)
 {
 	CDDerivedMesh *cddm = (CDDerivedMesh*) dm;
 
+	if(!ob) {
+		cddm->pbvh= NULL;
+		return NULL;
+	}
+
+	if(!ob->sculpt)
+		return NULL;
+	if(ob->sculpt->pbvh)
+		cddm->pbvh= ob->sculpt->pbvh;
+
 	if(!cddm->pbvh && ob->type == OB_MESH) {
 		Mesh *me= ob->data;
 
@@ -290,7 +301,7 @@ static void cdDM_drawUVEdges(DerivedMesh *dm)
 	}
 }
 
-static void cdDM_drawEdges(DerivedMesh *dm, int drawLooseEdges)
+static void cdDM_drawEdges(DerivedMesh *dm, int drawLooseEdges, int drawAllEdges)
 {
 	CDDerivedMesh *cddm = (CDDerivedMesh*) dm;
 	MVert *mvert = cddm->mvert;
@@ -301,7 +312,7 @@ static void cdDM_drawEdges(DerivedMesh *dm, int drawLooseEdges)
 		DEBUG_VBO( "Using legacy code. cdDM_drawEdges\n" );
 		glBegin(GL_LINES);
 		for(i = 0; i < dm->numEdgeData; i++, medge++) {
-			if((medge->flag&ME_EDGEDRAW)
+			if((drawAllEdges || (medge->flag&ME_EDGEDRAW))
 			   && (drawLooseEdges || !(medge->flag&ME_LOOSEEDGE))) {
 				glVertex3fv(mvert[medge->v1].co);
 				glVertex3fv(mvert[medge->v2].co);
@@ -317,7 +328,7 @@ static void cdDM_drawEdges(DerivedMesh *dm, int drawLooseEdges)
 		GPU_edge_setup(dm);
 		if( !GPU_buffer_legacy(dm) ) {
 			for(i = 0; i < dm->numEdgeData; i++, medge++) {
-				if((medge->flag&ME_EDGEDRAW)
+				if((drawAllEdges || (medge->flag&ME_EDGEDRAW))
 				   && (drawLooseEdges || !(medge->flag&ME_LOOSEEDGE))) {
 					draw = 1;
 				} 
@@ -1354,7 +1365,6 @@ static void cdDM_foreachMappedFaceCenter(
 
 static void cdDM_free_internal(CDDerivedMesh *cddm)
 {
-	if(cddm->pbvh) BLI_pbvh_free(cddm->pbvh);
 	if(cddm->fmap) MEM_freeN(cddm->fmap);
 	if(cddm->fmap_mem) MEM_freeN(cddm->fmap_mem);
 }
diff --git a/source/blender/blenkernel/intern/modifier.c b/source/blender/blenkernel/intern/modifier.c
index ad1fb80801f..e0aa47eac88 100644
--- a/source/blender/blenkernel/intern/modifier.c
+++ b/source/blender/blenkernel/intern/modifier.c
@@ -68,6 +68,7 @@
 #include "BKE_mesh.h"
 #include "BKE_modifier.h"
 #include "BKE_object.h"
+#include "BKE_paint.h"
 #include "BKE_particle.h"
 #include "BKE_pointcache.h"
 #include "BKE_scene.h"
@@ -9064,10 +9065,10 @@ static DerivedMesh *multiresModifier_applyModifier(ModifierData *md, Object *ob,
 		result->release(result);
 		result= cddm;
 	}
-	else if(ob->mode & OB_MODE_SCULPT) {
+	else if((ob->mode & OB_MODE_SCULPT) && ob->sculpt) {
 		/* would be created on the fly too, just nicer this
 		   way on first stroke after e.g. switching levels */
-		result->getPBVH(ob, result);
+		ob->sculpt->pbvh= result->getPBVH(ob, result);
 	}
 
 	return result;
diff --git a/source/blender/blenkernel/intern/multires.c b/source/blender/blenkernel/intern/multires.c
index 0d508a4c4d1..c70d12bcb75 100644
--- a/source/blender/blenkernel/intern/multires.c
+++ b/source/blender/blenkernel/intern/multires.c
@@ -34,14 +34,15 @@
 #include "DNA_object_types.h"
 #include "DNA_scene_types.h"
 
-#include "BLI_math.h"
 #include "BLI_blenlib.h"
+#include "BLI_math.h"
 #include "BLI_pbvh.h"
 
 #include "BKE_cdderivedmesh.h"
 #include "BKE_mesh.h"
 #include "BKE_modifier.h"
 #include "BKE_multires.h"
+#include "BKE_paint.h"
 #include "BKE_scene.h"
 #include "BKE_subsurf.h"
 #include "BKE_utildefines.h"
@@ -109,11 +110,16 @@ void multires_mark_as_modified(Object *ob)
 
 void multires_force_update(Object *ob)
 {
-
-	if(ob && ob->derivedFinal) {
-		ob->derivedFinal->needsFree =1;
-		ob->derivedFinal->release(ob->derivedFinal);
-		ob->derivedFinal = NULL;
+	if(ob) {
+		if(ob->derivedFinal) {
+			ob->derivedFinal->needsFree =1;
+			ob->derivedFinal->release(ob->derivedFinal);
+			ob->derivedFinal = NULL;
+		}
+		if(ob->sculpt && ob->sculpt->pbvh) {
+			BLI_pbvh_free(ob->sculpt->pbvh);
+			ob->sculpt->pbvh= NULL;
+		}
 	}
 }
 
diff --git a/source/blender/blenkernel/intern/object.c b/source/blender/blenkernel/intern/object.c
index b1cb83c0a51..64abffa5119 100644
--- a/source/blender/blenkernel/intern/object.c
+++ b/source/blender/blenkernel/intern/object.c
@@ -58,8 +58,9 @@
 #include "DNA_world_types.h"
 
 #include "BLI_blenlib.h"
-#include "BLI_math.h"
 #include "BLI_editVert.h"
+#include "BLI_math.h"
+#include "BLI_pbvh.h"
 
 #include "BKE_utildefines.h"
 
@@ -231,23 +232,26 @@ void object_free_display(Object *ob)
 	freedisplist(&ob->disp);
 }
 
-void free_sculptsession(SculptSession **ssp)
+void free_sculptsession(Object *ob)
 {
-	if(ssp && *ssp) {
-		SculptSession *ss = *ssp;
+	if(ob && ob->sculpt) {
+		SculptSession *ss = ob->sculpt;
+		DerivedMesh *dm= ob->derivedFinal;
+
+		if(ss->pbvh)
+			BLI_pbvh_free(ss->pbvh);
+		if(dm && dm->getPBVH)
+			dm->getPBVH(NULL, dm); /* signal to clear */
 
 		if(ss->texcache)
 			MEM_freeN(ss->texcache);
 
-		if(ss->layer_disps)
-			MEM_freeN(ss->layer_disps);
-
 		if(ss->layer_co)
 			MEM_freeN(ss->layer_co);
 
 		MEM_freeN(ss);
 
-		*ssp = NULL;
+		ob->sculpt = NULL;
 	}
 }
 
@@ -306,7 +310,7 @@ void free_object(Object *ob)
 	if(ob->bsoft) bsbFree(ob->bsoft);
 	if(ob->gpulamp.first) GPU_lamp_free(ob);
 
-	free_sculptsession(&ob->sculpt);
+	free_sculptsession(ob);
 
 	if(ob->pc_ids.first) BLI_freelistN(&ob->pc_ids);
 }
diff --git a/source/blender/blenkernel/intern/subsurf_ccg.c b/source/blender/blenkernel/intern/subsurf_ccg.c
index 937351b4992..0cc574984b8 100644
--- a/source/blender/blenkernel/intern/subsurf_ccg.c
+++ b/source/blender/blenkernel/intern/subsurf_ccg.c
@@ -42,11 +42,12 @@
 #include "DNA_scene_types.h"
 
 #include "BKE_cdderivedmesh.h"
-#include "BKE_utildefines.h"
 #include "BKE_global.h"
 #include "BKE_mesh.h"
+#include "BKE_paint.h"
 #include "BKE_scene.h"
 #include "BKE_subsurf.h"
+#include "BKE_utildefines.h"
 
 #include "BLI_blenlib.h"
 #include "BLI_edgehash.h"
@@ -1114,7 +1115,7 @@ static void ccgDM_drawVerts(DerivedMesh *dm) {
 	ccgFaceIterator_free(fi);
 	glEnd();
 }
-static void ccgDM_drawEdges(DerivedMesh *dm, int drawLooseEdges) {
+static void ccgDM_drawEdges(DerivedMesh *dm, int drawLooseEdges, int drawAllEdges) {
 	CCGDerivedMesh *ccgdm = (CCGDerivedMesh*) dm;
 	CCGSubSurf *ss = ccgdm->ss;
 	CCGEdgeIterator *ei = ccgSubSurf_getEdgeIterator(ss);
@@ -1219,7 +1220,7 @@ static void ccgDM_glNormalFast(float *a, float *b, float *c, float *d)
 
 static void ccgdm_pbvh_update(CCGDerivedMesh *ccgdm)
 {
-	if(ccgdm->pbvh) {
+	if(ccgdm->pbvh && ccgdm->multires.mmd) {
 		CCGFace **faces;
 		int totface;
 
@@ -1935,12 +1936,13 @@ static void ccgDM_release(DerivedMesh *dm) {
 				ccgdm->multires.update(dm);
 		}
 
-		if(ccgdm->pbvh) BLI_pbvh_free(ccgdm->pbvh);
 		if(ccgdm->gridFaces) MEM_freeN(ccgdm->gridFaces);
 		if(ccgdm->gridData) MEM_freeN(ccgdm->gridData);
 		if(ccgdm->gridAdjacency) MEM_freeN(ccgdm->gridAdjacency);
 		if(ccgdm->gridOffset) MEM_freeN(ccgdm->gridOffset);
 		if(ccgdm->freeSS) ccgSubSurf_free(ccgdm->ss);
+		if(ccgdm->fmap) MEM_freeN(ccgdm->fmap);
+		if(ccgdm->fmap_mem) MEM_freeN(ccgdm->fmap_mem);
 		MEM_freeN(ccgdm->edgeFlags);
 		MEM_freeN(ccgdm->faceFlags);
 		MEM_freeN(ccgdm->vertMap);
@@ -2185,11 +2187,35 @@ static int *ccgDM_getGridOffset(DerivedMesh *dm)
 	return ccgdm->gridOffset;
 }
 
+static ListBase *ccgDM_getFaceMap(Object *ob, DerivedMesh *dm)
+{
+	CCGDerivedMesh *ccgdm= (CCGDerivedMesh*)dm;
+
+	if(!ccgdm->multires.mmd && !ccgdm->fmap && ob->type == OB_MESH) {
+		Mesh *me= ob->data;
+
+		create_vert_face_map(&ccgdm->fmap, &ccgdm->fmap_mem, me->mface,
+				     me->totvert, me->totface);
+	}
+
+	return ccgdm->fmap;
+}
+
 static struct PBVH *ccgDM_getPBVH(Object *ob, DerivedMesh *dm)
 {
 	CCGDerivedMesh *ccgdm= (CCGDerivedMesh*)dm;
 	int gridSize, numGrids;
 
+	if(!ob) {
+		ccgdm->pbvh= NULL;
+		return NULL;
+	}
+
+	if(!ob->sculpt)
+		return NULL;
+	if(ob->sculpt->pbvh)
+		ccgdm->pbvh= ob->sculpt->pbvh;
+
 	if(ccgdm->pbvh)
 		return ccgdm->pbvh;
 
@@ -2199,14 +2225,14 @@ static struct PBVH *ccgDM_getPBVH(Object *ob, DerivedMesh *dm)
 		gridSize = ccgDM_getGridSize(dm);
 		numGrids = ccgDM_getNumGrids(dm);
 
-		ccgdm->pbvh = BLI_pbvh_new();
+		ob->sculpt->pbvh= ccgdm->pbvh = BLI_pbvh_new();
 		BLI_pbvh_build_grids(ccgdm->pbvh, ccgdm->gridData, ccgdm->gridAdjacency,
 			numGrids, gridSize, (void**)ccgdm->gridFaces);
 	}
 	else if(ob->type == OB_MESH) {
 		Mesh *me= ob->data;
 
-		ccgdm->pbvh = BLI_pbvh_new();
+		ob->sculpt->pbvh= ccgdm->pbvh = BLI_pbvh_new();
 		BLI_pbvh_build_mesh(ccgdm->pbvh, me->mface, me->mvert,
 				   me->totface, me->totvert);
 	}
@@ -2265,6 +2291,7 @@ static CCGDerivedMesh *getCCGDerivedMesh(CCGSubSurf *ss,
 	ccgdm->dm.getGridData = ccgDM_getGridData;
 	ccgdm->dm.getGridAdjacency = ccgDM_getGridAdjacency;
 	ccgdm->dm.getGridOffset = ccgDM_getGridOffset;
+	ccgdm->dm.getFaceMap = ccgDM_getFaceMap;
 	ccgdm->dm.getPBVH = ccgDM_getPBVH;
 
 	ccgdm->dm.getVertCos = ccgdm_getVertCos;