Todo issue: sculpting on deformed mesh

Used a crazyspace approach (like in edit mode), but only modifiers with
deformMatricies are allowed atm (currently shapekeys and armature modifiers only).
All the rest modifiers had an warning message that they aren't applied because
of sculpt mode. Deformation of multires is also unsupported.

With all this restictions users will always see the actual "layer" (or maybe
mesh state would be more correct word) they are sculpting on.

Internal changes:
- All modifiers could have deformMatricies callback (the same as deformMatriciesEM but
  for non-edit mode usage)
- Added function to build crazyspace for sculpting (sculpt_get_deform_matrices), but it
  could be generalized for usage in other painting modes (particle edit mode, i.e)

Todo:
- Implement crazyspace correction to support all kinds of deformation modifiers
- Maybe deformation of multires isn't so difficult?
- And maybe we could avoid extra bad-level-stub for ED_sculpt_modifiers_changed
  without code duplicating?

3 files changed, 25 insertions, 11 deletions

diff --git a/source/blender/blenlib/BLI_pbvh.h b/source/blender/blenlib/BLI_pbvh.h
index 4797aeb2364..f89068c885e 100644
--- a/source/blender/blenlib/BLI_pbvh.h
+++ b/source/blender/blenlib/BLI_pbvh.h
@@ -118,8 +118,6 @@ float BLI_pbvh_node_get_tmin(PBVHNode* node);
 void BLI_pbvh_update(PBVH *bvh, int flags, float (*face_nors)[3]);
 void BLI_pbvh_redraw_BB(PBVH *bvh, float bb_min[3], float bb_max[3]);
 void BLI_pbvh_get_grid_updates(PBVH *bvh, int clear, void ***gridfaces, int *totface);
-void BLI_pbvh_grids_update(PBVH *bvh, struct DMGridData **grids,
-	struct DMGridAdjacency *gridadj, void **gridfaces);
 
 /* vertex deformer */
 float (*BLI_pbvh_get_vertCos(struct PBVH *pbvh))[3];
diff --git a/source/blender/blenlib/intern/math_matrix.c b/source/blender/blenlib/intern/math_matrix.c
index 77ef825ee34..75134358c31 100644
--- a/source/blender/blenlib/intern/math_matrix.c
+++ b/source/blender/blenlib/intern/math_matrix.c
@@ -109,6 +109,20 @@ void copy_m4_m3(float m1[][4], float m2[][3])	/* no clear */
 
 }
 
+void swap_m3m3(float m1[][3], float m2[][3])
+{
+	float t;
+	int i, j;
+
+	for(i = 0; i < 3; i++) {
+		for (j = 0; j < 3; j++) {
+			t        = m1[i][j];
+			m1[i][j] = m2[i][j];
+			m2[i][j] = t;
+		}
+	}
+}
+
 void swap_m4m4(float m1[][4], float m2[][4])
 {
 	float t;
diff --git a/source/blender/blenlib/intern/pbvh.c b/source/blender/blenlib/intern/pbvh.c
index 8727fab7086..9be42f9f5fc 100644
--- a/source/blender/blenlib/intern/pbvh.c
+++ b/source/blender/blenlib/intern/pbvh.c
@@ -1475,13 +1475,6 @@ void BLI_pbvh_draw(PBVH *bvh, float (*planes)[4], float (*face_nors)[3], int smo
 	}
 }
 
-void BLI_pbvh_grids_update(PBVH *bvh, DMGridData **grids, DMGridAdjacency *gridadj, void **gridfaces)
-{
-	bvh->grids= grids;
-	bvh->gridadj= gridadj;
-	bvh->gridfaces= gridfaces;
-}
-
 float (*BLI_pbvh_get_vertCos(PBVH *pbvh))[3]
 {
 	int a;
@@ -1520,13 +1513,22 @@ void BLI_pbvh_apply_vertCos(PBVH *pbvh, float (*vertCos)[3])
 	}
 
 	if (pbvh->verts) {
+		MVert *mvert= pbvh->verts;
 		/* copy new verts coords */
-		for (a= 0; a < pbvh->totvert; ++a) {
-			copy_v3_v3(pbvh->verts[a].co, vertCos[a]);
+		for (a= 0; a < pbvh->totvert; ++a, ++mvert) {
+			copy_v3_v3(mvert->co, vertCos[a]);
+			mvert->flag |= ME_VERT_PBVH_UPDATE;
 		}
 
 		/* coordinates are new -- normals should also be updated */
 		mesh_calc_normals(pbvh->verts, pbvh->totvert, pbvh->faces, pbvh->totprim, NULL);
+
+		for (a= 0; a < pbvh->totnode; ++a)
+			BLI_pbvh_node_mark_update(&pbvh->nodes[a]);
+
+		BLI_pbvh_update(pbvh, PBVH_UpdateBB, NULL);
+		BLI_pbvh_update(pbvh, PBVH_UpdateOriginalBB, NULL);
+
 	}
 }