Fix T46726 shading issues in sculpt mode.

This fixes two issues:

* Normals were not being recalculated correctly when not using optimized
drawing for CDDerivedMesh (Multires actually handles that correctly).

* Loop normals (autosmooth option) were also not being calculated. Doing
this calculation is not desirable, since it can't be done correctly
without a severe performance hit. This is easy to test by doing a
dependency flush on the mesh after each scuplt stroke step. Instead they
are now disabled during sculpting.

3 files changed, 45 insertions, 24 deletions

diff --git a/source/blender/blenkernel/intern/cdderivedmesh.c b/source/blender/blenkernel/intern/cdderivedmesh.c
index 3b86c42d8c9..be4f0011e97 100644
--- a/source/blender/blenkernel/intern/cdderivedmesh.c
+++ b/source/blender/blenkernel/intern/cdderivedmesh.c
@@ -344,9 +344,6 @@ static void cdDM_update_normals_from_pbvh(DerivedMesh *dm)
 	CDDerivedMesh *cddm = (CDDerivedMesh *) dm;
 	float (*face_nors)[3];
 
-	if (!cddm->pbvh || !cddm->pbvh_draw || !dm->numPolyData)
-		return;
-
 	face_nors = CustomData_get_layer(&dm->polyData, CD_NORMAL);
 
 	BKE_pbvh_update(cddm->pbvh, PBVH_UpdateNormals, face_nors);
@@ -446,18 +443,18 @@ static void cdDM_drawFacesSolid(
 	CDDerivedMesh *cddm = (CDDerivedMesh *) dm;
 	int a;
 
-	if (cddm->pbvh && cddm->pbvh_draw) {
-		if (BKE_pbvh_has_faces(cddm->pbvh)) {
+	if (cddm->pbvh) {
+		if (cddm->pbvh_draw && BKE_pbvh_has_faces(cddm->pbvh)) {
 			float (*face_nors)[3] = CustomData_get_layer(&dm->faceData, CD_NORMAL);
 
-			cdDM_update_normals_from_pbvh(dm);
-
 			BKE_pbvh_draw(cddm->pbvh, partial_redraw_planes, face_nors,
 			              setMaterial, false, false);
 			glShadeModel(GL_FLAT);
+			return;
+		}
+		else {
+			cdDM_update_normals_from_pbvh(dm);
 		}
-
-		return;
 	}
 	
 	GPU_vertex_setup(dm);
@@ -505,14 +502,18 @@ static void cdDM_drawFacesTex_common(
 	 *       textured view, but object itself will be displayed gray
 	 *       (the same as it'll display without UV maps in textured view)
 	 */
-	if (cddm->pbvh && cddm->pbvh_draw && BKE_pbvh_type(cddm->pbvh) == PBVH_BMESH) {
-		if (BKE_pbvh_has_faces(cddm->pbvh)) {
-			cdDM_update_normals_from_pbvh(dm);
+	if (cddm->pbvh) {
+		if (cddm->pbvh_draw &&
+		    BKE_pbvh_type(cddm->pbvh) == PBVH_BMESH &&
+		    BKE_pbvh_has_faces(cddm->pbvh))
+		{
 			GPU_set_tpage(NULL, false, false);
 			BKE_pbvh_draw(cddm->pbvh, NULL, NULL, NULL, false, false);
+			return;
+		}
+		else {
+			cdDM_update_normals_from_pbvh(dm);
 		}
-
-		return;
 	}
 
 	colType = CD_TEXTURE_MLOOPCOL;
@@ -878,14 +879,18 @@ static void cdDM_drawMappedFacesGLSL(
 	 *       will skip using textures (dyntopo currently destroys UV anyway) and
 	 *       works fine for matcap
 	 */
-	if (cddm->pbvh && cddm->pbvh_draw && BKE_pbvh_type(cddm->pbvh) == PBVH_BMESH) {
-		if (BKE_pbvh_has_faces(cddm->pbvh)) {
-			cdDM_update_normals_from_pbvh(dm);
+	if (cddm->pbvh) {
+		if (cddm->pbvh_draw &&
+		    BKE_pbvh_type(cddm->pbvh) == PBVH_BMESH &&
+		    BKE_pbvh_has_faces(cddm->pbvh))
+		{
 			setMaterial(1, &gattribs);
 			BKE_pbvh_draw(cddm->pbvh, NULL, NULL, NULL, false, false);
+			return;
+		}
+		else {
+			cdDM_update_normals_from_pbvh(dm);
 		}
-
-		return;
 	}
 
 	matnr = -1;
@@ -1147,14 +1152,18 @@ static void cdDM_drawMappedFacesMat(
 	 *       works fine for matcap
 	 */
 
-	if (cddm->pbvh && cddm->pbvh_draw && BKE_pbvh_type(cddm->pbvh) == PBVH_BMESH) {
-		if (BKE_pbvh_has_faces(cddm->pbvh)) {
-			cdDM_update_normals_from_pbvh(dm);
+	if (cddm->pbvh) {
+		if (cddm->pbvh_draw &&
+		    BKE_pbvh_type(cddm->pbvh) == PBVH_BMESH &&
+		    BKE_pbvh_has_faces(cddm->pbvh))
+		{
 			setMaterial(userData, 1, &gattribs);
 			BKE_pbvh_draw(cddm->pbvh, NULL, NULL, NULL, false, false);
+			return;
+		}
+		else {
+			cdDM_update_normals_from_pbvh(dm);
 		}
-
-		return;
 	}
 
 	matnr = -1;
@@ -1347,6 +1356,7 @@ static void cdDM_buffer_copy_vertex(
 static void cdDM_buffer_copy_normal(
         DerivedMesh *dm, short *varray)
 {
+	CDDerivedMesh *cddm = (CDDerivedMesh *)dm;
 	int i, j, totpoly;
 	int start;
 
@@ -1362,6 +1372,10 @@ static void cdDM_buffer_copy_normal(
 	mloop = dm->getLoopArray(dm);
 	totpoly = dm->getNumPolys(dm);
 
+	/* we are in sculpt mode, disable loop normals (since they won't get updated) */
+	if (cddm->pbvh)
+		lnors = NULL;
+
 	start = 0;
 	for (i = 0; i < totpoly; i++, mpoly++) {
 		const bool smoothnormal = (mpoly->flag & ME_SMOOTH) != 0;
diff --git a/source/blender/blenkernel/intern/subsurf_ccg.c b/source/blender/blenkernel/intern/subsurf_ccg.c
index a2c625a7ec3..5814a06bb9f 100644
--- a/source/blender/blenkernel/intern/subsurf_ccg.c
+++ b/source/blender/blenkernel/intern/subsurf_ccg.c
@@ -1912,6 +1912,10 @@ static void ccgDM_buffer_copy_normal(
 	int shademodel;
 	int start = 0;
 
+	/* we are in sculpt mode, disable loop normals (since they won't get updated) */
+	if (ccgdm->pbvh)
+		lnors = NULL;
+
 	CCG_key_top_level(&key, ss);
 
 	for (i = 0; i < totface; i++) {
diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index 4aded2d3e05..fad7f089dca 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -5149,6 +5149,9 @@ static int sculpt_mode_toggle_exec(bContext *C, wmOperator *op)
 		paint_cursor_start(C, sculpt_poll_view3d);
 	}
 
+	if (ob->derivedFinal) /* VBO no longer valid */
+		GPU_drawobject_free(ob->derivedFinal);
+
 	WM_event_add_notifier(C, NC_SCENE | ND_MODE, scene);
 
 	return OPERATOR_FINISHED;