Fix T39517,

Issue here is that "show diffuse" option does not respect its intended
purpose which is to be used only for masking.

There are a couple of caveats here:

Dyntopo and multires -always- have mask data enabled, and thus as soon
as one goes to dyntopo mode or adds a multires modifier he would get the
default grey color instead.

Matcaps would break when nodes asked for a diffuse material color (this
was broken before too). Solved by adding global material state for when
matcaps are enabled. Also matcaps don't always played well with VBOs
off.

Added a few more missing updates for mask operators to notify
show_diffuse property as changed. This was also needed on rebuilding
dyntopo pbvh.

Also make zero mask color duller again after artist feedback.

4 files changed, 43 insertions, 14 deletions

diff --git a/source/blender/gpu/GPU_buffers.h b/source/blender/gpu/GPU_buffers.h
index 0629bae26f6..f880bfbda6f 100644
--- a/source/blender/gpu/GPU_buffers.h
+++ b/source/blender/gpu/GPU_buffers.h
@@ -180,7 +180,7 @@ void GPU_update_bmesh_pbvh_buffers(GPU_PBVH_Buffers *buffers,
                               struct GHash *bm_faces,
                               struct GSet *bm_unique_verts,
                               struct GSet *bm_other_verts,
-                              int show_diffuse_color);
+                              bool show_diffuse_color);
 
 void GPU_update_grid_pbvh_buffers(GPU_PBVH_Buffers *buffers, struct CCGElem **grids,
                              const struct DMFlagMat *grid_flag_mats,
diff --git a/source/blender/gpu/GPU_draw.h b/source/blender/gpu/GPU_draw.h
index 7e2f4655cb1..ee1eabc2a15 100644
--- a/source/blender/gpu/GPU_draw.h
+++ b/source/blender/gpu/GPU_draw.h
@@ -76,6 +76,7 @@ int GPU_enable_material(int nr, void *attribs);
 void GPU_disable_material(void);
 
 void GPU_material_diffuse_get(int nr, float diff[4]);
+bool GPU_material_use_matcaps_get(void);
 
 void GPU_set_material_alpha_blend(int alphablend);
 int GPU_get_material_alpha_blend(void);
diff --git a/source/blender/gpu/intern/gpu_buffers.c b/source/blender/gpu/intern/gpu_buffers.c
index 7875cad289a..28e117a0ff9 100644
--- a/source/blender/gpu/intern/gpu_buffers.c
+++ b/source/blender/gpu/intern/gpu_buffers.c
@@ -1354,6 +1354,7 @@ struct GPU_PBVH_Buffers {
 	int smooth;
 
 	bool show_diffuse_color;
+	bool use_matcaps;
 	float diffuse_color[4];
 };
 typedef enum {
@@ -1443,12 +1444,15 @@ void GPU_update_mesh_pbvh_buffers(GPU_PBVH_Buffers *buffers, MVert *mvert,
 
 	buffers->vmask = vmask;
 	buffers->show_diffuse_color = show_diffuse_color;
+	buffers->use_matcaps = GPU_material_use_matcaps_get();
 
 	if (buffers->vert_buf) {
 		int totelem = (buffers->smooth ? totvert : (buffers->tot_tri * 3));
-		float diffuse_color[4] = {1.0f, 1.0f, 1.0f, 1.0f};
+		float diffuse_color[4] = {0.8f, 0.8f, 0.8f, 0.8f};
 
-		if (buffers->show_diffuse_color) {
+		if (buffers->use_matcaps)
+			diffuse_color[0] = diffuse_color[1] = diffuse_color[2] = 1.0;
+		else if (show_diffuse_color) {
 			MFace *f = buffers->mface + buffers->face_indices[0];
 
 			GPU_material_diffuse_get(f->mat_nr + 1, diffuse_color);
@@ -1583,6 +1587,7 @@ GPU_PBVH_Buffers *GPU_build_pbvh_mesh_buffers(int (*face_vert_indices)[4],
 	buffers->smooth = mface[face_indices[0]].flag & ME_SMOOTH;
 
 	buffers->show_diffuse_color = false;
+	buffers->use_matcaps = false;
 
 	/* Count the number of visible triangles */
 	for (i = 0, tottri = 0; i < totface; ++i) {
@@ -1658,15 +1663,18 @@ void GPU_update_grid_pbvh_buffers(GPU_PBVH_Buffers *buffers, CCGElem **grids,
 	int i, j, k, x, y;
 
 	buffers->show_diffuse_color = show_diffuse_color;
+	buffers->use_matcaps = GPU_material_use_matcaps_get();
 
 	/* Build VBO */
 	if (buffers->vert_buf) {
 		int totvert = key->grid_area * totgrid;
 		int smooth = grid_flag_mats[grid_indices[0]].flag & ME_SMOOTH;
 		const int has_mask = key->has_mask;
-		float diffuse_color[4] = {1.0f, 1.0f, 1.0f, 1.0f};
+		float diffuse_color[4] = {0.8f, 0.8f, 0.8f, 1.0f};
 
-		if (buffers->show_diffuse_color) {
+		if (buffers->use_matcaps)
+			diffuse_color[0] = diffuse_color[1] = diffuse_color[2] = 1.0;
+		else if (show_diffuse_color) {
 			const DMFlagMat *flags = &grid_flag_mats[grid_indices[0]];
 
 			GPU_material_diffuse_get(flags->mat_nr + 1, diffuse_color);
@@ -1903,6 +1911,7 @@ GPU_PBVH_Buffers *GPU_build_grid_pbvh_buffers(int *grid_indices, int totgrid,
 	buffers->totgrid = totgrid;
 
 	buffers->show_diffuse_color = false;
+	buffers->use_matcaps = false;
 
 	/* Count the number of quads */
 	totquad = gpu_count_grid_quads(grid_hidden, grid_indices, totgrid, gridsize);
@@ -2023,17 +2032,18 @@ void GPU_update_bmesh_pbvh_buffers(GPU_PBVH_Buffers *buffers,
                               GHash *bm_faces,
                               GSet *bm_unique_verts,
                               GSet *bm_other_verts,
-                              int show_diffuse_color)
+                              bool show_diffuse_color)
 {
 	VertexBufferFormat *vert_data;
 	void *tri_data;
 	int tottri, totvert, maxvert = 0;
-	float diffuse_color[4] = {1.0f, 1.0f, 1.0f, 1.0f};
+	float diffuse_color[4] = {0.8f, 0.8f, 0.8f, 1.0f};
 
 	/* TODO, make mask layer optional for bmesh buffer */
 	const int cd_vert_mask_offset = CustomData_get_offset(&bm->vdata, CD_PAINT_MASK);
 
 	buffers->show_diffuse_color = show_diffuse_color;
+	buffers->use_matcaps = GPU_material_use_matcaps_get();
 
 	if (!buffers->vert_buf || (buffers->smooth && !buffers->index_buf))
 		return;
@@ -2061,8 +2071,10 @@ void GPU_update_bmesh_pbvh_buffers(GPU_PBVH_Buffers *buffers,
 		return;
 	}
 
-	if (show_diffuse_color) {
-		/* due to dynamc nature of dyntopo, only get first material */
+	if (buffers->use_matcaps)
+		diffuse_color[0] = diffuse_color[1] = diffuse_color[2] = 1.0;
+	else if (show_diffuse_color) {
+		/* due to dynamic nature of dyntopo, only get first material */
 		GHashIterator gh_iter;
 		BMFace *f;
 		BLI_ghashIterator_init(&gh_iter, bm_faces);
@@ -2215,6 +2227,7 @@ GPU_PBVH_Buffers *GPU_build_bmesh_pbvh_buffers(int smooth_shading)
 	buffers->use_bmesh = true;
 	buffers->smooth = smooth_shading;
 	buffers->show_diffuse_color = false;
+	buffers->use_matcaps = false;
 
 	return buffers;
 }
@@ -2225,9 +2238,11 @@ static void gpu_draw_buffers_legacy_mesh(GPU_PBVH_Buffers *buffers)
 	int i, j;
 	const int has_mask = (buffers->vmask != NULL);
 	const MFace *face = &buffers->mface[buffers->face_indices[0]];
-	float diffuse_color[4] = {1.0f, 1.0f, 1.0f, 1.0f};
+	float diffuse_color[4] = {0.8f, 0.8f, 0.8f, 1.0f};
 
-	if (buffers->show_diffuse_color)
+	if (buffers->use_matcaps)
+		diffuse_color[0] = diffuse_color[1] = diffuse_color[2] = 1.0;
+	else if (buffers->show_diffuse_color)
 		GPU_material_diffuse_get(face->mat_nr + 1, diffuse_color);
 
 	if (has_mask) {
@@ -2297,9 +2312,11 @@ static void gpu_draw_buffers_legacy_grids(GPU_PBVH_Buffers *buffers)
 	int i, j, x, y, gridsize = buffers->gridkey.grid_size;
 	const int has_mask = key->has_mask;
 	const DMFlagMat *flags = &buffers->grid_flag_mats[buffers->grid_indices[0]];
-	float diffuse_color[4] = {1.0f, 1.0f, 1.0f, 1.0f};
+	float diffuse_color[4] = {0.8f, 0.8f, 0.8f, 1.0f};
 
-	if (buffers->show_diffuse_color)
+	if (buffers->use_matcaps)
+		diffuse_color[0] = diffuse_color[1] = diffuse_color[2] = 1.0;
+	else if (buffers->show_diffuse_color)
 		GPU_material_diffuse_get(flags->mat_nr + 1, diffuse_color);
 
 	if (has_mask) {
@@ -2517,6 +2534,9 @@ bool GPU_pbvh_buffers_diffuse_changed(GPU_PBVH_Buffers *buffers, GHash *bm_faces
 	if (buffers->show_diffuse_color != show_diffuse_color)
 		return true;
 
+	if (buffers->use_matcaps != GPU_material_use_matcaps_get())
+		return true;
+
 	if (buffers->show_diffuse_color == false)
 		return false;
 
diff --git a/source/blender/gpu/intern/gpu_draw.c b/source/blender/gpu/intern/gpu_draw.c
index fba7e028830..64ab701acba 100644
--- a/source/blender/gpu/intern/gpu_draw.c
+++ b/source/blender/gpu/intern/gpu_draw.c
@@ -1342,6 +1342,7 @@ static struct GPUMaterialState {
 	GPUBlendMode *alphablend;
 	GPUBlendMode alphablend_fixed[FIXEDMAT];
 	bool use_alpha_pass, is_alpha_pass;
+	bool use_matcaps;
 
 	int lastmatnr, lastretval;
 	GPUBlendMode lastalphablend;
@@ -1409,6 +1410,7 @@ void GPU_begin_object_materials(View3D *v3d, RegionView3D *rv3d, Scene *scene, O
 	GMS.lastmatnr = -1;
 	GMS.lastretval = -1;
 	GMS.lastalphablend = GPU_BLEND_SOLID;
+	GMS.use_matcaps = use_matcap;
 
 	GMS.backface_culling = (v3d->flag2 & V3D_BACKFACE_CULLING) != 0;
 
@@ -1444,7 +1446,7 @@ void GPU_begin_object_materials(View3D *v3d, RegionView3D *rv3d, Scene *scene, O
 	if (use_matcap) {
 		GMS.gmatbuf[0] = v3d->defmaterial;
 		GPU_material_matcap(scene, v3d->defmaterial);
-		
+
 		/* do material 1 too, for displists! */
 		memcpy(&GMS.matbuf[1], &GMS.matbuf[0], sizeof(GPUMaterialFixed));
 	
@@ -1656,6 +1658,12 @@ void GPU_material_diffuse_get(int nr, float diff[4])
 	}
 }
 
+bool GPU_material_use_matcaps_get(void)
+{
+	return GMS.use_matcaps;
+}
+
+
 void GPU_end_object_materials(void)
 {
 	GPU_disable_material();