Sculpting: Sdd an option to hide mask in viewport

Brushes themselves are still affected by the mask, but the viewport is not
showing the mask. This way it's easier to see details while sculpting.

Studio request by Julien Kaspar

12 files changed, 94 insertions, 14 deletions

diff --git a/source/blender/blenkernel/BKE_paint.h b/source/blender/blenkernel/BKE_paint.h
index 07c4598c856..cc02d8b547e 100644
--- a/source/blender/blenkernel/BKE_paint.h
+++ b/source/blender/blenkernel/BKE_paint.h
@@ -194,6 +194,7 @@ typedef struct SculptSession {
 	/* PBVH acceleration structure */
 	struct PBVH *pbvh;
 	bool show_diffuse_color;
+	bool show_mask;
 
 	/* Painting on deformed mesh */
 	bool modifiers_active; /* object is deformed with some modifiers */
diff --git a/source/blender/blenkernel/BKE_pbvh.h b/source/blender/blenkernel/BKE_pbvh.h
index 2daa2ef7182..f619133ee4b 100644
--- a/source/blender/blenkernel/BKE_pbvh.h
+++ b/source/blender/blenkernel/BKE_pbvh.h
@@ -370,5 +370,6 @@ bool BKE_pbvh_node_vert_update_check_any(PBVH *bvh, PBVHNode *node);
 //void BKE_pbvh_node_BB_expand(PBVHNode *node, float co[3]);
 
 void pbvh_show_diffuse_color_set(PBVH *bvh, bool show_diffuse_color);
+void pbvh_show_mask_set(PBVH *bvh, bool show_mask);
 
 #endif /* __BKE_PBVH_H__ */
diff --git a/source/blender/blenkernel/intern/cdderivedmesh.c b/source/blender/blenkernel/intern/cdderivedmesh.c
index 13949e6777d..bbc66b9ecf2 100644
--- a/source/blender/blenkernel/intern/cdderivedmesh.c
+++ b/source/blender/blenkernel/intern/cdderivedmesh.c
@@ -287,6 +287,7 @@ static PBVH *cdDM_getPBVH(Object *ob, DerivedMesh *dm)
 		                     ob->sculpt->cd_face_node_offset);
 
 		pbvh_show_diffuse_color_set(cddm->pbvh, ob->sculpt->show_diffuse_color);
+		pbvh_show_mask_set(cddm->pbvh, ob->sculpt->show_mask);
 	}
 		
 
@@ -317,6 +318,7 @@ static PBVH *cdDM_getPBVH(Object *ob, DerivedMesh *dm)
 		        looptri, looptris_num);
 
 		pbvh_show_diffuse_color_set(cddm->pbvh, ob->sculpt->show_diffuse_color);
+		pbvh_show_mask_set(cddm->pbvh, ob->sculpt->show_mask);
 
 		deformed = check_sculpt_object_deformed(ob, true);
 
diff --git a/source/blender/blenkernel/intern/paint.c b/source/blender/blenkernel/intern/paint.c
index d1e9516f6d2..c6aa9c40fce 100644
--- a/source/blender/blenkernel/intern/paint.c
+++ b/source/blender/blenkernel/intern/paint.c
@@ -884,6 +884,7 @@ void BKE_sculpt_update_mesh_elements(Scene *scene, Sculpt *sd, Object *ob,
 
 	ss->modifiers_active = sculpt_modifiers_active(scene, sd, ob);
 	ss->show_diffuse_color = (sd->flags & SCULPT_SHOW_DIFFUSE) != 0;
+	ss->show_mask = (sd->flags & SCULPT_HIDE_MASK) == 0;
 
 	ss->building_vp_handle = false;
 
@@ -938,6 +939,7 @@ void BKE_sculpt_update_mesh_elements(Scene *scene, Sculpt *sd, Object *ob,
 	ss->pmap = (need_pmap && dm->getPolyMap) ? dm->getPolyMap(ob, dm) : NULL;
 
 	pbvh_show_diffuse_color_set(ss->pbvh, ss->show_diffuse_color);
+	pbvh_show_mask_set(ss->pbvh, ss->show_mask);
 
 	if (ss->modifiers_active) {
 		if (!ss->orig_cos) {
diff --git a/source/blender/blenkernel/intern/pbvh.c b/source/blender/blenkernel/intern/pbvh.c
index 3a7bade1ee3..d926c96c0d8 100644
--- a/source/blender/blenkernel/intern/pbvh.c
+++ b/source/blender/blenkernel/intern/pbvh.c
@@ -1107,6 +1107,7 @@ static int pbvh_get_buffers_update_flags(PBVH *bvh)
 {
 	int update_flags = 0;
 	update_flags |= bvh->show_diffuse_color ? GPU_PBVH_BUFFERS_SHOW_DIFFUSE_COLOR : 0;
+	update_flags |= bvh->show_mask ? GPU_PBVH_BUFFERS_SHOW_MASK : 0;
 	return update_flags;
 }
 
@@ -2109,6 +2110,16 @@ static void pbvh_node_check_diffuse_changed(PBVH *bvh, PBVHNode *node)
 		node->flag |= PBVH_UpdateDrawBuffers;
 }
 
+static void pbvh_node_check_mask_changed(PBVH *bvh, PBVHNode *node)
+{
+	if (!node->draw_buffers) {
+		return;
+	}
+	if (GPU_pbvh_buffers_mask_changed(node->draw_buffers, bvh->show_mask)) {
+		node->flag |= PBVH_UpdateDrawBuffers;
+	}
+}
+
 void BKE_pbvh_draw(PBVH *bvh, float (*planes)[4], float (*fnors)[3],
                    DMSetMaterial setMaterial, bool wireframe, bool fast)
 {
@@ -2116,8 +2127,10 @@ void BKE_pbvh_draw(PBVH *bvh, float (*planes)[4], float (*fnors)[3],
 	PBVHNode **nodes;
 	int totnode;
 
-	for (int a = 0; a < bvh->totnode; a++)
+	for (int a = 0; a < bvh->totnode; a++) {
 		pbvh_node_check_diffuse_changed(bvh, &bvh->nodes[a]);
+		pbvh_node_check_mask_changed(bvh, &bvh->nodes[a]);
+	}
 
 	BKE_pbvh_search_gather(bvh, update_search_cb, SET_INT_IN_POINTER(PBVH_UpdateNormals | PBVH_UpdateDrawBuffers),
 	                       &nodes, &totnode);
@@ -2367,3 +2380,8 @@ void pbvh_show_diffuse_color_set(PBVH *bvh, bool show_diffuse_color)
 
 	bvh->show_diffuse_color = !has_mask || show_diffuse_color;
 }
+
+void pbvh_show_mask_set(PBVH *bvh, bool show_mask)
+{
+	bvh->show_mask = show_mask;
+}
diff --git a/source/blender/blenkernel/intern/pbvh_intern.h b/source/blender/blenkernel/intern/pbvh_intern.h
index e05a3068682..afd539b2273 100644
--- a/source/blender/blenkernel/intern/pbvh_intern.h
+++ b/source/blender/blenkernel/intern/pbvh_intern.h
@@ -164,6 +164,7 @@ struct PBVH {
 	bool deformed;
 
 	bool show_diffuse_color;
+	bool show_mask;
 
 	/* Dynamic topology */
 	BMesh *bm;
diff --git a/source/blender/blenkernel/intern/subsurf_ccg.c b/source/blender/blenkernel/intern/subsurf_ccg.c
index 1b174cf4654..8bace2a8d65 100644
--- a/source/blender/blenkernel/intern/subsurf_ccg.c
+++ b/source/blender/blenkernel/intern/subsurf_ccg.c
@@ -4512,8 +4512,10 @@ static struct PBVH *ccgDM_getPBVH(Object *ob, DerivedMesh *dm)
 		                    looptri, looptris_num);
 	}
 
-	if (ccgdm->pbvh)
+	if (ccgdm->pbvh != NULL) {
 		pbvh_show_diffuse_color_set(ccgdm->pbvh, ob->sculpt->show_diffuse_color);
+		pbvh_show_mask_set(ccgdm->pbvh, ob->sculpt->show_mask);
+	}
 
 	/* For vertex paint, keep track of ccgdm */
 	if (!(ob->mode & OB_MODE_SCULPT) && ccgdm->pbvh) {
diff --git a/source/blender/editors/sculpt_paint/paint_ops.c b/source/blender/editors/sculpt_paint/paint_ops.c
index 73cb31bb1bd..004d2757a71 100644
--- a/source/blender/editors/sculpt_paint/paint_ops.c
+++ b/source/blender/editors/sculpt_paint/paint_ops.c
@@ -1277,6 +1277,10 @@ void ED_keymap_paint(wmKeyConfig *keyconf)
 
 	WM_keymap_add_item(keymap, "PAINT_OT_mask_lasso_gesture", LEFTMOUSE, KM_PRESS, KM_CTRL | KM_SHIFT, 0);
 
+	/* Toggle mask visibility */
+	kmi = WM_keymap_add_item(keymap, "WM_OT_context_toggle", MKEY, KM_PRESS, KM_CTRL, 0);
+	RNA_string_set(kmi->ptr, "data_path", "scene.tool_settings.sculpt.show_mask");
+
 	/* Toggle dynamic topology */
 	WM_keymap_add_item(keymap, "SCULPT_OT_dynamic_topology_toggle", DKEY, KM_PRESS, KM_CTRL, 0);
 
diff --git a/source/blender/gpu/GPU_buffers.h b/source/blender/gpu/GPU_buffers.h
index 1423361e8cb..6ffaa29ead6 100644
--- a/source/blender/gpu/GPU_buffers.h
+++ b/source/blender/gpu/GPU_buffers.h
@@ -239,6 +239,7 @@ GPU_PBVH_Buffers *GPU_pbvh_bmesh_buffers_build(bool smooth_shading);
 
 enum {
 	GPU_PBVH_BUFFERS_SHOW_DIFFUSE_COLOR = (1 << 0),
+	GPU_PBVH_BUFFERS_SHOW_MASK = (1 << 1),
 };
 
 void GPU_pbvh_mesh_buffers_update(
@@ -272,6 +273,7 @@ void GPU_pbvh_BB_draw_init(void);
 void GPU_pbvh_BB_draw_end(void);
 
 bool GPU_pbvh_buffers_diffuse_changed(GPU_PBVH_Buffers *buffers, struct GSet *bm_faces, bool show_diffuse_color);
+bool GPU_pbvh_buffers_mask_changed(GPU_PBVH_Buffers *buffers, bool show_mask);
 
 void GPU_pbvh_buffers_free(GPU_PBVH_Buffers *buffers);
 void GPU_pbvh_multires_buffers_free(struct GridCommonGPUBuffer **grid_common_gpu_buffer);
diff --git a/source/blender/gpu/intern/gpu_buffers.c b/source/blender/gpu/intern/gpu_buffers.c
index 09d6f80abb8..afe7a4c4d27 100644
--- a/source/blender/gpu/intern/gpu_buffers.c
+++ b/source/blender/gpu/intern/gpu_buffers.c
@@ -993,6 +993,8 @@ struct GPU_PBVH_Buffers {
 	bool smooth;
 
 	bool show_diffuse_color;
+	bool show_mask;
+
 	bool use_matcaps;
 	float diffuse_color[4];
 };
@@ -1037,11 +1039,13 @@ void GPU_pbvh_mesh_buffers_update(
         const int update_flags)
 {
 	const bool show_diffuse_color = (update_flags & GPU_PBVH_BUFFERS_SHOW_DIFFUSE_COLOR) != 0;
+	const bool show_mask = (update_flags & GPU_PBVH_BUFFERS_SHOW_MASK) != 0;
 	VertexBufferFormat *vert_data;
 	int i;
 
 	buffers->vmask = vmask;
 	buffers->show_diffuse_color = show_diffuse_color;
+	buffers->show_mask = show_mask;
 	buffers->use_matcaps = GPU_material_use_matcaps_get();
 
 	{
@@ -1086,7 +1090,7 @@ void GPU_pbvh_mesh_buffers_update(
 					for (uint j = 0; j < 3; j++) {
 						VertexBufferFormat *out = vert_data + face_vert_indices[i][j];
 
-						if (vmask) {
+						if (vmask && show_mask) {
 							uint v_index = buffers->mloop[lt->tri[j]].v;
 							gpu_color_from_mask_copy(vmask[v_index], diffuse_color, out->color);
 						}
@@ -1122,7 +1126,7 @@ void GPU_pbvh_mesh_buffers_update(
 					}
 
 					uchar color_ub[3];
-					if (vmask) {
+					if (vmask && show_mask) {
 						float fmask = (vmask[vtri[0]] + vmask[vtri[1]] + vmask[vtri[2]]) / 3.0f;
 						gpu_color_from_mask_copy(fmask, diffuse_color, color_ub);
 					}
@@ -1170,6 +1174,7 @@ GPU_PBVH_Buffers *GPU_pbvh_mesh_buffers_build(
 	buffers->smooth = mpoly[looptri[face_indices[0]].poly].flag & ME_SMOOTH;
 
 	buffers->show_diffuse_color = false;
+	buffers->show_mask = true;
 	buffers->use_matcaps = false;
 
 	/* Count the number of visible triangles */
@@ -1245,10 +1250,12 @@ void GPU_pbvh_grid_buffers_update(
         const int update_flags)
 {
 	const bool show_diffuse_color = (update_flags & GPU_PBVH_BUFFERS_SHOW_DIFFUSE_COLOR) != 0;
+	const bool show_mask = (update_flags & GPU_PBVH_BUFFERS_SHOW_MASK) != 0;
 	VertexBufferFormat *vert_data;
 	int i, j, k, x, y;
 
 	buffers->show_diffuse_color = show_diffuse_color;
+	buffers->show_mask = show_mask;
 	buffers->use_matcaps = GPU_material_use_matcaps_get();
 	buffers->smooth = grid_flag_mats[grid_indices[0]].flag & ME_SMOOTH;
 
@@ -1281,7 +1288,7 @@ void GPU_pbvh_grid_buffers_update(
 						if (buffers->smooth) {
 							normal_float_to_short_v3(vd->no, CCG_elem_no(key, elem));
 
-							if (has_mask) {
+							if (has_mask && show_mask) {
 								gpu_color_from_mask_copy(*CCG_elem_mask(key, elem),
 								                         diffuse_color, vd->color);
 							}
@@ -1476,6 +1483,7 @@ GPU_PBVH_Buffers *GPU_pbvh_grid_buffers_build(
 	buffers->totgrid = totgrid;
 
 	buffers->show_diffuse_color = false;
+	buffers->show_mask = true;
 	buffers->use_matcaps = false;
 
 	/* Count the number of quads */
@@ -1547,7 +1555,8 @@ static void gpu_bmesh_vert_to_buffer_copy(BMVert *v,
                                           const float fno[3],
                                           const float *fmask,
                                           const int cd_vert_mask_offset,
-                                          const float diffuse_color[4])
+                                          const float diffuse_color[4],
+                                          const bool show_mask)
 {
 	if (!BM_elem_flag_test(v, BM_ELEM_HIDDEN)) {
 		VertexBufferFormat *vd = &vert_data[*v_index];
@@ -1556,11 +1565,16 @@ static void gpu_bmesh_vert_to_buffer_copy(BMVert *v,
 		copy_v3_v3(vd->co, v->co);
 		normal_float_to_short_v3(vd->no, fno ? fno : v->no);
 
-		gpu_color_from_mask_copy(
-		        fmask ? *fmask :
-		                BM_ELEM_CD_GET_FLOAT(v, cd_vert_mask_offset),
-		        diffuse_color,
-		        vd->color);
+		float effective_mask;
+		if (show_mask) {
+			effective_mask = fmask ? *fmask
+			                       : BM_ELEM_CD_GET_FLOAT(v, cd_vert_mask_offset);
+		}
+		else {
+			effective_mask = 0.0f;
+		}
+
+		gpu_color_from_mask_copy(effective_mask, diffuse_color, vd->color);
 
 		/* Assign index for use in the triangle index buffer */
 		/* note: caller must set:  bm->elem_index_dirty |= BM_VERT; */
@@ -1618,6 +1632,7 @@ void GPU_pbvh_bmesh_buffers_update(
         const int update_flags)
 {
 	const bool show_diffuse_color = (update_flags & GPU_PBVH_BUFFERS_SHOW_DIFFUSE_COLOR) != 0;
+	const bool show_mask = (update_flags & GPU_PBVH_BUFFERS_SHOW_MASK) != 0;
 	VertexBufferFormat *vert_data;
 	void *tri_data;
 	int tottri, totvert, maxvert = 0;
@@ -1627,6 +1642,7 @@ void GPU_pbvh_bmesh_buffers_update(
 	const int cd_vert_mask_offset = CustomData_get_offset(&bm->vdata, CD_PAINT_MASK);
 
 	buffers->show_diffuse_color = show_diffuse_color;
+	buffers->show_mask = show_mask;
 	buffers->use_matcaps = GPU_material_use_matcaps_get();
 
 	/* Count visible triangles */
@@ -1677,13 +1693,15 @@ void GPU_pbvh_bmesh_buffers_update(
 			GSET_ITER (gs_iter, bm_unique_verts) {
 				gpu_bmesh_vert_to_buffer_copy(BLI_gsetIterator_getKey(&gs_iter),
 				                              vert_data, &v_index, NULL, NULL,
-				                              cd_vert_mask_offset, diffuse_color);
+				                              cd_vert_mask_offset, diffuse_color,
+				                              show_mask);
 			}
 
 			GSET_ITER (gs_iter, bm_other_verts) {
 				gpu_bmesh_vert_to_buffer_copy(BLI_gsetIterator_getKey(&gs_iter),
 				                              vert_data, &v_index, NULL, NULL,
-				                              cd_vert_mask_offset, diffuse_color);
+				                              cd_vert_mask_offset, diffuse_color,
+				                              show_mask);
 			}
 
 			maxvert = v_index;
@@ -1715,7 +1733,8 @@ void GPU_pbvh_bmesh_buffers_update(
 					for (i = 0; i < 3; i++) {
 						gpu_bmesh_vert_to_buffer_copy(v[i], vert_data,
 						                              &v_index, f->no, &fmask,
-						                              cd_vert_mask_offset, diffuse_color);
+						                              cd_vert_mask_offset, diffuse_color,
+						                              show_mask);
 					}
 				}
 			}
@@ -1810,6 +1829,7 @@ GPU_PBVH_Buffers *GPU_pbvh_bmesh_buffers_build(bool smooth_shading)
 	buffers->use_bmesh = true;
 	buffers->smooth = smooth_shading;
 	buffers->show_diffuse_color = false;
+	buffers->show_mask = true;
 	buffers->use_matcaps = false;
 
 	return buffers;
@@ -2000,6 +2020,11 @@ bool GPU_pbvh_buffers_diffuse_changed(
 	return !equals_v3v3(diffuse_color, buffers->diffuse_color);
 }
 
+bool GPU_pbvh_buffers_mask_changed(GPU_PBVH_Buffers *buffers, bool show_mask)
+{
+	return (buffers->show_mask != show_mask);
+}
+
 void GPU_pbvh_buffers_free(GPU_PBVH_Buffers *buffers)
 {
 	if (buffers) {
diff --git a/source/blender/makesdna/DNA_scene_types.h b/source/blender/makesdna/DNA_scene_types.h
index c9d4a8a5ca3..6535124a1be 100644
--- a/source/blender/makesdna/DNA_scene_types.h
+++ b/source/blender/makesdna/DNA_scene_types.h
@@ -2119,6 +2119,9 @@ typedef enum eSculptFlags {
 	/* If set, dynamic-topology detail size will be constant in object space */
 	SCULPT_DYNTOPO_DETAIL_CONSTANT = (1 << 13),
 	SCULPT_DYNTOPO_DETAIL_BRUSH = (1 << 14),
+
+	/* Don't display mask in viewport, but still use it for strokes. */
+	SCULPT_HIDE_MASK = (1 << 15),
 } eSculptFlags;
 
 typedef enum eImageePaintMode {
diff --git a/source/blender/makesrna/intern/rna_sculpt_paint.c b/source/blender/makesrna/intern/rna_sculpt_paint.c
index 9da27233e0e..6dbfc3634b2 100644
--- a/source/blender/makesrna/intern/rna_sculpt_paint.c
+++ b/source/blender/makesrna/intern/rna_sculpt_paint.c
@@ -287,6 +287,20 @@ static void rna_Sculpt_ShowDiffuseColor_update(Main *UNUSED(bmain), Scene *scene
 	}
 }
 
+static void rna_Sculpt_ShowMask_update(Main *UNUSED(bmain), Scene *scene, PointerRNA *UNUSED(ptr))
+{
+	Object *object = (scene->basact) ? scene->basact->object : NULL;
+	if (object == NULL || object->sculpt == NULL) {
+		return;
+	}
+	Sculpt *sd = scene->toolsettings->sculpt;
+	object->sculpt->show_mask = ((sd->flags & SCULPT_HIDE_MASK) == 0);
+	if (object->sculpt->pbvh != NULL) {
+		pbvh_show_mask_set(object->sculpt->pbvh, object->sculpt->show_mask);
+	}
+	WM_main_add_notifier(NC_OBJECT | ND_DRAW, object);
+}
+
 static char *rna_Sculpt_path(PointerRNA *UNUSED(ptr))
 {
 	return BLI_strdup("tool_settings.sculpt");
@@ -594,6 +608,11 @@ static void rna_def_sculpt(BlenderRNA  *brna)
 	                         "Show diffuse color of object and overlay sculpt mask on top of it");
 	RNA_def_property_update(prop, NC_OBJECT | ND_DRAW, "rna_Sculpt_ShowDiffuseColor_update");
 
+	prop = RNA_def_property(srna, "show_mask", PROP_BOOLEAN, PROP_NONE);
+	RNA_def_property_boolean_negative_sdna(prop, NULL, "flags", SCULPT_HIDE_MASK);
+	RNA_def_property_ui_text(prop, "Show Mask", "Show mask as overlay on object");
+	RNA_def_property_update(prop, NC_OBJECT | ND_DRAW, "rna_Sculpt_ShowMask_update");
+
 	prop = RNA_def_property(srna, "detail_size", PROP_FLOAT, PROP_PIXEL);
 	RNA_def_property_ui_range(prop, 0.5, 40.0, 10, 2);
 	RNA_def_property_ui_text(prop, "Detail Size", "Maximum edge length for dynamic topology sculpting (in pixels)");