Sculpt: Optimize Mask Overlay drawing

* Remove support for diffuse color in the pbvh buffers.
* Upload raw data to GPU.
* Only draw nodes that have mask data when drawing the overlay.

This should fix T56466

1 files changed, 55 insertions, 213 deletions

diff --git a/source/blender/gpu/intern/gpu_buffers.c b/source/blender/gpu/intern/gpu_buffers.c
index 16590785af6..458486584a7 100644
--- a/source/blender/gpu/intern/gpu_buffers.c
+++ b/source/blender/gpu/intern/gpu_buffers.c
@@ -85,7 +85,6 @@ struct GPU_PBVH_Buffers {
 
 	const int *face_indices;
 	int        face_indices_len;
-	const float *vmask;
 
 	/* grid pointers */
 	CCGKey gridkey;
@@ -105,25 +104,13 @@ struct GPU_PBVH_Buffers {
 	 * smooth-shaded or all faces are flat-shaded */
 	bool smooth;
 
-	bool show_diffuse_color;
 	bool show_mask;
-
-	float diffuse_color[4];
 };
 
 static struct {
-	uint pos, nor, col;
+	uint pos, nor, msk;
 } g_vbo_id = {0};
 
-static void gpu_material_diffuse_get(int UNUSED(nr), float diff[4])
-{
-	/* TODO: sculpt diffuse color option not supported in 2.8 yet. */
-	diff[0] = 0.8f;
-	diff[1] = 0.8f;
-	diff[2] = 0.8f;
-	diff[3] = 1.0f;
-}
-
 /* Allocates a non-initialized buffer to be sent to GPU.
  * Return is false it indicates that the memory map failed. */
 static bool gpu_pbvh_vert_buf_data_set(GPU_PBVH_Buffers *buffers, unsigned int vert_len)
@@ -136,7 +123,7 @@ static bool gpu_pbvh_vert_buf_data_set(GPU_PBVH_Buffers *buffers, unsigned int v
 		if (format.attr_len == 0) {
 			g_vbo_id.pos = GPU_vertformat_attr_add(&format, "pos", GPU_COMP_F32, 3, GPU_FETCH_FLOAT);
 			g_vbo_id.nor = GPU_vertformat_attr_add(&format, "nor", GPU_COMP_I16, 3, GPU_FETCH_INT_TO_FLOAT_UNIT);
-			g_vbo_id.col = GPU_vertformat_attr_add(&format, "color", GPU_COMP_U8, 3, GPU_FETCH_INT_TO_FLOAT_UNIT);
+			g_vbo_id.msk = GPU_vertformat_attr_add(&format, "msk", GPU_COMP_F32, 1, GPU_FETCH_FLOAT);
 		}
 #if 0
 		buffers->vert_buf = GPU_vertbuf_create_with_format_ex(&format, GPU_USAGE_DYNAMIC);
@@ -175,67 +162,17 @@ static void gpu_pbvh_batch_init(GPU_PBVH_Buffers *buffers)
 	}
 }
 
-static float gpu_color_from_mask(float mask)
-{
-	return 1.0f - mask * 0.75f;
-}
-
-static void gpu_color_from_mask_copy(float mask, const float diffuse_color[4], unsigned char out[3])
-{
-	float mask_color;
-
-	mask_color = gpu_color_from_mask(mask) * 255.0f;
-
-	out[0] = diffuse_color[0] * mask_color;
-	out[1] = diffuse_color[1] * mask_color;
-	out[2] = diffuse_color[2] * mask_color;
-}
-
-static void gpu_color_from_mask_quad_copy(const CCGKey *key,
-                                          CCGElem *a, CCGElem *b,
-                                          CCGElem *c, CCGElem *d,
-                                          const float *diffuse_color,
-                                          unsigned char out[3])
-{
-	float mask_color =
-	    gpu_color_from_mask((*CCG_elem_mask(key, a) +
-	                         *CCG_elem_mask(key, b) +
-	                         *CCG_elem_mask(key, c) +
-	                         *CCG_elem_mask(key, d)) * 0.25f) * 255.0f;
-
-	out[0] = diffuse_color[0] * mask_color;
-	out[1] = diffuse_color[1] * mask_color;
-	out[2] = diffuse_color[2] * mask_color;
-}
-
 void GPU_pbvh_mesh_buffers_update(
         GPU_PBVH_Buffers *buffers, const MVert *mvert,
         const int *vert_indices, int totvert, const float *vmask,
         const int (*face_vert_indices)[3],
         const int update_flags)
 {
-	const bool show_diffuse_color = (update_flags & GPU_PBVH_BUFFERS_SHOW_DIFFUSE_COLOR) != 0;
 	const bool show_mask = (update_flags & GPU_PBVH_BUFFERS_SHOW_MASK) != 0;
-
-	buffers->vmask = vmask;
-	buffers->show_diffuse_color = show_diffuse_color;
-	buffers->show_mask = show_mask;
+	bool empty_mask = true;
 
 	{
 		int totelem = (buffers->smooth ? totvert : (buffers->tot_tri * 3));
-		float diffuse_color[4] = {0.8f, 0.8f, 0.8f, 0.8f};
-
-		if (show_diffuse_color) {
-			const MLoopTri *lt = &buffers->looptri[buffers->face_indices[0]];
-			const MPoly *mp = &buffers->mpoly[lt->poly];
-
-			gpu_material_diffuse_get(mp->mat_nr + 1, diffuse_color);
-		}
-
-		copy_v4_v4(buffers->diffuse_color, diffuse_color);
-
-		uchar diffuse_color_ub[4];
-		rgba_float_to_uchar(diffuse_color_ub, diffuse_color);
 
 		/* Build VBO */
 		if (gpu_pbvh_vert_buf_data_set(buffers, totelem)) {
@@ -249,18 +186,15 @@ void GPU_pbvh_mesh_buffers_update(
 					GPU_vertbuf_attr_set(buffers->vert_buf, g_vbo_id.nor, i, v->no);
 				}
 
-				for (uint i = 0; i < buffers->face_indices_len; i++) {
-					const MLoopTri *lt = &buffers->looptri[buffers->face_indices[i]];
-					for (uint j = 0; j < 3; j++) {
-						int vidx = face_vert_indices[i][j];
-						if (vmask && show_mask) {
+				if (vmask && show_mask) {
+					for (uint i = 0; i < buffers->face_indices_len; i++) {
+						const MLoopTri *lt = &buffers->looptri[buffers->face_indices[i]];
+						for (uint j = 0; j < 3; j++) {
+							int vidx = face_vert_indices[i][j];
 							int v_index = buffers->mloop[lt->tri[j]].v;
-							uchar color_ub[3];
-							gpu_color_from_mask_copy(vmask[v_index], diffuse_color, color_ub);
-							GPU_vertbuf_attr_set(buffers->vert_buf, g_vbo_id.col, vidx, color_ub);
-						}
-						else {
-							GPU_vertbuf_attr_set(buffers->vert_buf, g_vbo_id.col, vidx, diffuse_color_ub);
+							float fmask = vmask[v_index];
+							GPU_vertbuf_attr_set(buffers->vert_buf, g_vbo_id.msk, vidx, &fmask);
+							empty_mask = empty_mask && (fmask == 0.0f);
 						}
 					}
 				}
@@ -291,13 +225,9 @@ void GPU_pbvh_mesh_buffers_update(
 						mpoly_prev = lt->poly;
 					}
 
-					uchar color_ub[3];
+					float fmask = 0.0f;
 					if (vmask && show_mask) {
-						float fmask = (vmask[vtri[0]] + vmask[vtri[1]] + vmask[vtri[2]]) / 3.0f;
-						gpu_color_from_mask_copy(fmask, diffuse_color, color_ub);
-					}
-					else {
-						copy_v3_v3_uchar(color_ub, diffuse_color_ub);
+						fmask = (vmask[vtri[0]] + vmask[vtri[1]] + vmask[vtri[2]]) / 3.0f;
 					}
 
 					for (uint j = 0; j < 3; j++) {
@@ -305,10 +235,12 @@ void GPU_pbvh_mesh_buffers_update(
 
 						GPU_vertbuf_attr_set(buffers->vert_buf, g_vbo_id.pos, vbo_index, v->co);
 						GPU_vertbuf_attr_set(buffers->vert_buf, g_vbo_id.nor, vbo_index, no);
-						GPU_vertbuf_attr_set(buffers->vert_buf, g_vbo_id.col, vbo_index, color_ub);
+						GPU_vertbuf_attr_set(buffers->vert_buf, g_vbo_id.msk, vbo_index, &fmask);
 
 						vbo_index++;
 					}
+
+					empty_mask = empty_mask && (fmask == 0.0f);
 				}
 			}
 
@@ -316,6 +248,8 @@ void GPU_pbvh_mesh_buffers_update(
 		}
 	}
 
+	buffers->show_mask = !empty_mask;
+	printf("buffers->show_mask %d\n", buffers->show_mask);
 	buffers->mvert = mvert;
 }
 
@@ -339,8 +273,7 @@ GPU_PBVH_Buffers *GPU_pbvh_mesh_buffers_build(
 	buffers->smooth = (mpoly[0].flag & ME_SMOOTH) != 0;
 #endif
 
-	buffers->show_diffuse_color = false;
-	buffers->show_mask = true;
+	buffers->show_mask = false;
 
 	/* Count the number of visible triangles */
 	for (i = 0, tottri = 0; i < face_indices_len; ++i) {
@@ -407,26 +340,15 @@ void GPU_pbvh_grid_buffers_update(
         int totgrid, const CCGKey *key,
         const int update_flags)
 {
-	const bool show_diffuse_color = (update_flags & GPU_PBVH_BUFFERS_SHOW_DIFFUSE_COLOR) != 0;
 	const bool show_mask = (update_flags & GPU_PBVH_BUFFERS_SHOW_MASK) != 0;
+	bool empty_mask = true;
 	int i, j, k, x, y;
 
-	buffers->show_diffuse_color = show_diffuse_color;
-	buffers->show_mask = show_mask;
 	buffers->smooth = grid_flag_mats[grid_indices[0]].flag & ME_SMOOTH;
 
 	/* Build VBO */
 	if (buffers->index_buf) {
 		const int has_mask = key->has_mask;
-		float diffuse_color[4] = {0.8f, 0.8f, 0.8f, 1.0f};
-
-		if (show_diffuse_color) {
-			const DMFlagMat *flags = &grid_flag_mats[grid_indices[0]];
-
-			gpu_material_diffuse_get(flags->mat_nr + 1, diffuse_color);
-		}
-
-		copy_v4_v4(buffers->diffuse_color, diffuse_color);
 
 		uint vbo_index_offset = 0;
 		/* Build VBO */
@@ -445,16 +367,10 @@ void GPU_pbvh_grid_buffers_update(
 							normal_float_to_short_v3(no_short, CCG_elem_no(key, elem));
 							GPU_vertbuf_attr_set(buffers->vert_buf, g_vbo_id.nor, vbo_index, no_short);
 
-							if (has_mask) {
-								uchar color_ub[3];
-								if (show_mask) {
-									gpu_color_from_mask_copy(*CCG_elem_mask(key, elem),
-									                         diffuse_color, color_ub);
-								}
-								else {
-									unit_float_to_uchar_clamp_v3(color_ub, diffuse_color);
-								}
-								GPU_vertbuf_attr_set(buffers->vert_buf, g_vbo_id.col, vbo_index, color_ub);
+							if (has_mask && show_mask) {
+								float fmask = *CCG_elem_mask(key, elem);
+								GPU_vertbuf_attr_set(buffers->vert_buf, g_vbo_id.msk, vbo_index, &fmask);
+								empty_mask = empty_mask && (fmask == 0.0f);
 							}
 						}
 						vbo_index += 1;
@@ -483,21 +399,13 @@ void GPU_pbvh_grid_buffers_update(
 							normal_float_to_short_v3(no_short, fno);
 							GPU_vertbuf_attr_set(buffers->vert_buf, g_vbo_id.nor, vbo_index, no_short);
 
-							if (has_mask) {
-								uchar color_ub[3];
-								if (show_mask) {
-									gpu_color_from_mask_quad_copy(key,
-									                              elems[0],
-									                              elems[1],
-									                              elems[2],
-									                              elems[3],
-									                              diffuse_color,
-									                              color_ub);
-								}
-								else {
-									unit_float_to_uchar_clamp_v3(color_ub, diffuse_color);
-								}
-								GPU_vertbuf_attr_set(buffers->vert_buf, g_vbo_id.col, vbo_index, color_ub);
+							if (has_mask && show_mask) {
+								float fmask = (*CCG_elem_mask(key, elems[0]) +
+								               *CCG_elem_mask(key, elems[1]) +
+								               *CCG_elem_mask(key, elems[2]) +
+								               *CCG_elem_mask(key, elems[3])) * 0.25f;
+								GPU_vertbuf_attr_set(buffers->vert_buf, g_vbo_id.msk, vbo_index, &fmask);
+								empty_mask = empty_mask && (fmask == 0.0f);
 							}
 						}
 					}
@@ -515,6 +423,7 @@ void GPU_pbvh_grid_buffers_update(
 	buffers->totgrid = totgrid;
 	buffers->grid_flag_mats = grid_flag_mats;
 	buffers->gridkey = *key;
+	buffers->show_mask = !empty_mask;
 
 	//printf("node updated %p\n", buffers);
 }
@@ -628,8 +537,7 @@ GPU_PBVH_Buffers *GPU_pbvh_grid_buffers_build(
 	buffers->grid_hidden = grid_hidden;
 	buffers->totgrid = totgrid;
 
-	buffers->show_diffuse_color = false;
-	buffers->show_mask = true;
+	buffers->show_mask = false;
 
 	/* Count the number of quads */
 	totquad = BKE_pbvh_count_grid_quads(grid_hidden, grid_indices, totgrid, gridsize);
@@ -689,36 +597,23 @@ static void gpu_bmesh_vert_to_buffer_copy__gwn(
         const float fno[3],
         const float *fmask,
         const int cd_vert_mask_offset,
-        const float diffuse_color[4],
-        const bool show_mask)
+        const bool show_mask,
+        bool *empty_mask)
 {
 	if (!BM_elem_flag_test(v, BM_ELEM_HIDDEN)) {
 
 		/* Set coord, normal, and mask */
 		GPU_vertbuf_attr_set(vert_buf, g_vbo_id.pos, *v_index, v->co);
 
-		{
-			short no_short[3];
-			normal_float_to_short_v3(no_short, fno ? fno : v->no);
-			GPU_vertbuf_attr_set(vert_buf, g_vbo_id.nor, *v_index, no_short);
-		}
-
-		{
-			uchar color_ub[3];
-			float effective_mask;
-			if (show_mask) {
-				effective_mask = fmask ? *fmask
-				                       : BM_ELEM_CD_GET_FLOAT(v, cd_vert_mask_offset);
-			}
-			else {
-				effective_mask = 0.0f;
-			}
+		short no_short[3];
+		normal_float_to_short_v3(no_short, fno ? fno : v->no);
+		GPU_vertbuf_attr_set(vert_buf, g_vbo_id.nor, *v_index, no_short);
 
-			gpu_color_from_mask_copy(
-			        effective_mask,
-			        diffuse_color,
-			        color_ub);
-			GPU_vertbuf_attr_set(vert_buf, g_vbo_id.col, *v_index, color_ub);
+		if (show_mask) {
+			float effective_mask = fmask ? *fmask
+			                             : BM_ELEM_CD_GET_FLOAT(v, cd_vert_mask_offset);
+			GPU_vertbuf_attr_set(vert_buf, g_vbo_id.msk, *v_index, &effective_mask);
+			*empty_mask = *empty_mask && (effective_mask == 0.0f);
 		}
 
 		/* Assign index for use in the triangle index buffer */
@@ -776,17 +671,13 @@ void GPU_pbvh_bmesh_buffers_update(
         GSet *bm_other_verts,
         const int update_flags)
 {
-	const bool show_diffuse_color = (update_flags & GPU_PBVH_BUFFERS_SHOW_DIFFUSE_COLOR) != 0;
 	const bool show_mask = (update_flags & GPU_PBVH_BUFFERS_SHOW_MASK) != 0;
 	int tottri, totvert, maxvert = 0;
-	float diffuse_color[4] = {0.8f, 0.8f, 0.8f, 1.0f};
+	bool empty_mask = true;
 
 	/* TODO, make mask layer optional for bmesh buffer */
 	const int cd_vert_mask_offset = CustomData_get_offset(&bm->vdata, CD_PAINT_MASK);
 
-	buffers->show_diffuse_color = show_diffuse_color;
-	buffers->show_mask = show_mask;
-
 	/* Count visible triangles */
 	tottri = gpu_bmesh_face_visible_count(bm_faces);
 
@@ -805,17 +696,6 @@ void GPU_pbvh_bmesh_buffers_update(
 		return;
 	}
 
-	if (show_diffuse_color) {
-		/* due to dynamic nature of dyntopo, only get first material */
-		GSetIterator gs_iter;
-		BMFace *f;
-		BLI_gsetIterator_init(&gs_iter, bm_faces);
-		f = BLI_gsetIterator_getKey(&gs_iter);
-		gpu_material_diffuse_get(f->mat_nr + 1, diffuse_color);
-	}
-
-	copy_v4_v4(buffers->diffuse_color, diffuse_color);
-
 	/* Fill vertex buffer */
 	if (gpu_pbvh_vert_buf_data_set(buffers, totvert)) {
 		int v_index = 0;
@@ -831,16 +711,16 @@ void GPU_pbvh_bmesh_buffers_update(
 				gpu_bmesh_vert_to_buffer_copy__gwn(
 				        BLI_gsetIterator_getKey(&gs_iter),
 				        buffers->vert_buf, &v_index, NULL, NULL,
-				        cd_vert_mask_offset, diffuse_color,
-				        show_mask);
+				        cd_vert_mask_offset,
+				        show_mask, &empty_mask);
 			}
 
 			GSET_ITER (gs_iter, bm_other_verts) {
 				gpu_bmesh_vert_to_buffer_copy__gwn(
 				        BLI_gsetIterator_getKey(&gs_iter),
 				        buffers->vert_buf, &v_index, NULL, NULL,
-				        cd_vert_mask_offset, diffuse_color,
-				        show_mask);
+				        cd_vert_mask_offset,
+				        show_mask, &empty_mask);
 			}
 
 			maxvert = v_index;
@@ -855,7 +735,7 @@ void GPU_pbvh_bmesh_buffers_update(
 
 				if (!BM_elem_flag_test(f, BM_ELEM_HIDDEN)) {
 					BMVert *v[3];
-					float fmask = 0;
+					float fmask = 0.0f;
 					int i;
 
 #if 0
@@ -873,8 +753,8 @@ void GPU_pbvh_bmesh_buffers_update(
 						gpu_bmesh_vert_to_buffer_copy__gwn(
 						        v[i], buffers->vert_buf,
 						        &v_index, f->no, &fmask,
-						        cd_vert_mask_offset, diffuse_color,
-						        show_mask);
+						        cd_vert_mask_offset,
+						        show_mask, &empty_mask);
 					}
 				}
 			}
@@ -934,6 +814,8 @@ void GPU_pbvh_bmesh_buffers_update(
 		buffers->is_index_buf_global = false;
 	}
 
+	buffers->show_mask = !empty_mask;
+
 	gpu_pbvh_batch_init(buffers);
 }
 
@@ -944,7 +826,6 @@ GPU_PBVH_Buffers *GPU_pbvh_bmesh_buffers_build(bool smooth_shading)
 	buffers = MEM_callocN(sizeof(GPU_PBVH_Buffers), "GPU_Buffers");
 	buffers->use_bmesh = true;
 	buffers->smooth = smooth_shading;
-	buffers->show_diffuse_color = false;
 	buffers->show_mask = true;
 
 	return buffers;
@@ -956,48 +837,9 @@ GPUBatch *GPU_pbvh_buffers_batch_get(GPU_PBVH_Buffers *buffers, bool fast)
 	        buffers->triangles_fast : buffers->triangles;
 }
 
-bool GPU_pbvh_buffers_diffuse_changed(GPU_PBVH_Buffers *buffers, GSet *bm_faces, bool show_diffuse_color)
-{
-	float diffuse_color[4];
-
-	if (buffers->show_diffuse_color != show_diffuse_color)
-		return true;
-
-	if (buffers->show_diffuse_color == false)
-		return false;
-
-	if (buffers->looptri) {
-		const MLoopTri *lt = &buffers->looptri[buffers->face_indices[0]];
-		const MPoly *mp = &buffers->mpoly[lt->poly];
-
-		gpu_material_diffuse_get(mp->mat_nr + 1, diffuse_color);
-	}
-	else if (buffers->use_bmesh) {
-		/* due to dynamic nature of dyntopo, only get first material */
-		if (BLI_gset_len(bm_faces) > 0) {
-			GSetIterator gs_iter;
-			BMFace *f;
-
-			BLI_gsetIterator_init(&gs_iter, bm_faces);
-			f = BLI_gsetIterator_getKey(&gs_iter);
-			gpu_material_diffuse_get(f->mat_nr + 1, diffuse_color);
-		}
-		else {
-			return false;
-		}
-	}
-	else {
-		const DMFlagMat *flags = &buffers->grid_flag_mats[buffers->grid_indices[0]];
-
-		gpu_material_diffuse_get(flags->mat_nr + 1, diffuse_color);
-	}
-
-	return !equals_v3v3(diffuse_color, buffers->diffuse_color);
-}
-
-bool GPU_pbvh_buffers_mask_changed(GPU_PBVH_Buffers *buffers, bool show_mask)
+bool GPU_pbvh_buffers_has_mask(GPU_PBVH_Buffers *buffers)
 {
-	return (buffers->show_mask != show_mask);
+	return buffers->show_mask;
 }
 
 void GPU_pbvh_buffers_free(GPU_PBVH_Buffers *buffers)