Sculpt: Render Mask and Face Sets with modifiers active

This removes the limitation of the sculpt overlays not being visible
with modifiers active.

Reviewed By: fclem

Maniphest Tasks: T68900

Differential Revision: https://developer.blender.org/D8673

5 files changed, 137 insertions, 2 deletions

diff --git a/source/blender/draw/engines/overlay/overlay_sculpt.c b/source/blender/draw/engines/overlay/overlay_sculpt.c
index b36252ead30..c6c617fd29b 100644
--- a/source/blender/draw/engines/overlay/overlay_sculpt.c
+++ b/source/blender/draw/engines/overlay/overlay_sculpt.c
@@ -22,6 +22,7 @@
 
 #include "DRW_render.h"
 
+#include "draw_cache_impl.h"
 #include "overlay_private.h"
 
 #include "BKE_paint.h"
@@ -48,14 +49,21 @@ void OVERLAY_sculpt_cache_populate(OVERLAY_Data *vedata, Object *ob)
 {
   OVERLAY_PrivateData *pd = vedata->stl->pd;
   const DRWContextState *draw_ctx = DRW_context_state_get();
+  struct GPUBatch *sculpt_overlays;
   PBVH *pbvh = ob->sculpt->pbvh;
 
   const bool use_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d);
 
-  if (use_pbvh || !ob->sculpt->deform_modifiers_active || ob->sculpt->shapekey_active) {
-    if (!use_pbvh || pbvh_has_mask(pbvh) || pbvh_has_face_sets(pbvh)) {
+  if (pbvh_has_mask(pbvh) || pbvh_has_face_sets(pbvh)) {
+    if (use_pbvh) {
       DRW_shgroup_call_sculpt(pd->sculpt_mask_grp, ob, false, true);
     }
+    else {
+      sculpt_overlays = DRW_mesh_batch_cache_get_sculpt_overlays(ob->data);
+      if (sculpt_overlays) {
+        DRW_shgroup_call(pd->sculpt_mask_grp, sculpt_overlays, ob);
+      }
+    }
   }
 }
 
diff --git a/source/blender/draw/intern/draw_cache_extract.h b/source/blender/draw/intern/draw_cache_extract.h
index 5f77dac98be..2094da0328b 100644
--- a/source/blender/draw/intern/draw_cache_extract.h
+++ b/source/blender/draw/intern/draw_cache_extract.h
@@ -55,6 +55,7 @@ typedef struct DRW_MeshCDMask {
   uint32_t sculpt_vcol : 8;
   uint32_t orco : 1;
   uint32_t tan_orco : 1;
+  uint32_t sculpt_overlays : 1;
   /** Edit uv layer is from the base edit mesh as
    *  modifiers could remove it. (see T68857) */
   uint32_t edit_uv : 1;
@@ -103,6 +104,7 @@ typedef struct MeshBufferCache {
     GPUVertBuf *uv;
     GPUVertBuf *tan;
     GPUVertBuf *vcol;
+    GPUVertBuf *sculpt_data;
     GPUVertBuf *orco;
     /* Only for edit mode. */
     GPUVertBuf *edit_data; /* extend */
@@ -170,6 +172,7 @@ typedef enum DRWBatchFlag {
   MBC_WIRE_LOOPS = (1 << 24),
   MBC_WIRE_LOOPS_UVS = (1 << 25),
   MBC_SKIN_ROOTS = (1 << 26),
+  MBC_SCULPT_OVERLAYS = (1 << 27),
 } DRWBatchFlag;
 
 #define MBC_EDITUV \
@@ -219,6 +222,7 @@ typedef struct MeshBatchCache {
     GPUBatch *wire_edges;     /* Individual edges with face normals. */
     GPUBatch *wire_loops;     /* Loops around faces. no edges between selected faces */
     GPUBatch *wire_loops_uvs; /* Same as wire_loops but only has uvs. */
+    GPUBatch *sculpt_overlays;
   } batch;
 
   GPUBatch **surface_per_mat;
diff --git a/source/blender/draw/intern/draw_cache_extract_mesh.c b/source/blender/draw/intern/draw_cache_extract_mesh.c
index cb716b3130a..657cb2061bd 100644
--- a/source/blender/draw/intern/draw_cache_extract_mesh.c
+++ b/source/blender/draw/intern/draw_cache_extract_mesh.c
@@ -53,6 +53,7 @@
 #include "BKE_mesh_tangent.h"
 #include "BKE_modifier.h"
 #include "BKE_object_deform.h"
+#include "BKE_paint.h"
 
 #include "atomic_ops.h"
 
@@ -2565,6 +2566,106 @@ static const MeshExtract extract_tan_hq = {
 /** \} */
 
 /* ---------------------------------------------------------------------- */
+/** \name Extract Sculpt Data
+ * \{ */
+
+static void *extract_sculpt_data_init(const MeshRenderData *mr,
+                                      struct MeshBatchCache *UNUSED(cache),
+                                      void *buf)
+{
+  GPUVertFormat format = {0};
+
+  CustomData *cd_ldata = (mr->extract_type == MR_EXTRACT_BMESH) ? &mr->bm->ldata : &mr->me->ldata;
+  CustomData *cd_vdata = (mr->extract_type == MR_EXTRACT_BMESH) ? &mr->bm->vdata : &mr->me->vdata;
+  CustomData *cd_pdata = (mr->extract_type == MR_EXTRACT_BMESH) ? &mr->bm->pdata : &mr->me->pdata;
+
+  float *cd_mask = CustomData_get_layer(cd_vdata, CD_PAINT_MASK);
+  int *cd_face_set = CustomData_get_layer(cd_pdata, CD_SCULPT_FACE_SETS);
+
+  if (format.attr_len == 0) {
+    GPU_vertformat_attr_add(&format, "fset", GPU_COMP_U8, 4, GPU_FETCH_INT_TO_FLOAT_UNIT);
+    GPU_vertformat_attr_add(&format, "msk", GPU_COMP_F32, 1, GPU_FETCH_FLOAT);
+  }
+
+  GPUVertBuf *vbo = buf;
+  GPU_vertbuf_init_with_format(vbo, &format);
+  GPU_vertbuf_data_alloc(vbo, mr->loop_len);
+
+  typedef struct gpuSculptData {
+    uint8_t face_set_color[4];
+    float mask;
+  } gpuSculptData;
+
+  gpuSculptData *vbo_data = (gpuSculptData *)GPU_vertbuf_get_data(vbo);
+  MLoop *loops = CustomData_get_layer(cd_ldata, CD_MLOOP);
+
+  if (mr->extract_type == MR_EXTRACT_BMESH) {
+    int cd_mask_ofs = CustomData_get_offset(cd_vdata, CD_PAINT_MASK);
+    int cd_face_set_ofs = CustomData_get_offset(cd_pdata, CD_SCULPT_FACE_SETS);
+    BMIter f_iter;
+    BMFace *efa;
+    BM_ITER_MESH (efa, &f_iter, mr->bm, BM_FACES_OF_MESH) {
+      BMLoop *l_iter, *l_first;
+      l_iter = l_first = BM_FACE_FIRST_LOOP(efa);
+      do {
+        float v_mask = 0.0f;
+        if (cd_mask) {
+          v_mask = BM_ELEM_CD_GET_FLOAT(l_iter->v, cd_mask_ofs);
+        }
+        vbo_data->mask = v_mask;
+        uchar face_set_color[4] = {UCHAR_MAX, UCHAR_MAX, UCHAR_MAX, UCHAR_MAX};
+        if (cd_face_set) {
+          const int face_set_id = BM_ELEM_CD_GET_INT(l_iter->f, cd_face_set_ofs);
+          if (face_set_id != mr->me->face_sets_color_default) {
+            BKE_paint_face_set_overlay_color_get(
+                face_set_id, mr->me->face_sets_color_seed, face_set_color);
+          }
+        }
+        copy_v3_v3_uchar(vbo_data->face_set_color, face_set_color);
+        vbo_data++;
+      } while ((l_iter = l_iter->next) != l_first);
+    }
+  }
+  else {
+    int mp_loop = 0;
+    for (int mp_index = 0; mp_index < mr->poly_len; mp_index++) {
+      const MPoly *p = &mr->mpoly[mp_index];
+      for (int l = 0; l < p->totloop; l++) {
+        float v_mask = 0.0f;
+        if (cd_mask) {
+          v_mask = cd_mask[loops[mp_loop].v];
+        }
+        vbo_data->mask = v_mask;
+
+        uchar face_set_color[4] = {UCHAR_MAX, UCHAR_MAX, UCHAR_MAX, UCHAR_MAX};
+        if (cd_face_set) {
+          const int face_set_id = cd_face_set[mp_index];
+          /* Skip for the default color Face Set to render it white. */
+          if (face_set_id != mr->me->face_sets_color_default) {
+            BKE_paint_face_set_overlay_color_get(
+                face_set_id, mr->me->face_sets_color_seed, face_set_color);
+          }
+        }
+        copy_v3_v3_uchar(vbo_data->face_set_color, face_set_color);
+        mp_loop++;
+        vbo_data++;
+      }
+    }
+  }
+
+  return NULL;
+}
+
+static const MeshExtract extract_sculpt_data = {
+    .init = extract_sculpt_data_init,
+    .data_flag = 0,
+    /* TODO: enable threading. */
+    .use_threading = false,
+};
+
+/** \} */
+
+/* ---------------------------------------------------------------------- */
 /** \name Extract VCol
  * \{ */
 
@@ -5598,6 +5699,7 @@ void mesh_buffer_cache_create_requested(struct TaskGraph *task_graph,
   TEST_ASSIGN(VBO, vbo, uv);
   TEST_ASSIGN(VBO, vbo, tan);
   TEST_ASSIGN(VBO, vbo, vcol);
+  TEST_ASSIGN(VBO, vbo, sculpt_data);
   TEST_ASSIGN(VBO, vbo, orco);
   TEST_ASSIGN(VBO, vbo, edge_fac);
   TEST_ASSIGN(VBO, vbo, weights);
@@ -5691,6 +5793,7 @@ void mesh_buffer_cache_create_requested(struct TaskGraph *task_graph,
   EXTRACT(vbo, uv);
   EXTRACT(vbo, tan);
   EXTRACT(vbo, vcol);
+  EXTRACT(vbo, sculpt_data);
   EXTRACT(vbo, orco);
   EXTRACT(vbo, edge_fac);
   EXTRACT(vbo, weights);
diff --git a/source/blender/draw/intern/draw_cache_impl.h b/source/blender/draw/intern/draw_cache_impl.h
index 784e52cfa17..9e7a2e2916c 100644
--- a/source/blender/draw/intern/draw_cache_impl.h
+++ b/source/blender/draw/intern/draw_cache_impl.h
@@ -175,6 +175,7 @@ struct GPUBatch *DRW_mesh_batch_cache_get_surface_texpaint_single(struct Mesh *m
 struct GPUBatch *DRW_mesh_batch_cache_get_surface_vertpaint(struct Mesh *me);
 struct GPUBatch *DRW_mesh_batch_cache_get_surface_sculpt(struct Mesh *me);
 struct GPUBatch *DRW_mesh_batch_cache_get_surface_weights(struct Mesh *me);
+struct GPUBatch *DRW_mesh_batch_cache_get_sculpt_overlays(struct Mesh *me);
 /* edit-mesh drawing */
 struct GPUBatch *DRW_mesh_batch_cache_get_edit_triangles(struct Mesh *me);
 struct GPUBatch *DRW_mesh_batch_cache_get_edit_vertices(struct Mesh *me);
diff --git a/source/blender/draw/intern/draw_cache_impl_mesh.c b/source/blender/draw/intern/draw_cache_impl_mesh.c
index d090832dc4b..6df0c7cc5ad 100644
--- a/source/blender/draw/intern/draw_cache_impl_mesh.c
+++ b/source/blender/draw/intern/draw_cache_impl_mesh.c
@@ -891,6 +891,17 @@ int DRW_mesh_material_count_get(Mesh *me)
   return mesh_render_mat_len_get(me);
 }
 
+GPUBatch *DRW_mesh_batch_cache_get_sculpt_overlays(Mesh *me)
+{
+  MeshBatchCache *cache = mesh_batch_cache_get(me);
+
+  cache->cd_needed.sculpt_overlays = 1;
+  mesh_batch_cache_add_request(cache, MBC_SCULPT_OVERLAYS);
+  DRW_batch_request(&cache->batch.sculpt_overlays);
+
+  return cache->batch.sculpt_overlays;
+}
+
 /** \} */
 
 /* ---------------------------------------------------------------------- */
@@ -1242,6 +1253,9 @@ void DRW_mesh_batch_cache_create_requested(struct TaskGraph *task_graph,
         if (cache->cd_used.orco != cache->cd_needed.orco) {
           GPU_VERTBUF_DISCARD_SAFE(mbuffercache->vbo.orco);
         }
+        if (cache->cd_used.sculpt_overlays != cache->cd_needed.sculpt_overlays) {
+          GPU_VERTBUF_DISCARD_SAFE(mbuffercache->vbo.sculpt_data);
+        }
         if (((cache->cd_used.vcol & cache->cd_needed.vcol) != cache->cd_needed.vcol) ||
             ((cache->cd_used.sculpt_vcol & cache->cd_needed.sculpt_vcol) !=
              cache->cd_needed.sculpt_vcol)) {
@@ -1333,6 +1347,11 @@ void DRW_mesh_batch_cache_create_requested(struct TaskGraph *task_graph,
   if (DRW_batch_requested(cache->batch.all_verts, GPU_PRIM_POINTS)) {
     DRW_vbo_request(cache->batch.all_verts, &mbufcache->vbo.pos_nor);
   }
+  if (DRW_batch_requested(cache->batch.sculpt_overlays, GPU_PRIM_TRIS)) {
+    DRW_ibo_request(cache->batch.sculpt_overlays, &mbufcache->ibo.tris);
+    DRW_vbo_request(cache->batch.sculpt_overlays, &mbufcache->vbo.pos_nor);
+    DRW_vbo_request(cache->batch.sculpt_overlays, &mbufcache->vbo.sculpt_data);
+  }
   if (DRW_batch_requested(cache->batch.all_edges, GPU_PRIM_LINES)) {
     DRW_ibo_request(cache->batch.all_edges, &mbufcache->ibo.lines);
     DRW_vbo_request(cache->batch.all_edges, &mbufcache->vbo.pos_nor);