Sculpt: Render Mask and Face Sets with modifiers active

This removes the limitation of the sculpt overlays not being visible
with modifiers active.

Reviewed By: fclem

Maniphest Tasks: T68900

Differential Revision: https://developer.blender.org/D8673

11 files changed, 166 insertions, 24 deletions

diff --git a/source/blender/blenkernel/BKE_paint.h b/source/blender/blenkernel/BKE_paint.h
index dfc75e2fd54..5af9c35cab6 100644
--- a/source/blender/blenkernel/BKE_paint.h
+++ b/source/blender/blenkernel/BKE_paint.h
@@ -204,6 +204,8 @@ bool paint_is_bmesh_face_hidden(struct BMFace *f);
 /* paint masks */
 float paint_grid_paint_mask(const struct GridPaintMask *gpm, uint level, uint x, uint y);
 
+void BKE_paint_face_set_overlay_color_get(const int face_set, const int seed, uchar r_color[4]);
+
 /* stroke related */
 bool paint_calculate_rake_rotation(struct UnifiedPaintSettings *ups,
                                    struct Brush *brush,
diff --git a/source/blender/blenkernel/intern/DerivedMesh.c b/source/blender/blenkernel/intern/DerivedMesh.c
index 681e46ed24e..bfdc8ad37e5 100644
--- a/source/blender/blenkernel/intern/DerivedMesh.c
+++ b/source/blender/blenkernel/intern/DerivedMesh.c
@@ -1054,7 +1054,6 @@ static void mesh_calc_modifiers(struct Depsgraph *depsgraph,
         }
         continue;
       }
-      BKE_modifier_set_error(md, "Sculpt: Hide, Mask and optimized display disabled");
     }
 
     if (need_mapping && !BKE_modifier_supports_mapping(md)) {
diff --git a/source/blender/blenkernel/intern/customdata.c b/source/blender/blenkernel/intern/customdata.c
index c5cc8448d1b..4757b0d5c1e 100644
--- a/source/blender/blenkernel/intern/customdata.c
+++ b/source/blender/blenkernel/intern/customdata.c
@@ -1994,7 +1994,8 @@ const CustomData_MeshMasks CD_MASK_EDITMESH = {
 };
 const CustomData_MeshMasks CD_MASK_DERIVEDMESH = {
     .vmask = (CD_MASK_ORIGINDEX | CD_MASK_MDEFORMVERT | CD_MASK_SHAPEKEY | CD_MASK_MVERT_SKIN |
-              CD_MASK_ORCO | CD_MASK_CLOTH_ORCO | CD_MASK_PROP_ALL | CD_MASK_PROP_COLOR),
+              CD_MASK_PAINT_MASK | CD_MASK_ORCO | CD_MASK_CLOTH_ORCO | CD_MASK_PROP_ALL |
+              CD_MASK_PROP_COLOR),
     .emask = (CD_MASK_ORIGINDEX | CD_MASK_FREESTYLE_EDGE | CD_MASK_PROP_ALL),
     .fmask = (CD_MASK_ORIGINDEX | CD_MASK_ORIGSPACE | CD_MASK_PREVIEW_MCOL | CD_MASK_TANGENT),
     .lmask = (CD_MASK_MLOOPUV | CD_MASK_MLOOPCOL | CD_MASK_CUSTOMLOOPNORMAL |
diff --git a/source/blender/blenkernel/intern/paint.c b/source/blender/blenkernel/intern/paint.c
index c9b3b3cc516..3ac9db0eb78 100644
--- a/source/blender/blenkernel/intern/paint.c
+++ b/source/blender/blenkernel/intern/paint.c
@@ -38,6 +38,7 @@
 #include "DNA_workspace_types.h"
 
 #include "BLI_bitmap.h"
+#include "BLI_hash.h"
 #include "BLI_listbase.h"
 #include "BLI_math_vector.h"
 #include "BLI_utildefines.h"
@@ -2101,3 +2102,21 @@ bool BKE_sculptsession_use_pbvh_draw(const Object *ob, const View3D *v3d)
   /* Multires and dyntopo always draw directly from the PBVH. */
   return true;
 }
+
+/* Returns the Face Set random color for rendering in the overlay given its ID and a color seed. */
+#define GOLDEN_RATIO_CONJUGATE 0.618033988749895f
+void BKE_paint_face_set_overlay_color_get(const int face_set, const int seed, uchar r_color[4])
+{
+  float rgba[4];
+  float random_mod_hue = GOLDEN_RATIO_CONJUGATE * (abs(face_set) + (seed % 10));
+  random_mod_hue = random_mod_hue - floorf(random_mod_hue);
+  const float random_mod_sat = BLI_hash_int_01(abs(face_set) + seed + 1);
+  const float random_mod_val = BLI_hash_int_01(abs(face_set) + seed + 2);
+  hsv_to_rgb(random_mod_hue,
+             0.6f + (random_mod_sat * 0.25f),
+             1.0f - (random_mod_val * 0.35f),
+             &rgba[0],
+             &rgba[1],
+             &rgba[2]);
+  rgba_float_to_uchar(r_color, rgba);
+}
diff --git a/source/blender/draw/engines/overlay/overlay_sculpt.c b/source/blender/draw/engines/overlay/overlay_sculpt.c
index b36252ead30..c6c617fd29b 100644
--- a/source/blender/draw/engines/overlay/overlay_sculpt.c
+++ b/source/blender/draw/engines/overlay/overlay_sculpt.c
@@ -22,6 +22,7 @@
 
 #include "DRW_render.h"
 
+#include "draw_cache_impl.h"
 #include "overlay_private.h"
 
 #include "BKE_paint.h"
@@ -48,14 +49,21 @@ void OVERLAY_sculpt_cache_populate(OVERLAY_Data *vedata, Object *ob)
 {
   OVERLAY_PrivateData *pd = vedata->stl->pd;
   const DRWContextState *draw_ctx = DRW_context_state_get();
+  struct GPUBatch *sculpt_overlays;
   PBVH *pbvh = ob->sculpt->pbvh;
 
   const bool use_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d);
 
-  if (use_pbvh || !ob->sculpt->deform_modifiers_active || ob->sculpt->shapekey_active) {
-    if (!use_pbvh || pbvh_has_mask(pbvh) || pbvh_has_face_sets(pbvh)) {
+  if (pbvh_has_mask(pbvh) || pbvh_has_face_sets(pbvh)) {
+    if (use_pbvh) {
       DRW_shgroup_call_sculpt(pd->sculpt_mask_grp, ob, false, true);
     }
+    else {
+      sculpt_overlays = DRW_mesh_batch_cache_get_sculpt_overlays(ob->data);
+      if (sculpt_overlays) {
+        DRW_shgroup_call(pd->sculpt_mask_grp, sculpt_overlays, ob);
+      }
+    }
   }
 }
 
diff --git a/source/blender/draw/intern/draw_cache_extract.h b/source/blender/draw/intern/draw_cache_extract.h
index 5f77dac98be..2094da0328b 100644
--- a/source/blender/draw/intern/draw_cache_extract.h
+++ b/source/blender/draw/intern/draw_cache_extract.h
@@ -55,6 +55,7 @@ typedef struct DRW_MeshCDMask {
   uint32_t sculpt_vcol : 8;
   uint32_t orco : 1;
   uint32_t tan_orco : 1;
+  uint32_t sculpt_overlays : 1;
   /** Edit uv layer is from the base edit mesh as
    *  modifiers could remove it. (see T68857) */
   uint32_t edit_uv : 1;
@@ -103,6 +104,7 @@ typedef struct MeshBufferCache {
     GPUVertBuf *uv;
     GPUVertBuf *tan;
     GPUVertBuf *vcol;
+    GPUVertBuf *sculpt_data;
     GPUVertBuf *orco;
     /* Only for edit mode. */
     GPUVertBuf *edit_data; /* extend */
@@ -170,6 +172,7 @@ typedef enum DRWBatchFlag {
   MBC_WIRE_LOOPS = (1 << 24),
   MBC_WIRE_LOOPS_UVS = (1 << 25),
   MBC_SKIN_ROOTS = (1 << 26),
+  MBC_SCULPT_OVERLAYS = (1 << 27),
 } DRWBatchFlag;
 
 #define MBC_EDITUV \
@@ -219,6 +222,7 @@ typedef struct MeshBatchCache {
     GPUBatch *wire_edges;     /* Individual edges with face normals. */
     GPUBatch *wire_loops;     /* Loops around faces. no edges between selected faces */
     GPUBatch *wire_loops_uvs; /* Same as wire_loops but only has uvs. */
+    GPUBatch *sculpt_overlays;
   } batch;
 
   GPUBatch **surface_per_mat;
diff --git a/source/blender/draw/intern/draw_cache_extract_mesh.c b/source/blender/draw/intern/draw_cache_extract_mesh.c
index cb716b3130a..657cb2061bd 100644
--- a/source/blender/draw/intern/draw_cache_extract_mesh.c
+++ b/source/blender/draw/intern/draw_cache_extract_mesh.c
@@ -53,6 +53,7 @@
 #include "BKE_mesh_tangent.h"
 #include "BKE_modifier.h"
 #include "BKE_object_deform.h"
+#include "BKE_paint.h"
 
 #include "atomic_ops.h"
 
@@ -2565,6 +2566,106 @@ static const MeshExtract extract_tan_hq = {
 /** \} */
 
 /* ---------------------------------------------------------------------- */
+/** \name Extract Sculpt Data
+ * \{ */
+
+static void *extract_sculpt_data_init(const MeshRenderData *mr,
+                                      struct MeshBatchCache *UNUSED(cache),
+                                      void *buf)
+{
+  GPUVertFormat format = {0};
+
+  CustomData *cd_ldata = (mr->extract_type == MR_EXTRACT_BMESH) ? &mr->bm->ldata : &mr->me->ldata;
+  CustomData *cd_vdata = (mr->extract_type == MR_EXTRACT_BMESH) ? &mr->bm->vdata : &mr->me->vdata;
+  CustomData *cd_pdata = (mr->extract_type == MR_EXTRACT_BMESH) ? &mr->bm->pdata : &mr->me->pdata;
+
+  float *cd_mask = CustomData_get_layer(cd_vdata, CD_PAINT_MASK);
+  int *cd_face_set = CustomData_get_layer(cd_pdata, CD_SCULPT_FACE_SETS);
+
+  if (format.attr_len == 0) {
+    GPU_vertformat_attr_add(&format, "fset", GPU_COMP_U8, 4, GPU_FETCH_INT_TO_FLOAT_UNIT);
+    GPU_vertformat_attr_add(&format, "msk", GPU_COMP_F32, 1, GPU_FETCH_FLOAT);
+  }
+
+  GPUVertBuf *vbo = buf;
+  GPU_vertbuf_init_with_format(vbo, &format);
+  GPU_vertbuf_data_alloc(vbo, mr->loop_len);
+
+  typedef struct gpuSculptData {
+    uint8_t face_set_color[4];
+    float mask;
+  } gpuSculptData;
+
+  gpuSculptData *vbo_data = (gpuSculptData *)GPU_vertbuf_get_data(vbo);
+  MLoop *loops = CustomData_get_layer(cd_ldata, CD_MLOOP);
+
+  if (mr->extract_type == MR_EXTRACT_BMESH) {
+    int cd_mask_ofs = CustomData_get_offset(cd_vdata, CD_PAINT_MASK);
+    int cd_face_set_ofs = CustomData_get_offset(cd_pdata, CD_SCULPT_FACE_SETS);
+    BMIter f_iter;
+    BMFace *efa;
+    BM_ITER_MESH (efa, &f_iter, mr->bm, BM_FACES_OF_MESH) {
+      BMLoop *l_iter, *l_first;
+      l_iter = l_first = BM_FACE_FIRST_LOOP(efa);
+      do {
+        float v_mask = 0.0f;
+        if (cd_mask) {
+          v_mask = BM_ELEM_CD_GET_FLOAT(l_iter->v, cd_mask_ofs);
+        }
+        vbo_data->mask = v_mask;
+        uchar face_set_color[4] = {UCHAR_MAX, UCHAR_MAX, UCHAR_MAX, UCHAR_MAX};
+        if (cd_face_set) {
+          const int face_set_id = BM_ELEM_CD_GET_INT(l_iter->f, cd_face_set_ofs);
+          if (face_set_id != mr->me->face_sets_color_default) {
+            BKE_paint_face_set_overlay_color_get(
+                face_set_id, mr->me->face_sets_color_seed, face_set_color);
+          }
+        }
+        copy_v3_v3_uchar(vbo_data->face_set_color, face_set_color);
+        vbo_data++;
+      } while ((l_iter = l_iter->next) != l_first);
+    }
+  }
+  else {
+    int mp_loop = 0;
+    for (int mp_index = 0; mp_index < mr->poly_len; mp_index++) {
+      const MPoly *p = &mr->mpoly[mp_index];
+      for (int l = 0; l < p->totloop; l++) {
+        float v_mask = 0.0f;
+        if (cd_mask) {
+          v_mask = cd_mask[loops[mp_loop].v];
+        }
+        vbo_data->mask = v_mask;
+
+        uchar face_set_color[4] = {UCHAR_MAX, UCHAR_MAX, UCHAR_MAX, UCHAR_MAX};
+        if (cd_face_set) {
+          const int face_set_id = cd_face_set[mp_index];
+          /* Skip for the default color Face Set to render it white. */
+          if (face_set_id != mr->me->face_sets_color_default) {
+            BKE_paint_face_set_overlay_color_get(
+                face_set_id, mr->me->face_sets_color_seed, face_set_color);
+          }
+        }
+        copy_v3_v3_uchar(vbo_data->face_set_color, face_set_color);
+        mp_loop++;
+        vbo_data++;
+      }
+    }
+  }
+
+  return NULL;
+}
+
+static const MeshExtract extract_sculpt_data = {
+    .init = extract_sculpt_data_init,
+    .data_flag = 0,
+    /* TODO: enable threading. */
+    .use_threading = false,
+};
+
+/** \} */
+
+/* ---------------------------------------------------------------------- */
 /** \name Extract VCol
  * \{ */
 
@@ -5598,6 +5699,7 @@ void mesh_buffer_cache_create_requested(struct TaskGraph *task_graph,
   TEST_ASSIGN(VBO, vbo, uv);
   TEST_ASSIGN(VBO, vbo, tan);
   TEST_ASSIGN(VBO, vbo, vcol);
+  TEST_ASSIGN(VBO, vbo, sculpt_data);
   TEST_ASSIGN(VBO, vbo, orco);
   TEST_ASSIGN(VBO, vbo, edge_fac);
   TEST_ASSIGN(VBO, vbo, weights);
@@ -5691,6 +5793,7 @@ void mesh_buffer_cache_create_requested(struct TaskGraph *task_graph,
   EXTRACT(vbo, uv);
   EXTRACT(vbo, tan);
   EXTRACT(vbo, vcol);
+  EXTRACT(vbo, sculpt_data);
   EXTRACT(vbo, orco);
   EXTRACT(vbo, edge_fac);
   EXTRACT(vbo, weights);
diff --git a/source/blender/draw/intern/draw_cache_impl.h b/source/blender/draw/intern/draw_cache_impl.h
index 784e52cfa17..9e7a2e2916c 100644
--- a/source/blender/draw/intern/draw_cache_impl.h
+++ b/source/blender/draw/intern/draw_cache_impl.h
@@ -175,6 +175,7 @@ struct GPUBatch *DRW_mesh_batch_cache_get_surface_texpaint_single(struct Mesh *m
 struct GPUBatch *DRW_mesh_batch_cache_get_surface_vertpaint(struct Mesh *me);
 struct GPUBatch *DRW_mesh_batch_cache_get_surface_sculpt(struct Mesh *me);
 struct GPUBatch *DRW_mesh_batch_cache_get_surface_weights(struct Mesh *me);
+struct GPUBatch *DRW_mesh_batch_cache_get_sculpt_overlays(struct Mesh *me);
 /* edit-mesh drawing */
 struct GPUBatch *DRW_mesh_batch_cache_get_edit_triangles(struct Mesh *me);
 struct GPUBatch *DRW_mesh_batch_cache_get_edit_vertices(struct Mesh *me);
diff --git a/source/blender/draw/intern/draw_cache_impl_mesh.c b/source/blender/draw/intern/draw_cache_impl_mesh.c
index d090832dc4b..6df0c7cc5ad 100644
--- a/source/blender/draw/intern/draw_cache_impl_mesh.c
+++ b/source/blender/draw/intern/draw_cache_impl_mesh.c
@@ -891,6 +891,17 @@ int DRW_mesh_material_count_get(Mesh *me)
   return mesh_render_mat_len_get(me);
 }
 
+GPUBatch *DRW_mesh_batch_cache_get_sculpt_overlays(Mesh *me)
+{
+  MeshBatchCache *cache = mesh_batch_cache_get(me);
+
+  cache->cd_needed.sculpt_overlays = 1;
+  mesh_batch_cache_add_request(cache, MBC_SCULPT_OVERLAYS);
+  DRW_batch_request(&cache->batch.sculpt_overlays);
+
+  return cache->batch.sculpt_overlays;
+}
+
 /** \} */
 
 /* ---------------------------------------------------------------------- */
@@ -1242,6 +1253,9 @@ void DRW_mesh_batch_cache_create_requested(struct TaskGraph *task_graph,
         if (cache->cd_used.orco != cache->cd_needed.orco) {
           GPU_VERTBUF_DISCARD_SAFE(mbuffercache->vbo.orco);
         }
+        if (cache->cd_used.sculpt_overlays != cache->cd_needed.sculpt_overlays) {
+          GPU_VERTBUF_DISCARD_SAFE(mbuffercache->vbo.sculpt_data);
+        }
         if (((cache->cd_used.vcol & cache->cd_needed.vcol) != cache->cd_needed.vcol) ||
             ((cache->cd_used.sculpt_vcol & cache->cd_needed.sculpt_vcol) !=
              cache->cd_needed.sculpt_vcol)) {
@@ -1333,6 +1347,11 @@ void DRW_mesh_batch_cache_create_requested(struct TaskGraph *task_graph,
   if (DRW_batch_requested(cache->batch.all_verts, GPU_PRIM_POINTS)) {
     DRW_vbo_request(cache->batch.all_verts, &mbufcache->vbo.pos_nor);
   }
+  if (DRW_batch_requested(cache->batch.sculpt_overlays, GPU_PRIM_TRIS)) {
+    DRW_ibo_request(cache->batch.sculpt_overlays, &mbufcache->ibo.tris);
+    DRW_vbo_request(cache->batch.sculpt_overlays, &mbufcache->vbo.pos_nor);
+    DRW_vbo_request(cache->batch.sculpt_overlays, &mbufcache->vbo.sculpt_data);
+  }
   if (DRW_batch_requested(cache->batch.all_edges, GPU_PRIM_LINES)) {
     DRW_ibo_request(cache->batch.all_edges, &mbufcache->ibo.lines);
     DRW_vbo_request(cache->batch.all_edges, &mbufcache->vbo.pos_nor);
diff --git a/source/blender/editors/space_view3d/view3d_draw.c b/source/blender/editors/space_view3d/view3d_draw.c
index 0b5daece556..77abcdefdc4 100644
--- a/source/blender/editors/space_view3d/view3d_draw.c
+++ b/source/blender/editors/space_view3d/view3d_draw.c
@@ -2360,6 +2360,10 @@ void ED_view3d_datamask(const bContext *C,
       (v3d->overlay.edit_flag & V3D_OVERLAY_EDIT_WEIGHT)) {
     r_cddata_masks->vmask |= CD_MASK_MDEFORMVERT;
   }
+  if ((CTX_data_mode_enum(C) == CTX_MODE_SCULPT)) {
+    r_cddata_masks->vmask |= CD_MASK_PAINT_MASK;
+    r_cddata_masks->pmask |= CD_MASK_SCULPT_FACE_SETS;
+  }
 }
 
 /* Goes over all modes and view3d settings. */
diff --git a/source/blender/gpu/intern/gpu_buffers.c b/source/blender/gpu/intern/gpu_buffers.c
index 606af5b1a15..13d0139e406 100644
--- a/source/blender/gpu/intern/gpu_buffers.c
+++ b/source/blender/gpu/intern/gpu_buffers.c
@@ -195,24 +195,6 @@ static void gpu_pbvh_batch_init(GPU_PBVH_Buffers *buffers, GPUPrimType prim)
 /** \name Mesh PBVH
  * \{ */
 
-/* Returns the Face Set random color for rendering in the overlay given its ID and a color seed. */
-#define GOLDEN_RATIO_CONJUGATE 0.618033988749895f
-static void face_set_overlay_color_get(const int face_set, const int seed, uchar *r_color)
-{
-  float rgba[4];
-  float random_mod_hue = GOLDEN_RATIO_CONJUGATE * (abs(face_set) + (seed % 10));
-  random_mod_hue = random_mod_hue - floorf(random_mod_hue);
-  const float random_mod_sat = BLI_hash_int_01(abs(face_set) + seed + 1);
-  const float random_mod_val = BLI_hash_int_01(abs(face_set) + seed + 2);
-  hsv_to_rgb(random_mod_hue,
-             0.6f + (random_mod_sat * 0.25f),
-             1.0f - (random_mod_val * 0.35f),
-             &rgba[0],
-             &rgba[1],
-             &rgba[2]);
-  rgba_float_to_uchar(r_color, rgba);
-}
-
 static bool gpu_pbvh_is_looptri_visible(const MLoopTri *lt,
                                         const MVert *mvert,
                                         const MLoop *mloop,
@@ -290,7 +272,7 @@ void GPU_pbvh_mesh_buffers_update(GPU_PBVH_Buffers *buffers,
           const int fset = abs(sculpt_face_sets[lt->poly]);
           /* Skip for the default color Face Set to render it white. */
           if (fset != face_sets_color_default) {
-            face_set_overlay_color_get(fset, face_sets_color_seed, face_set_color);
+            BKE_paint_face_set_overlay_color_get(fset, face_sets_color_seed, face_set_color);
             default_face_set = false;
           }
         }
@@ -672,7 +654,7 @@ void GPU_pbvh_grid_buffers_update(GPU_PBVH_Buffers *buffers,
         const int fset = abs(sculpt_face_sets[face_index]);
         /* Skip for the default color Face Set to render it white. */
         if (fset != face_sets_color_default) {
-          face_set_overlay_color_get(fset, face_sets_color_seed, face_set_color);
+          BKE_paint_face_set_overlay_color_get(fset, face_sets_color_seed, face_set_color);
           default_face_set = false;
         }
       }