Sculpt: Separate hide status from face sets, use generic attribute

Whether faces are hidden and face sets are orthogonal concepts, but
currently sculpt mode stores them together in the face set array.
This means that if anything is hidden, there must be face sets,
and if there are face sets, we have to keep track of what is hidden.
In other words, it adds a bunch of redundant work and state tracking.

On the user level it's nice that face sets and hiding are consistent,
but we don't need to store them together to accomplish that.

This commit uses the `".hide_poly"` attribute from rB2480b55f216c to
read and change hiding in sculpt mode. Face sets don't need to be
negative anymore, and a bunch of "face set <-> hide status" conversion
can be removed. Plus some other benefits:
 - We don't need to allocate either array quite as much.
 - The hide status can be read from 1/4 the memory as face sets.
 - Updates when entering or exiting sculpt mode can be removed.
 - More opportunities for early-outs when nothing is hidden.
 - Separating concerns makes sculpt code more obvious.
 - It will be easier to convert face sets into a generic int attribute.

 Differential Revision: https://developer.blender.org/D15950

1 files changed, 26 insertions, 119 deletions

diff --git a/source/blender/blenkernel/intern/paint.cc b/source/blender/blenkernel/intern/paint.cc
index 7ad9fd3a7c3..ea1bd3c1cc3 100644
--- a/source/blender/blenkernel/intern/paint.cc
+++ b/source/blender/blenkernel/intern/paint.cc
@@ -1712,6 +1712,8 @@ static void sculpt_update_object(
     ss->face_sets = nullptr;
   }
 
+  ss->hide_poly = (bool *)CustomData_get_layer_named(&me->pdata, CD_PROP_BOOL, ".hide_poly");
+
   ss->subdiv_ccg = me_eval->runtime.subdiv_ccg;
 
   PBVH *pbvh = BKE_sculpt_object_pbvh_ensure(depsgraph, ob);
@@ -1720,6 +1722,7 @@ static void sculpt_update_object(
 
   BKE_pbvh_subdiv_cgg_set(ss->pbvh, ss->subdiv_ccg);
   BKE_pbvh_face_sets_set(ss->pbvh, ss->face_sets);
+  BKE_pbvh_update_hide_attributes_from_mesh(ss->pbvh);
 
   BKE_pbvh_face_sets_color_set(ss->pbvh, me->face_sets_color_seed, me->face_sets_color_default);
 
@@ -1940,29 +1943,21 @@ int *BKE_sculpt_face_sets_ensure(Mesh *mesh)
           &mesh->pdata, CD_SCULPT_FACE_SETS, CD_CONSTRUCT, nullptr, mesh->totpoly)),
       mesh->totpoly};
 
-  /* Initialize the new face sets with a default valid visible ID and set the default
-   * color to render it white. */
-  if (hide_poly.is_single() && !hide_poly.get_internal_single()) {
-    face_sets.fill(1);
-  }
-  else {
-    const int face_sets_default_visible_id = 1;
-    const int face_sets_default_hidden_id = -2;
-
-    const VArraySpan<bool> hide_poly_span{hide_poly};
-    for (const int i : face_sets.index_range()) {
-      /* Assign a new hidden ID to hidden faces. This way we get at initial split in two Face Sets
-       * between hidden and visible faces based on the previous mesh visibly from other mode that
-       * can be useful in some cases. */
-      face_sets[i] = hide_poly_span[i] ? face_sets_default_hidden_id :
-                                         face_sets_default_visible_id;
-    }
-  }
-
+  face_sets.fill(1);
   mesh->face_sets_color_default = 1;
   return face_sets.data();
 }
 
+bool *BKE_sculpt_hide_poly_ensure(Mesh *mesh)
+{
+  if (bool *hide_poly = static_cast<bool *>(
+          CustomData_get_layer_named(&mesh->pdata, CD_PROP_BOOL, ".hide_poly"))) {
+    return hide_poly;
+  }
+  return static_cast<bool *>(CustomData_add_layer_named(
+      &mesh->pdata, CD_PROP_BOOL, CD_SET_DEFAULT, nullptr, mesh->totpoly, ".hide_poly"));
+}
+
 int BKE_sculpt_mask_layers_ensure(Object *ob, MultiresModifierData *mmd)
 {
   Mesh *me = static_cast<Mesh *>(ob->data);
@@ -2082,11 +2077,11 @@ static bool check_sculpt_object_deformed(Object *object, const bool for_construc
   return deformed;
 }
 
-void BKE_sculpt_face_sets_update_from_base_mesh_visibility(Mesh *mesh)
+void BKE_sculpt_sync_face_visibility_to_grids(Mesh *mesh, SubdivCCG *subdiv_ccg)
 {
   using namespace blender;
   using namespace blender::bke;
-  if (!CustomData_has_layer(&mesh->pdata, CD_SCULPT_FACE_SETS)) {
+  if (!subdiv_ccg) {
     return;
   }
 
@@ -2094,70 +2089,19 @@ void BKE_sculpt_face_sets_update_from_base_mesh_visibility(Mesh *mesh)
   const VArray<bool> hide_poly = attributes.lookup_or_default<bool>(
       ".hide_poly", ATTR_DOMAIN_FACE, false);
   if (hide_poly.is_single() && !hide_poly.get_internal_single()) {
+    /* Nothing is hidden, so we can just remove all visibility bitmaps. */
+    for (const int i : hide_poly.index_range()) {
+      BKE_subdiv_ccg_grid_hidden_free(subdiv_ccg, i);
+    }
     return;
   }
 
-  MutableSpan<int> face_sets{
-      static_cast<int *>(CustomData_get_layer(&mesh->pdata, CD_SCULPT_FACE_SETS)), mesh->totpoly};
-
-  for (const int i : hide_poly.index_range()) {
-    face_sets[i] = hide_poly[i] ? -std::abs(face_sets[i]) : std::abs(face_sets[i]);
-  }
-}
-
-static void set_hide_poly_from_face_sets(Mesh &mesh)
-{
-  using namespace blender;
-  using namespace blender::bke;
-  if (!CustomData_has_layer(&mesh.pdata, CD_SCULPT_FACE_SETS)) {
-    return;
-  }
-
-  const Span<int> face_sets{
-      static_cast<const int *>(CustomData_get_layer(&mesh.pdata, CD_SCULPT_FACE_SETS)),
-      mesh.totpoly};
-
-  MutableAttributeAccessor attributes = mesh.attributes_for_write();
-  if (std::all_of(
-          face_sets.begin(), face_sets.end(), [&](const int value) { return value > 0; })) {
-    attributes.remove(".hide_poly");
-    return;
-  }
-
-  SpanAttributeWriter<bool> hide_poly = attributes.lookup_or_add_for_write_only_span<bool>(
-      ".hide_poly", ATTR_DOMAIN_FACE);
-  if (!hide_poly) {
-    return;
-  }
-  for (const int i : hide_poly.span.index_range()) {
-    hide_poly.span[i] = face_sets[i] < 0;
-  }
-  hide_poly.finish();
-}
-
-void BKE_sculpt_sync_face_sets_visibility_to_base_mesh(Mesh *mesh)
-{
-  set_hide_poly_from_face_sets(*mesh);
-  BKE_mesh_flush_hidden_from_polys(mesh);
-}
-
-void BKE_sculpt_sync_face_sets_visibility_to_grids(Mesh *mesh, SubdivCCG *subdiv_ccg)
-{
-  const int *face_sets = static_cast<const int *>(
-      CustomData_get_layer(&mesh->pdata, CD_SCULPT_FACE_SETS));
-  if (!face_sets) {
-    return;
-  }
-
-  if (!subdiv_ccg) {
-    return;
-  }
-
+  const VArraySpan<bool> hide_poly_span(hide_poly);
   CCGKey key;
   BKE_subdiv_ccg_key_top_level(&key, subdiv_ccg);
   for (int i = 0; i < mesh->totloop; i++) {
     const int face_index = BKE_subdiv_ccg_grid_to_face_index(subdiv_ccg, i);
-    const bool is_hidden = (face_sets[face_index] < 0);
+    const bool is_hidden = hide_poly_span[face_index];
 
     /* Avoid creating and modifying the grid_hidden bitmap if the base mesh face is visible and
      * there is not bitmap for the grid. This is because missing grid_hidden implies grid is fully
@@ -2173,41 +2117,6 @@ void BKE_sculpt_sync_face_sets_visibility_to_grids(Mesh *mesh, SubdivCCG *subdiv
   }
 }
 
-void BKE_sculpt_sync_face_set_visibility(Mesh *mesh, SubdivCCG *subdiv_ccg)
-{
-  BKE_sculpt_face_sets_update_from_base_mesh_visibility(mesh);
-  BKE_sculpt_sync_face_sets_visibility_to_base_mesh(mesh);
-  BKE_sculpt_sync_face_sets_visibility_to_grids(mesh, subdiv_ccg);
-}
-
-void BKE_sculpt_ensure_orig_mesh_data(Object *object)
-{
-  Mesh *mesh = BKE_mesh_from_object(object);
-  BLI_assert(object->mode == OB_MODE_SCULPT);
-
-  /* Copy the current mesh visibility to the Face Sets. */
-  BKE_sculpt_face_sets_update_from_base_mesh_visibility(mesh);
-
-  /* Tessfaces aren't used and will become invalid. */
-  BKE_mesh_tessface_clear(mesh);
-
-  /* We always need to flush updates from depsgraph here, since at the very least
-   * `BKE_sculpt_face_sets_update_from_base_mesh_visibility()` will have updated some data layer of
-   * the mesh.
-   *
-   * All known potential sources of updates:
-   *   - Addition of, or changes to, the `CD_SCULPT_FACE_SETS` data layer
-   *     (`BKE_sculpt_face_sets_update_from_base_mesh_visibility`).
-   *   - Addition of a `CD_PAINT_MASK` data layer (`BKE_sculpt_mask_layers_ensure`).
-   *   - Object has any active modifier (modifier stack can be different in Sculpt mode).
-   *   - Multires:
-   *     + Differences of subdiv levels between sculpt and object modes
-   *       (`mmd->sculptlvl != mmd->lvl`).
-   *     + Addition of a `CD_GRID_PAINT_MASK` data layer (`BKE_sculpt_mask_layers_ensure`).
-   */
-  DEG_id_tag_update(&object->id, ID_RECALC_GEOMETRY);
-}
-
 static PBVH *build_pbvh_for_dynamic_topology(Object *ob)
 {
   PBVH *pbvh = BKE_pbvh_new();
@@ -2239,8 +2148,6 @@ static PBVH *build_pbvh_from_regular_mesh(Object *ob, Mesh *me_eval_deform, bool
   BKE_mesh_recalc_looptri(
       loops.data(), polys.data(), verts.data(), me->totloop, me->totpoly, looptri);
 
-  BKE_sculpt_sync_face_set_visibility(me, nullptr);
-
   BKE_pbvh_build_mesh(pbvh,
                       me,
                       polys.data(),
@@ -2275,7 +2182,7 @@ static PBVH *build_pbvh_from_ccg(Object *ob, SubdivCCG *subdiv_ccg, bool respect
   BKE_pbvh_respect_hide_set(pbvh, respect_hide);
 
   Mesh *base_mesh = BKE_mesh_from_object(ob);
-  BKE_sculpt_sync_face_set_visibility(base_mesh, subdiv_ccg);
+  BKE_sculpt_sync_face_visibility_to_grids(base_mesh, subdiv_ccg);
 
   BKE_pbvh_build_grids(pbvh,
                        subdiv_ccg->grids,
@@ -2369,10 +2276,10 @@ bool BKE_sculptsession_use_pbvh_draw(const Object *ob, const View3D *UNUSED(v3d)
 void BKE_paint_face_set_overlay_color_get(const int face_set, const int seed, uchar r_color[4])
 {
   float rgba[4];
-  float random_mod_hue = GOLDEN_RATIO_CONJUGATE * (abs(face_set) + (seed % 10));
+  float random_mod_hue = GOLDEN_RATIO_CONJUGATE * (face_set + (seed % 10));
   random_mod_hue = random_mod_hue - floorf(random_mod_hue);
-  const float random_mod_sat = BLI_hash_int_01(abs(face_set) + seed + 1);
-  const float random_mod_val = BLI_hash_int_01(abs(face_set) + seed + 2);
+  const float random_mod_sat = BLI_hash_int_01(face_set + seed + 1);
+  const float random_mod_val = BLI_hash_int_01(face_set + seed + 2);
   hsv_to_rgb(random_mod_hue,
              0.6f + (random_mod_sat * 0.25f),
              1.0f - (random_mod_val * 0.35f),