Sculpt: Separate hide status from face sets, use generic attribute

Whether faces are hidden and face sets are orthogonal concepts, but
currently sculpt mode stores them together in the face set array.
This means that if anything is hidden, there must be face sets,
and if there are face sets, we have to keep track of what is hidden.
In other words, it adds a bunch of redundant work and state tracking.

On the user level it's nice that face sets and hiding are consistent,
but we don't need to store them together to accomplish that.

This commit uses the `".hide_poly"` attribute from rB2480b55f216c to
read and change hiding in sculpt mode. Face sets don't need to be
negative anymore, and a bunch of "face set <-> hide status" conversion
can be removed. Plus some other benefits:
 - We don't need to allocate either array quite as much.
 - The hide status can be read from 1/4 the memory as face sets.
 - Updates when entering or exiting sculpt mode can be removed.
 - More opportunities for early-outs when nothing is hidden.
 - Separating concerns makes sculpt code more obvious.
 - It will be easier to convert face sets into a generic int attribute.

 Differential Revision: https://developer.blender.org/D15950

1 files changed, 13 insertions, 28 deletions

diff --git a/source/blender/blenkernel/BKE_paint.h b/source/blender/blenkernel/BKE_paint.h
index eef91bacc2f..774765c3ca1 100644
--- a/source/blender/blenkernel/BKE_paint.h
+++ b/source/blender/blenkernel/BKE_paint.h
@@ -529,13 +529,16 @@ typedef struct SculptSession {
   /* Mesh Face Sets */
   /* Total number of polys of the base mesh. */
   int totfaces;
-  /* Face sets store its visibility in the sign of the integer, using the absolute value as the
-   * Face Set ID. Positive IDs are visible, negative IDs are hidden.
-   * The 0 ID is not used by the tools or the visibility system, it is just used when creating new
+  /* The 0 ID is not used by the tools or the visibility system, it is just used when creating new
    * geometry (the trim tool, for example) to detect which geometry was just added, so it can be
    * assigned a valid Face Set after creation. Tools are not intended to run with Face Sets IDs set
    * to 0. */
   int *face_sets;
+  /**
+   * A reference to the ".hide_poly" attribute, to store whether (base) polygons are hidden.
+   * May be null.
+   */
+  bool *hide_poly;
 
   /* BMesh for dynamic topology sculpting */
   struct BMesh *bm;
@@ -697,6 +700,12 @@ void BKE_sculpt_update_object_after_eval(struct Depsgraph *depsgraph, struct Obj
 struct MultiresModifierData *BKE_sculpt_multires_active(const struct Scene *scene,
                                                         struct Object *ob);
 int *BKE_sculpt_face_sets_ensure(struct Mesh *mesh);
+/**
+ * Create the attribute used to store face visibility and retrieve its data.
+ * Note that changes to the face visibility have to be propagated to other domains
+ * (see #SCULPT_visibility_sync_all_from_faces).
+ */
+bool *BKE_sculpt_hide_poly_ensure(struct Mesh *mesh);
 int BKE_sculpt_mask_layers_ensure(struct Object *ob, struct MultiresModifierData *mmd);
 void BKE_sculpt_toolsettings_data_ensure(struct Scene *scene);
 
@@ -704,31 +713,7 @@ struct PBVH *BKE_sculpt_object_pbvh_ensure(struct Depsgraph *depsgraph, struct O
 
 void BKE_sculpt_bvh_update_from_ccg(struct PBVH *pbvh, struct SubdivCCG *subdiv_ccg);
 
-/**
- * This ensure that all elements in the mesh (both vertices and grids) have their visibility
- * updated according to the face sets.
- */
-void BKE_sculpt_sync_face_set_visibility(struct Mesh *mesh, struct SubdivCCG *subdiv_ccg);
-
-/**
- * Individual function to sync the Face Set visibility to mesh and grids.
- */
-void BKE_sculpt_sync_face_sets_visibility_to_base_mesh(struct Mesh *mesh);
-void BKE_sculpt_sync_face_sets_visibility_to_grids(struct Mesh *mesh,
-                                                   struct SubdivCCG *subdiv_ccg);
-
-/**
- * If a face set layer exists, initialize its visibility (sign) from the mesh's hidden values.
- */
-void BKE_sculpt_face_sets_update_from_base_mesh_visibility(struct Mesh *mesh);
-
-/**
- * Makes sculpt data consistent with other data on the mesh.
- *
- * \note IDs are expected to be original ones here, and calling code should ensure it updates its
- * depsgraph properly after calling this function if it needs up-to-date evaluated data.
- */
-void BKE_sculpt_ensure_orig_mesh_data(struct Object *object);
+void BKE_sculpt_sync_face_visibility_to_grids(struct Mesh *mesh, struct SubdivCCG *subdiv_ccg);
 
 /**
  * Test if PBVH can be used directly for drawing, which is faster than