Mesh: Move hide flags to generic attributes

This commit moves the hide status of mesh vertices, edges, and faces
from the `ME_FLAG` to optional generic boolean attributes. Storing this
data as generic attributes can significantly simplify and improve code,
as described in T95965.

The attributes are called `.hide_vert`, `.hide_edge`, and `.hide_poly`,
using the attribute name semantics discussed in T97452. The `.` prefix
means they are "UI attributes", so they still contain original data
edited by users, but they aren't meant to be accessed procedurally by
the user in arbitrary situations. They are also be hidden in the
spreadsheet and the attribute list by default,

Until 4.0, the attributes are still written to and read from the mesh
in the old way, so neither forward nor backward compatibility are
affected. This means memory requirements will be increased by one byte
per element when the hide status is used. When the flags are removed
completely, requirements will decrease when hiding is unused.

Further notes:
 * Some code can be further simplified to skip some processing when the
   hide attributes don't exist.
 * The data is still stored in flags for `BMesh`, necessitating some
   complexity in the conversion to and from `Mesh`.
 * Access to the "hide" property of mesh elements in RNA is slower.
   The separate boolean arrays should be used where possible.

Ref T95965

Differential Revision: https://developer.blender.org/D14685

4 files changed, 29 insertions, 19 deletions

diff --git a/source/blender/editors/sculpt_paint/paint_hide.c b/source/blender/editors/sculpt_paint/paint_hide.c
index 944b3f953a0..c904d533db8 100644
--- a/source/blender/editors/sculpt_paint/paint_hide.c
+++ b/source/blender/editors/sculpt_paint/paint_hide.c
@@ -78,6 +78,12 @@ static void partialvis_update_mesh(Object *ob,
   BKE_pbvh_node_get_verts(pbvh, node, &vert_indices, &mvert);
   paint_mask = CustomData_get_layer(&me->vdata, CD_PAINT_MASK);
 
+  bool *hide_vert = CustomData_get_layer_named(&me->vdata, CD_PROP_BOOL, ".hide_vert");
+  if (hide_vert == NULL) {
+    hide_vert = CustomData_add_layer_named(
+        &me->vdata, CD_PROP_BOOL, CD_CALLOC, NULL, me->totvert, ".hide_vert");
+  }
+
   SCULPT_undo_push_node(ob, node, SCULPT_UNDO_HIDDEN);
 
   for (i = 0; i < totvert; i++) {
@@ -86,16 +92,11 @@ static void partialvis_update_mesh(Object *ob,
 
     /* Hide vertex if in the hide volume. */
     if (is_effected(area, planes, v->co, vmask)) {
-      if (action == PARTIALVIS_HIDE) {
-        v->flag |= ME_HIDE;
-      }
-      else {
-        v->flag &= ~ME_HIDE;
-      }
+      hide_vert[vert_indices[i]] = (action == PARTIALVIS_HIDE);
       any_changed = true;
     }
 
-    if (!(v->flag & ME_HIDE)) {
+    if (!hide_vert[vert_indices[i]]) {
       any_visible = true;
     }
   }
diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index 02d21941a25..ffa931268fd 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -344,9 +344,11 @@ int SCULPT_active_face_set_get(SculptSession *ss)
 void SCULPT_vertex_visible_set(SculptSession *ss, PBVHVertRef vertex, bool visible)
 {
   switch (BKE_pbvh_type(ss->pbvh)) {
-    case PBVH_FACES:
-      SET_FLAG_FROM_TEST(ss->mvert[vertex.i].flag, !visible, ME_HIDE);
+    case PBVH_FACES: {
+      bool *hide_vert = BKE_pbvh_get_vert_hide_for_write(ss->pbvh);
+      hide_vert[vertex.i] = visible;
       break;
+    }
     case PBVH_BMESH: {
       BMVert *v = (BMVert *)vertex.i;
       BM_elem_flag_set(v, BM_ELEM_HIDDEN, !visible);
@@ -360,8 +362,10 @@ void SCULPT_vertex_visible_set(SculptSession *ss, PBVHVertRef vertex, bool visib
 bool SCULPT_vertex_visible_get(SculptSession *ss, PBVHVertRef vertex)
 {
   switch (BKE_pbvh_type(ss->pbvh)) {
-    case PBVH_FACES:
-      return !(ss->mvert[vertex.i].flag & ME_HIDE);
+    case PBVH_FACES: {
+      const bool *hide_vert = BKE_pbvh_get_vert_hide(ss->pbvh);
+      return hide_vert == NULL || !hide_vert[vertex.i];
+    }
     case PBVH_BMESH:
       return !BM_elem_flag_test((BMVert *)vertex.i, BM_ELEM_HIDDEN);
     case PBVH_GRIDS: {
diff --git a/source/blender/editors/sculpt_paint/sculpt_dyntopo.c b/source/blender/editors/sculpt_paint/sculpt_dyntopo.c
index 4f91d2215fb..40b4b74a441 100644
--- a/source/blender/editors/sculpt_paint/sculpt_dyntopo.c
+++ b/source/blender/editors/sculpt_paint/sculpt_dyntopo.c
@@ -227,8 +227,9 @@ static void SCULPT_dynamic_topology_disable_ex(
     me->face_sets_color_default = 1;
 
     /* Sync the visibility to vertices manually as the pmap is still not initialized. */
-    for (int i = 0; i < me->totvert; i++) {
-      me->mvert[i].flag &= ~ME_HIDE;
+    bool *hide_vert = (bool *)CustomData_get_layer_named(&me->vdata, CD_PROP_BOOL, ".hide_vert");
+    if (hide_vert != NULL) {
+      memset(hide_vert, 0, sizeof(bool) * me->totvert);
     }
   }
 
diff --git a/source/blender/editors/sculpt_paint/sculpt_undo.c b/source/blender/editors/sculpt_paint/sculpt_undo.c
index dc814d6d22f..04b2b2f04bf 100644
--- a/source/blender/editors/sculpt_paint/sculpt_undo.c
+++ b/source/blender/editors/sculpt_paint/sculpt_undo.c
@@ -346,14 +346,13 @@ static bool sculpt_undo_restore_hidden(bContext *C, SculptUndoNode *unode, bool
   SculptSession *ss = ob->sculpt;
   SubdivCCG *subdiv_ccg = ss->subdiv_ccg;
 
-  if (unode->maxvert) {
-    MVert *mvert = ss->mvert;
+  bool *hide_vert = BKE_pbvh_get_vert_hide_for_write(ss->pbvh);
 
+  if (unode->maxvert) {
     for (int i = 0; i < unode->totvert; i++) {
-      MVert *v = &mvert[unode->index[i]];
-      if ((BLI_BITMAP_TEST(unode->vert_hidden, i) != 0) != ((v->flag & ME_HIDE) != 0)) {
+      if ((BLI_BITMAP_TEST(unode->vert_hidden, i) != 0) != hide_vert[i]) {
         BLI_BITMAP_FLIP(unode->vert_hidden, i);
-        v->flag ^= ME_HIDE;
+        hide_vert[unode->index[i]] = !hide_vert[i];
         modified_vertices[unode->index[i]] = true;
       }
     }
@@ -1247,6 +1246,11 @@ static void sculpt_undo_store_hidden(Object *ob, SculptUndoNode *unode)
   PBVH *pbvh = ob->sculpt->pbvh;
   PBVHNode *node = unode->node;
 
+  const bool *hide_vert = BKE_pbvh_get_vert_hide(pbvh);
+  if (hide_vert == NULL) {
+    return;
+  }
+
   if (unode->grids) {
     /* Already stored during allocation. */
   }
@@ -1258,7 +1262,7 @@ static void sculpt_undo_store_hidden(Object *ob, SculptUndoNode *unode)
     BKE_pbvh_node_num_verts(pbvh, node, NULL, &allvert);
     BKE_pbvh_node_get_verts(pbvh, node, &vert_indices, &mvert);
     for (int i = 0; i < allvert; i++) {
-      BLI_BITMAP_SET(unode->vert_hidden, i, mvert[vert_indices[i]].flag & ME_HIDE);
+      BLI_BITMAP_SET(unode->vert_hidden, i, hide_vert[vert_indices[i]]);
     }
   }
 }