Mesh: Move hide flags to generic attributes

This commit moves the hide status of mesh vertices, edges, and faces
from the `ME_FLAG` to optional generic boolean attributes. Storing this
data as generic attributes can significantly simplify and improve code,
as described in T95965.

The attributes are called `.hide_vert`, `.hide_edge`, and `.hide_poly`,
using the attribute name semantics discussed in T97452. The `.` prefix
means they are "UI attributes", so they still contain original data
edited by users, but they aren't meant to be accessed procedurally by
the user in arbitrary situations. They are also be hidden in the
spreadsheet and the attribute list by default,

Until 4.0, the attributes are still written to and read from the mesh
in the old way, so neither forward nor backward compatibility are
affected. This means memory requirements will be increased by one byte
per element when the hide status is used. When the flags are removed
completely, requirements will decrease when hiding is unused.

Further notes:
 * Some code can be further simplified to skip some processing when the
   hide attributes don't exist.
 * The data is still stored in flags for `BMesh`, necessitating some
   complexity in the conversion to and from `Mesh`.
 * Access to the "hide" property of mesh elements in RNA is slower.
   The separate boolean arrays should be used where possible.

Ref T95965

Differential Revision: https://developer.blender.org/D14685

2 files changed, 14 insertions, 9 deletions

diff --git a/source/blender/gpu/intern/gpu_buffers.c b/source/blender/gpu/intern/gpu_buffers.c
index 2a0f624ac45..d64b8b4118a 100644
--- a/source/blender/gpu/intern/gpu_buffers.c
+++ b/source/blender/gpu/intern/gpu_buffers.c
@@ -211,11 +211,11 @@ static void gpu_pbvh_batch_init(GPU_PBVH_Buffers *buffers, GPUPrimType prim)
  * \{ */
 
 static bool gpu_pbvh_is_looptri_visible(const MLoopTri *lt,
-                                        const MVert *mvert,
+                                        const bool *hide_vert,
                                         const MLoop *mloop,
                                         const int *sculpt_face_sets)
 {
-  return (!paint_is_face_hidden(lt, mvert, mloop) && sculpt_face_sets &&
+  return (!paint_is_face_hidden(lt, hide_vert, mloop) && sculpt_face_sets &&
           sculpt_face_sets[lt->poly] > SCULPT_FACE_SET_NONE);
 }
 
@@ -233,6 +233,9 @@ void GPU_pbvh_mesh_buffers_update(PBVHGPUFormat *vbo_id,
   GPUAttrRef vcol_refs[MAX_GPU_ATTR];
   GPUAttrRef cd_uvs[MAX_GPU_ATTR];
 
+  const bool *hide_vert = (bool *)CustomData_get_layer_named(
+      &mesh->vdata, CD_PROP_BOOL, ".hide_vert");
+
   const CustomDataLayer *actcol = BKE_id_attributes_active_color_get(&mesh->id);
   eAttrDomain actcol_domain = actcol ? BKE_id_attribute_domain(&mesh->id, actcol) :
                                        ATTR_DOMAIN_AUTO;
@@ -310,7 +313,7 @@ void GPU_pbvh_mesh_buffers_update(PBVHGPUFormat *vbo_id,
           for (uint i = 0; i < buffers->face_indices_len; i++) {
             const MLoopTri *lt = &buffers->looptri[buffers->face_indices[i]];
 
-            if (!gpu_pbvh_is_looptri_visible(lt, mvert, buffers->mloop, sculpt_face_sets)) {
+            if (!gpu_pbvh_is_looptri_visible(lt, hide_vert, buffers->mloop, sculpt_face_sets)) {
               continue;
             }
 
@@ -350,7 +353,7 @@ void GPU_pbvh_mesh_buffers_update(PBVHGPUFormat *vbo_id,
               buffers->mloop[lt->tri[2]].v,
           };
 
-          if (!gpu_pbvh_is_looptri_visible(lt, mvert, buffers->mloop, sculpt_face_sets)) {
+          if (!gpu_pbvh_is_looptri_visible(lt, hide_vert, buffers->mloop, sculpt_face_sets)) {
             continue;
           }
 
@@ -390,7 +393,7 @@ void GPU_pbvh_mesh_buffers_update(PBVHGPUFormat *vbo_id,
             buffers->mloop[lt->tri[2]].v,
         };
 
-        if (!gpu_pbvh_is_looptri_visible(lt, mvert, buffers->mloop, sculpt_face_sets)) {
+        if (!gpu_pbvh_is_looptri_visible(lt, hide_vert, buffers->mloop, sculpt_face_sets)) {
           continue;
         }
 
@@ -454,7 +457,6 @@ void GPU_pbvh_mesh_buffers_update(PBVHGPUFormat *vbo_id,
 }
 
 GPU_PBVH_Buffers *GPU_pbvh_mesh_buffers_build(const Mesh *mesh,
-                                              const MVert *vertices,
                                               const MLoopTri *looptri,
                                               const int *sculpt_face_sets,
                                               const int *face_indices,
@@ -469,6 +471,9 @@ GPU_PBVH_Buffers *GPU_pbvh_mesh_buffers_build(const Mesh *mesh,
 
   buffers = MEM_callocN(sizeof(GPU_PBVH_Buffers), "GPU_Buffers");
 
+  const bool *hide_vert = (bool *)CustomData_get_layer_named(
+      &mesh->vdata, CD_PROP_BOOL, ".hide_vert");
+
   /* smooth or flat for all */
   buffers->smooth = mpoly[looptri[face_indices[0]].poly].flag & ME_SMOOTH;
 
@@ -477,7 +482,7 @@ GPU_PBVH_Buffers *GPU_pbvh_mesh_buffers_build(const Mesh *mesh,
   /* Count the number of visible triangles */
   for (i = 0, tottri = 0; i < face_indices_len; i++) {
     const MLoopTri *lt = &looptri[face_indices[i]];
-    if (gpu_pbvh_is_looptri_visible(lt, vertices, mloop, sculpt_face_sets)) {
+    if (gpu_pbvh_is_looptri_visible(lt, hide_vert, mloop, sculpt_face_sets)) {
       int r_edges[3];
       BKE_mesh_looptri_get_real_edges(mesh, lt, r_edges);
       for (int j = 0; j < 3; j++) {
@@ -510,7 +515,7 @@ GPU_PBVH_Buffers *GPU_pbvh_mesh_buffers_build(const Mesh *mesh,
     const MLoopTri *lt = &looptri[face_indices[i]];
 
     /* Skip hidden faces */
-    if (!gpu_pbvh_is_looptri_visible(lt, vertices, mloop, sculpt_face_sets)) {
+    if (!gpu_pbvh_is_looptri_visible(lt, hide_vert, mloop, sculpt_face_sets)) {
       continue;
     }
 
diff --git a/source/blender/gpu/intern/gpu_shader_builder_stubs.cc b/source/blender/gpu/intern/gpu_shader_builder_stubs.cc
index d8af2fc584d..3a14c060484 100644
--- a/source/blender/gpu/intern/gpu_shader_builder_stubs.cc
+++ b/source/blender/gpu/intern/gpu_shader_builder_stubs.cc
@@ -137,7 +137,7 @@ eAttrDomain BKE_id_attribute_domain(const struct ID *UNUSED(id),
 /** \name Stubs of BKE_paint.h
  * \{ */
 bool paint_is_face_hidden(const struct MLoopTri *UNUSED(lt),
-                          const struct MVert *UNUSED(mvert),
+                          const bool *UNUSED(hide_vert),
                           const struct MLoop *UNUSED(mloop))
 {
   BLI_assert_unreachable();