Mesh: Move hide flags to generic attributes

This commit moves the hide status of mesh vertices, edges, and faces
from the `ME_FLAG` to optional generic boolean attributes. Storing this
data as generic attributes can significantly simplify and improve code,
as described in T95965.

The attributes are called `.hide_vert`, `.hide_edge`, and `.hide_poly`,
using the attribute name semantics discussed in T97452. The `.` prefix
means they are "UI attributes", so they still contain original data
edited by users, but they aren't meant to be accessed procedurally by
the user in arbitrary situations. They are also be hidden in the
spreadsheet and the attribute list by default,

Until 4.0, the attributes are still written to and read from the mesh
in the old way, so neither forward nor backward compatibility are
affected. This means memory requirements will be increased by one byte
per element when the hide status is used. When the flags are removed
completely, requirements will decrease when hiding is unused.

Further notes:
 * Some code can be further simplified to skip some processing when the
   hide attributes don't exist.
 * The data is still stored in flags for `BMesh`, necessitating some
   complexity in the conversion to and from `Mesh`.
 * Access to the "hide" property of mesh elements in RNA is slower.
   The separate boolean arrays should be used where possible.

Ref T95965

Differential Revision: https://developer.blender.org/D14685

2 files changed, 25 insertions, 17 deletions

diff --git a/source/blender/editors/space_view3d/view3d_iterators.c b/source/blender/editors/space_view3d/view3d_iterators.c
index 35d4746608b..6256eeb9621 100644
--- a/source/blender/editors/space_view3d/view3d_iterators.c
+++ b/source/blender/editors/space_view3d/view3d_iterators.c
@@ -205,6 +205,7 @@ typedef struct foreachScreenObjectVert_userData {
   void (*func)(void *userData, MVert *mv, const float screen_co[2], int index);
   void *userData;
   ViewContext vc;
+  const bool *hide_vert;
   eV3DProjTest clip_flag;
 } foreachScreenObjectVert_userData;
 
@@ -262,18 +263,19 @@ static void meshobject_foreachScreenVert__mapFunc(void *userData,
                                                   const float UNUSED(no[3]))
 {
   foreachScreenObjectVert_userData *data = userData;
+  if (data->hide_vert && data->hide_vert[index]) {
+    return;
+  }
   struct MVert *mv = &((Mesh *)(data->vc.obact->data))->mvert[index];
 
-  if (!(mv->flag & ME_HIDE)) {
-    float screen_co[2];
-
-    if (ED_view3d_project_float_object(data->vc.region, co, screen_co, data->clip_flag) !=
-        V3D_PROJ_RET_OK) {
-      return;
-    }
+  float screen_co[2];
 
-    data->func(data->userData, mv, screen_co, index);
+  if (ED_view3d_project_float_object(data->vc.region, co, screen_co, data->clip_flag) !=
+      V3D_PROJ_RET_OK) {
+    return;
   }
+
+  data->func(data->userData, mv, screen_co, index);
 }
 
 void meshobject_foreachScreenVert(
@@ -297,6 +299,8 @@ void meshobject_foreachScreenVert(
   data.func = func;
   data.userData = userData;
   data.clip_flag = clip_flag;
+  data.hide_vert = (const bool *)CustomData_get_layer_named(
+      &me->vdata, CD_PROP_BOOL, ".hide_vert");
 
   if (clip_flag & V3D_PROJ_TEST_CLIP_BB) {
     ED_view3d_clipping_local(vc->rv3d, vc->obact->obmat);
diff --git a/source/blender/editors/space_view3d/view3d_select.c b/source/blender/editors/space_view3d/view3d_select.c
index 571c39f30cb..cd609f7e8cf 100644
--- a/source/blender/editors/space_view3d/view3d_select.c
+++ b/source/blender/editors/space_view3d/view3d_select.c
@@ -335,14 +335,16 @@ static bool edbm_backbuf_check_and_select_verts_obmode(Mesh *me,
                                                        const eSelectOp sel_op)
 {
   MVert *mv = me->mvert;
-  uint index;
   bool changed = false;
 
   const BLI_bitmap *select_bitmap = esel->select_bitmap;
 
   if (mv) {
-    for (index = 0; index < me->totvert; index++, mv++) {
-      if (!(mv->flag & ME_HIDE)) {
+    const bool *hide_vert = (const bool *)CustomData_get_layer_named(
+        &me->vdata, CD_PROP_BOOL, ".hide_vert");
+
+    for (int index = 0; index < me->totvert; index++, mv++) {
+      if (!(hide_vert && hide_vert[index])) {
         const bool is_select = mv->flag & SELECT;
         const bool is_inside = BLI_BITMAP_TEST_BOOL(select_bitmap, index);
         const int sel_op_result = ED_select_op_action_deselected(sel_op, is_select, is_inside);
@@ -362,14 +364,16 @@ static bool edbm_backbuf_check_and_select_faces_obmode(Mesh *me,
                                                        const eSelectOp sel_op)
 {
   MPoly *mpoly = me->mpoly;
-  uint index;
   bool changed = false;
 
   const BLI_bitmap *select_bitmap = esel->select_bitmap;
 
   if (mpoly) {
-    for (index = 0; index < me->totpoly; index++, mpoly++) {
-      if (!(mpoly->flag & ME_HIDE)) {
+    const bool *hide_poly = (const bool *)CustomData_get_layer_named(
+        &me->vdata, CD_PROP_BOOL, ".hide_poly");
+
+    for (int index = 0; index < me->totpoly; index++, mpoly++) {
+      if (!(hide_poly && hide_poly[index])) {
         const bool is_select = mpoly->flag & ME_FACE_SEL;
         const bool is_inside = BLI_BITMAP_TEST_BOOL(select_bitmap, index);
         const int sel_op_result = ED_select_op_action_deselected(sel_op, is_select, is_inside);
@@ -1260,7 +1264,7 @@ static bool do_lasso_select_paintface(ViewContext *vc,
   }
 
   if (changed) {
-    paintface_flush_flags(vc->C, ob, SELECT);
+    paintface_flush_flags(vc->C, ob, true, false);
   }
   return changed;
 }
@@ -3192,7 +3196,7 @@ static bool do_paintface_box_select(ViewContext *vc,
   }
 
   if (changed) {
-    paintface_flush_flags(vc->C, vc->obact, SELECT);
+    paintface_flush_flags(vc->C, vc->obact, true, false);
   }
   return changed;
 }
@@ -4093,7 +4097,7 @@ static bool paint_facesel_circle_select(ViewContext *vc,
   }
 
   if (changed) {
-    paintface_flush_flags(vc->C, ob, SELECT);
+    paintface_flush_flags(vc->C, ob, true, false);
   }
   return changed;
 }