Mesh: Move selection flags to generic attributes

Using the attribute name semantics from T97452, this patch moves the
selection status of mesh elements from the `SELECT` of vertices, and
edges, and the `ME_FACE_SEL` of faces to generic boolean attribute
Storing this data as generic attributes can significantly simplify and
improve code, as described in T95965.

The attributes are called `.select_vert`, `.select_edge`, and
`.select_poly`. The `.` prefix means they are "UI attributes",so they
still contain original data edited by users, but they aren't meant to
be accessed procedurally by the user in arbitrary situations. They are
also be hidden in the spreadsheet and the attribute list.

Until 4.0, the attributes are still written to and read from the mesh
in the old way, so neither forward nor backward compatibility are
affected. This means memory requirements will be increased by one byte
per element when selection is used. When the flags are removed
completely, requirements will decrease.

Further notes:
* The `MVert` flag is empty at runtime now, so it can be ignored.
* `BMesh` is unchanged, otherwise the change would be much larger.
* Many tests have slightly different results, since the selection
  attribute uses more generic propagation. Previously you couldn't
  really rely on edit mode selections being propagated procedurally.
  Now it mostly works as expected.

Similar to 2480b55f216c
Ref T95965

Differential Revision: https://developer.blender.org/D15795

1 files changed, 64 insertions, 31 deletions

diff --git a/source/blender/editors/sculpt_paint/paint_vertex.cc b/source/blender/editors/sculpt_paint/paint_vertex.cc
index c38a79cb6bb..9abc3c5112e 100644
--- a/source/blender/editors/sculpt_paint/paint_vertex.cc
+++ b/source/blender/editors/sculpt_paint/paint_vertex.cc
@@ -32,6 +32,7 @@
 #include "RNA_prototypes.h"
 
 #include "BKE_attribute.h"
+#include "BKE_attribute.hh"
 #include "BKE_brush.h"
 #include "BKE_colortools.h"
 #include "BKE_context.h"
@@ -1958,6 +1959,10 @@ static void do_wpaint_brush_blur_task_cb_ex(void *__restrict userdata,
   const float *sculpt_normal_frontface = SCULPT_brush_frontface_normal_from_falloff_shape(
       ss, data->brush->falloff_shape);
 
+  const blender::bke::AttributeAccessor attributes = data->me->attributes();
+  const blender::VArray<bool> select_vert = attributes.lookup_or_default<bool>(
+      ".select_vert", ATTR_DOMAIN_POINT, false);
+
   /* For each vertex */
   PBVHVertexIter vd;
   BKE_pbvh_vertex_iter_begin (ss->pbvh, data->nodes[n], vd, PBVH_ITER_UNIQUE) {
@@ -1967,9 +1972,8 @@ static void do_wpaint_brush_blur_task_cb_ex(void *__restrict userdata,
        * Otherwise, take the current vert. */
       const int v_index = has_grids ? ss->mloop[vd.grid_indices[vd.g]].v : vd.vert_indices[vd.i];
       const float grid_alpha = has_grids ? 1.0f / vd.gridsize : 1.0f;
-      const char v_flag = ss->mvert[v_index].flag;
       /* If the vertex is selected */
-      if (!(use_face_sel || use_vert_sel) || v_flag & SELECT) {
+      if (!(use_face_sel || use_vert_sel) || select_vert[v_index]) {
         /* Get the average poly weight */
         int total_hit_loops = 0;
         float weight_final = 0.0f;
@@ -2045,6 +2049,10 @@ static void do_wpaint_brush_smear_task_cb_ex(void *__restrict userdata,
   sub_v3_v3v3(brush_dir, cache->location, cache->last_location);
   project_plane_v3_v3v3(brush_dir, brush_dir, cache->view_normal);
 
+  const blender::bke::AttributeAccessor attributes = data->me->attributes();
+  const blender::VArray<bool> select_vert = attributes.lookup_or_default<bool>(
+      ".select_vert", ATTR_DOMAIN_POINT, false);
+
   if (cache->is_last_valid && (normalize_v3(brush_dir) != 0.0f)) {
 
     SculptBrushTest test;
@@ -2065,7 +2073,7 @@ static void do_wpaint_brush_smear_task_cb_ex(void *__restrict userdata,
         const MVert *mv_curr = &ss->mvert[v_index];
 
         /* If the vertex is selected */
-        if (!(use_face_sel || use_vert_sel) || mv_curr->flag & SELECT) {
+        if (!(use_face_sel || use_vert_sel) || select_vert[v_index]) {
           float brush_strength = cache->bstrength;
           const float angle_cos = (use_normal && vd.no) ?
                                       dot_v3v3(sculpt_normal_frontface, vd.no) :
@@ -2158,6 +2166,10 @@ static void do_wpaint_brush_draw_task_cb_ex(void *__restrict userdata,
   const float *sculpt_normal_frontface = SCULPT_brush_frontface_normal_from_falloff_shape(
       ss, data->brush->falloff_shape);
 
+  const blender::bke::AttributeAccessor attributes = data->me->attributes();
+  const blender::VArray<bool> select_vert = attributes.lookup_or_default<bool>(
+      ".select_vert", ATTR_DOMAIN_POINT, false);
+
   /* For each vertex */
   PBVHVertexIter vd;
   BKE_pbvh_vertex_iter_begin (ss->pbvh, data->nodes[n], vd, PBVH_ITER_UNIQUE) {
@@ -2169,9 +2181,8 @@ static void do_wpaint_brush_draw_task_cb_ex(void *__restrict userdata,
       const int v_index = has_grids ? ss->mloop[vd.grid_indices[vd.g]].v : vd.vert_indices[vd.i];
       const float grid_alpha = has_grids ? 1.0f / vd.gridsize : 1.0f;
 
-      const char v_flag = ss->mvert[v_index].flag;
       /* If the vertex is selected */
-      if (!(use_face_sel || use_vert_sel) || v_flag & SELECT) {
+      if (!(use_face_sel || use_vert_sel) || select_vert[v_index]) {
         float brush_strength = cache->bstrength;
         const float angle_cos = (use_normal && vd.no) ? dot_v3v3(sculpt_normal_frontface, vd.no) :
                                                         1.0f;
@@ -2224,6 +2235,10 @@ static void do_wpaint_brush_calc_average_weight_cb_ex(
   const float *sculpt_normal_frontface = SCULPT_brush_frontface_normal_from_falloff_shape(
       ss, data->brush->falloff_shape);
 
+  const blender::bke::AttributeAccessor attributes = data->me->attributes();
+  const blender::VArray<bool> select_vert = attributes.lookup_or_default<bool>(
+      ".select_vert", ATTR_DOMAIN_POINT, false);
+
   /* For each vertex */
   PBVHVertexIter vd;
   BKE_pbvh_vertex_iter_begin (ss->pbvh, data->nodes[n], vd, PBVH_ITER_UNIQUE) {
@@ -2234,10 +2249,9 @@ static void do_wpaint_brush_calc_average_weight_cb_ex(
       if (angle_cos > 0.0 &&
           BKE_brush_curve_strength(data->brush, sqrtf(test.dist), cache->radius) > 0.0) {
         const int v_index = has_grids ? ss->mloop[vd.grid_indices[vd.g]].v : vd.vert_indices[vd.i];
-        const char v_flag = ss->mvert[v_index].flag;
 
         /* If the vertex is selected. */
-        if (!(use_face_sel || use_vert_sel) || v_flag & SELECT) {
+        if (!(use_face_sel || use_vert_sel) || select_vert[v_index]) {
           const MDeformVert *dv = &data->wpi->dvert[v_index];
           accum->len += 1;
           accum->value += wpaint_get_active_weight(dv, data->wpi);
@@ -2956,6 +2970,11 @@ static void do_vpaint_brush_blur_loops(bContext *C,
 
   Color *previous_color = static_cast<Color *>(ss->cache->prev_colors_vpaint);
 
+  const blender::VArray<bool> select_vert = me->attributes().lookup_or_default<bool>(
+      ".select_vert", ATTR_DOMAIN_POINT, false);
+  const blender::VArray<bool> select_poly = me->attributes().lookup_or_default<bool>(
+      ".select_poly", ATTR_DOMAIN_FACE, false);
+
   blender::threading::parallel_for(IndexRange(totnode), 1LL, [&](IndexRange range) {
     for (int n : range) {
       const PBVHType pbvh_type = BKE_pbvh_type(ss->pbvh);
@@ -2987,10 +3006,9 @@ static void do_vpaint_brush_blur_loops(bContext *C,
           const int v_index = has_grids ? ss->mloop[vd.grid_indices[vd.g]].v :
                                           vd.vert_indices[vd.i];
           const float grid_alpha = has_grids ? 1.0f / vd.gridsize : 1.0f;
-          const MVert *mv = &ss->mvert[v_index];
 
           /* If the vertex is selected for painting. */
-          if (!use_vert_sel || mv->flag & SELECT) {
+          if (!use_vert_sel || select_vert[v_index]) {
             float brush_strength = cache->bstrength;
             const float angle_cos = (use_normal && vd.no) ?
                                         dot_v3v3(sculpt_normal_frontface, vd.no) :
@@ -3011,7 +3029,7 @@ static void do_vpaint_brush_blur_loops(bContext *C,
               for (int j = 0; j < gmap->vert_to_poly[v_index].count; j++) {
                 int p_index = gmap->vert_to_poly[v_index].indices[j];
                 const MPoly *mp = &ss->mpoly[p_index];
-                if (!use_face_sel || mp->flag & ME_FACE_SEL) {
+                if (!use_face_sel || select_poly[p_index]) {
                   total_hit_loops += mp->totloop;
                   for (int k = 0; k < mp->totloop; k++) {
                     const uint l_index = mp->loopstart + k;
@@ -3045,8 +3063,7 @@ static void do_vpaint_brush_blur_loops(bContext *C,
                   const int p_index = gmap->vert_to_poly[v_index].indices[j];
                   const int l_index = gmap->vert_to_loop[v_index].indices[j];
                   BLI_assert(ss->mloop[l_index].v == v_index);
-                  const MPoly *mp = &ss->mpoly[p_index];
-                  if (!use_face_sel || mp->flag & ME_FACE_SEL) {
+                  if (!use_face_sel || select_poly[p_index]) {
                     Color color_orig(0, 0, 0, 0); /* unused when array is nullptr */
 
                     if (previous_color != nullptr) {
@@ -3098,6 +3115,11 @@ static void do_vpaint_brush_blur_verts(bContext *C,
 
   Color *previous_color = static_cast<Color *>(ss->cache->prev_colors_vpaint);
 
+  const blender::VArray<bool> select_vert = me->attributes().lookup_or_default<bool>(
+      ".select_vert", ATTR_DOMAIN_POINT, false);
+  const blender::VArray<bool> select_poly = me->attributes().lookup_or_default<bool>(
+      ".select_poly", ATTR_DOMAIN_FACE, false);
+
   blender::threading::parallel_for(IndexRange(totnode), 1LL, [&](IndexRange range) {
     for (int n : range) {
       const PBVHType pbvh_type = BKE_pbvh_type(ss->pbvh);
@@ -3129,10 +3151,9 @@ static void do_vpaint_brush_blur_verts(bContext *C,
           const int v_index = has_grids ? ss->mloop[vd.grid_indices[vd.g]].v :
                                           vd.vert_indices[vd.i];
           const float grid_alpha = has_grids ? 1.0f / vd.gridsize : 1.0f;
-          const MVert *mv = &ss->mvert[v_index];
 
           /* If the vertex is selected for painting. */
-          if (!use_vert_sel || mv->flag & SELECT) {
+          if (!use_vert_sel || select_vert[v_index]) {
             float brush_strength = cache->bstrength;
             const float angle_cos = (use_normal && vd.no) ?
                                         dot_v3v3(sculpt_normal_frontface, vd.no) :
@@ -3153,7 +3174,7 @@ static void do_vpaint_brush_blur_verts(bContext *C,
               for (int j = 0; j < gmap->vert_to_poly[v_index].count; j++) {
                 int p_index = gmap->vert_to_poly[v_index].indices[j];
                 const MPoly *mp = &ss->mpoly[p_index];
-                if (!use_face_sel || mp->flag & ME_FACE_SEL) {
+                if (!use_face_sel || select_poly[p_index]) {
                   total_hit_loops += mp->totloop;
                   for (int k = 0; k < mp->totloop; k++) {
                     const uint l_index = mp->loopstart + k;
@@ -3190,8 +3211,7 @@ static void do_vpaint_brush_blur_verts(bContext *C,
 
                   BLI_assert(ss->mloop[gmap->vert_to_loop[v_index].indices[j]].v == v_index);
 
-                  const MPoly *mp = &ss->mpoly[p_index];
-                  if (!use_face_sel || mp->flag & ME_FACE_SEL) {
+                  if (!use_face_sel || select_poly[p_index]) {
                     Color color_orig(0, 0, 0, 0); /* unused when array is nullptr */
 
                     if (previous_color != nullptr) {
@@ -3247,6 +3267,11 @@ static void do_vpaint_brush_smear(bContext *C,
   Color *color_prev_smear = static_cast<Color *>(vpd->smear.color_prev);
   Color *color_prev = reinterpret_cast<Color *>(ss->cache->prev_colors_vpaint);
 
+  const blender::VArray<bool> select_vert = me->attributes().lookup_or_default<bool>(
+      ".select_vert", ATTR_DOMAIN_POINT, false);
+  const blender::VArray<bool> select_poly = me->attributes().lookup_or_default<bool>(
+      ".select_poly", ATTR_DOMAIN_FACE, false);
+
   blender::threading::parallel_for(IndexRange(totnode), 1LL, [&](IndexRange range) {
     for (int n : range) {
       float brush_size_pressure, brush_alpha_value, brush_alpha_pressure;
@@ -3283,7 +3308,7 @@ static void do_vpaint_brush_smear(bContext *C,
             const MVert *mv_curr = &ss->mvert[v_index];
 
             /* if the vertex is selected for painting. */
-            if (!use_vert_sel || mv_curr->flag & SELECT) {
+            if (!use_vert_sel || select_vert[v_index]) {
               /* Calc the dot prod. between ray norm on surf and current vert
                * (ie splash prevention factor), and only paint front facing verts. */
               float brush_strength = cache->bstrength;
@@ -3312,7 +3337,7 @@ static void do_vpaint_brush_smear(bContext *C,
                   BLI_assert(ss->mloop[l_index].v == v_index);
                   UNUSED_VARS_NDEBUG(l_index);
                   const MPoly *mp = &ss->mpoly[p_index];
-                  if (!use_face_sel || mp->flag & ME_FACE_SEL) {
+                  if (!use_face_sel || select_poly[p_index]) {
                     const MLoop *ml_other = &ss->mloop[mp->loopstart];
                     for (int k = 0; k < mp->totloop; k++, ml_other++) {
                       const uint v_other_index = ml_other->v;
@@ -3366,8 +3391,7 @@ static void do_vpaint_brush_smear(bContext *C,
                       BLI_assert(ss->mloop[l_index].v == v_index);
                     }
 
-                    const MPoly *mp = &ss->mpoly[p_index];
-                    if (!use_face_sel || mp->flag & ME_FACE_SEL) {
+                    if (!use_face_sel || select_poly[p_index]) {
                       /* Get the previous element color */
                       Color color_orig(0, 0, 0, 0); /* unused when array is nullptr */
 
@@ -3413,6 +3437,9 @@ static void calculate_average_color(VPaintData<Color, Traits, domain> *vpd,
 {
   using Blend = typename Traits::BlendType;
 
+  const blender::VArray<bool> select_vert = me->attributes().lookup_or_default<bool>(
+      ".select_vert", ATTR_DOMAIN_POINT, false);
+
   VPaintAverageAccum<Blend> *accum = (VPaintAverageAccum<Blend> *)MEM_mallocN(
       sizeof(*accum) * totnode, __func__);
   blender::threading::parallel_for(IndexRange(totnode), 1LL, [&](IndexRange range) {
@@ -3443,8 +3470,7 @@ static void calculate_average_color(VPaintData<Color, Traits, domain> *vpd,
                                           vd.vert_indices[vd.i];
           if (BKE_brush_curve_strength(brush, 0.0, cache->radius) > 0.0) {
             /* If the vertex is selected for painting. */
-            const MVert *mv = &ss->mvert[v_index];
-            if (!use_vert_sel || mv->flag & SELECT) {
+            if (!use_vert_sel || select_vert[v_index]) {
               accum2->len += gmap->vert_to_loop[v_index].count;
               /* if a vertex is within the brush region, then add its color to the blend. */
               for (int j = 0; j < gmap->vert_to_loop[v_index].count; j++) {
@@ -3529,6 +3555,11 @@ static void vpaint_do_draw(bContext *C,
 
   Color *previous_color = static_cast<Color *>(ss->cache->prev_colors_vpaint);
 
+  const blender::VArray<bool> select_vert = me->attributes().lookup_or_default<bool>(
+      ".select_vert", ATTR_DOMAIN_POINT, false);
+  const blender::VArray<bool> select_poly = me->attributes().lookup_or_default<bool>(
+      ".select_poly", ATTR_DOMAIN_FACE, false);
+
   blender::threading::parallel_for(IndexRange(totnode), 1LL, [&](IndexRange range) {
     for (int n : range) {
       const bool has_grids = (pbvh_type == PBVH_GRIDS);
@@ -3562,10 +3593,9 @@ static void vpaint_do_draw(bContext *C,
           const int v_index = has_grids ? ss->mloop[vd.grid_indices[vd.g]].v :
                                           vd.vert_indices[vd.i];
           const float grid_alpha = has_grids ? 1.0f / vd.gridsize : 1.0f;
-          const MVert *mv = &ss->mvert[v_index];
 
           /* If the vertex is selected for painting. */
-          if (!use_vert_sel || mv->flag & SELECT) {
+          if (!use_vert_sel || select_vert[v_index]) {
             /* Calc the dot prod. between ray norm on surf and current vert
              * (ie splash prevention factor), and only paint front facing verts. */
             float brush_strength = cache->bstrength;
@@ -3617,8 +3647,7 @@ static void vpaint_do_draw(bContext *C,
                   const int p_index = gmap->vert_to_poly[v_index].indices[j];
                   const int l_index = gmap->vert_to_loop[v_index].indices[j];
                   BLI_assert(ss->mloop[l_index].v == v_index);
-                  const MPoly *mp = &ss->mpoly[p_index];
-                  if (!use_face_sel || mp->flag & ME_FACE_SEL) {
+                  if (!use_face_sel || select_poly[p_index]) {
                     Color color_orig = Color(0, 0, 0, 0); /* unused when array is nullptr */
 
                     if (previous_color != nullptr) {
@@ -4045,6 +4074,11 @@ static bool vertex_color_set(Object *ob, ColorPaint4f paintcol_in, CustomDataLay
     return false;
   }
 
+  const blender::VArray<bool> select_vert = me->attributes().lookup_or_default<bool>(
+      ".select_vert", ATTR_DOMAIN_POINT, false);
+  const blender::VArray<bool> select_poly = me->attributes().lookup_or_default<bool>(
+      ".select_poly", ATTR_DOMAIN_FACE, false);
+
   Color paintcol = fromFloat<Color>(paintcol_in);
 
   const bool use_face_sel = (me->editflag & ME_EDIT_PAINT_FACE_SEL) != 0;
@@ -4088,21 +4122,20 @@ static bool vertex_color_set(Object *ob, ColorPaint4f paintcol_in, CustomDataLay
   }
   else {
     Color *color_layer = static_cast<Color *>(layer->data);
-    const Span<MVert> verts = me->verts();
     const Span<MPoly> polys = me->polys();
     const Span<MLoop> loops = me->loops();
 
     for (const int i : polys.index_range()) {
-      const MPoly &poly = polys[i];
-      if (use_face_sel && !(poly.flag & ME_FACE_SEL)) {
+      if (use_face_sel && !select_poly[i]) {
         continue;
       }
+      const MPoly &poly = polys[i];
 
       int j = 0;
       do {
         uint vidx = loops[poly.loopstart + j].v;
 
-        if (!(use_vert_sel && !(verts[vidx].flag & SELECT))) {
+        if (!(use_vert_sel && !(select_vert[vidx]))) {
           if constexpr (domain == ATTR_DOMAIN_CORNER) {
             color_layer[poly.loopstart + j] = paintcol;
           }