Fix T96080: hiding elements does not work with GPU subdiv

Faces, edges, and vertices are still shown when GPU subdivision is
actived. This is because the related edit mode flags were ignored by the
subdivision code.

The flags are now passed to the various compute shaders mostly as part of
the extra coarse data, also used for e.g. selection. For loose edges, a
temporary buffer is created when extracting them. Loose vertices are
already taken into account as it reuses the routines for coarse mesh
extraction, although `MeshRenderData.use_hide` was not initialized,
which is fixed now.

2 files changed, 44 insertions, 5 deletions

diff --git a/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_lines.cc b/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_lines.cc
index ce3ca428469..3cecaf81b8a 100644
--- a/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_lines.cc
+++ b/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_lines.cc
@@ -160,7 +160,7 @@ static void extract_lines_init_subdiv(const DRWSubdivCache *subdiv_cache,
 }
 
 static void extract_lines_loose_geom_subdiv(const DRWSubdivCache *subdiv_cache,
-                                            const MeshRenderData *UNUSED(mr),
+                                            const MeshRenderData *mr,
                                             void *buffer,
                                             void *UNUSED(data))
 {
@@ -169,8 +169,31 @@ static void extract_lines_loose_geom_subdiv(const DRWSubdivCache *subdiv_cache,
     return;
   }
 
+  /* Update flags for loose edges, points are already handled. */
+  static GPUVertFormat format;
+  if (format.attr_len == 0) {
+    GPU_vertformat_attr_add(&format, "data", GPU_COMP_U32, 1, GPU_FETCH_INT);
+  }
+
+  GPUVertBuf *flags = GPU_vertbuf_calloc();
+  GPU_vertbuf_init_with_format(flags, &format);
+
+  Span<DRWSubdivLooseEdge> loose_edges = draw_subdiv_cache_get_loose_edges(subdiv_cache);
+  GPU_vertbuf_data_alloc(flags, loose_edges.size());
+
+  uint *flags_data = static_cast<uint *>(GPU_vertbuf_get_data(flags));
+
+  const MEdge *medge = mr->medge;
+
+  for (DRWSubdivLooseEdge edge : loose_edges) {
+    *flags_data++ = (medge[edge.coarse_edge_index].flag & ME_HIDE) != 0;
+  }
+
   GPUIndexBuf *ibo = static_cast<GPUIndexBuf *>(buffer);
-  draw_subdiv_build_lines_loose_buffer(subdiv_cache, ibo, static_cast<uint>(loose_geom.edge_len));
+  draw_subdiv_build_lines_loose_buffer(
+      subdiv_cache, ibo, flags, static_cast<uint>(loose_geom.edge_len));
+
+  GPU_vertbuf_discard(flags);
 }
 
 constexpr MeshExtract create_extractor_lines()
diff --git a/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_points.cc b/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_points.cc
index 272963f3fd5..503ce0e79e9 100644
--- a/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_points.cc
+++ b/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_points.cc
@@ -156,7 +156,8 @@ static void extract_points_init_subdiv(const DRWSubdivCache *subdiv_cache,
 static void extract_points_iter_subdiv_common(GPUIndexBufBuilder *elb,
                                               const MeshRenderData *mr,
                                               const DRWSubdivCache *subdiv_cache,
-                                              uint subdiv_quad_index)
+                                              uint subdiv_quad_index,
+                                              bool for_bmesh)
 {
   int *subdiv_loop_vert_index = (int *)GPU_vertbuf_get_data(subdiv_cache->verts_orig_index);
   uint start_loop_idx = subdiv_quad_index * 4;
@@ -172,6 +173,21 @@ static void extract_points_iter_subdiv_common(GPUIndexBufBuilder *elb,
       continue;
     }
 
+    if (for_bmesh) {
+      const BMVert *mv = BM_vert_at_index(mr->bm, coarse_vertex_index);
+      if (BM_elem_flag_test(mv, BM_ELEM_HIDDEN)) {
+        GPU_indexbuf_set_point_restart(elb, coarse_vertex_index);
+        continue;
+      }
+    }
+    else {
+      const MVert *mv = &mr->mvert[coarse_vertex_index];
+      if (mr->use_hide && (mv->flag & ME_HIDE)) {
+        GPU_indexbuf_set_point_restart(elb, coarse_vertex_index);
+        continue;
+      }
+    }
+
     GPU_indexbuf_set_point_vert(elb, coarse_vertex_index, i);
   }
 }
@@ -183,7 +199,7 @@ static void extract_points_iter_subdiv_bm(const DRWSubdivCache *subdiv_cache,
                                           const BMFace *UNUSED(coarse_quad))
 {
   GPUIndexBufBuilder *elb = static_cast<GPUIndexBufBuilder *>(_data);
-  extract_points_iter_subdiv_common(elb, mr, subdiv_cache, subdiv_quad_index);
+  extract_points_iter_subdiv_common(elb, mr, subdiv_cache, subdiv_quad_index, true);
 }
 
 static void extract_points_iter_subdiv_mesh(const DRWSubdivCache *subdiv_cache,
@@ -193,7 +209,7 @@ static void extract_points_iter_subdiv_mesh(const DRWSubdivCache *subdiv_cache,
                                             const MPoly *UNUSED(coarse_quad))
 {
   GPUIndexBufBuilder *elb = static_cast<GPUIndexBufBuilder *>(_data);
-  extract_points_iter_subdiv_common(elb, mr, subdiv_cache, subdiv_quad_index);
+  extract_points_iter_subdiv_common(elb, mr, subdiv_cache, subdiv_quad_index, false);
 }
 
 static void extract_points_loose_geom_subdiv(const DRWSubdivCache *subdiv_cache,