Fix T96080: hiding elements does not work with GPU subdiv

Faces, edges, and vertices are still shown when GPU subdivision is
actived. This is because the related edit mode flags were ignored by the
subdivision code.

The flags are now passed to the various compute shaders mostly as part of
the extra coarse data, also used for e.g. selection. For loose edges, a
temporary buffer is created when extracting them. Loose vertices are
already taken into account as it reuses the routines for coarse mesh
extraction, although `MeshRenderData.use_hide` was not initialized,
which is fixed now.

8 files changed, 153 insertions, 33 deletions

diff --git a/source/blender/draw/intern/draw_cache_impl_subdivision.cc b/source/blender/draw/intern/draw_cache_impl_subdivision.cc
index 8eb0509d615..e3842ec2231 100644
--- a/source/blender/draw/intern/draw_cache_impl_subdivision.cc
+++ b/source/blender/draw/intern/draw_cache_impl_subdivision.cc
@@ -588,8 +588,9 @@ void draw_subdiv_cache_free(DRWSubdivCache *cache)
 #define SUBDIV_COARSE_FACE_FLAG_SMOOTH 1u
 #define SUBDIV_COARSE_FACE_FLAG_SELECT 2u
 #define SUBDIV_COARSE_FACE_FLAG_ACTIVE 4u
+#define SUBDIV_COARSE_FACE_FLAG_HIDDEN 8u
 
-#define SUBDIV_COARSE_FACE_FLAG_OFFSET 29u
+#define SUBDIV_COARSE_FACE_FLAG_OFFSET 28u
 
 #define SUBDIV_COARSE_FACE_FLAG_SMOOTH_MASK \
   (SUBDIV_COARSE_FACE_FLAG_SMOOTH << SUBDIV_COARSE_FACE_FLAG_OFFSET)
@@ -597,10 +598,12 @@ void draw_subdiv_cache_free(DRWSubdivCache *cache)
   (SUBDIV_COARSE_FACE_FLAG_SELECT << SUBDIV_COARSE_FACE_FLAG_OFFSET)
 #define SUBDIV_COARSE_FACE_FLAG_ACTIVE_MASK \
   (SUBDIV_COARSE_FACE_FLAG_ACTIVE << SUBDIV_COARSE_FACE_FLAG_OFFSET)
+#define SUBDIV_COARSE_FACE_FLAG_HIDDEN_MASK \
+  (SUBDIV_COARSE_FACE_FLAG_HIDDEN << SUBDIV_COARSE_FACE_FLAG_OFFSET)
 
 #define SUBDIV_COARSE_FACE_LOOP_START_MASK \
   ~((SUBDIV_COARSE_FACE_FLAG_SMOOTH | SUBDIV_COARSE_FACE_FLAG_SELECT | \
-     SUBDIV_COARSE_FACE_FLAG_ACTIVE) \
+     SUBDIV_COARSE_FACE_FLAG_ACTIVE | SUBDIV_COARSE_FACE_FLAG_HIDDEN) \
     << SUBDIV_COARSE_FACE_FLAG_OFFSET)
 
 static uint32_t compute_coarse_face_flag(BMFace *f, BMFace *efa_act)
@@ -617,6 +620,9 @@ static uint32_t compute_coarse_face_flag(BMFace *f, BMFace *efa_act)
   if (BM_elem_flag_test(f, BM_ELEM_SELECT)) {
     flag |= SUBDIV_COARSE_FACE_FLAG_SELECT;
   }
+  if (BM_elem_flag_test(f, BM_ELEM_HIDDEN)) {
+    flag |= SUBDIV_COARSE_FACE_FLAG_HIDDEN;
+  }
   if (f == efa_act) {
     flag |= SUBDIV_COARSE_FACE_FLAG_ACTIVE;
   }
@@ -647,6 +653,9 @@ static void draw_subdiv_cache_extra_coarse_face_data_mesh(Mesh *mesh, uint32_t *
     if ((mesh->mpoly[i].flag & ME_FACE_SEL) != 0) {
       flag |= SUBDIV_COARSE_FACE_FLAG_SELECT;
     }
+    if ((mesh->mpoly[i].flag & ME_HIDE) != 0) {
+      flag |= SUBDIV_COARSE_FACE_FLAG_HIDDEN;
+    }
     flags_data[i] = (uint)(mesh->mpoly[i].loopstart) | (flag << SUBDIV_COARSE_FACE_FLAG_OFFSET);
   }
 }
@@ -1119,12 +1128,17 @@ struct DRWSubdivUboStorage {
   uint coarse_face_select_mask;
   uint coarse_face_smooth_mask;
   uint coarse_face_active_mask;
+  uint coarse_face_hidden_mask;
   uint coarse_face_loopstart_mask;
 
   /* Number of elements to process in the compute shader (can be the coarse quad count, or the
    * final vertex count, depending on which compute pass we do). This is used to early out in case
    * of out of bond accesses as compute dispatch are of fixed size. */
   uint total_dispatch_size;
+
+  int _pad0;
+  int _pad2;
+  int _pad3;
 };
 
 static_assert((sizeof(DRWSubdivUboStorage) % 16) == 0,
@@ -1151,6 +1165,7 @@ static void draw_subdiv_init_ubo_storage(const DRWSubdivCache *cache,
   ubo->coarse_face_smooth_mask = SUBDIV_COARSE_FACE_FLAG_SMOOTH_MASK;
   ubo->coarse_face_select_mask = SUBDIV_COARSE_FACE_FLAG_SELECT_MASK;
   ubo->coarse_face_active_mask = SUBDIV_COARSE_FACE_FLAG_ACTIVE_MASK;
+  ubo->coarse_face_hidden_mask = SUBDIV_COARSE_FACE_FLAG_HIDDEN_MASK;
   ubo->coarse_face_loopstart_mask = SUBDIV_COARSE_FACE_LOOP_START_MASK;
   ubo->total_dispatch_size = total_dispatch_size;
 }
@@ -1552,12 +1567,11 @@ void draw_subdiv_build_tris_buffer(const DRWSubdivCache *cache,
       do_single_material ? SHADER_BUFFER_TRIS : SHADER_BUFFER_TRIS_MULTIPLE_MATERIALS, defines);
   GPU_shader_bind(shader);
 
-  if (!do_single_material) {
-    /* subdiv_polygon_offset is always at binding point 0 for each shader using it. */
-    GPU_vertbuf_bind_as_ssbo(cache->subdiv_polygon_offset_buffer, 0);
-  }
+  int binding_point = 0;
 
-  int binding_point = 1;
+  /* subdiv_polygon_offset is always at binding point 0 for each shader using it. */
+  GPU_vertbuf_bind_as_ssbo(cache->subdiv_polygon_offset_buffer, binding_point++);
+  GPU_vertbuf_bind_as_ssbo(cache->extra_coarse_face_data, binding_point++);
 
   /* Outputs */
   GPU_indexbuf_bind_as_ssbo(subdiv_tris, binding_point++);
@@ -1646,11 +1660,13 @@ void draw_subdiv_build_fdots_buffers(const DRWSubdivCache *cache,
 
 void draw_subdiv_build_lines_buffer(const DRWSubdivCache *cache, GPUIndexBuf *lines_indices)
 {
-  GPUShader *shader = get_subdiv_shader(SHADER_BUFFER_LINES, nullptr);
+  GPUShader *shader = get_subdiv_shader(SHADER_BUFFER_LINES, "#define SUBDIV_POLYGON_OFFSET\n");
   GPU_shader_bind(shader);
 
   int binding_point = 0;
+  GPU_vertbuf_bind_as_ssbo(cache->subdiv_polygon_offset_buffer, binding_point++);
   GPU_vertbuf_bind_as_ssbo(cache->edges_orig_index, binding_point++);
+  GPU_vertbuf_bind_as_ssbo(cache->extra_coarse_face_data, binding_point++);
   GPU_indexbuf_bind_as_ssbo(lines_indices, binding_point++);
   BLI_assert(binding_point <= MAX_GPU_SUBDIV_SSBOS);
 
@@ -1665,12 +1681,14 @@ void draw_subdiv_build_lines_buffer(const DRWSubdivCache *cache, GPUIndexBuf *li
 
 void draw_subdiv_build_lines_loose_buffer(const DRWSubdivCache *cache,
                                           GPUIndexBuf *lines_indices,
+                                          GPUVertBuf *lines_flags,
                                           uint num_loose_edges)
 {
   GPUShader *shader = get_subdiv_shader(SHADER_BUFFER_LINES_LOOSE, "#define LINES_LOOSE\n");
   GPU_shader_bind(shader);
 
-  GPU_indexbuf_bind_as_ssbo(lines_indices, 1);
+  GPU_indexbuf_bind_as_ssbo(lines_indices, 3);
+  GPU_vertbuf_bind_as_ssbo(lines_flags, 4);
 
   drw_subdiv_compute_dispatch(cache, shader, 0, 0, num_loose_edges);
 
@@ -1905,7 +1923,7 @@ static bool draw_subdiv_create_requested_buffers(const Scene *scene,
                                                  const bool do_final,
                                                  const bool do_uvedit,
                                                  const ToolSettings *ts,
-                                                 const bool /*use_hide*/,
+                                                 const bool use_hide,
                                                  OpenSubdiv_EvaluatorCache *evaluator_cache)
 {
   SubsurfModifierData *smd = reinterpret_cast<SubsurfModifierData *>(
@@ -1981,6 +1999,7 @@ static bool draw_subdiv_create_requested_buffers(const Scene *scene,
 
   MeshRenderData *mr = mesh_render_data_create(
       ob, mesh, is_editmode, is_paint_mode, is_mode_active, obmat, do_final, do_uvedit, ts);
+  mr->use_hide = use_hide;
 
   draw_subdiv_cache_update_extra_coarse_face_data(draw_cache, mesh_eval, mr);
 
diff --git a/source/blender/draw/intern/draw_subdivision.h b/source/blender/draw/intern/draw_subdivision.h
index b7cd520f54f..2689159d259 100644
--- a/source/blender/draw/intern/draw_subdivision.h
+++ b/source/blender/draw/intern/draw_subdivision.h
@@ -263,6 +263,7 @@ void draw_subdiv_build_lines_buffer(const DRWSubdivCache *cache,
 
 void draw_subdiv_build_lines_loose_buffer(const DRWSubdivCache *cache,
                                           struct GPUIndexBuf *lines_indices,
+                                          GPUVertBuf *lines_flags,
                                           uint num_loose_edges);
 
 void draw_subdiv_build_fdots_buffers(const DRWSubdivCache *cache,
diff --git a/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_lines.cc b/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_lines.cc
index ce3ca428469..3cecaf81b8a 100644
--- a/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_lines.cc
+++ b/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_lines.cc
@@ -160,7 +160,7 @@ static void extract_lines_init_subdiv(const DRWSubdivCache *subdiv_cache,
 }
 
 static void extract_lines_loose_geom_subdiv(const DRWSubdivCache *subdiv_cache,
-                                            const MeshRenderData *UNUSED(mr),
+                                            const MeshRenderData *mr,
                                             void *buffer,
                                             void *UNUSED(data))
 {
@@ -169,8 +169,31 @@ static void extract_lines_loose_geom_subdiv(const DRWSubdivCache *subdiv_cache,
     return;
   }
 
+  /* Update flags for loose edges, points are already handled. */
+  static GPUVertFormat format;
+  if (format.attr_len == 0) {
+    GPU_vertformat_attr_add(&format, "data", GPU_COMP_U32, 1, GPU_FETCH_INT);
+  }
+
+  GPUVertBuf *flags = GPU_vertbuf_calloc();
+  GPU_vertbuf_init_with_format(flags, &format);
+
+  Span<DRWSubdivLooseEdge> loose_edges = draw_subdiv_cache_get_loose_edges(subdiv_cache);
+  GPU_vertbuf_data_alloc(flags, loose_edges.size());
+
+  uint *flags_data = static_cast<uint *>(GPU_vertbuf_get_data(flags));
+
+  const MEdge *medge = mr->medge;
+
+  for (DRWSubdivLooseEdge edge : loose_edges) {
+    *flags_data++ = (medge[edge.coarse_edge_index].flag & ME_HIDE) != 0;
+  }
+
   GPUIndexBuf *ibo = static_cast<GPUIndexBuf *>(buffer);
-  draw_subdiv_build_lines_loose_buffer(subdiv_cache, ibo, static_cast<uint>(loose_geom.edge_len));
+  draw_subdiv_build_lines_loose_buffer(
+      subdiv_cache, ibo, flags, static_cast<uint>(loose_geom.edge_len));
+
+  GPU_vertbuf_discard(flags);
 }
 
 constexpr MeshExtract create_extractor_lines()
diff --git a/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_points.cc b/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_points.cc
index 272963f3fd5..503ce0e79e9 100644
--- a/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_points.cc
+++ b/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_points.cc
@@ -156,7 +156,8 @@ static void extract_points_init_subdiv(const DRWSubdivCache *subdiv_cache,
 static void extract_points_iter_subdiv_common(GPUIndexBufBuilder *elb,
                                               const MeshRenderData *mr,
                                               const DRWSubdivCache *subdiv_cache,
-                                              uint subdiv_quad_index)
+                                              uint subdiv_quad_index,
+                                              bool for_bmesh)
 {
   int *subdiv_loop_vert_index = (int *)GPU_vertbuf_get_data(subdiv_cache->verts_orig_index);
   uint start_loop_idx = subdiv_quad_index * 4;
@@ -172,6 +173,21 @@ static void extract_points_iter_subdiv_common(GPUIndexBufBuilder *elb,
       continue;
     }
 
+    if (for_bmesh) {
+      const BMVert *mv = BM_vert_at_index(mr->bm, coarse_vertex_index);
+      if (BM_elem_flag_test(mv, BM_ELEM_HIDDEN)) {
+        GPU_indexbuf_set_point_restart(elb, coarse_vertex_index);
+        continue;
+      }
+    }
+    else {
+      const MVert *mv = &mr->mvert[coarse_vertex_index];
+      if (mr->use_hide && (mv->flag & ME_HIDE)) {
+        GPU_indexbuf_set_point_restart(elb, coarse_vertex_index);
+        continue;
+      }
+    }
+
     GPU_indexbuf_set_point_vert(elb, coarse_vertex_index, i);
   }
 }
@@ -183,7 +199,7 @@ static void extract_points_iter_subdiv_bm(const DRWSubdivCache *subdiv_cache,
                                           const BMFace *UNUSED(coarse_quad))
 {
   GPUIndexBufBuilder *elb = static_cast<GPUIndexBufBuilder *>(_data);
-  extract_points_iter_subdiv_common(elb, mr, subdiv_cache, subdiv_quad_index);
+  extract_points_iter_subdiv_common(elb, mr, subdiv_cache, subdiv_quad_index, true);
 }
 
 static void extract_points_iter_subdiv_mesh(const DRWSubdivCache *subdiv_cache,
@@ -193,7 +209,7 @@ static void extract_points_iter_subdiv_mesh(const DRWSubdivCache *subdiv_cache,
                                             const MPoly *UNUSED(coarse_quad))
 {
   GPUIndexBufBuilder *elb = static_cast<GPUIndexBufBuilder *>(_data);
-  extract_points_iter_subdiv_common(elb, mr, subdiv_cache, subdiv_quad_index);
+  extract_points_iter_subdiv_common(elb, mr, subdiv_cache, subdiv_quad_index, false);
 }
 
 static void extract_points_loose_geom_subdiv(const DRWSubdivCache *subdiv_cache,
diff --git a/source/blender/draw/intern/shaders/common_subdiv_ibo_lines_comp.glsl b/source/blender/draw/intern/shaders/common_subdiv_ibo_lines_comp.glsl
index 3cbb9f980f3..5084dcc0746 100644
--- a/source/blender/draw/intern/shaders/common_subdiv_ibo_lines_comp.glsl
+++ b/source/blender/draw/intern/shaders/common_subdiv_ibo_lines_comp.glsl
@@ -1,22 +1,41 @@
 
 /* To be compiled with common_subdiv_lib.glsl */
 
-layout(std430, binding = 0) readonly buffer inputEdgeOrigIndex
+layout(std430, binding = 1) readonly buffer inputEdgeOrigIndex
 {
   int input_origindex[];
 };
 
-layout(std430, binding = 1) writeonly buffer outputLinesIndices
+layout(std430, binding = 2) readonly restrict buffer extraCoarseFaceData
+{
+  uint extra_coarse_face_data[];
+};
+
+layout(std430, binding = 3) writeonly buffer outputLinesIndices
 {
   uint output_lines[];
 };
 
+layout(std430, binding = 4) readonly buffer LinesLooseFlags
+{
+  uint lines_loose_flags[];
+};
+
 #ifndef LINES_LOOSE
-void emit_line(uint line_offset, uint start_loop_index, uint corner_index)
+
+bool is_face_hidden(uint coarse_quad_index)
+{
+  return (extra_coarse_face_data[coarse_quad_index] & coarse_face_hidden_mask) != 0;
+}
+
+void emit_line(uint line_offset, uint quad_index, uint start_loop_index, uint corner_index)
 {
   uint vertex_index = start_loop_index + corner_index;
 
-  if (input_origindex[vertex_index] == ORIGINDEX_NONE && optimal_display) {
+  uint coarse_quad_index = coarse_polygon_index_from_subdiv_quad_index(quad_index,
+                                                                       coarse_poly_count);
+
+  if (is_face_hidden(coarse_quad_index) || (input_origindex[vertex_index] == ORIGINDEX_NONE && optimal_display)) {
     output_lines[line_offset + 0] = 0xffffffff;
     output_lines[line_offset + 1] = 0xffffffff;
   }
@@ -41,8 +60,17 @@ void main()
   /* In the loose lines case, we execute for each line, with two vertices per line. */
   uint line_offset = edge_loose_offset + index * 2;
   uint loop_index = num_subdiv_loops + index * 2;
-  output_lines[line_offset] = loop_index;
-  output_lines[line_offset + 1] = loop_index + 1;
+
+  if (lines_loose_flags[index] != 0) {
+      /* Line is hidden. */
+      output_lines[line_offset] = 0xffffffff;
+      output_lines[line_offset + 1] = 0xffffffff;
+  }
+  else {
+      output_lines[line_offset] = loop_index;
+      output_lines[line_offset + 1] = loop_index + 1;
+  }
+
 #else
   /* We execute for each quad, so the start index of the loop is quad_index * 4. */
   uint start_loop_index = index * 4;
@@ -51,7 +79,7 @@ void main()
   uint start_line_index = index * 8;
 
   for (int i = 0; i < 4; i++) {
-    emit_line(start_line_index + i * 2, start_loop_index, i);
+    emit_line(start_line_index + i * 2, index, start_loop_index, i);
   }
 #endif
 }
diff --git a/source/blender/draw/intern/shaders/common_subdiv_ibo_tris_comp.glsl b/source/blender/draw/intern/shaders/common_subdiv_ibo_tris_comp.glsl
index 3dccc82541e..f1275b36c34 100644
--- a/source/blender/draw/intern/shaders/common_subdiv_ibo_tris_comp.glsl
+++ b/source/blender/draw/intern/shaders/common_subdiv_ibo_tris_comp.glsl
@@ -3,18 +3,28 @@
 
 /* Generate triangles from subdivision quads indices. */
 
-layout(std430, binding = 1) writeonly buffer outputTriangles
+layout(std430, binding = 1) readonly restrict buffer extraCoarseFaceData
+{
+  uint extra_coarse_face_data[];
+};
+
+layout(std430, binding = 2) writeonly buffer outputTriangles
 {
   uint output_tris[];
 };
 
 #ifndef SINGLE_MATERIAL
-layout(std430, binding = 2) readonly buffer inputPolygonMatOffset
+layout(std430, binding = 3) readonly buffer inputPolygonMatOffset
 {
   int polygon_mat_offset[];
 };
 #endif
 
+bool is_face_hidden(uint coarse_quad_index)
+{
+  return (extra_coarse_face_data[coarse_quad_index] & coarse_face_hidden_mask) != 0;
+}
+
 void main()
 {
   uint quad_index = get_global_invocation_index();
@@ -24,20 +34,31 @@ void main()
 
   uint loop_index = quad_index * 4;
 
+  uint coarse_quad_index = coarse_polygon_index_from_subdiv_quad_index(quad_index,
+                                                                       coarse_poly_count);
+
 #ifdef SINGLE_MATERIAL
   uint triangle_loop_index = quad_index * 6;
 #else
-  uint coarse_quad_index = coarse_polygon_index_from_subdiv_quad_index(quad_index,
-                                                                       coarse_poly_count);
   int mat_offset = polygon_mat_offset[coarse_quad_index];
 
   int triangle_loop_index = (int(quad_index) + mat_offset) * 6;
 #endif
 
-  output_tris[triangle_loop_index + 0] = loop_index + 0;
-  output_tris[triangle_loop_index + 1] = loop_index + 1;
-  output_tris[triangle_loop_index + 2] = loop_index + 2;
-  output_tris[triangle_loop_index + 3] = loop_index + 0;
-  output_tris[triangle_loop_index + 4] = loop_index + 2;
-  output_tris[triangle_loop_index + 5] = loop_index + 3;
+  if (is_face_hidden(coarse_quad_index)) {
+      output_tris[triangle_loop_index + 0] = 0xffffffff;
+      output_tris[triangle_loop_index + 1] = 0xffffffff;
+      output_tris[triangle_loop_index + 2] = 0xffffffff;
+      output_tris[triangle_loop_index + 3] = 0xffffffff;
+      output_tris[triangle_loop_index + 4] = 0xffffffff;
+      output_tris[triangle_loop_index + 5] = 0xffffffff;
+  }
+  else {
+      output_tris[triangle_loop_index + 0] = loop_index + 0;
+      output_tris[triangle_loop_index + 1] = loop_index + 1;
+      output_tris[triangle_loop_index + 2] = loop_index + 2;
+      output_tris[triangle_loop_index + 3] = loop_index + 0;
+      output_tris[triangle_loop_index + 4] = loop_index + 2;
+      output_tris[triangle_loop_index + 5] = loop_index + 3;
+   }
 }
diff --git a/source/blender/draw/intern/shaders/common_subdiv_lib.glsl b/source/blender/draw/intern/shaders/common_subdiv_lib.glsl
index ce324249446..55e4ac20271 100644
--- a/source/blender/draw/intern/shaders/common_subdiv_lib.glsl
+++ b/source/blender/draw/intern/shaders/common_subdiv_lib.glsl
@@ -31,6 +31,7 @@ layout(std140) uniform shader_data
   uint coarse_face_select_mask;
   uint coarse_face_smooth_mask;
   uint coarse_face_active_mask;
+  uint coarse_face_hidden_mask;
   uint coarse_face_loopstart_mask;
 
   /* Total number of elements to process. */
diff --git a/source/blender/draw/intern/shaders/common_subdiv_patch_evaluation_comp.glsl b/source/blender/draw/intern/shaders/common_subdiv_patch_evaluation_comp.glsl
index 65cf4ebb90f..e8d98428a8d 100644
--- a/source/blender/draw/intern/shaders/common_subdiv_patch_evaluation_comp.glsl
+++ b/source/blender/draw/intern/shaders/common_subdiv_patch_evaluation_comp.glsl
@@ -341,6 +341,11 @@ float get_face_flag(uint coarse_quad_index)
   return 0.0;
 }
 
+bool is_face_hidden(uint coarse_quad_index)
+{
+  return (extra_coarse_face_data[coarse_quad_index] & coarse_face_hidden_mask) != 0;
+}
+
 void main()
 {
   /* We execute for each coarse quad. */
@@ -371,7 +376,13 @@ void main()
 
   output_verts[coarse_quad_index] = vert;
   output_nors[coarse_quad_index] = fnor;
-  output_indices[coarse_quad_index] = coarse_quad_index;
+
+  if (is_face_hidden(coarse_quad_index)) {
+      output_indices[coarse_quad_index] = 0xffffffff;
+  }
+  else {
+      output_indices[coarse_quad_index] = coarse_quad_index;
+  }
 }
 #else
 void main()