Fix T96080: hiding elements does not work with GPU subdiv

Faces, edges, and vertices are still shown when GPU subdivision is
actived. This is because the related edit mode flags were ignored by the
subdivision code.

The flags are now passed to the various compute shaders mostly as part of
the extra coarse data, also used for e.g. selection. For loose edges, a
temporary buffer is created when extracting them. Loose vertices are
already taken into account as it reuses the routines for coarse mesh
extraction, although `MeshRenderData.use_hide` was not initialized,
which is fixed now.

4 files changed, 79 insertions, 18 deletions

diff --git a/source/blender/draw/intern/shaders/common_subdiv_ibo_lines_comp.glsl b/source/blender/draw/intern/shaders/common_subdiv_ibo_lines_comp.glsl
index 3cbb9f980f3..5084dcc0746 100644
--- a/source/blender/draw/intern/shaders/common_subdiv_ibo_lines_comp.glsl
+++ b/source/blender/draw/intern/shaders/common_subdiv_ibo_lines_comp.glsl
@@ -1,22 +1,41 @@
 
 /* To be compiled with common_subdiv_lib.glsl */
 
-layout(std430, binding = 0) readonly buffer inputEdgeOrigIndex
+layout(std430, binding = 1) readonly buffer inputEdgeOrigIndex
 {
   int input_origindex[];
 };
 
-layout(std430, binding = 1) writeonly buffer outputLinesIndices
+layout(std430, binding = 2) readonly restrict buffer extraCoarseFaceData
+{
+  uint extra_coarse_face_data[];
+};
+
+layout(std430, binding = 3) writeonly buffer outputLinesIndices
 {
   uint output_lines[];
 };
 
+layout(std430, binding = 4) readonly buffer LinesLooseFlags
+{
+  uint lines_loose_flags[];
+};
+
 #ifndef LINES_LOOSE
-void emit_line(uint line_offset, uint start_loop_index, uint corner_index)
+
+bool is_face_hidden(uint coarse_quad_index)
+{
+  return (extra_coarse_face_data[coarse_quad_index] & coarse_face_hidden_mask) != 0;
+}
+
+void emit_line(uint line_offset, uint quad_index, uint start_loop_index, uint corner_index)
 {
   uint vertex_index = start_loop_index + corner_index;
 
-  if (input_origindex[vertex_index] == ORIGINDEX_NONE && optimal_display) {
+  uint coarse_quad_index = coarse_polygon_index_from_subdiv_quad_index(quad_index,
+                                                                       coarse_poly_count);
+
+  if (is_face_hidden(coarse_quad_index) || (input_origindex[vertex_index] == ORIGINDEX_NONE && optimal_display)) {
     output_lines[line_offset + 0] = 0xffffffff;
     output_lines[line_offset + 1] = 0xffffffff;
   }
@@ -41,8 +60,17 @@ void main()
   /* In the loose lines case, we execute for each line, with two vertices per line. */
   uint line_offset = edge_loose_offset + index * 2;
   uint loop_index = num_subdiv_loops + index * 2;
-  output_lines[line_offset] = loop_index;
-  output_lines[line_offset + 1] = loop_index + 1;
+
+  if (lines_loose_flags[index] != 0) {
+      /* Line is hidden. */
+      output_lines[line_offset] = 0xffffffff;
+      output_lines[line_offset + 1] = 0xffffffff;
+  }
+  else {
+      output_lines[line_offset] = loop_index;
+      output_lines[line_offset + 1] = loop_index + 1;
+  }
+
 #else
   /* We execute for each quad, so the start index of the loop is quad_index * 4. */
   uint start_loop_index = index * 4;
@@ -51,7 +79,7 @@ void main()
   uint start_line_index = index * 8;
 
   for (int i = 0; i < 4; i++) {
-    emit_line(start_line_index + i * 2, start_loop_index, i);
+    emit_line(start_line_index + i * 2, index, start_loop_index, i);
   }
 #endif
 }
diff --git a/source/blender/draw/intern/shaders/common_subdiv_ibo_tris_comp.glsl b/source/blender/draw/intern/shaders/common_subdiv_ibo_tris_comp.glsl
index 3dccc82541e..f1275b36c34 100644
--- a/source/blender/draw/intern/shaders/common_subdiv_ibo_tris_comp.glsl
+++ b/source/blender/draw/intern/shaders/common_subdiv_ibo_tris_comp.glsl
@@ -3,18 +3,28 @@
 
 /* Generate triangles from subdivision quads indices. */
 
-layout(std430, binding = 1) writeonly buffer outputTriangles
+layout(std430, binding = 1) readonly restrict buffer extraCoarseFaceData
+{
+  uint extra_coarse_face_data[];
+};
+
+layout(std430, binding = 2) writeonly buffer outputTriangles
 {
   uint output_tris[];
 };
 
 #ifndef SINGLE_MATERIAL
-layout(std430, binding = 2) readonly buffer inputPolygonMatOffset
+layout(std430, binding = 3) readonly buffer inputPolygonMatOffset
 {
   int polygon_mat_offset[];
 };
 #endif
 
+bool is_face_hidden(uint coarse_quad_index)
+{
+  return (extra_coarse_face_data[coarse_quad_index] & coarse_face_hidden_mask) != 0;
+}
+
 void main()
 {
   uint quad_index = get_global_invocation_index();
@@ -24,20 +34,31 @@ void main()
 
   uint loop_index = quad_index * 4;
 
+  uint coarse_quad_index = coarse_polygon_index_from_subdiv_quad_index(quad_index,
+                                                                       coarse_poly_count);
+
 #ifdef SINGLE_MATERIAL
   uint triangle_loop_index = quad_index * 6;
 #else
-  uint coarse_quad_index = coarse_polygon_index_from_subdiv_quad_index(quad_index,
-                                                                       coarse_poly_count);
   int mat_offset = polygon_mat_offset[coarse_quad_index];
 
   int triangle_loop_index = (int(quad_index) + mat_offset) * 6;
 #endif
 
-  output_tris[triangle_loop_index + 0] = loop_index + 0;
-  output_tris[triangle_loop_index + 1] = loop_index + 1;
-  output_tris[triangle_loop_index + 2] = loop_index + 2;
-  output_tris[triangle_loop_index + 3] = loop_index + 0;
-  output_tris[triangle_loop_index + 4] = loop_index + 2;
-  output_tris[triangle_loop_index + 5] = loop_index + 3;
+  if (is_face_hidden(coarse_quad_index)) {
+      output_tris[triangle_loop_index + 0] = 0xffffffff;
+      output_tris[triangle_loop_index + 1] = 0xffffffff;
+      output_tris[triangle_loop_index + 2] = 0xffffffff;
+      output_tris[triangle_loop_index + 3] = 0xffffffff;
+      output_tris[triangle_loop_index + 4] = 0xffffffff;
+      output_tris[triangle_loop_index + 5] = 0xffffffff;
+  }
+  else {
+      output_tris[triangle_loop_index + 0] = loop_index + 0;
+      output_tris[triangle_loop_index + 1] = loop_index + 1;
+      output_tris[triangle_loop_index + 2] = loop_index + 2;
+      output_tris[triangle_loop_index + 3] = loop_index + 0;
+      output_tris[triangle_loop_index + 4] = loop_index + 2;
+      output_tris[triangle_loop_index + 5] = loop_index + 3;
+   }
 }
diff --git a/source/blender/draw/intern/shaders/common_subdiv_lib.glsl b/source/blender/draw/intern/shaders/common_subdiv_lib.glsl
index ce324249446..55e4ac20271 100644
--- a/source/blender/draw/intern/shaders/common_subdiv_lib.glsl
+++ b/source/blender/draw/intern/shaders/common_subdiv_lib.glsl
@@ -31,6 +31,7 @@ layout(std140) uniform shader_data
   uint coarse_face_select_mask;
   uint coarse_face_smooth_mask;
   uint coarse_face_active_mask;
+  uint coarse_face_hidden_mask;
   uint coarse_face_loopstart_mask;
 
   /* Total number of elements to process. */
diff --git a/source/blender/draw/intern/shaders/common_subdiv_patch_evaluation_comp.glsl b/source/blender/draw/intern/shaders/common_subdiv_patch_evaluation_comp.glsl
index 65cf4ebb90f..e8d98428a8d 100644
--- a/source/blender/draw/intern/shaders/common_subdiv_patch_evaluation_comp.glsl
+++ b/source/blender/draw/intern/shaders/common_subdiv_patch_evaluation_comp.glsl
@@ -341,6 +341,11 @@ float get_face_flag(uint coarse_quad_index)
   return 0.0;
 }
 
+bool is_face_hidden(uint coarse_quad_index)
+{
+  return (extra_coarse_face_data[coarse_quad_index] & coarse_face_hidden_mask) != 0;
+}
+
 void main()
 {
   /* We execute for each coarse quad. */
@@ -371,7 +376,13 @@ void main()
 
   output_verts[coarse_quad_index] = vert;
   output_nors[coarse_quad_index] = fnor;
-  output_indices[coarse_quad_index] = coarse_quad_index;
+
+  if (is_face_hidden(coarse_quad_index)) {
+      output_indices[coarse_quad_index] = 0xffffffff;
+  }
+  else {
+      output_indices[coarse_quad_index] = coarse_quad_index;
+  }
 }
 #else
 void main()