Fix T96356: artefacts with GPU subdivision and vertex paint mask

The lines paint mask IBO extraction was not implemented for GPU subdivision.
For it to work, we also now need to preserve the subdivision loop to
subdivision edge map, which until now was overwritten to store coarse edges
(the map to coarse edges is still preserved).

Also the paint flag stored in the 4th dimension of the loop normal buffer
was not properly set for flat shaded faces, leading to other kind of
artefacts and render issues.

6 files changed, 126 insertions, 8 deletions

diff --git a/source/blender/draw/intern/draw_cache_extract_mesh.cc b/source/blender/draw/intern/draw_cache_extract_mesh.cc
index c36a8ca5e2b..0180fe51cb6 100644
--- a/source/blender/draw/intern/draw_cache_extract_mesh.cc
+++ b/source/blender/draw/intern/draw_cache_extract_mesh.cc
@@ -858,6 +858,7 @@ static void mesh_buffer_cache_create_requested_subdiv(MeshBatchCache *cache,
   EXTRACT_ADD_REQUESTED(vbo, tan);
   EXTRACT_ADD_REQUESTED(vbo, edituv_stretch_area);
   EXTRACT_ADD_REQUESTED(vbo, edituv_stretch_angle);
+  EXTRACT_ADD_REQUESTED(ibo, lines_paint_mask);
   EXTRACT_ADD_REQUESTED(ibo, lines_adjacency);
   EXTRACT_ADD_REQUESTED(vbo, vcol);
   EXTRACT_ADD_REQUESTED(vbo, weights);
diff --git a/source/blender/draw/intern/draw_cache_impl_subdivision.cc b/source/blender/draw/intern/draw_cache_impl_subdivision.cc
index c7368c90213..dccad081f64 100644
--- a/source/blender/draw/intern/draw_cache_impl_subdivision.cc
+++ b/source/blender/draw/intern/draw_cache_impl_subdivision.cc
@@ -559,6 +559,7 @@ void draw_subdiv_cache_free(DRWSubdivCache *cache)
   GPU_VERTBUF_DISCARD_SAFE(cache->subdiv_polygon_offset_buffer);
   GPU_VERTBUF_DISCARD_SAFE(cache->extra_coarse_face_data);
   MEM_SAFE_FREE(cache->subdiv_loop_subdiv_vert_index);
+  MEM_SAFE_FREE(cache->subdiv_loop_subdiv_edge_index);
   MEM_SAFE_FREE(cache->subdiv_loop_poly_index);
   MEM_SAFE_FREE(cache->subdiv_polygon_offset);
   GPU_VERTBUF_DISCARD_SAFE(cache->subdiv_vertex_face_adjacency_offsets);
@@ -647,6 +648,9 @@ static void draw_subdiv_cache_extra_coarse_face_data_mesh(Mesh *mesh, uint32_t *
     if ((mesh->mpoly[i].flag & ME_SMOOTH) != 0) {
       flag = SUBDIV_COARSE_FACE_FLAG_SMOOTH;
     }
+    if ((mesh->mpoly[i].flag & ME_FACE_SEL) != 0) {
+      flag = SUBDIV_COARSE_FACE_FLAG_SELECT;
+    }
     flags_data[i] = (uint)(mesh->mpoly[i].loopstart) | (flag << SUBDIV_COARSE_FACE_FLAG_OFFSET);
   }
 }
@@ -735,6 +739,7 @@ struct DRWCacheBuildingContext {
   int *subdiv_loop_vert_index;
   int *subdiv_loop_subdiv_vert_index;
   int *subdiv_loop_edge_index;
+  int *subdiv_loop_subdiv_edge_index;
   int *subdiv_loop_poly_index;
 
   /* Temporary buffers used during traversal. */
@@ -796,6 +801,9 @@ static bool draw_subdiv_topology_info_cb(const SubdivForeachContext *foreach_con
   cache->subdiv_loop_subdiv_vert_index = static_cast<int *>(
       MEM_mallocN(cache->num_subdiv_loops * sizeof(int), "subdiv_loop_subdiv_vert_index"));
 
+  cache->subdiv_loop_subdiv_edge_index = static_cast<int *>(
+      MEM_mallocN(cache->num_subdiv_loops * sizeof(int), "subdiv_loop_subdiv_edge_index"));
+
   cache->subdiv_loop_poly_index = static_cast<int *>(
       MEM_mallocN(cache->num_subdiv_loops * sizeof(int), "subdiv_loop_poly_index"));
 
@@ -804,6 +812,7 @@ static bool draw_subdiv_topology_info_cb(const SubdivForeachContext *foreach_con
   ctx->subdiv_loop_vert_index = (int *)GPU_vertbuf_get_data(cache->verts_orig_index);
   ctx->subdiv_loop_edge_index = (int *)GPU_vertbuf_get_data(cache->edges_orig_index);
   ctx->subdiv_loop_subdiv_vert_index = cache->subdiv_loop_subdiv_vert_index;
+  ctx->subdiv_loop_subdiv_edge_index = cache->subdiv_loop_subdiv_edge_index;
   ctx->subdiv_loop_poly_index = cache->subdiv_loop_poly_index;
 
   ctx->v_origindex = static_cast<int *>(
@@ -902,9 +911,7 @@ static void draw_subdiv_loop_cb(const SubdivForeachContext *foreach_context,
   int coarse_vertex_index = ctx->vert_origindex_map[subdiv_vertex_index];
 
   ctx->subdiv_loop_subdiv_vert_index[subdiv_loop_index] = subdiv_vertex_index;
-  /* For now index the subdiv_edge_index, it will be replaced by the actual coarse edge index
-   * at the end of the traversal as some edges are only then traversed. */
-  ctx->subdiv_loop_edge_index[subdiv_loop_index] = subdiv_edge_index;
+  ctx->subdiv_loop_subdiv_edge_index[subdiv_loop_index] = subdiv_edge_index;
   ctx->subdiv_loop_poly_index[subdiv_loop_index] = coarse_poly_index;
   ctx->subdiv_loop_vert_index[subdiv_loop_index] = coarse_vertex_index;
 }
@@ -930,12 +937,13 @@ static void do_subdiv_traversal(DRWCacheBuildingContext *cache_building_context,
                                      cache_building_context->settings,
                                      cache_building_context->coarse_mesh);
 
-  /* Now that traversal is done, we can set up the right original indices for the loop-to-edge map.
+  /* Now that traversal is done, we can set up the right original indices for the
+   * subdiv-loop-to-coarse-edge map.
    */
   for (int i = 0; i < cache_building_context->cache->num_subdiv_loops; i++) {
     cache_building_context->subdiv_loop_edge_index[i] =
         cache_building_context
-            ->edge_origindex_map[cache_building_context->subdiv_loop_edge_index[i]];
+            ->edge_origindex_map[cache_building_context->subdiv_loop_subdiv_edge_index[i]];
   }
 }
 
diff --git a/source/blender/draw/intern/draw_subdivision.h b/source/blender/draw/intern/draw_subdivision.h
index a2ca1c1735c..ffed4dfc189 100644
--- a/source/blender/draw/intern/draw_subdivision.h
+++ b/source/blender/draw/intern/draw_subdivision.h
@@ -145,6 +145,8 @@ typedef struct DRWSubdivCache {
 
   /* Maps subdivision loop to subdivided vertex index. */
   int *subdiv_loop_subdiv_vert_index;
+  /* Maps subdivision loop to subdivided edge index. */
+  int *subdiv_loop_subdiv_edge_index;
   /* Maps subdivision loop to original coarse poly index. */
   int *subdiv_loop_poly_index;
 
diff --git a/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_lines_paint_mask.cc b/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_lines_paint_mask.cc
index f7eb5022cdc..8a40aeb8e56 100644
--- a/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_lines_paint_mask.cc
+++ b/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_lines_paint_mask.cc
@@ -27,6 +27,7 @@
 
 #include "MEM_guardedalloc.h"
 
+#include "draw_subdivision.h"
 #include "extract_mesh.h"
 
 namespace blender::draw {
@@ -103,12 +104,88 @@ static void extract_lines_paint_mask_finish(const MeshRenderData *UNUSED(mr),
   MEM_freeN(data->select_map);
 }
 
+static void extract_lines_paint_mask_init_subdiv(const DRWSubdivCache *subdiv_cache,
+                                                 const MeshRenderData *mr,
+                                                 MeshBatchCache *UNUSED(cache),
+                                                 void *UNUSED(buf),
+                                                 void *tls_data)
+{
+  MeshExtract_LinePaintMask_Data *data = static_cast<MeshExtract_LinePaintMask_Data *>(tls_data);
+  data->select_map = BLI_BITMAP_NEW(mr->edge_len, __func__);
+  GPU_indexbuf_init(&data->elb,
+                    GPU_PRIM_LINES,
+                    subdiv_cache->num_subdiv_edges,
+                    subdiv_cache->num_subdiv_loops * 2);
+}
+
+static void extract_lines_paint_mask_iter_subdiv_mesh(const DRWSubdivCache *subdiv_cache,
+                                                      const MeshRenderData *mr,
+                                                      void *_data,
+                                                      uint subdiv_quad_index,
+                                                      const MPoly *coarse_quad)
+{
+  MeshExtract_LinePaintMask_Data *data = static_cast<MeshExtract_LinePaintMask_Data *>(_data);
+  int *subdiv_loop_edge_index = (int *)GPU_vertbuf_get_data(subdiv_cache->edges_orig_index);
+  int *subdiv_loop_subdiv_edge_index = subdiv_cache->subdiv_loop_subdiv_edge_index;
+
+  uint start_loop_idx = subdiv_quad_index * 4;
+  uint end_loop_idx = (subdiv_quad_index + 1) * 4;
+  for (uint loop_idx = start_loop_idx; loop_idx < end_loop_idx; loop_idx++) {
+    const uint coarse_edge_index = (uint)subdiv_loop_edge_index[loop_idx];
+    const uint subdiv_edge_index = (uint)subdiv_loop_subdiv_edge_index[loop_idx];
+
+    if (coarse_edge_index == -1u) {
+      GPU_indexbuf_set_line_restart(&data->elb, subdiv_edge_index);
+    }
+    else {
+      const MEdge *me = &mr->medge[coarse_edge_index];
+      if (!((mr->use_hide && (me->flag & ME_HIDE)) ||
+            ((mr->extract_type == MR_EXTRACT_MAPPED) && (mr->e_origindex) &&
+             (mr->e_origindex[coarse_edge_index] == ORIGINDEX_NONE)))) {
+        const uint ml_index_other = (loop_idx == end_loop_idx) ? start_loop_idx : loop_idx + 1;
+        if (coarse_quad->flag & ME_FACE_SEL) {
+          if (BLI_BITMAP_TEST_AND_SET_ATOMIC(data->select_map, coarse_edge_index)) {
+            /* Hide edge as it has more than 2 selected loop. */
+            GPU_indexbuf_set_line_restart(&data->elb, subdiv_edge_index);
+          }
+          else {
+            /* First selected loop. Set edge visible, overwriting any unselected loop. */
+            GPU_indexbuf_set_line_verts(&data->elb, subdiv_edge_index, loop_idx, ml_index_other);
+          }
+        }
+        else {
+          /* Set these unselected loop only if this edge has no other selected loop. */
+          if (!BLI_BITMAP_TEST(data->select_map, coarse_edge_index)) {
+            GPU_indexbuf_set_line_verts(&data->elb, subdiv_edge_index, loop_idx, ml_index_other);
+          }
+        }
+      }
+      else {
+        GPU_indexbuf_set_line_restart(&data->elb, subdiv_edge_index);
+      }
+    }
+  }
+}
+
+static void extract_lines_paint_mask_finish_subdiv(
+    const struct DRWSubdivCache *UNUSED(subdiv_cache),
+    const MeshRenderData *mr,
+    struct MeshBatchCache *cache,
+    void *buf,
+    void *_data)
+{
+  extract_lines_paint_mask_finish(mr, cache, buf, _data);
+}
+
 constexpr MeshExtract create_extractor_lines_paint_mask()
 {
   MeshExtract extractor = {nullptr};
   extractor.init = extract_lines_paint_mask_init;
   extractor.iter_poly_mesh = extract_lines_paint_mask_iter_poly_mesh;
   extractor.finish = extract_lines_paint_mask_finish;
+  extractor.init_subdiv = extract_lines_paint_mask_init_subdiv;
+  extractor.iter_subdiv_mesh = extract_lines_paint_mask_iter_subdiv_mesh;
+  extractor.finish_subdiv = extract_lines_paint_mask_finish_subdiv;
   extractor.data_type = MR_DATA_NONE;
   extractor.data_size = sizeof(MeshExtract_LinePaintMask_Data);
   extractor.use_threading = false;
diff --git a/source/blender/draw/intern/shaders/common_subdiv_lib.glsl b/source/blender/draw/intern/shaders/common_subdiv_lib.glsl
index a620905622a..44048e73ee5 100644
--- a/source/blender/draw/intern/shaders/common_subdiv_lib.glsl
+++ b/source/blender/draw/intern/shaders/common_subdiv_lib.glsl
@@ -107,6 +107,10 @@ struct PosNorLoop {
   float flag;
 };
 
+struct LoopNormal {
+  float nx, ny, nz, flag;
+};
+
 vec3 get_vertex_pos(PosNorLoop vertex_data)
 {
   return vec3(vertex_data.x, vertex_data.y, vertex_data.z);
@@ -117,6 +121,16 @@ vec3 get_vertex_nor(PosNorLoop vertex_data)
   return vec3(vertex_data.nx, vertex_data.ny, vertex_data.nz);
 }
 
+LoopNormal get_normal_and_flag(PosNorLoop vertex_data)
+{
+  LoopNormal loop_nor;
+  loop_nor.nx = vertex_data.nx;
+  loop_nor.ny = vertex_data.ny;
+  loop_nor.nz = vertex_data.nz;
+  loop_nor.flag = vertex_data.flag;
+  return loop_nor;
+}
+
 void set_vertex_pos(inout PosNorLoop vertex_data, vec3 pos)
 {
   vertex_data.x = pos.x;
diff --git a/source/blender/draw/intern/shaders/common_subdiv_vbo_lnor_comp.glsl b/source/blender/draw/intern/shaders/common_subdiv_vbo_lnor_comp.glsl
index 8b4a5dd14f3..c3aa74ec18c 100644
--- a/source/blender/draw/intern/shaders/common_subdiv_vbo_lnor_comp.glsl
+++ b/source/blender/draw/intern/shaders/common_subdiv_vbo_lnor_comp.glsl
@@ -13,9 +13,14 @@ layout(std430, binding = 2) readonly buffer extraCoarseFaceData
 
 layout(std430, binding = 3) writeonly buffer outputLoopNormals
 {
-  vec3 output_lnor[];
+  LoopNormal output_lnor[];
 };
 
+bool is_face_selected(uint coarse_quad_index)
+{
+  return (extra_coarse_face_data[coarse_quad_index] & coarse_face_select_mask) != 0;
+}
+
 void main()
 {
   /* We execute for each quad. */
@@ -34,7 +39,7 @@ void main()
     /* Face is smooth, use vertex normals. */
     for (int i = 0; i < 4; i++) {
       PosNorLoop pos_nor_loop = pos_nor[start_loop_index + i];
-      output_lnor[start_loop_index + i] = get_vertex_nor(pos_nor_loop);
+      output_lnor[start_loop_index + i] = get_normal_and_flag(pos_nor_loop);
     }
   }
   else {
@@ -50,8 +55,19 @@ void main()
     add_newell_cross_v3_v3v3(face_normal, v3, v0);
 
     face_normal = normalize(face_normal);
+
+    LoopNormal loop_normal;
+    loop_normal.nx = face_normal.x;
+    loop_normal.ny = face_normal.y;
+    loop_normal.nz = face_normal.z;
+    loop_normal.flag = 0.0;
+
+    if (is_face_selected(coarse_quad_index)) {
+      loop_normal.flag = 1.0;
+    }
+
     for (int i = 0; i < 4; i++) {
-      output_lnor[start_loop_index + i] = face_normal;
+      output_lnor[start_loop_index + i] = loop_normal;
     }
   }
 }