Fix T96356: artefacts with GPU subdivision and vertex paint mask

The lines paint mask IBO extraction was not implemented for GPU subdivision.
For it to work, we also now need to preserve the subdivision loop to
subdivision edge map, which until now was overwritten to store coarse edges
(the map to coarse edges is still preserved).

Also the paint flag stored in the 4th dimension of the loop normal buffer
was not properly set for flat shaded faces, leading to other kind of
artefacts and render issues.

1 files changed, 77 insertions, 0 deletions

diff --git a/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_lines_paint_mask.cc b/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_lines_paint_mask.cc
index 9d799feabc6..286c7ea9c43 100644
--- a/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_lines_paint_mask.cc
+++ b/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_lines_paint_mask.cc
@@ -11,6 +11,7 @@
 
 #include "MEM_guardedalloc.h"
 
+#include "draw_subdivision.h"
 #include "extract_mesh.h"
 
 namespace blender::draw {
@@ -87,12 +88,88 @@ static void extract_lines_paint_mask_finish(const MeshRenderData *UNUSED(mr),
   MEM_freeN(data->select_map);
 }
 
+static void extract_lines_paint_mask_init_subdiv(const DRWSubdivCache *subdiv_cache,
+                                                 const MeshRenderData *mr,
+                                                 MeshBatchCache *UNUSED(cache),
+                                                 void *UNUSED(buf),
+                                                 void *tls_data)
+{
+  MeshExtract_LinePaintMask_Data *data = static_cast<MeshExtract_LinePaintMask_Data *>(tls_data);
+  data->select_map = BLI_BITMAP_NEW(mr->edge_len, __func__);
+  GPU_indexbuf_init(&data->elb,
+                    GPU_PRIM_LINES,
+                    subdiv_cache->num_subdiv_edges,
+                    subdiv_cache->num_subdiv_loops * 2);
+}
+
+static void extract_lines_paint_mask_iter_subdiv_mesh(const DRWSubdivCache *subdiv_cache,
+                                                      const MeshRenderData *mr,
+                                                      void *_data,
+                                                      uint subdiv_quad_index,
+                                                      const MPoly *coarse_quad)
+{
+  MeshExtract_LinePaintMask_Data *data = static_cast<MeshExtract_LinePaintMask_Data *>(_data);
+  int *subdiv_loop_edge_index = (int *)GPU_vertbuf_get_data(subdiv_cache->edges_orig_index);
+  int *subdiv_loop_subdiv_edge_index = subdiv_cache->subdiv_loop_subdiv_edge_index;
+
+  uint start_loop_idx = subdiv_quad_index * 4;
+  uint end_loop_idx = (subdiv_quad_index + 1) * 4;
+  for (uint loop_idx = start_loop_idx; loop_idx < end_loop_idx; loop_idx++) {
+    const uint coarse_edge_index = (uint)subdiv_loop_edge_index[loop_idx];
+    const uint subdiv_edge_index = (uint)subdiv_loop_subdiv_edge_index[loop_idx];
+
+    if (coarse_edge_index == -1u) {
+      GPU_indexbuf_set_line_restart(&data->elb, subdiv_edge_index);
+    }
+    else {
+      const MEdge *me = &mr->medge[coarse_edge_index];
+      if (!((mr->use_hide && (me->flag & ME_HIDE)) ||
+            ((mr->extract_type == MR_EXTRACT_MAPPED) && (mr->e_origindex) &&
+             (mr->e_origindex[coarse_edge_index] == ORIGINDEX_NONE)))) {
+        const uint ml_index_other = (loop_idx == end_loop_idx) ? start_loop_idx : loop_idx + 1;
+        if (coarse_quad->flag & ME_FACE_SEL) {
+          if (BLI_BITMAP_TEST_AND_SET_ATOMIC(data->select_map, coarse_edge_index)) {
+            /* Hide edge as it has more than 2 selected loop. */
+            GPU_indexbuf_set_line_restart(&data->elb, subdiv_edge_index);
+          }
+          else {
+            /* First selected loop. Set edge visible, overwriting any unselected loop. */
+            GPU_indexbuf_set_line_verts(&data->elb, subdiv_edge_index, loop_idx, ml_index_other);
+          }
+        }
+        else {
+          /* Set these unselected loop only if this edge has no other selected loop. */
+          if (!BLI_BITMAP_TEST(data->select_map, coarse_edge_index)) {
+            GPU_indexbuf_set_line_verts(&data->elb, subdiv_edge_index, loop_idx, ml_index_other);
+          }
+        }
+      }
+      else {
+        GPU_indexbuf_set_line_restart(&data->elb, subdiv_edge_index);
+      }
+    }
+  }
+}
+
+static void extract_lines_paint_mask_finish_subdiv(
+    const struct DRWSubdivCache *UNUSED(subdiv_cache),
+    const MeshRenderData *mr,
+    struct MeshBatchCache *cache,
+    void *buf,
+    void *_data)
+{
+  extract_lines_paint_mask_finish(mr, cache, buf, _data);
+}
+
 constexpr MeshExtract create_extractor_lines_paint_mask()
 {
   MeshExtract extractor = {nullptr};
   extractor.init = extract_lines_paint_mask_init;
   extractor.iter_poly_mesh = extract_lines_paint_mask_iter_poly_mesh;
   extractor.finish = extract_lines_paint_mask_finish;
+  extractor.init_subdiv = extract_lines_paint_mask_init_subdiv;
+  extractor.iter_subdiv_mesh = extract_lines_paint_mask_iter_subdiv_mesh;
+  extractor.finish_subdiv = extract_lines_paint_mask_finish_subdiv;
   extractor.data_type = MR_DATA_NONE;
   extractor.data_size = sizeof(MeshExtract_LinePaintMask_Data);
   extractor.use_threading = false;