Fix T96356: artefacts with GPU subdivision and vertex paint mask

The lines paint mask IBO extraction was not implemented for GPU subdivision.
For it to work, we also now need to preserve the subdivision loop to
subdivision edge map, which until now was overwritten to store coarse edges
(the map to coarse edges is still preserved).

Also the paint flag stored in the 4th dimension of the loop normal buffer
was not properly set for flat shaded faces, leading to other kind of
artefacts and render issues.

2 files changed, 33 insertions, 3 deletions

diff --git a/source/blender/draw/intern/shaders/common_subdiv_lib.glsl b/source/blender/draw/intern/shaders/common_subdiv_lib.glsl
index 5d71c5e4bb8..ce324249446 100644
--- a/source/blender/draw/intern/shaders/common_subdiv_lib.glsl
+++ b/source/blender/draw/intern/shaders/common_subdiv_lib.glsl
@@ -107,6 +107,10 @@ struct PosNorLoop {
   float flag;
 };
 
+struct LoopNormal {
+  float nx, ny, nz, flag;
+};
+
 vec3 get_vertex_pos(PosNorLoop vertex_data)
 {
   return vec3(vertex_data.x, vertex_data.y, vertex_data.z);
@@ -117,6 +121,16 @@ vec3 get_vertex_nor(PosNorLoop vertex_data)
   return vec3(vertex_data.nx, vertex_data.ny, vertex_data.nz);
 }
 
+LoopNormal get_normal_and_flag(PosNorLoop vertex_data)
+{
+  LoopNormal loop_nor;
+  loop_nor.nx = vertex_data.nx;
+  loop_nor.ny = vertex_data.ny;
+  loop_nor.nz = vertex_data.nz;
+  loop_nor.flag = vertex_data.flag;
+  return loop_nor;
+}
+
 void set_vertex_pos(inout PosNorLoop vertex_data, vec3 pos)
 {
   vertex_data.x = pos.x;
diff --git a/source/blender/draw/intern/shaders/common_subdiv_vbo_lnor_comp.glsl b/source/blender/draw/intern/shaders/common_subdiv_vbo_lnor_comp.glsl
index b7e04e240fb..b7bcfd2d369 100644
--- a/source/blender/draw/intern/shaders/common_subdiv_vbo_lnor_comp.glsl
+++ b/source/blender/draw/intern/shaders/common_subdiv_vbo_lnor_comp.glsl
@@ -13,9 +13,14 @@ layout(std430, binding = 2) readonly buffer extraCoarseFaceData
 
 layout(std430, binding = 3) writeonly buffer outputLoopNormals
 {
-  vec3 output_lnor[];
+  LoopNormal output_lnor[];
 };
 
+bool is_face_selected(uint coarse_quad_index)
+{
+  return (extra_coarse_face_data[coarse_quad_index] & coarse_face_select_mask) != 0;
+}
+
 void main()
 {
   /* We execute for each quad. */
@@ -34,7 +39,7 @@ void main()
     /* Face is smooth, use vertex normals. */
     for (int i = 0; i < 4; i++) {
       PosNorLoop pos_nor_loop = pos_nor[start_loop_index + i];
-      output_lnor[start_loop_index + i] = get_vertex_nor(pos_nor_loop);
+      output_lnor[start_loop_index + i] = get_normal_and_flag(pos_nor_loop);
     }
   }
   else {
@@ -50,8 +55,19 @@ void main()
     add_newell_cross_v3_v3v3(face_normal, v3, v0);
 
     face_normal = normalize(face_normal);
+
+    LoopNormal loop_normal;
+    loop_normal.nx = face_normal.x;
+    loop_normal.ny = face_normal.y;
+    loop_normal.nz = face_normal.z;
+    loop_normal.flag = 0.0;
+
+    if (is_face_selected(coarse_quad_index)) {
+      loop_normal.flag = 1.0;
+    }
+
     for (int i = 0; i < 4; i++) {
-      output_lnor[start_loop_index + i] = face_normal;
+      output_lnor[start_loop_index + i] = loop_normal;
     }
   }
 }