Fix: Viewer node overlay alpha broken on NVidia GPUs

On NVidia GPUs, when the blue channel was between 0 and 0.1,
the overlay's alpha would increase, making it invisible.

With the `overlay_line_fb` frame buffer bound in `overlay_engine.cc`
there are two outputs to write to: the color and the line output, which
is used for making smooth antialiased lines. The overlay is in its
current position in the order of overlays so that it draws on top of
curve wire lines. Not writing to that second output is undefined
behavior, so fix it by writing zeros. In the future, the overlay
could be smoothed on curves using that second texture.

Thanks to Clément for the explanation of the issue!

2 files changed, 6 insertions, 0 deletions

diff --git a/source/blender/draw/engines/overlay/shaders/infos/overlay_viewer_attribute_info.hh b/source/blender/draw/engines/overlay/shaders/infos/overlay_viewer_attribute_info.hh
index aca80992622..20d4797b76f 100644
--- a/source/blender/draw/engines/overlay/shaders/infos/overlay_viewer_attribute_info.hh
+++ b/source/blender/draw/engines/overlay/shaders/infos/overlay_viewer_attribute_info.hh
@@ -11,6 +11,7 @@ GPU_SHADER_CREATE_INFO(overlay_viewer_attribute_mesh)
     .vertex_source("overlay_viewer_attribute_mesh_vert.glsl")
     .fragment_source("overlay_viewer_attribute_frag.glsl")
     .fragment_out(0, Type::VEC4, "out_color")
+    .fragment_out(1, Type::VEC4, "lineOutput")
     .vertex_in(0, Type::VEC3, "pos")
     .vertex_in(1, Type::VEC4, "attribute_value")
     .vertex_out(overlay_viewer_attribute_iface)
@@ -25,6 +26,7 @@ GPU_SHADER_CREATE_INFO(overlay_viewer_attribute_pointcloud)
     .vertex_source("overlay_viewer_attribute_pointcloud_vert.glsl")
     .fragment_source("overlay_viewer_attribute_frag.glsl")
     .fragment_out(0, Type::VEC4, "out_color")
+    .fragment_out(1, Type::VEC4, "lineOutput")
     .vertex_in(3, Type::VEC4, "attribute_value")
     .vertex_out(overlay_viewer_attribute_iface)
     .additional_info("overlay_viewer_attribute_common", "draw_pointcloud");
@@ -38,6 +40,7 @@ GPU_SHADER_CREATE_INFO(overlay_viewer_attribute_curve)
     .vertex_source("overlay_viewer_attribute_curve_vert.glsl")
     .fragment_source("overlay_viewer_attribute_frag.glsl")
     .fragment_out(0, Type::VEC4, "out_color")
+    .fragment_out(1, Type::VEC4, "lineOutput")
     .vertex_in(0, Type::VEC3, "pos")
     .vertex_in(1, Type::VEC4, "attribute_value")
     .vertex_out(overlay_viewer_attribute_iface)
@@ -52,6 +55,7 @@ GPU_SHADER_CREATE_INFO(overlay_viewer_attribute_curves)
     .vertex_source("overlay_viewer_attribute_curves_vert.glsl")
     .fragment_source("overlay_viewer_attribute_frag.glsl")
     .fragment_out(0, Type::VEC4, "out_color")
+    .fragment_out(1, Type::VEC4, "lineOutput")
     .sampler(0, ImageType::FLOAT_BUFFER, "color_tx")
     .push_constant(Type::BOOL, "is_point_domain")
     .vertex_out(overlay_viewer_attribute_iface)
diff --git a/source/blender/draw/engines/overlay/shaders/overlay_viewer_attribute_frag.glsl b/source/blender/draw/engines/overlay/shaders/overlay_viewer_attribute_frag.glsl
index 6176a6b96ba..e5752ada940 100644
--- a/source/blender/draw/engines/overlay/shaders/overlay_viewer_attribute_frag.glsl
+++ b/source/blender/draw/engines/overlay/shaders/overlay_viewer_attribute_frag.glsl
@@ -3,4 +3,6 @@ void main()
 {
   out_color = finalColor;
   out_color.a *= opacity;
+  /* Writing to this second texture is necessary to avoid undefined behavior. */
+  lineOutput = vec4(0.0);
 }