gtk4: Premultiply alpha in GL textures

GTK expects GL textures to have premultiplied alpha. The ones we get
from GStreamer don't, leading to incorrect rendering of semitransparent
frames.

GTK 4.12 gained an API to set a different GL texture format, but it
won't help for older GTK versions. Plus, at the time of writing, it
causes a very slow download/upload path in GTK.

So, use a GTK GL shader node to premultiply the alpha without leaving
the GPU.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/1312>

3 files changed, 54 insertions, 7 deletions

diff --git a/video/gtk4/src/sink/frame.rs b/video/gtk4/src/sink/frame.rs
index 4a53e1a07..d5223a613 100644
--- a/video/gtk4/src/sink/frame.rs
+++ b/video/gtk4/src/sink/frame.rs
@@ -42,6 +42,7 @@ pub(crate) struct Texture {
     pub width: f32,
     pub height: f32,
     pub global_alpha: f32,
+    pub has_alpha: bool,
 }
 
 struct FrameWrapper(gst_video::VideoFrame<gst_video::video_frame::Readable>);
@@ -149,6 +150,7 @@ impl Frame {
 
         let width = self.frame.width();
         let height = self.frame.height();
+        let has_alpha = self.frame.format_info().has_alpha();
         let (texture, pixel_aspect_ratio) = {
             #[cfg(not(any(target_os = "macos", target_os = "windows", feature = "gst_gl")))]
             {
@@ -183,9 +185,11 @@ impl Frame {
             width: width as f32 * pixel_aspect_ratio as f32,
             height: height as f32,
             global_alpha: 1.0,
+            has_alpha,
         });
 
         for overlay in self.overlays {
+            let has_alpha = overlay.frame.format_info().has_alpha();
             let (texture, _pixel_aspect_ratio) =
                 video_frame_to_memory_texture(overlay.frame, cached_textures, &mut used_textures);
 
@@ -196,6 +200,7 @@ impl Frame {
                 width: overlay.width as f32,
                 height: overlay.height as f32,
                 global_alpha: overlay.global_alpha,
+                has_alpha,
             });
         }
 
diff --git a/video/gtk4/src/sink/paintable/imp.rs b/video/gtk4/src/sink/paintable/imp.rs
index 6763ce5a4..dedfcdf18 100644
--- a/video/gtk4/src/sink/paintable/imp.rs
+++ b/video/gtk4/src/sink/paintable/imp.rs
@@ -11,7 +11,7 @@
 
 use gtk::prelude::*;
 use gtk::subclass::prelude::*;
-use gtk::{gdk, glib, graphene};
+use gtk::{gdk, glib, graphene, gsk};
 
 use crate::sink::frame::{Frame, Texture};
 
@@ -28,11 +28,25 @@ static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
     )
 });
 
-#[derive(Default, Debug)]
+#[derive(Debug)]
 pub struct Paintable {
     paintables: RefCell<Vec<Texture>>,
     cached_textures: RefCell<HashMap<usize, gdk::Texture>>,
     gl_context: RefCell<Option<gdk::GLContext>>,
+    premult_shader: gsk::GLShader,
+}
+
+impl Default for Paintable {
+    fn default() -> Self {
+        Self {
+            paintables: Default::default(),
+            cached_textures: Default::default(),
+            gl_context: Default::default(),
+            premult_shader: gsk::GLShader::from_bytes(&glib::Bytes::from_static(include_bytes!(
+                "premult.glsl"
+            ))),
+        }
+    }
 }
 
 #[glib::object_subclass]
@@ -145,14 +159,31 @@ impl PaintableImpl for Paintable {
                 width: paintable_width,
                 height: paintable_height,
                 global_alpha,
+                has_alpha,
             } in &*paintables
             {
                 snapshot.push_opacity(*global_alpha as f64);
-                snapshot.append_texture(
-                    texture,
-                    &graphene::Rect::new(*x, *y, *paintable_width, *paintable_height),
-                );
-                snapshot.pop();
+
+                let bounds = graphene::Rect::new(*x, *y, *paintable_width, *paintable_height);
+
+                // Only premultiply GL textures that expect to be in premultiplied RGBA format.
+                let do_premult = texture.is::<gdk::GLTexture>() && *has_alpha;
+                if do_premult {
+                    snapshot.push_gl_shader(
+                        &self.premult_shader,
+                        &bounds,
+                        gsk::ShaderArgsBuilder::new(&self.premult_shader, None).to_args(),
+                    );
+                }
+
+                snapshot.append_texture(texture, &bounds);
+
+                if do_premult {
+                    snapshot.gl_shader_pop_texture(); // pop texture appended above from the shader
+                    snapshot.pop(); // pop shader
+                }
+
+                snapshot.pop(); // pop opacity
             }
         } else {
             gst::trace!(CAT, imp: self, "Snapshotting black frame");
diff --git a/video/gtk4/src/sink/paintable/premult.glsl b/video/gtk4/src/sink/paintable/premult.glsl
new file mode 100644
index 000000000..f2ab7ec55
--- /dev/null
+++ b/video/gtk4/src/sink/paintable/premult.glsl
@@ -0,0 +1,11 @@
+uniform sampler2D u_texture1;
+
+void mainImage(
+    out vec4 fragColor,
+    in vec2 fragCoord,
+    in vec2 resolution,
+    in vec2 uv
+) {
+  fragColor = GskTexture(u_texture1, uv);
+  fragColor.rgb = fragColor.rgb * fragColor.a;
+}