diff options
author | Sebastian Dröge <sebastian@centricular.com> | 2021-10-17 20:23:43 +0300 |
---|---|---|
committer | Sebastian Dröge <slomo@coaxion.net> | 2021-10-17 22:03:24 +0300 |
commit | 70f0aa975801dbb1c5a0f11944f136b45925ecde (patch) | |
tree | efc278969b3e63af5dc7512936bcbc9d4a9af4de /video | |
parent | 54c8f5b3aba19c6ac9c97bbd3424b173c6b3c628 (diff) |
gtk4: Add support for rendering overlay composition rectangles directly via GTK
Diffstat (limited to 'video')
-rw-r--r-- | video/gtk4/Cargo.toml | 6 | ||||
-rw-r--r-- | video/gtk4/examples/gtksink.rs | 15 | ||||
-rw-r--r-- | video/gtk4/src/sink/frame.rs | 192 | ||||
-rw-r--r-- | video/gtk4/src/sink/imp.rs | 46 | ||||
-rw-r--r-- | video/gtk4/src/sink/paintable/imp.rs | 124 | ||||
-rw-r--r-- | video/gtk4/src/sink/paintable/mod.rs | 28 |
6 files changed, 290 insertions, 121 deletions
diff --git a/video/gtk4/Cargo.toml b/video/gtk4/Cargo.toml index f9a03f42..9d1dbad3 100644 --- a/video/gtk4/Cargo.toml +++ b/video/gtk4/Cargo.toml @@ -10,9 +10,9 @@ description = "GTK 4 Sink element and Paintable widget" [dependencies] gtk = { package = "gtk4", git = "https://github.com/gtk-rs/gtk4-rs" } -gst_video = {package="gstreamer-video", git="https://gitlab.freedesktop.org/gstreamer/gstreamer-rs"} -gst_base = {package="gstreamer-base", git="https://gitlab.freedesktop.org/gstreamer/gstreamer-rs"} -gst = {package="gstreamer", git="https://gitlab.freedesktop.org/gstreamer/gstreamer-rs"} +gst = { package = "gstreamer", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", features = ["v1_16"] } +gst_base = { package = "gstreamer-base", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } +gst_video = { package = "gstreamer-video", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } once_cell = "1.0" fragile = "1.0.0" diff --git a/video/gtk4/examples/gtksink.rs b/video/gtk4/examples/gtksink.rs index 0016998a..82985b9c 100644 --- a/video/gtk4/examples/gtksink.rs +++ b/video/gtk4/examples/gtksink.rs @@ -9,6 +9,9 @@ fn create_ui(app: >k::Application) { let pipeline = gst::Pipeline::new(None); let src = gst::ElementFactory::make("videotestsrc", None).unwrap(); + let overlay = gst::ElementFactory::make("clockoverlay", None).unwrap(); + overlay.set_property("font-desc", "Monospace 42").unwrap(); + let sink = gst::ElementFactory::make("gtk4paintablesink", None).unwrap(); let paintable = sink .property("paintable") @@ -16,8 +19,16 @@ fn create_ui(app: >k::Application) { .get::<gdk::Paintable>() .unwrap(); - pipeline.add_many(&[&src, &sink]).unwrap(); - src.link(&sink).unwrap(); + pipeline.add_many(&[&src, &overlay, &sink]).unwrap(); + src.link_filtered( + &overlay, + &gst::Caps::builder("video/x-raw") + .field("width", 640) + .field("height", 480) + .build(), + ) + .unwrap(); + overlay.link(&sink).unwrap(); let window = gtk::ApplicationWindow::new(app); window.set_default_size(640, 480); diff --git a/video/gtk4/src/sink/frame.rs b/video/gtk4/src/sink/frame.rs index 777fc6f9..5fe11b18 100644 --- a/video/gtk4/src/sink/frame.rs +++ b/video/gtk4/src/sink/frame.rs @@ -11,77 +11,167 @@ use gtk::prelude::*; use gtk::{gdk, glib}; +use std::collections::{HashMap, HashSet}; use std::convert::AsRef; #[derive(Debug)] -pub struct Frame(pub gst_video::VideoFrame<gst_video::video_frame::Readable>); +pub struct Frame { + pub frame: gst_video::VideoFrame<gst_video::video_frame::Readable>, + pub overlays: Vec<Overlay>, +} #[derive(Debug)] -pub struct Paintable { - pub paintable: gdk::Paintable, - pub pixel_aspect_ratio: f64, +pub struct Overlay { + pub frame: gst_video::VideoFrame<gst_video::video_frame::Readable>, + pub x: i32, + pub y: i32, + pub width: u32, + pub height: u32, + pub global_alpha: f32, } -impl Paintable { - pub fn width(&self) -> i32 { - f64::round(self.paintable.intrinsic_width() as f64 * self.pixel_aspect_ratio) as i32 - } - - pub fn height(&self) -> i32 { - self.paintable.intrinsic_height() - } +#[derive(Debug)] +pub struct Texture { + pub texture: gdk::Texture, + pub x: f32, + pub y: f32, + pub width: f32, + pub height: f32, + pub global_alpha: f32, } -impl AsRef<[u8]> for Frame { +struct FrameWrapper(gst_video::VideoFrame<gst_video::video_frame::Readable>); +impl AsRef<[u8]> for FrameWrapper { fn as_ref(&self) -> &[u8] { self.0.plane_data(0).unwrap() } } -impl From<Frame> for Paintable { - fn from(f: Frame) -> Paintable { - let format = match f.0.format() { - gst_video::VideoFormat::Bgra => gdk::MemoryFormat::B8g8r8a8, - gst_video::VideoFormat::Argb => gdk::MemoryFormat::A8r8g8b8, - gst_video::VideoFormat::Rgba => gdk::MemoryFormat::R8g8b8a8, - gst_video::VideoFormat::Abgr => gdk::MemoryFormat::A8b8g8r8, - gst_video::VideoFormat::Rgb => gdk::MemoryFormat::R8g8b8, - gst_video::VideoFormat::Bgr => gdk::MemoryFormat::B8g8r8, - _ => unreachable!(), - }; - let width = f.0.width() as i32; - let height = f.0.height() as i32; - let rowstride = f.0.plane_stride()[0] as usize; - - let pixel_aspect_ratio = - (*f.0.info().par().numer() as f64) / (*f.0.info().par().denom() as f64); - - Paintable { - paintable: gdk::MemoryTexture::new( - width, - height, - format, - &glib::Bytes::from_owned(f), - rowstride, - ) - .upcast(), - pixel_aspect_ratio, - } +fn video_frame_to_memory_texture( + frame: gst_video::VideoFrame<gst_video::video_frame::Readable>, + cached_textures: &mut HashMap<usize, gdk::Texture>, + used_textures: &mut HashSet<usize>, +) -> (gdk::Texture, f64) { + let texture_id = frame.plane_data(0).unwrap().as_ptr() as usize; + + let pixel_aspect_ratio = + (*frame.info().par().numer() as f64) / (*frame.info().par().denom() as f64); + + if let Some(texture) = cached_textures.get(&texture_id) { + used_textures.insert(texture_id); + return (texture.clone(), pixel_aspect_ratio); } + + let format = match frame.format() { + gst_video::VideoFormat::Bgra => gdk::MemoryFormat::B8g8r8a8, + gst_video::VideoFormat::Argb => gdk::MemoryFormat::A8r8g8b8, + gst_video::VideoFormat::Rgba => gdk::MemoryFormat::R8g8b8a8, + gst_video::VideoFormat::Abgr => gdk::MemoryFormat::A8b8g8r8, + gst_video::VideoFormat::Rgb => gdk::MemoryFormat::R8g8b8, + gst_video::VideoFormat::Bgr => gdk::MemoryFormat::B8g8r8, + _ => unreachable!(), + }; + let width = frame.width(); + let height = frame.height(); + let rowstride = frame.plane_stride()[0] as usize; + + let texture = gdk::MemoryTexture::new( + width as i32, + height as i32, + format, + &glib::Bytes::from_owned(FrameWrapper(frame)), + rowstride, + ) + .upcast::<gdk::Texture>(); + + cached_textures.insert(texture_id, texture.clone()); + used_textures.insert(texture_id); + + (texture, pixel_aspect_ratio) } impl Frame { - pub fn new(buffer: &gst::Buffer, info: &gst_video::VideoInfo) -> Self { - let video_frame = - gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info).unwrap(); - Self(video_frame) - } + pub fn into_textures(self, cached_textures: &mut HashMap<usize, gdk::Texture>) -> Vec<Texture> { + let mut textures = Vec::with_capacity(1 + self.overlays.len()); + let mut used_textures = HashSet::with_capacity(1 + self.overlays.len()); - pub fn width(&self) -> u32 { - self.0.width() + let width = self.frame.width(); + let height = self.frame.height(); + let (texture, pixel_aspect_ratio) = + video_frame_to_memory_texture(self.frame, cached_textures, &mut used_textures); + + textures.push(Texture { + texture, + x: 0.0, + y: 0.0, + width: width as f32 * pixel_aspect_ratio as f32, + height: height as f32, + global_alpha: 1.0, + }); + + for overlay in self.overlays { + let (texture, _pixel_aspect_ratio) = + video_frame_to_memory_texture(overlay.frame, cached_textures, &mut used_textures); + + textures.push(Texture { + texture, + x: overlay.x as f32, + y: overlay.y as f32, + width: overlay.width as f32, + height: overlay.height as f32, + global_alpha: overlay.global_alpha, + }); + } + + // Remove textures that were not used this time + cached_textures.retain(|id, _| used_textures.contains(id)); + + textures } +} + +impl Frame { + pub fn new(buffer: &gst::Buffer, info: &gst_video::VideoInfo) -> Result<Self, gst::FlowError> { + let frame = gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info) + .map_err(|_| gst::FlowError::Error)?; + + let overlays = frame + .buffer() + .iter_meta::<gst_video::VideoOverlayCompositionMeta>() + .map(|meta| { + meta.overlay() + .iter() + .filter_map(|rect| { + let buffer = rect + .pixels_unscaled_argb(gst_video::VideoOverlayFormatFlags::GLOBAL_ALPHA); + let (x, y, width, height) = rect.render_rectangle(); + let global_alpha = rect.global_alpha(); + + let vmeta = buffer.meta::<gst_video::VideoMeta>().unwrap(); + let info = gst_video::VideoInfo::builder( + vmeta.format(), + vmeta.width(), + vmeta.height(), + ) + .build() + .unwrap(); + let frame = + gst_video::VideoFrame::from_buffer_readable(buffer, &info).ok()?; + + Some(Overlay { + frame, + x, + y, + width, + height, + global_alpha, + }) + }) + .collect::<Vec<_>>() + }) + .flatten() + .collect(); - pub fn height(&self) -> u32 { - self.0.height() + Ok(Self { frame, overlays }) } } diff --git a/video/gtk4/src/sink/imp.rs b/video/gtk4/src/sink/imp.rs index 944a4fe8..2fd71500 100644 --- a/video/gtk4/src/sink/imp.rs +++ b/video/gtk4/src/sink/imp.rs @@ -132,16 +132,34 @@ impl ElementImpl for PaintableSink { fn pad_templates() -> &'static [gst::PadTemplate] { static PAD_TEMPLATES: Lazy<Vec<gst::PadTemplate>> = Lazy::new(|| { // Those are the supported formats by a gdk::Texture - let caps = gst_video::video_make_raw_caps(&[ - gst_video::VideoFormat::Bgra, - gst_video::VideoFormat::Argb, - gst_video::VideoFormat::Rgba, - gst_video::VideoFormat::Abgr, - gst_video::VideoFormat::Rgb, - gst_video::VideoFormat::Bgr, - ]) - .any_features() - .build(); + let mut caps = gst::Caps::new_empty(); + { + let caps = caps.get_mut().unwrap(); + + for features in [ + None, + Some(&["memory:SystemMemory", "meta:GstVideoOverlayComposition"][..]), + Some(&["meta:GstVideoOverlayComposition"][..]), + ] { + let mut c = gst_video::video_make_raw_caps(&[ + gst_video::VideoFormat::Bgra, + gst_video::VideoFormat::Argb, + gst_video::VideoFormat::Rgba, + gst_video::VideoFormat::Abgr, + gst_video::VideoFormat::Rgb, + gst_video::VideoFormat::Bgr, + ]) + .build(); + + if let Some(features) = features { + c.get_mut() + .unwrap() + .set_features_simple(Some(gst::CapsFeatures::new(features))); + } + + caps.append(c); + } + } vec![gst::PadTemplate::new( "sink", @@ -209,6 +227,9 @@ impl BaseSinkImpl for PaintableSink { ) -> Result<(), gst::ErrorMessage> { query.add_allocation_meta::<gst_video::VideoMeta>(None); + // TODO: Provide a preferred "window size" here for higher-resolution rendering + query.add_allocation_meta::<gst_video::VideoOverlayCompositionMeta>(None); + self.parent_propose_allocation(element, query) } } @@ -227,7 +248,10 @@ impl VideoSinkImpl for PaintableSink { gst::FlowError::NotNegotiated })?; - let frame = Frame::new(buffer, info); + let frame = Frame::new(buffer, info).map_err(|err| { + gst_error!(CAT, obj: element, "Failed to map video frame"); + err + })?; self.pending_frame.lock().unwrap().replace(frame); let sender = self.sender.lock().unwrap(); diff --git a/video/gtk4/src/sink/paintable/imp.rs b/video/gtk4/src/sink/paintable/imp.rs index d226f051..a0ba33e1 100644 --- a/video/gtk4/src/sink/paintable/imp.rs +++ b/video/gtk4/src/sink/paintable/imp.rs @@ -13,11 +13,12 @@ use gtk::prelude::*; use gtk::subclass::prelude::*; use gtk::{gdk, glib, graphene}; -use gst::gst_trace; +use gst::{gst_debug, gst_trace}; -use crate::sink::frame::Paintable; +use crate::sink::frame::{Frame, Texture}; use std::cell::RefCell; +use std::collections::HashMap; use once_cell::sync::Lazy; @@ -31,7 +32,8 @@ pub(super) static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| { #[derive(Default)] pub struct SinkPaintable { - pub paintable: RefCell<Option<Paintable>>, + paintables: RefCell<Vec<Texture>>, + cached_textures: RefCell<HashMap<usize, gdk::Texture>>, } #[glib::object_subclass] @@ -46,52 +48,86 @@ impl ObjectImpl for SinkPaintable {} impl PaintableImpl for SinkPaintable { fn intrinsic_height(&self, _paintable: &Self::Type) -> i32 { - if let Some(Paintable { ref paintable, .. }) = *self.paintable.borrow() { - paintable.intrinsic_height() + if let Some(paintable) = self.paintables.borrow().first() { + f32::round(paintable.height) as i32 } else { 0 } } fn intrinsic_width(&self, _paintable: &Self::Type) -> i32 { - if let Some(Paintable { - ref paintable, - pixel_aspect_ratio, - }) = *self.paintable.borrow() - { - f64::round(paintable.intrinsic_width() as f64 * pixel_aspect_ratio) as i32 + if let Some(paintable) = self.paintables.borrow().first() { + f32::round(paintable.width) as i32 } else { 0 } } fn intrinsic_aspect_ratio(&self, _paintable: &Self::Type) -> f64 { - if let Some(Paintable { - ref paintable, - pixel_aspect_ratio, - }) = *self.paintable.borrow() - { - paintable.intrinsic_aspect_ratio() * pixel_aspect_ratio + if let Some(paintable) = self.paintables.borrow().first() { + paintable.width as f64 / paintable.height as f64 } else { 0.0 } } - fn current_image(&self, _paintable: &Self::Type) -> gdk::Paintable { - if let Some(Paintable { ref paintable, .. }) = *self.paintable.borrow() { - paintable.clone() - } else { - gdk::Paintable::new_empty(0, 0).expect("Couldn't create empty paintable") - } - } - fn snapshot(&self, paintable: &Self::Type, snapshot: &gdk::Snapshot, width: f64, height: f64) { - if let Some(Paintable { ref paintable, .. }) = *self.paintable.borrow() { + let snapshot = snapshot.downcast_ref::<gtk::Snapshot>().unwrap(); + + let paintables = self.paintables.borrow(); + + if !paintables.is_empty() { gst_trace!(CAT, obj: paintable, "Snapshotting frame"); - paintable.snapshot(snapshot, width, height); + + let (frame_width, frame_height) = + paintables.first().map(|p| (p.width, p.height)).unwrap(); + + let mut scale_x = width / frame_width as f64; + let mut scale_y = height / frame_height as f64; + let mut trans_x = 0.0; + let mut trans_y = 0.0; + + // TODO: Property for keeping aspect ratio or not + if (scale_x - scale_y).abs() > f64::EPSILON { + if scale_x > scale_y { + trans_x = + ((frame_width as f64 * scale_x) - (frame_width as f64 * scale_y)) / 2.0; + scale_x = scale_y; + } else { + trans_y = + ((frame_height as f64 * scale_y) - (frame_height as f64 * scale_x)) / 2.0; + scale_y = scale_x; + } + } + + if trans_x != 0.0 || trans_y != 0.0 { + snapshot.append_color( + &gdk::RGBA::BLACK, + &graphene::Rect::new(0f32, 0f32, width as f32, height as f32), + ); + } + + snapshot.translate(&graphene::Point::new(trans_x as f32, trans_y as f32)); + snapshot.scale(scale_x as f32, scale_y as f32); + + for Texture { + texture, + x, + y, + width: paintable_width, + height: paintable_height, + global_alpha, + } in &*paintables + { + snapshot.push_opacity(*global_alpha as f64); + snapshot.append_texture( + texture, + &graphene::Rect::new(*x, *y, *paintable_width, *paintable_height), + ); + snapshot.pop(); + } } else { gst_trace!(CAT, obj: paintable, "Snapshotting black frame"); - let snapshot = snapshot.downcast_ref::<gtk::Snapshot>().unwrap(); snapshot.append_color( &gdk::RGBA::BLACK, &graphene::Rect::new(0f32, 0f32, width as f32, height as f32), @@ -99,3 +135,35 @@ impl PaintableImpl for SinkPaintable { } } } + +impl SinkPaintable { + pub(super) fn handle_frame_changed(&self, obj: &super::SinkPaintable, frame: Option<Frame>) { + if let Some(frame) = frame { + gst_trace!(CAT, obj: obj, "Received new frame"); + + let new_paintables = frame.into_textures(&mut *self.cached_textures.borrow_mut()); + let new_size = new_paintables + .first() + .map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32)) + .unwrap(); + + let old_paintables = self.paintables.replace(new_paintables); + let old_size = old_paintables + .first() + .map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32)); + + if Some(new_size) != old_size { + gst_debug!( + CAT, + obj: obj, + "Size changed from {:?} to {:?}", + old_size, + new_size, + ); + obj.invalidate_size(); + } + + obj.invalidate_contents(); + } + } +} diff --git a/video/gtk4/src/sink/paintable/mod.rs b/video/gtk4/src/sink/paintable/mod.rs index 19914f4a..8f15eb92 100644 --- a/video/gtk4/src/sink/paintable/mod.rs +++ b/video/gtk4/src/sink/paintable/mod.rs @@ -9,14 +9,11 @@ // // SPDX-License-Identifier: MPL-2.0 -use crate::sink::frame::{Frame, Paintable}; +use crate::sink::frame::Frame; -use gtk::prelude::*; use gtk::subclass::prelude::*; use gtk::{gdk, glib}; -use gst::{gst_debug, gst_trace}; - mod imp; glib::wrapper! { @@ -39,27 +36,6 @@ impl Default for SinkPaintable { impl SinkPaintable { pub(crate) fn handle_frame_changed(&self, frame: Option<Frame>) { let self_ = imp::SinkPaintable::from_instance(self); - if let Some(frame) = frame { - gst_trace!(imp::CAT, obj: self, "Received new frame"); - - let paintable: Paintable = frame.into(); - let new_size = (paintable.width(), paintable.height()); - - let old_paintable = self_.paintable.replace(Some(paintable)); - let old_size = old_paintable.map(|p| (p.width(), p.height())); - - if Some(new_size) != old_size { - gst_debug!( - imp::CAT, - obj: self, - "Size changed from {:?} to {:?}", - old_size, - new_size, - ); - self.invalidate_size(); - } - - self.invalidate_contents(); - } + self_.handle_frame_changed(self, frame); } } |