closedcaption: implement a tttojson element

That new element can for example be used to serialize the output
of awstranscribe, ready for further editing

5 files changed, 372 insertions, 53 deletions

diff --git a/video/closedcaption/src/lib.rs b/video/closedcaption/src/lib.rs
index d813254ca..84fa5b7ac 100644
--- a/video/closedcaption/src/lib.rs
+++ b/video/closedcaption/src/lib.rs
@@ -17,20 +17,6 @@
 
 #![recursion_limit = "128"]
 
-use serde::{Deserialize, Serialize};
-
-#[derive(
-    Serialize, Deserialize, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Clone, Copy, glib::GEnum,
-)]
-#[repr(u32)]
-#[genum(type_name = "GstTtToCea608Mode")]
-enum Cea608Mode {
-    PopOn,
-    RollUp2,
-    RollUp3,
-    RollUp4,
-}
-
 #[allow(non_camel_case_types, non_upper_case_globals, unused)]
 #[allow(clippy::redundant_static_lifetimes, clippy::unreadable_literal)]
 #[allow(clippy::useless_transmute, clippy::trivially_copy_pass_by_ref)]
@@ -47,6 +33,8 @@ mod parser_utils;
 mod scc_enc;
 mod scc_parse;
 mod tttocea608;
+mod tttojson;
+mod ttutils;
 
 fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
     mcc_parse::register(plugin)?;
@@ -57,6 +45,7 @@ fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
     tttocea608::register(plugin)?;
     cea608overlay::register(plugin)?;
     ccdetect::register(plugin)?;
+    tttojson::register(plugin)?;
     Ok(())
 }
 
diff --git a/video/closedcaption/src/tttocea608/imp.rs b/video/closedcaption/src/tttocea608/imp.rs
index 7eef979cc..562a83d73 100644
--- a/video/closedcaption/src/tttocea608/imp.rs
+++ b/video/closedcaption/src/tttocea608/imp.rs
@@ -27,45 +27,7 @@ use once_cell::sync::Lazy;
 use crate::ffi;
 use std::sync::Mutex;
 
-use crate::Cea608Mode;
-
-use serde::Deserialize;
-
-#[derive(Clone, Copy, Deserialize, Debug, PartialEq)]
-enum TextStyle {
-    White,
-    Green,
-    Blue,
-    Cyan,
-    Red,
-    Yellow,
-    Magenta,
-    ItalicWhite,
-}
-
-#[derive(Deserialize, Debug)]
-struct Chunk {
-    style: TextStyle,
-    underline: bool,
-    text: String,
-}
-
-#[derive(Deserialize, Debug)]
-struct Line {
-    column: Option<u32>,
-    row: Option<u32>,
-    chunks: Vec<Chunk>,
-    /* In roll-up modes, new lines don't introduce a carriage return by
-     * default, but that can be overridden */
-    carriage_return: Option<bool>,
-}
-
-#[derive(Deserialize, Debug)]
-struct Lines {
-    lines: Vec<Line>,
-    mode: Option<Cea608Mode>,
-    clear: Option<bool>,
-}
+use crate::ttutils::{Cea608Mode, Chunk, Line, Lines, TextStyle};
 
 fn is_basicna(cc_data: u16) -> bool {
     0x0000 != (0x6000 & cc_data)
diff --git a/video/closedcaption/src/tttojson/imp.rs b/video/closedcaption/src/tttojson/imp.rs
new file mode 100644
index 000000000..c31664cd1
--- /dev/null
+++ b/video/closedcaption/src/tttojson/imp.rs
@@ -0,0 +1,266 @@
+// Copyright (C) 2020 Mathieu Duponchelle <mathieu@centricular.com>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
+// Boston, MA 02110-1335, USA.
+
+use glib::subclass;
+use glib::subclass::prelude::*;
+use gst::gst_log;
+use gst::prelude::*;
+use gst::subclass::prelude::*;
+
+use once_cell::sync::Lazy;
+
+use std::sync::Mutex;
+
+use crate::ttutils::{Cea608Mode, Chunk, Line, Lines, TextStyle};
+
+static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
+    gst::DebugCategory::new(
+        "tttojson",
+        gst::DebugColorFlags::empty(),
+        Some("Timed Text to JSON"),
+    )
+});
+
+const DEFAULT_MODE: Cea608Mode = Cea608Mode::RollUp2;
+
+static PROPERTIES: [subclass::Property; 1] = [subclass::Property("mode", |name| {
+    glib::ParamSpec::enum_(
+        name,
+        "Mode",
+        "Which mode to operate in",
+        Cea608Mode::static_type(),
+        DEFAULT_MODE as i32,
+        glib::ParamFlags::READWRITE,
+    )
+})];
+
+#[derive(Debug, Clone)]
+struct Settings {
+    mode: Cea608Mode,
+}
+
+impl Default for Settings {
+    fn default() -> Self {
+        Settings { mode: DEFAULT_MODE }
+    }
+}
+
+pub struct TtToJson {
+    srcpad: gst::Pad,
+    sinkpad: gst::Pad,
+    settings: Mutex<Settings>,
+}
+
+impl TtToJson {
+    fn sink_chain(
+        &self,
+        _pad: &gst::Pad,
+        element: &super::TtToJson,
+        buffer: gst::Buffer,
+    ) -> Result<gst::FlowSuccess, gst::FlowError> {
+        let pts = buffer.get_pts();
+        let duration = buffer.get_duration();
+
+        let buffer = buffer.into_mapped_buffer_readable().map_err(|_| {
+            gst::element_error!(
+                element,
+                gst::ResourceError::Read,
+                ["Failed to map buffer readable"]
+            );
+
+            gst::FlowError::Error
+        })?;
+
+        let text = std::str::from_utf8(buffer.as_slice()).map_err(|err| {
+            gst::element_error!(
+                element,
+                gst::ResourceError::Read,
+                ["Failed to map decode as utf8: {}", err]
+            );
+
+            gst::FlowError::Error
+        })?;
+
+        let mode = self.settings.lock().unwrap().mode;
+
+        let mut lines = Lines {
+            lines: Vec::new(),
+            mode: Some(mode),
+            clear: Some(false),
+        };
+
+        for phrase in text.lines() {
+            lines.lines.push(Line {
+                carriage_return: Some(true),
+                column: Some(0),
+                row: Some(13),
+                chunks: vec![Chunk {
+                    // Default CEA 608 styling
+                    style: TextStyle::White,
+                    underline: false,
+                    text: phrase.to_string(),
+                }],
+            });
+        }
+
+        let json = serde_json::to_string(&lines).map_err(|err| {
+            gst::element_error!(
+                element,
+                gst::ResourceError::Write,
+                ["Failed to serialize as json {}", err]
+            );
+
+            gst::FlowError::Error
+        })?;
+
+        let mut buf = gst::Buffer::from_mut_slice(json.into_bytes());
+        {
+            let buf_mut = buf.get_mut().unwrap();
+            buf_mut.set_pts(pts);
+            buf_mut.set_duration(duration);
+        }
+
+        self.srcpad.push(buf)
+    }
+
+    fn sink_event(&self, pad: &gst::Pad, element: &super::TtToJson, event: gst::Event) -> bool {
+        use gst::EventView;
+
+        gst_log!(CAT, obj: pad, "Handling event {:?}", event);
+
+        match event.view() {
+            EventView::Caps(_) => {
+                // We send our own caps downstream
+                let caps = gst::Caps::builder("application/x-json")
+                    .field("format", &"cea608")
+                    .build();
+                self.srcpad.push_event(gst::event::Caps::new(&caps))
+            }
+            EventView::Eos(_) => pad.event_default(Some(element), event),
+            _ => pad.event_default(Some(element), event),
+        }
+    }
+}
+
+impl ElementImpl for TtToJson {}
+
+impl ObjectSubclass for TtToJson {
+    const NAME: &'static str = "RsTtToJson";
+    type Type = super::TtToJson;
+    type ParentType = gst::Element;
+    type Instance = gst::subclass::ElementInstanceStruct<Self>;
+    type Class = subclass::simple::ClassStruct<Self>;
+
+    glib::object_subclass!();
+
+    fn with_class(klass: &Self::Class) -> Self {
+        let templ = klass.get_pad_template("sink").unwrap();
+        let sinkpad = gst::Pad::builder_with_template(&templ, Some("sink"))
+            .chain_function(|pad, parent, buffer| {
+                TtToJson::catch_panic_pad_function(
+                    parent,
+                    || Err(gst::FlowError::Error),
+                    |enc, element| enc.sink_chain(pad, element, buffer),
+                )
+            })
+            .event_function(|pad, parent, event| {
+                TtToJson::catch_panic_pad_function(
+                    parent,
+                    || false,
+                    |enc, element| enc.sink_event(pad, element, event),
+                )
+            })
+            .build();
+
+        let templ = klass.get_pad_template("src").unwrap();
+        let srcpad = gst::Pad::builder_with_template(&templ, Some("src")).build();
+
+        Self {
+            srcpad,
+            sinkpad,
+            settings: Mutex::new(Settings::default()),
+        }
+    }
+
+    fn class_init(klass: &mut Self::Class) {
+        klass.set_metadata(
+            "Timed text to JSON encoder",
+            "Encoder/ClosedCaption",
+            "Encodes Timed Text to JSON",
+            "Mathieu Duponchelle <mathieu@centricular.com>",
+        );
+
+        let caps = gst::Caps::builder("text/x-raw")
+            .field("format", &"utf8")
+            .build();
+        let sink_pad_template = gst::PadTemplate::new(
+            "sink",
+            gst::PadDirection::Sink,
+            gst::PadPresence::Always,
+            &caps,
+        )
+        .unwrap();
+        klass.add_pad_template(sink_pad_template);
+
+        let caps = gst::Caps::builder("application/x-json").build();
+        let src_pad_template = gst::PadTemplate::new(
+            "src",
+            gst::PadDirection::Src,
+            gst::PadPresence::Always,
+            &caps,
+        )
+        .unwrap();
+        klass.add_pad_template(src_pad_template);
+
+        klass.install_properties(&PROPERTIES);
+    }
+}
+
+impl ObjectImpl for TtToJson {
+    fn constructed(&self, obj: &Self::Type) {
+        self.parent_constructed(obj);
+
+        obj.add_pad(&self.sinkpad).unwrap();
+        obj.add_pad(&self.srcpad).unwrap();
+    }
+
+    fn set_property(&self, _obj: &Self::Type, id: usize, value: &glib::Value) {
+        let prop = &PROPERTIES[id];
+
+        match *prop {
+            subclass::Property("mode", ..) => {
+                let mut settings = self.settings.lock().unwrap();
+                settings.mode = value
+                    .get_some::<Cea608Mode>()
+                    .expect("type checked upstream");
+            }
+            _ => unimplemented!(),
+        }
+    }
+
+    fn get_property(&self, _obj: &Self::Type, id: usize) -> glib::Value {
+        let prop = &PROPERTIES[id];
+
+        match *prop {
+            subclass::Property("mode", ..) => {
+                let settings = self.settings.lock().unwrap();
+                settings.mode.to_value()
+            }
+            _ => unimplemented!(),
+        }
+    }
+}
diff --git a/video/closedcaption/src/tttojson/mod.rs b/video/closedcaption/src/tttojson/mod.rs
new file mode 100644
index 000000000..6c1c3b411
--- /dev/null
+++ b/video/closedcaption/src/tttojson/mod.rs
@@ -0,0 +1,36 @@
+// Copyright (C) 2020 Mathieu Duponchelle <mathieu@centricular.com>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
+// Boston, MA 02110-1335, USA.
+
+use glib::prelude::*;
+
+mod imp;
+
+glib::wrapper! {
+    pub struct TtToJson(ObjectSubclass<imp::TtToJson>) @extends gst::Element, gst::Object;
+}
+
+unsafe impl Send for TtToJson {}
+unsafe impl Sync for TtToJson {}
+
+pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
+    gst::Element::register(
+        Some(plugin),
+        "tttojson",
+        gst::Rank::None,
+        TtToJson::static_type(),
+    )
+}
diff --git a/video/closedcaption/src/ttutils.rs b/video/closedcaption/src/ttutils.rs
new file mode 100644
index 000000000..3ff84f387
--- /dev/null
+++ b/video/closedcaption/src/ttutils.rs
@@ -0,0 +1,66 @@
+// Copyright (C) 2021 Mathieu Duponchelle <mathieu@centricular.com>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
+// Boston, MA 02110-1335, USA.
+
+use serde::{Deserialize, Serialize};
+
+#[derive(
+    Serialize, Deserialize, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Clone, Copy, glib::GEnum,
+)]
+#[repr(u32)]
+#[genum(type_name = "GstTtToCea608Mode")]
+pub enum Cea608Mode {
+    PopOn,
+    RollUp2,
+    RollUp3,
+    RollUp4,
+}
+
+#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq)]
+pub enum TextStyle {
+    White,
+    Green,
+    Blue,
+    Cyan,
+    Red,
+    Yellow,
+    Magenta,
+    ItalicWhite,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct Chunk {
+    pub style: TextStyle,
+    pub underline: bool,
+    pub text: String,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct Line {
+    pub column: Option<u32>,
+    pub row: Option<u32>,
+    pub chunks: Vec<Chunk>,
+    /* In roll-up modes, new lines don't introduce a carriage return by
+     * default, but that can be overridden */
+    pub carriage_return: Option<bool>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct Lines {
+    pub lines: Vec<Line>,
+    pub mode: Option<Cea608Mode>,
+    pub clear: Option<bool>,
+}