diff options
| -rw-r--r-- | Cargo.toml | 2 | ||||
| -rw-r--r-- | docs/plugins/gst_plugins_cache.json | 123 | ||||
| -rw-r--r-- | meson.build | 1 | ||||
| -rw-r--r-- | mux/mp4/Cargo.toml | 50 | ||||
| l--------- | mux/mp4/LICENSE | 1 | ||||
| -rw-r--r-- | mux/mp4/build.rs | 3 | ||||
| -rw-r--r-- | mux/mp4/src/lib.rs | 34 | ||||
| -rw-r--r-- | mux/mp4/src/mp4mux/boxes.rs | 1601 | ||||
| -rw-r--r-- | mux/mp4/src/mp4mux/imp.rs | 1305 | ||||
| -rw-r--r-- | mux/mp4/src/mp4mux/mod.rs | 134 | ||||
| -rw-r--r-- | mux/mp4/tests/tests.rs | 132 |
11 files changed, 3386 insertions, 0 deletions
diff --git a/Cargo.toml b/Cargo.toml index dff1cf5dc..a8cf771dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ members = [ "mux/flavors", "mux/fmp4", + "mux/mp4", "net/aws", "net/hlssink3", @@ -63,6 +64,7 @@ default-members = [ "generic/threadshare", "mux/fmp4", + "mux/mp4", "net/aws", "net/hlssink3", diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index 2e79a5f3c..19bbbdbd8 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -2497,6 +2497,129 @@ "tracers": {}, "url": "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs" }, + "mp4": { + "description": "GStreamer Rust MP4 Plugin", + "elements": { + "isomp4mux": { + "author": "Sebastian Dröge <sebastian@centricular.com>", + "description": "ISO MP4 muxer", + "hierarchy": [ + "GstISOMP4Mux", + "GstRsMP4Mux", + "GstAggregator", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "klass": "Codec/Muxer", + "pad-templates": { + "sink_%%u": { + "caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-vp9:\n profile: { (string)0, (string)1, (string)2, (string)3 }\n chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\naudio/x-opus:\nchannel-mapping-family: [ 0, 255 ]\n channels: [ 1, 8 ]\n rate: [ 1, 2147483647 ]\n", + "direction": "sink", + "presence": "request", + "type": "GstRsMP4MuxPad" + }, + "src": { + "caps": "video/quicktime:\n variant: iso\n", + "direction": "src", + "presence": "always" + } + }, + "rank": "marginal" + } + }, + "filename": "gstmp4", + "license": "MPL", + "other-types": { + "GstRsMP4Mux": { + "hierarchy": [ + "GstRsMP4Mux", + "GstAggregator", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "kind": "object", + "properties": { + "interleave-bytes": { + "blurb": "Interleave between streams in bytes", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "18446744073709551615", + "min": "0", + "mutable": "ready", + "readable": true, + "type": "guint64", + "writable": true + }, + "interleave-time": { + "blurb": "Interleave between streams in nanoseconds", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "500000000", + "max": "18446744073709551615", + "min": "0", + "mutable": "ready", + "readable": true, + "type": "guint64", + "writable": true + }, + "movie-timescale": { + "blurb": "Timescale to use for the movie (units per second, 0 is automatic)", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "-1", + "min": "0", + "mutable": "ready", + "readable": true, + "type": "guint", + "writable": true + } + } + }, + "GstRsMP4MuxPad": { + "hierarchy": [ + "GstRsMP4MuxPad", + "GstAggregatorPad", + "GstPad", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "kind": "object", + "properties": { + "trak-timescale": { + "blurb": "Timescale to use for the track (units per second, 0 is automatic)", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "-1", + "min": "0", + "mutable": "ready", + "readable": true, + "type": "guint", + "writable": true + } + } + } + }, + "package": "gst-plugin-mp4", + "source": "gst-plugin-mp4", + "tracers": {}, + "url": "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs" + }, "ndi": { "description": "GStreamer NewTek NDI Plugin", "elements": { diff --git a/meson.build b/meson.build index 97d7e6c0a..d76f8770b 100644 --- a/meson.build +++ b/meson.build @@ -49,6 +49,7 @@ plugins = { # sodium has an external dependency, see below 'gst-plugin-threadshare': 'libgstthreadshare', + 'gst-plugin-mp4': 'libgstmp4', 'gst-plugin-fmp4': 'libgstfmp4', 'gst-plugin-aws': 'libgstaws', diff --git a/mux/mp4/Cargo.toml b/mux/mp4/Cargo.toml new file mode 100644 index 000000000..b84f7960c --- /dev/null +++ b/mux/mp4/Cargo.toml @@ -0,0 +1,50 @@ +[package] +name = "gst-plugin-mp4" +version = "0.10.0-alpha.1" +authors = ["Sebastian Dröge <sebastian@centricular.com>"] +license = "MPL-2.0" +description = "GStreamer Rust MP4 Plugin" +repository = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs" +edition = "2021" +rust-version = "1.63" + +[dependencies] +anyhow = "1" +gst = { package = "gstreamer", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } +gst-base = { package = "gstreamer-base", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } +gst-audio = { package = "gstreamer-audio", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } +gst-video = { package = "gstreamer-video", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } +gst-pbutils = { package = "gstreamer-pbutils", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } +once_cell = "1.0" + +[lib] +name = "gstmp4" +crate-type = ["cdylib", "rlib"] +path = "src/lib.rs" + +[dev-dependencies] +tempfile = "3" +url = "1" + +[build-dependencies] +gst-plugin-version-helper = { path="../../version-helper" } + +[features] +default = ["v1_18"] +static = [] +capi = [] +v1_18 = ["gst-video/v1_18"] +doc = ["gst/v1_18"] + +[package.metadata.capi] +min_version = "0.8.0" + +[package.metadata.capi.header] +enabled = false + +[package.metadata.capi.library] +install_subdir = "gstreamer-1.0" +versioning = false + +[package.metadata.capi.pkg_config] +requires_private = "gstreamer-1.0, gstreamer-base-1.0, gstreamer-audio-1.0, gstreamer-video-1.0, gobject-2.0, glib-2.0, gmodule-2.0" diff --git a/mux/mp4/LICENSE b/mux/mp4/LICENSE new file mode 120000 index 000000000..eb5d24fe9 --- /dev/null +++ b/mux/mp4/LICENSE @@ -0,0 +1 @@ +../../LICENSE-MPL-2.0
\ No newline at end of file diff --git a/mux/mp4/build.rs b/mux/mp4/build.rs new file mode 100644 index 000000000..cda12e57e --- /dev/null +++ b/mux/mp4/build.rs @@ -0,0 +1,3 @@ +fn main() { + gst_plugin_version_helper::info() +} diff --git a/mux/mp4/src/lib.rs b/mux/mp4/src/lib.rs new file mode 100644 index 000000000..16b003689 --- /dev/null +++ b/mux/mp4/src/lib.rs @@ -0,0 +1,34 @@ +// Copyright (C) 2022 Sebastian Dröge <sebastian@centricular.com> +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// <https://mozilla.org/MPL/2.0/>. +// +// SPDX-License-Identifier: MPL-2.0 +#![allow(clippy::non_send_fields_in_send_ty, unused_doc_comments)] + +/** + * plugin-mp4: + * + * Since: plugins-rs-0.10.0 + */ +use gst::glib; + +mod mp4mux; + +fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + mp4mux::register(plugin) +} + +gst::plugin_define!( + mp4, + env!("CARGO_PKG_DESCRIPTION"), + plugin_init, + concat!(env!("CARGO_PKG_VERSION"), "-", env!("COMMIT_ID")), + // FIXME: MPL-2.0 is only allowed since 1.18.3 (as unknown) and 1.20 (as known) + "MPL", + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_REPOSITORY"), + env!("BUILD_REL_DATE") +); diff --git a/mux/mp4/src/mp4mux/boxes.rs b/mux/mp4/src/mp4mux/boxes.rs new file mode 100644 index 000000000..5f1214661 --- /dev/null +++ b/mux/mp4/src/mp4mux/boxes.rs @@ -0,0 +1,1601 @@ +// Copyright (C) 2022 Sebastian Dröge <sebastian@centricular.com> +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// <https://mozilla.org/MPL/2.0/>. +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::prelude::*; + +use anyhow::{anyhow, bail, Context, Error}; + +use std::str::FromStr; + +fn write_box<T, F: FnOnce(&mut Vec<u8>) -> Result<T, Error>>( + vec: &mut Vec<u8>, + fourcc: impl std::borrow::Borrow<[u8; 4]>, + content_func: F, +) -> Result<T, Error> { + // Write zero size ... + let size_pos = vec.len(); + vec.extend([0u8; 4]); + vec.extend(fourcc.borrow()); + + let res = content_func(vec)?; + + // ... and update it here later. + let size: u32 = vec + .len() + .checked_sub(size_pos) + .expect("vector shrunk") + .try_into() + .context("too big box content")?; + vec[size_pos..][..4].copy_from_slice(&size.to_be_bytes()); + + Ok(res) +} + +const FULL_BOX_VERSION_0: u8 = 0; +const FULL_BOX_VERSION_1: u8 = 1; + +const FULL_BOX_FLAGS_NONE: u32 = 0; + +fn write_full_box<T, F: FnOnce(&mut Vec<u8>) -> Result<T, Error>>( + vec: &mut Vec<u8>, + fourcc: impl std::borrow::Borrow<[u8; 4]>, + version: u8, + flags: u32, + content_func: F, +) -> Result<T, Error> { + write_box(vec, fourcc, move |vec| { + assert_eq!(flags >> 24, 0); + vec.extend(((u32::from(version) << 24) | flags).to_be_bytes()); + content_func(vec) + }) +} + +/// Creates `ftyp` box +pub(super) fn create_ftyp(variant: super::Variant) -> Result<gst::Buffer, Error> { + let mut v = vec![]; + + let (brand, compatible_brands) = match variant { + super::Variant::ISO => (b"isom", vec![b"mp41", b"mp42"]), + }; + + write_box(&mut v, b"ftyp", |v| { + // major brand + v.extend(brand); + // minor version + v.extend(0u32.to_be_bytes()); + // compatible brands + v.extend(compatible_brands.into_iter().flatten()); + + Ok(()) + })?; + + Ok(gst::Buffer::from_mut_slice(v)) +} + +/// Creates `mdat` box *header*. +pub(super) fn create_mdat_header(size: Option<u64>) -> Result<gst::Buffer, Error> { + let mut v = vec![]; + + if let Some(size) = size { + if let Ok(size) = u32::try_from(size + 8) { + v.extend(8u32.to_be_bytes()); + v.extend(b"free"); + v.extend(size.to_be_bytes()); + v.extend(b"mdat"); + } else { + v.extend(1u32.to_be_bytes()); + v.extend(b"mdat"); + v.extend((size + 16).to_be_bytes()); + } + } else { + v.extend(8u32.to_be_bytes()); + v.extend(b"free"); + v.extend(0u32.to_be_bytes()); + v.extend(b"mdat"); + } + + Ok(gst::Buffer::from_mut_slice(v)) +} + +/// Creates `moov` box +pub(super) fn create_moov(header: super::Header) -> Result<gst::Buffer, Error> { + let mut v = vec![]; + + write_box(&mut v, b"moov", |v| write_moov(v, &header))?; + + Ok(gst::Buffer::from_mut_slice(v)) +} + +fn write_moov(v: &mut Vec<u8>, header: &super::Header) -> Result<(), Error> { + use gst::glib; + + let base = glib::DateTime::from_utc(1904, 1, 1, 0, 0, 0.0)?; + let now = glib::DateTime::now_utc()?; + let creation_time = + u64::try_from(now.difference(&base).as_seconds()).expect("time before 1904"); + + write_full_box(v, b"mvhd", FULL_BOX_VERSION_1, FULL_BOX_FLAGS_NONE, |v| { + write_mvhd(v, header, creation_time) + })?; + for (idx, stream) in header.streams.iter().enumerate() { + write_box(v, b"trak", |v| { + write_trak(v, header, idx, stream, creation_time) + })?; + } + + Ok(()) +} + +fn stream_to_timescale(stream: &super::Stream) -> u32 { + if stream.trak_timescale > 0 { + stream.trak_timescale + } else { + let s = stream.caps.structure(0).unwrap(); + + if let Ok(fps) = s.get::<gst::Fraction>("framerate") { + if fps.numer() == 0 { + return 10_000; + } + + if fps.denom() != 1 && fps.denom() != 1001 { + if let Some(fps) = (fps.denom() as u64) + .nseconds() + .mul_div_round(1_000_000_000, fps.numer() as u64) + .and_then(gst_video::guess_framerate) + { + return (fps.numer() as u32) + .mul_div_round(100, fps.denom() as u32) + .unwrap_or(10_000); + } + } + + (fps.numer() as u32) + .mul_div_round(100, fps.denom() as u32) + .unwrap_or(10_000) + } else if let Ok(rate) = s.get::<i32>("rate") { + rate as u32 + } else { + 10_000 + } + } +} + +fn header_to_timescale(header: &super::Header) -> u32 { + if header.movie_timescale > 0 { + header.movie_timescale + } else { + // Use the reference track timescale + stream_to_timescale(&header.streams[0]) + } +} + +fn write_mvhd(v: &mut Vec<u8>, header: &super::Header, creation_time: u64) -> Result<(), Error> { + let timescale = header_to_timescale(header); + + // Creation time + v.extend(creation_time.to_be_bytes()); + // Modification time + v.extend(creation_time.to_be_bytes()); + // Timescale + v.extend(timescale.to_be_bytes()); + // Duration + let min_earliest_pts = header.streams.iter().map(|s| s.earliest_pts).min().unwrap(); + let max_end_pts = header + .streams + .iter() + .map(|stream| stream.end_pts) + .max() + .unwrap(); + let duration = (max_end_pts - min_earliest_pts) + .nseconds() + .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds()) + .context("too big track duration")?; + v.extend(duration.to_be_bytes()); + + // Rate 1.0 + v.extend((1u32 << 16).to_be_bytes()); + // Volume 1.0 + v.extend((1u16 << 8).to_be_bytes()); + // Reserved + v.extend([0u8; 2 + 2 * 4]); + + // Matrix + v.extend( + [ + (1u32 << 16).to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + (1u32 << 16).to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + (16384u32 << 16).to_be_bytes(), + ] + .into_iter() + .flatten(), + ); + + // Pre defined + v.extend([0u8; 6 * 4]); + + // Next track id + v.extend((header.streams.len() as u32 + 1).to_be_bytes()); + + Ok(()) +} + +const TKHD_FLAGS_TRACK_ENABLED: u32 = 0x1; +const TKHD_FLAGS_TRACK_IN_MOVIE: u32 = 0x2; +const TKHD_FLAGS_TRACK_IN_PREVIEW: u32 = 0x4; + +fn write_trak( + v: &mut Vec<u8>, + header: &super::Header, + idx: usize, + stream: &super::Stream, + creation_time: u64, +) -> Result<(), Error> { + write_full_box( + v, + b"tkhd", + FULL_BOX_VERSION_1, + TKHD_FLAGS_TRACK_ENABLED | TKHD_FLAGS_TRACK_IN_MOVIE | TKHD_FLAGS_TRACK_IN_PREVIEW, + |v| write_tkhd(v, header, idx, stream, creation_time), + )?; + + write_box(v, b"mdia", |v| write_mdia(v, header, stream, creation_time))?; + write_box(v, b"edts", |v| write_edts(v, header, stream))?; + + Ok(()) +} + +fn write_tkhd( + v: &mut Vec<u8>, + header: &super::Header, + idx: usize, + stream: &super::Stream, + creation_time: u64, +) -> Result<(), Error> { + // Creation time + v.extend(creation_time.to_be_bytes()); + // Modification time + v.extend(creation_time.to_be_bytes()); + // Track ID + v.extend((idx as u32 + 1).to_be_bytes()); + // Reserved + v.extend(0u32.to_be_bytes()); + // Duration + + // Track header duration is in movie header timescale + let timescale = header_to_timescale(header); + + let min_earliest_pts = header.streams.iter().map(|s| s.earliest_pts).min().unwrap(); + // Duration is the end PTS of this stream up to the beginning of the earliest stream + let duration = stream.end_pts - min_earliest_pts; + let duration = duration + .nseconds() + .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds()) + .context("too big track duration")?; + v.extend(duration.to_be_bytes()); + + // Reserved + v.extend([0u8; 2 * 4]); + + // Layer + v.extend(0u16.to_be_bytes()); + // Alternate group + v.extend(0u16.to_be_bytes()); + + // Volume + let s = stream.caps.structure(0).unwrap(); + match s.name() { + "audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { + v.extend((1u16 << 8).to_be_bytes()) + } + _ => v.extend(0u16.to_be_bytes()), + } + + // Reserved + v.extend([0u8; 2]); + + // Matrix + v.extend( + [ + (1u32 << 16).to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + (1u32 << 16).to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + (16384u32 << 16).to_be_bytes(), + ] + .into_iter() + .flatten(), + ); + + // Width/height + match s.name() { + "video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => { + let width = s.get::<i32>("width").context("video caps without width")? as u32; + let height = s + .get::<i32>("height") + .context("video caps without height")? as u32; + let par = s + .get::<gst::Fraction>("pixel-aspect-ratio") + .unwrap_or_else(|_| gst::Fraction::new(1, 1)); + + let width = std::cmp::min( + width + .mul_div_round(par.numer() as u32, par.denom() as u32) + .unwrap_or(u16::MAX as u32), + u16::MAX as u32, + ); + let height = std::cmp::min(height, u16::MAX as u32); + + v.extend((width << 16).to_be_bytes()); + v.extend((height << 16).to_be_bytes()); + } + _ => v.extend([0u8; 2 * 4]), + } + + Ok(()) +} + +fn write_mdia( + v: &mut Vec<u8>, + header: &super::Header, + stream: &super::Stream, + creation_time: u64, +) -> Result<(), Error> { + write_full_box(v, b"mdhd", FULL_BOX_VERSION_1, FULL_BOX_FLAGS_NONE, |v| { + write_mdhd(v, header, stream, creation_time) + })?; + write_full_box(v, b"hdlr", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { + write_hdlr(v, header, stream) + })?; + + // TODO: write elng if needed + + write_box(v, b"minf", |v| write_minf(v, header, stream))?; + + Ok(()) +} + +fn language_code(lang: impl std::borrow::Borrow<[u8; 3]>) -> u16 { + let lang = lang.borrow(); + + // TODO: Need to relax this once we get the language code from tags + assert!(lang.iter().all(u8::is_ascii_lowercase)); + + (((lang[0] as u16 - 0x60) & 0x1F) << 10) + + (((lang[1] as u16 - 0x60) & 0x1F) << 5) + + ((lang[2] as u16 - 0x60) & 0x1F) +} + +fn write_mdhd( + v: &mut Vec<u8>, + _header: &super::Header, + stream: &super::Stream, + creation_time: u64, +) -> Result<(), Error> { + let timescale = stream_to_timescale(stream); + + // Creation time + v.extend(creation_time.to_be_bytes()); + // Modification time + v.extend(creation_time.to_be_bytes()); + // Timescale + v.extend(timescale.to_be_bytes()); + // Duration + let duration = stream + .chunks + .iter() + .flat_map(|c| c.samples.iter().map(|b| b.duration.nseconds())) + .sum::<u64>() + .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds()) + .context("too big track duration")?; + v.extend(duration.to_be_bytes()); + + // Language as ISO-639-2/T + // TODO: get actual language from the tags + v.extend(language_code(b"und").to_be_bytes()); + + // Pre-defined + v.extend([0u8; 2]); + + Ok(()) +} + +fn write_hdlr( + v: &mut Vec<u8>, + _header: &super::Header, + stream: &super::Stream, +) -> Result<(), Error> { + // Pre-defined + v.extend([0u8; 4]); + + let s = stream.caps.structure(0).unwrap(); + let (handler_type, name) = match s.name() { + "video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => { + (b"vide", b"VideoHandler\0".as_slice()) + } + "audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { + (b"soun", b"SoundHandler\0".as_slice()) + } + _ => unreachable!(), + }; + + // Handler type + v.extend(handler_type); + + // Reserved + v.extend([0u8; 3 * 4]); + + // Name + v.extend(name); + + Ok(()) +} + +fn write_minf( + v: &mut Vec<u8>, + header: &super::Header, + stream: &super::Stream, +) -> Result<(), Error> { + let s = stream.caps.structure(0).unwrap(); + + match s.name() { + "video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => { + // Flags are always 1 for unspecified reasons + write_full_box(v, b"vmhd", FULL_BOX_VERSION_0, 1, |v| write_vmhd(v, header))? + } + "audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { + write_full_box(v, b"smhd", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { + write_smhd(v, header) + })? + } + _ => unreachable!(), + } + + write_box(v, b"dinf", |v| write_dinf(v, header))?; + + write_box(v, b"stbl", |v| write_stbl(v, header, stream))?; + + Ok(()) +} + +fn write_vmhd(v: &mut Vec<u8>, _header: &super::Header) -> Result<(), Error> { + // Graphics mode + v.extend([0u8; 2]); + + // opcolor + v.extend([0u8; 2 * 3]); + + Ok(()) +} + +fn write_smhd(v: &mut Vec<u8>, _header: &super::Header) -> Result<(), Error> { + // Balance + v.extend([0u8; 2]); + + // Reserved + v.extend([0u8; 2]); + + Ok(()) +} + +fn write_dinf(v: &mut Vec<u8>, header: &super::Header) -> Result<(), Error> { + write_full_box(v, b"dref", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { + write_dref(v, header) + })?; + + Ok(()) +} + +const DREF_FLAGS_MEDIA_IN_SAME_FILE: u32 = 0x1; + +fn write_dref(v: &mut Vec<u8>, _header: &super::Header) -> Result<(), Error> { + // Entry count + v.extend(1u32.to_be_bytes()); + + write_full_box( + v, + b"url ", + FULL_BOX_VERSION_0, + DREF_FLAGS_MEDIA_IN_SAME_FILE, + |_v| Ok(()), + )?; + + Ok(()) +} + +fn write_stbl( + v: &mut Vec<u8>, + header: &super::Header, + stream: &super::Stream, +) -> Result<(), Error> { + write_full_box(v, b"stsd", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { + write_stsd(v, header, stream) + })?; + write_full_box(v, b"stts", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { + write_stts(v, header, stream) + })?; + + // If there are any composition time offsets we need to write the ctts box. If any are negative + // we need to write version 1 of the box, otherwise version 0 is sufficient. + let mut need_ctts = None; + if stream.delta_frames.requires_dts() { + for composition_time_offset in stream.chunks.iter().flat_map(|c| { + c.samples.iter().map(|b| { + b.composition_time_offset + .expect("not all samples have a composition time offset") + }) + }) { + if composition_time_offset < 0 { + need_ctts = Some(1); + break; + } else { + need_ctts = Some(0); + } + } + } + if let Some(need_ctts) = need_ctts { + let version = if need_ctts == 0 { + FULL_BOX_VERSION_0 + } else { + FULL_BOX_VERSION_1 + }; + + write_full_box(v, b"ctts", version, FULL_BOX_FLAGS_NONE, |v| { + write_ctts(v, header, stream, version) + })?; + + write_full_box(v, b"cslg", FULL_BOX_VERSION_1, FULL_BOX_FLAGS_NONE, |v| { + write_cslg(v, header, stream) + })?; + } + + // If any sample is not a sync point, write the stss box + if !stream.delta_frames.intra_only() + && stream + .chunks + .iter() + .flat_map(|c| c.samples.iter().map(|b| b.sync_point)) + .any(|sync_point| !sync_point) + { + write_full_box(v, b"stss", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { + write_stss(v, header, stream) + })?; + } + + write_full_box(v, b"stsz", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { + write_stsz(v, header, stream) + })?; + + write_full_box(v, b"stsc", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { + write_stsc(v, header, stream) + })?; + + if stream.chunks.last().unwrap().offset > u32::MAX as u64 { + write_full_box(v, b"co64", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { + write_stco(v, header, stream, true) + })?; + } else { + write_full_box(v, b"stco", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { + write_stco(v, header, stream, false) + })?; + } + + Ok(()) +} + +fn write_stsd( + v: &mut Vec<u8>, + header: &super::Header, + stream: &super::Stream, +) -> Result<(), Error> { + // Entry count + v.extend(1u32.to_be_bytes()); + + let s = stream.caps.structure(0).unwrap(); + match s.name() { + "video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => { + write_visual_sample_entry(v, header, stream)? + } + "audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { + write_audio_sample_entry(v, header, stream)? + } + _ => unreachable!(), + } + + Ok(()) +} + +fn write_sample_entry_box<T, F: FnOnce(&mut Vec<u8>) -> Result<T, Error>>( + v: &mut Vec<u8>, + fourcc: impl std::borrow::Borrow<[u8; 4]>, + content_func: F, +) -> Result<T, Error> { + write_box(v, fourcc, move |v| { + // Reserved + v.extend([0u8; 6]); + + // Data reference index + v.extend(1u16.to_be_bytes()); + + content_func(v) + }) +} + +fn write_visual_sample_entry( + v: &mut Vec<u8>, + _header: &super::Header, + stream: &super::Stream, +) -> Result<(), Error> { + let s = stream.caps.structure(0).unwrap(); + let fourcc = match s.name() { + "video/x-h264" => { + let stream_format = s.get::<&str>("stream-format").context("no stream-format")?; + match stream_format { + "avc" => b"avc1", + "avc3" => b"avc3", + _ => unreachable!(), + } + } + "video/x-h265" => { + let stream_format = s.get::<&str>("stream-format").context("no stream-format")?; + match stream_format { + "hvc1" => b"hvc1", + "hev1" => b"hev1", + _ => unreachable!(), + } + } + "image/jpeg" => b"jpeg", + "video/x-vp9" => b"vp09", + _ => unreachable!(), + }; + + write_sample_entry_box(v, fourcc, move |v| { + // pre-defined + v.extend([0u8; 2]); + // Reserved + v.extend([0u8; 2]); + // pre-defined + v.extend([0u8; 3 * 4]); + + // Width + let width = + u16::try_from(s.get::<i32>("width").context("no width")?).context("too big width")?; + v.extend(width.to_be_bytes()); + + // Height + let height = u16::try_from(s.get::<i32>("height").context("no height")?) + .context("too big height")?; + v.extend(height.to_be_bytes()); + + // Horizontal resolution + v.extend(0x00480000u32.to_be_bytes()); + + // Vertical resolution + v.extend(0x00480000u32.to_be_bytes()); + + // Reserved + v.extend([0u8; 4]); + + // Frame count + v.extend(1u16.to_be_bytes()); + + // Compressor name + v.extend([0u8; 32]); + + // Depth + v.extend(0x0018u16.to_be_bytes()); + + // Pre-defined + v.extend((-1i16).to_be_bytes()); + + // Codec specific boxes + match s.name() { + "video/x-h264" => { + let codec_data = s + .get::<&gst::BufferRef>("codec_data") + .context("no codec_data")?; + let map = codec_data + .map_readable() + .context("codec_data not mappable")?; + write_box(v, b"avcC", move |v| { + v.extend_from_slice(&map); + Ok(()) + })?; + } + "video/x-h265" => { + let codec_data = s + .get::<&gst::BufferRef>("codec_data") + .context("no codec_data")?; + let map = codec_data + .map_readable() + .context("codec_data not mappable")?; + write_box(v, b"hvcC", move |v| { + v.extend_from_slice(&map); + Ok(()) + })?; + } + "video/x-vp9" => { + let profile: u8 = match s.get::<&str>("profile").expect("no vp9 profile") { + "0" => Some(0), + "1" => Some(1), + "2" => Some(2), + "3" => Some(3), + _ => None, + } + .context("unsupported vp9 profile")?; + let colorimetry = gst_video::VideoColorimetry::from_str( + s.get::<&str>("colorimetry").expect("no colorimetry"), + ) + .context("failed to parse colorimetry")?; + let video_full_range = + colorimetry.range() == gst_video::VideoColorRange::Range0_255; + let chroma_format: u8 = + match s.get::<&str>("chroma-format").expect("no chroma-format") { + "4:2:0" => + // chroma-site is optional + { + match s + .get::<&str>("chroma-site") + .ok() + .and_then(|cs| gst_video::VideoChromaSite::from_str(cs).ok()) + { + Some(gst_video::VideoChromaSite::V_COSITED) => Some(0), + // COSITED + _ => Some(1), + } + } + "4:2:2" => Some(2), + "4:4:4" => Some(3), + _ => None, + } + .context("unsupported chroma-format")?; + let bit_depth: u8 = { + let bit_depth_luma = s.get::<u32>("bit-depth-luma").expect("no bit-depth-luma"); + let bit_depth_chroma = s + .get::<u32>("bit-depth-chroma") + .expect("no bit-depth-chroma"); + if bit_depth_luma != bit_depth_chroma { + return Err(anyhow!("bit-depth-luma and bit-depth-chroma have different values which is an unsupported configuration")); + } + bit_depth_luma as u8 + }; + write_full_box(v, b"vpcC", 1, 0, move |v| { + v.push(profile); + // XXX: hardcoded level 1 + v.push(10); + let mut byte: u8 = 0; + byte |= (bit_depth & 0xF) << 4; + byte |= (chroma_format & 0x7) << 1; + byte |= video_full_range as u8; + v.push(byte); + v.push(colorimetry.primaries().to_iso() as u8); + v.push(colorimetry.transfer().to_iso() as u8); + v.push(colorimetry.matrix().to_iso() as u8); + // 16-bit length field for codec initialization, unused + v.push(0); + v.push(0); + Ok(()) + })?; + } + "image/jpeg" => { + // Nothing to do here + } + _ => unreachable!(), + } + + if let Ok(par) = s.get::<gst::Fraction>("pixel-aspect-ratio") { + write_box(v, b"pasp", move |v| { + v.extend((par.numer() as u32).to_be_bytes()); + v.extend((par.denom() as u32).to_be_bytes()); + Ok(()) + })?; + } + + if let Some(colorimetry) = s + .get::<&str>("colorimetry") + .ok() + .and_then(|c| c.parse::<gst_video::VideoColorimetry>().ok()) + { + write_box(v, b"colr", move |v| { + v.extend(b"nclx"); + let (primaries, transfer, matrix) = { + #[cfg(feature = "v1_18")] + { + ( + (colorimetry.primaries().to_iso() as u16), + (colorimetry.transfer().to_iso() as u16), + (colorimetry.matrix().to_iso() as u16), + ) + } + #[cfg(not(feature = "v1_18"))] + { + let primaries = match colorimetry.primaries() { + gst_video::VideoColorPrimaries::Bt709 => 1u16, + gst_video::VideoColorPrimaries::Bt470m => 4u16, + gst_video::VideoColorPrimaries::Bt470bg => 5u16, + gst_video::VideoColorPrimaries::Smpte170m => 6u16, + gst_video::VideoColorPrimaries::Smpte240m => 7u16, + gst_video::VideoColorPrimaries::Film => 8u16, + gst_video::VideoColorPrimaries::Bt2020 => 9u16, + _ => 2, + }; + let transfer = match colorimetry.transfer() { + gst_video::VideoTransferFunction::Bt709 => 1u16, + gst_video::VideoTransferFunction::Gamma22 => 4u16, + gst_video::VideoTransferFunction::Gamma28 => 5u16, + gst_video::VideoTransferFunction::Smpte240m => 7u16, + gst_video::VideoTransferFunction::Gamma10 => 8u16, + gst_video::VideoTransferFunction::Log100 => 9u16, + gst_video::VideoTransferFunction::Log316 => 10u16, + gst_video::VideoTransferFunction::Srgb => 13u16, + gst_video::VideoTransferFunction::Bt202012 => 15u16, + _ => 2, + }; + let matrix = match colorimetry.matrix() { + gst_video::VideoColorMatrix::Rgb => 0u16, + gst_video::VideoColorMatrix::Bt709 => 1u16, + gst_video::VideoColorMatrix::Fcc => 4u16, + gst_video::VideoColorMatrix::Bt601 => 6u16, + gst_video::VideoColorMatrix::Smpte240m => 7u16, + gst_video::VideoColorMatrix::Bt2020 => 9u16, + _ => 2, + }; + + (primaries, transfer, matrix) + } + }; + + let full_range = match colorimetry.range() { + gst_video::VideoColorRange::Range0_255 => 0x80u8, + gst_video::VideoColorRange::Range16_235 => 0x00u8, + _ => 0x00, + }; + + v.extend(primaries.to_be_bytes()); + v.extend(transfer.to_be_bytes()); + v.extend(matrix.to_be_bytes()); + v.push(full_range); + + Ok(()) + })?; + } + + #[cfg(feature = "v1_18")] + { + if let Ok(cll) = gst_video::VideoContentLightLevel::from_caps(&stream.caps) { + write_box(v, b"clli", move |v| { + v.extend((cll.max_content_light_level() as u16).to_be_bytes()); + v.extend((cll.max_frame_average_light_level() as u16).to_be_bytes()); + Ok(()) + })?; + } + + if let Ok(mastering) = gst_video::VideoMasteringDisplayInfo::from_caps(&stream.caps) { + write_box(v, b"mdcv", move |v| { + for primary in mastering.display_primaries() { + v.extend(primary.x.to_be_bytes()); + v.extend(primary.y.to_be_bytes()); + } + v.extend(mastering.white_point().x.to_be_bytes()); + v.extend(mastering.white_point().y.to_be_bytes()); + v.extend(mastering.max_display_mastering_luminance().to_be_bytes()); + v.extend(mastering.max_display_mastering_luminance().to_be_bytes()); + Ok(()) + })?; + } + } + + // Write fiel box for codecs that require it + if ["image/jpeg"].contains(&s.name()) { + let interlace_mode = s + .get::<&str>("interlace-mode") + .ok() + .map(gst_video::VideoInterlaceMode::from_string) + .unwrap_or(gst_video::VideoInterlaceMode::Progressive); + let field_order = s + .get::<&str>("field-order") + .ok() + .map(gst_video::VideoFieldOrder::from_string) + .unwrap_or(gst_video::VideoFieldOrder::Unknown); + + write_box(v, b"fiel", move |v| { + let (interlace, field_order) = match interlace_mode { + gst_video::VideoInterlaceMode::Progressive => (1, 0), + gst_video::VideoInterlaceMode::Interleaved + if field_order == gst_video::VideoFieldOrder::TopFieldFirst => + { + (2, 9) + } + gst_video::VideoInterlaceMode::Interleaved => (2, 14), + _ => (0, 0), + }; + + v.push(interlace); + v.push(field_order); + Ok(()) + })?; + } + + // TODO: write btrt bitrate box based on tags + + Ok(()) + })?; + + Ok(()) +} + +fn write_audio_sample_entry( + v: &mut Vec<u8>, + _header: &super::Header, + stream: &super::Stream, +) -> Result<(), Error> { + let s = stream.caps.structure(0).unwrap(); + let fourcc = match s.name() { + "audio/mpeg" => b"mp4a", + "audio/x-opus" => b"Opus", + "audio/x-alaw" => b"alaw", + "audio/x-mulaw" => b"ulaw", + "audio/x-adpcm" => { + let layout = s.get::<&str>("layout").context("no ADPCM layout field")?; + + match layout { + "g726" => b"ms\x00\x45", + _ => unreachable!(), + } + } + _ => unreachable!(), + }; + + let sample_size = match s.name() { + "audio/x-adpcm" => { + let bitrate = s.get::<i32>("bitrate").context("no ADPCM bitrate field")?; + (bitrate / 8000) as u16 + } + _ => 16u16, + }; + + write_sample_entry_box(v, fourcc, move |v| { + // Reserved + v.extend([0u8; 2 * 4]); + + // Channel count + let channels = u16::try_from(s.get::<i32>("channels").context("no channels")?) + .context("too many channels")?; + v.extend(channels.to_be_bytes()); + + // Sample size + v.extend(sample_size.to_be_bytes()); + + // Pre-defined + v.extend([0u8; 2]); + + // Reserved + v.extend([0u8; 2]); + + // Sample rate + let rate = u16::try_from(s.get::<i32>("rate").context("no rate")?).unwrap_or(0); + v.extend((u32::from(rate) << 16).to_be_bytes()); + + // Codec specific boxes + match s.name() { + "audio/mpeg" => { + let codec_data = s + .get::<&gst::BufferRef>("codec_data") + .context("no codec_data")?; + let map = codec_data + .map_readable() + .context("codec_data not mappable")?; + if map.len() < 2 { + bail!("too small codec_data"); + } + write_esds_aac(v, &map)?; + } + "audio/x-opus" => { + write_dops(v, &stream.caps)?; + } + "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { + // Nothing to do here + } + _ => unreachable!(), + } + + // If rate did not fit into 16 bits write a full `srat` box + if rate == 0 { + let rate = s.get::<i32>("rate").context("no rate")?; + // FIXME: This is defined as full box? + write_full_box( + v, + b"srat", + FULL_BOX_VERSION_0, + FULL_BOX_FLAGS_NONE, + move |v| { + v.extend((rate as u32).to_be_bytes()); + Ok(()) + }, + )?; + } + + // TODO: write btrt bitrate box based on tags + + // TODO: chnl box for channel ordering? probably not needed for AAC + + Ok(()) + })?; + + Ok(()) +} + +fn write_esds_aac(v: &mut Vec<u8>, codec_data: &[u8]) -> Result<(), Error> { + let calculate_len = |mut len| { + if len > 260144641 { + bail!("too big descriptor length"); + } + + if len == 0 { + return Ok(([0; 4], 1)); + } + + let mut idx = 0; + let mut lens = [0u8; 4]; + while len > 0 { + lens[idx] = ((if len > 0x7f { 0x80 } else { 0x00 }) | (len & 0x7f)) as u8; + idx += 1; + len >>= 7; + } + + Ok((lens, idx)) + }; + + write_full_box( + v, + b"esds", + FULL_BOX_VERSION_0, + FULL_BOX_FLAGS_NONE, + move |v| { + // Calculate all lengths bottom up + + // Decoder specific info + let decoder_specific_info_len = calculate_len(codec_data.len())?; + + // Decoder config + let decoder_config_len = + calculate_len(13 + 1 + decoder_specific_info_len.1 + codec_data.len())?; + + // SL config + let sl_config_len = calculate_len(1)?; + + // ES descriptor + let es_descriptor_len = calculate_len( + 3 + 1 + + decoder_config_len.1 + + 13 + + 1 + + decoder_specific_info_len.1 + + codec_data.len() + + 1 + + sl_config_len.1 + + 1, + )?; + + // ES descriptor tag + v.push(0x03); + + // Length + v.extend_from_slice(&es_descriptor_len.0[..(es_descriptor_len.1)]); + + // Track ID + v.extend(1u16.to_be_bytes()); + // Flags + v.push(0u8); + + // Decoder config descriptor + v.push(0x04); + + // Length + v.extend_from_slice(&decoder_config_len.0[..(decoder_config_len.1)]); + + // Object type ESDS_OBJECT_TYPE_MPEG4_P3 + v.push(0x40); + // Stream type ESDS_STREAM_TYPE_AUDIO + v.push((0x05 << 2) | 0x01); + + // Buffer size db? + v.extend([0u8; 3]); + + // Max bitrate + v.extend(0u32.to_be_bytes()); + + // Avg bitrate + v.extend(0u32.to_be_bytes()); + + // Decoder specific info + v.push(0x05); + + // Length + v.extend_from_slice(&decoder_specific_info_len.0[..(decoder_specific_info_len.1)]); + v.extend_from_slice(codec_data); + + // SL config descriptor + v.push(0x06); + + // Length: 1 (tag) + 1 (length) + 1 (predefined) + v.extend_from_slice(&sl_config_len.0[..(sl_config_len.1)]); + + // Predefined + v.push(0x02); + Ok(()) + }, + ) +} + +fn write_dops(v: &mut Vec<u8>, caps: &gst::Caps) -> Result<(), Error> { + let rate; + let channels; + let channel_mapping_family; + let stream_count; + let coupled_count; + let pre_skip; + let output_gain; + let mut channel_mapping = [0; 256]; + + // TODO: Use audio clipping meta to calculate pre_skip + + if let Some(header) = caps + .structure(0) + .unwrap() + .get::<gst::ArrayRef>("streamheader") + .ok() + .and_then(|a| a.get(0).and_then(|v| v.get::<gst::Buffer>().ok())) + { + ( + rate, + channels, + channel_mapping_family, + stream_count, + coupled_count, + pre_skip, + output_gain, + ) = gst_pbutils::codec_utils_opus_parse_header(&header, Some(&mut channel_mapping)) + .unwrap(); + } else { + ( + rate, + channels, + channel_mapping_family, + stream_count, + coupled_count, + ) = gst_pbutils::codec_utils_opus_parse_caps(caps, Some(&mut channel_mapping)).unwrap(); + output_gain = 0; + pre_skip = 0; + } + + write_box(v, b"dOps", move |v| { + // Version number + v.push(0); + v.push(channels); + v.extend(pre_skip.to_le_bytes()); + v.extend(rate.to_le_bytes()); + v.extend(output_gain.to_le_bytes()); + v.push(channel_mapping_family); + if channel_mapping_family > 0 { + v.push(stream_count); + v.push(coupled_count); + v.extend(&channel_mapping[..channels as usize]); + } + + Ok(()) + }) +} + +fn write_stts( + v: &mut Vec<u8>, + _header: &super::Header, + stream: &super::Stream, +) -> Result<(), Error> { + let timescale = stream_to_timescale(stream); + + let entry_count_position = v.len(); + // Entry count, rewritten in the end + v.extend(0u32.to_be_bytes()); + + let mut last_duration: Option<u32> = None; + let mut sample_count = 0u32; + let mut num_entries = 0u32; + for duration in stream + .chunks + .iter() + .flat_map(|c| c.samples.iter().map(|b| b.duration)) + { + let duration = u32::try_from( + duration + .nseconds() + .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds()) + .context("too big sample duration")?, + ) + .context("too big sample duration")?; + + if last_duration.map_or(true, |last_duration| last_duration != duration) { + if let Some(last_duration) = last_duration { + v.extend(sample_count.to_be_bytes()); + v.extend(last_duration.to_be_bytes()); + num_entries += 1; + } + + last_duration = Some(duration); + sample_count = 1; + } else { + sample_count += 1; + } + } + + if let Some(last_duration) = last_duration { + v.extend(sample_count.to_be_bytes()); + v.extend(last_duration.to_be_bytes()); + num_entries += 1; + } + + // Rewrite entry count + v[entry_count_position..][..4].copy_from_slice(&num_entries.to_be_bytes()); + + Ok(()) +} + +fn write_ctts( + v: &mut Vec<u8>, + _header: &super::Header, + stream: &super::Stream, + version: u8, +) -> Result<(), Error> { + let timescale = stream_to_timescale(stream); + + let entry_count_position = v.len(); + // Entry count, rewritten in the end + v.extend(0u32.to_be_bytes()); + + let mut last_composition_time_offset = None; + let mut sample_count = 0u32; + let mut num_entries = 0u32; + for composition_time_offset in stream + .chunks + .iter() + .flat_map(|c| c.samples.iter().map(|b| b.composition_time_offset)) + { + let composition_time_offset = composition_time_offset + .expect("not all samples have a composition time offset") + .mul_div_round(timescale as i64, gst::ClockTime::SECOND.nseconds() as i64) + .context("too big sample composition time offset")?; + + if last_composition_time_offset.map_or(true, |last_composition_time_offset| { + last_composition_time_offset != composition_time_offset + }) { + if let Some(last_composition_time_offset) = last_composition_time_offset { + v.extend(sample_count.to_be_bytes()); + if version == FULL_BOX_VERSION_0 { + let last_composition_time_offset = u32::try_from(last_composition_time_offset) + .context("too big sample composition time offset")?; + + v.extend(last_composition_time_offset.to_be_bytes()); + } else { + let last_composition_time_offset = i32::try_from(last_composition_time_offset) + .context("too big sample composition time offset")?; + v.extend(last_composition_time_offset.to_be_bytes()); + } + num_entries += 1; + } + + last_composition_time_offset = Some(composition_time_offset); + sample_count = 1; + } else { + sample_count += 1; + } + } + + if let Some(last_composition_time_offset) = last_composition_time_offset { + v.extend(sample_count.to_be_bytes()); + if version == FULL_BOX_VERSION_0 { + let last_composition_time_offset = u32::try_from(last_composition_time_offset) + .context("too big sample composition time offset")?; + + v.extend(last_composition_time_offset.to_be_bytes()); + } else { + let last_composition_time_offset = i32::try_from(last_composition_time_offset) + .context("too big sample composition time offset")?; + v.extend(last_composition_time_offset.to_be_bytes()); + } + num_entries += 1; + } + + // Rewrite entry count + v[entry_count_position..][..4].copy_from_slice(&num_entries.to_be_bytes()); + + Ok(()) +} + +fn write_cslg( + v: &mut Vec<u8>, + _header: &super::Header, + stream: &super::Stream, +) -> Result<(), Error> { + let timescale = stream_to_timescale(stream); + + let (min_ctts, max_ctts) = stream + .chunks + .iter() + .flat_map(|c| { + c.samples.iter().map(|b| { + b.composition_time_offset + .expect("not all samples have a composition time offset") + }) + }) + .fold((None, None), |(min, max), ctts| { + ( + if min.map_or(true, |min| ctts < min) { + Some(ctts) + } else { + min + }, + if max.map_or(true, |max| ctts > max) { + Some(ctts) + } else { + max + }, + ) + }); + let min_ctts = min_ctts + .unwrap() + .mul_div_round(timescale as i64, gst::ClockTime::SECOND.nseconds() as i64) + .context("too big composition time offset")?; + let max_ctts = max_ctts + .unwrap() + .mul_div_round(timescale as i64, gst::ClockTime::SECOND.nseconds() as i64) + .context("too big composition time offset")?; + + // Composition to DTS shift + v.extend((-min_ctts).to_be_bytes()); + + // least decode to display delta + v.extend(min_ctts.to_be_bytes()); + + // greatest decode to display delta + v.extend(max_ctts.to_be_bytes()); + + // composition start time + let composition_start_time = stream + .earliest_pts + .nseconds() + .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds() as u64) + .context("too earliest PTS")?; + v.extend(composition_start_time.to_be_bytes()); + + // composition end time + let composition_end_time = stream + .end_pts + .nseconds() + .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds() as u64) + .context("too end PTS")?; + v.extend(composition_end_time.to_be_bytes()); + + Ok(()) +} + +fn write_stss( + v: &mut Vec<u8>, + _header: &super::Header, + stream: &super::Stream, +) -> Result<(), Error> { + let entry_count_position = v.len(); + // Entry count, rewritten in the end + v.extend(0u32.to_be_bytes()); + + let mut num_entries = 0u32; + for (idx, _sync_point) in stream + .chunks + .iter() + .flat_map(|c| c.samples.iter().map(|b| b.sync_point)) + .enumerate() + .filter(|(_idx, sync_point)| *sync_point) + { + v.extend((idx as u32 + 1).to_be_bytes()); + num_entries += 1; + } + + // Rewrite entry count + v[entry_count_position..][..4].copy_from_slice(&num_entries.to_be_bytes()); + + Ok(()) +} + +fn write_stsz( + v: &mut Vec<u8>, + _header: &super::Header, + stream: &super::Stream, +) -> Result<(), Error> { + let first_sample_size = stream.chunks[0].samples[0].size; + + if stream + .chunks + .iter() + .flat_map(|c| c.samples.iter().map(|b| b.size)) + .all(|size| size == first_sample_size) + { + // Sample size + v.extend(first_sample_size.to_be_bytes()); + + // Sample count + let sample_count = stream + .chunks + .iter() + .map(|c| c.samples.len() as u32) + .sum::<u32>(); + v.extend(sample_count.to_be_bytes()); + } else { + // Sample size + v.extend(0u32.to_be_bytes()); + + // Sample count, will be rewritten later + let sample_count_position = v.len(); + let mut sample_count = 0u32; + v.extend(0u32.to_be_bytes()); + + for size in stream + .chunks + .iter() + .flat_map(|c| c.samples.iter().map(|b| b.size)) + { + v.extend(size.to_be_bytes()); + sample_count += 1; + } + + v[sample_count_position..][..4].copy_from_slice(&sample_count.to_be_bytes()); + } + + Ok(()) +} + +fn write_stsc( + v: &mut Vec<u8>, + _header: &super::Header, + stream: &super::Stream, +) -> Result<(), Error> { + let entry_count_position = v.len(); + // Entry count, rewritten in the end + v.extend(0u32.to_be_bytes()); + + let mut num_entries = 0u32; + let mut first_chunk = 1u32; + let mut samples_per_chunk: Option<u32> = None; + for (idx, chunk) in stream.chunks.iter().enumerate() { + if samples_per_chunk.map_or(true, |samples_per_chunk| { + samples_per_chunk != chunk.samples.len() as u32 + }) { + if let Some(samples_per_chunk) = samples_per_chunk { + v.extend(first_chunk.to_be_bytes()); + v.extend(samples_per_chunk.to_be_bytes()); + // sample description index + v.extend(1u32.to_be_bytes()); + num_entries += 1; + } + samples_per_chunk = Some(chunk.samples.len() as u32); + first_chunk = idx as u32 + 1; + } + } + + if let Some(samples_per_chunk) = samples_per_chunk { + v.extend(first_chunk.to_be_bytes()); + v.extend(samples_per_chunk.to_be_bytes()); + // sample description index + v.extend(1u32.to_be_bytes()); + num_entries += 1; + } + + // Rewrite entry count + v[entry_count_position..][..4].copy_from_slice(&num_entries.to_be_bytes()); + + Ok(()) +} + +fn write_stco( + v: &mut Vec<u8>, + _header: &super::Header, + stream: &super::Stream, + co64: bool, +) -> Result<(), Error> { + // Entry count + v.extend((stream.chunks.len() as u32).to_be_bytes()); + + for chunk in &stream.chunks { + if co64 { + v.extend(chunk.offset.to_be_bytes()); + } else { + v.extend(u32::try_from(chunk.offset).unwrap().to_be_bytes()); + } + } + + Ok(()) +} + +fn write_edts( + v: &mut Vec<u8>, + header: &super::Header, + stream: &super::Stream, +) -> Result<(), Error> { + write_full_box(v, b"elst", FULL_BOX_VERSION_1, 0, |v| { + write_elst(v, header, stream) + })?; + + Ok(()) +} + +fn write_elst( + v: &mut Vec<u8>, + header: &super::Header, + stream: &super::Stream, +) -> Result<(), Error> { + // In movie header timescale + let timescale = header_to_timescale(header); + + let min_earliest_pts = header.streams.iter().map(|s| s.earliest_pts).min().unwrap(); + + if min_earliest_pts != stream.earliest_pts { + // Entry count + v.extend(2u32.to_be_bytes()); + + // First entry for the gap + + // Edit duration + let gap = (stream.earliest_pts - min_earliest_pts) + .nseconds() + .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds()) + .context("too big gap")?; + v.extend(gap.to_be_bytes()); + + // Media time + v.extend((-1i64).to_be_bytes()); + + // Media rate + v.extend(1u16.to_be_bytes()); + v.extend(0u16.to_be_bytes()); + } else { + // Entry count + v.extend(1u32.to_be_bytes()); + } + + // Edit duration + let duration = (stream.end_pts - stream.earliest_pts) + .nseconds() + .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds()) + .context("too big track duration")?; + v.extend(duration.to_be_bytes()); + + // Media time + if let Some(gst::Signed::Negative(start_dts)) = stream.start_dts { + let shift = (stream.earliest_pts + start_dts) + .nseconds() + .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds()) + .context("too big track duration")?; + + v.extend(shift.to_be_bytes()); + } else { + v.extend(0u64.to_be_bytes()); + } + + // Media rate + v.extend(1u16.to_be_bytes()); + v.extend(0u16.to_be_bytes()); + + Ok(()) +} diff --git a/mux/mp4/src/mp4mux/imp.rs b/mux/mp4/src/mp4mux/imp.rs new file mode 100644 index 000000000..b8ec1d57e --- /dev/null +++ b/mux/mp4/src/mp4mux/imp.rs @@ -0,0 +1,1305 @@ +// Copyright (C) 2022 Sebastian Dröge <sebastian@centricular.com> +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// <https://mozilla.org/MPL/2.0/>. +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::glib; +use gst::prelude::*; +use gst::subclass::prelude::*; +use gst_base::prelude::*; +use gst_base::subclass::prelude::*; + +use std::sync::Mutex; + +use once_cell::sync::Lazy; + +use super::boxes; + +static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| { + gst::DebugCategory::new( + "mp4mux", + gst::DebugColorFlags::empty(), + Some("MP4Mux Element"), + ) +}); + +const DEFAULT_INTERLEAVE_BYTES: Option<u64> = None; +const DEFAULT_INTERLEAVE_TIME: Option<gst::ClockTime> = Some(gst::ClockTime::from_mseconds(500)); + +#[derive(Debug, Clone)] +struct Settings { + interleave_bytes: Option<u64>, + interleave_time: Option<gst::ClockTime>, + movie_timescale: u32, +} + +impl Default for Settings { + fn default() -> Self { + Settings { + interleave_bytes: DEFAULT_INTERLEAVE_BYTES, + interleave_time: DEFAULT_INTERLEAVE_TIME, + movie_timescale: 0, + } + } +} + +struct PendingBuffer { + buffer: gst::Buffer, + timestamp: gst::Signed<gst::ClockTime>, + pts: gst::ClockTime, + composition_time_offset: Option<i64>, + duration: Option<gst::ClockTime>, +} + +struct Stream { + /// Sink pad for this stream. + sinkpad: super::MP4MuxPad, + + /// Currently configured caps for this stream. + caps: gst::Caps, + /// Whether this stream is intra-only and has frame reordering. + delta_frames: super::DeltaFrames, + + /// Already written out chunks with their samples for this stream + chunks: Vec<super::Chunk>, + + /// Queued time in the latest chunk. + queued_chunk_time: gst::ClockTime, + /// Queue bytes in the latest chunk. + queued_chunk_bytes: u64, + + /// Currently pending buffer, DTS or PTS running time and duration + /// + /// If the duration is set then the next buffer is already queued up and the duration was + /// calculated based on that. + pending_buffer: Option<PendingBuffer>, + + /// Start DTS. + start_dts: Option<gst::Signed<gst::ClockTime>>, + + /// Earliest PTS. + earliest_pts: Option<gst::ClockTime>, + /// Current end PTS. + end_pts: Option<gst::ClockTime>, +} + +#[derive(Default)] +struct State { + /// List of streams when the muxer was started. + streams: Vec<Stream>, + + /// Index of stream that is currently selected to fill a chunk. + current_stream_idx: Option<usize>, + + /// Current writing offset since the beginning of the stream. + current_offset: u64, + + /// Offset of the `mdat` box from the beginning of the stream. + mdat_offset: Option<u64>, + + /// Size of the `mdat` as written so far. + mdat_size: u64, +} + +#[derive(Default)] +pub(crate) struct MP4Mux { + state: Mutex<State>, + settings: Mutex<Settings>, +} + +impl MP4Mux { + /// Queue a buffer and calculate its duration. + /// + /// Returns `Ok(())` if a buffer with duration is known or if the stream is EOS and a buffer is + /// queued, i.e. if this stream is ready to be processed. + /// + /// Returns `Err(Eos)` if nothing is queued and the stream is EOS. + /// + /// Returns `Err(AGGREGATOR_FLOW_NEED_DATA)` if more data is needed. + /// + /// Returns `Err(Error)` on errors. + fn queue_buffer(&self, stream: &mut Stream) -> Result<(), gst::FlowError> { + // Loop up to two times here to first retrieve the current buffer and then potentially + // already calculate its duration based on the next queued buffer. + loop { + match stream.pending_buffer { + Some(PendingBuffer { + duration: Some(_), .. + }) => return Ok(()), + Some(PendingBuffer { + timestamp, + pts, + ref buffer, + ref mut duration, + .. + }) => { + // Already have a pending buffer but no duration, so try to get that now + let buffer = match stream.sinkpad.peek_buffer() { + Some(buffer) => buffer, + None => { + if stream.sinkpad.is_eos() { + let dur = buffer.duration().unwrap_or(gst::ClockTime::ZERO); + gst::trace!( + CAT, + obj: stream.sinkpad, + "Stream is EOS, using {dur} as duration for queued buffer", + ); + + let pts = pts + dur; + if stream.end_pts.map_or(true, |end_pts| end_pts < pts) { + gst::trace!(CAT, obj: stream.sinkpad, "Stream end PTS {pts}"); + stream.end_pts = Some(pts); + } + + *duration = Some(dur); + + return Ok(()); + } else { + gst::trace!(CAT, obj: stream.sinkpad, "Stream has no buffer queued"); + return Err(gst_base::AGGREGATOR_FLOW_NEED_DATA); + } + } + }; + + if stream.delta_frames.requires_dts() && buffer.dts().is_none() { + gst::error!(CAT, obj: stream.sinkpad, "Require DTS for video streams"); + return Err(gst::FlowError::Error); + } + + if stream.delta_frames.intra_only() + && buffer.flags().contains(gst::BufferFlags::DELTA_UNIT) + { + gst::error!(CAT, obj: stream.sinkpad, "Intra-only stream with delta units"); + return Err(gst::FlowError::Error); + } + + let pts_position = buffer.pts().ok_or_else(|| { + gst::error!(CAT, obj: stream.sinkpad, "Require timestamped buffers"); + gst::FlowError::Error + })?; + + let next_timestamp_position = if stream.delta_frames.requires_dts() { + // Was checked above + buffer.dts().unwrap() + } else { + pts_position + }; + + let segment = match stream.sinkpad.segment().downcast::<gst::ClockTime>().ok() { + Some(segment) => segment, + None => { + gst::error!(CAT, obj: stream.sinkpad, "Got buffer before segment"); + return Err(gst::FlowError::Error); + } + }; + + // If the stream has no valid running time, assume it's before everything else. + let next_timestamp = match segment.to_running_time_full(next_timestamp_position) + { + None => { + gst::error!(CAT, obj: stream.sinkpad, "Stream has no valid running time"); + return Err(gst::FlowError::Error); + } + Some(running_time) => running_time, + }; + + gst::trace!( + CAT, + obj: stream.sinkpad, + "Stream has buffer with timestamp {next_timestamp} queued", + ); + + let dur = next_timestamp + .saturating_sub(timestamp) + .positive() + .unwrap_or_else(|| { + gst::warning!( + CAT, + obj: stream.sinkpad, + "Stream timestamps going backwards {next_timestamp} < {timestamp}", + ); + gst::ClockTime::ZERO + }); + + gst::trace!( + CAT, + obj: stream.sinkpad, + "Using {dur} as duration for queued buffer", + ); + + let pts = pts + dur; + if stream.end_pts.map_or(true, |end_pts| end_pts < pts) { + gst::trace!(CAT, obj: stream.sinkpad, "Stream end PTS {pts}"); + stream.end_pts = Some(pts); + } + + *duration = Some(dur); + + return Ok(()); + } + None => { + // Have no buffer queued at all yet + + let buffer = match stream.sinkpad.pop_buffer() { + Some(buffer) => buffer, + None => { + if stream.sinkpad.is_eos() { + gst::trace!( + CAT, + obj: stream.sinkpad, + "Stream is EOS", + ); + + return Err(gst::FlowError::Eos); + } else { + gst::trace!(CAT, obj: stream.sinkpad, "Stream has no buffer queued"); + return Err(gst_base::AGGREGATOR_FLOW_NEED_DATA); + } + } + }; + + if stream.delta_frames.requires_dts() && buffer.dts().is_none() { + gst::error!(CAT, obj: stream.sinkpad, "Require DTS for video streams"); + return Err(gst::FlowError::Error); + } + + if stream.delta_frames.intra_only() + && buffer.flags().contains(gst::BufferFlags::DELTA_UNIT) + { + gst::error!(CAT, obj: stream.sinkpad, "Intra-only stream with delta units"); + return Err(gst::FlowError::Error); + } + + let pts_position = buffer.pts().ok_or_else(|| { + gst::error!(CAT, obj: stream.sinkpad, "Require timestamped buffers"); + gst::FlowError::Error + })?; + let dts_position = buffer.dts(); + + let segment = match stream + .sinkpad + .segment() + .clone() + .downcast::<gst::ClockTime>() + .ok() + { + Some(segment) => segment, + None => { + gst::error!(CAT, obj: stream.sinkpad, "Got buffer before segment"); + return Err(gst::FlowError::Error); + } + }; + + let pts = match segment.to_running_time_full(pts_position) { + None => { + gst::error!(CAT, obj: stream.sinkpad, "Stream has no valid PTS running time"); + return Err(gst::FlowError::Error); + } + Some(running_time) => running_time, + }.positive().unwrap_or_else(|| { + gst::error!(CAT, obj: stream.sinkpad, "Stream has negative PTS running time"); + gst::ClockTime::ZERO + }); + + let dts = match dts_position { + None => None, + Some(dts_position) => match segment.to_running_time_full(dts_position) { + None => { + gst::error!(CAT, obj: stream.sinkpad, "Stream has no valid DTS running time"); + return Err(gst::FlowError::Error); + } + Some(running_time) => Some(running_time), + }, + }; + + let timestamp = if stream.delta_frames.requires_dts() { + // Was checked above + let dts = dts.unwrap(); + + if stream.start_dts.is_none() { + gst::debug!(CAT, obj: stream.sinkpad, "Stream start DTS {dts}"); + stream.start_dts = Some(dts); + } + + dts + } else { + gst::Signed::Positive(pts) + }; + + if stream + .earliest_pts + .map_or(true, |earliest_pts| earliest_pts > pts) + { + gst::debug!(CAT, obj: stream.sinkpad, "Stream earliest PTS {pts}"); + stream.earliest_pts = Some(pts); + } + + let composition_time_offset = if stream.delta_frames.requires_dts() { + let pts = gst::Signed::Positive(pts); + let dts = dts.unwrap(); // set above + + if pts > dts { + Some(i64::try_from((pts - dts).nseconds().positive().unwrap()).map_err(|_| { + gst::error!(CAT, obj: stream.sinkpad, "Too big PTS/DTS difference"); + gst::FlowError::Error + })?) + } else { + let diff = i64::try_from((dts - pts).nseconds().positive().unwrap()).map_err(|_| { + gst::error!(CAT, obj: stream.sinkpad, "Too big PTS/DTS difference"); + gst::FlowError::Error + })?; + Some(-diff) + } + } else { + None + }; + + gst::trace!( + CAT, + obj: stream.sinkpad, + "Stream has buffer of size {} with timestamp {timestamp} pending", + buffer.size(), + ); + + stream.pending_buffer = Some(PendingBuffer { + buffer, + timestamp, + pts, + composition_time_offset, + duration: None, + }); + } + } + } + } + + fn find_earliest_stream( + &self, + settings: &Settings, + state: &mut State, + ) -> Result<Option<usize>, gst::FlowError> { + if let Some(current_stream_idx) = state.current_stream_idx { + // If a stream was previously selected, check if another buffer from + // this stream can be consumed or if that would exceed the interleave. + + let single_stream = state.streams.len() == 1; + let stream = &mut state.streams[current_stream_idx]; + + match self.queue_buffer(stream) { + Ok(_) => { + assert!(matches!( + stream.pending_buffer, + Some(PendingBuffer { + duration: Some(_), + .. + }) + )); + + if single_stream + || (settings.interleave_bytes.map_or(true, |interleave_bytes| { + interleave_bytes >= stream.queued_chunk_bytes + }) && settings.interleave_time.map_or(true, |interleave_time| { + interleave_time >= stream.queued_chunk_time + })) + { + gst::trace!(CAT, + obj: stream.sinkpad, + "Continuing current chunk: single stream {}, or {} >= {} and {} >= {}", + single_stream, + gst::format::Bytes::from_u64(stream.queued_chunk_bytes), + settings.interleave_bytes.map(gst::format::Bytes::from_u64).display(), + stream.queued_chunk_time, settings.interleave_time.display(), + ); + return Ok(Some(current_stream_idx)); + } + + state.current_stream_idx = None; + gst::debug!(CAT, + obj: stream.sinkpad, + "Switching to next chunk: {} < {} and {} < {}", + gst::format::Bytes::from_u64(stream.queued_chunk_bytes), + settings.interleave_bytes.map(gst::format::Bytes::from_u64).display(), + stream.queued_chunk_time, settings.interleave_time.display(), + ); + } + Err(gst::FlowError::Eos) => { + gst::debug!(CAT, obj: stream.sinkpad, "Stream is EOS, switching to next stream"); + state.current_stream_idx = None; + } + Err(err) => { + return Err(err); + } + } + } + + // Otherwise find the next earliest stream here + let mut earliest_stream = None; + let mut all_have_data_or_eos = true; + let mut all_eos = true; + + for (idx, stream) in state.streams.iter_mut().enumerate() { + // First queue a buffer on each stream and try to get the duration + + match self.queue_buffer(stream) { + Ok(_) => { + assert!(matches!( + stream.pending_buffer, + Some(PendingBuffer { + duration: Some(_), + .. + }) + )); + + let timestamp = stream.pending_buffer.as_ref().unwrap().timestamp; + + gst::trace!(CAT, + obj: stream.sinkpad, + "Stream at timestamp {timestamp}", + ); + + all_eos = false; + + if earliest_stream + .as_ref() + .map_or(true, |(_idx, _stream, earliest_timestamp)| { + *earliest_timestamp > timestamp + }) + { + earliest_stream = Some((idx, stream, timestamp)); + } + } + Err(gst::FlowError::Eos) => { + all_eos &= true; + continue; + } + Err(gst_base::AGGREGATOR_FLOW_NEED_DATA) => { + all_have_data_or_eos = false; + continue; + } + Err(err) => { + return Err(err); + } + } + } + + if !all_have_data_or_eos { + gst::trace!(CAT, imp: self, "Not all streams have a buffer or are EOS"); + Err(gst_base::AGGREGATOR_FLOW_NEED_DATA) + } else if all_eos { + gst::info!(CAT, imp: self, "All streams are EOS"); + Err(gst::FlowError::Eos) + } else if let Some((idx, stream, earliest_timestamp)) = earliest_stream { + gst::debug!( + CAT, + obj: stream.sinkpad, + "Stream is earliest stream with timestamp {earliest_timestamp}", + ); + + gst::debug!( + CAT, + obj: stream.sinkpad, + "Starting new chunk at offset {}", + state.current_offset, + ); + + stream.chunks.push(super::Chunk { + offset: state.current_offset, + samples: Vec::new(), + }); + stream.queued_chunk_time = gst::ClockTime::ZERO; + stream.queued_chunk_bytes = 0; + + state.current_stream_idx = Some(idx); + Ok(Some(idx)) + } else { + unreachable!() + } + } + + fn drain_buffers( + &self, + settings: &Settings, + state: &mut State, + buffers: &mut gst::BufferListRef, + ) -> Result<(), gst::FlowError> { + // Now we can start handling buffers + while let Some(idx) = self.find_earliest_stream(settings, state)? { + let stream = &mut state.streams[idx]; + + let buffer = stream.pending_buffer.take().unwrap(); + let duration = buffer.duration.unwrap(); + let composition_time_offset = buffer.composition_time_offset; + let mut buffer = buffer.buffer; + + stream.queued_chunk_time += duration; + stream.queued_chunk_bytes += buffer.size() as u64; + + stream + .chunks + .last_mut() + .unwrap() + .samples + .push(super::Sample { + sync_point: !buffer.flags().contains(gst::BufferFlags::DELTA_UNIT), + duration, + composition_time_offset, + size: buffer.size() as u32, + }); + + { + let buffer = buffer.make_mut(); + buffer.set_dts(None); + buffer.set_pts(None); + buffer.set_duration(duration); + buffer.unset_flags(gst::BufferFlags::all()); + } + + state.current_offset += buffer.size() as u64; + state.mdat_size += buffer.size() as u64; + buffers.add(buffer); + } + + Ok(()) + } + + fn create_streams(&self, state: &mut State) -> Result<(), gst::FlowError> { + gst::info!(CAT, imp: self, "Creating streams"); + + for pad in self + .obj() + .sink_pads() + .into_iter() + .map(|pad| pad.downcast::<super::MP4MuxPad>().unwrap()) + { + let caps = match pad.current_caps() { + Some(caps) => caps, + None => { + gst::warning!(CAT, obj: pad, "Skipping pad without caps"); + continue; + } + }; + + gst::info!(CAT, obj: pad, "Configuring caps {:?}", caps); + + let s = caps.structure(0).unwrap(); + + let mut delta_frames = super::DeltaFrames::IntraOnly; + match s.name() { + "video/x-h264" | "video/x-h265" => { + if !s.has_field_with_type("codec_data", gst::Buffer::static_type()) { + gst::error!(CAT, obj: pad, "Received caps without codec_data"); + return Err(gst::FlowError::NotNegotiated); + } + delta_frames = super::DeltaFrames::Bidirectional; + } + "video/x-vp9" => { + if !s.has_field_with_type("colorimetry", str::static_type()) { + gst::error!(CAT, obj: pad, "Received caps without colorimetry"); + return Err(gst::FlowError::NotNegotiated); + } + delta_frames = super::DeltaFrames::PredictiveOnly; + } + "image/jpeg" => (), + "audio/mpeg" => { + if !s.has_field_with_type("codec_data", gst::Buffer::static_type()) { + gst::error!(CAT, obj: pad, "Received caps without codec_data"); + return Err(gst::FlowError::NotNegotiated); + } + } + "audio/x-opus" => { + if let Some(header) = s + .get::<gst::ArrayRef>("streamheader") + .ok() + .and_then(|a| a.get(0).and_then(|v| v.get::<gst::Buffer>().ok())) + { + if gst_pbutils::codec_utils_opus_parse_header(&header, None).is_err() { + gst::error!(CAT, obj: pad, "Received invalid Opus header"); + return Err(gst::FlowError::NotNegotiated); + } + } else if gst_pbutils::codec_utils_opus_parse_caps(&caps, None).is_err() { + gst::error!(CAT, obj: pad, "Received invalid Opus caps"); + return Err(gst::FlowError::NotNegotiated); + } + } + "audio/x-alaw" | "audio/x-mulaw" => (), + "audio/x-adpcm" => (), + "application/x-onvif-metadata" => (), + _ => unreachable!(), + } + + state.streams.push(Stream { + sinkpad: pad, + caps, + delta_frames, + chunks: Vec::new(), + pending_buffer: None, + queued_chunk_time: gst::ClockTime::ZERO, + queued_chunk_bytes: 0, + start_dts: None, + earliest_pts: None, + end_pts: None, + }); + } + + if state.streams.is_empty() { + gst::error!(CAT, imp: self, "No streams available"); + return Err(gst::FlowError::Error); + } + + // Sort video streams first and then audio streams and then metadata streams, and each group by pad name. + state.streams.sort_by(|a, b| { + let order_of_caps = |caps: &gst::CapsRef| { + let s = caps.structure(0).unwrap(); + + if s.name().starts_with("video/") { + 0 + } else if s.name().starts_with("audio/") { + 1 + } else if s.name().starts_with("application/x-onvif-metadata") { + 2 + } else { + unimplemented!(); + } + }; + + let st_a = order_of_caps(&a.caps); + let st_b = order_of_caps(&b.caps); + + if st_a == st_b { + return a.sinkpad.name().cmp(&b.sinkpad.name()); + } + + st_a.cmp(&st_b) + }); + + Ok(()) + } +} + +#[glib::object_subclass] +impl ObjectSubclass for MP4Mux { + const NAME: &'static str = "GstRsMP4Mux"; + type Type = super::MP4Mux; + type ParentType = gst_base::Aggregator; + type Class = Class; +} + +impl ObjectImpl for MP4Mux { + fn properties() -> &'static [glib::ParamSpec] { + static PROPERTIES: Lazy<Vec<glib::ParamSpec>> = Lazy::new(|| { + vec![ + glib::ParamSpecUInt64::builder("interleave-bytes") + .nick("Interleave Bytes") + .blurb("Interleave between streams in bytes") + .default_value(DEFAULT_INTERLEAVE_BYTES.unwrap_or(0)) + .mutable_ready() + .build(), + glib::ParamSpecUInt64::builder("interleave-time") + .nick("Interleave Time") + .blurb("Interleave between streams in nanoseconds") + .default_value( + DEFAULT_INTERLEAVE_TIME + .map(gst::ClockTime::nseconds) + .unwrap_or(u64::MAX), + ) + .mutable_ready() + .build(), + glib::ParamSpecUInt::builder("movie-timescale") + .nick("Movie Timescale") + .blurb("Timescale to use for the movie (units per second, 0 is automatic)") + .mutable_ready() + .build(), + ] + }); + + &PROPERTIES + } + + fn set_property(&self, _id: usize, value: &glib::Value, pspec: &glib::ParamSpec) { + match pspec.name() { + "interleave-bytes" => { + let mut settings = self.settings.lock().unwrap(); + settings.interleave_bytes = match value.get().expect("type checked upstream") { + 0 => None, + v => Some(v), + }; + } + + "interleave-time" => { + let mut settings = self.settings.lock().unwrap(); + settings.interleave_time = match value.get().expect("type checked upstream") { + Some(gst::ClockTime::ZERO) | None => None, + v => v, + }; + } + + "movie-timescale" => { + let mut settings = self.settings.lock().unwrap(); + settings.movie_timescale = value.get().expect("type checked upstream"); + } + + _ => unimplemented!(), + } + } + + fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { + match pspec.name() { + "interleave-bytes" => { + let settings = self.settings.lock().unwrap(); + settings.interleave_bytes.unwrap_or(0).to_value() + } + + "interleave-time" => { + let settings = self.settings.lock().unwrap(); + settings.interleave_time.to_value() + } + + "movie-timescale" => { + let settings = self.settings.lock().unwrap(); + settings.movie_timescale.to_value() + } + + _ => unimplemented!(), + } + } +} + +impl GstObjectImpl for MP4Mux {} + +impl ElementImpl for MP4Mux { + fn request_new_pad( + &self, + templ: &gst::PadTemplate, + name: Option<&str>, + caps: Option<&gst::Caps>, + ) -> Option<gst::Pad> { + let state = self.state.lock().unwrap(); + if !state.streams.is_empty() { + gst::error!( + CAT, + imp: self, + "Can't request new pads after start was generated" + ); + return None; + } + + self.parent_request_new_pad(templ, name, caps) + } +} + +impl AggregatorImpl for MP4Mux { + fn next_time(&self) -> Option<gst::ClockTime> { + None + } + + fn sink_query( + &self, + aggregator_pad: &gst_base::AggregatorPad, + query: &mut gst::QueryRef, + ) -> bool { + use gst::QueryViewMut; + + gst::trace!(CAT, obj: aggregator_pad, "Handling query {:?}", query); + + match query.view_mut() { + QueryViewMut::Caps(q) => { + let allowed_caps = aggregator_pad + .current_caps() + .unwrap_or_else(|| aggregator_pad.pad_template_caps()); + + if let Some(filter_caps) = q.filter() { + let res = filter_caps + .intersect_with_mode(&allowed_caps, gst::CapsIntersectMode::First); + q.set_result(&res); + } else { + q.set_result(&allowed_caps); + } + + true + } + _ => self.parent_sink_query(aggregator_pad, query), + } + } + + fn sink_event_pre_queue( + &self, + aggregator_pad: &gst_base::AggregatorPad, + mut event: gst::Event, + ) -> Result<gst::FlowSuccess, gst::FlowError> { + use gst::EventView; + + gst::trace!(CAT, obj: aggregator_pad, "Handling event {:?}", event); + + match event.view() { + EventView::Segment(ev) => { + if ev.segment().format() != gst::Format::Time { + gst::warning!( + CAT, + obj: aggregator_pad, + "Received non-TIME segment, replacing with default TIME segment" + ); + let segment = gst::FormattedSegment::<gst::ClockTime>::new(); + event = gst::event::Segment::builder(&segment) + .seqnum(event.seqnum()) + .build(); + } + self.parent_sink_event_pre_queue(aggregator_pad, event) + } + _ => self.parent_sink_event_pre_queue(aggregator_pad, event), + } + } + + fn sink_event(&self, aggregator_pad: &gst_base::AggregatorPad, event: gst::Event) -> bool { + use gst::EventView; + + gst::trace!(CAT, obj: aggregator_pad, "Handling event {:?}", event); + + match event.view() { + EventView::Tag(_ev) => { + // TODO: Maybe store for putting into the header at the end? + + self.parent_sink_event(aggregator_pad, event) + } + _ => self.parent_sink_event(aggregator_pad, event), + } + } + + fn src_query(&self, query: &mut gst::QueryRef) -> bool { + use gst::QueryViewMut; + + gst::trace!(CAT, imp: self, "Handling query {:?}", query); + + match query.view_mut() { + QueryViewMut::Seeking(q) => { + // We can't really handle seeking, it would break everything + q.set(false, gst::ClockTime::ZERO, gst::ClockTime::NONE); + true + } + _ => self.parent_src_query(query), + } + } + + fn src_event(&self, event: gst::Event) -> bool { + use gst::EventView; + + gst::trace!(CAT, imp: self, "Handling event {:?}", event); + + match event.view() { + EventView::Seek(_ev) => false, + _ => self.parent_src_event(event), + } + } + + fn flush(&self) -> Result<gst::FlowSuccess, gst::FlowError> { + let mut state = self.state.lock().unwrap(); + for stream in &mut state.streams { + stream.pending_buffer = None; + } + drop(state); + + self.parent_flush() + } + + fn stop(&self) -> Result<(), gst::ErrorMessage> { + gst::trace!(CAT, imp: self, "Stopping"); + + let _ = self.parent_stop(); + + *self.state.lock().unwrap() = State::default(); + + Ok(()) + } + + fn start(&self) -> Result<(), gst::ErrorMessage> { + gst::trace!(CAT, imp: self, "Starting"); + + self.parent_start()?; + + // Always output a BYTES segment + let segment = gst::FormattedSegment::<gst::format::Bytes>::new(); + self.obj().update_segment(&segment); + + *self.state.lock().unwrap() = State::default(); + + Ok(()) + } + + fn negotiate(&self) -> bool { + true + } + + fn aggregate(&self, _timeout: bool) -> Result<gst::FlowSuccess, gst::FlowError> { + let settings = self.settings.lock().unwrap().clone(); + let mut state = self.state.lock().unwrap(); + + let mut buffers = gst::BufferList::new(); + let mut caps = None; + + // If no streams were created yet, collect all streams now and write the mdat. + if state.streams.is_empty() { + // First check if downstream is seekable. If not we can't rewrite the mdat box header! + drop(state); + + let mut q = gst::query::Seeking::new(gst::Format::Bytes); + if self.obj().src_pad().peer_query(&mut q) { + if !q.result().0 { + gst::element_imp_error!( + self, + gst::StreamError::Mux, + ["Downstream is not seekable"] + ); + return Err(gst::FlowError::Error); + } + } else { + // Can't query downstream, have to assume downstream is seekable + gst::warning!(CAT, imp: self, "Can't query downstream for seekability"); + } + + state = self.state.lock().unwrap(); + self.create_streams(&mut state)?; + + // Create caps now to be sent before any buffers + caps = Some( + gst::Caps::builder("video/quicktime") + .field("variant", "iso") + .build(), + ); + + gst::info!( + CAT, + imp: self, + "Creating ftyp box at offset {}", + state.current_offset + ); + + // ... and then create the ftyp box plus mdat box header so we can start outputting + // actual data + let buffers = buffers.get_mut().unwrap(); + + let ftyp = boxes::create_ftyp(self.obj().class().as_ref().variant).map_err(|err| { + gst::error!(CAT, imp: self, "Failed to create ftyp box: {err}"); + gst::FlowError::Error + })?; + state.current_offset += ftyp.size() as u64; + buffers.add(ftyp); + + gst::info!( + CAT, + imp: self, + "Creating mdat box header at offset {}", + state.current_offset + ); + state.mdat_offset = Some(state.current_offset); + let mdat = boxes::create_mdat_header(None).map_err(|err| { + gst::error!(CAT, imp: self, "Failed to create mdat box header: {err}"); + gst::FlowError::Error + })?; + state.current_offset += mdat.size() as u64; + state.mdat_size = 0; + buffers.add(mdat); + } + + let res = match self.drain_buffers(&settings, &mut state, buffers.get_mut().unwrap()) { + Ok(_) => Ok(gst::FlowSuccess::Ok), + Err(err @ gst::FlowError::Eos) | Err(err @ gst_base::AGGREGATOR_FLOW_NEED_DATA) => { + Err(err) + } + Err(err) => return Err(err), + }; + + if res == Err(gst::FlowError::Eos) { + // Create moov box now and append it to the buffers + + gst::info!( + CAT, + imp: self, + "Creating moov box now, mdat ends at offset {} with size {}", + state.current_offset, + state.mdat_size + ); + + let mut streams = Vec::with_capacity(state.streams.len()); + for stream in state.streams.drain(..) { + let pad_settings = stream.sinkpad.imp().settings.lock().unwrap().clone(); + let (earliest_pts, end_pts) = match Option::zip(stream.earliest_pts, stream.end_pts) + { + Some(res) => res, + None => continue, // empty stream + }; + + streams.push(super::Stream { + caps: stream.caps.clone(), + delta_frames: stream.delta_frames, + trak_timescale: pad_settings.trak_timescale, + start_dts: stream.start_dts, + earliest_pts, + end_pts, + chunks: stream.chunks, + }); + } + + let moov = boxes::create_moov(super::Header { + variant: self.obj().class().as_ref().variant, + movie_timescale: settings.movie_timescale, + streams, + }) + .map_err(|err| { + gst::error!(CAT, imp: self, "Failed to create moov box: {err}"); + gst::FlowError::Error + })?; + state.current_offset += moov.size() as u64; + buffers.get_mut().unwrap().add(moov); + } + + drop(state); + + if let Some(ref caps) = caps { + self.obj().set_src_caps(caps); + } + + if !buffers.is_empty() { + if let Err(err) = self.obj().finish_buffer_list(buffers) { + gst::error!(CAT, imp: self, "Failed pushing buffer: {:?}", err); + return Err(err); + } + } + + if res == Err(gst::FlowError::Eos) { + let mut state = self.state.lock().unwrap(); + + if let Some(mdat_offset) = state.mdat_offset { + gst::info!( + CAT, + imp: self, + "Rewriting mdat box header at offset {mdat_offset} with size {} now", + state.mdat_size, + ); + let mut segment = gst::FormattedSegment::<gst::format::Bytes>::new(); + segment.set_start(gst::format::Bytes::from_u64(mdat_offset)); + state.current_offset = mdat_offset; + let mdat = boxes::create_mdat_header(Some(state.mdat_size)).map_err(|err| { + gst::error!(CAT, imp: self, "Failed to create mdat box header: {err}"); + gst::FlowError::Error + })?; + drop(state); + + self.obj().update_segment(&segment); + if let Err(err) = self.obj().finish_buffer(mdat) { + gst::error!( + CAT, + imp: self, + "Failed pushing updated mdat box header buffer downstream: {:?}", + err, + ); + } + } + } + + res + } +} + +#[repr(C)] +pub(crate) struct Class { + parent: gst_base::ffi::GstAggregatorClass, + variant: super::Variant, +} + +unsafe impl ClassStruct for Class { + type Type = MP4Mux; +} + +impl std::ops::Deref for Class { + type Target = glib::Class<gst_base::Aggregator>; + + fn deref(&self) -> &Self::Target { + unsafe { &*(&self.parent as *const _ as *const _) } + } +} + +unsafe impl<T: MP4MuxImpl> IsSubclassable<T> for super::MP4Mux { + fn class_init(class: &mut glib::Class<Self>) { + Self::parent_class_init::<T>(class); + + let class = class.as_mut(); + class.variant = T::VARIANT; + } +} + +pub(crate) trait MP4MuxImpl: AggregatorImpl { + const VARIANT: super::Variant; +} + +#[derive(Default)] +pub(crate) struct ISOMP4Mux; + +#[glib::object_subclass] +impl ObjectSubclass for ISOMP4Mux { + const NAME: &'static str = "GstISOMP4Mux"; + type Type = super::ISOMP4Mux; + type ParentType = super::MP4Mux; +} + +impl ObjectImpl for ISOMP4Mux {} + +impl GstObjectImpl for ISOMP4Mux {} + +impl ElementImpl for ISOMP4Mux { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static ELEMENT_METADATA: Lazy<gst::subclass::ElementMetadata> = Lazy::new(|| { + gst::subclass::ElementMetadata::new( + "ISOMP4Mux", + "Codec/Muxer", + "ISO MP4 muxer", + "Sebastian Dröge <sebastian@centricular.com>", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static PAD_TEMPLATES: Lazy<Vec<gst::PadTemplate>> = Lazy::new(|| { + let src_pad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &gst::Caps::builder("video/quicktime") + .field("variant", "iso") + .build(), + ) + .unwrap(); + + let sink_pad_template = gst::PadTemplate::with_gtype( + "sink_%u", + gst::PadDirection::Sink, + gst::PadPresence::Request, + &[ + gst::Structure::builder("video/x-h264") + .field("stream-format", gst::List::new(["avc", "avc3"])) + .field("alignment", "au") + .field("width", gst::IntRange::new(1, u16::MAX as i32)) + .field("height", gst::IntRange::new(1, u16::MAX as i32)) + .build(), + gst::Structure::builder("video/x-h265") + .field("stream-format", gst::List::new(["hvc1", "hev1"])) + .field("alignment", "au") + .field("width", gst::IntRange::new(1, u16::MAX as i32)) + .field("height", gst::IntRange::new(1, u16::MAX as i32)) + .build(), + gst::Structure::builder("video/x-vp9") + .field("profile", gst::List::new(["0", "1", "2", "3"])) + .field("chroma-format", gst::List::new(["4:2:0", "4:2:2", "4:4:4"])) + .field("bit-depth-luma", gst::List::new([8u32, 10u32, 12u32])) + .field("bit-depth-chroma", gst::List::new([8u32, 10u32, 12u32])) + .field("width", gst::IntRange::new(1, u16::MAX as i32)) + .field("height", gst::IntRange::new(1, u16::MAX as i32)) + .build(), + gst::Structure::builder("audio/mpeg") + .field("mpegversion", 4i32) + .field("stream-format", "raw") + .field("channels", gst::IntRange::new(1, u16::MAX as i32)) + .field("rate", gst::IntRange::new(1, i32::MAX)) + .build(), + gst::Structure::builder("audio/x-opus") + .field("channel-mapping-family", gst::IntRange::new(0i32, 255)) + .field("channels", gst::IntRange::new(1i32, 8)) + .field("rate", gst::IntRange::new(1, i32::MAX)) + .build(), + ] + .into_iter() + .collect::<gst::Caps>(), + super::MP4MuxPad::static_type(), + ) + .unwrap(); + + vec![src_pad_template, sink_pad_template] + }); + + PAD_TEMPLATES.as_ref() + } +} + +impl AggregatorImpl for ISOMP4Mux {} + +impl MP4MuxImpl for ISOMP4Mux { + const VARIANT: super::Variant = super::Variant::ISO; +} + +#[derive(Default, Clone)] +struct PadSettings { + trak_timescale: u32, +} + +#[derive(Default)] +pub(crate) struct MP4MuxPad { + settings: Mutex<PadSettings>, +} + +#[glib::object_subclass] +impl ObjectSubclass for MP4MuxPad { + const NAME: &'static str = "GstRsMP4MuxPad"; + type Type = super::MP4MuxPad; + type ParentType = gst_base::AggregatorPad; +} + +impl ObjectImpl for MP4MuxPad { + fn properties() -> &'static [glib::ParamSpec] { + static PROPERTIES: Lazy<Vec<glib::ParamSpec>> = Lazy::new(|| { + vec![glib::ParamSpecUInt::builder("trak-timescale") + .nick("Track Timescale") + .blurb("Timescale to use for the track (units per second, 0 is automatic)") + .mutable_ready() + .build()] + }); + + &PROPERTIES + } + + fn set_property(&self, _id: usize, value: &glib::Value, pspec: &glib::ParamSpec) { + match pspec.name() { + "trak-timescale" => { + let mut settings = self.settings.lock().unwrap(); + settings.trak_timescale = value.get().expect("type checked upstream"); + } + + _ => unimplemented!(), + } + } + + fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { + match pspec.name() { + "trak-timescale" => { + let settings = self.settings.lock().unwrap(); + settings.trak_timescale.to_value() + } + + _ => unimplemented!(), + } + } +} + +impl GstObjectImpl for MP4MuxPad {} + +impl PadImpl for MP4MuxPad {} + +impl AggregatorPadImpl for MP4MuxPad { + fn flush(&self, aggregator: &gst_base::Aggregator) -> Result<gst::FlowSuccess, gst::FlowError> { + let mux = aggregator.downcast_ref::<super::MP4Mux>().unwrap(); + let mut mux_state = mux.imp().state.lock().unwrap(); + + for stream in &mut mux_state.streams { + if stream.sinkpad == *self.obj() { + stream.pending_buffer = None; + break; + } + } + + drop(mux_state); + + self.parent_flush(aggregator) + } +} diff --git a/mux/mp4/src/mp4mux/mod.rs b/mux/mp4/src/mp4mux/mod.rs new file mode 100644 index 000000000..1e6ad28be --- /dev/null +++ b/mux/mp4/src/mp4mux/mod.rs @@ -0,0 +1,134 @@ +// Copyright (C) 2022 Sebastian Dröge <sebastian@centricular.com> +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// <https://mozilla.org/MPL/2.0/>. +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::glib; +use gst::prelude::*; + +mod boxes; +mod imp; + +glib::wrapper! { + pub(crate) struct MP4MuxPad(ObjectSubclass<imp::MP4MuxPad>) @extends gst_base::AggregatorPad, gst::Pad, gst::Object; +} + +glib::wrapper! { + pub(crate) struct MP4Mux(ObjectSubclass<imp::MP4Mux>) @extends gst_base::Aggregator, gst::Element, gst::Object; +} + +glib::wrapper! { + pub(crate) struct ISOMP4Mux(ObjectSubclass<imp::ISOMP4Mux>) @extends MP4Mux, gst_base::Aggregator, gst::Element, gst::Object; +} + +pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + #[cfg(feature = "doc")] + { + MP4Mux::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty()); + MP4MuxPad::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty()); + } + gst::Element::register( + Some(plugin), + "isomp4mux", + gst::Rank::Marginal, + ISOMP4Mux::static_type(), + )?; + + Ok(()) +} + +#[derive(Debug, Copy, Clone)] +pub(crate) enum DeltaFrames { + /// Only single completely decodable frames + IntraOnly, + /// Frames may depend on past frames + PredictiveOnly, + /// Frames may depend on past or future frames + Bidirectional, +} + +impl DeltaFrames { + /// Whether dts is required to order samples differently from presentation order + pub(crate) fn requires_dts(&self) -> bool { + matches!(self, Self::Bidirectional) + } + /// Whether this coding structure does not allow delta flags on samples + pub(crate) fn intra_only(&self) -> bool { + matches!(self, Self::IntraOnly) + } +} + +#[derive(Debug)] +pub(crate) struct Sample { + /// Sync point + sync_point: bool, + + /// Sample duration + duration: gst::ClockTime, + + /// Composition time offset + /// + /// This is `None` for streams that have no concept of DTS. + composition_time_offset: Option<i64>, + + /// Size + size: u32, +} + +#[derive(Debug)] +pub(crate) struct Chunk { + /// Chunk start offset + offset: u64, + + /// Samples of this stream that are part of this chunk + samples: Vec<Sample>, +} + +#[derive(Debug)] +pub(crate) struct Stream { + /// Caps of this stream + caps: gst::Caps, + + /// If this stream has delta frames, and if so if it can have B frames. + delta_frames: DeltaFrames, + + /// Pre-defined trak timescale if not 0. + trak_timescale: u32, + + /// Start DTS + /// + /// If this is negative then an edit list entry is needed to + /// make all sample times positive. + /// + /// This is `None` for streams that have no concept of DTS. + start_dts: Option<gst::Signed<gst::ClockTime>>, + + /// Earliest PTS + /// + /// If this is >0 then an edit list entry is needed to shift + earliest_pts: gst::ClockTime, + + /// End PTS + end_pts: gst::ClockTime, + + /// All the chunks stored for this stream + chunks: Vec<Chunk>, +} + +#[derive(Debug)] +pub(crate) struct Header { + #[allow(dead_code)] + variant: Variant, + /// Pre-defined movie timescale if not 0. + movie_timescale: u32, + streams: Vec<Stream>, +} + +#[allow(clippy::upper_case_acronyms)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Variant { + ISO, +} diff --git a/mux/mp4/tests/tests.rs b/mux/mp4/tests/tests.rs new file mode 100644 index 000000000..4a766ce7d --- /dev/null +++ b/mux/mp4/tests/tests.rs @@ -0,0 +1,132 @@ +// Copyright (C) 2022 Sebastian Dröge <sebastian@centricular.com> +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// <https://mozilla.org/MPL/2.0/>. +// +// SPDX-License-Identifier: MPL-2.0 +// + +use gst::prelude::*; +use gst_pbutils::prelude::*; + +fn init() { + use std::sync::Once; + static INIT: Once = Once::new(); + + INIT.call_once(|| { + gst::init().unwrap(); + gstmp4::plugin_register_static().unwrap(); + }); +} + +#[test] +fn test_basic() { + init(); + + struct Pipeline(gst::Pipeline); + impl std::ops::Deref for Pipeline { + type Target = gst::Pipeline; + + fn deref(&self) -> &Self::Target { + &self.0 + } + } + impl Drop for Pipeline { + fn drop(&mut self) { + let _ = self.0.set_state(gst::State::Null); + } + } + + let pipeline = match gst::parse_launch( + "videotestsrc num-buffers=99 ! x264enc ! mux. \ + audiotestsrc num-buffers=140 ! fdkaacenc ! mux. \ + isomp4mux name=mux ! filesink name=sink \ + ", + ) { + Ok(pipeline) => Pipeline(pipeline.downcast::<gst::Pipeline>().unwrap()), + Err(_) => return, + }; + + let dir = tempfile::TempDir::new().unwrap(); + let mut location = dir.path().to_owned(); + location.push("test.mp4"); + + let sink = pipeline.by_name("sink").unwrap(); + sink.set_property("location", location.to_str().expect("Non-UTF8 filename")); + + pipeline + .set_state(gst::State::Playing) + .expect("Unable to set the pipeline to the `Playing` state"); + + for msg in pipeline.bus().unwrap().iter_timed(gst::ClockTime::NONE) { + use gst::MessageView; + + match msg.view() { + MessageView::Eos(..) => break, + MessageView::Error(err) => { + panic!( + "Error from {:?}: {} ({:?})", + err.src().map(|s| s.path_string()), + err.error(), + err.debug() + ); + } + _ => (), + } + } + + pipeline + .set_state(gst::State::Null) + .expect("Unable to set the pipeline to the `Null` state"); + + drop(pipeline); + + let discoverer = gst_pbutils::Discoverer::new(gst::ClockTime::from_seconds(5)) + .expect("Failed to create discoverer"); + let info = discoverer + .discover_uri( + url::Url::from_file_path(&location) + .expect("Failed to convert filename to URL") + .as_str(), + ) + .expect("Failed to discover MP4 file"); + + assert_eq!(info.duration(), Some(gst::ClockTime::from_mseconds(3_300))); + + let audio_streams = info.audio_streams(); + assert_eq!(audio_streams.len(), 1); + let audio_stream = audio_streams[0] + .downcast_ref::<gst_pbutils::DiscovererAudioInfo>() + .unwrap(); + assert_eq!(audio_stream.channels(), 1); + assert_eq!(audio_stream.sample_rate(), 44_100); + let caps = audio_stream.caps().unwrap(); + assert!( + caps.can_intersect( + &gst::Caps::builder("audio/mpeg") + .any_features() + .field("mpegversion", 4i32) + .build() + ), + "Unexpected audio caps {:?}", + caps + ); + + let video_streams = info.video_streams(); + assert_eq!(video_streams.len(), 1); + let video_stream = video_streams[0] + .downcast_ref::<gst_pbutils::DiscovererVideoInfo>() + .unwrap(); + assert_eq!(video_stream.width(), 320); + assert_eq!(video_stream.height(), 240); + assert_eq!(video_stream.framerate(), gst::Fraction::new(30, 1)); + assert_eq!(video_stream.par(), gst::Fraction::new(1, 1)); + assert!(!video_stream.is_interlaced()); + let caps = video_stream.caps().unwrap(); + assert!( + caps.can_intersect(&gst::Caps::builder("video/x-h264").any_features().build()), + "Unexpected video caps {:?}", + caps + ); +} |