fmp4mux: Write header as late as possible

Especially do it only after the initial fragment's information are all
known.

3 files changed, 35 insertions, 27 deletions

diff --git a/generic/fmp4/src/fmp4mux/boxes.rs b/generic/fmp4/src/fmp4mux/boxes.rs
index 597ed8089..40b0567b6 100644
--- a/generic/fmp4/src/fmp4mux/boxes.rs
+++ b/generic/fmp4/src/fmp4mux/boxes.rs
@@ -356,7 +356,7 @@ pub(super) fn create_fmp4_header(cfg: super::HeaderConfiguration) -> Result<gst:
     let mut v = vec![];
 
     let (brand, compatible_brands) =
-        brands_from_variant_and_caps(cfg.variant, cfg.streams.iter().map(|s| s.1));
+        brands_from_variant_and_caps(cfg.variant, cfg.streams.iter().map(|s| &s.1));
 
     write_box(&mut v, b"ftyp", |v| {
         // major brand
@@ -454,7 +454,7 @@ fn write_mvhd(
     // Modification time
     v.extend(creation_time.to_be_bytes());
     // Timescale: uses the reference track timescale
-    v.extend(caps_to_timescale(cfg.streams[0].1).to_be_bytes());
+    v.extend(caps_to_timescale(&cfg.streams[0].1).to_be_bytes());
     // Duration
     v.extend(0u64.to_be_bytes());
 
@@ -1400,7 +1400,7 @@ fn write_mvex(v: &mut Vec<u8>, cfg: &super::HeaderConfiguration) -> Result<(), E
 
 fn write_mehd(v: &mut Vec<u8>, cfg: &super::HeaderConfiguration) -> Result<(), Error> {
     // Use the reference track timescale
-    let timescale = caps_to_timescale(cfg.streams[0].1);
+    let timescale = caps_to_timescale(&cfg.streams[0].1);
 
     let duration = cfg
         .duration
@@ -1443,7 +1443,7 @@ pub(super) fn create_fmp4_fragment_header(
     let mut v = vec![];
 
     let (brand, compatible_brands) =
-        brands_from_variant_and_caps(cfg.variant, cfg.streams.iter().map(|s| s.1));
+        brands_from_variant_and_caps(cfg.variant, cfg.streams.iter().map(|s| &s.1));
 
     write_box(&mut v, b"styp", |v| {
         // major brand
diff --git a/generic/fmp4/src/fmp4mux/imp.rs b/generic/fmp4/src/fmp4mux/imp.rs
index eb10d5675..b4a41dfd6 100644
--- a/generic/fmp4/src/fmp4mux/imp.rs
+++ b/generic/fmp4/src/fmp4mux/imp.rs
@@ -503,7 +503,7 @@ impl FMP4Mux {
         settings: &Settings,
         timeout: bool,
         at_eos: bool,
-    ) -> Result<Option<gst::BufferList>, gst::FlowError> {
+    ) -> Result<(Option<gst::Caps>, Option<gst::BufferList>), gst::FlowError> {
         let class = element.class();
 
         if at_eos {
@@ -513,7 +513,7 @@ impl FMP4Mux {
         } else {
             for stream in &state.streams {
                 if !stream.fragment_filled && !stream.sinkpad.is_eos() {
-                    return Ok(None);
+                    return Ok((None, None));
                 }
             }
         }
@@ -558,7 +558,7 @@ impl FMP4Mux {
             stream.fragment_filled = false;
 
             if gops.is_empty() {
-                streams.push((&stream.sinkpad, &stream.caps, None));
+                streams.push((stream.sinkpad.clone(), stream.caps.clone(), None));
             } else {
                 let first_gop = gops.first().unwrap();
                 let last_gop = gops.last().unwrap();
@@ -622,8 +622,8 @@ impl FMP4Mux {
                 };
 
                 streams.push((
-                    &stream.sinkpad,
-                    &stream.caps,
+                    stream.sinkpad.clone(),
+                    stream.caps.clone(),
                     Some(super::FragmentTimingInfo {
                         start_time,
                         intra_only: stream.intra_only,
@@ -698,6 +698,15 @@ impl FMP4Mux {
             }
         }
 
+        // Create header now if it was not created before and return the caps
+        let mut caps = None;
+        if state.stream_header.is_none() {
+            let (_, new_caps) = self
+                .update_header(element, state, &settings, false)?
+                .unwrap();
+            caps = Some(new_caps);
+        }
+
         // Interleave buffers according to the settings into a single vec
         let mut interleaved_buffers =
             Vec::with_capacity(drain_buffers.iter().map(|bs| bs.len()).sum());
@@ -873,7 +882,7 @@ impl FMP4Mux {
         }
 
         if settings.write_mfra && at_eos {
-            match boxes::create_mfra(streams[0].1, &state.fragment_offsets) {
+            match boxes::create_mfra(&streams[0].1, &state.fragment_offsets) {
                 Ok(mut mfra) => {
                     {
                         let mfra = mfra.get_mut().unwrap();
@@ -895,7 +904,7 @@ impl FMP4Mux {
         // TODO: Write edit list at EOS
         // TODO: Rewrite bitrates at EOS
 
-        Ok(buffer_list)
+        Ok((caps, buffer_list))
     }
 
     fn create_streams(
@@ -1020,7 +1029,7 @@ impl FMP4Mux {
         let streams = state
             .streams
             .iter()
-            .map(|s| (&s.sinkpad, &s.caps))
+            .map(|s| (s.sinkpad.clone(), s.caps.clone()))
             .collect::<Vec<_>>();
 
         let mut buffer = boxes::create_fmp4_header(super::HeaderConfiguration {
@@ -1461,7 +1470,7 @@ impl AggregatorImpl for FMP4Mux {
         let mut all_eos = true;
         let mut upstream_events = vec![];
 
-        let buffers = {
+        let (caps, buffers) = {
             let mut state = self.state.lock().unwrap();
 
             // Create streams
@@ -1469,17 +1478,6 @@ impl AggregatorImpl for FMP4Mux {
                 self.create_streams(aggregator, &mut state)?;
             }
 
-            // Stream header in the beginning and set output caps.
-            if state.stream_header.is_none() {
-                let (_, caps) = self
-                    .update_header(aggregator, &mut state, &settings, false)?
-                    .unwrap();
-
-                drop(state);
-                aggregator.set_src_caps(&caps);
-                state = self.state.lock().unwrap();
-            }
-
             // Queue buffers from all streams that are not filled for the current fragment yet
             let fragment_start_pts = state.fragment_start_pts;
             for (idx, stream) in state.streams.iter_mut().enumerate() {
@@ -1616,6 +1614,16 @@ impl AggregatorImpl for FMP4Mux {
             sinkpad.push_event(event);
         }
 
+        if let Some(caps) = caps {
+            gst::debug!(
+                CAT,
+                obj: aggregator,
+                "Setting caps on source pad: {:?}",
+                caps
+            );
+            aggregator.set_src_caps(&caps);
+        }
+
         if let Some(buffers) = buffers {
             gst::trace!(CAT, obj: aggregator, "Pushing buffer list {:?}", buffers);
             aggregator.finish_buffer_list(buffers)?;
diff --git a/generic/fmp4/src/fmp4mux/mod.rs b/generic/fmp4/src/fmp4mux/mod.rs
index 50b1a4194..a9b649a61 100644
--- a/generic/fmp4/src/fmp4mux/mod.rs
+++ b/generic/fmp4/src/fmp4mux/mod.rs
@@ -69,7 +69,7 @@ pub(crate) struct HeaderConfiguration<'a> {
     update: bool,
     /// First caps must be the video/reference stream. Must be in the order the tracks are going to
     /// be used later for the fragments too.
-    streams: &'a [(&'a gst_base::AggregatorPad, &'a gst::Caps)],
+    streams: &'a [(gst_base::AggregatorPad, gst::Caps)],
     write_mehd: bool,
     duration: Option<gst::ClockTime>,
 }
@@ -79,8 +79,8 @@ pub(crate) struct FragmentHeaderConfiguration<'a> {
     variant: Variant,
     sequence_number: u32,
     streams: &'a [(
-        &'a gst_base::AggregatorPad,
-        &'a gst::Caps,
+        gst_base::AggregatorPad,
+        gst::Caps,
         Option<FragmentTimingInfo>,
     )],
     buffers: &'a [Buffer],