Audaspace: added audio file streams functionality.

On the blender side this commit fixes importing video files with audio
and video streams that do not share the same start time and duration.

Differential Revision: https://developer.blender.org/D12353

9 files changed, 161 insertions, 55 deletions

diff --git a/source/blender/blenkernel/BKE_sound.h b/source/blender/blenkernel/BKE_sound.h
index fa58813c5f8..8796e2c18f3 100644
--- a/source/blender/blenkernel/BKE_sound.h
+++ b/source/blender/blenkernel/BKE_sound.h
@@ -96,13 +96,24 @@ typedef struct SoundInfo {
     eSoundChannels channels;
   } specs;
   float length;
-  double start_offset;
 } SoundInfo;
 
+typedef struct SoundStreamInfo {
+  double duration;
+  double start;
+} SoundStreamInfo;
+
 /* Get information about given sound. Returns truth on success., false if sound can not be loaded
  * or if the codes is not supported. */
 bool BKE_sound_info_get(struct Main *main, struct bSound *sound, SoundInfo *sound_info);
 
+/* Get information about given sound. Returns truth on success., false if sound can not be loaded
+ * or if the codes is not supported. */
+bool BKE_sound_stream_info_get(struct Main *main,
+                               const char *filepath,
+                               int stream,
+                               SoundStreamInfo *sound_info);
+
 #if defined(WITH_AUDASPACE)
 AUD_Device *BKE_sound_mixdown(const struct Scene *scene,
                               AUD_DeviceSpecs specs,
diff --git a/source/blender/blenkernel/intern/sound.c b/source/blender/blenkernel/intern/sound.c
index c61fa793367..ccb10f080e3 100644
--- a/source/blender/blenkernel/intern/sound.c
+++ b/source/blender/blenkernel/intern/sound.c
@@ -1213,7 +1213,6 @@ static bool sound_info_from_playback_handle(void *playback_handle, SoundInfo *so
   AUD_SoundInfo info = AUD_getInfo(playback_handle);
   sound_info->specs.channels = (eSoundChannels)info.specs.channels;
   sound_info->length = info.length;
-  sound_info->start_offset = info.start_offset;
   return true;
 }
 
@@ -1231,6 +1230,44 @@ bool BKE_sound_info_get(struct Main *main, struct bSound *sound, SoundInfo *soun
   return result;
 }
 
+bool BKE_sound_stream_info_get(struct Main *main, const char *filepath, int stream, SoundStreamInfo *sound_info)
+{
+  const char *path;
+  char str[FILE_MAX];
+  AUD_Sound *sound;
+  AUD_StreamInfo *stream_infos;
+  int stream_count;
+
+  BLI_strncpy(str, filepath, sizeof(str));
+  path = BKE_main_blendfile_path(main);
+  BLI_path_abs(str, path);
+
+  sound = AUD_Sound_file(str);
+  if (!sound) {
+    return false;
+  }
+
+  stream_count = AUD_Sound_getFileStreams(sound, &stream_infos);
+
+  AUD_Sound_free(sound);
+
+  if (!stream_infos) {
+    return false;
+  }
+
+  if ((stream < 0) || (stream >= stream_count)) {
+    free(stream_infos);
+    return false;
+  }
+
+  sound_info->start = stream_infos[stream].start;
+  sound_info->duration = stream_infos[stream].duration;
+
+  free(stream_infos);
+
+  return true;
+}
+
 #else /* WITH_AUDASPACE */
 
 #  include "BLI_utildefines.h"
@@ -1400,6 +1437,14 @@ bool BKE_sound_info_get(struct Main *UNUSED(main),
   return false;
 }
 
+bool BKE_sound_stream_info_get(struct Main *UNUSED(main),
+                               const char *UNUSED(filepath),
+                               int UNUSED(stream),
+                               SoundStreamInfo *UNUSED(sound_info))
+{
+  return false;
+}
+
 #endif /* WITH_AUDASPACE */
 
 void BKE_sound_reset_scene_runtime(Scene *scene)
diff --git a/source/blender/editors/space_graph/graph_edit.c b/source/blender/editors/space_graph/graph_edit.c
index 2955c4ef7ae..872b17372de 100644
--- a/source/blender/editors/space_graph/graph_edit.c
+++ b/source/blender/editors/space_graph/graph_edit.c
@@ -1098,7 +1098,8 @@ static int graphkeys_sound_bake_exec(bContext *C, wmOperator *op)
                                     RNA_boolean_get(op->ptr, "use_square"),
                                     RNA_float_get(op->ptr, "sthreshold"),
                                     FPS,
-                                    &sbi.length);
+                                    &sbi.length,
+                                    0);
 
   if (sbi.samples == NULL) {
     BKE_report(op->reports, RPT_ERROR, "Unsupported audio format");
diff --git a/source/blender/editors/space_sequencer/sequencer_add.c b/source/blender/editors/space_sequencer/sequencer_add.c
index 081f0241e94..bdfa639b327 100644
--- a/source/blender/editors/space_sequencer/sequencer_add.c
+++ b/source/blender/editors/space_sequencer/sequencer_add.c
@@ -47,6 +47,7 @@
 #include "BKE_mask.h"
 #include "BKE_movieclip.h"
 #include "BKE_report.h"
+#include "BKE_sound.h"
 
 #include "IMB_imbuf.h"
 
@@ -643,7 +644,15 @@ static void sequencer_add_movie_multiple_strips(bContext *C,
     BLI_strncpy(load_data->name, file_only, sizeof(load_data->name));
     Sequence *seq_movie = NULL;
     Sequence *seq_sound = NULL;
-    double video_start_offset;
+    double video_start_offset = -1;
+    double audio_start_offset = 0;
+
+    if (RNA_boolean_get(op->ptr, "sound")) {
+      SoundStreamInfo sound_info;
+      if (BKE_sound_stream_info_get(bmain, load_data->path, 0, &sound_info)) {
+        audio_start_offset = video_start_offset = sound_info.start;
+      }
+    }
 
     load_data->channel++;
     seq_movie = SEQ_add_movie_strip(bmain, scene, ed->seqbasep, load_data, &video_start_offset);
@@ -653,9 +662,30 @@ static void sequencer_add_movie_multiple_strips(bContext *C,
     }
     else {
       if (RNA_boolean_get(op->ptr, "sound")) {
-        seq_sound = SEQ_add_sound_strip(bmain, scene, ed->seqbasep, load_data, video_start_offset);
+        int minimum_frame_offset = MIN2(video_start_offset, audio_start_offset) * FPS;
+
+        int video_frame_offset = video_start_offset * FPS;
+        int audio_frame_offset = audio_start_offset * FPS;
+
+        double video_frame_remainder = video_start_offset * FPS - video_frame_offset;
+        double audio_frame_remainder = audio_start_offset * FPS - audio_frame_offset;
+
+        double audio_skip = (video_frame_remainder - audio_frame_remainder) / FPS;
+
+        video_frame_offset -= minimum_frame_offset;
+        audio_frame_offset -= minimum_frame_offset;
+
+        load_data->start_frame += audio_frame_offset;
+        seq_sound = SEQ_add_sound_strip(bmain, scene, ed->seqbasep, load_data, audio_skip);
+
+        int min_startdisp = MIN2(seq_movie->startdisp, seq_sound->startdisp);
+        int max_enddisp = MAX2(seq_movie->enddisp, seq_sound->enddisp);
+
+        load_data->start_frame += max_enddisp - min_startdisp - audio_frame_offset;
+      }
+      else {
+        load_data->start_frame += seq_movie->enddisp - seq_movie->startdisp;
       }
-      load_data->start_frame += seq_movie->enddisp - seq_movie->startdisp;
       seq_load_apply_generic_options(C, op, seq_sound);
       seq_load_apply_generic_options(C, op, seq_movie);
       seq_build_proxy(C, seq_movie);
@@ -672,7 +702,15 @@ static bool sequencer_add_movie_single_strip(bContext *C, wmOperator *op, SeqLoa
 
   Sequence *seq_movie = NULL;
   Sequence *seq_sound = NULL;
-  double video_start_offset;
+  double video_start_offset = -1;
+  double audio_start_offset = 0;
+
+  if (RNA_boolean_get(op->ptr, "sound")) {
+    SoundStreamInfo sound_info;
+    if (BKE_sound_stream_info_get(bmain, load_data->path, 0, &sound_info)) {
+      audio_start_offset = video_start_offset = sound_info.start;
+    }
+  }
 
   load_data->channel++;
   seq_movie = SEQ_add_movie_strip(bmain, scene, ed->seqbasep, load_data, &video_start_offset);
@@ -683,7 +721,21 @@ static bool sequencer_add_movie_single_strip(bContext *C, wmOperator *op, SeqLoa
     return false;
   }
   if (RNA_boolean_get(op->ptr, "sound")) {
-    seq_sound = SEQ_add_sound_strip(bmain, scene, ed->seqbasep, load_data, video_start_offset);
+    int minimum_frame_offset = MIN2(video_start_offset, audio_start_offset) * FPS;
+
+    int video_frame_offset = video_start_offset * FPS;
+    int audio_frame_offset = audio_start_offset * FPS;
+
+    double video_frame_remainder = video_start_offset * FPS - video_frame_offset;
+    double audio_frame_remainder = audio_start_offset * FPS - audio_frame_offset;
+
+    double audio_skip = (video_frame_remainder - audio_frame_remainder) / FPS;
+
+    video_frame_offset -= minimum_frame_offset;
+    audio_frame_offset -= minimum_frame_offset;
+
+    load_data->start_frame += audio_frame_offset;
+    seq_sound = SEQ_add_sound_strip(bmain, scene, ed->seqbasep, load_data, audio_skip);
   }
   seq_load_apply_generic_options(C, op, seq_sound);
   seq_load_apply_generic_options(C, op, seq_movie);
diff --git a/source/blender/editors/space_sequencer/sequencer_draw.c b/source/blender/editors/space_sequencer/sequencer_draw.c
index b56ad48cec2..bf817005a08 100644
--- a/source/blender/editors/space_sequencer/sequencer_draw.c
+++ b/source/blender/editors/space_sequencer/sequencer_draw.c
@@ -420,6 +420,10 @@ static void draw_seq_waveform_overlay(View2D *v2d,
       float sample_offset = start_sample + i * samples_per_pix;
       int p = sample_offset;
 
+      if (p < 0) {
+        continue;
+      }
+
       if (p >= waveform->length) {
         break;
       }
diff --git a/source/blender/imbuf/intern/anim_movie.c b/source/blender/imbuf/intern/anim_movie.c
index dbca16ca82b..13f9356751e 100644
--- a/source/blender/imbuf/intern/anim_movie.c
+++ b/source/blender/imbuf/intern/anim_movie.c
@@ -664,11 +664,6 @@ static int startffmpeg(struct anim *anim)
     anim->duration_in_frames = (int)(stream_dur * av_q2d(frame_rate) + 0.5f);
   }
 
-  double ctx_start = 0;
-  if (pFormatCtx->start_time != AV_NOPTS_VALUE) {
-    ctx_start = (double)pFormatCtx->start_time / AV_TIME_BASE;
-  }
-
   frs_num = frame_rate.num;
   frs_den = frame_rate.den;
 
@@ -683,7 +678,7 @@ static int startffmpeg(struct anim *anim)
   anim->frs_sec_base = frs_den;
   /* Save the relative start time for the video. IE the start time in relation to where playback
    * starts. */
-  anim->start_offset = video_start - ctx_start;
+  anim->start_offset = video_start;
 
   anim->params = 0;
 
diff --git a/source/blender/makesrna/intern/rna_sequencer_api.c b/source/blender/makesrna/intern/rna_sequencer_api.c
index a0564d3435b..b43b57a35be 100644
--- a/source/blender/makesrna/intern/rna_sequencer_api.c
+++ b/source/blender/makesrna/intern/rna_sequencer_api.c
@@ -323,8 +323,8 @@ static Sequence *rna_Sequences_new_movie(ID *id,
   SEQ_add_load_data_init(&load_data, name, file, frame_start, channel);
   load_data.fit_method = fit_method;
   load_data.allow_invalid_file = true;
-  double video_start_offset;
-  Sequence *seq = SEQ_add_movie_strip(bmain, scene, seqbase, &load_data, &video_start_offset);
+  double start_offset = -1;
+  Sequence *seq = SEQ_add_movie_strip(bmain, scene, seqbase, &load_data, &start_offset);
 
   DEG_relations_tag_update(bmain);
   DEG_id_tag_update(&scene->id, ID_RECALC_SEQUENCER_STRIPS);
diff --git a/source/blender/sequencer/SEQ_add.h b/source/blender/sequencer/SEQ_add.h
index 4025f1a4a04..d2a731d9953 100644
--- a/source/blender/sequencer/SEQ_add.h
+++ b/source/blender/sequencer/SEQ_add.h
@@ -88,7 +88,7 @@ struct Sequence *SEQ_add_movie_strip(struct Main *bmain,
                                      struct Scene *scene,
                                      struct ListBase *seqbase,
                                      struct SeqLoadData *load_data,
-                                     double *r_video_start_offset);
+                                     double *r_start_offset);
 struct Sequence *SEQ_add_scene_strip(struct Scene *scene,
                                      struct ListBase *seqbase,
                                      struct SeqLoadData *load_data);
diff --git a/source/blender/sequencer/intern/strip_add.c b/source/blender/sequencer/intern/strip_add.c
index 9081c655d2f..3cf7a4ebf4d 100644
--- a/source/blender/sequencer/intern/strip_add.c
+++ b/source/blender/sequencer/intern/strip_add.c
@@ -403,26 +403,8 @@ Sequence *SEQ_add_sound_strip(Main *bmain,
     return NULL;
   }
 
-  /* If this sound it part of a video, then the sound might start after the video.
-   * In this case we need to then offset the start frame of the audio so it syncs up
-   * properly with the video.
-   */
-  int start_frame_offset = info.start_offset * FPS;
-  double start_frame_offset_remainer = (info.start_offset * FPS - start_frame_offset) / FPS;
-
-  if (start_frame_offset_remainer > FLT_EPSILON) {
-    /* We can't represent a fraction of a frame, so skip the first frame fraction of sound so we
-     * start on a "whole" frame.
-     */
-    start_frame_offset++;
-  }
-
-  sound->offset_time += start_frame_offset_remainer;
-
-  Sequence *seq = SEQ_sequence_alloc(seqbase,
-                                     load_data->start_frame + start_frame_offset,
-                                     load_data->channel,
-                                     SEQ_TYPE_SOUND_RAM);
+  Sequence *seq = SEQ_sequence_alloc(
+      seqbase, load_data->start_frame, load_data->channel, SEQ_TYPE_SOUND_RAM);
   seq->sound = sound;
   seq->scene_sound = NULL;
 
@@ -508,7 +490,7 @@ Sequence *SEQ_add_movie_strip(Main *bmain,
                               Scene *scene,
                               ListBase *seqbase,
                               SeqLoadData *load_data,
-                              double *r_video_start_offset)
+                              double *r_start_offset)
 {
   char path[sizeof(load_data->path)];
   BLI_strncpy(path, load_data->path, sizeof(path));
@@ -554,8 +536,40 @@ Sequence *SEQ_add_movie_strip(Main *bmain,
     return NULL;
   }
 
+  int video_frame_offset = 0;
+  float video_fps = 0.0f;
+
+  if (anim_arr[0] != NULL) {
+    short fps_denom;
+    float fps_num;
+
+    IMB_anim_get_fps(anim_arr[0], &fps_denom, &fps_num, true);
+
+    video_fps = fps_denom / fps_num;
+
+    /* Adjust scene's frame rate settings to match. */
+    if (load_data->flags & SEQ_LOAD_MOVIE_SYNC_FPS) {
+      scene->r.frs_sec = fps_denom;
+      scene->r.frs_sec_base = fps_num;
+    }
+
+    double video_start_offset = IMD_anim_get_offset(anim_arr[0]);
+    int minimum_frame_offset;
+
+    if (*r_start_offset >= 0) {
+      minimum_frame_offset = MIN2(video_start_offset, *r_start_offset) * FPS;
+    }
+    else {
+      minimum_frame_offset = video_start_offset * FPS;
+    }
+
+    video_frame_offset = video_start_offset * FPS - minimum_frame_offset;
+
+    *r_start_offset = video_start_offset;
+  }
+
   Sequence *seq = SEQ_sequence_alloc(
-      seqbase, load_data->start_frame, load_data->channel, SEQ_TYPE_MOVIE);
+      seqbase, load_data->start_frame + video_frame_offset, load_data->channel, SEQ_TYPE_MOVIE);
 
   /* Multiview settings. */
   if (load_data->use_multiview) {
@@ -579,27 +593,11 @@ Sequence *SEQ_add_movie_strip(Main *bmain,
 
   seq->blend_mode = SEQ_TYPE_CROSS; /* so alpha adjustment fade to the strip below */
 
-  float video_fps = 0.0f;
-
   if (anim_arr[0] != NULL) {
     seq->len = IMB_anim_get_duration(anim_arr[0], IMB_TC_RECORD_RUN);
-    *r_video_start_offset = IMD_anim_get_offset(anim_arr[0]);
 
     IMB_anim_load_metadata(anim_arr[0]);
 
-    short fps_denom;
-    float fps_num;
-
-    IMB_anim_get_fps(anim_arr[0], &fps_denom, &fps_num, true);
-
-    video_fps = fps_denom / fps_num;
-
-    /* Adjust scene's frame rate settings to match. */
-    if (load_data->flags & SEQ_LOAD_MOVIE_SYNC_FPS) {
-      scene->r.frs_sec = fps_denom;
-      scene->r.frs_sec_base = fps_num;
-    }
-
     /* Set initial scale based on load_data->fit_method. */
     orig_width = IMB_anim_get_image_width(anim_arr[0]);
     orig_height = IMB_anim_get_image_height(anim_arr[0]);