Audaspace: added audio file streams functionality.

On the blender side this commit fixes importing video files with audio
and video streams that do not share the same start time and duration.

Differential Revision: https://developer.blender.org/D12353

3 files changed, 63 insertions, 6 deletions

diff --git a/source/blender/editors/space_graph/graph_edit.c b/source/blender/editors/space_graph/graph_edit.c
index 2955c4ef7ae..872b17372de 100644
--- a/source/blender/editors/space_graph/graph_edit.c
+++ b/source/blender/editors/space_graph/graph_edit.c
@@ -1098,7 +1098,8 @@ static int graphkeys_sound_bake_exec(bContext *C, wmOperator *op)
                                     RNA_boolean_get(op->ptr, "use_square"),
                                     RNA_float_get(op->ptr, "sthreshold"),
                                     FPS,
-                                    &sbi.length);
+                                    &sbi.length,
+                                    0);
 
   if (sbi.samples == NULL) {
     BKE_report(op->reports, RPT_ERROR, "Unsupported audio format");
diff --git a/source/blender/editors/space_sequencer/sequencer_add.c b/source/blender/editors/space_sequencer/sequencer_add.c
index 081f0241e94..bdfa639b327 100644
--- a/source/blender/editors/space_sequencer/sequencer_add.c
+++ b/source/blender/editors/space_sequencer/sequencer_add.c
@@ -47,6 +47,7 @@
 #include "BKE_mask.h"
 #include "BKE_movieclip.h"
 #include "BKE_report.h"
+#include "BKE_sound.h"
 
 #include "IMB_imbuf.h"
 
@@ -643,7 +644,15 @@ static void sequencer_add_movie_multiple_strips(bContext *C,
     BLI_strncpy(load_data->name, file_only, sizeof(load_data->name));
     Sequence *seq_movie = NULL;
     Sequence *seq_sound = NULL;
-    double video_start_offset;
+    double video_start_offset = -1;
+    double audio_start_offset = 0;
+
+    if (RNA_boolean_get(op->ptr, "sound")) {
+      SoundStreamInfo sound_info;
+      if (BKE_sound_stream_info_get(bmain, load_data->path, 0, &sound_info)) {
+        audio_start_offset = video_start_offset = sound_info.start;
+      }
+    }
 
     load_data->channel++;
     seq_movie = SEQ_add_movie_strip(bmain, scene, ed->seqbasep, load_data, &video_start_offset);
@@ -653,9 +662,30 @@ static void sequencer_add_movie_multiple_strips(bContext *C,
     }
     else {
       if (RNA_boolean_get(op->ptr, "sound")) {
-        seq_sound = SEQ_add_sound_strip(bmain, scene, ed->seqbasep, load_data, video_start_offset);
+        int minimum_frame_offset = MIN2(video_start_offset, audio_start_offset) * FPS;
+
+        int video_frame_offset = video_start_offset * FPS;
+        int audio_frame_offset = audio_start_offset * FPS;
+
+        double video_frame_remainder = video_start_offset * FPS - video_frame_offset;
+        double audio_frame_remainder = audio_start_offset * FPS - audio_frame_offset;
+
+        double audio_skip = (video_frame_remainder - audio_frame_remainder) / FPS;
+
+        video_frame_offset -= minimum_frame_offset;
+        audio_frame_offset -= minimum_frame_offset;
+
+        load_data->start_frame += audio_frame_offset;
+        seq_sound = SEQ_add_sound_strip(bmain, scene, ed->seqbasep, load_data, audio_skip);
+
+        int min_startdisp = MIN2(seq_movie->startdisp, seq_sound->startdisp);
+        int max_enddisp = MAX2(seq_movie->enddisp, seq_sound->enddisp);
+
+        load_data->start_frame += max_enddisp - min_startdisp - audio_frame_offset;
+      }
+      else {
+        load_data->start_frame += seq_movie->enddisp - seq_movie->startdisp;
       }
-      load_data->start_frame += seq_movie->enddisp - seq_movie->startdisp;
       seq_load_apply_generic_options(C, op, seq_sound);
       seq_load_apply_generic_options(C, op, seq_movie);
       seq_build_proxy(C, seq_movie);
@@ -672,7 +702,15 @@ static bool sequencer_add_movie_single_strip(bContext *C, wmOperator *op, SeqLoa
 
   Sequence *seq_movie = NULL;
   Sequence *seq_sound = NULL;
-  double video_start_offset;
+  double video_start_offset = -1;
+  double audio_start_offset = 0;
+
+  if (RNA_boolean_get(op->ptr, "sound")) {
+    SoundStreamInfo sound_info;
+    if (BKE_sound_stream_info_get(bmain, load_data->path, 0, &sound_info)) {
+      audio_start_offset = video_start_offset = sound_info.start;
+    }
+  }
 
   load_data->channel++;
   seq_movie = SEQ_add_movie_strip(bmain, scene, ed->seqbasep, load_data, &video_start_offset);
@@ -683,7 +721,21 @@ static bool sequencer_add_movie_single_strip(bContext *C, wmOperator *op, SeqLoa
     return false;
   }
   if (RNA_boolean_get(op->ptr, "sound")) {
-    seq_sound = SEQ_add_sound_strip(bmain, scene, ed->seqbasep, load_data, video_start_offset);
+    int minimum_frame_offset = MIN2(video_start_offset, audio_start_offset) * FPS;
+
+    int video_frame_offset = video_start_offset * FPS;
+    int audio_frame_offset = audio_start_offset * FPS;
+
+    double video_frame_remainder = video_start_offset * FPS - video_frame_offset;
+    double audio_frame_remainder = audio_start_offset * FPS - audio_frame_offset;
+
+    double audio_skip = (video_frame_remainder - audio_frame_remainder) / FPS;
+
+    video_frame_offset -= minimum_frame_offset;
+    audio_frame_offset -= minimum_frame_offset;
+
+    load_data->start_frame += audio_frame_offset;
+    seq_sound = SEQ_add_sound_strip(bmain, scene, ed->seqbasep, load_data, audio_skip);
   }
   seq_load_apply_generic_options(C, op, seq_sound);
   seq_load_apply_generic_options(C, op, seq_movie);
diff --git a/source/blender/editors/space_sequencer/sequencer_draw.c b/source/blender/editors/space_sequencer/sequencer_draw.c
index b56ad48cec2..bf817005a08 100644
--- a/source/blender/editors/space_sequencer/sequencer_draw.c
+++ b/source/blender/editors/space_sequencer/sequencer_draw.c
@@ -420,6 +420,10 @@ static void draw_seq_waveform_overlay(View2D *v2d,
       float sample_offset = start_sample + i * samples_per_pix;
       int p = sample_offset;
 
+      if (p < 0) {
+        continue;
+      }
+
       if (p >= waveform->length) {
         break;
       }