Fix T94287: gaps between strips when adding movies

Currently, audio and video strips are synchronized based on data from
media stream, which is nice, but this causes gaps between strips.
This synchronization was implemented by moving movie strip position
relative to sound, which doesn't make much sense for user which is
mostly interested in editing video.

Code was bit hard to read, so it has been simplified. Ideally video
stream time would be easily accessible so synchronization could be done
at any time, but this is not necessary at this point.

Reviewed By: zeddb

Differential Revision: https://developer.blender.org/D13948

4 files changed, 39 insertions, 96 deletions

diff --git a/source/blender/editors/space_sequencer/sequencer_add.c b/source/blender/editors/space_sequencer/sequencer_add.c
index d9d801b56ac..15ee35f375d 100644
--- a/source/blender/editors/space_sequencer/sequencer_add.c
+++ b/source/blender/editors/space_sequencer/sequencer_add.c
@@ -637,6 +637,7 @@ static void sequencer_add_movie_clamp_sound_strip_length(Scene *scene,
   }
 
   SEQ_transform_set_right_handle_frame(seq_sound, SEQ_transform_get_right_handle_frame(seq_movie));
+  SEQ_transform_set_left_handle_frame(seq_sound, SEQ_transform_get_left_handle_frame(seq_movie));
   SEQ_time_update_sequence(scene, seqbase, seq_sound);
 }
 
@@ -658,52 +659,20 @@ static void sequencer_add_movie_multiple_strips(bContext *C,
     BLI_strncpy(load_data->name, file_only, sizeof(load_data->name));
     Sequence *seq_movie = NULL;
     Sequence *seq_sound = NULL;
-    double video_start_offset = -1;
-    double audio_start_offset = 0;
-
-    if (RNA_boolean_get(op->ptr, "sound")) {
-      SoundStreamInfo sound_info;
-      if (BKE_sound_stream_info_get(bmain, load_data->path, 0, &sound_info)) {
-        audio_start_offset = video_start_offset = sound_info.start;
-      }
-    }
 
     load_data->channel++;
-    seq_movie = SEQ_add_movie_strip(bmain, scene, ed->seqbasep, load_data, &video_start_offset);
+    seq_movie = SEQ_add_movie_strip(bmain, scene, ed->seqbasep, load_data);
     load_data->channel--;
     if (seq_movie == NULL) {
       BKE_reportf(op->reports, RPT_ERROR, "File '%s' could not be loaded", load_data->path);
     }
     else {
       if (RNA_boolean_get(op->ptr, "sound")) {
-        int minimum_frame_offset = MIN2(video_start_offset, audio_start_offset) * FPS;
-
-        int video_frame_offset = video_start_offset * FPS;
-        int audio_frame_offset = audio_start_offset * FPS;
-
-        double video_frame_remainder = video_start_offset * FPS - video_frame_offset;
-        double audio_frame_remainder = audio_start_offset * FPS - audio_frame_offset;
-
-        double audio_skip = (video_frame_remainder - audio_frame_remainder) / FPS;
-
-        video_frame_offset -= minimum_frame_offset;
-        audio_frame_offset -= minimum_frame_offset;
-
-        load_data->start_frame += audio_frame_offset;
-        seq_sound = SEQ_add_sound_strip(bmain, scene, ed->seqbasep, load_data, audio_skip);
-
-        int min_startdisp = 0, max_enddisp = 0;
-        if (seq_sound != NULL) {
-          min_startdisp = MIN2(seq_movie->startdisp, seq_sound->startdisp);
-          max_enddisp = MAX2(seq_movie->enddisp, seq_sound->enddisp);
-        }
-
-        load_data->start_frame += max_enddisp - min_startdisp - audio_frame_offset;
+        seq_sound = SEQ_add_sound_strip(bmain, scene, ed->seqbasep, load_data);
+        sequencer_add_movie_clamp_sound_strip_length(scene, ed->seqbasep, seq_movie, seq_sound);
       }
-      else {
-        load_data->start_frame += seq_movie->enddisp - seq_movie->startdisp;
-      }
-      sequencer_add_movie_clamp_sound_strip_length(scene, ed->seqbasep, seq_movie, seq_sound);
+
+      load_data->start_frame += seq_movie->enddisp - seq_movie->startdisp;
       seq_load_apply_generic_options(C, op, seq_sound);
       seq_load_apply_generic_options(C, op, seq_movie);
       SEQ_collection_append_strip(seq_movie, r_movie_strips);
@@ -723,18 +692,9 @@ static bool sequencer_add_movie_single_strip(bContext *C,
 
   Sequence *seq_movie = NULL;
   Sequence *seq_sound = NULL;
-  double video_start_offset = -1;
-  double audio_start_offset = 0;
-
-  if (RNA_boolean_get(op->ptr, "sound")) {
-    SoundStreamInfo sound_info;
-    if (BKE_sound_stream_info_get(bmain, load_data->path, 0, &sound_info)) {
-      audio_start_offset = video_start_offset = sound_info.start;
-    }
-  }
 
   load_data->channel++;
-  seq_movie = SEQ_add_movie_strip(bmain, scene, ed->seqbasep, load_data, &video_start_offset);
+  seq_movie = SEQ_add_movie_strip(bmain, scene, ed->seqbasep, load_data);
   load_data->channel--;
 
   if (seq_movie == NULL) {
@@ -742,23 +702,9 @@ static bool sequencer_add_movie_single_strip(bContext *C,
     return false;
   }
   if (RNA_boolean_get(op->ptr, "sound")) {
-    int minimum_frame_offset = MIN2(video_start_offset, audio_start_offset) * FPS;
-
-    int video_frame_offset = video_start_offset * FPS;
-    int audio_frame_offset = audio_start_offset * FPS;
-
-    double video_frame_remainder = video_start_offset * FPS - video_frame_offset;
-    double audio_frame_remainder = audio_start_offset * FPS - audio_frame_offset;
-
-    double audio_skip = (video_frame_remainder - audio_frame_remainder) / FPS;
-
-    video_frame_offset -= minimum_frame_offset;
-    audio_frame_offset -= minimum_frame_offset;
-
-    load_data->start_frame += audio_frame_offset;
-    seq_sound = SEQ_add_sound_strip(bmain, scene, ed->seqbasep, load_data, audio_skip);
+    seq_sound = SEQ_add_sound_strip(bmain, scene, ed->seqbasep, load_data);
+    sequencer_add_movie_clamp_sound_strip_length(scene, ed->seqbasep, seq_movie, seq_sound);
   }
-  sequencer_add_movie_clamp_sound_strip_length(scene, ed->seqbasep, seq_movie, seq_sound);
   seq_load_apply_generic_options(C, op, seq_sound);
   seq_load_apply_generic_options(C, op, seq_movie);
   SEQ_collection_append_strip(seq_movie, r_movie_strips);
@@ -905,7 +851,7 @@ static void sequencer_add_sound_multiple_strips(bContext *C,
     RNA_string_get(&itemptr, "name", file_only);
     BLI_join_dirfile(load_data->path, sizeof(load_data->path), dir_only, file_only);
     BLI_strncpy(load_data->name, file_only, sizeof(load_data->name));
-    Sequence *seq = SEQ_add_sound_strip(bmain, scene, ed->seqbasep, load_data, 0.0f);
+    Sequence *seq = SEQ_add_sound_strip(bmain, scene, ed->seqbasep, load_data);
     if (seq == NULL) {
       BKE_reportf(op->reports, RPT_ERROR, "File '%s' could not be loaded", load_data->path);
     }
@@ -923,7 +869,7 @@ static bool sequencer_add_sound_single_strip(bContext *C, wmOperator *op, SeqLoa
   Scene *scene = CTX_data_scene(C);
   Editing *ed = SEQ_editing_ensure(scene);
 
-  Sequence *seq = SEQ_add_sound_strip(bmain, scene, ed->seqbasep, load_data, 0.0f);
+  Sequence *seq = SEQ_add_sound_strip(bmain, scene, ed->seqbasep, load_data);
   if (seq == NULL) {
     BKE_reportf(op->reports, RPT_ERROR, "File '%s' could not be loaded", load_data->path);
     return false;
diff --git a/source/blender/makesrna/intern/rna_sequencer_api.c b/source/blender/makesrna/intern/rna_sequencer_api.c
index 7989c316c4c..e2e2bc2d5e8 100644
--- a/source/blender/makesrna/intern/rna_sequencer_api.c
+++ b/source/blender/makesrna/intern/rna_sequencer_api.c
@@ -328,8 +328,7 @@ static Sequence *rna_Sequences_new_movie(ID *id,
   SEQ_add_load_data_init(&load_data, name, file, frame_start, channel);
   load_data.fit_method = fit_method;
   load_data.allow_invalid_file = true;
-  double start_offset = -1;
-  Sequence *seq = SEQ_add_movie_strip(bmain, scene, seqbase, &load_data, &start_offset);
+  Sequence *seq = SEQ_add_movie_strip(bmain, scene, seqbase, &load_data);
 
   DEG_relations_tag_update(bmain);
   DEG_id_tag_update(&scene->id, ID_RECALC_SEQUENCER_STRIPS);
@@ -378,7 +377,7 @@ static Sequence *rna_Sequences_new_sound(ID *id,
   SeqLoadData load_data;
   SEQ_add_load_data_init(&load_data, name, file, frame_start, channel);
   load_data.allow_invalid_file = true;
-  Sequence *seq = SEQ_add_sound_strip(bmain, scene, seqbase, &load_data, 0.0f);
+  Sequence *seq = SEQ_add_sound_strip(bmain, scene, seqbase, &load_data);
 
   if (seq == NULL) {
     BKE_report(reports, RPT_ERROR, "Sequences.new_sound: unable to open sound file");
diff --git a/source/blender/sequencer/SEQ_add.h b/source/blender/sequencer/SEQ_add.h
index 85f44ab914f..23ad845fda1 100644
--- a/source/blender/sequencer/SEQ_add.h
+++ b/source/blender/sequencer/SEQ_add.h
@@ -64,7 +64,8 @@ typedef struct SeqLoadData {
   bool use_multiview;
   char views_format;
   struct Stereo3dFormat *stereo3d_format;
-  bool allow_invalid_file; /* Used by RNA API to create placeholder strips. */
+  bool allow_invalid_file;     /* Used by RNA API to create placeholder strips. */
+  double r_video_stream_start; /* For AV synchronization. Set by `SEQ_add_movie_strip`. */
 } SeqLoadData;
 
 /**
@@ -108,8 +109,7 @@ struct Sequence *SEQ_add_image_strip(struct Main *bmain,
 struct Sequence *SEQ_add_sound_strip(struct Main *bmain,
                                      struct Scene *scene,
                                      struct ListBase *seqbase,
-                                     struct SeqLoadData *load_data,
-                                     double audio_offset);
+                                     struct SeqLoadData *load_data);
 /**
  * Add meta strip.
  *
@@ -133,8 +133,7 @@ struct Sequence *SEQ_add_meta_strip(struct Scene *scene,
 struct Sequence *SEQ_add_movie_strip(struct Main *bmain,
                                      struct Scene *scene,
                                      struct ListBase *seqbase,
-                                     struct SeqLoadData *load_data,
-                                     double *r_start_offset);
+                                     struct SeqLoadData *load_data);
 /**
  * Add scene strip.
  *
diff --git a/source/blender/sequencer/intern/strip_add.c b/source/blender/sequencer/intern/strip_add.c
index f342765eec9..93950782295 100644
--- a/source/blender/sequencer/intern/strip_add.c
+++ b/source/blender/sequencer/intern/strip_add.c
@@ -287,14 +287,24 @@ Sequence *SEQ_add_image_strip(Main *bmain, Scene *scene, ListBase *seqbase, SeqL
 }
 
 #ifdef WITH_AUDASPACE
-Sequence *SEQ_add_sound_strip(Main *bmain,
-                              Scene *scene,
-                              ListBase *seqbase,
-                              SeqLoadData *load_data,
-                              const double audio_offset)
+
+static void seq_add_sound_av_sync(Main *bmain, Scene *scene, Sequence *seq, SeqLoadData *load_data)
+{
+  SoundStreamInfo sound_stream;
+  if (!BKE_sound_stream_info_get(bmain, load_data->path, 0, &sound_stream)) {
+    return;
+  }
+
+  const double av_stream_offset = sound_stream.start - load_data->r_video_stream_start;
+  const int frame_offset = av_stream_offset * FPS;
+  /* Set subframe offset. */
+  seq->sound->offset_time = ((double)frame_offset / FPS) - av_stream_offset;
+  SEQ_transform_translate_sequence(scene, seq, frame_offset);
+}
+
+Sequence *SEQ_add_sound_strip(Main *bmain, Scene *scene, ListBase *seqbase, SeqLoadData *load_data)
 {
   bSound *sound = BKE_sound_new_file(bmain, load_data->path); /* Handles relative paths. */
-  sound->offset_time = audio_offset;
   SoundInfo info;
   bool sound_loaded = BKE_sound_info_get(bmain, sound, &info);
 
@@ -337,6 +347,8 @@ Sequence *SEQ_add_sound_strip(Main *bmain,
     }
   }
 
+  seq_add_sound_av_sync(bmain, scene, seq, load_data);
+
   /* Set Last active directory. */
   BLI_strncpy(scene->ed->act_sounddir, strip->dir, FILE_MAXDIR);
   seq_add_set_name(scene, seq, load_data);
@@ -373,8 +385,7 @@ Sequence *SEQ_add_meta_strip(Scene *scene, ListBase *seqbase, SeqLoadData *load_
   return seqm;
 }
 
-Sequence *SEQ_add_movie_strip(
-    Main *bmain, Scene *scene, ListBase *seqbase, SeqLoadData *load_data, double *r_start_offset)
+Sequence *SEQ_add_movie_strip(Main *bmain, Scene *scene, ListBase *seqbase, SeqLoadData *load_data)
 {
   char path[sizeof(load_data->path)];
   BLI_strncpy(path, load_data->path, sizeof(path));
@@ -420,8 +431,8 @@ Sequence *SEQ_add_movie_strip(
     return NULL;
   }
 
-  int video_frame_offset = 0;
   float video_fps = 0.0f;
+  load_data->r_video_stream_start = 0.0;
 
   if (anim_arr[0] != NULL) {
     short fps_denom;
@@ -437,23 +448,11 @@ Sequence *SEQ_add_movie_strip(
       scene->r.frs_sec_base = fps_num;
     }
 
-    double video_start_offset = IMD_anim_get_offset(anim_arr[0]);
-    int minimum_frame_offset;
-
-    if (*r_start_offset >= 0) {
-      minimum_frame_offset = MIN2(video_start_offset, *r_start_offset) * FPS;
-    }
-    else {
-      minimum_frame_offset = video_start_offset * FPS;
-    }
-
-    video_frame_offset = video_start_offset * FPS - minimum_frame_offset;
-
-    *r_start_offset = video_start_offset;
+    load_data->r_video_stream_start = IMD_anim_get_offset(anim_arr[0]);
   }
 
   Sequence *seq = SEQ_sequence_alloc(
-      seqbase, load_data->start_frame + video_frame_offset, load_data->channel, SEQ_TYPE_MOVIE);
+      seqbase, load_data->start_frame, load_data->channel, SEQ_TYPE_MOVIE);
 
   /* Multiview settings. */
   if (load_data->use_multiview) {