VSE: Build proxies only for slow movies

This change applies only for automatic proxy building, when strip
is added to timeline. Manual building process is not affected.

Don't build proxy file if movie is already fast enough to seek.
To determine seek performance, check if whole GOP can be decoded
in 100 milliseconds.

To consider some variation in GOP size, large number of packets are
read, assuming that each packet will produce 1 frame. While this is not
technically correct, it does give quite accurate estimate of maximum GOP
size.

This test will ensure consistent performance on wide array of machines.
Check should be done in order of few milliseconds.

Reviewed By: sergey

Differential Revision: https://developer.blender.org/D11671

8 files changed, 168 insertions, 35 deletions

diff --git a/source/blender/editors/space_clip/clip_ops.c b/source/blender/editors/space_clip/clip_ops.c
index ef522e57d02..03b6d8c7381 100644
--- a/source/blender/editors/space_clip/clip_ops.c
+++ b/source/blender/editors/space_clip/clip_ops.c
@@ -1560,7 +1560,8 @@ static int clip_rebuild_proxy_exec(bContext *C, wmOperator *UNUSED(op))
                                                        clip->proxy.build_size_flag,
                                                        clip->proxy.quality,
                                                        true,
-                                                       NULL);
+                                                       NULL,
+                                                       false);
   }
 
   WM_jobs_customdata_set(wm_job, pj, proxy_freejob);
diff --git a/source/blender/editors/space_sequencer/sequencer_add.c b/source/blender/editors/space_sequencer/sequencer_add.c
index 9f31e55439d..72c39839739 100644
--- a/source/blender/editors/space_sequencer/sequencer_add.c
+++ b/source/blender/editors/space_sequencer/sequencer_add.c
@@ -59,6 +59,7 @@
 
 #include "SEQ_add.h"
 #include "SEQ_effects.h"
+#include "SEQ_iterator.h"
 #include "SEQ_proxy.h"
 #include "SEQ_relations.h"
 #include "SEQ_render.h"
@@ -601,29 +602,28 @@ static IMB_Proxy_Size seq_get_proxy_size_flags(bContext *C)
   return proxy_sizes;
 }
 
-static void seq_build_proxy(bContext *C, Sequence *seq)
+static void seq_build_proxy(bContext *C, SeqCollection *movie_strips)
 {
   if (U.sequencer_proxy_setup != USER_SEQ_PROXY_SETUP_AUTOMATIC) {
     return;
   }
 
-  /* Enable and set proxy size. */
-  SEQ_proxy_set(seq, true);
-  seq->strip->proxy->build_size_flags = seq_get_proxy_size_flags(C);
-  seq->strip->proxy->build_flags |= SEQ_PROXY_SKIP_EXISTING;
-
-  /* Build proxy. */
-  GSet *file_list = BLI_gset_new(BLI_ghashutil_strhash_p, BLI_ghashutil_strcmp, "file list");
   wmJob *wm_job = ED_seq_proxy_wm_job_get(C);
   ProxyJob *pj = ED_seq_proxy_job_get(C, wm_job);
-  SEQ_proxy_rebuild_context(pj->main, pj->depsgraph, pj->scene, seq, file_list, &pj->queue);
-  BLI_gset_free(file_list, MEM_freeN);
+
+  Sequence *seq;
+  SEQ_ITERATOR_FOREACH (seq, movie_strips) {
+    /* Enable and set proxy size. */
+    SEQ_proxy_set(seq, true);
+    seq->strip->proxy->build_size_flags = seq_get_proxy_size_flags(C);
+    seq->strip->proxy->build_flags |= SEQ_PROXY_SKIP_EXISTING;
+    SEQ_proxy_rebuild_context(pj->main, pj->depsgraph, pj->scene, seq, NULL, &pj->queue, true);
+  }
 
   if (!WM_jobs_is_running(wm_job)) {
     G.is_break = false;
     WM_jobs_start(CTX_wm_manager(C), wm_job);
   }
-
   ED_area_tag_redraw(CTX_wm_area(C));
 }
 
@@ -642,7 +642,8 @@ static void sequencer_add_movie_clamp_sound_strip_length(Scene *scene,
 
 static void sequencer_add_movie_multiple_strips(bContext *C,
                                                 wmOperator *op,
-                                                SeqLoadData *load_data)
+                                                SeqLoadData *load_data,
+                                                SeqCollection *r_movie_strips)
 {
   Main *bmain = CTX_data_main(C);
   Scene *scene = CTX_data_scene(C);
@@ -705,13 +706,16 @@ static void sequencer_add_movie_multiple_strips(bContext *C,
       sequencer_add_movie_clamp_sound_strip_length(scene, ed->seqbasep, seq_movie, seq_sound);
       seq_load_apply_generic_options(C, op, seq_sound);
       seq_load_apply_generic_options(C, op, seq_movie);
-      seq_build_proxy(C, seq_movie);
+      SEQ_collection_append_strip(seq_movie, r_movie_strips);
     }
   }
   RNA_END;
 }
 
-static bool sequencer_add_movie_single_strip(bContext *C, wmOperator *op, SeqLoadData *load_data)
+static bool sequencer_add_movie_single_strip(bContext *C,
+                                             wmOperator *op,
+                                             SeqLoadData *load_data,
+                                             SeqCollection *r_movie_strips)
 {
   Main *bmain = CTX_data_main(C);
   Scene *scene = CTX_data_scene(C);
@@ -757,7 +761,7 @@ static bool sequencer_add_movie_single_strip(bContext *C, wmOperator *op, SeqLoa
   sequencer_add_movie_clamp_sound_strip_length(scene, ed->seqbasep, seq_movie, seq_sound);
   seq_load_apply_generic_options(C, op, seq_sound);
   seq_load_apply_generic_options(C, op, seq_movie);
-  seq_build_proxy(C, seq_movie);
+  SEQ_collection_append_strip(seq_movie, r_movie_strips);
 
   return true;
 }
@@ -774,21 +778,25 @@ static int sequencer_add_movie_strip_exec(bContext *C, wmOperator *op)
     ED_sequencer_deselect_all(scene);
   }
 
+  SeqCollection *movie_strips = SEQ_collection_create(__func__);
   const int tot_files = RNA_property_collection_length(op->ptr,
                                                        RNA_struct_find_property(op->ptr, "files"));
   if (tot_files > 1) {
-    sequencer_add_movie_multiple_strips(C, op, &load_data);
+    sequencer_add_movie_multiple_strips(C, op, &load_data, movie_strips);
   }
   else {
-    if (!sequencer_add_movie_single_strip(C, op, &load_data)) {
-      sequencer_add_cancel(C, op);
-      return OPERATOR_CANCELLED;
-    }
+    sequencer_add_movie_single_strip(C, op, &load_data, movie_strips);
+  }
+
+  if (SEQ_collection_len(movie_strips) == 0) {
+    SEQ_collection_free(movie_strips);
+    return OPERATOR_CANCELLED;
   }
 
   /* Free custom data. */
   sequencer_add_cancel(C, op);
 
+  seq_build_proxy(C, movie_strips);
   DEG_relations_tag_update(bmain);
   DEG_id_tag_update(&scene->id, ID_RECALC_SEQUENCER_STRIPS);
   WM_event_add_notifier(C, NC_SCENE | ND_SEQUENCER, scene);
diff --git a/source/blender/editors/space_sequencer/sequencer_proxy.c b/source/blender/editors/space_sequencer/sequencer_proxy.c
index 0a8eb7cb88f..fb561025da2 100644
--- a/source/blender/editors/space_sequencer/sequencer_proxy.c
+++ b/source/blender/editors/space_sequencer/sequencer_proxy.c
@@ -85,7 +85,7 @@ static void seq_proxy_build_job(const bContext *C, ReportList *reports)
     }
 
     bool success = SEQ_proxy_rebuild_context(
-        pj->main, pj->depsgraph, pj->scene, seq, file_list, &pj->queue);
+        pj->main, pj->depsgraph, pj->scene, seq, file_list, &pj->queue, false);
 
     if (!success && (seq->strip->proxy->build_flags & SEQ_PROXY_SKIP_EXISTING) != 0) {
       BKE_reportf(reports, RPT_WARNING, "Overwrite is not checked for %s, skipping", seq->name);
@@ -137,7 +137,7 @@ static int sequencer_rebuild_proxy_exec(bContext *C, wmOperator *UNUSED(op))
       short stop = 0, do_update;
       float progress;
 
-      SEQ_proxy_rebuild_context(bmain, depsgraph, scene, seq, file_list, &queue);
+      SEQ_proxy_rebuild_context(bmain, depsgraph, scene, seq, file_list, &queue, false);
 
       for (link = queue.first; link; link = link->next) {
         struct SeqIndexBuildContext *context = link->data;
diff --git a/source/blender/imbuf/IMB_imbuf.h b/source/blender/imbuf/IMB_imbuf.h
index 65d7631445d..a557d7dc6d1 100644
--- a/source/blender/imbuf/IMB_imbuf.h
+++ b/source/blender/imbuf/IMB_imbuf.h
@@ -378,8 +378,9 @@ struct IndexBuildContext *IMB_anim_index_rebuild_context(struct anim *anim,
                                                          IMB_Timecode_Type tcs_in_use,
                                                          IMB_Proxy_Size proxy_sizes_in_use,
                                                          int quality,
-                                                         bool overwrite,
-                                                         struct GSet *file_list);
+                                                         const bool overwrite,
+                                                         struct GSet *file_list,
+                                                         bool build_only_on_bad_performance);
 
 /**
  * Will rebuild all used indices and proxies at once.
@@ -431,6 +432,7 @@ bool IMB_anim_can_produce_frames(const struct anim *anim);
 int ismovie(const char *filepath);
 int IMB_anim_get_image_width(struct anim *anim);
 int IMB_anim_get_image_height(struct anim *anim);
+bool IMB_get_gop_decode_time(struct anim *anim);
 
 /**
  *
diff --git a/source/blender/imbuf/intern/indexer.c b/source/blender/imbuf/intern/indexer.c
index 7aeee2740c0..67724a3bd3b 100644
--- a/source/blender/imbuf/intern/indexer.c
+++ b/source/blender/imbuf/intern/indexer.c
@@ -37,6 +37,8 @@
 #  include "BLI_winstuff.h"
 #endif
 
+#include "PIL_time.h"
+
 #include "IMB_anim.h"
 #include "IMB_indexer.h"
 #include "imbuf.h"
@@ -814,12 +816,16 @@ typedef struct FFmpegIndexBuilderContext {
   double pts_time_base;
   int frameno, frameno_gapless;
   int start_pts_set;
+
+  bool build_only_on_bad_performance;
+  bool building_cancelled;
 } FFmpegIndexBuilderContext;
 
 static IndexBuildContext *index_ffmpeg_create_context(struct anim *anim,
                                                       IMB_Timecode_Type tcs_in_use,
                                                       IMB_Proxy_Size proxy_sizes_in_use,
-                                                      int quality)
+                                                      int quality,
+                                                      bool build_only_on_bad_performance)
 {
   FFmpegIndexBuilderContext *context = MEM_callocN(sizeof(FFmpegIndexBuilderContext),
                                                    "FFmpeg index builder context");
@@ -831,6 +837,7 @@ static IndexBuildContext *index_ffmpeg_create_context(struct anim *anim,
   context->proxy_sizes_in_use = proxy_sizes_in_use;
   context->num_proxy_sizes = IMB_PROXY_MAX_SLOT;
   context->num_indexers = IMB_TC_MAX_SLOT;
+  context->build_only_on_bad_performance = build_only_on_bad_performance;
 
   memset(context->proxy_ctx, 0, sizeof(context->proxy_ctx));
   memset(context->indexer, 0, sizeof(context->indexer));
@@ -936,15 +943,17 @@ static void index_rebuild_ffmpeg_finish(FFmpegIndexBuilderContext *context, int
 {
   int i;
 
+  const bool do_rollback = stop || context->building_cancelled;
+
   for (i = 0; i < context->num_indexers; i++) {
     if (context->tcs_in_use & tc_types[i]) {
-      IMB_index_builder_finish(context->indexer[i], stop);
+      IMB_index_builder_finish(context->indexer[i], do_rollback);
     }
   }
 
   for (i = 0; i < context->num_proxy_sizes; i++) {
     if (context->proxy_sizes_in_use & proxy_sizes[i]) {
-      free_proxy_output_ffmpeg(context->proxy_ctx[i], stop);
+      free_proxy_output_ffmpeg(context->proxy_ctx[i], do_rollback);
     }
   }
 
@@ -1095,6 +1104,111 @@ static int index_rebuild_ffmpeg(FFmpegIndexBuilderContext *context,
   return 1;
 }
 
+/* Get number of frames, that can be decoded in specified time period. */
+static int indexer_performance_get_decode_rate(FFmpegIndexBuilderContext *context,
+                                               const double time_period)
+{
+  AVFrame *in_frame = av_frame_alloc();
+  AVPacket *packet = av_packet_alloc();
+
+  const double start = PIL_check_seconds_timer();
+  int frames_decoded = 0;
+
+  while (av_read_frame(context->iFormatCtx, packet) >= 0) {
+    if (packet->stream_index != context->videoStream) {
+      continue;
+    }
+
+    int ret = avcodec_send_packet(context->iCodecCtx, packet);
+    while (ret >= 0) {
+      ret = avcodec_receive_frame(context->iCodecCtx, in_frame);
+
+      if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
+        break;
+      }
+
+      if (ret < 0) {
+        fprintf(stderr, "Error decoding proxy frame: %s\n", av_err2str(ret));
+        break;
+      }
+      frames_decoded++;
+    }
+
+    const double end = PIL_check_seconds_timer();
+
+    if (end > start + time_period) {
+      break;
+    }
+  }
+
+  avcodec_flush_buffers(context->iCodecCtx);
+  av_seek_frame(context->iFormatCtx, -1, 0, AVSEEK_FLAG_BACKWARD);
+  return frames_decoded;
+}
+
+/* Read up to 10k movie packets and return max GOP size detected.
+ * Number of packets is arbitrary. It should be as large as possible, but processed within
+ * reasonable time period, so detected GOP size is as close to real as possible. */
+static int indexer_performance_get_max_gop_size(FFmpegIndexBuilderContext *context)
+{
+  AVPacket *packet = av_packet_alloc();
+
+  const int packets_max = 10000;
+  int packet_index = 0;
+  int max_gop = 0;
+  int cur_gop = 0;
+
+  while (av_read_frame(context->iFormatCtx, packet) >= 0) {
+    if (packet->stream_index != context->videoStream) {
+      continue;
+    }
+    packet_index++;
+    cur_gop++;
+
+    if (packet->flags & AV_PKT_FLAG_KEY) {
+      max_gop = max_ii(max_gop, cur_gop);
+      cur_gop = 0;
+    }
+
+    if (packet_index > packets_max) {
+      break;
+    }
+  }
+
+  av_seek_frame(context->iFormatCtx, -1, 0, AVSEEK_FLAG_BACKWARD);
+  return max_gop;
+}
+
+/* Assess scrubbing performance of provided file. This function is not meant to be very exact.
+ * It compares number number of frames decoded in reasonable time with largest detected GOP size.
+ * Because seeking happens in single GOP, it means, that maximum seek time can be detected this
+ * way.
+ * Since proxies use GOP size of 10 frames, skip building if detected GOP size is less or
+ * equal.
+ */
+static bool indexer_need_to_build_proxy(FFmpegIndexBuilderContext *context)
+{
+  if (!context->build_only_on_bad_performance) {
+    return true;
+  }
+
+  /* Make sure, that file is not cold read. */
+  indexer_performance_get_decode_rate(context, 0.1);
+  /* Get decode rate per 100ms. This is arbitrary, but seems to be good baseline cadence of
+   * seeking. */
+  const int decode_rate = indexer_performance_get_decode_rate(context, 0.1);
+  const int max_gop_size = indexer_performance_get_max_gop_size(context);
+
+  if (max_gop_size <= 10 || max_gop_size < decode_rate) {
+    printf("Skipping proxy building for %s: Decoding performance is already good.\n",
+           context->iFormatCtx->url);
+    context->building_cancelled = true;
+    return false;
+  }
+
+  return true;
+}
+
 #endif
 
 /* ----------------------------------------------------------------------
@@ -1274,7 +1388,8 @@ IndexBuildContext *IMB_anim_index_rebuild_context(struct anim *anim,
                                                   IMB_Proxy_Size proxy_sizes_in_use,
                                                   int quality,
                                                   const bool overwrite,
-                                                  GSet *file_list)
+                                                  GSet *file_list,
+                                                  bool build_only_on_bad_performance)
 {
   IndexBuildContext *context = NULL;
   IMB_Proxy_Size proxy_sizes_to_build = proxy_sizes_in_use;
@@ -1328,7 +1443,8 @@ IndexBuildContext *IMB_anim_index_rebuild_context(struct anim *anim,
   switch (anim->curtype) {
 #ifdef WITH_FFMPEG
     case ANIM_FFMPEG:
-      context = index_ffmpeg_create_context(anim, tcs_in_use, proxy_sizes_to_build, quality);
+      context = index_ffmpeg_create_context(
+          anim, tcs_in_use, proxy_sizes_to_build, quality, build_only_on_bad_performance);
       break;
 #endif
 #ifdef WITH_AVI
@@ -1358,7 +1474,9 @@ void IMB_anim_index_rebuild(struct IndexBuildContext *context,
   switch (context->anim_type) {
 #ifdef WITH_FFMPEG
     case ANIM_FFMPEG:
-      index_rebuild_ffmpeg((FFmpegIndexBuilderContext *)context, stop, do_update, progress);
+      if (indexer_need_to_build_proxy((FFmpegIndexBuilderContext *)context)) {
+        index_rebuild_ffmpeg((FFmpegIndexBuilderContext *)context, stop, do_update, progress);
+      }
       break;
 #endif
 #ifdef WITH_AVI
diff --git a/source/blender/makesrna/intern/rna_space.c b/source/blender/makesrna/intern/rna_space.c
index ec9b67deeac..e2411f5f4ae 100644
--- a/source/blender/makesrna/intern/rna_space.c
+++ b/source/blender/makesrna/intern/rna_space.c
@@ -2340,7 +2340,8 @@ static void seq_build_proxy(bContext *C, PointerRNA *ptr)
     seq->strip->proxy->build_size_flags |= SEQ_rendersize_to_proxysize(sseq->render_size);
 
     /* Build proxy. */
-    SEQ_proxy_rebuild_context(pj->main, pj->depsgraph, pj->scene, seq, file_list, &pj->queue);
+    SEQ_proxy_rebuild_context(
+        pj->main, pj->depsgraph, pj->scene, seq, file_list, &pj->queue, true);
   }
 
   BLI_gset_free(file_list, MEM_freeN);
diff --git a/source/blender/sequencer/SEQ_proxy.h b/source/blender/sequencer/SEQ_proxy.h
index 7bfe932ff1c..164b279245c 100644
--- a/source/blender/sequencer/SEQ_proxy.h
+++ b/source/blender/sequencer/SEQ_proxy.h
@@ -42,7 +42,8 @@ bool SEQ_proxy_rebuild_context(struct Main *bmain,
                                struct Scene *scene,
                                struct Sequence *seq,
                                struct GSet *file_list,
-                               struct ListBase *queue);
+                               struct ListBase *queue,
+                               bool build_only_on_bad_performance);
 void SEQ_proxy_rebuild(struct SeqIndexBuildContext *context,
                        short *stop,
                        short *do_update,
diff --git a/source/blender/sequencer/intern/proxy.c b/source/blender/sequencer/intern/proxy.c
index cc0ea0634a3..5982f89a287 100644
--- a/source/blender/sequencer/intern/proxy.c
+++ b/source/blender/sequencer/intern/proxy.c
@@ -415,7 +415,8 @@ bool SEQ_proxy_rebuild_context(Main *bmain,
                                Scene *scene,
                                Sequence *seq,
                                struct GSet *file_list,
-                               ListBase *queue)
+                               ListBase *queue,
+                               bool build_only_on_bad_performance)
 {
   SeqIndexBuildContext *context;
   Sequence *nseq;
@@ -476,7 +477,8 @@ bool SEQ_proxy_rebuild_context(Main *bmain,
                                                                 context->size_flags,
                                                                 context->quality,
                                                                 context->overwrite,
-                                                                file_list);
+                                                                file_list,
+                                                                build_only_on_bad_performance);
       }
       if (!context->index_context) {
         MEM_freeN(context);