Fix slow tracking of long sequences

The performance issue was noticeable when tracking a lot of tracks
which are using keyframe pattern matching. What was happening is that
at some cache gets filled in and the furthest away frame gets removed
from the cache: the frame at marker's keyframe gets removed and needs
to be re-read from disk on the next tracking step.

This change makes it so frames at markers' keyframes are not removed
from cache during tracking.

Steps to easily reproduce:
- Set cache size to 512 Mb.
- Open image sequence in clip editor
- Detect features
- Track all markers

Originally was reported by Rik, thanks!

3 files changed, 81 insertions, 0 deletions

diff --git a/source/blender/blenkernel/BKE_tracking.h b/source/blender/blenkernel/BKE_tracking.h
index 8cac196accc..29eb180a2ab 100644
--- a/source/blender/blenkernel/BKE_tracking.h
+++ b/source/blender/blenkernel/BKE_tracking.h
@@ -510,6 +510,7 @@ void BKE_tracking_refine_marker(struct MovieClip *clip,
 struct AutoTrackContext *BKE_autotrack_context_new(struct MovieClip *clip,
                                                    struct MovieClipUser *user,
                                                    bool is_backwards);
+void BKE_autotrack_context_start(struct AutoTrackContext *context);
 bool BKE_autotrack_context_step(struct AutoTrackContext *context);
 void BKE_autotrack_context_sync(struct AutoTrackContext *context);
 void BKE_autotrack_context_sync_user(struct AutoTrackContext *context, struct MovieClipUser *user);
diff --git a/source/blender/blenkernel/intern/tracking_auto.c b/source/blender/blenkernel/intern/tracking_auto.c
index 21a56c44c9b..6145e51920f 100644
--- a/source/blender/blenkernel/intern/tracking_auto.c
+++ b/source/blender/blenkernel/intern/tracking_auto.c
@@ -24,6 +24,9 @@
 #include "BKE_movieclip.h"
 #include "BKE_tracking.h"
 
+#include "IMB_imbuf.h"
+#include "IMB_imbuf_types.h"
+
 #include "libmv-capi.h"
 #include "tracking_private.h"
 
@@ -99,6 +102,12 @@ typedef struct AutoTrackContext {
   /* Accessor for images of clip. Used by the autotrack context. */
   TrackingImageAccessor *image_accessor;
 
+  /* Image buffers acquired for markers which are using keyframe pattern matching.
+   * These image buffers are user-referenced and flagged as persistent so that they don't get
+   * removed from the movie cache during tracking. */
+  int num_referenced_image_buffers;
+  ImBuf **referenced_image_buffers;
+
   /* --------------------------------------------------------------------
    * Variant part.
    * Denotes tracing state and tracking result.
@@ -554,6 +563,59 @@ AutoTrackContext *BKE_autotrack_context_new(MovieClip *clip,
 /** \} */
 
 /* -------------------------------------------------------------------- */
+/** \name Context tracking start.
+ *
+ * Called from possible job once before performing tracking steps.
+ * \{ */
+
+static void reference_keyframed_image_buffers(AutoTrackContext *context)
+{
+  /* NOTE: This is potentially over-allocating, but it simplifies memory manipulation.
+   * In practice this is unlikely to be noticed in the profiler as the memory footprint of this
+   * data is way less of what the tracking process will use. */
+  context->referenced_image_buffers = MEM_calloc_arrayN(
+      context->num_autotrack_markers, sizeof(ImBuf *), __func__);
+
+  context->num_referenced_image_buffers = 0;
+
+  for (int i = 0; i < context->num_autotrack_markers; ++i) {
+    const AutoTrackMarker *autotrack_marker = &context->autotrack_markers[i];
+    const int clip_index = autotrack_marker->libmv_marker.clip;
+    const int track_index = autotrack_marker->libmv_marker.track;
+
+    const AutoTrackClip *autotrack_clip = &context->autotrack_clips[clip_index];
+    const AutoTrackTrack *autotrack_track = &context->all_autotrack_tracks[track_index];
+    const MovieTrackingTrack *track = autotrack_track->track;
+
+    if (track->pattern_match != TRACK_MATCH_KEYFRAME) {
+      continue;
+    }
+
+    const int scene_frame = BKE_movieclip_remap_clip_to_scene_frame(
+        autotrack_clip->clip, autotrack_marker->libmv_marker.reference_frame);
+
+    MovieClipUser user_at_keyframe;
+    BKE_movieclip_user_set_frame(&user_at_keyframe, scene_frame);
+    user_at_keyframe.render_size = MCLIP_PROXY_RENDER_SIZE_FULL;
+    user_at_keyframe.render_flag = 0;
+
+    /* Keep reference to the image buffer so that we can manipulate its flags later on.
+     * Also request the movie cache to not remove the image buffer from the cache. */
+    ImBuf *ibuf = BKE_movieclip_get_ibuf(autotrack_clip->clip, &user_at_keyframe);
+    ibuf->userflags |= IB_PERSISTENT;
+
+    context->referenced_image_buffers[context->num_referenced_image_buffers++] = ibuf;
+  }
+}
+
+void BKE_autotrack_context_start(AutoTrackContext *context)
+{
+  reference_keyframed_image_buffers(context);
+}
+
+/** \} */
+
+/* -------------------------------------------------------------------- */
 /** \name Threaded context step (tracking process).
  * \{ */
 
@@ -799,6 +861,20 @@ void BKE_autotrack_context_finish(AutoTrackContext *context)
   }
 }
 
+static void release_keyframed_image_buffers(AutoTrackContext *context)
+{
+  for (int i = 0; i < context->num_referenced_image_buffers; ++i) {
+    ImBuf *ibuf = context->referenced_image_buffers[i];
+
+    /* Restore flag. It is not expected that anyone else is setting this flag on image buffers from
+     * movie clip, so can simply clear the flag. */
+    ibuf->userflags &= ~IB_PERSISTENT;
+    IMB_freeImBuf(ibuf);
+  }
+
+  MEM_freeN(context->referenced_image_buffers);
+}
+
 void BKE_autotrack_context_free(AutoTrackContext *context)
 {
   if (context->autotrack != NULL) {
@@ -809,6 +885,8 @@ void BKE_autotrack_context_free(AutoTrackContext *context)
     tracking_image_accessor_destroy(context->image_accessor);
   }
 
+  release_keyframed_image_buffers(context);
+
   MEM_SAFE_FREE(context->all_autotrack_tracks);
   MEM_SAFE_FREE(context->autotrack_markers);
 
diff --git a/source/blender/editors/space_clip/tracking_ops_track.c b/source/blender/editors/space_clip/tracking_ops_track.c
index f9cbce40deb..d5223d57490 100644
--- a/source/blender/editors/space_clip/tracking_ops_track.c
+++ b/source/blender/editors/space_clip/tracking_ops_track.c
@@ -211,6 +211,8 @@ static void track_markers_startjob(
   TrackMarkersJob *tmj = (TrackMarkersJob *)tmv;
   int framenr = tmj->sfra;
 
+  BKE_autotrack_context_start(tmj->context);
+
   while (framenr != tmj->efra) {
     if (tmj->delay > 0) {
       /* Tracking should happen with fixed fps. Calculate time