VSE strip thumbnails

Draw thumbnails as strip overlay. This works for movie and image strips.
To draw thumbnails, this overlay has to be enabled and strips must be
tall enough.

The thumbnails are loaded from source file using separate thread and
stored in cache.

Drawing code uses only images stored in cache, and if any is missing,
background rendering job is started. If job can not render thumbnail,
to prevent endless loop of creating job for missing image it sets
`SEQ_FLAG_SKIP_THUMBNAILS` bit of `Sequence` flag.

To prevent visual glitches during timeline panning and zooming, `View2D`
flag `V2D_IS_NAVIGATING` is implemented. If bit is set, drawing code
will look for set of evenly distributed thumbnails that should be
guaranteed to exist and also set of previously displayed thumbnails.
Due to volatile nature of cache these thumbnails can be missing anyway,
in which case no new thumbnails will be drawn for particular strip.

Cache capacity is limited to 5000 thumbnails and performs cleanup of
non visible images when limit is reached.

ref T89143

Reviewed By: ISS

Differential Revision: https://developer.blender.org/D12266

7 files changed, 292 insertions, 5 deletions

diff --git a/source/blender/sequencer/SEQ_render.h b/source/blender/sequencer/SEQ_render.h
index c138daf1318..50280db55ff 100644
--- a/source/blender/sequencer/SEQ_render.h
+++ b/source/blender/sequencer/SEQ_render.h
@@ -27,6 +27,8 @@
 extern "C" {
 #endif
 
+#define SEQ_RENDER_THUMB_SIZE 256
+
 struct ListBase;
 struct Main;
 struct Scene;
@@ -67,6 +69,25 @@ struct ImBuf *SEQ_render_give_ibuf(const SeqRenderData *context,
 struct ImBuf *SEQ_render_give_ibuf_direct(const SeqRenderData *context,
                                           float timeline_frame,
                                           struct Sequence *seq);
+void SEQ_render_thumbnails(const struct SeqRenderData *context,
+                           struct Sequence *seq,
+                           struct Sequence *seq_orig,
+                           float start_frame,
+                           float frame_step,
+                           rctf *view_area,
+                           short *stop);
+struct ImBuf *SEQ_get_thumbnail(const struct SeqRenderData *context,
+                                struct Sequence *seq,
+                                float timeline_frame,
+                                rcti *crop,
+                                bool clipped);
+int SEQ_render_thumbnails_guaranteed_set_frame_step_get(const struct Sequence *seq);
+void SEQ_render_thumbnails_base_set(const struct SeqRenderData *context,
+                                    struct Sequence *seq,
+                                    struct Sequence *seq_orig,
+                                    rctf *view_area,
+                                    short *stop);
+
 void SEQ_render_init_colorspace(struct Sequence *seq);
 void SEQ_render_new_render_data(struct Main *bmain,
                                 struct Depsgraph *depsgraph,
diff --git a/source/blender/sequencer/SEQ_utils.h b/source/blender/sequencer/SEQ_utils.h
index 09de7bc67e9..d30a1b2d7ae 100644
--- a/source/blender/sequencer/SEQ_utils.h
+++ b/source/blender/sequencer/SEQ_utils.h
@@ -34,6 +34,7 @@ struct Mask;
 struct Scene;
 struct Sequence;
 struct StripElem;
+struct SeqRenderData;
 
 void SEQ_sort(struct ListBase *seqbase);
 void SEQ_sequence_base_unique_name_recursive(struct Scene *scene,
@@ -57,7 +58,6 @@ void SEQ_set_scale_to_fit(const struct Sequence *seq,
                           const int preview_height,
                           const eSeqImageFitMethod fit_method);
 void SEQ_ensure_unique_name(struct Sequence *seq, struct Scene *scene);
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/blender/sequencer/intern/image_cache.c b/source/blender/sequencer/intern/image_cache.c
index 86bd840ce31..86c198075e9 100644
--- a/source/blender/sequencer/intern/image_cache.c
+++ b/source/blender/sequencer/intern/image_cache.c
@@ -104,6 +104,7 @@
 #define DCACHE_IMAGES_PER_FILE 100
 #define DCACHE_CURRENT_VERSION 2
 #define COLORSPACE_NAME_MAX 64 /* XXX: defined in imb intern */
+#define THUMB_CACHE_LIMIT 5000
 
 typedef struct DiskCacheHeaderEntry {
   unsigned char encoding;
@@ -148,6 +149,7 @@ typedef struct SeqCache {
   struct BLI_mempool *items_pool;
   struct SeqCacheKey *last_key;
   SeqDiskCache *disk_cache;
+  int thumbnail_count;
 } SeqCache;
 
 typedef struct SeqCacheItem {
@@ -776,7 +778,7 @@ static float seq_cache_timeline_frame_to_frame_index(Sequence *seq, float timeli
   /* With raw images, map timeline_frame to strip input media frame range. This means that static
    * images or extended frame range of movies will only generate one cache entry. No special
    * treatment in converting frame index to timeline_frame is needed. */
-  if (type == SEQ_CACHE_STORE_RAW) {
+  if (type == SEQ_CACHE_STORE_RAW || type == SEQ_CACHE_STORE_THUMBNAIL) {
     return seq_give_frame_index(seq, timeline_frame);
   }
 
@@ -875,7 +877,7 @@ static void seq_cache_put_ex(Scene *scene, SeqCacheKey *key, ImBuf *ibuf)
   if (BLI_ghash_reinsert(cache->hash, key, item, seq_cache_keyfree, seq_cache_valfree)) {
     IMB_refImBuf(ibuf);
 
-    if (!key->is_temp_cache) {
+    if (!key->is_temp_cache || key->type != SEQ_CACHE_STORE_THUMBNAIL) {
       cache->last_key = key;
     }
   }
@@ -1161,6 +1163,7 @@ static void seq_cache_create(Main *bmain, Scene *scene)
     cache->hash = BLI_ghash_new(seq_cache_hashhash, seq_cache_hashcmp, "SeqCache hash");
     cache->last_key = NULL;
     cache->bmain = bmain;
+    cache->thumbnail_count = 0;
     BLI_mutex_init(&cache->iterator_mutex);
     scene->ed->cache = cache;
 
@@ -1217,7 +1220,7 @@ void seq_cache_free_temp_cache(Scene *scene, short id, int timeline_frame)
     SeqCacheKey *key = BLI_ghashIterator_getKey(&gh_iter);
     BLI_ghashIterator_step(&gh_iter);
 
-    if (key->is_temp_cache && key->task_id == id) {
+    if (key->is_temp_cache && key->task_id == id && key->type != SEQ_CACHE_STORE_THUMBNAIL) {
       /* Use frame_index here to avoid freeing raw images if they are used for multiple frames. */
       float frame_index = seq_cache_timeline_frame_to_frame_index(
           key->seq, timeline_frame, key->type);
@@ -1278,6 +1281,7 @@ void SEQ_cache_cleanup(Scene *scene)
     BLI_ghash_remove(cache->hash, key, seq_cache_keyfree, seq_cache_valfree);
   }
   cache->last_key = NULL;
+  cache->thumbnail_count = 0;
   seq_cache_unlock(scene);
 }
 
@@ -1345,6 +1349,46 @@ void seq_cache_cleanup_sequence(Scene *scene,
   seq_cache_unlock(scene);
 }
 
+void seq_cache_thumbnail_cleanup(Scene *scene, rctf *view_area_safe)
+{
+  /* Add offsets to the left and right end to keep some frames in cache. */
+  view_area_safe->xmax += 200;
+  view_area_safe->xmin -= 200;
+  view_area_safe->ymin -= 1;
+  view_area_safe->ymax += 1;
+
+  SeqCache *cache = seq_cache_get_from_scene(scene);
+  if (!cache) {
+    return;
+  }
+
+  GHashIterator gh_iter;
+  BLI_ghashIterator_init(&gh_iter, cache->hash);
+  while (!BLI_ghashIterator_done(&gh_iter)) {
+    SeqCacheKey *key = BLI_ghashIterator_getKey(&gh_iter);
+    BLI_ghashIterator_step(&gh_iter);
+
+    const int frame_index = key->timeline_frame - key->seq->startdisp;
+    const int frame_step = SEQ_render_thumbnails_guaranteed_set_frame_step_get(key->seq);
+    const int relative_base_frame = round_fl_to_int((frame_index / (float)frame_step)) *
+                                    frame_step;
+    const int nearest_guaranted_absolute_frame = relative_base_frame + key->seq->startdisp;
+
+    if (nearest_guaranted_absolute_frame == key->timeline_frame) {
+      continue;
+    }
+
+    if ((key->type & SEQ_CACHE_STORE_THUMBNAIL) &&
+        (key->timeline_frame > view_area_safe->xmax ||
+         key->timeline_frame < view_area_safe->xmin || key->seq->machine > view_area_safe->ymax ||
+         key->seq->machine < view_area_safe->ymin)) {
+      BLI_ghash_remove(cache->hash, key, seq_cache_keyfree, seq_cache_valfree);
+      cache->thumbnail_count--;
+    }
+  }
+  cache->last_key = NULL;
+}
+
 struct ImBuf *seq_cache_get(const SeqRenderData *context,
                             Sequence *seq,
                             float timeline_frame,
@@ -1436,6 +1480,37 @@ bool seq_cache_put_if_possible(
   return false;
 }
 
+void seq_cache_thumbnail_put(
+    const SeqRenderData *context, Sequence *seq, float timeline_frame, ImBuf *i, rctf *view_area)
+{
+  Scene *scene = context->scene;
+
+  if (!scene->ed->cache) {
+    seq_cache_create(context->bmain, scene);
+  }
+
+  seq_cache_lock(scene);
+  SeqCache *cache = seq_cache_get_from_scene(scene);
+  SeqCacheKey *key = seq_cache_allocate_key(
+      cache, context, seq, timeline_frame, SEQ_CACHE_STORE_THUMBNAIL);
+
+  /* Prevent reinserting, it breaks cache key linking. */
+  if (BLI_ghash_haskey(cache->hash, key)) {
+    seq_cache_unlock(scene);
+    return;
+  }
+
+  /* Limit cache to THUMB_CACHE_LIMIT (5000) images stored. */
+  if (cache->thumbnail_count >= THUMB_CACHE_LIMIT) {
+    rctf view_area_safe = *view_area;
+    seq_cache_thumbnail_cleanup(scene, &view_area_safe);
+  }
+
+  seq_cache_put_ex(scene, key, i);
+  cache->thumbnail_count++;
+  seq_cache_unlock(scene);
+}
+
 void seq_cache_put(
     const SeqRenderData *context, Sequence *seq, float timeline_frame, int type, ImBuf *i)
 {
diff --git a/source/blender/sequencer/intern/image_cache.h b/source/blender/sequencer/intern/image_cache.h
index 63c559caee9..60031311985 100644
--- a/source/blender/sequencer/intern/image_cache.h
+++ b/source/blender/sequencer/intern/image_cache.h
@@ -46,6 +46,11 @@ void seq_cache_put(const struct SeqRenderData *context,
                    float timeline_frame,
                    int type,
                    struct ImBuf *i);
+void seq_cache_thumbnail_put(const struct SeqRenderData *context,
+                             struct Sequence *seq,
+                             float timeline_frame,
+                             struct ImBuf *i,
+                             rctf *view_area);
 bool seq_cache_put_if_possible(const struct SeqRenderData *context,
                                struct Sequence *seq,
                                float timeline_frame,
@@ -60,6 +65,7 @@ void seq_cache_cleanup_sequence(struct Scene *scene,
                                 struct Sequence *seq_changed,
                                 int invalidate_types,
                                 bool force_seq_changed_range);
+void seq_cache_thumbnail_cleanup(Scene *scene, rctf *view_area);
 bool seq_cache_is_full(void);
 
 #ifdef __cplusplus
diff --git a/source/blender/sequencer/intern/render.c b/source/blender/sequencer/intern/render.c
index e42a3f4d6e9..b2642228c91 100644
--- a/source/blender/sequencer/intern/render.c
+++ b/source/blender/sequencer/intern/render.c
@@ -434,6 +434,31 @@ static void sequencer_image_crop_init(const Sequence *seq,
   BLI_rctf_init(r_crop, left, in->x - right, bottom, in->y - top);
 }
 
+static void sequencer_thumbnail_transform(ImBuf *in, ImBuf *out)
+{
+  float image_scale_factor = (float)out->x / in->x;
+  float transform_matrix[3][3];
+
+  /* Set to keep same loc,scale,rot but change scale to thumb size limit. */
+  const float scale_x = 1 * image_scale_factor;
+  const float scale_y = 1 * image_scale_factor;
+  const float image_center_offs_x = (out->x - in->x) / 2;
+  const float image_center_offs_y = (out->y - in->y) / 2;
+  const float pivot[2] = {in->x / 2, in->y / 2};
+  loc_rot_size_to_mat3(transform_matrix,
+                       (const float[]){image_center_offs_x, image_center_offs_y},
+                       0,
+                       (const float[]){scale_x, scale_y});
+  transform_pivot_set_m3(transform_matrix, pivot);
+  invert_m3(transform_matrix);
+
+  /* No crop. */
+  rctf source_crop;
+  BLI_rctf_init(&source_crop, 0, in->x, 0, in->y);
+
+  IMB_transform(in, out, transform_matrix, &source_crop, IMB_FILTER_NEAREST);
+}
+
 static void sequencer_preprocess_transform_crop(
     ImBuf *in, ImBuf *out, const SeqRenderData *context, Sequence *seq, const bool is_proxy_image)
 {
@@ -1896,7 +1921,164 @@ ImBuf *SEQ_render_give_ibuf_direct(const SeqRenderData *context,
   seq_render_state_init(&state);
 
   ImBuf *ibuf = seq_render_strip(context, &state, seq, timeline_frame);
-
   return ibuf;
 }
+
+/* Gets the direct image from source and scales to thumbnail size. */
+static ImBuf *seq_get_uncached_thumbnail(const SeqRenderData *context,
+                                         SeqRenderState *state,
+                                         Sequence *seq,
+                                         float timeline_frame)
+{
+  bool is_proxy_image = false;
+  ImBuf *ibuf = do_render_strip_uncached(context, state, seq, timeline_frame, &is_proxy_image);
+
+  if (ibuf == NULL) {
+    return NULL;
+  }
+
+  float aspect_ratio = (float)ibuf->x / ibuf->y;
+  int rectx, recty;
+  /* Calculate new dimensions - THUMB_SIZE (256) for x or y. */
+  if (ibuf->x > ibuf->y) {
+    rectx = SEQ_RENDER_THUMB_SIZE;
+    recty = round_fl_to_int(rectx / aspect_ratio);
+  }
+  else {
+    recty = SEQ_RENDER_THUMB_SIZE;
+    rectx = round_fl_to_int(recty * aspect_ratio);
+  }
+
+  /* Scale ibuf to thumbnail size. */
+  ImBuf *scaled_ibuf = IMB_allocImBuf(rectx, recty, 32, ibuf->rect_float ? IB_rectfloat : IB_rect);
+  sequencer_thumbnail_transform(ibuf, scaled_ibuf);
+  seq_imbuf_assign_spaces(context->scene, scaled_ibuf);
+  IMB_freeImBuf(ibuf);
+
+  return scaled_ibuf;
+}
+
+/* Get cached thumbnails. */
+ImBuf *SEQ_get_thumbnail(
+    const SeqRenderData *context, Sequence *seq, float timeline_frame, rcti *crop, bool clipped)
+{
+  ImBuf *ibuf = seq_cache_get(context, seq, roundf(timeline_frame), SEQ_CACHE_STORE_THUMBNAIL);
+
+  if (!clipped || ibuf == NULL) {
+    return ibuf;
+  }
+
+  /* Do clipping. */
+  ImBuf *ibuf_cropped = IMB_dupImBuf(ibuf);
+  if (crop->xmin < 0 || crop->ymin < 0) {
+    crop->xmin = 0;
+    crop->ymin = 0;
+  }
+  if (crop->xmax >= ibuf->x || crop->ymax >= ibuf->y) {
+    crop->xmax = ibuf->x - 1;
+    crop->ymax = ibuf->y - 1;
+  }
+  IMB_rect_crop(ibuf_cropped, crop);
+  IMB_freeImBuf(ibuf);
+  return ibuf_cropped;
+}
+
+/* Render the series of thumbnails and store in cache. */
+void SEQ_render_thumbnails(const SeqRenderData *context,
+                           Sequence *seq,
+                           Sequence *seq_orig,
+                           float start_frame,
+                           float frame_step,
+                           rctf *view_area,
+                           short *stop)
+{
+  SeqRenderState state;
+  seq_render_state_init(&state);
+
+  /* Adding the hold offset value (seq->anim_startofs) to the start frame. Position of image not
+   * affected, but frame loaded affected. */
+  start_frame = start_frame - frame_step;
+  float upper_thumb_bound = (seq->endstill) ? (seq->start + seq->len) : seq->enddisp;
+  upper_thumb_bound = (upper_thumb_bound > view_area->xmax) ? view_area->xmax + frame_step :
+                                                              upper_thumb_bound;
+
+  while ((start_frame < upper_thumb_bound) & !*stop) {
+    ImBuf *ibuf = seq_cache_get(
+        context, seq_orig, round_fl_to_int(start_frame), SEQ_CACHE_STORE_THUMBNAIL);
+    if (ibuf) {
+      IMB_freeImBuf(ibuf);
+      start_frame += frame_step;
+      continue;
+    }
+
+    ibuf = seq_get_uncached_thumbnail(context, &state, seq, round_fl_to_int(start_frame));
+
+    if (ibuf) {
+      seq_cache_thumbnail_put(context, seq_orig, round_fl_to_int(start_frame), ibuf, view_area);
+      IMB_freeImBuf(ibuf);
+      seq_orig->flag &= ~SEQ_FLAG_SKIP_THUMBNAILS;
+    }
+    else {
+      /* Can not open source file. */
+      seq_orig->flag |= SEQ_FLAG_SKIP_THUMBNAILS;
+      return;
+    }
+
+    start_frame += frame_step;
+  }
+}
+
+/* Get frame step for equally spaced thumbnails. These thumbnails should always be present in
+ * memory, so they can be used when zooming.*/
+int SEQ_render_thumbnails_guaranteed_set_frame_step_get(const Sequence *seq)
+{
+  const int content_len = (seq->enddisp - seq->startdisp - seq->startstill - seq->endstill);
+
+  /* Arbitrary, but due to performance reasons should be as low as possible. */
+  const int thumbnails_base_set_count = min_ii(content_len / 100, 30);
+  if (thumbnails_base_set_count <= 0) {
+    return 0;
+  }
+  return content_len / thumbnails_base_set_count;
+}
+
+/* Render set of evenly spaced thumbnails that are drawn when zooming. */
+void SEQ_render_thumbnails_base_set(
+    const SeqRenderData *context, Sequence *seq, Sequence *seq_orig, rctf *view_area, short *stop)
+{
+  SeqRenderState state;
+  seq_render_state_init(&state);
+
+  int timeline_frame = seq->startdisp;
+  const int frame_step = SEQ_render_thumbnails_guaranteed_set_frame_step_get(seq);
+
+  while (timeline_frame < seq->enddisp && !*stop) {
+    ImBuf *ibuf = seq_cache_get(
+        context, seq_orig, roundf(timeline_frame), SEQ_CACHE_STORE_THUMBNAIL);
+    if (ibuf) {
+      IMB_freeImBuf(ibuf);
+
+      if (frame_step == 0) {
+        return;
+      }
+
+      timeline_frame += frame_step;
+      continue;
+    }
+
+    ibuf = seq_get_uncached_thumbnail(context, &state, seq, timeline_frame);
+
+    if (ibuf) {
+      seq_cache_thumbnail_put(context, seq_orig, timeline_frame, ibuf, view_area);
+      IMB_freeImBuf(ibuf);
+    }
+
+    if (frame_step == 0) {
+      return;
+    }
+
+    timeline_frame += frame_step;
+  }
+}
+
 /** \} */
diff --git a/source/blender/sequencer/intern/sequencer.c b/source/blender/sequencer/intern/sequencer.c
index d3f8411cf0a..382bd51aae1 100644
--- a/source/blender/sequencer/intern/sequencer.c
+++ b/source/blender/sequencer/intern/sequencer.c
@@ -957,6 +957,8 @@ static bool seq_read_lib_cb(Sequence *seq, void *user_data)
   BLI_listbase_clear(&seq->anims);
 
   SEQ_modifier_blend_read_lib(reader, sce, &seq->modifiers);
+
+  seq->flag &= ~SEQ_FLAG_SKIP_THUMBNAILS;
   return true;
 }
 
diff --git a/source/blender/sequencer/intern/utils.c b/source/blender/sequencer/intern/utils.c
index 1d3e7e4a223..8421aab5217 100644
--- a/source/blender/sequencer/intern/utils.c
+++ b/source/blender/sequencer/intern/utils.c
@@ -42,6 +42,7 @@
 #include "SEQ_edit.h"
 #include "SEQ_iterator.h"
 #include "SEQ_relations.h"
+#include "SEQ_render.h"
 #include "SEQ_select.h"
 #include "SEQ_sequencer.h"
 #include "SEQ_time.h"