Sequencer: Transform ImBuf Processor.

Inside the sequencer the cropping and transform of images/buffers were
implemented locally. This reduced the optimizations that a compiler
could do and added confusing code styles. This patch adds
`IMB_transform` to reduce the confusion and increases compiler
optimizations as more code can be inlined and we can keep track of
indices inside the inner loop.

This increases end-user performance by 30% when playing back aa video
in VSE.

Reviewed By: ISS, zeddb

Differential Revision: https://developer.blender.org/D11549

6 files changed, 187 insertions, 177 deletions

diff --git a/release/datafiles/locale b/release/datafiles/locale
-Subproject 5ab29b1331d2103dae634b987f121c4599459d7
+Subproject 4833954c0ac85cc407e1d5a153aa11b1d1823ec
diff --git a/release/scripts/addons b/release/scripts/addons
-Subproject cdabac54c4fe7c6f8df125814442762aa539172
+Subproject f86f25e62217264495d05f116ccb09d575fe984
diff --git a/source/blender/imbuf/IMB_imbuf.h b/source/blender/imbuf/IMB_imbuf.h
index 9c84127105a..651c69726ff 100644
--- a/source/blender/imbuf/IMB_imbuf.h
+++ b/source/blender/imbuf/IMB_imbuf.h
@@ -70,6 +70,7 @@ extern "C" {
  */
 struct ImBuf;
 struct rcti;
+struct rctf;
 
 /**
  *
@@ -323,6 +324,11 @@ typedef enum IMB_Proxy_Size {
   IMB_PROXY_MAX_SLOT = 4,
 } IMB_Proxy_Size;
 
+typedef enum eIMBInterpolationFilterMode {
+  IMB_FILTER_NEAREST,
+  IMB_FILTER_BILINEAR,
+} eIMBInterpolationFilterMode;
+
 /* Defaults to BL_proxy within the directory of the animation. */
 void IMB_anim_set_index_dir(struct anim *anim, const char *dir);
 void IMB_anim_get_fname(struct anim *anim, char *file, int size);
@@ -732,6 +738,12 @@ void IMB_processor_apply_threaded_scanlines(int total_scanlines,
                                             ScanlineThreadFunc do_thread,
                                             void *custom_data);
 
+void IMB_transform(struct ImBuf *src,
+                   struct ImBuf *dst,
+                   float transform_matrix[3][3],
+                   struct rctf *src_crop,
+                   const eIMBInterpolationFilterMode filter);
+
 /* ffmpeg */
 void IMB_ffmpeg_init(void);
 const char *IMB_ffmpeg_last_error(void);
diff --git a/source/blender/imbuf/intern/imageprocess.c b/source/blender/imbuf/intern/imageprocess.c
index 1fe3a7717fb..c88a9b81ff2 100644
--- a/source/blender/imbuf/intern/imageprocess.c
+++ b/source/blender/imbuf/intern/imageprocess.c
@@ -350,6 +350,137 @@ void nearest_interpolation(ImBuf *in, ImBuf *out, float u, float v, int xout, in
 }
 
 /* -------------------------------------------------------------------- */
+/** \name Image transform
+ * \{ */
+typedef struct TransformUserData {
+  ImBuf *src;
+  ImBuf *dst;
+  float start_uv[2];
+  float add_x[2];
+  float add_y[2];
+  rctf src_crop;
+} TransformUserData;
+
+static void imb_transform_calc_start_uv(const float transform_matrix[3][3], float r_start_uv[2])
+{
+  float orig[2];
+  orig[0] = 0.0f;
+  orig[1] = 0.0f;
+  mul_v2_m3v2(r_start_uv, transform_matrix, orig);
+}
+
+static void imb_transform_calc_add_x(const float transform_matrix[3][3],
+                                     const float start_uv[2],
+                                     const int width,
+                                     float r_add_x[2])
+{
+  float uv_max_x[2];
+  uv_max_x[0] = width;
+  uv_max_x[1] = 0.0f;
+  mul_v2_m3v2(r_add_x, transform_matrix, uv_max_x);
+  sub_v2_v2(r_add_x, start_uv);
+  mul_v2_fl(r_add_x, 1.0f / width);
+}
+
+static void imb_transform_calc_add_y(const float transform_matrix[3][3],
+                                     const float start_uv[2],
+                                     const int height,
+                                     float r_add_y[2])
+{
+  float uv_max_y[2];
+  uv_max_y[0] = 0.0f;
+  uv_max_y[1] = height;
+  mul_v2_m3v2(r_add_y, transform_matrix, uv_max_y);
+  sub_v2_v2(r_add_y, start_uv);
+  mul_v2_fl(r_add_y, 1.0f / height);
+}
+
+typedef void (*InterpolationColorFunction)(
+    struct ImBuf *in, unsigned char outI[4], float outF[4], float u, float v);
+BLI_INLINE void imb_transform_scanlines(const TransformUserData *user_data,
+                                        int start_scanline,
+                                        int num_scanlines,
+                                        InterpolationColorFunction interpolation)
+{
+  const int width = user_data->dst->x;
+
+  float next_line_start_uv[2];
+  madd_v2_v2v2fl(next_line_start_uv, user_data->start_uv, user_data->add_y, start_scanline);
+
+  unsigned char *outI = NULL;
+  float *outF = NULL;
+  pixel_from_buffer(user_data->dst, &outI, &outF, 0, start_scanline);
+
+  for (int yi = start_scanline; yi < start_scanline + num_scanlines; yi++) {
+    float uv[2];
+    copy_v2_v2(uv, next_line_start_uv);
+    add_v2_v2(next_line_start_uv, user_data->add_y);
+    for (int xi = 0; xi < width; xi++) {
+      if (uv[0] >= user_data->src_crop.xmin && uv[0] < user_data->src_crop.xmax &&
+          uv[1] >= user_data->src_crop.ymin && uv[1] < user_data->src_crop.ymax) {
+        interpolation(user_data->src, outI, outF, uv[0], uv[1]);
+      }
+      add_v2_v2(uv, user_data->add_x);
+      if (outI) {
+        outI += 4;
+      }
+      if (outF) {
+        outF += 4;
+      }
+    }
+  }
+}
+
+static void imb_transform_nearest_scanlines(void *custom_data,
+                                            int start_scanline,
+                                            int num_scanlines)
+{
+  const TransformUserData *user_data = custom_data;
+  imb_transform_scanlines(user_data, start_scanline, num_scanlines, nearest_interpolation_color);
+}
+
+static void imb_transform_bilinear_scanlines(void *custom_data,
+                                             int start_scanline,
+                                             int num_scanlines)
+{
+  const TransformUserData *user_data = custom_data;
+  imb_transform_scanlines(user_data, start_scanline, num_scanlines, bilinear_interpolation_color);
+}
+
+static ScanlineThreadFunc imb_transform_scanline_func(const eIMBInterpolationFilterMode filter)
+{
+  ScanlineThreadFunc scanline_func = NULL;
+  switch (filter) {
+    case IMB_FILTER_NEAREST:
+      scanline_func = imb_transform_nearest_scanlines;
+      break;
+    case IMB_FILTER_BILINEAR:
+      scanline_func = imb_transform_bilinear_scanlines;
+      break;
+  }
+  return scanline_func;
+}
+
+void IMB_transform(struct ImBuf *src,
+                   struct ImBuf *dst,
+                   float transform_matrix[3][3],
+                   struct rctf *src_crop,
+                   const eIMBInterpolationFilterMode filter)
+{
+  TransformUserData user_data;
+  user_data.src = src;
+  user_data.dst = dst;
+  user_data.src_crop = *src_crop;
+  imb_transform_calc_start_uv(transform_matrix, user_data.start_uv);
+  imb_transform_calc_add_x(transform_matrix, user_data.start_uv, src->x, user_data.add_x);
+  imb_transform_calc_add_y(transform_matrix, user_data.start_uv, src->y, user_data.add_y);
+  ScanlineThreadFunc scanline_func = imb_transform_scanline_func(filter);
+  IMB_processor_apply_threaded_scanlines(dst->y, scanline_func, &user_data);
+}
+
+/** \} */
+
+/* -------------------------------------------------------------------- */
 /** \name Threaded Image Processing
  * \{ */
 
diff --git a/source/blender/sequencer/intern/render.c b/source/blender/sequencer/intern/render.c
index 73a9569fcd5..b0a8605e922 100644
--- a/source/blender/sequencer/intern/render.c
+++ b/source/blender/sequencer/intern/render.c
@@ -37,6 +37,7 @@
 #include "BLI_linklist.h"
 #include "BLI_listbase.h"
 #include "BLI_path_util.h"
+#include "BLI_rect.h"
 
 #include "BKE_anim_data.h"
 #include "BKE_animsys.h"
@@ -470,29 +471,6 @@ static bool seq_input_have_to_preprocess(const SeqRenderData *context,
   return false;
 }
 
-typedef struct ImageTransformThreadInitData {
-  ImBuf *ibuf_source;
-  ImBuf *ibuf_out;
-  Sequence *seq;
-  float preview_scale_factor;
-  bool is_proxy_image;
-  bool for_render;
-} ImageTransformThreadInitData;
-
-typedef struct ImageTransformThreadData {
-  ImBuf *ibuf_source;
-  ImBuf *ibuf_out;
-  Sequence *seq;
-  /* image_scale_factor is used to scale proxies to correct preview size. */
-  float image_scale_factor;
-  /* Preview scale factor is needed to correct translation to match preview size. */
-  float preview_scale_factor;
-  float crop_scale_factor;
-  bool for_render;
-  int start_line;
-  int tot_line;
-} ImageTransformThreadData;
-
 /**
  * Effect, mask and scene in strip input strips are rendered in preview resolution.
  * They are already down-scaled. #input_preprocess() does not expect this to happen.
@@ -512,49 +490,21 @@ static bool seq_need_scale_to_render_size(const Sequence *seq, bool is_proxy_ima
   return false;
 }
 
-static void sequencer_image_crop_transform_init(void *handle_v,
-                                                int start_line,
-                                                int tot_line,
-                                                void *init_data_v)
-{
-  ImageTransformThreadData *handle = (ImageTransformThreadData *)handle_v;
-  const ImageTransformThreadInitData *init_data = (ImageTransformThreadInitData *)init_data_v;
-
-  handle->ibuf_source = init_data->ibuf_source;
-  handle->ibuf_out = init_data->ibuf_out;
-  handle->seq = init_data->seq;
-
-  handle->preview_scale_factor = init_data->preview_scale_factor;
-  if (seq_need_scale_to_render_size(init_data->seq, init_data->is_proxy_image)) {
-    handle->image_scale_factor = 1.0f;
-  }
-  else {
-    handle->image_scale_factor = handle->preview_scale_factor;
-  }
-
-  /* Proxy image is smaller, so crop values must be corrected by proxy scale factor.
-   * Proxy scale factor always matches preview_scale_factor. */
-  handle->crop_scale_factor = seq_need_scale_to_render_size(init_data->seq,
-                                                            init_data->is_proxy_image) ?
-                                  init_data->preview_scale_factor :
-                                  1.0f;
-
-  handle->for_render = init_data->for_render;
-  handle->start_line = start_line;
-  handle->tot_line = tot_line;
-}
-
-static void sequencer_image_crop_transform_matrix(const ImageTransformThreadData *data,
+static void sequencer_image_crop_transform_matrix(const Sequence *seq,
+                                                  const ImBuf *in,
+                                                  const ImBuf *out,
+                                                  const float image_scale_factor,
+                                                  const float preview_scale_factor,
                                                   float r_transform_matrix[3][3])
 {
-  const StripTransform *transform = data->seq->strip->transform;
-  const float scale_x = transform->scale_x * data->image_scale_factor;
-  const float scale_y = transform->scale_y * data->image_scale_factor;
-  const float image_center_offs_x = (data->ibuf_out->x - data->ibuf_source->x) / 2;
-  const float image_center_offs_y = (data->ibuf_out->y - data->ibuf_source->y) / 2;
-  const float translate_x = transform->xofs * data->preview_scale_factor + image_center_offs_x;
-  const float translate_y = transform->yofs * data->preview_scale_factor + image_center_offs_y;
-  const float pivot[2] = {data->ibuf_source->x / 2, data->ibuf_source->y / 2};
+  const StripTransform *transform = seq->strip->transform;
+  const float scale_x = transform->scale_x * image_scale_factor;
+  const float scale_y = transform->scale_y * image_scale_factor;
+  const float image_center_offs_x = (out->x - in->x) / 2;
+  const float image_center_offs_y = (out->y - in->y) / 2;
+  const float translate_x = transform->xofs * preview_scale_factor + image_center_offs_x;
+  const float translate_y = transform->yofs * preview_scale_factor + image_center_offs_y;
+  const float pivot[2] = {in->x / 2, in->y / 2};
   loc_rot_size_to_mat3(r_transform_matrix,
                        (const float[]){translate_x, translate_y},
                        transform->rotation,
@@ -563,108 +513,44 @@ static void sequencer_image_crop_transform_matrix(const ImageTransformThreadData
   invert_m3(r_transform_matrix);
 }
 
-static void sequencer_image_crop_transform_start_uv(const ImageTransformThreadData *data,
-                                                    const float transform_matrix[3][3],
-                                                    float r_start_uv[2])
+static void sequencer_image_crop_init(const Sequence *seq,
+                                             const ImBuf *in,
+                                             float crop_scale_factor,
+                                             rctf *r_crop)
 {
-  float orig[2];
-  orig[0] = 0.0f;
-  orig[1] = data->start_line;
-  mul_v2_m3v2(r_start_uv, transform_matrix, orig);
-}
+  const StripCrop *c = seq->strip->crop;
+  const int left = c->left * crop_scale_factor;
+  const int right = c->right * crop_scale_factor;
+  const int top = c->top * crop_scale_factor;
+  const int bottom = c->bottom * crop_scale_factor;
 
-static void sequencer_image_crop_transform_uv_min(const float transform_matrix[3][3],
-                                                  float r_uv_min[2])
-{
-  float orig[2];
-  orig[0] = 0.0f;
-  orig[1] = 0.0f;
-  mul_v2_m3v2(r_uv_min, transform_matrix, orig);
+  BLI_rctf_init(r_crop, left, in->x - right, bottom, in->y - top);
 }
 
-static void sequencer_image_crop_transform_delta_x(const ImageTransformThreadData *data,
-                                                   const float transform_matrix[3][3],
-                                                   const float uv_min[2],
-                                                   float r_add_x[2])
+static void sequencer_preprocess_transform_crop(
+    ImBuf *in, ImBuf *out, const SeqRenderData *context, Sequence *seq, const bool is_proxy_image)
 {
-  float uv_max_x[2];
-  uv_max_x[0] = data->ibuf_out->x;
-  uv_max_x[1] = 0.0f;
-  mul_v2_m3v2(r_add_x, transform_matrix, uv_max_x);
-  sub_v2_v2(r_add_x, uv_min);
-  mul_v2_fl(r_add_x, 1.0f / data->ibuf_out->x);
-}
+  const Scene *scene = context->scene;
+  const float preview_scale_factor = context->preview_render_size == SEQ_RENDER_SIZE_SCENE ?
+                                         (float)scene->r.size / 100 :
+                                         SEQ_rendersize_to_scale_factor(
+                                             context->preview_render_size);
+  const bool do_scale_to_render_size = seq_need_scale_to_render_size(seq, is_proxy_image);
+  const float image_scale_factor = do_scale_to_render_size ? 1.0f : preview_scale_factor;
 
-static void sequencer_image_crop_transform_delta_y(const ImageTransformThreadData *data,
-                                                   const float transform_matrix[3][3],
-                                                   const float uv_min[2],
-                                                   float r_add_y[2])
-{
-  float uv_max_y[2];
-  uv_max_y[0] = 0.0f;
-  uv_max_y[1] = data->ibuf_out->y;
-  mul_v2_m3v2(r_add_y, transform_matrix, uv_max_y);
-  sub_v2_v2(r_add_y, uv_min);
-  mul_v2_fl(r_add_y, 1.0f / data->ibuf_out->y);
-}
-
-static void sequencer_image_crop_transform_interpolation_coefs(
-    const ImageTransformThreadData *data, float r_start_uv[2], float r_add_x[2], float r_add_y[2])
-{
   float transform_matrix[3][3];
-  sequencer_image_crop_transform_matrix(data, transform_matrix);
-  sequencer_image_crop_transform_start_uv(data, transform_matrix, r_start_uv);
-  float uv_min[2];
-  sequencer_image_crop_transform_uv_min(transform_matrix, uv_min);
-  sequencer_image_crop_transform_delta_x(data, transform_matrix, uv_min, r_add_x);
-  sequencer_image_crop_transform_delta_y(data, transform_matrix, uv_min, r_add_y);
-}
+  sequencer_image_crop_transform_matrix(
+      seq, in, out, image_scale_factor, preview_scale_factor, transform_matrix);
 
-static void *sequencer_image_crop_transform_do_thread(void *data_v)
-{
-  const ImageTransformThreadData *data = data_v;
-
-  float last_uv[2];
-  float add_x[2];
-  float add_y[2];
-  sequencer_image_crop_transform_interpolation_coefs(data_v, last_uv, add_x, add_y);
-
-  /* Image crop is done by offsetting image boundary limits. */
-  const StripCrop *c = data->seq->strip->crop;
-  const int left = c->left * data->crop_scale_factor;
-  const int right = c->right * data->crop_scale_factor;
-  const int top = c->top * data->crop_scale_factor;
-  const int bottom = c->bottom * data->crop_scale_factor;
-
-  const float source_pixel_range_max[2] = {data->ibuf_source->x - right,
-                                           data->ibuf_source->y - top};
-  const float source_pixel_range_min[2] = {left, bottom};
-
-  const int width = data->ibuf_out->x;
-
-  float uv[2];
-  for (int yi = data->start_line; yi < data->start_line + data->tot_line; yi++) {
-    copy_v2_v2(uv, last_uv);
-    add_v2_v2(last_uv, add_y);
-    for (int xi = 0; xi < width; xi++) {
-      if (source_pixel_range_min[0] >= uv[0] || uv[0] >= source_pixel_range_max[0] ||
-          source_pixel_range_min[1] >= uv[1] || uv[1] >= source_pixel_range_max[1]) {
-        add_v2_v2(uv, add_x);
-        continue;
-      }
-
-      if (data->for_render) {
-        bilinear_interpolation(data->ibuf_source, data->ibuf_out, uv[0], uv[1], xi, yi);
-      }
-      else {
-        nearest_interpolation(data->ibuf_source, data->ibuf_out, uv[0], uv[1], xi, yi);
-      }
-
-      add_v2_v2(uv, add_x);
-    }
-  }
+  /* Proxy image is smaller, so crop values must be corrected by proxy scale factor.
+   * Proxy scale factor always matches preview_scale_factor. */
+  rctf source_crop;
+  const float crop_scale_factor = do_scale_to_render_size ? preview_scale_factor : 1.0f;
+  sequencer_image_crop_init(seq, in, crop_scale_factor, &source_crop);
 
-  return NULL;
+  const eIMBInterpolationFilterMode filter = context->for_render ? IMB_FILTER_BILINEAR :
+                                                                   IMB_FILTER_NEAREST;
+  IMB_transform(in, out, transform_matrix, &source_crop, filter);
 }
 
 static void multibuf(ImBuf *ibuf, const float fmul)
@@ -726,27 +612,8 @@ static ImBuf *input_preprocess(const SeqRenderData *context,
     const int y = context->recty;
     preprocessed_ibuf = IMB_allocImBuf(x, y, 32, ibuf->rect_float ? IB_rectfloat : IB_rect);
 
-    ImageTransformThreadInitData init_data = {NULL};
-    init_data.ibuf_source = ibuf;
-    init_data.ibuf_out = preprocessed_ibuf;
-    init_data.seq = seq;
-    init_data.is_proxy_image = is_proxy_image;
-
-    /* Get scale factor if preview resolution doesn't match project resolution. */
-    if (context->preview_render_size == SEQ_RENDER_SIZE_SCENE) {
-      init_data.preview_scale_factor = (float)scene->r.size / 100;
-    }
-    else {
-      init_data.preview_scale_factor = SEQ_rendersize_to_scale_factor(
-          context->preview_render_size);
-    }
+    sequencer_preprocess_transform_crop(ibuf, preprocessed_ibuf, context, seq, is_proxy_image);
 
-    init_data.for_render = context->for_render;
-    IMB_processor_apply_threaded(context->recty,
-                                 sizeof(ImageTransformThreadData),
-                                 &init_data,
-                                 sequencer_image_crop_transform_init,
-                                 sequencer_image_crop_transform_do_thread);
     seq_imbuf_assign_spaces(scene, preprocessed_ibuf);
     IMB_metadata_copy(preprocessed_ibuf, ibuf);
     IMB_freeImBuf(ibuf);
diff --git a/source/tools b/source/tools
-Subproject f99d29ae3e6ad44d45d79309454c45f8088781a
+Subproject 01f51a0e551ab730f0934dc6488613690ac4bf8