VSE: Use early out for aplha over blending

When scaling down image, users expect to see background, which doesn't
currently happen in VSE. This is because strips use cross blend mode by
default, because alpha over is much slower. Reason is, because any area
of image can be transparent, and therefore it can't have early out
implemented in a way that cross blend mode can.

Flag images rendered by codecs that don't support transparency as fully
opaque and implement a form of early out for alpha over blend mode.

When rendering image stack, 2-input effects are ignored on the
"way down". Alpha over needs rendered overlay image to decide whether it
will use only overlay or background too. Therefore overlay can be
rendered safely before it is used. Image flags can be checked and it can
be freed if needed. Freeing doesn't cause any performance degradation,
because image is always stored in cache.

This feature does not improve blend mode performance. In summary, it
only allowes for having alpha over blend mode on background images
without suffering from lower performance.

Reviewed By: sergey

Differential Revision: https://developer.blender.org/D12914

1 files changed, 65 insertions, 0 deletions

diff --git a/source/blender/sequencer/intern/render.c b/source/blender/sequencer/intern/render.c
index 12d97d7e3e3..2246336369b 100644
--- a/source/blender/sequencer/intern/render.c
+++ b/source/blender/sequencer/intern/render.c
@@ -466,6 +466,45 @@ static void sequencer_thumbnail_transform(ImBuf *in, ImBuf *out)
   IMB_transform(in, out, transform_matrix, &source_crop, IMB_FILTER_NEAREST);
 }
 
+/* Check whether transform introduces transparent ares in the result (happens when the transformed
+ * image does not fully cover the render frame).
+ *
+ * The check is done by checking whether all corners of viewport fit inside of the transformed
+ * image. If they do not the image will have transparent areas. */
+static bool seq_image_transform_transparency_gained(const SeqRenderData *context, Sequence *seq)
+{
+  Scene *scene = context->scene;
+  const int x = context->rectx;
+  const int y = context->recty;
+
+  float seq_image_quad[4][2];
+  SEQ_image_transform_final_quad_get(scene, seq, seq_image_quad);
+  for (int i = 0; i < 4; i++) {
+    add_v2_v2(seq_image_quad[i], (float[]){x / 2, y / 2});
+  }
+
+  return !isect_point_quad_v2((float[]){x, y},
+                              seq_image_quad[0],
+                              seq_image_quad[1],
+                              seq_image_quad[2],
+                              seq_image_quad[3]) ||
+         !isect_point_quad_v2((float[]){0, y},
+                              seq_image_quad[0],
+                              seq_image_quad[1],
+                              seq_image_quad[2],
+                              seq_image_quad[3]) ||
+         !isect_point_quad_v2((float[]){x, 0},
+                              seq_image_quad[0],
+                              seq_image_quad[1],
+                              seq_image_quad[2],
+                              seq_image_quad[3]) ||
+         !isect_point_quad_v2((float[]){0, 0},
+                              seq_image_quad[0],
+                              seq_image_quad[1],
+                              seq_image_quad[2],
+                              seq_image_quad[3]);
+}
+
 static void sequencer_preprocess_transform_crop(
     ImBuf *in, ImBuf *out, const SeqRenderData *context, Sequence *seq, const bool is_proxy_image)
 {
@@ -490,6 +529,13 @@ static void sequencer_preprocess_transform_crop(
   const eIMBInterpolationFilterMode filter = context->for_render ? IMB_FILTER_BILINEAR :
                                                                    IMB_FILTER_NEAREST;
   IMB_transform(in, out, transform_matrix, &source_crop, filter);
+
+  if (!seq_image_transform_transparency_gained(context, seq)) {
+    out->planes = in->planes;
+  }
+  else {
+    out->planes = R_IMF_PLANES_RGBA;
+  }
 }
 
 static void multibuf(ImBuf *ibuf, const float fmul)
@@ -525,6 +571,10 @@ static void multibuf(ImBuf *ibuf, const float fmul)
       rt_float += 4;
     }
   }
+
+  if (ELEM(ibuf->planes, R_IMF_PLANES_BW, R_IMF_PLANES_RGB) && fmul < 1.0f) {
+    ibuf->planes = R_IMF_PLANES_RGBA;
+  }
 }
 
 static ImBuf *input_preprocess(const SeqRenderData *context,
@@ -1804,6 +1854,20 @@ static ImBuf *seq_render_strip_stack(const SeqRenderData *context,
 
     early_out = seq_get_early_out_for_blend_mode(seq);
 
+    /* Early out for alpha over. It requires image to be rendered, so it can't use
+     * `seq_get_early_out_for_blend_mode`. */
+    if (out == NULL && seq->blend_mode == SEQ_TYPE_ALPHAOVER && seq->blend_opacity == 100.0f) {
+      ImBuf *test = seq_render_strip(context, state, seq, timeline_frame);
+      if (ELEM(test->planes, R_IMF_PLANES_BW, R_IMF_PLANES_RGB)) {
+        early_out = EARLY_USE_INPUT_2;
+      }
+      else {
+        early_out = EARLY_DO_EFFECT;
+      }
+      /* Free the image. It is stored in cache, so this doesn't affect performance. */
+      IMB_freeImBuf(test);
+    }
+
     switch (early_out) {
       case EARLY_NO_INPUT:
       case EARLY_USE_INPUT_2:
@@ -1828,6 +1892,7 @@ static ImBuf *seq_render_strip_stack(const SeqRenderData *context,
         }
         break;
     }
+
     if (out) {
       break;
     }