VSE: Use early out for aplha over blending

When scaling down image, users expect to see background, which doesn't
currently happen in VSE. This is because strips use cross blend mode by
default, because alpha over is much slower. Reason is, because any area
of image can be transparent, and therefore it can't have early out
implemented in a way that cross blend mode can.

Flag images rendered by codecs that don't support transparency as fully
opaque and implement a form of early out for alpha over blend mode.

When rendering image stack, 2-input effects are ignored on the
"way down". Alpha over needs rendered overlay image to decide whether it
will use only overlay or background too. Therefore overlay can be
rendered safely before it is used. Image flags can be checked and it can
be freed if needed. Freeing doesn't cause any performance degradation,
because image is always stored in cache.

This feature does not improve blend mode performance. In summary, it
only allowes for having alpha over blend mode on background images
without suffering from lower performance.

Reviewed By: sergey

Differential Revision: https://developer.blender.org/D12914

3 files changed, 80 insertions, 2 deletions

diff --git a/source/blender/sequencer/intern/effects.c b/source/blender/sequencer/intern/effects.c
index 37d1f6832bb..6a6889c3679 100644
--- a/source/blender/sequencer/intern/effects.c
+++ b/source/blender/sequencer/intern/effects.c
@@ -2942,6 +2942,9 @@ static ImBuf *do_solid_color(const SeqRenderData *context,
       }
     }
   }
+
+  out->planes = R_IMF_PLANES_RGB;
+
   return out;
 }
 
@@ -4024,6 +4027,14 @@ static int early_out_mul_input2(Sequence *UNUSED(seq), float facf0, float facf1)
   return EARLY_DO_EFFECT;
 }
 
+static int early_out_mul_input1(Sequence *UNUSED(seq), float facf0, float facf1)
+{
+  if (facf0 == 0.0f && facf1 == 0.0f) {
+    return EARLY_USE_INPUT_2;
+  }
+  return EARLY_DO_EFFECT;
+}
+
 static void get_default_fac_noop(Sequence *UNUSED(seq),
                                  float UNUSED(timeline_frame),
                                  float *facf0,
@@ -4134,6 +4145,7 @@ static struct SeqEffectHandle get_sequence_effect_impl(int seq_type)
       rval.multithreaded = true;
       rval.init = init_alpha_over_or_under;
       rval.execute_slice = do_alphaover_effect;
+      rval.early_out = early_out_mul_input1;
       break;
     case SEQ_TYPE_OVERDROP:
       rval.multithreaded = true;
diff --git a/source/blender/sequencer/intern/render.c b/source/blender/sequencer/intern/render.c
index 12d97d7e3e3..2246336369b 100644
--- a/source/blender/sequencer/intern/render.c
+++ b/source/blender/sequencer/intern/render.c
@@ -466,6 +466,45 @@ static void sequencer_thumbnail_transform(ImBuf *in, ImBuf *out)
   IMB_transform(in, out, transform_matrix, &source_crop, IMB_FILTER_NEAREST);
 }
 
+/* Check whether transform introduces transparent ares in the result (happens when the transformed
+ * image does not fully cover the render frame).
+ *
+ * The check is done by checking whether all corners of viewport fit inside of the transformed
+ * image. If they do not the image will have transparent areas. */
+static bool seq_image_transform_transparency_gained(const SeqRenderData *context, Sequence *seq)
+{
+  Scene *scene = context->scene;
+  const int x = context->rectx;
+  const int y = context->recty;
+
+  float seq_image_quad[4][2];
+  SEQ_image_transform_final_quad_get(scene, seq, seq_image_quad);
+  for (int i = 0; i < 4; i++) {
+    add_v2_v2(seq_image_quad[i], (float[]){x / 2, y / 2});
+  }
+
+  return !isect_point_quad_v2((float[]){x, y},
+                              seq_image_quad[0],
+                              seq_image_quad[1],
+                              seq_image_quad[2],
+                              seq_image_quad[3]) ||
+         !isect_point_quad_v2((float[]){0, y},
+                              seq_image_quad[0],
+                              seq_image_quad[1],
+                              seq_image_quad[2],
+                              seq_image_quad[3]) ||
+         !isect_point_quad_v2((float[]){x, 0},
+                              seq_image_quad[0],
+                              seq_image_quad[1],
+                              seq_image_quad[2],
+                              seq_image_quad[3]) ||
+         !isect_point_quad_v2((float[]){0, 0},
+                              seq_image_quad[0],
+                              seq_image_quad[1],
+                              seq_image_quad[2],
+                              seq_image_quad[3]);
+}
+
 static void sequencer_preprocess_transform_crop(
     ImBuf *in, ImBuf *out, const SeqRenderData *context, Sequence *seq, const bool is_proxy_image)
 {
@@ -490,6 +529,13 @@ static void sequencer_preprocess_transform_crop(
   const eIMBInterpolationFilterMode filter = context->for_render ? IMB_FILTER_BILINEAR :
                                                                    IMB_FILTER_NEAREST;
   IMB_transform(in, out, transform_matrix, &source_crop, filter);
+
+  if (!seq_image_transform_transparency_gained(context, seq)) {
+    out->planes = in->planes;
+  }
+  else {
+    out->planes = R_IMF_PLANES_RGBA;
+  }
 }
 
 static void multibuf(ImBuf *ibuf, const float fmul)
@@ -525,6 +571,10 @@ static void multibuf(ImBuf *ibuf, const float fmul)
       rt_float += 4;
     }
   }
+
+  if (ELEM(ibuf->planes, R_IMF_PLANES_BW, R_IMF_PLANES_RGB) && fmul < 1.0f) {
+    ibuf->planes = R_IMF_PLANES_RGBA;
+  }
 }
 
 static ImBuf *input_preprocess(const SeqRenderData *context,
@@ -1804,6 +1854,20 @@ static ImBuf *seq_render_strip_stack(const SeqRenderData *context,
 
     early_out = seq_get_early_out_for_blend_mode(seq);
 
+    /* Early out for alpha over. It requires image to be rendered, so it can't use
+     * `seq_get_early_out_for_blend_mode`. */
+    if (out == NULL && seq->blend_mode == SEQ_TYPE_ALPHAOVER && seq->blend_opacity == 100.0f) {
+      ImBuf *test = seq_render_strip(context, state, seq, timeline_frame);
+      if (ELEM(test->planes, R_IMF_PLANES_BW, R_IMF_PLANES_RGB)) {
+        early_out = EARLY_USE_INPUT_2;
+      }
+      else {
+        early_out = EARLY_DO_EFFECT;
+      }
+      /* Free the image. It is stored in cache, so this doesn't affect performance. */
+      IMB_freeImBuf(test);
+    }
+
     switch (early_out) {
       case EARLY_NO_INPUT:
       case EARLY_USE_INPUT_2:
@@ -1828,6 +1892,7 @@ static ImBuf *seq_render_strip_stack(const SeqRenderData *context,
         }
         break;
     }
+
     if (out) {
       break;
     }
diff --git a/source/blender/sequencer/intern/strip_transform.c b/source/blender/sequencer/intern/strip_transform.c
index becf44a7a8e..1a67150b60c 100644
--- a/source/blender/sequencer/intern/strip_transform.c
+++ b/source/blender/sequencer/intern/strip_transform.c
@@ -520,7 +520,8 @@ static void seq_image_transform_quad_get_ex(const Scene *scene,
 }
 
 /**
- * Get 4 corner points of strip image, optionally without rotation component applied
+ * Get 4 corner points of strip image, optionally without rotation component applied.
+ * Corner vectors are in viewport space.
  *
  * \param scene: Scene in which strips are located
  * \param seq: Sequence to calculate image transform origin
@@ -536,7 +537,7 @@ void SEQ_image_transform_quad_get(const Scene *scene,
 }
 
 /**
- * Get 4 corner points of strip image.
+ * Get 4 corner points of strip image. Corner vectors are in viewport space.
  *
  * \param scene: Scene in which strips are located
  * \param seq: Sequence to calculate image transform origin