Fix T88237: Prefetch crash on rendering scene strip

Prefetch needs to avoid rendering scene strips, because

 - Rendering in background needs own dependency graph, which fails to
   initialize from evaluated data.
 - This locks UI and can make it unresponsive for long time periods.

In T88237 prefetch failed to avoid scene strip, because of effect strip
was attached to scene strip.

Ensure, that no effect that is attached to scene strip either directly
or indirectly would be rendered.

Reviewed By: sergey

Differential Revision: https://developer.blender.org/D11247

1 files changed, 75 insertions, 47 deletions

diff --git a/source/blender/sequencer/intern/prefetch.c b/source/blender/sequencer/intern/prefetch.c
index 15609a76f5c..dd2d828415c 100644
--- a/source/blender/sequencer/intern/prefetch.c
+++ b/source/blender/sequencer/intern/prefetch.c
@@ -53,7 +53,9 @@
 #include "DEG_depsgraph_debug.h"
 #include "DEG_depsgraph_query.h"
 
+#include "SEQ_iterator.h"
 #include "SEQ_prefetch.h"
+#include "SEQ_relations.h"
 #include "SEQ_render.h"
 #include "SEQ_sequencer.h"
 
@@ -359,67 +361,93 @@ void seq_prefetch_free(Scene *scene)
   scene->ed->prefetch_job = NULL;
 }
 
-/* Skip frame if we need to render 3D scene strip. Rendering 3D scene requires main lock or setting
- * up render job that doesn't have API to do openGL renders which can be used for sequencer. */
-static bool seq_prefetch_do_skip_frame(PrefetchJob *pfjob, ListBase *seqbase)
+static bool seq_prefetch_seq_has_disk_cache(PrefetchJob *pfjob,
+                                            Sequence *seq,
+                                            bool can_have_final_image)
 {
-  float cfra = seq_prefetch_cfra(pfjob);
-  Sequence *seq_arr[MAXSEQ + 1];
-  int count = seq_get_shown_sequences(seqbase, cfra, 0, seq_arr);
   SeqRenderData *ctx = &pfjob->context_cpy;
-  ImBuf *ibuf = NULL;
+  float cfra = seq_prefetch_cfra(pfjob);
 
-  /* Disable prefetching 3D scene strips, but check for disk cache. */
-  for (int i = 0; i < count; i++) {
-    if (seq_arr[i]->type == SEQ_TYPE_META &&
-        seq_prefetch_do_skip_frame(pfjob, &seq_arr[i]->seqbase)) {
-      return true;
-    }
+  ImBuf *ibuf = seq_cache_get(ctx, seq, cfra, SEQ_CACHE_STORE_PREPROCESSED);
+  if (ibuf != NULL) {
+    IMB_freeImBuf(ibuf);
+    return true;
+  }
 
-    if (seq_arr[i]->type == SEQ_TYPE_SCENE && (seq_arr[i]->flag & SEQ_SCENE_STRIPS) == 0) {
-      int cached_types = 0;
+  ibuf = seq_cache_get(ctx, seq, cfra, SEQ_CACHE_STORE_RAW);
+  if (ibuf != NULL) {
+    IMB_freeImBuf(ibuf);
+    return true;
+  }
 
-      ibuf = seq_cache_get(ctx, seq_arr[i], cfra, SEQ_CACHE_STORE_FINAL_OUT);
-      if (ibuf != NULL) {
-        cached_types |= SEQ_CACHE_STORE_FINAL_OUT;
-        IMB_freeImBuf(ibuf);
-        ibuf = NULL;
-      }
+  if (!can_have_final_image) {
+    return false;
+  }
 
-      ibuf = seq_cache_get(ctx, seq_arr[i], cfra, SEQ_CACHE_STORE_COMPOSITE);
-      if (ibuf != NULL) {
-        cached_types |= SEQ_CACHE_STORE_COMPOSITE;
-        IMB_freeImBuf(ibuf);
-        ibuf = NULL;
-      }
+  ibuf = seq_cache_get(ctx, seq, cfra, SEQ_CACHE_STORE_FINAL_OUT);
+  if (ibuf != NULL) {
+    IMB_freeImBuf(ibuf);
+    return true;
+  }
 
-      ibuf = seq_cache_get(ctx, seq_arr[i], cfra, SEQ_CACHE_STORE_PREPROCESSED);
-      if (ibuf != NULL) {
-        cached_types |= SEQ_CACHE_STORE_PREPROCESSED;
-        IMB_freeImBuf(ibuf);
-        ibuf = NULL;
-      }
+  return false;
+}
 
-      ibuf = seq_cache_get(ctx, seq_arr[i], cfra, SEQ_CACHE_STORE_RAW);
-      if (ibuf != NULL) {
-        cached_types |= SEQ_CACHE_STORE_RAW;
-        IMB_freeImBuf(ibuf);
-        ibuf = NULL;
-      }
+static bool seq_prefetch_scene_strip_is_rendered(PrefetchJob *pfjob,
+                                                 ListBase *seqbase,
+                                                 SeqCollection *scene_strips,
+                                                 bool is_recursive_check)
+{
+  float cfra = seq_prefetch_cfra(pfjob);
+  Sequence *seq_arr[MAXSEQ + 1];
+  int count = seq_get_shown_sequences(seqbase, cfra, 0, seq_arr);
 
-      if ((cached_types & (SEQ_CACHE_STORE_RAW | SEQ_CACHE_STORE_PREPROCESSED)) != 0) {
-        continue;
-      }
+  /* Iterate over rendered strips. */
+  for (int i = 0; i < count; i++) {
+    Sequence *seq = seq_arr[i];
+    if (seq->type == SEQ_TYPE_META &&
+        seq_prefetch_scene_strip_is_rendered(pfjob, &seq->seqbase, scene_strips, true)) {
+      return true;
+    }
+
+    /* Disable prefetching 3D scene strips, but check for disk cache. */
+    if (seq->type == SEQ_TYPE_SCENE && (seq->flag & SEQ_SCENE_STRIPS) == 0 &&
+        !seq_prefetch_seq_has_disk_cache(pfjob, seq, !is_recursive_check)) {
+      return true;
+    }
 
-      /* It is only safe to use these cache types if strip is last in stack. */
-      if (i == count - 1 && (cached_types & SEQ_CACHE_STORE_FINAL_OUT) != 0) {
-        continue;
+    /* Check if strip is effect of scene strip or uses it as modifier. This is recursive check. */
+    Sequence *seq_scene;
+    SEQ_ITERATOR_FOREACH (seq_scene, scene_strips) {
+      if (SEQ_relations_render_loop_check(seq, seq_scene)) {
+        return true;
       }
+    }
+  }
+  return false;
+}
 
-      return true;
+static SeqCollection *query_scene_strips(ListBase *seqbase)
+{
+  SeqCollection *collection = SEQ_query_all_strips_recursive(seqbase);
+  LISTBASE_FOREACH (Sequence *, seq, seqbase) {
+    if (seq->type != SEQ_TYPE_SCENE || (seq->flag & SEQ_SCENE_STRIPS) != 0) {
+      SEQ_collection_remove_strip(seq, collection);
     }
   }
+  return collection;
+}
 
+/* Prefetch must avoid rendering scene strips, because rendering in background locks UI and can
+ * make it unresponsive for long time periods. */
+static bool seq_prefetch_must_skip_frame(PrefetchJob *pfjob, ListBase *seqbase)
+{
+  SeqCollection *scene_strips = query_scene_strips(seqbase);
+  if (seq_prefetch_scene_strip_is_rendered(pfjob, seqbase, scene_strips, false)) {
+    SEQ_collection_free(scene_strips);
+    return true;
+  }
+  SEQ_collection_free(scene_strips);
   return false;
 }
 
@@ -464,7 +492,7 @@ static void *seq_prefetch_frames(void *job)
     pfjob->scene_eval->ed->prefetch_job = pfjob;
 
     ListBase *seqbase = SEQ_active_seqbase_get(SEQ_editing_get(pfjob->scene, false));
-    if (seq_prefetch_do_skip_frame(pfjob, seqbase)) {
+    if (seq_prefetch_must_skip_frame(pfjob, seqbase)) {
       pfjob->num_frames_prefetched++;
       continue;
     }