Fix T71300: Crash on rendering scene recursively from sequencer

Adding recursive scenes has been disabled, but old files still can be opened.

Add check if scene will render itself.
Opening such file will produce warning on open and error on running render.

Reviewed By: campbellbarton

Differential Revision: https://developer.blender.org/D7562

3 files changed, 74 insertions, 0 deletions

diff --git a/source/blender/blenkernel/BKE_sequencer.h b/source/blender/blenkernel/BKE_sequencer.h
index 939fcb33b2d..5f0c10c9a5f 100644
--- a/source/blender/blenkernel/BKE_sequencer.h
+++ b/source/blender/blenkernel/BKE_sequencer.h
@@ -35,6 +35,7 @@ struct GSet;
 struct ImBuf;
 struct Main;
 struct Mask;
+struct ReportList;
 struct Scene;
 struct Sequence;
 struct SequenceModifierData;
@@ -598,6 +599,7 @@ void BKE_sequencer_color_balance_apply(struct StripColorBalance *cb,
                                        struct ImBuf *mask_input);
 
 void BKE_sequencer_all_free_anim_ibufs(struct Scene *scene, int cfra);
+bool BKE_sequencer_check_scene_recursion(struct Scene *scene, struct ReportList *reports);
 
 #ifdef __cplusplus
 }
diff --git a/source/blender/blenkernel/intern/sequencer.c b/source/blender/blenkernel/intern/sequencer.c
index d3d885fccd9..46edcea26c0 100644
--- a/source/blender/blenkernel/intern/sequencer.c
+++ b/source/blender/blenkernel/intern/sequencer.c
@@ -39,6 +39,7 @@
 #include "DNA_sequence_types.h"
 #include "DNA_sound_types.h"
 #include "DNA_space_types.h"
+#include "DNA_windowmanager_types.h"
 
 #include "BLI_fileops.h"
 #include "BLI_linklist.h"
@@ -69,6 +70,7 @@
 #include "BKE_main.h"
 #include "BKE_mask.h"
 #include "BKE_movieclip.h"
+#include "BKE_report.h"
 #include "BKE_scene.h"
 #include "BKE_sequencer.h"
 #include "BKE_sequencer_offscreen.h"
@@ -3462,6 +3464,11 @@ static ImBuf *seq_render_scene_strip(const SeqRenderData *context,
     return NULL;
   }
 
+  /* Prevent rendering scene recursively. */
+  if (seq->scene == context->scene) {
+    return NULL;
+  }
+
   scene = seq->scene;
   frame = (double)scene->r.sfra + (double)nr + (double)seq->anim_startofs;
 
@@ -5970,3 +5977,62 @@ void BKE_sequencer_all_free_anim_ibufs(Scene *scene, int cfra)
   sequencer_all_free_anim_ibufs(&ed->seqbase, cfra);
   BKE_sequencer_cache_cleanup(scene);
 }
+
+static bool sequencer_seq_generates_image(Sequence *seq)
+{
+  switch (seq->type) {
+    case SEQ_TYPE_IMAGE:
+    case SEQ_TYPE_SCENE:
+    case SEQ_TYPE_MOVIE:
+    case SEQ_TYPE_MOVIECLIP:
+    case SEQ_TYPE_MASK:
+    case SEQ_TYPE_COLOR:
+    case SEQ_TYPE_TEXT:
+      return true;
+  }
+  return false;
+}
+
+static Sequence *sequencer_check_scene_recursion(Scene *scene, ListBase *seqbase)
+{
+  LISTBASE_FOREACH (Sequence *, seq, seqbase) {
+    if (seq->type == SEQ_TYPE_SCENE && seq->scene == scene) {
+      return seq;
+    }
+
+    if (seq->type == SEQ_TYPE_META && sequencer_check_scene_recursion(scene, &seq->seqbase)) {
+      return seq;
+    }
+  }
+
+  return NULL;
+}
+
+bool BKE_sequencer_check_scene_recursion(Scene *scene, ReportList *reports)
+{
+  Editing *ed = BKE_sequencer_editing_get(scene, false);
+  if (ed == NULL) {
+    return false;
+  }
+
+  Sequence *recursive_seq = sequencer_check_scene_recursion(scene, &ed->seqbase);
+
+  if (recursive_seq != NULL) {
+    BKE_reportf(reports,
+                RPT_WARNING,
+                "Recursion detected in video sequencer. Strip %s at frame %d will not be rendered",
+                recursive_seq->name + 2,
+                recursive_seq->startdisp);
+
+    LISTBASE_FOREACH (Sequence *, seq, &ed->seqbase) {
+      if (seq->type != SEQ_TYPE_SCENE && sequencer_seq_generates_image(seq)) {
+        /* There are other strips to render, so render them. */
+        return false;
+      }
+    }
+    /* No other strips to render - cancel operator. */
+    return true;
+  }
+
+  return false;
+}
diff --git a/source/blender/editors/render/render_internal.c b/source/blender/editors/render/render_internal.c
index 1237c2c2eda..17049fdb28b 100644
--- a/source/blender/editors/render/render_internal.c
+++ b/source/blender/editors/render/render_internal.c
@@ -945,6 +945,12 @@ static int screen_render_invoke(bContext *C, wmOperator *op, const wmEvent *even
     return OPERATOR_CANCELLED;
   }
 
+  /* Reports are done inside check function, and it will return false if there are other strips to
+   * render. */
+  if ((scene->r.scemode & R_DOSEQ) && BKE_sequencer_check_scene_recursion(scene, op->reports)) {
+    return OPERATOR_CANCELLED;
+  }
+
   /* stop all running jobs, except screen one. currently previews frustrate Render */
   WM_jobs_kill_all_except(CTX_wm_manager(C), CTX_wm_screen(C));