Fix T78028: crash with grease pencil and save buffers

Perform grease pencil rendering delayed in this case, as there are no render
buffers available for compositing. This keeps memory usage lower, but does
involve multiple depsgraph evaluation. This seems in line with the intent of
the save buffers feature, to use minimal memory.

3 files changed, 87 insertions, 44 deletions

diff --git a/source/blender/draw/intern/draw_manager.c b/source/blender/draw/intern/draw_manager.c
index f995582149a..a3c8a4a669f 100644
--- a/source/blender/draw/intern/draw_manager.c
+++ b/source/blender/draw/intern/draw_manager.c
@@ -1735,11 +1735,10 @@ static void DRW_render_gpencil_to_image(RenderEngine *engine,
 
 void DRW_render_gpencil(struct RenderEngine *engine, struct Depsgraph *depsgraph)
 {
-  /* Early out if there are no grease pencil objects, especially important
-   * to avoid failing in in background renders without OpenGL context. */
-  if (!DRW_render_check_grease_pencil(depsgraph)) {
-    return;
-  }
+  /* This function should only be called if there are are grease pencil objects,
+   * especially important to avoid failing in in background renders without OpenGL
+   * context. */
+  BLI_assert(DRW_render_check_grease_pencil(depsgraph));
 
   Scene *scene = DEG_get_evaluated_scene(depsgraph);
   ViewLayer *view_layer = DEG_get_evaluated_view_layer(depsgraph);
diff --git a/source/blender/render/extern/include/RE_engine.h b/source/blender/render/extern/include/RE_engine.h
index e53f33eacff..a153c622af8 100644
--- a/source/blender/render/extern/include/RE_engine.h
+++ b/source/blender/render/extern/include/RE_engine.h
@@ -149,6 +149,7 @@ typedef struct RenderEngine {
 
   /* Depsgraph */
   struct Depsgraph *depsgraph;
+  bool has_grease_pencil;
 
   /* callback for render pass query */
   ThreadMutex update_render_passes_mutex;
diff --git a/source/blender/render/intern/source/external_engine.c b/source/blender/render/intern/source/external_engine.c
index 440c54f5eeb..31b20008b52 100644
--- a/source/blender/render/intern/source/external_engine.c
+++ b/source/blender/render/intern/source/external_engine.c
@@ -617,6 +617,8 @@ static void engine_depsgraph_init(RenderEngine *engine, ViewLayer *view_layer)
   else {
     BKE_scene_graph_update_for_newframe(engine->depsgraph);
   }
+
+  engine->has_grease_pencil = DRW_render_check_grease_pencil(engine->depsgraph);
 }
 
 static void engine_depsgraph_free(RenderEngine *engine)
@@ -749,10 +751,63 @@ bool RE_bake_engine(Render *re,
 
 /* Render */
 
+static void engine_render_view_layer(Render *re,
+                                     RenderEngine *engine,
+                                     ViewLayer *view_layer_iter,
+                                     const bool use_engine,
+                                     const bool use_grease_pencil)
+{
+  /* Lock UI so scene can't be edited while we read from it in this render thread. */
+  if (re->draw_lock) {
+    re->draw_lock(re->dlh, 1);
+  }
+
+  /* Create depsgraph with scene evaluated at render resolution. */
+  ViewLayer *view_layer = BLI_findstring(
+      &re->scene->view_layers, view_layer_iter->name, offsetof(ViewLayer, name));
+  engine_depsgraph_init(engine, view_layer);
+
+  /* Sync data to engine, within draw lock so scene data can be accessed safely. */
+  if (use_engine) {
+    if (engine->type->update) {
+      engine->type->update(engine, re->main, engine->depsgraph);
+    }
+  }
+
+  if (re->draw_lock) {
+    re->draw_lock(re->dlh, 0);
+  }
+
+  /* Perform render with engine. */
+  if (use_engine) {
+    if (engine->type->flag & RE_USE_GPU_CONTEXT) {
+      DRW_render_context_enable(engine->re);
+    }
+
+    engine->type->render(engine, engine->depsgraph);
+
+    if (engine->type->flag & RE_USE_GPU_CONTEXT) {
+      DRW_render_context_disable(engine->re);
+    }
+  }
+
+  /* Optionally composite grease pencil over render result. */
+  if (engine->has_grease_pencil && use_grease_pencil && !re->result->do_exr_tile) {
+    /* NOTE: External engine might have been requested to free its
+     * dependency graph, which is only allowed if there is no grease
+     * pencil (pipeline is taking care of that). */
+    if (!RE_engine_test_break(engine) && engine->depsgraph != NULL) {
+      DRW_render_gpencil(engine, engine->depsgraph);
+    }
+  }
+
+  /* Free dependency graph, if engine has not done it already. */
+  engine_depsgraph_free(engine);
+}
+
 int RE_engine_render(Render *re, int do_all)
 {
   RenderEngineType *type = RE_engines_find(re->r.engine);
-  RenderEngine *engine;
   bool persistent_data = (re->r.mode & R_PERSISTENT_DATA) != 0;
 
   /* verify if we can render */
@@ -815,7 +870,7 @@ int RE_engine_render(Render *re, int do_all)
   re->i.totface = re->i.totvert = re->i.totlamp = 0;
 
   /* render */
-  engine = re->engine;
+  RenderEngine *engine = re->engine;
 
   if (!engine) {
     engine = RE_engine_create(type);
@@ -853,45 +908,16 @@ int RE_engine_render(Render *re, int do_all)
     re->draw_lock(re->dlh, 0);
   }
 
+  /* Render view layers. */
+  bool delay_grease_pencil = false;
+
   if (type->render) {
     FOREACH_VIEW_LAYER_TO_RENDER_BEGIN (re, view_layer_iter) {
-      if (re->draw_lock) {
-        re->draw_lock(re->dlh, 1);
-      }
-
-      ViewLayer *view_layer = BLI_findstring(
-          &re->scene->view_layers, view_layer_iter->name, offsetof(ViewLayer, name));
-      engine_depsgraph_init(engine, view_layer);
-
-      if (type->update) {
-        type->update(engine, re->main, engine->depsgraph);
-      }
-
-      if (re->draw_lock) {
-        re->draw_lock(re->dlh, 0);
-      }
-
-      if (engine->type->flag & RE_USE_GPU_CONTEXT) {
-        DRW_render_context_enable(engine->re);
-      }
-
-      type->render(engine, engine->depsgraph);
-
-      if (engine->type->flag & RE_USE_GPU_CONTEXT) {
-        DRW_render_context_disable(engine->re);
-      }
+      engine_render_view_layer(re, engine, view_layer_iter, true, true);
 
-      /* Grease pencil render over previous render result.
-       *
-       * NOTE: External engine might have been requested to free its
-       * dependency graph, which is only allowed if there is no grease
-       * pencil (pipeline is taking care of that).
-       */
-      if (!RE_engine_test_break(engine) && engine->depsgraph != NULL) {
-        DRW_render_gpencil(engine, engine->depsgraph);
-      }
-
-      engine_depsgraph_free(engine);
+      /* With save buffers there is no render buffer in memory for compositing, delay
+       * grease pencil in that case. */
+      delay_grease_pencil = engine->has_grease_pencil && re->result->do_exr_tile;
 
       if (RE_engine_test_break(engine)) {
         break;
@@ -900,6 +926,7 @@ int RE_engine_render(Render *re, int do_all)
     FOREACH_VIEW_LAYER_TO_RENDER_END;
   }
 
+  /* Clear tile data */
   engine->tile_x = 0;
   engine->tile_y = 0;
   engine->flag &= ~RE_ENGINE_RENDERING;
@@ -908,10 +935,26 @@ int RE_engine_render(Render *re, int do_all)
 
   BLI_rw_mutex_lock(&re->partsmutex, THREAD_LOCK_WRITE);
 
+  /* For save buffers, read back from disk. */
   if (re->result->do_exr_tile) {
     render_result_exr_file_end(re, engine);
   }
 
+  /* Perform delayed grease pencil rendering. */
+  if (delay_grease_pencil) {
+    BLI_rw_mutex_unlock(&re->partsmutex);
+
+    FOREACH_VIEW_LAYER_TO_RENDER_BEGIN (re, view_layer_iter) {
+      engine_render_view_layer(re, engine, view_layer_iter, false, true);
+      if (RE_engine_test_break(engine)) {
+        break;
+      }
+    }
+    FOREACH_VIEW_LAYER_TO_RENDER_END;
+
+    BLI_rw_mutex_lock(&re->partsmutex, THREAD_LOCK_WRITE);
+  }
+
   /* re->engine becomes zero if user changed active render engine during render */
   if (!persistent_data || !re->engine) {
     RE_engine_free(engine);
@@ -983,7 +1026,7 @@ void RE_engine_free_blender_memory(RenderEngine *engine)
    *
    * TODO(sergey): Find better solution for this.
    */
-  if (DRW_render_check_grease_pencil(engine->depsgraph)) {
+  if (engine->has_grease_pencil) {
     return;
   }
   DEG_graph_free(engine->depsgraph);