Render: faster animation and re-rendering with Persistent Data

For Cycles, when enabling the Persistent Data option, the full render data
will be preserved from frame-to-frame in animation renders and between
re-renders of the scene. This means that any modifier evaluation, BVH
building, OpenGL vertex buffer uploads, etc, can be done only once for
unchanged objects. This comes at an increased memory cost.

Previously there option was named Persistent Images and had a more limited
impact on render time and memory.

When using multiple view layers, only data from a single view layer is
preserved to keep memory usage somewhat under control. However objects
shared between view layers are preserved, and so this can speedup such
renders as well, even single frame renders.

For Eevee and Workbench this option is not available, however these engines
will now always reuse the depsgraph for animation and multiple view layers.
This can significantly speed up rendering.

These engines do not support sharing the depsgraph between re-renders, due
to technical issues regarding OpenGL contexts. Support for this could be added
if those are solved, see the code comments for details.

4 files changed, 155 insertions, 53 deletions

diff --git a/source/blender/render/RE_engine.h b/source/blender/render/RE_engine.h
index c31a41f66d5..f6ab7fd9d3c 100644
--- a/source/blender/render/RE_engine.h
+++ b/source/blender/render/RE_engine.h
@@ -224,6 +224,8 @@ void RE_engine_register_pass(struct RenderEngine *engine,
                              const char *chanid,
                              eNodeSocketDatatype type);
 
+bool RE_engine_use_persistent_data(struct RenderEngine *engine);
+
 /* Engine Types */
 
 void RE_engines_init(void);
diff --git a/source/blender/render/RE_pipeline.h b/source/blender/render/RE_pipeline.h
index 27dcd9e70ed..7420ee64a81 100644
--- a/source/blender/render/RE_pipeline.h
+++ b/source/blender/render/RE_pipeline.h
@@ -184,14 +184,14 @@ void RE_InitRenderCB(struct Render *re);
 void RE_FreeRender(struct Render *re);
 /* only called on exit */
 void RE_FreeAllRender(void);
-/* Free memory used by persistent data.
- * Invoked when loading new file.
- */
-void RE_FreeAllPersistentData(void);
-/* only call on file load */
+
+/* On file load, free render results. */
 void RE_FreeAllRenderResults(void);
-/* for external render engines that can keep persistent data */
-void RE_FreePersistentData(void);
+/* On file load or changes engines, free persistent render data.
+ * Assumes no engines are currently rendering. */
+void RE_FreeAllPersistentData(void);
+/* Free persistent render data, optionally only for the given scene. */
+void RE_FreePersistentData(const Scene *scene);
 
 /* get results and statistics */
 void RE_FreeRenderResult(struct RenderResult *rr);
diff --git a/source/blender/render/intern/engine.c b/source/blender/render/intern/engine.c
index a43a78f5d3d..ccb9996a143 100644
--- a/source/blender/render/intern/engine.c
+++ b/source/blender/render/intern/engine.c
@@ -140,6 +140,27 @@ RenderEngine *RE_engine_create(RenderEngineType *type)
   return engine;
 }
 
+static void engine_depsgraph_free(RenderEngine *engine)
+{
+  if (engine->depsgraph) {
+    /* Need GPU context since this might free GPU buffers. This function can
+     * only be called from a render thread. We do not currently support
+     * persistent data with GPU contexts for that reason. */
+    const bool use_gpu_context = (engine->type->flag & RE_USE_GPU_CONTEXT);
+    if (use_gpu_context) {
+      BLI_assert(!BLI_thread_is_main());
+      DRW_render_context_enable(engine->re);
+    }
+
+    DEG_graph_free(engine->depsgraph);
+    engine->depsgraph = NULL;
+
+    if (use_gpu_context) {
+      DRW_render_context_disable(engine->re);
+    }
+  }
+}
+
 void RE_engine_free(RenderEngine *engine)
 {
 #ifdef WITH_PYTHON
@@ -148,6 +169,8 @@ void RE_engine_free(RenderEngine *engine)
   }
 #endif
 
+  engine_depsgraph_free(engine);
+
   BLI_mutex_end(&engine->update_render_passes_mutex);
 
   MEM_freeN(engine);
@@ -598,34 +621,94 @@ RenderData *RE_engine_get_render_data(Render *re)
   return &re->r;
 }
 
+bool RE_engine_use_persistent_data(RenderEngine *engine)
+{
+  /* See engine_depsgraph_free() for why preserving the depsgraph for
+   * re-renders is not supported with GPU contexts. */
+  return (engine->re->r.mode & R_PERSISTENT_DATA) && !(engine->type->flag & RE_USE_GPU_CONTEXT);
+}
+
+static bool engine_keep_depsgraph(RenderEngine *engine)
+{
+  /* For persistent data or GPU engines like Eevee, reuse the depsgraph between
+   * view layers and animation frames. For renderers like Cycles that create
+   * their own copy of the scene, persistent data must be explicitly enabled to
+   * keep memory usage low by default. */
+  return (engine->re->r.mode & R_PERSISTENT_DATA) || (engine->type->flag & RE_USE_GPU_CONTEXT);
+}
+
 /* Depsgraph */
 static void engine_depsgraph_init(RenderEngine *engine, ViewLayer *view_layer)
 {
   Main *bmain = engine->re->main;
   Scene *scene = engine->re->scene;
+  bool reuse_depsgraph = false;
+
+  /* Reuse depsgraph from persistent data if possible. */
+  if (engine->depsgraph) {
+    if (DEG_get_bmain(engine->depsgraph) != bmain ||
+        DEG_get_input_scene(engine->depsgraph) != scene) {
+      /* If bmain or scene changes, we need a completely new graph. */
+      engine_depsgraph_free(engine);
+    }
+    else if (DEG_get_input_view_layer(engine->depsgraph) != view_layer) {
+      /* If only view layer changed, reuse depsgraph in the hope of reusing
+       * objects shared between view layers. */
+      DEG_graph_replace_owners(engine->depsgraph, bmain, scene, view_layer);
+      DEG_graph_tag_relations_update(engine->depsgraph);
+    }
 
-  engine->depsgraph = DEG_graph_new(bmain, scene, view_layer, DAG_EVAL_RENDER);
-  DEG_debug_name_set(engine->depsgraph, "RENDER");
+    reuse_depsgraph = true;
+  }
+
+  if (!engine->depsgraph) {
+    /* Ensure we only use persistent data for one scene / view layer at a time,
+     * to avoid excessive memory usage. */
+    RE_FreePersistentData(NULL);
+
+    /* Create new depsgraph if not cached with persistent data. */
+    engine->depsgraph = DEG_graph_new(bmain, scene, view_layer, DAG_EVAL_RENDER);
+    DEG_debug_name_set(engine->depsgraph, "RENDER");
+  }
 
   if (engine->re->r.scemode & R_BUTS_PREVIEW) {
+    /* Update for preview render. */
     Depsgraph *depsgraph = engine->depsgraph;
     DEG_graph_relations_update(depsgraph);
+
+    /* Need GPU context since this might free GPU buffers. */
+    const bool use_gpu_context = (engine->type->flag & RE_USE_GPU_CONTEXT) && reuse_depsgraph;
+    if (use_gpu_context) {
+      DRW_render_context_enable(engine->re);
+    }
+
     DEG_evaluate_on_framechange(depsgraph, CFRA);
-    DEG_ids_check_recalc(bmain, depsgraph, scene, view_layer, true);
-    DEG_ids_clear_recalc(bmain, depsgraph);
+
+    if (use_gpu_context) {
+      DRW_render_context_disable(engine->re);
+    }
   }
   else {
+    /* Go through update with full Python callbacks for regular render. */
     BKE_scene_graph_update_for_newframe(engine->depsgraph);
   }
 
   engine->has_grease_pencil = DRW_render_check_grease_pencil(engine->depsgraph);
 }
 
-static void engine_depsgraph_free(RenderEngine *engine)
+static void engine_depsgraph_exit(RenderEngine *engine)
 {
-  DEG_graph_free(engine->depsgraph);
-
-  engine->depsgraph = NULL;
+  if (engine->depsgraph) {
+    if (engine_keep_depsgraph(engine)) {
+      /* Clear recalc flags since the engine should have handled the updates for the currently
+       * rendered framed by now. */
+      DEG_ids_clear_recalc(engine->depsgraph);
+    }
+    else {
+      /* Free immediately to save memory. */
+      engine_depsgraph_free(engine);
+    }
+  }
 }
 
 void RE_engine_frame_set(RenderEngine *engine, int frame, float subframe)
@@ -670,7 +753,6 @@ bool RE_bake_engine(Render *re,
 {
   RenderEngineType *type = RE_engines_find(re->r.engine);
   RenderEngine *engine;
-  bool persistent_data = (re->r.mode & R_PERSISTENT_DATA) != 0;
 
   /* set render info */
   re->i.cfra = re->scene->r.cfra;
@@ -729,13 +811,13 @@ bool RE_bake_engine(Render *re,
   engine->tile_y = 0;
   engine->flag &= ~RE_ENGINE_RENDERING;
 
+  /* Free depsgraph outside of parts mutex lock, since this locks OpenGL context
+   * while the the UI drawing might also lock the OpenGL context and parts mutex. */
+  engine_depsgraph_free(engine);
   BLI_rw_mutex_lock(&re->partsmutex, THREAD_LOCK_WRITE);
 
-  /* re->engine becomes zero if user changed active render engine during render */
-  if (!persistent_data || !re->engine) {
-    RE_engine_free(engine);
-    re->engine = NULL;
-  }
+  RE_engine_free(engine);
+  re->engine = NULL;
 
   RE_parts_free(re);
   BLI_rw_mutex_unlock(&re->partsmutex);
@@ -778,13 +860,14 @@ static void engine_render_view_layer(Render *re,
 
   /* Perform render with engine. */
   if (use_engine) {
-    if (engine->type->flag & RE_USE_GPU_CONTEXT) {
+    const bool use_gpu_context = (engine->type->flag & RE_USE_GPU_CONTEXT);
+    if (use_gpu_context) {
       DRW_render_context_enable(engine->re);
     }
 
     engine->type->render(engine, engine->depsgraph);
 
-    if (engine->type->flag & RE_USE_GPU_CONTEXT) {
+    if (use_gpu_context) {
       DRW_render_context_disable(engine->re);
     }
   }
@@ -800,13 +883,12 @@ static void engine_render_view_layer(Render *re,
   }
 
   /* Free dependency graph, if engine has not done it already. */
-  engine_depsgraph_free(engine);
+  engine_depsgraph_exit(engine);
 }
 
 bool RE_engine_render(Render *re, bool do_all)
 {
   RenderEngineType *type = RE_engines_find(re->r.engine);
-  bool persistent_data = (re->r.mode & R_PERSISTENT_DATA) != 0;
 
   /* verify if we can render */
   if (!type->render) {
@@ -953,7 +1035,13 @@ bool RE_engine_render(Render *re, bool do_all)
   }
 
   /* re->engine becomes zero if user changed active render engine during render */
-  if (!persistent_data || !re->engine) {
+  if (!engine_keep_depsgraph(engine) || !re->engine) {
+    /* Free depsgraph outside of parts mutex lock, since this locks OpenGL context
+     * while the the UI drawing might also lock the OpenGL context and parts mutex. */
+    BLI_rw_mutex_unlock(&re->partsmutex);
+    engine_depsgraph_free(engine);
+    BLI_rw_mutex_lock(&re->partsmutex, THREAD_LOCK_WRITE);
+
     RE_engine_free(engine);
     re->engine = NULL;
   }
@@ -1023,9 +1111,8 @@ void RE_engine_free_blender_memory(RenderEngine *engine)
    *
    * TODO(sergey): Find better solution for this.
    */
-  if (engine->has_grease_pencil) {
+  if (engine->has_grease_pencil || engine_keep_depsgraph(engine)) {
     return;
   }
-  DEG_graph_free(engine->depsgraph);
-  engine->depsgraph = NULL;
+  engine_depsgraph_free(engine);
 }
diff --git a/source/blender/render/intern/pipeline.c b/source/blender/render/intern/pipeline.c
index 92bec9c6fd4..5a2ac6ed348 100644
--- a/source/blender/render/intern/pipeline.c
+++ b/source/blender/render/intern/pipeline.c
@@ -662,17 +662,6 @@ void RE_FreeAllRender(void)
 #endif
 }
 
-void RE_FreeAllPersistentData(void)
-{
-  Render *re;
-  for (re = RenderGlobal.renderlist.first; re != NULL; re = re->next) {
-    if ((re->r.mode & R_PERSISTENT_DATA) != 0 && re->engine != NULL) {
-      RE_engine_free(re->engine);
-      re->engine = NULL;
-    }
-  }
-}
-
 /* on file load, free all re */
 void RE_FreeAllRenderResults(void)
 {
@@ -687,19 +676,39 @@ void RE_FreeAllRenderResults(void)
   }
 }
 
-void RE_FreePersistentData(void)
+void RE_FreeAllPersistentData(void)
 {
   Render *re;
+  for (re = RenderGlobal.renderlist.first; re != NULL; re = re->next) {
+    if (re->engine != NULL) {
+      BLI_assert(!(re->engine->flag & RE_ENGINE_RENDERING));
+      RE_engine_free(re->engine);
+      re->engine = NULL;
+    }
+  }
+}
 
-  /* render engines can be kept around for quick re-render, this clears all */
-  for (re = RenderGlobal.renderlist.first; re; re = re->next) {
-    if (re->engine) {
-      /* if engine is currently rendering, just tag it to be freed when render is finished */
-      if (!(re->engine->flag & RE_ENGINE_RENDERING)) {
-        RE_engine_free(re->engine);
-      }
+static void re_free_persistent_data(Render *re)
+{
+  /* If engine is currently rendering, just wait for it to be freed when it finishes rendering. */
+  if (re->engine && !(re->engine->flag & RE_ENGINE_RENDERING)) {
+    RE_engine_free(re->engine);
+    re->engine = NULL;
+  }
+}
 
-      re->engine = NULL;
+void RE_FreePersistentData(const Scene *scene)
+{
+  /* Render engines can be kept around for quick re-render, this clears all or one scene. */
+  if (scene) {
+    Render *re = RE_GetSceneRender(scene);
+    if (re) {
+      re_free_persistent_data(re);
+    }
+  }
+  else {
+    for (Render *re = RenderGlobal.renderlist.first; re; re = re->next) {
+      re_free_persistent_data(re);
     }
   }
 }
@@ -2669,13 +2678,17 @@ void RE_PreviewRender(Render *re, Main *bmain, Scene *sce)
 
 void RE_CleanAfterRender(Render *re)
 {
-  /* Destroy the opengl context in the correct thread. */
-  RE_gl_context_destroy(re);
+  if (re->engine && !RE_engine_use_persistent_data(re->engine)) {
+    RE_engine_free(re->engine);
+    re->engine = NULL;
+  }
   if (re->pipeline_depsgraph != NULL) {
     DEG_graph_free(re->pipeline_depsgraph);
+    re->pipeline_depsgraph = NULL;
+    re->pipeline_scene_eval = NULL;
   }
-  re->pipeline_depsgraph = NULL;
-  re->pipeline_scene_eval = NULL;
+  /* Destroy the opengl context in the correct thread. */
+  RE_gl_context_destroy(re);
 }
 
 /* note; repeated win/disprect calc... solve that nicer, also in compo */