Cycles: Free Blender side data as soon as we don't need it

Currently this is possible after built-in images are loaded in memory.
Allows to save memory used by dependency graph and copy-on-write.

In practice this lowers peak system memory usage from 52GB to 42GB on
a production file of spring 03_035_A.lighting.

Note, that this only applies to F12 and command line renders.

Bigger note, that this optimization is currently only possible if
there are no grease pencil objects to be rendered.

5 files changed, 53 insertions, 2 deletions

diff --git a/intern/cycles/blender/blender_session.cpp b/intern/cycles/blender/blender_session.cpp
index 28b0f4faf63..17da8b43d29 100644
--- a/intern/cycles/blender/blender_session.cpp
+++ b/intern/cycles/blender/blender_session.cpp
@@ -470,6 +470,11 @@ void BlenderSession::render(BL::Depsgraph& b_depsgraph_)
 		                &python_thread_state);
 		builtin_images_load();
 
+		/* Attempt to free all data which is held by Blender side, since at this
+		 * point we knwo that we've got everything to render current view layer.
+		 */
+		free_blender_memory_if_possible();
+
 		/* Make sure all views have different noise patterns. - hardcoded value just to make it random */
 		if(view_index != 0) {
 			scene->integrator->seed += hash_int_2d(scene->integrator->seed, hash_int(view_index * 0xdeadbeef));
@@ -1406,4 +1411,16 @@ void BlenderSession::update_resumable_tile_manager(int num_samples)
 	session->tile_manager.range_num_samples = range_num_samples;
 }
 
+void BlenderSession::free_blender_memory_if_possible()
+{
+	if (!background) {
+		/* During interactive render we can not free anything: attempts to save
+		 * memory would cause things to be allocated and evaluated for every
+		 * updated sample.
+		 */
+		return;
+	}
+	b_engine.free_blender_memory();
+}
+
 CCL_NAMESPACE_END
diff --git a/intern/cycles/blender/blender_session.h b/intern/cycles/blender/blender_session.h
index 99b14a09a5c..fdeba7b2f81 100644
--- a/intern/cycles/blender/blender_session.h
+++ b/intern/cycles/blender/blender_session.h
@@ -106,6 +106,9 @@ public:
 	BL::BlendData b_data;
 	BL::RenderSettings b_render;
 	BL::Depsgraph b_depsgraph;
+	/* NOTE: Blender's scene might become invalid after call
+	 * free_blender_memory_if_possible().
+	 */
 	BL::Scene b_scene;
 	BL::SpaceView3D b_v3d;
 	BL::RegionView3D b_rv3d;
@@ -172,6 +175,12 @@ protected:
 
 	/* Update tile manager to reflect resumable render settings. */
 	void update_resumable_tile_manager(int num_samples);
+
+	/* Is used after each render layer synchronization is done with the goal
+	 * of freeing render engine data which is held from Blender side (for
+	 * example, dependency graph).
+	 */
+	void free_blender_memory_if_possible();
 };
 
 CCL_NAMESPACE_END
diff --git a/source/blender/makesrna/intern/rna_render.c b/source/blender/makesrna/intern/rna_render.c
index 1e9f68c577b..46b77b80d49 100644
--- a/source/blender/makesrna/intern/rna_render.c
+++ b/source/blender/makesrna/intern/rna_render.c
@@ -685,6 +685,9 @@ static void rna_def_render_engine(BlenderRNA *brna)
 	parm = RNA_def_int(func, "pixel_size", 0, 1, 8, "Pixel Size", "", 1, 8);
 	RNA_def_function_return(func, parm);
 
+	RNA_def_function(srna, "free_blender_memory", "RE_engine_free_blender_memory");
+	RNA_def_function_ui_description(func, "Free Blender side memory of render engine");
+
 	RNA_define_verify_sdna(0);
 
 	prop = RNA_def_property(srna, "is_animation", PROP_BOOLEAN, PROP_NONE);
diff --git a/source/blender/render/extern/include/RE_engine.h b/source/blender/render/extern/include/RE_engine.h
index 1ca889f2347..faa4c8b3184 100644
--- a/source/blender/render/extern/include/RE_engine.h
+++ b/source/blender/render/extern/include/RE_engine.h
@@ -193,4 +193,6 @@ struct RenderData *RE_engine_get_render_data(struct Render *re);
 void RE_bake_engine_set_engine_parameters(
         struct Render *re, struct Main *bmain, struct Scene *scene);
 
+void RE_engine_free_blender_memory(struct RenderEngine *engine);
+
 #endif /* __RE_ENGINE_H__ */
diff --git a/source/blender/render/intern/source/external_engine.c b/source/blender/render/intern/source/external_engine.c
index a06389bfcd4..758600e89aa 100644
--- a/source/blender/render/intern/source/external_engine.c
+++ b/source/blender/render/intern/source/external_engine.c
@@ -738,8 +738,13 @@ int RE_engine_render(Render *re, int do_all)
 
 			type->render(engine, engine->depsgraph);
 
-			/* grease pencil render over previous render result */
-			if (!RE_engine_test_break(engine)) {
+			/* Grease pencil render over previous render result.
+			 *
+			 * NOTE: External engine might have been requested to free its
+			 * dependency graph, which is only allowed if there is no grease
+			 * pencil (pipeline is taking care of that).
+			 */
+			if (!RE_engine_test_break(engine) && engine->depsgraph != NULL) {
 				DRW_render_gpencil(engine, engine->depsgraph);
 			}
 
@@ -813,3 +818,18 @@ void RE_engine_register_pass(struct RenderEngine *engine, struct Scene *scene, s
 		}
 	}
 }
+
+void RE_engine_free_blender_memory(RenderEngine *engine)
+{
+	/* Weak way to save memory, but not crash grease pencil.
+	 *
+	 * TODO(sergey): Find better solution for this.
+	 * TODO(sergey): Try to find solution which does not involve looping over
+	 * all the objects.
+	 */
+	if (DRW_render_check_grease_pencil(engine->depsgraph)) {
+		return;
+	}
+	DEG_graph_free(engine->depsgraph);
+	engine->depsgraph = NULL;
+}