Fix #35561: freestyle + read full sample layers = crash.

Now add_freestyle() in pipeline.c takes a second argument to enable/disable
stroke rendering.  When stroke rendering is disabled, the function allocates
data structures but does not perform stroke rendering.  The allocated data
structures (mostly left unpopulated with data elements) are intended to allow
for the Read Full Sample Layers (Shift-R) command in the compositor.

8 files changed, 37 insertions, 18 deletions

diff --git a/source/blender/freestyle/FRS_freestyle.h b/source/blender/freestyle/FRS_freestyle.h
index b10a73277bf..7be51b37f7a 100644
--- a/source/blender/freestyle/FRS_freestyle.h
+++ b/source/blender/freestyle/FRS_freestyle.h
@@ -45,7 +45,7 @@ void FRS_set_context(struct bContext *C);
 void FRS_read_file(struct bContext *C);
 int FRS_is_freestyle_enabled(struct SceneRenderLayer *srl);
 void FRS_init_stroke_rendering(struct Render *re);
-struct Render *FRS_do_stroke_rendering(struct Render *re, struct SceneRenderLayer *srl);
+struct Render *FRS_do_stroke_rendering(struct Render *re, struct SceneRenderLayer *srl, int render);
 void FRS_finish_stroke_rendering(struct Render *re);
 void FRS_composite_result(struct Render *re, struct SceneRenderLayer *srl, struct Render *freestyle_render);
 void FRS_exit(void);
diff --git a/source/blender/freestyle/intern/application/Controller.cpp b/source/blender/freestyle/intern/application/Controller.cpp
index 436cdbfe6b5..dbf3fa8349e 100644
--- a/source/blender/freestyle/intern/application/Controller.cpp
+++ b/source/blender/freestyle/intern/application/Controller.cpp
@@ -833,17 +833,18 @@ void Controller::ResetRenderCount()
 	_render_count = 0;
 }
 
-Render *Controller::RenderStrokes(Render *re)
+Render *Controller::RenderStrokes(Render *re, bool render)
 {
 	_Chrono.start();
 	BlenderStrokeRenderer *blenderRenderer = new BlenderStrokeRenderer(re, ++_render_count);
-	_Canvas->Render(blenderRenderer);
+	if (render)
+		_Canvas->Render(blenderRenderer);
 	real d = _Chrono.stop();
 	if (G.debug & G_DEBUG_FREESTYLE) {
 		cout << "Temporary scene generation: " << d << endl;
 	}
 	_Chrono.start();
-	Render *freestyle_render = blenderRenderer->RenderScene(re);
+	Render *freestyle_render = blenderRenderer->RenderScene(re, render);
 	d = _Chrono.stop();
 	if (G.debug & G_DEBUG_FREESTYLE) {
 		cout << "Stroke rendering  : " << d << endl;
diff --git a/source/blender/freestyle/intern/application/Controller.h b/source/blender/freestyle/intern/application/Controller.h
index 3315fe7e20b..c8eaaf5486b 100644
--- a/source/blender/freestyle/intern/application/Controller.h
+++ b/source/blender/freestyle/intern/application/Controller.h
@@ -86,7 +86,7 @@ public:
 	void toggleEdgeTesselationNature(Nature::EdgeNature iNature);
 	void DrawStrokes();
 	void ResetRenderCount();
-	Render *RenderStrokes(Render *re);
+	Render *RenderStrokes(Render *re, bool render);
 	void SwapStyleModules(unsigned i1, unsigned i2);
 	void InsertStyleModule(unsigned index, const char *iFileName);
 	void InsertStyleModule(unsigned index, const char *iName, struct Text *iText);
diff --git a/source/blender/freestyle/intern/blender_interface/BlenderStrokeRenderer.cpp b/source/blender/freestyle/intern/blender_interface/BlenderStrokeRenderer.cpp
index 294da904497..096447de214 100644
--- a/source/blender/freestyle/intern/blender_interface/BlenderStrokeRenderer.cpp
+++ b/source/blender/freestyle/intern/blender_interface/BlenderStrokeRenderer.cpp
@@ -476,7 +476,7 @@ Object *BlenderStrokeRenderer::NewMesh() const
 	return ob;
 }
 
-Render *BlenderStrokeRenderer::RenderScene(Render *re)
+Render *BlenderStrokeRenderer::RenderScene(Render *re, bool render)
 {
 	Camera *camera = (Camera *)freestyle_scene->camera->data;
 	if (camera->clipend < _z)
@@ -489,7 +489,7 @@ Render *BlenderStrokeRenderer::RenderScene(Render *re)
 
 	Render *freestyle_render = RE_NewRender(freestyle_scene->id.name);
 
-	RE_RenderFreestyleStrokes(freestyle_render, freestyle_bmain, freestyle_scene);
+	RE_RenderFreestyleStrokes(freestyle_render, freestyle_bmain, freestyle_scene, render);
 
 	return freestyle_render;
 }
diff --git a/source/blender/freestyle/intern/blender_interface/BlenderStrokeRenderer.h b/source/blender/freestyle/intern/blender_interface/BlenderStrokeRenderer.h
index 4d34f70f689..ae0e6bed3b2 100644
--- a/source/blender/freestyle/intern/blender_interface/BlenderStrokeRenderer.h
+++ b/source/blender/freestyle/intern/blender_interface/BlenderStrokeRenderer.h
@@ -51,7 +51,7 @@ public:
 
 	Object *NewMesh() const;
 
-	Render *RenderScene(Render *re);
+	Render *RenderScene(Render *re, bool render);
 
 protected:
 	Main *freestyle_bmain;
diff --git a/source/blender/freestyle/intern/blender_interface/FRS_freestyle.cpp b/source/blender/freestyle/intern/blender_interface/FRS_freestyle.cpp
index d5002d6fa39..8cb44d05b84 100644
--- a/source/blender/freestyle/intern/blender_interface/FRS_freestyle.cpp
+++ b/source/blender/freestyle/intern/blender_interface/FRS_freestyle.cpp
@@ -580,12 +580,15 @@ void FRS_init_stroke_rendering(Render *re)
 	controller->ResetRenderCount();
 }
 
-Render *FRS_do_stroke_rendering(Render *re, SceneRenderLayer *srl)
+Render *FRS_do_stroke_rendering(Render *re, SceneRenderLayer *srl, int render)
 {
 	Main bmain = {0};
 	Render *freestyle_render = NULL;
 	Text *text, *next_text;
 
+	if (!render)
+		return controller->RenderStrokes(re, false);
+
 	RenderMonitor monitor(re);
 	controller->setRenderMonitor(&monitor);
 
@@ -619,7 +622,7 @@ Render *FRS_do_stroke_rendering(Render *re, SceneRenderLayer *srl)
 		re->i.infostr = NULL;
 		freestyle_scene = re->scene;
 		controller->DrawStrokes();
-		freestyle_render = controller->RenderStrokes(re);
+		freestyle_render = controller->RenderStrokes(re, true);
 		controller->CloseFile();
 		freestyle_scene = NULL;
 
diff --git a/source/blender/render/extern/include/RE_pipeline.h b/source/blender/render/extern/include/RE_pipeline.h
index 52d42cd20a4..118d0036464 100644
--- a/source/blender/render/extern/include/RE_pipeline.h
+++ b/source/blender/render/extern/include/RE_pipeline.h
@@ -226,7 +226,7 @@ void RE_TileProcessor(struct Render *re);
 void RE_BlenderFrame(struct Render *re, struct Main *bmain, struct Scene *scene, struct SceneRenderLayer *srl, struct Object *camera_override, unsigned int lay, int frame, const short write_still);
 void RE_BlenderAnim(struct Render *re, struct Main *bmain, struct Scene *scene, struct Object *camera_override, unsigned int lay, int sfra, int efra, int tfra);
 #ifdef WITH_FREESTYLE
-void RE_RenderFreestyleStrokes(struct Render *re, struct Main *bmain, struct Scene *scene);
+void RE_RenderFreestyleStrokes(struct Render *re, struct Main *bmain, struct Scene *scene, int render);
 #endif
 
 /* error reporting */
diff --git a/source/blender/render/intern/source/pipeline.c b/source/blender/render/intern/source/pipeline.c
index c8f8f844e5f..068df215edb 100644
--- a/source/blender/render/intern/source/pipeline.c
+++ b/source/blender/render/intern/source/pipeline.c
@@ -1081,7 +1081,7 @@ static void threaded_tile_processor(Render *re)
 }
 
 #ifdef WITH_FREESTYLE
-static void add_freestyle(Render *re);
+static void add_freestyle(Render *re, int render);
 static void free_all_freestyle_renders(void);
 #endif
 
@@ -1097,7 +1097,7 @@ void RE_TileProcessor(Render *re)
 	/* Freestyle */
 	if (re->r.mode & R_EDGE_FRS) {
 		if (!re->test_break(re->tbh)) {
-			add_freestyle(re);
+			add_freestyle(re, 1);
 	
 			free_all_freestyle_renders();
 			
@@ -1150,7 +1150,7 @@ static void do_render_3d(Render *re)
 	/* Freestyle */
 	if (re->r.mode & R_EDGE_FRS)
 		if (!re->test_break(re->tbh))
-			add_freestyle(re);
+			add_freestyle(re, 1);
 #endif
 	
 	/* do left-over 3d post effects (flares) */
@@ -1633,7 +1633,7 @@ static void render_composit_stats(void *UNUSED(arg), char *str)
 
 #ifdef WITH_FREESTYLE
 /* invokes Freestyle stroke rendering */
-static void add_freestyle(Render *re)
+static void add_freestyle(Render *re, int render)
 {
 	SceneRenderLayer *srl, *actsrl;
 	LinkData *link;
@@ -1659,7 +1659,7 @@ static void add_freestyle(Render *re)
 		if ((re->r.scemode & R_SINGLE_LAYER) && srl != actsrl)
 			continue;
 		if (FRS_is_freestyle_enabled(srl)) {
-			link->data = (void *)FRS_do_stroke_rendering(re, srl);
+			link->data = (void *)FRS_do_stroke_rendering(re, srl, render);
 		}
 	}
 
@@ -1853,6 +1853,11 @@ void RE_MergeFullSample(Render *re, Main *bmain, Scene *sce, bNodeTree *ntree)
 	for (scene = re->main->scene.first; scene; scene = scene->id.next)
 		scene->id.flag |= LIB_DOIT;
 	
+#ifdef WITH_FREESTYLE
+	for (scene = re->freestyle_bmain.scene.first; scene; scene = scene->id.next)
+		scene->id.flag &= ~LIB_DOIT;
+#endif
+
 	for (node = ntree->nodes.first; node; node = node->next) {
 		if (node->type == CMP_NODE_R_LAYERS) {
 			Scene *nodescene = (Scene *)node->id;
@@ -1876,7 +1881,16 @@ void RE_MergeFullSample(Render *re, Main *bmain, Scene *sce, bNodeTree *ntree)
 	re->display_init(re->dih, re->result);
 	re->display_clear(re->dch, re->result);
 	
+#ifdef WITH_FREESTYLE
+	if (re->r.mode & R_EDGE_FRS)
+		add_freestyle(re, 0);
+#endif
+
 	do_merge_fullsample(re, ntree);
+
+#ifdef WITH_FREESTYLE
+	free_all_freestyle_renders();
+#endif
 }
 
 /* returns fully composited render-result on given time step (in RenderData) */
@@ -2445,11 +2459,12 @@ void RE_BlenderFrame(Render *re, Main *bmain, Scene *scene, SceneRenderLayer *sr
 }
 
 #ifdef WITH_FREESTYLE
-void RE_RenderFreestyleStrokes(Render *re, Main *bmain, Scene *scene)
+void RE_RenderFreestyleStrokes(Render *re, Main *bmain, Scene *scene, int render)
 {
 	re->result_ok= 0;
 	if (render_initialize_from_main(re, bmain, scene, NULL, NULL, scene->lay, 0, 0)) {
-		do_render_fields_blur_3d(re);
+		if (render)
+			do_render_fields_blur_3d(re);
 	}
 	re->result_ok = 1;
 }