Freestyle: Add option for rendering Freestyle to a separate pass

This allows for more flexibility in Compositing compared to the
hardcoded alpha-over that is currently used.

Reviewed By: brecht

Differential Revision: https://developer.blender.org/D6829

9 files changed, 54 insertions, 18 deletions

diff --git a/source/blender/freestyle/intern/blender_interface/FRS_freestyle.cpp b/source/blender/freestyle/intern/blender_interface/FRS_freestyle.cpp
index fdf9d4bac3d..0877107a240 100644
--- a/source/blender/freestyle/intern/blender_interface/FRS_freestyle.cpp
+++ b/source/blender/freestyle/intern/blender_interface/FRS_freestyle.cpp
@@ -549,7 +549,15 @@ void FRS_composite_result(Render *re, ViewLayer *view_layer, Render *freestyle_r
     }
     return;
   }
-  dest = RE_RenderLayerGetPass(rl, RE_PASSNAME_COMBINED, re->viewname);
+
+  if (view_layer->freestyle_config.flags & FREESTYLE_AS_RENDER_PASS) {
+    RE_create_render_pass(
+        re->result, RE_PASSNAME_FREESTYLE, 4, "RGBA", view_layer->name, re->viewname);
+    dest = RE_RenderLayerGetPass(rl, RE_PASSNAME_FREESTYLE, re->viewname);
+  }
+  else {
+    dest = RE_RenderLayerGetPass(rl, RE_PASSNAME_COMBINED, re->viewname);
+  }
   if (!dest) {
     if (G.debug & G_DEBUG_FREESTYLE) {
       cout << "No destination result image to composite to" << endl;
diff --git a/source/blender/makesdna/DNA_freestyle_types.h b/source/blender/makesdna/DNA_freestyle_types.h
index 77cf95f4fb8..8b3d8090c89 100644
--- a/source/blender/makesdna/DNA_freestyle_types.h
+++ b/source/blender/makesdna/DNA_freestyle_types.h
@@ -44,6 +44,7 @@ enum {
   FREESTYLE_ADVANCED_OPTIONS_FLAG = 1 << 4,
   FREESTYLE_CULLING = 1 << 5,
   FREESTYLE_VIEW_MAP_CACHE = 1 << 6,
+  FREESTYLE_AS_RENDER_PASS = 1 << 7,
 };
 
 /* FreestyleConfig::mode */
diff --git a/source/blender/makesdna/DNA_scene_types.h b/source/blender/makesdna/DNA_scene_types.h
index 667511f6514..9cb2663386a 100644
--- a/source/blender/makesdna/DNA_scene_types.h
+++ b/source/blender/makesdna/DNA_scene_types.h
@@ -317,6 +317,8 @@ typedef enum eScenePassType {
 #define RE_PASSNAME_SUBSURFACE_INDIRECT "SubsurfaceInd"
 #define RE_PASSNAME_SUBSURFACE_COLOR "SubsurfaceCol"
 
+#define RE_PASSNAME_FREESTYLE "Freestyle"
+
 /* View - MultiView */
 typedef struct SceneRenderView {
   struct SceneRenderView *next, *prev;
diff --git a/source/blender/makesrna/intern/rna_scene.c b/source/blender/makesrna/intern/rna_scene.c
index 262048e55c2..cc0903ed2c8 100644
--- a/source/blender/makesrna/intern/rna_scene.c
+++ b/source/blender/makesrna/intern/rna_scene.c
@@ -1661,6 +1661,17 @@ void rna_Scene_freestyle_update(Main *UNUSED(bmain), Scene *UNUSED(scene), Point
   DEG_id_tag_update(&scene->id, 0);
 }
 
+void rna_Scene_use_freestyle_update(Main *UNUSED(bmain), Scene *UNUSED(scene), PointerRNA *ptr)
+{
+  Scene *scene = (Scene *)ptr->owner_id;
+
+  DEG_id_tag_update(&scene->id, 0);
+
+  if (scene->nodetree) {
+    ntreeCompositUpdateRLayers(scene->nodetree);
+  }
+}
+
 void rna_Scene_use_view_map_cache_update(Main *UNUSED(bmain),
                                          Scene *UNUSED(scene),
                                          PointerRNA *UNUSED(ptr))
@@ -4626,6 +4637,14 @@ void rna_def_freestyle_settings(BlenderRNA *brna)
   RNA_def_property_update(
       prop, NC_SCENE | ND_RENDER_OPTIONS, "rna_Scene_use_view_map_cache_update");
 
+  prop = RNA_def_property(srna, "as_render_pass", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_sdna(prop, NULL, "flags", FREESTYLE_AS_RENDER_PASS);
+  RNA_def_property_ui_text(
+      prop,
+      "As Render Pass",
+      "Renders Freestyle output to a separate pass instead of overlaying it on the Combined pass");
+  RNA_def_property_update(prop, NC_SCENE | ND_RENDER_OPTIONS, "rna_ViewLayer_pass_update");
+
   prop = RNA_def_property(srna, "sphere_radius", PROP_FLOAT, PROP_NONE);
   RNA_def_property_float_sdna(prop, NULL, "sphere_radius");
   RNA_def_property_range(prop, 0.0, 1000.0);
@@ -5691,7 +5710,7 @@ static void rna_def_scene_render_data(BlenderRNA *brna)
   RNA_def_property_clear_flag(prop, PROP_ANIMATABLE);
   RNA_def_property_boolean_sdna(prop, NULL, "mode", R_EDGE_FRS);
   RNA_def_property_ui_text(prop, "Edge", "Draw stylized strokes using Freestyle");
-  RNA_def_property_update(prop, NC_SCENE | ND_RENDER_OPTIONS, "rna_Scene_freestyle_update");
+  RNA_def_property_update(prop, NC_SCENE | ND_RENDER_OPTIONS, "rna_Scene_use_freestyle_update");
 
   /* threads */
   prop = RNA_def_property(srna, "threads", PROP_INT, PROP_NONE);
diff --git a/source/blender/nodes/composite/nodes/node_composite_image.c b/source/blender/nodes/composite/nodes/node_composite_image.c
index e9a9aa37cfa..b74f325a3fa 100644
--- a/source/blender/nodes/composite/nodes/node_composite_image.c
+++ b/source/blender/nodes/composite/nodes/node_composite_image.c
@@ -315,6 +315,12 @@ static void cmp_node_rlayer_create_outputs(bNodeTree *ntree,
             engine, scene, view_layer, cmp_node_rlayer_create_outputs_cb, NULL);
         RE_engine_free(engine);
 
+        if ((scene->r.mode & R_EDGE_FRS) &&
+            (view_layer->freestyle_config.flags & FREESTYLE_AS_RENDER_PASS)) {
+          ntreeCompositRegisterPass(
+              scene->nodetree, scene, view_layer, RE_PASSNAME_FREESTYLE, SOCK_RGBA);
+        }
+
         MEM_freeN(data);
         node->storage = NULL;
 
diff --git a/source/blender/render/extern/include/RE_pipeline.h b/source/blender/render/extern/include/RE_pipeline.h
index dc18bac541f..40984b6d3b2 100644
--- a/source/blender/render/extern/include/RE_pipeline.h
+++ b/source/blender/render/extern/include/RE_pipeline.h
@@ -228,6 +228,13 @@ struct RenderPass *RE_create_gp_pass(struct RenderResult *rr,
                                      const char *layername,
                                      const char *viewname);
 
+void RE_create_render_pass(struct RenderResult *rr,
+                           const char *name,
+                           int channels,
+                           const char *chan_id,
+                           const char *layername,
+                           const char *viewname);
+
 /* obligatory initialize call, disprect is optional */
 void RE_InitState(struct Render *re,
                   struct Render *source,
diff --git a/source/blender/render/intern/include/render_result.h b/source/blender/render/intern/include/render_result.h
index 24356194f5d..3c089335115 100644
--- a/source/blender/render/intern/include/render_result.h
+++ b/source/blender/render/intern/include/render_result.h
@@ -66,12 +66,6 @@ void render_result_merge(struct RenderResult *rr, struct RenderResult *rrpart);
 /* Add Passes */
 
 void render_result_clone_passes(struct Render *re, struct RenderResult *rr, const char *viewname);
-void render_result_add_pass(struct RenderResult *rr,
-                            const char *name,
-                            int channels,
-                            const char *chan_id,
-                            const char *layername,
-                            const char *viewname);
 
 /* Free */
 
diff --git a/source/blender/render/intern/source/external_engine.c b/source/blender/render/intern/source/external_engine.c
index d3080ebe3ed..90058da5f0c 100644
--- a/source/blender/render/intern/source/external_engine.c
+++ b/source/blender/render/intern/source/external_engine.c
@@ -258,7 +258,7 @@ void RE_engine_add_pass(RenderEngine *engine,
     return;
   }
 
-  render_result_add_pass(re->result, name, channels, chan_id, layername, NULL);
+  RE_create_render_pass(re->result, name, channels, chan_id, layername, NULL);
 }
 
 void RE_engine_end_result(
diff --git a/source/blender/render/intern/source/render_result.c b/source/blender/render/intern/source/render_result.c
index 0bf03347b80..120787a8d13 100644
--- a/source/blender/render/intern/source/render_result.c
+++ b/source/blender/render/intern/source/render_result.c
@@ -520,12 +520,12 @@ void render_result_clone_passes(Render *re, RenderResult *rr, const char *viewna
   }
 }
 
-void render_result_add_pass(RenderResult *rr,
-                            const char *name,
-                            int channels,
-                            const char *chan_id,
-                            const char *layername,
-                            const char *viewname)
+void RE_create_render_pass(RenderResult *rr,
+                           const char *name,
+                           int channels,
+                           const char *chan_id,
+                           const char *layername,
+                           const char *viewname)
 {
   RenderLayer *rl;
   RenderPass *rp;
@@ -1234,7 +1234,7 @@ void render_result_exr_file_begin(Render *re, RenderEngine *engine)
        * mutex locked to avoid deadlock with Python GIL. */
       BLI_rw_mutex_lock(&re->resultmutex, THREAD_LOCK_WRITE);
       for (RenderPass *pass = templates.first; pass; pass = pass->next) {
-        render_result_add_pass(
+        RE_create_render_pass(
             re->result, pass->name, pass->channels, pass->chan_id, rl->name, NULL);
       }
       BLI_rw_mutex_unlock(&re->resultmutex);
@@ -1277,8 +1277,7 @@ void render_result_exr_file_end(Render *re, RenderEngine *engine)
      * mutex locked to avoid deadlock with Python GIL. */
     BLI_rw_mutex_lock(&re->resultmutex, THREAD_LOCK_WRITE);
     for (RenderPass *pass = templates.first; pass; pass = pass->next) {
-      render_result_add_pass(
-          re->result, pass->name, pass->channels, pass->chan_id, rl->name, NULL);
+      RE_create_render_pass(re->result, pass->name, pass->channels, pass->chan_id, rl->name, NULL);
     }
 
     BLI_freelistN(&templates);