Cycles: use reference count to detect used shaders

Shaders are only compiled if they are used by some other Node (Geometry, Light, etc.).
This usage detection is done before updating the Scene, however it fails at detecting
Shaders used by Procedurals not known to Cycles (e.g. ones defined by third party
applications), as Procedurals are only updated after the shaders are compiled.

To remedy this, we now use the Node reference counting mechanism to detect whether a
Shader is used and therefore should be compiled.

This removes `ShaderManager::update_shaders_used` as it is not needed anymore, however,
since it would also update the Shader ids, this is now performed in
`ShaderManager::device_update`, and a new virtual `device_update_specific` method was
added to handle device updates for SVM and OSL.

Reviewed By: brecht

Differential Revision: https://developer.blender.org/D10965

7 files changed, 53 insertions, 65 deletions

diff --git a/intern/cycles/render/osl.cpp b/intern/cycles/render/osl.cpp
index 53c67049571..9bd6e3a5e2d 100644
--- a/intern/cycles/render/osl.cpp
+++ b/intern/cycles/render/osl.cpp
@@ -91,10 +91,10 @@ void OSLShaderManager::reset(Scene * /*scene*/)
   shading_system_init();
 }
 
-void OSLShaderManager::device_update(Device *device,
-                                     DeviceScene *dscene,
-                                     Scene *scene,
-                                     Progress &progress)
+void OSLShaderManager::device_update_specific(Device *device,
+                                              DeviceScene *dscene,
+                                              Scene *scene,
+                                              Progress &progress)
 {
   if (!need_update())
     return;
@@ -1149,7 +1149,7 @@ void OSLCompiler::compile(OSLGlobals *og, Shader *shader)
     shader->has_integrator_dependency = false;
 
     /* generate surface shader */
-    if (shader->used && graph && output->input("Surface")->link) {
+    if (shader->reference_count() && graph && output->input("Surface")->link) {
       shader->osl_surface_ref = compile_type(shader, shader->graph, SHADER_TYPE_SURFACE);
 
       if (has_bump)
@@ -1165,7 +1165,7 @@ void OSLCompiler::compile(OSLGlobals *og, Shader *shader)
     }
 
     /* generate volume shader */
-    if (shader->used && graph && output->input("Volume")->link) {
+    if (shader->reference_count() && graph && output->input("Volume")->link) {
       shader->osl_volume_ref = compile_type(shader, shader->graph, SHADER_TYPE_VOLUME);
       shader->has_volume = true;
     }
@@ -1173,7 +1173,7 @@ void OSLCompiler::compile(OSLGlobals *og, Shader *shader)
       shader->osl_volume_ref = OSL::ShaderGroupRef();
 
     /* generate displacement shader */
-    if (shader->used && graph && output->input("Displacement")->link) {
+    if (shader->reference_count() && graph && output->input("Displacement")->link) {
       shader->osl_displacement_ref = compile_type(shader, shader->graph, SHADER_TYPE_DISPLACEMENT);
       shader->has_displacement = true;
     }
diff --git a/intern/cycles/render/osl.h b/intern/cycles/render/osl.h
index ea2d0ca492c..c9eaf26e70a 100644
--- a/intern/cycles/render/osl.h
+++ b/intern/cycles/render/osl.h
@@ -79,7 +79,10 @@ class OSLShaderManager : public ShaderManager {
     return true;
   }
 
-  void device_update(Device *device, DeviceScene *dscene, Scene *scene, Progress &progress);
+  void device_update_specific(Device *device,
+                              DeviceScene *dscene,
+                              Scene *scene,
+                              Progress &progress) override;
   void device_free(Device *device, DeviceScene *dscene, Scene *scene);
 
   /* osl compile and query */
diff --git a/intern/cycles/render/scene.cpp b/intern/cycles/render/scene.cpp
index dca545fa56c..96dda8cb1b1 100644
--- a/intern/cycles/render/scene.cpp
+++ b/intern/cycles/render/scene.cpp
@@ -523,9 +523,6 @@ bool Scene::update(Progress &progress, bool &kernel_switch_needed)
 {
   /* update scene */
   if (need_update()) {
-    /* Updated used shader tag so we know which features are need for the kernel. */
-    shader_manager->update_shaders_used(this);
-
     /* Update max_closures. */
     KernelIntegrator *kintegrator = &dscene.data.integrator;
     if (params.background) {
@@ -606,7 +603,7 @@ int Scene::get_max_closure_count()
   int max_closures = 0;
   for (int i = 0; i < shaders.size(); i++) {
     Shader *shader = shaders[i];
-    if (shader->used) {
+    if (shader->reference_count()) {
       int num_closures = shader->graph->get_num_closures();
       max_closures = max(max_closures, num_closures);
     }
@@ -767,9 +764,10 @@ template<> void Scene::delete_node_impl(ParticleSystem *node)
   particle_system_manager->tag_update(this);
 }
 
-template<> void Scene::delete_node_impl(Shader * /*node*/)
+template<> void Scene::delete_node_impl(Shader *shader)
 {
   /* don't delete unused shaders, not supported */
+  shader->clear_reference_count();
 }
 
 template<> void Scene::delete_node_impl(Procedural *node)
@@ -836,9 +834,12 @@ template<> void Scene::delete_nodes(const set<ParticleSystem *> &nodes, const No
   particle_system_manager->tag_update(this);
 }
 
-template<> void Scene::delete_nodes(const set<Shader *> & /*nodes*/, const NodeOwner * /*owner*/)
+template<> void Scene::delete_nodes(const set<Shader *> &nodes, const NodeOwner * /*owner*/)
 {
   /* don't delete unused shaders, not supported */
+  for (Shader *shader : nodes) {
+    shader->clear_reference_count();
+  }
 }
 
 template<> void Scene::delete_nodes(const set<Procedural *> &nodes, const NodeOwner *owner)
diff --git a/intern/cycles/render/shader.cpp b/intern/cycles/render/shader.cpp
index 44a48cd2839..59b60904746 100644
--- a/intern/cycles/render/shader.cpp
+++ b/intern/cycles/render/shader.cpp
@@ -16,7 +16,6 @@
 
 #include "device/device.h"
 
-#include "render/alembic.h"
 #include "render/background.h"
 #include "render/camera.h"
 #include "render/colorspace.h"
@@ -27,6 +26,7 @@
 #include "render/nodes.h"
 #include "render/object.h"
 #include "render/osl.h"
+#include "render/procedural.h"
 #include "render/scene.h"
 #include "render/shader.h"
 #include "render/svm.h"
@@ -218,7 +218,6 @@ Shader::Shader() : Node(get_node_type())
   displacement_method = DISPLACE_BUMP;
 
   id = -1;
-  used = false;
 
   need_update_uvs = true;
   need_update_attribute = true;
@@ -382,8 +381,9 @@ void Shader::tag_used(Scene *scene)
 {
   /* if an unused shader suddenly gets used somewhere, it needs to be
    * recompiled because it was skipped for compilation before */
-  if (!used) {
+  if (!reference_count()) {
     tag_modified();
+    /* We do not reference here as the shader will be referenced when added to a socket. */
     scene->shader_manager->tag_update(scene, ShaderManager::SHADER_MODIFIED);
   }
 }
@@ -461,52 +461,28 @@ int ShaderManager::get_shader_id(Shader *shader, bool smooth)
   return id;
 }
 
-void ShaderManager::update_shaders_used(Scene *scene)
+void ShaderManager::device_update(Device *device,
+                                  DeviceScene *dscene,
+                                  Scene *scene,
+                                  Progress &progress)
 {
   if (!need_update()) {
     return;
   }
 
-  /* figure out which shaders are in use, so SVM/OSL can skip compiling them
-   * for speed and avoid loading image textures into memory */
   uint id = 0;
   foreach (Shader *shader, scene->shaders) {
-    shader->used = false;
     shader->id = id++;
   }
 
-  scene->default_surface->used = true;
-  scene->default_light->used = true;
-  scene->default_background->used = true;
-  scene->default_empty->used = true;
+  /* Those shaders should always be compiled as they are used as fallback if a shader cannot be
+   * found, e.g. bad shader index for the triangle shaders on a Mesh. */
+  assert(scene->default_surface->reference_count() != 0);
+  assert(scene->default_light->reference_count() != 0);
+  assert(scene->default_background->reference_count() != 0);
+  assert(scene->default_empty->reference_count() != 0);
 
-  if (scene->background->get_shader())
-    scene->background->get_shader()->used = true;
-
-#ifdef WITH_ALEMBIC
-  foreach (Procedural *procedural, scene->procedurals) {
-    AlembicProcedural *abc_proc = static_cast<AlembicProcedural *>(procedural);
-
-    foreach (Node *abc_node, abc_proc->get_objects()) {
-      AlembicObject *abc_object = static_cast<AlembicObject *>(abc_node);
-
-      foreach (Node *node, abc_object->get_used_shaders()) {
-        Shader *shader = static_cast<Shader *>(node);
-        shader->used = true;
-      }
-    }
-  }
-#endif
-
-  foreach (Geometry *geom, scene->geometry)
-    foreach (Node *node, geom->get_used_shaders()) {
-      Shader *shader = static_cast<Shader *>(node);
-      shader->used = true;
-    }
-
-  foreach (Light *light, scene->lights)
-    if (light->get_shader())
-      const_cast<Shader *>(light->get_shader())->used = true;
+  device_update_specific(device, dscene, scene, progress);
 }
 
 void ShaderManager::device_update_common(Device *device,
@@ -639,6 +615,7 @@ void ShaderManager::add_default(Scene *scene)
     Shader *shader = scene->create_node<Shader>();
     shader->name = "default_surface";
     shader->set_graph(graph);
+    shader->reference();
     scene->default_surface = shader;
     shader->tag_update(scene);
   }
@@ -657,6 +634,8 @@ void ShaderManager::add_default(Scene *scene)
     shader->set_graph(graph);
     scene->default_volume = shader;
     shader->tag_update(scene);
+    /* No default reference for the volume to avoid compiling volume kernels if there are no actual
+     * volumes in the scene */
   }
 
   /* default light */
@@ -673,6 +652,7 @@ void ShaderManager::add_default(Scene *scene)
     Shader *shader = scene->create_node<Shader>();
     shader->name = "default_light";
     shader->set_graph(graph);
+    shader->reference();
     scene->default_light = shader;
     shader->tag_update(scene);
   }
@@ -684,6 +664,7 @@ void ShaderManager::add_default(Scene *scene)
     Shader *shader = scene->create_node<Shader>();
     shader->name = "default_background";
     shader->set_graph(graph);
+    shader->reference();
     scene->default_background = shader;
     shader->tag_update(scene);
   }
@@ -695,6 +676,7 @@ void ShaderManager::add_default(Scene *scene)
     Shader *shader = scene->create_node<Shader>();
     shader->name = "default_empty";
     shader->set_graph(graph);
+    shader->reference();
     scene->default_empty = shader;
     shader->tag_update(scene);
   }
@@ -735,7 +717,7 @@ void ShaderManager::get_requested_features(Scene *scene,
   requested_features->nodes_features = 0;
   for (int i = 0; i < scene->shaders.size(); i++) {
     Shader *shader = scene->shaders[i];
-    if (!shader->used) {
+    if (!shader->reference_count()) {
       continue;
     }
 
diff --git a/intern/cycles/render/shader.h b/intern/cycles/render/shader.h
index f47d64f346c..50c8bed4669 100644
--- a/intern/cycles/render/shader.h
+++ b/intern/cycles/render/shader.h
@@ -132,7 +132,6 @@ class Shader : public Node {
 
   /* determined before compiling */
   uint id;
-  bool used;
 
 #ifdef WITH_OSL
   /* osl shading state references */
@@ -187,10 +186,11 @@ class ShaderManager {
   }
 
   /* device update */
-  virtual void device_update(Device *device,
-                             DeviceScene *dscene,
-                             Scene *scene,
-                             Progress &progress) = 0;
+  void device_update(Device *device, DeviceScene *dscene, Scene *scene, Progress &progress);
+  virtual void device_update_specific(Device *device,
+                                      DeviceScene *dscene,
+                                      Scene *scene,
+                                      Progress &progress) = 0;
   virtual void device_free(Device *device, DeviceScene *dscene, Scene *scene) = 0;
 
   void device_update_common(Device *device, DeviceScene *dscene, Scene *scene, Progress &progress);
@@ -208,7 +208,6 @@ class ShaderManager {
   static void add_default(Scene *scene);
 
   /* Selective nodes compilation. */
-  void update_shaders_used(Scene *scene);
   void get_requested_features(Scene *scene, DeviceRequestedFeatures *requested_features);
 
   static void free_memory();
diff --git a/intern/cycles/render/svm.cpp b/intern/cycles/render/svm.cpp
index fce604234f1..5c793c5c016 100644
--- a/intern/cycles/render/svm.cpp
+++ b/intern/cycles/render/svm.cpp
@@ -69,10 +69,10 @@ void SVMShaderManager::device_update_shader(Scene *scene,
           << summary.full_report();
 }
 
-void SVMShaderManager::device_update(Device *device,
-                                     DeviceScene *dscene,
-                                     Scene *scene,
-                                     Progress &progress)
+void SVMShaderManager::device_update_specific(Device *device,
+                                              DeviceScene *dscene,
+                                              Scene *scene,
+                                              Progress &progress)
 {
   if (!need_update())
     return;
@@ -776,7 +776,7 @@ void SVMCompiler::compile_type(Shader *shader, ShaderGraph *graph, ShaderType ty
     add_node(NODE_ENTER_BUMP_EVAL, bump_state_offset);
   }
 
-  if (shader->used) {
+  if (shader->reference_count()) {
     CompilerState state(graph);
     if (clin->link) {
       bool generate = false;
diff --git a/intern/cycles/render/svm.h b/intern/cycles/render/svm.h
index a4ca68e1d8d..85b4b9c419f 100644
--- a/intern/cycles/render/svm.h
+++ b/intern/cycles/render/svm.h
@@ -46,7 +46,10 @@ class SVMShaderManager : public ShaderManager {
 
   void reset(Scene *scene);
 
-  void device_update(Device *device, DeviceScene *dscene, Scene *scene, Progress &progress);
+  void device_update_specific(Device *device,
+                              DeviceScene *dscene,
+                              Scene *scene,
+                              Progress &progress) override;
   void device_free(Device *device, DeviceScene *dscene, Scene *scene);
 
  protected: