Cycles: internal support for the concept of procedurals

Procedurals are nodes in the scene that can generate an arbitrary number of
other nodes at render time. This will be used to implement an Alembic procedural
that can load an Alembic file into Cycles nodes. In the future we also expect to
have a USD procedural.

Direct loading of such files at render time is a standard feature in other
production renderers. Reasons to support this are memory usage and performance,
delayed loading of heavy scene data until rendering, Cycles standalone rendering
using standard file formats beyond our XML files, and shared functionality for
Cycles integration in multiple 3D apps.

Ref T79174, D3089

9 files changed, 204 insertions, 1 deletions

diff --git a/intern/cycles/blender/blender_sync.cpp b/intern/cycles/blender/blender_sync.cpp
index 6b2039e7e21..aae2c35da4b 100644
--- a/intern/cycles/blender/blender_sync.cpp
+++ b/intern/cycles/blender/blender_sync.cpp
@@ -24,6 +24,7 @@
 #include "render/mesh.h"
 #include "render/nodes.h"
 #include "render/object.h"
+#include "render/procedural.h"
 #include "render/scene.h"
 #include "render/shader.h"
 
@@ -58,6 +59,7 @@ BlenderSync::BlenderSync(BL::RenderEngine &b_engine,
       b_scene(b_scene),
       shader_map(scene),
       object_map(scene),
+      procedural_map(scene),
       geometry_map(scene),
       light_map(scene),
       particle_system_map(scene),
diff --git a/intern/cycles/render/CMakeLists.txt b/intern/cycles/render/CMakeLists.txt
index 9663e25cd93..9fcec4df5c2 100644
--- a/intern/cycles/render/CMakeLists.txt
+++ b/intern/cycles/render/CMakeLists.txt
@@ -48,6 +48,7 @@ set(SRC
   mesh_displace.cpp
   mesh_subdivision.cpp
   nodes.cpp
+  procedural.cpp
   object.cpp
   osl.cpp
   particles.cpp
@@ -90,6 +91,7 @@ set(SRC_HEADERS
   object.h
   osl.h
   particles.h
+  procedural.h
   curves.h
   scene.h
   session.h
diff --git a/intern/cycles/render/procedural.cpp b/intern/cycles/render/procedural.cpp
new file mode 100644
index 00000000000..1307a35dcf2
--- /dev/null
+++ b/intern/cycles/render/procedural.cpp
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2011-2018 Blender Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "procedural.h"
+
+#include "render/scene.h"
+#include "render/stats.h"
+
+#include "util/util_foreach.h"
+#include "util/util_progress.h"
+
+CCL_NAMESPACE_BEGIN
+
+NODE_ABSTRACT_DEFINE(Procedural)
+{
+  NodeType *type = NodeType::add("procedural_base", NULL);
+  return type;
+}
+
+Procedural::Procedural(const NodeType *type) : Node(type)
+{
+}
+
+Procedural::~Procedural()
+{
+}
+
+ProceduralManager::ProceduralManager()
+{
+  need_update_ = true;
+}
+
+ProceduralManager::~ProceduralManager()
+{
+}
+
+void ProceduralManager::update(Scene *scene, Progress &progress)
+{
+  if (!need_update()) {
+    return;
+  }
+
+  progress.set_status("Updating Procedurals");
+
+  scoped_callback_timer timer([scene](double time) {
+    if (scene->update_stats) {
+      scene->update_stats->procedurals.times.add_entry({"update", time});
+    }
+  });
+
+  foreach (Procedural *procedural, scene->procedurals) {
+    if (progress.get_cancel()) {
+      return;
+    }
+
+    procedural->generate(scene, progress);
+  }
+
+  if (progress.get_cancel()) {
+    return;
+  }
+
+  need_update_ = false;
+}
+
+void ProceduralManager::tag_update()
+{
+  need_update_ = true;
+}
+
+bool ProceduralManager::need_update() const
+{
+  return need_update_;
+}
+
+CCL_NAMESPACE_END
diff --git a/intern/cycles/render/procedural.h b/intern/cycles/render/procedural.h
new file mode 100644
index 00000000000..985b8d69979
--- /dev/null
+++ b/intern/cycles/render/procedural.h
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2011-2018 Blender Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "graph/node.h"
+
+CCL_NAMESPACE_BEGIN
+
+class Progress;
+class Scene;
+
+/* A Procedural is a Node which can create other Nodes before rendering starts.
+ *
+ * The Procedural is supposed to be the owner of any nodes that it creates. It can also create
+ * Nodes directly in the Scene (through Scene.create_node), it should still be set as the owner of
+ * those Nodes.
+ */
+class Procedural : public Node, public NodeOwner {
+ public:
+  NODE_ABSTRACT_DECLARE
+
+  explicit Procedural(const NodeType *type);
+  virtual ~Procedural();
+
+  /* Called each time the ProceduralManager is tagged for an update, this function is the entry
+   * point for the data generated by this Procedural. */
+  virtual void generate(Scene *scene, Progress &progress) = 0;
+
+  /* Create a node and set this Procedural as the owner. */
+  template<typename T> T *create_node()
+  {
+    T *node = new T();
+    node->set_owner(this);
+    return node;
+  }
+
+  /* Delete a Node created and owned by this Procedural. */
+  template<typename T> void delete_node(T *node)
+  {
+    assert(node->get_owner() == this);
+    delete node;
+  }
+};
+
+class ProceduralManager {
+  bool need_update_;
+
+ public:
+  ProceduralManager();
+  ~ProceduralManager();
+
+  void update(Scene *scene, Progress &progress);
+
+  void tag_update();
+
+  bool need_update() const;
+};
+
+CCL_NAMESPACE_END
diff --git a/intern/cycles/render/scene.cpp b/intern/cycles/render/scene.cpp
index 1c403abd0a2..93a80de1ede 100644
--- a/intern/cycles/render/scene.cpp
+++ b/intern/cycles/render/scene.cpp
@@ -29,6 +29,7 @@
 #include "render/object.h"
 #include "render/osl.h"
 #include "render/particles.h"
+#include "render/procedural.h"
 #include "render/scene.h"
 #include "render/session.h"
 #include "render/shader.h"
@@ -114,6 +115,7 @@ Scene::Scene(const SceneParams &params_, Device *device)
   image_manager = new ImageManager(device->info);
   particle_system_manager = new ParticleSystemManager();
   bake_manager = new BakeManager();
+  procedural_manager = new ProceduralManager();
   kernels_loaded = false;
 
   /* TODO(sergey): Check if it's indeed optimal value for the split kernel. */
@@ -142,6 +144,9 @@ void Scene::free_memory(bool final)
 
   foreach (Shader *s, shaders)
     delete s;
+  /* delete procedurals before other types as they may hold pointers to those types */
+  foreach (Procedural *p, procedurals)
+    delete p;
   foreach (Geometry *g, geometry)
     delete g;
   foreach (Object *o, objects)
@@ -156,6 +161,7 @@ void Scene::free_memory(bool final)
   objects.clear();
   lights.clear();
   particle_systems.clear();
+  procedurals.clear();
 
   if (device) {
     camera->device_free(device, &dscene, this);
@@ -195,6 +201,7 @@ void Scene::free_memory(bool final)
     delete image_manager;
     delete bake_manager;
     delete update_stats;
+    delete procedural_manager;
   }
 }
 
@@ -236,6 +243,11 @@ void Scene::device_update(Device *device_, Progress &progress)
   if (progress.get_cancel() || device->have_error())
     return;
 
+  procedural_manager->update(this, progress);
+
+  if (progress.get_cancel())
+    return;
+
   progress.set_status("Updating Background");
   background->device_update(device, &dscene, this);
 
@@ -391,7 +403,7 @@ bool Scene::need_data_update()
           light_manager->need_update() || lookup_tables->need_update() ||
           integrator->is_modified() || shader_manager->need_update() ||
           particle_system_manager->need_update() || bake_manager->need_update() ||
-          film->is_modified());
+          film->is_modified() || procedural_manager->need_update());
 }
 
 bool Scene::need_reset()
@@ -416,6 +428,7 @@ void Scene::reset()
   geometry_manager->tag_update(this, GeometryManager::UPDATE_ALL);
   light_manager->tag_update(this, LightManager::UPDATE_ALL);
   particle_system_manager->tag_update(this);
+  procedural_manager->tag_update();
 }
 
 void Scene::device_free()
@@ -726,6 +739,12 @@ template<> void Scene::delete_node_impl(Shader * /*node*/)
   /* don't delete unused shaders, not supported */
 }
 
+template<> void Scene::delete_node_impl(Procedural *node)
+{
+  delete_node_from_array(procedurals, node);
+  procedural_manager->tag_update();
+}
+
 template<typename T>
 static void remove_nodes_in_set(const set<T *> &nodes_set,
                                 vector<T *> &nodes_array,
@@ -780,4 +799,10 @@ template<> void Scene::delete_nodes(const set<Shader *> & /*nodes*/, const NodeO
   /* don't delete unused shaders, not supported */
 }
 
+template<> void Scene::delete_nodes(const set<Procedural *> &nodes, const NodeOwner *owner)
+{
+  remove_nodes_in_set(nodes, procedurals, owner);
+  procedural_manager->tag_update();
+}
+
 CCL_NAMESPACE_END
diff --git a/intern/cycles/render/scene.h b/intern/cycles/render/scene.h
index 27e9a131bbd..2deb3cd91e5 100644
--- a/intern/cycles/render/scene.h
+++ b/intern/cycles/render/scene.h
@@ -53,6 +53,8 @@ class Object;
 class ObjectManager;
 class ParticleSystemManager;
 class ParticleSystem;
+class Procedural;
+class ProceduralManager;
 class CurveSystemManager;
 class Shader;
 class ShaderManager;
@@ -236,6 +238,7 @@ class Scene : public NodeOwner {
   vector<Light *> lights;
   vector<ParticleSystem *> particle_systems;
   vector<Pass> passes;
+  vector<Procedural *> procedurals;
 
   /* data managers */
   ImageManager *image_manager;
@@ -245,6 +248,7 @@ class Scene : public NodeOwner {
   ObjectManager *object_manager;
   ParticleSystemManager *particle_system_manager;
   BakeManager *bake_manager;
+  ProceduralManager *procedural_manager;
 
   /* default shaders */
   Shader *default_surface;
@@ -395,6 +399,8 @@ template<> void Scene::delete_node_impl(ParticleSystem *node);
 
 template<> void Scene::delete_node_impl(Shader *node);
 
+template<> void Scene::delete_node_impl(Procedural *node);
+
 template<> void Scene::delete_nodes(const set<Light *> &nodes, const NodeOwner *owner);
 
 template<> void Scene::delete_nodes(const set<Geometry *> &nodes, const NodeOwner *owner);
@@ -405,6 +411,8 @@ template<> void Scene::delete_nodes(const set<ParticleSystem *> &nodes, const No
 
 template<> void Scene::delete_nodes(const set<Shader *> &nodes, const NodeOwner *owner);
 
+template<> void Scene::delete_nodes(const set<Procedural *> &nodes, const NodeOwner *owner);
+
 CCL_NAMESPACE_END
 
 #endif /*  __SCENE_H__ */
diff --git a/intern/cycles/render/shader.cpp b/intern/cycles/render/shader.cpp
index db765878f07..8f2030c6fe9 100644
--- a/intern/cycles/render/shader.cpp
+++ b/intern/cycles/render/shader.cpp
@@ -366,6 +366,7 @@ void Shader::tag_update(Scene *scene)
   if (attributes.modified(prev_attributes)) {
     need_update_attribute = true;
     scene->geometry_manager->tag_update(scene, GeometryManager::SHADER_ATTRIBUTE_MODIFIED);
+    scene->procedural_manager->tag_update();
   }
 
   if (has_volume != prev_has_volume || volume_step_rate != prev_volume_step_rate) {
diff --git a/intern/cycles/render/stats.cpp b/intern/cycles/render/stats.cpp
index 1a840a906a5..2c6273842e2 100644
--- a/intern/cycles/render/stats.cpp
+++ b/intern/cycles/render/stats.cpp
@@ -375,6 +375,7 @@ string SceneUpdateStats::full_report()
   result += "Particles:\n" + particles.full_report(1);
   result += "SVM:\n" + svm.full_report(1);
   result += "Tables:\n" + tables.full_report(1);
+  result += "Procedurals:\n" + procedurals.full_report(1);
   return result;
 }
 
@@ -394,6 +395,7 @@ void SceneUpdateStats::clear()
   scene.times.clear();
   svm.times.clear();
   tables.times.clear();
+  procedurals.times.clear();
 }
 
 CCL_NAMESPACE_END
diff --git a/intern/cycles/render/stats.h b/intern/cycles/render/stats.h
index a6be27db4c2..93d029bba61 100644
--- a/intern/cycles/render/stats.h
+++ b/intern/cycles/render/stats.h
@@ -219,6 +219,7 @@ class SceneUpdateStats {
   UpdateTimeStats scene;
   UpdateTimeStats svm;
   UpdateTimeStats tables;
+  UpdateTimeStats procedurals;
 
   string full_report();