Cycles: Add CPU+GPU rendering support with OptiX

Adds support for building multiple BVH types in order to support using both CPU and OptiX
devices for rendering simultaneously. Primitive packing for Embree and OptiX is now
standalone, so it only needs to be run once and can be shared between the two. Additionally,
BVH building was made a device call, so that each device backend can decide how to
perform the building. The multi-device for instance creates a special multi-BVH that holds
references to several sub-BVHs, one for each sub-device.

Reviewed By: brecht, kevindietrich

Differential Revision: https://developer.blender.org/D9718

8 files changed, 139 insertions, 30 deletions

diff --git a/intern/cycles/render/geometry.cpp b/intern/cycles/render/geometry.cpp
index 12f4eaf0b79..64b98a91853 100644
--- a/intern/cycles/render/geometry.cpp
+++ b/intern/cycles/render/geometry.cpp
@@ -15,8 +15,7 @@
  */
 
 #include "bvh/bvh.h"
-#include "bvh/bvh_build.h"
-#include "bvh/bvh_embree.h"
+#include "bvh/bvh2.h"
 
 #include "device/device.h"
 
@@ -41,6 +40,7 @@
 #include "util/util_foreach.h"
 #include "util/util_logging.h"
 #include "util/util_progress.h"
+#include "util/util_task.h"
 
 CCL_NAMESPACE_BEGIN
 
@@ -162,7 +162,8 @@ int Geometry::motion_step(float time) const
 
 bool Geometry::need_build_bvh(BVHLayout layout) const
 {
-  return !transform_applied || has_surface_bssrdf || layout == BVH_LAYOUT_OPTIX;
+  return is_instanced() || layout == BVH_LAYOUT_OPTIX || layout == BVH_LAYOUT_MULTI_OPTIX ||
+         layout == BVH_LAYOUT_MULTI_OPTIX_EMBREE;
 }
 
 bool Geometry::is_instanced() const
@@ -218,7 +219,7 @@ void Geometry::compute_bvh(
       bvh->geometry = geometry;
       bvh->objects = objects;
 
-      bvh->refit(*progress);
+      device->build_bvh(bvh, *progress, true);
     }
     else {
       progress->set_status(msg, "Building BVH");
@@ -235,7 +236,7 @@ void Geometry::compute_bvh(
 
       delete bvh;
       bvh = BVH::create(bparams, geometry, objects, device);
-      MEM_GUARDED_CALL(progress, bvh->build, *progress);
+      MEM_GUARDED_CALL(progress, device->build_bvh, bvh, *progress, false);
     }
   }
 
@@ -1162,25 +1163,66 @@ void GeometryManager::device_update_bvh(Device *device,
 
   VLOG(1) << "Using " << bvh_layout_name(bparams.bvh_layout) << " layout.";
 
-  BVH *bvh = BVH::create(bparams, scene->geometry, scene->objects, device);
-  bvh->build(progress, &device->stats);
+  delete scene->bvh;
+  BVH *bvh = scene->bvh = BVH::create(bparams, scene->geometry, scene->objects, device);
+  device->build_bvh(bvh, progress, false);
 
   if (progress.get_cancel()) {
-#ifdef WITH_EMBREE
-    if (dscene->data.bvh.scene) {
-      BVHEmbree::destroy(dscene->data.bvh.scene);
-      dscene->data.bvh.scene = NULL;
-    }
-#endif
-    delete bvh;
     return;
   }
 
+  PackedBVH pack;
+  if (bparams.bvh_layout == BVH_LAYOUT_BVH2) {
+    pack = std::move(static_cast<BVH2 *>(bvh)->pack);
+  }
+  else {
+    progress.set_status("Updating Scene BVH", "Packing BVH primitives");
+
+    size_t num_prims = 0;
+    size_t num_tri_verts = 0;
+    foreach (Geometry *geom, scene->geometry) {
+      if (geom->geometry_type == Geometry::MESH || geom->geometry_type == Geometry::VOLUME) {
+        Mesh *mesh = static_cast<Mesh *>(geom);
+        num_prims += mesh->num_triangles();
+        num_tri_verts += 3 * mesh->num_triangles();
+      }
+      else if (geom->is_hair()) {
+        Hair *hair = static_cast<Hair *>(geom);
+        num_prims += hair->num_segments();
+      }
+    }
+
+    pack.root_index = -1;
+    pack.prim_tri_index.reserve(num_prims);
+    pack.prim_tri_verts.reserve(num_tri_verts);
+    pack.prim_type.reserve(num_prims);
+    pack.prim_index.reserve(num_prims);
+    pack.prim_object.reserve(num_prims);
+    pack.prim_visibility.reserve(num_prims);
+
+    // Merge visibility flags of all objects and find object index for non-instanced geometry
+    unordered_map<const Geometry *, pair<int, uint>> geometry_to_object_info;
+    geometry_to_object_info.reserve(scene->geometry.size());
+    foreach (Object *ob, scene->objects) {
+      const Geometry *const geom = ob->get_geometry();
+      pair<int, uint> &info = geometry_to_object_info[geom];
+      info.second |= ob->visibility_for_tracing();
+      if (!geom->is_instanced()) {
+        info.first = ob->get_device_index();
+      }
+    }
+
+    // Iterate over scene mesh list instead of objects, since 'optix_prim_offset' was calculated
+    // based on that list, which may be ordered differently from the object list.
+    foreach (Geometry *geom, scene->geometry) {
+      const pair<int, uint> &info = geometry_to_object_info[geom];
+      geom->pack_primitives(pack, info.first, info.second);
+    }
+  }
+
   /* copy to device */
   progress.set_status("Updating Scene BVH", "Copying BVH to device");
 
-  PackedBVH &pack = bvh->pack;
-
   if (pack.nodes.size()) {
     dscene->bvh_nodes.steal_data(pack.nodes);
     dscene->bvh_nodes.copy_to_device();
@@ -1226,10 +1268,8 @@ void GeometryManager::device_update_bvh(Device *device,
   dscene->data.bvh.bvh_layout = bparams.bvh_layout;
   dscene->data.bvh.use_bvh_steps = (scene->params.num_bvh_time_steps != 0);
   dscene->data.bvh.curve_subdivisions = scene->params.curve_subdivisions();
-
-  bvh->copy_to_device(progress, dscene);
-
-  delete bvh;
+  /* The scene handle is set in 'CPUDevice::const_copy_to' and 'OptiXDevice::const_copy_to' */
+  dscene->data.bvh.scene = NULL;
 }
 
 void GeometryManager::device_update_preprocess(Device *device, Scene *scene, Progress &progress)
@@ -1653,14 +1693,6 @@ void GeometryManager::device_update(Device *device,
 
 void GeometryManager::device_free(Device *device, DeviceScene *dscene)
 {
-#ifdef WITH_EMBREE
-  if (dscene->data.bvh.scene) {
-    if (dscene->data.bvh.bvh_layout == BVH_LAYOUT_EMBREE)
-      BVHEmbree::destroy(dscene->data.bvh.scene);
-    dscene->data.bvh.scene = NULL;
-  }
-#endif
-
   dscene->bvh_nodes.free();
   dscene->bvh_leaf_nodes.free();
   dscene->object_node.free();
diff --git a/intern/cycles/render/geometry.h b/intern/cycles/render/geometry.h
index 1c101540464..d3daf0cc809 100644
--- a/intern/cycles/render/geometry.h
+++ b/intern/cycles/render/geometry.h
@@ -41,6 +41,7 @@ class Scene;
 class SceneParams;
 class Shader;
 class Volume;
+struct PackedBVH;
 
 /* Geometry
  *
@@ -124,6 +125,8 @@ class Geometry : public Node {
                    int n,
                    int total);
 
+  virtual void pack_primitives(PackedBVH &pack, int object, uint visibility) = 0;
+
   /* Check whether the geometry should have own BVH built separately. Briefly,
    * own BVH is needed for geometry, if:
    *
diff --git a/intern/cycles/render/hair.cpp b/intern/cycles/render/hair.cpp
index d67bd209142..a64b8242337 100644
--- a/intern/cycles/render/hair.cpp
+++ b/intern/cycles/render/hair.cpp
@@ -14,6 +14,8 @@
  * limitations under the License.
  */
 
+#include "bvh/bvh.h"
+
 #include "render/hair.h"
 #include "render/curves.h"
 #include "render/scene.h"
@@ -492,4 +494,35 @@ void Hair::pack_curves(Scene *scene,
   }
 }
 
+void Hair::pack_primitives(PackedBVH &pack, int object, uint visibility)
+{
+  if (curve_first_key.empty())
+    return;
+
+  const size_t num_prims = num_segments();
+  pack.prim_tri_index.reserve(pack.prim_tri_index.size() + num_prims);
+  pack.prim_type.reserve(pack.prim_type.size() + num_prims);
+  pack.prim_visibility.reserve(pack.prim_visibility.size() + num_prims);
+  pack.prim_index.reserve(pack.prim_index.size() + num_prims);
+  pack.prim_object.reserve(pack.prim_object.size() + num_prims);
+  // 'pack.prim_time' is unused by Embree and OptiX
+
+  uint type = has_motion_blur() ?
+                  ((curve_shape == CURVE_RIBBON) ? PRIMITIVE_MOTION_CURVE_RIBBON :
+                                                   PRIMITIVE_MOTION_CURVE_THICK) :
+                  ((curve_shape == CURVE_RIBBON) ? PRIMITIVE_CURVE_RIBBON : PRIMITIVE_CURVE_THICK);
+
+  for (size_t j = 0; j < num_curves(); ++j) {
+    Curve curve = get_curve(j);
+    for (size_t k = 0; k < curve.num_segments(); ++k) {
+      pack.prim_tri_index.push_back_reserved(-1);
+      pack.prim_type.push_back_reserved(PRIMITIVE_PACK_SEGMENT(type, k));
+      pack.prim_visibility.push_back_reserved(visibility);
+      // Each curve segment points back to its curve index
+      pack.prim_index.push_back_reserved(j + prim_offset);
+      pack.prim_object.push_back_reserved(object);
+    }
+  }
+}
+
 CCL_NAMESPACE_END
diff --git a/intern/cycles/render/hair.h b/intern/cycles/render/hair.h
index 32c5b00e879..d86a34fb87b 100644
--- a/intern/cycles/render/hair.h
+++ b/intern/cycles/render/hair.h
@@ -145,6 +145,8 @@ class Hair : public Geometry {
 
   /* BVH */
   void pack_curves(Scene *scene, float4 *curve_key_co, float4 *curve_data, size_t curvekey_offset);
+
+  void pack_primitives(PackedBVH &pack, int object, uint visibility);
 };
 
 CCL_NAMESPACE_END
diff --git a/intern/cycles/render/mesh.cpp b/intern/cycles/render/mesh.cpp
index 43e664a686f..a0afdd3b841 100644
--- a/intern/cycles/render/mesh.cpp
+++ b/intern/cycles/render/mesh.cpp
@@ -805,4 +805,35 @@ void Mesh::pack_patches(uint *patch_data, uint vert_offset, uint face_offset, ui
   }
 }
 
+void Mesh::pack_primitives(PackedBVH &pack, int object, uint visibility)
+{
+  if (triangles.empty())
+    return;
+
+  const size_t num_prims = num_triangles();
+  pack.prim_tri_index.reserve(pack.prim_tri_index.size() + num_prims);
+  pack.prim_tri_verts.reserve(pack.prim_tri_verts.size() + num_prims * 3);
+  pack.prim_type.reserve(pack.prim_type.size() + num_prims);
+  pack.prim_visibility.reserve(pack.prim_visibility.size() + num_prims);
+  pack.prim_index.reserve(pack.prim_index.size() + num_prims);
+  pack.prim_object.reserve(pack.prim_object.size() + num_prims);
+  // 'pack.prim_time' is unused by Embree and OptiX
+
+  uint type = has_motion_blur() ? PRIMITIVE_MOTION_TRIANGLE : PRIMITIVE_TRIANGLE;
+
+  for (size_t k = 0; k < num_prims; ++k) {
+    pack.prim_tri_index.push_back_reserved(pack.prim_tri_verts.size());
+
+    const Mesh::Triangle t = get_triangle(k);
+    pack.prim_tri_verts.push_back_reserved(float3_to_float4(verts[t.v[0]]));
+    pack.prim_tri_verts.push_back_reserved(float3_to_float4(verts[t.v[1]]));
+    pack.prim_tri_verts.push_back_reserved(float3_to_float4(verts[t.v[2]]));
+
+    pack.prim_type.push_back_reserved(type);
+    pack.prim_visibility.push_back_reserved(visibility);
+    pack.prim_index.push_back_reserved(k + prim_offset);
+    pack.prim_object.push_back_reserved(object);
+  }
+}
+
 CCL_NAMESPACE_END
diff --git a/intern/cycles/render/mesh.h b/intern/cycles/render/mesh.h
index e2746e560da..b0a16fdfd8f 100644
--- a/intern/cycles/render/mesh.h
+++ b/intern/cycles/render/mesh.h
@@ -184,9 +184,8 @@ class Mesh : public Geometry {
   unordered_multimap<int, int>
       vert_stitching_map; /* stitching index -> multiple real vert indices */
 
-  friend class BVH;
+  friend class BVH2;
   friend class BVHBuild;
-  friend class BVHEmbree;
   friend class BVHSpatialSplit;
   friend class DiagSplit;
   friend class EdgeDice;
@@ -233,6 +232,8 @@ class Mesh : public Geometry {
                   size_t tri_offset);
   void pack_patches(uint *patch_data, uint vert_offset, uint face_offset, uint corner_offset);
 
+  void pack_primitives(PackedBVH &pack, int object, uint visibility) override;
+
   void tessellate(DiagSplit *split);
 
   SubdFace get_subd_face(size_t index) const;
diff --git a/intern/cycles/render/scene.cpp b/intern/cycles/render/scene.cpp
index 98c256a43b5..b7720b7aa99 100644
--- a/intern/cycles/render/scene.cpp
+++ b/intern/cycles/render/scene.cpp
@@ -16,6 +16,7 @@
 
 #include <stdlib.h>
 
+#include "bvh/bvh.h"
 #include "device/device.h"
 #include "render/background.h"
 #include "render/bake.h"
@@ -100,6 +101,7 @@ Scene::Scene(const SceneParams &params_, Device *device)
 {
   memset((void *)&dscene.data, 0, sizeof(dscene.data));
 
+  bvh = NULL;
   camera = create_node<Camera>();
   dicing_camera = create_node<Camera>();
   lookup_tables = new LookupTables();
@@ -135,6 +137,9 @@ Scene::~Scene()
 
 void Scene::free_memory(bool final)
 {
+  delete bvh;
+  bvh = NULL;
+
   foreach (Shader *s, shaders)
     delete s;
   foreach (Geometry *g, geometry)
diff --git a/intern/cycles/render/scene.h b/intern/cycles/render/scene.h
index 6686327dc49..27e9a131bbd 100644
--- a/intern/cycles/render/scene.h
+++ b/intern/cycles/render/scene.h
@@ -38,6 +38,7 @@ CCL_NAMESPACE_BEGIN
 
 class AttributeRequestSet;
 class Background;
+class BVH;
 class Camera;
 class Device;
 class DeviceInfo;
@@ -220,6 +221,7 @@ class Scene : public NodeOwner {
   string name;
 
   /* data */
+  BVH *bvh;
   Camera *camera;
   Camera *dicing_camera;
   LookupTables *lookup_tables;