Cycles: Separate Embree device for each CPU Device.

Before, Cycles was using a shared Embree device across all instances.
This could result in crashes when viewport rendering and material
preview were using Cycles simultaneously.

Fixes issue T80042

Maniphest Tasks: T80042

Differential Revision: https://developer.blender.org/D8772

7 files changed, 51 insertions, 69 deletions

diff --git a/intern/cycles/bvh/bvh.cpp b/intern/cycles/bvh/bvh.cpp
index e9e67fd1305..2a1114229da 100644
--- a/intern/cycles/bvh/bvh.cpp
+++ b/intern/cycles/bvh/bvh.cpp
@@ -98,14 +98,15 @@ BVH::BVH(const BVHParams &params_,
 
 BVH *BVH::create(const BVHParams &params,
                  const vector<Geometry *> &geometry,
-                 const vector<Object *> &objects)
+                 const vector<Object *> &objects,
+                 const Device *device)
 {
   switch (params.bvh_layout) {
     case BVH_LAYOUT_BVH2:
       return new BVH2(params, geometry, objects);
     case BVH_LAYOUT_EMBREE:
 #ifdef WITH_EMBREE
-      return new BVHEmbree(params, geometry, objects);
+      return new BVHEmbree(params, geometry, objects, device);
 #else
       break;
 #endif
diff --git a/intern/cycles/bvh/bvh.h b/intern/cycles/bvh/bvh.h
index 6639e06b0bc..033b1fd8e04 100644
--- a/intern/cycles/bvh/bvh.h
+++ b/intern/cycles/bvh/bvh.h
@@ -89,7 +89,8 @@ class BVH {
 
   static BVH *create(const BVHParams &params,
                      const vector<Geometry *> &geometry,
-                     const vector<Object *> &objects);
+                     const vector<Object *> &objects,
+                     const Device *device);
   virtual ~BVH()
   {
   }
diff --git a/intern/cycles/bvh/bvh_embree.cpp b/intern/cycles/bvh/bvh_embree.cpp
index 4ef873634f0..b6f454da851 100644
--- a/intern/cycles/bvh/bvh_embree.cpp
+++ b/intern/cycles/bvh/bvh_embree.cpp
@@ -36,6 +36,8 @@
 
 #  include "bvh/bvh_embree.h"
 
+#  include "device/device.h"
+
 /* Kernel includes are necessary so that the filter function for Embree can access the packed BVH.
  */
 #  include "kernel/bvh/bvh_embree.h"
@@ -298,12 +300,6 @@ static bool rtc_progress_func(void *user_ptr, const double n)
   return !progress->get_cancel();
 }
 
-/* This is to have a shared device between all BVH instances.
-   It would be useful to actually to use a separte RTCDevice per Cycles instance. */
-RTCDevice BVHEmbree::rtc_shared_device = NULL;
-int BVHEmbree::rtc_shared_users = 0;
-thread_mutex BVHEmbree::rtc_shared_mutex;
-
 static size_t count_primitives(Geometry *geom)
 {
   if (geom->type == Geometry::MESH) {
@@ -320,11 +316,13 @@ static size_t count_primitives(Geometry *geom)
 
 BVHEmbree::BVHEmbree(const BVHParams &params_,
                      const vector<Geometry *> &geometry_,
-                     const vector<Object *> &objects_)
+                     const vector<Object *> &objects_,
+                     const Device *device)
     : BVH(params_, geometry_, objects_),
       scene(NULL),
       mem_used(0),
       top_level(NULL),
+      rtc_device((RTCDevice)device->bvh_device()),
       stats(NULL),
       curve_subdivisions(params.curve_subdivisions),
       build_quality(RTC_BUILD_QUALITY_REFIT),
@@ -332,47 +330,8 @@ BVHEmbree::BVHEmbree(const BVHParams &params_,
 {
   _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON);
   _MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON);
-  thread_scoped_lock lock(rtc_shared_mutex);
-  if (rtc_shared_users == 0) {
-    rtc_shared_device = rtcNewDevice("verbose=0");
-    /* Check here if Embree was built with the correct flags. */
-    ssize_t ret = rtcGetDeviceProperty(rtc_shared_device, RTC_DEVICE_PROPERTY_RAY_MASK_SUPPORTED);
-    if (ret != 1) {
-      assert(0);
-      VLOG(1) << "Embree is compiled without the RTC_DEVICE_PROPERTY_RAY_MASK_SUPPORTED flag."
-                 "Ray visibility will not work.";
-    }
-    ret = rtcGetDeviceProperty(rtc_shared_device, RTC_DEVICE_PROPERTY_FILTER_FUNCTION_SUPPORTED);
-    if (ret != 1) {
-      assert(0);
-      VLOG(1)
-          << "Embree is compiled without the RTC_DEVICE_PROPERTY_FILTER_FUNCTION_SUPPORTED flag."
-             "Renders may not look as expected.";
-    }
-    ret = rtcGetDeviceProperty(rtc_shared_device, RTC_DEVICE_PROPERTY_CURVE_GEOMETRY_SUPPORTED);
-    if (ret != 1) {
-      assert(0);
-      VLOG(1)
-          << "Embree is compiled without the RTC_DEVICE_PROPERTY_CURVE_GEOMETRY_SUPPORTED flag. "
-             "Line primitives will not be rendered.";
-    }
-    ret = rtcGetDeviceProperty(rtc_shared_device, RTC_DEVICE_PROPERTY_TRIANGLE_GEOMETRY_SUPPORTED);
-    if (ret != 1) {
-      assert(0);
-      VLOG(1) << "Embree is compiled without the RTC_DEVICE_PROPERTY_TRIANGLE_GEOMETRY_SUPPORTED "
-                 "flag. "
-                 "Triangle primitives will not be rendered.";
-    }
-    ret = rtcGetDeviceProperty(rtc_shared_device, RTC_DEVICE_PROPERTY_BACKFACE_CULLING_ENABLED);
-    if (ret != 0) {
-      assert(0);
-      VLOG(1) << "Embree is compiled with the RTC_DEVICE_PROPERTY_BACKFACE_CULLING_ENABLED flag. "
-                 "Renders may not look as expected.";
-    }
-  }
-  ++rtc_shared_users;
-
-  rtcSetDeviceErrorFunction(rtc_shared_device, rtc_error_func, NULL);
+
+  rtcSetDeviceErrorFunction(rtc_device, rtc_error_func, NULL);
 
   pack.root_index = -1;
 }
@@ -390,12 +349,6 @@ void BVHEmbree::destroy(RTCScene scene)
     rtcReleaseScene(scene);
     scene = NULL;
   }
-  thread_scoped_lock lock(rtc_shared_mutex);
-  --rtc_shared_users;
-  if (rtc_shared_users == 0) {
-    rtcReleaseDevice(rtc_shared_device);
-    rtc_shared_device = NULL;
-  }
 }
 
 void BVHEmbree::delete_rtcScene()
@@ -421,9 +374,9 @@ void BVHEmbree::delete_rtcScene()
 
 void BVHEmbree::build(Progress &progress, Stats *stats_)
 {
-  assert(rtc_shared_device);
+  assert(rtc_device);
   stats = stats_;
-  rtcSetDeviceMemoryMonitorFunction(rtc_shared_device, rtc_memory_monitor_func, stats);
+  rtcSetDeviceMemoryMonitorFunction(rtc_device, rtc_memory_monitor_func, stats);
 
   progress.set_substatus("Building BVH");
 
@@ -434,7 +387,7 @@ void BVHEmbree::build(Progress &progress, Stats *stats_)
 
   const bool dynamic = params.bvh_type == SceneParams::BVH_DYNAMIC;
 
-  scene = rtcNewScene(rtc_shared_device);
+  scene = rtcNewScene(rtc_device);
   const RTCSceneFlags scene_flags = (dynamic ? RTC_SCENE_FLAG_DYNAMIC : RTC_SCENE_FLAG_NONE) |
                                     RTC_SCENE_FLAG_COMPACT | RTC_SCENE_FLAG_ROBUST;
   rtcSetSceneFlags(scene, scene_flags);
@@ -561,7 +514,7 @@ void BVHEmbree::add_instance(Object *ob, int i)
   const size_t num_motion_steps = min(num_object_motion_steps, RTC_MAX_TIME_STEP_COUNT);
   assert(num_object_motion_steps <= RTC_MAX_TIME_STEP_COUNT);
 
-  RTCGeometry geom_id = rtcNewGeometry(rtc_shared_device, RTC_GEOMETRY_TYPE_INSTANCE);
+  RTCGeometry geom_id = rtcNewGeometry(rtc_device, RTC_GEOMETRY_TYPE_INSTANCE);
   rtcSetGeometryInstancedScene(geom_id, instance_bvh->scene);
   rtcSetGeometryTimeStepCount(geom_id, num_motion_steps);
 
@@ -615,7 +568,7 @@ void BVHEmbree::add_triangles(const Object *ob, const Mesh *mesh, int i)
   assert(num_geometry_motion_steps <= RTC_MAX_TIME_STEP_COUNT);
 
   const size_t num_triangles = mesh->num_triangles();
-  RTCGeometry geom_id = rtcNewGeometry(rtc_shared_device, RTC_GEOMETRY_TYPE_TRIANGLE);
+  RTCGeometry geom_id = rtcNewGeometry(rtc_device, RTC_GEOMETRY_TYPE_TRIANGLE);
   rtcSetGeometryBuildQuality(geom_id, build_quality);
   rtcSetGeometryTimeStepCount(geom_id, num_motion_steps);
 
@@ -804,7 +757,7 @@ void BVHEmbree::add_curves(const Object *ob, const Hair *hair, int i)
                                    RTC_GEOMETRY_TYPE_FLAT_CATMULL_ROM_CURVE :
                                    RTC_GEOMETRY_TYPE_ROUND_CATMULL_ROM_CURVE);
 
-  RTCGeometry geom_id = rtcNewGeometry(rtc_shared_device, type);
+  RTCGeometry geom_id = rtcNewGeometry(rtc_device, type);
   rtcSetGeometryTessellationRate(geom_id, curve_subdivisions + 1);
   unsigned *rtc_indices = (unsigned *)rtcSetNewGeometryBuffer(
       geom_id, RTC_BUFFER_TYPE_INDEX, 0, RTC_FORMAT_UINT, sizeof(int), num_segments);
diff --git a/intern/cycles/bvh/bvh_embree.h b/intern/cycles/bvh/bvh_embree.h
index f60a1ca0102..841f02eccec 100644
--- a/intern/cycles/bvh/bvh_embree.h
+++ b/intern/cycles/bvh/bvh_embree.h
@@ -50,7 +50,8 @@ class BVHEmbree : public BVH {
   friend class BVH;
   BVHEmbree(const BVHParams &params,
             const vector<Geometry *> &geometry,
-            const vector<Object *> &objects);
+            const vector<Object *> &objects,
+            const Device *device);
 
   virtual void pack_nodes(const BVHNode *) override;
   virtual void refit_nodes() override;
@@ -73,9 +74,7 @@ class BVHEmbree : public BVH {
   void update_tri_vertex_buffer(RTCGeometry geom_id, const Mesh *mesh);
   void update_curve_vertex_buffer(RTCGeometry geom_id, const Hair *hair);
 
-  static RTCDevice rtc_shared_device;
-  static int rtc_shared_users;
-  static thread_mutex rtc_shared_mutex;
+  RTCDevice rtc_device;
 
   Stats *stats;
   vector<RTCScene> delayed_delete_scenes;
diff --git a/intern/cycles/device/device.h b/intern/cycles/device/device.h
index 115b05e3911..0d0b9759c64 100644
--- a/intern/cycles/device/device.h
+++ b/intern/cycles/device/device.h
@@ -373,6 +373,12 @@ class Device {
     return NULL;
   }
 
+  /* Device specific pointer for BVH creation. Currently only used by Embree. */
+  virtual void *bvh_device() const
+  {
+    return NULL;
+  }
+
   /* load/compile kernels, must be called before adding tasks */
   virtual bool load_kernels(const DeviceRequestedFeatures & /*requested_features*/)
   {
diff --git a/intern/cycles/device/device_cpu.cpp b/intern/cycles/device/device_cpu.cpp
index ee3a3ddea64..d37ed046c1e 100644
--- a/intern/cycles/device/device_cpu.cpp
+++ b/intern/cycles/device/device_cpu.cpp
@@ -24,6 +24,10 @@
 #  include <OSL/oslexec.h>
 #endif
 
+#ifdef WITH_EMBREE
+#  include <embree3/rtcore.h>
+#endif
+
 #include "device/device.h"
 #include "device/device_denoising.h"
 #include "device/device_intern.h"
@@ -183,6 +187,9 @@ class CPUDevice : public Device {
   oidn::FilterRef oidn_filter;
 #endif
   thread_spin_lock oidn_task_lock;
+#ifdef WITH_EMBREE
+  RTCDevice embree_device;
+#endif
 
   bool use_split_kernel;
 
@@ -302,6 +309,9 @@ class CPUDevice : public Device {
 #ifdef WITH_OSL
     kernel_globals.osl = &osl_globals;
 #endif
+#ifdef WITH_EMBREE
+    embree_device = rtcNewDevice("verbose=0");
+#endif
     use_split_kernel = DebugFlags().cpu.split_kernel;
     if (use_split_kernel) {
       VLOG(1) << "Will be using split kernel.";
@@ -339,6 +349,9 @@ class CPUDevice : public Device {
 
   ~CPUDevice()
   {
+#ifdef WITH_EMBREE
+    rtcReleaseDevice(embree_device);
+#endif
     task_pool.cancel();
     texture_info.free();
   }
@@ -523,6 +536,15 @@ class CPUDevice : public Device {
 #endif
   }
 
+  void *bvh_device() const override
+  {
+#ifdef WITH_EMBREE
+    return embree_device;
+#else
+    return NULL;
+#endif
+  }
+
   void thread_run(DeviceTask &task)
   {
     if (task.type == DeviceTask::RENDER)
diff --git a/intern/cycles/render/geometry.cpp b/intern/cycles/render/geometry.cpp
index 145b1fa492c..f57b76b10ac 100644
--- a/intern/cycles/render/geometry.cpp
+++ b/intern/cycles/render/geometry.cpp
@@ -214,7 +214,7 @@ void Geometry::compute_bvh(
       bparams.curve_subdivisions = params->curve_subdivisions();
 
       delete bvh;
-      bvh = BVH::create(bparams, geometry, objects);
+      bvh = BVH::create(bparams, geometry, objects, device);
       MEM_GUARDED_CALL(progress, bvh->build, *progress);
     }
   }
@@ -1029,7 +1029,7 @@ void GeometryManager::device_update_bvh(Device *device,
 
   VLOG(1) << "Using " << bvh_layout_name(bparams.bvh_layout) << " layout.";
 
-  BVH *bvh = BVH::create(bparams, scene->geometry, scene->objects);
+  BVH *bvh = BVH::create(bparams, scene->geometry, scene->objects, device);
   bvh->build(progress, &device->stats);
 
   if (progress.get_cancel()) {