Fix T87793: Cycles OptiX crash hiding objects in viewport render

7 files changed, 87 insertions, 4 deletions

diff --git a/intern/cycles/bvh/bvh_optix.cpp b/intern/cycles/bvh/bvh_optix.cpp
index d630e8965dc..cd266f72f89 100644
--- a/intern/cycles/bvh/bvh_optix.cpp
+++ b/intern/cycles/bvh/bvh_optix.cpp
@@ -17,6 +17,8 @@
 
 #ifdef WITH_OPTIX
 
+#  include "device/device.h"
+
 #  include "bvh/bvh_optix.h"
 
 CCL_NAMESPACE_BEGIN
@@ -26,6 +28,7 @@ BVHOptiX::BVHOptiX(const BVHParams &params_,
                    const vector<Object *> &objects_,
                    Device *device)
     : BVH(params_, geometry_, objects_),
+      device(device),
       traversable_handle(0),
       as_data(device, params_.top_level ? "optix tlas" : "optix blas", false),
       motion_transform_data(device, "optix motion transform", false)
@@ -34,7 +37,9 @@ BVHOptiX::BVHOptiX(const BVHParams &params_,
 
 BVHOptiX::~BVHOptiX()
 {
-  // Acceleration structure memory is freed via the 'as_data' destructor
+  // Acceleration structure memory is delayed freed on device, since deleting the
+  // BVH may happen while still being used for rendering.
+  device->release_optix_bvh(this);
 }
 
 CCL_NAMESPACE_END
diff --git a/intern/cycles/bvh/bvh_optix.h b/intern/cycles/bvh/bvh_optix.h
index aa514beae0d..ba5d90471d1 100644
--- a/intern/cycles/bvh/bvh_optix.h
+++ b/intern/cycles/bvh/bvh_optix.h
@@ -28,6 +28,7 @@ CCL_NAMESPACE_BEGIN
 
 class BVHOptiX : public BVH {
  public:
+  Device *device;
   uint64_t traversable_handle;
   device_only_memory<char> as_data;
   device_only_memory<char> motion_transform_data;
diff --git a/intern/cycles/device/device.h b/intern/cycles/device/device.h
index 3c697b350a2..bdf18d09b31 100644
--- a/intern/cycles/device/device.h
+++ b/intern/cycles/device/device.h
@@ -426,6 +426,9 @@ class Device {
   /* acceleration structure building */
   virtual void build_bvh(BVH *bvh, Progress &progress, bool refit);
 
+  /* OptiX specific destructor. */
+  virtual void release_optix_bvh(BVH *){};
+
 #ifdef WITH_NETWORK
   /* networking */
   void server_run();
diff --git a/intern/cycles/device/device_memory.cpp b/intern/cycles/device/device_memory.cpp
index 9eee86b0814..80a05fc32fe 100644
--- a/intern/cycles/device/device_memory.cpp
+++ b/intern/cycles/device/device_memory.cpp
@@ -35,10 +35,54 @@ device_memory::device_memory(Device *device, const char *name, MemoryType type)
       device_pointer(0),
       host_pointer(0),
       shared_pointer(0),
-      shared_counter(0)
+      shared_counter(0),
+      original_device_ptr(0),
+      original_device_size(0),
+      original_device(0),
+      need_realloc_(false),
+      modified(false)
 {
 }
 
+device_memory::device_memory(device_memory &&other) noexcept
+    : data_type(other.data_type),
+      data_elements(other.data_elements),
+      data_size(other.data_size),
+      device_size(other.device_size),
+      data_width(other.data_width),
+      data_height(other.data_height),
+      data_depth(other.data_depth),
+      type(other.type),
+      name(other.name),
+      device(other.device),
+      device_pointer(other.device_pointer),
+      host_pointer(other.host_pointer),
+      shared_pointer(other.shared_pointer),
+      shared_counter(other.shared_counter),
+      original_device_ptr(other.original_device_ptr),
+      original_device_size(other.original_device_size),
+      original_device(other.original_device),
+      need_realloc_(other.need_realloc_),
+      modified(other.modified)
+{
+  other.data_elements = 0;
+  other.data_size = 0;
+  other.device_size = 0;
+  other.data_width = 0;
+  other.data_height = 0;
+  other.data_depth = 0;
+  other.device = 0;
+  other.device_pointer = 0;
+  other.host_pointer = 0;
+  other.shared_pointer = 0;
+  other.shared_counter = 0;
+  other.original_device_ptr = 0;
+  other.original_device_size = 0;
+  other.original_device = 0;
+  other.need_realloc_ = false;
+  other.modified = false;
+}
+
 device_memory::~device_memory()
 {
   assert(shared_pointer == 0);
diff --git a/intern/cycles/device/device_memory.h b/intern/cycles/device/device_memory.h
index 97459b9ae6a..80f4d7b0468 100644
--- a/intern/cycles/device/device_memory.h
+++ b/intern/cycles/device/device_memory.h
@@ -238,6 +238,7 @@ class device_memory {
 
   /* Only create through subclasses. */
   device_memory(Device *device, const char *name, MemoryType type);
+  device_memory(device_memory &&other) noexcept;
 
   /* No copying allowed. */
   device_memory(const device_memory &) = delete;
@@ -277,6 +278,10 @@ template<typename T> class device_only_memory : public device_memory {
     data_elements = max(device_type_traits<T>::num_elements, 1);
   }
 
+  device_only_memory(device_only_memory &&other) noexcept : device_memory(std::move(other))
+  {
+  }
+
   virtual ~device_only_memory()
   {
     free();
diff --git a/intern/cycles/device/device_optix.cpp b/intern/cycles/device/device_optix.cpp
index fcf8fab9cc4..bb6027254f9 100644
--- a/intern/cycles/device/device_optix.cpp
+++ b/intern/cycles/device/device_optix.cpp
@@ -193,6 +193,9 @@ class OptiXDevice : public CUDADevice {
   device_only_memory<unsigned char> denoiser_state;
   int denoiser_input_passes = 0;
 
+  vector<device_only_memory<char>> delayed_free_bvh_memory;
+  thread_mutex delayed_free_bvh_mutex;
+
  public:
   OptiXDevice(DeviceInfo &info_, Stats &stats_, Profiler &profiler_, bool background_)
       : CUDADevice(info_, stats_, profiler_, background_),
@@ -258,6 +261,8 @@ class OptiXDevice : public CUDADevice {
     // Make CUDA context current
     const CUDAContextScope scope(cuContext);
 
+    free_bvh_memory_delayed();
+
     sbt_data.free();
     texture_info.free();
     launch_params.free();
@@ -1265,6 +1270,8 @@ class OptiXDevice : public CUDADevice {
       return;
     }
 
+    free_bvh_memory_delayed();
+
     BVHOptiX *const bvh_optix = static_cast<BVHOptiX *>(bvh);
 
     progress.set_substatus("Building OptiX acceleration structure");
@@ -1735,6 +1742,24 @@ class OptiXDevice : public CUDADevice {
     }
   }
 
+  void release_optix_bvh(BVH *bvh) override
+  {
+    thread_scoped_lock lock(delayed_free_bvh_mutex);
+    /* Do delayed free of BVH memory, since geometry holding BVH might be deleted
+     * while GPU is still rendering. */
+    BVHOptiX *const bvh_optix = static_cast<BVHOptiX *>(bvh);
+
+    delayed_free_bvh_memory.emplace_back(std::move(bvh_optix->as_data));
+    delayed_free_bvh_memory.emplace_back(std::move(bvh_optix->motion_transform_data));
+    bvh_optix->traversable_handle = 0;
+  }
+
+  void free_bvh_memory_delayed()
+  {
+    thread_scoped_lock lock(delayed_free_bvh_mutex);
+    delayed_free_bvh_memory.free_memory();
+  }
+
   void const_copy_to(const char *name, void *host, size_t size) override
   {
     // Set constant memory for CUDA module
diff --git a/intern/cycles/util/util_vector.h b/intern/cycles/util/util_vector.h
index 04fb33368d9..87cd4de8438 100644
--- a/intern/cycles/util/util_vector.h
+++ b/intern/cycles/util/util_vector.h
@@ -43,8 +43,8 @@ class vector : public std::vector<value_type, allocator_type> {
   /* Try as hard as possible to use zero memory. */
   void free_memory()
   {
-    BaseClass::resize(0);
-    BaseClass::shrink_to_fit();
+    vector<value_type, allocator_type> empty;
+    BaseClass::swap(empty);
   }
 
   /* Some external API might demand working with std::vector. */