Cycles: Increase instance limit for OptiX acceleration structure building

For a while now OptiX had support for 28-bits of instance IDs, instead of the initial 24-bits (see also
value reported by OPTIX_DEVICE_PROPERTY_LIMIT_MAX_INSTANCE_ID). This change makes use of
that and also adds an error reported when the number of instances an OptiX acceleration structure is
created with goes beyond the limit, to make this clear instead of just rendering an image with artifacts.

Manifest Tasks: T81431

2 files changed, 16 insertions, 2 deletions

diff --git a/intern/cycles/device/device_optix.cpp b/intern/cycles/device/device_optix.cpp
index 673fc1752bb..de98e3f3594 100644
--- a/intern/cycles/device/device_optix.cpp
+++ b/intern/cycles/device/device_optix.cpp
@@ -1519,6 +1519,16 @@ class OptiXDevice : public CUDADevice {
       bvh_optix->traversable_handle = 0;
       bvh_optix->motion_transform_data.free();
 
+#  if OPTIX_ABI_VERSION < 23
+      if (bvh->objects.size() > 0x7FFFFF) {
+#  else
+      if (bvh->objects.size() > 0x7FFFFFF) {
+#  endif
+        progress.set_error(
+            "Failed to build OptiX acceleration structure because there are too many instances");
+        return;
+      }
+
       // Fill instance descriptions
 #  if OPTIX_ABI_VERSION < 41
       device_vector<OptixAabb> aabbs(this, "optix tlas aabbs", MEM_READ_ONLY);
@@ -1681,7 +1691,11 @@ class OptiXDevice : public CUDADevice {
             instance.flags = OPTIX_INSTANCE_FLAG_DISABLE_TRANSFORM;
             // Non-instanced objects read ID from prim_object, so
             // distinguish them from instanced objects with high bit set
+#  if OPTIX_ABI_VERSION < 23
             instance.instanceId |= 0x800000;
+#  else
+            instance.instanceId |= 0x8000000;
+#  endif
           }
         }
       }
diff --git a/intern/cycles/kernel/kernels/optix/kernel_optix.cu b/intern/cycles/kernel/kernels/optix/kernel_optix.cu
index 8ccd2555091..0c2c84fdbdf 100644
--- a/intern/cycles/kernel/kernels/optix/kernel_optix.cu
+++ b/intern/cycles/kernel/kernels/optix/kernel_optix.cu
@@ -47,9 +47,9 @@ template<bool always = false> ccl_device_forceinline uint get_object_id()
   // Choose between always returning object ID or only for instances
   if (always)
     // Can just remove the high bit since instance always contains object ID
-    return object & 0x7FFFFF;
+    return object & 0x7FFFFFF;  // OPTIX_ABI_VERSION >= 23 ? 0x7FFFFFF : 0x7FFFFF
   // Set to OBJECT_NONE if this is not an instanced object
-  else if (object & 0x800000)
+  else if (object & 0x8000000)  // OPTIX_ABI_VERSION >= 23 ? 0x8000000 : 0x800000
     object = OBJECT_NONE;
   return object;
 }