Cycles: reduce shadow terminator artifacts

Offset rays from the flat surface to match where they would be for a smooth
surface as specified by the normals. In the shading panel there is now a
Shading Offset (existing option) and Geometry Offset (new).

The Geometry Offset works as follows:
* 0: disabled
* 0.001: only terminated triangles (normal points to the light, geometry
  doesn't) are affected
* 0.1 (default): triangles at grazing angles are affected, and the effect
  fades out
* 1: all triangles are affected

Limitations:
* The artifact is still visible in some cases, it could be that some quads
  require to be treated specifically as quads.
* Inconsistent normals cause artifacts.
* If small objects cast shadows to a big low poly surface, the shadows can
  appear to be in a wrong place - because the surface moved slightly above
  the geometry. This can be noticed only at grazing angles to light.
* Approximated surfaces of two non-intersecting low-poly objects can overlap
  that causes off-the-wall shadows.

Generally, using one or a few levels of subdivision can get rid of artifacts
faster than before.

Differential Revision: https://developer.blender.org/D11065

10 files changed, 138 insertions, 22 deletions

diff --git a/intern/cycles/blender/addon/properties.py b/intern/cycles/blender/addon/properties.py
index cda1355eb2d..71e2f9fc3a5 100644
--- a/intern/cycles/blender/addon/properties.py
+++ b/intern/cycles/blender/addon/properties.py
@@ -1254,12 +1254,19 @@ class CyclesObjectSettings(bpy.types.PropertyGroup):
     )
 
     shadow_terminator_offset: FloatProperty(
-        name="Shadow Terminator Offset",
+        name="Shadow Terminator Shading Offset",
         description="Push the shadow terminator towards the light to hide artifacts on low poly geometry",
         min=0.0, max=1.0,
         default=0.0,
     )
 
+    shadow_terminator_geometry_offset: FloatProperty(
+        name="Shadow Terminator Geometry Offset",
+        description="Offset rays from the surface to reduce shadow terminator artifact on low poly geometry. Only affects triangles at grazing angles to light",
+        min=0.0, max=1.0,
+        default=0.1,
+    )
+
     is_shadow_catcher: BoolProperty(
         name="Shadow Catcher",
         description="Only render shadows on this object, for compositing renders into real footage",
diff --git a/intern/cycles/blender/addon/ui.py b/intern/cycles/blender/addon/ui.py
index 62a2fa7f036..3d990467f04 100644
--- a/intern/cycles/blender/addon/ui.py
+++ b/intern/cycles/blender/addon/ui.py
@@ -1223,20 +1223,31 @@ class CYCLES_OBJECT_PT_shading(CyclesButtonsPanel, Panel):
 
     @classmethod
     def poll(cls, context):
-        return CyclesButtonsPanel.poll(context) and (context.object)
+        if not CyclesButtonsPanel.poll(context):
+            return False
+
+        ob = context.object
+        return ob and has_geometry_visibility(ob)
+
+    def draw(self, context):
+        pass
+
+
+class CYCLES_OBJECT_PT_shading_shadow_terminator(CyclesButtonsPanel, Panel):
+    bl_label = "Shadow Terminator"
+    bl_parent_id = "CYCLES_OBJECT_PT_shading"
+    bl_context = "object"
 
     def draw(self, context):
         layout = self.layout
         layout.use_property_split = True
 
-        flow = layout.grid_flow(row_major=False, columns=0, even_columns=True, even_rows=False, align=False)
-        layout = self.layout
+        flow = layout.grid_flow(row_major=False, columns=0, even_columns=True, even_rows=False, align=True)
+
         ob = context.object
         cob = ob.cycles
-
-        if has_geometry_visibility(ob):
-            col = flow.column()
-            col.prop(cob, "shadow_terminator_offset")
+        flow.prop(cob, "shadow_terminator_geometry_offset", text="Geometry Offset")
+        flow.prop(cob, "shadow_terminator_offset", text="Shading Offset")
 
 
 class CYCLES_OBJECT_PT_visibility(CyclesButtonsPanel, Panel):
@@ -2316,6 +2327,7 @@ classes = (
     CYCLES_PT_context_material,
     CYCLES_OBJECT_PT_motion_blur,
     CYCLES_OBJECT_PT_shading,
+    CYCLES_OBJECT_PT_shading_shadow_terminator,
     CYCLES_OBJECT_PT_visibility,
     CYCLES_OBJECT_PT_visibility_ray_visibility,
     CYCLES_OBJECT_PT_visibility_culling,
diff --git a/intern/cycles/blender/blender_object.cpp b/intern/cycles/blender/blender_object.cpp
index cb84013c551..635392fb3d4 100644
--- a/intern/cycles/blender/blender_object.cpp
+++ b/intern/cycles/blender/blender_object.cpp
@@ -290,8 +290,12 @@ Object *BlenderSync::sync_object(BL::Depsgraph &b_depsgraph,
   bool is_shadow_catcher = get_boolean(cobject, "is_shadow_catcher");
   object->set_is_shadow_catcher(is_shadow_catcher);
 
-  float shadow_terminator_offset = get_float(cobject, "shadow_terminator_offset");
-  object->set_shadow_terminator_offset(shadow_terminator_offset);
+  float shadow_terminator_shading_offset = get_float(cobject, "shadow_terminator_offset");
+  object->set_shadow_terminator_shading_offset(shadow_terminator_shading_offset);
+
+  float shadow_terminator_geometry_offset = get_float(cobject,
+                                                      "shadow_terminator_geometry_offset");
+  object->set_shadow_terminator_geometry_offset(shadow_terminator_geometry_offset);
 
   /* sync the asset name for Cryptomatte */
   BL::Object parent = b_ob.parent();
diff --git a/intern/cycles/kernel/bvh/bvh_util.h b/intern/cycles/kernel/bvh/bvh_util.h
index a694e4dc259..168c2939136 100644
--- a/intern/cycles/kernel/bvh/bvh_util.h
+++ b/intern/cycles/kernel/bvh/bvh_util.h
@@ -71,6 +71,85 @@ ccl_device_inline float3 ray_offset(float3 P, float3 Ng)
 #endif
 }
 
+/* This function should be used to compute a modified ray start position for
+ * rays leaving from a surface. The algorithm slightly distorts flat surface
+ * of a triangle. Surface is lifted by amount h along normal n in the incident
+ * point. */
+
+ccl_device_inline float3 smooth_surface_offset(KernelGlobals *kg, ShaderData *sd, float3 Ng)
+{
+  float3 V[3], N[3];
+  triangle_vertices_and_normals(kg, sd->prim, V, N);
+
+  const float u = sd->u, v = sd->v;
+  const float w = 1 - u - v;
+  float3 P = V[0] * u + V[1] * v + V[2] * w; /* Local space */
+  float3 n = N[0] * u + N[1] * v + N[2] * w; /* We get away without normalization */
+  n = transform_direction(&(sd->ob_tfm), n); /* Normal x scale, world space */
+
+  /* Parabolic approximation */
+  float a = dot(N[2] - N[0], V[0] - V[2]);
+  float b = dot(N[2] - N[1], V[1] - V[2]);
+  float c = dot(N[1] - N[0], V[1] - V[0]);
+  float h = a * u * (u - 1) + (a + b + c) * u * v + b * v * (v - 1);
+
+  /* Check flipped normals */
+  if (dot(n, Ng) > 0) {
+    /* Local linear envelope */
+    float h0 = max(max(dot(V[1] - V[0], N[0]), dot(V[2] - V[0], N[0])), 0.0f);
+    float h1 = max(max(dot(V[0] - V[1], N[1]), dot(V[2] - V[1], N[1])), 0.0f);
+    float h2 = max(max(dot(V[0] - V[2], N[2]), dot(V[1] - V[2], N[2])), 0.0f);
+    h0 = max(dot(V[0] - P, N[0]) + h0, 0.0f);
+    h1 = max(dot(V[1] - P, N[1]) + h1, 0.0f);
+    h2 = max(dot(V[2] - P, N[2]) + h2, 0.0f);
+    h = max(min(min(h0, h1), h2), h * 0.5f);
+  }
+  else {
+    float h0 = max(max(dot(V[0] - V[1], N[0]), dot(V[0] - V[2], N[0])), 0.0f);
+    float h1 = max(max(dot(V[1] - V[0], N[1]), dot(V[1] - V[2], N[1])), 0.0f);
+    float h2 = max(max(dot(V[2] - V[0], N[2]), dot(V[2] - V[1], N[2])), 0.0f);
+    h0 = max(dot(P - V[0], N[0]) + h0, 0.0f);
+    h1 = max(dot(P - V[1], N[1]) + h1, 0.0f);
+    h2 = max(dot(P - V[2], N[2]) + h2, 0.0f);
+    h = min(-min(min(h0, h1), h2), h * 0.5f);
+  }
+
+  return n * h;
+}
+
+/* Ray offset to avoid shadow terminator artifact. */
+
+ccl_device_inline float3 ray_offset_shadow(KernelGlobals *kg, ShaderData *sd, float3 L)
+{
+  float NL = dot(sd->N, L);
+  bool transmit = (NL < 0.0f);
+  float3 Ng = (transmit ? -sd->Ng : sd->Ng);
+  float3 P = ray_offset(sd->P, Ng);
+
+  if ((sd->type & PRIMITIVE_ALL_TRIANGLE) && (sd->shader & SHADER_SMOOTH_NORMAL)) {
+    const float offset_cutoff =
+        kernel_tex_fetch(__objects, sd->object).shadow_terminator_geometry_offset;
+    /* Do ray offset (heavy stuff) only for close to be terminated triangles:
+     * offset_cutoff = 0.1f means that 10-20% of rays will be affected. Also
+     * make a smooth transition near the threshold. */
+    if (offset_cutoff > 0.0f) {
+      float NgL = dot(Ng, L);
+      float offset_amount = 0.0f;
+      if (NL < offset_cutoff) {
+        offset_amount = clamp(2.0f - (NgL + NL) / offset_cutoff, 0.0f, 1.0f);
+      }
+      else {
+        offset_amount = clamp(1.0f - NgL / offset_cutoff, 0.0f, 1.0f);
+      }
+      if (offset_amount > 0.0f) {
+        P += smooth_surface_offset(kg, sd, Ng) * offset_amount;
+      }
+    }
+  }
+
+  return P;
+}
+
 #if defined(__VOLUME_RECORD_ALL__) || (defined(__SHADOW_RECORD_ALL__) && defined(__KERNEL_CPU__))
 /* ToDo: Move to another file? */
 ccl_device int intersections_compare(const void *a, const void *b)
diff --git a/intern/cycles/kernel/closure/bsdf.h b/intern/cycles/kernel/closure/bsdf.h
index 6070fd983f5..6f2f2ebb202 100644
--- a/intern/cycles/kernel/closure/bsdf.h
+++ b/intern/cycles/kernel/closure/bsdf.h
@@ -462,7 +462,7 @@ ccl_device_inline int bsdf_sample(KernelGlobals *kg,
   else {
     /* Shadow terminator offset. */
     const float frequency_multiplier =
-        kernel_tex_fetch(__objects, sd->object).shadow_terminator_offset;
+        kernel_tex_fetch(__objects, sd->object).shadow_terminator_shading_offset;
     if (frequency_multiplier > 1.0f) {
       *eval *= shift_cos_in(dot(*omega_in, sc->N), frequency_multiplier);
     }
@@ -488,12 +488,9 @@ ccl_device_inline
               const float3 omega_in,
               float *pdf)
 {
-  /* For curves use the smooth normal, particularly for ribbons the geometric
-   * normal gives too much darkening otherwise. */
-  const float3 Ng = (sd->type & PRIMITIVE_ALL_CURVE) ? sd->N : sd->Ng;
   float3 eval;
 
-  if (dot(Ng, omega_in) >= 0.0f) {
+  if (dot(sd->N, omega_in) >= 0.0f) {
     switch (sc->type) {
       case CLOSURE_BSDF_DIFFUSE_ID:
       case CLOSURE_BSDF_BSSRDF_ID:
@@ -589,7 +586,7 @@ ccl_device_inline
     }
     /* Shadow terminator offset. */
     const float frequency_multiplier =
-        kernel_tex_fetch(__objects, sd->object).shadow_terminator_offset;
+        kernel_tex_fetch(__objects, sd->object).shadow_terminator_shading_offset;
     if (frequency_multiplier > 1.0f) {
       eval *= shift_cos_in(dot(omega_in, sc->N), frequency_multiplier);
     }
diff --git a/intern/cycles/kernel/geom/geom_triangle.h b/intern/cycles/kernel/geom/geom_triangle.h
index ebc5abc017a..0a9460aa166 100644
--- a/intern/cycles/kernel/geom/geom_triangle.h
+++ b/intern/cycles/kernel/geom/geom_triangle.h
@@ -75,6 +75,19 @@ ccl_device_inline void triangle_vertices(KernelGlobals *kg, int prim, float3 P[3
   P[2] = float4_to_float3(kernel_tex_fetch(__prim_tri_verts, tri_vindex.w + 2));
 }
 
+/* Triangle vertex locations and vertex normals */
+
+ccl_device_inline void triangle_vertices_and_normals(KernelGlobals *kg, int prim, float3 P[3], float3 N[3])
+{
+  const uint4 tri_vindex = kernel_tex_fetch(__tri_vindex, prim);
+  P[0] = float4_to_float3(kernel_tex_fetch(__prim_tri_verts, tri_vindex.w + 0));
+  P[1] = float4_to_float3(kernel_tex_fetch(__prim_tri_verts, tri_vindex.w + 1));
+  P[2] = float4_to_float3(kernel_tex_fetch(__prim_tri_verts, tri_vindex.w + 2));
+  N[0] = float4_to_float3(kernel_tex_fetch(__tri_vnormal, tri_vindex.x));
+  N[1] = float4_to_float3(kernel_tex_fetch(__tri_vnormal, tri_vindex.y));
+  N[2] = float4_to_float3(kernel_tex_fetch(__tri_vnormal, tri_vindex.z));
+}
+
 /* Interpolate smooth vertex normal from vertices */
 
 ccl_device_inline float3
diff --git a/intern/cycles/kernel/kernel_emission.h b/intern/cycles/kernel/kernel_emission.h
index 96ecc624067..aebf2ec8e28 100644
--- a/intern/cycles/kernel/kernel_emission.h
+++ b/intern/cycles/kernel/kernel_emission.h
@@ -176,8 +176,7 @@ ccl_device_noinline_cpu bool direct_emission(KernelGlobals *kg,
 
   if (ls->shader & SHADER_CAST_SHADOW) {
     /* setup ray */
-    bool transmit = (dot(sd->Ng, ls->D) < 0.0f);
-    ray->P = ray_offset(sd->P, (transmit) ? -sd->Ng : sd->Ng);
+    ray->P = ray_offset_shadow(kg, sd, ls->D);
 
     if (ls->t == FLT_MAX) {
       /* distant light */
diff --git a/intern/cycles/kernel/kernel_types.h b/intern/cycles/kernel/kernel_types.h
index 86b3b3b3365..85d75b36e5f 100644
--- a/intern/cycles/kernel/kernel_types.h
+++ b/intern/cycles/kernel/kernel_types.h
@@ -1479,7 +1479,8 @@ typedef struct KernelObject {
   float cryptomatte_object;
   float cryptomatte_asset;
 
-  float shadow_terminator_offset;
+  float shadow_terminator_shading_offset;
+  float shadow_terminator_geometry_offset;
   float pad1, pad2, pad3;
 } KernelObject;
 static_assert_align(KernelObject, 16);
diff --git a/intern/cycles/render/object.cpp b/intern/cycles/render/object.cpp
index f65f8bc6e90..5fe4e9ed57f 100644
--- a/intern/cycles/render/object.cpp
+++ b/intern/cycles/render/object.cpp
@@ -93,7 +93,8 @@ NODE_DEFINE(Object)
   SOCKET_POINT(dupli_generated, "Dupli Generated", zero_float3());
   SOCKET_POINT2(dupli_uv, "Dupli UV", zero_float2());
   SOCKET_TRANSFORM_ARRAY(motion, "Motion", array<Transform>());
-  SOCKET_FLOAT(shadow_terminator_offset, "Terminator Offset", 0.0f);
+  SOCKET_FLOAT(shadow_terminator_shading_offset, "Shadow Terminator Shading Offset", 0.0f);
+  SOCKET_FLOAT(shadow_terminator_geometry_offset, "Shadow Terminator Geometry Offset", 0.1f);
   SOCKET_STRING(asset_name, "Asset Name", ustring());
 
   SOCKET_BOOLEAN(is_shadow_catcher, "Shadow Catcher", false);
@@ -507,7 +508,9 @@ void ObjectManager::device_update_object_transform(UpdateObjectTransformState *s
     kobject.cryptomatte_asset = util_hash_to_float(hash_asset);
   }
 
-  kobject.shadow_terminator_offset = 1.0f / (1.0f - 0.5f * ob->shadow_terminator_offset);
+  kobject.shadow_terminator_shading_offset = 1.0f /
+                                             (1.0f - 0.5f * ob->shadow_terminator_shading_offset);
+  kobject.shadow_terminator_geometry_offset = ob->shadow_terminator_geometry_offset;
 
   /* Object flag. */
   if (ob->use_holdout) {
diff --git a/intern/cycles/render/object.h b/intern/cycles/render/object.h
index e4bc7ac3d8e..ebb7733c2aa 100644
--- a/intern/cycles/render/object.h
+++ b/intern/cycles/render/object.h
@@ -64,7 +64,8 @@ class Object : public Node {
   NODE_SOCKET_API(bool, hide_on_missing_motion)
   NODE_SOCKET_API(bool, use_holdout)
   NODE_SOCKET_API(bool, is_shadow_catcher)
-  NODE_SOCKET_API(float, shadow_terminator_offset)
+  NODE_SOCKET_API(float, shadow_terminator_shading_offset)
+  NODE_SOCKET_API(float, shadow_terminator_geometry_offset)
 
   NODE_SOCKET_API(float3, dupli_generated)
   NODE_SOCKET_API(float2, dupli_uv)