Workbench: Add Curvature overlay for better visibility of surface detail for e.g. sculpting

The approach is fairly simple, just apply an edge detection filter to the view normal and scale the brightness based on that.

The overlay is disabled at object boundaries to avoid dark lines around objects.

Generally, this implementation follows the proposal of @monio at https://blender.community/c/rightclickselect/J9bbbc.
The changes are:
- Dynamic filter radius (on high-DPI displays, a radius of two is used)
- Options to reduce the strength of both ridges and valleys
- Tweaked function for the strength reduction (the original method actually had a local maximum, resulting in a brighter line inside valleys)
- Multiplication for blending instead of overlay, which doesn't work reliably with scene-referred intensities
- Renamed to point out the distinction between it and the SSAO-based cavity overlay

Reviewers: jbakker

Reviewed By: jbakker

Subscribers: billreynish, manitwo, linko, monio

Differential Revision: https://developer.blender.org/D3617

1 files changed, 40 insertions, 0 deletions

diff --git a/source/blender/draw/engines/workbench/shaders/workbench_curvature_lib.glsl b/source/blender/draw/engines/workbench/shaders/workbench_curvature_lib.glsl
new file mode 100644
index 00000000000..6b693675f84
--- /dev/null
+++ b/source/blender/draw/engines/workbench/shaders/workbench_curvature_lib.glsl
@@ -0,0 +1,40 @@
+#ifndef CURVATURE_OFFSET
+#  define CURVATURE_OFFSET 1
+#endif
+
+float curvature_soft_clamp(float curvature, float control)
+{
+    if (curvature < 0.5 / control)
+        return curvature * (1.0 - curvature * control);
+    return 0.25 / control;
+}
+
+float calculate_curvature(usampler2D objectId, sampler2D normalBuffer, ivec2 texel, float ridge, float valley)
+{
+    uint object_up    = texelFetchOffset(objectId, texel, 0, ivec2(0,  CURVATURE_OFFSET)).r;
+    uint object_down  = texelFetchOffset(objectId, texel, 0, ivec2(0, -CURVATURE_OFFSET)).r;
+    uint object_left  = texelFetchOffset(objectId, texel, 0, ivec2(-CURVATURE_OFFSET, 0)).r;
+    uint object_right = texelFetchOffset(objectId, texel, 0, ivec2( CURVATURE_OFFSET, 0)).r;
+
+    if((object_up != object_down) || (object_right != object_left)) {
+        return 0.0;
+    }
+
+    vec2 normal_up    = texelFetchOffset(normalBuffer, texel, 0, ivec2(0,  CURVATURE_OFFSET)).rg;
+    vec2 normal_down  = texelFetchOffset(normalBuffer, texel, 0, ivec2(0, -CURVATURE_OFFSET)).rg;
+    vec2 normal_left  = texelFetchOffset(normalBuffer, texel, 0, ivec2(-CURVATURE_OFFSET, 0)).rg;
+    vec2 normal_right = texelFetchOffset(normalBuffer, texel, 0, ivec2( CURVATURE_OFFSET, 0)).rg;
+
+#ifdef WORKBENCH_ENCODE_NORMALS
+    normal_up    = normal_decode(normal_up   ).rg;
+    normal_down  = normal_decode(normal_down ).rg;
+    normal_left  = normal_decode(normal_left ).rg;
+    normal_right = normal_decode(normal_right).rg;
+#endif
+
+    float normal_diff = ((normal_up.g - normal_down.g) + (normal_right.r - normal_left.r));
+
+    if (normal_diff < 0)
+        return -2.0 * curvature_soft_clamp(-normal_diff, valley);
+    return 2.0 * curvature_soft_clamp(normal_diff, ridge);
+}