Workbench: Add Curvature overlay for better visibility of surface detail for e.g. sculpting

The approach is fairly simple, just apply an edge detection filter to the view normal and scale the brightness based on that.

The overlay is disabled at object boundaries to avoid dark lines around objects.

Generally, this implementation follows the proposal of @monio at https://blender.community/c/rightclickselect/J9bbbc.
The changes are:
- Dynamic filter radius (on high-DPI displays, a radius of two is used)
- Options to reduce the strength of both ridges and valleys
- Tweaked function for the strength reduction (the original method actually had a local maximum, resulting in a brighter line inside valleys)
- Multiplication for blending instead of overlay, which doesn't work reliably with scene-referred intensities
- Renamed to point out the distinction between it and the SSAO-based cavity overlay

Reviewers: jbakker

Reviewed By: jbakker

Subscribers: billreynish, manitwo, linko, monio

Differential Revision: https://developer.blender.org/D3617

3 files changed, 49 insertions, 2 deletions

diff --git a/source/blender/draw/engines/workbench/shaders/workbench_curvature_lib.glsl b/source/blender/draw/engines/workbench/shaders/workbench_curvature_lib.glsl
new file mode 100644
index 00000000000..6b693675f84
--- /dev/null
+++ b/source/blender/draw/engines/workbench/shaders/workbench_curvature_lib.glsl
@@ -0,0 +1,40 @@
+#ifndef CURVATURE_OFFSET
+#  define CURVATURE_OFFSET 1
+#endif
+
+float curvature_soft_clamp(float curvature, float control)
+{
+    if (curvature < 0.5 / control)
+        return curvature * (1.0 - curvature * control);
+    return 0.25 / control;
+}
+
+float calculate_curvature(usampler2D objectId, sampler2D normalBuffer, ivec2 texel, float ridge, float valley)
+{
+    uint object_up    = texelFetchOffset(objectId, texel, 0, ivec2(0,  CURVATURE_OFFSET)).r;
+    uint object_down  = texelFetchOffset(objectId, texel, 0, ivec2(0, -CURVATURE_OFFSET)).r;
+    uint object_left  = texelFetchOffset(objectId, texel, 0, ivec2(-CURVATURE_OFFSET, 0)).r;
+    uint object_right = texelFetchOffset(objectId, texel, 0, ivec2( CURVATURE_OFFSET, 0)).r;
+
+    if((object_up != object_down) || (object_right != object_left)) {
+        return 0.0;
+    }
+
+    vec2 normal_up    = texelFetchOffset(normalBuffer, texel, 0, ivec2(0,  CURVATURE_OFFSET)).rg;
+    vec2 normal_down  = texelFetchOffset(normalBuffer, texel, 0, ivec2(0, -CURVATURE_OFFSET)).rg;
+    vec2 normal_left  = texelFetchOffset(normalBuffer, texel, 0, ivec2(-CURVATURE_OFFSET, 0)).rg;
+    vec2 normal_right = texelFetchOffset(normalBuffer, texel, 0, ivec2( CURVATURE_OFFSET, 0)).rg;
+
+#ifdef WORKBENCH_ENCODE_NORMALS
+    normal_up    = normal_decode(normal_up   ).rg;
+    normal_down  = normal_decode(normal_down ).rg;
+    normal_left  = normal_decode(normal_left ).rg;
+    normal_right = normal_decode(normal_right).rg;
+#endif
+
+    float normal_diff = ((normal_up.g - normal_down.g) + (normal_right.r - normal_left.r));
+
+    if (normal_diff < 0)
+        return -2.0 * curvature_soft_clamp(-normal_diff, valley);
+    return 2.0 * curvature_soft_clamp(normal_diff, ridge);
+}
diff --git a/source/blender/draw/engines/workbench/shaders/workbench_data_lib.glsl b/source/blender/draw/engines/workbench/shaders/workbench_data_lib.glsl
index 7ba6b3a5193..3a538f4f2ac 100644
--- a/source/blender/draw/engines/workbench/shaders/workbench_data_lib.glsl
+++ b/source/blender/draw/engines/workbench/shaders/workbench_data_lib.glsl
@@ -13,5 +13,7 @@ struct WorldData {
 	int num_lights;
 	int matcap_orientation;
 	float background_alpha;
-	int pad[1];
+	float curvature_ridge;
+	float curvature_valley;
+	int pad[3];
 };
diff --git a/source/blender/draw/engines/workbench/shaders/workbench_deferred_composite_frag.glsl b/source/blender/draw/engines/workbench/shaders/workbench_deferred_composite_frag.glsl
index 508aeb1f0c1..f15e6b613a9 100644
--- a/source/blender/draw/engines/workbench/shaders/workbench_deferred_composite_frag.glsl
+++ b/source/blender/draw/engines/workbench/shaders/workbench_deferred_composite_frag.glsl
@@ -103,12 +103,17 @@ void main()
 #endif
 	vec3 shaded_color = diffuse_light * diffuse_color.rgb + specular_color;
 
-#ifdef V3D_SHADING_CAVITY
+#ifdef V3D_SHADING_SSAO
 	vec2 cavity = texelFetch(cavityBuffer, texel, 0).rg;
 	shaded_color *= 1.0 - cavity.x;
 	shaded_color *= 1.0 + cavity.y;
 #endif
 
+#ifdef V3D_SHADING_CURVATURE
+	float curvature = calculate_curvature(objectId, normalBuffer, texel, world_data.curvature_ridge, world_data.curvature_valley);
+	shaded_color *= curvature + 1.0;
+#endif
+
 #ifdef V3D_SHADING_SHADOW
 	float light_factor = -dot(normal_viewport, world_data.shadow_direction_vs.xyz);
 	/* The step function might be ok for meshes but it's