Workbench: Add Curvature overlay for better visibility of surface detail for e.g. sculpting

The approach is fairly simple, just apply an edge detection filter to the view normal and scale the brightness based on that.

The overlay is disabled at object boundaries to avoid dark lines around objects.

Generally, this implementation follows the proposal of @monio at https://blender.community/c/rightclickselect/J9bbbc.
The changes are:
- Dynamic filter radius (on high-DPI displays, a radius of two is used)
- Options to reduce the strength of both ridges and valleys
- Tweaked function for the strength reduction (the original method actually had a local maximum, resulting in a brighter line inside valleys)
- Multiplication for blending instead of overlay, which doesn't work reliably with scene-referred intensities
- Renamed to point out the distinction between it and the SSAO-based cavity overlay

Reviewers: jbakker

Reviewed By: jbakker

Subscribers: billreynish, manitwo, linko, monio

Differential Revision: https://developer.blender.org/D3617

9 files changed, 88 insertions, 15 deletions

diff --git a/source/blender/draw/CMakeLists.txt b/source/blender/draw/CMakeLists.txt
index 2097fb66c8b..05865ba5636 100644
--- a/source/blender/draw/CMakeLists.txt
+++ b/source/blender/draw/CMakeLists.txt
@@ -231,6 +231,7 @@ data_to_c_simple(engines/workbench/shaders/workbench_forward_depth_frag.glsl SRC
 data_to_c_simple(engines/workbench/shaders/workbench_forward_transparent_accum_frag.glsl SRC)
 data_to_c_simple(engines/workbench/shaders/workbench_ghost_resolve_frag.glsl SRC)
 data_to_c_simple(engines/workbench/shaders/workbench_object_outline_lib.glsl SRC)
+data_to_c_simple(engines/workbench/shaders/workbench_curvature_lib.glsl SRC)
 data_to_c_simple(engines/workbench/shaders/workbench_prepass_vert.glsl SRC)
 data_to_c_simple(engines/workbench/shaders/workbench_prepass_frag.glsl SRC)
 data_to_c_simple(engines/workbench/shaders/workbench_shadow_vert.glsl SRC)
diff --git a/source/blender/draw/engines/workbench/shaders/workbench_curvature_lib.glsl b/source/blender/draw/engines/workbench/shaders/workbench_curvature_lib.glsl
new file mode 100644
index 00000000000..6b693675f84
--- /dev/null
+++ b/source/blender/draw/engines/workbench/shaders/workbench_curvature_lib.glsl
@@ -0,0 +1,40 @@
+#ifndef CURVATURE_OFFSET
+#  define CURVATURE_OFFSET 1
+#endif
+
+float curvature_soft_clamp(float curvature, float control)
+{
+    if (curvature < 0.5 / control)
+        return curvature * (1.0 - curvature * control);
+    return 0.25 / control;
+}
+
+float calculate_curvature(usampler2D objectId, sampler2D normalBuffer, ivec2 texel, float ridge, float valley)
+{
+    uint object_up    = texelFetchOffset(objectId, texel, 0, ivec2(0,  CURVATURE_OFFSET)).r;
+    uint object_down  = texelFetchOffset(objectId, texel, 0, ivec2(0, -CURVATURE_OFFSET)).r;
+    uint object_left  = texelFetchOffset(objectId, texel, 0, ivec2(-CURVATURE_OFFSET, 0)).r;
+    uint object_right = texelFetchOffset(objectId, texel, 0, ivec2( CURVATURE_OFFSET, 0)).r;
+
+    if((object_up != object_down) || (object_right != object_left)) {
+        return 0.0;
+    }
+
+    vec2 normal_up    = texelFetchOffset(normalBuffer, texel, 0, ivec2(0,  CURVATURE_OFFSET)).rg;
+    vec2 normal_down  = texelFetchOffset(normalBuffer, texel, 0, ivec2(0, -CURVATURE_OFFSET)).rg;
+    vec2 normal_left  = texelFetchOffset(normalBuffer, texel, 0, ivec2(-CURVATURE_OFFSET, 0)).rg;
+    vec2 normal_right = texelFetchOffset(normalBuffer, texel, 0, ivec2( CURVATURE_OFFSET, 0)).rg;
+
+#ifdef WORKBENCH_ENCODE_NORMALS
+    normal_up    = normal_decode(normal_up   ).rg;
+    normal_down  = normal_decode(normal_down ).rg;
+    normal_left  = normal_decode(normal_left ).rg;
+    normal_right = normal_decode(normal_right).rg;
+#endif
+
+    float normal_diff = ((normal_up.g - normal_down.g) + (normal_right.r - normal_left.r));
+
+    if (normal_diff < 0)
+        return -2.0 * curvature_soft_clamp(-normal_diff, valley);
+    return 2.0 * curvature_soft_clamp(normal_diff, ridge);
+}
diff --git a/source/blender/draw/engines/workbench/shaders/workbench_data_lib.glsl b/source/blender/draw/engines/workbench/shaders/workbench_data_lib.glsl
index 7ba6b3a5193..3a538f4f2ac 100644
--- a/source/blender/draw/engines/workbench/shaders/workbench_data_lib.glsl
+++ b/source/blender/draw/engines/workbench/shaders/workbench_data_lib.glsl
@@ -13,5 +13,7 @@ struct WorldData {
 	int num_lights;
 	int matcap_orientation;
 	float background_alpha;
-	int pad[1];
+	float curvature_ridge;
+	float curvature_valley;
+	int pad[3];
 };
diff --git a/source/blender/draw/engines/workbench/shaders/workbench_deferred_composite_frag.glsl b/source/blender/draw/engines/workbench/shaders/workbench_deferred_composite_frag.glsl
index 508aeb1f0c1..f15e6b613a9 100644
--- a/source/blender/draw/engines/workbench/shaders/workbench_deferred_composite_frag.glsl
+++ b/source/blender/draw/engines/workbench/shaders/workbench_deferred_composite_frag.glsl
@@ -103,12 +103,17 @@ void main()
 #endif
 	vec3 shaded_color = diffuse_light * diffuse_color.rgb + specular_color;
 
-#ifdef V3D_SHADING_CAVITY
+#ifdef V3D_SHADING_SSAO
 	vec2 cavity = texelFetch(cavityBuffer, texel, 0).rg;
 	shaded_color *= 1.0 - cavity.x;
 	shaded_color *= 1.0 + cavity.y;
 #endif
 
+#ifdef V3D_SHADING_CURVATURE
+	float curvature = calculate_curvature(objectId, normalBuffer, texel, world_data.curvature_ridge, world_data.curvature_valley);
+	shaded_color *= curvature + 1.0;
+#endif
+
 #ifdef V3D_SHADING_SHADOW
 	float light_factor = -dot(normal_viewport, world_data.shadow_direction_vs.xyz);
 	/* The step function might be ok for meshes but it's
diff --git a/source/blender/draw/engines/workbench/workbench_data.c b/source/blender/draw/engines/workbench/workbench_data.c
index aa99883bb6b..94b721dd935 100644
--- a/source/blender/draw/engines/workbench/workbench_data.c
+++ b/source/blender/draw/engines/workbench/workbench_data.c
@@ -84,6 +84,9 @@ void workbench_private_data_init(WORKBENCH_PrivateData *wpd)
 	copy_v3_v3(wd->object_outline_color, wpd->shading.object_outline_color);
 	wd->object_outline_color[3] = 1.0f;
 
+	wd->curvature_ridge = 0.5f / max_ff(wpd->shading.curvature_ridge_factor*wpd->shading.curvature_ridge_factor, 1e-4f);
+	wd->curvature_valley = 0.7f / max_ff(wpd->shading.curvature_valley_factor*wpd->shading.curvature_valley_factor, 1e-4f);
+
 	wpd->world_ubo = DRW_uniformbuffer_create(sizeof(WORKBENCH_UBO_World), &wpd->world_data);
 
 	/* Cavity settings */
diff --git a/source/blender/draw/engines/workbench/workbench_deferred.c b/source/blender/draw/engines/workbench/workbench_deferred.c
index 66dcfdd66d0..ce14ee66ab2 100644
--- a/source/blender/draw/engines/workbench/workbench_deferred.c
+++ b/source/blender/draw/engines/workbench/workbench_deferred.c
@@ -104,6 +104,7 @@ extern char datatoc_workbench_cavity_lib_glsl[];
 extern char datatoc_workbench_common_lib_glsl[];
 extern char datatoc_workbench_data_lib_glsl[];
 extern char datatoc_workbench_object_outline_lib_glsl[];
+extern char datatoc_workbench_curvature_lib_glsl[];
 extern char datatoc_workbench_world_light_lib_glsl[];
 
 extern char datatoc_gpu_shader_depth_only_frag_glsl[];
@@ -124,6 +125,9 @@ static char *workbench_build_composite_frag(WORKBENCH_PrivateData *wpd)
 	if (wpd->shading.flag & V3D_SHADING_OBJECT_OUTLINE) {
 		BLI_dynstr_append(ds, datatoc_workbench_object_outline_lib_glsl);
 	}
+	if (CURVATURE_ENABLED(wpd)) {
+		BLI_dynstr_append(ds, datatoc_workbench_curvature_lib_glsl);
+	}
 
 	BLI_dynstr_append(ds, datatoc_workbench_deferred_composite_frag_glsl);
 
@@ -499,7 +503,7 @@ static void workbench_composite_uniforms(WORKBENCH_PrivateData *wpd, DRWShadingG
 	if (NORMAL_VIEWPORT_COMP_PASS_ENABLED(wpd)) {
 		DRW_shgroup_uniform_texture_ref(grp, "normalBuffer", &e_data.normal_buffer_tx);
 	}
-	if (CAVITY_ENABLED(wpd)) {
+	if (SSAO_ENABLED(wpd)) {
 		DRW_shgroup_uniform_texture_ref(grp, "cavityBuffer", &e_data.cavity_buffer_tx);
 	}
 	if (SPECULAR_HIGHLIGHT_ENABLED(wpd) || MATCAP_ENABLED(wpd)) {
@@ -903,7 +907,7 @@ void workbench_deferred_draw_scene(WORKBENCH_Data *vedata)
 		DRW_draw_pass(psl->ghost_resolve_pass);
 	}
 
-	if (CAVITY_ENABLED(wpd)) {
+	if (SSAO_ENABLED(wpd)) {
 		GPU_framebuffer_bind(fbl->cavity_fb);
 		DRW_draw_pass(psl->cavity_pass);
 	}
diff --git a/source/blender/draw/engines/workbench/workbench_forward.c b/source/blender/draw/engines/workbench/workbench_forward.c
index aa75e1746c7..7a03984b196 100644
--- a/source/blender/draw/engines/workbench/workbench_forward.c
+++ b/source/blender/draw/engines/workbench/workbench_forward.c
@@ -75,6 +75,7 @@ extern char datatoc_workbench_data_lib_glsl[];
 extern char datatoc_workbench_background_lib_glsl[];
 extern char datatoc_workbench_checkerboard_depth_frag_glsl[];
 extern char datatoc_workbench_object_outline_lib_glsl[];
+extern char datatoc_workbench_curvature_lib_glsl[];
 extern char datatoc_workbench_prepass_vert_glsl[];
 extern char datatoc_workbench_common_lib_glsl[];
 extern char datatoc_workbench_world_light_lib_glsl[];
@@ -123,6 +124,7 @@ static char *workbench_build_forward_composite_frag(void)
 	BLI_dynstr_append(ds, datatoc_workbench_common_lib_glsl);
 	BLI_dynstr_append(ds, datatoc_workbench_background_lib_glsl);
 	BLI_dynstr_append(ds, datatoc_workbench_object_outline_lib_glsl);
+	BLI_dynstr_append(ds, datatoc_workbench_curvature_lib_glsl);
 	BLI_dynstr_append(ds, datatoc_workbench_forward_composite_frag_glsl);
 
 	str = BLI_dynstr_get_cstring(ds);
diff --git a/source/blender/draw/engines/workbench/workbench_materials.c b/source/blender/draw/engines/workbench/workbench_materials.c
index 2faa9e288f8..6f43e1f8707 100644
--- a/source/blender/draw/engines/workbench/workbench_materials.c
+++ b/source/blender/draw/engines/workbench/workbench_materials.c
@@ -55,8 +55,17 @@ char *workbench_material_build_defines(WORKBENCH_PrivateData *wpd, bool use_text
 	if (wpd->shading.flag & V3D_SHADING_SHADOW) {
 		BLI_dynstr_appendf(ds, "#define V3D_SHADING_SHADOW\n");
 	}
-	if (CAVITY_ENABLED(wpd)) {
-		BLI_dynstr_appendf(ds, "#define V3D_SHADING_CAVITY\n");
+	if (SSAO_ENABLED(wpd)) {
+		BLI_dynstr_appendf(ds, "#define V3D_SHADING_SSAO\n");
+	}
+	if (CURVATURE_ENABLED(wpd)) {
+		BLI_dynstr_appendf(ds, "#define V3D_SHADING_CURVATURE\n");
+		if (U.pixelsize > 1.5f) {
+			BLI_dynstr_appendf(ds, "#define CURVATURE_OFFSET 2\n");
+		}
+		else {
+			BLI_dynstr_appendf(ds, "#define CURVATURE_OFFSET 1\n");
+		}
 	}
 	if (SPECULAR_HIGHLIGHT_ENABLED(wpd)) {
 		BLI_dynstr_appendf(ds, "#define V3D_SHADING_SPECULAR_HIGHLIGHT\n");
@@ -139,13 +148,16 @@ int workbench_material_get_shader_index(WORKBENCH_PrivateData *wpd, bool use_tex
 	/* 1 bit V3D_SHADING_SPECULAR_HIGHLIGHT */
 	SET_FLAG_FROM_TEST(index, wpd->shading.flag & V3D_SHADING_SPECULAR_HIGHLIGHT, 1 << 3);
 	SET_FLAG_FROM_TEST(index, wpd->shading.flag & V3D_SHADING_SHADOW, 1 << 4);
-	SET_FLAG_FROM_TEST(index, wpd->shading.flag & V3D_SHADING_CAVITY, 1 << 5);
+	SET_FLAG_FROM_TEST(index, SSAO_ENABLED(wpd), 1 << 5);
 	SET_FLAG_FROM_TEST(index, wpd->shading.flag & V3D_SHADING_OBJECT_OUTLINE, 1 << 6);
+	bool uses_curvature = CURVATURE_ENABLED(wpd);
+	SET_FLAG_FROM_TEST(index, uses_curvature, 1 << 7);
+	SET_FLAG_FROM_TEST(index, uses_curvature && (U.pixelsize > 1.5f), 1 << 8);
 	/* 2 bits STUDIOLIGHT_ORIENTATION */
-	SET_FLAG_FROM_TEST(index, wpd->studio_light->flag & STUDIOLIGHT_ORIENTATION_WORLD, 1 << 7);
-	SET_FLAG_FROM_TEST(index, wpd->studio_light->flag & STUDIOLIGHT_ORIENTATION_VIEWNORMAL, 1 << 8);
+	SET_FLAG_FROM_TEST(index, wpd->studio_light->flag & STUDIOLIGHT_ORIENTATION_WORLD, 1 << 9);
+	SET_FLAG_FROM_TEST(index, wpd->studio_light->flag & STUDIOLIGHT_ORIENTATION_VIEWNORMAL, 1 << 10);
 	/* 1 bit for hair */
-	SET_FLAG_FROM_TEST(index, is_hair, 1 << 9);
+	SET_FLAG_FROM_TEST(index, is_hair, 1 << 11);
 	return index;
 }
 
diff --git a/source/blender/draw/engines/workbench/workbench_private.h b/source/blender/draw/engines/workbench/workbench_private.h
index 93b3bddfb72..08b6e4a218f 100644
--- a/source/blender/draw/engines/workbench/workbench_private.h
+++ b/source/blender/draw/engines/workbench/workbench_private.h
@@ -49,7 +49,8 @@
 #define STUDIOLIGHT_ORIENTATION_WORLD_ENABLED(wpd) (STUDIOLIGHT_ENABLED(wpd) && (wpd->studio_light->flag & STUDIOLIGHT_ORIENTATION_WORLD))
 #define STUDIOLIGHT_ORIENTATION_CAMERA_ENABLED(wpd) (STUDIOLIGHT_ENABLED(wpd) && (wpd->studio_light->flag & STUDIOLIGHT_ORIENTATION_CAMERA))
 #define STUDIOLIGHT_ORIENTATION_VIEWNORMAL_ENABLED(wpd) (MATCAP_ENABLED(wpd) && (wpd->studio_light->flag & STUDIOLIGHT_ORIENTATION_VIEWNORMAL))
-#define CAVITY_ENABLED(wpd) (wpd->shading.flag & V3D_SHADING_CAVITY)
+#define SSAO_ENABLED(wpd) ((wpd->shading.flag & V3D_SHADING_CAVITY) && ((wpd->shading.cavity_type == V3D_SHADING_CAVITY_SSAO) || (wpd->shading.cavity_type == V3D_SHADING_CAVITY_BOTH)))
+#define CURVATURE_ENABLED(wpd) ((wpd->shading.flag & V3D_SHADING_CAVITY) && ((wpd->shading.cavity_type == V3D_SHADING_CAVITY_CURVATURE) || (wpd->shading.cavity_type == V3D_SHADING_CAVITY_BOTH)))
 #define SHADOW_ENABLED(wpd) (wpd->shading.flag & V3D_SHADING_SHADOW)
 #define GHOST_ENABLED(psl) (!DRW_pass_is_empty(psl->ghost_prepass_pass) || !DRW_pass_is_empty(psl->ghost_prepass_hair_pass))
 
@@ -59,9 +60,10 @@
                              ((IS_NAVIGATING(wpd) || wpd->is_playback) && (wpd->user_preferences->gpu_viewport_quality >= GPU_VIEWPORT_QUALITY_TAA8))))
 #define TAA_ENABLED(wpd) (DRW_state_is_image_render() || (wpd->user_preferences->gpu_viewport_quality >= GPU_VIEWPORT_QUALITY_TAA8 && !IS_NAVIGATING(wpd) && !wpd->is_playback))
 #define SPECULAR_HIGHLIGHT_ENABLED(wpd) ((wpd->shading.flag & V3D_SHADING_SPECULAR_HIGHLIGHT) && (!STUDIOLIGHT_ORIENTATION_VIEWNORMAL_ENABLED(wpd)))
-#define OBJECT_ID_PASS_ENABLED(wpd) (wpd->shading.flag & V3D_SHADING_OBJECT_OUTLINE)
-#define NORMAL_VIEWPORT_COMP_PASS_ENABLED(wpd) (MATCAP_ENABLED(wpd) || STUDIOLIGHT_ENABLED(wpd) || SHADOW_ENABLED(wpd) || SPECULAR_HIGHLIGHT_ENABLED(wpd))
-#define NORMAL_VIEWPORT_PASS_ENABLED(wpd) (NORMAL_VIEWPORT_COMP_PASS_ENABLED(wpd) || CAVITY_ENABLED(wpd))
+#define OBJECT_OUTLINE_ENABLED(wpd) (wpd->shading.flag & V3D_SHADING_OBJECT_OUTLINE)
+#define OBJECT_ID_PASS_ENABLED(wpd) (OBJECT_OUTLINE_ENABLED(wpd) || CURVATURE_ENABLED(wpd))
+#define NORMAL_VIEWPORT_COMP_PASS_ENABLED(wpd) (MATCAP_ENABLED(wpd) || STUDIOLIGHT_ENABLED(wpd) || SHADOW_ENABLED(wpd) || SPECULAR_HIGHLIGHT_ENABLED(wpd) || CURVATURE_ENABLED(wpd))
+#define NORMAL_VIEWPORT_PASS_ENABLED(wpd) (NORMAL_VIEWPORT_COMP_PASS_ENABLED(wpd) || SSAO_ENABLED(wpd))
 #define NORMAL_ENCODING_ENABLED() (true)
 
 
@@ -150,7 +152,9 @@ typedef struct WORKBENCH_UBO_World {
 	int num_lights;
 	int matcap_orientation;
 	float background_alpha;
-	int pad[1];
+	float curvature_ridge;
+	float curvature_valley;
+	int pad[3];
 } WORKBENCH_UBO_World;
 BLI_STATIC_ASSERT_ALIGN(WORKBENCH_UBO_World, 16)