Object Mode: Rework outline drawing.

This changes quite a few things.
- Outline is now per object.
- No more outline at object intersection (fix hairs problem).
- Simplify the code quite a bit.

We use a R16UI buffer to save one id per object outline. We convert this id
to color when detecting the outline.

Added textureGatherOffsets to the code but could not test on current hardware
so leaving it commented for now.

1 files changed, 47 insertions, 57 deletions

diff --git a/source/blender/draw/modes/shaders/object_outline_detect_frag.glsl b/source/blender/draw/modes/shaders/object_outline_detect_frag.glsl
index dc0ea938436..6b0f66d7c94 100644
--- a/source/blender/draw/modes/shaders/object_outline_detect_frag.glsl
+++ b/source/blender/draw/modes/shaders/object_outline_detect_frag.glsl
@@ -3,77 +3,67 @@ in vec4 uvcoordsvar;
 
 out vec4 FragColor;
 
-uniform sampler2D outlineColor;
+uniform usampler2D outlineId;
 uniform sampler2D outlineDepth;
 uniform sampler2D sceneDepth;
 
+uniform int idOffsets[5];
+
 uniform float alphaOcclu;
 uniform vec2 viewportSize;
 
-void search_outline(ivec2 uv, vec4 ref_col, inout bool ref_occlu, inout bool outline)
+vec4 convert_id_to_color(int id)
 {
-	if (!outline) {
-		vec4 color = texelFetch(outlineColor, uv, 0).rgba;
-		if (color != ref_col) {
-			outline = true;
-		}
-		else {
-			float depth = texelFetch(outlineDepth, uv, 0).r;
-			float scene_depth = texelFetch(sceneDepth, uv, 0).r;
-			bool occlu = (depth > scene_depth);
-
-			if (occlu != ref_occlu && !ref_occlu) {
-				outline = true;
-			}
-		}
+	if (id == 0) {
+		return vec4(0.0);
+	}
+	if (id < idOffsets[1]) {
+		return colorActive;
+	}
+	else if (id < idOffsets[2]) {
+		return colorGroupActive;
+	}
+	else if (id < idOffsets[3]) {
+		return colorSelect;
+	}
+	else if (id < idOffsets[4]) {
+		return colorGroup;
+	}
+	else {
+		return colorTransform;
 	}
 }
 
+const ivec2 ofs[4] = ivec2[4](
+    ivec2( 1,  0), ivec2( 0,  1),
+    ivec2(-1,  0), ivec2( 0, -1)
+);
+
 void main()
 {
-	ivec2 uv = ivec2(gl_FragCoord.xy);
-
-	vec4 color[4];
-	/* Idea : Use a 16bit ID to identify the color
-	 * and store the colors in a UBO. And fetch all ids
-	 * for discontinuity check with one textureGather \o/ */
-	vec4 ref_col = texelFetch(outlineColor, uv, 0).rgba;
-	color[0] = texelFetchOffset(outlineColor, uv, 0, ivec2( 1,  0)).rgba;
-	color[1] = texelFetchOffset(outlineColor, uv, 0, ivec2( 0,  1)).rgba;
-	color[2] = texelFetchOffset(outlineColor, uv, 0, ivec2(-1,  0)).rgba;
-	color[3] = texelFetchOffset(outlineColor, uv, 0, ivec2( 0, -1)).rgba;
-
-	/* TODO GATHER */
-	vec4 depths;
-	float depth = texelFetch(outlineDepth, uv, 0).r;
-	depths.x = texelFetchOffset(outlineDepth, uv, 0, ivec2( 1,  0)).r;
-	depths.y = texelFetchOffset(outlineDepth, uv, 0, ivec2( 0,  1)).r;
-	depths.z = texelFetchOffset(outlineDepth, uv, 0, ivec2(-1,  0)).r;
-	depths.w = texelFetchOffset(outlineDepth, uv, 0, ivec2( 0, -1)).r;
-
-	vec4 scene_depths;
-	float scene_depth = texelFetch(sceneDepth, uv, 0).r;
-	scene_depths.x = texelFetchOffset(sceneDepth, uv, 0, ivec2( 1,  0)).r;
-	scene_depths.y = texelFetchOffset(sceneDepth, uv, 0, ivec2( 0,  1)).r;
-	scene_depths.z = texelFetchOffset(sceneDepth, uv, 0, ivec2(-1,  0)).r;
-	scene_depths.w = texelFetchOffset(sceneDepth, uv, 0, ivec2( 0, -1)).r;
+	ivec2 texel = ivec2(gl_FragCoord.xy);
+	vec2 uv = gl_FragCoord.xy / vec2(textureSize(outlineId, 0).xy);
 
-	bool ref_occlu = (depth > scene_depth);
-	bool outline = false;
-
-#if 1
-	bvec4 occlu = (!ref_occlu) ? notEqual(greaterThan(depths, scene_depths), bvec4(ref_occlu)) : bvec4(false);
-	outline = (!outline) ? (color[0] != ref_col) || occlu.x : true;
-	outline = (!outline) ? (color[1] != ref_col) || occlu.y : true;
-	outline = (!outline) ? (color[2] != ref_col) || occlu.z : true;
-	outline = (!outline) ? (color[3] != ref_col) || occlu.w : true;
+	uvec4 id;
+	uint ref_id = texelFetch(outlineId, texel, 0).r;
+#if 0 /* commented out until being tested */
+	id = textureGatherOffsets(outlineId, uv, ofs);
 #else
-	search_outline(uv + ivec2( 1,  0), ref_col, ref_occlu, outline);
-	search_outline(uv + ivec2( 0,  1), ref_col, ref_occlu, outline);
-	search_outline(uv + ivec2(-1,  0), ref_col, ref_occlu, outline);
-	search_outline(uv + ivec2( 0, -1), ref_col, ref_occlu, outline);
+	id.x = texelFetchOffset(outlineId, texel, 0, ofs[0]).r;
+	id.y = texelFetchOffset(outlineId, texel, 0, ofs[1]).r;
+	id.z = texelFetchOffset(outlineId, texel, 0, ofs[2]).r;
+	id.w = texelFetchOffset(outlineId, texel, 0, ofs[3]).r;
 #endif
 
-	FragColor = ref_col;
-	FragColor.a *= (outline) ? (ref_occlu) ? alphaOcclu : 1.0 : 0.0;
+	float ref_depth = texelFetch(outlineDepth, texel, 0).r;
+	float scene_depth = texelFetch(sceneDepth, texel, 0).r;
+
+	/* Avoid bad cases of zfighting for occlusion only. */
+	const float epsilon = 3.0 / 8388608.0;
+	bool occluded = (ref_depth > scene_depth + epsilon);
+	bool outline = any(notEqual(id, uvec4(ref_id)));
+
+	FragColor = convert_id_to_color(int(ref_id));
+	FragColor.a *= (occluded) ? alphaOcclu : 1.0;
+	FragColor.a = (outline) ? FragColor.a : 0.0;
 }