Object Mode: Rework outline drawing.

This changes quite a few things.
- Outline is now per object.
- No more outline at object intersection (fix hairs problem).
- Simplify the code quite a bit.

We use a R16UI buffer to save one id per object outline. We convert this id
to color when detecting the outline.

Added textureGatherOffsets to the code but could not test on current hardware
so leaving it commented for now.

4 files changed, 137 insertions, 94 deletions

diff --git a/source/blender/draw/CMakeLists.txt b/source/blender/draw/CMakeLists.txt
index d292e4d9405..d2e70300561 100644
--- a/source/blender/draw/CMakeLists.txt
+++ b/source/blender/draw/CMakeLists.txt
@@ -229,6 +229,7 @@ data_to_c_simple(modes/shaders/object_empty_image_vert.glsl SRC)
 data_to_c_simple(modes/shaders/object_outline_resolve_frag.glsl SRC)
 data_to_c_simple(modes/shaders/object_outline_expand_frag.glsl SRC)
 data_to_c_simple(modes/shaders/object_outline_detect_frag.glsl SRC)
+data_to_c_simple(modes/shaders/object_outline_prepass_frag.glsl SRC)
 data_to_c_simple(modes/shaders/object_grid_frag.glsl SRC)
 data_to_c_simple(modes/shaders/object_grid_vert.glsl SRC)
 data_to_c_simple(modes/shaders/object_lightprobe_grid_vert.glsl SRC)
diff --git a/source/blender/draw/modes/object_mode.c b/source/blender/draw/modes/object_mode.c
index 939507a4f21..0d9e46d1268 100644
--- a/source/blender/draw/modes/object_mode.c
+++ b/source/blender/draw/modes/object_mode.c
@@ -40,6 +40,8 @@
 
 #include "BIF_gl.h"
 
+#include "BLI_string_utils.h"
+
 #include "BKE_anim.h"
 #include "BKE_camera.h"
 #include "BKE_curve.h"
@@ -69,6 +71,7 @@ extern struct GPUUniformBuffer *globals_ubo; /* draw_common.c */
 extern struct GPUTexture *globals_ramp; /* draw_common.c */
 extern GlobalsUboStorage ts;
 
+extern char datatoc_object_outline_prepass_frag_glsl[];
 extern char datatoc_object_outline_resolve_frag_glsl[];
 extern char datatoc_object_outline_detect_frag_glsl[];
 extern char datatoc_object_outline_expand_frag_glsl[];
@@ -223,6 +226,13 @@ typedef struct OBJECT_PrivateData {
 	DRWShadingGroup *points_select;
 	DRWShadingGroup *points_select_group;
 	DRWShadingGroup *points_transform;
+
+	/* Outlines id offset */
+	int id_ofs_active;
+	int id_ofs_active_group;
+	int id_ofs_select;
+	int id_ofs_select_group;
+	int id_ofs_transform;
 } OBJECT_PrivateData; /* Transient data */
 
 static struct {
@@ -232,6 +242,7 @@ static struct {
 	struct Gwn_VertFormat *empty_image_wire_format;
 
 	/* fullscreen shaders */
+	GPUShader *outline_prepass_sh;
 	GPUShader *outline_resolve_sh;
 	GPUShader *outline_resolve_aa_sh;
 	GPUShader *outline_detect_sh;
@@ -259,6 +270,7 @@ static struct {
 	bool draw_grid;
 	/* Temp buffer textures */
 	struct GPUTexture *outlines_depth_tx;
+	struct GPUTexture *outlines_id_tx;
 	struct GPUTexture *outlines_color_tx;
 	struct GPUTexture *outlines_blur_tx;
 } e_data = {NULL}; /* Engine data */
@@ -288,14 +300,20 @@ static void OBJECT_engine_init(void *vedata)
 	if (DRW_state_is_fbo()) {
 		e_data.outlines_depth_tx = DRW_texture_pool_query_2D(size[0], size[1], DRW_TEX_DEPTH_24,
 		                                                     &draw_engine_object_type);
-		e_data.outlines_color_tx = DRW_texture_pool_query_2D(size[0], size[1], DRW_TEX_RGBA_8,
-		                                                     &draw_engine_object_type);
+		/* XXX TODO DRW_TEX_R_16U can overflow, it would cause no harm
+		 * (only bad colored or missing outlines) but we should
+		 * use 32bits only if the scene have that many objects */
+		e_data.outlines_id_tx = DRW_texture_pool_query_2D(size[0], size[1], DRW_TEX_R_16U,
+		                                                  &draw_engine_object_type);
 
 		GPU_framebuffer_ensure_config(&fbl->outlines_fb, {
 			GPU_ATTACHMENT_TEXTURE(e_data.outlines_depth_tx),
-			GPU_ATTACHMENT_TEXTURE(e_data.outlines_color_tx)
+			GPU_ATTACHMENT_TEXTURE(e_data.outlines_id_tx)
 		});
 
+		e_data.outlines_color_tx = DRW_texture_pool_query_2D(size[0], size[1], DRW_TEX_RGBA_8,
+		                                                     &draw_engine_object_type);
+
 		GPU_framebuffer_ensure_config(&fbl->expand_fb, {
 			GPU_ATTACHMENT_NONE,
 			GPU_ATTACHMENT_TEXTURE(e_data.outlines_color_tx)
@@ -310,66 +328,58 @@ static void OBJECT_engine_init(void *vedata)
 		});
 	}
 
+	/* Shaders */
 	if (!e_data.outline_resolve_sh) {
+		/* Outline */
+		e_data.outline_prepass_sh = DRW_shader_create_3D(datatoc_object_outline_prepass_frag_glsl, NULL);
+
 		e_data.outline_resolve_sh = DRW_shader_create_fullscreen(datatoc_object_outline_resolve_frag_glsl, NULL);
-	}
 
-	if (!e_data.outline_resolve_aa_sh) {
 		e_data.outline_resolve_aa_sh = DRW_shader_create_with_lib(
 		            datatoc_common_fullscreen_vert_glsl, NULL,
 		            datatoc_object_outline_resolve_frag_glsl,
 		            datatoc_common_fxaa_lib_glsl,
 		            "#define FXAA_ALPHA\n"
 		            "#define USE_FXAA\n");
-	}
 
-	if (!e_data.outline_detect_sh) {
-		e_data.outline_detect_sh = DRW_shader_create_fullscreen(datatoc_object_outline_detect_frag_glsl, NULL);
-	}
+		e_data.outline_detect_sh = DRW_shader_create_with_lib(
+		            datatoc_common_fullscreen_vert_glsl, NULL,
+		            datatoc_object_outline_detect_frag_glsl,
+		            datatoc_common_globals_lib_glsl,
+		            NULL);
 
-	if (!e_data.outline_fade_sh) {
 		e_data.outline_fade_sh = DRW_shader_create_fullscreen(datatoc_object_outline_expand_frag_glsl, NULL);
-	}
 
-	if (!e_data.object_empty_image_sh) {
+		/* Empty images */
 		e_data.object_empty_image_sh = DRW_shader_create_with_lib(
 		           datatoc_object_empty_image_vert_glsl, NULL,
 		           datatoc_object_empty_image_frag_glsl,
 		           datatoc_common_globals_lib_glsl, NULL);
-	}
 
-	if (!e_data.object_empty_image_wire_sh) {
 		e_data.object_empty_image_wire_sh = DRW_shader_create_with_lib(
 		           datatoc_object_empty_image_vert_glsl, NULL,
 		           datatoc_object_empty_image_frag_glsl,
 		           datatoc_common_globals_lib_glsl,
 		           "#define USE_WIRE\n");
-	}
 
-	if (!e_data.grid_sh) {
+		/* Grid */
 		e_data.grid_sh = DRW_shader_create_with_lib(
 		        datatoc_object_grid_vert_glsl, NULL,
 		        datatoc_object_grid_frag_glsl,
 		        datatoc_common_globals_lib_glsl, NULL);
-	}
 
-	if (!e_data.part_prim_sh) {
+		/* Particles */
 		e_data.part_prim_sh = DRW_shader_create(
 		        datatoc_object_particle_prim_vert_glsl, NULL, datatoc_gpu_shader_flat_color_frag_glsl, NULL);
-	}
 
-	if (!e_data.part_axis_sh) {
 		e_data.part_axis_sh = DRW_shader_create(
 		        datatoc_object_particle_prim_vert_glsl, NULL, datatoc_gpu_shader_flat_color_frag_glsl,
 		        "#define USE_AXIS\n");
-	}
 
-	if (!e_data.part_dot_sh) {
 		e_data.part_dot_sh = DRW_shader_create(
 		        datatoc_object_particle_dot_vert_glsl, NULL, datatoc_object_particle_dot_frag_glsl, NULL);
-	}
 
-	if (!e_data.lightprobe_grid_sh) {
+		/* Lightprobes */
 		e_data.lightprobe_grid_sh = DRW_shader_create(
 		        datatoc_object_lightprobe_grid_vert_glsl, NULL, datatoc_gpu_shader_uniform_color_frag_glsl, NULL);
 	}
@@ -558,6 +568,7 @@ static void OBJECT_engine_free(void)
 	MEM_SAFE_FREE(e_data.particle_format);
 	MEM_SAFE_FREE(e_data.empty_image_format);
 	MEM_SAFE_FREE(e_data.empty_image_wire_format);
+	DRW_SHADER_FREE_SAFE(e_data.outline_prepass_sh);
 	DRW_SHADER_FREE_SAFE(e_data.outline_resolve_sh);
 	DRW_SHADER_FREE_SAFE(e_data.outline_resolve_aa_sh);
 	DRW_SHADER_FREE_SAFE(e_data.outline_detect_sh);
@@ -571,10 +582,10 @@ static void OBJECT_engine_free(void)
 	DRW_SHADER_FREE_SAFE(e_data.lightprobe_grid_sh);
 }
 
-static DRWShadingGroup *shgroup_outline(DRWPass *pass, const float col[4], GPUShader *sh)
+static DRWShadingGroup *shgroup_outline(DRWPass *pass, const int *ofs, GPUShader *sh)
 {
 	DRWShadingGroup *grp = DRW_shgroup_create(sh, pass);
-	DRW_shgroup_uniform_vec4(grp, "color", col, 1);
+	DRW_shgroup_uniform_int(grp, "baseId", ofs, 1);
 
 	return grp;
 }
@@ -602,12 +613,19 @@ static DRWShadingGroup *shgroup_theme_id_to_outline_or(
 {
 	switch (theme_id) {
 		case TH_ACTIVE:
+			stl->g_data->id_ofs_active++;
 			return stl->g_data->outlines_active;
 		case TH_SELECT:
+			stl->g_data->id_ofs_select++;
 			return stl->g_data->outlines_select;
-		case TH_GROUP_ACTIVE:
+		case TH_GROUP:
+			stl->g_data->id_ofs_select_group++;
 			return stl->g_data->outlines_select_group;
+		case TH_GROUP_ACTIVE:
+			stl->g_data->id_ofs_active_group++;
+			return stl->g_data->outlines_active_group;
 		case TH_TRANSFORM:
+			stl->g_data->id_ofs_transform++;
 			return stl->g_data->outlines_transform;
 		default:
 			return fallback;
@@ -622,8 +640,10 @@ static DRWShadingGroup *shgroup_theme_id_to_wire_or(
 			return stl->g_data->wire_active;
 		case TH_SELECT:
 			return stl->g_data->wire_select;
-		case TH_GROUP_ACTIVE:
+		case TH_GROUP:
 			return stl->g_data->wire_select_group;
+		case TH_GROUP_ACTIVE:
+			return stl->g_data->wire_active_group;
 		case TH_TRANSFORM:
 			return stl->g_data->wire_transform;
 		default:
@@ -639,8 +659,10 @@ static DRWShadingGroup *shgroup_theme_id_to_point_or(
 			return stl->g_data->points_active;
 		case TH_SELECT:
 			return stl->g_data->points_select;
-		case TH_GROUP_ACTIVE:
+		case TH_GROUP:
 			return stl->g_data->points_select_group;
+		case TH_GROUP_ACTIVE:
+			return stl->g_data->points_active_group;
 		case TH_TRANSFORM:
 			return stl->g_data->points_transform;
 		default:
@@ -786,18 +808,24 @@ static void OBJECT_cache_init(void *vedata)
 		DRWState state = DRW_STATE_WRITE_COLOR | DRW_STATE_WRITE_DEPTH | DRW_STATE_DEPTH_LESS | DRW_STATE_WIRE;
 		psl->outlines = DRW_pass_create("Outlines Depth Pass", state);
 
-		GPUShader *sh = GPU_shader_get_builtin_shader(GPU_SHADER_3D_UNIFORM_COLOR);
+		GPUShader *sh = e_data.outline_prepass_sh;
 
 		/* Select */
-		stl->g_data->outlines_select = shgroup_outline(psl->outlines, ts.colorSelect, sh);
-		stl->g_data->outlines_select_group = shgroup_outline(psl->outlines, ts.colorGroupActive, sh);
+		stl->g_data->outlines_select = shgroup_outline(psl->outlines, &stl->g_data->id_ofs_select, sh);
+		stl->g_data->outlines_select_group = shgroup_outline(psl->outlines, &stl->g_data->id_ofs_select_group, sh);
 
 		/* Transform */
-		stl->g_data->outlines_transform = shgroup_outline(psl->outlines, ts.colorTransform, sh);
+		stl->g_data->outlines_transform = shgroup_outline(psl->outlines, &stl->g_data->id_ofs_transform, sh);
 
 		/* Active */
-		stl->g_data->outlines_active = shgroup_outline(psl->outlines, ts.colorActive, sh);
-		stl->g_data->outlines_active_group = shgroup_outline(psl->outlines, ts.colorGroupActive, sh);
+		stl->g_data->outlines_active = shgroup_outline(psl->outlines, &stl->g_data->id_ofs_active, sh);
+		stl->g_data->outlines_active_group = shgroup_outline(psl->outlines, &stl->g_data->id_ofs_active_group, sh);
+
+		stl->g_data->id_ofs_select = 0;
+		stl->g_data->id_ofs_select_group = 0;
+		stl->g_data->id_ofs_active = 0;
+		stl->g_data->id_ofs_active_group = 0;
+		stl->g_data->id_ofs_transform = 0;
 	}
 
 	{
@@ -822,10 +850,12 @@ static void OBJECT_cache_init(void *vedata)
 		psl->outlines_search = DRW_pass_create("Outlines Detect Pass", state);
 
 		DRWShadingGroup *grp = DRW_shgroup_create(e_data.outline_detect_sh, psl->outlines_search);
-		DRW_shgroup_uniform_texture_ref(grp, "outlineColor", &e_data.outlines_color_tx);
+		DRW_shgroup_uniform_texture_ref(grp, "outlineId", &e_data.outlines_id_tx);
 		DRW_shgroup_uniform_texture_ref(grp, "outlineDepth", &e_data.outlines_depth_tx);
 		DRW_shgroup_uniform_texture_ref(grp, "sceneDepth", &dtxl->depth);
+		DRW_shgroup_uniform_block(grp, "globalsBlock", globals_ubo);
 		DRW_shgroup_uniform_float(grp, "alphaOcclu", &alphaOcclu, 1);
+		DRW_shgroup_uniform_int(grp, "idOffsets", &stl->g_data->id_ofs_active, 5);
 		DRW_shgroup_call_add(grp, quad, NULL);
 
 		psl->outlines_expand = DRW_pass_create("Outlines Expand Pass", state);
@@ -2038,6 +2068,7 @@ static void OBJECT_draw_scene(void *vedata)
 	OBJECT_PassList *psl = ((OBJECT_Data *)vedata)->psl;
 	OBJECT_StorageList *stl = ((OBJECT_Data *)vedata)->stl;
 	OBJECT_FramebufferList *fbl = ((OBJECT_Data *)vedata)->fbl;
+	OBJECT_PrivateData *g_data = stl->g_data;
 	DefaultFramebufferList *dfbl = DRW_viewport_framebuffer_list_get();
 	DefaultTextureList *dtxl = DRW_viewport_texture_list_get();
 
@@ -2045,7 +2076,18 @@ static void OBJECT_draw_scene(void *vedata)
 
 	if (DRW_state_is_fbo()) {
 		DRW_stats_group_start("Outlines");
-		
+
+		int id_ct_select = g_data->id_ofs_select;
+		int id_ct_select_group = g_data->id_ofs_select_group;
+		int id_ct_active = g_data->id_ofs_active;
+		int id_ct_active_group = g_data->id_ofs_active_group;
+
+		g_data->id_ofs_active = 1;
+		g_data->id_ofs_active_group = g_data->id_ofs_active + id_ct_active + 1;
+		g_data->id_ofs_select = g_data->id_ofs_active_group + id_ct_active_group + 1;
+		g_data->id_ofs_select_group = g_data->id_ofs_select + id_ct_select + 1;
+		g_data->id_ofs_transform = g_data->id_ofs_select_group + id_ct_select_group + 1;
+
 		/* Render filled polygon on a separate framebuffer */
 		GPU_framebuffer_bind(fbl->outlines_fb);
 		GPU_framebuffer_clear_color_depth(fbl->outlines_fb, clearcol, 1.0f);
diff --git a/source/blender/draw/modes/shaders/object_outline_detect_frag.glsl b/source/blender/draw/modes/shaders/object_outline_detect_frag.glsl
index dc0ea938436..6b0f66d7c94 100644
--- a/source/blender/draw/modes/shaders/object_outline_detect_frag.glsl
+++ b/source/blender/draw/modes/shaders/object_outline_detect_frag.glsl
@@ -3,77 +3,67 @@ in vec4 uvcoordsvar;
 
 out vec4 FragColor;
 
-uniform sampler2D outlineColor;
+uniform usampler2D outlineId;
 uniform sampler2D outlineDepth;
 uniform sampler2D sceneDepth;
 
+uniform int idOffsets[5];
+
 uniform float alphaOcclu;
 uniform vec2 viewportSize;
 
-void search_outline(ivec2 uv, vec4 ref_col, inout bool ref_occlu, inout bool outline)
+vec4 convert_id_to_color(int id)
 {
-	if (!outline) {
-		vec4 color = texelFetch(outlineColor, uv, 0).rgba;
-		if (color != ref_col) {
-			outline = true;
-		}
-		else {
-			float depth = texelFetch(outlineDepth, uv, 0).r;
-			float scene_depth = texelFetch(sceneDepth, uv, 0).r;
-			bool occlu = (depth > scene_depth);
-
-			if (occlu != ref_occlu && !ref_occlu) {
-				outline = true;
-			}
-		}
+	if (id == 0) {
+		return vec4(0.0);
+	}
+	if (id < idOffsets[1]) {
+		return colorActive;
+	}
+	else if (id < idOffsets[2]) {
+		return colorGroupActive;
+	}
+	else if (id < idOffsets[3]) {
+		return colorSelect;
+	}
+	else if (id < idOffsets[4]) {
+		return colorGroup;
+	}
+	else {
+		return colorTransform;
 	}
 }
 
+const ivec2 ofs[4] = ivec2[4](
+    ivec2( 1,  0), ivec2( 0,  1),
+    ivec2(-1,  0), ivec2( 0, -1)
+);
+
 void main()
 {
-	ivec2 uv = ivec2(gl_FragCoord.xy);
-
-	vec4 color[4];
-	/* Idea : Use a 16bit ID to identify the color
-	 * and store the colors in a UBO. And fetch all ids
-	 * for discontinuity check with one textureGather \o/ */
-	vec4 ref_col = texelFetch(outlineColor, uv, 0).rgba;
-	color[0] = texelFetchOffset(outlineColor, uv, 0, ivec2( 1,  0)).rgba;
-	color[1] = texelFetchOffset(outlineColor, uv, 0, ivec2( 0,  1)).rgba;
-	color[2] = texelFetchOffset(outlineColor, uv, 0, ivec2(-1,  0)).rgba;
-	color[3] = texelFetchOffset(outlineColor, uv, 0, ivec2( 0, -1)).rgba;
-
-	/* TODO GATHER */
-	vec4 depths;
-	float depth = texelFetch(outlineDepth, uv, 0).r;
-	depths.x = texelFetchOffset(outlineDepth, uv, 0, ivec2( 1,  0)).r;
-	depths.y = texelFetchOffset(outlineDepth, uv, 0, ivec2( 0,  1)).r;
-	depths.z = texelFetchOffset(outlineDepth, uv, 0, ivec2(-1,  0)).r;
-	depths.w = texelFetchOffset(outlineDepth, uv, 0, ivec2( 0, -1)).r;
-
-	vec4 scene_depths;
-	float scene_depth = texelFetch(sceneDepth, uv, 0).r;
-	scene_depths.x = texelFetchOffset(sceneDepth, uv, 0, ivec2( 1,  0)).r;
-	scene_depths.y = texelFetchOffset(sceneDepth, uv, 0, ivec2( 0,  1)).r;
-	scene_depths.z = texelFetchOffset(sceneDepth, uv, 0, ivec2(-1,  0)).r;
-	scene_depths.w = texelFetchOffset(sceneDepth, uv, 0, ivec2( 0, -1)).r;
+	ivec2 texel = ivec2(gl_FragCoord.xy);
+	vec2 uv = gl_FragCoord.xy / vec2(textureSize(outlineId, 0).xy);
 
-	bool ref_occlu = (depth > scene_depth);
-	bool outline = false;
-
-#if 1
-	bvec4 occlu = (!ref_occlu) ? notEqual(greaterThan(depths, scene_depths), bvec4(ref_occlu)) : bvec4(false);
-	outline = (!outline) ? (color[0] != ref_col) || occlu.x : true;
-	outline = (!outline) ? (color[1] != ref_col) || occlu.y : true;
-	outline = (!outline) ? (color[2] != ref_col) || occlu.z : true;
-	outline = (!outline) ? (color[3] != ref_col) || occlu.w : true;
+	uvec4 id;
+	uint ref_id = texelFetch(outlineId, texel, 0).r;
+#if 0 /* commented out until being tested */
+	id = textureGatherOffsets(outlineId, uv, ofs);
 #else
-	search_outline(uv + ivec2( 1,  0), ref_col, ref_occlu, outline);
-	search_outline(uv + ivec2( 0,  1), ref_col, ref_occlu, outline);
-	search_outline(uv + ivec2(-1,  0), ref_col, ref_occlu, outline);
-	search_outline(uv + ivec2( 0, -1), ref_col, ref_occlu, outline);
+	id.x = texelFetchOffset(outlineId, texel, 0, ofs[0]).r;
+	id.y = texelFetchOffset(outlineId, texel, 0, ofs[1]).r;
+	id.z = texelFetchOffset(outlineId, texel, 0, ofs[2]).r;
+	id.w = texelFetchOffset(outlineId, texel, 0, ofs[3]).r;
 #endif
 
-	FragColor = ref_col;
-	FragColor.a *= (outline) ? (ref_occlu) ? alphaOcclu : 1.0 : 0.0;
+	float ref_depth = texelFetch(outlineDepth, texel, 0).r;
+	float scene_depth = texelFetch(sceneDepth, texel, 0).r;
+
+	/* Avoid bad cases of zfighting for occlusion only. */
+	const float epsilon = 3.0 / 8388608.0;
+	bool occluded = (ref_depth > scene_depth + epsilon);
+	bool outline = any(notEqual(id, uvec4(ref_id)));
+
+	FragColor = convert_id_to_color(int(ref_id));
+	FragColor.a *= (occluded) ? alphaOcclu : 1.0;
+	FragColor.a = (outline) ? FragColor.a : 0.0;
 }
diff --git a/source/blender/draw/modes/shaders/object_outline_prepass_frag.glsl b/source/blender/draw/modes/shaders/object_outline_prepass_frag.glsl
new file mode 100644
index 00000000000..776adb787ad
--- /dev/null
+++ b/source/blender/draw/modes/shaders/object_outline_prepass_frag.glsl
@@ -0,0 +1,10 @@
+uniform int callId;
+uniform int baseId;
+
+/* using uint because 16bit uint can contain more ids than int. */
+out uint outId;
+
+void main()
+{
+	outId = uint(baseId + callId);
+}