Overlay: Improve Outline diagonal Antialiasing

I doubt we can do much better. Most of the aliasing comes from the edge
detection which does not use a lot of samples. We could use more samples
but then the detection becomes way more complex and expensive.

The second issue comes from the reconstruction (AA pass) that only bleed
adjacent pixels in if their line direction is perpendicular to the offset.
This makes corner gaps on certain diagonals.

2 files changed, 74 insertions, 39 deletions

diff --git a/source/blender/draw/engines/overlay/shaders/antialiasing_frag.glsl b/source/blender/draw/engines/overlay/shaders/antialiasing_frag.glsl
index 4784d420e1d..0d01f67c6ea 100644
--- a/source/blender/draw/engines/overlay/shaders/antialiasing_frag.glsl
+++ b/source/blender/draw/engines/overlay/shaders/antialiasing_frag.glsl
@@ -144,6 +144,8 @@ void main()
     vec4 lines = vec4(neightbor_line0.z, neightbor_line1.z, neightbor_line2.z, neightbor_line3.z);
     /* Count number of line neighbors. */
     float blend = dot(vec4(0.25), step(0.001, lines));
+    /* Only do blend if there is more than 2 neighbor. This avoid loosing too much AA. */
+    blend = clamp(blend * 2.0 - 1.0, 0.0, 1.0);
     fragColor = mix(fragColor, fragColor / fragColor.a, blend);
   }
 #endif
diff --git a/source/blender/draw/engines/overlay/shaders/outline_detect_frag.glsl b/source/blender/draw/engines/overlay/shaders/outline_detect_frag.glsl
index f5a3aa2c332..c77c89396af 100644
--- a/source/blender/draw/engines/overlay/shaders/outline_detect_frag.glsl
+++ b/source/blender/draw/engines/overlay/shaders/outline_detect_frag.glsl
@@ -57,6 +57,34 @@ bvec4 gather_edges(vec2 uv, uint ref)
   return notEqual(ids, uvec4(ref));
 }
 
+/* Clockwise */
+vec2 rotate_90(vec2 v)
+{
+  return vec2(v.y, -v.x);
+}
+vec2 rotate_180(vec2 v)
+{
+  return vec2(-v.x, -v.y);
+}
+vec2 rotate_270(vec2 v)
+{
+  return vec2(-v.y, v.x);
+}
+
+/* Counter-Clockwise */
+bvec4 rotate_90(bvec4 v)
+{
+  return v.yzwx;
+}
+bvec4 rotate_180(bvec4 v)
+{
+  return v.zwxy;
+}
+bvec4 rotate_270(bvec4 v)
+{
+  return v.wxyz;
+}
+
 /* Apply offset to line endpoint based on surrounding edges infos. */
 bool line_offset(bvec2 edges, vec2 ofs, inout vec2 line_point)
 {
@@ -79,6 +107,7 @@ bool line_offset(bvec2 edges, vec2 ofs, inout vec2 line_point)
 /* Use surrounding edges to approximate the outline direction to create smooth lines. */
 void straight_line_dir(bvec4 edges1, bvec4 edges2, out vec2 line_start, out vec2 line_end)
 {
+  /* Y_POS as reference. Other cases are rotated to match reference. */
   line_end = vec2(1.5, 0.5 + PROXIMITY_OFS);
   line_start = vec2(-line_end.x, line_end.y);
 
@@ -92,53 +121,50 @@ void straight_line_dir(bvec4 edges1, bvec4 edges2, out vec2 line_start, out vec2
   }
 }
 
-/* Compute line direction vector from the bottom left corner. */
-void diag_dir(bvec4 edges, out vec2 line_start, out vec2 line_end)
+vec2 diag_offset(bvec4 edges)
 {
-  /* TODO Improve diagonal antialiasing. */
+  /* X_NEG | Y_POS as reference. Other cases are rotated to match reference.
+   * So the line is comming from bottom left. */
   if (all(edges.wz)) {
-    line_start = vec2(-3.0, -0.5 + PROXIMITY_OFS);
-    line_end = vec2(3.0, 0.5 + PROXIMITY_OFS);
+    /* Horizontal line. */
+    return vec2(2.5, 0.5);
   }
   else if (all(not(edges.xw))) {
-    line_start = vec2(-0.5 - PROXIMITY_OFS, -3.0);
-    line_end = vec2(0.5 - PROXIMITY_OFS, 3.0);
+    /* Vertical line. */
+    return vec2(0.5, 2.5);
   }
   else if (edges.w) {
-    line_start = vec2(-1.0, -0.5 - PROXIMITY_OFS);
-    line_end = vec2(2.0, 0.5 - PROXIMITY_OFS);
+    /* Less horizontal Line. */
+    return vec2(2.5, 0.5);
   }
   else {
-    line_start = vec2(-0.6, -0.5 + PROXIMITY_OFS);
-    line_end = vec2(0.6 - PROXIMITY_OFS, 0.5);
+    /* Less vertical Line. */
+    return vec2(0.5, 2.5);
   }
 }
-/* Clockwise */
-vec2 rotate_90(vec2 v)
-{
-  return vec2(v.y, -v.x);
-}
-vec2 rotate_180(vec2 v)
-{
-  return vec2(-v.x, -v.y);
-}
-vec2 rotate_270(vec2 v)
-{
-  return vec2(-v.y, v.x);
-}
 
-/* Counter-Clockwise */
-bvec4 rotate_90(bvec4 v)
-{
-  return v.yzwx;
-}
-bvec4 rotate_180(bvec4 v)
-{
-  return v.zwxy;
-}
-bvec4 rotate_270(bvec4 v)
+/* Compute line direction vector from the bottom left corner. */
+void diag_dir(bvec4 edges1, bvec4 edges2, out vec2 line_start, out vec2 line_end)
 {
-  return v.wxyz;
+  /* Negate instead of rotating back the result of diag_offset. */
+  edges2 = not(edges2);
+  edges2 = rotate_180(edges2);
+  line_end = diag_offset(edges1);
+  line_end += diag_offset(edges2);
+
+  if (line_end.x == line_end.y) {
+    /* Perfect diagonal line. Push line start towards edge. */
+    line_start = vec2(-1.0, 1.0) * PROXIMITY_OFS * 0.4;
+  }
+  else if (line_end.x > line_end.y) {
+    /* Horizontal Line. Lower line start. */
+    line_start = vec2(0.0, PROXIMITY_OFS);
+  }
+  else {
+    /* Vertical Line. Push line start to the right. */
+    line_start = -vec2(PROXIMITY_OFS, 0.0);
+  }
+  line_end += line_start;
 }
 
 void main()
@@ -292,26 +318,33 @@ void main()
       /* Diagonal */
     case DIAG_XNEG_YPOS:
       extra_edges = gather_edges(uvs + ofs.xy * vec2(1.5), ref);
-      diag_dir(extra_edges, line_start, line_end);
+      extra_edges2 = gather_edges(uvs + ofs.xy * vec2(-1.5), ref);
+      diag_dir(extra_edges, extra_edges2, line_start, line_end);
       break;
     case DIAG_XPOS_YNEG:
       extra_edges = gather_edges(uvs - ofs.xy * vec2(1.5), ref);
+      extra_edges2 = gather_edges(uvs - ofs.xy * vec2(-1.5), ref);
       extra_edges = rotate_180(extra_edges);
-      diag_dir(extra_edges, line_start, line_end);
+      extra_edges2 = rotate_180(extra_edges2);
+      diag_dir(extra_edges, extra_edges2, line_start, line_end);
       line_start = rotate_180(line_start);
       line_end = rotate_180(line_end);
       break;
     case DIAG_XPOS_YPOS:
       extra_edges = gather_edges(uvs + ofs.xy * vec2(1.5, -1.5), ref);
+      extra_edges2 = gather_edges(uvs - ofs.xy * vec2(1.5, -1.5), ref);
       extra_edges = rotate_90(extra_edges);
-      diag_dir(extra_edges, line_start, line_end);
+      extra_edges2 = rotate_90(extra_edges2);
+      diag_dir(extra_edges, extra_edges2, line_start, line_end);
       line_start = rotate_90(line_start);
       line_end = rotate_90(line_end);
       break;
     case DIAG_XNEG_YNEG:
       extra_edges = gather_edges(uvs - ofs.xy * vec2(1.5, -1.5), ref);
+      extra_edges2 = gather_edges(uvs + ofs.xy * vec2(1.5, -1.5), ref);
       extra_edges = rotate_270(extra_edges);
-      diag_dir(extra_edges, line_start, line_end);
+      extra_edges2 = rotate_270(extra_edges2);
+      diag_dir(extra_edges, extra_edges2, line_start, line_end);
       line_start = rotate_270(line_start);
       line_end = rotate_270(line_end);
       break;