Edit Mesh: Rework new implementation and use geometry shader to draw lines

This make it (theoriticaly) compatible with all supported hardware with
consistent results.

Also we now draw the lines with analytic anti-aliasing instead of relying
on MSAA (which offers less benefits in our case).

The remaining aliasing comes from edges cut in half by the mesh which is
not rendered with MSAA. Hopefully this is not too much distracting and only
happen if the face is almost parallel to the view.

1 files changed, 27 insertions, 14 deletions

diff --git a/source/blender/draw/modes/shaders/edit_mesh_overlay_frag.glsl b/source/blender/draw/modes/shaders/edit_mesh_overlay_frag.glsl
index bf41a0a43ae..d48c5158872 100644
--- a/source/blender/draw/modes/shaders/edit_mesh_overlay_frag.glsl
+++ b/source/blender/draw/modes/shaders/edit_mesh_overlay_frag.glsl
@@ -1,23 +1,36 @@
 
-#ifdef FLAT
-flat in vec4 finalColor;
-#else
-in vec4 finalColor;
-#  ifdef EDGE
-flat in int selectOveride;
-#  endif
-#endif
+#define M_1_SQRTPI   0.5641895835477563    /* 1/sqrt(pi) */
+
+/**
+ * We want to know how much a pixel is covered by a line.
+ * We replace the square pixel with acircle of the same area and try to find the intersection area.
+ * The area we search is the circular segment. https://en.wikipedia.org/wiki/Circular_segment
+ * The formula for the area uses inverse trig function and is quite complexe.
+ * Instead, we approximate it by using the smoothstep function and a 1.05 factor to the disc radius.
+ **/
+#define DISC_RADIUS (M_1_SQRTPI * 1.05)
+#define GRID_LINE_SMOOTH_START (0.5 - DISC_RADIUS)
+#define GRID_LINE_SMOOTH_END (0.5 + DISC_RADIUS)
+
+uniform float edgeScale;
+
+flat in vec4 finalColorOuter_f;
+in vec4 finalColor_f;
+in float edgeCoord_f;
 
 out vec4 FragColor;
 
 void main()
 {
-#if defined(EDGE) && !defined(FLAT)
-	vec4 prim_col = mix(colorEditMeshMiddle, colorEdgeSelect, finalColor.a);
-	prim_col = (selectOveride != 0) ? prim_col : finalColor;
-	prim_col.a = 1.0;
+	float dist = abs(edgeCoord_f) - max(sizeEdge * edgeScale - 0.5, 0.0);
+	float dist_outer = dist - max(sizeEdge * edgeScale, 1.0);
+#if 1
+	float mix_w = smoothstep(GRID_LINE_SMOOTH_START, GRID_LINE_SMOOTH_END, dist);
+	float mix_w_outer = smoothstep(GRID_LINE_SMOOTH_START, GRID_LINE_SMOOTH_END, dist_outer);
 #else
-#  define prim_col finalColor
+	float mix_w = step(0.5, dist);
+	float mix_w_outer = step(0.5, dist_outer);
 #endif
-	FragColor = prim_col;
+	FragColor = mix(finalColorOuter_f, finalColor_f, 1.0 - mix_w * finalColorOuter_f.a);
+	FragColor.a *= 1.0 - (finalColorOuter_f.a > 0.0 ? mix_w_outer : mix_w);
 }