Clay Engine: Edit Mesh overlay fix jaggy edges.

We do a strip of triangles around the triangle of interest and modify the variying vars to "trick" the fragment shader into rendering what we need.
This keeps the compatibility for both shaders.
Vertex still get half displayed when angle at the vertex is very acute.

This patch is only for the simple, no vert clipped case.

This is 2.5x slower than without the trick (on my hardware with 1million tris).

2 files changed, 111 insertions, 21 deletions

diff --git a/source/blender/gpu/shaders/gpu_shader_edit_overlay_frag.glsl b/source/blender/gpu/shaders/gpu_shader_edit_overlay_frag.glsl
index 9a1c8511eb1..dcee761650b 100644
--- a/source/blender/gpu/shaders/gpu_shader_edit_overlay_frag.glsl
+++ b/source/blender/gpu/shaders/gpu_shader_edit_overlay_frag.glsl
@@ -23,6 +23,9 @@ smooth in vec3 vertexColor;
  *   in the first 2 components of the first vec4.
  *   This needs noperspective interpolation.
  *   The rest is filled with vertex screen positions.
+ *   eData1.zw actually contain v2
+ *   eData2.xy actually contain v1
+ *   eData2.zw actually contain v0
  *
  * - Hard case : two 2d edge corner are described by each
  *   vec4 as origin and direction. This is constant over
@@ -117,12 +120,12 @@ void main()
 		e.xy = eData1.xy;
 
 		/* computing missing distance */
-		vec2 dir = normalize(eData1.zw - eData2.xy);
-		e.z = distToEdge(eData1.zw, dir);
+		vec2 dir = normalize(eData2.zw - eData2.xy);
+		e.z = distToEdge(eData2.zw, dir);
 
-		p.x = distance(eData1.zw, gl_FragCoord.xy);
+		p.x = distance(eData2.zw, gl_FragCoord.xy);
 		p.y = distance(eData2.xy, gl_FragCoord.xy);
-		p.z = distance(eData2.zw, gl_FragCoord.xy);
+		p.z = distance(eData1.zw, gl_FragCoord.xy);
 	}
 	else {
 		ivec3 eidxs = clipEdgeIdx[clipCase - 1];
diff --git a/source/blender/gpu/shaders/gpu_shader_edit_overlay_geom.glsl b/source/blender/gpu/shaders/gpu_shader_edit_overlay_geom.glsl
index f7ee3f006e4..7f37607ecc1 100644
--- a/source/blender/gpu/shaders/gpu_shader_edit_overlay_geom.glsl
+++ b/source/blender/gpu/shaders/gpu_shader_edit_overlay_geom.glsl
@@ -5,8 +5,22 @@
 /* This shader follows the principles of
  * http://developer.download.nvidia.com/SDK/10/direct3d/Source/SolidWireframe/Doc/SolidWireframe.pdf */
 
+#define EDGE_FIX
+
 layout(triangles) in;
+
+#ifdef EDGE_FIX
+/* To fix the edge artifacts, we render
+ * an outline strip around the screenspace
+ * triangle. Order is important.
+ * TODO diagram
+ */
+const float fixupSize = 6.0; /* in pixels */
+
+layout(triangle_strip, max_vertices=17) out;
+#else
 layout(triangle_strip, max_vertices=3) out;
+#endif
 
 uniform mat4 ProjectionMatrix;
 uniform vec2 viewportSize;
@@ -48,16 +62,13 @@ const ivec4 clipPointsIdx[6] = ivec4[6](
 );
 
 /* project to screen space */
-vec2 proj(int v)
+vec2 proj(vec4 pos)
 {
-	vec4 pos = pPos[v];
 	return (0.5 * (pos.xy / pos.w) + 0.5) * viewportSize;
 }
 
-float dist(vec2 pos[3], int v)
+float dist(vec2 pos[3], vec2 vpos, int v)
 {
-	/* current vertex position */
-	vec2 vpos = pos[v];
 	/* endpoints of opposite edge */
 	vec2 e1 = pos[(v + 1) % 3];
 	vec2 e2 = pos[(v + 2) % 3];
@@ -87,13 +98,13 @@ vec4 getClipData(vec2 pos[3], ivec2 vidx)
 	return vec4(A, Adir);
 }
 
-void doVertex(int v)
+void doVertex(int v, vec4 pos)
 {
 #ifdef VERTEX_SELECTION
 	vertexColor = getVertexColor(v);
 #endif
 
-	gl_Position = pPos[v];
+	gl_Position = pos;
 
 	EmitVertex();
 }
@@ -116,10 +127,11 @@ void main()
 		return;
 
 	/* Edge */
+	ivec3 eflag; vec3 ecrease, esharp;
 	for (int v = 0; v < 3; ++v) {
-		flag[v] = vData[v].y;// | (vData[v].x << 8);
-		edgesCrease[v] = vData[v].z / 255.0;
-		edgesSharp[v] = vData[v].w / 255.0;
+		flag[v] = eflag[v] = vData[v].y | (vData[v].x << 8);
+		edgesCrease[v] = ecrease[v] = vData[v].z / 255.0;
+		edgesSharp[v] = esharp[v] = vData[v].w / 255.0;
 	}
 
 	/* Face */
@@ -134,24 +146,99 @@ void main()
 	}
 
 	/* Vertex */
-	vec2 pos[3] = vec2[3](proj(0), proj(1), proj(2));
+	vec2 pos[3] = vec2[3](proj(pPos[0]), proj(pPos[1]), proj(pPos[2]));
 
 	/* Simple case : compute edge distances in geometry shader */
 	if (clipCase == 0) {
 
 		/* Packing screen positions and 2 distances */
-		eData1 = vec4(0.0, 0.0, pos[0]);
-		eData2 = vec4(pos[1], pos[2]);
+		eData1 = vec4(0.0, 0.0, pos[2]);
+		eData2 = vec4(pos[1], pos[0]);
 
 		/* Only pass the first 2 distances */
 		for (int v = 0; v < 2; ++v) {
-			eData1[v] = dist(pos, v);
-			doVertex(v);
+			eData1[v] = dist(pos, pos[v], v);
+			doVertex(v, pPos[v]);
 			eData1[v] = 0.0;
 		}
 
 		/* and the last vertex */
-		doVertex(2);
+		doVertex(2, pPos[2]);
+
+#ifdef EDGE_FIX
+		vec2 fixvec[3];
+
+		for (int v = 0; v < 3; ++v) {
+			vec2 v1 = pos[v];
+			vec2 v2 = pos[(v + 1) % 3];
+			vec2 v3 = pos[(v + 2) % 3];
+
+			/* Edge normalized vector */
+			vec2 dir = normalize(v2 - v1);
+			/* perpendicular to dir */
+			vec2 perp = vec2(-dir.y, dir.x);
+
+			/* Backface case */
+			if (dot(perp, v3 - v1) > 0) {
+				perp = -perp;
+			}
+
+			fixvec[v] = perp * fixupSize;
+
+			/* Make it view independant */
+			fixvec[v] /= viewportSize;
+
+			/* Perspective */
+			if (ProjectionMatrix[3][3] == 0.0) {
+				/* vPos[v].z is negative and we don't want
+				 * our fixvec to be flipped */
+				fixvec[v] *= -vPos[v].z;
+			}
+		}
+
+		/* to not let face color bleed */
+		faceColor = vec4(0.0, 0.0, 0.0, 0.0);
+
+		/* we don't want other edges : make them far*/
+		eData1 = vec4(1e10);
+
+		/* Start with the same last vertex to create a
+		 * degenerate triangle in order to "create"
+		 * a new triangle strip */
+		for (int i = 2; i < 7; ++i) {
+			int vbe = (i - 1) % 3;
+			int vaf = (i + 1) % 3;
+			int v = i % 3;
+
+			/* first 2 vertex should not drax edges */
+			flag[2] = (vData[v].x << 8);
+			doVertex(v, pPos[v]);
+
+			vec4 fixPos = pPos[v] + vec4(fixvec[v], 0.0, 0.0);
+			doVertex(v, fixPos);
+
+			/* do a final corner tri but not another edge */
+			if(i == 6) continue;
+
+			/* Now one triangle only shade one edge
+			 * so we use the edge distance calculated
+			 * in the fragment shader, the third edge;
+			 * we do this because we need flat interp to
+			 * draw a continuous triangle strip */
+			eData2.xy = pos[vaf];
+			eData2.zw = pos[v];
+			flag[0] = (vData[v].x << 8);
+			flag[1] = (vData[vaf].x << 8);
+			flag[2] = vData[vbe].y;
+			edgesCrease[2] = ecrease[vbe];
+			edgesSharp[2] = esharp[vbe];
+
+			doVertex(vaf, pPos[vaf]);
+
+			fixPos = pPos[vaf] + vec4(fixvec[v], 0.0, 0.0);
+			doVertex(vaf, fixPos);
+		}
+#endif
 	}
 	/* Harder case : compute visible edges vectors */
 	else {
@@ -161,7 +248,7 @@ void main()
 		eData2 = getClipData(pos, vindices.yw);
 
 		for (int v = 0; v < 3; ++v)
-			doVertex(v);
+			doVertex(v, pPos[v]);
 	}
 
 	EndPrimitive();