Edit Mesh: Simplify the overlay shader.

Remove all the clip cases and just pass the vertices screen position to the
fragment shader.

This does put a bit more pressure on the fragment shader but it seems to be
faster than before. And it simplify the code a lot.

It seems to fix some long standing issue on some intel GPU.

4 files changed, 104 insertions, 295 deletions

diff --git a/source/blender/draw/modes/shaders/edit_mesh_overlay_frag.glsl b/source/blender/draw/modes/shaders/edit_mesh_overlay_frag.glsl
index 4627643cf86..9fa6220da4d 100644
--- a/source/blender/draw/modes/shaders/edit_mesh_overlay_frag.glsl
+++ b/source/blender/draw/modes/shaders/edit_mesh_overlay_frag.glsl
@@ -10,11 +10,11 @@
  * TODO Refine the range to only affect GPUs. */
 
 uniform float faceAlphaMod;
+
 flat in vec3 edgesCrease;
 flat in vec3 edgesBweight;
 flat in vec4 faceColor;
 flat in ivec3 flag;
-flat in int clipCase;
 #ifdef VERTEX_SELECTION
 in vec3 vertexColor;
 #endif
@@ -22,25 +22,7 @@ in vec3 vertexColor;
 in float facing;
 #endif
 
-/* We use a vec4[2] interface to pass edge data
- * (without fragmenting memory accesses)
- *
- * There are 2 cases :
- *
- * - Simple case : geometry shader return edge distances
- *   in the first 2 components of the first vec4.
- *   This needs noperspective interpolation.
- *   The rest is filled with vertex screen positions.
- *   eData2[0] actually contain v2
- *   eData2[1] actually contain v1
- *   eData2[2] actually contain v0
- *
- * - Hard case : two 2d edge corner are described by each
- *   vec4 as origin and direction. This is constant over
- *   the triangle and use to detect the correct case. */
-
-noperspective in vec2 eData1;
-flat in vec2 eData2[3];
+flat in vec2 ssPos[3];
 
 out vec4 FragColor;
 
@@ -49,38 +31,21 @@ out vec4 FragColor;
 
 #define LARGE_EDGE_SIZE 2.0
 
-
 /* Style Parameters in pixel */
 
-/* Array to retrieve vert/edge indices */
-const ivec3 clipEdgeIdx[6] = ivec3[6](
-	ivec3(1, 0, 2),
-	ivec3(2, 0, 1),
-	ivec3(2, 1, 0),
-	ivec3(2, 1, 0),
-	ivec3(2, 0, 1),
-	ivec3(1, 0, 2)
-);
-
-const ivec3 clipPointIdx[6] = ivec3[6](
-	ivec3(0, 1, 2),
-	ivec3(0, 2, 1),
-	ivec3(0, 2, 1),
-	ivec3(1, 2, 0),
-	ivec3(1, 2, 0),
-	ivec3(2, 1, 0)
-);
-
-void colorDist(vec4 color, float dist)
+void distToEdgeAndPoint(vec2 dir, vec2 ori, out float edge, out float point)
 {
-	FragColor = (dist < 0) ? color : FragColor;
+	dir = normalize(dir.xy);
+	vec2 of = gl_FragCoord.xy - ori;
+	point = dot(of, of);
+	float dof = dot(dir, of);
+	edge = sqrt(abs(point - dof * dof));
+	point = sqrt(point);
 }
 
-float distToEdge(vec2 o, vec2 dir)
+void colorDist(vec4 color, float dist)
 {
-	vec2 af = gl_FragCoord.xy - o;
-	float daf = dot(dir, af);
-	return sqrt(abs(dot(af, af) - daf * daf));
+	FragColor = (dist < 0) ? color : FragColor;
 }
 
 #ifdef ANTI_ALIASING
@@ -99,42 +64,13 @@ void main()
 	vec3 e, p;
 
 	/* Step 1 : Computing Distances */
-
-	if (clipCase == 0) {
-		e.xy = eData1;
-
-		/* computing missing distance */
-		vec2 dir = normalize(eData2[2] - eData2[1]);
-		e.z = distToEdge(eData2[2], dir);
-
-		p.x = distance(eData2[2], gl_FragCoord.xy);
-		p.y = distance(eData2[1], gl_FragCoord.xy);
-		p.z = distance(eData2[0], gl_FragCoord.xy);
-	}
-	else {
-		ivec3 eidxs = clipEdgeIdx[clipCase - 1];
-		ivec3 pidxs = clipPointIdx[clipCase - 1];
-
-		e[eidxs.x] = distToEdge(eData1, eData2[0]);
-		e[eidxs.y] = distToEdge(eData2[1], eData2[2]);
-
-		/* Three edges visible cases */
-		if (clipCase == 1 || clipCase == 2 || clipCase == 4) {
-			e[eidxs.z] = distToEdge(eData1, normalize(eData2[1] - eData1));
-			p[pidxs.y] = distance(eData2[1], gl_FragCoord.xy);
-		}
-		else {
-			e[eidxs.z] = 1e10; /* off screen */
-			p[pidxs.y] = 1e10; /* off screen */
-		}
-
-		p[pidxs.x] = distance(eData1, gl_FragCoord.xy);
-		p[pidxs.z] = 1e10; /* off screen */
-	}
+	distToEdgeAndPoint((ssPos[1] - ssPos[0]) + 1e-8, ssPos[0], e.z, p.x);
+	distToEdgeAndPoint((ssPos[2] - ssPos[1]) + 1e-8, ssPos[1], e.x, p.y);
+	distToEdgeAndPoint((ssPos[0] - ssPos[2]) + 1e-8, ssPos[2], e.y, p.z);
 
 	/* Step 2 : coloring (order dependant) */
 
-	/* First */
+	/* Face */
 	FragColor = faceColor;
 	FragColor.a *= faceAlphaMod;
 
diff --git a/source/blender/draw/modes/shaders/edit_mesh_overlay_geom_edge.glsl b/source/blender/draw/modes/shaders/edit_mesh_overlay_geom_edge.glsl
index 0089f3b004c..88cb6c07a94 100644
--- a/source/blender/draw/modes/shaders/edit_mesh_overlay_geom_edge.glsl
+++ b/source/blender/draw/modes/shaders/edit_mesh_overlay_geom_edge.glsl
@@ -24,7 +24,6 @@ flat out vec3 edgesCrease;
 flat out vec3 edgesBweight;
 flat out vec4 faceColor;
 flat out ivec3 flag;
-flat out int clipCase;
 #ifdef VERTEX_SELECTION
 out vec3 vertexColor;
 #endif
@@ -33,41 +32,17 @@ out float facing;
 #endif
 
 /* See fragment shader */
-noperspective out vec2 eData1;
-flat out vec2 eData2[3];
+flat out vec2 ssPos[3];
 
 #define FACE_ACTIVE     (1 << 2)
 #define FACE_SELECTED   (1 << 3)
 
-/* Table 1. Triangle Projection Cases */
-const ivec4 clipPointsIdx[6] = ivec4[6](
-	ivec4(0, 1, 2, 2),
-	ivec4(0, 2, 1, 1),
-	ivec4(0, 0, 1, 2),
-	ivec4(1, 2, 0, 0),
-	ivec4(1, 1, 0, 2),
-	ivec4(2, 2, 0, 1)
-);
-
 /* project to screen space */
 vec2 proj(vec4 pos)
 {
 	return (0.5 * (pos.xy / pos.w) + 0.5) * viewportSize;
 }
 
-float dist(vec2 pos[3], vec2 vpos, int v)
-{
-	/* endpoints of opposite edge */
-	vec2 e1 = pos[(v + 1) % 3];
-	vec2 e2 = pos[(v + 2) % 3];
-	/* Edge normalized vector */
-	vec2 dir = normalize(e2 - e1);
-	/* perpendicular to dir */
-	vec2 orthogonal = vec2(-dir.y, dir.x);
-
-	return abs(dot(vpos - e1, orthogonal));
-}
-
 void doVertex(int v, vec4 pos)
 {
 #ifdef VERTEX_SELECTION
@@ -85,8 +60,6 @@ void doVertex(int v, vec4 pos)
 
 void main()
 {
-	clipCase = 0;
-
 	/* Face */
 	faceColor = vec4(0.0);
 
@@ -117,19 +90,16 @@ void main()
 	}
 
 	/* Edge / Vert data */
-	eData1 = vec2(1e10);
-	eData2[0] = vec2(1e10);
-	eData2[2] = pos[0];
-	eData2[1] = pos[1];
-	flag[0] = (vData[0].x << 8);
+	ssPos[0] = ssPos[2] = pos[0];
+	ssPos[1] = pos[1];
+	flag[0] = flag[2] = (vData[0].x << 8);
 	flag[1] = (vData[1].x << 8);
-	flag[2] = 0;
 
 	doVertex(0, pPos[0] + vec4(-dirs1.xy, 0.0, 0.0));
 	doVertex(0, pPos[0] + vec4( dirs1.zw, 0.0, 0.0));
 	doVertex(0, pPos[0] + vec4(-dirs1.zw, 0.0, 0.0));
 
-	flag[2] = vData[0].y | (vData[0].x << 8);
+	flag[2] |= vData[0].y;
 	edgesCrease[2] = vData[0].z / 255.0;
 	edgesBweight[2] = vData[0].w / 255.0;
 
diff --git a/source/blender/draw/modes/shaders/edit_mesh_overlay_geom_tri.glsl b/source/blender/draw/modes/shaders/edit_mesh_overlay_geom_tri.glsl
index 890144982ab..e0ec6e50dc2 100644
--- a/source/blender/draw/modes/shaders/edit_mesh_overlay_geom_tri.glsl
+++ b/source/blender/draw/modes/shaders/edit_mesh_overlay_geom_tri.glsl
@@ -52,49 +52,17 @@ out float facing;
 #endif
 
 /* See fragment shader */
-noperspective out vec2 eData1;
-flat out vec2 eData2[3];
+flat out vec2 ssPos[3];
 
 #define FACE_ACTIVE     (1 << 2)
 #define FACE_SELECTED   (1 << 3)
 
-/* Table 1. Triangle Projection Cases */
-const ivec4 clipPointsIdx[6] = ivec4[6](
-	ivec4(0, 1, 2, 2),
-	ivec4(0, 2, 1, 1),
-	ivec4(0, 0, 1, 2),
-	ivec4(1, 2, 0, 0),
-	ivec4(1, 1, 0, 2),
-	ivec4(2, 2, 0, 1)
-);
-
 /* project to screen space */
 vec2 proj(vec4 pos)
 {
 	return (0.5 * (pos.xy / pos.w) + 0.5) * viewportSize;
 }
 
-float dist(vec2 pos[3], vec2 vpos, int v)
-{
-	/* endpoints of opposite edge */
-	vec2 e1 = pos[(v + 1) % 3];
-	vec2 e2 = pos[(v + 2) % 3];
-	/* Edge normalized vector */
-	vec2 dir = normalize(e2 - e1);
-	/* perpendicular to dir */
-	vec2 orthogonal = vec2(-dir.y, dir.x);
-
-	return abs(dot(vpos - e1, orthogonal));
-}
-
-vec4 getClipData(vec2 pos[3], ivec2 vidx)
-{
-	vec2 A = pos[vidx.x];
-	vec2 Adir = normalize(A - pos[vidx.y]);
-
-	return vec4(A, Adir);
-}
-
 void doVertex(int v)
 {
 #ifdef VERTEX_SELECTION
@@ -127,21 +95,6 @@ void doLoopStrip(int v, vec3 offset)
 
 void main()
 {
-	/* First we detect which case we are in */
-	clipCase = 0;
-
-	/* if perspective */
-	if (ProjectionMatrix[3][3] == 0.0) {
-		/* See Table 1. Triangle Projection Cases  */
-		clipCase += int(pPos[0].z / pPos[0].w < -1 || vPos[0].z > 0.0) * 4;
-		clipCase += int(pPos[1].z / pPos[1].w < -1 || vPos[1].z > 0.0) * 2;
-		clipCase += int(pPos[2].z / pPos[2].w < -1 || vPos[2].z > 0.0) * 1;
-	}
-
-	/* If triangle is behind nearplane, early out */
-	if (clipCase == 7)
-		return;
-
 	/* Edge */
 	ivec3 eflag; vec3 ecrease, ebweight;
 	for (int v = 0; v < 3; ++v) {
@@ -159,138 +112,97 @@ void main()
 		faceColor = colorFace;
 
 	/* Vertex */
-	vec2 pos[3] = vec2[3](proj(pPos[0]), proj(pPos[1]), proj(pPos[2]));
+	vec2 pos[3];
+	ssPos[0] = pos[0] = proj(pPos[0]);
+	ssPos[1] = pos[1] = proj(pPos[1]);
+	ssPos[2] = pos[2] = proj(pPos[2]);
 
-	/* Simple case : compute edge distances in geometry shader */
-	if (clipCase == 0) {
+	doVertex(0);
+	doVertex(1);
+	doVertex(2);
 
-		/* Packing screen positions and 2 distances */
-		eData2[0] = pos[2];
-		eData2[1] = pos[1];
-		eData2[2] = pos[0];
+#ifdef EDGE_FIX
+	vec2 fixvec[6];
+	vec2 fixvecaf[6];
+	vec2 cornervec[3];
+
+	/* This fix the case when 2 vertices are perfectly aligned
+	 * and corner vectors have nowhere to go.
+	 * ie: length(cornervec[i]) == 0 */
+	const float epsilon = 1e-2; /* in pixel so not that much */
+	const vec2 bias[3] = vec2[3](
+		vec2( epsilon,  epsilon),
+		vec2(-epsilon,  epsilon),
+		vec2(     0.0, -epsilon)
+	);
+
+	for (int i = 0; i < 3; ++i) {
+		int i1 = (i + 1) % 3;
+		int i2 = (i + 2) % 3;
+
+		vec2 v1 = ssPos[i] + bias[i];
+		vec2 v2 = ssPos[i1] + bias[i1];
+		vec2 v3 = ssPos[i2] + bias[i2];
+
+		/* Edge normalized vector */
+		vec2 dir = normalize(v2 - v1);
+		vec2 dir2 = normalize(v3 - v1);
+
+		cornervec[i] = -normalize(dir + dir2);
+
+		/* perpendicular to dir */
+		vec2 perp = vec2(-dir.y, dir.x);
+
+		/* Backface case */
+		if (dot(perp, dir2) > 0) {
+			perp = -perp;
+		}
 
-		/* Only pass the first 2 distances */
-		for (int v = 0; v < 2; ++v) {
-			eData1[v] = dist(pos, pos[v], v);
-			doVertex(v);
-			eData1[v] = 0.0;
+		/* Make it view independent */
+		perp *= sizeEdgeFix / viewportSize;
+		cornervec[i] *= sizeEdgeFix / viewportSize;
+		fixvec[i] = fixvecaf[i] = perp;
+
+		/* Perspective */
+		if (ProjectionMatrix[3][3] == 0.0) {
+			/* vPos[i].z is negative and we don't want
+			 * our fixvec to be flipped */
+			fixvec[i] *= -vPos[i].z;
+			fixvecaf[i] *= -vPos[i1].z;
+			cornervec[i] *= -vPos[i].z;
 		}
+	}
 
-		/* and the last vertex */
-		doVertex(2);
+	/* to not let face color bleed */
+	faceColor.a = 0.0;
 
-#ifdef EDGE_FIX
-		vec2 fixvec[6];
-		vec2 fixvecaf[6];
-		vec2 cornervec[3];
-
-		/* This fix the case when 2 vertices are perfectly aligned
-		 * and corner vectors have nowhere to go.
-		 * ie: length(cornervec[i]) == 0 */
-		const float epsilon = 1e-2; /* in pixel so not that much */
-		const vec2 bias[3] = vec2[3](
-			vec2( epsilon,  epsilon),
-			vec2(-epsilon,  epsilon),
-			vec2(     0.0, -epsilon)
-		);
-
-		for (int i = 0; i < 3; ++i) {
-			int i1 = (i + 1) % 3;
-			int i2 = (i + 2) % 3;
-
-			vec2 v1 = pos[i] + bias[i];
-			vec2 v2 = pos[i1] + bias[i1];
-			vec2 v3 = pos[i2] + bias[i2];
-
-			/* Edge normalized vector */
-			vec2 dir = normalize(v2 - v1);
-			vec2 dir2 = normalize(v3 - v1);
-
-			cornervec[i] = -normalize(dir + dir2);
-
-			/* perpendicular to dir */
-			vec2 perp = vec2(-dir.y, dir.x);
-
-			/* Backface case */
-			if (dot(perp, dir2) > 0) {
-				perp = -perp;
-			}
-
-			/* Make it view independent */
-			perp *= sizeEdgeFix / viewportSize;
-			cornervec[i] *= sizeEdgeFix / viewportSize;
-			fixvec[i] = fixvecaf[i] = perp;
-
-			/* Perspective */
-			if (ProjectionMatrix[3][3] == 0.0) {
-				/* vPos[i].z is negative and we don't want
-				 * our fixvec to be flipped */
-				fixvec[i] *= -vPos[i].z;
-				fixvecaf[i] *= -vPos[i1].z;
-				cornervec[i] *= -vPos[i].z;
-			}
-		}
+	/* Start with the same last vertex to create a
+	 * degenerate triangle in order to "create"
+	 * a new triangle strip */
+	for (int i = 2; i < 5; ++i) {
+		int vbe = (i - 1) % 3;
+		int vaf = (i + 1) % 3;
+		int v = i % 3;
+
+		doLoopStrip(v, vec3(fixvec[v], Z_OFFSET));
 
-		/* to not let face color bleed */
-		faceColor.a = 0.0;
-
-		/* we don't want other edges : make them far */
-		eData1 = vec2(1e10);
-		eData2[0] = vec2(1e10);
-
-		/* Start with the same last vertex to create a
-		 * degenerate triangle in order to "create"
-		 * a new triangle strip */
-		for (int i = 2; i < 5; ++i) {
-			int vbe = (i - 1) % 3;
-			int vaf = (i + 1) % 3;
-			int v = i % 3;
-
-			/* Position of the "hidden" third vertex */
-			eData2[0] = pos[vbe];
-			doLoopStrip(v, vec3(fixvec[v], Z_OFFSET));
-
-			/* Now one triangle only shade one edge
-			 * so we use the edge distance calculated
-			 * in the fragment shader, the third edge;
-			 * we do this because we need flat interp to
-			 * draw a continuous triangle strip */
-			eData2[1] = pos[vaf];
-			eData2[2] = pos[v];
-			flag[0] = (vData[v].x << 8);
-			flag[1] = (vData[vaf].x << 8);
-			flag[2] = eflag[vbe];
-			edgesCrease[2] = ecrease[vbe];
-			edgesBweight[2] = ebweight[vbe];
-
-			doLoopStrip(vaf, vec3(fixvecaf[v], Z_OFFSET));
-
-			/* corner vertices should not draw edges but draw point only */
-			flag[2] = (vData[vbe].x << 8);
+		/* Only shade the edge that we are currently drawing.
+		 * (fix corner bleeding) */
+		flag[vbe] |= (EDGE_EXISTS & eflag[vbe]);
+		flag[vaf] &= ~EDGE_EXISTS;
+		flag[v]   &= ~EDGE_EXISTS;
+		doLoopStrip(vaf, vec3(fixvecaf[v], Z_OFFSET));
+
+		/* corner vertices should not draw edges but draw point only */
+		flag[vbe] &= ~EDGE_EXISTS;
 #ifdef VERTEX_SELECTION
-			doLoopStrip(vaf, vec3(cornervec[vaf], Z_OFFSET));
+		doLoopStrip(vaf, vec3(cornervec[vaf], Z_OFFSET));
 #endif
-		}
+	}
 
-		/* finish the loop strip */
-		doLoopStrip(2, vec3(fixvec[2], Z_OFFSET));
+	/* finish the loop strip */
+	doLoopStrip(2, vec3(fixvec[2], Z_OFFSET));
 #endif
-	}
-	/* Harder case : compute visible edges vectors */
-	else {
-		ivec4 vindices = clipPointsIdx[clipCase - 1];
-
-		vec4 tmp;
-		tmp = getClipData(pos, vindices.xz);
-		eData1 = tmp.xy;
-		eData2[0] = tmp.zw;
-		tmp = getClipData(pos, vindices.yw);
-		eData2[1] = tmp.xy;
-		eData2[2] = tmp.zw;
-
-		for (int v = 0; v < 3; ++v)
-			doVertex(v);
-	}
 
 	EndPrimitive();
 }
diff --git a/source/blender/draw/modes/shaders/edit_mesh_overlay_loosevert_vert.glsl b/source/blender/draw/modes/shaders/edit_mesh_overlay_loosevert_vert.glsl
index 53e5b09cb70..13d824c2410 100644
--- a/source/blender/draw/modes/shaders/edit_mesh_overlay_loosevert_vert.glsl
+++ b/source/blender/draw/modes/shaders/edit_mesh_overlay_loosevert_vert.glsl
@@ -17,14 +17,12 @@ flat out vec3 edgesCrease;
 flat out vec3 edgesBweight;
 flat out vec4 faceColor;
 flat out ivec3 flag;
-flat out int clipCase;
 #ifdef VERTEX_SELECTION
 out vec3 vertexColor;
 #endif
 
 /* See fragment shader */
-noperspective out vec2 eData1;
-flat out vec2 eData2[3];
+flat out vec2 ssPos[3];
 
 /* project to screen space */
 vec2 proj(vec4 pos)
@@ -34,7 +32,6 @@ vec2 proj(vec4 pos)
 
 void main()
 {
-	clipCase = 0;
 	edgesCrease = vec3(0.0);
 	edgesBweight = vec3(0.0);
 
@@ -47,14 +44,8 @@ void main()
 	vertexColor = vec3(0.0);
 #endif
 
-	/* only vertex position 0 is used */
-	eData1 = vec2(1e10);
-	eData2[0] = vec2(1e10);
-	eData2[1] = vec2(1e10);
-	eData2[2] = proj(pPos);
-
-	flag[0] = (data.x << 8);
-	flag[1] = flag[2] = 0;
+	ssPos[0] = ssPos[1] = ssPos[2] = proj(pPos);
+	flag[0] = flag[1] = flag[2] = (data.x << 8);
 
 	gl_PointSize = sizeEdgeFix;
 	gl_Position = pPos;