Clay Engine: Edit mesh overlays

Based on the previous overlay shader from merwin.
This shader takes care of clipped vertex cases and do all edit mode face info in one pass (except face centers).
As the shading is done one the triangle itself the visual can't go beyond the surface of the mesh. That leads to half displayed edges on the outline of the mesh.
This problem can be fixed by a second pass.

This is work in progress.

6 files changed, 392 insertions, 0 deletions

diff --git a/source/blender/gpu/CMakeLists.txt b/source/blender/gpu/CMakeLists.txt
index 2d82eb692ed..29326a7bd3e 100644
--- a/source/blender/gpu/CMakeLists.txt
+++ b/source/blender/gpu/CMakeLists.txt
@@ -191,6 +191,9 @@ data_to_c_simple(shaders/gpu_shader_edges_overlay_vert.glsl SRC)
 data_to_c_simple(shaders/gpu_shader_edges_overlay_geom.glsl SRC)
 data_to_c_simple(shaders/gpu_shader_edges_overlay_simple_geom.glsl SRC)
 data_to_c_simple(shaders/gpu_shader_edges_overlay_frag.glsl SRC)
+data_to_c_simple(shaders/gpu_shader_edit_overlay_frag.glsl SRC)
+data_to_c_simple(shaders/gpu_shader_edit_overlay_geom.glsl SRC)
+data_to_c_simple(shaders/gpu_shader_edit_overlay_vert.glsl SRC)
 data_to_c_simple(shaders/gpu_shader_text_vert.glsl SRC)
 data_to_c_simple(shaders/gpu_shader_text_frag.glsl SRC)
 data_to_c_simple(shaders/gpu_shader_keyframe_diamond_vert.glsl SRC)
diff --git a/source/blender/gpu/GPU_shader.h b/source/blender/gpu/GPU_shader.h
index 36a59102df8..2a3edd4c42a 100644
--- a/source/blender/gpu/GPU_shader.h
+++ b/source/blender/gpu/GPU_shader.h
@@ -100,6 +100,7 @@ typedef enum GPUBuiltinShader {
 	GPU_SHADER_EDGES_FRONT_BACK_PERSP,
 	GPU_SHADER_EDGES_FRONT_BACK_ORTHO,
 	GPU_SHADER_EDGES_OVERLAY_SIMPLE,
+	GPU_SHADER_EDGES_OVERLAY_EDIT,
 	GPU_SHADER_EDGES_OVERLAY,
 	GPU_SHADER_KEYFRAME_DIAMOND,
 	GPU_SHADER_SIMPLE_LIGHTING,
diff --git a/source/blender/gpu/intern/gpu_shader.c b/source/blender/gpu/intern/gpu_shader.c
index 4f5c5428508..d8508551f0c 100644
--- a/source/blender/gpu/intern/gpu_shader.c
+++ b/source/blender/gpu/intern/gpu_shader.c
@@ -109,6 +109,9 @@ extern char datatoc_gpu_shader_edges_overlay_vert_glsl[];
 extern char datatoc_gpu_shader_edges_overlay_geom_glsl[];
 extern char datatoc_gpu_shader_edges_overlay_simple_geom_glsl[];
 extern char datatoc_gpu_shader_edges_overlay_frag_glsl[];
+extern char datatoc_gpu_shader_edit_overlay_frag_glsl[];
+extern char datatoc_gpu_shader_edit_overlay_geom_glsl[];
+extern char datatoc_gpu_shader_edit_overlay_vert_glsl[];
 extern char datatoc_gpu_shader_text_vert_glsl[];
 extern char datatoc_gpu_shader_text_frag_glsl[];
 extern char datatoc_gpu_shader_keyframe_diamond_vert_glsl[];
@@ -667,6 +670,9 @@ GPUShader *GPU_shader_get_builtin_shader(GPUBuiltinShader shader)
 		                                        datatoc_gpu_shader_flat_color_frag_glsl },
 		[GPU_SHADER_EDGES_OVERLAY_SIMPLE] = { datatoc_gpu_shader_3D_vert_glsl, datatoc_gpu_shader_edges_overlay_frag_glsl,
 		                                      datatoc_gpu_shader_edges_overlay_simple_geom_glsl },
+		[GPU_SHADER_EDGES_OVERLAY_EDIT] = { datatoc_gpu_shader_edit_overlay_vert_glsl,
+		                                    datatoc_gpu_shader_edit_overlay_frag_glsl,
+		                                    datatoc_gpu_shader_edit_overlay_geom_glsl },
 		[GPU_SHADER_EDGES_OVERLAY] = { datatoc_gpu_shader_edges_overlay_vert_glsl,
 		                               datatoc_gpu_shader_edges_overlay_frag_glsl,
 		                               datatoc_gpu_shader_edges_overlay_geom_glsl },
diff --git a/source/blender/gpu/shaders/gpu_shader_edit_overlay_frag.glsl b/source/blender/gpu/shaders/gpu_shader_edit_overlay_frag.glsl
new file mode 100644
index 00000000000..9a1c8511eb1
--- /dev/null
+++ b/source/blender/gpu/shaders/gpu_shader_edit_overlay_frag.glsl
@@ -0,0 +1,191 @@
+
+/* Solid Wirefram implementation
+ * Mike Erwin, Clément Foucault */
+
+/* This shader follows the principles of
+ * http://developer.download.nvidia.com/SDK/10/direct3d/Source/SolidWireframe/Doc/SolidWireframe.pdf */
+
+flat in vec3 edgesCrease;
+flat in vec3 edgesBweight;
+flat in ivec3 flag;
+flat in vec4 faceColor;
+flat in int clipCase;
+// #ifdef VERTEX_SELECTION
+smooth in vec3 vertexColor;
+// #endif
+
+/* We use a vec4[2] interface to pass edge data
+ * (without fragmenting memory accesses)
+ *
+ * There is 2 cases :
+ *
+ * - Simple case : geometry shader return edge distances
+ *   in the first 2 components of the first vec4.
+ *   This needs noperspective interpolation.
+ *   The rest is filled with vertex screen positions.
+ *
+ * - Hard case : two 2d edge corner are described by each
+ *   vec4 as origin and direction. This is constant over
+ *   the triangle and use to detect the correct case. */
+
+noperspective in vec4 eData1;
+flat in vec4 eData2;
+
+out vec4 FragColor;
+
+#define EDGE_EXISTS		(1 << 0)
+#define EDGE_ACTIVE		(1 << 1)
+#define EDGE_SELECTED	(1 << 2)
+#define EDGE_SEAM		(1 << 3)
+#define EDGE_SHARP		(1 << 4)
+/* Vertex flag is shifted and combined with the edge flag */
+#define VERTEX_ACTIVE	(1 << (0 + 8))
+#define VERTEX_SELECTED	(1 << (1 + 8))
+
+/* Style Parameters in pixel */
+const float transitionWidth = 1.0;
+
+/* Width : minimum central size
+ * Outline : additional info */
+const float edgeWidth = 0.2;
+const float edgeOutlineWidth = 1.5;
+
+const float vertexWidth = 2.0;
+const float vertexOutline = 2.0;
+
+/* Array to retreive vert/edge indices */
+const ivec3 clipEdgeIdx[6] = ivec3[6](
+	ivec3(1, 0, 2),
+	ivec3(2, 0, 1),
+	ivec3(2, 1, 0),
+	ivec3(2, 1, 0),
+	ivec3(2, 0, 1),
+	ivec3(1, 0, 2)
+);
+
+const ivec3 clipPointIdx[6] = ivec3[6](
+	ivec3(0, 1, 2),
+	ivec3(0, 2, 1),
+	ivec3(0, 2, 1),
+	ivec3(1, 2, 0),
+	ivec3(1, 2, 0),
+	ivec3(2, 1, 0)
+);
+
+float getEdgeSize(int v)
+{
+	float size = 0.0;
+
+	size += ((flag[v] & EDGE_EXISTS) != 0) ? edgeWidth : 0;
+	size += ((flag[v] & EDGE_SEAM) != 0) ? edgeOutlineWidth : 0;
+	size += ((flag[v] & EDGE_SHARP) != 0) ? edgeOutlineWidth : 0;
+	size += (edgesCrease[v] > 0.0) ? edgeOutlineWidth : 0;
+	size += (edgesBweight[v] > 0.0) ? edgeOutlineWidth : 0;
+
+	return size;
+}
+
+float getVertexSize(int v)
+{
+	float size = vertexWidth;
+
+	size += ((flag[v] & VERTEX_ACTIVE) != 0) ? vertexOutline : 0;
+
+	return size;
+}
+
+void colorDist(vec4 color, float width, inout float dist)
+{
+	FragColor = mix(color, FragColor, smoothstep(0.0, transitionWidth, dist));
+	dist += width;
+}
+
+float distToEdge(vec2 o, vec2 dir)
+{
+	vec2 af = gl_FragCoord.xy - o;
+	float daf = dot(dir, af);
+	return sqrt(abs(dot(af, af) - daf * daf));
+}
+
+void main()
+{
+	vec3 e, p;
+
+	/* Step 1 : Computing Distances */
+
+	if (clipCase == 0) {
+		e.xy = eData1.xy;
+
+		/* computing missing distance */
+		vec2 dir = normalize(eData1.zw - eData2.xy);
+		e.z = distToEdge(eData1.zw, dir);
+
+		p.x = distance(eData1.zw, gl_FragCoord.xy);
+		p.y = distance(eData2.xy, gl_FragCoord.xy);
+		p.z = distance(eData2.zw, gl_FragCoord.xy);
+	}
+	else {
+		ivec3 eidxs = clipEdgeIdx[clipCase - 1];
+		ivec3 pidxs = clipPointIdx[clipCase - 1];
+
+		e[eidxs.x] = distToEdge(eData1.xy, eData1.zw);
+		e[eidxs.y] = distToEdge(eData2.xy, eData2.zw);
+
+		/* Three edges visible cases */
+		if (clipCase == 1 || clipCase == 2 || clipCase == 4) {
+			e[eidxs.z] = distToEdge(eData1.xy, normalize(eData2.xy - eData1.xy));
+			p[pidxs.y] = distance(eData2.xy, gl_FragCoord.xy);
+		}
+		else {
+			e[eidxs.z] = 1e10; /* off screen */
+			p[pidxs.y] = 1e10; /* off screen */
+		}
+
+		p[pidxs.x] = distance(eData1.xy, gl_FragCoord.xy);
+		p[pidxs.z] = 1e10; /* off screen */
+	}
+
+	/* Step 2 : coloring (order dependant) */
+
+	/* First */
+	FragColor = faceColor;
+
+	/* Edges */
+	for (int v = 0; v < 3; ++v) {
+		float edgeness = e[v] - getEdgeSize(v);
+
+		/* Ordered from outer to inner */
+		if (edgesBweight[v] > 0.0)
+			colorDist(vec4(0.0, 0.4, 1.0, edgesBweight[v]), edgeOutlineWidth, edgeness);
+
+		if (edgesCrease[v] > 0.0)
+			colorDist(vec4(0.0, 1.0, 1.0, edgesCrease[v]), edgeOutlineWidth, edgeness);
+
+		if ((flag[v] & EDGE_SEAM) != 0)
+			colorDist(vec4(1.0, 0.0, 0.0, 1.0), edgeOutlineWidth, edgeness);
+
+		if ((flag[v] & EDGE_SHARP) != 0)
+			colorDist(vec4(0.0, 0.5, 1.0, 1.0), edgeOutlineWidth, edgeness);
+
+		if ((flag[v] & EDGE_ACTIVE) != 0)
+			colorDist(vec4(1.0, 1.0, 0.0, 1.0), edgeWidth, edgeness);
+		else if ((flag[v] & EDGE_SELECTED) != 0)
+			colorDist(vec4(0.0, 1.0, 0.0, 1.0), edgeWidth, edgeness);
+		else if ((flag[v] & EDGE_EXISTS) != 0)
+			colorDist(vec4(0.0, 0.0, 0.0, 1.0), edgeWidth, edgeness);
+	}
+
+	/* Points */
+	for (int v = 0; v < 3; ++v) {
+		float size = p[v] - getVertexSize(v);
+
+		/* Ordered from outer to inner */
+		if ((flag[v] & VERTEX_ACTIVE) != 0)
+			colorDist(vec4(1.0, 0.0, 0.0, 1.0), vertexOutline, size);
+
+		if ((flag[v] & VERTEX_SELECTED) != 0)
+			colorDist(vec4(0.0, 1.0, 0.0, 1.0), vertexWidth, size);
+		else
+			colorDist(vec4(0.0, 0.0, 0.0, 1.0), vertexWidth, size);
+	}
+}
diff --git a/source/blender/gpu/shaders/gpu_shader_edit_overlay_geom.glsl b/source/blender/gpu/shaders/gpu_shader_edit_overlay_geom.glsl
new file mode 100644
index 00000000000..7b1393c69c6
--- /dev/null
+++ b/source/blender/gpu/shaders/gpu_shader_edit_overlay_geom.glsl
@@ -0,0 +1,168 @@
+
+/* Solid Wirefram implementation
+ * Mike Erwin, Clément Foucault */
+
+/* This shader follows the principles of
+ * http://developer.download.nvidia.com/SDK/10/direct3d/Source/SolidWireframe/Doc/SolidWireframe.pdf */
+
+layout(triangles) in;
+layout(triangle_strip, max_vertices=3) out;
+
+uniform mat4 ProjectionMatrix;
+uniform vec2 viewportSize;
+
+in vec4 vPos[];
+in vec4 pPos[];
+in ivec4 vData[];
+
+/* these are the same for all vertices
+ * and does not need interpolation */
+flat out vec3 edgesCrease;
+flat out vec3 edgesSharp;
+flat out ivec3 flag;
+flat out vec4 faceColor;
+flat out int clipCase;
+// #ifdef VERTEX_SELECTION
+smooth out vec3 vertexColor;
+// #endif
+
+/* See fragment shader */
+noperspective out vec4 eData1;
+flat out vec4 eData2;
+
+
+#define VERTEX_ACTIVE	(1 << 0)
+#define VERTEX_SELECTED	(1 << 1)
+
+#define FACE_ACTIVE		(1 << 2)
+#define FACE_SELECTED	(1 << 3)
+
+/* Table 1. Triangle Projection Cases */
+const ivec4 clipPointsIdx[6] = ivec4[6](
+	ivec4(0, 1, 2, 2),
+	ivec4(0, 2, 1, 1),
+	ivec4(0, 0, 1, 2),
+	ivec4(1, 2, 0, 0),
+	ivec4(1, 1, 0, 2),
+	ivec4(2, 2, 0, 1)
+);
+
+/* project to screen space */
+vec2 proj(int v)
+{
+	vec4 pos = pPos[v];
+	return (0.5 * (pos.xy / pos.w) + 0.5) * viewportSize;
+}
+
+float dist(vec2 pos[3], int v)
+{
+	/* current vertex position */
+	vec2 vpos = pos[v];
+	/* endpoints of opposite edge */
+	vec2 e1 = pos[(v + 1) % 3];
+	vec2 e2 = pos[(v + 2) % 3];
+	/* Edge normalized vector */
+	vec2 dir = normalize(e2 - e1);
+	/* perpendicular to dir */
+	vec2 orthogonal = vec2(-dir.y, dir.x);
+
+	return abs(dot(vpos - e1, orthogonal));
+}
+
+vec3 getVertexColor(int v)
+{
+	if ((vData[v].x & VERTEX_ACTIVE) != 0)
+		return vec3(0.0, 1.0, 0.0);
+	else if ((vData[v].x & VERTEX_SELECTED) != 0)
+		return vec3(1.0, 0.0, 0.0);
+	else
+		return vec3(0.0, 0.0, 0.0);
+}
+
+vec4 getClipData(vec2 pos[3], ivec2 vidx)
+{
+	vec2 A = pos[vidx.x];
+	vec2 Adir = normalize(A - pos[vidx.y]);
+
+	return vec4(A, Adir);
+}
+
+void doVertex(int v)
+{
+#ifdef VERTEX_SELECTION
+	vertexColor = getVertexColor(v);
+#endif
+
+	gl_Position = pPos[v];
+
+	EmitVertex();
+}
+
+void main()
+{
+	/* First we detect which case we are in */
+	clipCase = 0;
+
+	/* if perspective */
+	if (ProjectionMatrix[3][3] == 0.0) {
+		/* See Table 1. Triangle Projection Cases  */
+		clipCase += int(pPos[0].z / pPos[0].w < -1 || vPos[0].z > 0.0) * 4;
+		clipCase += int(pPos[1].z / pPos[1].w < -1 || vPos[1].z > 0.0) * 2;
+		clipCase += int(pPos[2].z / pPos[2].w < -1 || vPos[2].z > 0.0) * 1;
+	}
+
+	/* If triangle is behind nearplane, early out */
+	if (clipCase == 7)
+		return;
+
+	/* Edge */
+	for (int v = 0; v < 3; ++v) {
+		flag[v] = vData[v].y;// | (vData[v].x << 8);
+		edgesCrease[v] = vData[v].z / 255.0;
+		edgesSharp[v] = vData[v].w / 255.0;
+	}
+
+	/* Face */
+	if ((vData[0].x & FACE_ACTIVE) != 0) {
+		faceColor = vec4(0.1, 1.0, 0.0, 0.2);
+	}
+	else if ((vData[0].x & FACE_SELECTED) != 0) {
+		faceColor = vec4(1.0, 0.2, 0.0, 0.2);
+	}
+	else {
+		faceColor = vec4(0.0, 0.0, 0.0, 0.2);
+	}
+
+	/* Vertex */
+	vec2 pos[3] = vec2[3](proj(0), proj(1), proj(2));
+
+	/* Simple case : compute edge distances in geometry shader */
+	if (clipCase == 0) {
+
+		/* Packing screen positions and 2 distances */
+		eData1 = vec4(0.0, 0.0, pos[0]);
+		eData2 = vec4(pos[1], pos[2]);
+
+		/* Only pass the first 2 distances */
+		for (int v = 0; v < 2; ++v) {
+			eData1[v] = dist(pos, v);
+			doVertex(v);
+			eData1[v] = 0.0;
+		}
+
+		/* and the last vertex
+		doVertex(2);
+	}
+	/* Harder case : compute visible edges vectors */
+	else {
+		ivec4 vindices = clipPointsIdx[clipCase - 1];
+
+		eData1 = getClipData(pos, vindices.xz);
+		eData2 = getClipData(pos, vindices.yw);
+
+		for (int v = 0; v < 3; ++v)
+			doVertex(v);
+	}
+
+	EndPrimitive();
+}
diff --git a/source/blender/gpu/shaders/gpu_shader_edit_overlay_vert.glsl b/source/blender/gpu/shaders/gpu_shader_edit_overlay_vert.glsl
new file mode 100644
index 00000000000..5bc9a8bef85
--- /dev/null
+++ b/source/blender/gpu/shaders/gpu_shader_edit_overlay_vert.glsl
@@ -0,0 +1,23 @@
+
+/* Solid Wirefram implementation
+ * Mike Erwin, Clément Foucault */
+
+/* This shader follows the principles of
+ * http://developer.download.nvidia.com/SDK/10/direct3d/Source/SolidWireframe/Doc/SolidWireframe.pdf */
+
+uniform mat4 ModelViewMatrix;
+uniform mat4 ModelViewProjectionMatrix;
+
+in vec3 pos;
+in ivec4 data;
+
+out vec4 vPos;
+out vec4 pPos;
+out ivec4 vData;
+
+void main()
+{
+	vPos = ModelViewMatrix * vec4(pos, 1.0);
+	pPos = ModelViewProjectionMatrix * vec4(pos, 1.0);
+	vData = data;
+}