Edit Mode overlays: separate multiple shaders for loose edges and verts

6 files changed, 243 insertions, 56 deletions

diff --git a/source/blender/gpu/CMakeLists.txt b/source/blender/gpu/CMakeLists.txt
index 66e792c9749..95b15d664ab 100644
--- a/source/blender/gpu/CMakeLists.txt
+++ b/source/blender/gpu/CMakeLists.txt
@@ -192,7 +192,9 @@ data_to_c_simple(shaders/gpu_shader_edges_overlay_geom.glsl SRC)
 data_to_c_simple(shaders/gpu_shader_edges_overlay_simple_geom.glsl SRC)
 data_to_c_simple(shaders/gpu_shader_edges_overlay_frag.glsl SRC)
 data_to_c_simple(shaders/gpu_shader_edit_overlay_frag.glsl SRC)
-data_to_c_simple(shaders/gpu_shader_edit_overlay_geom.glsl SRC)
+data_to_c_simple(shaders/gpu_shader_edit_overlay_geom_tri.glsl SRC)
+data_to_c_simple(shaders/gpu_shader_edit_overlay_geom_edge.glsl SRC)
+data_to_c_simple(shaders/gpu_shader_edit_overlay_geom_vert.glsl SRC)
 data_to_c_simple(shaders/gpu_shader_edit_overlay_vert.glsl SRC)
 data_to_c_simple(shaders/gpu_shader_text_vert.glsl SRC)
 data_to_c_simple(shaders/gpu_shader_text_frag.glsl SRC)
diff --git a/source/blender/gpu/GPU_shader.h b/source/blender/gpu/GPU_shader.h
index 1e03d9abe25..55fa4749221 100644
--- a/source/blender/gpu/GPU_shader.h
+++ b/source/blender/gpu/GPU_shader.h
@@ -100,7 +100,9 @@ typedef enum GPUBuiltinShader {
 	GPU_SHADER_EDGES_FRONT_BACK_PERSP,
 	GPU_SHADER_EDGES_FRONT_BACK_ORTHO,
 	GPU_SHADER_EDGES_OVERLAY_SIMPLE,
-	GPU_SHADER_EDGES_OVERLAY_EDIT,
+	GPU_SHADER_EDGES_OVERLAY_EDIT_TRI,
+	GPU_SHADER_EDGES_OVERLAY_EDIT_VERT,
+	GPU_SHADER_EDGES_OVERLAY_EDIT_EDGE,
 	GPU_SHADER_EDGES_OVERLAY,
 	GPU_SHADER_KEYFRAME_DIAMOND,
 	GPU_SHADER_SIMPLE_LIGHTING,
diff --git a/source/blender/gpu/intern/gpu_shader.c b/source/blender/gpu/intern/gpu_shader.c
index 4bfae6523a9..c633fd0835e 100644
--- a/source/blender/gpu/intern/gpu_shader.c
+++ b/source/blender/gpu/intern/gpu_shader.c
@@ -110,7 +110,9 @@ extern char datatoc_gpu_shader_edges_overlay_geom_glsl[];
 extern char datatoc_gpu_shader_edges_overlay_simple_geom_glsl[];
 extern char datatoc_gpu_shader_edges_overlay_frag_glsl[];
 extern char datatoc_gpu_shader_edit_overlay_frag_glsl[];
-extern char datatoc_gpu_shader_edit_overlay_geom_glsl[];
+extern char datatoc_gpu_shader_edit_overlay_geom_tri_glsl[];
+extern char datatoc_gpu_shader_edit_overlay_geom_edge_glsl[];
+extern char datatoc_gpu_shader_edit_overlay_geom_vert_glsl[];
 extern char datatoc_gpu_shader_edit_overlay_vert_glsl[];
 extern char datatoc_gpu_shader_text_vert_glsl[];
 extern char datatoc_gpu_shader_text_frag_glsl[];
@@ -670,9 +672,15 @@ GPUShader *GPU_shader_get_builtin_shader(GPUBuiltinShader shader)
 		                                        datatoc_gpu_shader_flat_color_frag_glsl },
 		[GPU_SHADER_EDGES_OVERLAY_SIMPLE] = { datatoc_gpu_shader_3D_vert_glsl, datatoc_gpu_shader_edges_overlay_frag_glsl,
 		                                      datatoc_gpu_shader_edges_overlay_simple_geom_glsl },
-		[GPU_SHADER_EDGES_OVERLAY_EDIT] = { datatoc_gpu_shader_edit_overlay_vert_glsl,
-		                                    datatoc_gpu_shader_edit_overlay_frag_glsl,
-		                                    datatoc_gpu_shader_edit_overlay_geom_glsl },
+		[GPU_SHADER_EDGES_OVERLAY_EDIT_TRI] = { datatoc_gpu_shader_edit_overlay_vert_glsl,
+		                                        datatoc_gpu_shader_edit_overlay_frag_glsl,
+		                                        datatoc_gpu_shader_edit_overlay_geom_tri_glsl },
+		[GPU_SHADER_EDGES_OVERLAY_EDIT_EDGE] = { datatoc_gpu_shader_edit_overlay_vert_glsl,
+		                                         datatoc_gpu_shader_edit_overlay_frag_glsl,
+		                                         datatoc_gpu_shader_edit_overlay_geom_edge_glsl },
+		[GPU_SHADER_EDGES_OVERLAY_EDIT_VERT] = { datatoc_gpu_shader_edit_overlay_vert_glsl,
+		                                         datatoc_gpu_shader_edit_overlay_frag_glsl,
+		                                         datatoc_gpu_shader_edit_overlay_geom_vert_glsl },
 		[GPU_SHADER_EDGES_OVERLAY] = { datatoc_gpu_shader_edges_overlay_vert_glsl,
 		                               datatoc_gpu_shader_edges_overlay_frag_glsl,
 		                               datatoc_gpu_shader_edges_overlay_geom_glsl },
diff --git a/source/blender/gpu/shaders/gpu_shader_edit_overlay_geom_edge.glsl b/source/blender/gpu/shaders/gpu_shader_edit_overlay_geom_edge.glsl
new file mode 100644
index 00000000000..834d90be637
--- /dev/null
+++ b/source/blender/gpu/shaders/gpu_shader_edit_overlay_geom_edge.glsl
@@ -0,0 +1,147 @@
+
+/* Solid Wirefram implementation
+ * Mike Erwin, Clément Foucault */
+
+/* This shader follows the principles of
+ * http://developer.download.nvidia.com/SDK/10/direct3d/Source/SolidWireframe/Doc/SolidWireframe.pdf */
+
+layout(lines) in;
+layout(triangle_strip, max_vertices=6) out;
+
+const float fixupSize = 9.5; /* in pixels */
+
+uniform mat4 ProjectionMatrix;
+uniform vec2 viewportSize;
+
+in vec4 vPos[];
+in vec4 pPos[];
+in ivec4 vData[];
+
+/* these are the same for all vertices
+ * and does not need interpolation */
+flat out vec3 edgesCrease;
+flat out vec3 edgesSharp;
+flat out ivec3 flag;
+flat out vec4 faceColor;
+flat out int clipCase;
+#ifdef VERTEX_SELECTION
+smooth out vec3 vertexColor;
+#endif
+
+/* See fragment shader */
+noperspective out vec4 eData1;
+flat out vec4 eData2;
+
+#define VERTEX_ACTIVE   (1 << 0)
+#define VERTEX_SELECTED (1 << 1)
+
+#define FACE_ACTIVE     (1 << 2)
+#define FACE_SELECTED   (1 << 3)
+
+/* Table 1. Triangle Projection Cases */
+const ivec4 clipPointsIdx[6] = ivec4[6](
+	ivec4(0, 1, 2, 2),
+	ivec4(0, 2, 1, 1),
+	ivec4(0, 0, 1, 2),
+	ivec4(1, 2, 0, 0),
+	ivec4(1, 1, 0, 2),
+	ivec4(2, 2, 0, 1)
+);
+
+/* project to screen space */
+vec2 proj(vec4 pos)
+{
+	return (0.5 * (pos.xy / pos.w) + 0.5) * viewportSize;
+}
+
+float dist(vec2 pos[3], vec2 vpos, int v)
+{
+	/* endpoints of opposite edge */
+	vec2 e1 = pos[(v + 1) % 3];
+	vec2 e2 = pos[(v + 2) % 3];
+	/* Edge normalized vector */
+	vec2 dir = normalize(e2 - e1);
+	/* perpendicular to dir */
+	vec2 orthogonal = vec2(-dir.y, dir.x);
+
+	return abs(dot(vpos - e1, orthogonal));
+}
+
+vec3 getVertexColor(int v)
+{
+	if ((vData[v].x & VERTEX_ACTIVE) != 0)
+		return vec3(0.0, 1.0, 0.0);
+	else if ((vData[v].x & VERTEX_SELECTED) != 0)
+		return vec3(1.0, 0.0, 0.0);
+	else
+		return vec3(0.0, 0.0, 0.0);
+}
+
+void doVertex(int v, vec4 pos)
+{
+#ifdef VERTEX_SELECTION
+	vertexColor = getVertexColor(v);
+#endif
+
+	gl_Position = pos;
+
+	EmitVertex();
+}
+
+void main()
+{
+	clipCase = 0;
+
+	/* Face */
+	faceColor = vec4(0.0);
+
+	/* Proj Vertex */
+	vec2 pos[2] = vec2[2](proj(pPos[0]), proj(pPos[1]));
+
+	/* little optimization use a vec4 to vectorize
+	 * following operations */
+	vec4 dirs1, dirs2;
+
+	/* Edge normalized vector */
+	dirs1.xy = normalize(pos[1] - pos[0]);
+
+	/* perpendicular to dir */
+	dirs1.zw = vec2(-dirs1.y, dirs1.x);
+
+	/* Make it view independant */
+	dirs1 *= fixupSize / viewportSize.xyxy;
+
+	dirs2 = dirs1;
+
+	/* Perspective */
+	if (ProjectionMatrix[3][3] == 0.0) {
+		/* vPos[i].z is negative and we don't want
+		 * our fixvec to be flipped */
+		dirs1 *= -vPos[0].z;
+		dirs2 *= -vPos[1].z;
+	}
+
+	/* Edge / Vert data */
+	eData1 = vec4(1e10);
+	eData2.zw = pos[0];
+	eData2.xy = pos[1];
+	flag[0] = (vData[0].x << 8);
+	flag[1] = (vData[1].x << 8);
+	flag[2] = 0;
+
+	doVertex(0, pPos[0] + vec4(-dirs1.xy, 0.0, 0.0));
+	doVertex(0, pPos[0] + vec4( dirs1.zw, 0.0, 0.0));
+	doVertex(0, pPos[0] + vec4(-dirs1.zw, 0.0, 0.0));
+
+	flag[2] = vData[0].y | (vData[0].x << 8);
+	edgesCrease[2] = vData[0].z / 255.0;
+	edgesSharp[2] = vData[0].w / 255.0;
+
+	doVertex(1, pPos[1] + vec4( dirs2.zw, 0.0, 0.0));
+	doVertex(1, pPos[1] + vec4(-dirs2.zw, 0.0, 0.0));
+
+	flag[2] = 0;
+	doVertex(1, pPos[1] + vec4( dirs2.xy, 0.0, 0.0));
+
+	EndPrimitive();
+}
diff --git a/source/blender/gpu/shaders/gpu_shader_edit_overlay_geom.glsl b/source/blender/gpu/shaders/gpu_shader_edit_overlay_geom_tri.glsl
index 8bd46d8810c..f3966b40a6b 100644
--- a/source/blender/gpu/shaders/gpu_shader_edit_overlay_geom.glsl
+++ b/source/blender/gpu/shaders/gpu_shader_edit_overlay_geom_tri.glsl
@@ -5,8 +5,7 @@
 /* This shader follows the principles of
  * http://developer.download.nvidia.com/SDK/10/direct3d/Source/SolidWireframe/Doc/SolidWireframe.pdf */
 
-#define EDGE_FIX
-
+//#define EDGE_FIX
 layout(triangles) in;
 
 #ifdef EDGE_FIX
@@ -39,9 +38,9 @@ flat out vec3 edgesSharp;
 flat out ivec3 flag;
 flat out vec4 faceColor;
 flat out int clipCase;
-// #ifdef VERTEX_SELECTION
+#ifdef VERTEX_SELECTION
 smooth out vec3 vertexColor;
-// #endif
+#endif
 
 /* See fragment shader */
 noperspective out vec4 eData1;
@@ -50,10 +49,9 @@ flat out vec4 eData2;
 
 #define VERTEX_ACTIVE   (1 << 0)
 #define VERTEX_SELECTED (1 << 1)
-#define VERTEX_LOOSE    (1 << 2)
 
-#define FACE_ACTIVE     (1 << 3)
-#define FACE_SELECTED   (1 << 4)
+#define FACE_ACTIVE     (1 << 2)
+#define FACE_SELECTED   (1 << 3)
 
 /* Table 1. Triangle Projection Cases */
 const ivec4 clipPointsIdx[6] = ivec4[6](
@@ -113,19 +111,6 @@ void doVertex(int v, vec4 pos)
 	EmitVertex();
 }
 
-void doLooseVertex(int v, vec4 pos, vec2 fixvec)
-{
-	doVertex(v, pos + vec4( fixvec.x,  fixvec.y, 0.0, 0.0));
-
-	/* Quad */
-	doVertex(v, pos + vec4( fixvec.x,  fixvec.y, 0.0, 0.0));
-	doVertex(v, pos + vec4(-fixvec.x,  fixvec.y, 0.0, 0.0));
-	doVertex(v, pos + vec4( fixvec.x, -fixvec.y, 0.0, 0.0));
-	doVertex(v, pos + vec4(-fixvec.x, -fixvec.y, 0.0, 0.0));
-
-	doVertex(v, pos + vec4(-fixvec.x, -fixvec.y, 0.0, 0.0));
-}
-
 void main()
 {
 	/* First we detect which case we are in */
@@ -165,38 +150,9 @@ void main()
 	/* Vertex */
 	vec2 pos[3] = vec2[3](proj(pPos[0]), proj(pPos[1]), proj(pPos[2]));
 
-	/* Loose Vertex : emit quads linked by degenerate triangles */
-	if ((vData[0].x & VERTEX_LOOSE) != 0) {
-		vec2 fixvec[3];
-
-		/* there is no face */
-		faceColor = vec4(0.0);
-
-		/* and don't forget to overide clipCase */
-		clipCase = 0;
 
-		/* only verterx position 0 is used */
-		eData1 = eData2 = vec4(1e10);
-
-		vec2 dir = vec2(1.0) * fixupSize;
-		/* Make it view independant */
-		dir /= viewportSize;
-
-		for (int v = 0; v < 3; ++v) {
-			fixvec[v] = dir;
-			if (ProjectionMatrix[3][3] == 0.0) {
-				fixvec[v] *= -vPos[v].z;
-			}
-		}
-
-		for (int v = 0; v < 3; ++v) {
-			eData2.zw = pos[v];
-			flag[0] = (vData[v].x << 8);
-			doLooseVertex(v, pPos[v], fixvec[v]);
-		}
-	}
 	/* Simple case : compute edge distances in geometry shader */
-	else if (clipCase == 0) {
+	if (clipCase == 0) {
 
 		/* Packing screen positions and 2 distances */
 		eData1 = vec4(0.0, 0.0, pos[2]);
diff --git a/source/blender/gpu/shaders/gpu_shader_edit_overlay_geom_vert.glsl b/source/blender/gpu/shaders/gpu_shader_edit_overlay_geom_vert.glsl
new file mode 100644
index 00000000000..a114a29eca4
--- /dev/null
+++ b/source/blender/gpu/shaders/gpu_shader_edit_overlay_geom_vert.glsl
@@ -0,0 +1,72 @@
+
+/* Solid Wirefram implementation
+ * Mike Erwin, Clément Foucault */
+
+/* This shader follows the principles of
+ * http://developer.download.nvidia.com/SDK/10/direct3d/Source/SolidWireframe/Doc/SolidWireframe.pdf */
+
+layout(points) in;
+layout(triangle_strip, max_vertices=4) out;
+
+const float fixupSize = 9.5; /* in pixels */
+
+uniform mat4 ProjectionMatrix;
+uniform vec2 viewportSize;
+
+in vec4 vPos[];
+in vec4 pPos[];
+in ivec4 vData[];
+
+/* these are the same for all vertices
+ * and does not need interpolation */
+flat out ivec3 flag;
+flat out vec4 faceColor;
+flat out int clipCase;
+
+/* See fragment shader */
+noperspective out vec4 eData1;
+flat out vec4 eData2;
+
+/* project to screen space */
+vec2 proj(vec4 pos)
+{
+	return (0.5 * (pos.xy / pos.w) + 0.5) * viewportSize;
+}
+
+void doVertex(vec4 pos)
+{
+	gl_Position = pos;
+	EmitVertex();
+}
+
+void main()
+{
+	clipCase = 0;
+
+	/* there is no face */
+	faceColor = vec4(0.0);
+
+	/* only verterx position 0 is used */
+	eData1 = eData2 = vec4(1e10);
+	flag = ivec3(0);
+
+	vec2 dir = vec2(1.0) * fixupSize;
+	/* Make it view independant */
+	dir /= viewportSize;
+
+	if (ProjectionMatrix[3][3] == 0.0) {
+		dir *= -vPos[0].z;
+	}
+
+	eData2.zw = proj(pPos[0]);
+
+	flag[0] = (vData[0].x << 8);
+
+	/* Quad */
+	doVertex(pPos[0] + vec4( dir.x,  dir.y, 0.0, 0.0));
+	doVertex(pPos[0] + vec4(-dir.x,  dir.y, 0.0, 0.0));
+	doVertex(pPos[0] + vec4( dir.x, -dir.y, 0.0, 0.0));
+	doVertex(pPos[0] + vec4(-dir.x, -dir.y, 0.0, 0.0));
+
+	EndPrimitive();
+}