Edit Mesh: Improve mesh cage drawing / fix errors and cleanup

* Fixes vertices with bad coloring. Now vertices draw with depth write so
they occlude the underlying face geom overlay, avoiding double drawing the
vertex.

* Decrease the z_offset of edges so they don't poke too much through
geometry. Also delete this offset in ortho view.

* Add zoffset to active and selected vertices so they always draw on top
if they overlap a non selected vertex.

* Fix alpha of edge_fix in vertex selection mode

5 files changed, 41 insertions, 40 deletions

diff --git a/source/blender/draw/modes/edit_mesh_mode.c b/source/blender/draw/modes/edit_mesh_mode.c
index 4ab3c1f992a..fabff950c60 100644
--- a/source/blender/draw/modes/edit_mesh_mode.c
+++ b/source/blender/draw/modes/edit_mesh_mode.c
@@ -355,42 +355,45 @@ static DRWPass *edit_mesh_create_overlay_pass(
 	        "Edit Mesh Face Overlay Pass",
 	        DRW_STATE_WRITE_COLOR | DRW_STATE_POINT | statemod);
 
-	*r_face_shgrp = DRW_shgroup_create(tri_sh, pass);
-	DRW_shgroup_uniform_block(*r_face_shgrp, "globalsBlock", globals_ubo);
-	DRW_shgroup_uniform_vec2(*r_face_shgrp, "viewportSize", DRW_viewport_size_get(), 1);
-	DRW_shgroup_uniform_float(*r_face_shgrp, "faceAlphaMod", face_alpha, 1);
-	DRW_shgroup_uniform_float(*r_face_shgrp, "edgeScale", edge_width_scale, 1);
-	DRW_shgroup_uniform_ivec4(*r_face_shgrp, "dataMask", data_mask, 1);
-	DRW_shgroup_uniform_bool_copy(*r_face_shgrp, "doEdges", do_edges);
-	if (!fast_mode) {
-		DRW_shgroup_uniform_bool_copy(*r_face_shgrp, "isXray", xray);
-	}
-
-	*r_ledges_shgrp = DRW_shgroup_create(ledge_sh, pass);
-	DRW_shgroup_uniform_block(*r_ledges_shgrp, "globalsBlock", globals_ubo);
-	DRW_shgroup_uniform_vec2(*r_ledges_shgrp, "viewportSize", DRW_viewport_size_get(), 1);
-	DRW_shgroup_uniform_float(*r_ledges_shgrp, "edgeScale", edge_width_scale, 1);
-	DRW_shgroup_uniform_ivec4(*r_ledges_shgrp, "dataMask", data_mask, 1);
-	DRW_shgroup_uniform_bool_copy(*r_ledges_shgrp, "doEdges", do_edges);
-
 	if ((tsettings->selectmode & SCE_SELECT_VERTEX) != 0) {
 		*r_lverts_shgrp = DRW_shgroup_create(e_data.overlay_lvert_sh, pass);
 		DRW_shgroup_uniform_block(*r_lverts_shgrp, "globalsBlock", globals_ubo);
 		DRW_shgroup_uniform_vec2(*r_lverts_shgrp, "viewportSize", DRW_viewport_size_get(), 1);
 		DRW_shgroup_uniform_float(*r_lverts_shgrp, "edgeScale", edge_width_scale, 1);
+		DRW_shgroup_state_enable(*r_lverts_shgrp, DRW_STATE_WRITE_DEPTH);
 
 		*r_verts_shgrp = DRW_shgroup_create(e_data.overlay_vert_sh, pass);
 		DRW_shgroup_uniform_block(*r_verts_shgrp, "globalsBlock", globals_ubo);
 		DRW_shgroup_uniform_vec2(*r_verts_shgrp, "viewportSize", DRW_viewport_size_get(), 1);
 		DRW_shgroup_uniform_float(*r_verts_shgrp, "edgeScale", edge_width_scale, 1);
+		DRW_shgroup_state_enable(*r_verts_shgrp, DRW_STATE_WRITE_DEPTH);
 	}
 
 	if ((tsettings->selectmode & SCE_SELECT_FACE) != 0) {
 		*r_facedot_shgrp = DRW_shgroup_create(e_data.overlay_facedot_sh, pass);
 		DRW_shgroup_uniform_block(*r_facedot_shgrp, "globalsBlock", globals_ubo);
 		DRW_shgroup_uniform_float(*r_facedot_shgrp, "edgeScale", edge_width_scale, 1);
+		DRW_shgroup_state_enable(*r_facedot_shgrp, DRW_STATE_WRITE_DEPTH);
 	}
 
+	*r_face_shgrp = DRW_shgroup_create(tri_sh, pass);
+	DRW_shgroup_uniform_block(*r_face_shgrp, "globalsBlock", globals_ubo);
+	DRW_shgroup_uniform_vec2(*r_face_shgrp, "viewportSize", DRW_viewport_size_get(), 1);
+	DRW_shgroup_uniform_float(*r_face_shgrp, "faceAlphaMod", face_alpha, 1);
+	DRW_shgroup_uniform_float(*r_face_shgrp, "edgeScale", edge_width_scale, 1);
+	DRW_shgroup_uniform_ivec4(*r_face_shgrp, "dataMask", data_mask, 1);
+	DRW_shgroup_uniform_bool_copy(*r_face_shgrp, "doEdges", do_edges);
+	if (!fast_mode) {
+		DRW_shgroup_uniform_bool_copy(*r_face_shgrp, "isXray", xray);
+	}
+
+	*r_ledges_shgrp = DRW_shgroup_create(ledge_sh, pass);
+	DRW_shgroup_uniform_block(*r_ledges_shgrp, "globalsBlock", globals_ubo);
+	DRW_shgroup_uniform_vec2(*r_ledges_shgrp, "viewportSize", DRW_viewport_size_get(), 1);
+	DRW_shgroup_uniform_float(*r_ledges_shgrp, "edgeScale", edge_width_scale, 1);
+	DRW_shgroup_uniform_ivec4(*r_ledges_shgrp, "dataMask", data_mask, 1);
+	DRW_shgroup_uniform_bool_copy(*r_ledges_shgrp, "doEdges", do_edges);
+
 	return pass;
 }
 
diff --git a/source/blender/draw/modes/shaders/edit_mesh_overlay_frag.glsl b/source/blender/draw/modes/shaders/edit_mesh_overlay_frag.glsl
index 7fa822bc697..8d7a653c2fe 100644
--- a/source/blender/draw/modes/shaders/edit_mesh_overlay_frag.glsl
+++ b/source/blender/draw/modes/shaders/edit_mesh_overlay_frag.glsl
@@ -120,35 +120,24 @@ void main()
 
 #ifdef VERTEX_SELECTION
 			vec4 inner_edge_color = vec4(vertexColor, 1.0);
-#  ifdef EDGE_FIX
-			inner_edge_color *= isXray ? 1.0 : EDGE_FIX_ALPHA;
-#  endif
-			colorDistEdge(vec4(vertexColor, 1.0), innerEdge);
 #else
 			vec4 inner_edge_color = EDIT_MESH_edge_color_inner(flag[v], (flag[0] & FACE_ACTIVE_) != 0);
-#  ifdef EDGE_FIX
+#endif
+#ifdef EDGE_FIX
 			inner_edge_color *= isXray ? 1.0 : EDGE_FIX_ALPHA;
-#  endif
-			colorDistEdge(inner_edge_color, innerEdge);
 #endif
+			colorDistEdge(inner_edge_color, innerEdge);
 		}
 	}
 
 #if defined(VERTEX_SELECTION) && defined(EDGE_FIX)
 	/* Points */
 	for (int v = 0; v < 3; ++v) {
-		if ((flag[v] & EDGE_VERTEX_EXISTS) == 0) {
-			/* Leave as-is, no vertex. */
-		}
-		else {
+		if ((flag[v] & EDGE_VERTEX_EXISTS) != 0) {
 			float size = p[v] - sizeVertex;
-
 			vec4 point_color = colorVertex;
 			point_color = ((flag[v] & EDGE_VERTEX_SELECTED) != 0) ? colorVertexSelect : point_color;
 			point_color = ((flag[v] & EDGE_VERTEX_ACTIVE) != 0) ? vec4(colorEditMeshActive.xyz, 1.0) : point_color;
-#  ifdef EDGE_FIX
-			point_color *= isXray ? 1.0 : EDGE_FIX_ALPHA;
-#  endif
 			colorDist(point_color, size);
 		}
 	}
diff --git a/source/blender/draw/modes/shaders/edit_mesh_overlay_geom_tri.glsl b/source/blender/draw/modes/shaders/edit_mesh_overlay_geom_tri.glsl
index e97bc7d1f67..a04494d2ec4 100644
--- a/source/blender/draw/modes/shaders/edit_mesh_overlay_geom_tri.glsl
+++ b/source/blender/draw/modes/shaders/edit_mesh_overlay_geom_tri.glsl
@@ -37,7 +37,7 @@ out float facing;
 #endif
 
 #ifdef ANTI_ALIASING
-#define Z_OFFSET -0.0013
+#define Z_OFFSET -0.0005
 #else
 #define Z_OFFSET 0.0
 #endif
@@ -80,7 +80,8 @@ void doVertexOfs(int v, vec2 fixvec)
 #ifdef VERTEX_FACING
 	facing = v_facing[v];
 #endif
-	gl_Position = pPos[v] + vec4(fixvec * pPos[v].w, Z_OFFSET, 0.0);
+	float z_ofs = Z_OFFSET * ((ProjectionMatrix[3][3] == 0.0) ? 1.0 : 0.0);
+	gl_Position = pPos[v] + vec4(fixvec * pPos[v].w, z_ofs, 0.0);
 
 	EmitVertex();
 }
@@ -91,7 +92,7 @@ void mask_edge_flag(int v, ivec3 eflag)
 
 	/* Only shade the edge that we are currently drawing.
 	 * (fix corner bleeding) */
-	flag = eflag;
+	flag = eflag & ~EDGE_VERTEX_EXISTS;
 	flag[vaf] &= ~EDGE_EXISTS;
 	flag[v]   &= ~EDGE_EXISTS;
 }
diff --git a/source/blender/draw/modes/shaders/edit_mesh_overlay_points_vert.glsl b/source/blender/draw/modes/shaders/edit_mesh_overlay_points_vert.glsl
index 722411571e4..3886213ce49 100644
--- a/source/blender/draw/modes/shaders/edit_mesh_overlay_points_vert.glsl
+++ b/source/blender/draw/modes/shaders/edit_mesh_overlay_points_vert.glsl
@@ -3,7 +3,7 @@ uniform mat3 NormalMatrix;
 uniform mat4 ProjectionMatrix;
 uniform mat4 ModelViewMatrix;
 uniform mat4 ModelViewProjectionMatrix;
-uniform float ofs = 1e-4;
+uniform float ofs = 3e-5;
 
 in vec3 pos;
 in ivec4 data;
@@ -21,7 +21,15 @@ void main()
 
 	gl_PointSize = sizeVertex * 2.0;
 	gl_Position = ModelViewProjectionMatrix * vec4(pos, 1.0);
-	gl_Position.z -= ofs;
+	gl_Position.z -= ofs * ((ProjectionMatrix[3][3] == 0.0) ? 1.0 : 0.0);
+
+	/* Make selected and active vertex always on top. */
+	if ((data.x & VERTEX_SELECTED) != 0) {
+		gl_Position.z -= 1e-7;
+	}
+	if ((data.x & VERTEX_ACTIVE) != 0) {
+		gl_Position.z -= 1e-7;
+	}
 
 #ifdef VERTEX_FACING
 	vec4 vPos = ModelViewMatrix * vec4(pos, 1.0);
diff --git a/source/blender/draw/modes/shaders/edit_mesh_overlay_vert.glsl b/source/blender/draw/modes/shaders/edit_mesh_overlay_vert.glsl
index f1ebb419d60..e4268188e09 100644
--- a/source/blender/draw/modes/shaders/edit_mesh_overlay_vert.glsl
+++ b/source/blender/draw/modes/shaders/edit_mesh_overlay_vert.glsl
@@ -29,7 +29,7 @@ out float vFacing;
 void main()
 {
 	pPos = ModelViewProjectionMatrix * vec4(pos, 1.0);
-	pPos.z -= ofs;
+	pPos.z -= ofs * ((ProjectionMatrix[3][3] == 0.0) ? 1.0 : 0.0);
 	vData = data & dataMask;
 #  ifdef VERTEX_FACING
 	vec4 vpos = ModelViewMatrix * vec4(pos, 1.0);
@@ -59,7 +59,7 @@ out vec3 barycentric;
 void main()
 {
 	gl_Position = ModelViewProjectionMatrix * vec4(pos, 1.0);
-	gl_Position.z -= ofs;
+	gl_Position.z -= ofs * ((ProjectionMatrix[3][3] == 0.0) ? 1.0 : 0.0);
 
 	int v_0 = (gl_VertexID / 3) * 3;
 	int vidx = gl_VertexID % 3;