Edit Mesh: Refactor Edit cage drawing to use old style drawing

This is work in progress. Look is not final.

This align data VBO data structure used for edti cage drawing to the one
use for normal drawing.

We no longer use barycentric coords to draw the lines an just rasterize
line primitives for edge drawing. This is a bit slower than using the
previous fast method but faster than the "correct" (edge artifact free)
method. This also make the code way simpler.

This also makes it possible to reuse possible and normal vbos used for
shading if the edit cage matches the

This also touches the UV batch code to share as much render data as
possible. The code also prepare for edit cage "modified" drawing cage (with
modifier applied) but is not enabled since selection and operators does not
work with modified cage yet.

1 files changed, 67 insertions, 83 deletions

diff --git a/source/blender/draw/modes/shaders/edit_mesh_overlay_vert.glsl b/source/blender/draw/modes/shaders/edit_mesh_overlay_vert.glsl
index 8b9bcb33fae..2c906412ed2 100644
--- a/source/blender/draw/modes/shaders/edit_mesh_overlay_vert.glsl
+++ b/source/blender/draw/modes/shaders/edit_mesh_overlay_vert.glsl
@@ -1,115 +1,99 @@
 
-/* Solid Wirefram implementation
- * Mike Erwin, Clément Foucault */
-
 uniform mat3 NormalMatrix;
 uniform mat4 ProjectionMatrix;
 uniform mat4 ModelViewMatrix;
 uniform mat4 ModelViewProjectionMatrix;
 uniform mat4 ModelMatrix;
+uniform float faceAlphaMod;
 uniform ivec4 dataMask = ivec4(0xFF);
 
+in ivec4 data;
 in vec3 pos;
-#ifdef VERTEX_FACING
+#ifndef FACEDOT
 in vec3 vnor;
+#else
+in vec4 norAndFlag;
+#  define vnor norAndFlag.xyz
 #endif
 
-#ifdef EDGE_FIX
-in ivec4 data;
+#ifdef EDGE
+flat out vec4 finalColorStipple;
+flat out float base_dist;
+out float dist;
+#endif
 
-out vec4 pPos;
-out ivec4 vData;
-#  ifdef VERTEX_FACING
-out float vFacing;
+#ifdef FLAT
+flat out vec4 finalColor;
+#else
+out vec4 finalColor;
+#  ifdef EDGE
+flat out int selectOveride;
 #  endif
+#endif
 
 void main()
 {
-	pPos = ModelViewProjectionMatrix * vec4(pos, 1.0);
-	vData = data & dataMask;
-#  ifdef VERTEX_FACING
-	vec4 vpos = ModelViewMatrix * vec4(pos, 1.0);
-	vec3 view_normal = normalize(NormalMatrix * vnor);
-	vec3 view_vec = (ProjectionMatrix[3][3] == 0.0)
-		? normalize(vpos.xyz)
-		: vec3(0.0, 0.0, 1.0);
-	vFacing = dot(view_vec, view_normal);
-#  endif
+	gl_Position = ModelViewProjectionMatrix * vec4(pos, 1.0);
 
-#  ifdef USE_WORLD_CLIP_PLANES
-	world_clip_planes_calc_clip_distance((ModelMatrix * vec4(pos, 1.0)).xyz);
-#  endif
-}
+	ivec4 m_data = data & dataMask;
 
-#else /* EDGE_FIX */
-
-/* Consecutive data of the nth vertex.
- * Only valid for first vertex in the triangle.
- * Assuming GL_FRIST_VERTEX_CONVENTION. */
-in ivec4 data0;
-in ivec4 data1;
-in ivec4 data2;
-
-flat out vec3 edgesCrease;
-flat out vec3 edgesBweight;
-flat out vec4 faceColor;
-flat out ivec3 flag;
-#  ifdef VERTEX_SELECTION
-out vec3 vertexColor;
-#  endif
-#  ifdef VERTEX_FACING
-out float facing;
+#if defined(VERT)
+	finalColor = EDIT_MESH_vertex_color(m_data.y);
+	gl_PointSize = sizeVertex * 2.0;
+	gl_Position.z -= 3e-5 * ((ProjectionMatrix[3][3] == 0.0) ? 1.0 : 0.0);
+	/* Make selected and active vertex always on top. */
+	if ((data.x & VERT_SELECTED) != 0) {
+		gl_Position.z -= 1e-7;
+	}
+	if ((data.x & VERT_ACTIVE) != 0) {
+		gl_Position.z -= 1e-7;
+	}
+
+#elif defined(EDGE)
+#  ifdef FLAT
+	finalColor = EDIT_MESH_edge_color_inner(m_data.y);
+#  else
+	finalColor = EDIT_MESH_edge_vertex_color(m_data.y);
+	selectOveride = (m_data.y & EDGE_SELECTED);
 #  endif
 
-out vec3 barycentric;
+	float crease = float(m_data.z) / 255.0;
+	float bweight = float(m_data.w) / 255.0;
+	finalColorStipple = EDIT_MESH_edge_color_outer(m_data.y, m_data.x, crease, bweight);
+	base_dist = dist = float(gl_VertexID % 128);
 
-void main()
-{
-	gl_Position = ModelViewProjectionMatrix * vec4(pos, 1.0);
+#elif defined(FACE)
+	finalColor = EDIT_MESH_face_color(m_data.x);
+	finalColor.a *= faceAlphaMod;
 
-	int v_0 = (gl_VertexID / 3) * 3;
-	int vidx = gl_VertexID % 3;
-	barycentric = vec3(equal(ivec3(0, 1, 2), ivec3(vidx)));
-
-	/* Edge */
-	ivec4 vData[3] = ivec4[3](data0, data1, data2);
-	ivec3 eflag;
-	for (int v = 0; v < 3; ++v) {
-		vData[v] = vData[v] & dataMask;
-		flag[v] = eflag[v] = vData[v].y | (vData[v].x << 8);
-		edgesCrease[v] = vData[v].z / 255.0;
-		edgesBweight[v] = vData[v].w / 255.0;
-	}
+#elif defined(FACEDOT)
+	finalColor = EDIT_MESH_facedot_color(norAndFlag.w);
+	/* Bias Facedot Z position in clipspace. */
+	gl_Position.z -= 0.00035;
+	gl_PointSize = sizeFaceDot;
 
-	/* Face */
-	if ((vData[0].x & FACE_ACTIVE) != 0) {
-		faceColor = colorFaceSelect;
-	}
-	else if ((vData[0].x & FACE_SELECTED) != 0) {
-		faceColor = colorFaceSelect;
-	}
-	else if ((vData[0].x & FACE_FREESTYLE) != 0) {
-		faceColor = colorFaceFreestyle;
-	}
-	else {
-		faceColor = colorFace;
-	}
+#endif
 
-#  ifdef VERTEX_SELECTION
-	vertexColor = EDIT_MESH_vertex_color(data0.x).rgb;
-#  endif
-#  ifdef VERTEX_FACING
-	vec4 vPos = ModelViewMatrix * vec4(pos, 1.0);
+#ifndef FACE
+	vec4 vpos = ModelViewMatrix * vec4(pos, 1.0);
 	vec3 view_normal = normalize(NormalMatrix * vnor);
 	vec3 view_vec = (ProjectionMatrix[3][3] == 0.0)
-		? normalize(vPos.xyz)
+		? normalize(vpos.xyz)
 		: vec3(0.0, 0.0, 1.0);
-	facing = dot(view_vec, view_normal);
-#  endif
+	float facing = dot(view_vec, view_normal);
+	facing = 1.0 - abs(facing) * 0.3;
 
-#  ifdef USE_WORLD_CLIP_PLANES
-	world_clip_planes_calc_clip_distance((ModelMatrix * vec4(pos, 1.0)).xyz);
+	finalColor = mix(colorEditMeshMiddle, finalColor, facing);
+	finalColor.a = 1.0;
+
+#  if defined(EDGE) && !defined(FLAT)
+	/* Hack to blend color in pixel shader in case of overide.  */
+	finalColor.a = facing;
 #  endif
-}
 
 #endif
+
+#ifdef USE_WORLD_CLIP_PLANES
+	world_clip_planes_calc_clip_distance((ModelMatrix * vec4(pos, 1.0)).xyz);
+#endif
+}