Edit Mode: Rework display to differentiate selection modes

This removes the large edges and instead use colors to hint in which
selection mode the user is.

The component in each individual selection mode is more prominent to add
more emphasis. The other components are less prominent and dimmed.

A minor default theme changes needed to be made to keep clarity in all
combinations.

Forcing old behavior (no selection mode hit) could be done quite easily
by just making the 2 booleans true (selectFaces and selectEdges).

Reviewers: campbellbarton, billreynish

Reviewed By: campbellbarton, billreynish

Subscribers: ThinkingPolygons

Maniphest Tasks: T1234

Differential Revision: https://developer.blender.org/D4526

5 files changed, 24 insertions, 33 deletions

diff --git a/source/blender/draw/modes/edit_mesh_mode.c b/source/blender/draw/modes/edit_mesh_mode.c
index b3b6acf9b7d..6839c8fcbec 100644
--- a/source/blender/draw/modes/edit_mesh_mode.c
+++ b/source/blender/draw/modes/edit_mesh_mode.c
@@ -336,8 +336,8 @@ static DRWPass *edit_mesh_create_overlay_pass(float *face_alpha,
   DRW_shgroup_uniform_block(grp, "globalsBlock", G_draw.block_ubo);
   DRW_shgroup_uniform_float(grp, "faceAlphaMod", face_alpha, 1);
   DRW_shgroup_uniform_ivec4(grp, "dataMask", data_mask, 1);
-  DRW_shgroup_uniform_bool_copy(grp, "doEdges", do_edges);
   DRW_shgroup_uniform_float_copy(grp, "ofs", 0.0f);
+  DRW_shgroup_uniform_bool_copy(grp, "selectFaces", select_face);
   if (rv3d->rflag & RV3D_CLIPPING) {
     DRW_shgroup_world_clip_planes_from_rv3d(grp, rv3d);
   }
@@ -354,7 +354,8 @@ static DRWPass *edit_mesh_create_overlay_pass(float *face_alpha,
   DRW_shgroup_uniform_ivec4(grp, "dataMask", data_mask, 1);
   DRW_shgroup_uniform_bool_copy(grp, "doEdges", do_edges);
   DRW_shgroup_uniform_float_copy(grp, "ofs", depth_ofs);
-  DRW_shgroup_uniform_float_copy(grp, "edgeScale", show_wide_edge ? 1.75f : 1.0f);
+  DRW_shgroup_uniform_bool_copy(grp, "selectEdges", select_edge);
+
   DRW_shgroup_state_enable(grp, DRW_STATE_OFFSET_NEGATIVE);
   /* To match blender loop structure. */
   DRW_shgroup_state_enable(grp, DRW_STATE_FIRST_VERTEX_CONVENTION);
@@ -422,9 +423,6 @@ static void EDIT_MESH_cache_init(void *vedata)
         stl->g_data->do_faces = false;
         stl->g_data->do_zbufclip = false;
       }
-      if ((tsettings->selectmode & SCE_SELECT_FACE) == 0) {
-        stl->g_data->data_mask[0] &= ~VFLAG_FACE_ACTIVE;
-      }
       if ((v3d->overlay.edit_flag & V3D_OVERLAY_EDIT_SEAMS) == 0) {
         stl->g_data->data_mask[1] &= ~VFLAG_EDGE_SEAM;
       }
@@ -552,6 +550,7 @@ static void EDIT_MESH_cache_init(void *vedata)
 
     /* however we loose the front faces value (because we need the depth of occluded wires and
      * faces are alpha blended ) so we recover them in a new pass. */
+    bool select_face = (tsettings->selectmode & SCE_SELECT_FACE) != 0;
     psl->facefill_occlude = DRW_pass_create(
         "Front Face Color", DRW_STATE_WRITE_COLOR | DRW_STATE_DEPTH_LESS_EQUAL | DRW_STATE_BLEND);
     stl->g_data->facefill_occluded_shgrp = DRW_shgroup_create(sh_data->overlay_facefill,
diff --git a/source/blender/draw/modes/shaders/edit_mesh_overlay_common_lib.glsl b/source/blender/draw/modes/shaders/edit_mesh_overlay_common_lib.glsl
index ffabfd3fcee..50e9c3677f9 100644
--- a/source/blender/draw/modes/shaders/edit_mesh_overlay_common_lib.glsl
+++ b/source/blender/draw/modes/shaders/edit_mesh_overlay_common_lib.glsl
@@ -1,5 +1,7 @@
 
 uniform bool doEdges = true;
+uniform bool selectFaces = true;
+uniform bool selectEdges = true;
 
 vec4 EDIT_MESH_edge_color_outer(int edge_flag, int face_flag, float crease, float bweight)
 {
@@ -15,8 +17,8 @@ vec4 EDIT_MESH_edge_color_outer(int edge_flag, int face_flag, float crease, floa
 vec4 EDIT_MESH_edge_color_inner(int edge_flag)
 {
   vec4 color = colorWireEdit;
-  color = ((edge_flag & EDGE_SELECTED) != 0) ? colorEdgeSelect : color;
-  color = ((edge_flag & EDGE_ACTIVE) != 0) ? colorEditMeshActive : color;
+  color = (doEdges && ((edge_flag & EDGE_SELECTED) != 0)) ? colorEdgeSelect : color;
+  color = (doEdges && ((edge_flag & EDGE_ACTIVE) != 0)) ? colorEditMeshActive : color;
   return color;
 }
 
@@ -43,18 +45,15 @@ vec4 EDIT_MESH_vertex_color(int vertex_flag)
 
 vec4 EDIT_MESH_face_color(int face_flag)
 {
-  if ((face_flag & FACE_ACTIVE) != 0) {
-    return mix(colorFaceSelect, colorEditMeshActive, 0.5);
-  }
-  else if ((face_flag & FACE_SELECTED) != 0) {
-    return colorFaceSelect;
-  }
-  else if ((face_flag & FACE_FREESTYLE) != 0) {
-    return colorFaceFreestyle;
-  }
-  else {
-    return colorFace;
-  }
+  vec4 color = colorFace;
+  color = ((face_flag & FACE_FREESTYLE) != 0) ? colorFaceFreestyle : color;
+  color = ((face_flag & FACE_SELECTED) != 0) ? colorFaceSelect : color;
+  color = ((face_flag & FACE_ACTIVE) != 0) ? mix(colorFaceSelect, colorEditMeshActive, 0.5) :
+                                             color;
+  color.a *= ((face_flag & (FACE_FREESTYLE | FACE_SELECTED | FACE_ACTIVE)) == 0 || selectFaces) ?
+                 1.0 :
+                 0.5;
+  return color;
 }
 
 vec4 EDIT_MESH_facedot_color(float facedot_flag)
diff --git a/source/blender/draw/modes/shaders/edit_mesh_overlay_frag.glsl b/source/blender/draw/modes/shaders/edit_mesh_overlay_frag.glsl
index 3418732afc2..5f1008b2f72 100644
--- a/source/blender/draw/modes/shaders/edit_mesh_overlay_frag.glsl
+++ b/source/blender/draw/modes/shaders/edit_mesh_overlay_frag.glsl
@@ -12,8 +12,6 @@
 #define GRID_LINE_SMOOTH_START (0.5 - DISC_RADIUS)
 #define GRID_LINE_SMOOTH_END (0.5 + DISC_RADIUS)
 
-uniform float edgeScale;
-
 flat in vec4 finalColorOuter_f;
 in vec4 finalColor_f;
 noperspective in float edgeCoord_f;
@@ -22,8 +20,8 @@ out vec4 FragColor;
 
 void main()
 {
-  float dist = abs(edgeCoord_f) - max(sizeEdge * edgeScale - 0.5, 0.0);
-  float dist_outer = dist - max(sizeEdge * edgeScale, 1.0);
+  float dist = abs(edgeCoord_f) - max(sizeEdge - 0.5, 0.0);
+  float dist_outer = dist - max(sizeEdge, 1.0);
 #ifdef USE_SMOOTH_WIRE
   float mix_w = smoothstep(GRID_LINE_SMOOTH_START, GRID_LINE_SMOOTH_END, dist);
   float mix_w_outer = smoothstep(GRID_LINE_SMOOTH_START, GRID_LINE_SMOOTH_END, dist_outer);
@@ -33,4 +31,5 @@ void main()
 #endif
   FragColor = mix(finalColorOuter_f, finalColor_f, 1.0 - mix_w * finalColorOuter_f.a);
   FragColor.a *= 1.0 - (finalColorOuter_f.a > 0.0 ? mix_w_outer : mix_w);
+  FragColor.a *= (selectEdges) ? 1.0 : 0.4;
 }
diff --git a/source/blender/draw/modes/shaders/edit_mesh_overlay_geom.glsl b/source/blender/draw/modes/shaders/edit_mesh_overlay_geom.glsl
index 6e59de12260..f25724d2e91 100644
--- a/source/blender/draw/modes/shaders/edit_mesh_overlay_geom.glsl
+++ b/source/blender/draw/modes/shaders/edit_mesh_overlay_geom.glsl
@@ -4,7 +4,6 @@ layout(triangle_strip, max_vertices = 4) out;
 
 uniform vec2 viewportSize;
 uniform vec2 viewportSizeInv;
-uniform float edgeScale;
 
 in vec4 finalColor[2];
 in vec4 finalColorOuter[2];
@@ -57,9 +56,9 @@ void main()
   line = abs(line) * viewportSize;
 
   finalColorOuter_f = finalColorOuter[0];
-  float half_size = sizeEdge * edgeScale;
+  float half_size = sizeEdge;
   /* Enlarge edge for flag display. */
-  half_size += (finalColorOuter_f.a > 0.0) ? max(sizeEdge * edgeScale, 1.0) : 0.0;
+  half_size += (finalColorOuter_f.a > 0.0) ? max(sizeEdge, 1.0) : 0.0;
 
 #ifdef USE_SMOOTH_WIRE
   /* Add 1 px for AA */
diff --git a/source/blender/draw/modes/shaders/edit_mesh_overlay_vert.glsl b/source/blender/draw/modes/shaders/edit_mesh_overlay_vert.glsl
index 8c54470cd5a..6b93193c9ae 100644
--- a/source/blender/draw/modes/shaders/edit_mesh_overlay_vert.glsl
+++ b/source/blender/draw/modes/shaders/edit_mesh_overlay_vert.glsl
@@ -74,22 +74,17 @@ void main()
 
 #endif
 
-#if !defined(FACE) && !defined(EDGE_DECORATION)
+#if !defined(FACE)
   /* Facing based color blend */
   vec4 vpos = ModelViewMatrix * vec4(pos, 1.0);
   vec3 view_normal = normalize(NormalMatrix * vnor + 1e-4);
   vec3 view_vec = (ProjectionMatrix[3][3] == 0.0) ? normalize(vpos.xyz) : vec3(0.0, 0.0, 1.0);
   float facing = dot(view_vec, view_normal);
-  facing = 1.0 - abs(facing) * 0.3;
+  facing = 1.0 - abs(facing) * 0.2;
 
   finalColor = mix(colorEditMeshMiddle, finalColor, facing);
   finalColor.a = 1.0;
 
-#  if defined(EDGE) && !defined(FLAT)
-  /* Hack to blend color in pixel shader in case of overide.  */
-  finalColor.a = facing;
-#  endif
-
 #endif
 
 #ifdef USE_WORLD_CLIP_PLANES