UV: Edge selection support

This patch adds edge selection support for UV editing (refer T76545).
Developed as a part of GSoC 2021 project - UV Editor Improvements.

Previously, selections in the UV editor always flushed down to vertices
and this caused multiple issues such as T76343, T78757 and T26676.
This patch fixes that by adding edge selection support for all UV
operators and adding support for flushing selections between vertices
and edges. Updating UV select modes is now done using a separate
operator, which also handles select mode flushing and undo for UV
select modes. Drawing edges (in UV edge mode) is also updated to match
the edit-mesh display in the 3D viewport.

Notes on technical changes made with this patch:
* MLOOPUV_EDGESEL flag is restored (was removed in rB9fa29fe7652a).
* Support for flushing selection between vertices and edges.
* Restored the BMLoopUV.select_edge boolean in the Python API.
* New operator to update UV select modes and flushing.
* UV select mode is now part of editmesh undo.

TODOs added with this patch:
* Edge support for shortest path operator (currently uses vertex path logic).
* Change default theme color instead of reducing contrast with edge-select.
* Proper UV element selections for Reveal Hidden operator.

Reviewed By: campbellbarton

Differential Revision: https://developer.blender.org/D12028

6 files changed, 60 insertions, 10 deletions

diff --git a/source/blender/draw/engines/overlay/overlay_edit_uv.c b/source/blender/draw/engines/overlay/overlay_edit_uv.c
index 8f566970337..93c75bde1e7 100644
--- a/source/blender/draw/engines/overlay/overlay_edit_uv.c
+++ b/source/blender/draw/engines/overlay/overlay_edit_uv.c
@@ -113,6 +113,11 @@ void OVERLAY_edit_uv_init(OVERLAY_Data *vedata)
   const bool do_uv_overlay = is_image_type && is_uv_editor && has_edit_object;
   const bool show_modified_uvs = sima->flag & SI_DRAWSHADOW;
   const bool is_tiled_image = image && (image->source == IMA_SRC_TILED);
+  const bool do_edges_only = (ts->uv_flag & UV_SYNC_SELECTION) ?
+                                  /* NOTE: Ignore #SCE_SELECT_EDGE because a single selected edge
+                                   * on the mesh may cause singe UV vertices to be selected. */
+                                  false :
+                                  (ts->uv_selectmode == UV_SELECT_EDGE);
   const bool do_faces = ((sima->flag & SI_NO_DRAWFACES) == 0);
   const bool do_face_dots = (ts->uv_flag & UV_SYNC_SELECTION) ?
                                 (ts->selectmode & SCE_SELECT_FACE) != 0 :
@@ -124,6 +129,7 @@ void OVERLAY_edit_uv_init(OVERLAY_Data *vedata)
                                   (brush->imagepaint_tool == PAINT_TOOL_CLONE) &&
                                   brush->clone.image;
 
+  pd->edit_uv.do_verts = show_overlays && (!do_edges_only);
   pd->edit_uv.do_faces = show_overlays && do_faces && !do_uvstretching_overlay;
   pd->edit_uv.do_face_dots = show_overlays && do_faces && do_face_dots;
   pd->edit_uv.do_uv_overlay = show_overlays && do_uv_overlay;
@@ -183,7 +189,11 @@ void OVERLAY_edit_uv_cache_init(OVERLAY_Data *vedata)
       DRW_PASS_CREATE(psl->edit_uv_edges_ps,
                       DRW_STATE_WRITE_COLOR | DRW_STATE_WRITE_DEPTH | DRW_STATE_DEPTH_LESS_EQUAL |
                           DRW_STATE_BLEND_ALPHA);
-      GPUShader *sh = OVERLAY_shader_edit_uv_edges_get();
+      const bool do_edges_only = (ts->uv_flag & UV_SYNC_SELECTION) ?
+                                     false :
+                                     (ts->uv_selectmode & UV_SELECT_EDGE);
+      GPUShader *sh = do_edges_only ? OVERLAY_shader_edit_uv_edges_for_edge_select_get() :
+                                      OVERLAY_shader_edit_uv_edges_get();
       if (pd->edit_uv.do_uv_shadow_overlay) {
         pd->edit_uv_shadow_edges_grp = DRW_shgroup_create(sh, psl->edit_uv_edges_ps);
         DRW_shgroup_uniform_block(pd->edit_uv_shadow_edges_grp, "globalsBlock", G_draw.block_ubo);
@@ -211,11 +221,14 @@ void OVERLAY_edit_uv_cache_init(OVERLAY_Data *vedata)
   }
 
   if (pd->edit_uv.do_uv_overlay) {
-    /* uv verts */
-    {
+    if (pd->edit_uv.do_verts || pd->edit_uv.do_face_dots) {
       DRW_PASS_CREATE(psl->edit_uv_verts_ps,
                       DRW_STATE_WRITE_COLOR | DRW_STATE_WRITE_DEPTH | DRW_STATE_DEPTH_LESS_EQUAL |
                           DRW_STATE_BLEND_ALPHA);
+    }
+
+    /* uv verts */
+    if (pd->edit_uv.do_verts) {
       GPUShader *sh = OVERLAY_shader_edit_uv_verts_get();
       pd->edit_uv_verts_grp = DRW_shgroup_create(sh, psl->edit_uv_verts_ps);
 
@@ -430,9 +443,11 @@ static void overlay_edit_uv_cache_populate(OVERLAY_Data *vedata, Object *ob)
       if (geom) {
         DRW_shgroup_call_obmat(pd->edit_uv_edges_grp, geom, NULL);
       }
-      geom = DRW_mesh_batch_cache_get_edituv_verts(ob, ob->data);
-      if (geom) {
-        DRW_shgroup_call_obmat(pd->edit_uv_verts_grp, geom, NULL);
+      if (pd->edit_uv.do_verts) {
+        geom = DRW_mesh_batch_cache_get_edituv_verts(ob, ob->data);
+        if (geom) {
+          DRW_shgroup_call_obmat(pd->edit_uv_verts_grp, geom, NULL);
+        }
       }
       if (pd->edit_uv.do_faces) {
         geom = DRW_mesh_batch_cache_get_edituv_faces(ob, ob->data);
diff --git a/source/blender/draw/engines/overlay/overlay_private.h b/source/blender/draw/engines/overlay/overlay_private.h
index 3cc70527659..94eccbcb1f3 100644
--- a/source/blender/draw/engines/overlay/overlay_private.h
+++ b/source/blender/draw/engines/overlay/overlay_private.h
@@ -359,6 +359,7 @@ typedef struct OVERLAY_PrivateData {
     bool do_stencil_overlay;
     bool do_mask_overlay;
 
+    bool do_verts;
     bool do_faces;
     bool do_face_dots;
 
@@ -699,6 +700,7 @@ GPUShader *OVERLAY_shader_edit_mesh_vert(void);
 GPUShader *OVERLAY_shader_edit_particle_strand(void);
 GPUShader *OVERLAY_shader_edit_particle_point(void);
 GPUShader *OVERLAY_shader_edit_uv_edges_get(void);
+GPUShader *OVERLAY_shader_edit_uv_edges_for_edge_select_get(void);
 GPUShader *OVERLAY_shader_edit_uv_face_get(void);
 GPUShader *OVERLAY_shader_edit_uv_face_dots_get(void);
 GPUShader *OVERLAY_shader_edit_uv_verts_get(void);
diff --git a/source/blender/draw/engines/overlay/overlay_shader.c b/source/blender/draw/engines/overlay/overlay_shader.c
index da2c2cff2f6..639b20feae4 100644
--- a/source/blender/draw/engines/overlay/overlay_shader.c
+++ b/source/blender/draw/engines/overlay/overlay_shader.c
@@ -167,6 +167,7 @@ typedef struct OVERLAY_Shaders {
   GPUShader *edit_uv_verts;
   GPUShader *edit_uv_faces;
   GPUShader *edit_uv_edges;
+  GPUShader *edit_uv_edges_for_edge_select;
   GPUShader *edit_uv_face_dots;
   GPUShader *edit_uv_stretching_angle;
   GPUShader *edit_uv_stretching_area;
@@ -1605,6 +1606,20 @@ GPUShader *OVERLAY_shader_edit_uv_edges_get(void)
   return sh_data->edit_uv_edges;
 }
 
+GPUShader *OVERLAY_shader_edit_uv_edges_for_edge_select_get(void)
+{
+  OVERLAY_Shaders *sh_data = &e_data.sh_data[0];
+  if (!sh_data->edit_uv_edges_for_edge_select) {
+    sh_data->edit_uv_edges_for_edge_select = DRW_shader_create_with_shaderlib(
+        datatoc_edit_uv_edges_vert_glsl,
+        datatoc_edit_uv_edges_geom_glsl,
+        datatoc_edit_uv_edges_frag_glsl,
+        e_data.lib,
+        "#define USE_EDGE_SELECT\n");
+  }
+  return sh_data->edit_uv_edges_for_edge_select;
+}
+
 GPUShader *OVERLAY_shader_edit_uv_face_get(void)
 {
   OVERLAY_Shaders *sh_data = &e_data.sh_data[0];
diff --git a/source/blender/draw/engines/overlay/shaders/edit_uv_edges_frag.glsl b/source/blender/draw/engines/overlay/shaders/edit_uv_edges_frag.glsl
index 8f90c1acbbb..b6793e5ab28 100644
--- a/source/blender/draw/engines/overlay/shaders/edit_uv_edges_frag.glsl
+++ b/source/blender/draw/engines/overlay/shaders/edit_uv_edges_frag.glsl
@@ -35,7 +35,13 @@ void main()
   float line_distance = distance(stipplePos_f, stippleStart_f) / max(dd.x, dd.y);
 
   if (lineStyle == OVERLAY_UV_LINE_STYLE_OUTLINE) {
+#ifdef USE_EDGE_SELECT
+    /* TODO(@campbellbarton): The current wire-edit color contrast enough against the selection.
+     * Look into changing the default theme color instead of reducing contrast with edge-select. */
+    inner_color = (selectionFac_f != 0.0) ? colorEdgeSelect : (colorWireEdit * 0.5);
+#else
     inner_color = mix(colorWireEdit, colorEdgeSelect, selectionFac_f);
+#endif
     outer_color = vec4(vec3(0.0), 1.0);
   }
   else if (lineStyle == OVERLAY_UV_LINE_STYLE_DASH) {
diff --git a/source/blender/draw/engines/overlay/shaders/edit_uv_edges_geom.glsl b/source/blender/draw/engines/overlay/shaders/edit_uv_edges_geom.glsl
index 4f8d553a220..089cfdaa710 100644
--- a/source/blender/draw/engines/overlay/shaders/edit_uv_edges_geom.glsl
+++ b/source/blender/draw/engines/overlay/shaders/edit_uv_edges_geom.glsl
@@ -53,11 +53,19 @@ void main()
   vec2 line_dir = normalize(line);
   vec2 line_perp = vec2(-line_dir.y, line_dir.x);
   vec2 edge_ofs = line_perp * sizeViewportInv * ceil(half_size);
+#ifdef USE_EDGE_SELECT
+  /* No blending with edge selection. */
+  float selectFac0 = selectionFac[0];
+  float selectFac1 = selectionFac[0];
+#else
+  float selectFac0 = selectionFac[0];
+  float selectFac1 = selectionFac[1];
+#endif
 
-  do_vertex(pos0, selectionFac[0], stippleStart[0], stipplePos[0], half_size, edge_ofs.xy);
-  do_vertex(pos0, selectionFac[0], stippleStart[0], stipplePos[0], -half_size, -edge_ofs.xy);
-  do_vertex(pos1, selectionFac[1], stippleStart[1], stipplePos[1], half_size, edge_ofs.xy);
-  do_vertex(pos1, selectionFac[1], stippleStart[1], stipplePos[1], -half_size, -edge_ofs.xy);
+  do_vertex(pos0, selectFac0, stippleStart[0], stipplePos[0], half_size, edge_ofs.xy);
+  do_vertex(pos0, selectFac0, stippleStart[0], stipplePos[0], -half_size, -edge_ofs.xy);
+  do_vertex(pos1, selectFac1, stippleStart[1], stipplePos[1], half_size, edge_ofs.xy);
+  do_vertex(pos1, selectFac1, stippleStart[1], stipplePos[1], -half_size, -edge_ofs.xy);
 
   EndPrimitive();
 }
diff --git a/source/blender/draw/engines/overlay/shaders/edit_uv_edges_vert.glsl b/source/blender/draw/engines/overlay/shaders/edit_uv_edges_vert.glsl
index 7627a287a05..a291b93603c 100644
--- a/source/blender/draw/engines/overlay/shaders/edit_uv_edges_vert.glsl
+++ b/source/blender/draw/engines/overlay/shaders/edit_uv_edges_vert.glsl
@@ -19,7 +19,11 @@ void main()
   gl_Position.xy = floor(gl_Position.xy * half_viewport_res) / half_viewport_res +
                    half_pixel_offset;
 
+#ifdef USE_EDGE_SELECT
+  bool is_select = (flag & EDGE_UV_SELECT) != 0;
+#else
   bool is_select = (flag & VERT_UV_SELECT) != 0;
+#endif
   selectionFac = is_select ? 1.0 : 0.0;
   /* Move selected edges to the top
    * Vertices are between 0.0 and 0.2, Edges between 0.2 and 0.4