View 3D: move picking arguments into a struct & minor refactor

- Add SelectPick_Params struct to make picking logic more
  straightforward and easier to extend.

- Use `eSelectOp` instead of booleans (extend, deselect, toggle)
  which were used to represent 4 states (which wasn't obvious).

- Handle deselect_all when pocking instead of view3d_select_exec,
  de-duplicate de-selection which was already needed in when replacing
  the selection in picking functions.

- Handle outliner update & notifiers in the picking functions
  instead of view3d_select_exec.

- Fix particle select deselect_all option which did nothing.

4 files changed, 176 insertions, 108 deletions

diff --git a/source/blender/editors/mesh/editface.c b/source/blender/editors/mesh/editface.c
index 8a1e12c76c5..9832f7ba9cf 100644
--- a/source/blender/editors/mesh/editface.c
+++ b/source/blender/editors/mesh/editface.c
@@ -366,59 +366,71 @@ bool paintface_minmax(Object *ob, float r_min[3], float r_max[3])
   return ok;
 }
 
-bool paintface_mouse_select(
-    struct bContext *C, Object *ob, const int mval[2], bool extend, bool deselect, bool toggle)
+bool paintface_mouse_select(struct bContext *C,
+                            const int mval[2],
+                            const struct SelectPick_Params *params,
+                            Object *ob)
 {
   Mesh *me;
-  MPoly *mpoly_sel;
+  MPoly *mpoly_sel = NULL;
   uint index;
+  bool changed = false;
+  bool found = false;
 
   /* Get the face under the cursor */
   me = BKE_mesh_from_object(ob);
 
-  if (!ED_mesh_pick_face(C, ob, mval, ED_MESH_PICK_DEFAULT_FACE_DIST, &index)) {
-    return false;
-  }
-
-  if (index >= me->totpoly) {
-    return false;
-  }
-
-  mpoly_sel = me->mpoly + index;
-  if (mpoly_sel->flag & ME_HIDE) {
-    return false;
+  if (ED_mesh_pick_face(C, ob, mval, ED_MESH_PICK_DEFAULT_FACE_DIST, &index)) {
+    if (index < me->totpoly) {
+      mpoly_sel = me->mpoly + index;
+      if ((mpoly_sel->flag & ME_HIDE) == 0) {
+        found = true;
+      }
+    }
   }
 
-  /* clear flags */
-  if (!extend && !deselect && !toggle) {
-    paintface_deselect_all_visible(C, ob, SEL_DESELECT, false);
+  if ((params->sel_op == SEL_OP_SET) && (found || params->deselect_all)) {
+    changed |= paintface_deselect_all_visible(C, ob, SEL_DESELECT, false);
   }
 
-  me->act_face = (int)index;
+  if (found) {
+    me->act_face = (int)index;
 
-  if (extend) {
-    mpoly_sel->flag |= ME_FACE_SEL;
-  }
-  else if (deselect) {
-    mpoly_sel->flag &= ~ME_FACE_SEL;
-  }
-  else if (toggle) {
-    if (mpoly_sel->flag & ME_FACE_SEL) {
-      mpoly_sel->flag &= ~ME_FACE_SEL;
-    }
-    else {
-      mpoly_sel->flag |= ME_FACE_SEL;
+    switch (params->sel_op) {
+      case SEL_OP_ADD: {
+        mpoly_sel->flag |= ME_FACE_SEL;
+        break;
+      }
+      case SEL_OP_SUB: {
+        mpoly_sel->flag &= ~ME_FACE_SEL;
+        break;
+      }
+      case SEL_OP_XOR: {
+        if (mpoly_sel->flag & ME_FACE_SEL) {
+          mpoly_sel->flag &= ~ME_FACE_SEL;
+        }
+        else {
+          mpoly_sel->flag |= ME_FACE_SEL;
+        }
+        break;
+      }
+      case SEL_OP_SET: {
+        mpoly_sel->flag |= ME_FACE_SEL;
+        break;
+      }
+      case SEL_OP_AND: {
+        BLI_assert_unreachable(); /* Doesn't make sense for picking. */
+        break;
+      }
     }
-  }
-  else {
-    mpoly_sel->flag |= ME_FACE_SEL;
-  }
 
-  /* image window redraw */
+    /* image window redraw */
 
-  paintface_flush_flags(C, ob, SELECT);
-  ED_region_tag_redraw(CTX_wm_region(C)); /* XXX: should redraw all 3D views. */
-  return true;
+    paintface_flush_flags(C, ob, SELECT);
+    ED_region_tag_redraw(CTX_wm_region(C)); /* XXX: should redraw all 3D views. */
+    changed = true;
+  }
+  return changed || found;
 }
 
 void paintvert_flush_flags(Object *ob)
diff --git a/source/blender/editors/mesh/editmesh_path.c b/source/blender/editors/mesh/editmesh_path.c
index dbda9fa7746..f2e7150e791 100644
--- a/source/blender/editors/mesh/editmesh_path.c
+++ b/source/blender/editors/mesh/editmesh_path.c
@@ -28,6 +28,7 @@
 #include "ED_mesh.h"
 #include "ED_object.h"
 #include "ED_screen.h"
+#include "ED_select_utils.h"
 #include "ED_uvedit.h"
 #include "ED_view3d.h"
 
@@ -700,7 +701,10 @@ static int edbm_shortest_path_pick_invoke(bContext *C, wmOperator *op, const wmE
     /* TODO(dfelinto): right now we try to find the closest element twice.
      * The ideal is to refactor EDBM_select_pick so it doesn't
      * have to pick the nearest vert/edge/face again. */
-    EDBM_select_pick(C, event->mval, true, false, false);
+    const struct SelectPick_Params params = {
+        .sel_op = SEL_OP_ADD,
+    };
+    EDBM_select_pick(C, event->mval, &params);
     return OPERATOR_FINISHED;
   }
 
diff --git a/source/blender/editors/mesh/editmesh_select.c b/source/blender/editors/mesh/editmesh_select.c
index 8784b1a90d9..4d73a0460b2 100644
--- a/source/blender/editors/mesh/editmesh_select.c
+++ b/source/blender/editors/mesh/editmesh_select.c
@@ -2016,7 +2016,7 @@ void MESH_OT_select_interior_faces(wmOperatorType *ot)
  * Gets called via generic mouse select operator.
  * \{ */
 
-bool EDBM_select_pick(bContext *C, const int mval[2], bool extend, bool deselect, bool toggle)
+bool EDBM_select_pick(bContext *C, const int mval[2], const struct SelectPick_Params *params)
 {
   ViewContext vc;
 
@@ -2033,100 +2033,148 @@ bool EDBM_select_pick(bContext *C, const int mval[2], bool extend, bool deselect
   uint bases_len = 0;
   Base **bases = BKE_view_layer_array_from_bases_in_edit_mode(vc.view_layer, vc.v3d, &bases_len);
 
-  bool ok = false;
+  bool changed = false;
+  const bool found = unified_findnearest(
+      &vc, bases, bases_len, &base_index_active, &eve, &eed, &efa);
 
-  if (unified_findnearest(&vc, bases, bases_len, &base_index_active, &eve, &eed, &efa)) {
+  if ((params->sel_op == SEL_OP_SET) && (found || params->deselect_all)) {
+    /* Deselect everything. */
+    for (uint base_index = 0; base_index < bases_len; base_index++) {
+      Base *base_iter = bases[base_index];
+      Object *ob_iter = base_iter->object;
+      EDBM_flag_disable_all(BKE_editmesh_from_object(ob_iter), BM_ELEM_SELECT);
+      DEG_id_tag_update(ob_iter->data, ID_RECALC_SELECT);
+      WM_event_add_notifier(C, NC_GEOM | ND_SELECT, ob_iter->data);
+    }
+    changed = true;
+  }
+
+  if (found) {
     Base *basact = bases[base_index_active];
     ED_view3d_viewcontext_init_object(&vc, basact->object);
 
-    /* Deselect everything */
-    if (extend == false && deselect == false && toggle == false) {
-      for (uint base_index = 0; base_index < bases_len; base_index++) {
-        Base *base_iter = bases[base_index];
-        Object *ob_iter = base_iter->object;
-        EDBM_flag_disable_all(BKE_editmesh_from_object(ob_iter), BM_ELEM_SELECT);
-        if (basact->object != ob_iter) {
-          DEG_id_tag_update(ob_iter->data, ID_RECALC_SELECT);
-          WM_event_add_notifier(C, NC_GEOM | ND_SELECT, ob_iter->data);
-        }
-      }
-    }
-
     if (efa) {
-      if (extend) {
-        /* set the last selected face */
-        BM_mesh_active_face_set(vc.em->bm, efa);
-
-        /* Work-around: deselect first, so we can guarantee it will */
-        /* be active even if it was already selected */
-        BM_select_history_remove(vc.em->bm, efa);
-        BM_face_select_set(vc.em->bm, efa, false);
-        BM_select_history_store(vc.em->bm, efa);
-        BM_face_select_set(vc.em->bm, efa, true);
-      }
-      else if (deselect) {
-        BM_select_history_remove(vc.em->bm, efa);
-        BM_face_select_set(vc.em->bm, efa, false);
-      }
-      else {
-        /* set the last selected face */
-        BM_mesh_active_face_set(vc.em->bm, efa);
+      switch (params->sel_op) {
+        case SEL_OP_ADD: {
+          BM_mesh_active_face_set(vc.em->bm, efa);
 
-        if (!BM_elem_flag_test(efa, BM_ELEM_SELECT)) {
+          /* Work-around: deselect first, so we can guarantee it will
+           * be active even if it was already selected. */
+          BM_select_history_remove(vc.em->bm, efa);
+          BM_face_select_set(vc.em->bm, efa, false);
           BM_select_history_store(vc.em->bm, efa);
           BM_face_select_set(vc.em->bm, efa, true);
+          break;
         }
-        else if (toggle) {
+        case SEL_OP_SUB: {
           BM_select_history_remove(vc.em->bm, efa);
           BM_face_select_set(vc.em->bm, efa, false);
+          break;
+        }
+        case SEL_OP_XOR: {
+          BM_mesh_active_face_set(vc.em->bm, efa);
+          if (!BM_elem_flag_test(efa, BM_ELEM_SELECT)) {
+            BM_select_history_store(vc.em->bm, efa);
+            BM_face_select_set(vc.em->bm, efa, true);
+          }
+          else {
+            BM_select_history_remove(vc.em->bm, efa);
+            BM_face_select_set(vc.em->bm, efa, false);
+          }
+          break;
+        }
+        case SEL_OP_SET: {
+          BM_mesh_active_face_set(vc.em->bm, efa);
+          if (!BM_elem_flag_test(efa, BM_ELEM_SELECT)) {
+            BM_select_history_store(vc.em->bm, efa);
+            BM_face_select_set(vc.em->bm, efa, true);
+          }
+          break;
+        }
+        case SEL_OP_AND: {
+          BLI_assert_unreachable(); /* Doesn't make sense for picking. */
+          break;
         }
       }
     }
     else if (eed) {
-      if (extend) {
-        /* Work-around: deselect first, so we can guarantee it will */
-        /* be active even if it was already selected */
-        BM_select_history_remove(vc.em->bm, eed);
-        BM_edge_select_set(vc.em->bm, eed, false);
-        BM_select_history_store(vc.em->bm, eed);
-        BM_edge_select_set(vc.em->bm, eed, true);
-      }
-      else if (deselect) {
-        BM_select_history_remove(vc.em->bm, eed);
-        BM_edge_select_set(vc.em->bm, eed, false);
-      }
-      else {
-        if (!BM_elem_flag_test(eed, BM_ELEM_SELECT)) {
+
+      switch (params->sel_op) {
+        case SEL_OP_ADD: {
+          /* Work-around: deselect first, so we can guarantee it will
+           * be active even if it was already selected. */
+          BM_select_history_remove(vc.em->bm, eed);
+          BM_edge_select_set(vc.em->bm, eed, false);
           BM_select_history_store(vc.em->bm, eed);
           BM_edge_select_set(vc.em->bm, eed, true);
+          break;
         }
-        else if (toggle) {
+        case SEL_OP_SUB: {
           BM_select_history_remove(vc.em->bm, eed);
           BM_edge_select_set(vc.em->bm, eed, false);
+          break;
+        }
+        case SEL_OP_XOR: {
+          if (!BM_elem_flag_test(eed, BM_ELEM_SELECT)) {
+            BM_select_history_store(vc.em->bm, eed);
+            BM_edge_select_set(vc.em->bm, eed, true);
+          }
+          else {
+            BM_select_history_remove(vc.em->bm, eed);
+            BM_edge_select_set(vc.em->bm, eed, false);
+          }
+          break;
+        }
+        case SEL_OP_SET: {
+          if (!BM_elem_flag_test(eve, BM_ELEM_SELECT)) {
+            BM_select_history_store(vc.em->bm, eve);
+            BM_vert_select_set(vc.em->bm, eve, true);
+          }
+          break;
+        }
+        case SEL_OP_AND: {
+          BLI_assert_unreachable(); /* Doesn't make sense for picking. */
+          break;
         }
       }
     }
     else if (eve) {
-      if (extend) {
-        /* Work-around: deselect first, so we can guarantee it will */
-        /* be active even if it was already selected */
-        BM_select_history_remove(vc.em->bm, eve);
-        BM_vert_select_set(vc.em->bm, eve, false);
-        BM_select_history_store(vc.em->bm, eve);
-        BM_vert_select_set(vc.em->bm, eve, true);
-      }
-      else if (deselect) {
-        BM_select_history_remove(vc.em->bm, eve);
-        BM_vert_select_set(vc.em->bm, eve, false);
-      }
-      else {
-        if (!BM_elem_flag_test(eve, BM_ELEM_SELECT)) {
+      switch (params->sel_op) {
+        case SEL_OP_ADD: {
+          /* Work-around: deselect first, so we can guarantee it will
+           * be active even if it was already selected. */
+          BM_select_history_remove(vc.em->bm, eve);
+          BM_vert_select_set(vc.em->bm, eve, false);
           BM_select_history_store(vc.em->bm, eve);
           BM_vert_select_set(vc.em->bm, eve, true);
+          break;
         }
-        else if (toggle) {
+        case SEL_OP_SUB: {
           BM_select_history_remove(vc.em->bm, eve);
           BM_vert_select_set(vc.em->bm, eve, false);
+          break;
+        }
+        case SEL_OP_XOR: {
+          if (!BM_elem_flag_test(eve, BM_ELEM_SELECT)) {
+            BM_select_history_store(vc.em->bm, eve);
+            BM_vert_select_set(vc.em->bm, eve, true);
+          }
+          else {
+            BM_select_history_remove(vc.em->bm, eve);
+            BM_vert_select_set(vc.em->bm, eve, false);
+          }
+          break;
+        }
+        case SEL_OP_SET: {
+          if (!BM_elem_flag_test(eve, BM_ELEM_SELECT)) {
+            BM_select_history_store(vc.em->bm, eve);
+            BM_vert_select_set(vc.em->bm, eve, true);
+          }
+          break;
+        }
+        case SEL_OP_AND: {
+          BLI_assert_unreachable(); /* Doesn't make sense for picking. */
+          break;
         }
       }
     }
@@ -2168,12 +2216,12 @@ bool EDBM_select_pick(bContext *C, const int mval[2], bool extend, bool deselect
     DEG_id_tag_update(vc.obedit->data, ID_RECALC_SELECT);
     WM_event_add_notifier(C, NC_GEOM | ND_SELECT, vc.obedit->data);
 
-    ok = true;
+    changed = true;
   }
 
   MEM_freeN(bases);
 
-  return ok;
+  return changed;
 }
 
 /** \} */
diff --git a/source/blender/editors/mesh/editmesh_tools.c b/source/blender/editors/mesh/editmesh_tools.c
index 54cc3efe986..10d121ffd32 100644
--- a/source/blender/editors/mesh/editmesh_tools.c
+++ b/source/blender/editors/mesh/editmesh_tools.c
@@ -57,6 +57,7 @@
 #include "ED_object.h"
 #include "ED_outliner.h"
 #include "ED_screen.h"
+#include "ED_select_utils.h"
 #include "ED_transform.h"
 #include "ED_uvedit.h"
 #include "ED_view3d.h"
@@ -8540,7 +8541,10 @@ static int edbm_point_normals_modal(bContext *C, wmOperator *op, const wmEvent *
       case EDBM_CLNOR_MODAL_POINTTO_SET_USE_SELECTED:
         new_mode = EDBM_CLNOR_POINTTO_MODE_COORDINATES;
         view3d_operator_needs_opengl(C);
-        if (EDBM_select_pick(C, event->mval, false, false, false)) {
+        const struct SelectPick_Params params = {
+            .sel_op = SEL_OP_SET,
+        };
+        if (EDBM_select_pick(C, event->mval, &params)) {
           /* Point to newly selected active. */
           ED_object_calc_active_center_for_editmode(obedit, false, target);