Transform Snap Refactor: Use enum for cage snap options

This allows the addition of the `SNAP_GEOM_CAGE` option.
Currently unused.

11 files changed, 102 insertions, 78 deletions

diff --git a/source/blender/editors/curve/editcurve.c b/source/blender/editors/curve/editcurve.c
index 767d822aa39..535ccaa06fd 100644
--- a/source/blender/editors/curve/editcurve.c
+++ b/source/blender/editors/curve/editcurve.c
@@ -5576,7 +5576,7 @@ static int add_vertex_invoke(bContext *C, wmOperator *op, const wmEvent *event)
           SCE_SNAP_MODE_FACE,
           &(const struct SnapObjectParams){
               .snap_select = (vc.obedit != NULL) ? SNAP_NOT_ACTIVE : SNAP_ALL,
-              .use_object_edit_cage = false,
+              .edit_mode_type = SNAP_GEOM_FINAL,
           },
           mval,
           NULL,
diff --git a/source/blender/editors/gizmo_library/gizmo_types/move3d_gizmo.c b/source/blender/editors/gizmo_library/gizmo_types/move3d_gizmo.c
index 364444f99ae..68322ed56af 100644
--- a/source/blender/editors/gizmo_library/gizmo_types/move3d_gizmo.c
+++ b/source/blender/editors/gizmo_library/gizmo_types/move3d_gizmo.c
@@ -292,7 +292,7 @@ static int gizmo_move_modal(bContext *C,
               (SCE_SNAP_MODE_VERTEX | SCE_SNAP_MODE_EDGE | SCE_SNAP_MODE_FACE),
               &(const struct SnapObjectParams){
                   .snap_select = SNAP_ALL,
-                  .use_object_edit_cage = true,
+                  .edit_mode_type = SNAP_GEOM_EDIT,
                   .use_occlusion_test = true,
               },
               mval_fl,
diff --git a/source/blender/editors/gizmo_library/gizmo_types/snap3d_gizmo.c b/source/blender/editors/gizmo_library/gizmo_types/snap3d_gizmo.c
index 788022a0f1d..6c274a0fbf9 100644
--- a/source/blender/editors/gizmo_library/gizmo_types/snap3d_gizmo.c
+++ b/source/blender/editors/gizmo_library/gizmo_types/snap3d_gizmo.c
@@ -376,7 +376,7 @@ short ED_gizmotypes_snap_3d_update(wmGizmo *gz,
                                                            snap_elements,
                                                            &(const struct SnapObjectParams){
                                                                .snap_select = SNAP_ALL,
-                                                               .use_object_edit_cage = true,
+                                                               .edit_mode_type = SNAP_GEOM_EDIT,
                                                                .use_occlusion_test = true,
                                                            },
                                                            mval_fl,
diff --git a/source/blender/editors/include/ED_transform_snap_object_context.h b/source/blender/editors/include/ED_transform_snap_object_context.h
index b7174964ef6..a63c1e86a50 100644
--- a/source/blender/editors/include/ED_transform_snap_object_context.h
+++ b/source/blender/editors/include/ED_transform_snap_object_context.h
@@ -39,12 +39,18 @@ struct View3D;
 
 /* ED_transform_snap_object_*** API */
 
-typedef enum eSnapSelect {
+typedef enum {
   SNAP_ALL = 0,
   SNAP_NOT_SELECTED = 1,
   SNAP_NOT_ACTIVE = 2,
 } eSnapSelect;
 
+typedef enum {
+  SNAP_GEOM_FINAL = 0,
+  SNAP_GEOM_CAGE = 1,
+  SNAP_GEOM_EDIT = 2, /* Bmesh for mesh-type. */
+} eSnapEditType;
+
 /** used for storing multiple hits */
 struct SnapObjectHitDepth {
   struct SnapObjectHitDepth *next, *prev;
@@ -64,10 +70,10 @@ struct SnapObjectHitDepth {
 
 /** parameters that define which objects will be used to snap. */
 struct SnapObjectParams {
-  /* special context sensitive handling for the active or selected object */
+  /* Special context sensitive handling for the active or selected object. */
   char snap_select;
-  /* use editmode cage */
-  unsigned int use_object_edit_cage : 1;
+  /* Geometry for snapping in edit mode. */
+  char edit_mode_type;
   /* snap to the closest element, use when using more than one snap type */
   unsigned int use_occlusion_test : 1;
   /* exclude back facing geometry from snapping */
diff --git a/source/blender/editors/mesh/editmesh_utils.c b/source/blender/editors/mesh/editmesh_utils.c
index 2e98f0558f3..19c9909039c 100644
--- a/source/blender/editors/mesh/editmesh_utils.c
+++ b/source/blender/editors/mesh/editmesh_utils.c
@@ -1721,7 +1721,7 @@ void EDBM_project_snap_verts(
                                                     SCE_SNAP_MODE_FACE,
                                                     &(const struct SnapObjectParams){
                                                         .snap_select = SNAP_NOT_ACTIVE,
-                                                        .use_object_edit_cage = false,
+                                                        .edit_mode_type = SNAP_GEOM_FINAL,
                                                         .use_occlusion_test = true,
                                                     },
                                                     mval,
diff --git a/source/blender/editors/space_view3d/view3d_edit.c b/source/blender/editors/space_view3d/view3d_edit.c
index 5f3d71cc190..358373ea4d1 100644
--- a/source/blender/editors/space_view3d/view3d_edit.c
+++ b/source/blender/editors/space_view3d/view3d_edit.c
@@ -5065,7 +5065,7 @@ void ED_view3d_cursor3d_position_rotation(bContext *C,
                                                    SCE_SNAP_MODE_FACE,
                                                    &(const struct SnapObjectParams){
                                                        .snap_select = SNAP_ALL,
-                                                       .use_object_edit_cage = false,
+                                                       .edit_mode_type = SNAP_GEOM_FINAL,
                                                        .use_occlusion_test = true,
                                                    },
                                                    mval_fl,
diff --git a/source/blender/editors/space_view3d/view3d_gizmo_ruler.c b/source/blender/editors/space_view3d/view3d_gizmo_ruler.c
index e202276831c..0d8382d8cdd 100644
--- a/source/blender/editors/space_view3d/view3d_gizmo_ruler.c
+++ b/source/blender/editors/space_view3d/view3d_gizmo_ruler.c
@@ -338,7 +338,7 @@ static bool view3d_ruler_item_mousemove(struct Depsgraph *depsgraph,
                                                   SCE_SNAP_MODE_FACE,
                                                   &(const struct SnapObjectParams){
                                                       .snap_select = SNAP_ALL,
-                                                      .use_object_edit_cage = true,
+                                                      .edit_mode_type = SNAP_GEOM_EDIT,
                                                   },
                                                   mval_fl,
                                                   NULL,
@@ -352,7 +352,7 @@ static bool view3d_ruler_item_mousemove(struct Depsgraph *depsgraph,
                                              depsgraph,
                                              &(const struct SnapObjectParams){
                                                  .snap_select = SNAP_ALL,
-                                                 .use_object_edit_cage = true,
+                                                 .edit_mode_type = SNAP_GEOM_EDIT,
                                              },
                                              ray_start,
                                              ray_normal,
diff --git a/source/blender/editors/space_view3d/view3d_placement.c b/source/blender/editors/space_view3d/view3d_placement.c
index 8172742a756..c7a2c580f36 100644
--- a/source/blender/editors/space_view3d/view3d_placement.c
+++ b/source/blender/editors/space_view3d/view3d_placement.c
@@ -323,7 +323,7 @@ static bool idp_poject_surface_normal(SnapObjectContext *snap_context,
                                                  SCE_SNAP_MODE_FACE,
                                                  &(const struct SnapObjectParams){
                                                      .snap_select = SNAP_ALL,
-                                                     .use_object_edit_cage = true,
+                                                     .edit_mode_type = SNAP_GEOM_EDIT,
                                                  },
                                                  mval_fl,
                                                  NULL,
@@ -941,7 +941,7 @@ static void view3d_interactive_add_calc_plane(bContext *C,
                                                   SCE_SNAP_MODE_FACE,
                                                   &(const struct SnapObjectParams){
                                                       .snap_select = SNAP_ALL,
-                                                      .use_object_edit_cage = true,
+                                                      .edit_mode_type = SNAP_GEOM_EDIT,
                                                   },
                                                   mval_fl,
                                                   NULL,
diff --git a/source/blender/editors/space_view3d/view3d_walk.c b/source/blender/editors/space_view3d/view3d_walk.c
index cbd65e3175d..5007bbf976d 100644
--- a/source/blender/editors/space_view3d/view3d_walk.c
+++ b/source/blender/editors/space_view3d/view3d_walk.c
@@ -407,7 +407,7 @@ static bool walk_floor_distance_get(RegionView3D *rv3d,
       &(const struct SnapObjectParams){
           .snap_select = SNAP_ALL,
           /* Avoid having to convert the edit-mesh to a regular mesh. */
-          .use_object_edit_cage = true,
+          .edit_mode_type = SNAP_GEOM_EDIT,
       },
       ray_start,
       ray_normal,
@@ -982,7 +982,7 @@ static float getVelocityZeroTime(const float gravity, const float velocity)
 
 static int walkApply(bContext *C, WalkInfo *walk, bool is_confirm)
 {
-#define WALK_ROTATE_RELATIVE_FAC 2.2f           /* More is faster, relative to region size. */
+#define WALK_ROTATE_RELATIVE_FAC 2.2f /* More is faster, relative to region size. */
 #define WALK_ROTATE_CONSTANT_FAC DEG2RAD(0.15f) /* More is faster, radians per-pixel. */
 #define WALK_TOP_LIMIT DEG2RADF(85.0f)
 #define WALK_BOTTOM_LIMIT DEG2RADF(-80.0f)
diff --git a/source/blender/editors/transform/transform_snap.c b/source/blender/editors/transform/transform_snap.c
index d0f91802fff..193954fffb6 100644
--- a/source/blender/editors/transform/transform_snap.c
+++ b/source/blender/editors/transform/transform_snap.c
@@ -345,7 +345,7 @@ void applyProject(TransInfo *t)
                 SCE_SNAP_MODE_FACE,
                 &(const struct SnapObjectParams){
                     .snap_select = t->tsnap.modeSelect,
-                    .use_object_edit_cage = (t->flag & T_EDIT) != 0,
+                    .edit_mode_type = (t->flag & T_EDIT) != 0 ? SNAP_GEOM_EDIT : SNAP_GEOM_FINAL,
                     .use_occlusion_test = false,
                     .use_backface_culling = t->tsnap.use_backface_culling,
                 },
@@ -1167,7 +1167,7 @@ short snapObjectsTransform(
       t->settings->snap_mode,
       &(const struct SnapObjectParams){
           .snap_select = t->tsnap.modeSelect,
-          .use_object_edit_cage = (t->flag & T_EDIT) != 0,
+          .edit_mode_type = (t->flag & T_EDIT) != 0 ? SNAP_GEOM_EDIT : SNAP_GEOM_FINAL,
           .use_occlusion_test = t->settings->snap_mode != SCE_SNAP_MODE_FACE,
           .use_backface_culling = t->tsnap.use_backface_culling,
       },
@@ -1201,7 +1201,7 @@ bool peelObjectsTransform(TransInfo *t,
       t->depsgraph,
       &(const struct SnapObjectParams){
           .snap_select = t->tsnap.modeSelect,
-          .use_object_edit_cage = (t->flag & T_EDIT) != 0,
+          .edit_mode_type = (t->flag & T_EDIT) != 0 ? SNAP_GEOM_EDIT : SNAP_GEOM_FINAL,
       },
       mval,
       -1.0f,
diff --git a/source/blender/editors/transform/transform_snap_object.c b/source/blender/editors/transform/transform_snap_object.c
index 58198f21ba2..841c3e86008 100644
--- a/source/blender/editors/transform/transform_snap_object.c
+++ b/source/blender/editors/transform/transform_snap_object.c
@@ -149,6 +149,33 @@ static bool editmesh_eval_final_is_bmesh(const BMEditMesh *em)
   return (em->mesh_eval_final->runtime.wrapper_type == ME_WRAPPER_TYPE_BMESH);
 }
 
+/* Mesh used for snapping.
+ * If NULL the BMesh should be used. */
+static Mesh *mesh_for_snap(Object *ob, eSnapEditType edit_mode_type, bool *r_use_hide)
+{
+  Mesh *me = ob->data;
+  bool use_hide = false;
+  if (BKE_object_is_in_editmode(ob)) {
+    if ((edit_mode_type == SNAP_GEOM_EDIT) || editmesh_eval_final_is_bmesh(me->edit_mesh)) {
+      return NULL;
+    }
+
+    BMEditMesh *em = BKE_editmesh_from_object(ob);
+    if ((edit_mode_type == SNAP_GEOM_FINAL) && em->mesh_eval_final) {
+      me = em->mesh_eval_final;
+      use_hide = true;
+    }
+    else if ((edit_mode_type == SNAP_GEOM_CAGE) && em->mesh_eval_cage) {
+      me = em->mesh_eval_cage;
+      use_hide = true;
+    }
+  }
+  if (r_use_hide) {
+    *r_use_hide = use_hide;
+  }
+  return me;
+}
+
 /** \} */
 
 /* -------------------------------------------------------------------- */
@@ -369,7 +396,7 @@ static SnapObjectData *snap_object_data_editmesh_get(SnapObjectContext *sctx,
 typedef void (*IterSnapObjsCallback)(SnapObjectContext *sctx,
                                      Object *ob,
                                      float obmat[4][4],
-                                     bool use_obedit,
+                                     eSnapEditType edit_mode_type,
                                      bool use_backface_culling,
                                      bool is_object_active,
                                      void *data);
@@ -386,7 +413,7 @@ static void iter_snap_objects(SnapObjectContext *sctx,
   ViewLayer *view_layer = DEG_get_input_view_layer(depsgraph);
   const View3D *v3d = sctx->v3d_data.v3d;
   const eSnapSelect snap_select = params->snap_select;
-  const bool use_object_edit_cage = params->use_object_edit_cage;
+  const eSnapEditType edit_mode_type = params->edit_mode_type;
   const bool use_backface_culling = params->use_backface_culling;
 
   Base *base_act = view_layer->basact;
@@ -422,7 +449,7 @@ static void iter_snap_objects(SnapObjectContext *sctx,
         sob_callback(sctx,
                      dupli_ob->ob,
                      dupli_ob->mat,
-                     use_object_edit_cage,
+                     edit_mode_type,
                      use_backface_culling,
                      is_object_active,
                      data);
@@ -433,7 +460,7 @@ static void iter_snap_objects(SnapObjectContext *sctx,
     sob_callback(sctx,
                  obj_eval,
                  obj_eval->obmat,
-                 use_object_edit_cage,
+                 edit_mode_type,
                  use_backface_culling,
                  is_object_active,
                  data);
@@ -966,7 +993,7 @@ struct RaycastObjUserData {
 static void raycast_obj_fn(SnapObjectContext *sctx,
                            Object *ob,
                            float obmat[4][4],
-                           bool use_obedit,
+                           eSnapEditType edit_mode_type,
                            bool use_backface_culling,
                            bool is_object_active,
                            void *data)
@@ -979,7 +1006,8 @@ static void raycast_obj_fn(SnapObjectContext *sctx,
 
   bool retval = false;
   if (use_occlusion_test) {
-    if (use_obedit && sctx->use_v3d && XRAY_FLAG_ENABLED(sctx->v3d_data.v3d)) {
+    if ((edit_mode_type == SNAP_GEOM_EDIT) && sctx->use_v3d &&
+        XRAY_FLAG_ENABLED(sctx->v3d_data.v3d)) {
       /* Use of occlude geometry in editing mode disabled. */
       return;
     }
@@ -993,33 +1021,25 @@ static void raycast_obj_fn(SnapObjectContext *sctx,
 
   switch (ob->type) {
     case OB_MESH: {
-      Mesh *me = ob->data;
       bool use_hide = false;
-      if (BKE_object_is_in_editmode(ob)) {
-        if (use_obedit || editmesh_eval_final_is_bmesh(me->edit_mesh)) {
-          /* Operators only update the editmesh looptris of the original mesh. */
-          BMEditMesh *em_orig = BKE_editmesh_from_object(DEG_get_original_object(ob));
-          retval = raycastEditMesh(sctx,
-                                   dt->ray_start,
-                                   dt->ray_dir,
-                                   ob,
-                                   em_orig,
-                                   obmat,
-                                   ob_index,
-                                   use_backface_culling,
-                                   ray_depth,
-                                   dt->r_loc,
-                                   dt->r_no,
-                                   dt->r_index,
-                                   dt->r_hit_list);
-          break;
-        }
-
-        BMEditMesh *em = BKE_editmesh_from_object(ob);
-        if (em->mesh_eval_final) {
-          me = em->mesh_eval_final;
-          use_hide = true;
-        }
+      Mesh *me = mesh_for_snap(ob, edit_mode_type, &use_hide);
+      if (me == NULL) {
+        /* Operators only update the editmesh looptris of the original mesh. */
+        BMEditMesh *em_orig = BKE_editmesh_from_object(DEG_get_original_object(ob));
+        retval = raycastEditMesh(sctx,
+                                 dt->ray_start,
+                                 dt->ray_dir,
+                                 ob,
+                                 em_orig,
+                                 obmat,
+                                 ob_index,
+                                 use_backface_culling,
+                                 ray_depth,
+                                 dt->r_loc,
+                                 dt->r_no,
+                                 dt->r_index,
+                                 dt->r_hit_list);
+        break;
       }
       retval = raycastMesh(sctx,
                            dt->ray_start,
@@ -1771,7 +1791,6 @@ static short snap_mesh_edge_verts_mixed(SnapObjectContext *sctx,
 static short snapArmature(SnapData *snapdata,
                           Object *ob,
                           const float obmat[4][4],
-                          bool use_obedit,
                           /* read/write args */
                           float *dist_px,
                           /* return args */
@@ -1792,7 +1811,7 @@ static short snapArmature(SnapData *snapdata,
   dist_squared_to_projected_aabb_precalc(
       &neasrest_precalc, lpmat, snapdata->win_size, snapdata->mval);
 
-  use_obedit = use_obedit && BKE_object_is_in_editmode(ob);
+  bool use_obedit = ((bArmature *)ob->data)->edbo != NULL;
 
   if (use_obedit == false) {
     /* Test BoundBox */
@@ -2663,7 +2682,7 @@ struct SnapObjUserData {
 static void snap_obj_fn(SnapObjectContext *sctx,
                         Object *ob,
                         float obmat[4][4],
-                        bool use_obedit,
+                        eSnapEditType edit_mode_type,
                         bool use_backface_culling,
                         bool UNUSED(is_object_active),
                         void *data)
@@ -2673,28 +2692,21 @@ static void snap_obj_fn(SnapObjectContext *sctx,
 
   switch (ob->type) {
     case OB_MESH: {
-      Mesh *me = ob->data;
-      if (BKE_object_is_in_editmode(ob)) {
-        if (use_obedit || editmesh_eval_final_is_bmesh(me->edit_mesh)) {
-          /* Operators only update the editmesh looptris of the original mesh. */
-          BMEditMesh *em_orig = BKE_editmesh_from_object(DEG_get_original_object(ob));
-          retval = snapEditMesh(sctx,
-                                dt->snapdata,
-                                ob,
-                                em_orig,
-                                obmat,
-                                use_backface_culling,
-                                dt->dist_px,
-                                dt->r_loc,
-                                dt->r_no,
-                                dt->r_index);
-          break;
-        }
-
-        BMEditMesh *em = BKE_editmesh_from_object(ob);
-        if (em->mesh_eval_final) {
-          me = em->mesh_eval_final;
-        }
+      Mesh *me = mesh_for_snap(ob, edit_mode_type, NULL);
+      if (me == NULL) {
+        /* Operators only update the editmesh looptris of the original mesh. */
+        BMEditMesh *em_orig = BKE_editmesh_from_object(DEG_get_original_object(ob));
+        retval = snapEditMesh(sctx,
+                              dt->snapdata,
+                              ob,
+                              em_orig,
+                              obmat,
+                              use_backface_culling,
+                              dt->dist_px,
+                              dt->r_loc,
+                              dt->r_no,
+                              dt->r_index);
+        break;
       }
       else if (ob->dt == OB_BOUNDBOX) {
         /* Do not snap to objects that are in bounding box display mode */
@@ -2715,11 +2727,17 @@ static void snap_obj_fn(SnapObjectContext *sctx,
     }
     case OB_ARMATURE:
       retval = snapArmature(
-          dt->snapdata, ob, obmat, use_obedit, dt->dist_px, dt->r_loc, dt->r_no, dt->r_index);
+          dt->snapdata, ob, obmat, dt->dist_px, dt->r_loc, dt->r_no, dt->r_index);
       break;
     case OB_CURVE:
-      retval = snapCurve(
-          dt->snapdata, ob, obmat, use_obedit, dt->dist_px, dt->r_loc, dt->r_no, dt->r_index);
+      retval = snapCurve(dt->snapdata,
+                         ob,
+                         obmat,
+                         edit_mode_type == SNAP_GEOM_EDIT,
+                         dt->dist_px,
+                         dt->r_loc,
+                         dt->r_no,
+                         dt->r_index);
       break; /* Use ATTR_FALLTHROUGH if we want to snap to the generated mesh. */
     case OB_SURF:
     case OB_FONT: {