Mask: Use control point position when handle is selected

This is something what comes after an experiment, which makes behavior
more desirable. Basically, for Lock-to-Selection functionality always
use control point position if any of control point itself or handles
are selected.

Initial patch from Sebastian, modification from Sergey.

Differential Revision: https://developer.blender.org/D10265

4 files changed, 36 insertions, 20 deletions

diff --git a/source/blender/editors/include/ED_mask.h b/source/blender/editors/include/ED_mask.h
index b20dae694fc..247911bdc55 100644
--- a/source/blender/editors/include/ED_mask.h
+++ b/source/blender/editors/include/ED_mask.h
@@ -66,7 +66,7 @@ void ED_mask_cursor_location_get(struct ScrArea *area, float cursor[2]);
 bool ED_mask_selected_minmax(const struct bContext *C,
                              float min[2],
                              float max[2],
-                             bool include_handles);
+                             bool handles_as_control_point);
 
 /* mask_draw.c */
 void ED_mask_draw(const struct bContext *C, const char draw_flag, const char draw_type);
diff --git a/source/blender/editors/mask/mask_query.c b/source/blender/editors/mask/mask_query.c
index eace146dbe9..401b6eac4f2 100644
--- a/source/blender/editors/mask/mask_query.c
+++ b/source/blender/editors/mask/mask_query.c
@@ -604,7 +604,22 @@ void ED_mask_point_pos__reverse(
   *yr = co[1];
 }
 
-bool ED_mask_selected_minmax(const bContext *C, float min[2], float max[2], bool include_handles)
+static void handle_position_for_minmax(const MaskSplinePoint *point,
+                                       eMaskWhichHandle which_handle,
+                                       bool handles_as_control_point,
+                                       float r_handle[2])
+{
+  if (handles_as_control_point) {
+    copy_v2_v2(r_handle, point->bezt.vec[1]);
+    return;
+  }
+  BKE_mask_point_handle(point, which_handle, r_handle);
+}
+
+bool ED_mask_selected_minmax(const bContext *C,
+                             float min[2],
+                             float max[2],
+                             bool handles_as_control_point)
 {
   Depsgraph *depsgraph = CTX_data_depsgraph_pointer(C);
   Mask *mask = CTX_data_edit_mask(C);
@@ -641,22 +656,22 @@ bool ED_mask_selected_minmax(const bContext *C, float min[2], float max[2], bool
           ok = true;
         }
 
-        if (!include_handles) {
-          /* Ignore handles. */
-        }
-        else if (BKE_mask_point_handles_mode_get(point) == MASK_HANDLE_MODE_STICK) {
-          BKE_mask_point_handle(deform_point, MASK_WHICH_HANDLE_STICK, handle);
+        if (BKE_mask_point_handles_mode_get(point) == MASK_HANDLE_MODE_STICK) {
+          handle_position_for_minmax(
+              deform_point, MASK_WHICH_HANDLE_STICK, handles_as_control_point, handle);
           minmax_v2v2_v2(min, max, handle);
           ok = true;
         }
         else {
           if ((bezt->f1 & SELECT) && (bezt->h1 != HD_VECT)) {
-            BKE_mask_point_handle(deform_point, MASK_WHICH_HANDLE_LEFT, handle);
+            handle_position_for_minmax(
+                deform_point, MASK_WHICH_HANDLE_LEFT, handles_as_control_point, handle);
             minmax_v2v2_v2(min, max, handle);
             ok = true;
           }
           if ((bezt->f3 & SELECT) && (bezt->h2 != HD_VECT)) {
-            BKE_mask_point_handle(deform_point, MASK_WHICH_HANDLE_RIGHT, handle);
+            handle_position_for_minmax(
+                deform_point, MASK_WHICH_HANDLE_RIGHT, handles_as_control_point, handle);
             minmax_v2v2_v2(min, max, handle);
             ok = true;
           }
diff --git a/source/blender/editors/space_clip/clip_utils.c b/source/blender/editors/space_clip/clip_utils.c
index 271ec219809..bb79eb34129 100644
--- a/source/blender/editors/space_clip/clip_utils.c
+++ b/source/blender/editors/space_clip/clip_utils.c
@@ -484,7 +484,7 @@ static bool tracking_has_selection(SpaceClip *space_clip)
   return false;
 }
 
-static bool mask_has_selection(const bContext *C, bool include_handles)
+static bool mask_has_selection(const bContext *C)
 {
   Mask *mask = CTX_data_edit_mask(C);
   if (mask == NULL) {
@@ -506,10 +506,7 @@ static bool mask_has_selection(const bContext *C, bool include_handles)
           return true;
         }
 
-        if (!include_handles) {
-          /* Ignore handles. */
-        }
-        else if (BKE_mask_point_handles_mode_get(point) == MASK_HANDLE_MODE_STICK) {
+        if (BKE_mask_point_handles_mode_get(point) == MASK_HANDLE_MODE_STICK) {
           return true;
         }
         else {
@@ -527,14 +524,17 @@ static bool mask_has_selection(const bContext *C, bool include_handles)
   return false;
 }
 
-static bool selected_boundbox(const bContext *C, float min[2], float max[2], bool include_handles)
+static bool selected_boundbox(const bContext *C,
+                              float min[2],
+                              float max[2],
+                              bool handles_as_control_point)
 {
   SpaceClip *sc = CTX_wm_space_clip(C);
   if (sc->mode == SC_MODE_TRACKING) {
     return selected_tracking_boundbox(sc, min, max);
   }
 
-  if (ED_mask_selected_minmax(C, min, max, include_handles)) {
+  if (ED_mask_selected_minmax(C, min, max, handles_as_control_point)) {
     MovieClip *clip = ED_space_clip_get_clip(sc);
     int width, height;
     ED_space_clip_get_size(sc, &width, &height);
@@ -563,12 +563,13 @@ bool clip_view_calculate_view_selection(
 
   /* NOTE: The `fit` argument is set to truth when doing "View to Selected" operator, and it set to
    * false when this function is used for Lock-to-Selection functionality. When locking to
-   * selection the handles are to be ignored. So we can derive the `include_handles` from `fit`.
+   * selection the handles are to use control point position. So we can derive the
+   * `handles_as_control_point` from `fit`.
    *
    * TODO(sergey): Make such decision more explicit. Maybe pass use-case for the calculation to
    * tell operator from lock-to-selection apart. */
   float min[2], max[2];
-  if (!selected_boundbox(C, min, max, fit)) {
+  if (!selected_boundbox(C, min, max, !fit)) {
     return false;
   }
 
@@ -622,7 +623,7 @@ bool clip_view_has_locked_selection(const bContext *C)
     return tracking_has_selection(space_clip);
   }
 
-  return mask_has_selection(C, false);
+  return mask_has_selection(C);
 }
 
 void clip_draw_sfra_efra(View2D *v2d, Scene *scene)
diff --git a/source/blender/editors/space_image/image_ops.c b/source/blender/editors/space_image/image_ops.c
index e7f468f2f26..a3052a919e7 100644
--- a/source/blender/editors/space_image/image_ops.c
+++ b/source/blender/editors/space_image/image_ops.c
@@ -898,7 +898,7 @@ static int image_view_selected_exec(bContext *C, wmOperator *UNUSED(op))
     }
   }
   else if (ED_space_image_check_show_maskedit(sima, obedit)) {
-    if (!ED_mask_selected_minmax(C, min, max, true)) {
+    if (!ED_mask_selected_minmax(C, min, max, false)) {
       return OPERATOR_CANCELLED;
     }
   }