Fix wrong mouse tolerance in mask editor

There are two issues. The biggest one was that a pixel value was used
to compare a squared distance meaning the tolerance was too strict.
The other issue was that the tolerance in pixels was different to what
the tracking mode is using.

The user-level change is that now it should be easier to tweak the
mask shape.

2 files changed, 5 insertions, 3 deletions

diff --git a/source/blender/editors/mask/mask_add.c b/source/blender/editors/mask/mask_add.c
index 0b9261eac4f..d10c420e28c 100644
--- a/source/blender/editors/mask/mask_add.c
+++ b/source/blender/editors/mask/mask_add.c
@@ -230,7 +230,7 @@ static bool add_vertex_subdivide(const bContext *C, Mask *mask, const float co[2
   MaskLayer *mask_layer;
   MaskSpline *spline;
   MaskSplinePoint *point = NULL;
-  const float threshold = 9;
+  const float threshold = 12;
   float tangent[2];
   float u;
 
@@ -593,7 +593,7 @@ static int add_feather_vertex_exec(bContext *C, wmOperator *op)
   MaskLayer *mask_layer;
   MaskSpline *spline;
   MaskSplinePoint *point = NULL;
-  const float threshold = 9;
+  const float threshold = 12;
   float co[2], u;
 
   RNA_float_get_array(op->ptr, "location", co);
diff --git a/source/blender/editors/mask/mask_query.c b/source/blender/editors/mask/mask_query.c
index afe457a8502..89524a7b9e2 100644
--- a/source/blender/editors/mask/mask_query.c
+++ b/source/blender/editors/mask/mask_query.c
@@ -45,6 +45,8 @@ bool ED_mask_find_nearest_diff_point(const bContext *C,
                                      float *r_u,
                                      float *r_score)
 {
+  const float threshold_sq = threshold * threshold;
+
   ScrArea *area = CTX_wm_area(C);
   ARegion *region = CTX_wm_region(C);
 
@@ -139,7 +141,7 @@ bool ED_mask_find_nearest_diff_point(const bContext *C,
     }
   }
 
-  if (point && dist_best_sq < threshold) {
+  if (point && dist_best_sq < threshold_sq) {
     if (r_mask_layer) {
       *r_mask_layer = point_mask_layer;
     }