Curves: improve Comb brush

New supported features:
* 3D/spherical brush that samples a good position on the curves.
* Falloff.

The custom falloff curve mapping is not yet available in the ui because that
requires some more ui reorganization. This is better done when we have
a better understanding of what settings we need exactly.

Currently, the depth of the 3d brush is only sampled once per stroke, when
first pressing LMB. Sometimes it is expected that the depth of the brush can
change within a single brush. However, implementing that in a good way
is not straight forward and might need additional options. Therefore that
will be handled separately. Some experimentation results are in D14376.

Ref T96445.

Differential Revision: https://developer.blender.org/D14376

2 files changed, 32 insertions, 26 deletions

diff --git a/source/blender/blenlib/BLI_math_geom.h b/source/blender/blenlib/BLI_math_geom.h
index 2f3fbd59b5f..c31e3045c97 100644
--- a/source/blender/blenlib/BLI_math_geom.h
+++ b/source/blender/blenlib/BLI_math_geom.h
@@ -322,18 +322,22 @@ double closest_to_line_v2_db(double r_close[2],
 float closest_to_line_v3(float r_close[3], const float p[3], const float l1[3], const float l2[3]);
 /**
  * Point closest to v1 on line v2-v3 in 2D.
+ *
+ * \return A value in [0, 1] that corresponds to the position of #r_close on the line segment.
  */
-void closest_to_line_segment_v2(float r_close[2],
-                                const float p[2],
-                                const float l1[2],
-                                const float l2[2]);
+float closest_to_line_segment_v2(float r_close[2],
+                                 const float p[2],
+                                 const float l1[2],
+                                 const float l2[2]);
 /**
  * Point closest to v1 on line v2-v3 in 3D.
+ *
+ * \return A value in [0, 1] that corresponds to the position of #r_close on the line segment.
  */
-void closest_to_line_segment_v3(float r_close[3],
-                                const float p[3],
-                                const float l1[3],
-                                const float l2[3]);
+float closest_to_line_segment_v3(float r_close[3],
+                                 const float p[3],
+                                 const float l1[3],
+                                 const float l2[3]);
 void closest_to_plane_normalized_v3(float r_close[3], const float plane[4], const float pt[3]);
 /**
  * Find the closest point on a plane.
diff --git a/source/blender/blenlib/intern/math_geom.c b/source/blender/blenlib/intern/math_geom.c
index bc3ed099fd5..e1ec22063e0 100644
--- a/source/blender/blenlib/intern/math_geom.c
+++ b/source/blender/blenlib/intern/math_geom.c
@@ -294,46 +294,48 @@ float dist_to_line_segment_v2(const float p[2], const float l1[2], const float l
   return sqrtf(dist_squared_to_line_segment_v2(p, l1, l2));
 }
 
-void closest_to_line_segment_v2(float r_close[2],
-                                const float p[2],
-                                const float l1[2],
-                                const float l2[2])
+float closest_to_line_segment_v2(float r_close[2],
+                                 const float p[2],
+                                 const float l1[2],
+                                 const float l2[2])
 {
   float lambda, cp[2];
 
   lambda = closest_to_line_v2(cp, p, l1, l2);
 
   /* flip checks for !finite case (when segment is a point) */
-  if (!(lambda > 0.0f)) {
+  if (lambda <= 0.0f) {
     copy_v2_v2(r_close, l1);
+    return 0.0f;
   }
-  else if (!(lambda < 1.0f)) {
+  if (lambda >= 1.0f) {
     copy_v2_v2(r_close, l2);
+    return 1.0f;
   }
-  else {
-    copy_v2_v2(r_close, cp);
-  }
+  copy_v2_v2(r_close, cp);
+  return lambda;
 }
 
-void closest_to_line_segment_v3(float r_close[3],
-                                const float p[3],
-                                const float l1[3],
-                                const float l2[3])
+float closest_to_line_segment_v3(float r_close[3],
+                                 const float p[3],
+                                 const float l1[3],
+                                 const float l2[3])
 {
   float lambda, cp[3];
 
   lambda = closest_to_line_v3(cp, p, l1, l2);
 
   /* flip checks for !finite case (when segment is a point) */
-  if (!(lambda > 0.0f)) {
+  if (lambda <= 0.0f) {
     copy_v3_v3(r_close, l1);
+    return 0.0f;
   }
-  else if (!(lambda < 1.0f)) {
+  if (lambda >= 1.0f) {
     copy_v3_v3(r_close, l2);
+    return 1.0f;
   }
-  else {
-    copy_v3_v3(r_close, cp);
-  }
+  copy_v3_v3(r_close, cp);
+  return lambda;
 }
 
 void closest_to_plane_v3(float r_close[3], const float plane[4], const float pt[3])