Fix T65620: Sculpting brush size jumping.

The PBVHs raycast function calls `isect_ray_tri_epsilon_v3` with epsilon `0.1` which is inaccurate and may result in the problem presented in T65620.
The solution is to use `isect_ray_tri_watertight_v3` instead `isect_ray_tri_epsilon_v3`.
This can positively affect other areas as well.

Reviewers: brecht, campbellbarton

Differential Revision: https://developer.blender.org/D5083

5 files changed, 42 insertions, 39 deletions

diff --git a/source/blender/blenkernel/BKE_pbvh.h b/source/blender/blenkernel/BKE_pbvh.h
index 0a6f3a7d4d9..4edc5e651d8 100644
--- a/source/blender/blenkernel/BKE_pbvh.h
+++ b/source/blender/blenkernel/BKE_pbvh.h
@@ -133,12 +133,12 @@ bool BKE_pbvh_node_raycast(PBVH *bvh,
                            float (*origco)[3],
                            bool use_origco,
                            const float ray_start[3],
-                           const float ray_normal[3],
+                           struct IsectRayPrecalc *isect_precalc,
                            float *depth);
 
 bool BKE_pbvh_bmesh_node_raycast_detail(PBVHNode *node,
                                         const float ray_start[3],
-                                        const float ray_normal[3],
+                                        struct IsectRayPrecalc *isect_precalc,
                                         float *depth,
                                         float *r_edge_length);
 
diff --git a/source/blender/blenkernel/intern/pbvh.c b/source/blender/blenkernel/intern/pbvh.c
index 44dbf7a1e47..ffc4ec65d4d 100644
--- a/source/blender/blenkernel/intern/pbvh.c
+++ b/source/blender/blenkernel/intern/pbvh.c
@@ -1635,7 +1635,7 @@ void BKE_pbvh_raycast(PBVH *bvh,
 }
 
 bool ray_face_intersection_quad(const float ray_start[3],
-                                const float ray_normal[3],
+                                struct IsectRayPrecalc *isect_precalc,
                                 const float t0[3],
                                 const float t1[3],
                                 const float t2[3],
@@ -1644,9 +1644,9 @@ bool ray_face_intersection_quad(const float ray_start[3],
 {
   float depth_test;
 
-  if ((isect_ray_tri_epsilon_v3(ray_start, ray_normal, t0, t1, t2, &depth_test, NULL, 0.1f) &&
+  if ((isect_ray_tri_watertight_v3(ray_start, isect_precalc, t0, t1, t2, &depth_test, NULL) &&
        (depth_test < *depth)) ||
-      (isect_ray_tri_epsilon_v3(ray_start, ray_normal, t0, t2, t3, &depth_test, NULL, 0.1f) &&
+      (isect_ray_tri_watertight_v3(ray_start, isect_precalc, t0, t2, t3, &depth_test, NULL) &&
        (depth_test < *depth))) {
     *depth = depth_test;
     return true;
@@ -1657,15 +1657,14 @@ bool ray_face_intersection_quad(const float ray_start[3],
 }
 
 bool ray_face_intersection_tri(const float ray_start[3],
-                               const float ray_normal[3],
+                               struct IsectRayPrecalc *isect_precalc,
                                const float t0[3],
                                const float t1[3],
                                const float t2[3],
                                float *depth)
 {
   float depth_test;
-
-  if ((isect_ray_tri_epsilon_v3(ray_start, ray_normal, t0, t1, t2, &depth_test, NULL, 0.1f) &&
+  if ((isect_ray_tri_watertight_v3(ray_start, isect_precalc, t0, t1, t2, &depth_test, NULL) &&
        (depth_test < *depth))) {
     *depth = depth_test;
     return true;
@@ -1754,7 +1753,7 @@ static bool pbvh_faces_node_raycast(PBVH *bvh,
                                     const PBVHNode *node,
                                     float (*origco)[3],
                                     const float ray_start[3],
-                                    const float ray_normal[3],
+                                    struct IsectRayPrecalc *isect_precalc,
                                     float *depth)
 {
   const MVert *vert = bvh->verts;
@@ -1774,7 +1773,7 @@ static bool pbvh_faces_node_raycast(PBVH *bvh,
     if (origco) {
       /* intersect with backuped original coordinates */
       hit |= ray_face_intersection_tri(ray_start,
-                                       ray_normal,
+                                       isect_precalc,
                                        origco[face_verts[0]],
                                        origco[face_verts[1]],
                                        origco[face_verts[2]],
@@ -1783,7 +1782,7 @@ static bool pbvh_faces_node_raycast(PBVH *bvh,
     else {
       /* intersect with current coordinates */
       hit |= ray_face_intersection_tri(ray_start,
-                                       ray_normal,
+                                       isect_precalc,
                                        vert[mloop[lt->tri[0]].v].co,
                                        vert[mloop[lt->tri[1]].v].co,
                                        vert[mloop[lt->tri[2]].v].co,
@@ -1798,7 +1797,7 @@ static bool pbvh_grids_node_raycast(PBVH *bvh,
                                     PBVHNode *node,
                                     float (*origco)[3],
                                     const float ray_start[3],
-                                    const float ray_normal[3],
+                                    struct IsectRayPrecalc *isect_precalc,
                                     float *depth)
 {
   const int totgrid = node->totprim;
@@ -1826,7 +1825,7 @@ static bool pbvh_grids_node_raycast(PBVH *bvh,
 
         if (origco) {
           hit |= ray_face_intersection_quad(ray_start,
-                                            ray_normal,
+                                            isect_precalc,
                                             origco[y * gridsize + x],
                                             origco[y * gridsize + x + 1],
                                             origco[(y + 1) * gridsize + x + 1],
@@ -1835,7 +1834,7 @@ static bool pbvh_grids_node_raycast(PBVH *bvh,
         }
         else {
           hit |= ray_face_intersection_quad(ray_start,
-                                            ray_normal,
+                                            isect_precalc,
                                             CCG_grid_elem_co(&bvh->gridkey, grid, x, y),
                                             CCG_grid_elem_co(&bvh->gridkey, grid, x + 1, y),
                                             CCG_grid_elem_co(&bvh->gridkey, grid, x + 1, y + 1),
@@ -1858,7 +1857,7 @@ bool BKE_pbvh_node_raycast(PBVH *bvh,
                            float (*origco)[3],
                            bool use_origco,
                            const float ray_start[3],
-                           const float ray_normal[3],
+                           struct IsectRayPrecalc *isect_precalc,
                            float *depth)
 {
   bool hit = false;
@@ -1869,13 +1868,13 @@ bool BKE_pbvh_node_raycast(PBVH *bvh,
 
   switch (bvh->type) {
     case PBVH_FACES:
-      hit |= pbvh_faces_node_raycast(bvh, node, origco, ray_start, ray_normal, depth);
+      hit |= pbvh_faces_node_raycast(bvh, node, origco, ray_start, isect_precalc, depth);
       break;
     case PBVH_GRIDS:
-      hit |= pbvh_grids_node_raycast(bvh, node, origco, ray_start, ray_normal, depth);
+      hit |= pbvh_grids_node_raycast(bvh, node, origco, ray_start, isect_precalc, depth);
       break;
     case PBVH_BMESH:
-      hit = pbvh_bmesh_node_raycast(node, ray_start, ray_normal, depth, use_origco);
+      hit = pbvh_bmesh_node_raycast(node, ray_start, isect_precalc, depth, use_origco);
       break;
   }
 
diff --git a/source/blender/blenkernel/intern/pbvh_bmesh.c b/source/blender/blenkernel/intern/pbvh_bmesh.c
index cb4505880e2..1d8088c6605 100644
--- a/source/blender/blenkernel/intern/pbvh_bmesh.c
+++ b/source/blender/blenkernel/intern/pbvh_bmesh.c
@@ -1508,7 +1508,7 @@ static bool pbvh_bmesh_collapse_short_edges(EdgeQueueContext *eq_ctx,
 
 bool pbvh_bmesh_node_raycast(PBVHNode *node,
                              const float ray_start[3],
-                             const float ray_normal[3],
+                             struct IsectRayPrecalc *isect_precalc,
                              float *depth,
                              bool use_original)
 {
@@ -1518,7 +1518,7 @@ bool pbvh_bmesh_node_raycast(PBVHNode *node,
     for (int i = 0; i < node->bm_tot_ortri; i++) {
       const int *t = node->bm_ortri[i];
       hit |= ray_face_intersection_tri(ray_start,
-                                       ray_normal,
+                                       isect_precalc,
                                        node->bm_orco[t[0]],
                                        node->bm_orco[t[1]],
                                        node->bm_orco[t[2]],
@@ -1537,7 +1537,7 @@ bool pbvh_bmesh_node_raycast(PBVHNode *node,
 
         BM_face_as_array_vert_tri(f, v_tri);
         hit |= ray_face_intersection_tri(
-            ray_start, ray_normal, v_tri[0]->co, v_tri[1]->co, v_tri[2]->co, depth);
+            ray_start, isect_precalc, v_tri[0]->co, v_tri[1]->co, v_tri[2]->co, depth);
       }
     }
   }
@@ -1547,7 +1547,7 @@ bool pbvh_bmesh_node_raycast(PBVHNode *node,
 
 bool BKE_pbvh_bmesh_node_raycast_detail(PBVHNode *node,
                                         const float ray_start[3],
-                                        const float ray_normal[3],
+                                        struct IsectRayPrecalc *isect_precalc,
                                         float *depth,
                                         float *r_edge_length)
 {
@@ -1568,7 +1568,7 @@ bool BKE_pbvh_bmesh_node_raycast_detail(PBVHNode *node,
       bool hit_local;
       BM_face_as_array_vert_tri(f, v_tri);
       hit_local = ray_face_intersection_tri(
-          ray_start, ray_normal, v_tri[0]->co, v_tri[1]->co, v_tri[2]->co, depth);
+          ray_start, isect_precalc, v_tri[0]->co, v_tri[1]->co, v_tri[2]->co, depth);
 
       if (hit_local) {
         f_hit = f;
diff --git a/source/blender/blenkernel/intern/pbvh_intern.h b/source/blender/blenkernel/intern/pbvh_intern.h
index 74d7312e6a7..dc9d796075b 100644
--- a/source/blender/blenkernel/intern/pbvh_intern.h
+++ b/source/blender/blenkernel/intern/pbvh_intern.h
@@ -174,14 +174,14 @@ void BBC_update_centroid(BBC *bbc);
 int BB_widest_axis(const BB *bb);
 void pbvh_grow_nodes(PBVH *bvh, int totnode);
 bool ray_face_intersection_quad(const float ray_start[3],
-                                const float ray_normal[3],
+                                struct IsectRayPrecalc *isect_precalc,
                                 const float *t0,
                                 const float *t1,
                                 const float *t2,
                                 const float *t3,
                                 float *depth);
 bool ray_face_intersection_tri(const float ray_start[3],
-                               const float ray_normal[3],
+                               struct IsectRayPrecalc *isect_precalc,
                                const float *t0,
                                const float *t1,
                                const float *t2,
@@ -208,7 +208,7 @@ void pbvh_update_BB_redraw(PBVH *bvh, PBVHNode **nodes, int totnode, int flag);
 /* pbvh_bmesh.c */
 bool pbvh_bmesh_node_raycast(PBVHNode *node,
                              const float ray_start[3],
-                             const float ray_normal[3],
+                             struct IsectRayPrecalc *isect_precalc,
                              float *dist,
                              bool use_original);
 bool pbvh_bmesh_node_nearest_to_ray(PBVHNode *node,
diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index c902b6a0551..5a7fb1abb7b 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -1730,17 +1730,21 @@ typedef struct SculptDoBrushSmoothGridDataChunk {
 
 typedef struct {
   SculptSession *ss;
-  const float *ray_start, *ray_normal;
+  const float *ray_start;
   bool hit;
   float depth;
   bool original;
+
+  struct IsectRayPrecalc isect_precalc;
 } SculptRaycastData;
 
 typedef struct {
-  const float *ray_start, *ray_normal;
+  const float *ray_start;
   bool hit;
   float depth;
   float edge_length;
+
+  struct IsectRayPrecalc isect_precalc;
 } SculptDetailRaycastData;
 
 typedef struct {
@@ -4949,7 +4953,7 @@ static void sculpt_raycast_cb(PBVHNode *node, void *data_v, float *tmin)
                               origco,
                               use_origco,
                               srd->ray_start,
-                              srd->ray_normal,
+                              &srd->isect_precalc,
                               &srd->depth)) {
       srd->hit = 1;
       *tmin = srd->depth;
@@ -4995,7 +4999,7 @@ static void sculpt_raycast_detail_cb(PBVHNode *node, void *data_v, float *tmin)
   if (BKE_pbvh_node_get_tmin(node) < *tmin) {
     SculptDetailRaycastData *srd = data_v;
     if (BKE_pbvh_bmesh_node_raycast_detail(
-            node, srd->ray_start, srd->ray_normal, &srd->depth, &srd->edge_length)) {
+            node, srd->ray_start, &srd->isect_precalc, &srd->depth, &srd->edge_length)) {
       srd->hit = 1;
       *tmin = srd->depth;
     }
@@ -5067,14 +5071,14 @@ bool sculpt_stroke_get_location(bContext *C, float out[3], const float mouse[2])
 
   bool hit = false;
   {
-    SculptRaycastData srd = {
-        .original = original,
-        .ss = ob->sculpt,
-        .hit = 0,
-        .ray_start = ray_start,
-        .ray_normal = ray_normal,
-        .depth = depth,
-    };
+    SculptRaycastData srd;
+    srd.ss = ob->sculpt;
+    srd.ray_start = ray_start;
+    srd.hit = 0;
+    srd.depth = depth;
+    srd.original = original;
+    isect_ray_tri_watertight_v3_precalc(&srd.isect_precalc, ray_normal);
+
     BKE_pbvh_raycast(ss->pbvh, sculpt_raycast_cb, &srd, ray_start, ray_normal, srd.original);
     if (srd.hit) {
       hit = true;
@@ -6372,9 +6376,9 @@ static void sample_detail(bContext *C, int mx, int my)
   SculptDetailRaycastData srd;
   srd.hit = 0;
   srd.ray_start = ray_start;
-  srd.ray_normal = ray_normal;
   srd.depth = depth;
   srd.edge_length = 0.0f;
+  isect_ray_tri_watertight_v3_precalc(&srd.isect_precalc, ray_normal);
 
   BKE_pbvh_raycast(ob->sculpt->pbvh, sculpt_raycast_detail_cb, &srd, ray_start, ray_normal, false);