Sculpt: New brush cursor, active vertex and normal radius

This commit includes the new brush cursor, active vertex updates and the normal radius brush property for all sculpt brushes.
  -The new brush cursor previews the real stroke radius over the mesh and the sampled sculpt normal.
  -The active vertex is used in sculpt tools and brushes as a starting point for an operation, similar to a preselection. It is also mirrored following the enabled symmetry options to preview the stroke symmetry.
  -The normal radius brush property limits the radius that is going to be used to sample the sculpt normal and area center. It controls how closely the cursor follows the surface and it improves the behavior of most brushes, making them suitable for hard surface sculpting.

Reviewed By: campbellbarton, brecht

Differential Revision: https://developer.blender.org/D3594

4 files changed, 71 insertions, 6 deletions

diff --git a/source/blender/blenkernel/intern/brush.c b/source/blender/blenkernel/intern/brush.c
index b2d3ccfebc3..83e4a582ff1 100644
--- a/source/blender/blenkernel/intern/brush.c
+++ b/source/blender/blenkernel/intern/brush.c
@@ -72,6 +72,7 @@ static void brush_defaults(Brush *brush)
   brush->autosmooth_factor = 0.0f;
   brush->topology_rake_factor = 0.0f;
   brush->crease_pinch_factor = 0.5f;
+  brush->normal_radius_factor = 0.5f;
   brush->sculpt_plane = SCULPT_DISP_DIR_AREA;
   /* How far above or below the plane that is found by averaging the faces. */
   brush->plane_offset = 0.0f;
diff --git a/source/blender/blenkernel/intern/pbvh.c b/source/blender/blenkernel/intern/pbvh.c
index cea68a0c525..7a8c082842e 100644
--- a/source/blender/blenkernel/intern/pbvh.c
+++ b/source/blender/blenkernel/intern/pbvh.c
@@ -1753,14 +1753,21 @@ static bool pbvh_faces_node_raycast(PBVH *bvh,
                                     const PBVHNode *node,
                                     float (*origco)[3],
                                     const float ray_start[3],
+                                    const float ray_normal[3],
                                     struct IsectRayPrecalc *isect_precalc,
-                                    float *depth)
+                                    float *depth,
+                                    int *r_active_vertex_index,
+                                    float *r_face_normal)
 {
   const MVert *vert = bvh->verts;
   const MLoop *mloop = bvh->mloop;
   const int *faces = node->prim_indices;
   int i, totface = node->totprim;
   bool hit = false;
+  float min_depth = FLT_MAX;
+  float location[3] = {0.0f};
+  float nearest_vertex_co[3];
+  copy_v3_fl(nearest_vertex_co, 0.0f);
 
   for (i = 0; i < totface; ++i) {
     const MLoopTri *lt = &bvh->looptri[faces[i]];
@@ -1787,6 +1794,22 @@ static bool pbvh_faces_node_raycast(PBVH *bvh,
                                        vert[mloop[lt->tri[1]].v].co,
                                        vert[mloop[lt->tri[2]].v].co,
                                        depth);
+
+      if (hit && *depth < min_depth) {
+        min_depth = *depth;
+        normal_tri_v3(r_face_normal,
+                      vert[mloop[lt->tri[0]].v].co,
+                      vert[mloop[lt->tri[1]].v].co,
+                      vert[mloop[lt->tri[2]].v].co);
+        madd_v3_v3v3fl(location, ray_start, ray_normal, *depth);
+        for (int j = 0; j < 3; j++) {
+          if (len_squared_v3v3(location, vert[mloop[lt->tri[j]].v].co) <
+              len_squared_v3v3(location, nearest_vertex_co)) {
+            copy_v3_v3(nearest_vertex_co, vert[mloop[lt->tri[j]].v].co);
+            *r_active_vertex_index = mloop[lt->tri[j]].v;
+          }
+        }
+      }
     }
   }
 
@@ -1857,8 +1880,11 @@ bool BKE_pbvh_node_raycast(PBVH *bvh,
                            float (*origco)[3],
                            bool use_origco,
                            const float ray_start[3],
+                           const float ray_normal[3],
                            struct IsectRayPrecalc *isect_precalc,
-                           float *depth)
+                           float *depth,
+                           int *active_vertex_index,
+                           float *face_normal)
 {
   bool hit = false;
 
@@ -1868,13 +1894,29 @@ bool BKE_pbvh_node_raycast(PBVH *bvh,
 
   switch (bvh->type) {
     case PBVH_FACES:
-      hit |= pbvh_faces_node_raycast(bvh, node, origco, ray_start, isect_precalc, depth);
+      hit |= pbvh_faces_node_raycast(bvh,
+                                     node,
+                                     origco,
+                                     ray_start,
+                                     ray_normal,
+                                     isect_precalc,
+                                     depth,
+                                     active_vertex_index,
+                                     face_normal);
       break;
     case PBVH_GRIDS:
       hit |= pbvh_grids_node_raycast(bvh, node, origco, ray_start, isect_precalc, depth);
       break;
     case PBVH_BMESH:
-      hit = pbvh_bmesh_node_raycast(node, ray_start, isect_precalc, depth, use_origco);
+      BM_mesh_elem_index_ensure(bvh->bm, BM_VERT);
+      hit = pbvh_bmesh_node_raycast(node,
+                                    ray_start,
+                                    ray_normal,
+                                    isect_precalc,
+                                    depth,
+                                    use_origco,
+                                    active_vertex_index,
+                                    face_normal);
       break;
   }
 
diff --git a/source/blender/blenkernel/intern/pbvh_bmesh.c b/source/blender/blenkernel/intern/pbvh_bmesh.c
index 1d8088c6605..c04e172f116 100644
--- a/source/blender/blenkernel/intern/pbvh_bmesh.c
+++ b/source/blender/blenkernel/intern/pbvh_bmesh.c
@@ -1508,12 +1508,18 @@ static bool pbvh_bmesh_collapse_short_edges(EdgeQueueContext *eq_ctx,
 
 bool pbvh_bmesh_node_raycast(PBVHNode *node,
                              const float ray_start[3],
+                             const float ray_normal[3],
                              struct IsectRayPrecalc *isect_precalc,
                              float *depth,
-                             bool use_original)
+                             bool use_original,
+                             int *r_active_vertex_index,
+                             float *r_face_normal)
 {
   bool hit = false;
 
+  float min_depth = FLT_MAX;
+  float nearest_vertex_co[3] = {0.0f};
+  float location[3] = {0.0f};
   if (use_original && node->bm_tot_ortri) {
     for (int i = 0; i < node->bm_tot_ortri; i++) {
       const int *t = node->bm_ortri[i];
@@ -1538,6 +1544,19 @@ bool pbvh_bmesh_node_raycast(PBVHNode *node,
         BM_face_as_array_vert_tri(f, v_tri);
         hit |= ray_face_intersection_tri(
             ray_start, isect_precalc, v_tri[0]->co, v_tri[1]->co, v_tri[2]->co, depth);
+
+        if (hit && *depth < min_depth) {
+          min_depth = *depth;
+          normal_tri_v3(r_face_normal, v_tri[0]->co, v_tri[1]->co, v_tri[2]->co);
+          madd_v3_v3v3fl(location, ray_start, ray_normal, *depth);
+          for (int j = 0; j < 3; j++) {
+            if (len_squared_v3v3(location, v_tri[j]->co) <
+                len_squared_v3v3(location, nearest_vertex_co)) {
+              copy_v3_v3(nearest_vertex_co, v_tri[j]->co);
+              *r_active_vertex_index = BM_elem_index_get(v_tri[j]);
+            }
+          }
+        }
       }
     }
   }
diff --git a/source/blender/blenkernel/intern/pbvh_intern.h b/source/blender/blenkernel/intern/pbvh_intern.h
index e74a8d43c68..bad103743eb 100644
--- a/source/blender/blenkernel/intern/pbvh_intern.h
+++ b/source/blender/blenkernel/intern/pbvh_intern.h
@@ -206,9 +206,12 @@ void pbvh_update_BB_redraw(PBVH *bvh, PBVHNode **nodes, int totnode, int flag);
 /* pbvh_bmesh.c */
 bool pbvh_bmesh_node_raycast(PBVHNode *node,
                              const float ray_start[3],
+                             const float ray_normal[3],
                              struct IsectRayPrecalc *isect_precalc,
                              float *dist,
-                             bool use_original);
+                             bool use_original,
+                             int *r_active_vertex_index,
+                             float *r_face_normal);
 bool pbvh_bmesh_node_nearest_to_ray(PBVHNode *node,
                                     const float ray_start[3],
                                     const float ray_normal[3],