Fix Face Sets painting and selection precision

This fixes the following issues:
- Previously, the face set from the active vertex was used directly. Vertices always return the most recently created face set, so in some cases there may be some face sets that were not possible to select as active. Now the active face set is set in the ray intersection, so it always matches the face under the cursor.
- When drawing face sets they were set per vertex, so it was not possible to paint one face at a time. Now face sets are painted per poly when using the brush on meshes, testing the distance to the center of each poly.
- The code for the active vertex on PBVH_GRIDS was not correct, so I also fixed that to test if everything was working correctly.
{F8441699}

Reviewed By: jbakker

Differential Revision: https://developer.blender.org/D7303

3 files changed, 20 insertions, 2 deletions

diff --git a/source/blender/blenkernel/BKE_paint.h b/source/blender/blenkernel/BKE_paint.h
index 8c925ee2ae1..f4b485ae53a 100644
--- a/source/blender/blenkernel/BKE_paint.h
+++ b/source/blender/blenkernel/BKE_paint.h
@@ -334,6 +334,9 @@ typedef struct SculptSession {
   /* Cursor data and active vertex for tools */
   int active_vertex_index;
 
+  int active_face_index;
+  int active_grid_index;
+
   float cursor_radius;
   float cursor_location[3];
   float cursor_normal[3];
diff --git a/source/blender/blenkernel/BKE_pbvh.h b/source/blender/blenkernel/BKE_pbvh.h
index b4f16bfd899..2020d45ae86 100644
--- a/source/blender/blenkernel/BKE_pbvh.h
+++ b/source/blender/blenkernel/BKE_pbvh.h
@@ -155,6 +155,7 @@ bool BKE_pbvh_node_raycast(PBVH *bvh,
                            struct IsectRayPrecalc *isect_precalc,
                            float *depth,
                            int *active_vertex_index,
+                           int *active_face_grid_index,
                            float *face_normal);
 
 bool BKE_pbvh_bmesh_node_raycast_detail(PBVHNode *node,
diff --git a/source/blender/blenkernel/intern/pbvh.c b/source/blender/blenkernel/intern/pbvh.c
index ac0c9d030cf..879f1e507ba 100644
--- a/source/blender/blenkernel/intern/pbvh.c
+++ b/source/blender/blenkernel/intern/pbvh.c
@@ -2121,6 +2121,7 @@ static bool pbvh_faces_node_raycast(PBVH *bvh,
                                     struct IsectRayPrecalc *isect_precalc,
                                     float *depth,
                                     int *r_active_vertex_index,
+                                    int *r_active_face_index,
                                     float *r_face_normal)
 {
   const MVert *vert = bvh->verts;
@@ -2166,6 +2167,7 @@ static bool pbvh_faces_node_raycast(PBVH *bvh,
           if (len_squared_v3v3(location, co[j]) < len_squared_v3v3(location, nearest_vertex_co)) {
             copy_v3_v3(nearest_vertex_co, co[j]);
             *r_active_vertex_index = mloop[lt->tri[j]].v;
+            *r_active_face_index = lt->poly;
           }
         }
       }
@@ -2183,6 +2185,7 @@ static bool pbvh_grids_node_raycast(PBVH *bvh,
                                     struct IsectRayPrecalc *isect_precalc,
                                     float *depth,
                                     int *r_active_vertex_index,
+                                    int *r_active_grid_index,
                                     float *r_face_normal)
 {
   const int totgrid = node->totprim;
@@ -2236,15 +2239,23 @@ static bool pbvh_grids_node_raycast(PBVH *bvh,
           if (r_active_vertex_index) {
             float location[3] = {0.0};
             madd_v3_v3v3fl(location, ray_start, ray_normal, *depth);
+
+            const int x_it[4] = {0, 1, 1, 0};
+            const int y_it[4] = {0, 0, 1, 1};
+
             for (int j = 0; j < 4; j++) {
               if (len_squared_v3v3(location, co[j]) <
                   len_squared_v3v3(location, nearest_vertex_co)) {
                 copy_v3_v3(nearest_vertex_co, co[j]);
-                *r_active_vertex_index = gridkey->grid_area * grid_index + y * gridkey->grid_size +
-                                         x;
+
+                *r_active_vertex_index = gridkey->grid_area * grid_index +
+                                         (y + y_it[j]) * gridkey->grid_size + (x + x_it[j]);
               }
             }
           }
+          if (r_active_grid_index) {
+            *r_active_grid_index = grid_index;
+          }
         }
       }
     }
@@ -2266,6 +2277,7 @@ bool BKE_pbvh_node_raycast(PBVH *bvh,
                            struct IsectRayPrecalc *isect_precalc,
                            float *depth,
                            int *active_vertex_index,
+                           int *active_face_grid_index,
                            float *face_normal)
 {
   bool hit = false;
@@ -2284,6 +2296,7 @@ bool BKE_pbvh_node_raycast(PBVH *bvh,
                                      isect_precalc,
                                      depth,
                                      active_vertex_index,
+                                     active_face_grid_index,
                                      face_normal);
       break;
     case PBVH_GRIDS:
@@ -2295,6 +2308,7 @@ bool BKE_pbvh_node_raycast(PBVH *bvh,
                                      isect_precalc,
                                      depth,
                                      active_vertex_index,
+                                     active_face_grid_index,
                                      face_normal);
       break;
     case PBVH_BMESH: