Fix Face Sets painting and selection precision

This fixes the following issues:
- Previously, the face set from the active vertex was used directly. Vertices always return the most recently created face set, so in some cases there may be some face sets that were not possible to select as active. Now the active face set is set in the ray intersection, so it always matches the face under the cursor.
- When drawing face sets they were set per vertex, so it was not possible to paint one face at a time. Now face sets are painted per poly when using the brush on meshes, testing the distance to the center of each poly.
- The code for the active vertex on PBVH_GRIDS was not correct, so I also fixed that to test if everything was working correctly.
{F8441699}

Reviewed By: jbakker

Differential Revision: https://developer.blender.org/D7303

1 files changed, 16 insertions, 2 deletions

diff --git a/source/blender/blenkernel/intern/pbvh.c b/source/blender/blenkernel/intern/pbvh.c
index ac0c9d030cf..879f1e507ba 100644
--- a/source/blender/blenkernel/intern/pbvh.c
+++ b/source/blender/blenkernel/intern/pbvh.c
@@ -2121,6 +2121,7 @@ static bool pbvh_faces_node_raycast(PBVH *bvh,
                                     struct IsectRayPrecalc *isect_precalc,
                                     float *depth,
                                     int *r_active_vertex_index,
+                                    int *r_active_face_index,
                                     float *r_face_normal)
 {
   const MVert *vert = bvh->verts;
@@ -2166,6 +2167,7 @@ static bool pbvh_faces_node_raycast(PBVH *bvh,
           if (len_squared_v3v3(location, co[j]) < len_squared_v3v3(location, nearest_vertex_co)) {
             copy_v3_v3(nearest_vertex_co, co[j]);
             *r_active_vertex_index = mloop[lt->tri[j]].v;
+            *r_active_face_index = lt->poly;
           }
         }
       }
@@ -2183,6 +2185,7 @@ static bool pbvh_grids_node_raycast(PBVH *bvh,
                                     struct IsectRayPrecalc *isect_precalc,
                                     float *depth,
                                     int *r_active_vertex_index,
+                                    int *r_active_grid_index,
                                     float *r_face_normal)
 {
   const int totgrid = node->totprim;
@@ -2236,15 +2239,23 @@ static bool pbvh_grids_node_raycast(PBVH *bvh,
           if (r_active_vertex_index) {
             float location[3] = {0.0};
             madd_v3_v3v3fl(location, ray_start, ray_normal, *depth);
+
+            const int x_it[4] = {0, 1, 1, 0};
+            const int y_it[4] = {0, 0, 1, 1};
+
             for (int j = 0; j < 4; j++) {
               if (len_squared_v3v3(location, co[j]) <
                   len_squared_v3v3(location, nearest_vertex_co)) {
                 copy_v3_v3(nearest_vertex_co, co[j]);
-                *r_active_vertex_index = gridkey->grid_area * grid_index + y * gridkey->grid_size +
-                                         x;
+
+                *r_active_vertex_index = gridkey->grid_area * grid_index +
+                                         (y + y_it[j]) * gridkey->grid_size + (x + x_it[j]);
               }
             }
           }
+          if (r_active_grid_index) {
+            *r_active_grid_index = grid_index;
+          }
         }
       }
     }
@@ -2266,6 +2277,7 @@ bool BKE_pbvh_node_raycast(PBVH *bvh,
                            struct IsectRayPrecalc *isect_precalc,
                            float *depth,
                            int *active_vertex_index,
+                           int *active_face_grid_index,
                            float *face_normal)
 {
   bool hit = false;
@@ -2284,6 +2296,7 @@ bool BKE_pbvh_node_raycast(PBVH *bvh,
                                      isect_precalc,
                                      depth,
                                      active_vertex_index,
+                                     active_face_grid_index,
                                      face_normal);
       break;
     case PBVH_GRIDS:
@@ -2295,6 +2308,7 @@ bool BKE_pbvh_node_raycast(PBVH *bvh,
                                      isect_precalc,
                                      depth,
                                      active_vertex_index,
+                                     active_face_grid_index,
                                      face_normal);
       break;
     case PBVH_BMESH: