Fix T65027: Snap 3D cursor on hidden faces doesn't work in Edit Mode.

I'm not very fond of adding new types of bvhtrees.
But this is probably the most efficient solution.

3 files changed, 63 insertions, 19 deletions

diff --git a/source/blender/blenkernel/BKE_bvhutils.h b/source/blender/blenkernel/BKE_bvhutils.h
index 4b64b6fa269..c88a64097bb 100644
--- a/source/blender/blenkernel/BKE_bvhutils.h
+++ b/source/blender/blenkernel/BKE_bvhutils.h
@@ -216,17 +216,18 @@ float bvhtree_sphereray_tri_intersection(const BVHTreeRay *ray,
 
 /* Using local coordinates */
 enum {
-  BVHTREE_FROM_VERTS = 0,
-  BVHTREE_FROM_EDGES = 1,
-  BVHTREE_FROM_FACES = 2,
-  BVHTREE_FROM_LOOPTRI = 3,
-
-  BVHTREE_FROM_LOOSEVERTS = 4,
-  BVHTREE_FROM_LOOSEEDGES = 5,
-
-  BVHTREE_FROM_EM_VERTS = 6,
-  BVHTREE_FROM_EM_EDGES = 7,
-  BVHTREE_FROM_EM_LOOPTRI = 8,
+  BVHTREE_FROM_VERTS,
+  BVHTREE_FROM_EDGES,
+  BVHTREE_FROM_FACES,
+  BVHTREE_FROM_LOOPTRI,
+  BVHTREE_FROM_LOOPTRI_NO_HIDDEN,
+
+  BVHTREE_FROM_LOOSEVERTS,
+  BVHTREE_FROM_LOOSEEDGES,
+
+  BVHTREE_FROM_EM_VERTS,
+  BVHTREE_FROM_EM_EDGES,
+  BVHTREE_FROM_EM_LOOPTRI,
 };
 
 bool bvhcache_find(const BVHCache *cache, int type, BVHTree **r_tree);
diff --git a/source/blender/blenkernel/intern/bvhutils.c b/source/blender/blenkernel/intern/bvhutils.c
index 8600d60c5c6..e51d15ee152 100644
--- a/source/blender/blenkernel/intern/bvhutils.c
+++ b/source/blender/blenkernel/intern/bvhutils.c
@@ -1168,6 +1168,35 @@ static BLI_bitmap *loose_edges_map_get(const MEdge *medge,
   return loose_edges_mask;
 }
 
+static BLI_bitmap *looptri_no_hidden_map_get(const MPoly *mpoly,
+                                             const int looptri_len,
+                                             int *r_looptri_active_len)
+{
+  BLI_bitmap *looptri_mask = BLI_BITMAP_NEW(looptri_len, __func__);
+
+  int looptri_no_hidden_len = 0;
+  int looptri_iter = 0;
+  const MPoly *mp = mpoly;
+  while (looptri_iter != looptri_len) {
+    int mp_totlooptri = mp->totloop - 2;
+    if (mp->flag & ME_HIDE) {
+      looptri_iter += mp_totlooptri;
+    }
+    else {
+      while (mp_totlooptri--) {
+        BLI_BITMAP_ENABLE(looptri_mask, looptri_iter);
+        looptri_iter++;
+        looptri_no_hidden_len++;
+      }
+    }
+    mp++;
+  }
+
+  *r_looptri_active_len = looptri_no_hidden_len;
+
+  return looptri_mask;
+}
+
 /**
  * Builds or queries a bvhcache for the cache bvhtree of the request type.
  */
@@ -1292,6 +1321,7 @@ BVHTree *BKE_bvhtree_from_mesh_get(struct BVHTreeFromMesh *data,
       break;
 
     case BVHTREE_FROM_LOOPTRI:
+    case BVHTREE_FROM_LOOPTRI_NO_HIDDEN:
       data_cp.nearest_callback = mesh_looptri_nearest_point;
       data_cp.raycast_callback = mesh_looptri_spherecast;
 
@@ -1306,10 +1336,14 @@ BVHTree *BKE_bvhtree_from_mesh_get(struct BVHTreeFromMesh *data,
         data_cp.cached = bvhcache_find(
             mesh->runtime.bvh_cache, BVHTREE_FROM_LOOPTRI, &data_cp.tree);
         if (data_cp.cached == false) {
-          int looptri_num = BKE_mesh_runtime_looptri_len(mesh);
-          /* this assert checks we have looptris,
-           * if not caller should use DM_ensure_looptri() */
-          BLI_assert(!(looptri_num == 0 && mesh->totpoly != 0));
+          BLI_bitmap *looptri_mask = NULL;
+          int looptri_mask_active_len = -1;
+          int looptri_len = BKE_mesh_runtime_looptri_len(mesh);
+
+          if (type == BVHTREE_FROM_LOOPTRI_NO_HIDDEN) {
+            looptri_mask = looptri_no_hidden_map_get(
+                mesh->mpoly, looptri_len, &looptri_mask_active_len);
+          }
 
           data_cp.tree = bvhtree_from_mesh_looptri_create_tree(0.0,
                                                                tree_type,
@@ -1317,9 +1351,9 @@ BVHTree *BKE_bvhtree_from_mesh_get(struct BVHTreeFromMesh *data,
                                                                data_cp.vert,
                                                                data_cp.loop,
                                                                data_cp.looptri,
-                                                               looptri_num,
-                                                               NULL,
-                                                               -1);
+                                                               looptri_len,
+                                                               looptri_mask,
+                                                               looptri_mask_active_len);
 
           /* Save on cache for later use */
           /* printf("BVHTree built and saved on cache\n"); */
diff --git a/source/blender/editors/transform/transform_snap_object.c b/source/blender/editors/transform/transform_snap_object.c
index 829365367ee..f81d01ca754 100644
--- a/source/blender/editors/transform/transform_snap_object.c
+++ b/source/blender/editors/transform/transform_snap_object.c
@@ -337,6 +337,7 @@ static bool raycastMesh(SnapObjectContext *sctx,
                         Mesh *me,
                         float obmat[4][4],
                         const unsigned int ob_index,
+                        bool use_hide,
                         /* read/write args */
                         float *ray_depth,
                         /* return args */
@@ -419,7 +420,12 @@ static bool raycastMesh(SnapObjectContext *sctx,
   }
 
   if (treedata->tree == NULL) {
-    BKE_bvhtree_from_mesh_get(treedata, me, BVHTREE_FROM_LOOPTRI, 4);
+    if (use_hide) {
+      BKE_bvhtree_from_mesh_get(treedata, me, BVHTREE_FROM_LOOPTRI_NO_HIDDEN, 4);
+    }
+    else {
+      BKE_bvhtree_from_mesh_get(treedata, me, BVHTREE_FROM_LOOPTRI, 4);
+    }
 
     /* required for snapping with occlusion. */
     treedata->edge = me->medge;
@@ -723,6 +729,7 @@ static bool raycastObj(SnapObjectContext *sctx,
       }
 
       Mesh *me = ob->data;
+      bool use_hide = false;
       if (BKE_object_is_in_editmode(ob)) {
         BMEditMesh *em = BKE_editmesh_from_object(ob);
         if (use_obedit) {
@@ -742,6 +749,7 @@ static bool raycastObj(SnapObjectContext *sctx,
         }
         else if (em->mesh_eval_final) {
           me = em->mesh_eval_final;
+          use_hide = true;
         }
       }
       retval = raycastMesh(sctx,
@@ -751,6 +759,7 @@ static bool raycastObj(SnapObjectContext *sctx,
                            me,
                            obmat,
                            ob_index,
+                           use_hide,
                            ray_depth,
                            r_loc,
                            r_no,