Fix part of T70295: sculpt drawing not clipping PBVH behind the camera

Use all 6 clipping planes for drawing.

5 files changed, 42 insertions, 36 deletions

diff --git a/source/blender/blenkernel/BKE_pbvh.h b/source/blender/blenkernel/BKE_pbvh.h
index 899dddf31e8..3930fec71db 100644
--- a/source/blender/blenkernel/BKE_pbvh.h
+++ b/source/blender/blenkernel/BKE_pbvh.h
@@ -63,6 +63,11 @@ typedef enum {
   PBVH_UpdateTopology = 256,
 } PBVHNodeFlags;
 
+typedef struct PBVHFrustumPlanes {
+  float (*planes)[4];
+  int num_planes;
+} PBVHFrustumPlanes;
+
 /* Callbacks */
 
 /* returns 1 if the search should continue from this node, 0 otherwise */
@@ -167,7 +172,7 @@ bool BKE_pbvh_node_find_nearest_to_ray(PBVH *bvh,
 /* Drawing */
 
 void BKE_pbvh_draw_cb(PBVH *bvh,
-                      float (*planes)[4],
+                      PBVHFrustumPlanes *frustum,
                       void (*draw_fn)(void *user_data, struct GPU_PBVH_Buffers *buffers),
                       void *user_data);
 
@@ -245,10 +250,10 @@ void BKE_pbvh_node_get_original_BB(PBVHNode *node, float bb_min[3], float bb_max
 
 float BKE_pbvh_node_get_tmin(PBVHNode *node);
 
-/* test if AABB is at least partially inside the planes' volume */
-bool BKE_pbvh_node_planes_contain_AABB(PBVHNode *node, void *data);
-/* test if AABB is at least partially outside the planes' volume */
-bool BKE_pbvh_node_planes_exclude_AABB(PBVHNode *node, void *data);
+/* test if AABB is at least partially inside the PBVHFrustumPlanes volume */
+bool BKE_pbvh_node_frustum_contain_AABB(PBVHNode *node, void *frustum);
+/* test if AABB is at least partially outside the PBVHFrustumPlanes volume */
+bool BKE_pbvh_node_frustum_exclude_AABB(PBVHNode *node, void *frustum);
 
 struct GSet *BKE_pbvh_bmesh_node_unique_verts(PBVHNode *node);
 struct GSet *BKE_pbvh_bmesh_node_other_verts(PBVHNode *node);
diff --git a/source/blender/blenkernel/intern/pbvh.c b/source/blender/blenkernel/intern/pbvh.c
index eac18c79891..636a5c62e4d 100644
--- a/source/blender/blenkernel/intern/pbvh.c
+++ b/source/blender/blenkernel/intern/pbvh.c
@@ -2167,13 +2167,14 @@ typedef enum {
  * Returns true if the AABB is at least partially within the frustum
  * (ok, not a real frustum), false otherwise.
  */
-static PlaneAABBIsect test_planes_aabb(const float bb_min[3],
-                                       const float bb_max[3],
-                                       const float (*planes)[4])
+static PlaneAABBIsect test_frustum_aabb(const float bb_min[3],
+                                        const float bb_max[3],
+                                        PBVHFrustumPlanes *frustum)
 {
   PlaneAABBIsect ret = ISECT_INSIDE;
+  float(*planes)[4] = frustum->planes;
 
-  for (int i = 0; i < 4; i++) {
+  for (int i = 0; i < frustum->num_planes; i++) {
     float vmin[3], vmax[3];
 
     for (int axis = 0; axis < 3; axis++) {
@@ -2198,24 +2199,24 @@ static PlaneAABBIsect test_planes_aabb(const float bb_min[3],
   return ret;
 }
 
-bool BKE_pbvh_node_planes_contain_AABB(PBVHNode *node, void *data)
+bool BKE_pbvh_node_frustum_contain_AABB(PBVHNode *node, void *data)
 {
   const float *bb_min, *bb_max;
   /* BKE_pbvh_node_get_BB */
   bb_min = node->vb.bmin;
   bb_max = node->vb.bmax;
 
-  return test_planes_aabb(bb_min, bb_max, data) != ISECT_OUTSIDE;
+  return test_frustum_aabb(bb_min, bb_max, data) != ISECT_OUTSIDE;
 }
 
-bool BKE_pbvh_node_planes_exclude_AABB(PBVHNode *node, void *data)
+bool BKE_pbvh_node_frustum_exclude_AABB(PBVHNode *node, void *data)
 {
   const float *bb_min, *bb_max;
   /* BKE_pbvh_node_get_BB */
   bb_min = node->vb.bmin;
   bb_max = node->vb.bmax;
 
-  return test_planes_aabb(bb_min, bb_max, data) != ISECT_INSIDE;
+  return test_frustum_aabb(bb_min, bb_max, data) != ISECT_INSIDE;
 }
 
 typedef struct PBVHNodeDrawCallbackData {
@@ -2282,7 +2283,7 @@ void BKE_pbvh_update_draw_buffers(PBVH *bvh, bool show_vcol)
  * Version of #BKE_pbvh_draw that runs a callback.
  */
 void BKE_pbvh_draw_cb(PBVH *bvh,
-                      float (*planes)[4],
+                      PBVHFrustumPlanes *frustum,
                       void (*draw_fn)(void *user_data, GPU_PBVH_Buffers *buffers),
                       void *user_data)
 {
@@ -2291,9 +2292,9 @@ void BKE_pbvh_draw_cb(PBVH *bvh,
       .user_data = user_data,
   };
 
-  if (planes) {
+  if (frustum) {
     BKE_pbvh_search_callback(
-        bvh, BKE_pbvh_node_planes_contain_AABB, planes, pbvh_node_draw_cb, &draw_data);
+        bvh, BKE_pbvh_node_frustum_contain_AABB, frustum, pbvh_node_draw_cb, &draw_data);
   }
   else {
     BKE_pbvh_search_callback(bvh, NULL, NULL, pbvh_node_draw_cb, &draw_data);
diff --git a/source/blender/draw/intern/draw_manager_data.c b/source/blender/draw/intern/draw_manager_data.c
index 302d726f74e..7887f6874b3 100644
--- a/source/blender/draw/intern/draw_manager_data.c
+++ b/source/blender/draw/intern/draw_manager_data.c
@@ -888,25 +888,17 @@ static void sculpt_debug_cb(void *user_data,
 }
 #endif
 
-static void drw_sculpt_get_frustum_planes(Object *ob, float planes[4][4])
+static void drw_sculpt_get_frustum_planes(Object *ob, float planes[6][4])
 {
   /* TODO: take into account partial redraw for clipping planes. */
-  float frustum_planes[6][4];
-  DRW_view_frustum_planes_get(DRW_view_default_get(), frustum_planes);
-
-  /* PBVH only needs X/Y clipping planes, no Z depth.
-   * TODO: support Z depth clipping in PBVH. */
-  copy_v4_v4(planes[0], frustum_planes[0]);
-  copy_v4_v4(planes[1], frustum_planes[1]);
-  copy_v4_v4(planes[2], frustum_planes[3]);
-  copy_v4_v4(planes[3], frustum_planes[5]);
+  DRW_view_frustum_planes_get(DRW_view_default_get(), planes);
 
   /* Transform clipping planes to object space. Transforming a plane with a
    * 4x4 matrix is done by multiplying with the tranpose inverse. The inverse
    * cancels out here since we transform by inverse(obmat). */
   float tmat[4][4];
   transpose_m4_m4(tmat, ob->obmat);
-  for (int i = 0; i < 4; i++) {
+  for (int i = 0; i < 6; i++) {
     mul_m4_v4(tmat, planes[i]);
   }
 }
@@ -919,8 +911,9 @@ static void drw_sculpt_generate_calls(DRWSculptCallbackData *scd, bool use_vcol)
     return;
   }
 
-  float planes[4][4];
+  float planes[6][4];
   drw_sculpt_get_frustum_planes(scd->ob, planes);
+  PBVHFrustumPlanes frustum = {.planes = planes, .num_planes = 6};
 
   scd->fast_mode = false;
 
@@ -936,7 +929,7 @@ static void drw_sculpt_generate_calls(DRWSculptCallbackData *scd, bool use_vcol)
   BKE_pbvh_update_normals(pbvh, mesh->runtime.subdiv_ccg);
   BKE_pbvh_update_draw_buffers(pbvh, use_vcol);
 
-  BKE_pbvh_draw_cb(pbvh, planes, (void (*)(void *, GPU_PBVH_Buffers *))sculpt_draw_cb, scd);
+  BKE_pbvh_draw_cb(pbvh, &frustum, (void (*)(void *, GPU_PBVH_Buffers *))sculpt_draw_cb, scd);
 
 #ifdef SCULPT_DEBUG_BUFFERS
   int node_nr = 0;
diff --git a/source/blender/editors/sculpt_paint/paint_hide.c b/source/blender/editors/sculpt_paint/paint_hide.c
index 3463ceb7341..3e55a5d1b36 100644
--- a/source/blender/editors/sculpt_paint/paint_hide.c
+++ b/source/blender/editors/sculpt_paint/paint_hide.c
@@ -323,17 +323,18 @@ static void get_pbvh_nodes(
   /* select search callback */
   switch (mode) {
     case PARTIALVIS_INSIDE:
-      cb = BKE_pbvh_node_planes_contain_AABB;
+      cb = BKE_pbvh_node_frustum_contain_AABB;
       break;
     case PARTIALVIS_OUTSIDE:
-      cb = BKE_pbvh_node_planes_exclude_AABB;
+      cb = BKE_pbvh_node_frustum_exclude_AABB;
       break;
     case PARTIALVIS_ALL:
     case PARTIALVIS_MASKED:
       break;
   }
 
-  BKE_pbvh_search_gather(pbvh, cb, clip_planes, nodes, totnode);
+  PBVHFrustumPlanes frustum = {.planes = clip_planes, .num_planes = 4};
+  BKE_pbvh_search_gather(pbvh, cb, &frustum, nodes, totnode);
 }
 
 static int hide_show_exec(bContext *C, wmOperator *op)
@@ -364,6 +365,8 @@ static int hide_show_exec(bContext *C, wmOperator *op)
   get_pbvh_nodes(pbvh, &nodes, &totnode, clip_planes, area);
   pbvh_type = BKE_pbvh_type(pbvh);
 
+  negate_m4(clip_planes);
+
   /* start undo */
   switch (action) {
     case PARTIALVIS_HIDE:
diff --git a/source/blender/editors/sculpt_paint/paint_mask.c b/source/blender/editors/sculpt_paint/paint_mask.c
index 3a8ea3c296c..52514fa8922 100644
--- a/source/blender/editors/sculpt_paint/paint_mask.c
+++ b/source/blender/editors/sculpt_paint/paint_mask.c
@@ -314,8 +314,10 @@ bool ED_sculpt_mask_box_select(struct bContext *C, ViewContext *vc, const rcti *
         flip_plane(clip_planes_final[j], clip_planes[j], symmpass);
       }
 
-      BKE_pbvh_search_gather(
-          pbvh, BKE_pbvh_node_planes_contain_AABB, clip_planes_final, &nodes, &totnode);
+      PBVHFrustumPlanes frustum = {.planes = clip_planes_final, .num_planes = 4};
+      BKE_pbvh_search_gather(pbvh, BKE_pbvh_node_frustum_contain_AABB, &frustum, &nodes, &totnode);
+
+      negate_m4(clip_planes_final);
 
       MaskTaskData data = {
           .ob = ob,
@@ -482,7 +484,6 @@ static int paint_mask_gesture_lasso_exec(bContext *C, wmOperator *op)
                                   &data);
 
     ED_view3d_clipping_calc(&bb, clip_planes, vc.ar, vc.obact, &data.rect);
-    negate_m4(clip_planes);
 
     BKE_sculpt_update_object_for_edit(depsgraph, ob, false, true);
     pbvh = ob->sculpt->pbvh;
@@ -504,8 +505,11 @@ static int paint_mask_gesture_lasso_exec(bContext *C, wmOperator *op)
 
         /* gather nodes inside lasso's enclosing rectangle
          * (should greatly help with bigger meshes) */
+        PBVHFrustumPlanes frustum = {.planes = clip_planes_final, .num_planes = 4};
         BKE_pbvh_search_gather(
-            pbvh, BKE_pbvh_node_planes_contain_AABB, clip_planes_final, &nodes, &totnode);
+            pbvh, BKE_pbvh_node_frustum_contain_AABB, &frustum, &nodes, &totnode);
+
+        negate_m4(clip_planes_final);
 
         data.task_data.ob = ob;
         data.task_data.pbvh = pbvh;