Fix T72721: Add visibility flags updates to the PBVH

Currently, there its a function that sets manually the fully_hidden flag
of the nodes from the visibility operators in paint_hide.c. The undo
code was not updating the flag, so the visibility state of the nodes was
incorrect after preforming undo operations. This sometimes was drawing
fully hidden nodes with empty buffers, causing artifacts in the
geometry.

I added a function to mark nodes which visibility state changed (similar
as we are updating the mask flags and the nodes bounding boxes). This
way, the tools, operators and undo code don't have to update the
visibility flags, making everything much simpler to understand and
maintain.

I did not remove the flag update code from the current visibility
operators in this patch, but after reimplementing them (and all the new
ones) in the new visibility system, all visibility updates should be
done using this method and the BKE_pbvh_node_fully_hidden_set function
should be removed.

Reviewed By: jbakker

Maniphest Tasks: T72721

Differential Revision: https://developer.blender.org/D6767

1 files changed, 8 insertions, 5 deletions

diff --git a/source/blender/blenkernel/BKE_pbvh.h b/source/blender/blenkernel/BKE_pbvh.h
index 9eb10d296de..a26148d11bf 100644
--- a/source/blender/blenkernel/BKE_pbvh.h
+++ b/source/blender/blenkernel/BKE_pbvh.h
@@ -66,13 +66,14 @@ typedef enum {
   PBVH_UpdateDrawBuffers = 1 << 4,
   PBVH_UpdateRedraw = 1 << 5,
   PBVH_UpdateMask = 1 << 6,
+  PBVH_UpdateVisibility = 1 << 8,
 
-  PBVH_RebuildDrawBuffers = 1 << 7,
-  PBVH_FullyHidden = 1 << 8,
-  PBVH_FullyMasked = 1 << 9,
-  PBVH_FullyUnmasked = 1 << 10,
+  PBVH_RebuildDrawBuffers = 1 << 9,
+  PBVH_FullyHidden = 1 << 10,
+  PBVH_FullyMasked = 1 << 11,
+  PBVH_FullyUnmasked = 1 << 12,
 
-  PBVH_UpdateTopology = 1 << 11,
+  PBVH_UpdateTopology = 1 << 13,
 } PBVHNodeFlags;
 
 typedef struct PBVHFrustumPlanes {
@@ -246,6 +247,7 @@ void BKE_pbvh_node_mark_update_mask(PBVHNode *node);
 void BKE_pbvh_node_mark_rebuild_draw(PBVHNode *node);
 void BKE_pbvh_node_mark_redraw(PBVHNode *node);
 void BKE_pbvh_node_mark_normals_update(PBVHNode *node);
+void BKE_pbvh_node_mark_visibility_update(PBVHNode *node);
 void BKE_pbvh_node_mark_topology_update(PBVHNode *node);
 void BKE_pbvh_node_fully_hidden_set(PBVHNode *node, int fully_hidden);
 void BKE_pbvh_node_fully_masked_set(PBVHNode *node, int fully_masked);
@@ -286,6 +288,7 @@ void BKE_pbvh_bmesh_after_stroke(PBVH *bvh);
 
 void BKE_pbvh_update_bounds(PBVH *bvh, int flags);
 void BKE_pbvh_update_vertex_data(PBVH *bvh, int flags);
+void BKE_pbvh_update_visibility(PBVH *bvh);
 void BKE_pbvh_update_normals(PBVH *bvh, struct SubdivCCG *subdiv_ccg);
 void BKE_pbvh_redraw_BB(PBVH *bvh, float bb_min[3], float bb_max[3]);
 void BKE_pbvh_get_grid_updates(PBVH *bvh, bool clear, void ***r_gridfaces, int *r_totface);