Sculpt: Use mpoly flags to sync Face Sets visibility

Previously, all Face Set visibility logic was using mvert flags directly
to store the visibility state on the vertices while sculpting. As Face
Sets are a poly attribute, it is much simpler to use mpoly flags and let
BKE_mesh_flush_hidden_from_polys handle the vertex visibility, even for
Multires.

Now all operators that update the Face Set visibility state will always
copy the visibility to the mesh (using poly flags) and the grids, all
using the same code.

This should fix a lot of visibility glitches and bugs like the following:
 - Sculpt visibility reset when changing multires levels.
 - Multires visibility not updating in edit mode.
 - Single face visibible when surrounded by visibile face set, even when
the face set was hidden.

Reviewed By: sergey

Differential Revision: https://developer.blender.org/D9175

6 files changed, 40 insertions, 67 deletions

diff --git a/source/blender/blenkernel/BKE_paint.h b/source/blender/blenkernel/BKE_paint.h
index 0d3241ba6b8..150d0d9b011 100644
--- a/source/blender/blenkernel/BKE_paint.h
+++ b/source/blender/blenkernel/BKE_paint.h
@@ -612,6 +612,11 @@ void BKE_sculpt_bvh_update_from_ccg(struct PBVH *pbvh, struct SubdivCCG *subdiv_
  * updated according to the face sets. */
 void BKE_sculpt_sync_face_set_visibility(struct Mesh *mesh, struct SubdivCCG *subdiv_ccg);
 
+/* Individual function to sync the Face Set visibility to mesh and grids. */
+void BKE_sculpt_sync_face_sets_visibility_to_base_mesh(struct Mesh *mesh);
+void BKE_sculpt_sync_face_sets_visibility_to_grids(struct Mesh *mesh,
+                                                   struct SubdivCCG *subdiv_ccg);
+
 /* Ensures that a Face Set data-layers exists. If it does not, it creates one respecting the
  * visibility stored in the vertices of the mesh. If it does, it copies the visibility from the
  * mesh to the Face Sets. */
diff --git a/source/blender/blenkernel/intern/paint.c b/source/blender/blenkernel/intern/paint.c
index 2488895c779..88415140a5b 100644
--- a/source/blender/blenkernel/intern/paint.c
+++ b/source/blender/blenkernel/intern/paint.c
@@ -1894,63 +1894,41 @@ void BKE_sculpt_face_sets_ensure_from_base_mesh_visibility(Mesh *mesh)
 
   int *face_sets = CustomData_get_layer(&mesh->pdata, CD_SCULPT_FACE_SETS);
 
-  /* Show the only the face sets that have all visible vertices. */
   for (int i = 0; i < mesh->totpoly; i++) {
-    for (int l = 0; l < mesh->mpoly[i].totloop; l++) {
-      MLoop *loop = &mesh->mloop[mesh->mpoly[i].loopstart + l];
-      if (mesh->mvert[loop->v].flag & ME_HIDE) {
-        if (initialize_new_face_sets) {
-          /* When initializing a new Face Set data-layer, assign a new hidden Face Set ID to hidden
-           * vertices. This way, we get at initial split in two Face Sets between hidden and
-           * visible vertices based on the previous mesh visibly from other mode that can be
-           * useful in some cases. */
-          face_sets[i] = face_sets_default_hidden_id;
-        }
-        else {
-          /* Otherwise, set the already existing Face Set ID to hidden. */
-          face_sets[i] = -abs(face_sets[i]);
-        }
-        break;
-      }
+    if (!(mesh->mpoly[i].flag & ME_HIDE)) {
+      continue;
+    }
+
+    if (initialize_new_face_sets) {
+      /* When initializing a new Face Set data-layer, assign a new hidden Face Set ID to hidden
+       * vertices. This way, we get at initial split in two Face Sets between hidden and
+       * visible vertices based on the previous mesh visibly from other mode that can be
+       * useful in some cases. */
+      face_sets[i] = face_sets_default_hidden_id;
+    }
+    else {
+      /* Otherwise, set the already existing Face Set ID to hidden. */
+      face_sets[i] = -abs(face_sets[i]);
     }
   }
 }
 
-static void sculpt_sync_face_sets_visibility_to_base_mesh(Mesh *mesh)
+void BKE_sculpt_sync_face_sets_visibility_to_base_mesh(Mesh *mesh)
 {
   int *face_sets = CustomData_get_layer(&mesh->pdata, CD_SCULPT_FACE_SETS);
   if (!face_sets) {
     return;
   }
 
-  /* Enabled if the vertex should be visible according to the Face Sets. */
-  BLI_bitmap *visible_vertex = BLI_BITMAP_NEW(mesh->totvert, "visible vertices");
-  /* Enabled if the visibility of this vertex can be affected by the Face Sets to avoid modifying
-   * disconnected geometry. */
-  BLI_bitmap *modified_vertex = BLI_BITMAP_NEW(mesh->totvert, "modified vertices");
-
   for (int i = 0; i < mesh->totpoly; i++) {
     const bool is_face_set_visible = face_sets[i] >= 0;
-    for (int l = 0; l < mesh->mpoly[i].totloop; l++) {
-      MLoop *loop = &mesh->mloop[mesh->mpoly[i].loopstart + l];
-      if (is_face_set_visible) {
-        BLI_BITMAP_ENABLE(visible_vertex, loop->v);
-      }
-      BLI_BITMAP_ENABLE(modified_vertex, loop->v);
-    }
-  }
-
-  for (int i = 0; i < mesh->totvert; i++) {
-    if (BLI_BITMAP_TEST(modified_vertex, i) && !BLI_BITMAP_TEST(visible_vertex, i)) {
-      mesh->mvert[i].flag |= ME_HIDE;
-    }
+    SET_FLAG_FROM_TEST(mesh->mpoly[i].flag, !is_face_set_visible, ME_HIDE);
   }
 
-  MEM_SAFE_FREE(visible_vertex);
-  MEM_SAFE_FREE(modified_vertex);
+  BKE_mesh_flush_hidden_from_polys(mesh);
 }
 
-static void sculpt_sync_face_sets_visibility_to_grids(Mesh *mesh, SubdivCCG *subdiv_ccg)
+void BKE_sculpt_sync_face_sets_visibility_to_grids(Mesh *mesh, SubdivCCG *subdiv_ccg)
 {
   int *face_sets = CustomData_get_layer(&mesh->pdata, CD_SCULPT_FACE_SETS);
   if (!face_sets) {
@@ -1984,8 +1962,8 @@ static void sculpt_sync_face_sets_visibility_to_grids(Mesh *mesh, SubdivCCG *sub
 void BKE_sculpt_sync_face_set_visibility(struct Mesh *mesh, struct SubdivCCG *subdiv_ccg)
 {
   BKE_sculpt_face_sets_ensure_from_base_mesh_visibility(mesh);
-  sculpt_sync_face_sets_visibility_to_base_mesh(mesh);
-  sculpt_sync_face_sets_visibility_to_grids(mesh, subdiv_ccg);
+  BKE_sculpt_sync_face_sets_visibility_to_base_mesh(mesh);
+  BKE_sculpt_sync_face_sets_visibility_to_grids(mesh, subdiv_ccg);
 }
 
 static PBVH *build_pbvh_for_dynamic_topology(Object *ob)
diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index 4e15728b081..f512da504eb 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -544,22 +544,19 @@ bool SCULPT_vertex_has_face_set(SculptSession *ss, int index, int face_set)
   return true;
 }
 
-static void sculpt_visibility_sync_face_sets_to_vertex(SculptSession *ss, int index)
-{
-  SCULPT_vertex_visible_set(ss, index, SCULPT_vertex_any_face_set_visible_get(ss, index));
-}
-
-void SCULPT_visibility_sync_all_face_sets_to_vertices(SculptSession *ss)
+void SCULPT_visibility_sync_all_face_sets_to_vertices(Object *ob)
 {
+  SculptSession *ss = ob->sculpt;
+  Mesh *mesh = BKE_object_get_original_mesh(ob);
   switch (BKE_pbvh_type(ss->pbvh)) {
     case PBVH_FACES: {
-      for (int i = 0; i < ss->totvert; i++) {
-        sculpt_visibility_sync_face_sets_to_vertex(ss, i);
-      }
+      BKE_sculpt_sync_face_sets_visibility_to_base_mesh(mesh);
       break;
     }
     case PBVH_GRIDS: {
-      BKE_pbvh_sync_face_sets_to_grids(ss->pbvh);
+      BKE_sculpt_sync_face_sets_visibility_to_base_mesh(mesh);
+      BKE_sculpt_sync_face_sets_visibility_to_grids(mesh, ss->subdiv_ccg);
+      break;
     }
     case PBVH_BMESH:
       break;
@@ -8176,6 +8173,9 @@ void ED_object_sculptmode_enter_ex(Main *bmain,
    * freed memory. */
   BKE_object_free_derived_caches(ob);
 
+  /* Copy the current mesh visibility to the Face Sets. */
+  BKE_sculpt_face_sets_ensure_from_base_mesh_visibility(me);
+
   sculpt_init_session(depsgraph, scene, ob);
 
   /* Mask layer is required. */
diff --git a/source/blender/editors/sculpt_paint/sculpt_face_set.c b/source/blender/editors/sculpt_paint/sculpt_face_set.c
index dae4b068e93..152d23dfa5b 100644
--- a/source/blender/editors/sculpt_paint/sculpt_face_set.c
+++ b/source/blender/editors/sculpt_paint/sculpt_face_set.c
@@ -720,7 +720,7 @@ static int sculpt_face_set_init_exec(bContext *C, wmOperator *op)
   SCULPT_undo_push_end();
 
   /* Sync face sets visibility and vertex visibility as now all Face Sets are visible. */
-  SCULPT_visibility_sync_all_face_sets_to_vertices(ss);
+  SCULPT_visibility_sync_all_face_sets_to_vertices(ob);
 
   for (int i = 0; i < totnode; i++) {
     BKE_pbvh_node_mark_update_visibility(nodes[i]);
@@ -889,12 +889,6 @@ static int sculpt_face_sets_change_visibility_exec(bContext *C, wmOperator *op)
   if (mode == SCULPT_FACE_SET_VISIBILITY_SHOW_ACTIVE) {
     SCULPT_face_sets_visibility_all_set(ss, false);
     SCULPT_face_set_visibility_set(ss, active_face_set, true);
-    for (int i = 0; i < tot_vert; i++) {
-      SCULPT_vertex_visible_set(ss,
-                                i,
-                                SCULPT_vertex_visible_get(ss, i) &&
-                                    SCULPT_vertex_has_face_set(ss, i, active_face_set));
-    }
   }
 
   if (mode == SCULPT_FACE_SET_VISIBILITY_HIDE_ACTIVE) {
@@ -918,7 +912,7 @@ static int sculpt_face_sets_change_visibility_exec(bContext *C, wmOperator *op)
   }
 
   /* Sync face sets visibility and vertex visibility. */
-  SCULPT_visibility_sync_all_face_sets_to_vertices(ss);
+  SCULPT_visibility_sync_all_face_sets_to_vertices(ob);
 
   SCULPT_undo_push_end();
 
@@ -930,10 +924,6 @@ static int sculpt_face_sets_change_visibility_exec(bContext *C, wmOperator *op)
 
   MEM_SAFE_FREE(nodes);
 
-  if (BKE_pbvh_type(pbvh) == PBVH_FACES) {
-    BKE_mesh_flush_hidden_from_verts(ob->data);
-  }
-
   ED_region_tag_redraw(region);
   DEG_id_tag_update(&ob->id, ID_RECALC_SHADING);
 
@@ -1194,7 +1184,7 @@ static int sculpt_face_set_edit_invoke(bContext *C, wmOperator *op, const wmEven
   SCULPT_undo_push_end();
 
   /* Sync face sets visibility and vertex visibility as now all Face Sets are visible. */
-  SCULPT_visibility_sync_all_face_sets_to_vertices(ss);
+  SCULPT_visibility_sync_all_face_sets_to_vertices(ob);
 
   for (int i = 0; i < totnode; i++) {
     BKE_pbvh_node_mark_update_visibility(nodes[i]);
diff --git a/source/blender/editors/sculpt_paint/sculpt_intern.h b/source/blender/editors/sculpt_paint/sculpt_intern.h
index 1fddfc7d6db..ec4d594293a 100644
--- a/source/blender/editors/sculpt_paint/sculpt_intern.h
+++ b/source/blender/editors/sculpt_paint/sculpt_intern.h
@@ -186,7 +186,7 @@ bool SCULPT_vertex_is_boundary(const SculptSession *ss, const int index);
 void SCULPT_vertex_visible_set(SculptSession *ss, int index, bool visible);
 bool SCULPT_vertex_visible_get(SculptSession *ss, int index);
 
-void SCULPT_visibility_sync_all_face_sets_to_vertices(struct SculptSession *ss);
+void SCULPT_visibility_sync_all_face_sets_to_vertices(struct Object *ob);
 void SCULPT_visibility_sync_all_vertex_to_face_sets(struct SculptSession *ss);
 
 /* Face Sets API */
diff --git a/source/blender/editors/sculpt_paint/sculpt_undo.c b/source/blender/editors/sculpt_paint/sculpt_undo.c
index c67a3145be8..fa9eb43891c 100644
--- a/source/blender/editors/sculpt_paint/sculpt_undo.c
+++ b/source/blender/editors/sculpt_paint/sculpt_undo.c
@@ -647,7 +647,7 @@ static void sculpt_undo_restore_list(bContext *C, Depsgraph *depsgraph, ListBase
 
       BKE_sculpt_update_object_for_edit(depsgraph, ob, true, need_mask, false);
 
-      SCULPT_visibility_sync_all_face_sets_to_vertices(ss);
+      SCULPT_visibility_sync_all_face_sets_to_vertices(ob);
 
       BKE_pbvh_update_vertex_data(ss->pbvh, PBVH_UpdateVisibility);