Fix T76865: Vertex paint draws hidden but cannot be painted onto

4 files changed, 64 insertions, 21 deletions

diff --git a/source/blender/blenkernel/BKE_pbvh.h b/source/blender/blenkernel/BKE_pbvh.h
index 1779385b97b..1eeb340d071 100644
--- a/source/blender/blenkernel/BKE_pbvh.h
+++ b/source/blender/blenkernel/BKE_pbvh.h
@@ -308,6 +308,8 @@ void BKE_pbvh_face_sets_set(PBVH *bvh, int *face_sets);
 
 void BKE_pbvh_face_sets_color_set(PBVH *bvh, int seed, int color_default);
 
+void BKE_pbvh_respect_hide_set(PBVH *bvh, bool respect_hide);
+
 /* vertex deformer */
 float (*BKE_pbvh_vert_coords_alloc(struct PBVH *pbvh))[3];
 void BKE_pbvh_vert_coords_apply(struct PBVH *pbvh, const float (*vertCos)[3], const int totvert);
@@ -333,6 +335,7 @@ typedef struct PBVHVertexIter {
   int gy;
   int i;
   int index;
+  bool respect_hide;
 
   /* grid */
   struct CCGKey key;
@@ -401,9 +404,15 @@ void pbvh_vertex_iter_init(PBVH *bvh, PBVHNode *node, PBVHVertexIter *vi, int mo
         } \
         else if (vi.mverts) { \
           vi.mvert = &vi.mverts[vi.vert_indices[vi.gx]]; \
-          vi.visible = !(vi.mvert->flag & ME_HIDE); \
-          if (mode == PBVH_ITER_UNIQUE && !vi.visible) \
-            continue; \
+          if (vi.respect_hide) { \
+            vi.visible = !(vi.mvert->flag & ME_HIDE); \
+            if (mode == PBVH_ITER_UNIQUE && !vi.visible) { \
+              continue; \
+            } \
+          } \
+          else { \
+            BLI_assert(vi.visible); \
+          } \
           vi.co = vi.mvert->co; \
           vi.no = vi.mvert->no; \
           vi.index = vi.vert_indices[vi.i]; \
diff --git a/source/blender/blenkernel/intern/paint.c b/source/blender/blenkernel/intern/paint.c
index a7f973dfdbb..0c9b1721f79 100644
--- a/source/blender/blenkernel/intern/paint.c
+++ b/source/blender/blenkernel/intern/paint.c
@@ -1480,6 +1480,7 @@ static void sculpt_update_object(
   SculptSession *ss = ob->sculpt;
   Mesh *me = BKE_object_get_original_mesh(ob);
   MultiresModifierData *mmd = BKE_sculpt_multires_active(scene, ob);
+  const bool use_face_sets = (ob->mode & OB_MODE_SCULPT) != 0;
 
   ss->deform_modifiers_active = sculpt_modifiers_active(scene, sd, ob);
   ss->show_mask = (sd->flags & SCULPT_HIDE_MASK) == 0;
@@ -1535,17 +1536,22 @@ static void sculpt_update_object(
   }
 
   /* Sculpt Face Sets. */
-  if (!CustomData_has_layer(&me->pdata, CD_SCULPT_FACE_SETS)) {
-    ss->face_sets = CustomData_add_layer(
-        &me->pdata, CD_SCULPT_FACE_SETS, CD_CALLOC, NULL, me->totpoly);
-    for (int i = 0; i < me->totpoly; i++) {
-      ss->face_sets[i] = 1;
-    }
+  if (use_face_sets) {
+    if (!CustomData_has_layer(&me->pdata, CD_SCULPT_FACE_SETS)) {
+      ss->face_sets = CustomData_add_layer(
+          &me->pdata, CD_SCULPT_FACE_SETS, CD_CALLOC, NULL, me->totpoly);
+      for (int i = 0; i < me->totpoly; i++) {
+        ss->face_sets[i] = 1;
+      }
 
-    /* Set the default face set color if the datalayer did not exist. */
-    me->face_sets_color_default = 1;
+      /* Set the default face set color if the datalayer did not exist. */
+      me->face_sets_color_default = 1;
+    }
+    ss->face_sets = CustomData_get_layer(&me->pdata, CD_SCULPT_FACE_SETS);
+  }
+  else {
+    ss->face_sets = NULL;
   }
-  ss->face_sets = CustomData_get_layer(&me->pdata, CD_SCULPT_FACE_SETS);
 
   ss->subdiv_ccg = me_eval->runtime.subdiv_ccg;
 
@@ -1805,11 +1811,12 @@ static PBVH *build_pbvh_for_dynamic_topology(Object *ob)
   return pbvh;
 }
 
-static PBVH *build_pbvh_from_regular_mesh(Object *ob, Mesh *me_eval_deform)
+static PBVH *build_pbvh_from_regular_mesh(Object *ob, Mesh *me_eval_deform, bool respect_hide)
 {
   Mesh *me = BKE_object_get_original_mesh(ob);
   const int looptris_num = poly_to_tri_count(me->totpoly, me->totloop);
   PBVH *pbvh = BKE_pbvh_new();
+  BKE_pbvh_respect_hide_set(pbvh, respect_hide);
 
   MLoopTri *looptri = MEM_malloc_arrayN(looptris_num, sizeof(*looptri), __func__);
 
@@ -1841,11 +1848,12 @@ static PBVH *build_pbvh_from_regular_mesh(Object *ob, Mesh *me_eval_deform)
   return pbvh;
 }
 
-static PBVH *build_pbvh_from_ccg(Object *ob, SubdivCCG *subdiv_ccg)
+static PBVH *build_pbvh_from_ccg(Object *ob, SubdivCCG *subdiv_ccg, bool respect_hide)
 {
   CCGKey key;
   BKE_subdiv_ccg_key_top_level(&key, subdiv_ccg);
   PBVH *pbvh = BKE_pbvh_new();
+  BKE_pbvh_respect_hide_set(pbvh, respect_hide);
   BKE_pbvh_build_grids(pbvh,
                        subdiv_ccg->grids,
                        subdiv_ccg->num_grids,
@@ -1863,6 +1871,14 @@ PBVH *BKE_sculpt_object_pbvh_ensure(Depsgraph *depsgraph, Object *ob)
   if (ob == NULL || ob->sculpt == NULL) {
     return NULL;
   }
+
+  bool respect_hide = true;
+  if (ob->mode & (OB_MODE_VERTEX_PAINT | OB_MODE_WEIGHT_PAINT)) {
+    if (!(BKE_paint_select_vert_test(ob) || BKE_paint_select_face_test(ob))) {
+      respect_hide = false;
+    }
+  }
+
   PBVH *pbvh = ob->sculpt->pbvh;
   if (pbvh != NULL) {
     /* NOTE: It is possible that grids were re-allocated due to modifier
@@ -1886,11 +1902,11 @@ PBVH *BKE_sculpt_object_pbvh_ensure(Depsgraph *depsgraph, Object *ob)
     Object *object_eval = DEG_get_evaluated_object(depsgraph, ob);
     Mesh *mesh_eval = object_eval->data;
     if (mesh_eval->runtime.subdiv_ccg != NULL) {
-      pbvh = build_pbvh_from_ccg(ob, mesh_eval->runtime.subdiv_ccg);
+      pbvh = build_pbvh_from_ccg(ob, mesh_eval->runtime.subdiv_ccg, respect_hide);
     }
     else if (ob->type == OB_MESH) {
       Mesh *me_eval_deform = object_eval->runtime.mesh_deform_eval;
-      pbvh = build_pbvh_from_regular_mesh(ob, me_eval_deform);
+      pbvh = build_pbvh_from_regular_mesh(ob, me_eval_deform, respect_hide);
     }
   }
 
diff --git a/source/blender/blenkernel/intern/pbvh.c b/source/blender/blenkernel/intern/pbvh.c
index d925a2b4db9..17edf94a53c 100644
--- a/source/blender/blenkernel/intern/pbvh.c
+++ b/source/blender/blenkernel/intern/pbvh.c
@@ -295,6 +295,10 @@ static void build_mesh_leaf_node(PBVH *bvh, PBVHNode *node)
 
   node->face_vert_indices = (const int(*)[3])face_vert_indices;
 
+  if (bvh->respect_hide == false) {
+    has_visible = true;
+  }
+
   for (int i = 0; i < totface; i++) {
     const MLoopTri *lt = &bvh->looptri[node->prim_indices[i]];
     for (int j = 0; j < 3; j++) {
@@ -302,8 +306,10 @@ static void build_mesh_leaf_node(PBVH *bvh, PBVHNode *node)
           bvh, map, &node->face_verts, &node->uniq_verts, bvh->mloop[lt->tri[j]].v);
     }
 
-    if (!paint_is_face_hidden(lt, bvh->verts, bvh->mloop)) {
-      has_visible = true;
+    if (has_visible == false) {
+      if (!paint_is_face_hidden(lt, bvh->verts, bvh->mloop)) {
+        has_visible = true;
+      }
     }
   }
 
@@ -666,7 +672,7 @@ void BKE_pbvh_build_grids(PBVH *bvh,
 PBVH *BKE_pbvh_new(void)
 {
   PBVH *bvh = MEM_callocN(sizeof(PBVH), "pbvh");
-
+  bvh->respect_hide = true;
   return bvh;
 }
 
@@ -2117,7 +2123,7 @@ static bool pbvh_faces_node_raycast(PBVH *bvh,
     const MLoopTri *lt = &bvh->looptri[faces[i]];
     const int *face_verts = node->face_vert_indices[i];
 
-    if (paint_is_face_hidden(lt, vert, mloop)) {
+    if (bvh->respect_hide && paint_is_face_hidden(lt, vert, mloop)) {
       continue;
     }
 
@@ -2426,7 +2432,7 @@ static bool pbvh_faces_node_nearest_to_ray(PBVH *bvh,
     const MLoopTri *lt = &bvh->looptri[faces[i]];
     const int *face_verts = node->face_vert_indices[i];
 
-    if (paint_is_face_hidden(lt, vert, mloop)) {
+    if (bvh->respect_hide && paint_is_face_hidden(lt, vert, mloop)) {
       continue;
     }
 
@@ -2900,6 +2906,12 @@ void pbvh_vertex_iter_init(PBVH *bvh, PBVHNode *node, PBVHVertexIter *vi, int mo
   vi->fno = NULL;
   vi->mvert = NULL;
 
+  vi->respect_hide = bvh->respect_hide;
+  if (bvh->respect_hide == false) {
+    /* The same value for all vertices. */
+    vi->visible = true;
+  }
+
   BKE_pbvh_node_get_grids(bvh, node, &grid_indices, &totgrid, NULL, &gridsize, &grids);
   BKE_pbvh_node_num_verts(bvh, node, &uniq_verts, &totvert);
   BKE_pbvh_node_get_verts(bvh, node, &vert_indices, &verts);
@@ -3014,3 +3026,8 @@ void BKE_pbvh_face_sets_set(PBVH *bvh, int *face_sets)
 {
   bvh->face_sets = face_sets;
 }
+
+void BKE_pbvh_respect_hide_set(PBVH *bvh, bool respect_hide)
+{
+  bvh->respect_hide = respect_hide;
+}
diff --git a/source/blender/blenkernel/intern/pbvh_intern.h b/source/blender/blenkernel/intern/pbvh_intern.h
index d3e42ac7705..7397f939894 100644
--- a/source/blender/blenkernel/intern/pbvh_intern.h
+++ b/source/blender/blenkernel/intern/pbvh_intern.h
@@ -160,6 +160,7 @@ struct PBVH {
   bool deformed;
   bool show_mask;
   bool show_face_sets;
+  bool respect_hide;
 
   /* Dynamic topology */
   BMesh *bm;