Weight & Vertex Paint: always respect edit mode hiding on faces.

In some cases it is mandatory to be able to hide parts of the mesh
in order to paint certain areas. The Mask modifier doesn't work in
weight paint, and edit mode hiding requires using selection, which
is not always convenient.

This makes the weight and vertex paint modes always respect edit mode
hiding like sculpt mode. The change in behavior affects drawing and
building paint PBVH. Thus it affects brushes, but not menu operators
like Smooth or Normalize.

In addition, this makes the Alt-H shortcut available even without
any selection enabled, and implements Hide for vertex selection.

Differential Revision: https://developer.blender.org/D14163

5 files changed, 112 insertions, 11 deletions

diff --git a/source/blender/editors/include/ED_mesh.h b/source/blender/editors/include/ED_mesh.h
index 30a98129ee6..52044109702 100644
--- a/source/blender/editors/include/ED_mesh.h
+++ b/source/blender/editors/include/ED_mesh.h
@@ -425,6 +425,9 @@ void paintvert_select_ungrouped(struct Object *ob, bool extend, bool flush_flags
 void paintvert_flush_flags(struct Object *ob);
 void paintvert_tag_select_update(struct bContext *C, struct Object *ob);
 
+void paintvert_hide(struct bContext *C, struct Object *ob, bool unselected);
+void paintvert_reveal(struct bContext *C, struct Object *ob, bool select);
+
 /* mirrtopo */
 typedef struct MirrTopoStore_t {
   intptr_t *index_lookup;
diff --git a/source/blender/editors/mesh/editface.cc b/source/blender/editors/mesh/editface.cc
index 69fe69fe117..b69cd8b8606 100644
--- a/source/blender/editors/mesh/editface.cc
+++ b/source/blender/editors/mesh/editface.cc
@@ -551,3 +551,54 @@ void paintvert_select_ungrouped(Object *ob, bool extend, bool flush_flags)
     paintvert_flush_flags(ob);
   }
 }
+
+void paintvert_hide(bContext *C, Object *ob, const bool unselected)
+{
+  Mesh *const me = BKE_mesh_from_object(ob);
+
+  if (me == NULL || me->totvert == 0) {
+    return;
+  }
+
+  for (int i = 0; i < me->totvert; i++) {
+    MVert *const mvert = &me->mvert[i];
+
+    if ((mvert->flag & ME_HIDE) == 0) {
+      if (((mvert->flag & SELECT) == 0) == unselected) {
+        mvert->flag |= ME_HIDE;
+      }
+    }
+
+    if (mvert->flag & ME_HIDE) {
+      mvert->flag &= ~SELECT;
+    }
+  }
+
+  BKE_mesh_flush_hidden_from_verts(me);
+
+  paintvert_flush_flags(ob);
+  paintvert_tag_select_update(C, ob);
+}
+
+void paintvert_reveal(bContext *C, Object *ob, const bool select)
+{
+  Mesh *const me = BKE_mesh_from_object(ob);
+
+  if (me == NULL || me->totvert == 0) {
+    return;
+  }
+
+  for (int i = 0; i < me->totvert; i++) {
+    MVert *const mvert = &me->mvert[i];
+
+    if (mvert->flag & ME_HIDE) {
+      SET_FLAG_FROM_TEST(mvert->flag, select, SELECT);
+      mvert->flag &= ~ME_HIDE;
+    }
+  }
+
+  BKE_mesh_flush_hidden_from_verts(me);
+
+  paintvert_flush_flags(ob);
+  paintvert_tag_select_update(C, ob);
+}
diff --git a/source/blender/editors/sculpt_paint/paint_intern.h b/source/blender/editors/sculpt_paint/paint_intern.h
index ea17114efa5..02c3b5be8b9 100644
--- a/source/blender/editors/sculpt_paint/paint_intern.h
+++ b/source/blender/editors/sculpt_paint/paint_intern.h
@@ -370,10 +370,12 @@ void PAINT_OT_face_select_linked(struct wmOperatorType *ot);
 void PAINT_OT_face_select_linked_pick(struct wmOperatorType *ot);
 void PAINT_OT_face_select_all(struct wmOperatorType *ot);
 void PAINT_OT_face_select_hide(struct wmOperatorType *ot);
-void PAINT_OT_face_select_reveal(struct wmOperatorType *ot);
+
+void PAINT_OT_face_vert_reveal(struct wmOperatorType *ot);
 
 void PAINT_OT_vert_select_all(struct wmOperatorType *ot);
 void PAINT_OT_vert_select_ungrouped(struct wmOperatorType *ot);
+void PAINT_OT_vert_select_hide(struct wmOperatorType *ot);
 
 bool vert_paint_poll(struct bContext *C);
 bool mask_paint_poll(struct bContext *C);
diff --git a/source/blender/editors/sculpt_paint/paint_ops.c b/source/blender/editors/sculpt_paint/paint_ops.c
index ce6b397af15..994ae4011b4 100644
--- a/source/blender/editors/sculpt_paint/paint_ops.c
+++ b/source/blender/editors/sculpt_paint/paint_ops.c
@@ -1454,6 +1454,7 @@ void ED_operatortypes_paint(void)
   /* vertex selection */
   WM_operatortype_append(PAINT_OT_vert_select_all);
   WM_operatortype_append(PAINT_OT_vert_select_ungrouped);
+  WM_operatortype_append(PAINT_OT_vert_select_hide);
 
   /* vertex */
   WM_operatortype_append(PAINT_OT_vertex_paint_toggle);
@@ -1472,7 +1473,8 @@ void ED_operatortypes_paint(void)
   WM_operatortype_append(PAINT_OT_face_select_linked_pick);
   WM_operatortype_append(PAINT_OT_face_select_all);
   WM_operatortype_append(PAINT_OT_face_select_hide);
-  WM_operatortype_append(PAINT_OT_face_select_reveal);
+
+  WM_operatortype_append(PAINT_OT_face_vert_reveal);
 
   /* partial visibility */
   WM_operatortype_append(PAINT_OT_hide_show);
diff --git a/source/blender/editors/sculpt_paint/paint_utils.c b/source/blender/editors/sculpt_paint/paint_utils.c
index 3f26f590b70..1f272882100 100644
--- a/source/blender/editors/sculpt_paint/paint_utils.c
+++ b/source/blender/editors/sculpt_paint/paint_utils.c
@@ -749,25 +749,68 @@ void PAINT_OT_face_select_hide(wmOperatorType *ot)
       ot->srna, "unselected", 0, "Unselected", "Hide unselected rather than selected objects");
 }
 
-static int face_select_reveal_exec(bContext *C, wmOperator *op)
+static int vert_select_hide_exec(bContext *C, wmOperator *op)
+{
+  const bool unselected = RNA_boolean_get(op->ptr, "unselected");
+  Object *ob = CTX_data_active_object(C);
+  paintvert_hide(C, ob, unselected);
+  ED_region_tag_redraw(CTX_wm_region(C));
+  return OPERATOR_FINISHED;
+}
+
+void PAINT_OT_vert_select_hide(wmOperatorType *ot)
+{
+  ot->name = "Vertex Select Hide";
+  ot->description = "Hide selected vertices";
+  ot->idname = "PAINT_OT_vert_select_hide";
+
+  ot->exec = vert_select_hide_exec;
+  ot->poll = vert_paint_poll;
+
+  ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
+
+  RNA_def_boolean(
+      ot->srna, "unselected", 0, "Unselected", "Hide unselected rather than selected vertices");
+}
+
+static int face_vert_reveal_exec(bContext *C, wmOperator *op)
 {
   const bool select = RNA_boolean_get(op->ptr, "select");
   Object *ob = CTX_data_active_object(C);
-  paintface_reveal(C, ob, select);
+
+  if (BKE_paint_select_vert_test(ob)) {
+    paintvert_reveal(C, ob, select);
+  }
+  else {
+    paintface_reveal(C, ob, select);
+  }
+
   ED_region_tag_redraw(CTX_wm_region(C));
   return OPERATOR_FINISHED;
 }
 
-void PAINT_OT_face_select_reveal(wmOperatorType *ot)
+static bool face_vert_reveal_poll(bContext *C)
 {
-  ot->name = "Face Select Reveal";
-  ot->description = "Reveal hidden faces";
-  ot->idname = "PAINT_OT_face_select_reveal";
+  Object *ob = CTX_data_active_object(C);
 
-  ot->exec = face_select_reveal_exec;
-  ot->poll = facemask_paint_poll;
+  /* Allow using this operator when no selection is enabled but hiding is applied. */
+  return BKE_paint_select_elem_test(ob) || BKE_paint_always_hide_test(ob);
+}
+
+void PAINT_OT_face_vert_reveal(wmOperatorType *ot)
+{
+  ot->name = "Reveal Faces/Vertices";
+  ot->description = "Reveal hidden faces and vertices";
+  ot->idname = "PAINT_OT_face_vert_reveal";
+
+  ot->exec = face_vert_reveal_exec;
+  ot->poll = face_vert_reveal_poll;
 
   ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
 
-  RNA_def_boolean(ot->srna, "select", true, "Select", "");
+  RNA_def_boolean(ot->srna,
+                  "select",
+                  true,
+                  "Select",
+                  "Specifies whether the newly revealed geometry should be selected");
 }