Sculpt: Auto-masking UI improvements

Add auto-masking as a popover in the header while in Sculpt mode,
following the design in T101593.

These properties were present in the Options panel (and popover),
they have been removed from there.

Moreover, this commit makes the auto-masking section in Brush settings
match the new popover.

In the future this popover can be used for other modes that support
auto-masking such as Grease Pencil.

See D16145 for details and screenshots.

Reviewed By: JulienKaspar

Differential Revision: https://developer.blender.org/D16145

5 files changed, 138 insertions, 80 deletions

diff --git a/release/scripts/startup/bl_ui/properties_paint_common.py b/release/scripts/startup/bl_ui/properties_paint_common.py
index 72a87703bd5..0e49a506e73 100644
--- a/release/scripts/startup/bl_ui/properties_paint_common.py
+++ b/release/scripts/startup/bl_ui/properties_paint_common.py
@@ -928,60 +928,80 @@ def brush_settings_advanced(layout, context, brush, popover=False):
     use_frontface = False
 
     if mode == 'SCULPT':
+        sculpt = context.tool_settings.sculpt
         capabilities = brush.sculpt_capabilities
         use_accumulate = capabilities.has_accumulate
         use_frontface = True
 
         col = layout.column(heading="Auto-Masking", align=True)
 
-        # topology automasking
+        col = layout.column(align=True)
         col.prop(brush, "use_automasking_topology", text="Topology")
-
-        # face masks automasking
         col.prop(brush, "use_automasking_face_sets", text="Face Sets")
 
-        # boundary edges/face sets automasking
+        layout.separator()
+
+        col = layout.column(align=True)
         col.prop(brush, "use_automasking_boundary_edges", text="Mesh Boundary")
         col.prop(brush, "use_automasking_boundary_face_sets", text="Face Sets Boundary")
-        col.prop(brush, "use_automasking_cavity", text="Cavity")
-        col.prop(brush, "use_automasking_cavity_inverted", text="Cavity (Inverted)")
-        col.prop(brush, "use_automasking_start_normal", text="Area Normal")
-        col.prop(brush, "use_automasking_view_normal", text="View Normal")
 
-        col.separator()
-        col.prop(brush, "automasking_boundary_edges_propagation_steps")
+        if brush.use_automasking_boundary_edges or brush.use_automasking_boundary_face_sets:
+            col = layout.column()
+            col.use_property_split = False
+            split = col.split(factor=0.4)
+            col = split.column()
+            split.prop(brush, "automasking_boundary_edges_propagation_steps")
 
-        sculpt = context.tool_settings.sculpt
+        layout.separator()
 
-        if brush.use_automasking_start_normal:
-            col.separator()
+        col = layout.column(align=True)
+        row = col.row()
+        row.prop(brush, "use_automasking_cavity", text="Cavity")
 
-            col.prop(sculpt, "automasking_start_normal_limit")
-            col.prop(sculpt, "automasking_start_normal_falloff")
+        is_cavity_active = brush.use_automasking_cavity or brush.use_automasking_cavity_inverted
 
-        if brush.use_automasking_view_normal:
-            col.separator()
+        if is_cavity_active:
+            row.operator("sculpt.mask_from_cavity", text="Create Mask")
 
-            col.prop(brush, "use_automasking_view_occlusion", text="Occlusion")
-            col.prop(sculpt, "automasking_view_normal_limit")
-            col.prop(sculpt, "automasking_view_normal_falloff")
+        col.prop(brush, "use_automasking_cavity_inverted", text="Cavity (inverted)")
 
-        if brush.use_automasking_cavity or brush.use_automasking_cavity_inverted:
-            col.separator()
+        if is_cavity_active:
+            col = layout.column(align=True)
+            col.prop(brush, "automasking_cavity_factor", text="Factor")
+            col.prop(brush, "automasking_cavity_blur_steps", text="Blur")
 
-            col.prop(brush, "automasking_cavity_factor", text="Cavity Factor")
-            col.prop(brush, "automasking_cavity_blur_steps", text="Cavity Blur")
-            col.prop(brush, "use_automasking_custom_cavity_curve", text="Use Curve")
+            col = layout.column()
+            col.prop(brush, "use_automasking_custom_cavity_curve", text="Custom Curve")
 
             if brush.use_automasking_custom_cavity_curve:
                 col.template_curve_mapping(brush, "automasking_cavity_curve")
 
         layout.separator()
 
+        col = layout.column(align=True)
+        col.prop(brush, "use_automasking_view_normal", text="View Normal")
+
+        if brush.use_automasking_view_normal:
+            col.prop(brush, "use_automasking_view_occlusion", text="Occlusion")
+            subcol = col.column(align=True)
+            subcol.active = not brush.use_automasking_view_occlusion
+            subcol.prop(sculpt, "automasking_view_normal_limit", text="Limit")
+            subcol.prop(sculpt, "automasking_view_normal_falloff", text="Falloff")
+
+        col = layout.column()
+        col.prop(brush, "use_automasking_start_normal", text="Area Normal")
+
+        if brush.use_automasking_start_normal:
+            col = layout.column(align=True)
+            col.prop(sculpt, "automasking_start_normal_limit", text="Limit")
+            col.prop(sculpt, "automasking_start_normal_falloff", text="Falloff")
+
+        layout.separator()
+
         # sculpt plane settings
         if capabilities.has_sculpt_plane:
             layout.prop(brush, "sculpt_plane")
-            col = layout.column(heading="Use Original", align=True)
+            col = layout.column(heading="Original", align=True)
             col.prop(brush, "use_original_normal", text="Normal")
             col.prop(brush, "use_original_plane", text="Plane")
             layout.separator()
diff --git a/release/scripts/startup/bl_ui/space_view3d.py b/release/scripts/startup/bl_ui/space_view3d.py
index 3a02492635a..ea257498e11 100644
--- a/release/scripts/startup/bl_ui/space_view3d.py
+++ b/release/scripts/startup/bl_ui/space_view3d.py
@@ -838,12 +838,18 @@ class VIEW3D_HT_header(Header):
                         text="Guides",
                     )
 
-            layout.separator_spacer()
+        elif object_mode == 'SCULPT':
+            layout.popover(
+                panel="VIEW3D_PT_sculpt_automasking",
+                text="",
+                icon="MOD_MASK"
+            )
+
         else:
             # Transform settings depending on tool header visibility
             VIEW3D_HT_header.draw_xform_template(layout, context)
 
-            layout.separator_spacer()
+        layout.separator_spacer()
 
         # Viewport Settings
         layout.popover(
@@ -7680,6 +7686,77 @@ class VIEW3D_PT_paint_weight_context_menu(Panel):
         )
 
 
+class VIEW3D_PT_sculpt_automasking(Panel):
+    bl_space_type = 'VIEW_3D'
+    bl_region_type = 'HEADER'
+    bl_label = "Auto-Masking"
+    bl_ui_units_x = 10
+
+    def draw(self, context):
+        layout = self.layout
+
+        tool_settings = context.tool_settings
+        sculpt = tool_settings.sculpt
+        layout.label(text="Auto-Masking")
+
+        col = layout.column(align=True)
+        col.prop(sculpt, "use_automasking_topology", text="Topology")
+        col.prop(sculpt, "use_automasking_face_sets", text="Face Sets")
+
+        col.separator()
+
+        col = layout.column(align=True)
+        col.prop(sculpt, "use_automasking_boundary_edges", text="Mesh Boundary")
+        col.prop(sculpt, "use_automasking_boundary_face_sets", text="Face Sets Boundary")
+
+        if sculpt.use_automasking_boundary_edges or sculpt.use_automasking_boundary_face_sets:
+            col.prop(sculpt.brush, "automasking_boundary_edges_propagation_steps")
+
+        col.separator()
+
+        col = layout.column(align=True)
+        row = col.row()
+        row.prop(sculpt, "use_automasking_cavity", text="Cavity")
+
+        is_cavity_active = sculpt.use_automasking_cavity or sculpt.use_automasking_cavity_inverted
+
+        if is_cavity_active:
+            row.operator("sculpt.mask_from_cavity", text="Create Mask")
+
+        col.prop(sculpt, "use_automasking_cavity_inverted", text="Cavity (inverted)")
+
+        if is_cavity_active:
+            col = layout.column(align=True)
+            col.prop(sculpt, "automasking_cavity_factor", text="Factor")
+            col.prop(sculpt, "automasking_cavity_blur_steps", text="Blur")
+
+            col = layout.column()
+            col.prop(sculpt, "use_automasking_custom_cavity_curve", text="Custom Curve")
+
+            if sculpt.use_automasking_custom_cavity_curve:
+                col.template_curve_mapping(sculpt, "automasking_cavity_curve")
+
+        col.separator()
+
+        col = layout.column(align=True)
+        col.prop(sculpt, "use_automasking_view_normal", text="View Normal")
+
+        if sculpt.use_automasking_view_normal:
+            col.prop(sculpt, "use_automasking_view_occlusion", text="Occlusion")
+            subcol = col.column(align=True)
+            subcol.active = not sculpt.use_automasking_view_occlusion
+            subcol.prop(sculpt, "automasking_view_normal_limit", text="Limit")
+            subcol.prop(sculpt, "automasking_view_normal_falloff", text="Falloff")
+
+        col = layout.column()
+        col.prop(sculpt, "use_automasking_start_normal", text="Area Normal")
+
+        if sculpt.use_automasking_start_normal:
+            col = layout.column(align=True)
+            col.prop(sculpt, "automasking_start_normal_limit", text="Limit")
+            col.prop(sculpt, "automasking_start_normal_falloff", text="Falloff")
+
+
 class VIEW3D_PT_sculpt_context_menu(Panel):
     # Only for popover, these are dummy values.
     bl_space_type = 'VIEW_3D'
@@ -8070,6 +8147,7 @@ classes = (
     VIEW3D_PT_gpencil_sculpt_context_menu,
     VIEW3D_PT_gpencil_weight_context_menu,
     VIEW3D_PT_gpencil_draw_context_menu,
+    VIEW3D_PT_sculpt_automasking,
     VIEW3D_PT_sculpt_context_menu,
     TOPBAR_PT_gpencil_materials,
     TOPBAR_PT_gpencil_vertexcolor,
diff --git a/release/scripts/startup/bl_ui/space_view3d_toolbar.py b/release/scripts/startup/bl_ui/space_view3d_toolbar.py
index 3357676bf2f..328e45f79b9 100644
--- a/release/scripts/startup/bl_ui/space_view3d_toolbar.py
+++ b/release/scripts/startup/bl_ui/space_view3d_toolbar.py
@@ -964,52 +964,6 @@ class VIEW3D_PT_sculpt_options(Panel, View3DPaintPanel):
         col.prop(sculpt, "use_sculpt_delay_updates")
         col.prop(sculpt, "use_deform_only")
 
-        col.separator()
-
-        col = layout.column(heading="Auto-Masking", align=True)
-
-        col.prop(sculpt, "use_automasking_topology", text="Topology")
-        col.prop(sculpt, "use_automasking_face_sets", text="Face Sets")
-        col.prop(sculpt, "use_automasking_boundary_edges", text="Mesh Boundary")
-        col.prop(sculpt, "use_automasking_boundary_face_sets", text="Face Sets Boundary")
-        col.prop(sculpt, "use_automasking_cavity", text="Cavity")
-        col.prop(sculpt, "use_automasking_cavity_inverted", text="Cavity (Inverted)")
-        col.prop(sculpt, "use_automasking_start_normal", text="Area Normal")
-        col.prop(sculpt, "use_automasking_view_normal", text="View Normal")
-
-        if sculpt.use_automasking_start_normal:
-            col.separator()
-
-            col.prop(sculpt, "automasking_start_normal_limit")
-            col.prop(sculpt, "automasking_start_normal_falloff")
-
-        if sculpt.use_automasking_view_normal:
-            col.separator()
-
-            col.prop(sculpt, "use_automasking_view_occlusion", text="Occlusion")
-            col.prop(sculpt, "automasking_view_normal_limit")
-            col.prop(sculpt, "automasking_view_normal_falloff")
-
-        col.separator()
-        col.prop(sculpt.brush, "automasking_boundary_edges_propagation_steps")
-
-        if sculpt.use_automasking_cavity or sculpt.use_automasking_cavity_inverted:
-            col.separator()
-
-            col2 = col.column()
-            props = col2.operator("sculpt.mask_from_cavity", text="Mask From Cavity")
-            props.use_automask_settings = True
-
-            col2 = col.column()
-
-            col2.prop(sculpt, "automasking_cavity_factor", text="Cavity Factor")
-            col2.prop(sculpt, "automasking_cavity_blur_steps", text="Cavity Blur")
-
-            col2.prop(sculpt, "use_automasking_custom_cavity_curve", text="Use Curve")
-
-            if sculpt.use_automasking_custom_cavity_curve:
-                col2.template_curve_mapping(sculpt, "automasking_cavity_curve")
-
 
 class VIEW3D_PT_sculpt_options_gravity(Panel, View3DPaintPanel):
     bl_context = ".sculpt_mode"  # dot on purpose (access from topbar)
diff --git a/source/blender/makesrna/intern/rna_brush.c b/source/blender/makesrna/intern/rna_brush.c
index 5e4b08d8a41..b44b0620329 100644
--- a/source/blender/makesrna/intern/rna_brush.c
+++ b/source/blender/makesrna/intern/rna_brush.c
@@ -3240,18 +3240,21 @@ static void rna_def_brush(BlenderRNA *brna)
     RNA_def_property_update(prop, 0, "rna_Brush_update");
   } while ((++entry)->identifier);
 
-  prop = RNA_def_property(srna, "automasking_cavity_factor", PROP_FLOAT, PROP_NONE);
+  prop = RNA_def_property(srna, "automasking_cavity_factor", PROP_FLOAT, PROP_FACTOR);
   RNA_def_property_float_sdna(prop, NULL, "automasking_cavity_factor");
+  RNA_def_property_float_default(prop, 1.0f);
   RNA_def_property_ui_text(prop, "Cavity Factor", "The contrast of the cavity mask");
-  RNA_def_property_ui_range(prop, 0.0f, 1.0f, 0.1, 3);
   RNA_def_property_range(prop, 0.0f, 5.0f);
+  RNA_def_property_ui_range(prop, 0.0f, 1.0f, 0.1, 3);
   RNA_def_property_override_flag(prop, PROPOVERRIDE_OVERRIDABLE_LIBRARY);
   RNA_def_property_update(prop, 0, "rna_Brush_update");
 
   prop = RNA_def_property(srna, "automasking_cavity_blur_steps", PROP_INT, PROP_NONE);
   RNA_def_property_int_sdna(prop, NULL, "automasking_cavity_blur_steps");
+  RNA_def_property_int_default(prop, 0);
   RNA_def_property_ui_text(prop, "Blur Steps", "The number of times the cavity mask is blurred");
-  RNA_def_property_range(prop, 0.0f, 25.0f);
+  RNA_def_property_range(prop, 0, 25);
+  RNA_def_property_ui_range(prop, 0, 10, 1, 1);
   RNA_def_property_override_flag(prop, PROPOVERRIDE_OVERRIDABLE_LIBRARY);
   RNA_def_property_update(prop, 0, "rna_Brush_update");
 
diff --git a/source/blender/makesrna/intern/rna_sculpt_paint.c b/source/blender/makesrna/intern/rna_sculpt_paint.c
index d8ad3d1f41e..440309849ab 100644
--- a/source/blender/makesrna/intern/rna_sculpt_paint.c
+++ b/source/blender/makesrna/intern/rna_sculpt_paint.c
@@ -927,17 +927,20 @@ static void rna_def_sculpt(BlenderRNA *brna)
     RNA_def_property_update(prop, NC_SCENE | ND_TOOLSETTINGS, NULL);
   } while ((++entry)->identifier);
 
-  prop = RNA_def_property(srna, "automasking_cavity_factor", PROP_FLOAT, PROP_NONE);
+  prop = RNA_def_property(srna, "automasking_cavity_factor", PROP_FLOAT, PROP_FACTOR);
   RNA_def_property_float_sdna(prop, NULL, "automasking_cavity_factor");
   RNA_def_property_ui_text(prop, "Cavity Factor", "The contrast of the cavity mask");
-  RNA_def_property_ui_range(prop, 0.0f, 1.0f, 0.1, 3);
+  RNA_def_property_float_default(prop, 1.0f);
   RNA_def_property_range(prop, 0.0f, 5.0f);
+  RNA_def_property_ui_range(prop, 0.0f, 1.0f, 0.1, 3);
   RNA_def_property_update(prop, NC_SCENE | ND_TOOLSETTINGS, NULL);
 
   prop = RNA_def_property(srna, "automasking_cavity_blur_steps", PROP_INT, PROP_NONE);
   RNA_def_property_int_sdna(prop, NULL, "automasking_cavity_blur_steps");
   RNA_def_property_ui_text(prop, "Blur Steps", "The number of times the cavity mask is blurred");
-  RNA_def_property_range(prop, 0.0f, 25.0f);
+  RNA_def_property_int_default(prop, 0);
+  RNA_def_property_range(prop, 0, 25);
+  RNA_def_property_ui_range(prop, 0, 10, 1, 1);
   RNA_def_property_update(prop, NC_SCENE | ND_TOOLSETTINGS, NULL);
 
   prop = RNA_def_property(srna, "automasking_cavity_curve", PROP_POINTER, PROP_NONE);