Sculpt Vertex Colors: Initial implementation

Sculpt Vertex Colors is a painting system that runs inside sculpt mode, reusing all its tools and optimizations. This provides much better performance, easier to maintain code and more advanced features (new brush engine, filters, symmetry options, masks and face sets compatibility...). This is also the initial step for future features like vertex painting in Multires and brushes that can sculpt and paint at the same time.

This commit includes:
  - SCULPT_UNDO_COLOR for undo support in sculpt mode
  - SCULPT_UPDATE_COLOR and PBVH flags and rendering
  - Sculpt Color API functions
  - Sculpt capability for sculpt tools (only enabled in the Paint Brush for now)
  - Rendering support in workbench (default to Sculpt Vertex Colors except in Vertex Paint)
  - Conversion operator between MPropCol (Sculpt Vertex Colors) and MLoopCol (Vertex Paint)
  - Remesher reprojection in the Voxel Remehser
  - Paint Brush and Smear Brush with color smoothing in alt-smooth mode
  - Parameters for the new brush engine (density, opacity, flow, wet paint mixing, tip scale) implemented in Sculpt Vertex Colors
  - Color Filter
  - Color picker (uses S shortcut, replaces smooth)
  - Color selector in the top bar

Reviewed By: brecht

Maniphest Tasks: T72866

Differential Revision: https://developer.blender.org/D5975

3 files changed, 272 insertions, 5 deletions

diff --git a/source/blender/makesrna/RNA_access.h b/source/blender/makesrna/RNA_access.h
index 0a26be3a753..0b4f82ca263 100644
--- a/source/blender/makesrna/RNA_access.h
+++ b/source/blender/makesrna/RNA_access.h
@@ -387,6 +387,8 @@ extern StructRNA RNA_MeshIntProperty;
 extern StructRNA RNA_MeshLoop;
 extern StructRNA RNA_MeshLoopColor;
 extern StructRNA RNA_MeshLoopColorLayer;
+extern StructRNA RNA_MeshVertColor;
+extern StructRNA RNA_MeshVertColorLayer;
 extern StructRNA RNA_MeshLoopTriangle;
 extern StructRNA RNA_MeshPolygon;
 extern StructRNA RNA_MeshSequenceCacheModifier;
diff --git a/source/blender/makesrna/intern/rna_brush.c b/source/blender/makesrna/intern/rna_brush.c
index cc5cd97a8a0..59656b48cfe 100644
--- a/source/blender/makesrna/intern/rna_brush.c
+++ b/source/blender/makesrna/intern/rna_brush.c
@@ -99,6 +99,8 @@ const EnumPropertyItem rna_enum_brush_sculpt_tool_items[] = {
     {SCULPT_TOOL_CLOTH, "CLOTH", ICON_BRUSH_SCULPT_DRAW, "Cloth", ""},
     {SCULPT_TOOL_SIMPLIFY, "SIMPLIFY", ICON_BRUSH_DATA, "Simplify", ""},
     {SCULPT_TOOL_MASK, "MASK", ICON_BRUSH_MASK, "Mask", ""},
+    {SCULPT_TOOL_PAINT, "PAINT", ICON_BRUSH_SCULPT_DRAW, "Paint", ""},
+    {SCULPT_TOOL_SMEAR, "SMEAR", ICON_BRUSH_SCULPT_DRAW, "Smear", ""},
     {SCULPT_TOOL_DRAW_FACE_SETS, "DRAW_FACE_SETS", ICON_BRUSH_MASK, "Draw Face Sets", ""},
     {0, NULL, 0, NULL, NULL},
 };
@@ -314,7 +316,8 @@ static bool rna_BrushCapabilitiesSculpt_has_topology_rake_get(PointerRNA *ptr)
 static bool rna_BrushCapabilitiesSculpt_has_auto_smooth_get(PointerRNA *ptr)
 {
   Brush *br = (Brush *)ptr->data;
-  return !ELEM(br->sculpt_tool, SCULPT_TOOL_MASK, SCULPT_TOOL_SMOOTH);
+  return !ELEM(
+      br->sculpt_tool, SCULPT_TOOL_MASK, SCULPT_TOOL_SMOOTH, SCULPT_TOOL_PAINT, SCULPT_TOOL_SMEAR);
 }
 
 static bool rna_BrushCapabilitiesSculpt_has_height_get(PointerRNA *ptr)
@@ -409,6 +412,12 @@ static bool rna_BrushCapabilitiesSculpt_has_sculpt_plane_get(PointerRNA *ptr)
                SCULPT_TOOL_SMOOTH);
 }
 
+static bool rna_BrushCapabilitiesSculpt_has_color_get(PointerRNA *ptr)
+{
+  Brush *br = (Brush *)ptr->data;
+  return ELEM(br->sculpt_tool, SCULPT_TOOL_PAINT);
+}
+
 static bool rna_BrushCapabilitiesSculpt_has_secondary_color_get(PointerRNA *ptr)
 {
   Brush *br = (Brush *)ptr->data;
@@ -1053,6 +1062,7 @@ static void rna_def_sculpt_capabilities(BlenderRNA *brna)
   SCULPT_TOOL_CAPABILITY(has_plane_offset, "Has Plane Offset");
   SCULPT_TOOL_CAPABILITY(has_random_texture_angle, "Has Random Texture Angle");
   SCULPT_TOOL_CAPABILITY(has_sculpt_plane, "Has Sculpt Plane");
+  SCULPT_TOOL_CAPABILITY(has_color, "Has Color");
   SCULPT_TOOL_CAPABILITY(has_secondary_color, "Has Secondary Color");
   SCULPT_TOOL_CAPABILITY(has_smooth_stroke, "Has Smooth Stroke");
   SCULPT_TOOL_CAPABILITY(has_space_attenuation, "Has Space Attenuation");
@@ -2221,6 +2231,46 @@ static void rna_def_brush(BlenderRNA *brna)
       prop, "Strength", "How powerful the effect of the brush is when applied");
   RNA_def_property_update(prop, 0, "rna_Brush_update");
 
+  prop = RNA_def_property(srna, "flow", PROP_FLOAT, PROP_FACTOR);
+  RNA_def_property_float_sdna(prop, NULL, "flow");
+  RNA_def_property_range(prop, 0.0f, 1.0f);
+  RNA_def_property_ui_range(prop, 0.0f, 1.0f, 0.001, 3);
+  RNA_def_property_ui_text(prop, "Flow", "Amount of paint that is applied per stroke sample");
+  RNA_def_property_update(prop, 0, "rna_Brush_update");
+
+  prop = RNA_def_property(srna, "wet_mix", PROP_FLOAT, PROP_FACTOR);
+  RNA_def_property_float_sdna(prop, NULL, "wet_mix");
+  RNA_def_property_range(prop, 0.0f, 1.0f);
+  RNA_def_property_ui_range(prop, 0.0f, 1.0f, 0.001, 3);
+  RNA_def_property_ui_text(
+      prop, "Wet Mix", "Amount of paint that is picked from the surface into the brush color");
+  RNA_def_property_update(prop, 0, "rna_Brush_update");
+
+  prop = RNA_def_property(srna, "wet_persistence", PROP_FLOAT, PROP_FACTOR);
+  RNA_def_property_float_sdna(prop, NULL, "wet_persistence");
+  RNA_def_property_range(prop, 0.0f, 1.0f);
+  RNA_def_property_ui_range(prop, 0.0f, 1.0f, 0.001, 3);
+  RNA_def_property_ui_text(
+      prop,
+      "Wet Persistence",
+      "Amount of wet paint that stays in the brush after applyig paint to the surface");
+  RNA_def_property_update(prop, 0, "rna_Brush_update");
+
+  prop = RNA_def_property(srna, "density", PROP_FLOAT, PROP_FACTOR);
+  RNA_def_property_float_sdna(prop, NULL, "density");
+  RNA_def_property_range(prop, 0.0f, 1.0f);
+  RNA_def_property_ui_range(prop, 0.0f, 1.0f, 0.001, 3);
+  RNA_def_property_ui_text(
+      prop, "Density", "Amount of random elements that are going to be affected by the brush");
+  RNA_def_property_update(prop, 0, "rna_Brush_update");
+
+  prop = RNA_def_property(srna, "tip_scale_x", PROP_FLOAT, PROP_FACTOR);
+  RNA_def_property_float_sdna(prop, NULL, "tip_scale_x");
+  RNA_def_property_range(prop, 0.0f, 1.0f);
+  RNA_def_property_ui_range(prop, 0.0f, 1.0f, 0.001, 3);
+  RNA_def_property_ui_text(prop, "Tip Scale X", "Scale of the brush tip in the X axis");
+  RNA_def_property_update(prop, 0, "rna_Brush_update");
+
   prop = RNA_def_property(srna, "dash_ratio", PROP_FLOAT, PROP_FACTOR);
   RNA_def_property_float_sdna(prop, NULL, "dash_ratio");
   RNA_def_property_range(prop, 0.0f, 1.0f);
diff --git a/source/blender/makesrna/intern/rna_mesh.c b/source/blender/makesrna/intern/rna_mesh.c
index 5f986c28964..62318925a0b 100644
--- a/source/blender/makesrna/intern/rna_mesh.c
+++ b/source/blender/makesrna/intern/rna_mesh.c
@@ -758,6 +758,47 @@ static void rna_MeshLoopColorLayer_active_set(PointerRNA *ptr, bool value)
   rna_CustomDataLayer_active_set(ptr, rna_mesh_ldata(ptr), value, CD_MLOOPCOL, 0);
 }
 
+/* sculpt_vertex_color_layers */
+
+DEFINE_CUSTOMDATA_LAYER_COLLECTION(sculpt_vertex_color, vdata, CD_PROP_COLOR)
+DEFINE_CUSTOMDATA_LAYER_COLLECTION_ACTIVEITEM(
+    sculpt_vertex_color, vdata, CD_PROP_COLOR, active, MeshVertColorLayer)
+
+
+static void rna_MeshVertColorLayer_data_begin(CollectionPropertyIterator *iter, PointerRNA *ptr)
+{
+  Mesh *me = rna_mesh(ptr);
+  CustomDataLayer *layer = (CustomDataLayer *)ptr->data;
+  rna_iterator_array_begin(
+      iter, layer->data, sizeof(MPropCol), (me->edit_mesh) ? 0 : me->totvert, 0, NULL);
+}
+
+static int rna_MeshVertColorLayer_data_length(PointerRNA *ptr)
+{
+  Mesh *me = rna_mesh(ptr);
+  return (me->edit_mesh) ? 0 : me->totvert;
+}
+
+static bool rna_MeshVertColorLayer_active_render_get(PointerRNA *ptr)
+{
+  return rna_CustomDataLayer_active_get(ptr, rna_mesh_vdata(ptr), CD_PROP_COLOR, 1);
+}
+
+static bool rna_MeshVertColorLayer_active_get(PointerRNA *ptr)
+{
+  return rna_CustomDataLayer_active_get(ptr, rna_mesh_vdata(ptr), CD_PROP_COLOR, 0);
+}
+
+static void rna_MeshVertColorLayer_active_render_set(PointerRNA *ptr, bool value)
+{
+  rna_CustomDataLayer_active_set(ptr, rna_mesh_vdata(ptr), value, CD_PROP_COLOR, 1);
+}
+
+static void rna_MeshVertColorLayer_active_set(PointerRNA *ptr, bool value)
+{
+  rna_CustomDataLayer_active_set(ptr, rna_mesh_vdata(ptr), value, CD_PROP_COLOR, 0);
+}
+
 static int rna_float_layer_check(CollectionPropertyIterator *UNUSED(iter), void *data)
 {
   CustomDataLayer *layer = (CustomDataLayer *)data;
@@ -1218,6 +1259,19 @@ static char *rna_MeshColor_path(PointerRNA *ptr)
   return rna_LoopCustomData_data_path(ptr, "vertex_colors", CD_MLOOPCOL);
 }
 
+static char *rna_MeshVertColorLayer_path(PointerRNA *ptr)
+{
+  CustomDataLayer *cdl = ptr->data;
+  char name_esc[sizeof(cdl->name) * 2];
+  BLI_strescape(name_esc, cdl->name, sizeof(name_esc));
+  return BLI_sprintfN("sculpt_vertex_colors[\"%s\"]", name_esc);
+}
+
+static char *rna_MeshVertColor_path(PointerRNA *ptr)
+{
+  return rna_VertCustomData_data_path(ptr, "sculpt_vertex_colors", CD_PROP_COLOR);
+}
+
 /**** Float Property Layer API ****/
 static char *rna_MeshVertexFloatPropertyLayer_path(PointerRNA *ptr)
 {
@@ -1439,6 +1493,33 @@ static void rna_Mesh_vertex_color_remove(struct Mesh *me,
   }
 }
 
+static PointerRNA rna_Mesh_sculpt_vertex_color_new(struct Mesh *me,
+                                                   const char *name,
+                                                   const bool do_init)
+{
+  PointerRNA ptr;
+  CustomData *vdata;
+  CustomDataLayer *cdl = NULL;
+  int index = ED_mesh_sculpt_color_add(me, name, false, do_init);
+
+  if (index != -1) {
+    vdata = rna_mesh_vdata_helper(me);
+    cdl = &vdata->layers[CustomData_get_layer_index_n(vdata, CD_PROP_COLOR, index)];
+  }
+
+  RNA_pointer_create(&me->id, &RNA_MeshVertColorLayer, cdl, &ptr);
+  return ptr;
+}
+
+static void rna_Mesh_sculpt_vertex_color_remove(struct Mesh *me,
+                                                ReportList *reports,
+                                                CustomDataLayer *layer)
+{
+  if (ED_mesh_sculpt_color_remove_named(me, layer->name) == false) {
+    BKE_reportf(reports, RPT_ERROR, "Sculpt vertex color '%s' not found", layer->name);
+  }
+}
+
 #  define DEFINE_CUSTOMDATA_PROPERTY_API( \
       elemname, datatype, cd_prop_type, cdata, countvar, layertype) \
     static PointerRNA rna_Mesh_##elemname##_##datatype##_property_new(struct Mesh *me, \
@@ -1510,10 +1591,6 @@ static void UNUSED_FUNCTION(rna_mesh_unused)(void)
   (void)rna_Mesh_uv_layer_render_index_get;
   (void)rna_Mesh_uv_layer_render_index_set;
   (void)rna_Mesh_uv_layer_render_set;
-  (void)rna_Mesh_vertex_color_render_get;
-  (void)rna_Mesh_vertex_color_render_index_get;
-  (void)rna_Mesh_vertex_color_render_index_set;
-  (void)rna_Mesh_vertex_color_render_set;
   (void)rna_Mesh_face_map_index_range;
   (void)rna_Mesh_face_map_active_index_set;
   (void)rna_Mesh_face_map_active_index_get;
@@ -2034,6 +2111,65 @@ static void rna_def_mloopcol(BlenderRNA *brna)
   RNA_def_property_update(prop, 0, "rna_Mesh_update_data");
 }
 
+static void rna_def_MPropCol(BlenderRNA *brna)
+{
+  StructRNA *srna;
+  PropertyRNA *prop;
+
+  srna = RNA_def_struct(brna, "MeshVertColorLayer", NULL);
+  RNA_def_struct_ui_text(srna,
+                         "Mesh Sculpt Vertex Color Layer",
+                         "Layer of sculpt vertex colors in a Mesh data-block");
+  RNA_def_struct_sdna(srna, "CustomDataLayer");
+  RNA_def_struct_path_func(srna, "rna_MeshVertColorLayer_path");
+  RNA_def_struct_ui_icon(srna, ICON_GROUP_VCOL);
+
+  prop = RNA_def_property(srna, "name", PROP_STRING, PROP_NONE);
+  RNA_def_struct_name_property(srna, prop);
+  RNA_def_property_string_funcs(prop, NULL, NULL, "rna_MeshVertexLayer_name_set");
+  RNA_def_property_ui_text(prop, "Name", "Name of Sculpt Vertex color layer");
+  RNA_def_property_update(prop, 0, "rna_Mesh_update_data");
+
+  prop = RNA_def_property(srna, "active", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_funcs(
+      prop, "rna_MeshVertColorLayer_active_get", "rna_MeshVertColorLayer_active_set");
+  RNA_def_property_ui_text(
+      prop, "Active", "Sets the sculpt vertex color layer as active for display and editing");
+  RNA_def_property_update(prop, 0, "rna_Mesh_update_data");
+
+  prop = RNA_def_property(srna, "active_render", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_sdna(prop, NULL, "active_rnd", 0);
+  RNA_def_property_boolean_funcs(prop,
+                                 "rna_MeshVertColorLayer_active_render_get",
+                                 "rna_MeshVertColorLayer_active_render_set");
+  RNA_def_property_ui_text(
+      prop, "Active Render", "Sets the sculpt vertex color layer as active for rendering");
+  RNA_def_property_update(prop, 0, "rna_Mesh_update_data");
+
+  prop = RNA_def_property(srna, "data", PROP_COLLECTION, PROP_NONE);
+  RNA_def_property_struct_type(prop, "MeshVertColor");
+  RNA_def_property_ui_text(prop, "Data", "");
+  RNA_def_property_collection_funcs(prop,
+                                    "rna_MeshVertColorLayer_data_begin",
+                                    "rna_iterator_array_next",
+                                    "rna_iterator_array_end",
+                                    "rna_iterator_array_get",
+                                    "rna_MeshVertColorLayer_data_length",
+                                    NULL,
+                                    NULL,
+                                    NULL);
+
+  srna = RNA_def_struct(brna, "MeshVertColor", NULL);
+  RNA_def_struct_sdna(srna, "MPropCol");
+  RNA_def_struct_ui_text(srna, "Mesh Sculpt Vertex Color", "Vertex colors in a Mesh");
+  RNA_def_struct_path_func(srna, "rna_MeshVertColor_path");
+
+  prop = RNA_def_property(srna, "color", PROP_FLOAT, PROP_COLOR);
+  RNA_def_property_array(prop, 4);
+  RNA_def_property_range(prop, 0.0f, 1.0f);
+  RNA_def_property_ui_text(prop, "Color", "");
+  RNA_def_property_update(prop, 0, "rna_Mesh_update_data");
+}
 static void rna_def_mproperties(BlenderRNA *brna)
 {
   StructRNA *srna;
@@ -2373,6 +2509,60 @@ static void rna_def_loop_colors(BlenderRNA *brna, PropertyRNA *cprop)
   RNA_def_property_update(prop, 0, "rna_Mesh_update_data_edit_active_color");
 }
 
+static void rna_def_vert_colors(BlenderRNA *brna, PropertyRNA *cprop)
+{
+  StructRNA *srna;
+  PropertyRNA *prop;
+
+  FunctionRNA *func;
+  PropertyRNA *parm;
+
+  RNA_def_property_srna(cprop, "VertColors");
+  srna = RNA_def_struct(brna, "VertColors", NULL);
+  RNA_def_struct_sdna(srna, "Mesh");
+  RNA_def_struct_ui_text(srna, "Vert Colors", "Collection of sculpt vertex colors");
+
+  func = RNA_def_function(srna, "new", "rna_Mesh_sculpt_vertex_color_new");
+  RNA_def_function_ui_description(func, "Add a sculpt vertex color layer to Mesh");
+  RNA_def_string(func, "name", "Col", 0, "", "Sculpt Vertex color name");
+  RNA_def_boolean(func,
+                  "do_init",
+                  true,
+                  "",
+                  "Whether new layer's data should be initialized by copying current active one");
+  parm = RNA_def_pointer(func, "layer", "MeshVertColorLayer", "", "The newly created layer");
+  RNA_def_parameter_flags(parm, 0, PARM_RNAPTR);
+  RNA_def_function_return(func, parm);
+
+  func = RNA_def_function(srna, "remove", "rna_Mesh_sculpt_vertex_color_remove");
+  RNA_def_function_ui_description(func, "Remove a vertex color layer");
+  RNA_def_function_flag(func, FUNC_USE_REPORTS);
+  parm = RNA_def_pointer(func, "layer", "MeshVertColorLayer", "", "The layer to remove");
+  RNA_def_parameter_flags(parm, PROP_NEVER_NULL, PARM_REQUIRED);
+  RNA_def_property_clear_flag(parm, PROP_THICK_WRAP);
+
+  prop = RNA_def_property(srna, "active", PROP_POINTER, PROP_NONE);
+  RNA_def_property_struct_type(prop, "MeshVertColorLayer");
+  RNA_def_property_pointer_funcs(prop,
+                                 "rna_Mesh_sculpt_vertex_color_active_get",
+                                 "rna_Mesh_sculpt_vertex_color_active_set",
+                                 NULL,
+                                 NULL);
+  RNA_def_property_flag(prop, PROP_EDITABLE | PROP_NEVER_UNLINK);
+  RNA_def_property_ui_text(
+      prop, "Active Sculpt Vertex Color Layer", "Active sculpt vertex color layer");
+  RNA_def_property_update(prop, 0, "rna_Mesh_update_data_edit_active_color");
+
+  prop = RNA_def_property(srna, "active_index", PROP_INT, PROP_UNSIGNED);
+  RNA_def_property_int_funcs(prop,
+                             "rna_Mesh_sculpt_vertex_color_active_index_get",
+                             "rna_Mesh_sculpt_vertex_color_active_index_set",
+                             "rna_Mesh_sculpt_vertex_color_index_range");
+  RNA_def_property_ui_text(
+      prop, "Active Sculpt Vertex Color Index", "Active sculpt vertex color index");
+  RNA_def_property_update(prop, 0, "rna_Mesh_update_data_edit_active_color");
+}
+
 static void rna_def_uv_layers(BlenderRNA *brna, PropertyRNA *cprop)
 {
   StructRNA *srna;
@@ -2839,6 +3029,23 @@ static void rna_def_mesh(BlenderRNA *brna)
   RNA_def_property_ui_text(prop, "Vertex Colors", "All vertex colors");
   rna_def_loop_colors(brna, prop);
 
+  /* Sculpt Vertex colors */
+
+  prop = RNA_def_property(srna, "sculpt_vertex_colors", PROP_COLLECTION, PROP_NONE);
+  RNA_def_property_collection_sdna(prop, NULL, "vdata.layers", "vdata.totlayer");
+  RNA_def_property_collection_funcs(prop,
+                                    "rna_Mesh_sculpt_vertex_colors_begin",
+                                    NULL,
+                                    NULL,
+                                    NULL,
+                                    "rna_Mesh_sculpt_vertex_colors_length",
+                                    NULL,
+                                    NULL,
+                                    NULL);
+  RNA_def_property_struct_type(prop, "MeshVertColorLayer");
+  RNA_def_property_ui_text(prop, "Sculpt Vertex Colors", "All vertex colors");
+  rna_def_vert_colors(brna, prop);
+
   /* TODO, edge customdata layers (bmesh py api can access already) */
   prop = RNA_def_property(srna, "vertex_layers_float", PROP_COLLECTION, PROP_NONE);
   RNA_def_property_collection_sdna(prop, NULL, "vdata.layers", "vdata.totlayer");
@@ -3033,6 +3240,13 @@ static void rna_def_mesh(BlenderRNA *brna)
       prop, "Preserve Face Sets", "Keep the current Face Sets on the new mesh");
   RNA_def_property_update(prop, 0, "rna_Mesh_update_draw");
 
+  prop = RNA_def_property(srna, "use_remesh_preserve_vertex_colors", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_sdna(prop, NULL, "flag", ME_REMESH_REPROJECT_VERTEX_COLORS);
+  RNA_def_property_boolean_default(prop, false);
+  RNA_def_property_ui_text(
+      prop, "Preserve Vertex Colors", "Keep the current vertex colors on the new mesh");
+  RNA_def_property_update(prop, 0, "rna_Mesh_update_draw");
+
   prop = RNA_def_property(srna, "remesh_mode", PROP_ENUM, PROP_NONE);
   RNA_def_property_enum_sdna(prop, NULL, "remesh_mode");
   RNA_def_property_enum_items(prop, rna_enum_mesh_remesh_mode_items);
@@ -3183,6 +3397,7 @@ void RNA_def_mesh(BlenderRNA *brna)
   rna_def_mpolygon(brna);
   rna_def_mloopuv(brna);
   rna_def_mloopcol(brna);
+  rna_def_MPropCol(brna);
   rna_def_mproperties(brna);
   rna_def_face_map(brna);
 }