Fix T86455: vertex color baking issue with sculpt vertex colors

Baking to Vertex Colors would always bake to sculpt vertex colors (if
such a layer is present) even if those are not enabled in the
experimental preferences. This would bake without an error but leave the
user without a result to look in the viewport.

Now check if sculpt vertex colors are enabled and only bake to them in
that case.

Maniphest Tasks: T86455

Differential Revision: https://developer.blender.org/D10692

1 files changed, 6 insertions, 3 deletions

diff --git a/source/blender/editors/object/object_bake_api.c b/source/blender/editors/object/object_bake_api.c
index 610551e8539..d64769567f7 100644
--- a/source/blender/editors/object/object_bake_api.c
+++ b/source/blender/editors/object/object_bake_api.c
@@ -450,8 +450,9 @@ static bool bake_object_check(ViewLayer *view_layer,
 
   if (target == R_BAKE_TARGET_VERTEX_COLORS) {
     MPropCol *mcol = CustomData_get_layer(&me->vdata, CD_PROP_COLOR);
+    const bool mcol_valid = (mcol != NULL && U.experimental.use_sculpt_vertex_colors);
     MLoopCol *mloopcol = CustomData_get_layer(&me->ldata, CD_MLOOPCOL);
-    if (mcol == NULL && mloopcol == NULL) {
+    if (mloopcol == NULL && !mcol_valid) {
       BKE_reportf(reports,
                   RPT_ERROR,
                   "No vertex colors layer found in the object \"%s\"",
@@ -933,8 +934,9 @@ static bool bake_targets_init_vertex_colors(BakeTargets *targets, Object *ob, Re
 
   Mesh *me = ob->data;
   MPropCol *mcol = CustomData_get_layer(&me->vdata, CD_PROP_COLOR);
+  const bool mcol_valid = (mcol != NULL && U.experimental.use_sculpt_vertex_colors);
   MLoopCol *mloopcol = CustomData_get_layer(&me->ldata, CD_MLOOPCOL);
-  if (mcol == NULL && mloopcol == NULL) {
+  if (mloopcol == NULL && !mcol_valid) {
     BKE_report(reports, RPT_ERROR, "No vertex colors layer found to bake to");
     return false;
   }
@@ -1043,6 +1045,7 @@ static bool bake_targets_output_vertex_colors(BakeTargets *targets, Object *ob,
 {
   Mesh *me = ob->data;
   MPropCol *mcol = CustomData_get_layer(&me->vdata, CD_PROP_COLOR);
+  const bool mcol_valid = (mcol != NULL && U.experimental.use_sculpt_vertex_colors);
   MLoopCol *mloopcol = CustomData_get_layer(&me->ldata, CD_MLOOPCOL);
   const int num_channels = targets->num_channels;
   const float *result = targets->result;
@@ -1052,7 +1055,7 @@ static bool bake_targets_output_vertex_colors(BakeTargets *targets, Object *ob,
   BLI_assert(me->totloop == me_split->totloop);
   UNUSED_VARS_NDEBUG(me_split);
 
-  if (mcol) {
+  if (mcol_valid) {
     const int totvert = me->totvert;
     const int totloop = me->totloop;