Solidify modifier: add option to assign shell & rim geometry to selected vertex groups.

This commit gives the solidify modifier the ability to assign the newly created shell
and rim geometries to selected vertex groups. This expands the procedural control over
the modifier stack by letting users apply modifiers to the shell geometry without affecting
the original geometry.

This will be especially helpful for NPR users that use solidify to create backface
culling lines on their characters giving them the ability to add displace noise
and other effects.

Differential Revision: https://developer.blender.org/D6903

5 files changed, 102 insertions, 1 deletions

diff --git a/source/blender/makesdna/DNA_modifier_types.h b/source/blender/makesdna/DNA_modifier_types.h
index 405f12d5fb9..7c8dc7ffd52 100644
--- a/source/blender/makesdna/DNA_modifier_types.h
+++ b/source/blender/makesdna/DNA_modifier_types.h
@@ -1148,6 +1148,8 @@ typedef struct SolidifyModifierData {
 
   /** Name of vertex group to use, MAX_VGROUP_NAME. */
   char defgrp_name[64];
+  char shell_defgrp_name[64];
+  char rim_defgrp_name[64];
   /** New surface offset leve.l*/
   float offset;
   /** Midpoint of the offset . */
diff --git a/source/blender/makesrna/intern/rna_modifier.c b/source/blender/makesrna/intern/rna_modifier.c
index 51b50647e1d..91eaae6cfc8 100644
--- a/source/blender/makesrna/intern/rna_modifier.c
+++ b/source/blender/makesrna/intern/rna_modifier.c
@@ -797,6 +797,8 @@ RNA_MOD_VGROUP_NAME_SET(Shrinkwrap, vgroup_name);
 RNA_MOD_VGROUP_NAME_SET(SimpleDeform, vgroup_name);
 RNA_MOD_VGROUP_NAME_SET(Smooth, defgrp_name);
 RNA_MOD_VGROUP_NAME_SET(Solidify, defgrp_name);
+RNA_MOD_VGROUP_NAME_SET(Solidify, shell_defgrp_name);
+RNA_MOD_VGROUP_NAME_SET(Solidify, rim_defgrp_name);
 RNA_MOD_VGROUP_NAME_SET(UVWarp, vgroup_name);
 RNA_MOD_VGROUP_NAME_SET(Warp, defgrp_name);
 RNA_MOD_VGROUP_NAME_SET(Wave, defgrp_name);
@@ -4449,6 +4451,18 @@ static void rna_def_modifier_solidify(BlenderRNA *brna)
   RNA_def_property_string_funcs(prop, NULL, NULL, "rna_SolidifyModifier_defgrp_name_set");
   RNA_def_property_update(prop, 0, "rna_Modifier_update");
 
+  prop = RNA_def_property(srna, "shell_vertex_group", PROP_STRING, PROP_NONE);
+  RNA_def_property_string_sdna(prop, NULL, "shell_defgrp_name");
+  RNA_def_property_ui_text(prop, "Shell Vertex Group", "Vertex group that the generated shell geometry will be weighted to");
+  RNA_def_property_string_funcs(prop, NULL, NULL, "rna_SolidifyModifier_shell_defgrp_name_set");
+  RNA_def_property_update(prop, 0, "rna_Modifier_update");
+
+  prop = RNA_def_property(srna, "rim_vertex_group", PROP_STRING, PROP_NONE);
+  RNA_def_property_string_sdna(prop, NULL, "rim_defgrp_name");
+  RNA_def_property_ui_text(prop, "Rim Vertex Group", "Vertex group that the generated rim geometry will be weighted to");
+  RNA_def_property_string_funcs(prop, NULL, NULL, "rna_SolidifyModifier_rim_defgrp_name_set");
+  RNA_def_property_update(prop, 0, "rna_Modifier_update");
+
   prop = RNA_def_property(srna, "use_rim", PROP_BOOLEAN, PROP_NONE);
   RNA_def_property_boolean_sdna(prop, NULL, "flag", MOD_SOLIDIFY_RIM);
   RNA_def_property_ui_text(prop,
diff --git a/source/blender/modifiers/intern/MOD_solidify.c b/source/blender/modifiers/intern/MOD_solidify.c
index 41852272787..0eed2335ed7 100644
--- a/source/blender/modifiers/intern/MOD_solidify.c
+++ b/source/blender/modifiers/intern/MOD_solidify.c
@@ -62,7 +62,8 @@ static void requiredDataMask(Object *UNUSED(ob),
   SolidifyModifierData *smd = (SolidifyModifierData *)md;
 
   /* ask for vertexgroups if we need them */
-  if (smd->defgrp_name[0] != '\0') {
+  if (smd->defgrp_name[0] != '\0' || smd->shell_defgrp_name[0] != '\0' ||
+      smd->rim_defgrp_name[0] != '\0') {
     r_cddata_masks->vmask |= CD_MASK_MDEFORMVERT;
   }
 }
diff --git a/source/blender/modifiers/intern/MOD_solidify_extrude.c b/source/blender/modifiers/intern/MOD_solidify_extrude.c
index 8febf78fef5..ecd62e5e1fe 100644
--- a/source/blender/modifiers/intern/MOD_solidify_extrude.c
+++ b/source/blender/modifiers/intern/MOD_solidify_extrude.c
@@ -240,6 +240,9 @@ Mesh *MOD_solidify_extrude_applyModifier(ModifierData *md,
   MDeformVert *dvert;
   const bool defgrp_invert = (smd->flag & MOD_SOLIDIFY_VGROUP_INV) != 0;
   int defgrp_index;
+  const int shell_defgrp_index = BKE_object_defgroup_name_index(ctx->object,
+                                                                smd->shell_defgrp_name);
+  const int rim_defgrp_index = BKE_object_defgroup_name_index(ctx->object, smd->rim_defgrp_name);
 
   /* array size is doubled in case of using a shell */
   const uint stride = do_shell ? 2 : 1;
@@ -886,6 +889,36 @@ Mesh *MOD_solidify_extrude_applyModifier(ModifierData *md,
     }
   }
 
+  /* Add vertex weights for rim and shell vgroups. */
+  if (shell_defgrp_index != -1 || rim_defgrp_index != -1) {
+    dvert = CustomData_duplicate_referenced_layer(&result->vdata, CD_MDEFORMVERT, result->totvert);
+    /* If no vertices were ever added to an object's vgroup, dvert might be NULL. */
+    if (dvert == NULL) {
+      /* Add a valid data layer! */
+      dvert = CustomData_add_layer(
+          &result->vdata, CD_MDEFORMVERT, CD_CALLOC, NULL, result->totvert);
+    }
+    /* Ultimate security check. */
+    if (!dvert) {
+      return result;
+    }
+    result->dvert = dvert;
+
+    if (rim_defgrp_index != -1) {
+      for (uint i = 0; i < rimVerts; i++) {
+        BKE_defvert_ensure_index(&result->dvert[new_vert_arr[i]], rim_defgrp_index)->weight = 1.0f;
+        BKE_defvert_ensure_index(&result->dvert[(do_shell ? new_vert_arr[i] : i) + numVerts],
+                                 rim_defgrp_index)
+            ->weight = 1.0f;
+      }
+    }
+
+    if (shell_defgrp_index != -1) {
+      for (uint i = numVerts; i < result->totvert; i++) {
+        BKE_defvert_ensure_index(&result->dvert[i], shell_defgrp_index)->weight = 1.0f;
+      }
+    }
+  }
   if (smd->flag & MOD_SOLIDIFY_RIM) {
     uint i;
 
diff --git a/source/blender/modifiers/intern/MOD_solidify_nonmanifold.c b/source/blender/modifiers/intern/MOD_solidify_nonmanifold.c
index 22fc56bdeaf..0f9f5952a26 100644
--- a/source/blender/modifiers/intern/MOD_solidify_nonmanifold.c
+++ b/source/blender/modifiers/intern/MOD_solidify_nonmanifold.c
@@ -178,6 +178,9 @@ Mesh *MOD_solidify_nonmanifold_applyModifier(ModifierData *md,
   MDeformVert *dvert;
   const bool defgrp_invert = (smd->flag & MOD_SOLIDIFY_VGROUP_INV) != 0;
   int defgrp_index;
+  const int shell_defgrp_index = BKE_object_defgroup_name_index(ctx->object,
+                                                                smd->shell_defgrp_name);
+  const int rim_defgrp_index = BKE_object_defgroup_name_index(ctx->object, smd->rim_defgrp_name);
 
   MOD_get_vgroup(ctx->object, mesh, smd->defgrp_name, &dvert, &defgrp_index);
 
@@ -1778,6 +1781,18 @@ Mesh *MOD_solidify_nonmanifold_applyModifier(ModifierData *md,
   int *origindex_edge = CustomData_get_layer(&result->edata, CD_ORIGINDEX);
   int *origindex_poly = CustomData_get_layer(&result->pdata, CD_ORIGINDEX);
 
+  /* Checks that result has dvert data. */
+  if (shell_defgrp_index != -1 || rim_defgrp_index != -1) {
+    dvert = CustomData_duplicate_referenced_layer(&result->vdata, CD_MDEFORMVERT, result->totvert);
+    /* If no vertices were ever added to an object's vgroup, dvert might be NULL. */
+    if (dvert == NULL) {
+      /* Add a valid data layer! */
+      dvert = CustomData_add_layer(
+          &result->vdata, CD_MDEFORMVERT, CD_CALLOC, NULL, result->totvert);
+    }
+    result->dvert = dvert;
+  }
+
   /* Make_new_verts. */
   {
     gs_ptr = orig_vert_groups_arr;
@@ -2101,12 +2116,20 @@ Mesh *MOD_solidify_nonmanifold_applyModifier(ModifierData *md,
         MEdge *open_face_edge;
         uint open_face_edge_index;
         if (!do_flip) {
+          if (rim_defgrp_index != -1) {
+            BKE_defvert_ensure_index(&result->dvert[medge[edge1->new_edge].v1], rim_defgrp_index)
+                ->weight = 1.0f;
+          }
           CustomData_copy_data(&mesh->ldata, &result->ldata, loop1, (int)loop_index, 1);
           mloop[loop_index].v = medge[edge1->new_edge].v1;
           mloop[loop_index++].e = edge1->new_edge;
 
           if (!v2_singularity) {
             open_face_edge_index = edge1->link_edge_groups[1]->open_face_edge;
+            if (rim_defgrp_index != -1) {
+              BKE_defvert_ensure_index(&result->dvert[medge[edge1->new_edge].v2], rim_defgrp_index)
+                  ->weight = 1.0f;
+            }
             CustomData_copy_data(&mesh->ldata, &result->ldata, loop2, (int)loop_index, 1);
             mloop[loop_index].v = medge[edge1->new_edge].v2;
             open_face_edge = medge + open_face_edge_index;
@@ -2118,12 +2141,20 @@ Mesh *MOD_solidify_nonmanifold_applyModifier(ModifierData *md,
             }
           }
 
+          if (rim_defgrp_index != -1) {
+            BKE_defvert_ensure_index(&result->dvert[medge[edge2->new_edge].v2], rim_defgrp_index)
+                ->weight = 1.0f;
+          }
           CustomData_copy_data(&mesh->ldata, &result->ldata, loop2, (int)loop_index, 1);
           mloop[loop_index].v = medge[edge2->new_edge].v2;
           mloop[loop_index++].e = edge2->new_edge;
 
           if (!v1_singularity) {
             open_face_edge_index = edge2->link_edge_groups[0]->open_face_edge;
+            if (rim_defgrp_index != -1) {
+              BKE_defvert_ensure_index(&result->dvert[medge[edge2->new_edge].v1], rim_defgrp_index)
+                  ->weight = 1.0f;
+            }
             CustomData_copy_data(&mesh->ldata, &result->ldata, loop1, (int)loop_index, 1);
             mloop[loop_index].v = medge[edge2->new_edge].v1;
             open_face_edge = medge + open_face_edge_index;
@@ -2138,6 +2169,10 @@ Mesh *MOD_solidify_nonmanifold_applyModifier(ModifierData *md,
         else {
           if (!v1_singularity) {
             open_face_edge_index = edge1->link_edge_groups[0]->open_face_edge;
+            if (rim_defgrp_index != -1) {
+              BKE_defvert_ensure_index(&result->dvert[medge[edge1->new_edge].v1], rim_defgrp_index)
+                  ->weight = 1.0f;
+            }
             CustomData_copy_data(&mesh->ldata, &result->ldata, loop1, (int)loop_index, 1);
             mloop[loop_index].v = medge[edge1->new_edge].v1;
             open_face_edge = medge + open_face_edge_index;
@@ -2149,12 +2184,20 @@ Mesh *MOD_solidify_nonmanifold_applyModifier(ModifierData *md,
             }
           }
 
+          if (rim_defgrp_index != -1) {
+            BKE_defvert_ensure_index(&result->dvert[medge[edge2->new_edge].v1], rim_defgrp_index)
+                ->weight = 1.0f;
+          }
           CustomData_copy_data(&mesh->ldata, &result->ldata, loop1, (int)loop_index, 1);
           mloop[loop_index].v = medge[edge2->new_edge].v1;
           mloop[loop_index++].e = edge2->new_edge;
 
           if (!v2_singularity) {
             open_face_edge_index = edge2->link_edge_groups[1]->open_face_edge;
+            if (rim_defgrp_index != -1) {
+              BKE_defvert_ensure_index(&result->dvert[medge[edge2->new_edge].v2], rim_defgrp_index)
+                  ->weight = 1.0f;
+            }
             CustomData_copy_data(&mesh->ldata, &result->ldata, loop2, (int)loop_index, 1);
             mloop[loop_index].v = medge[edge2->new_edge].v2;
             open_face_edge = medge + open_face_edge_index;
@@ -2166,6 +2209,10 @@ Mesh *MOD_solidify_nonmanifold_applyModifier(ModifierData *md,
             }
           }
 
+          if (rim_defgrp_index != -1) {
+            BKE_defvert_ensure_index(&result->dvert[medge[edge1->new_edge].v2], rim_defgrp_index)
+                ->weight = 1.0f;
+          }
           CustomData_copy_data(&mesh->ldata, &result->ldata, loop2, (int)loop_index, 1);
           mloop[loop_index].v = medge[edge1->new_edge].v2;
           mloop[loop_index++].e = edge1->new_edge;
@@ -2243,6 +2290,10 @@ Mesh *MOD_solidify_nonmanifold_applyModifier(ModifierData *md,
           mpoly[poly_index].flag = fr->face->flag;
           if (fr->reversed != do_flip) {
             for (int l = (int)k - 1; l >= 0; l--) {
+              if (shell_defgrp_index != -1) {
+                BKE_defvert_ensure_index(&result->dvert[face_verts[l]], shell_defgrp_index)
+                    ->weight = 1.0f;
+              }
               CustomData_copy_data(
                   &mesh->ldata, &result->ldata, (int)face_loops[l], (int)loop_index, 1);
               mloop[loop_index].v = face_verts[l];