Cloth: add a vertex group setting to exclude from object collision.

This can be useful as a workaround on the boundary with the pinned
vertices in some situations among other things, and completely copies
the existing design of the self collision vertex group setting.

Differential Revision: https://developer.blender.org/D10043

7 files changed, 51 insertions, 6 deletions

diff --git a/release/scripts/startup/bl_ui/properties_physics_cloth.py b/release/scripts/startup/bl_ui/properties_physics_cloth.py
index 79089b7cb89..3a5ca1e04f5 100644
--- a/release/scripts/startup/bl_ui/properties_physics_cloth.py
+++ b/release/scripts/startup/bl_ui/properties_physics_cloth.py
@@ -345,6 +345,7 @@ class PHYSICS_PT_cloth_object_collision(PhysicButtonsPanel, Panel):
 
         cloth = context.cloth.collision_settings
         md = context.cloth
+        ob = context.object
 
         layout.active = cloth.use_collision and cloth_panel_enabled(md)
 
@@ -357,6 +358,9 @@ class PHYSICS_PT_cloth_object_collision(PhysicButtonsPanel, Panel):
         col.prop(cloth, "impulse_clamp")
 
         col = flow.column()
+        col.prop_search(cloth, "vertex_group_object_collisions", ob, "vertex_groups", text="Vertex Group")
+
+        col = flow.column()
         col.prop(cloth, "collection")
 
 
diff --git a/source/blender/blenkernel/BKE_cloth.h b/source/blender/blenkernel/BKE_cloth.h
index 20e2122a195..e9267955d70 100644
--- a/source/blender/blenkernel/BKE_cloth.h
+++ b/source/blender/blenkernel/BKE_cloth.h
@@ -50,6 +50,7 @@ struct Scene;
 typedef enum eClothVertexFlag {
   CLOTH_VERT_FLAG_PINNED = (1 << 0),
   CLOTH_VERT_FLAG_NOSELFCOLL = (1 << 1), /* vertex NOT used for self collisions */
+  CLOTH_VERT_FLAG_NOOBJCOLL = (1 << 2),  /* vertex NOT used for object collisions */
 } eClothVertexFlag;
 
 typedef struct ClothHairData {
diff --git a/source/blender/blenkernel/intern/cloth.c b/source/blender/blenkernel/intern/cloth.c
index e9df562a15f..c2d6d5c7594 100644
--- a/source/blender/blenkernel/intern/cloth.c
+++ b/source/blender/blenkernel/intern/cloth.c
@@ -611,6 +611,8 @@ int cloth_uses_vgroup(ClothModifierData *clmd)
 {
   return (((clmd->coll_parms->flags & CLOTH_COLLSETTINGS_FLAG_SELF) &&
            (clmd->coll_parms->vgroup_selfcol > 0)) ||
+          ((clmd->coll_parms->flags & CLOTH_COLLSETTINGS_FLAG_ENABLED) &&
+           (clmd->coll_parms->vgroup_objcol > 0)) ||
           (clmd->sim_parms->vgroup_pressure > 0) || (clmd->sim_parms->vgroup_struct > 0) ||
           (clmd->sim_parms->vgroup_bend > 0) || (clmd->sim_parms->vgroup_shrink > 0) ||
           (clmd->sim_parms->vgroup_intern > 0) || (clmd->sim_parms->vgroup_mass > 0));
@@ -644,8 +646,8 @@ static void cloth_apply_vgroup(ClothModifierData *clmd, Mesh *mesh)
       verts->shrink_factor = 0.0f;
 
       /* Reset vertex flags */
-      verts->flags &= ~CLOTH_VERT_FLAG_PINNED;
-      verts->flags &= ~CLOTH_VERT_FLAG_NOSELFCOLL;
+      verts->flags &= ~(CLOTH_VERT_FLAG_PINNED | CLOTH_VERT_FLAG_NOSELFCOLL |
+                        CLOTH_VERT_FLAG_NOOBJCOLL);
 
       MDeformVert *dvert = CustomData_get(&mesh->vdata, i, CD_MDEFORMVERT);
       if (dvert) {
@@ -682,6 +684,12 @@ static void cloth_apply_vgroup(ClothModifierData *clmd, Mesh *mesh)
             }
           }
 
+          if (dvert->dw[j].def_nr == (clmd->coll_parms->vgroup_objcol - 1)) {
+            if (dvert->dw[j].weight > 0.0f) {
+              verts->flags |= CLOTH_VERT_FLAG_NOOBJCOLL;
+            }
+          }
+
           if (dvert->dw[j].def_nr == (clmd->sim_parms->vgroup_shrink - 1)) {
             /* Used for linear interpolation between min and max
              * shrink factor based on weight. */
diff --git a/source/blender/blenkernel/intern/collision.c b/source/blender/blenkernel/intern/collision.c
index b4d668c7f50..1c24dae430c 100644
--- a/source/blender/blenkernel/intern/collision.c
+++ b/source/blender/blenkernel/intern/collision.c
@@ -1012,7 +1012,7 @@ static bool cloth_bvh_collision_is_active(const ClothModifierData *UNUSED(clmd),
   const int flags_a = verts[tri_a->tri[0]].flags & verts[tri_a->tri[1]].flags &
                       verts[tri_a->tri[2]].flags;
 
-  if (flags_a & CLOTH_VERT_FLAG_PINNED) {
+  if (flags_a & (CLOTH_VERT_FLAG_PINNED | CLOTH_VERT_FLAG_NOOBJCOLL)) {
     return false;
   }
 
diff --git a/source/blender/makesdna/DNA_cloth_types.h b/source/blender/makesdna/DNA_cloth_types.h
index 11993d95c2c..467174c4f32 100644
--- a/source/blender/makesdna/DNA_cloth_types.h
+++ b/source/blender/makesdna/DNA_cloth_types.h
@@ -240,9 +240,11 @@ typedef struct ClothCollSettings {
   char _pad[4];
   /** Only use colliders from this group of objects. */
   struct Collection *group;
-  /** Vgroup to paint which vertices are used for self collisions. */
+  /** Vgroup to paint which vertices are not used for self collisions. */
   short vgroup_selfcol;
-  char _pad2[6];
+  /** Vgroup to paint which vertices are not used for object collisions. */
+  short vgroup_objcol;
+  char _pad2[4];
   /** Impulse clamp for object collisions. */
   float clamp;
   /** Impulse clamp for self collisions. */
diff --git a/source/blender/makesdna/DNA_modifier_defaults.h b/source/blender/makesdna/DNA_modifier_defaults.h
index fcb582ef837..a5466641be4 100644
--- a/source/blender/makesdna/DNA_modifier_defaults.h
+++ b/source/blender/makesdna/DNA_modifier_defaults.h
@@ -179,6 +179,7 @@
     .loop_count = 2, \
     .group = NULL, \
     .vgroup_selfcol = 0, \
+    .vgroup_objcol = 0, \
     .clamp = 0.0f, \
     .self_clamp = 0.0f, \
   }
diff --git a/source/blender/makesrna/intern/rna_cloth.c b/source/blender/makesrna/intern/rna_cloth.c
index b0e0b577629..2bc00dd5af5 100644
--- a/source/blender/makesrna/intern/rna_cloth.c
+++ b/source/blender/makesrna/intern/rna_cloth.c
@@ -396,6 +396,24 @@ static void rna_CollSettings_selfcol_vgroup_set(PointerRNA *ptr, const char *val
   rna_object_vgroup_name_index_set(ptr, value, &coll->vgroup_selfcol);
 }
 
+static void rna_CollSettings_objcol_vgroup_get(PointerRNA *ptr, char *value)
+{
+  ClothCollSettings *coll = (ClothCollSettings *)ptr->data;
+  rna_object_vgroup_name_index_get(ptr, value, coll->vgroup_objcol);
+}
+
+static int rna_CollSettings_objcol_vgroup_length(PointerRNA *ptr)
+{
+  ClothCollSettings *coll = (ClothCollSettings *)ptr->data;
+  return rna_object_vgroup_name_index_length(ptr, coll->vgroup_objcol);
+}
+
+static void rna_CollSettings_objcol_vgroup_set(PointerRNA *ptr, const char *value)
+{
+  ClothCollSettings *coll = (ClothCollSettings *)ptr->data;
+  rna_object_vgroup_name_index_set(ptr, value, &coll->vgroup_objcol);
+}
+
 static PointerRNA rna_ClothSettings_rest_shape_key_get(PointerRNA *ptr)
 {
   Object *ob = (Object *)ptr->owner_id;
@@ -1163,7 +1181,18 @@ static void rna_def_cloth_collision_settings(BlenderRNA *brna)
   RNA_def_property_ui_text(
       prop,
       "Selfcollision Vertex Group",
-      "Vertex group to define vertices which are not used during self collisions");
+      "Triangles with all vertices in this group are not used during self collisions");
+  RNA_def_property_update(prop, 0, "rna_cloth_update");
+
+  prop = RNA_def_property(srna, "vertex_group_object_collisions", PROP_STRING, PROP_NONE);
+  RNA_def_property_string_funcs(prop,
+                                "rna_CollSettings_objcol_vgroup_get",
+                                "rna_CollSettings_objcol_vgroup_length",
+                                "rna_CollSettings_objcol_vgroup_set");
+  RNA_def_property_ui_text(
+      prop,
+      "Collision Vertex Group",
+      "Triangles with all vertices in this group are not used during object collisions");
   RNA_def_property_update(prop, 0, "rna_cloth_update");
 
   prop = RNA_def_property(srna, "self_impulse_clamp", PROP_FLOAT, PROP_NONE);