Fix T77685: object transforms from rigid body simulation are ignored by modifiers

This does not fix all the cases in the bug report, because there are multiple
different issues. Only the first two are fixed. The third is probably a known
issue for now.

Before this patch, the rigid body simulation was always done after modifiers
are evaluated, because to perform the simulation, the final geometry of the
object was required. However, the geometry is not required in all cases,
depending on the selected collisions shape.

This patch changes it so that when the simulation does not need the
evaluated geometry, the simulation will be done before the modifiers
are evaluated. This gives the modifiers access to the simulated positions.
When the rigid body simulation does depend on the evaluated geometry,
it will still be performed after modifiers are evaluated.

The simulation will be performed after modifiers are evaluated, iff
the collision shape is "Convex Hull" or "Mesh" and the source is set
to "Deform" or "Final".

Reviewers: sergey

Differential Revision: https://developer.blender.org/D8487

2 files changed, 19 insertions, 3 deletions

diff --git a/source/blender/depsgraph/intern/builder/deg_builder_relations.cc b/source/blender/depsgraph/intern/builder/deg_builder_relations.cc
index c5c509ee853..ccd7cadc8e8 100644
--- a/source/blender/depsgraph/intern/builder/deg_builder_relations.cc
+++ b/source/blender/depsgraph/intern/builder/deg_builder_relations.cc
@@ -1013,6 +1013,9 @@ void DepsgraphRelationBuilder::build_object_pointcache(Object *object)
     /* Check which components needs the point cache. */
     int flag = -1;
     if (ptcache_id->type == PTCACHE_TYPE_RIGIDBODY) {
+      if (object->rigidbody_object->type == RBO_TYPE_PASSIVE) {
+        continue;
+      }
       flag = FLAG_TRANSFORM;
       OperationKey transform_key(
           &object->id, NodeType::TRANSFORM, OperationCode::TRANSFORM_SIMULATION_INIT);
@@ -1713,8 +1716,7 @@ void DepsgraphRelationBuilder::build_rigidbody(Scene *scene)
       /* Geometry must be known to create the rigid body. RBO_MESH_BASE
        * uses the non-evaluated mesh, so then the evaluation is
        * unnecessary. */
-      if (object->rigidbody_object != nullptr &&
-          object->rigidbody_object->mesh_source != RBO_MESH_BASE) {
+      if (rigidbody_object_depends_on_evaluated_geometry(object->rigidbody_object)) {
         /* NOTE: We prefer this relation to be never killed, to avoid
          * access partially evaluated mesh from solver. */
         ComponentKey object_geometry_key(&object->id, NodeType::GEOMETRY);
diff --git a/source/blender/depsgraph/intern/builder/deg_builder_relations_impl.h b/source/blender/depsgraph/intern/builder/deg_builder_relations_impl.h
index d4c28060878..b853ecd8e56 100644
--- a/source/blender/depsgraph/intern/builder/deg_builder_relations_impl.h
+++ b/source/blender/depsgraph/intern/builder/deg_builder_relations_impl.h
@@ -27,6 +27,7 @@
 
 #include "DNA_ID.h"
 #include "DNA_object_types.h"
+#include "DNA_rigidbody_types.h"
 
 namespace blender {
 namespace deg {
@@ -126,6 +127,19 @@ Relation *DepsgraphRelationBuilder::add_node_handle_relation(const KeyType &key_
   return nullptr;
 }
 
+static bool rigidbody_object_depends_on_evaluated_geometry(const RigidBodyOb *rbo)
+{
+  if (rbo == nullptr) {
+    return false;
+  }
+  if (ELEM(rbo->shape, RB_SHAPE_CONVEXH, RB_SHAPE_TRIMESH)) {
+    if (rbo->mesh_source != RBO_MESH_BASE) {
+      return true;
+    }
+  }
+  return false;
+}
+
 template<typename KeyTo>
 Relation *DepsgraphRelationBuilder::add_depends_on_transform_relation(ID *id,
                                                                       const KeyTo &key_to,
@@ -134,7 +148,7 @@ Relation *DepsgraphRelationBuilder::add_depends_on_transform_relation(ID *id,
 {
   if (GS(id->name) == ID_OB) {
     Object *object = reinterpret_cast<Object *>(id);
-    if (object->rigidbody_object != nullptr) {
+    if (rigidbody_object_depends_on_evaluated_geometry(object->rigidbody_object)) {
       OperationKey transform_key(&object->id, NodeType::TRANSFORM, OperationCode::TRANSFORM_EVAL);
       return add_relation(transform_key, key_to, description, flags);
     }