Fix T101906: Modifier apply not working if target object is in excluded collection

The issue was introduced by the optimization of hidden objects and modifiers
in the f12f7800c296.

The solution here detects that either an object is hidden or the modifier is
disabled and does special tricks to ensure the dependencies are evaluated.
This is done by constructing a separate minimal dependency graph needed for
the object on which the modifier is being applied on. This minimal dependency
graph will not perform visibility optimization, making it so modifier
dependencies are ensured to be evaluated.

The downside of such approach is that some dependencies which are not needed
for the modifier are still evaluated. There is no currently an easy way to
avoid this. At least not without introducing possible race conditions with
other dependency graphs.

If the performance of applying modifiers in such cases becomes a problem the
possible solution would be to create a temporary object with a single modifier
so that only minimal set of dependencies is pulled in the minimal dependency
graph.

Differential Revision: https://developer.blender.org/D16421

1 files changed, 42 insertions, 17 deletions

diff --git a/source/blender/editors/object/object_modifier.cc b/source/blender/editors/object/object_modifier.cc
index 85a35861329..37fdcf41815 100644
--- a/source/blender/editors/object/object_modifier.cc
+++ b/source/blender/editors/object/object_modifier.cc
@@ -942,31 +942,56 @@ bool ED_object_modifier_apply(Main *bmain,
   Object *ob_eval = DEG_get_evaluated_object(depsgraph, ob);
   ModifierData *md_eval = (ob_eval) ? BKE_modifiers_findby_name(ob_eval, md->name) : md;
 
-  /* Allow apply of a non-real-time modifier, by first re-enabling real-time. */
-  int prev_mode = md_eval->mode;
-  md_eval->mode |= eModifierMode_Realtime;
+  Depsgraph *apply_depsgraph = depsgraph;
+  Depsgraph *local_depsgraph = nullptr;
+
+  /* If the object is hidden or the modifier is not enabled for the viewport is disabled a special
+   * handling is required. This is because the viewport dependency graph optimizes out evaluation
+   * of objects which are used by hidden objects and disabled modifiers.
+   *
+   * The idea is to create a dependency graph which does not perform those optimizations. */
+  if ((ob_eval->base_flag & BASE_ENABLED_VIEWPORT) == 0 ||
+      (md_eval->mode & eModifierMode_Realtime) == 0) {
+    ViewLayer *view_layer = DEG_get_input_view_layer(depsgraph);
+
+    local_depsgraph = DEG_graph_new(bmain, scene, view_layer, DAG_EVAL_VIEWPORT);
+    DEG_disable_visibility_optimization(local_depsgraph);
+
+    ID *ids[] = {&ob->id};
+
+    DEG_graph_build_from_ids(local_depsgraph, ids, 1);
+    DEG_evaluate_on_refresh(local_depsgraph);
+
+    apply_depsgraph = local_depsgraph;
+
+    /* The evaluated object and modifier are now from the different dependency graph. */
+    ob_eval = DEG_get_evaluated_object(local_depsgraph, ob);
+    md_eval = BKE_modifiers_findby_name(ob_eval, md->name);
+
+    /* Force mode on the evaluated modifier, enforcing the modifier evaluation in the apply()
+     * functions. */
+    md_eval->mode |= eModifierMode_Realtime;
+  }
 
+  bool did_apply = false;
   if (mode == MODIFIER_APPLY_SHAPE) {
-    if (!modifier_apply_shape(bmain, reports, depsgraph, scene, ob, md_eval)) {
-      md_eval->mode = prev_mode;
-      return false;
-    }
+    did_apply = modifier_apply_shape(bmain, reports, apply_depsgraph, scene, ob, md_eval);
   }
   else {
-    if (!modifier_apply_obdata(reports, depsgraph, scene, ob, md_eval)) {
-      md_eval->mode = prev_mode;
-      return false;
-    }
+    did_apply = modifier_apply_obdata(reports, apply_depsgraph, scene, ob, md_eval);
   }
 
-  md_eval->mode = prev_mode;
-
-  if (!keep_modifier) {
-    BKE_modifier_remove_from_list(ob, md);
-    BKE_modifier_free(md);
+  if (did_apply) {
+    if (!keep_modifier) {
+      BKE_modifier_remove_from_list(ob, md);
+      BKE_modifier_free(md);
+    }
+    BKE_object_free_derived_caches(ob);
   }
 
-  BKE_object_free_derived_caches(ob);
+  if (local_depsgraph != nullptr) {
+    DEG_graph_free(local_depsgraph);
+  }
 
   return true;
 }