Fix T101906: Modifier apply not working if target object is in excluded collection

The issue was introduced by the optimization of hidden objects and modifiers
in the f12f7800c296.

The solution here detects that either an object is hidden or the modifier is
disabled and does special tricks to ensure the dependencies are evaluated.
This is done by constructing a separate minimal dependency graph needed for
the object on which the modifier is being applied on. This minimal dependency
graph will not perform visibility optimization, making it so modifier
dependencies are ensured to be evaluated.

The downside of such approach is that some dependencies which are not needed
for the modifier are still evaluated. There is no currently an easy way to
avoid this. At least not without introducing possible race conditions with
other dependency graphs.

If the performance of applying modifiers in such cases becomes a problem the
possible solution would be to create a temporary object with a single modifier
so that only minimal set of dependencies is pulled in the minimal dependency
graph.

Differential Revision: https://developer.blender.org/D16421

6 files changed, 67 insertions, 19 deletions

diff --git a/source/blender/depsgraph/DEG_depsgraph.h b/source/blender/depsgraph/DEG_depsgraph.h
index a8b21e4c153..48a6a5cda74 100644
--- a/source/blender/depsgraph/DEG_depsgraph.h
+++ b/source/blender/depsgraph/DEG_depsgraph.h
@@ -221,6 +221,14 @@ bool DEG_is_active(const struct Depsgraph *depsgraph);
 void DEG_make_active(struct Depsgraph *depsgraph);
 void DEG_make_inactive(struct Depsgraph *depsgraph);
 
+/**
+ * Disable the visibility optimization making it so IDs which affect hidden objects or disabled
+ * modifiers are still evaluated.
+ *
+ * For example, this ensures that an object which is needed by a modifier is ignoring checks about
+ * whether the object is hidden or the modifier is disabled. */
+void DEG_disable_visibility_optimization(struct Depsgraph *depsgraph);
+
 /** \} */
 
 /* -------------------------------------------------------------------- */
diff --git a/source/blender/depsgraph/DEG_depsgraph_build.h b/source/blender/depsgraph/DEG_depsgraph_build.h
index 201a534f535..ffeb5e897ab 100644
--- a/source/blender/depsgraph/DEG_depsgraph_build.h
+++ b/source/blender/depsgraph/DEG_depsgraph_build.h
@@ -56,6 +56,9 @@ void DEG_graph_build_for_render_pipeline(struct Depsgraph *graph);
  */
 void DEG_graph_build_for_compositor_preview(struct Depsgraph *graph, struct bNodeTree *nodetree);
 
+/**
+ * Builds the minimal dependency graph needed for evaluation of the given IDs.
+ */
 void DEG_graph_build_from_ids(struct Depsgraph *graph, struct ID **ids, int num_ids);
 
 /** Tag relations from the given graph for update. */
diff --git a/source/blender/depsgraph/intern/depsgraph.cc b/source/blender/depsgraph/intern/depsgraph.cc
index 316d0b615c6..4d7d537b450 100644
--- a/source/blender/depsgraph/intern/depsgraph.cc
+++ b/source/blender/depsgraph/intern/depsgraph.cc
@@ -58,6 +58,7 @@ Depsgraph::Depsgraph(Main *bmain, Scene *scene, ViewLayer *view_layer, eEvaluati
       ctime(BKE_scene_ctime_get(scene)),
       scene_cow(nullptr),
       is_active(false),
+      use_visibility_optimization(true),
       is_evaluating(false),
       is_render_pipeline_depsgraph(false),
       use_editors_update(false)
@@ -334,3 +335,9 @@ void DEG_make_inactive(struct Depsgraph *depsgraph)
   deg::Depsgraph *deg_graph = reinterpret_cast<deg::Depsgraph *>(depsgraph);
   deg_graph->is_active = false;
 }
+
+void DEG_disable_visibility_optimization(struct Depsgraph *depsgraph)
+{
+  deg::Depsgraph *deg_graph = reinterpret_cast<deg::Depsgraph *>(depsgraph);
+  deg_graph->use_visibility_optimization = false;
+}
\ No newline at end of file
diff --git a/source/blender/depsgraph/intern/depsgraph.h b/source/blender/depsgraph/intern/depsgraph.h
index 2f88199384d..042cb045c6f 100644
--- a/source/blender/depsgraph/intern/depsgraph.h
+++ b/source/blender/depsgraph/intern/depsgraph.h
@@ -147,6 +147,9 @@ struct Depsgraph {
    * to read stuff from. */
   bool is_active;
 
+  /* Optimize out evaluation of operations which affect hidden objects or disabled modifiers. */
+  bool use_visibility_optimization;
+
   DepsgraphDebug debug;
 
   bool is_evaluating;
diff --git a/source/blender/depsgraph/intern/eval/deg_eval_visibility.cc b/source/blender/depsgraph/intern/eval/deg_eval_visibility.cc
index a056ba1dfa7..0ee4052eff3 100644
--- a/source/blender/depsgraph/intern/eval/deg_eval_visibility.cc
+++ b/source/blender/depsgraph/intern/eval/deg_eval_visibility.cc
@@ -37,7 +37,8 @@ void deg_evaluate_object_node_visibility(::Depsgraph *depsgraph, IDNode *id_node
   const int required_flags = (graph->mode == DAG_EVAL_VIEWPORT) ? BASE_ENABLED_VIEWPORT :
                                                                   BASE_ENABLED_RENDER;
 
-  const bool is_enabled = object->base_flag & required_flags;
+  const bool is_enabled = !graph->use_visibility_optimization ||
+                          object->base_flag & required_flags;
 
   if (id_node->is_enabled_on_eval != is_enabled) {
     id_node->is_enabled_on_eval = is_enabled;
@@ -73,7 +74,8 @@ void deg_evaluate_object_modifiers_mode_node_visibility(::Depsgraph *depsgraph,
                    "Modifier node in depsgraph is not found. Likely due to missing "
                    "DEG_relations_tag_update().");
 
-    const bool modifier_enabled = modifier->mode & modifier_mode;
+    const bool modifier_enabled = !graph->use_visibility_optimization ||
+                                  (modifier->mode & modifier_mode);
     const int mute_flag = modifier_enabled ? 0 : DEPSOP_FLAG_MUTE;
     if ((modifier_node->flag & DEPSOP_FLAG_MUTE) != mute_flag) {
       modifier_node->flag &= ~DEPSOP_FLAG_MUTE;
diff --git a/source/blender/editors/object/object_modifier.cc b/source/blender/editors/object/object_modifier.cc
index 85a35861329..37fdcf41815 100644
--- a/source/blender/editors/object/object_modifier.cc
+++ b/source/blender/editors/object/object_modifier.cc
@@ -942,31 +942,56 @@ bool ED_object_modifier_apply(Main *bmain,
   Object *ob_eval = DEG_get_evaluated_object(depsgraph, ob);
   ModifierData *md_eval = (ob_eval) ? BKE_modifiers_findby_name(ob_eval, md->name) : md;
 
-  /* Allow apply of a non-real-time modifier, by first re-enabling real-time. */
-  int prev_mode = md_eval->mode;
-  md_eval->mode |= eModifierMode_Realtime;
+  Depsgraph *apply_depsgraph = depsgraph;
+  Depsgraph *local_depsgraph = nullptr;
+
+  /* If the object is hidden or the modifier is not enabled for the viewport is disabled a special
+   * handling is required. This is because the viewport dependency graph optimizes out evaluation
+   * of objects which are used by hidden objects and disabled modifiers.
+   *
+   * The idea is to create a dependency graph which does not perform those optimizations. */
+  if ((ob_eval->base_flag & BASE_ENABLED_VIEWPORT) == 0 ||
+      (md_eval->mode & eModifierMode_Realtime) == 0) {
+    ViewLayer *view_layer = DEG_get_input_view_layer(depsgraph);
+
+    local_depsgraph = DEG_graph_new(bmain, scene, view_layer, DAG_EVAL_VIEWPORT);
+    DEG_disable_visibility_optimization(local_depsgraph);
+
+    ID *ids[] = {&ob->id};
+
+    DEG_graph_build_from_ids(local_depsgraph, ids, 1);
+    DEG_evaluate_on_refresh(local_depsgraph);
+
+    apply_depsgraph = local_depsgraph;
+
+    /* The evaluated object and modifier are now from the different dependency graph. */
+    ob_eval = DEG_get_evaluated_object(local_depsgraph, ob);
+    md_eval = BKE_modifiers_findby_name(ob_eval, md->name);
+
+    /* Force mode on the evaluated modifier, enforcing the modifier evaluation in the apply()
+     * functions. */
+    md_eval->mode |= eModifierMode_Realtime;
+  }
 
+  bool did_apply = false;
   if (mode == MODIFIER_APPLY_SHAPE) {
-    if (!modifier_apply_shape(bmain, reports, depsgraph, scene, ob, md_eval)) {
-      md_eval->mode = prev_mode;
-      return false;
-    }
+    did_apply = modifier_apply_shape(bmain, reports, apply_depsgraph, scene, ob, md_eval);
   }
   else {
-    if (!modifier_apply_obdata(reports, depsgraph, scene, ob, md_eval)) {
-      md_eval->mode = prev_mode;
-      return false;
-    }
+    did_apply = modifier_apply_obdata(reports, apply_depsgraph, scene, ob, md_eval);
   }
 
-  md_eval->mode = prev_mode;
-
-  if (!keep_modifier) {
-    BKE_modifier_remove_from_list(ob, md);
-    BKE_modifier_free(md);
+  if (did_apply) {
+    if (!keep_modifier) {
+      BKE_modifier_remove_from_list(ob, md);
+      BKE_modifier_free(md);
+    }
+    BKE_object_free_derived_caches(ob);
   }
 
-  BKE_object_free_derived_caches(ob);
+  if (local_depsgraph != nullptr) {
+    DEG_graph_free(local_depsgraph);
+  }
 
   return true;
 }