Fix T87535, T87295: issues with new persistent data option

Some persistent data code was disable due to a deeper design issue, which
meant some updates were not communicated to renderers.

Dependency graph updates work in two passes, once where Blender scene
animation updates are done, then app handler scripts can run to make further
scene modifications, and then the depsgraph is updated again to take those
into account.

Previously the viewport would update renderers twice when such app handler
scripts were present. Now both viewport and persistent data rendering update
the renderers only once, accumulating updates from both passes.

6 files changed, 62 insertions, 26 deletions

diff --git a/source/blender/blenkernel/intern/scene.c b/source/blender/blenkernel/intern/scene.c
index 44c0fd5144b..47b6817d429 100644
--- a/source/blender/blenkernel/intern/scene.c
+++ b/source/blender/blenkernel/intern/scene.c
@@ -2636,6 +2636,7 @@ static void scene_graph_update_tagged(Depsgraph *depsgraph, Main *bmain, bool on
 
   Scene *scene = DEG_get_input_scene(depsgraph);
   ViewLayer *view_layer = DEG_get_input_view_layer(depsgraph);
+  bool used_multiple_passes = false;
 
   bool run_callbacks = DEG_id_type_any_updated(depsgraph);
   if (run_callbacks) {
@@ -2672,8 +2673,6 @@ static void scene_graph_update_tagged(Depsgraph *depsgraph, Main *bmain, bool on
        * If there are no relations changed by the callback this call will do nothing. */
       DEG_graph_relations_update(depsgraph);
     }
-    /* Inform editors about possible changes. */
-    DEG_editors_update(bmain, depsgraph, scene, view_layer, false);
 
     /* If user callback did not tag anything for update we can skip second iteration.
      * Otherwise we update scene once again, but without running callbacks to bring
@@ -2682,8 +2681,17 @@ static void scene_graph_update_tagged(Depsgraph *depsgraph, Main *bmain, bool on
       break;
     }
 
+    /* Clear recalc flags for second pass, but back them up for editors update. */
+    DEG_ids_clear_recalc(depsgraph, true);
+    used_multiple_passes = true;
     run_callbacks = false;
   }
+
+  /* Inform editors about changes, using recalc flags from both passes. */
+  if (used_multiple_passes) {
+    DEG_ids_restore_recalc(depsgraph);
+  }
+  DEG_editors_update(depsgraph, false);
 }
 
 void BKE_scene_graph_update_tagged(Depsgraph *depsgraph, Main *bmain)
@@ -2702,6 +2710,7 @@ void BKE_scene_graph_update_for_newframe(Depsgraph *depsgraph)
   Scene *scene = DEG_get_input_scene(depsgraph);
   ViewLayer *view_layer = DEG_get_input_view_layer(depsgraph);
   Main *bmain = DEG_get_bmain(depsgraph);
+  bool used_multiple_passes = false;
 
   /* Keep this first. */
   BKE_callback_exec_id(bmain, &scene->id, BKE_CB_EVT_FRAME_CHANGE_PRE);
@@ -2738,16 +2747,23 @@ void BKE_scene_graph_update_for_newframe(Depsgraph *depsgraph)
       DEG_graph_relations_update(depsgraph);
     }
 
-    /* Inform editors about possible changes. */
-    DEG_editors_update(bmain, depsgraph, scene, view_layer, true);
-
     /* If user callback did not tag anything for update we can skip second iteration.
      * Otherwise we update scene once again, but without running callbacks to bring
      * scene to a fully evaluated state with user modifications taken into account. */
     if (DEG_is_fully_evaluated(depsgraph)) {
       break;
     }
+
+    /* Clear recalc flags for second pass, but back them up for editors update. */
+    DEG_ids_clear_recalc(depsgraph, true);
+    used_multiple_passes = true;
+  }
+
+  /* Inform editors about changes, using recalc flags from both passes. */
+  if (used_multiple_passes) {
+    DEG_ids_restore_recalc(depsgraph);
   }
+  DEG_editors_update(depsgraph, true);
 }
 
 /**
diff --git a/source/blender/depsgraph/DEG_depsgraph.h b/source/blender/depsgraph/DEG_depsgraph.h
index f7aeca7e75f..134e71ecb6a 100644
--- a/source/blender/depsgraph/DEG_depsgraph.h
+++ b/source/blender/depsgraph/DEG_depsgraph.h
@@ -145,14 +145,14 @@ void DEG_enable_editors_update(struct Depsgraph *depsgraph);
 
 /* Check if something was changed in the database and inform editors about this,
  * then clear recalc flags. */
-void DEG_editors_update(struct Main *bmain,
-                        struct Depsgraph *depsgraph,
-                        struct Scene *scene,
-                        struct ViewLayer *view_layer,
-                        bool time);
+void DEG_editors_update(struct Depsgraph *depsgraph, bool time);
 
 /* Clear recalc flags after editors or renderers have handled updates. */
-void DEG_ids_clear_recalc(Depsgraph *depsgraph);
+void DEG_ids_clear_recalc(Depsgraph *depsgraph, const bool backup);
+
+/* Restore recalc flags, backed up by a previous call to DEG_ids_clear_recalc.
+ * This also clears the backup. */
+void DEG_ids_restore_recalc(Depsgraph *depsgraph);
 
 /* ************************************************ */
 /* Evaluation Engine API */
diff --git a/source/blender/depsgraph/intern/depsgraph_tag.cc b/source/blender/depsgraph/intern/depsgraph_tag.cc
index d658fcce08a..608ec6292f6 100644
--- a/source/blender/depsgraph/intern/depsgraph_tag.cc
+++ b/source/blender/depsgraph/intern/depsgraph_tag.cc
@@ -824,23 +824,26 @@ void DEG_enable_editors_update(Depsgraph *depsgraph)
 
 /* Check if something was changed in the database and inform
  * editors about this. */
-void DEG_editors_update(
-    Main *bmain, Depsgraph *depsgraph, Scene *scene, ViewLayer *view_layer, bool time)
+void DEG_editors_update(Depsgraph *depsgraph, bool time)
 {
   deg::Depsgraph *graph = (deg::Depsgraph *)depsgraph;
+  if (!graph->use_editors_update) {
+    return;
+  }
 
-  if (graph->use_editors_update) {
-    bool updated = time || DEG_id_type_any_updated(depsgraph);
+  Scene *scene = DEG_get_input_scene(depsgraph);
+  ViewLayer *view_layer = DEG_get_input_view_layer(depsgraph);
+  Main *bmain = DEG_get_bmain(depsgraph);
+  bool updated = time || DEG_id_type_any_updated(depsgraph);
 
-    DEGEditorUpdateContext update_ctx = {nullptr};
-    update_ctx.bmain = bmain;
-    update_ctx.depsgraph = depsgraph;
-    update_ctx.scene = scene;
-    update_ctx.view_layer = view_layer;
-    deg::deg_editors_scene_update(&update_ctx, updated);
-  }
+  DEGEditorUpdateContext update_ctx = {nullptr};
+  update_ctx.bmain = bmain;
+  update_ctx.depsgraph = depsgraph;
+  update_ctx.scene = scene;
+  update_ctx.view_layer = view_layer;
+  deg::deg_editors_scene_update(&update_ctx, updated);
 
-  DEG_ids_clear_recalc(depsgraph);
+  DEG_ids_clear_recalc(depsgraph, false);
 }
 
 static void deg_graph_clear_id_recalc_flags(ID *id)
@@ -854,7 +857,7 @@ static void deg_graph_clear_id_recalc_flags(ID *id)
   /* XXX And what about scene's master collection here? */
 }
 
-void DEG_ids_clear_recalc(Depsgraph *depsgraph)
+void DEG_ids_clear_recalc(Depsgraph *depsgraph, const bool backup)
 {
   deg::Depsgraph *deg_graph = reinterpret_cast<deg::Depsgraph *>(depsgraph);
   /* TODO(sergey): Re-implement POST_UPDATE_HANDLER_WORKAROUND using entry_tags
@@ -864,6 +867,9 @@ void DEG_ids_clear_recalc(Depsgraph *depsgraph)
   }
   /* Go over all ID nodes nodes, clearing tags. */
   for (deg::IDNode *id_node : deg_graph->id_nodes) {
+    if (backup) {
+      id_node->id_cow_recalc_backup |= id_node->id_cow->recalc;
+    }
     /* TODO: we clear original ID recalc flags here, but this may not work
      * correctly when there are multiple depsgraph with others still using
      * the recalc flag. */
@@ -875,3 +881,13 @@ void DEG_ids_clear_recalc(Depsgraph *depsgraph)
   }
   memset(deg_graph->id_type_updated, 0, sizeof(deg_graph->id_type_updated));
 }
+
+void DEG_ids_restore_recalc(Depsgraph *depsgraph)
+{
+  deg::Depsgraph *deg_graph = reinterpret_cast<deg::Depsgraph *>(depsgraph);
+
+  for (deg::IDNode *id_node : deg_graph->id_nodes) {
+    id_node->id_cow->recalc |= id_node->id_cow_recalc_backup;
+    id_node->id_cow_recalc_backup = 0;
+  }
+}
diff --git a/source/blender/depsgraph/intern/node/deg_node_id.cc b/source/blender/depsgraph/intern/node/deg_node_id.cc
index 8e159a7ff08..688afe141e9 100644
--- a/source/blender/depsgraph/intern/node/deg_node_id.cc
+++ b/source/blender/depsgraph/intern/node/deg_node_id.cc
@@ -90,6 +90,7 @@ void IDNode::init(const ID *id, const char *UNUSED(subdata))
   is_collection_fully_expanded = false;
   has_base = false;
   is_user_modified = false;
+  id_cow_recalc_backup = 0;
 
   visible_components_mask = 0;
   previously_visible_components_mask = 0;
diff --git a/source/blender/depsgraph/intern/node/deg_node_id.h b/source/blender/depsgraph/intern/node/deg_node_id.h
index e2d3b3fc36f..073469598dc 100644
--- a/source/blender/depsgraph/intern/node/deg_node_id.h
+++ b/source/blender/depsgraph/intern/node/deg_node_id.h
@@ -120,6 +120,9 @@ struct IDNode : public Node {
   /* Accumulated flag from operation. Is initialized and used during updates flush. */
   bool is_user_modified;
 
+  /* Accumulate recalc flags from multiple update passes. */
+  int id_cow_recalc_backup;
+
   IDComponentsMask visible_components_mask;
   IDComponentsMask previously_visible_components_mask;
 
diff --git a/source/blender/render/intern/engine.c b/source/blender/render/intern/engine.c
index 09ad875b479..2edfaf09358 100644
--- a/source/blender/render/intern/engine.c
+++ b/source/blender/render/intern/engine.c
@@ -702,7 +702,7 @@ static void engine_depsgraph_exit(RenderEngine *engine)
     if (engine_keep_depsgraph(engine)) {
       /* Clear recalc flags since the engine should have handled the updates for the currently
        * rendered framed by now. */
-      DEG_ids_clear_recalc(engine->depsgraph);
+      DEG_ids_clear_recalc(engine->depsgraph, false);
     }
     else {
       /* Free immediately to save memory. */
@@ -718,7 +718,7 @@ void RE_engine_frame_set(RenderEngine *engine, int frame, float subframe)
   }
 
   /* Clear recalc flags before update so engine can detect what changed. */
-  DEG_ids_clear_recalc(engine->depsgraph);
+  DEG_ids_clear_recalc(engine->depsgraph, false);
 
   Render *re = engine->re;
   double cfra = (double)frame + (double)subframe;