Fix missing Cycles 3D viewport updates when editing materials, lamps.

This introduces a new depsgraph API for getting updated datablocks,
rather than getting it from bpy.data.

* depsgraph.ids_updated gives a list of all datablocks in the depsgraph
  which have been updated.
* depsgraph.id_type_updated('TYPE') is true if any datablock of the given
  type has been added, removed or modified.

More API updates are coming to properly handle multiple depsgraphs and
finer update granularity, but this should make Cycles work again.

7 files changed, 172 insertions, 67 deletions

diff --git a/source/blender/depsgraph/DEG_depsgraph.h b/source/blender/depsgraph/DEG_depsgraph.h
index 589df5df878..d79ee9aa86b 100644
--- a/source/blender/depsgraph/DEG_depsgraph.h
+++ b/source/blender/depsgraph/DEG_depsgraph.h
@@ -173,7 +173,7 @@ void DEG_graph_id_tag_update(struct Main *bmain,
  */
 void DEG_id_type_tag(struct Main *bmain, short id_type);
 
-void DEG_ids_clear_recalc(struct Main *bmain);
+void DEG_ids_clear_recalc(struct Main *bmain, Depsgraph *depsgraph);
 
 /* Update Flushing ------------------------------- */
 
diff --git a/source/blender/depsgraph/DEG_depsgraph_query.h b/source/blender/depsgraph/DEG_depsgraph_query.h
index 99c5d2dc291..7a4d9a19335 100644
--- a/source/blender/depsgraph/DEG_depsgraph_query.h
+++ b/source/blender/depsgraph/DEG_depsgraph_query.h
@@ -66,7 +66,8 @@ float DEG_get_ctime(const Depsgraph *graph);
 /* ********************* DEG evaluated data ******************* */
 
 /* Check if given ID type was tagged for update. */
-bool DEG_id_type_tagged(struct Main *bmain, short id_type);
+bool DEG_id_type_updated(const struct Depsgraph *depsgraph, short id_type);
+bool DEG_id_type_any_updated(const struct Depsgraph *depsgraph);
 
 /* Get additional evaluation flags for the given ID. */
 short DEG_get_eval_flags_for_id(const struct Depsgraph *graph, struct ID *id);
@@ -96,7 +97,7 @@ struct Object *DEG_get_original_object(struct Object *object);
 /* Get original version of given evaluated ID datablock. */
 struct ID *DEG_get_original_id(struct ID *id);
 
-/* ************************ DEG iterators ********************* */
+/* ************************ DEG object iterators ********************* */
 
 enum {
 	DEG_ITER_OBJECT_FLAG_LINKED_DIRECTLY   = (1 << 0),
@@ -181,6 +182,21 @@ void DEG_iterator_objects_end(struct BLI_Iterator *iter);
 #define DEG_OBJECT_ITER_FOR_RENDER_ENGINE_END                             \
 	DEG_OBJECT_ITER_END
 
+
+/* ************************ DEG ID iterators ********************* */
+
+typedef struct DEGIDIterData {
+	struct Depsgraph *graph;
+	bool only_updated;
+
+	size_t id_node_index;
+	size_t num_id_nodes;
+} DEGIDIterData;
+
+void DEG_iterator_ids_begin(struct BLI_Iterator *iter, DEGIDIterData *data);
+void DEG_iterator_ids_next(struct BLI_Iterator *iter);
+void DEG_iterator_ids_end(struct BLI_Iterator *iter);
+
 /* ************************ DEG traversal ********************* */
 
 typedef void (*DEGForeachIDCallback)(ID *id, void *user_data);
diff --git a/source/blender/depsgraph/intern/depsgraph.h b/source/blender/depsgraph/intern/depsgraph.h
index 7b6af9deee1..a368158a6c4 100644
--- a/source/blender/depsgraph/intern/depsgraph.h
+++ b/source/blender/depsgraph/intern/depsgraph.h
@@ -36,6 +36,10 @@
 
 #pragma once
 
+#include <stdlib.h>
+
+#include "BKE_library.h" /* for MAX_LIBARRAY */
+
 #include "BLI_threads.h"  /* for SpinLock */
 
 #include "DEG_depsgraph.h"
@@ -174,6 +178,9 @@ struct Depsgraph {
 	/* Indicates whether relations needs to be updated. */
 	bool need_update;
 
+	/* Indicates which ID types were updated. */
+	char id_type_updated[MAX_LIBARRAY];
+
 	/* Quick-Access Temp Data ............. */
 
 	/* Nodes which have been tagged as "directly modified". */
diff --git a/source/blender/depsgraph/intern/depsgraph_query.cc b/source/blender/depsgraph/intern/depsgraph_query.cc
index 734b0ef931a..06fbe980620 100644
--- a/source/blender/depsgraph/intern/depsgraph_query.cc
+++ b/source/blender/depsgraph/intern/depsgraph_query.cc
@@ -80,9 +80,24 @@ float DEG_get_ctime(const Depsgraph *graph)
 }
 
 
-bool DEG_id_type_tagged(Main *bmain, short id_type)
+bool DEG_id_type_updated(const Depsgraph *graph, short id_type)
 {
-	return bmain->id_tag_update[BKE_idcode_to_index(id_type)] != 0;
+	const DEG::Depsgraph *deg_graph = reinterpret_cast<const DEG::Depsgraph *>(graph);
+	return deg_graph->id_type_updated[BKE_idcode_to_index(id_type)] != 0;
+}
+
+bool DEG_id_type_any_updated(const Depsgraph *graph)
+{
+	const DEG::Depsgraph *deg_graph = reinterpret_cast<const DEG::Depsgraph *>(graph);
+
+	/* Loop over all ID types. */
+	for (int id_type_index = 0; id_type_index < MAX_LIBARRAY; id_type_index++) {
+		if (deg_graph->id_type_updated[id_type_index]) {
+			return true;
+		}
+	}
+
+	return false;
 }
 
 short DEG_get_eval_flags_for_id(const Depsgraph *graph, ID *id)
diff --git a/source/blender/depsgraph/intern/depsgraph_query_iter.cc b/source/blender/depsgraph/intern/depsgraph_query_iter.cc
index 1726c7c855a..843d379058a 100644
--- a/source/blender/depsgraph/intern/depsgraph_query_iter.cc
+++ b/source/blender/depsgraph/intern/depsgraph_query_iter.cc
@@ -41,6 +41,7 @@ extern "C" {
 #include "BKE_anim.h"
 #include "BKE_idprop.h"
 #include "BKE_layer.h"
+#include "BKE_node.h"
 #include "BKE_object.h"
 } /* extern "C" */
 
@@ -196,13 +197,13 @@ void DEG_iterator_objects_begin(BLI_Iterator *iter, DEGObjectIterData *data)
 	DEG::Depsgraph *deg_graph = reinterpret_cast<DEG::Depsgraph *>(depsgraph);
 	const size_t num_id_nodes = deg_graph->id_nodes.size();
 
+	iter->data = data;
+
 	if (num_id_nodes == 0) {
-		iter->data = NULL;
 		iter->valid = false;
 		return;
 	}
 
-	iter->data = data;
 	data->dupli_parent = NULL;
 	data->dupli_list = NULL;
 	data->dupli_object_next = NULL;
@@ -269,3 +270,72 @@ void DEG_iterator_objects_end(BLI_Iterator *iter)
 	(void) iter;
 #endif
 }
+
+/* ************************ DEG ID ITERATOR ********************* */
+
+static void DEG_iterator_ids_step(BLI_Iterator *iter, DEG::IDDepsNode *id_node, bool only_updated)
+{
+	ID *id_cow = id_node->id_cow;
+
+	if (only_updated && !(id_cow->recalc & ID_RECALC_ALL)) {
+		bNodeTree *ntree = ntreeFromID(id_cow);
+
+		/* Nodetree is considered part of the datablock. */
+		if (!(ntree && (ntree->id.recalc & ID_RECALC_ALL))) {
+			iter->skip = true;
+			return;
+		}
+	}
+
+	iter->current = id_cow;
+	iter->skip = false;
+}
+
+void DEG_iterator_ids_begin(BLI_Iterator *iter, DEGIDIterData *data)
+{
+	Depsgraph *depsgraph = data->graph;
+	DEG::Depsgraph *deg_graph = reinterpret_cast<DEG::Depsgraph *>(depsgraph);
+	const size_t num_id_nodes = deg_graph->id_nodes.size();
+
+	iter->data = data;
+
+	if ((num_id_nodes == 0) ||
+	    (data->only_updated && !DEG_id_type_any_updated(depsgraph))) {
+		iter->valid = false;
+		return;
+	}
+
+	data->id_node_index = 0;
+	data->num_id_nodes = num_id_nodes;
+
+	DEG::IDDepsNode *id_node = deg_graph->id_nodes[data->id_node_index];
+	DEG_iterator_ids_step(iter, id_node, data->only_updated);
+
+	if (iter->skip) {
+		DEG_iterator_ids_next(iter);
+	}
+}
+
+void DEG_iterator_ids_next(BLI_Iterator *iter)
+{
+	DEGIDIterData *data = (DEGIDIterData *)iter->data;
+	Depsgraph *depsgraph = data->graph;
+	DEG::Depsgraph *deg_graph = reinterpret_cast<DEG::Depsgraph *>(depsgraph);
+
+	do {
+		iter->skip = false;
+
+		++data->id_node_index;
+		if (data->id_node_index == data->num_id_nodes) {
+			iter->valid = false;
+			return;
+		}
+
+		DEG::IDDepsNode *id_node = deg_graph->id_nodes[data->id_node_index];
+		DEG_iterator_ids_step(iter, id_node, data->only_updated);
+	} while (iter->skip);
+}
+
+void DEG_iterator_ids_end(BLI_Iterator *UNUSED(iter))
+{
+}
diff --git a/source/blender/depsgraph/intern/depsgraph_tag.cc b/source/blender/depsgraph/intern/depsgraph_tag.cc
index b1645b0cb49..8e63bc85499 100644
--- a/source/blender/depsgraph/intern/depsgraph_tag.cc
+++ b/source/blender/depsgraph/intern/depsgraph_tag.cc
@@ -64,6 +64,7 @@ extern "C" {
 } /* extern "C" */
 
 #include "DEG_depsgraph.h"
+#include "DEG_depsgraph_query.h"
 
 #include "intern/builder/deg_builder.h"
 #include "intern/eval/deg_eval_flush.h"
@@ -621,7 +622,20 @@ void DEG_id_type_tag(Main *bmain, short id_type)
 		DEG_id_type_tag(bmain, ID_SCE);
 	}
 
-	bmain->id_tag_update[BKE_idcode_to_index(id_type)] = 1;
+	int id_type_index = BKE_idcode_to_index(id_type);
+
+	LISTBASE_FOREACH (Scene *, scene, &bmain->scene) {
+		LISTBASE_FOREACH (ViewLayer *, view_layer, &scene->view_layers) {
+			Depsgraph *depsgraph =
+			        (Depsgraph *)BKE_scene_get_depsgraph(scene,
+			                                             view_layer,
+			                                             false);
+			if (depsgraph != NULL) {
+				DEG::Depsgraph *deg_graph = reinterpret_cast<DEG::Depsgraph *>(depsgraph);
+				deg_graph->id_type_updated[id_type_index] = 1;
+			}
+		}
+	}
 }
 
 void DEG_graph_flush_update(Main *bmain, Depsgraph *depsgraph)
@@ -663,58 +677,62 @@ void DEG_ids_check_recalc(Main *bmain,
                           ViewLayer *view_layer,
                           bool time)
 {
-	ListBase *lbarray[MAX_LIBARRAY];
-	int a;
-	bool updated = false;
-
-	/* Loop over all ID types. */
-	a  = set_listbasepointers(bmain, lbarray);
-	while (a--) {
-		ListBase *lb = lbarray[a];
-		ID *id = (ID *)lb->first;
-
-		if (id && bmain->id_tag_update[BKE_idcode_to_index(GS(id->name))]) {
-			updated = true;
-			break;
-		}
-	}
+	bool updated = time || DEG_id_type_any_updated(depsgraph);
 
 	DEGEditorUpdateContext update_ctx = {NULL};
 	update_ctx.bmain = bmain;
 	update_ctx.depsgraph = depsgraph;
 	update_ctx.scene = scene;
 	update_ctx.view_layer = view_layer;
-	DEG::deg_editors_scene_update(&update_ctx, (updated || time));
+	DEG::deg_editors_scene_update(&update_ctx, updated);
+}
+
+static void deg_graph_clear_id_node_func(
+        void *__restrict data_v,
+        const int i,
+        const ParallelRangeTLS *__restrict /*tls*/)
+{
+	/* TODO: we clear original ID recalc flags here, but this may not work
+	 * correctly when there are multiple depsgraph with others still using
+	 * the recalc flag. */
+	DEG::Depsgraph *deg_graph = reinterpret_cast<DEG::Depsgraph *>(data_v);
+	DEG::IDDepsNode *id_node = deg_graph->id_nodes[i];
+	id_node->id_cow->recalc &= ~ID_RECALC_ALL;
+	id_node->id_orig->recalc &= ~ID_RECALC_ALL;
+
+	/* Clear embedded node trees too. */
+	bNodeTree *ntree_cow = ntreeFromID(id_node->id_cow);
+	if (ntree_cow) {
+		ntree_cow->id.recalc &= ~ID_RECALC_ALL;
+	}
+	bNodeTree *ntree_orig = ntreeFromID(id_node->id_orig);
+	if (ntree_orig) {
+		ntree_orig->id.recalc &= ~ID_RECALC_ALL;
+	}
 }
 
-void DEG_ids_clear_recalc(Main *bmain)
+void DEG_ids_clear_recalc(Main *UNUSED(bmain),
+                          Depsgraph *depsgraph)
 {
-	ListBase *lbarray[MAX_LIBARRAY];
-	bNodeTree *ntree;
-	int a;
+	DEG::Depsgraph *deg_graph = reinterpret_cast<DEG::Depsgraph *>(depsgraph);
 
 	/* TODO(sergey): Re-implement POST_UPDATE_HANDLER_WORKAROUND using entry_tags
 	 * and id_tags storage from the new dependency graph.
 	 */
 
-	/* Loop over all ID types. */
-	a  = set_listbasepointers(bmain, lbarray);
-	while (a--) {
-		ListBase *lb = lbarray[a];
-		ID *id = (ID *)lb->first;
-
-		if (id && bmain->id_tag_update[BKE_idcode_to_index(GS(id->name))]) {
-			for (; id; id = (ID *)id->next) {
-				id->recalc &= ~ID_RECALC_ALL;
-
-				/* Some ID's contain semi-datablock nodetree */
-				ntree = ntreeFromID(id);
-				if (ntree != NULL) {
-					ntree->id.recalc &= ~ID_RECALC_ALL;
-				}
-			}
-		}
+	if (!DEG_id_type_any_updated(depsgraph)) {
+		return;
 	}
 
-	memset(bmain->id_tag_update, 0, sizeof(bmain->id_tag_update));
+	/* Go over all ID nodes nodes, clearing tags. */
+	const int num_id_nodes = deg_graph->id_nodes.size();
+	ParallelRangeSettings settings;
+	BLI_parallel_range_settings_defaults(&settings);
+	settings.min_iter_per_thread = 1024;
+	BLI_task_parallel_range(0, num_id_nodes,
+	                        deg_graph,
+	                        deg_graph_clear_id_node_func,
+	                        &settings);
+
+	memset(deg_graph->id_type_updated, 0, sizeof(deg_graph->id_type_updated));
 }
diff --git a/source/blender/depsgraph/intern/eval/deg_eval_flush.cc b/source/blender/depsgraph/intern/eval/deg_eval_flush.cc
index 02f10130fed..3330c802aa9 100644
--- a/source/blender/depsgraph/intern/eval/deg_eval_flush.cc
+++ b/source/blender/depsgraph/intern/eval/deg_eval_flush.cc
@@ -313,16 +313,6 @@ static void graph_clear_operation_func(
 	node->flag &= ~(DEPSOP_FLAG_DIRECTLY_MODIFIED | DEPSOP_FLAG_NEEDS_UPDATE);
 }
 
-static void graph_clear_id_node_func(
-        void *__restrict data_v,
-        const int i,
-        const ParallelRangeTLS *__restrict /*tls*/)
-{
-	Depsgraph *graph = (Depsgraph *)data_v;
-	IDDepsNode *id_node = graph->id_nodes[i];
-	id_node->id_cow->recalc &= ~ID_RECALC_ALL;
-}
-
 /* Clear tags from all operation nodes. */
 void deg_graph_clear_tags(Depsgraph *graph)
 {
@@ -337,17 +327,6 @@ void deg_graph_clear_tags(Depsgraph *graph)
 		                        graph_clear_operation_func,
 		                        &settings);
 	}
-	/* Go over all ID nodes nodes, clearing tags. */
-	{
-		const int num_id_nodes = graph->id_nodes.size();
-		ParallelRangeSettings settings;
-		BLI_parallel_range_settings_defaults(&settings);
-		settings.min_iter_per_thread = 1024;
-		BLI_task_parallel_range(0, num_id_nodes,
-		                        graph,
-		                        graph_clear_id_node_func,
-		                        &settings);
-	}
 	/* Clear any entry tags which haven't been flushed. */
 	BLI_gset_clear(graph->entry_tags, NULL);
 }