Merge branch 'master' into multi_previews_id

6 files changed, 151 insertions, 75 deletions

diff --git a/source/blender/blenkernel/intern/anim_sys.c b/source/blender/blenkernel/intern/anim_sys.c
index e3764adb969..a5abc6beff8 100644
--- a/source/blender/blenkernel/intern/anim_sys.c
+++ b/source/blender/blenkernel/intern/anim_sys.c
@@ -1657,7 +1657,7 @@ static bool animsys_write_rna_setting(PathResolvedRNA *anim_rna, const float val
 		/* for cases like duplifarmes it's only a temporary so don't
 		 * notify anyone of updates */
 		if (!(id->tag & LIB_TAG_ANIM_NO_RECALC)) {
-			id->tag |= LIB_TAG_ID_RECALC;
+			BKE_id_tag_set_atomic(id, LIB_TAG_ID_RECALC);
 			DAG_id_type_tag(G.main, GS(id->name));
 		}
 	}
diff --git a/source/blender/blenkernel/intern/armature.c b/source/blender/blenkernel/intern/armature.c
index cc508aa0511..badfeaee6bd 100644
--- a/source/blender/blenkernel/intern/armature.c
+++ b/source/blender/blenkernel/intern/armature.c
@@ -1795,6 +1795,7 @@ static void pose_proxy_synchronize(Object *ob, Object *from, int layer_protected
 			
 			/* copy posechannel to temp, but restore important pointers */
 			pchanw = *pchanp;
+			pchanw.bone = pchan->bone;
 			pchanw.prev = pchan->prev;
 			pchanw.next = pchan->next;
 			pchanw.parent = pchan->parent;
diff --git a/source/blender/blenkernel/intern/library.c b/source/blender/blenkernel/intern/library.c
index 5d28a84ef18..e5b066a3c26 100644
--- a/source/blender/blenkernel/intern/library.c
+++ b/source/blender/blenkernel/intern/library.c
@@ -129,6 +129,8 @@
 #include "IMB_imbuf.h"
 #include "IMB_imbuf_types.h"
 
+#include "atomic_ops.h"
+
 /* GS reads the memory pointed at in a specific ordering. 
  * only use this definition, makes little and big endian systems
  * work fine, in conjunction with MAKE_ID */
@@ -262,9 +264,12 @@ void id_fake_user_clear(ID *id)
 }
 
 static int id_expand_local_callback(
-        void *UNUSED(user_data), struct ID *UNUSED(id_self), struct ID **id_pointer, int UNUSED(cd_flag))
+        void *UNUSED(user_data), struct ID *id_self, struct ID **id_pointer, int UNUSED(cd_flag))
 {
-	if (*id_pointer) {
+	/* Can hapen that we get unlinkable ID here, e.g. with shapekey referring to itself (through drivers)...
+	 * Just skip it, shape key can only be either indirectly linked, or fully local, period.
+	 * And let's curse one more time that stupid useless shapekey ID type! */
+	if (*id_pointer && *id_pointer != id_self && BKE_idcode_is_linkable(GS((*id_pointer)->name))) {
 		id_lib_extern(*id_pointer);
 	}
 
@@ -1627,31 +1632,28 @@ void BKE_main_id_clear_newpoins(Main *bmain)
  * \param untagged_only If true, only make local datablocks not tagged with LIB_TAG_PRE_EXISTING.
  * \param set_fake If true, set fake user on all localized datablocks (except group and objects ones).
  */
-/* XXX TODO This function should probably be reworked.
- *
- * Old (2.77) version was simply making (tagging) datablocks as local, without actually making any check whether
+/* Note: Old (2.77) version was simply making (tagging) datablocks as local, without actually making any check whether
  * they were also indirectly used or not...
  *
- * Current version uses regular id_make_local callback, but this is not super-efficient since this ends up
+ * Current version uses regular id_make_local callback, which is not super-efficient since this ends up
  * duplicating some IDs and then removing original ones (due to missing knowledge of which ID uses some other ID).
  *
- * We could first check all IDs and detect those to be made local that are only used by other local or future-local
- * datablocks, and directly tag those as local (instead of going through id_make_local) maybe...
- *
- * We'll probably need at some point a true dependency graph between datablocks, but for now this should work
- * good enough (performances is not a critical point here anyway).
+ * However, we now have a first check that allows us to use 'direct localization' of a lot of IDs, so performances
+ * are now *reasonably* OK.
  */
 void BKE_library_make_local(
         Main *bmain, const Library *lib, GHash *old_to_new_ids, const bool untagged_only, const bool set_fake)
 {
 	ListBase *lbarray[MAX_LIBARRAY];
-	ID *id, *id_next;
+	ID *id;
 	int a;
 
+	LinkNode *todo_ids = NULL;
 	LinkNode *copied_ids = NULL;
 	LinkNode *linked_loop_candidates = NULL;
-	MemArena *linklist_mem = BLI_memarena_new(256 * sizeof(copied_ids), __func__);
+	MemArena *linklist_mem = BLI_memarena_new(512 * sizeof(*todo_ids), __func__);
 
+	/* Step 1: Detect datablocks to make local. */
 	for (a = set_listbasepointers(bmain, lbarray); a--; ) {
 		id = lbarray[a]->first;
 
@@ -1659,54 +1661,80 @@ void BKE_library_make_local(
 		 * by real datablocks responsible of them. */
 		const bool do_skip = (id && !BKE_idcode_is_linkable(GS(id->name)));
 
-		for (; id; id = id_next) {
+		for (; id; id = id->next) {
 			id->newid = NULL;
 			id->tag &= ~LIB_TAG_DOIT;
-			id_next = id->next;  /* id is possibly being inserted again */
 
-			/* The check on the second line (LIB_TAG_PRE_EXISTING) is done so its
+			if (id->lib == NULL) {
+				id->tag &= ~(LIB_TAG_EXTERN | LIB_TAG_INDIRECT | LIB_TAG_NEW);
+			}
+			/* The check on the fourth line (LIB_TAG_PRE_EXISTING) is done so its
 			 * possible to tag data you don't want to be made local, used for
 			 * appending data, so any libdata already linked wont become local
-			 * (very nasty to discover all your links are lost after appending)  
-			 * */
-			if (!do_skip && id->tag & (LIB_TAG_EXTERN | LIB_TAG_INDIRECT | LIB_TAG_NEW) &&
-			    ((untagged_only == false) || !(id->tag & LIB_TAG_PRE_EXISTING)))
+			 * (very nasty to discover all your links are lost after appending).
+			 * Also, never ever make proxified objects local, would not make any sense. */
+			else if (!do_skip && id->tag & (LIB_TAG_EXTERN | LIB_TAG_INDIRECT | LIB_TAG_NEW) &&
+			         ELEM(lib, NULL, id->lib) &&
+			         !(GS(id->name) == ID_OB && ((Object *)id)->proxy_from != NULL) &&
+			         ((untagged_only == false) || !(id->tag & LIB_TAG_PRE_EXISTING)))
 			{
-				if (lib == NULL || id->lib == lib) {
-					if (id->lib) {
-						/* In this specific case, we do want to make ID local even if it has no local usage yet... */
-						if (GS(id->name) == ID_OB) {
-							/* Special case for objects because we don't want proxy pointers to be
-							 * cleared yet. This will happen down the road in this function.
-							 */
-							BKE_object_make_local_ex(bmain, (Object*)id, true, false);
-						}
-						else {
-							id_make_local(bmain, id, false, true);
-						}
-
-						if (id->newid) {
-							BLI_linklist_prepend_arena(&copied_ids, id, linklist_mem);
-						}
-					}
-					else {
-						id->tag &= ~(LIB_TAG_EXTERN | LIB_TAG_INDIRECT | LIB_TAG_NEW);
-					}
-				}
+				BLI_linklist_prepend_arena(&todo_ids, id, linklist_mem);
+				id->tag |= LIB_TAG_DOIT;
+			}
+		}
+	}
 
-				if (set_fake) {
-					if (!ELEM(GS(id->name), ID_OB, ID_GR)) {
-						/* do not set fake user on objects, groups (instancing) */
-						id_fake_user_set(id);
-					}
-				}
+	/* Step 2: Check which datablocks we can directly make local (because they are only used by already, or future,
+	 * local data), others will need to be duplicated and further processed later. */
+	BKE_library_indirectly_used_data_tag_clear(bmain);
+
+	/* Step 3: Make IDs local, either directly (quick and simple), or using generic process,
+	 * which involves more complex checks and might instead create a local copy of original linked ID. */
+	for (LinkNode *it = todo_ids, *it_next; it; it = it_next) {
+		it_next = it->next;
+		id = it->link;
+
+		if (id->tag & LIB_TAG_DOIT) {
+			/* We know all users of this object are local or will be made fully local, even if currently there are
+			 * some indirect usages. So instead of making a copy that se'll likely get rid of later, directly make
+			 * that data block local. Saves a tremendous amount of time with complex scenes... */
+			id_clear_lib_data_ex(bmain, id, true);
+			BKE_id_expand_local(id);
+			id->tag &= ~LIB_TAG_DOIT;
+		}
+		else {
+			/* In this specific case, we do want to make ID local even if it has no local usage yet... */
+			if (GS(id->name) == ID_OB) {
+				/* Special case for objects because we don't want proxy pointers to be
+				 * cleared yet. This will happen down the road in this function.
+				 */
+				BKE_object_make_local_ex(bmain, (Object*)id, true, false);
+			}
+			else {
+				id_make_local(bmain, id, false, true);
+			}
+
+			if (id->newid) {
+				/* Reuse already allocated LinkNode (transferring it from todo_ids to copied_ids). */
+				BLI_linklist_prepend_nlink(&copied_ids, id, it);
+			}
+		}
+
+		if (set_fake) {
+			if (!ELEM(GS(id->name), ID_OB, ID_GR)) {
+				/* do not set fake user on objects, groups (instancing) */
+				id_fake_user_set(id);
 			}
 		}
 	}
 
-	/* We have to remap local usages of old (linked) ID to new (local) id in a second loop, as lbarray ordering is not
-	 * enough to ensure us we did catch all dependencies (e.g. if making local a parent object before its child...).
-	 * See T48907. */
+	/* At this point, we are done with directly made local IDs. Now we have to handle duplicated ones, since their
+	 * remaining linked original counterpart may not be needed anymore... */
+	todo_ids = NULL;
+
+	/* Step 4: We have to remap local usages of old (linked) ID to new (local) id in a separated loop,
+	 * as lbarray ordering is not enough to ensure us we did catch all dependencies
+	 * (e.g. if making local a parent object before its child...). See T48907. */
 	for (LinkNode *it = copied_ids; it; it = it->next) {
 		id = it->link;
 
@@ -1717,9 +1745,15 @@ void BKE_library_make_local(
 		if (old_to_new_ids) {
 			BLI_ghash_insert(old_to_new_ids, id, id->newid);
 		}
+
+		/* Special hack for groups... Thing is, since we can't instantiate them here, we need to ensure
+		 * they remain 'alive' (only instantiation is a real group 'user'... *sigh* See T49722. */
+		if (GS(id->name) == ID_GR && (id->tag & LIB_TAG_INDIRECT) != 0) {
+			id_us_ensure_real(id->newid);
+		}
 	}
 
-	/* Third step: remove datablocks that have been copied to be localized and are no more used in the end...
+	/* Step 5: remove datablocks that have been copied to be localized and are no more used in the end...
 	 * Note that we may have to loop more than once here, to tackle dependencies between linked objects... */
 	bool do_loop = true;
 	while (do_loop) {
@@ -1766,11 +1800,6 @@ void BKE_library_make_local(
 					ob->proxy = ob->proxy_from = ob->proxy_group = NULL;
 				}
 			}
-			/* Special hack for groups... Thing is, since we can't instantiate them here, we need to ensure
-			 * they remain 'alive' (only instantiation is a real group 'user'... *sigh* See T49722. */
-			else if (GS(id->name) == ID_GR && (id->tag & LIB_TAG_INDIRECT) != 0) {
-				id_us_ensure_real(id->newid);
-			}
 
 			if (!is_local) {
 				if (!is_lib) {  /* Not used at all, we can free it! */
@@ -1797,9 +1826,9 @@ void BKE_library_make_local(
 		}
 	}
 
-	/* Fourth step: Try to find circle dependencies between indirectly-linked-only datablocks.
+	/* Step 6: Try to find circle dependencies between indirectly-linked-only datablocks.
 	 * Those are fake 'usages' that prevent their deletion. See T49775 for nice ugly case. */
-	BKE_library_tag_unused_linked_data(bmain, false);
+	BKE_library_unused_linked_data_set_tag(bmain, false);
 	for (LinkNode *it = linked_loop_candidates; it; it = it->next) {
 		if (it->link == NULL) {
 			continue;
@@ -1812,11 +1841,17 @@ void BKE_library_make_local(
 		/* Note: in theory here we are only handling datablocks forming exclusive linked dependency-cycles-based
 		 * archipelagos, so no need to check again after we have deleted one, as done in previous step. */
 		if (id->tag & LIB_TAG_DOIT) {
+			/* Object's deletion rely on valid ob->data, but ob->data may have already been freed here...
+			 * Setting it to NULL may not be 100% correct, but should be safe and do the work. */
+			if (GS(id->name) == ID_OB) {
+				((Object *)id)->data = NULL;
+			}
+
 			/* Note: *in theory* IDs tagged here are fully *outside* of file scope, totally unused, so we can
 			 *       directly wipe them out without caring about clearing their usages.
 			 *       However, this is a highly-risky presumption, and nice crasher in case something goes wrong here.
 			 *       So for 2.78a will keep the safe option, and switch to more efficient one in master later. */
-#if 0
+#if 1
 			BKE_libblock_free_ex(bmain, id, false);
 #else
 			BKE_libblock_unlink(bmain, id, false, false);
@@ -1895,3 +1930,13 @@ void BKE_library_filepath_set(Library *lib, const char *filepath)
 		BLI_path_abs(lib->filepath, basepath);
 	}
 }
+
+void BKE_id_tag_set_atomic(ID *id, int tag)
+{
+	atomic_fetch_and_or_uint32((uint32_t *)&id->tag, tag);
+}
+
+void BKE_id_tag_clear_atomic(ID *id, int tag)
+{
+	atomic_fetch_and_and_uint32((uint32_t *)&id->tag, ~tag);
+}
diff --git a/source/blender/blenkernel/intern/library_query.c b/source/blender/blenkernel/intern/library_query.c
index cec7fbacd80..fa75c906fb1 100644
--- a/source/blender/blenkernel/intern/library_query.c
+++ b/source/blender/blenkernel/intern/library_query.c
@@ -1170,8 +1170,9 @@ void BKE_library_ID_test_usages(Main *bmain, void *idv, bool *is_used_local, boo
 	*is_used_linked = (iter.count_indirect != 0);
 }
 
-
-static int foreach_libblock_tag_unused_linked_data_callback(void *user_data, ID *self_id, ID **id_p, int UNUSED(cb_flag))
+/* ***** IDs usages.checking/tagging. ***** */
+static int foreach_libblock_used_linked_data_tag_clear_cb(
+        void *user_data, ID *self_id, ID **id_p, int UNUSED(cb_flag))
 {
 	bool *is_changed = user_data;
 
@@ -1206,7 +1207,7 @@ static int foreach_libblock_tag_unused_linked_data_callback(void *user_data, ID
  * \param do_init_tag if \a true, all linked data are checked, if \a false, only linked datablocks already tagged with
  *                    LIB_TAG_DOIT are checked.
  */
-void BKE_library_tag_unused_linked_data(Main *bmain, const bool do_init_tag)
+void BKE_library_unused_linked_data_set_tag(Main *bmain, const bool do_init_tag)
 {
 	ListBase *lb_array[MAX_LIBARRAY];
 
@@ -1232,7 +1233,38 @@ void BKE_library_tag_unused_linked_data(Main *bmain, const bool do_init_tag)
 
 		while (i--) {
 			for (ID *id = lb_array[i]->first; id; id = id->next) {
-				BKE_library_foreach_ID_link(id, foreach_libblock_tag_unused_linked_data_callback, &do_loop, IDWALK_NOP);
+				if (id->tag & LIB_TAG_DOIT) {
+					/* Unused ID (so far), no need to check it further. */
+					continue;
+				}
+				BKE_library_foreach_ID_link(id, foreach_libblock_used_linked_data_tag_clear_cb, &do_loop, IDWALK_NOP);
+			}
+		}
+	}
+}
+
+/**
+ * Untag linked data blocks used by other untagged linked datablocks.
+ * Used to detect datablocks that we can forcefully make local (instead of copying them to later get rid of original):
+ * All datablocks we want to make local are tagged by caller, after this function has ran caller knows datablocks still
+ * tagged can directly be made local, since they are only used by other datablocks that will also be made fully local.
+ */
+void BKE_library_indirectly_used_data_tag_clear(Main *bmain)
+{
+	ListBase *lb_array[MAX_LIBARRAY];
+
+	bool do_loop = true;
+	while (do_loop) {
+		int i = set_listbasepointers(bmain, lb_array);
+		do_loop = false;
+
+		while (i--) {
+			for (ID *id = lb_array[i]->first; id; id = id->next) {
+				if (id->lib == NULL || id->tag & LIB_TAG_DOIT) {
+					/* Local or non-indirectly-used ID (so far), no need to check it further. */
+					continue;
+				}
+				BKE_library_foreach_ID_link(id, foreach_libblock_used_linked_data_tag_clear_cb, &do_loop, IDWALK_NOP);
 			}
 		}
 	}
diff --git a/source/blender/blenkernel/intern/pbvh_bmesh.c b/source/blender/blenkernel/intern/pbvh_bmesh.c
index 55f9f384081..a821578db1a 100644
--- a/source/blender/blenkernel/intern/pbvh_bmesh.c
+++ b/source/blender/blenkernel/intern/pbvh_bmesh.c
@@ -148,8 +148,7 @@ BLI_INLINE void bm_face_as_array_index_tri(BMFace *f, int r_index[3])
  *
  * Its assumed that \a l_radial_first is never forming the target face.
  */
-static bool bm_face_exists_tri_from_loop_vert(
-        BMLoop *l_radial_first, BMVert *v_opposite, BMFace **r_face_existing)
+static BMFace *bm_face_exists_tri_from_loop_vert(BMLoop *l_radial_first, BMVert *v_opposite)
 {
 	BLI_assert(!ELEM(v_opposite, l_radial_first->v, l_radial_first->next->v, l_radial_first->prev->v));
 	if (l_radial_first->radial_next != l_radial_first) {
@@ -157,12 +156,11 @@ static bool bm_face_exists_tri_from_loop_vert(
 		do {
 			BLI_assert(l_radial_iter->f->len == 3);
 			if (l_radial_iter->prev->v == v_opposite) {
-				*r_face_existing = l_radial_iter->f;
-				return true;
+				return l_radial_iter->f;
 			}
 		} while ((l_radial_iter = l_radial_iter->radial_next) != l_radial_first);
 	}
-	return false;
+	return NULL;
 }
 
 /**
@@ -519,7 +517,7 @@ static BMFace *pbvh_bmesh_face_create(
 	PBVHNode *node = &bvh->nodes[node_index];
 
 	/* ensure we never add existing face */
-	BLI_assert(BM_face_exists(v_tri, 3, NULL) == false);
+	BLI_assert(!BM_face_exists(v_tri, 3));
 
 	BMFace *f = BM_face_create(bvh->bm, v_tri, e_tri, 3, f_example, BM_CREATE_NOP);
 	f->head.hflag = f_example->head.hflag;
@@ -1313,18 +1311,17 @@ static void pbvh_bmesh_collapse_edge(
 		 * deletion as well. Prevents extraneous "flaps" from being
 		 * created. */
 #if 0
-		if (UNLIKELY(BM_face_exists(v_tri, 3, &existing_face)))
+		if (UNLIKELY(existing_face = BM_face_exists(v_tri, 3)))
 #else
-		if (UNLIKELY(bm_face_exists_tri_from_loop_vert(l->next, v_conn, &existing_face)))
+		if (UNLIKELY(existing_face = bm_face_exists_tri_from_loop_vert(l->next, v_conn)))
 #endif
 		{
-			BLI_assert(existing_face);
 			BLI_buffer_append(deleted_faces, BMFace *, existing_face);
 		}
 		else {
 			BMVert *v_tri[3] = {v_conn, l->next->v, l->prev->v};
 
-			BLI_assert(BM_face_exists(v_tri, 3, NULL) == false);
+			BLI_assert(!BM_face_exists(v_tri, 3));
 			BMEdge *e_tri[3];
 			PBVHNode *n = pbvh_bmesh_node_from_face(bvh, f);
 			int ni = n - bvh->nodes;
diff --git a/source/blender/blenkernel/intern/sound.c b/source/blender/blenkernel/intern/sound.c
index 3132a8e27e7..f20885b1e8f 100644
--- a/source/blender/blenkernel/intern/sound.c
+++ b/source/blender/blenkernel/intern/sound.c
@@ -251,7 +251,8 @@ void BKE_sound_init(struct Main *bmain)
 void BKE_sound_init_main(struct Main *bmain)
 {
 #ifdef WITH_JACK
-	AUD_setSynchronizerCallback(sound_sync_callback, bmain);
+	if (sound_device)
+		AUD_setSynchronizerCallback(sound_sync_callback, bmain);
 #else
 	(void)bmain; /* unused */
 #endif