ID-Remap, step two: add some user-level tools.

This commit adds operators and Outliner menu entries to reload or relocate a library,
and to delete or replace a datablock.

RNA ID API is also extended to allow ID deletion and remapping from python.

Review task: D2027 (https://developer.blender.org/D2027).
Reviewed by campbellbarton, thanks a bunch.

1 files changed, 397 insertions, 3 deletions

diff --git a/source/blender/windowmanager/intern/wm_files_link.c b/source/blender/windowmanager/intern/wm_files_link.c
index 2e4a4b63b7a..3709e10b366 100644
--- a/source/blender/windowmanager/intern/wm_files_link.c
+++ b/source/blender/windowmanager/intern/wm_files_link.c
@@ -61,6 +61,7 @@
 #include "BKE_context.h"
 #include "BKE_depsgraph.h"
 #include "BKE_library.h"
+#include "BKE_library_remap.h"
 #include "BKE_global.h"
 #include "BKE_main.h"
 #include "BKE_report.h"
@@ -211,7 +212,8 @@ static WMLinkAppendDataItem *wm_link_append_data_item_add(
 }
 
 static void wm_link_do(
-        WMLinkAppendData *lapp_data, ReportList *reports, Main *bmain, Scene *scene, View3D *v3d)
+        WMLinkAppendData *lapp_data, ReportList *reports, Main *bmain, Scene *scene, View3D *v3d,
+        const bool use_placeholders, const bool force_indirect)
 {
 	Main *mainl;
 	BlendHandle *bh;
@@ -258,7 +260,9 @@ static void wm_link_do(
 				continue;
 			}
 
-			new_id = BLO_library_link_named_part_ex(mainl, &bh, item->idcode, item->name, flag, scene, v3d);
+			new_id = BLO_library_link_named_part_ex(
+			             mainl, &bh, item->idcode, item->name, flag, scene, v3d, use_placeholders, force_indirect);
+
 			if (new_id) {
 				/* If the link is sucessful, clear item's libs 'todo' flags.
 				 * This avoids trying to link same item with other libraries to come. */
@@ -401,7 +405,7 @@ static int wm_link_append_exec(bContext *C, wmOperator *op)
 	/* XXX We'd need re-entrant locking on Main for this to work... */
 	/* BKE_main_lock(bmain); */
 
-	wm_link_do(lapp_data, op->reports, bmain, scene, CTX_wm_view3d(C));
+	wm_link_do(lapp_data, op->reports, bmain, scene, CTX_wm_view3d(C), false, false);
 
 	/* BKE_main_unlock(bmain); */
 
@@ -518,3 +522,393 @@ void WM_OT_append(wmOperatorType *ot)
 	RNA_def_boolean(ot->srna, "use_recursive", true, "Localize All",
 	                "Localize all appended data, including those indirectly linked from other libraries");
 }
+
+/** \name Reload/relocate libraries.
+ *
+ * \{ */
+
+static int wm_lib_relocate_invoke(bContext *C, wmOperator *op, const wmEvent *UNUSED(event))
+{
+	Library *lib;
+	char lib_name[MAX_NAME];
+
+	RNA_string_get(op->ptr, "library", lib_name);
+	lib = (Library *)BKE_libblock_find_name_ex(CTX_data_main(C), ID_LI, lib_name);
+
+	if (lib) {
+		if (lib->parent) {
+			BKE_reportf(op->reports, RPT_ERROR_INVALID_INPUT,
+			            "Cannot relocate indirectly linked library '%s'", lib->filepath);
+			return OPERATOR_CANCELLED;
+		}
+		RNA_string_set(op->ptr, "filepath", lib->filepath);
+
+		WM_event_add_fileselect(C, op);
+
+		return OPERATOR_RUNNING_MODAL;
+	}
+
+	return OPERATOR_CANCELLED;
+}
+
+/* Note that IDs listed in lapp_data items *must* have been removed from bmain by caller. */
+static void lib_relocate_do(Main *bmain, WMLinkAppendData *lapp_data, ReportList *reports, const bool do_reload)
+{
+	ListBase *lbarray[MAX_LIBARRAY];
+	int lba_idx;
+
+	LinkNode *itemlink;
+	int item_idx;
+
+	BKE_main_id_tag_all(bmain, LIB_TAG_PRE_EXISTING, true);
+
+	/* We do not want any instanciation here! */
+	wm_link_do(lapp_data, reports, bmain, NULL, NULL, do_reload, do_reload);
+
+	BKE_main_lock(bmain);
+
+	/* We add back old id to bmain.
+	 * We need to do this in a first, separated loop, otherwise some of those may not be handled by
+	 * ID remapping, which means they would still reference old data to be deleted... */
+	for (item_idx = 0, itemlink = lapp_data->items.list; itemlink; item_idx++, itemlink = itemlink->next) {
+		WMLinkAppendDataItem *item = itemlink->link;
+		ID *old_id = item->customdata;
+
+		BLI_assert(old_id);
+		BLI_addtail(which_libbase(bmain, GS(old_id->name)), old_id);
+	}
+
+	/* Note that in reload case, we also want to replace indirect usages. */
+	const short remap_flags = ID_REMAP_SKIP_NEVER_NULL_USAGE | (do_reload ? 0 : ID_REMAP_SKIP_INDIRECT_USAGE);
+	for (item_idx = 0, itemlink = lapp_data->items.list; itemlink; item_idx++, itemlink = itemlink->next) {
+		WMLinkAppendDataItem *item = itemlink->link;
+		ID *old_id = item->customdata;
+		ID *new_id = item->new_id;
+
+		BLI_assert(old_id);
+		if (do_reload) {
+			/* Since we asked for placeholders in case of missing IDs, we expect to always get a valid one. */
+			BLI_assert(new_id);
+		}
+		if (new_id) {
+#ifdef PRINT_DEBUG
+			printf("before remap, old_id users: %d, new_id users: %d\n", old_id->us, new_id->us);
+#endif
+			BKE_libblock_remap_locked(bmain, old_id, new_id, remap_flags);
+
+			if (old_id->flag & LIB_FAKEUSER) {
+				id_fake_user_clear(old_id);
+				id_fake_user_set(new_id);
+			}
+
+#ifdef PRINT_DEBUG
+			printf("after remap, old_id users: %d, new_id users: %d\n", old_id->us, new_id->us);
+#endif
+
+			/* In some cases, new_id might become direct link, remove parent of library in this case. */
+			if (new_id->lib->parent && (new_id->tag & LIB_TAG_INDIRECT) == 0) {
+				if (do_reload) {
+					BLI_assert(0);  /* Should not happen in 'pure' reload case... */
+				}
+				new_id->lib->parent = NULL;
+			}
+		}
+
+		if (old_id->us > 0 && new_id && old_id->lib == new_id->lib) {
+			/* Note that this *should* not happen - but better be safe than sorry in this area, at least until we are
+			 * 100% sure this cannot ever happen.
+			 * Also, we can safely assume names were unique so far, so just replacing '.' by '~' should work,
+			 * but this does not totally rules out the possibility of name collision. */
+			size_t len = strlen(old_id->name);
+			size_t dot_pos;
+			bool has_num = false;
+
+			for (dot_pos = len; dot_pos--;) {
+				char c = old_id->name[dot_pos];
+				if (c == '.') {
+					break;
+				}
+				else if (c < '0' || c > '9') {
+					has_num = false;
+					break;
+				}
+				has_num = true;
+			}
+
+			if (has_num) {
+				old_id->name[dot_pos] = '~';
+			}
+			else {
+				len = MIN2(len, MAX_ID_NAME - 7);
+				BLI_strncpy(&old_id->name[len], "~000", 7);
+			}
+
+			id_sort_by_name(which_libbase(bmain, GS(old_id->name)), old_id);
+
+			BKE_reportf(reports, RPT_WARNING,
+			            "Lib Reload: Replacing all references to old datablock '%s' by reloaded one failed, "
+			            "old one (%d remaining users) had to be kept and was renamed to '%s'",
+			            new_id->name, old_id->us, old_id->name);
+		}
+	}
+
+	BKE_main_unlock(bmain);
+
+	for (item_idx = 0, itemlink = lapp_data->items.list; itemlink; item_idx++, itemlink = itemlink->next) {
+		WMLinkAppendDataItem *item = itemlink->link;
+		ID *old_id = item->customdata;
+
+		if (old_id->us == 0) {
+			BKE_libblock_free(bmain, old_id);
+		}
+	}
+
+	/* Some datablocks can get reloaded/replaced 'silently' because they are not linkable (shape keys e.g.),
+	 * so we need another loop here to clear old ones if possible. */
+	lba_idx = set_listbasepointers(bmain, lbarray);
+	while (lba_idx--) {
+		ID *id, *id_next;
+		for (id  = lbarray[lba_idx]->first; id; id = id_next) {
+			id_next = id->next;
+			/* XXX That check may be a bit to generic/permissive? */
+			if (id->lib && (id->flag & LIB_TAG_PRE_EXISTING) && id->us == 0) {
+				BKE_libblock_free(bmain, id);
+			}
+		}
+	}
+
+	/* Get rid of no more used libraries... */
+	BKE_main_id_tag_idcode(bmain, ID_LI, LIB_TAG_DOIT, true);
+	lba_idx = set_listbasepointers(bmain, lbarray);
+	while (lba_idx--) {
+		ID *id;
+		for (id = lbarray[lba_idx]->first; id; id = id->next) {
+			if (id->lib) {
+				id->lib->id.tag &= ~LIB_TAG_DOIT;
+			}
+		}
+	}
+	Library *lib, *lib_next;
+	for (lib = which_libbase(bmain, ID_LI)->first; lib; lib = lib_next) {
+		lib_next = lib->id.next;
+		if (lib->id.tag & LIB_TAG_DOIT) {
+			id_us_clear_real(&lib->id);
+			if (lib->id.us == 0) {
+				BKE_libblock_free(bmain, (ID *)lib);
+			}
+		}
+	}
+}
+
+static int wm_lib_relocate_exec_do(bContext *C, wmOperator *op, bool do_reload)
+{
+	Library *lib;
+	char lib_name[MAX_NAME];
+
+	RNA_string_get(op->ptr, "library", lib_name);
+	lib = (Library *)BKE_libblock_find_name_ex(CTX_data_main(C), ID_LI, lib_name);
+
+	if (lib) {
+		Main *bmain = CTX_data_main(C);
+		Scene *scene = CTX_data_scene(C);
+		PropertyRNA *prop;
+		WMLinkAppendData *lapp_data;
+
+		ListBase *lbarray[MAX_LIBARRAY];
+		int lba_idx;
+
+		char path[FILE_MAX], root[FILE_MAXDIR], libname[FILE_MAX], relname[FILE_MAX];
+		short flag = 0;
+
+		if (RNA_boolean_get(op->ptr, "relative_path")) {
+			flag |= FILE_RELPATH;
+		}
+
+		if (lib->parent && !do_reload) {
+			BKE_reportf(op->reports, RPT_ERROR_INVALID_INPUT,
+			            "Cannot relocate indirectly linked library '%s'", lib->filepath);
+			return OPERATOR_CANCELLED;
+		}
+
+		RNA_string_get(op->ptr, "directory", root);
+		RNA_string_get(op->ptr, "filename", libname);
+
+		if (!BLO_has_bfile_extension(libname)) {
+			BKE_report(op->reports, RPT_ERROR, "Not a library");
+			return OPERATOR_CANCELLED;
+		}
+
+		BLI_join_dirfile(path, sizeof(path), root, libname);
+
+		if (!BLI_exists(path)) {
+			BKE_reportf(op->reports, RPT_ERROR_INVALID_INPUT,
+			            "Trying to reload or relocate library '%s' to invalid path '%s'", lib->id.name, path);
+			return OPERATOR_CANCELLED;
+		}
+
+		if (BLI_path_cmp(lib->filepath, path) == 0) {
+#ifdef PRINT_DEBUG
+			printf("We are supposed to reload '%s' lib (%d)...\n", lib->filepath, lib->id.us);
+#endif
+
+			do_reload = true;
+
+			lapp_data = wm_link_append_data_new(flag);
+			wm_link_append_data_library_add(lapp_data, path);
+		}
+		else {
+			int totfiles = 0;
+
+#ifdef PRINT_DEBUG
+			printf("We are supposed to relocate '%s' lib to new '%s' one...\n", lib->filepath, libname);
+#endif
+
+			/* Check if something is indicated for relocate. */
+			prop = RNA_struct_find_property(op->ptr, "files");
+			if (prop) {
+				totfiles = RNA_property_collection_length(op->ptr, prop);
+				if (totfiles == 0) {
+					if (!libname[0]) {
+						BKE_report(op->reports, RPT_ERROR, "Nothing indicated");
+						return OPERATOR_CANCELLED;
+					}
+				}
+			}
+
+			lapp_data = wm_link_append_data_new(flag);
+
+			if (totfiles) {
+				RNA_BEGIN (op->ptr, itemptr, "files")
+				{
+					RNA_string_get(&itemptr, "name", relname);
+
+					BLI_join_dirfile(path, sizeof(path), root, relname);
+
+					if (BLI_path_cmp(path, lib->filepath) == 0 || !BLO_has_bfile_extension(relname)) {
+						continue;
+					}
+
+#ifdef PRINT_DEBUG
+					printf("\t candidate new lib to reload datablocks from: %s\n", path);
+#endif
+					wm_link_append_data_library_add(lapp_data, path);
+				}
+				RNA_END;
+			}
+			else {
+#ifdef PRINT_DEBUG
+				printf("\t candidate new lib to reload datablocks from: %s\n", path);
+#endif
+				wm_link_append_data_library_add(lapp_data, path);
+			}
+		}
+
+		lba_idx = set_listbasepointers(bmain, lbarray);
+		while (lba_idx--) {
+			ID *id = lbarray[lba_idx]->first;
+			const short idcode = id ? GS(id->name) : 0;
+
+			if (!id || !BKE_idcode_is_linkable(idcode)) {
+				/* No need to reload non-linkable datatypes, those will get relinked with their 'users ID'. */
+				continue;
+			}
+
+			for (; id; id = id->next) {
+				if (id->lib == lib) {
+					WMLinkAppendDataItem *item;
+
+					/* We remove it from current Main, and add it to items to link... */
+					/* Note that non-linkable IDs (like e.g. shapekeys) are also explicitely linked here... */
+					BLI_remlink(lbarray[lba_idx], id);
+					item = wm_link_append_data_item_add(lapp_data, id->name + 2, idcode, id);
+					BLI_BITMAP_SET_ALL(item->libraries, true, lapp_data->num_libraries);
+
+#ifdef PRINT_DEBUG
+					printf("\tdatablock to seek for: %s\n", id->name);
+#endif
+				}
+			}
+		}
+
+		lib_relocate_do(bmain, lapp_data, op->reports, do_reload);
+
+		wm_link_append_data_free(lapp_data);
+
+		BKE_main_lib_objects_recalc_all(bmain);
+		IMB_colormanagement_check_file_config(bmain);
+
+		/* important we unset, otherwise these object wont
+		 * link into other scenes from this blend file */
+		BKE_main_id_tag_all(bmain, LIB_TAG_PRE_EXISTING, false);
+
+		/* recreate dependency graph to include new objects */
+		DAG_scene_relations_rebuild(bmain, scene);
+
+		/* free gpu materials, some materials depend on existing objects, such as lamps so freeing correctly refreshes */
+		GPU_materials_free();
+
+		/* XXX TODO: align G.lib with other directory storage (like last opened image etc...) */
+		BLI_strncpy(G.lib, root, FILE_MAX);
+
+		WM_event_add_notifier(C, NC_WINDOW, NULL);
+
+		return OPERATOR_FINISHED;
+	}
+
+	return OPERATOR_CANCELLED;
+}
+
+static int wm_lib_relocate_exec(bContext *C, wmOperator *op)
+{
+	return wm_lib_relocate_exec_do(C, op, false);
+}
+
+void WM_OT_lib_relocate(wmOperatorType *ot)
+{
+	PropertyRNA *prop;
+
+	ot->name = "Relocate Library";
+	ot->idname = "WM_OT_lib_relocate";
+	ot->description = "Relocate the given library to one or several others";
+
+	ot->invoke = wm_lib_relocate_invoke;
+	ot->exec = wm_lib_relocate_exec;
+
+	ot->flag |= OPTYPE_UNDO;
+
+	prop = RNA_def_string(ot->srna, "library", NULL, MAX_NAME, "Library", "Library to relocate");
+	RNA_def_property_flag(prop, PROP_HIDDEN);
+
+	WM_operator_properties_filesel(
+	            ot, FILE_TYPE_FOLDER | FILE_TYPE_BLENDER, FILE_BLENDER, FILE_OPENFILE,
+	            WM_FILESEL_FILEPATH | WM_FILESEL_DIRECTORY | WM_FILESEL_FILENAME | WM_FILESEL_FILES | WM_FILESEL_RELPATH,
+	            FILE_DEFAULTDISPLAY, FILE_SORT_ALPHA);
+}
+
+static int wm_lib_reload_exec(bContext *C, wmOperator *op)
+{
+	return wm_lib_relocate_exec_do(C, op, true);
+}
+
+void WM_OT_lib_reload(wmOperatorType *ot)
+{
+	PropertyRNA *prop;
+
+	ot->name = "Reload Library";
+	ot->idname = "WM_OT_lib_reload";
+	ot->description = "Reload the given library";
+
+	ot->exec = wm_lib_reload_exec;
+
+	ot->flag |= OPTYPE_UNDO;
+
+	prop = RNA_def_string(ot->srna, "library", NULL, MAX_NAME, "Library", "Library to reload");
+	RNA_def_property_flag(prop, PROP_HIDDEN);
+
+	WM_operator_properties_filesel(
+	            ot, FILE_TYPE_FOLDER | FILE_TYPE_BLENDER, FILE_BLENDER, FILE_OPENFILE,
+	            WM_FILESEL_FILEPATH | WM_FILESEL_DIRECTORY | WM_FILESEL_FILENAME | WM_FILESEL_RELPATH,
+	            FILE_DEFAULTDISPLAY, FILE_SORT_ALPHA);
+}
+
+/** \} */