Final 'FileBrowser First Stage' merge.

It basically rewrites most of filelist.c, with some more limited changes in other areas of filebrowser.

From user perspective, it:
* Removes some info in 'long' drawing mode (owner, permissions) - OS-specific data that do not really matter in Blender!
* Makes short/long display 'fixed' size (among four choices, like thumbnails mode).
* Allows to list several layers of dirtree at once, in a flat way (inside .blend files and/or real directories).
* Consequently, adds datablocks types filtering.
* Uses way less RAM when listing big directories, especially in thumbnail mode (we are talking of several hundred of MiB spared).
* Generates thumbnails way faster.

From code perspective, it:
* Is ready for asset engine needs (on data structure level in filebrowser's listing).
* Simplifies and makes 'generic' file listing much lighter.
* Separates file listing in three different aspects:
** 'generic' filelisting (in BLI), which becomes a shallow wrapper around stat struct.
** 'filebrowser drawing' filelisting, which only contains current visible subset of the whole list (sliding window), with extra drawing data (strings for size, date/time, preview, etc.).
** 'asset-ready' filelisting, which is used for operations common to 'basic' filehandling and future asset-related one.
* Uses uuid's to handle file selection/state in the browser, instead of using flags in filelisting items.
* Uses much lighter BLI_task handling for previews, instead of heavy 'job' system (using the new 'notifier' timer to handle UI refresh, in similar way to jobs).
* Moves .blend datablocks preview handling to IMB_thumbnail (necessary to avoid storing all datablock previews at once, and gives better consistency and performances too).

Revision: https://developer.blender.org/D1316

Thanks to Campbell & Sergey for the reviews. :)

3 files changed, 131 insertions, 64 deletions

diff --git a/source/blender/windowmanager/WM_api.h b/source/blender/windowmanager/WM_api.h
index 274265773a4..745f211a0da 100644
--- a/source/blender/windowmanager/WM_api.h
+++ b/source/blender/windowmanager/WM_api.h
@@ -417,7 +417,7 @@ enum {
 	WM_JOB_TYPE_OBJECT_SIM_FLUID,
 	WM_JOB_TYPE_OBJECT_BAKE_TEXTURE,
 	WM_JOB_TYPE_OBJECT_BAKE,
-	WM_JOB_TYPE_FILESEL_THUMBNAIL,
+	WM_JOB_TYPE_FILESEL_READDIR,
 	WM_JOB_TYPE_CLIP_BUILD_PROXY,
 	WM_JOB_TYPE_CLIP_TRACK_MARKERS,
 	WM_JOB_TYPE_CLIP_SOLVE_CAMERA,
diff --git a/source/blender/windowmanager/WM_types.h b/source/blender/windowmanager/WM_types.h
index 109ccc27d79..3c0e99bddd0 100644
--- a/source/blender/windowmanager/WM_types.h
+++ b/source/blender/windowmanager/WM_types.h
@@ -361,6 +361,7 @@ typedef struct wmNotifier {
 #define ND_SPACE_NODE_VIEW		(17<<16)
 #define ND_SPACE_CHANGED		(18<<16) /*sent to a new editor type after it's replaced an old one*/
 #define ND_SPACE_CLIP			(19<<16)
+#define ND_SPACE_FILE_PREVIEW   (20<<16)
 
 /* subtype, 256 entries too */
 #define NOTE_SUBTYPE		0x0000FF00
diff --git a/source/blender/windowmanager/intern/wm_operators.c b/source/blender/windowmanager/intern/wm_operators.c
index 15aab9faa00..516164b540a 100644
--- a/source/blender/windowmanager/intern/wm_operators.c
+++ b/source/blender/windowmanager/intern/wm_operators.c
@@ -1262,6 +1262,8 @@ void WM_operator_properties_filesel(wmOperatorType *ot, int filter, short type,
 	RNA_def_property_flag(prop, PROP_HIDDEN | PROP_SKIP_SAVE);
 	prop = RNA_def_boolean(ot->srna, "filter_folder", (filter & FILE_TYPE_FOLDER) != 0, "Filter folders", "");
 	RNA_def_property_flag(prop, PROP_HIDDEN | PROP_SKIP_SAVE);
+	prop = RNA_def_boolean(ot->srna, "filter_blenlib", (filter & FILE_TYPE_BLENDERLIB) != 0, "Filter Blender IDs", "");
+	RNA_def_property_flag(prop, PROP_HIDDEN | PROP_SKIP_SAVE);
 
 	prop = RNA_def_int(ot->srna, "filemode", type, FILE_LOADLIB, FILE_SPECIAL,
 	                   "File Browser Mode", "The setting for the file browser mode to load a .blend file, a library or a special file",
@@ -2594,28 +2596,121 @@ static short wm_link_append_flag(wmOperator *op)
 	return flag;
 }
 
-static int wm_link_append_exec(bContext *C, wmOperator *op)
+/* Helper.
+ *     if `name` is non-NULL, we assume a single-item link/append.
+ *     else if `*todo_libraries` is NULL we assume first-run.
+ */
+static void wm_link_append_do_libgroup(
+        bContext *C, wmOperator *op, const char *root, const char *libname, char *group, char *name,
+        const short flag, GSet **todo_libraries)
 {
 	Main *bmain = CTX_data_main(C);
-	Scene *scene = CTX_data_scene(C);
-	Main *mainl = NULL;
+	Main *mainl;
 	BlendHandle *bh;
 	Library *lib;
+
+	char path[FILE_MAX_LIBEXTRA], relname[FILE_MAX];
+	int idcode;
+	const bool is_first_run = (*todo_libraries == NULL);
+
+	BLI_assert(group);
+	idcode = BKE_idcode_from_name(group);
+
+	bh = BLO_blendhandle_from_file(libname, op->reports);
+
+	if (bh == NULL) {
+		/* unlikely since we just browsed it, but possible
+		 * error reports will have been made by BLO_blendhandle_from_file() */
+		return;
+	}
+
+	/* here appending/linking starts */
+	mainl = BLO_library_append_begin(bmain, &bh, libname);
+	lib = mainl->curlib;
+	BLI_assert(lib);
+
+	if (mainl->versionfile < 250) {
+		BKE_reportf(op->reports, RPT_WARNING,
+		            "Linking or appending from a very old .blend file format (%d.%d), no animation conversion will "
+		            "be done! You may want to re-save your lib file with current Blender",
+		            mainl->versionfile, mainl->subversionfile);
+	}
+
+	if (name) {
+		BLO_library_append_named_part_ex(C, mainl, &bh, name, idcode, flag);
+	}
+	else {
+		if (is_first_run) {
+			*todo_libraries = BLI_gset_new(BLI_ghashutil_strhash_p, BLI_ghashutil_strcmp, __func__);
+		}
+
+		RNA_BEGIN (op->ptr, itemptr, "files")
+		{
+			char curr_libname[FILE_MAX];
+			int curr_idcode;
+
+			RNA_string_get(&itemptr, "name", relname);
+
+			BLI_join_dirfile(path, sizeof(path), root, relname);
+
+			if (BLO_library_path_explode(path, curr_libname, &group, &name)) {
+				if (!group || !name) {
+					continue;
+				}
+
+				curr_idcode = BKE_idcode_from_name(group);
+
+				if ((idcode == curr_idcode) && (BLI_path_cmp(curr_libname, libname) == 0)) {
+					BLO_library_append_named_part_ex(C, mainl, &bh, name, idcode, flag);
+				}
+				else if (is_first_run) {
+					BLI_join_dirfile(path, sizeof(path), curr_libname, group);
+					if (!BLI_gset_haskey(*todo_libraries, path)) {
+						BLI_gset_insert(*todo_libraries, BLI_strdup(path));
+					}
+				}
+			}
+		}
+		RNA_END;
+	}
+	BLO_library_append_end(C, mainl, &bh, idcode, flag);
+
+	BLO_blendhandle_close(bh);
+
+	/* mark all library linked objects to be updated */
+	BKE_main_lib_objects_recalc_all(bmain);
+	IMB_colormanagement_check_file_config(bmain);
+
+	/* append, rather than linking */
+	if ((flag & FILE_LINK) == 0) {
+		BLI_assert(BLI_findindex(&bmain->library, lib) != -1);
+		BKE_library_make_local(bmain, lib, true);
+	}
+}
+
+static int wm_link_append_exec(bContext *C, wmOperator *op)
+{
+	Main *bmain = CTX_data_main(C);
+	Scene *scene = CTX_data_scene(C);
 	PropertyRNA *prop;
-	char name[FILE_MAX], dir[FILE_MAX], libname[FILE_MAX];
-	char *group;
-	int idcode, totfiles = 0;
+	char path[FILE_MAX_LIBEXTRA], root[FILE_MAXDIR], libname[FILE_MAX], relname[FILE_MAX];
+	char *group, *name;
+	int totfiles = 0;
 	short flag;
 
-	RNA_string_get(op->ptr, "filename", name);
-	RNA_string_get(op->ptr, "directory", dir);
+	GSet *todo_libraries = NULL;
+
+	RNA_string_get(op->ptr, "filename", relname);
+	RNA_string_get(op->ptr, "directory", root);
+
+	BLI_join_dirfile(path, sizeof(path), root, relname);
 
 	/* test if we have a valid data */
-	if (BLO_library_path_explode(dir, libname, &group, NULL) == 0) {
+	if (!BLO_library_path_explode(path, libname, &group, &name)) {
 		BKE_report(op->reports, RPT_ERROR, "Not a library");
 		return OPERATOR_CANCELLED;
 	}
-	else if ((group == NULL) || (group[0] == '\0')) {
+	else if (!group) {
 		BKE_report(op->reports, RPT_ERROR, "Nothing indicated");
 		return OPERATOR_CANCELLED;
 	}
@@ -2629,35 +2724,17 @@ static int wm_link_append_exec(bContext *C, wmOperator *op)
 	if (prop) {
 		totfiles = RNA_property_collection_length(op->ptr, prop);
 		if (totfiles == 0) {
-			if (name[0] == '\0') {
+			if (!name) {
 				BKE_report(op->reports, RPT_ERROR, "Nothing indicated");
 				return OPERATOR_CANCELLED;
 			}
 		}
 	}
-	else if (name[0] == '\0') {
+	else if (!name) {
 		BKE_report(op->reports, RPT_ERROR, "Nothing indicated");
 		return OPERATOR_CANCELLED;
 	}
 
-	bh = BLO_blendhandle_from_file(libname, op->reports);
-
-	if (bh == NULL) {
-		/* unlikely since we just browsed it, but possible
-		 * error reports will have been made by BLO_blendhandle_from_file() */
-		return OPERATOR_CANCELLED;
-	}
-
-
-	/* from here down, no error returns */
-
-	idcode = BKE_idcode_from_name(group);
-
-	/* now we have or selected, or an indicated file */
-	if (RNA_boolean_get(op->ptr, "autoselect"))
-		BKE_scene_base_deselect_all(scene);
-
-	
 	flag = wm_link_append_flag(op);
 
 	/* sanity checks for flag */
@@ -2667,46 +2744,36 @@ static int wm_link_append_exec(bContext *C, wmOperator *op)
 		flag &= ~FILE_GROUP_INSTANCE;
 	}
 
+	/* from here down, no error returns */
 
+	/* now we have or selected, or an indicated file */
+	if (RNA_boolean_get(op->ptr, "autoselect"))
+		BKE_scene_base_deselect_all(scene);
+	
 	/* tag everything, all untagged data can be made local
 	 * its also generally useful to know what is new
 	 *
 	 * take extra care BKE_main_id_flag_all(LIB_LINK_TAG, false) is called after! */
 	BKE_main_id_flag_all(bmain, LIB_PRE_EXISTING, 1);
 
-	/* here appending/linking starts */
-	mainl = BLO_library_append_begin(bmain, &bh, libname);
-	lib = mainl->curlib;
-	BLI_assert(lib);
-
-	if (mainl->versionfile < 250) {
-		BKE_reportf(op->reports, RPT_WARNING,
-		            "Linking or appending from a very old .blend file format (%d.%d), no animation conversion will "
-		            "be done! You may want to re-save your lib file with current Blender",
-		            mainl->versionfile, mainl->subversionfile);
+	if (totfiles != 0) {
+		name = NULL;
 	}
 
-	if (totfiles == 0) {
-		BLO_library_append_named_part_ex(C, mainl, &bh, name, idcode, flag);
-	}
-	else {
-		RNA_BEGIN (op->ptr, itemptr, "files")
-		{
-			RNA_string_get(&itemptr, "name", name);
-			BLO_library_append_named_part_ex(C, mainl, &bh, name, idcode, flag);
+	wm_link_append_do_libgroup(C, op, root, libname, group, name, flag, &todo_libraries);
+
+	if (todo_libraries) {
+		GSetIterator libs_it;
+
+		GSET_ITER(libs_it, todo_libraries) {
+			char *libpath = (char *)BLI_gsetIterator_getKey(&libs_it);
+
+			BLO_library_path_explode(libpath, libname, &group, NULL);
+
+			wm_link_append_do_libgroup(C, op, root, libname, group, NULL, flag, &todo_libraries);
 		}
-		RNA_END;
-	}
-	BLO_library_append_end(C, mainl, &bh, idcode, flag);
-	
-	/* mark all library linked objects to be updated */
-	BKE_main_lib_objects_recalc_all(bmain);
-	IMB_colormanagement_check_file_config(bmain);
 
-	/* append, rather than linking */
-	if ((flag & FILE_LINK) == 0) {
-		BLI_assert(BLI_findindex(&bmain->library, lib) != -1);
-		BKE_library_make_local(bmain, lib, true);
+		BLI_gset_free(todo_libraries, MEM_freeN);
 	}
 
 	/* important we unset, otherwise these object wont
@@ -2718,10 +2785,9 @@ static int wm_link_append_exec(bContext *C, wmOperator *op)
 	
 	/* free gpu materials, some materials depend on existing objects, such as lamps so freeing correctly refreshes */
 	GPU_materials_free();
-	BLO_blendhandle_close(bh);
 
 	/* XXX TODO: align G.lib with other directory storage (like last opened image etc...) */
-	BLI_strncpy(G.lib, dir, FILE_MAX);
+	BLI_strncpy(G.lib, root, FILE_MAX);
 
 	WM_event_add_notifier(C, NC_WINDOW, NULL);
 
@@ -2761,7 +2827,7 @@ static void WM_OT_link(wmOperatorType *ot)
 	ot->flag |= OPTYPE_UNDO;
 
 	WM_operator_properties_filesel(
-	        ot, FILE_TYPE_FOLDER | FILE_TYPE_BLENDER, FILE_LOADLIB, FILE_OPENFILE,
+	        ot, FILE_TYPE_FOLDER | FILE_TYPE_BLENDER | FILE_TYPE_BLENDERLIB, FILE_LOADLIB, FILE_OPENFILE,
 	        WM_FILESEL_FILEPATH | WM_FILESEL_DIRECTORY | WM_FILESEL_FILENAME | WM_FILESEL_RELPATH | WM_FILESEL_FILES,
 	        FILE_DEFAULTDISPLAY);
 	
@@ -2781,7 +2847,7 @@ static void WM_OT_append(wmOperatorType *ot)
 	ot->flag |= OPTYPE_UNDO;
 
 	WM_operator_properties_filesel(
-		ot, FILE_TYPE_FOLDER | FILE_TYPE_BLENDER, FILE_LOADLIB, FILE_OPENFILE,
+		ot, FILE_TYPE_FOLDER | FILE_TYPE_BLENDER | FILE_TYPE_BLENDERLIB, FILE_LOADLIB, FILE_OPENFILE,
 		WM_FILESEL_FILEPATH | WM_FILESEL_DIRECTORY | WM_FILESEL_FILENAME | WM_FILESEL_FILES,
 		FILE_DEFAULTDISPLAY);