Adds support for utf paths on Windows.

Not all file formats/calls are supported yet. It will be expended.

Please from now on use BLI_fopen, BLI_* for file manipulations.
For non-windows systems BLI_fopen just calls fopen.
For Windows, the utf-8 string is translated to utf-16 string in order to call UTF version of the function.

9 files changed, 217 insertions, 63 deletions

diff --git a/source/blender/blenlib/BLI_fileops.h b/source/blender/blenlib/BLI_fileops.h
index 8fe8456e845..fa1a61998d7 100644
--- a/source/blender/blenlib/BLI_fileops.h
+++ b/source/blender/blenlib/BLI_fileops.h
@@ -33,6 +33,9 @@
 #ifndef __BLI_FILEOPS_H__
 #define __BLI_FILEOPS_H__
 
+#include <stdio.h>
+
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -65,6 +68,10 @@ unsigned int BLI_dir_contents(const char *dir, struct direntry **filelist);
 
 /* Files */
 
+FILE * BLI_fopen(const char * filename, const char * mode);
+void* BLI_gzopen(const char * filename, const char * mode);
+int    BLI_open(const char *filename, int oflag, int pmode);
+
 int    BLI_file_is_writable(const char *file);
 int    BLI_file_touch(const char *file);
 
diff --git a/source/blender/blenlib/BLI_winstuff.h b/source/blender/blenlib/BLI_winstuff.h
index d939882b105..c393b219785 100644
--- a/source/blender/blenlib/BLI_winstuff.h
+++ b/source/blender/blenlib/BLI_winstuff.h
@@ -124,7 +124,7 @@ struct dirent {
 
 typedef struct _DIR {
 	HANDLE handle;
-	WIN32_FIND_DATA data;
+	WIN32_FIND_DATAW data;
 	char path[MAX_PATH];
 	long dd_loc;
 	long dd_size;
diff --git a/source/blender/blenlib/CMakeLists.txt b/source/blender/blenlib/CMakeLists.txt
index ddcc344e939..c06a1240729 100644
--- a/source/blender/blenlib/CMakeLists.txt
+++ b/source/blender/blenlib/CMakeLists.txt
@@ -160,4 +160,10 @@ if(WITH_XDG_USER_DIRS)
 	add_definitions(-DWITH_XDG_USER_DIRS)
 endif()
 
+if(WIN32)
+	list(APPEND INC
+		../../../intern/utfconv
+	)
+endif()
+
 blender_add_lib(bf_blenlib "${SRC}" "${INC}" "${INC_SYS}")
diff --git a/source/blender/blenlib/SConscript b/source/blender/blenlib/SConscript
index dda8303eea7..ea51e5b6ab5 100644
--- a/source/blender/blenlib/SConscript
+++ b/source/blender/blenlib/SConscript
@@ -18,6 +18,7 @@ if env['WITH_BF_BINRELOC']:
 
 if env['OURPLATFORM'] in ('win32-vc', 'win32-mingw', 'linuxcross', 'win64-vc'):
     incs += ' ' + env['BF_PTHREADS_INC']
+    incs += ' ../../../intern/utfconv'
 
 if env['OURPLATFORM'] == 'linuxcross':
     if env['WITH_BF_OPENMP']:
diff --git a/source/blender/blenlib/intern/dynlib.c b/source/blender/blenlib/intern/dynlib.c
index 0bfa5e79eb8..c23feb975df 100644
--- a/source/blender/blenlib/intern/dynlib.c
+++ b/source/blender/blenlib/intern/dynlib.c
@@ -44,11 +44,17 @@ struct DynamicLibrary {
 #ifdef WIN32
 
 #include <windows.h>
+#include "utf_winfunc.h"
+#include "utfconv.h"
 
 DynamicLibrary *BLI_dynlib_open(char *name)
 {
 	DynamicLibrary *lib;
-	void *handle= LoadLibrary(name);
+	void *handle;
+
+	UTF16_ENCODE(name);
+	handle= LoadLibraryW(name_16);
+	UTF16_UN_ENCODE(name);
 
 	if(!handle)
 		return NULL;
diff --git a/source/blender/blenlib/intern/fileops.c b/source/blender/blenlib/intern/fileops.c
index ab7beeeb8f1..9527ec9466b 100644
--- a/source/blender/blenlib/intern/fileops.c
+++ b/source/blender/blenlib/intern/fileops.c
@@ -45,6 +45,8 @@
 #include <io.h>
 #include "BLI_winstuff.h"
 #include "BLI_callbacks.h"
+#include "utf_winfunc.h"
+#include "utfconv.h"
 #else
 #include <unistd.h> // for read close
 #include <sys/param.h>
@@ -77,11 +79,10 @@ int BLI_file_gzip(const char *from, const char *to)
 
 	/* level 1 is very close to 3 (the default) in terms of file size,
 	 * but about twice as fast, best use for speedy saving - campbell */
-	gzfile = gzopen(to, "wb1");
+	gzfile = BLI_gzopen(to, "wb1");
 	if(gzfile == NULL)
 		return -1;
-	
-	file = open(from, O_BINARY|O_RDONLY);
+	file = BLI_open(from, O_BINARY|O_RDONLY,0);
 	if(file < 0)
 		return -2;
 
@@ -121,8 +122,7 @@ char *BLI_file_ungzip_to_mem(const char *from_file, int *size_r)
 
 	size= 0;
 
-	gzfile = gzopen( from_file, "rb" );
-
+	gzfile = BLI_gzopen( from_file, "rb" );
 	for(;;) {
 		if(mem==NULL) {
 			mem= MEM_callocN(chunk_size, "BLI_ungzip_to_mem");
@@ -158,12 +158,12 @@ int BLI_file_is_writable(const char *filename)
 	int file;
 	
 	/* first try to open without creating */
-	file = open(filename, O_BINARY | O_RDWR, 0666);
+	file = BLI_open(filename, O_BINARY | O_RDWR, 0666);
 	
 	if (file < 0) {
 		/* now try to open and create. a test without actually
 		 * creating a file would be nice, but how? */
-		file = open(filename, O_BINARY | O_RDWR | O_CREAT, 0666);
+		file = BLI_open(filename, O_BINARY | O_RDWR | O_CREAT, 0666);
 		
 		if(file < 0) {
 			return 0;
@@ -183,13 +183,13 @@ int BLI_file_is_writable(const char *filename)
 
 int BLI_file_touch(const char *file)
 {
-	FILE *f = fopen(file,"r+b");
+	FILE *f = BLI_fopen(file,"r+b");
 	if (f != NULL) {
 		char c = getc(f);
 		rewind(f);
 		putc(c,f);
 	} else {
-		f = fopen(file,"wb");
+		f = BLI_fopen(file,"wb");
 	}
 	if (f) {
 		fclose(f);
@@ -202,21 +202,63 @@ int BLI_file_touch(const char *file)
 
 static char str[MAXPATHLEN+12];
 
+FILE * BLI_fopen(const char * filename, const char * mode)
+{
+	return ufopen(filename, mode);
+}
+
+gzFile BLI_gzopen(const char * filename, const char * mode)
+{
+	gzFile gzfile;
+	int fi;
+
+	if(!filename || !mode) {return 0;}
+	else
+
+	{
+			
+		wchar_t short_name_16 [256];
+		char short_name [256];
+		int i=0;
+		UTF16_ENCODE(filename);
+
+		GetShortPathNameW(filename_16,short_name_16,256);
+
+		for(i=0;i<256;i++) {short_name[i]=short_name_16[i];};
+
+
+		gzfile = gzopen(short_name,mode);
+
+		UTF16_UN_ENCODE(filename);
+
+	}
+	return gzfile;
+}
+
+int   BLI_open(const char *filename, int oflag, int pmode)
+{
+	return uopen(filename, oflag, pmode);
+}
+
 int BLI_delete(const char *file, int dir, int recursive)
 {
 	int err;
+	
+	UTF16_ENCODE(file)
 
 	if (recursive) {
 		callLocalErrorCallBack("Recursive delete is unsupported on Windows");
 		err= 1;
 	} else if (dir) {
-		err= !RemoveDirectory(file);
+		err= !RemoveDirectoryW(file_16);
 		if (err) printf ("Unable to remove directory");
 	} else {
-		err= !DeleteFile(file);
+		err= !DeleteFileW(file_16);
 		if (err) callLocalErrorCallBack("Unable to delete file");
 	}
 
+	UTF16_UN_ENCODE(file)
+
 	return err;
 }
 
@@ -235,8 +277,13 @@ int BLI_move(const char *file, const char *to)
 			strcat(str, BLI_last_slash(file) + 1);
 		}
 	}
+	
+	UTF16_ENCODE(file)
+	UTF16_ENCODE(str)
+	err= !MoveFileW(file_16, str_16);
+	UTF16_UN_ENCODE(str)
+	UTF16_UN_ENCODE(file)
 
-	err= !MoveFile(file, str);
 	if (err) {
 		callLocalErrorCallBack("Unable to move file");
 		printf(" Move from '%s' to '%s' failed\n", file, str);
@@ -262,8 +309,12 @@ int BLI_copy(const char *file, const char *to)
 		}
 	}
 
-	err= !CopyFile(file,str,FALSE);
-	
+	UTF16_ENCODE(file)
+	UTF16_ENCODE(str)
+	err= !CopyFileW(file_16,str_16,FALSE);
+	UTF16_UN_ENCODE(str)
+	UTF16_UN_ENCODE(file)
+
 	if (err) {
 		callLocalErrorCallBack("Unable to copy file!");
 		printf(" Copy from '%s' to '%s' failed\n", file, str);
@@ -292,7 +343,7 @@ void BLI_dir_create_recursive(const char *dirname)
 
 	BLI_strncpy(tmp, dirname, sizeof(tmp));
 	lslash= BLI_last_slash(tmp);
-
+	
 	if (lslash == tmp + strlen(tmp) - 1) {
 		*lslash = 0;
 	}
@@ -307,8 +358,8 @@ void BLI_dir_create_recursive(const char *dirname)
 	}
 	
 	if(dirname[0]) /* patch, this recursive loop tries to create a nameless directory */
-		if (!CreateDirectory(dirname, NULL))
-			callLocalErrorCallBack("Unable to create directory\n");
+		if (umkdir(dirname)==-1)
+			printf("Unable to create directory %s\n",dirname);
 }
 
 int BLI_rename(const char *from, const char *to)
@@ -318,8 +369,8 @@ int BLI_rename(const char *from, const char *to)
 	/* make sure the filenames are different (case insensitive) before removing */
 	if (BLI_exists(to) && BLI_strcasecmp(from, to))
 		if(BLI_delete(to, 0, 0)) return 1;
-
-	return rename(from, to);
+	
+	return urename(from, to);
 }
 
 #else /* The UNIX world */
@@ -499,6 +550,21 @@ static int delete_single_file(const char *from, const char *UNUSED(to))
 	return recursiveOp_Callback_OK;
 }
 
+FILE * BLI_fopen(const char * filename, const char * mode)
+{
+	return fopen(filename, mode);
+}
+
+gzFile BLI_gzopen(const char * filename, const char * mode)
+{
+	return gzopen(filename, mode);
+}
+
+int BLI_open(const char *filename, int oflag, int pmode)
+{
+	return open(filename, oflag, pmode);
+}
+
 int BLI_delete(const char *file, int dir, int recursive) 
 {
 	if(strchr(file, '"')) {
diff --git a/source/blender/blenlib/intern/path_util.c b/source/blender/blenlib/intern/path_util.c
index 9c80fa9ca15..ccf85934788 100644
--- a/source/blender/blenlib/intern/path_util.c
+++ b/source/blender/blenlib/intern/path_util.c
@@ -62,6 +62,7 @@
 #endif
 
 #ifdef WIN32
+#include "utf_winfunc.h"
 #  include <io.h>
 #  ifdef _WIN32_IE
 #    undef _WIN32_IE
@@ -825,10 +826,8 @@ const char *BLI_getDefaultDocumentFolder(void)
 		HRESULT hResult;
 
 		/* Check for %HOME% env var */
-
-		ret = getenv("HOME");
-		if(ret) {
-			if (BLI_is_dir(ret)) return ret;
+		if(uput_getenv("HOME",documentfolder,MAXPATHLEN)) {
+			if (BLI_is_dir(documentfolder)) return documentfolder;
 		}
 				
 		/* add user profile support for WIN 2K / NT.
@@ -1186,7 +1185,9 @@ void BLI_setenv(const char *env, const char*val)
 
 	/* non-free windows */
 #elif (defined(WIN32) || defined(WIN64)) /* not free windows */
-	_putenv_s(env, val);
+	uputenv(env, val);
+
+
 #else
 	/* linux/osx/bsd */
 	setenv(env, val, 1);
@@ -1774,17 +1775,23 @@ static void bli_where_am_i(char *fullname, const size_t maxlen, const char *name
 #endif
 
 #ifdef _WIN32
-	if(GetModuleFileName(0, fullname, maxlen)) {
+	wchar_t * fullname_16 = MEM_mallocN(maxlen*sizeof(wchar_t), "ProgramPath");
+	if(GetModuleFileNameW(0, fullname_16, maxlen)) {
+		conv_utf_16_to_8(fullname_16,fullname, maxlen);
 		if(!BLI_exists(fullname)) {
 			printf("path can't be found: \"%.*s\"\n", maxlen, fullname);
 			MessageBox(NULL, "path contains invalid characters or is too long (see console)", "Error", MB_OK);
 		}
+		MEM_freeN(fullname_16);
 		return;
 	}
+
+	MEM_freeN(fullname_16);
 #endif
 
 	/* unix and non linux */
 	if (name && name[0]) {
+
 		BLI_strncpy(fullname, name, maxlen);
 		if (name[0] == '.') {
 			char wdir[FILE_MAX]= "";
diff --git a/source/blender/blenlib/intern/storage.c b/source/blender/blenlib/intern/storage.c
index 29f11ee0550..363052c7255 100644
--- a/source/blender/blenlib/intern/storage.c
+++ b/source/blender/blenlib/intern/storage.c
@@ -78,6 +78,7 @@
 #include <io.h>
 #include <direct.h>
 #include "BLI_winstuff.h"
+#include "utfconv.h"
 #endif
 
 
@@ -86,10 +87,13 @@
 
 #include "DNA_listBase.h"
 
-#include "BLI_fileops.h"
 #include "BLI_listbase.h"
 #include "BLI_linklist.h"
+#include "BLI_fileops.h"
+
+#include "BLI_fileops_types.h"
 #include "BLI_string.h"
+#include "BLI_fileops.h"
 
 #include "BKE_utildefines.h"
 
@@ -210,13 +214,22 @@ static void bli_builddir(const char *dirname, const char *relname)
 		buf[rellen]='/';
 		rellen++;
 	}
-
+#ifndef WIN32
 	if (chdir(dirname) == -1) {
 		perror(dirname);
 		return;
 	}
+#else
+	UTF16_ENCODE(dirname)
+	if(!SetCurrentDirectoryW(dirname_16)){
+		perror(dirname);
+		free(dirname_16);
+		return;
+	}
+	UTF16_UN_ENCODE(dirname)
 
-	if ( (dir = (DIR *)opendir(".")) ) {
+#endif
+	if ( (dir = (DIR *)opendir(".")) ){
 		while ((fname = (struct dirent*) readdir(dir)) != NULL) {
 			dlink = (struct dirlink *)malloc(sizeof(struct dirlink));
 			if (dlink) {
@@ -250,11 +263,16 @@ static void bli_builddir(const char *dirname, const char *relname)
 					files[actnum].relname = dlink->name;
 					files[actnum].path = BLI_strdupcat(dirname, dlink->name);
 // use 64 bit file size, only needed for WIN32 and WIN64. 
-// Excluding other than current MSVC compiler until able to test.
+// Excluding other than current MSVC compiler until able to test
+#ifdef WIN32
+					{wchar_t * name_16 = alloc_utf16_from_8(dlink->name,0);
 #if (defined(WIN32) || defined(WIN64)) && (_MSC_VER>=1500)
-					_stat64(dlink->name,&files[actnum].s);
+					_wstat64(name_16,&files[actnum].s);
 #elif defined(__MINGW32__)
 					_stati64(dlink->name,&files[actnum].s);
+#endif
+					free(name_16);};
+
 #else
 					stat(dlink->name,&files[actnum].s);
 #endif
@@ -423,13 +441,13 @@ size_t BLI_file_descriptor_size(int file)
 	struct stat buf;
 
 	if (file <= 0) return (-1);
-	fstat(file, &buf);
+	fstat(file, &buf);//CHANGE
 	return (buf.st_size);
 }
 
 size_t BLI_file_size(const char *path)
 {
-	int size, file = open(path, O_BINARY|O_RDONLY);
+	int size, file = BLI_open(path, O_BINARY|O_RDONLY, 0);
 	
 	if (file == -1)
 		return -1;
@@ -442,27 +460,26 @@ size_t BLI_file_size(const char *path)
 
 int BLI_exists(const char *name)
 {
-#if defined(WIN32) && !defined(__MINGW32__)
+#if defined(WIN32) 
+#ifndef __MINGW32__
 	struct _stat64i32 st;
-	/* in Windows stat doesn't recognize dir ending on a slash 
-	 * To not break code where the ending slash is expected we
-	 * don't mess with the argument name directly here - elubie */
-	char tmp[FILE_MAX];
-	int len, res;
-	BLI_strncpy(tmp, name, FILE_MAX);
-	len = strlen(tmp);
-	if (len > 3 && ( tmp[len-1]=='\\' || tmp[len-1]=='/') ) tmp[len-1] = '\0';
-	res = _stat(tmp, &st);
-	if (res == -1) return(0);
-#elif defined(__MINGW32__)
+#else
 	struct _stati64 st;
-	char tmp[FILE_MAX];
+#endif
+	/*  in Windows stat doesn't recognize dir ending on a slash 
+		To not break code where the ending slash is expected we
+		don't mess with the argument name directly here - elubie */
+	wchar_t * tmp_16 = alloc_utf16_from_8(name, 0);
 	int len, res;
-	BLI_strncpy(tmp, name, FILE_MAX);
-	len = strlen(tmp);
-	if (len > 3 && ( tmp[len-1]=='\\' || tmp[len-1]=='/') ) tmp[len-1] = '\0';
-	res = _stati64(tmp, &st);
-	if (res) return(0);
+	len = wcslen(tmp_16);
+	if (len > 3 && ( tmp_16[len-1]==L'\\' || tmp_16[len-1]==L'/') ) tmp_16[len-1] = '\0';
+#ifndef __MINGW32__
+	res = _wstat(tmp_16, &st);
+#else
+	res = _wstati64(tmp_16, &st);
+#endif
+	free(tmp_16);
+	if (res == -1) return(0);
 #else
 	struct stat st;
 	if (stat(name,&st)) return(0);	
@@ -484,7 +501,7 @@ int BLI_is_file(const char *path)
 
 LinkNode *BLI_file_read_as_lines(const char *name)
 {
-	FILE *fp= fopen(name, "r");
+	FILE *fp= BLI_fopen(name, "r");
 	LinkNode *lines= NULL;
 	char *buf;
 	size_t size;
@@ -530,11 +547,23 @@ void BLI_file_free_lines(LinkNode *lines)
 
 int BLI_file_older(const char *file1, const char *file2)
 {
+#if WIN32
+	struct _stat st1, st2;
+
+	UTF16_ENCODE(file1)
+	UTF16_ENCODE(file2)
+	
+	if(_wstat(file1_16, &st1)) return 0;
+	if(_wstat(file2_16, &st2)) return 0;
+
+	UTF16_UN_ENCODE(file2)
+	UTF16_UN_ENCODE(file1)
+#else
 	struct stat st1, st2;
 
 	if(stat(file1, &st1)) return 0;
 	if(stat(file2, &st2)) return 0;
-
+#endif
 	return (st1.st_mtime < st2.st_mtime);
 }
 
diff --git a/source/blender/blenlib/intern/winstuff.c b/source/blender/blenlib/intern/winstuff.c
index 4665b08b202..7fc2e52255c 100644
--- a/source/blender/blenlib/intern/winstuff.c
+++ b/source/blender/blenlib/intern/winstuff.c
@@ -47,13 +47,16 @@
 #define WIN32_SKIP_HKEY_PROTECTION		// need to use HKEY
 #include "BLI_winstuff.h"
 
- /* FILE_MAX */
+#include "utf_winfunc.h"
+#include "utfconv.h"
+
+ /* FILE_MAXDIR + FILE_MAXFILE */
 
 int BLI_getInstallationDir( char * str )
 {
 	char dir[FILE_MAXDIR];
 	int a;
-	
+	/*change to utf support*/
 	GetModuleFileName(NULL,str,FILE_MAX);
 	BLI_split_dir_part(str, dir, sizeof(dir)); /* shouldn't be relative */
 	a = strlen(dir);
@@ -169,7 +172,9 @@ void RegisterBlendExtension(void)
 
 DIR *opendir (const char *path)
 {
-	if (GetFileAttributes(path) & FILE_ATTRIBUTE_DIRECTORY) {
+	wchar_t * path_16 = alloc_utf16_from_8(path, 0);
+
+	if (GetFileAttributesW(path_16) & FILE_ATTRIBUTE_DIRECTORY) {
 		DIR *newd= MEM_mallocN(sizeof(DIR), "opendir");
 
 		newd->handle = INVALID_HANDLE_VALUE;
@@ -180,29 +185,56 @@ DIR *opendir (const char *path)
 		newd->direntry.d_reclen= 0;
 		newd->direntry.d_name= NULL;
 		
+		free(path_16);
 		return newd;
 	} else {
+		free(path_16);
 		return NULL;
 	}
 }
 
-struct dirent *readdir(DIR *dp)
+static char * BLI_alloc_utf_8_from_16(wchar_t * in16, size_t add)
+{
+	size_t bsize = count_utf_8_from_16(in16);
+	char * out8 = NULL;
+	if(!bsize) return NULL;
+	out8 = (char*)MEM_mallocN(sizeof(char) * (bsize + add),"UTF-8 String");
+	conv_utf_16_to_8(in16,out8, bsize);
+	return out8;
+}
+
+static wchar_t * BLI_alloc_utf16_from_8(char * in8, size_t add)
 {
+	size_t bsize = count_utf_16_from_8(in8);
+	wchar_t * out16 = NULL;
+	if(!bsize) return NULL;
+	out16 =(wchar_t*) MEM_mallocN(sizeof(wchar_t) * (bsize + add), "UTF-16 String");
+	conv_utf_8_to_16(in8,out16, bsize);
+	return out16;
+}
+
+
+
+struct dirent *readdir(DIR *dp) {
+	char * FileName;
+	size_t size;
 	if (dp->direntry.d_name) {
 		MEM_freeN(dp->direntry.d_name);
 		dp->direntry.d_name= NULL;
 	}
 		
 	if (dp->handle==INVALID_HANDLE_VALUE) {
-		dp->handle= FindFirstFile(dp->path, &(dp->data));
+		wchar_t * path_16 = alloc_utf16_from_8(dp->path, 0);
+		dp->handle= FindFirstFileW(path_16, &(dp->data));
+		free(path_16);
 		if (dp->handle==INVALID_HANDLE_VALUE)
 			return NULL;
 			
-		dp->direntry.d_name= BLI_strdup(dp->data.cFileName);
-
+		dp->direntry.d_name= BLI_alloc_utf_8_from_16(dp->data.cFileName, 0);
+		
 		return &dp->direntry;
-	} else if (FindNextFile (dp->handle, &(dp->data))) {
-		dp->direntry.d_name= BLI_strdup(dp->data.cFileName);
+	} else if (FindNextFileW (dp->handle, &(dp->data))) {
+		dp->direntry.d_name= BLI_alloc_utf_8_from_16(dp->data.cFileName,0);
 
 		return &dp->direntry;
 	} else {