Adds support for utf paths on Windows.

Not all file formats/calls are supported yet. It will be expended.

Please from now on use BLI_fopen, BLI_* for file manipulations.
For non-windows systems BLI_fopen just calls fopen.
For Windows, the utf-8 string is translated to utf-16 string in order to call UTF version of the function.

6 files changed, 570 insertions, 0 deletions

diff --git a/intern/utfconv/CMakeLists.txt b/intern/utfconv/CMakeLists.txt
new file mode 100644
index 00000000000..9dd41c28406
--- /dev/null
+++ b/intern/utfconv/CMakeLists.txt
@@ -0,0 +1,24 @@
+set(INC
+	.
+)
+
+set(INC_SYS
+	${GLEW_INCLUDE_PATH}
+)
+
+set(SRC
+	utfconv.c
+
+	utfconv.h
+	#utf_func.h
+)
+
+
+if(WIN32)
+	list(APPEND SRC
+		utf_winfunc.c
+		utf_winfunc.h
+	)
+endif()
+
+blender_add_lib(bf_intern_utfconv "${SRC}" "${INC}" "${INC_SYS}")
\ No newline at end of file
diff --git a/intern/utfconv/SConscript b/intern/utfconv/SConscript
new file mode 100644
index 00000000000..1f0fab330d3
--- /dev/null
+++ b/intern/utfconv/SConscript
@@ -0,0 +1,12 @@
+#!/usr/bin/python
+Import ('env')
+
+sources = env.Glob('*.c')
+
+incs = '.'
+defs = ''
+
+
+
+if env['OURPLATFORM'] in ('win32-vc', 'win32-mingw', 'linuxcross', 'win64-vc'):
+    env.BlenderLib ('bf_utfconv', sources, Split(incs), Split(defs), libtype=['intern','player'], priority=[0,0] )
diff --git a/intern/utfconv/utf_winfunc.c b/intern/utfconv/utf_winfunc.c
new file mode 100644
index 00000000000..09f5dfff67e
--- /dev/null
+++ b/intern/utfconv/utf_winfunc.c
@@ -0,0 +1,156 @@
+/*
+ * ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version. 
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2009 Blender Foundation.
+ * All rights reserved.
+ * 
+ * Contributor(s): Alexandr Kuznetsov, Andrea Weikert
+ *
+ * ***** END GPL LICENSE BLOCK *****
+ */
+
+#ifdef WIN32
+
+#ifndef _WIN32_IE
+#define _WIN32_IE 0x0501
+#endif
+
+#include "utf_winfunc.h"
+#include <io.h>
+#include <Windows.h>
+#include <wchar.h>
+
+
+FILE * ufopen(const char * filename, const char * mode)
+{
+	FILE * f = NULL;
+	UTF16_ENCODE(filename);
+	UTF16_ENCODE (mode);
+
+	if(filename_16 && mode_16) f = _wfopen(filename_16, mode_16);
+	
+	UTF16_UN_ENCODE(mode);
+	UTF16_UN_ENCODE(filename);
+
+	if(!f) 
+	{
+		if(f=fopen(filename,mode))
+			printf("WARNING: %s is not utf path. Please update it.\n",filename);
+	}
+
+	return f;
+}
+
+int uopen(const char *filename, int oflag, int pmode)
+{
+	int f = -1;
+	UTF16_ENCODE(filename);
+	
+	if(filename_16) f = _wopen(filename_16, oflag, pmode);
+
+	UTF16_UN_ENCODE(filename);
+
+	if(f==-1) 
+	{
+		if((f=open(filename,oflag, pmode))!=-1)
+			printf("WARNING: %s is not utf path. Please update it.\n",filename);
+	}
+
+	return f;
+}
+
+int urename(const char *oldname, const char *newname )
+{
+	int r = -1;
+	UTF16_ENCODE(oldname);
+	UTF16_ENCODE (newname);
+
+	if(oldname_16 && newname_16) r = _wrename(oldname_16, newname_16);
+	
+	UTF16_UN_ENCODE(newname);
+	UTF16_UN_ENCODE(oldname);
+	return r;
+}
+
+int umkdir(const char *pathname)
+{
+
+	BOOL r = 0;
+	UTF16_ENCODE(pathname);
+	
+	if(pathname_16) r = CreateDirectoryW(pathname_16, NULL);
+
+	UTF16_UN_ENCODE(pathname);
+
+	return r?0:-1;
+}
+
+char * u_alloc_getenv(const char *varname)
+{
+	char * r = 0;
+	wchar_t * str;
+	UTF16_ENCODE(varname);
+	if(varname_16){ str = _wgetenv(varname_16);
+	r = alloc_utf_8_from_16(str,0);}
+	UTF16_UN_ENCODE(varname);
+
+	return r;
+}
+void  u_free_getenv(char *val)
+{
+	free(val);
+}
+
+int uput_getenv(const char *varname, char * value, size_t buffsize)
+{
+	int r = 0;
+	wchar_t * str;
+	if(!buffsize) return r;
+
+	UTF16_ENCODE(varname);
+		if(varname_16) 
+			{
+				str = _wgetenv(varname_16);
+				conv_utf_16_to_8(str, value, buffsize);
+				r = 1;
+			}
+	UTF16_UN_ENCODE(varname);
+
+	if(!r) value[0] = 0;
+
+	return r;
+}
+
+
+
+int uputenv(const char *name, const char *value)
+{
+	int r = -1;
+	UTF16_ENCODE(name)
+	UTF16_ENCODE(value)
+		if(name_16 && value_16) {
+			if(SetEnvironmentVariableW(name_16,value_16)!=0) 
+				r =0;
+			else r = -1;
+	}
+	UTF16_UN_ENCODE(value)
+	UTF16_UN_ENCODE(name)
+
+	return r;
+}
+
+#endif
\ No newline at end of file
diff --git a/intern/utfconv/utf_winfunc.h b/intern/utfconv/utf_winfunc.h
new file mode 100644
index 00000000000..5e88d2de901
--- /dev/null
+++ b/intern/utfconv/utf_winfunc.h
@@ -0,0 +1,44 @@
+/*
+ * ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version. 
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2009 Blender Foundation.
+ * All rights reserved.
+ * 
+ * Contributor(s): Alexandr Kuznetsov, Andrea Weikert
+ *
+ * ***** END GPL LICENSE BLOCK *****
+ */
+
+#ifdef WIN32
+
+
+#include "utfconv.h"
+#include <stdio.h>
+
+FILE * ufopen(const char * filename, const char * mode);
+int uopen(const char *filename, int oflag, int pmode);
+int urename(const char *oldname, const char *newname );
+
+char * u_alloc_getenv(const char *varname);
+void  u_free_getenv(char *val);
+
+int uput_getenv(const char *varname, char * value, size_t buffsize);
+int uputenv(const char *name, const char *value);
+
+int umkdir(const char *pathname);
+
+#endif
\ No newline at end of file
diff --git a/intern/utfconv/utfconv.c b/intern/utfconv/utfconv.c
new file mode 100644
index 00000000000..9aeca36e55a
--- /dev/null
+++ b/intern/utfconv/utfconv.c
@@ -0,0 +1,232 @@
+/*
+ * ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version. 
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2009 Blender Foundation.
+ * All rights reserved.
+ * 
+ * Contributor(s): Alexandr Kuznetsov, Andrea Weikert
+ *
+ * ***** END GPL LICENSE BLOCK *****
+ */
+
+#include "utfconv.h"
+
+size_t count_utf_8_from_16(wchar_t * string16)
+{
+	int i;
+	size_t count = 0;
+	wchar_t u = 0;
+	if(!string16) return 0;
+
+	for(i=0;u = string16[i];i++)
+	{
+		if(u < 0x0080) count+=1; else
+		if(u < 0x0800) count+=2; else
+		if(u < 0xD800) count+=3; else
+		if(u < 0xDC00) {
+			i++;
+			if((u = string16[i])==0) break;
+			if(u >= 0xDC00 && u < 0xE000)count+=4;
+		} else
+		if(u < 0xE000) /*illigal*/; else			
+		count+=3;
+	}
+
+	return ++count;
+}
+
+
+size_t count_utf_16_from_8(char * string8)
+{
+		size_t count = 0;
+		char u;
+		char type = 0;
+		unsigned int u32 = 0;
+
+		if(!string8) return 0;
+
+		for(;(u = *string8);string8++)
+		{
+			if(type==0)
+			{
+				if((u&0x01<<7) == 0)	 {count++; u32 = 0; continue;} 		//1 utf-8 char
+				if((u&0x07<<5) == 0xC0)  {type=1; u32 = u & 0x1F; continue;} 	//2 utf-8 char
+				if((u&0x0F<<4) == 0xE0)  {type=2; u32 = u & 0x0F; continue;} 	//3 utf-8 char
+				if((u&0x1F<<3) == 0xF0)  {type=3; u32 = u & 0x07; continue;} 	//4 utf-8 char
+					continue;
+			} else 
+			{
+				if((u & 0xC0) == 0x80) {u32=(u32<<6) | (u&0x3F); type--;} else
+				{u32 = 0; type = 0;};
+			}
+			if(type==0)
+			{
+				if((0 < u32 && u32 < 0xD800) || (0xE000 <= u32 && u32 < 0x10000)) count++; else
+				if(0x10000 <= u32 && u32 < 0x110000) count+=2;
+				u32 = 0;
+			}
+
+		}
+
+	return ++count;
+}
+
+
+
+
+int conv_utf_16_to_8(wchar_t * in16, char * out8, size_t size8)
+{
+	char * out8end = out8+size8;
+	wchar_t u = 0;
+	int err = 0;
+	if(!size8 || !in16 || !out8) return UTF_ERROR_NULL_IN;
+	out8end--;
+
+	for(; out8 < out8end && (u=*in16); in16++, out8++)
+	{
+		if(u < 0x0080) *out8 = u; else
+		if(u < 0x0800) {
+			if(out8 + 1 >= out8end) break;
+			*out8++=(0x3<<6) | (0x1F & (u>>6));
+			*out8  =(0x1<<7) | (0x3F & (u));		
+		}else
+		if(u < 0xD800 || u >= 0xE000) {
+			if(out8 + 2 >= out8end) break;
+			*out8++=(0x7<<5) | (0xF & (u>>12));
+			*out8++=(0x1<<7) | (0x3F & (u>>6));;
+			*out8  =(0x1<<7) | (0x3F & (u));
+		}else
+		if(u < 0xDC00) {
+			wchar_t u2 = *++in16;
+
+			if(!u2) break;
+			if(u2 >= 0xDC00 && u2 < 0xE000)
+			{
+				if(out8 + 3 >= out8end) break; else { 
+				unsigned int uc = 0x10000 + (u2 - 0xDC00) + ((u - 0xD800)<<10);
+
+				*out8++=(0xF<<4) | (0x7 & (uc>>18));
+				*out8++=(0x1<<7) | (0x3F & (uc>>12));
+				*out8++=(0x1<<7) | (0x3F & (uc>>6));
+				*out8  =(0x1<<7) | (0x3F & (uc));
+				}
+			} else {out8--; err|=UTF_ERROR_ILLCHAR;};
+		} else
+		if(u < 0xE000) {out8--; err|=UTF_ERROR_ILLCHAR;}
+
+
+	}
+
+	*out8=*out8end=0;
+
+	if(*in16) err|=UTF_ERROR_SMALL;
+
+	return err;
+}
+
+
+int conv_utf_8_to_16(char * in8, wchar_t * out16, size_t size16)
+{
+	char u;
+	char type = 0;
+	wchar_t u32 = 0;
+	wchar_t * out16end = out16+size16;
+	int err = 0;
+	if(!size16 || !in8 || !out16) return UTF_ERROR_NULL_IN;
+	out16end--;
+
+	for(;out16<out16end && (u = *in8);in8++)
+	{
+		if(type==0)
+		{
+			if((u&0x01<<7) == 0)	 {*out16=u; out16++; u32 = 0; continue;} 		//1 utf-8 char
+			if((u&0x07<<5) == 0xC0)  {type=1; u32 = u & 0x1F; continue;} 	//2 utf-8 char
+			if((u&0x0F<<4) == 0xE0)  {type=2; u32 = u & 0x0F; continue;} 	//3 utf-8 char
+			if((u&0x1F<<3) == 0xF0)  {type=3; u32 = u & 0x07; continue;} 	//4 utf-8 char
+			err|=UTF_ERROR_ILLCHAR; continue;
+		} else 
+		{
+			if((u & 0xC0) == 0x80) {u32=(u32<<6) | (u&0x3F); type--;} else
+			{u32 = 0; type = 0; err|=UTF_ERROR_ILLSEQ;};
+		}
+		if(type==0)
+		{
+			if((0 < u32 && u32 < 0xD800) || (0xE000 <= u32 && u32 < 0x10000)) {*out16=u32; out16++;}else
+			if(0x10000 <= u32 && u32 < 0x110000) {
+				if(out16 + 1 >= out16end) break;
+				u32-=0x10000;
+				*out16 = 0xD800 + (u32 >> 10);
+				out16++;
+				*out16 = 0xDC00 + (u32 & 0x3FF);
+				out16++;
+			};
+			u32 = 0;
+		}
+
+	}
+
+	*out16=*out16end=0;
+
+	if(*in8) err|=UTF_ERROR_SMALL;
+
+	return err;
+}
+
+int is_ascii(char * in8)
+{
+	for(in8; *in8; in8++) 
+		if(0x80 & *in8) return 0;
+
+	return 1;
+}
+
+void utf_8_cut_end(char * inout8, size_t maxcutpoint)
+{
+	const char * start = inout8;
+	char * cur = inout8 + maxcutpoint;
+	char cc;
+	if(!inout8) return;
+
+	cc = *cur;
+
+
+	
+	
+}
+
+
+
+char * alloc_utf_8_from_16(wchar_t * in16, size_t add)
+{
+	size_t bsize = count_utf_8_from_16(in16);
+	char * out8 = NULL;
+	if(!bsize) return NULL;
+	out8 = (char*)malloc(sizeof(char) * (bsize + add));
+	conv_utf_16_to_8(in16,out8, bsize);
+	return out8;
+}
+
+wchar_t * alloc_utf16_from_8(char * in8, size_t add)
+{
+	size_t bsize = count_utf_16_from_8(in8);
+	wchar_t * out16 = NULL;
+	if(!bsize) return NULL;
+	out16 =(wchar_t*) malloc(sizeof(wchar_t) * (bsize + add));
+	conv_utf_8_to_16(in8,out16, bsize);
+	return out16;
+}
+
diff --git a/intern/utfconv/utfconv.h b/intern/utfconv/utfconv.h
new file mode 100644
index 00000000000..185c76422e2
--- /dev/null
+++ b/intern/utfconv/utfconv.h
@@ -0,0 +1,102 @@
+/*
+ * ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version. 
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2009 Blender Foundation.
+ * All rights reserved.
+ * 
+ * Contributor(s): Alexandr Kuznetsov, Andrea Weikert
+ *
+ * ***** END GPL LICENSE BLOCK *****
+ */
+
+#include <wchar.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+
+
+#ifdef __cplusplus 
+extern 	"C" {
+#endif
+
+/**
+ * Counts how many bytes is requered for for future utf-8 string using utf-16
+ * @param string-16 pointer to working utf-16 string
+ * @return How many bytes must be allocated includeng NULL.
+ */
+size_t count_utf_8_from_16(wchar_t * string16);
+
+/**
+ * Counts how many wchar_t (two byte) is requered for for future utf-16 string using utf-8
+ * @param string-8 pointer to working utf-8 string
+ * @return How many bytes must be allocated includeng NULL.
+ */
+size_t count_utf_16_from_8(char * string8);
+
+/**
+ * conv_utf_*** errors
+ */
+#define UTF_ERROR_NULL_IN 1<<0 /* Error occures when requered parameter is missing*/
+#define UTF_ERROR_ILLCHAR 1<<1 /* Error if character is in illigal UTF rage*/
+#define UTF_ERROR_SMALL   1<<2 /* Passed size is to small. It gives legal string with character missing at the end*/
+#define UTF_ERROR_ILLSEQ  1<<3 /* Error if sequence is broken and doesn't finish*/
+
+/**
+ * Converts utf-16 string to allocated utf-8 string
+ * @params in16 utf-16 string to convert
+ * @params out8 utf-8 string to string the conversion
+ * @params size8 the allocated size in bytes of out8
+ * @return Returns any errors occured during conversion. See the block above,
+ */
+int conv_utf_16_to_8(wchar_t * in16, char * out8, size_t size8);
+
+/**
+ * Converts utf-8 string to allocated utf-16 string
+ * @params in8 utf-8 string to convert
+ * @params out16 utf-16 string to string the conversion
+ * @params size16 the allocated size in wchar_t (two byte) of out16
+ * @return Returns any errors occured during conversion. See the block above,
+ */
+int conv_utf_8_to_16(char * in8, wchar_t * out16, size_t size16);
+
+
+/**
+ * Allocates and converts the utf-8 string from utf-16
+ * @params in16 utf-16 string to convert
+ * @params add any additional size which will be allocated for new utf-8 string in bytes
+ * @return New allocated and converted utf-8 string or NULL if in16 is 0.
+ */
+char * alloc_utf_8_from_16(wchar_t * in16, size_t add);
+
+/**
+ * Allocates and converts the utf-16 string from utf-8
+ * @params in8 utf-8 string to convert
+ * @params add any additional size which will be allocated for new utf-16 string in wchar_t (two bytes)
+ * @return New allocated and converted utf-16 string or NULL if in8 is 0.
+ */
+wchar_t * alloc_utf16_from_8(char * in8, size_t add);
+
+/* Easy allocation and conversion of new utf-16 string. New string has _16 suffix. Must be deallocated with UTF16_UN_ENCODE in right order*/
+#define UTF16_ENCODE(in8str) if(1){\
+	wchar_t * in8str ## _16 = alloc_utf16_from_8((char*)in8str, 0);
+
+#define UTF16_UN_ENCODE(in8str) \
+	free(in8str ## _16 ); };
+
+#ifdef __cplusplus 
+}
+#endif
\ No newline at end of file