support full-width/composite characters and true color palette in terminal (#1386)

Added normal support for full-width (CJK etc) and composite (using diacritics) characters.
IMPORTANT: WINPORT API changed in binary incompatible way in order to implement this. 

7 files changed, 307 insertions, 143 deletions

diff --git a/far2l/src/mix/StrCells.cpp b/far2l/src/mix/StrCells.cpp
new file mode 100644
index 00000000..055b8f36
--- /dev/null
+++ b/far2l/src/mix/StrCells.cpp
@@ -0,0 +1,159 @@
+#include "headers.hpp"
+
+#include <utils.h>
+#include "StrCells.h"
+#include "config.hpp"
+
+size_t StrCellsCount(const wchar_t *pwz, size_t nw)
+{
+	size_t out = 0;
+	for (size_t i = 0; i < nw; ++i) {
+		if (IsCharFullWidth(pwz[i])) {
+			out+= 2;
+		} else if ((i == nw - 1 || !IsCharPrefix(pwz[i])) && (i == 0 || !IsCharSuffix(pwz[i]))) {
+			++out;
+		}
+	}
+	return out;
+}
+
+size_t StrZCellsCount(const wchar_t *pwz)
+{
+	return StrCellsCount(pwz, wcslen(pwz));
+}
+
+size_t StrSizeOfCells(const wchar_t *pwz, size_t n, size_t &ng, bool round_up)
+{
+	size_t i = 0, g = 0;
+	for (; g < ng && i < n; ++g) {
+		for (; i < n; ++i) {
+			if (!IsCharPrefix(pwz[i])) {
+				break;
+			}
+		}
+		if (i < n) {
+			if (IsCharFullWidth(pwz[i])) {
+				++g;
+				if (!round_up && g == ng) {
+					break;
+				}
+			}
+			++i;
+		}
+		for (; i < n; ++i) {
+			if (!IsCharSuffix(pwz[i])) {
+				break;
+			}
+		}
+	}
+	ng = g;
+	return i;
+}
+
+size_t StrSizeOfCell(const wchar_t *pwz, size_t n)
+{
+	size_t ng = 1;
+	return StrSizeOfCells(pwz, n, ng, true);
+}
+
+static struct TruncReplacement
+{
+	const wchar_t *wz;
+	size_t len;
+} s_trunc_replacement[2] = { {L"...", 3}, {L"…", 1} };
+
+static const struct TruncReplacement &ChooseTruncReplacement()
+{
+	return s_trunc_replacement[Opt.NoGraphics ? 0 : 1];
+}
+
+void StrCellsTruncateLeft(wchar_t *pwz, size_t &n, size_t ng)
+{
+	size_t vl = StrCellsCount(pwz, n);
+	const auto &rpl = ChooseTruncReplacement();
+	if (vl <= ng || n < rpl.len) {
+		return;
+	}
+
+	for (size_t ofs = rpl.len; ofs < n; ++ofs) {
+		if (!IsCharXxxfix(pwz[ofs]) && StrCellsCount(pwz + ofs, n - ofs) + rpl.len <= ng) {
+			n-= ofs;
+			wmemmove(pwz + rpl.len, pwz + ofs, n);
+			n+= rpl.len;
+			wmemcpy(pwz, rpl.wz, rpl.len); //…
+			return;
+		}
+	}
+	wcsncpy(pwz, rpl.wz, ng);
+	n = std::min(ng, rpl.len);
+}
+
+void StrCellsTruncateRight(wchar_t *pwz, size_t &n, size_t ng)
+{
+	size_t vl = StrCellsCount(pwz, n);
+	const auto &rpl = ChooseTruncReplacement();
+	if (vl <= ng || n < rpl.len) {
+		return;
+	}
+
+	n-= rpl.len; // pre-reserve space for ...
+	do {
+		while (n > 0 && IsCharXxxfix(pwz[n - 1])) {
+			--n;
+		}
+		if (n == 0) {
+			break;
+		}
+		--n;
+	} while (StrCellsCount(pwz, n) + rpl.len > ng);
+
+	wmemcpy(&pwz[n], rpl.wz, rpl.len);
+	n+= rpl.len;
+}
+
+void StrCellsTruncateCenter(wchar_t *pwz, size_t &n, size_t ng)
+{
+	size_t vl = StrCellsCount(pwz, n);
+	const auto &rpl = ChooseTruncReplacement();
+	if (vl <= ng || n < rpl.len) {
+		return;
+	}
+
+	auto cut_start = n / 2;
+	if (cut_start > 0) {
+		--cut_start;
+	}
+	if (cut_start > 0 && rpl.len > 1) {
+		--cut_start;
+	}
+	while (cut_start > 0 && IsCharXxxfix(pwz[cut_start])) {
+		--cut_start;
+	}
+	auto cut_end = cut_start + rpl.len;
+	while (cut_end < n && IsCharXxxfix(pwz[cut_end])) {
+		++cut_end;
+	}
+
+	while (StrCellsCount(pwz, cut_start) + StrCellsCount(pwz + cut_end, n - cut_end) + rpl.len > ng) {
+		if (cut_start > 0) {
+			--cut_start;
+			while (cut_start > 0 && IsCharXxxfix(pwz[cut_start])) {
+				--cut_start;
+			}
+			if (StrCellsCount(pwz, cut_start) + StrCellsCount(pwz + cut_end, n - cut_end) + rpl.len <= ng) {
+				break;
+			}
+		}
+		if (cut_end < n) {
+			++cut_end;
+			while (cut_end < n && IsCharXxxfix(pwz[cut_end])) {
+				++cut_end;
+			}
+		}
+	}
+
+	wmemmove(&pwz[cut_start + rpl.len], &pwz[cut_end], n - cut_end);
+	wmemcpy(&pwz[cut_start], rpl.wz, rpl.len);
+	n-= (cut_end - cut_start);
+	n+= rpl.len;
+}
diff --git a/far2l/src/mix/StrCells.h b/far2l/src/mix/StrCells.h
new file mode 100644
index 00000000..3840cab1
--- /dev/null
+++ b/far2l/src/mix/StrCells.h
@@ -0,0 +1,10 @@
+#pragma once
+
+size_t StrCellsCount(const wchar_t *pwz, size_t nw);
+size_t StrZCellsCount(const wchar_t *pwz);
+size_t StrSizeOfCells(const wchar_t *pwz, size_t nw, size_t &ng, bool round_up);
+size_t StrSizeOfCell(const wchar_t *pwz, size_t nw);
+
+void StrCellsTruncateLeft(wchar_t *pwz, size_t &n, size_t ng);
+void StrCellsTruncateRight(wchar_t *pwz, size_t &n, size_t ng);
+void StrCellsTruncateCenter(wchar_t *pwz, size_t &n, size_t ng);
diff --git a/far2l/src/mix/format.cpp b/far2l/src/mix/format.cpp
index 8ecf8132..2a36f985 100644
--- a/far2l/src/mix/format.cpp
+++ b/far2l/src/mix/format.cpp
@@ -38,35 +38,61 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 void BaseFormat::Reset()
 {
-	_Width=0;
-	_Precision=static_cast<size_t>(-1);
-	_FillChar=L' ';
-	_Align=fmt::A_RIGHT;
+	_Cells = false;
+	_Skip = 0;
+	_Expand = 0;
+	_Truncate = static_cast<size_t>(-1);
+	_FillChar = L' ';
+	_Align = fmt::A_RIGHT;
 }
 
-void BaseFormat::Put(LPCWSTR Data,size_t Length)
+void BaseFormat::Put(LPCWSTR Data, size_t Length)
 {
-	if (_Precision==static_cast<size_t>(-1))
-	{
-		_Precision=Length;
+	if (_Skip != 0) {
+		size_t SkipChars;
+		if (_Cells) {
+			size_t ng = _Skip;
+			SkipChars = StrSizeOfCells(Data, Length, ng, true);
+		} else {
+			SkipChars = _Skip;
+		}
+		if (SkipChars < Length) {
+			Data+= SkipChars;
+			Length-= SkipChars;
+		} else {
+			Data+= Length;
+			Length = 0;
+		}
 	}
 
-	FARString OutStr(Data,Min(_Precision,Length));
-
-	if (_Align==fmt::A_RIGHT)
+	if (_Truncate != static_cast<size_t>(-1))
 	{
-		while (OutStr.GetLength()<_Width)
+		if (_Cells)
 		{
-			OutStr.Insert(0,_FillChar);
+			size_t ng = _Truncate;
+			Length = StrSizeOfCells(Data, Length, ng, false);
+		}
+		else if (Length > _Truncate)
+		{
+			Length = _Truncate;
 		}
 	}
-	else
+
+	FARString OutStr(Data, Length);
+
+	size_t Count = _Cells ? OutStr.CellsCount() : OutStr.GetLength();
+
+	if (_Align == fmt::A_RIGHT)
 	{
-		while (OutStr.GetLength()<_Width)
+		for(;Count < _Expand; ++Count)
 		{
-			OutStr.Append(_FillChar);
+			OutStr.Insert(0, _FillChar);
 		}
 	}
+	else if (_Expand > Count)
+	{
+		OutStr.Append(_FillChar, _Expand - Count);
+	}
 
 	Commit(OutStr);
 	Reset();
@@ -109,18 +135,42 @@ BaseFormat& BaseFormat::operator<<(FARString& String)
 	return *this;
 }
 
-BaseFormat& BaseFormat::operator<<(const fmt::Width& Manipulator)
+BaseFormat& BaseFormat::operator<<(const fmt::Chars&)
+{
+	SetCells(false);
+	return *this;
+}
+
+BaseFormat& BaseFormat::operator<<(const fmt::Cells&)
 {
-	SetWidth(Manipulator.GetValue());
+	SetCells(true);
 	return *this;
 }
 
-BaseFormat& BaseFormat::operator<<(const fmt::Precision& Manipulator)
+BaseFormat& BaseFormat::operator<<(const fmt::Skip& Manipulator)
 {
-	SetPrecision(Manipulator.GetValue());
+	SetSkip(Manipulator.GetValue());
 	return *this;
 }
 
+BaseFormat& BaseFormat::operator<<(const fmt::Expand& Manipulator)
+{
+	SetExpand(Manipulator.GetValue());
+	return *this;
+}
+
+BaseFormat& BaseFormat::operator<<(const fmt::Truncate& Manipulator)
+{
+	SetTruncate(Manipulator.GetValue());
+	return *this;
+}
+
+BaseFormat& BaseFormat::operator<<(const fmt::Size& Manipulator)
+{
+	SetTruncate(Manipulator.GetValue());
+	SetExpand(Manipulator.GetValue());
+	return *this;
+}
 
 BaseFormat& BaseFormat::operator<<(const fmt::FillChar& Manipulator)
 {
diff --git a/far2l/src/mix/format.hpp b/far2l/src/mix/format.hpp
index e1130e19..ae56740f 100644
--- a/far2l/src/mix/format.hpp
+++ b/far2l/src/mix/format.hpp
@@ -37,19 +37,38 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 namespace fmt
 {
-	class Width
+	struct Cells { }; // makes modifiers operate in screen cells but not characters
+	struct Chars { }; // makes modifiers operate in characters but not screen cells
+
+	class Skip
+	{
+			size_t Value;
+		public:
+			Skip(size_t Value=static_cast<size_t>(-1)) {this->Value=Value;}
+			size_t GetValue()const {return Value;}
+	};
+
+	class Expand
+	{
+			size_t Value;
+		public:
+			Expand(size_t Value=0) {this->Value=Value;}
+			size_t GetValue()const {return Value;}
+	};
+
+	class Truncate
 	{
 			size_t Value;
 		public:
-			Width(size_t Value=0) {this->Value=Value;}
+			Truncate(size_t Value=static_cast<size_t>(-1)) {this->Value=Value;}
 			size_t GetValue()const {return Value;}
 	};
 
-	class Precision
+	class Size // same as Expand + Truncate with same parameter
 	{
 			size_t Value;
 		public:
-			Precision(size_t Value=static_cast<size_t>(-1)) {this->Value=Value;}
+			Size(size_t Value=static_cast<size_t>(-1)) {this->Value=Value;}
 			size_t GetValue()const {return Value;}
 	};
 
@@ -75,8 +94,10 @@ namespace fmt
 
 class BaseFormat
 {
-		size_t _Width;
-		size_t _Precision;
+		bool _Cells;
+		size_t _Skip;
+		size_t _Expand;
+		size_t _Truncate;
 		WCHAR _FillChar;
 		fmt::AlignType _Align;
 
@@ -91,10 +112,12 @@ class BaseFormat
 		virtual ~BaseFormat() {}
 
 		// attributes
-		void SetPrecision(size_t Precision=static_cast<size_t>(-1)) {_Precision=Precision;}
-		void SetWidth(size_t Width=0) {_Width=Width;}
-		void SetAlign(fmt::AlignType Align=fmt::A_RIGHT) {_Align=Align;}
-		void SetFillChar(WCHAR Char=L' ') {_FillChar=Char;}
+		inline void SetCells(bool Cells = false) { _Cells = Cells; }
+		inline void SetSkip(size_t Skip = 0) { _Skip = Skip; }
+		inline void SetTruncate(size_t Truncate = static_cast<size_t>(-1)) { _Truncate=Truncate; }
+		inline void SetExpand(size_t Expand = 0) { _Expand = Expand; }
+		inline void SetAlign(fmt::AlignType Align = fmt::A_RIGHT) { _Align = Align; }
+		inline void SetFillChar(WCHAR Char = L' ') { _FillChar = Char; }
 
 		// data
 		BaseFormat& operator<<(INT64 Value);
@@ -114,8 +137,12 @@ class BaseFormat
 		BaseFormat& operator<<(FARString& String);
 
 		// manipulators
-		BaseFormat& operator<<(const fmt::Width& Manipulator);
-		BaseFormat& operator<<(const fmt::Precision& Manipulator);
+		BaseFormat& operator<<(const fmt::Cells&);
+		BaseFormat& operator<<(const fmt::Chars&);
+		BaseFormat& operator<<(const fmt::Skip& Manipulator);
+		BaseFormat& operator<<(const fmt::Expand& Manipulator);
+		BaseFormat& operator<<(const fmt::Truncate& Manipulator);
+		BaseFormat& operator<<(const fmt::Size& Manipulator);
 		BaseFormat& operator<<(const fmt::LeftAlign& Manipulator);
 		BaseFormat& operator<<(const fmt::RightAlign& Manipulator);
 		BaseFormat& operator<<(const fmt::FillChar& Manipulator);
diff --git a/far2l/src/mix/panelmix.cpp b/far2l/src/mix/panelmix.cpp
index 7758a7e9..673cc30b 100644
--- a/far2l/src/mix/panelmix.cpp
+++ b/far2l/src/mix/panelmix.cpp
@@ -494,7 +494,7 @@ const FARString FormatStr_Attribute( DWORD FileAttributes, DWORD UnixMode, int W
 	}
 
 	if (Width > 0)
-		strResult<<fmt::Width(Width)<<fmt::Precision(Width);
+		strResult<<fmt::Expand(Width)<<fmt::Truncate(Width);
 
 	strResult<<OutStr;
 
@@ -550,7 +550,7 @@ const FARString FormatStr_DateTime(const FILETIME *FileTime,int ColumnType,DWORD
 
 	ConvertDate(*FileTime,strDateStr,strTimeStr,ColumnWidth,Brief,TextMonth,FullYear);
 
-	strResult<<fmt::Width(Width)<<fmt::Precision(Width);
+	strResult<<fmt::Expand(Width)<<fmt::Truncate(Width);
 	switch(ColumnType)
 	{
 		case DATE_COLUMN:
@@ -592,9 +592,9 @@ const FARString FormatStr_Size(int64_t FileSize, int64_t PhysicalSize, const FAR
 			PtrName=Msg::ListSymLink;
 		}
 
-		strResult<<fmt::Width(Width)<<fmt::Precision(Width);
+		strResult<<fmt::Expand(Width)<<fmt::Truncate(Width);
 		if (StrLength(PtrName) <= Width-2) {
-			// precombine into tmp string to avoid miseffect of fmt::Width etc (#1137)
+			// precombine into tmp string to avoid miseffect of fmt::Expand etc (#1137)
 			strResult<<FARString(L"<").Append(PtrName).Append(L">");
 		} else {
 			strResult<<PtrName;
diff --git a/far2l/src/mix/strmix.cpp b/far2l/src/mix/strmix.cpp
index 317f68f2..4e1434ac 100644
--- a/far2l/src/mix/strmix.cpp
+++ b/far2l/src/mix/strmix.cpp
@@ -225,20 +225,13 @@ wchar_t* WINAPI TruncStrFromEnd(wchar_t *Str,int MaxLength)
 {
 	assert(MaxLength >= 0);
 
-	MaxLength=Max(0, MaxLength);
+	MaxLength = Max(0, MaxLength);
 
-	if (Str)
-	{
-		int Length = StrLength(Str);
-
-		if (Length > MaxLength)
-		{
-			if (MaxLength>3)
-				wmemcpy(Str+MaxLength-3, L"...", 3);
-
-			Str[MaxLength]=0;
-		}
-	}
+	const size_t Len = StrLength(Str);
+	size_t n = Len;
+	StrCellsTruncateRight(Str, n, MaxLength);
+	assert(n <= Len);
+	Str[n] = 0;
 
 	return Str;
 }
@@ -248,27 +241,13 @@ wchar_t* WINAPI TruncStr(wchar_t *Str,int MaxLength)
 {
 	assert(MaxLength >= 0);
 
-	MaxLength=Max(0, MaxLength);
+	MaxLength = Max(0, MaxLength);
 
-	if (Str)
-	{
-		int Length=StrLength(Str);
-
-		if (MaxLength<0)
-			MaxLength=0;
-
-		if (Length > MaxLength)
-		{
-			if (MaxLength>3)
-			{
-				wchar_t *MovePos = Str+Length-MaxLength+3;
-				wmemmove(Str+3, MovePos, StrLength(MovePos)+1);
-				wmemcpy(Str,L"...",3);
-			}
-
-			Str[MaxLength]=0;
-		}
-	}
+	const size_t Len = StrLength(Str);
+	size_t n = Len;
+	StrCellsTruncateLeft(Str, n, MaxLength);
+	assert(n <= Len);
+	Str[n] = 0;
 
 	return Str;
 }
@@ -286,31 +265,13 @@ wchar_t* TruncStrFromCenter(wchar_t *Str, int MaxLength)
 {
 	assert(MaxLength >= 0);
 
-	MaxLength=Max(0, MaxLength);
-
-	if (Str)
-	{
-		int Length = StrLength(Str);
-
-		if (MaxLength < 0)
-			MaxLength=0;
-
-		if (Length > MaxLength)
-		{
-			const int DotsLen = 3;
-
-			if (MaxLength > DotsLen)
-			{
-				int Len1 = (MaxLength - DotsLen) / 2;
-				int Len2 = MaxLength - DotsLen - Len1;
-				wmemcpy(Str + Len1, L"...", DotsLen);
-				wmemmove(Str + Len1 + DotsLen, Str + Length - Len2, Len2);
-			}
-
-			Str[MaxLength] = 0;
-		}
-	}
+	MaxLength = Max(0, MaxLength);
 
+	const size_t Len = StrLength(Str);
+	size_t n = Len;
+	StrCellsTruncateCenter(Str, n, MaxLength);
+	assert(n <= Len);
+	Str[n] = 0;
 	return Str;
 }
 
@@ -324,44 +285,8 @@ FARString& TruncStrFromCenter(FARString &strStr, int MaxLength)
 
 wchar_t* WINAPI TruncPathStr(wchar_t *Str, int MaxLength)
 {
-	assert(MaxLength >= 0);
-
-	MaxLength=Max(0, MaxLength);
-
-	if (Str)
-	{
-		int nLength = (int)wcslen(Str);
-
-		if ((MaxLength > 0) && (nLength > MaxLength) && (nLength >= 2))
-		{
-			wchar_t *lpStart = nullptr;
-
-/*			if (*Str && (Str[1] == L':') && IsSlash(Str[2]))
-				lpStart = Str+3;
-			else*/
-			{
-				if ((Str[0] == GOOD_SLASH) && (Str[1] == GOOD_SLASH))
-				{
-					if ((lpStart = const_cast<wchar_t*>(FirstSlash(Str+2))) )
-					{
-						wchar_t *lpStart2=lpStart;
-
-						if ((lpStart-Str < nLength) && ((lpStart=const_cast<wchar_t*>(FirstSlash(lpStart2+1)))))
-							lpStart++;
-					}
-				}
-			}
-
-			if (!lpStart || (lpStart-Str > MaxLength-5))
-				return TruncStr(Str, MaxLength);
-
-			wchar_t *lpInPos = lpStart+3+(nLength-MaxLength);
-			wmemmove(lpStart+3, lpInPos, (wcslen(lpInPos)+1));
-			wmemcpy(lpStart, L"...", 3);
-		}
-	}
-
-	return Str;
+	//  TODO
+	return TruncStr(Str, MaxLength);
 }
 
 
@@ -515,7 +440,7 @@ FARString& CenterStr(const wchar_t *Src, FARString &strDest, int Length)
 	{
 		int Space = (Length - SrcLength) / 2;
 		FormatString FString;
-		FString << fmt::Width(Space) << L"" << strTempStr << fmt::Width(Length - Space - SrcLength) << L"";
+		FString << fmt::Expand(Space) << L"" << strTempStr << fmt::Expand(Length - Space - SrcLength) << L"";
 		strDest = std::move(FString.strValue());
 	}
 
@@ -524,18 +449,11 @@ FARString& CenterStr(const wchar_t *Src, FARString &strDest, int Length)
 
 FARString FixedSizeStr(FARString str, size_t Length, bool RAlign)
 {
-	if (str.GetLength() > Length)
+	if (str.CellsCount() > Length)
 	{
-		if (str.GetLength() > 2)
-		{
-			size_t RmLen = (str.GetLength() - Length) + 1;
-			size_t RmPos = (str.GetLength() - RmLen) / 2;
-			str.Replace(RmPos, RmLen, L"…", 1);
-		}
-		else
-			str = L"…";
+		TruncStr(str, Length);
 	}
-	else while (str.GetLength() < Length)
+	else while (str.CellsCount() < Length)
 	{
 		if (RAlign)
 			str.Insert(0, L" ", 1);
diff --git a/far2l/src/mix/strmix.hpp b/far2l/src/mix/strmix.hpp
index 2c84649e..3905f3ef 100644
--- a/far2l/src/mix/strmix.hpp
+++ b/far2l/src/mix/strmix.hpp
@@ -96,7 +96,7 @@ inline bool IsWordDiv(const wchar_t *WordDiv, wchar_t Chr)
 	{ return wcschr(WordDiv, Chr) != nullptr; }
 
 inline bool IsWordDivSTNR(const wchar_t *WordDiv, wchar_t Chr)
-	{ return wcschr(WordDiv, Chr) != nullptr || wcschr(L" \t\n\r", Chr) != nullptr; }
+	{ return wcschr(WordDiv, Chr) != nullptr || IsSpace(Chr) || IsEol(Chr); }
 
 //   WordDiv  - набор разделителей слова в кодировке OEM
 // возвращает указатель на начало слова