Gettext strip (#2946)

* Add Mnemonic strip tool.

* Fixes

* [PO] Post-process mnemonics in languages which contain mnemonics outside the text

* [Mac] Avoid diff noise on local builds when stripping mnemonics.

7 files changed, 513 insertions, 2 deletions

diff --git a/main/po/Makefile.am b/main/po/Makefile.am
index 30e7f62590..0b44e025eb 100644
--- a/main/po/Makefile.am
+++ b/main/po/Makefile.am
@@ -1,3 +1,4 @@
+include $(top_srcdir)/xbuild.include
 
 LC_BUILD=$(top_builddir)/build/locale
 #old automake versions don't set datarootdir or localedir
@@ -7,15 +8,31 @@ FILES = $(addsuffix .po, $(ALL_LINGUAS))
 GMO_FILES = $(patsubst %.po,$(LC_BUILD)/%/LC_MESSAGES/$(PACKAGE).mo,$(FILES))
 MO_FILES = $(foreach po,$(FILES), $(INSTALL_DIR)/$(basename $(po))/LC_MESSAGES/$(PACKAGE).mo)
 
-all: $(GMO_FILES)
+all: $(GMO_FILES) post-strip-mnemonics
 
 update-po:
 	$(MDTOOL_RUN) gettext-update -f:$(top_srcdir)/Main.sln
 
-$(GMO_FILES): $(LC_BUILD)/%/LC_MESSAGES/$(PACKAGE).mo: %.po
+$(GMO_FILES): $(LC_BUILD)/%/LC_MESSAGES/$(PACKAGE).mo: %.po strip-mnemonics
 	$(MKDIR_P) $(dir $@)
 	msgfmt '$<' -o '$@'
 
+if ENABLE_MACPLATFORM
+strip-mnemonics:
+	mkdir -p backup
+	cp *.po backup
+	$(XBUILD) "StripMnemonics/StripMnemonics.sln"
+	mono StripMnemonics.exe .
+
+post-strip-mnemonics:
+	mv backup/*.po .
+	rm -rf backup
+else
+strip-mnemonics:
+post-strip-mnemonics:
+endif
+
+
 statistics:
 	@for LANGFILE in $(ALL_LINGUAS); do \
 		echo "$$LANGFILE.po:"; \
diff --git a/main/po/StripMnemonics/POProcessor.cs b/main/po/StripMnemonics/POProcessor.cs
new file mode 100644
index 0000000000..172b2f41f2
--- /dev/null
+++ b/main/po/StripMnemonics/POProcessor.cs
@@ -0,0 +1,266 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace StripMnemonics
+{
+	public static class POProcessor
+	{
+		public static POFile Read (string poFile)
+		{
+			var po = new POFile {
+				FileName = poFile
+			};
+
+			using (var stream = File.OpenRead (poFile))
+			using (var reader = new StreamReader (stream)) {
+				string line = reader.ReadLine ();
+
+				// Read PO Copyright header.
+				while (line.StartsWith ("# ", StringComparison.Ordinal)) {
+					po.CopyrightHeader.Add (line);
+					line = reader.ReadLine ();
+				}
+
+				po.Header = ReadBlock (ref line, reader, po);
+				if (!po.NPluralsSet)
+				{
+					po.NPluralsSet = true;
+					po.NPlurals = 2;
+				}
+
+				while (!reader.EndOfStream) {
+					var block = ReadBlock (ref line, reader, po);
+					if (block != null)
+						po.Messages.Add (block);
+				}
+			}
+			return po;
+		}
+
+		static POBlock ReadBlock (ref string line, StreamReader reader, POFile po)
+		{
+			var map = new HashSet<string> {
+				{ "# " },
+				{ "#. " },
+				{ "#: " },
+				{ "#, " },
+				{ "#| msgid " },
+			};
+
+			var block = new POBlock ();
+			bool init = false;
+
+			do {
+				foreach (var mapping in map) {
+					if (line.StartsWith (mapping, StringComparison.Ordinal)) {
+						block.Metadata.Add(line);
+						goto read_next;
+					}
+				}
+
+				string actual;
+				bool readNext;
+				if ((actual = ReadString(ref line, reader, "msgid ", out readNext, po)) != null)
+				{
+					block.Id = actual;
+					init = true;
+					if (!readNext)
+						continue;
+				}
+
+				if ((actual = ReadString(ref line, reader, "msgid_plural ", out readNext, po)) != null)
+				{
+					block.IdPlural = actual;
+					if (!readNext)
+						continue;
+				}
+
+				if ((actual = ReadString(ref line, reader, "msgstr ", out readNext, po)) != null)
+				{
+					block.TranslatedString = actual;
+					if (!readNext)
+						continue;
+				}
+
+				if ((actual = ReadString(ref line, reader, "msgstr[0] ", out readNext, po)) != null)
+				{
+					block.SetTranslatedPlural(0, actual);
+					if (!readNext)
+						continue;
+				}
+
+				if ((actual = ReadString(ref line, reader, "msgstr[1] ", out readNext, po)) != null)
+				{
+					block.SetTranslatedPlural(1, actual);
+					if (!readNext)
+						continue;
+				}
+
+				if ((actual = ReadString(ref line, reader, "msgstr[2] ", out readNext, po)) != null)
+				{
+					block.SetTranslatedPlural(2, actual);
+					if (!readNext)
+						continue;
+				}
+
+			read_next:
+				line = reader.ReadLine ();
+			} while (!string.IsNullOrEmpty (line));
+
+			if (init)
+				return block;
+			return null;
+		}
+
+		static string ReadString(ref string line, StreamReader reader, string header, out bool readNext, POFile po)
+		{
+			if (!line.StartsWith(header, StringComparison.Ordinal))
+			{
+				readNext = false;
+				return null;
+			}
+
+			var actual = line.Substring(header.Length);
+			actual = actual.Substring(Math.Min (1, actual.Length), Math.Max (actual.Length - 2, 0));
+			bool isMultiline = string.IsNullOrEmpty(actual);
+			if (!isMultiline)
+			{
+				readNext = true;
+				return actual;
+			}
+
+			var multiLineString = new List<string>();
+
+			line = reader.ReadLine();
+			while (!string.IsNullOrEmpty(line) && line.StartsWith("\"", StringComparison.Ordinal))
+			{
+				var leadingStripped = line.Remove(0, 1);
+				var stripped = leadingStripped.Remove(leadingStripped.Length - 1, 1);
+				multiLineString.Add(stripped);
+
+				// Parse number of plural forms to write for each string.
+				if (line.StartsWith("\"Plural-Forms:", StringComparison.Ordinal))
+				{
+					var rest = line.Substring("\"Plural-Forms: ".Length).TrimEnd('\"').Split (';');
+					int plurals;
+					if (int.TryParse(rest[0].Trim().Substring("nplurals=".Length), out plurals))
+					    po.NPlurals = plurals;
+					po.NPluralsSet = true;
+				}
+				line = reader.ReadLine();
+			}
+			readNext = false;
+			return string.Concat (multiLineString);
+		}
+
+
+		public static void Write(POFile po, string outPath)
+		{
+			using (var stream = File.Open(outPath, FileMode.Create))
+			using (var writer = new StreamWriter(stream))
+			{
+				foreach (var line in po.CopyrightHeader)
+					writer.WriteLine(line);
+
+				writer.WriteLine("msgid \"\"");
+				writer.WriteLine("msgstr \"\"");
+				foreach (var line in po.Header.TranslatedString.Split(new string[] { "\\n" }, StringSplitOptions.RemoveEmptyEntries))
+					writer.WriteLine($"\"{line}\\n\"");
+
+				writer.WriteLine();
+				bool isMessages = po.FileName.EndsWith("messages.po", StringComparison.OrdinalIgnoreCase);
+				foreach (var block in po.Messages)
+				{
+					WriteBlock(block, writer, po, isMessages);
+					writer.WriteLine();
+				}
+			}
+		}
+
+		static IEnumerable<string> LineWrap(string text, string kind)
+		{
+			// Gettext wraps at 80 chars, including quotes.
+			// Also breaks on \n.
+
+			// Take into account kind, quotes and space.
+			if (text.Length + kind.Length + 3 > 80 || text.Contains("\\n"))
+				yield return kind + " \"\"";
+			else
+			{
+				yield return kind + $" \"{text}\"";
+				yield break;
+			}
+
+			var words = text.Split(new char[] { ' ' });
+			var sb = new StringBuilder();
+
+			for (int i = 0; i < words.Length; ++i)
+			{
+				// Add the space into account.
+				if (sb.Length + words[i].Length + 3 > 79)
+				{
+					yield return $"\"{sb.ToString()}\"";
+					sb.Clear();
+				}
+
+				if (words[i].Contains("\\n"))
+				{
+					var split = words[i].Split(new string[] { "\\n" }, StringSplitOptions.None);
+					for (int splitIndex = 0; splitIndex < split.Length - 1; ++splitIndex)
+					{
+						sb.Append(split[splitIndex]);
+						sb.Append("\\n");
+						yield return $"\"{sb.ToString()}\"";
+						sb.Clear();
+					}
+					sb.Append(split[split.Length - 1]);
+				}
+				else
+				{
+					sb.Append(words[i]);
+				}
+
+				if (i != words.Length - 1)
+					sb.Append(' ');
+			}
+			if (sb.Length > 0)
+				yield return $"\"{sb.ToString()}\"";
+			yield break;
+		}
+
+		static void WriteBlock(POBlock block, StreamWriter writer, POFile po, bool isMessages)
+		{
+			foreach (var item in block.Metadata)
+				writer.WriteLine(item);
+
+			foreach (var line in LineWrap(block.Id.Replace("\r\n", "\n"), "msgid"))
+				writer.WriteLine(line);
+
+			if (block.IdPlural != null)
+			{
+				foreach (var line in LineWrap(block.IdPlural.Replace("\r\n", "\n"), "msgid_plural"))
+					writer.WriteLine(line);
+
+				for (int i = 0; i < po.NPlurals; ++i)
+				{
+					var translatedPlural = block.GetTranslatedPlural(i);
+					if (translatedPlural == null)
+						continue;
+
+					string value = isMessages ? "" : translatedPlural.Replace("\r\n", "\n");
+					foreach (var line in LineWrap(value, $"msgstr[{i}]"))
+						writer.WriteLine(line);
+				}
+			}
+			else
+			{
+				string value = isMessages ? "" : block.TranslatedString.Replace("\r\n", "\n");
+				foreach (var line in LineWrap(value, "msgstr"))
+					writer.WriteLine(line);
+			}
+		}
+	}
+}
+
diff --git a/main/po/StripMnemonics/PoModel.cs b/main/po/StripMnemonics/PoModel.cs
new file mode 100644
index 0000000000..7516dd4e1a
--- /dev/null
+++ b/main/po/StripMnemonics/PoModel.cs
@@ -0,0 +1,91 @@
+using System;
+using System.Collections.Generic;
+
+namespace StripMnemonics
+{
+	[Serializable]
+	public class POBlock : IEquatable<POBlock>
+	{
+		string id = string.Empty;
+		public string Id
+		{
+			get { return id; }
+			set { id = value.Replace("\r\n", "\n").Replace("\n", "\r\n"); }
+		}
+
+		string idPlural = null;
+		public string IdPlural
+		{
+			get { return idPlural; }
+			set { idPlural = value.Replace("\r\n", "\n").Replace("\n", "\r\n"); }
+		}
+
+		string translatedString = string.Empty;
+		public string TranslatedString
+		{
+			get { return translatedString; }
+			set { translatedString = value.Replace("\r\n", "\n").Replace("\n", "\r\n"); }
+		}
+
+		readonly string[] translatedPlural = new string[3];
+		public string GetTranslatedPlural(int index)
+		{
+			return translatedPlural[index];
+		}
+
+		public void SetTranslatedPlural(int index, string value)
+		{
+			translatedPlural[index] = value.Replace("\r\n", "\n").Replace("\n", "\r\n");
+		}
+
+		public List<string> Metadata { get; } = new List<string>();
+
+		static bool IsPluralEqual(string a, string b)
+		{
+			return (string.IsNullOrEmpty(a) && string.IsNullOrEmpty(b)) || a == b;
+		}
+
+		public bool Equals(POBlock other)
+		{
+			return id == other.id &&
+					idPlural == other.idPlural &&
+					translatedString == other.translatedString &&
+					IsPluralEqual(translatedPlural[0], other.translatedPlural[0]) &&
+					IsPluralEqual(translatedPlural[1], other.translatedPlural[1]) &&
+					IsPluralEqual(translatedPlural[2], other.translatedPlural[2]);
+		}
+	}
+
+	[Serializable]
+	public class POFile : IEquatable<POFile>
+	{
+		public string FileName;
+		public List<string> CopyrightHeader { get; } = new List<string> ();
+		public int NPlurals;
+		public bool NPluralsSet;
+		public POBlock Header { get; set; }
+		public List<POBlock> Messages { get; } = new List<POBlock> ();
+
+		public bool Equals(POFile other)
+		{
+			bool basic = NPlurals == other.NPlurals &&
+						 Header.Equals(other.Header);
+
+			bool collections = true;
+			for (int i = 0; i < CopyrightHeader.Count; ++i)
+				collections &= CopyrightHeader[i] == other.CopyrightHeader[i];
+
+			bool messages = true;
+			for (int i = 0; i < Messages.Count; ++i)
+			{
+				var msg = Messages[i];
+				var otherMsg = other.Messages[i];
+				bool equals = msg.Equals(otherMsg);
+				messages &= equals;
+			}
+
+			return basic && collections && messages;
+		}
+	}
+}
+
diff --git a/main/po/StripMnemonics/Program.cs b/main/po/StripMnemonics/Program.cs
new file mode 100644
index 0000000000..ed5fe115af
--- /dev/null
+++ b/main/po/StripMnemonics/Program.cs
@@ -0,0 +1,53 @@
+using System;
+using System.IO;
+using System.Text.RegularExpressions;
+
+namespace StripMnemonics
+{
+	class MainClass
+	{
+		static string[] langs = {
+			"ja",
+			"ko",
+			"zh_CN",
+			"zh_TW",
+		};
+
+		static Regex reg = new Regex(@"\(_\w\)$", RegexOptions.Compiled);
+
+		static string StripMnemonics(string text)
+		{
+			if (reg.IsMatch(text)) {
+				return text.Substring(0, text.Length - 4);
+			}
+			return text;
+		}
+
+		static void PostProcess(string file)
+		{
+			var poFile = POProcessor.Read(file);
+
+			foreach (var block in poFile.Messages) {
+				if (block.IdPlural == null) {
+					block.TranslatedString = StripMnemonics(block.TranslatedString);
+				} else {
+					for (int i = 0; i < poFile.NPlurals; ++i)
+						block.SetTranslatedPlural(i, StripMnemonics(block.GetTranslatedPlural(i)));
+				}
+			}
+
+			POProcessor.Write(poFile, file);
+		}
+
+		public static void Main(string[] args)
+		{
+			if (args.Length != 1) {
+				Console.WriteLine("Usage: StripMnemonics.exe <po_dir>");
+				return;
+			}
+
+			foreach (var lang in langs)
+				PostProcess(Path.Combine (args[0], lang + ".po"));
+		}
+	}
+}
diff --git a/main/po/StripMnemonics/Properties/AssemblyInfo.cs b/main/po/StripMnemonics/Properties/AssemblyInfo.cs
new file mode 100644
index 0000000000..e3ca56650a
--- /dev/null
+++ b/main/po/StripMnemonics/Properties/AssemblyInfo.cs
@@ -0,0 +1,26 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+
+// Information about this assembly is defined by the following attributes. 
+// Change them to the values specific to your project.
+
+[assembly: AssemblyTitle("StripMnemonics")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("Microsoft")]
+[assembly: AssemblyProduct("")]
+[assembly: AssemblyCopyright("")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// The assembly version has the format "{Major}.{Minor}.{Build}.{Revision}".
+// The form "{Major}.{Minor}.*" will automatically update the build and revision,
+// and "{Major}.{Minor}.{Build}.*" will update just the revision.
+
+[assembly: AssemblyVersion("1.0.*")]
+
+// The following attributes are used to specify the signing key for the assembly, 
+// if desired. See the Mono documentation for more information about signing.
+
+//[assembly: AssemblyDelaySign(false)]
+//[assembly: AssemblyKeyFile("")]
diff --git a/main/po/StripMnemonics/StripMnemonics.csproj b/main/po/StripMnemonics/StripMnemonics.csproj
new file mode 100644
index 0000000000..4395d31558
--- /dev/null
+++ b/main/po/StripMnemonics/StripMnemonics.csproj
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">x86</Platform>
+    <ProjectGuid>{F5892E72-15A0-4BFA-B65C-B53397A86337}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <RootNamespace>StripMnemonics</RootNamespace>
+    <AssemblyName>StripMnemonics</AssemblyName>
+    <TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x86' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>..</OutputPath>
+    <DefineConstants>DEBUG;</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <ExternalConsole>true</ExternalConsole>
+    <PlatformTarget>x86</PlatformTarget>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x86' ">
+    <Optimize>true</Optimize>
+    <OutputPath>..</OutputPath>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <ExternalConsole>true</ExternalConsole>
+    <PlatformTarget>x86</PlatformTarget>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="System" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="Program.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="PoModel.cs" />
+    <Compile Include="POProcessor.cs" />
+  </ItemGroup>
+  <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
+</Project>
\ No newline at end of file
diff --git a/main/po/StripMnemonics/StripMnemonics.sln b/main/po/StripMnemonics/StripMnemonics.sln
new file mode 100644
index 0000000000..c6f9c28c15
--- /dev/null
+++ b/main/po/StripMnemonics/StripMnemonics.sln
@@ -0,0 +1,17 @@
+

+Microsoft Visual Studio Solution File, Format Version 12.00

+# Visual Studio 2012

+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StripMnemonics", "StripMnemonics.csproj", "{F5892E72-15A0-4BFA-B65C-B53397A86337}"

+EndProject

+Global

+	GlobalSection(SolutionConfigurationPlatforms) = preSolution

+		Debug|x86 = Debug|x86

+		Release|x86 = Release|x86

+	EndGlobalSection

+	GlobalSection(ProjectConfigurationPlatforms) = postSolution

+		{F5892E72-15A0-4BFA-B65C-B53397A86337}.Debug|x86.ActiveCfg = Debug|x86

+		{F5892E72-15A0-4BFA-B65C-B53397A86337}.Debug|x86.Build.0 = Debug|x86

+		{F5892E72-15A0-4BFA-B65C-B53397A86337}.Release|x86.ActiveCfg = Release|x86

+		{F5892E72-15A0-4BFA-B65C-B53397A86337}.Release|x86.Build.0 = Release|x86

+	EndGlobalSection

+EndGlobal