From 93b52c8fc6545e44172737ffca7308d9c3e0efd6 Mon Sep 17 00:00:00 2001 From: Marius Ungureanu Date: Fri, 1 Sep 2017 16:17:34 +0300 Subject: Gettext strip (#2946) * Add Mnemonic strip tool. * Fixes * [PO] Post-process mnemonics in languages which contain mnemonics outside the text * [Mac] Avoid diff noise on local builds when stripping mnemonics. --- main/po/Makefile.am | 21 +- main/po/StripMnemonics/POProcessor.cs | 266 ++++++++++++++++++++++ main/po/StripMnemonics/PoModel.cs | 91 ++++++++ main/po/StripMnemonics/Program.cs | 53 +++++ main/po/StripMnemonics/Properties/AssemblyInfo.cs | 26 +++ main/po/StripMnemonics/StripMnemonics.csproj | 41 ++++ main/po/StripMnemonics/StripMnemonics.sln | 17 ++ 7 files changed, 513 insertions(+), 2 deletions(-) create mode 100644 main/po/StripMnemonics/POProcessor.cs create mode 100644 main/po/StripMnemonics/PoModel.cs create mode 100644 main/po/StripMnemonics/Program.cs create mode 100644 main/po/StripMnemonics/Properties/AssemblyInfo.cs create mode 100644 main/po/StripMnemonics/StripMnemonics.csproj create mode 100644 main/po/StripMnemonics/StripMnemonics.sln (limited to 'main/po') diff --git a/main/po/Makefile.am b/main/po/Makefile.am index 30e7f62590..0b44e025eb 100644 --- a/main/po/Makefile.am +++ b/main/po/Makefile.am @@ -1,3 +1,4 @@ +include $(top_srcdir)/xbuild.include LC_BUILD=$(top_builddir)/build/locale #old automake versions don't set datarootdir or localedir @@ -7,15 +8,31 @@ FILES = $(addsuffix .po, $(ALL_LINGUAS)) GMO_FILES = $(patsubst %.po,$(LC_BUILD)/%/LC_MESSAGES/$(PACKAGE).mo,$(FILES)) MO_FILES = $(foreach po,$(FILES), $(INSTALL_DIR)/$(basename $(po))/LC_MESSAGES/$(PACKAGE).mo) -all: $(GMO_FILES) +all: $(GMO_FILES) post-strip-mnemonics update-po: $(MDTOOL_RUN) gettext-update -f:$(top_srcdir)/Main.sln -$(GMO_FILES): $(LC_BUILD)/%/LC_MESSAGES/$(PACKAGE).mo: %.po +$(GMO_FILES): $(LC_BUILD)/%/LC_MESSAGES/$(PACKAGE).mo: %.po strip-mnemonics $(MKDIR_P) $(dir $@) msgfmt '$<' -o '$@' +if ENABLE_MACPLATFORM +strip-mnemonics: + mkdir -p backup + cp *.po backup + $(XBUILD) "StripMnemonics/StripMnemonics.sln" + mono StripMnemonics.exe . + +post-strip-mnemonics: + mv backup/*.po . + rm -rf backup +else +strip-mnemonics: +post-strip-mnemonics: +endif + + statistics: @for LANGFILE in $(ALL_LINGUAS); do \ echo "$$LANGFILE.po:"; \ diff --git a/main/po/StripMnemonics/POProcessor.cs b/main/po/StripMnemonics/POProcessor.cs new file mode 100644 index 0000000000..172b2f41f2 --- /dev/null +++ b/main/po/StripMnemonics/POProcessor.cs @@ -0,0 +1,266 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace StripMnemonics +{ + public static class POProcessor + { + public static POFile Read (string poFile) + { + var po = new POFile { + FileName = poFile + }; + + using (var stream = File.OpenRead (poFile)) + using (var reader = new StreamReader (stream)) { + string line = reader.ReadLine (); + + // Read PO Copyright header. + while (line.StartsWith ("# ", StringComparison.Ordinal)) { + po.CopyrightHeader.Add (line); + line = reader.ReadLine (); + } + + po.Header = ReadBlock (ref line, reader, po); + if (!po.NPluralsSet) + { + po.NPluralsSet = true; + po.NPlurals = 2; + } + + while (!reader.EndOfStream) { + var block = ReadBlock (ref line, reader, po); + if (block != null) + po.Messages.Add (block); + } + } + return po; + } + + static POBlock ReadBlock (ref string line, StreamReader reader, POFile po) + { + var map = new HashSet { + { "# " }, + { "#. " }, + { "#: " }, + { "#, " }, + { "#| msgid " }, + }; + + var block = new POBlock (); + bool init = false; + + do { + foreach (var mapping in map) { + if (line.StartsWith (mapping, StringComparison.Ordinal)) { + block.Metadata.Add(line); + goto read_next; + } + } + + string actual; + bool readNext; + if ((actual = ReadString(ref line, reader, "msgid ", out readNext, po)) != null) + { + block.Id = actual; + init = true; + if (!readNext) + continue; + } + + if ((actual = ReadString(ref line, reader, "msgid_plural ", out readNext, po)) != null) + { + block.IdPlural = actual; + if (!readNext) + continue; + } + + if ((actual = ReadString(ref line, reader, "msgstr ", out readNext, po)) != null) + { + block.TranslatedString = actual; + if (!readNext) + continue; + } + + if ((actual = ReadString(ref line, reader, "msgstr[0] ", out readNext, po)) != null) + { + block.SetTranslatedPlural(0, actual); + if (!readNext) + continue; + } + + if ((actual = ReadString(ref line, reader, "msgstr[1] ", out readNext, po)) != null) + { + block.SetTranslatedPlural(1, actual); + if (!readNext) + continue; + } + + if ((actual = ReadString(ref line, reader, "msgstr[2] ", out readNext, po)) != null) + { + block.SetTranslatedPlural(2, actual); + if (!readNext) + continue; + } + + read_next: + line = reader.ReadLine (); + } while (!string.IsNullOrEmpty (line)); + + if (init) + return block; + return null; + } + + static string ReadString(ref string line, StreamReader reader, string header, out bool readNext, POFile po) + { + if (!line.StartsWith(header, StringComparison.Ordinal)) + { + readNext = false; + return null; + } + + var actual = line.Substring(header.Length); + actual = actual.Substring(Math.Min (1, actual.Length), Math.Max (actual.Length - 2, 0)); + bool isMultiline = string.IsNullOrEmpty(actual); + if (!isMultiline) + { + readNext = true; + return actual; + } + + var multiLineString = new List(); + + line = reader.ReadLine(); + while (!string.IsNullOrEmpty(line) && line.StartsWith("\"", StringComparison.Ordinal)) + { + var leadingStripped = line.Remove(0, 1); + var stripped = leadingStripped.Remove(leadingStripped.Length - 1, 1); + multiLineString.Add(stripped); + + // Parse number of plural forms to write for each string. + if (line.StartsWith("\"Plural-Forms:", StringComparison.Ordinal)) + { + var rest = line.Substring("\"Plural-Forms: ".Length).TrimEnd('\"').Split (';'); + int plurals; + if (int.TryParse(rest[0].Trim().Substring("nplurals=".Length), out plurals)) + po.NPlurals = plurals; + po.NPluralsSet = true; + } + line = reader.ReadLine(); + } + readNext = false; + return string.Concat (multiLineString); + } + + + public static void Write(POFile po, string outPath) + { + using (var stream = File.Open(outPath, FileMode.Create)) + using (var writer = new StreamWriter(stream)) + { + foreach (var line in po.CopyrightHeader) + writer.WriteLine(line); + + writer.WriteLine("msgid \"\""); + writer.WriteLine("msgstr \"\""); + foreach (var line in po.Header.TranslatedString.Split(new string[] { "\\n" }, StringSplitOptions.RemoveEmptyEntries)) + writer.WriteLine($"\"{line}\\n\""); + + writer.WriteLine(); + bool isMessages = po.FileName.EndsWith("messages.po", StringComparison.OrdinalIgnoreCase); + foreach (var block in po.Messages) + { + WriteBlock(block, writer, po, isMessages); + writer.WriteLine(); + } + } + } + + static IEnumerable LineWrap(string text, string kind) + { + // Gettext wraps at 80 chars, including quotes. + // Also breaks on \n. + + // Take into account kind, quotes and space. + if (text.Length + kind.Length + 3 > 80 || text.Contains("\\n")) + yield return kind + " \"\""; + else + { + yield return kind + $" \"{text}\""; + yield break; + } + + var words = text.Split(new char[] { ' ' }); + var sb = new StringBuilder(); + + for (int i = 0; i < words.Length; ++i) + { + // Add the space into account. + if (sb.Length + words[i].Length + 3 > 79) + { + yield return $"\"{sb.ToString()}\""; + sb.Clear(); + } + + if (words[i].Contains("\\n")) + { + var split = words[i].Split(new string[] { "\\n" }, StringSplitOptions.None); + for (int splitIndex = 0; splitIndex < split.Length - 1; ++splitIndex) + { + sb.Append(split[splitIndex]); + sb.Append("\\n"); + yield return $"\"{sb.ToString()}\""; + sb.Clear(); + } + sb.Append(split[split.Length - 1]); + } + else + { + sb.Append(words[i]); + } + + if (i != words.Length - 1) + sb.Append(' '); + } + if (sb.Length > 0) + yield return $"\"{sb.ToString()}\""; + yield break; + } + + static void WriteBlock(POBlock block, StreamWriter writer, POFile po, bool isMessages) + { + foreach (var item in block.Metadata) + writer.WriteLine(item); + + foreach (var line in LineWrap(block.Id.Replace("\r\n", "\n"), "msgid")) + writer.WriteLine(line); + + if (block.IdPlural != null) + { + foreach (var line in LineWrap(block.IdPlural.Replace("\r\n", "\n"), "msgid_plural")) + writer.WriteLine(line); + + for (int i = 0; i < po.NPlurals; ++i) + { + var translatedPlural = block.GetTranslatedPlural(i); + if (translatedPlural == null) + continue; + + string value = isMessages ? "" : translatedPlural.Replace("\r\n", "\n"); + foreach (var line in LineWrap(value, $"msgstr[{i}]")) + writer.WriteLine(line); + } + } + else + { + string value = isMessages ? "" : block.TranslatedString.Replace("\r\n", "\n"); + foreach (var line in LineWrap(value, "msgstr")) + writer.WriteLine(line); + } + } + } +} + diff --git a/main/po/StripMnemonics/PoModel.cs b/main/po/StripMnemonics/PoModel.cs new file mode 100644 index 0000000000..7516dd4e1a --- /dev/null +++ b/main/po/StripMnemonics/PoModel.cs @@ -0,0 +1,91 @@ +using System; +using System.Collections.Generic; + +namespace StripMnemonics +{ + [Serializable] + public class POBlock : IEquatable + { + string id = string.Empty; + public string Id + { + get { return id; } + set { id = value.Replace("\r\n", "\n").Replace("\n", "\r\n"); } + } + + string idPlural = null; + public string IdPlural + { + get { return idPlural; } + set { idPlural = value.Replace("\r\n", "\n").Replace("\n", "\r\n"); } + } + + string translatedString = string.Empty; + public string TranslatedString + { + get { return translatedString; } + set { translatedString = value.Replace("\r\n", "\n").Replace("\n", "\r\n"); } + } + + readonly string[] translatedPlural = new string[3]; + public string GetTranslatedPlural(int index) + { + return translatedPlural[index]; + } + + public void SetTranslatedPlural(int index, string value) + { + translatedPlural[index] = value.Replace("\r\n", "\n").Replace("\n", "\r\n"); + } + + public List Metadata { get; } = new List(); + + static bool IsPluralEqual(string a, string b) + { + return (string.IsNullOrEmpty(a) && string.IsNullOrEmpty(b)) || a == b; + } + + public bool Equals(POBlock other) + { + return id == other.id && + idPlural == other.idPlural && + translatedString == other.translatedString && + IsPluralEqual(translatedPlural[0], other.translatedPlural[0]) && + IsPluralEqual(translatedPlural[1], other.translatedPlural[1]) && + IsPluralEqual(translatedPlural[2], other.translatedPlural[2]); + } + } + + [Serializable] + public class POFile : IEquatable + { + public string FileName; + public List CopyrightHeader { get; } = new List (); + public int NPlurals; + public bool NPluralsSet; + public POBlock Header { get; set; } + public List Messages { get; } = new List (); + + public bool Equals(POFile other) + { + bool basic = NPlurals == other.NPlurals && + Header.Equals(other.Header); + + bool collections = true; + for (int i = 0; i < CopyrightHeader.Count; ++i) + collections &= CopyrightHeader[i] == other.CopyrightHeader[i]; + + bool messages = true; + for (int i = 0; i < Messages.Count; ++i) + { + var msg = Messages[i]; + var otherMsg = other.Messages[i]; + bool equals = msg.Equals(otherMsg); + messages &= equals; + } + + return basic && collections && messages; + } + } +} + diff --git a/main/po/StripMnemonics/Program.cs b/main/po/StripMnemonics/Program.cs new file mode 100644 index 0000000000..ed5fe115af --- /dev/null +++ b/main/po/StripMnemonics/Program.cs @@ -0,0 +1,53 @@ +using System; +using System.IO; +using System.Text.RegularExpressions; + +namespace StripMnemonics +{ + class MainClass + { + static string[] langs = { + "ja", + "ko", + "zh_CN", + "zh_TW", + }; + + static Regex reg = new Regex(@"\(_\w\)$", RegexOptions.Compiled); + + static string StripMnemonics(string text) + { + if (reg.IsMatch(text)) { + return text.Substring(0, text.Length - 4); + } + return text; + } + + static void PostProcess(string file) + { + var poFile = POProcessor.Read(file); + + foreach (var block in poFile.Messages) { + if (block.IdPlural == null) { + block.TranslatedString = StripMnemonics(block.TranslatedString); + } else { + for (int i = 0; i < poFile.NPlurals; ++i) + block.SetTranslatedPlural(i, StripMnemonics(block.GetTranslatedPlural(i))); + } + } + + POProcessor.Write(poFile, file); + } + + public static void Main(string[] args) + { + if (args.Length != 1) { + Console.WriteLine("Usage: StripMnemonics.exe "); + return; + } + + foreach (var lang in langs) + PostProcess(Path.Combine (args[0], lang + ".po")); + } + } +} diff --git a/main/po/StripMnemonics/Properties/AssemblyInfo.cs b/main/po/StripMnemonics/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..e3ca56650a --- /dev/null +++ b/main/po/StripMnemonics/Properties/AssemblyInfo.cs @@ -0,0 +1,26 @@ +using System.Reflection; +using System.Runtime.CompilerServices; + +// Information about this assembly is defined by the following attributes. +// Change them to the values specific to your project. + +[assembly: AssemblyTitle("StripMnemonics")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("Microsoft")] +[assembly: AssemblyProduct("")] +[assembly: AssemblyCopyright("")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// The assembly version has the format "{Major}.{Minor}.{Build}.{Revision}". +// The form "{Major}.{Minor}.*" will automatically update the build and revision, +// and "{Major}.{Minor}.{Build}.*" will update just the revision. + +[assembly: AssemblyVersion("1.0.*")] + +// The following attributes are used to specify the signing key for the assembly, +// if desired. See the Mono documentation for more information about signing. + +//[assembly: AssemblyDelaySign(false)] +//[assembly: AssemblyKeyFile("")] diff --git a/main/po/StripMnemonics/StripMnemonics.csproj b/main/po/StripMnemonics/StripMnemonics.csproj new file mode 100644 index 0000000000..4395d31558 --- /dev/null +++ b/main/po/StripMnemonics/StripMnemonics.csproj @@ -0,0 +1,41 @@ + + + + Debug + x86 + {F5892E72-15A0-4BFA-B65C-B53397A86337} + Exe + StripMnemonics + StripMnemonics + v4.6.1 + + + true + full + false + .. + DEBUG; + prompt + 4 + true + x86 + + + true + .. + prompt + 4 + true + x86 + + + + + + + + + + + + \ No newline at end of file diff --git a/main/po/StripMnemonics/StripMnemonics.sln b/main/po/StripMnemonics/StripMnemonics.sln new file mode 100644 index 0000000000..c6f9c28c15 --- /dev/null +++ b/main/po/StripMnemonics/StripMnemonics.sln @@ -0,0 +1,17 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 2012 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StripMnemonics", "StripMnemonics.csproj", "{F5892E72-15A0-4BFA-B65C-B53397A86337}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|x86 = Debug|x86 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {F5892E72-15A0-4BFA-B65C-B53397A86337}.Debug|x86.ActiveCfg = Debug|x86 + {F5892E72-15A0-4BFA-B65C-B53397A86337}.Debug|x86.Build.0 = Debug|x86 + {F5892E72-15A0-4BFA-B65C-B53397A86337}.Release|x86.ActiveCfg = Release|x86 + {F5892E72-15A0-4BFA-B65C-B53397A86337}.Release|x86.Build.0 = Release|x86 + EndGlobalSection +EndGlobal -- cgit v1.2.3