Update of the tools for the new "dynamic" i18n menu, to generate the languages file into locale dir...

4 files changed, 155 insertions, 54 deletions

diff --git a/release/scripts/modules/bl_i18n_utils/settings.py b/release/scripts/modules/bl_i18n_utils/settings.py
index 26a4cbaeb01..07dcf213fb1 100644
--- a/release/scripts/modules/bl_i18n_utils/settings.py
+++ b/release/scripts/modules/bl_i18n_utils/settings.py
@@ -31,6 +31,59 @@ import os.path
 # MISC
 ###############################################################################
 
+# The languages defined in Blender.
+LANGUAGES_CATEGORIES = (
+    # Min completeness level, UI english label.
+    ( 0.95, "Complete"),
+    ( 0.33, "In Progress"),
+    ( -1.0, "Starting"),
+)
+LANGUAGES = (
+    # ID, UI english label, ISO code.
+    ( 0, "Default (Default)", "DEFAULT"),
+    ( 1, "English (English)", "en_US"),
+    ( 2, "Japanese (日本語)", "ja_JP"),
+    ( 3, "Dutch (Nederlandse taal)", "nl_NL"),
+    ( 4, "Italian (Italiano)", "it_IT"),
+    ( 5, "German (Deutsch)", "de_DE"),
+    ( 6, "Finnish (Suomi)", "fi_FI"),
+    ( 7, "Swedish (Svenska)", "sv_SE"),
+    ( 8, "French (Français)", "fr_FR"),
+    ( 9, "Spanish (Español)", "es"),
+    (10, "Catalan (Català)", "ca_AD"),
+    (11, "Czech (Český)", "cs_CZ"),
+    (12, "Portuguese (Português)", "pt_PT"),
+    (13, "Simplified Chinese (简体中文)", "zh_CN"),
+    (14, "Traditional Chinese (繁體中文)", "zh_TW"),
+    (15, "Russian (Русский)", "ru_RU"),
+    (16, "Croatian (Hrvatski)", "hr_HR"),
+    (17, "Serbian (Српски)", "sr_RS"),
+    (18, "Ukrainian (Український)", "uk_UA"),
+    (19, "Polish (Polski)", "pl_PL"),
+    (20, "Romanian (Român)", "ro_RO"), # XXX No po's yet.
+    # Using the utf8 flipped form of Arabic (العربية).
+    (21, "Arabic (ﺔﻴﺑﺮﻌﻟﺍ)", "ar_EG"),
+    (22, "Bulgarian (Български)", "bg_BG"),
+    (23, "Greek (Ελληνικά)", "el_GR"),
+    (24, "Korean (한국 언어)", "ko_KR"), # XXX No po's yet.
+    (25, "Nepali (नेपाली)", "ne_NP"),
+    # Using the utf8 flipped form of Persian (فارسی).
+    (26, "Persian (ﯽﺳﺭﺎﻓ)", "fa_IR"),
+    (27, "Indonesian (Bahasa indonesia)", "id_ID"),
+    (28, "Serbian Latin (Srpski latinica)", "sr_RS@latin"),
+    (29, "Kyrgyz (Кыргыз тили)", "ky_KG"),
+    (30, "Turkish (Türkçe)", "tr_TR"),
+    (31, "Hungarian (Magyar)", "hu_HU"),
+    (32, "Brazilian Portuguese (Português do Brasil)", "pt_BR"),
+    # Using the utf8 flipped form of Hebrew (עִבְרִית)).
+    (33, "Hebrew (תירִבְעִ)", "he_IL"),
+    (34, "Estonian (Eestlane)", "et_EE"),
+    (35, "Esperanto (Esperanto)", "eo"),
+)
+
+# Name of language file used by Blender to generate translations' menu.
+LANGUAGES_FILE = "languages"
+
 # The min level of completeness for a po file to be imported from /branches
 # into /trunk, as a percentage. -1 means "import everything".
 IMPORT_MIN_LEVEL = -1
@@ -141,37 +194,6 @@ WARN_MSGID_NOT_CAPITALIZED_ALLOWED = {
     "iTaSC parameters",
     "vBVH",
     "rv",
-    "en_US",
-    "fr_FR",
-    "it_IT",
-    "ru_RU",
-    "zh_CN",
-    "es",
-    "zh_TW",
-    "ar_EG",
-    "pt",
-    "bg_BG",
-    "ca_AD",
-    "hr_HR",
-    "cs_CZ",
-    "nl_NL",
-    "fi_FI",
-    "de_DE",
-    "el_GR",
-    "id_ID",
-    "ja_JP",
-    "ky_KG",
-    "ko_KR",
-    "ne_NP",
-    "fa_IR",
-    "pl_PL",
-    "ro_RO",
-    "sr_RS",
-    "sr_RS@latin",
-    "sv_SE",
-    "uk_UA",
-    "tr_TR",
-    "hu_HU",
     "et_EE",
     "eo",
     "available with",                # Is part of multi-line msg.
@@ -185,6 +207,7 @@ WARN_MSGID_NOT_CAPITALIZED_ALLOWED = {
     "p0",
     "res",
 }
+WARN_MSGID_NOT_CAPITALIZED_ALLOWED |= set(lng[2] for lng in LANGUAGES)
 
 
 ###############################################################################
diff --git a/release/scripts/modules/bl_i18n_utils/update_mo.py b/release/scripts/modules/bl_i18n_utils/update_mo.py
index 4a68f19fab0..7f68736593c 100755
--- a/release/scripts/modules/bl_i18n_utils/update_mo.py
+++ b/release/scripts/modules/bl_i18n_utils/update_mo.py
@@ -69,9 +69,9 @@ def main():
                                                  "under {}.".format(TRUNK_MO_DIR))
     parser.add_argument('langs', metavar='ISO_code', nargs='*',
                         help="Restrict processed languages to those.")
-    parser.add_argument('po', help="Only process that po file (implies --mo).",
+    parser.add_argument('--po', help="Only process that po file (implies --mo).",
                         nargs='?')
-    parser.add_argument('mo', help="Mo file to generate (implies --po).",
+    parser.add_argument('--mo', help="Mo file to generate (implies --po).",
                         nargs='?')
     args = parser.parse_args()
 
@@ -97,6 +97,7 @@ def main():
                 t = process_po(po, lang)
                 if t:
                     ret = t
+
     return ret
 
 
diff --git a/release/scripts/modules/bl_i18n_utils/update_pot.py b/release/scripts/modules/bl_i18n_utils/update_pot.py
index 6a7efddda6c..c4d9a2ed20a 100755
--- a/release/scripts/modules/bl_i18n_utils/update_pot.py
+++ b/release/scripts/modules/bl_i18n_utils/update_pot.py
@@ -38,6 +38,9 @@ except:
     from . import (settings, utils)
 
 
+LANGUAGES_CATEGORIES = settings.LANGUAGES_CATEGORIES
+LANGUAGES = settings.LANGUAGES
+
 COMMENT_PREFIX = settings.COMMENT_PREFIX
 COMMENT_PREFIX_SOURCE = settings.COMMENT_PREFIX_SOURCE
 CONTEXT_PREFIX = settings.CONTEXT_PREFIX
@@ -261,7 +264,6 @@ def main():
             spell_cache = pickle.load(f)
     else:
         spell_cache = set()
-    print(len(spell_cache))
 
     print("Generating POT file {}…".format(FILE_NAME_POT))
     msgs, states = gen_empty_pot()
@@ -292,11 +294,22 @@ def main():
     print("\tMerged {} messages ({} were already present)."
           "".format(num_added, num_present))
 
+    print("\tAdding languages labels...")
+    messages = {(CONTEXT_DEFAULT, lng[1]):
+                ("Languages’ labels from bl_i18n_utils/settings.py",)
+                for lng in LANGUAGES}
+    messages.update({(CONTEXT_DEFAULT, cat[1]):
+                     ("Language categories’ labels from bl_i18n_utils/settings.py",)
+                     for cat in LANGUAGES_CATEGORIES})
+    num_added, num_present = merge_messages(msgs, states, messages,
+                                            True, spell_cache)
+    tot_messages += num_added
+    print("\tAdded {} language messages.".format(num_added))
+
     # Write back all messages into blender.pot.
     utils.write_messages(FILE_NAME_POT, msgs, states["comm_msg"],
                          states["fuzzy_msg"])
 
-    print(len(spell_cache))
     if SPELL_CACHE and spell_cache:
         with open(SPELL_CACHE, 'wb') as f:
             pickle.dump(spell_cache, f)
diff --git a/release/scripts/modules/bl_i18n_utils/update_trunk.py b/release/scripts/modules/bl_i18n_utils/update_trunk.py
index b7f8f375744..859045eafda 100755
--- a/release/scripts/modules/bl_i18n_utils/update_trunk.py
+++ b/release/scripts/modules/bl_i18n_utils/update_trunk.py
@@ -35,38 +35,62 @@ import shutil
 
 try:
     import settings
+    import utils
 except:
-    from . import settings
+    from . import (settings, utils)
 
+BRANCHES_DIR = settings.BRANCHES_DIR
 TRUNK_PO_DIR = settings.TRUNK_PO_DIR
 TRUNK_MO_DIR = settings.TRUNK_MO_DIR
 
+LANGUAGES_CATEGORIES = settings.LANGUAGES_CATEGORIES
+LANGUAGES = settings.LANGUAGES
+LANGUAGES_FILE = settings.LANGUAGES_FILE
+
 PY3 = settings.PYTHON3_EXEC
 
 
+def find_matching_po(languages, stats):
+    """Match languages defined in LANGUAGES setting to relevant po, if possible!"""
+    ret = []
+    for uid, label, org_key in languages:
+        key = org_key
+        if key not in stats:
+            # Try to simplify the key (eg from es_ES to es).
+            if '_' in org_key:
+                key = org_key[0:org_key.index('_')]
+            if '@' in org_key:
+                key = key + org_key[org_key.index('@'):]
+        if key in stats:
+            ret.append((stats[key], uid, label, org_key))
+        else:
+            # Mark invalid entries, so that we can put them in the languages file,
+            # but commented!
+            ret.append((0.0, -uid, label, org_key))
+    return ret
+
 def main():
     import argparse
-    parser = argparse.ArgumentParser(description="" \
-                        "Update trunk from branches:\n" \
-                        "* Remove po’s in trunk.\n" \
-                        "* Copy po’s from branches advanced enough.\n" \
-                        "* Clean po’s in trunk.\n" \
-                        "* Compile po’s in trunk in mo’s, keeping " \
-                        "track of those failing.\n" \
-                        "* Remove po’s and mo’s (and their dir’s) that " \
-                        "failed to compile or are no more present in trunk.")
-    parser.add_argument('-t', '--threshold', type=int,
-                        help="Import threshold, as a percentage.")
-    parser.add_argument('-p', '--po', action="store_false",
-                        help="Do not remove failing po’s.")
-    parser.add_argument('-m', '--mo', action="store_false",
-                        help="Do not remove failing mo’s.")
-    parser.add_argument('langs', metavar='ISO_code', nargs='*',
-                        help="Restrict processed languages to those.")
+    parser = argparse.ArgumentParser(description=""
+                        "Update trunk from branches:\n"
+                        "* Remove po’s in trunk.\n"
+                        "* Copy po’s from branches advanced enough.\n"
+                        "* Clean po’s in trunk.\n"
+                        "* Compile po’s in trunk in mo’s, keeping "
+                        "track of those failing.\n"
+                        "* Remove po’s and mo’s (and their dir’s) that "
+                        "failed to compile or are no more present in trunk."
+                        "* Generate languages file used by Blender's i18n.")
+    parser.add_argument('-t', '--threshold', type=int, help="Import threshold, as a percentage.")
+    parser.add_argument('-p', '--po', action="store_true", help="Remove failing po’s.")
+    parser.add_argument('-m', '--mo', action="store_true", help="Remove failing mo’s.")
+    parser.add_argument('langs', metavar='ISO_code', nargs='*', help="Restrict processed languages to those.")
     args = parser.parse_args()
 
     ret = 0
     failed = set()
+    # 'DEFAULT' and en_US are always valid, fully-translated "languages"!
+    stats = {"DEFAULT": 1.0, "en_US": 1.0}
 
     # Remove po’s in trunk.
     for po in os.listdir(TRUNK_PO_DIR):
@@ -89,7 +113,7 @@ def main():
 
     # Add in failed all mo’s no more having relevant po’s in trunk.
     for lang in os.listdir(TRUNK_MO_DIR):
-        if lang == ".svn":
+        if lang in {".svn", LANGUAGES_FILE}:
             continue  # !!!
         if not os.path.exists(os.path.join(TRUNK_PO_DIR, ".".join((lang, "po")))):
             failed.add(lang)
@@ -115,6 +139,46 @@ def main():
             if t:
                 ret = t
                 failed.add(lang)
+                continue
+
+            # Yes, I know, it's the third time we parse each po's here. :/
+            u1, u2, _stats = utils.parse_messages(os.path.join(BRANCHES_DIR, lang, po))
+            stats[lang] = _stats["trans_msg"] / _stats["tot_msg"]
+
+    # Generate languages file used by Blender's i18n system.
+    # First, match all entries in LANGUAGES to a lang in stats, if possible!
+    stats = find_matching_po(LANGUAGES, stats)
+    limits = sorted(LANGUAGES_CATEGORIES, key=lambda it: it[0], reverse=True)
+    print(limits)
+    idx = 0
+    stats = sorted(stats, key=lambda it: it[0], reverse=True)
+    print(stats)
+    langs_cats = [[] for i in range(len(limits))]
+    highest_uid = 0
+    for prop, uid, label, key in stats:
+        print(key, prop)
+        if prop < limits[idx][0]:
+            # Sub-sort languages by iso-codes.
+            langs_cats[idx].sort(key=lambda it: it[2])
+            print(langs_cats)
+            idx += 1
+        langs_cats[idx].append((uid, label, key))
+        if abs(uid) > highest_uid:
+            highest_uid = abs(uid)
+    # Sub-sort last group of languages by iso-codes!
+    langs_cats[idx].sort(key=lambda it: it[2])
+    with open(os.path.join(TRUNK_MO_DIR, LANGUAGES_FILE), 'w', encoding="utf-8") as f:
+        f.write("# Highest ID currently in use: {}\n".format(highest_uid))
+        for cat, langs_cat in zip(limits, langs_cats):
+            # Write "category menu label"...
+            f.write("0:{}:\n".format(cat[1]))
+            # ...and all matching language entries!
+            for uid, label, key in langs_cat:
+                if uid < 0:
+                    # Non-existing, commented entry!
+                    f.write("# No translation yet! #{}:{}:{}\n".format(-uid, label, key))
+                else:
+                    f.write("{}:{}:{}\n".format(uid, label, key))
 
     # Remove failing po’s, mo’s and related dir’s.
     for lang in failed: