Welcome to mirror list, hosted at ThFree Co, Russian Federation.

git.blender.org/blender.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBastien Montagne <montagne29@wanadoo.fr>2013-01-12 20:49:06 +0400
committerBastien Montagne <montagne29@wanadoo.fr>2013-01-12 20:49:06 +0400
commit55e3912f257a61ee32d75e7c8489ae4465975c49 (patch)
tree6e9254b95a7cbaf358254e8c18d2cc51c8a8b19e /release
parent68d83f41400468b4c15b2201e8138b60756d1f30 (diff)
Big refactor of bl_i18n_utils, switching to an object-based model. Still a bit wip.
Notes: * Most tools here have been updated, only a few (unused :p ) features should be missing. But some updates are rough for now, and performances are globally worse currently, will address this later (this is only really noticeable when you work over the whole repo, anyway ;) ). * This update breaks "edit translations" addon, will fix it later, once core (i.e. this module) is considered stable again!
Diffstat (limited to 'release')
-rw-r--r--release/scripts/modules/bl_i18n_utils/bl_process_msg.py307
-rwxr-xr-xrelease/scripts/modules/bl_i18n_utils/check_po.py119
-rwxr-xr-xrelease/scripts/modules/bl_i18n_utils/clean_po.py27
-rwxr-xr-xrelease/scripts/modules/bl_i18n_utils/import_po_from_branches.py56
-rwxr-xr-xrelease/scripts/modules/bl_i18n_utils/merge_po.py105
-rw-r--r--release/scripts/modules/bl_i18n_utils/settings.py74
-rwxr-xr-xrelease/scripts/modules/bl_i18n_utils/update_branches.py39
-rwxr-xr-xrelease/scripts/modules/bl_i18n_utils/update_languages_menu.py12
-rwxr-xr-xrelease/scripts/modules/bl_i18n_utils/update_mo.py10
-rwxr-xr-xrelease/scripts/modules/bl_i18n_utils/update_po.py59
-rwxr-xr-xrelease/scripts/modules/bl_i18n_utils/update_pot.py79
-rwxr-xr-xrelease/scripts/modules/bl_i18n_utils/update_trunk.py28
-rw-r--r--release/scripts/modules/bl_i18n_utils/utils.py875
13 files changed, 1097 insertions, 693 deletions
diff --git a/release/scripts/modules/bl_i18n_utils/bl_process_msg.py b/release/scripts/modules/bl_i18n_utils/bl_process_msg.py
index 7e9266d0530..531ad2faf50 100644
--- a/release/scripts/modules/bl_i18n_utils/bl_process_msg.py
+++ b/release/scripts/modules/bl_i18n_utils/bl_process_msg.py
@@ -23,27 +23,41 @@
# You should not directly use this script, rather use update_msg.py!
import os
+import re
+import collections
+import copy
# XXX Relative import does not work here when used from Blender...
from bl_i18n_utils import settings
+import bpy
-#classes = set()
-
+print(dir(settings))
SOURCE_DIR = settings.SOURCE_DIR
CUSTOM_PY_UI_FILES = [os.path.abspath(os.path.join(SOURCE_DIR, p)) for p in settings.CUSTOM_PY_UI_FILES]
FILE_NAME_MESSAGES = settings.FILE_NAME_MESSAGES
-COMMENT_PREFIX = settings.COMMENT_PREFIX
-CONTEXT_PREFIX = settings.CONTEXT_PREFIX
+MSG_COMMENT_PREFIX = settings.MSG_COMMENT_PREFIX
+MSG_CONTEXT_PREFIX = settings.MSG_CONTEXT_PREFIX
CONTEXT_DEFAULT = settings.CONTEXT_DEFAULT
+#CONTEXT_DEFAULT = bpy.app.i18n.contexts.default # XXX Not yet! :)
UNDOC_OPS_STR = settings.UNDOC_OPS_STR
NC_ALLOWED = settings.WARN_MSGID_NOT_CAPITALIZED_ALLOWED
+##### Utils #####
+
+# check for strings like ": %d"
+ignore_reg = re.compile(r"^(?:[-*.()/\\+:%xWXYZ0-9]|%d|%f|%s|%r|\s)*$")
+filter_message = ignore_reg.match
+
+
def check(check_ctxt, messages, key, msgsrc):
+ """
+ Performs a set of checks over the given key (context, message)...
+ """
if check_ctxt is None:
return
multi_rnatip = check_ctxt.get("multi_rnatip")
@@ -73,7 +87,79 @@ def check(check_ctxt, messages, key, msgsrc):
undoc_ops.add(key)
+def print_warnings(check_ctxt, messages):
+ if check_ctxt is not None:
+ print("WARNINGS:")
+ keys = set()
+ for c in check_ctxt.values():
+ keys |= c
+ # XXX Temp, see below
+ keys -= check_ctxt["multi_rnatip"]
+ for key in keys:
+ if key in check_ctxt["undoc_ops"]:
+ print("\tThe following operators are undocumented:")
+ else:
+ print("\t“{}”|“{}”:".format(*key))
+ if key in check_ctxt["multi_lines"]:
+ print("\t\t-> newline in this message!")
+ if key in check_ctxt["not_capitalized"]:
+ print("\t\t-> message not capitalized!")
+ if key in check_ctxt["end_point"]:
+ print("\t\t-> message with endpoint!")
+ # XXX Hide this one for now, too much false positives.
+# if key in check_ctxt["multi_rnatip"]:
+# print("\t\t-> tip used in several RNA items")
+ if key in check_ctxt["py_in_rna"]:
+ print("\t\t-> RNA message also used in py UI code:")
+ print("\t\t{}".format("\n\t\t".join(messages[key])))
+
+
+def enable_addons(addons={}, support={}, disable=False):
+ """
+ Enable (or disable) addons based either on a set of names, or a set of 'support' types.
+ Returns the list of all affected addons (as fake modules)!
+ """
+ import addon_utils
+ import bpy
+
+ userpref = bpy.context.user_preferences
+ used_ext = {ext.module for ext in userpref.addons}
+
+ ret = [mod for mod in addon_utils.modules(addon_utils.addons_fake_modules)
+ if ((addons and mod.__name__ in addons) or
+ (not addons and addon_utils.module_bl_info(mod)["support"] in support))]
+
+ for mod in ret:
+ module_name = mod.__name__
+ if disable:
+ if module_name not in used_ext:
+ continue
+ print(" Disabling module ", module_name)
+ bpy.ops.wm.addon_disable(module=module_name)
+ else:
+ if module_name in used_ext:
+ continue
+ print(" Enabling module ", module_name)
+ bpy.ops.wm.addon_enable(module=module_name)
+
+ # XXX There are currently some problems with bpy/rna...
+ # *Very* tricky to solve!
+ # So this is a hack to make all newly added operator visible by
+ # bpy.types.OperatorProperties.__subclasses__()
+ for cat in dir(bpy.ops):
+ cat = getattr(bpy.ops, cat)
+ for op in dir(cat):
+ getattr(cat, op).get_rna()
+
+ return ret
+
+
+##### RNA #####
+
def dump_messages_rna(messages, check_ctxt):
+ """
+ Dump into messages dict all RNA-defined UI messages (labels en tooltips).
+ """
import bpy
def classBlackList():
@@ -257,20 +343,17 @@ def dump_messages_rna(messages, check_ctxt):
# Parse everything (recursively parsing from bpy_struct "class"...).
processed = process_cls_list(type(bpy.context).__base__.__subclasses__())
print("{} classes processed!".format(processed))
-# import pickle
-# global classes
-# classes = {str(c) for c in classes}
-# with open("/home/i7deb64/Bureau/tpck_2", "wb") as f:
-# pickle.dump(classes, f, protocol=0)
from bpy_extras.keyconfig_utils import KM_HIERARCHY
walk_keymap_hierarchy(KM_HIERARCHY, "KM_HIERARCHY")
-def dump_messages_pytext(messages, check_ctxt):
- """ dumps text inlined in the python user interface: eg.
+##### Python source code #####
+def dump_py_messages_from_files(messages, check_ctxt, files):
+ """
+ Dump text inlined in the python files given, e.g. 'My Name' in:
layout.prop("someprop", text="My Name")
"""
import ast
@@ -278,7 +361,6 @@ def dump_messages_pytext(messages, check_ctxt):
# -------------------------------------------------------------------------
# Gather function names
- import bpy
# key: func_id
# val: [(arg_kw, arg_pos), (arg_kw, arg_pos), ...]
func_translate_args = {}
@@ -290,15 +372,12 @@ def dump_messages_pytext(messages, check_ctxt):
# E.g. we don’t want to get strings inside subscripts (blah["foo"])!
stopper_nodes = {ast.Subscript, }
+ # For now only consider functions from UILayout...
for func_id, func in bpy.types.UILayout.bl_rna.functions.items():
- # check it has a 'text' argument
+ # check it has one or more arguments as defined in translate_kw
for (arg_pos, (arg_kw, arg)) in enumerate(func.parameters.items()):
- if ((arg_kw in translate_kw) and
- (arg.is_output is False) and
- (arg.type == 'STRING')):
-
- func_translate_args.setdefault(func_id, []).append((arg_kw,
- arg_pos))
+ if ((arg_kw in translate_kw) and (arg.is_output is False) and (arg.type == 'STRING')):
+ func_translate_args.setdefault(func_id, []).append((arg_kw, arg_pos))
# print(func_translate_args)
check_ctxt_py = None
@@ -308,19 +387,20 @@ def dump_messages_pytext(messages, check_ctxt):
"not_capitalized": check_ctxt["not_capitalized"],
"end_point": check_ctxt["end_point"]}
- # -------------------------------------------------------------------------
- # Function definitions
-
+ # Helper function
def extract_strings(fp_rel, node):
- """ Recursively get strings, needed in case we have "Blah" + "Blah",
- passed as an argument in that case it wont evaluate to a string.
- However, break on some kind of stopper nodes, like e.g. Subscript.
"""
-
+ Recursively get strings, needed in case we have "Blah" + "Blah", passed as an argument in that case it won't
+ evaluate to a string. However, break on some kind of stopper nodes, like e.g. Subscript.
+ """
if type(node) == ast.Str:
eval_str = ast.literal_eval(node)
if eval_str:
- key = (CONTEXT_DEFAULT, eval_str)
+ # Parse optional context included in string!
+ if bpy.app.i18n.context_sep in eval_str:
+ key = eval_str.split(bpy.app.i18n.context_sep, 1)
+ else:
+ key = (CONTEXT_DEFAULT, eval_str)
msgsrc = "{}:{}".format(fp_rel, node.lineno)
check(check_ctxt_py, messages, key, msgsrc)
messages.setdefault(key, []).append(msgsrc)
@@ -330,10 +410,9 @@ def dump_messages_pytext(messages, check_ctxt):
if type(nd) not in stopper_nodes:
extract_strings(fp_rel, nd)
- def extract_strings_from_file(fp):
- filedata = open(fp, 'r', encoding="utf8")
- root_node = ast.parse(filedata.read(), fp, 'exec')
- filedata.close()
+ for fp in files:
+ with open(fp, 'r', encoding="utf8") as filedata:
+ root_node = ast.parse(filedata.read(), fp, 'exec')
fp_rel = os.path.relpath(fp, SOURCE_DIR)
@@ -361,72 +440,90 @@ def dump_messages_pytext(messages, check_ctxt):
if kw.arg == arg_kw:
extract_strings(fp_rel, kw.value)
- # -------------------------------------------------------------------------
- # Dump Messages
- mod_dir = os.path.join(SOURCE_DIR,
- "release",
- "scripts",
- "startup",
- "bl_ui")
+def dump_py_messages(messages, check_ctxt, addons):
+ mod_dir = os.path.join(SOURCE_DIR, "release", "scripts", "startup", "bl_ui")
- files = [os.path.join(mod_dir, fn)
- for fn in sorted(os.listdir(mod_dir))
- if not fn.startswith("_")
- if fn.endswith("py")
- ]
+ files = [os.path.join(mod_dir, fn) for fn in sorted(os.listdir(mod_dir))
+ if not fn.startswith("_") if fn.endswith("py")]
# Dummy Cycles has its py addon in its own dir!
files += CUSTOM_PY_UI_FILES
- for fp in files:
- extract_strings_from_file(fp)
+ # Add all addons we support in main translation file!
+ for mod in addons:
+ fn = mod.__file__
+ if os.path.basename(fn) == "__init__.py":
+ mod_dir = os.path.dirname(fn)
+ files += [fn for fn in sorted(os.listdir(mod_dir))
+ if os.path.isfile(fn) and os.path.splitext(fn)[1] == ".py"]
+ else:
+ files.append(fn)
+
+ dump_py_messages_from_files(messages, check_ctxt, files)
+##### Main functions! #####
+
def dump_messages(do_messages, do_checks):
- import collections
- import re
+ messages = getattr(collections, 'OrderedDict', dict)()
- def enable_addons():
- """For now, enable all official addons, before extracting msgids."""
- import addon_utils
- import bpy
+ messages[(CONTEXT_DEFAULT, "")] = []
- userpref = bpy.context.user_preferences
- used_ext = {ext.module for ext in userpref.addons}
- support = {"OFFICIAL"}
- # collect the categories that can be filtered on
- addons = [(mod, addon_utils.module_bl_info(mod)) for mod in
- addon_utils.modules(addon_utils.addons_fake_modules)]
+ # Enable all wanted addons.
+ # For now, enable all official addons, before extracting msgids.
+ addons = enable_addons(support={"OFFICIAL"})
- for mod, info in addons:
- module_name = mod.__name__
- if module_name in used_ext or info["support"] not in support:
- continue
- print(" Enabling module ", module_name)
- bpy.ops.wm.addon_enable(module=module_name)
+ check_ctxt = None
+ if do_checks:
+ check_ctxt = {"multi_rnatip": set(),
+ "multi_lines": set(),
+ "py_in_rna": set(),
+ "not_capitalized": set(),
+ "end_point": set(),
+ "undoc_ops": set()}
+
+ # get strings from RNA
+ dump_messages_rna(messages, check_ctxt)
- # XXX There are currently some problems with bpy/rna...
- # *Very* tricky to solve!
- # So this is a hack to make all newly added operator visible by
- # bpy.types.OperatorProperties.__subclasses__()
- for cat in dir(bpy.ops):
- cat = getattr(bpy.ops, cat)
- for op in dir(cat):
- getattr(cat, op).get_rna()
+ # get strings from UI layout definitions text="..." args
+ dump_py_messages(messages, check_ctxt, addons)
+
+ del messages[(CONTEXT_DEFAULT, "")]
+
+ print_warnings(check_ctxt, messages)
+
+ if do_messages:
+ print("Writing messages…")
+ num_written = 0
+ num_filtered = 0
+ with open(FILE_NAME_MESSAGES, 'w', encoding="utf8") as message_file:
+ for (ctx, key), value in messages.items():
+ # filter out junk values
+ if filter_message(key):
+ num_filtered += 1
+ continue
+
+ # Remove newlines in key and values!
+ message_file.write("\n".join(MSG_COMMENT_PREFIX + msgsrc.replace("\n", "") for msgsrc in value))
+ message_file.write("\n")
+ if ctx:
+ message_file.write(MSG_CONTEXT_PREFIX + ctx.replace("\n", "") + "\n")
+ message_file.write(key.replace("\n", "") + "\n")
+ num_written += 1
+
+ print("Written {} messages to: {} ({} were filtered out)."
+ "".format(num_written, FILE_NAME_MESSAGES, num_filtered))
- # check for strings like ": %d"
- ignore_reg = re.compile(r"^(?:[-*.()/\\+:%xWXYZ0-9]|%d|%f|%s|%r|\s)*$")
- filter_message = ignore_reg.match
+def dump_addon_messages(module_name, messages_formats, do_checks):
messages = getattr(collections, 'OrderedDict', dict)()
messages[(CONTEXT_DEFAULT, "")] = []
-
- # Enable all wanted addons.
- enable_addons()
+ minus_messages = copy.deepcopy(messages)
check_ctxt = None
+ minus_check_ctxt = None
if do_checks:
check_ctxt = {"multi_rnatip": set(),
"multi_lines": set(),
@@ -434,39 +531,44 @@ def dump_messages(do_messages, do_checks):
"not_capitalized": set(),
"end_point": set(),
"undoc_ops": set()}
+ minus_check_ctxt = copy.deepcopy(check_ctxt)
- # get strings from RNA
+ # Get current addon state (loaded or not):
+ was_loaded = addon_utils.check(module_name)[1]
+
+ # Enable our addon and get strings from RNA.
+ enable_addons(addons={module_name})
dump_messages_rna(messages, check_ctxt)
+ # Now disable our addon, and rescan RNA.
+ enable_addons(addons={module_name}, disable=True)
+ dump_messages_rna(minus_messages, minus_check_ctxt)
+
+ # Restore previous state if needed!
+ if was_loaded:
+ enable_addons(addons={module_name})
+
+ # and make the diff!
+ for key in minus_messages:
+ if k == (CONTEXT_DEFAULT, ""):
+ continue
+ del messages[k]
+
+ if check_ctxt:
+ for key in check_ctxt:
+ for warning in minus_check_ctxt[key]:
+ check_ctxt[key].remove(warning)
+
+ # and we are done with those!
+ del minus_messages
+ del minus_check_ctxt
+
# get strings from UI layout definitions text="..." args
dump_messages_pytext(messages, check_ctxt)
del messages[(CONTEXT_DEFAULT, "")]
- if do_checks:
- print("WARNINGS:")
- keys = set()
- for c in check_ctxt.values():
- keys |= c
- # XXX Temp, see below
- keys -= check_ctxt["multi_rnatip"]
- for key in keys:
- if key in check_ctxt["undoc_ops"]:
- print("\tThe following operators are undocumented:")
- else:
- print("\t“{}”|“{}”:".format(*key))
- if key in check_ctxt["multi_lines"]:
- print("\t\t-> newline in this message!")
- if key in check_ctxt["not_capitalized"]:
- print("\t\t-> message not capitalized!")
- if key in check_ctxt["end_point"]:
- print("\t\t-> message with endpoint!")
- # XXX Hide this one for now, too much false positives.
-# if key in check_ctxt["multi_rnatip"]:
-# print("\t\t-> tip used in several RNA items")
- if key in check_ctxt["py_in_rna"]:
- print("\t\t-> RNA message also used in py UI code:")
- print("\t\t{}".format("\n\t\t".join(messages[key])))
+ print_warnings
if do_messages:
print("Writing messages…")
@@ -491,6 +593,7 @@ def dump_messages(do_messages, do_checks):
"".format(num_written, FILE_NAME_MESSAGES, num_filtered))
+
def main():
try:
import bpy
diff --git a/release/scripts/modules/bl_i18n_utils/check_po.py b/release/scripts/modules/bl_i18n_utils/check_po.py
index 2e82047bb95..96f91ee0914 100755
--- a/release/scripts/modules/bl_i18n_utils/check_po.py
+++ b/release/scripts/modules/bl_i18n_utils/check_po.py
@@ -38,45 +38,47 @@ BRANCHES_DIR = settings.BRANCHES_DIR
FILE_NAME_POT = settings.FILE_NAME_POT
-def print_diff(ref_messages, messages, states):
+def print_diff(ref_msgs, msgs):
# Remove comments from messages list!
- messages = set(messages.keys()) - states["comm_msg"]
- unneeded = (messages - ref_messages)
- for msgid in unneeded:
- print('\tUnneeded message id "{}"'.format(msgid))
+ messages = set(msgs.msgs.keys()) - msgs.comm_msgs
+ unneeded = (messages - ref_msgs.msgs.keys())
+ for msgkey in unneeded:
+ print('\tUnneeded message context/id "{}"'.format(msgkey))
- missing = (ref_messages - messages)
- for msgid in missing:
- print('\tMissing message id "{}"'.format(msgid))
+ missing = (ref_msgs.msgs.keys() - messages)
+ for msgkey in missing:
+ print('\tMissing message context/id "{}"'.format(msgkey))
- for msgid in states["comm_msg"]:
- print('\tCommented message id "{}"'.format(msgid))
+ for msgid in msgs.comm_msgs:
+ print('\tCommented message context/id "{}"'.format(msgkey))
- print("\t{} unneeded messages, {} missing messages, {} commented messages." \
- "".format(len(unneeded), len(missing), len(states["comm_msg"])))
+ print("\t{} unneeded messages, {} missing messages, {} commented messages."
+ "".format(len(unneeded), len(missing), len(msgs.comm_msgs)))
return 0
-def process_po(ref_messages, po, glob_stats, do_stats, do_messages):
+#def process_po(ref_messages, po, glob_stats, do_stats, do_messages):
+def process_po(ref_messages, po, do_stats, do_messages):
print("Checking {}...".format(po))
ret = 0
- messages, states, stats = utils.parse_messages(po)
+ messages = utils.I18nMessages(kind='PO', src=po)
if do_messages:
- t = print_diff(ref_messages, messages, states)
+ t = print_diff(ref_messages, messages)
if t:
ret = t
if do_stats:
print("\tStats:")
- t = utils.print_stats(stats, glob_stats, prefix=" ")
+ t = messages.print_stats(prefix=" ")
if t:
ret = t
- if states["is_broken"]:
+ if messages.parsing_errors:
print("\tERROR! This .po is broken!")
ret = 1
return ret
+# XXX Quick update for new I18Nfoo objects, need rework!
def main():
import argparse
parser = argparse.ArgumentParser(description="Check po’s in branches " \
@@ -97,22 +99,21 @@ def main():
if args.pot:
global FILE_NAME_POT
FILE_NAME_POT = args.pot
- glob_stats = {"nbr" : 0.0,
- "lvl" : 0.0,
- "lvl_ttips" : 0.0,
- "lvl_trans_ttips" : 0.0,
- "lvl_ttips_in_trans": 0.0,
- "lvl_comm" : 0.0,
- "nbr_signs" : 0,
- "nbr_trans_signs" : 0,
- "contexts" : set()}
+ #glob_stats = {"nbr" : 0.0,
+ #"lvl" : 0.0,
+ #"lvl_ttips" : 0.0,
+ #"lvl_trans_ttips" : 0.0,
+ #"lvl_ttips_in_trans": 0.0,
+ #"lvl_comm" : 0.0,
+ #"nbr_signs" : 0,
+ #"nbr_trans_signs" : 0,
+ #"contexts" : set()}
ret = 0
pot_messages = None
if args.messages:
- pot_messages, u1, pot_stats = utils.parse_messages(FILE_NAME_POT)
- pot_messages = set(pot_messages.keys())
- glob_stats["nbr_signs"] = pot_stats["nbr_signs"]
+ pot_messages = utils.I18nMessages(kind='PO', src=FILE_NAME_POT)
+ #glob_stats["nbr_signs"] = pot_stats["nbr_signs"]
if args.langs:
for lang in args.langs:
@@ -121,16 +122,16 @@ def main():
else:
po = os.path.join(BRANCHES_DIR, lang, ".".join((lang, "po")))
if os.path.exists(po):
- t = process_po(pot_messages, po, glob_stats,
- args.stats, args.messages)
+ #t = process_po(pot_messages, po, glob_stats, args.stats, args.messages)
+ t = process_po(pot_messages, po, args.stats, args.messages)
if t:
ret = t
elif args.trunk:
for po in os.listdir(TRUNK_PO_DIR):
if po.endswith(".po"):
po = os.path.join(TRUNK_PO_DIR, po)
- t = process_po(pot_messages, po, glob_stats,
- args.stats, args.messages)
+ #t = process_po(pot_messages, po, glob_stats, args.stats, args.messages)
+ t = process_po(pot_messages, po, args.stats, args.messages)
if t:
ret = t
else:
@@ -138,35 +139,35 @@ def main():
for po in os.listdir(os.path.join(BRANCHES_DIR, lang)):
if po.endswith(".po"):
po = os.path.join(BRANCHES_DIR, lang, po)
- t = process_po(pot_messages, po, glob_stats,
- args.stats, args.messages)
+ #t = process_po(pot_messages, po, glob_stats, args.stats, args.messages)
+ t = process_po(pot_messages, po, args.stats, args.messages)
if t:
ret = t
- if args.stats and glob_stats["nbr"] != 0.0:
- nbr_contexts = len(glob_stats["contexts"] - {""})
- if nbr_contexts != 1:
- if nbr_contexts == 0:
- nbr_contexts = "No"
- _ctx_txt = "s are"
- else:
- _ctx_txt = " is"
- print("\nAverage stats for all {:.0f} processed files:\n"
- " {:>6.1%} done!\n"
- " {:>6.1%} of messages are tooltips.\n"
- " {:>6.1%} of tooltips are translated.\n"
- " {:>6.1%} of translated messages are tooltips.\n"
- " {:>6.1%} of messages are commented.\n"
- " The org msgids are currently made of {} signs.\n"
- " All processed translations are currently made of {} signs.\n"
- " {} specific context{} present:\n {}\n"
- "".format(glob_stats["nbr"], glob_stats["lvl"] / glob_stats["nbr"],
- glob_stats["lvl_ttips"] / glob_stats["nbr"],
- glob_stats["lvl_trans_ttips"] / glob_stats["nbr"],
- glob_stats["lvl_ttips_in_trans"] / glob_stats["nbr"],
- glob_stats["lvl_comm"] / glob_stats["nbr"], glob_stats["nbr_signs"],
- glob_stats["nbr_trans_signs"], nbr_contexts, _ctx_txt,
- "\n ".join(glob_stats["contexts"]-{""})))
+ #if args.stats and glob_stats["nbr"] != 0.0:
+ #nbr_contexts = len(glob_stats["contexts"] - {""})
+ #if nbr_contexts != 1:
+ #if nbr_contexts == 0:
+ #nbr_contexts = "No"
+ #_ctx_txt = "s are"
+ #else:
+ #_ctx_txt = " is"
+ #print("\nAverage stats for all {:.0f} processed files:\n"
+ #" {:>6.1%} done!\n"
+ #" {:>6.1%} of messages are tooltips.\n"
+ #" {:>6.1%} of tooltips are translated.\n"
+ #" {:>6.1%} of translated messages are tooltips.\n"
+ #" {:>6.1%} of messages are commented.\n"
+ #" The org msgids are currently made of {} signs.\n"
+ #" All processed translations are currently made of {} signs.\n"
+ #" {} specific context{} present:\n {}\n"
+ #"".format(glob_stats["nbr"], glob_stats["lvl"] / glob_stats["nbr"],
+ #glob_stats["lvl_ttips"] / glob_stats["nbr"],
+ #glob_stats["lvl_trans_ttips"] / glob_stats["nbr"],
+ #glob_stats["lvl_ttips_in_trans"] / glob_stats["nbr"],
+ #glob_stats["lvl_comm"] / glob_stats["nbr"], glob_stats["nbr_signs"],
+ #glob_stats["nbr_trans_signs"], nbr_contexts, _ctx_txt,
+ #"\n ".join(glob_stats["contexts"]-{""})))
return ret
diff --git a/release/scripts/modules/bl_i18n_utils/clean_po.py b/release/scripts/modules/bl_i18n_utils/clean_po.py
index 2924ad9fb74..da8d25cb9f4 100755
--- a/release/scripts/modules/bl_i18n_utils/clean_po.py
+++ b/release/scripts/modules/bl_i18n_utils/clean_po.py
@@ -39,30 +39,27 @@ BRANCHES_DIR = settings.BRANCHES_DIR
def do_clean(po, strict):
print("Cleaning {}...".format(po))
- messages, states, u1 = utils.parse_messages(po)
+ msgs = utils.I18nMessages(kind='PO', src=po)
- if strict and states["is_broken"]:
+ if strict and msgs.parsing_errors:
print("ERROR! This .po file is broken!")
return 1
- for msgkey in states["comm_msg"]:
- del messages[msgkey]
- utils.write_messages(po, messages, states["comm_msg"], states["fuzzy_msg"])
- print("Removed {} commented messages.".format(len(states["comm_msg"])))
+ nbr_rem = len(msgs.comm_msgs)
+ for msgkey in msgs.comm_msgs:
+ del msgs.msgs[msgkey]
+ msgs.write(kind='PO', dest=po)
+ print("Removed {} commented messages.".format(nbr_rem))
return 0
def main():
import argparse
- parser = argparse.ArgumentParser(description="Clean po’s in branches " \
- "or trunk (i.e. remove " \
- "all commented messages).")
- parser.add_argument('-t', '--trunk', action="store_true",
- help="Clean po’s in trunk rather than branches.")
- parser.add_argument('-s', '--strict', action="store_true",
- help="Raise an error if a po is broken.")
- parser.add_argument('langs', metavar='ISO_code', nargs='*',
- help="Restrict processed languages to those.")
+ parser = argparse.ArgumentParser(description="Clean po’s in branches or trunk (i.e. remove all commented "
+ "messages).")
+ parser.add_argument('-t', '--trunk', action="store_true", help="Clean po’s in trunk rather than branches.")
+ parser.add_argument('-s', '--strict', action="store_true", help="Raise an error if a po is broken.")
+ parser.add_argument('langs', metavar='ISO_code', nargs='*', help="Restrict processed languages to those.")
args = parser.parse_args()
ret = 0
diff --git a/release/scripts/modules/bl_i18n_utils/import_po_from_branches.py b/release/scripts/modules/bl_i18n_utils/import_po_from_branches.py
index 533dded3c57..f42b45e7d4e 100755
--- a/release/scripts/modules/bl_i18n_utils/import_po_from_branches.py
+++ b/release/scripts/modules/bl_i18n_utils/import_po_from_branches.py
@@ -26,7 +26,6 @@ import os
import shutil
import sys
import subprocess
-from codecs import open
try:
import settings
@@ -48,14 +47,10 @@ PY3 = settings.PYTHON3_EXEC
def main():
import argparse
- parser = argparse.ArgumentParser(description="Import advanced enough po’s " \
- "from branches to trunk.")
- parser.add_argument('-t', '--threshold', type=int,
- help="Import threshold, as a percentage.")
- parser.add_argument('-s', '--strict', action="store_true",
- help="Raise an error if a po is broken.")
- parser.add_argument('langs', metavar='ISO_code', nargs='*',
- help="Restrict processed languages to those.")
+ parser = argparse.ArgumentParser(description="Import advanced enough po’s from branches to trunk.")
+ parser.add_argument('-t', '--threshold', type=float, help="Import threshold, as a percentage.")
+ parser.add_argument('-s', '--strict', action="store_true", help="Raise an error if a po is broken.")
+ parser.add_argument('langs', metavar='ISO_code', nargs='*', help="Restrict processed languages to those.")
args = parser.parse_args()
ret = 0
@@ -70,51 +65,40 @@ def main():
po = os.path.join(BRANCHES_DIR, lang, ".".join((lang, "po")))
if os.path.exists(po):
po_is_rtl = os.path.join(BRANCHES_DIR, lang, RTL_PREPROCESS_FILE)
- msgs, state, stats = utils.parse_messages(po)
- tot_msgs = stats["tot_msg"]
- trans_msgs = stats["trans_msg"]
+ msgs = utils.I18nMessages(iso=lang, kind='PO', src=po)
lvl = 0.0
- if tot_msgs:
- lvl = float(trans_msgs) / float(tot_msgs)
+ if msgs.nbr_msgs:
+ lvl = msgs.nbr_trans_msgs / msgs.nbr_msgs
if lvl > threshold:
- if state["is_broken"] and args.strict:
- print("{:<10}: {:>6.1%} done, but BROKEN, skipped." \
- "".format(lang, lvl))
+ if msgs.parsing_errors and args.strict:
+ print("{:<10}: {:>6.1%} done, but BROKEN, skipped.".format(lang, lvl))
ret = 1
else:
if os.path.exists(po_is_rtl):
- out_po = os.path.join(TRUNK_PO_DIR,
- ".".join((lang, "po")))
- out_raw_po = os.path.join(TRUNK_PO_DIR,
- "_".join((lang, "raw.po")))
+ out_po = os.path.join(TRUNK_PO_DIR, ".".join((lang, "po")))
+ out_raw_po = os.path.join(TRUNK_PO_DIR, "_".join((lang, "raw.po")))
keys = []
trans = []
- for k, m in msgs.items():
+ for k, m in msgs.msgs.items():
keys.append(k)
- trans.append("".join(m["msgstr_lines"]))
+ trans.append(m.msgstr)
trans = rtl_preprocess.log2vis(trans)
for k, t in zip(keys, trans):
- # Mono-line for now...
- msgs[k]["msgstr_lines"] = [t]
- utils.write_messages(out_po, msgs, state["comm_msg"],
- state["fuzzy_msg"])
+ msgs[k].msgstr = t
+ msgs.write(kinf='PO', dest=out_po)
# Also copies org po!
shutil.copy(po, out_raw_po)
- print("{:<10}: {:>6.1%} done, enough translated " \
- "messages, processed and copied to trunk." \
+ print("{:<10}: {:>6.1%} done, enough translated messages, processed and copied to trunk."
"".format(lang, lvl))
else:
shutil.copy(po, TRUNK_PO_DIR)
- print("{:<10}: {:>6.1%} done, enough translated " \
- "messages, copied to trunk.".format(lang, lvl))
+ print("{:<10}: {:>6.1%} done, enough translated messages, copied to trunk.".format(lang, lvl))
else:
- if state["is_broken"] and args.strict:
- print("{:<10}: {:>6.1%} done, BROKEN and not enough " \
- "translated messages, skipped".format(lang, lvl))
+ if msgs.parsing_errors and args.strict:
+ print("{:<10}: {:>6.1%} done, BROKEN and not enough translated messages, skipped".format(lang, lvl))
ret = 1
else:
- print("{:<10}: {:>6.1%} done, not enough translated " \
- "messages, skipped.".format(lang, lvl))
+ print("{:<10}: {:>6.1%} done, not enough translated messages, skipped.".format(lang, lvl))
return ret
diff --git a/release/scripts/modules/bl_i18n_utils/merge_po.py b/release/scripts/modules/bl_i18n_utils/merge_po.py
index baf67de2e81..610be0f15bd 100755
--- a/release/scripts/modules/bl_i18n_utils/merge_po.py
+++ b/release/scripts/modules/bl_i18n_utils/merge_po.py
@@ -37,26 +37,22 @@ except:
from . import (settings, utils)
+# XXX This is a quick hack to make it work with new I18n... objects! To be reworked!
def main():
import argparse
- parser = argparse.ArgumentParser(description="" \
- "Merge one or more .po files into the first dest one.\n" \
- "If a msgkey (msgctxt, msgid) is present in more than " \
- "one merged po, the one in the first file wins, unless " \
- "it’s marked as fuzzy and one later is not.\n" \
- "The fuzzy flag is removed if necessary.\n" \
- "All other comments are never modified.\n" \
- "Commented messages in dst will always remain " \
- "commented, and commented messages are never merged " \
+ parser = argparse.ArgumentParser(description=""
+ "Merge one or more .po files into the first dest one.\n"
+ "If a msgkey (msgctxt, msgid) is present in more than one merged po, the one in the first file "
+ "wins, unless it’s marked as fuzzy and one later is not.\n"
+ "The fuzzy flag is removed if necessary.\n"
+ "All other comments are never modified.\n"
+ "Commented messages in dst will always remain commented, and commented messages are never merged "
"from sources.")
- parser.add_argument('-s', '--stats', action="store_true",
- help="Show statistics info.")
+ parser.add_argument('-s', '--stats', action="store_true", help="Show statistics info.")
parser.add_argument('-r', '--replace', action="store_true",
help="Replace existing messages of same \"level\" already in dest po.")
- parser.add_argument('dst', metavar='dst.po',
- help="The dest po into which merge the others.")
- parser.add_argument('src', metavar='src.po', nargs='+',
- help="The po's to merge into the dst.po one.")
+ parser.add_argument('dst', metavar='dst.po', help="The dest po into which merge the others.")
+ parser.add_argument('src', metavar='src.po', nargs='+', help="The po's to merge into the dst.po one.")
args = parser.parse_args()
ret = 0
@@ -67,89 +63,78 @@ def main():
nbr_added = 0
nbr_unfuzzied = 0
- dst_messages, dst_states, dst_stats = utils.parse_messages(args.dst)
- if dst_states["is_broken"]:
+ dst_msgs = utils.I18nMessages(kind='PO', src=args.dst)
+ if dst_msgs.parsing_errors:
print("Dest po is BROKEN, aborting.")
return 1
if args.stats:
print("Dest po, before merging:")
- utils.print_stats(dst_stats, prefix="\t")
- # If we don’t want to replace existing valid translations, pre-populate
- # done_msgkeys and done_fuzzy_msgkeys.
+ dst_msgs.print_stats(prefix="\t")
+ # If we don’t want to replace existing valid translations, pre-populate done_msgkeys and done_fuzzy_msgkeys.
if not args.replace:
- done_msgkeys = dst_states["trans_msg"].copy()
- done_fuzzy_msgkeys = dst_states["fuzzy_msg"].copy()
+ done_msgkeys = dst_msgs.trans_msgs.copy()
+ done_fuzzy_msgkeys = dst_msgs.fuzzy_msgs.copy()
for po in args.src:
- messages, states, stats = utils.parse_messages(po)
- if states["is_broken"]:
+ msgs = utils.I18nMessages(kind='PO', src=po)
+ if msgs.parsing_errors:
print("\tSrc po {} is BROKEN, skipping.".format(po))
ret = 1
continue
print("\tMerging {}...".format(po))
if args.stats:
print("\t\tMerged po stats:")
- utils.print_stats(stats, prefix="\t\t\t")
- for msgkey, val in messages.items():
+ msgs.print_stats(prefix="\t\t\t")
+ for msgkey, msg in msgs.msgs.items():
msgctxt, msgid = msgkey
# This msgkey has already been completely merged, or is a commented one,
# or the new message is commented, skip it.
- if msgkey in (done_msgkeys | dst_states["comm_msg"] | states["comm_msg"]):
+ if msgkey in (done_msgkeys | dst_msgs.comm_msgs | msgs.comm_msgs):
continue
- is_ttip = utils.is_tooltip(msgid)
+ is_ttip = msg.is_tooltip
# New messages does not yet exists in dest.
- if msgkey not in dst_messages:
- dst_messages[msgkey] = messages[msgkey]
- if msgkey in states["fuzzy_msg"]:
+ if msgkey not in dst_msgs.msgs:
+ dst_msgs[msgkey] = msgs.msgs[msgkey]
+ if msgkey in msgs.fuzzy_msgs:
done_fuzzy_msgkeys.add(msgkey)
- dst_states["fuzzy_msg"].add(msgkey)
- elif msgkey in states["trans_msg"]:
+ dst_msgs.fuzzy_msgs.add(msgkey)
+ elif msgkey in msgs.trans_msgs:
done_msgkeys.add(msgkey)
- dst_states["trans_msg"].add(msgkey)
- dst_stats["trans_msg"] += 1
- if is_ttip:
- dst_stats["trans_ttips"] += 1
+ dst_msgs.trans_msgs.add(msgkey)
nbr_added += 1
- dst_stats["tot_msg"] += 1
- if is_ttip:
- dst_stats["tot_ttips"] += 1
# From now on, the new messages is already in dst.
# New message is neither translated nor fuzzy, skip it.
- elif msgkey not in (states["trans_msg"] | states["fuzzy_msg"]):
+ elif msgkey not in (msgs.trans_msgs | msgs.fuzzy_msgs):
continue
# From now on, the new message is either translated or fuzzy!
# The new message is translated.
- elif msgkey in states["trans_msg"]:
- dst_messages[msgkey]["msgstr_lines"] = messages[msgkey]["msgstr_lines"]
+ elif msgkey in msgs.trans_msgs:
+ dst_msgs.msgs[msgkey].msgstr = msg.msgstr
done_msgkeys.add(msgkey)
done_fuzzy_msgkeys.discard(msgkey)
- if msgkey in dst_states["fuzzy_msg"]:
- dst_states["fuzzy_msg"].remove(msgkey)
+ if msgkey in dst_msgs.fuzzy_msgs:
+ dst_msgs.fuzzy_msgs.remove(msgkey)
nbr_unfuzzied += 1
- if msgkey not in dst_states["trans_msg"]:
- dst_states["trans_msg"].add(msgkey)
- dst_stats["trans_msg"] += 1
- if is_ttip:
- dst_stats["trans_ttips"] += 1
+ if msgkey not in dst_msgs.trans_msgs:
+ dst_msgs.trans_msgs.add(msgkey)
else:
nbr_replaced += 1
nbr_merged += 1
- # The new message is fuzzy, org one is fuzzy too,
- # and this msgkey has not yet been merged.
- elif msgkey not in (dst_states["trans_msg"] | done_fuzzy_msgkeys):
- dst_messages[msgkey]["msgstr_lines"] = messages[msgkey]["msgstr_lines"]
+ # The new message is fuzzy, org one is fuzzy too, and this msgkey has not yet been merged.
+ elif msgkey not in (dst_msgs.trans_msgs | done_fuzzy_msgkeys):
+ dst_msgs[msgkey].msgstr = msg.msgstr
done_fuzzy_msgkeys.add(msgkey)
- dst_states["fuzzy_msg"].add(msgkey)
+ dst_msgs.fuzzy_msgs.add(msgkey)
nbr_merged += 1
nbr_replaced += 1
- utils.write_messages(args.dst, dst_messages, dst_states["comm_msg"], dst_states["fuzzy_msg"])
+ dst_msgs.write(kind='PO', dest=args.dst)
- print("Merged completed. {} messages were merged (among which {} were replaced), " \
- "{} were added, {} were \"un-fuzzied\"." \
- "".format(nbr_merged, nbr_replaced, nbr_added, nbr_unfuzzied))
+ print("Merged completed. {} messages were merged (among which {} were replaced), {} were added, "
+ "{} were \"un-fuzzied\".".format(nbr_merged, nbr_replaced, nbr_added, nbr_unfuzzied))
if args.stats:
+ dst_msgs.update_info()
print("Final merged po stats:")
- utils.print_stats(dst_stats, prefix="\t")
+ dst_msgs.print_stats(prefix="\t")
return ret
diff --git a/release/scripts/modules/bl_i18n_utils/settings.py b/release/scripts/modules/bl_i18n_utils/settings.py
index 0f09e8238ab..eb5fd4cd0fa 100644
--- a/release/scripts/modules/bl_i18n_utils/settings.py
+++ b/release/scripts/modules/bl_i18n_utils/settings.py
@@ -96,13 +96,62 @@ IMPORT_MIN_LEVEL = -1
IMPORT_LANGUAGES_SKIP = {'am', 'bg', 'fi', 'el', 'et', 'ko', 'ne', 'pl', 'ro', 'uz', 'uz@cyrillic'}
# The comment prefix used in generated messages.txt file.
-COMMENT_PREFIX = "#~ "
+MSG_COMMENT_PREFIX = "#~ "
+
+# The comment prefix used in generated messages.txt file.
+MSG_CONTEXT_PREFIX = "MSGCTXT:"
+
+# The default comment prefix used in po's.
+PO_COMMENT_PREFIX= "# "
# The comment prefix used to mark sources of msgids, in po's.
-COMMENT_PREFIX_SOURCE = "#: "
+PO_COMMENT_PREFIX_SOURCE = "#: "
-# The comment prefix used in generated messages.txt file.
-CONTEXT_PREFIX = "MSGCTXT:"
+# The comment prefix used to mark sources of msgids, in po's.
+PO_COMMENT_PREFIX_SOURCE_CUSTOM = "#. :src: "
+
+# The comment prefix used to comment entries in po's.
+PO_COMMENT_PREFIX_MSG= "#~ "
+
+# The comment prefix used to mark fuzzy msgids, in po's.
+PO_COMMENT_FUZZY = "#, fuzzy"
+
+# The prefix used to define context, in po's.
+PO_MSGCTXT = "msgctxt "
+
+# The prefix used to define msgid, in po's.
+PO_MSGID = "msgid "
+
+# The prefix used to define msgstr, in po's.
+PO_MSGSTR = "msgstr "
+
+# The 'header' key of po files.
+PO_HEADER_KEY = ("", "")
+
+PO_HEADER_MSGSTR = (
+ "Project-Id-Version: Blender {blender_ver} (r{blender_rev})\\n\n"
+ "Report-Msgid-Bugs-To: \\n\n"
+ "POT-Creation-Date: {time}\\n\n"
+ "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\\n\n"
+ "Last-Translator: FULL NAME <EMAIL@ADDRESS>\\n\n"
+ "Language-Team: LANGUAGE <LL@li.org>\\n\n"
+ "Language: {iso}\\n\n"
+ "MIME-Version: 1.0\\n\n"
+ "Content-Type: text/plain; charset=UTF-8\\n\n"
+ "Content-Transfer-Encoding: 8bit\n"
+)
+PO_HEADER_COMMENT_COPYRIGHT = (
+ "# Blender's translation file (po format).\n"
+ "# Copyright (C) {year} The Blender Foundation.\n"
+ "# This file is distributed under the same license as the Blender package.\n"
+ "#\n"
+)
+PO_HEADER_COMMENT = (
+ "# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.\n"
+ "#"
+)
+
+TEMPLATE_ISO_ID = "__TEMPLATE__"
# Default context.
CONTEXT_DEFAULT = ""
@@ -121,8 +170,7 @@ PYGETTEXT_ALLOWED_EXTS = {".c", ".cpp", ".cxx", ".hpp", ".hxx", ".h"}
PYGETTEXT_MAX_MULTI_CTXT = 16
# Where to search contexts definitions, relative to SOURCE_DIR (defined below).
-PYGETTEXT_CONTEXTS_DEFSRC = os.path.join("source", "blender", "blenfont",
- "BLF_translation.h")
+PYGETTEXT_CONTEXTS_DEFSRC = os.path.join("source", "blender", "blenfont", "BLF_translation.h")
# Regex to extract contexts defined in BLF_translation.h
# XXX Not full-proof, but should be enough here!
@@ -146,11 +194,21 @@ _str_base = (
"(?P={_}2)" # And closing quote.
)
str_clean_re = _str_base.format(_="g", capt="P<clean>")
+_inbetween_str_re = (
+ # XXX Strings may have comments between their pieces too, not only spaces!
+ r"(?:\s*(?:"
+ # A C comment
+ r"/\*.*(?!\*/).\*/|"
+ # Or a C++ one!
+ r"//[^\n]*\n"
+ # And we are done!
+ r")?)*"
+)
# Here we have to consider two different cases (empty string and other).
_str_whole_re = (
_str_base.format(_="{_}1_", capt=":") +
# Optional loop start, this handles "split" strings...
- "(?:(?<=[\"'])\\s*(?=[\"'])(?:"
+ "(?:(?<=[\"'])" + _inbetween_str_re + "(?=[\"'])(?:"
+ _str_base.format(_="{_}2_", capt=":") +
# End of loop.
"))*"
@@ -231,6 +289,8 @@ WARN_MSGID_NOT_CAPITALIZED_ALLOWED = {
}
WARN_MSGID_NOT_CAPITALIZED_ALLOWED |= set(lng[2] for lng in LANGUAGES)
+PARSER_CACHE_HASH = 'sha1'
+
###############################################################################
# PATHS
diff --git a/release/scripts/modules/bl_i18n_utils/update_branches.py b/release/scripts/modules/bl_i18n_utils/update_branches.py
index 4c38a2f71fb..1a22a9835dd 100755
--- a/release/scripts/modules/bl_i18n_utils/update_branches.py
+++ b/release/scripts/modules/bl_i18n_utils/update_branches.py
@@ -43,42 +43,31 @@ FILE_NAME_POT = settings.FILE_NAME_POT
def main():
import argparse
- parser = argparse.ArgumentParser(description="" \
- "Update all branches:\n" \
- "* Generate a temp messages.txt file.\n" \
- "* Use it to generate a temp .pot file.\n" \
- "* Use it to update all .po’s in /branches.")
- parser.add_argument('--pproc-contexts', action="store_true",
- help="Pre-process po’s to avoid having plenty of "
- "fuzzy msgids just because a context was "
- "added/changed!")
- parser.add_argument('-c', '--no_checks', default=True,
- action="store_false",
- help="No checks over UI messages.")
+ parser = argparse.ArgumentParser(description="Update all branches:\n"
+ "* Generate a temp messages.txt file.\n"
+ "* Use it to generate a blender.pot file.\n"
+ "* Use it to update all .po’s in /branches.")
+ #parser.add_argument('--pproc-contexts', action="store_true",
+ #help="Pre-process po’s to avoid having plenty of fuzzy msgids just because a context was "
+ #"added/changed!")
+ parser.add_argument('-c', '--no_checks', default=True, action="store_false", help="No checks over UI messages.")
parser.add_argument('-a', '--add', action="store_true",
- help="Add missing po’s (useful only when one or " \
- "more languages are given!).")
- parser.add_argument('langs', metavar='ISO_code', nargs='*',
- help="Restrict processed languages to those.")
+ help="Add missing po’s (useful only when one or more languages are given!).")
+ parser.add_argument('langs', metavar='ISO_code', nargs='*', help="Restrict processed languages to those.")
args = parser.parse_args()
ret = 0
# Generate a temp messages file.
- dummy, msgfile = tempfile.mkstemp(suffix=".txt",
- prefix="blender_messages_")
+ dummy, msgfile = tempfile.mkstemp(suffix=".txt", prefix="blender_messages_")
os.close(dummy)
cmd = (PY3, "./update_msg.py", "-o", msgfile)
t = subprocess.call(cmd)
if t:
ret = t
- # Generate a temp pot file.
- # Back to having a pot file in trunk/po. It's quite useful for translators that want to start
+ # Generate blender.pot file in trunk/po. It's quite useful for translators that want to start
# a new translation and not not want to bother generating their own po from scratch!
-# dummy, potfile = tempfile.mkstemp(suffix=".pot",
-# prefix="blender_pot_")
-# os.close(dummy)
potfile = FILE_NAME_POT
cmd = [PY3, "./update_pot.py", "-i", msgfile, "-o", potfile]
if not args.no_checks:
@@ -93,8 +82,8 @@ def main():
if args.add:
cmd.append("-a")
cmd += args.langs
- if args.pproc_contexts:
- cmd.append("--pproc-contexts")
+ #if args.pproc_contexts:
+ #cmd.append("--pproc-contexts")
t = subprocess.call(cmd)
if t:
ret = t
diff --git a/release/scripts/modules/bl_i18n_utils/update_languages_menu.py b/release/scripts/modules/bl_i18n_utils/update_languages_menu.py
index 9b4cb20fadf..6263f1c1e64 100755
--- a/release/scripts/modules/bl_i18n_utils/update_languages_menu.py
+++ b/release/scripts/modules/bl_i18n_utils/update_languages_menu.py
@@ -73,11 +73,11 @@ def find_matching_po(languages, stats, forbidden):
def main():
import argparse
- parser = argparse.ArgumentParser(description=""
- "Update 'languages' text file used by Blender at runtime to build translations menu.")
+ parser = argparse.ArgumentParser(description="Update 'languages' text file used by Blender at runtime to build "
+ "translations menu.")
parser.add_argument('-m', '--min_translation', type=int, default=-100,
- help="Minimum level of translation, as a percentage "
- "(translations below this are commented out in menu).")
+ help="Minimum level of translation, as a percentage (translations below this are commented out "
+ "in menu).")
parser.add_argument('langs', metavar='ISO_code', nargs='*',
help="Unconditionally exclude those languages from the menu.")
args = parser.parse_args()
@@ -92,8 +92,8 @@ def main():
for po in os.listdir(TRUNK_PO_DIR):
if po.endswith(".po") and not po.endswith("_raw.po"):
lang = os.path.basename(po)[:-3]
- u1, u2, _stats = utils.parse_messages(os.path.join(TRUNK_PO_DIR, po))
- stats[lang] = _stats["trans_msg"] / _stats["tot_msg"]
+ msgs = utils.I18nMessages(kind='PO', src=os.path.join(TRUNK_PO_DIR, po))
+ stats[lang] = msgs.nbr_trans_msgs / msgs.nbr_msgs
# Generate languages file used by Blender's i18n system.
# First, match all entries in LANGUAGES to a lang in stats, if possible!
diff --git a/release/scripts/modules/bl_i18n_utils/update_mo.py b/release/scripts/modules/bl_i18n_utils/update_mo.py
index 7f68736593c..ab6fedc87a7 100755
--- a/release/scripts/modules/bl_i18n_utils/update_mo.py
+++ b/release/scripts/modules/bl_i18n_utils/update_mo.py
@@ -51,11 +51,11 @@ def process_po(po, lang, mo=None):
# show stats
cmd = (GETTEXT_MSGFMT_EXECUTABLE,
- "--statistics",
- po,
- "-o",
- mo or os.path.join(mo_dir, ".".join((DOMAIN, "mo"))),
- )
+ "--statistics",
+ po,
+ "-o",
+ mo or os.path.join(mo_dir, ".".join((DOMAIN, "mo"))),
+ )
print("Running ", " ".join(cmd))
ret = subprocess.call(cmd)
diff --git a/release/scripts/modules/bl_i18n_utils/update_po.py b/release/scripts/modules/bl_i18n_utils/update_po.py
index 4c6495c5cfe..a1fe20f9fce 100755
--- a/release/scripts/modules/bl_i18n_utils/update_po.py
+++ b/release/scripts/modules/bl_i18n_utils/update_po.py
@@ -41,6 +41,7 @@ TRUNK_PO_DIR = settings.TRUNK_PO_DIR
FILE_NAME_POT = settings.FILE_NAME_POT
+# XXX Not updated, not sure it's that much useful...
def pproc_newcontext_po(po, pot_messages, pot_stats):
print("Adding new contexts to {}...".format(po))
messages, state, stats = utils.parse_messages(po)
@@ -73,41 +74,39 @@ def pproc_newcontext_po(po, pot_messages, pot_stats):
def process_po(po, lang):
# update po file
- cmd = (GETTEXT_MSGMERGE_EXECUTABLE,
- "--update",
- "-w", "1", # XXX Ugly hack to prevent msgmerge merging
- # short source comments together!
- "--no-wrap",
- "--backup=none",
- "--lang={}".format(lang),
- po,
- FILE_NAME_POT,
- )
-
+ #cmd = (GETTEXT_MSGMERGE_EXECUTABLE,
+ #"--update",
+ #"-w", "1", # XXX Ugly hack to prevent msgmerge merging short source comments together!
+ #"--no-wrap",
+ #"--backup=none",
+ #"--lang={}".format(lang),
+ #po,
+ #FILE_NAME_POT,
+ #)
+
+ pot = utils.I18nMessages(kind='PO', src=FILE_NAME_POT)
+ msg = utils.I18nMessages(iso=lang, kind='PO', src=po)
print("Updating {}...".format(po))
- print("Running ", " ".join(cmd))
- ret = subprocess.call(cmd)
+ msg.update(pot)
+ msg.write(kind='PO', dest=po)
+ #print("Running ", " ".join(cmd))
+ #ret = subprocess.call(cmd)
print("Finished!\n")
- return ret
+ return 0
def main():
import argparse
parser = argparse.ArgumentParser(description="Write out messages.txt "
"from Blender.")
- parser.add_argument('-t', '--trunk', action="store_true",
- help="Update po’s in /trunk/po rather than /branches.")
- parser.add_argument('-i', '--input', metavar="File",
- help="Input pot file path.")
- parser.add_argument('--pproc-contexts', action="store_true",
- help="Pre-process po’s to avoid having plenty of "
- "fuzzy msgids just because a context was "
- "added/changed!")
+ parser.add_argument('-t', '--trunk', action="store_true", help="Update po’s in /trunk/po rather than /branches.")
+ parser.add_argument('-i', '--input', metavar="File", help="Input pot file path.")
+ #parser.add_argument('--pproc-contexts', action="store_true",
+ #help="Pre-process po’s to avoid having plenty of fuzzy msgids just because a context was "
+ #"added/changed!")
parser.add_argument('-a', '--add', action="store_true",
- help="Add missing po’s (useful only when one or "
- "more languages are given!).")
- parser.add_argument('langs', metavar='ISO_code', nargs='*',
- help="Restrict processed languages to those.")
+ help="Add missing po’s (useful only when one or more languages are given!).")
+ parser.add_argument('langs', metavar='ISO_code', nargs='*', help="Restrict processed languages to those.")
args = parser.parse_args()
if args.input:
@@ -115,9 +114,11 @@ def main():
FILE_NAME_POT = args.input
ret = 0
- if args.pproc_contexts:
- _ctxt_proc = pproc_newcontext_po
- pot_messages, _a, pot_stats = utils.parse_messages(FILE_NAME_POT)
+ #if args.pproc_contexts:
+ #_ctxt_proc = pproc_newcontext_po
+ #pot_messages, _a, pot_stats = utils.parse_messages(FILE_NAME_POT)
+ if 0:
+ pass
else:
_ctxt_proc = lambda a, b, c: 0
pot_messages, pot_stats = None, None
diff --git a/release/scripts/modules/bl_i18n_utils/update_pot.py b/release/scripts/modules/bl_i18n_utils/update_pot.py
index ecb5d837a09..e05e60937a8 100755
--- a/release/scripts/modules/bl_i18n_utils/update_pot.py
+++ b/release/scripts/modules/bl_i18n_utils/update_pot.py
@@ -23,6 +23,7 @@
# Update blender.pot file from messages.txt
import subprocess
+import collections
import os
import sys
import re
@@ -41,9 +42,11 @@ except:
LANGUAGES_CATEGORIES = settings.LANGUAGES_CATEGORIES
LANGUAGES = settings.LANGUAGES
-COMMENT_PREFIX = settings.COMMENT_PREFIX
-COMMENT_PREFIX_SOURCE = settings.COMMENT_PREFIX_SOURCE
-CONTEXT_PREFIX = settings.CONTEXT_PREFIX
+PO_COMMENT_PREFIX = settings.PO_COMMENT_PREFIX
+PO_COMMENT_PREFIX_SOURCE = settings.PO_COMMENT_PREFIX_SOURCE
+PO_COMMENT_PREFIX_SOURCE_CUSTOM = settings.PO_COMMENT_PREFIX_SOURCE_CUSTOM
+MSG_COMMENT_PREFIX = settings.MSG_COMMENT_PREFIX
+MSG_CONTEXT_PREFIX = settings.MSG_CONTEXT_PREFIX
FILE_NAME_MESSAGES = settings.FILE_NAME_MESSAGES
FILE_NAME_POT = settings.FILE_NAME_POT
SOURCE_DIR = settings.SOURCE_DIR
@@ -79,6 +82,10 @@ _clean_str = re.compile(settings.str_clean_re).finditer
clean_str = lambda s: "".join(m.group("clean") for m in _clean_str(s))
+def _new_messages():
+ return getattr(collections, "OrderedDict", dict)()
+
+
def check_file(path, rel_path, messages):
def process_entry(ctxt, msg):
# Context.
@@ -207,20 +214,22 @@ def get_svnrev():
def gen_empty_pot():
+ blender_ver = ""
blender_rev = get_svnrev().decode()
utctime = time.gmtime()
time_str = time.strftime("%Y-%m-%d %H:%M+0000", utctime)
year_str = time.strftime("%Y", utctime)
- return utils.gen_empty_messages(blender_rev, time_str, year_str)
+ return utils.I18nMessages.gen_empty_messages("__POT__", blender_ver, blender_rev, time_str, year_str)
escape_re = tuple(re.compile(r[0]) for r in settings.ESCAPE_RE)
escape = lambda s, n: escape_re[n].sub(settings.ESCAPE_RE[n][1], s)
-def merge_messages(msgs, states, messages, do_checks, spell_cache):
- num_added = num_present = 0
+def merge_messages(msgs, messages, do_checks, spell_cache):
+ num_added = 0
+ num_present = msgs.nbr_msgs
for (context, msgid), srcs in messages.items():
if do_checks:
err = spell_check(msgid, spell_cache)
@@ -233,19 +242,15 @@ def merge_messages(msgs, states, messages, do_checks, spell_cache):
for n in range(len(escape_re)):
msgid = escape(msgid, n)
- srcs = [COMMENT_PREFIX_SOURCE + s for s in srcs]
-
key = (context, msgid)
- if key not in msgs:
- msgs[key] = {"msgid_lines": [msgid],
- "msgstr_lines": [""],
- "comment_lines": srcs,
- "msgctxt_lines": [context]}
+ if key not in msgs.msgs:
+ msg = utils.I18nMessage([context], [msgid], [""], [])
+ msg.sources = srcs
+ msgs.msgs[key] = msg
num_added += 1
else:
- # We need to merge comments!
- msgs[key]["comment_lines"].extend(srcs)
- num_present += 1
+ # We need to merge sources!
+ msgs.msgs[key].sources += srcs
return num_added, num_present
@@ -270,7 +275,7 @@ def main():
print("Running fake py gettext…")
# Not using any more xgettext, simpler to do it ourself!
- messages = utils.new_messages()
+ messages = _new_messages()
py_xgettext(messages)
print("Finished, found {} messages.".format(len(messages)))
@@ -281,55 +286,49 @@ def main():
spell_cache = set()
print("Generating POT file {}…".format(FILE_NAME_POT))
- msgs, states = gen_empty_pot()
- tot_messages, _a = merge_messages(msgs, states, messages,
- True, spell_cache)
+ msgs = gen_empty_pot()
+ tot_messages, _a = merge_messages(msgs, messages, True, spell_cache)
# add messages collected automatically from RNA
print("\tMerging RNA messages from {}…".format(FILE_NAME_MESSAGES))
- messages = utils.new_messages()
+ messages.clear()
with open(FILE_NAME_MESSAGES, encoding="utf-8") as f:
srcs = []
context = ""
for line in f:
line = utils.stripeol(line)
- if line.startswith(COMMENT_PREFIX):
- srcs.append(line[len(COMMENT_PREFIX):].strip())
- elif line.startswith(CONTEXT_PREFIX):
- context = line[len(CONTEXT_PREFIX):].strip()
+ if line.startswith(MSG_COMMENT_PREFIX):
+ srcs.append(line[len(MSG_COMMENT_PREFIX):].strip())
+ elif line.startswith(MSG_CONTEXT_PREFIX):
+ context = line[len(MSG_CONTEXT_PREFIX):].strip()
else:
key = (context, line)
messages[key] = srcs
srcs = []
context = ""
- num_added, num_present = merge_messages(msgs, states, messages,
- True, spell_cache)
+ num_added, num_present = merge_messages(msgs, messages, True, spell_cache)
tot_messages += num_added
- print("\tMerged {} messages ({} were already present)."
- "".format(num_added, num_present))
+ print("\tMerged {} messages ({} were already present).".format(num_added, num_present))
print("\tAdding languages labels...")
- messages = {(CONTEXT_DEFAULT, lng[1]):
- ("Languages’ labels from bl_i18n_utils/settings.py",)
- for lng in LANGUAGES}
- messages.update({(CONTEXT_DEFAULT, cat[1]):
- ("Language categories’ labels from bl_i18n_utils/settings.py",)
- for cat in LANGUAGES_CATEGORIES})
- num_added, num_present = merge_messages(msgs, states, messages,
- True, spell_cache)
+ messages.clear()
+ messages.update(((CONTEXT_DEFAULT, lng[1]), ("Languages’ labels from bl_i18n_utils/settings.py",))
+ for lng in LANGUAGES)
+ messages.update(((CONTEXT_DEFAULT, cat[1]), ("Language categories’ labels from bl_i18n_utils/settings.py",))
+ for cat in LANGUAGES_CATEGORIES)
+ num_added, num_present = merge_messages(msgs, messages, True, spell_cache)
tot_messages += num_added
print("\tAdded {} language messages.".format(num_added))
# Write back all messages into blender.pot.
- utils.write_messages(FILE_NAME_POT, msgs, states["comm_msg"],
- states["fuzzy_msg"])
+ msgs.write('PO', FILE_NAME_POT)
if SPELL_CACHE and spell_cache:
with open(SPELL_CACHE, 'wb') as f:
pickle.dump(spell_cache, f)
- print("Finished, total: {} messages!".format(tot_messages - 1))
+ print("Finished, total: {} messages!".format(tot_messages))
return 0
diff --git a/release/scripts/modules/bl_i18n_utils/update_trunk.py b/release/scripts/modules/bl_i18n_utils/update_trunk.py
index b84a227ae0a..d7d1a9741cb 100755
--- a/release/scripts/modules/bl_i18n_utils/update_trunk.py
+++ b/release/scripts/modules/bl_i18n_utils/update_trunk.py
@@ -25,8 +25,7 @@
# * Copy po’s from branches advanced enough.
# * Clean po’s in trunk.
# * Compile po’s in trunk in mo’s, keeping track of those failing.
-# * Remove po’s, mo’s (and their dir’s) that failed to compile or
-# are no more present in trunk.
+# * Remove po’s, mo’s (and their dir’s) that failed to compile or are no more present in trunk.
import subprocess
import os
@@ -50,25 +49,6 @@ LANGUAGES_FILE = settings.LANGUAGES_FILE
PY3 = settings.PYTHON3_EXEC
-def find_matching_po(languages, stats):
- """Match languages defined in LANGUAGES setting to relevant po, if possible!"""
- ret = []
- for uid, label, org_key in languages:
- key = org_key
- if key not in stats:
- # Try to simplify the key (eg from es_ES to es).
- if '_' in org_key:
- key = org_key[0:org_key.index('_')]
- if '@' in org_key:
- key = key + org_key[org_key.index('@'):]
- if key in stats:
- ret.append((stats[key], uid, label, org_key))
- else:
- # Mark invalid entries, so that we can put them in the languages file,
- # but commented!
- ret.append((0.0, -uid, label, org_key))
- return ret
-
def main():
import argparse
parser = argparse.ArgumentParser(description=""
@@ -99,7 +79,8 @@ def main():
os.remove(po)
# Copy po’s from branches.
- cmd = [PY3, "./import_po_from_branches.py", "-s"]
+ #cmd = [PY3, "./import_po_from_branches.py", "-s"]
+ cmd = [PY3, "./import_po_from_branches.py"]
if args.threshold is not None:
cmd += ["-t", str(args.threshold)]
if args.langs:
@@ -124,7 +105,8 @@ def main():
if args.langs and lang not in args.langs:
continue
- cmd = [PY3, "./clean_po.py", "-t", "-s", lang]
+ #cmd = [PY3, "./clean_po.py", "-t", "-s", lang]
+ cmd = [PY3, "./clean_po.py", "-t", lang]
t = subprocess.call(cmd)
if t:
ret = t
diff --git a/release/scripts/modules/bl_i18n_utils/utils.py b/release/scripts/modules/bl_i18n_utils/utils.py
index 9481f750092..ff5ed4f93d7 100644
--- a/release/scripts/modules/bl_i18n_utils/utils.py
+++ b/release/scripts/modules/bl_i18n_utils/utils.py
@@ -20,357 +20,660 @@
# Some misc utilities...
+import collections
+import copy
import os
+import re
import sys
-import collections
from bl_i18n_utils import settings
-COMMENT_PREFIX = settings.COMMENT_PREFIX
+PO_COMMENT_PREFIX = settings.PO_COMMENT_PREFIX
+PO_COMMENT_PREFIX_MSG = settings.PO_COMMENT_PREFIX_MSG
+PO_COMMENT_PREFIX_SOURCE = settings.PO_COMMENT_PREFIX_SOURCE
+PO_COMMENT_PREFIX_SOURCE_CUSTOM = settings.PO_COMMENT_PREFIX_SOURCE_CUSTOM
+PO_COMMENT_FUZZY = settings.PO_COMMENT_FUZZY
+PO_MSGCTXT = settings.PO_MSGCTXT
+PO_MSGID = settings.PO_MSGID
+PO_MSGSTR = settings.PO_MSGSTR
+
+PO_HEADER_KEY = settings.PO_HEADER_KEY
+PO_HEADER_COMMENT = settings.PO_HEADER_COMMENT
+PO_HEADER_COMMENT_COPYRIGHT = settings.PO_HEADER_COMMENT_COPYRIGHT
+PO_HEADER_MSGSTR = settings.PO_HEADER_MSGSTR
+
+PARSER_CACHE_HASH = settings.PARSER_CACHE_HASH
+
WARN_NC = settings.WARN_MSGID_NOT_CAPITALIZED
NC_ALLOWED = settings.WARN_MSGID_NOT_CAPITALIZED_ALLOWED
+PARSER_CACHE_HASH = settings.PARSER_CACHE_HASH
+##### Misc Utils #####
+
def stripeol(s):
return s.rstrip("\n\r")
-# XXX For now, we assume that all messages > 30 chars are tooltips!
-def is_tooltip(msgid):
- return len(msgid) > 30
+_valid_po_path_re = re.compile(r"^\S+:[0-9]+$")
+def is_valid_po_path(path):
+ return bool(_valid_po_path_re.match(path))
+
+
+class I18nMessage:
+ """
+ Internal representation of a message.
+ """
+ __slots__ = ("msgctxt_lines", "msgid_lines", "msgstr_lines", "comment_lines", "is_fuzzy", "is_commented")
+
+ def __init__(self, msgctxt_lines=[], msgid_lines=[], msgstr_lines=[], comment_lines=[],
+ is_commented=False, is_fuzzy=False):
+ self.msgctxt_lines = msgctxt_lines
+ self.msgid_lines = msgid_lines
+ self.msgstr_lines = msgstr_lines
+ self.comment_lines = comment_lines
+ self.is_fuzzy = is_fuzzy
+ self.is_commented = is_commented
+
+ def _get_msgctxt(self):
+ return ("".join(self.msgctxt_lines)).replace("\\n", "\n")
+ def _set_msgctxt(self, ctxt):
+ self.msgctxt_lines = [ctxt]
+ msgctxt = property(_get_msgctxt, _set_msgctxt)
+
+ def _get_msgid(self):
+ return ("".join(self.msgid_lines)).replace("\\n", "\n")
+ def _set_msgid(self, msgid):
+ self.msgid_lines = [msgid]
+ msgid = property(_get_msgid, _set_msgid)
+
+ def _get_msgstr(self):
+ return ("".join(self.msgstr_lines)).replace("\\n", "\n")
+ def _set_msgstr(self, msgstr):
+ self.msgstr_lines = [msgstr]
+ msgstr = property(_get_msgstr, _set_msgstr)
+
+ def _get_sources(self):
+ lstrip1 = len(PO_COMMENT_PREFIX_SOURCE)
+ lstrip2 = len(PO_COMMENT_PREFIX_SOURCE_CUSTOM)
+ return ([l[lstrip1:] for l in self.comment_lines if l.startswith(PO_COMMENT_PREFIX_SOURCE)] +
+ [l[lstrip2:] for l in self.comment_lines if l.startswith(PO_COMMENT_PREFIX_SOURCE_CUSTOM)])
+ def _set_sources(self, sources):
+ # list.copy() is not available in py3.2 ...
+ cmmlines = []
+ cmmlines[:] = self.comment_lines
+ for l in cmmlines:
+ if l.startswith(PO_COMMENT_PREFIX_SOURCE) or l.startswith(PO_COMMENT_PREFIX_SOURCE_CUSTOM):
+ self.comment_lines.remove(l)
+ lines_src = []
+ lines_src_custom = []
+ for src in sources:
+ if is_valid_po_path(src):
+ lines_src.append(PO_COMMENT_PREFIX_SOURCE + src)
+ else:
+ lines_src_custom.append(PO_COMMENT_PREFIX_SOURCE_CUSTOM + src)
+ self.comment_lines += lines_src_custom + lines_src
+ sources = property(_get_sources, _set_sources)
+
+ def _get_is_tooltip(self):
+ # XXX For now, we assume that all messages > 30 chars are tooltips!
+ return len(self.msgid) > 30
+ is_tooltip = property(_get_is_tooltip)
+
+ def normalize(self, max_len=80):
+ """
+ Normalize this message, call this before exporting it...
+ Currently normalize msgctxt, msgid and msgstr lines to given max_len (if below 1, make them single line).
+ """
+ max_len -= 2 # The two quotes!
+ # We do not need the full power of textwrap... We just split first at escaped new lines, then into each line
+ # if needed... No word splitting, nor fancy spaces handling!
+ def _wrap(text, max_len, init_len):
+ if len(text) + init_len < max_len:
+ return [text]
+ lines = text.splitlines()
+ ret = []
+ for l in lines:
+ tmp = []
+ cur_len = 0
+ words = l.split(' ')
+ for w in words:
+ cur_len += len(w) + 1
+ if cur_len > (max_len - 1) and tmp:
+ ret.append(" ".join(tmp) + " ")
+ del tmp[:]
+ cur_len = len(w) + 1
+ tmp.append(w)
+ if tmp:
+ ret.append(" ".join(tmp))
+ return ret
+ if max_len < 1:
+ self.msgctxt_lines = self.msgctxt.replace("\n", "\\n\n").splitlines()
+ self.msgid_lines = self.msgid.replace("\n", "\\n\n").splitlines()
+ self.msgstr_lines = self.msgstr.replace("\n", "\\n\n").splitlines()
+ else:
+ init_len = len(PO_MSGCTXT) + 1
+ if self.is_commented:
+ init_len += len(PO_COMMENT_PREFIX_MSG)
+ self.msgctxt_lines = _wrap(self.msgctxt.replace("\n", "\\n\n"), max_len, init_len)
+ init_len = len(PO_MSGID) + 1
+ if self.is_commented:
+ init_len += len(PO_COMMENT_PREFIX_MSG)
+ self.msgid_lines = _wrap(self.msgid.replace("\n", "\\n\n"), max_len, init_len)
-def new_messages():
- return getattr(collections, 'OrderedDict', dict)()
+ init_len = len(PO_MSGSTR) + 1
+ if self.is_commented:
+ init_len += len(PO_COMMENT_PREFIX_MSG)
+ self.msgstr_lines = _wrap(self.msgstr.replace("\n", "\\n\n"), max_len, init_len)
-def parse_messages(fname):
+class I18nMessages:
"""
- Returns a tupple (messages, states, stats).
- messages is an odereddict of dicts
- {(ctxt, msgid): {msgid_lines:, msgstr_lines:,
- comment_lines:, msgctxt_lines:}}.
- states is a dict of three sets of (msgid, ctxt), and a boolean flag
- indicating the .po is somewhat broken
- {trans_msg:, fuzzy_msg:, comm_msg:, is_broken:}.
- stats is a dict of values
- {tot_msg:, trans_msg:, tot_ttips:, trans_ttips:, comm_msg:,
- nbr_signs:, nbr_trans_signs:, contexts: set()}.
- Note: This function will silently "arrange" mis-formated entries, thus
- using afterward write_messages() should always produce a po-valid file,
- though not correct!
+ Internal representation of messages for one language (iso code), with additional stats info.
"""
- tot_messages = 0
- tot_tooltips = 0
- trans_messages = 0
- trans_tooltips = 0
- comm_messages = 0
- nbr_signs = 0
- nbr_trans_signs = 0
- contexts = set()
- reading_msgid = False
- reading_msgstr = False
- reading_msgctxt = False
- reading_comment = False
- is_translated = False
- is_fuzzy = False
- is_commented = False
- is_broken = False
- msgid_lines = []
- msgstr_lines = []
- msgctxt_lines = []
- comment_lines = []
-
- messages = new_messages()
- translated_messages = set()
- fuzzy_messages = set()
- commented_messages = set()
-
- def clean_vars():
- nonlocal reading_msgid, reading_msgstr, reading_msgctxt, \
- reading_comment, is_fuzzy, is_translated, is_commented, \
- msgid_lines, msgstr_lines, msgctxt_lines, comment_lines
- reading_msgid = reading_msgstr = reading_msgctxt = \
- reading_comment = False
- is_tooltip = is_fuzzy = is_translated = is_commented = False
+
+ # Avoid parsing again!
+ # Keys should be (pseudo) file-names, values are tuples (hash, I18nMessages)
+ # Note: only used by po parser currently!
+ _parser_cache = {}
+
+ def __init__(self, iso="__POT__", kind=None, key=None, src=None):
+ self.iso = iso
+ self.msgs = self._new_messages()
+ self.trans_msgs = set()
+ self.fuzzy_msgs = set()
+ self.comm_msgs = set()
+ self.ttip_msgs = set()
+ self.contexts = set()
+ self.nbr_msgs = 0
+ self.nbr_trans_msgs = 0
+ self.nbr_ttips = 0
+ self.nbr_trans_ttips = 0
+ self.nbr_comm_msgs = 0
+ self.nbr_signs = 0
+ self.nbr_trans_signs = 0
+ self.parsing_errors = []
+ if kind and src:
+ self.parse(kind, key, src)
+ self.update_info()
+
+ @staticmethod
+ def _new_messages():
+ return getattr(collections, 'OrderedDict', dict)()
+
+ @classmethod
+ def gen_empty_messages(cls, iso, blender_ver, blender_rev, time, year, default_copyright=True):
+ """Generate an empty I18nMessages object (only header is present!)."""
+ msgstr = PO_HEADER_MSGSTR.format(blender_ver=str(blender_ver), blender_rev=int(blender_rev),
+ time=str(time), iso=str(iso))
+ comment = ""
+ if default_copyright:
+ comment = PO_HEADER_COMMENT_COPYRIGHT.format(year=str(year))
+ comment = comment + PO_HEADER_COMMENT
+
+ msgs = cls(iso=iso)
+ msgs.msgs[PO_HEADER_KEY] = I18nMessage([], [""], [msgstr], [comment], False, True)
+ msgs.update_info()
+
+ return msgs
+
+ def normalize(self, max_len=80):
+ for msg in self.msgs.values():
+ msg.normalize(max_len)
+
+ def merge(self, replace=False, *args):
+ pass
+
+ def update(self, ref, use_similar=0.8, keep_old_commented=True):
+ """
+ Update this I18nMessage with the ref one. Translations from ref are never used. Source comments from ref
+ completely replace current ones. If use_similar is not 0.0, it will try to match new messages in ref with an
+ existing one. Messages no more found in ref will be marked as commented if keep_old_commented is True,
+ or removed.
+ """
+ import difflib
+ similar_pool = {}
+ if use_similar > 0.0:
+ for key in self.msgs:
+ similar_pool.setdefault(key[1], set()).add(key)
+
+ msgs = self._new_messages()
+ for (key, msg) in ref.msgs.items():
+ if key in self.msgs:
+ msgs[key] = self.msgs[key]
+ msgs[key].sources = msg.sources
+ else:
+ skey = None
+ if use_similar > 0.0:
+ # try to find some close key in existing messages...
+ tmp = difflib.get_close_matches(key[1], similar_pool, n=1, cutoff=use_similar)
+ if tmp:
+ tmp = tmp[0]
+ # Try to get the same context, else just get one...
+ skey = (key[0], tmp)
+ if skey not in similar_pool[tmp]:
+ skey = tuple(similar_pool[tmp])[0]
+ msgs[key] = msg
+ if skey:
+ msgs[key].msgstr = self.msgs[skey].msgstr
+ msgs[key].is_fuzzy = True
+ # Add back all "old" and already commented messages as commented ones, if required.
+ if keep_old_commented:
+ for key, msg in self.msgs.items():
+ if key not in msgs:
+ msgs[key] = msg
+ msgs[key].is_commented = True
+ # And finalize the update!
+ self.msgs = msgs
+
+ def update_info(self):
+ self.trans_msgs.clear()
+ self.fuzzy_msgs.clear()
+ self.comm_msgs.clear()
+ self.ttip_msgs.clear()
+ self.contexts.clear()
+ self.nbr_signs = 0
+ self.nbr_trans_signs = 0
+ for key, msg in self.msgs.items():
+ if key == PO_HEADER_KEY:
+ continue
+ if msg.is_commented:
+ self.comm_msgs.add(key)
+ else:
+ if msg.msgstr:
+ self.trans_msgs.add(key)
+ if msg.is_fuzzy:
+ self.fuzzy_msgs.add(key)
+ if msg.is_tooltip:
+ self.ttip_msgs.add(key)
+ self.contexts.add(key[0])
+ self.nbr_signs += len(msg.msgid)
+ self.nbr_trans_signs += len(msg.msgstr)
+ self.nbr_msgs = len(self.msgs)
+ self.nbr_trans_msgs = len(self.trans_msgs)
+ self.nbr_ttips = len(self.ttip_msgs)
+ self.nbr_trans_ttips = len(self.ttip_msgs & self.trans_msgs)
+ self.nbr_comm_msgs = len(self.comm_msgs)
+
+ def print_stats(self, prefix=""):
+ """
+ Print out some stats about an I18nMessages object.
+ """
+ lvl = 0.0
+ lvl_ttips = 0.0
+ lvl_comm = 0.0
+ lvl_trans_ttips = 0.0
+ lvl_ttips_in_trans = 0.0
+ if self.nbr_msgs > 0:
+ lvl = float(self.nbr_trans_msgs) / float(self.nbr_msgs)
+ lvl_ttips = float(self.nbr_ttips) / float(self.nbr_msgs)
+ lvl_comm = float(self.nbr_comm_msgs) / float(self.nbr_msgs + self.nbr_comm_msgs)
+ if self.nbr_ttips > 0:
+ lvl_trans_ttips = float(self.nbr_trans_ttips) / float(self.nbr_ttips)
+ if self.nbr_trans_msgs > 0:
+ lvl_ttips_in_trans = float(self.nbr_trans_ttips) / float(self.nbr_trans_msgs)
+
+ lines = ("",
+ "{:>6.1%} done! ({} translated messages over {}).\n"
+ "".format(lvl, self.nbr_trans_msgs, self.nbr_msgs),
+ "{:>6.1%} of messages are tooltips ({} over {}).\n"
+ "".format(lvl_ttips, self.nbr_ttips, self.nbr_msgs),
+ "{:>6.1%} of tooltips are translated ({} over {}).\n"
+ "".format(lvl_trans_ttips, self.nbr_trans_ttips, self.nbr_ttips),
+ "{:>6.1%} of translated messages are tooltips ({} over {}).\n"
+ "".format(lvl_ttips_in_trans, self.nbr_trans_ttips, self.nbr_trans_msgs),
+ "{:>6.1%} of messages are commented ({} over {}).\n"
+ "".format(lvl_comm, self.nbr_comm_msgs, self.nbr_comm_msgs + self.nbr_msgs),
+ "This translation is currently made of {} signs.\n".format(self.nbr_trans_signs))
+ print(prefix.join(lines))
+
+ def parse(self, kind, key, src):
+ del self.parsing_errors[:]
+ self.parsers[kind](self, src, key)
+ if self.parsing_errors:
+ print("WARNING! Errors while parsing {}:".format(key))
+ for line, error in self.parsing_errors:
+ print(" Around line {}: {}".format(line, error))
+ print("The parser solved them as well as it could...")
+ self.update_info()
+
+ def parse_messages_from_po(self, src, key=None):
+ """
+ Parse a po file.
+ Note: This function will silently "arrange" mis-formated entries, thus using afterward write_messages() should
+ always produce a po-valid file, though not correct!
+ """
+ reading_msgid = False
+ reading_msgstr = False
+ reading_msgctxt = False
+ reading_comment = False
+ is_commented = False
+ is_fuzzy = False
+ msgctxt_lines = []
msgid_lines = []
msgstr_lines = []
- msgctxt_lines = []
comment_lines = []
- def finalize_message():
- nonlocal reading_msgid, reading_msgstr, reading_msgctxt, \
- reading_comment, is_fuzzy, is_translated, is_commented, \
- msgid_lines, msgstr_lines, msgctxt_lines, comment_lines, \
- messages, translated_messages, fuzzy_messages, \
- commented_messages, \
- tot_messages, tot_tooltips, trans_messages, trans_tooltips, \
- comm_messages, nbr_signs, nbr_trans_signs, contexts
-
- msgid = "".join(msgid_lines)
- msgctxt = "".join(msgctxt_lines)
- msgkey = (msgctxt, msgid)
- is_ttip = is_tooltip(msgid)
-
- # Never allow overriding existing msgid/msgctxt pairs!
- if msgkey in messages:
- clean_vars()
- return
-
- nbr_signs += len(msgid)
- if is_commented:
- commented_messages.add(msgkey)
- elif is_fuzzy:
- fuzzy_messages.add(msgkey)
- elif is_translated:
- translated_messages.add(msgkey)
- nbr_trans_signs += len("".join(msgstr_lines))
- messages[msgkey] = {"msgid_lines" : msgid_lines,
- "msgstr_lines" : msgstr_lines,
- "comment_lines": comment_lines,
- "msgctxt_lines": msgctxt_lines}
-
- if is_commented:
- comm_messages += 1
- else:
- tot_messages += 1
- if is_ttip:
- tot_tooltips += 1
- if not is_fuzzy and is_translated:
- trans_messages += 1
- if is_ttip:
- trans_tooltips += 1
- if msgctxt not in contexts:
- contexts.add(msgctxt)
-
- clean_vars()
-
- with open(fname, 'r', encoding="utf-8") as f:
- for line_nr, line in enumerate(f):
- line = stripeol(line)
+ # Helper function
+ def finalize_message(self, line_nr):
+ nonlocal reading_msgid, reading_msgstr, reading_msgctxt, reading_comment
+ nonlocal is_commented, is_fuzzy, msgid_lines, msgstr_lines, msgctxt_lines, comment_lines
+
+ msgid = "".join(msgid_lines)
+ msgctxt = "".join(msgctxt_lines)
+ msgkey = (msgctxt, msgid)
+
+ # Never allow overriding existing msgid/msgctxt pairs!
+ if msgkey in self.msgs:
+ self.parsing_errors.append((line_nr, "{} context/msgid is already in current messages!".format(msgkey)))
+ return
+
+ self.msgs[msgkey] = I18nMessage(msgctxt_lines, msgid_lines, msgstr_lines, comment_lines,
+ is_commented, is_fuzzy)
+
+ # Let's clean up and get ready for next message!
+ reading_msgid = reading_msgstr = reading_msgctxt = reading_comment = False
+ is_commented = is_fuzzy = False
+ msgctxt_lines = []
+ msgid_lines = []
+ msgstr_lines = []
+ comment_lines = []
+
+ # try to use src as file name...
+ if os.path.exists(src):
+ if not key:
+ key = src
+ with open(src, 'r', encoding="utf-8") as f:
+ src = f.read()
+
+ # Try to use values from cache!
+ curr_hash = None
+ if key and key in self._parser_cache:
+ old_hash, msgs = self._parser_cache[key]
+ import hashlib
+ curr_hash = hashlib.new(PARSER_CACHE_HASH, src.encode()).digest()
+ if curr_hash == old_hash:
+ self.msgs = copy.deepcopy(msgs) # we might edit self.msgs!
+ return
+
+ _comm_msgctxt = PO_COMMENT_PREFIX_MSG + PO_MSGCTXT
+ _len_msgctxt = len(PO_MSGCTXT + '"')
+ _len_comm_msgctxt = len(_comm_msgctxt + '"')
+ _comm_msgid = PO_COMMENT_PREFIX_MSG + PO_MSGID
+ _len_msgid = len(PO_MSGID + '"')
+ _len_comm_msgid = len(_comm_msgid + '"')
+ _comm_msgstr = PO_COMMENT_PREFIX_MSG + PO_MSGSTR
+ _len_msgstr = len(PO_MSGSTR + '"')
+ _len_comm_msgstr = len(_comm_msgstr + '"')
+ _len_comm_str = len(PO_COMMENT_PREFIX_MSG + '"')
+
+ # Main loop over all lines in src...
+ for line_nr, line in enumerate(src.splitlines()):
if line == "":
- finalize_message()
+ finalize_message(self, line_nr)
- elif line.startswith("msgctxt") or \
- line.startswith("".join((COMMENT_PREFIX, "msgctxt"))):
+ elif line.startswith(PO_MSGCTXT) or line.startswith(_comm_msgctxt):
reading_comment = False
reading_ctxt = True
- if line.startswith(COMMENT_PREFIX):
+ if line.startswith(PO_COMMENT_PREFIX_MSG):
is_commented = True
- line = line[9 + len(COMMENT_PREFIX):-1]
+ line = line[_len_comm_msgctxt:-1]
else:
- line = line[9:-1]
+ line = line[_len_msgctxt:-1]
msgctxt_lines.append(line)
- elif line.startswith("msgid") or \
- line.startswith("".join((COMMENT_PREFIX, "msgid"))):
+ elif line.startswith(PO_MSGID) or line.startswith(_comm_msgid):
reading_comment = False
reading_msgid = True
- if line.startswith(COMMENT_PREFIX):
+ if line.startswith(PO_COMMENT_PREFIX_MSG):
+ if not is_commented and reading_ctxt:
+ self.parsing_errors.append((line_nr, "commented msgid following regular msgctxt"))
is_commented = True
- line = line[7 + len(COMMENT_PREFIX):-1]
+ line = line[_len_comm_msgid:-1]
else:
- line = line[7:-1]
+ line = line[_len_msgid:-1]
+ reading_ctxt = False
msgid_lines.append(line)
- elif line.startswith("msgstr") or \
- line.startswith("".join((COMMENT_PREFIX, "msgstr"))):
+ elif line.startswith(PO_MSGSTR) or line.startswith(_comm_msgstr):
if not reading_msgid:
- is_broken = True
+ self.parsing_errors.append((line_nr, "msgstr without a prior msgid"))
else:
reading_msgid = False
reading_msgstr = True
- if line.startswith(COMMENT_PREFIX):
- line = line[8 + len(COMMENT_PREFIX):-1]
+ if line.startswith(PO_COMMENT_PREFIX_MSG):
+ line = line[_len_comm_msgstr:-1]
if not is_commented:
- is_broken = True
+ self.parsing_errors.append((line_nr, "commented msgstr following regular msgid"))
else:
- line = line[8:-1]
+ line = line[_len_msgstr:-1]
if is_commented:
- is_broken = True
+ self.parsing_errors.append((line_nr, "regular msgstr following commented msgid"))
msgstr_lines.append(line)
- if line:
- is_translated = True
- elif line.startswith("#"):
- if reading_msgid:
- if is_commented:
- msgid_lines.append(line[1 + len(COMMENT_PREFIX):-1])
- else:
- msgid_lines.append(line)
- is_broken = True
- elif reading_msgstr:
- if is_commented:
- msgstr_lines.append(line[1 + len(COMMENT_PREFIX):-1])
- else:
- msgstr_lines.append(line)
- is_broken = True
+ elif line.startswith(PO_COMMENT_PREFIX[0]):
+ if line.startswith(PO_COMMENT_PREFIX_MSG):
+ if reading_msgctxt:
+ if is_commented:
+ msgctxt_lines.append(line[_len_comm_str:-1])
+ else:
+ msgctxt_lines.append(line)
+ self.parsing_errors.append((line_nr, "commented string while reading regular msgctxt"))
+ elif reading_msgid:
+ if is_commented:
+ msgid_lines.append(line[_len_comm_str:-1])
+ else:
+ msgid_lines.append(line)
+ self.parsing_errors.append((line_nr, "commented string while reading regular msgid"))
+ elif reading_msgstr:
+ if is_commented:
+ msgstr_lines.append(line[_len_comm_str:-1])
+ else:
+ msgstr_lines.append(line)
+ self.parsing_errors.append((line_nr, "commented string while reading regular msgstr"))
else:
- if line.startswith("#, fuzzy"):
+ if reading_msgctxt or reading_msgid or reading_msgstr:
+ self.parsing_errors.append((line_nr,
+ "commented string within msgctxt, msgid or msgstr scope, ignored"))
+ elif line.startswith(PO_COMMENT_FUZZY):
is_fuzzy = True
else:
comment_lines.append(line)
reading_comment = True
else:
- if reading_msgid:
+ if reading_msgctxt:
+ msgctxt_lines.append(line[1:-1])
+ elif reading_msgid:
msgid_lines.append(line[1:-1])
elif reading_msgstr:
line = line[1:-1]
msgstr_lines.append(line)
- if not is_translated and line:
- is_translated = True
else:
- is_broken = True
+ self.parsing_errors.append((line_nr, "regular string outside msgctxt, msgid or msgstr scope"))
+ print(line)
# If no final empty line, last message is not finalized!
if reading_msgstr:
- finalize_message()
-
- return (messages,
- {"trans_msg": translated_messages,
- "fuzzy_msg": fuzzy_messages,
- "comm_msg" : commented_messages,
- "is_broken": is_broken},
- {"tot_msg" : tot_messages,
- "trans_msg" : trans_messages,
- "tot_ttips" : tot_tooltips,
- "trans_ttips" : trans_tooltips,
- "comm_msg" : comm_messages,
- "nbr_signs" : nbr_signs,
- "nbr_trans_signs": nbr_trans_signs,
- "contexts" : contexts})
-
-
-def write_messages(fname, messages, commented, fuzzy):
- "Write in fname file the content of messages (similar to parse_messages " \
- "returned values). commented and fuzzy are two sets containing msgid. " \
- "Returns the number of written messages."
- num = 0
- with open(fname, 'w', encoding="utf-8") as f:
- for msgkey, val in messages.items():
- msgctxt, msgid = msgkey
- f.write("\n".join(val["comment_lines"]))
- # Only mark as fuzzy if msgstr is not empty!
- if msgkey in fuzzy and "".join(val["msgstr_lines"]):
- f.write("\n#, fuzzy")
- if msgkey in commented:
- if msgctxt:
- f.write("\n{}msgctxt \"".format(COMMENT_PREFIX))
- f.write("\"\n{}\"".format(COMMENT_PREFIX).join(
- val["msgctxt_lines"]))
- f.write("\"")
- f.write("\n{}msgid \"".format(COMMENT_PREFIX))
- f.write("\"\n{}\"".format(COMMENT_PREFIX).join(
- val["msgid_lines"]))
- f.write("\"\n{}msgstr \"".format(COMMENT_PREFIX))
- f.write("\"\n{}\"".format(COMMENT_PREFIX).join(
- val["msgstr_lines"]))
- f.write("\"\n\n")
- else:
- if msgctxt:
- f.write("\nmsgctxt \"")
- f.write("\"\n\"".join(val["msgctxt_lines"]))
- f.write("\"")
- f.write("\nmsgid \"")
- f.write("\"\n\"".join(val["msgid_lines"]))
- f.write("\"\nmsgstr \"")
- f.write("\"\n\"".join(val["msgstr_lines"]))
- f.write("\"\n\n")
- num += 1
- return num
-
-
-def gen_empty_messages(blender_rev, time_str, year_str):
- """Generate an empty messages & state data (only header if present!)."""
- header_key = ("", "")
-
- messages = new_messages()
- messages[header_key] = {
- "msgid_lines": [""],
- "msgctxt_lines": [],
- "msgstr_lines": [
- "Project-Id-Version: Blender r{}\\n"
- "".format(blender_rev),
- "Report-Msgid-Bugs-To: \\n",
- "POT-Creation-Date: {}\\n"
- "".format(time_str),
- "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\\n",
- "Last-Translator: FULL NAME <EMAIL@ADDRESS>\\n",
- "Language-Team: LANGUAGE <LL@li.org>\\n",
- "Language: \\n",
- "MIME-Version: 1.0\\n",
- "Content-Type: text/plain; charset=UTF-8\\n",
- "Content-Transfer-Encoding: 8bit\\n"
- ],
- "comment_lines": [
- "# Blender's translation file (po format).",
- "# Copyright (C) {} The Blender Foundation."
- "".format(year_str),
- "# This file is distributed under the same "
- "# license as the Blender package.",
- "# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.",
- "#",
- ],
- }
+ finalize_message(self, line_nr)
+
+ if key:
+ if not curr_hash:
+ import hashlib
+ curr_hash = hashlib.new(PARSER_CACHE_HASH, src.encode()).digest()
+ self._parser_cache[key] = (curr_hash, self.msgs)
+
+ def write(self, kind, dest):
+ self.writers[kind](self, dest)
+
+ def write_messages_to_po(self, fname):
+ """
+ Write messages in fname po file.
+ """
+ self.normalize(max_len=0) # No wrapping for now...
+ with open(fname, 'w', encoding="utf-8") as f:
+ for msg in self.msgs.values():
+ f.write("\n".join(msg.comment_lines))
+ # Only mark as fuzzy if msgstr is not empty!
+ if msg.is_fuzzy and msg.msgstr:
+ f.write("\n" + PO_COMMENT_FUZZY)
+ _p = PO_COMMENT_PREFIX_MSG if msg.is_commented else ""
+ _pmsgctxt = _p + PO_MSGCTXT
+ _pmsgid = _p + PO_MSGID
+ _pmsgstr = _p + PO_MSGSTR
+ chunks = []
+ if msg.msgctxt:
+ if len(msg.msgctxt_lines) > 1:
+ chunks += [
+ "\n" + _pmsgctxt + "\"\"\n" + _p + "\"",
+ ("\"\n" + _p + "\"").join(msg.msgctxt_lines),
+ "\"",
+ ]
+ else:
+ chunks += ["\n" + _pmsgctxt + "\"" + msg.msgctxt + "\""]
+ if len(msg.msgid_lines) > 1:
+ print(msg.msgid_lines)
+ chunks += [
+ "\n" + _pmsgid + "\"\"\n" + _p + "\"",
+ ("\"\n" + _p + "\"").join(msg.msgid_lines),
+ "\"",
+ ]
+ else:
+ chunks += ["\n" + _pmsgid + "\"" + msg.msgid + "\""]
+ if len(msg.msgstr_lines) > 1:
+ chunks += [
+ "\n" + _pmsgstr + "\"\"\n" + _p + "\"",
+ ("\"\n" + _p + "\"").join(msg.msgstr_lines),
+ "\"",
+ ]
+ else:
+ chunks += ["\n" + _pmsgstr + "\"" + msg.msgstr + "\""]
+ chunks += ["\n\n"]
+ f.write("".join(chunks))
- states = {"trans_msg": set(),
- "fuzzy_msg": {header_key},
- "comm_msg": set(),
- "is_broken": False}
+ parsers = {
+ "PO": parse_messages_from_po,
+# "PYTUPLE": parse_messages_from_pytuple,
+ }
- return messages, states
+ writers = {
+ "PO": write_messages_to_po,
+ #"PYDICT": write_messages_to_pydict,
+ }
-def print_stats(stats, glob_stats=None, prefix=""):
+class I18n:
"""
- Print out some stats about a po file.
- glob_stats is for making global stats over several po's.
+ Internal representation of a whole translation set.
"""
- tot_msgs = stats["tot_msg"]
- trans_msgs = stats["trans_msg"]
- tot_ttips = stats["tot_ttips"]
- trans_ttips = stats["trans_ttips"]
- comm_msgs = stats["comm_msg"]
- nbr_signs = stats["nbr_signs"]
- nbr_trans_signs = stats["nbr_trans_signs"]
- contexts = stats["contexts"]
- lvl = lvl_ttips = lvl_trans_ttips = lvl_ttips_in_trans = lvl_comm = 0.0
-
- if tot_msgs > 0:
- lvl = float(trans_msgs) / float(tot_msgs)
- lvl_ttips = float(tot_ttips) / float(tot_msgs)
- lvl_comm = float(comm_msgs) / float(tot_msgs+comm_msgs)
- if tot_ttips > 0:
- lvl_trans_ttips = float(trans_ttips) / float(tot_ttips)
- if trans_msgs > 0:
- lvl_ttips_in_trans = float(trans_ttips) / float(trans_msgs)
-
- if glob_stats:
- glob_stats["nbr"] += 1.0
- glob_stats["lvl"] += lvl
- glob_stats["lvl_ttips"] += lvl_ttips
- glob_stats["lvl_trans_ttips"] += lvl_trans_ttips
- glob_stats["lvl_ttips_in_trans"] += lvl_ttips_in_trans
- glob_stats["lvl_comm"] += lvl_comm
- glob_stats["nbr_trans_signs"] += nbr_trans_signs
- if glob_stats["nbr_signs"] == 0:
- glob_stats["nbr_signs"] = nbr_signs
- glob_stats["contexts"] |= contexts
-
- lines = ("",
- "{:>6.1%} done! ({} translated messages over {}).\n"
- "".format(lvl, trans_msgs, tot_msgs),
- "{:>6.1%} of messages are tooltips ({} over {}).\n"
- "".format(lvl_ttips, tot_ttips, tot_msgs),
- "{:>6.1%} of tooltips are translated ({} over {}).\n"
- "".format(lvl_trans_ttips, trans_ttips, tot_ttips),
- "{:>6.1%} of translated messages are tooltips ({} over {}).\n"
- "".format(lvl_ttips_in_trans, trans_ttips, trans_msgs),
- "{:>6.1%} of messages are commented ({} over {}).\n"
- "".format(lvl_comm, comm_msgs, comm_msgs + tot_msgs),
- "This translation is currently made of {} signs.\n"
- "".format(nbr_trans_signs))
- print(prefix.join(lines))
- return 0
+
+ def __init__(self, src):
+ self.trans = {}
+ self.update_info()
+
+ def update_info(self):
+ self.nbr_trans = 0
+ self.lvl = 0.0
+ self.lvl_ttips = 0.0
+ self.lvl_trans_ttips = 0.0
+ self.lvl_ttips_in_trans = 0.0
+ self.lvl_comm = 0.0
+ self.nbr_signs = 0
+ self.nbr_trans_signs = 0
+ self.contexts = set()
+
+ if TEMPLATE_ISO_ID in self.trans:
+ self.nbr_trans = len(self.trans) - 1
+ self.nbr_signs = self.trans[TEMPLATE_ISO_ID].nbr_signs
+ else:
+ self.nbr_trans = len(self.trans)
+ for iso, msgs in self.trans.items():
+ msgs.update_info()
+ if msgs.nbr_msgs > 0:
+ self.lvl += float(msgs.nbr_trans_msgs) / float(msgs.nbr_msgs)
+ self.lvl_ttips += float(msgs.nbr_ttips) / float(msgs.nbr_msgs)
+ self.lvl_comm += float(msgs.nbr_comm_msgs) / float(msgs.nbr_msgs + msgs.nbr_comm_msgs)
+ if msgs.nbr_ttips > 0:
+ self.lvl_trans_ttips = float(msgs.nbr_trans_ttips) / float(msgs.nbr_ttips)
+ if msgs.nbr_trans_msgs > 0:
+ self.lvl_ttips_in_trans = float(msgs.nbr_trans_ttips) / float(msgs.nbr_trans_msgs)
+ if self.nbr_signs == 0:
+ self.nbr_signs = msgs.nbr_signs
+ self.nbr_trans_signs += msgs.nbr_trans_signs
+ self.contexts |= msgs.contexts
+
+ def print_stats(self, prefix="", print_msgs=True):
+ """
+ Print out some stats about an I18n object.
+ If print_msgs is True, it will also print all its translations' stats.
+ """
+ if print_msgs:
+ msgs_prefix = prefix + " "
+ for key, msgs in self.trans:
+ if key == TEMPLATE_ISO_ID:
+ continue
+ print(prefix + key + ":")
+ msgs.print_stats(prefix=msgs_prefix)
+ print(prefix)
+
+ nbr_contexts = len(self.contexts - {CONTEXT_DEFAULT})
+ if nbr_contexts != 1:
+ if nbr_contexts == 0:
+ nbr_contexts = "No"
+ _ctx_txt = "s are"
+ else:
+ _ctx_txt = " is"
+ lines = ("",
+ "Average stats for all {} translations:\n".format(self.nbr_trans),
+ " {:>6.1%} done!\n".format(self.lvl / self.nbr_trans),
+ " {:>6.1%} of messages are tooltips.\n".format(self.lvl_ttips / self.nbr_trans),
+ " {:>6.1%} of tooltips are translated.\n".format(self.lvl_trans_ttips / self.nbr_trans),
+ " {:>6.1%} of translated messages are tooltips.\n".format(self.lvl_ttips_in_trans / self.nbr_trans),
+ " {:>6.1%} of messages are commented.\n".format(self.lvl_comm / self.nbr_trans),
+ " The org msgids are currently made of {} signs.\n".format(self.nbr_signs),
+ " All processed translations are currently made of {} signs.\n".format(self.nbr_trans_signs),
+ " {} specific context{} present:\n {}\n"
+ "".format(self.nbr_contexts, _ctx_txt, "\n ".join(self.contexts - {CONTEXT_DEFAULT})),
+ "\n")
+ print(prefix.join(lines))
+
+
+##### Parsers #####
+
+#def parse_messages_from_pytuple(self, src, key=None):
+ #"""
+ #Returns a dict of tuples similar to the one returned by parse_messages_from_po (one per language, plus a 'pot'
+ #one keyed as '__POT__').
+ #"""
+ ## src may be either a string to be interpreted as py code, or a real tuple!
+ #if isinstance(src, str):
+ #src = eval(src)
+#
+ #curr_hash = None
+ #if key and key in _parser_cache:
+ #old_hash, ret = _parser_cache[key]
+ #import hashlib
+ #curr_hash = hashlib.new(PARSER_CACHE_HASH, str(src).encode()).digest()
+ #if curr_hash == old_hash:
+ #return ret
+#
+ #pot = new_messages()
+ #states = gen_states()
+ #stats = gen_stats()
+ #ret = {"__POT__": (pot, states, stats)}
+ #for msg in src:
+ #key = msg[0]
+ #messages[msgkey] = gen_message(msgid_lines, msgstr_lines, comment_lines, msgctxt_lines)
+ #pot[key] = gen_message(msgid_lines=[key[1]], msgstr_lines=[
+ #for lang, trans, (is_fuzzy, comments) in msg[2:]:
+ #if trans and not is_fuzzy:
+ #i18n_dict.setdefault(lang, dict())[key] = trans
+#
+ #if key:
+ #if not curr_hash:
+ #import hashlib
+ #curr_hash = hashlib.new(PARSER_CACHE_HASH, str(src).encode()).digest()
+ #_parser_cache[key] = (curr_hash, val)
+ #return ret \ No newline at end of file