improvements to translation message generator

- for rna stringsinclude the path as comment, eg:
  #~ bpy.types.VertexGroup.name

- skip strings like %dx%d or -X.

- for strings extracted from python scripts include file:line reference as with C files.

- have messages in order they appear in the file, files and classes sorted so it reads more logically.

- extract these comments from the messages.txt file into the pot file.

2 files changed, 135 insertions, 41 deletions

diff --git a/po/update_msg.py b/po/update_msg.py
index fc3e26b1980..aede99651b1 100644
--- a/po/update_msg.py
+++ b/po/update_msg.py
@@ -30,6 +30,7 @@ CURRENT_DIR = os.path.abspath(os.path.dirname(__file__))
 SOURCE_DIR = os.path.normpath(os.path.abspath(os.path.join(CURRENT_DIR, "..")))
 
 FILE_NAME_MESSAGES = os.path.join(CURRENT_DIR, "messages.txt")
+COMMENT_PREFIX = "#~ "
 
 
 def dump_messages_rna(messages):
@@ -38,51 +39,89 @@ def dump_messages_rna(messages):
     # -------------------------------------------------------------------------
     # Function definitions
 
-    def walkProperties(properties):
+    def walkProperties(bl_rna):
         import bpy
-        for prop in properties:
-            messages.add(prop.name)
-            messages.add(prop.description)
+
+        # get our parents properties not to export them multiple times
+        bl_rna_base = bl_rna.base
+        if bl_rna_base:
+            bl_rna_base_props = bl_rna_base.properties.values()
+        else:
+            bl_rna_base_props = ()
+
+        for prop in bl_rna.properties:
+            # only write this property is our parent hasn't got it.
+            if prop in bl_rna_base_props:
+                continue
+            if prop.identifier == "rna_type":
+                continue
+
+            msgsrc = "bpy.types.%s.%s" % (bl_rna.identifier, prop.identifier)
+            messages.setdefault(prop.name, []).append(msgsrc)
+            messages.setdefault(prop.description, []).append(msgsrc)
 
             if isinstance(prop, bpy.types.EnumProperty):
                 for item in prop.enum_items:
-                    messages.add(item.name)
-                    messages.add(item.description)
+                    msgsrc = "bpy.types.%s.%s, '%s'" % (bl_rna.identifier,
+                                                        prop.identifier,
+                                                        item.identifier,
+                                                        )
+                    messages.setdefault(item.name, []).append(msgsrc)
+                    messages.setdefault(item.description, []).append(msgsrc)
 
     def walkRNA(bl_rna):
+        msgsrc = "bpy.types.%s" % bl_rna.identifier
+
         if bl_rna.name and bl_rna.name != bl_rna.identifier:
-            messages.add(bl_rna.name)
+            messages.setdefault(bl_rna.name, []).append(msgsrc)
 
         if bl_rna.description:
-            messages.add(bl_rna.description)
+            messages.setdefault(bl_rna.description, []).append(msgsrc)
 
-        walkProperties(bl_rna.properties)
+        walkProperties(bl_rna)
 
     def walkClass(cls):
         walkRNA(cls.bl_rna)
 
-    def walk_keymap_hierarchy(hier):
+    def walk_keymap_hierarchy(hier, msgsrc_prev):
         for lvl in hier:
-            messages.add(lvl[0])
+            msgsrc = "%s.%s" % (msgsrc_prev, lvl[1])
+            messages.setdefault(lvl[0], []).append(msgsrc)
 
             if lvl[3]:
-                walk_keymap_hierarchy(lvl[3])
+                walk_keymap_hierarchy(lvl[3], msgsrc)
 
     # -------------------------------------------------------------------------
     # Dump Messages
 
-    for cls in type(bpy.context).__base__.__subclasses__():
+    def full_class_id(cls):
+        """ gives us 'ID.Lamp.AreaLamp' which is best for sorting.
+        """
+        cls_id = ""
+        bl_rna = cls.bl_rna
+        while bl_rna:
+            cls_id = "%s.%s" % (bl_rna.identifier, cls_id)
+            bl_rna = bl_rna.base
+        return cls_id
+
+    cls_list = type(bpy.context).__base__.__subclasses__()
+    cls_list.sort(key=full_class_id)
+    for cls in cls_list:
         walkClass(cls)
 
-    for cls in bpy.types.Space.__subclasses__():
+    cls_list = bpy.types.Space.__subclasses__()
+    cls_list.sort(key=full_class_id)
+    for cls in cls_list:
         walkClass(cls)
 
-    for cls in bpy.types.Operator.__subclasses__():
+    cls_list = bpy.types.Operator.__subclasses__()
+    cls_list.sort(key=full_class_id)
+    for cls in cls_list:
         walkClass(cls)
 
     from bpy_extras.keyconfig_utils import KM_HIERARCHY
 
-    walk_keymap_hierarchy(KM_HIERARCHY)
+    walk_keymap_hierarchy(KM_HIERARCHY, "KM_HIERARCHY")
 
 
     ## XXX. what is this supposed to do, we wrote the file already???
@@ -121,26 +160,30 @@ def dump_messages_pytext(messages):
     # -------------------------------------------------------------------------
     # Function definitions
 
-    def extract_strings(fp, node_container):
+    def extract_strings(fp_rel, node_container):
         """ Recursively get strings, needed incase we have "Blah" + "Blah",
             passed as an argument in that case it wont evaluate to a string.
         """
+
         for node in ast.walk(node_container):
             if type(node) == ast.Str:
                 eval_str = ast.literal_eval(node)
                 if eval_str:
-                    # print("%s:%d: %s" % (fp, node.lineno, eval_str))  # testing
-                    messages.add(eval_str)
+                    # print("%s:%d: %s" % (fp, node.lineno, eval_str))
+                    msgsrc = "%s:%s" % (fp_rel, node.lineno)
+                    messages.setdefault(eval_str, []).append(msgsrc)
 
-    def extract_strings_from_file(fn):
-        filedata = open(fn, 'r', encoding="utf8")
-        root_node = ast.parse(filedata.read(), fn, 'exec')
+    def extract_strings_from_file(fp):
+        filedata = open(fp, 'r', encoding="utf8")
+        root_node = ast.parse(filedata.read(), fp, 'exec')
         filedata.close()
 
+        fp_rel = os.path.relpath(fp, SOURCE_DIR)
+
         for node in ast.walk(root_node):
             if type(node) == ast.Call:
                 # print("found function at")
-                # print("%s:%d" % (fn, node.lineno))
+                # print("%s:%d" % (fp, node.lineno))
 
                 # lambda's
                 if type(node.func) == ast.Name:
@@ -155,29 +198,60 @@ def dump_messages_pytext(messages):
                 # do nothing if not found
                 for arg_kw, arg_pos in translate_args:
                     if arg_pos < len(node.args):
-                        extract_strings(fn, node.args[arg_pos])
+                        extract_strings(fp_rel, node.args[arg_pos])
                     else:
                         for kw in node.keywords:
                             if kw.arg == arg_kw:
-                                extract_strings(fn, kw.value)
+                                extract_strings(fp_rel, kw.value)
 
     # -------------------------------------------------------------------------
     # Dump Messages
 
-    mod_dir = os.path.join(SOURCE_DIR, "release", "scripts", "startup", "bl_ui")
+    mod_dir = os.path.join(SOURCE_DIR,
+                           "release",
+                           "scripts",
+                           "startup",
+                           "bl_ui")
 
-    files = [os.path.join(mod_dir, f)
-             for f in os.listdir(mod_dir)
-             if not f.startswith("_")
-             if f.endswith("py")
+    files = [os.path.join(mod_dir, fn)
+             for fn in sorted(os.listdir(mod_dir))
+             if not fn.startswith("_")
+             if fn.endswith("py")
              ]
 
-    for fn in files:
-        extract_strings_from_file(fn)
+    for fp in files:
+        extract_strings_from_file(fp)
 
 
 def dump_messages():
-    messages = {""}
+
+    def filter_message(msg):
+
+        # check for strings like ": %d"
+        msg_test = msg
+        for ignore in ("%d", "%s", "%r",  # string formatting
+                       "*", ".", "(", ")", "-", "/", "\\", "+", ":", "#", "%"
+                       "0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
+                       "x",  # used on its own eg: 100x200
+                       "X", "Y", "Z",  # used alone. no need to include
+                       ):
+            msg_test = msg_test.replace(ignore, "")
+        msg_test = msg_test.strip()
+        if not msg_test:
+            # print("Skipping: '%s'" % msg)
+            return True
+
+        # we could filter out different strings here
+
+        return False
+
+    if 1:
+        import collections
+        messages = collections.OrderedDict()
+    else:
+        messages = {}
+
+    messages[""] = []
 
     # get strings from RNA
     dump_messages_rna(messages)
@@ -185,10 +259,21 @@ def dump_messages():
     # get strings from UI layout definitions text="..." args
     dump_messages_pytext(messages)
 
-    messages.remove("")
+    del messages[""]
 
     message_file = open(FILE_NAME_MESSAGES, 'w', encoding="utf8")
-    message_file.writelines("\n".join(sorted(messages)))
+    # message_file.writelines("\n".join(sorted(messages)))
+
+    for key, value in messages.items():
+
+        # filter out junk values
+        if filter_message(key):
+            continue
+
+        for msgsrc in value:
+            message_file.write("%s%s\n" % (COMMENT_PREFIX, msgsrc))
+        message_file.write("%s\n" % key)
+
     message_file.close()
 
     print("Written %d messages to: %r" % (len(messages), FILE_NAME_MESSAGES))
diff --git a/po/update_pot.py b/po/update_pot.py
index 33f0b397462..22e1e7a1c1f 100755
--- a/po/update_pot.py
+++ b/po/update_pot.py
@@ -31,6 +31,7 @@ GETTEXT_XGETTEXT_EXECUTABLE = "xgettext"
 CURRENT_DIR = os.path.abspath(os.path.dirname(__file__))
 SOURCE_DIR = os.path.normpath(os.path.abspath(os.path.join(CURRENT_DIR, "..")))
 DOMAIN = "blender"
+COMMENT_PREFIX = "#~ "  # from update_msg.py
 
 FILE_NAME_POT = os.path.join(CURRENT_DIR, "blender.pot")
 FILE_NAME_MESSAGES = os.path.join(CURRENT_DIR, "messages.txt")
@@ -76,6 +77,7 @@ def main():
     # add messages collected automatically from RNA
     with open(FILE_NAME_POT, "a", "utf-8") as pot_handle:
         with open(FILE_NAME_MESSAGES, 'r', "utf-8") as handle:
+            msgsrc_ls = []
             while True:
                 line = handle.readline()
 
@@ -83,13 +85,20 @@ def main():
                     break
 
                 line = stripeol(line)
-                line = line.replace("\\", "\\\\")
-                line = line.replace("\"", "\\\"")
 
-                if not pot_messages.get(line):
-                    pot_handle.write("\n#: Automatically collected from RNA\n")
-                    pot_handle.write("msgid \"%s\"\n" % (line))
-                    pot_handle.write("msgstr \"\"\n")
+                # COMMENT_PREFIX
+                if line.startswith(COMMENT_PREFIX):
+                    msgsrc_ls.append(line[len(COMMENT_PREFIX):].strip())
+                else:
+                    line = line.replace("\\", "\\\\")
+                    line = line.replace("\"", "\\\"")
+
+                    if not pot_messages.get(line):
+                        for msgsrc in msgsrc_ls:
+                            pot_handle.write("#: %s\n" % msgsrc)
+                        pot_handle.write("msgid \"%s\"\n" % line)
+                        pot_handle.write("msgstr \"\"\n\n")
+                    msgsrc_ls[:] = []
 
 
 if __name__ == "__main__":