diff options
| -rw-r--r-- | data/hardcoded_categories.txt | 16 | ||||
| -rwxr-xr-x | tools/python/clean_strings_txt.py | 322 | ||||
| -rwxr-xr-x | tools/python/find_untranslated_strings.py | 44 |
3 files changed, 371 insertions, 11 deletions
diff --git a/data/hardcoded_categories.txt b/data/hardcoded_categories.txt new file mode 100644 index 0000000000..0456510b89 --- /dev/null +++ b/data/hardcoded_categories.txt @@ -0,0 +1,16 @@ +food +hotel +tourism +wifi +transport +fuel +parking +shop +atm +bank +entertainment +hospital +pharmacy +police +toilet +post diff --git a/tools/python/clean_strings_txt.py b/tools/python/clean_strings_txt.py new file mode 100755 index 0000000000..2d0942575c --- /dev/null +++ b/tools/python/clean_strings_txt.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python2.7 + +from __future__ import print_function + +import logging +import re +import subprocess +from argparse import ArgumentParser +from collections import defaultdict +from itertools import chain +from os.path import abspath, isabs + +from find_untranslated_strings import StringsTxt + +""" +This script determines which strings are used in the platform code (iOS and +Android) and removes all the other strings from strings.txt. For more information, +run this script with the -h option. +""" + + +OMIM_ROOT = "" + +MACRO_RE = re.compile('L\(.*?@\"(.*?)\"\)') +XML_RE = re.compile("value=\"(.*?)\"") +ANDROID_JAVA_RE = re.compile("R\.string\.([\w_]*)") +ANDROID_XML_RE = re.compile("@string/(.*?)\W") +IOS_CANDIDATES_RE = re.compile("(.*?):[^L\(]@\"([a-z0-9_]*?)\"") + +HARDCODED_CATEGORIES = None + +HARDCODED_COLORS = [ + "red", "yellow", "blue", "green", "purple", "orange", "brown", "pink" +] + + +def exec_shell(test, *flags): + spell = ["{0} {1}".format(test, list(*flags))] + + process = subprocess.Popen( + spell, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=True + ) + + logging.info(" ".join(spell)) + out, _ = process.communicate() + return filter(None, out.splitlines()) + + +def grep_ios(): + logging.info("Grepping ios") + grep = "grep -r -I 'L(\|localizedText\|localizedPlaceholder' {0}/iphone/*".format(OMIM_ROOT) + ret = exec_shell(grep) + return filter_ios_grep(ret) + + +def grep_android(): + logging.info("Grepping android") + grep = "grep -r -I 'R.string.' {0}/android/src".format(OMIM_ROOT) + ret = android_grep_wrapper(grep, ANDROID_JAVA_RE) + grep = "grep -r -I '@string/' {0}/android/res".format(OMIM_ROOT) + ret.update(android_grep_wrapper(grep, ANDROID_XML_RE)) + grep = "grep -r -I '@string/' {0}/android/AndroidManifest.xml".format(OMIM_ROOT) + ret.update(android_grep_wrapper(grep, ANDROID_XML_RE)) + + return parenthesize(ret) + + +def grep_ios_candidates(): + logging.info("Grepping ios candidates") + grep = "grep -nr -I '@\"' {0}/iphone/*".format(OMIM_ROOT) + ret = exec_shell(grep) + + strs = strings_from_grepped(ret, IOS_CANDIDATES_RE) + return strs + + +def android_grep_wrapper(grep, regex): + grepped = exec_shell(grep) + return strings_from_grepped(grepped, regex) + + +def filter_ios_grep(strings): + filtered = strings_from_grepped(strings, MACRO_RE) + filtered = parenthesize(process_ternary_operators(filtered)) + filtered.update(parenthesize(strings_from_grepped(strings, XML_RE))) + filtered.update(parenthesize(HARDCODED_CATEGORIES)) + filtered.update(parenthesize(HARDCODED_COLORS)) + return filtered + + +def process_ternary_operators(filtered): + return chain(*map(lambda s: s.split('" : @"'), filtered)) + + +def strings_from_grepped(grepped, regexp): + return set( + chain( + *filter( + None, map(lambda s: regexp.findall(s), grepped) + ) + ) + ) + + +def parenthesize(strings): + return set(map(lambda s: "[{0}]".format(s), strings)) + + +def write_filtered_strings_txt(filtered, filepath, languages=None): + logging.info("Writing strings to file {0}".format(filepath)) + strings_txt = StringsTxt() + strings_dict = {key : dict(strings_txt.translations[key]) for key in filtered} + strings_txt.translations = strings_dict + strings_txt.comments_and_tags = {} + strings_txt.write_formatted(filepath, languages=languages) + + +def get_args(): + parser = ArgumentParser( + description=""" + A script for cleaning up the strings.txt file. It can cleanup the file + inplace, that is all the unused strings will be removed from strings.txt, + or it can produce two separate files for ios and android. We can also + produce the compiled string resources specifically for each platform + that do not contain strings for other platforms or comments.""" + ) + + parser.add_argument( + "-s", "--single-file", + dest="single", default=False, + action="store_true", + help="""Create single cleaned up file for both platform. All strings + that are not used in the project will be thrown away. Otherwise, two + platform specific files will be produced.""" + ) + + parser.add_argument( + "-l", "--language", + dest="langs", default=None, + action="append", + help="""The languages to be included into the resulting strings.txt + file or files. If this param is empty, all languages from the current + strings.txt file will be preserved.""" + ) + + parser.add_argument( + "-g", "--generate-localizations", + dest="generate", default=False, + action="store_true", + help="Generate localizations for the platforms." + ) + + parser.add_argument( + "-o", "--output", + dest="output", default="strings.txt", + help="""The name for the resulting file. It will be saved to the + project folder. Only relevant if the -s option is set.""" + ) + + parser.add_argument( + "-m", "--missing-strings", + dest="missing", default=False, + action="store_true", + help="""Find the keys that are used in iOS, but are not translated + in strings.txt and exit.""" + ) + + parser.add_argument( + "-c", "--candidates", + dest="candidates", default=False, + action="store_true", + help="""Find the strings in iOS that are not in the L() macros, but that + look like they might be keys.""" + ) + + parser.add_argument( + "-r", "--root", + dest="omim_root", default=find_omim(), + help="Path to the root of the OMIM project" + ) + + parser.add_argument( + "-ct", "--categories", + dest="hardcoded_cagegories", + default="{0}/data/hardcoded_categories.txt".format(find_omim()), + help="""Path to the list of the categories that are displayed in the + interface, but are not taken from strings.txt""" + ) + + return parser.parse_args() + + +def do_multiple(args): + write_filtered_strings_txt( + grep_ios(), "{0}/ios_strings.txt".format(OMIM_ROOT), args.langs + ) + write_filtered_strings_txt( + grep_android(), "{0}/android_strings.txt".format(OMIM_ROOT), args.langs + ) + if args.generate: + print("Going to generate locs") + exec_shell( + "{0}/tools/unix/generate_localizations.sh".format(OMIM_ROOT), + "android_strings.txt", "ios_strings.txt" + ) + + +def generate_auto_tags(ios, android): + new_tags = defaultdict(list) + for i in ios: + new_tags[i].append("ios") + + for a in android: + new_tags[a].append("android") + + for key in new_tags: + new_tags[key] = ", ".join(new_tags[key]) + + return new_tags + + +def new_comments_and_tags(strings_txt, filtered, new_tags): + comments_and_tags = {key: strings_txt.comments_and_tags[key] for key in filtered} + for key in comments_and_tags: + comments_and_tags[key]["tags"] = new_tags[key] + return comments_and_tags + + +def do_single(args): + ios = grep_ios() + android = grep_android() + + new_tags = generate_auto_tags(ios, android) + + filtered = ios + filtered.update(android) + + strings_txt = StringsTxt() + strings_txt.translations = {key: dict(strings_txt.translations[key]) for key in filtered} + + strings_txt.comments_and_tags = new_comments_and_tags(strings_txt, filtered, new_tags) + + path = args.output if isabs(args.output) else "{0}/{1}".format(OMIM_ROOT, args.output) + strings_txt.write_formatted(languages=args.langs, target_file=path) + + if args.generate: + exec_shell( + "{}/unix/generate_localizations.sh".format(OMIM_ROOT), + args.output, args.output + ) + +def do_missing(args): + ios = set(grep_ios()) + strings_txt_keys = set(StringsTxt().translations.keys()) + missing = ios - strings_txt_keys + + if missing: + for m in missing: + logging.info(m) + exit(1) + logging.info("Ok. No missing strings.") + exit(0) + + +def do_candidates(args): + all_candidates = defaultdict(list) + for source, candidate in grep_ios_candidates(): + all_candidates[candidate].append(source) + + for candidate, sources in all_candidates.iteritems(): + print(candidate, sources) + + +def do_ios_suspects(args): + grep = "grep -re -I 'L(' {}/iphone/*".format(OMIM_ROOT) + suspects = exec_shell(grep) + SUSPECT_RE = re.compile(r"(.*?):.*?\WL\(([^@].*?)\)") + strings = strings_from_grepped(suspects, SUSPECT_RE) + for s in strings: + print(s) + + +def find_omim(): + my_path = abspath(__file__) + tools_index = my_path.rfind("/tools/python") + omim_path = my_path[:tools_index] + return omim_path + + +def read_hardcoded_categories(a_path): + with open(a_path) as infile: + return filter(None, [s.strip() for s in infile]) + + +if __name__ == "__main__": + global OMIM_ROOT, HARDCODED_CATEGORIES + + logging.basicConfig(level=logging.DEBUG) + args = get_args() + + OMIM_ROOT=args.omim_root + + HARDCODED_CATEGORIES = read_hardcoded_categories( + args.hardcoded_cagegories + ) + + args.langs = set(args.langs) if args.langs else None + + if args.missing: + do_missing(args) + exit(0) + + if args.candidates: + do_candidates(args) + exit(0) + + if args.single: + do_single(args) + else: + do_multiple(args) diff --git a/tools/python/find_untranslated_strings.py b/tools/python/find_untranslated_strings.py index 51084e4a00..a5ec529eb7 100755 --- a/tools/python/find_untranslated_strings.py +++ b/tools/python/find_untranslated_strings.py @@ -21,8 +21,12 @@ SIMILARITY_THRESHOLD = 20.0 #% class StringsTxt: - def __init__(self): - self.strings_path = join(dirname(argv[0]), "..", "..", "strings.txt") + def __init__(self, strings_path=None): + if not strings_path: + self.strings_path = join(dirname(argv[0]), "..", "..", "strings.txt") + else: + self.strings_path = strings_path + self.translations = defaultdict(dict) # dict<key, dict<lang, translation>> self.translations_by_language = defaultdict(dict) # dict<lang, dict<key, translation>> self.comments_and_tags = defaultdict(dict) @@ -31,6 +35,8 @@ class StringsTxt: self.duplicates = {} # dict<lang, TransAndKey> self.keys_in_order = [] self._read_file() + + def process_file(self): self._populate_translations_by_langs() self._find_duplicates() self.most_duplicated = [] @@ -163,29 +169,44 @@ class StringsTxt: print("{0}:\n{1}\n".format(lang, "\n".join(missing_keys))) - def write_formatted(self): + def write_formatted(self, target_file=None, languages=None): + before_block = "" + if target_file is None: + target_file = self.strings_path non_itunes_langs = sorted(list(self.all_langs - set(ITUNES_LANGS))) - with open(self.strings_path, "w") as outfile: + with open(target_file, "w") as outfile: for key in self.keys_in_order: + if not key: + continue if key in self.translations: tran = self.translations[key] else: - outfile.write("{0}\n\n".format(key)) + if key.startswith("[["): + outfile.write("{0}{1}\n".format(before_block, key)) + before_block = "\n" continue - outfile.write(" {0}\n".format(key)) + outfile.write("{0} {1}\n".format(before_block, key)) + before_block = "\n" + if key in self.comments_and_tags: for k, v in self.comments_and_tags[key].items(): outfile.write(" {0} = {1}\n".format(k, v)) + self._write_translations_for_langs(ITUNES_LANGS, tran, outfile, only_langs=languages) + self._write_translations_for_langs(non_itunes_langs, tran, outfile, only_langs=languages) - self._write_translations_for_langs(ITUNES_LANGS, tran, outfile) - self._write_translations_for_langs(non_itunes_langs, tran, outfile) - outfile.write("\n") + def _write_translations_for_langs(self, langs, tran, outfile, only_langs=None): + langs_to_write = [] + if only_langs: + for lang in only_langs: + if lang in langs: + langs_to_write.append(lang) + else: + langs_to_write = langs - def _write_translations_for_langs(self, langs, tran, outfile): - for lang in langs: + for lang in langs_to_write: if lang in tran: outfile.write(" {0} = {1}\n".format(lang, tran[lang])) @@ -284,6 +305,7 @@ class StringsTxt: if __name__ == "__main__": strings = StringsTxt() + strings.process_file() strings.print_statistics() strings.print_duplicates() strings.print_most_duplicated() |