#!/bin/python3 import argparse import json import os import sys # This allows for importing from the localization and util directories NOTE: Auto importing tools will also prepend the import paths with "tools." this will not work and needs to be removed from import paths sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) from util.time import ExecutionTimer; timer = ExecutionTimer() from dynamicVariables import ( extractVariablesFromDict, identifyLocaleDynamicVariableDifferences, prettyPrintIssuesTable, identifyAndPrintOldDynamicVariables, extractFormattingTags, ) from localization.localeTypes import generateLocalesType from util.logger import console from util.fileUtils import createMappedJsonFileDictionary, writeFile # These string keys are ignored for formatting tag checks ignored_strings_formatting = { "pl": [ # disappearingMessagesTurnedOffYouGroup in pl only has one bold word as the word combines both bold words "disappearingMessagesTurnedOffYouGroup"], "ru": [ # disappearingMessagesTurnedOffGroup in ru only has one bold word as the word combines both bold words "disappearingMessagesTurnedOffGroup"], "sr_CS": [ # disappearingMessagesTurnedOffGroup in sr_CS only has one bold word as the word combines both bold words "disappearingMessagesTurnedOffGroup"] } # If the --throw-error-on-missing flag is passed, the script will exit with an error if there are any missing keys or dynamic variables # This is useful for CI/CD pipelines to ensure that all translations are consistent parser = argparse.ArgumentParser(description="Generate locale files") parser.add_argument( "--error-on-problems", action="store_true", help="Exit with an error if there are any missing keys or dynamic variables", ) parser.add_argument( "--error-old-dynamic-variables", action="store_true", help="Exit with an error if there are any old dynamic variables", ) parser.add_argument( "--print-problems", action="store_true", help="Print the problems table", ) parser.add_argument( "--print-problem-strings", action="store_true", help="Print the problem strings and which locales they are in", ) parser.add_argument( "--print-problem-formatting-tag-strings", action="store_true", help="Print the problem strings and which locales they are in", ) parser.add_argument( "--write-problems", action="store_true", help="Write the problems to a file" ) parser.add_argument( "--problems-file", default="./tools/localization/output/problems.json", help="The file to write the problems to", ) parser.add_argument( "--print-old-dynamic-variables", action="store_true", help="The file to write the problems to", ) parser.add_argument("--en-only", action="store_true", help="Only check the en locale") parser.add_argument("--debug", action="store_true", help="Enable debug mode") parser.add_argument( "--dict-dir", type=str, default="./_locales" ) parser.add_argument( "--dict-file-name", type=str, default="messages.json", ) parser.add_argument( "--en-file-path", type=str, default="./_locales/en/messages.json", ) parser.add_argument( "--generate-types", action="store_true", help="Generate the types file", ) args = parser.parse_args() if args.debug: console.enableDebug() GENERATE_TYPES = args.generate_types OUTPUT_DIR = "./ts/localization" EN_FILE = args.en_file_path INPUT_DIR = args.dict_dir # Create a dictionary that maps locale names to their corresponding JSON file data locales, localeFiles = createMappedJsonFileDictionary(INPUT_DIR, args.dict_file_name) if args.en_only: locales = {"en": locales["en"]} # Generate the locales type and write it to a file if GENERATE_TYPES: generateTypesOutputMessage = generateLocalesType(locales["en"]) console.info(generateTypesOutputMessage) localeVariables = dict() localeVariablesOld = dict() locale_b_tags = dict() locale_br_tags = dict() locale_span_tags = dict() locale_disallowed_tags = dict() locale_improper_tags = dict() # Extract the dynamic variables from each locale and store them in a dictionary for locale, data in locales.items(): console.debug(f"Extracting dynamic variables for {locale}") ( localeVariables[locale], localeVariablesOld[locale], ) = extractVariablesFromDict(data) ( locale_b_tags[locale], locale_br_tags[locale], locale_span_tags[locale], locale_disallowed_tags[locale], locale_improper_tags[locale], ) = extractFormattingTags(data) problems = identifyLocaleDynamicVariableDifferences(localeVariables, locale_b_tags, locale_br_tags, locale_span_tags, locale_disallowed_tags, locale_improper_tags) found_old_dynamic_variables = identifyAndPrintOldDynamicVariables( localeVariablesOld, args.print_old_dynamic_variables ) # Wrapping up the script and printing out the results number_of_tag_problems = 0 if problems: message = "There are issues with the locales." if args.print_problem_strings: string_to_locales = {} for locale, locale_problems in problems.items(): if "additional_variables" in locale_problems: for problem_string in locale_problems["additional_variables"].keys(): if problem_string not in string_to_locales: string_to_locales[problem_string] = [locale] else: string_to_locales[problem_string].append(locale) if "missing_variables" in locale_problems: for problem_string in locale_problems["missing_variables"].keys(): if problem_string not in string_to_locales: string_to_locales[problem_string] = [locale] else: string_to_locales[problem_string].append(locale) if "missing_br_tags" in locale_problems: for problem_string, tag_issues in locale_problems["missing_br_tags"].items(): if tag_issues > 0: if problem_string not in string_to_locales: string_to_locales[problem_string] = [locale] else: string_to_locales[problem_string].append(locale) if "missing_b_tags" in locale_problems: for problem_string, tag_issues in locale_problems["missing_b_tags"].items(): if tag_issues > 0: if problem_string not in string_to_locales: string_to_locales[problem_string] = [locale] else: string_to_locales[problem_string].append(locale) if "missing_span_tags" in locale_problems: for problem_string, tag_issues in locale_problems["missing_span_tags"].items(): if tag_issues > 0: if problem_string not in string_to_locales: string_to_locales[problem_string] = [locale] else: string_to_locales[problem_string].append(locale) if "disallowed_tags" in locale_problems: for problem_string, tag_issues in locale_problems["disallowed_tags"].items(): if tag_issues > 0: if problem_string not in string_to_locales: string_to_locales[problem_string] = [locale] else: string_to_locales[problem_string].append(locale) if "improper_tags" in locale_problems: for problem_string, tag_issues in locale_problems["improper_tags"].items(): if tag_issues > 0: if problem_string not in string_to_locales: string_to_locales[problem_string] = [locale] else: string_to_locales[problem_string].append(locale) console.debug(f"Problem strings: {json.dumps(string_to_locales, indent=2)}") message += " See above for problem strings and which locales they are in." if args.print_problem_formatting_tag_strings: locales_to_strings = {} for locale, locale_problems in problems.items(): locale_missing_br_tags = set() locale_missing_b_tags = set() locale_missing_span_tags = set() locale_disallowed_tags = set() locale_improper_tags = set() if "missing_br_tags" in locale_problems: for problem_string, tag_issues in locale_problems["missing_br_tags"].items(): if tag_issues > 0: locale_missing_br_tags.add(problem_string) if "missing_b_tags" in locale_problems: for problem_string, tag_issues in locale_problems["missing_b_tags"].items(): if tag_issues > 0: locale_missing_b_tags.add(problem_string) if "missing_span_tags" in locale_problems: for problem_string, tag_issues in locale_problems["missing_span_tags"].items(): if tag_issues > 0: locale_missing_span_tags.add(problem_string) if "disallowed_tags" in locale_problems: for problem_string, tag_issues in locale_problems["disallowed_tags"].items(): if tag_issues > 0: locale_disallowed_tags.add(problem_string) if "improper_tags" in locale_problems: for problem_string, tag_issues in locale_problems["improper_tags"].items(): if tag_issues > 0: locale_improper_tags.add(problem_string) locales_to_strings[locale] = { "br": list(locale_missing_br_tags), "b": list(locale_missing_b_tags), "span": list(locale_missing_span_tags), "disallowed_tags": list(locale_disallowed_tags), "improper_tags": list(locale_improper_tags), } if locales_to_strings[locale]["br"] == []: del locales_to_strings[locale]["br"] if locales_to_strings[locale]["b"] == []: del locales_to_strings[locale]["b"] if locales_to_strings[locale]["span"] == []: del locales_to_strings[locale]["span"] if locales_to_strings[locale]["disallowed_tags"] == []: del locales_to_strings[locale]["disallowed_tags"] if locales_to_strings[locale]["improper_tags"] == []: del locales_to_strings[locale]["improper_tags"] console.info(f"Problem strings: {json.dumps(locales_to_strings, indent=2)}") message += " See above for problem strings and which locales they are in." for locale, locale_strings in locales_to_strings.items(): printed_locale = False printed_problem_strings = set() for tag_type, tag_strings in locale_strings.items(): if tag_strings: if locale in ignored_strings_formatting and tag_strings == ignored_strings_formatting[locale]: continue if not printed_locale: print(f"{locale}") printed_locale = True for tag_string in tag_strings: if tag_string not in printed_problem_strings: printed_problem_strings.add(tag_string) number_of_tag_problems += 1 print( f"- [{tag_string}](https://crowdin.com/editor/session-crossplatform-strings/300/en-{locale.replace('-','').replace('_','').lower()}?view=comfortable&filter=basic&value=3#q={tag_string})") print(f"Total Problems: {number_of_tag_problems}") if args.print_problems: prettyPrintIssuesTable(problems) message += " See above for details." if args.write_problems: writeFile(args.problems_file, json.dumps(problems, indent=2)) console.info(f"Problems written to {args.problems_file}") message += f" Problems written to {args.problems_file}" if not args.print_problems and not args.write_problems: message += " Run the script with --print-problems or --write-problems to see the problems." console.warn(message) if found_old_dynamic_variables: warning_message = ( "Old dynamic variables were found in the locales. Please update the locales to use the new dynamic variables. " ) if args.print_old_dynamic_variables: if args.print_problems: warning_message += "See above for details (before the problems table)." else: warning_message += "See above for details." else: warning_message += "Run the script with --print-old-dynamic-variables to see the old dynamic variables." console.warn(warning_message) console.debug("Locales generation complete") timer.stop() if args.error_on_problems: missing_keys_all = 0 additional_keys_all = 0 missing_variables_all = 0 additional_variables_all = 0 for locale_name, locale_issues in problems.items(): if locale_name == "en": continue missing_keys_all += len(locale_issues.get("missing_keys", [])) additional_keys_all += len(locale_issues.get("additional_keys", [])) missing_variables_all += sum( len(v) for v in locale_issues.get("missing_variables", {}).values() ) additional_variables_all += sum( len(v) for v in locale_issues.get("additional_variables", {}).values() ) EXIT_CODE = 0 if missing_keys_all > 0: console.log(f"Missing keys: {missing_keys_all}") if additional_keys_all > 0: console.log(f"Additional keys: {additional_keys_all}") if missing_variables_all > 0: console.log(f"Missing variables: {missing_variables_all}") EXIT_CODE = 1 if additional_variables_all > 0: console.log(f"Additional variables: {additional_variables_all}") EXIT_CODE = 1 if number_of_tag_problems > 0: console.log(f"Formatting issues: {number_of_tag_problems}") EXIT_CODE = 1 sys.exit(EXIT_CODE) sys.exit(0)