You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
353 lines
13 KiB
Python
353 lines
13 KiB
Python
#!/bin/python3
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
# This allows for importing from the localization and util directories NOTE: Auto importing tools will also prepend the import paths with "tools." this will not work and needs to be removed from import paths
|
|
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
|
|
|
from util.time import ExecutionTimer;
|
|
|
|
timer = ExecutionTimer()
|
|
|
|
from dynamicVariables import (
|
|
extractVariablesFromDict,
|
|
identifyLocaleDynamicVariableDifferences,
|
|
prettyPrintIssuesTable,
|
|
identifyAndPrintOldDynamicVariables, extractFormattingTags,
|
|
)
|
|
from localization.localeTypes import generateLocalesType
|
|
from util.logger import console
|
|
from util.fileUtils import createMappedJsonFileDictionary, writeFile
|
|
|
|
# These string keys are ignored for formatting tag checks
|
|
ignored_strings_formatting = {
|
|
"pl": [
|
|
# disappearingMessagesTurnedOffYouGroup in pl only has one bold word as the word combines both bold words
|
|
"disappearingMessagesTurnedOffYouGroup"],
|
|
"ru": [
|
|
# disappearingMessagesTurnedOffGroup in ru only has one bold word as the word combines both bold words
|
|
"disappearingMessagesTurnedOffGroup"],
|
|
"sr_CS": [
|
|
# disappearingMessagesTurnedOffGroup in sr_CS only has one bold word as the word combines both bold words
|
|
"disappearingMessagesTurnedOffGroup"]
|
|
}
|
|
|
|
# If the --throw-error-on-missing flag is passed, the script will exit with an error if there are any missing keys or dynamic variables
|
|
# This is useful for CI/CD pipelines to ensure that all translations are consistent
|
|
parser = argparse.ArgumentParser(description="Generate locale files")
|
|
parser.add_argument(
|
|
"--error-on-problems",
|
|
action="store_true",
|
|
help="Exit with an error if there are any missing keys or dynamic variables",
|
|
)
|
|
parser.add_argument(
|
|
"--error-old-dynamic-variables",
|
|
action="store_true",
|
|
help="Exit with an error if there are any old dynamic variables",
|
|
)
|
|
parser.add_argument(
|
|
"--print-problems",
|
|
action="store_true",
|
|
help="Print the problems table",
|
|
)
|
|
parser.add_argument(
|
|
"--print-problem-strings",
|
|
action="store_true",
|
|
help="Print the problem strings and which locales they are in",
|
|
)
|
|
parser.add_argument(
|
|
"--print-problem-formatting-tag-strings",
|
|
action="store_true",
|
|
help="Print the problem strings and which locales they are in",
|
|
)
|
|
parser.add_argument(
|
|
"--write-problems", action="store_true", help="Write the problems to a file"
|
|
)
|
|
parser.add_argument(
|
|
"--problems-file",
|
|
default="./tools/localization/output/problems.json",
|
|
help="The file to write the problems to",
|
|
)
|
|
parser.add_argument(
|
|
"--print-old-dynamic-variables",
|
|
action="store_true",
|
|
help="The file to write the problems to",
|
|
)
|
|
parser.add_argument("--en-only", action="store_true", help="Only check the en locale")
|
|
parser.add_argument("--debug", action="store_true", help="Enable debug mode")
|
|
parser.add_argument(
|
|
"--dict-dir",
|
|
type=str,
|
|
default="./_locales"
|
|
)
|
|
parser.add_argument(
|
|
"--dict-file-name",
|
|
type=str,
|
|
default="messages.json",
|
|
)
|
|
parser.add_argument(
|
|
"--en-file-path",
|
|
type=str,
|
|
default="./_locales/en/messages.json",
|
|
)
|
|
parser.add_argument(
|
|
"--generate-types",
|
|
action="store_true",
|
|
help="Generate the types file",
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
if args.debug:
|
|
console.enableDebug()
|
|
|
|
GENERATE_TYPES = args.generate_types
|
|
OUTPUT_DIR = "./ts/localization"
|
|
EN_FILE = args.en_file_path
|
|
INPUT_DIR = args.dict_dir
|
|
|
|
# Create a dictionary that maps locale names to their corresponding JSON file data
|
|
locales, localeFiles = createMappedJsonFileDictionary(INPUT_DIR, args.dict_file_name)
|
|
|
|
if args.en_only:
|
|
locales = {"en": locales["en"]}
|
|
|
|
# Generate the locales type and write it to a file
|
|
if GENERATE_TYPES:
|
|
generateTypesOutputMessage = generateLocalesType(locales["en"])
|
|
console.info(generateTypesOutputMessage)
|
|
|
|
localeVariables = dict()
|
|
localeVariablesOld = dict()
|
|
locale_b_tags = dict()
|
|
locale_br_tags = dict()
|
|
locale_span_tags = dict()
|
|
locale_disallowed_tags = dict()
|
|
locale_improper_tags = dict()
|
|
# Extract the dynamic variables from each locale and store them in a dictionary
|
|
for locale, data in locales.items():
|
|
console.debug(f"Extracting dynamic variables for {locale}")
|
|
(
|
|
localeVariables[locale],
|
|
localeVariablesOld[locale],
|
|
) = extractVariablesFromDict(data)
|
|
(
|
|
locale_b_tags[locale],
|
|
locale_br_tags[locale],
|
|
locale_span_tags[locale],
|
|
locale_disallowed_tags[locale],
|
|
locale_improper_tags[locale],
|
|
) = extractFormattingTags(data)
|
|
|
|
problems = identifyLocaleDynamicVariableDifferences(localeVariables, locale_b_tags,
|
|
locale_br_tags,
|
|
locale_span_tags, locale_disallowed_tags, locale_improper_tags)
|
|
|
|
found_old_dynamic_variables = identifyAndPrintOldDynamicVariables(
|
|
localeVariablesOld, args.print_old_dynamic_variables
|
|
)
|
|
|
|
# Wrapping up the script and printing out the results
|
|
number_of_tag_problems = 0
|
|
|
|
if problems:
|
|
message = "There are issues with the locales."
|
|
|
|
if args.print_problem_strings:
|
|
string_to_locales = {}
|
|
for locale, locale_problems in problems.items():
|
|
if "additional_variables" in locale_problems:
|
|
for problem_string in locale_problems["additional_variables"].keys():
|
|
if problem_string not in string_to_locales:
|
|
string_to_locales[problem_string] = [locale]
|
|
else:
|
|
string_to_locales[problem_string].append(locale)
|
|
if "missing_variables" in locale_problems:
|
|
for problem_string in locale_problems["missing_variables"].keys():
|
|
if problem_string not in string_to_locales:
|
|
string_to_locales[problem_string] = [locale]
|
|
else:
|
|
string_to_locales[problem_string].append(locale)
|
|
if "missing_br_tags" in locale_problems:
|
|
for problem_string, tag_issues in locale_problems["missing_br_tags"].items():
|
|
if tag_issues > 0:
|
|
if problem_string not in string_to_locales:
|
|
string_to_locales[problem_string] = [locale]
|
|
else:
|
|
string_to_locales[problem_string].append(locale)
|
|
if "missing_b_tags" in locale_problems:
|
|
for problem_string, tag_issues in locale_problems["missing_b_tags"].items():
|
|
if tag_issues > 0:
|
|
if problem_string not in string_to_locales:
|
|
string_to_locales[problem_string] = [locale]
|
|
else:
|
|
string_to_locales[problem_string].append(locale)
|
|
if "missing_span_tags" in locale_problems:
|
|
for problem_string, tag_issues in locale_problems["missing_span_tags"].items():
|
|
if tag_issues > 0:
|
|
if problem_string not in string_to_locales:
|
|
string_to_locales[problem_string] = [locale]
|
|
else:
|
|
string_to_locales[problem_string].append(locale)
|
|
if "disallowed_tags" in locale_problems:
|
|
for problem_string, tag_issues in locale_problems["disallowed_tags"].items():
|
|
if tag_issues > 0:
|
|
if problem_string not in string_to_locales:
|
|
string_to_locales[problem_string] = [locale]
|
|
else:
|
|
string_to_locales[problem_string].append(locale)
|
|
if "improper_tags" in locale_problems:
|
|
for problem_string, tag_issues in locale_problems["improper_tags"].items():
|
|
if tag_issues > 0:
|
|
if problem_string not in string_to_locales:
|
|
string_to_locales[problem_string] = [locale]
|
|
else:
|
|
string_to_locales[problem_string].append(locale)
|
|
|
|
console.debug(f"Problem strings: {json.dumps(string_to_locales, indent=2)}")
|
|
message += " See above for problem strings and which locales they are in."
|
|
|
|
if args.print_problem_formatting_tag_strings:
|
|
locales_to_strings = {}
|
|
for locale, locale_problems in problems.items():
|
|
locale_missing_br_tags = set()
|
|
locale_missing_b_tags = set()
|
|
locale_missing_span_tags = set()
|
|
locale_disallowed_tags = set()
|
|
locale_improper_tags = set()
|
|
if "missing_br_tags" in locale_problems:
|
|
for problem_string, tag_issues in locale_problems["missing_br_tags"].items():
|
|
if tag_issues > 0:
|
|
locale_missing_br_tags.add(problem_string)
|
|
if "missing_b_tags" in locale_problems:
|
|
for problem_string, tag_issues in locale_problems["missing_b_tags"].items():
|
|
if tag_issues > 0:
|
|
locale_missing_b_tags.add(problem_string)
|
|
if "missing_span_tags" in locale_problems:
|
|
for problem_string, tag_issues in locale_problems["missing_span_tags"].items():
|
|
if tag_issues > 0:
|
|
locale_missing_span_tags.add(problem_string)
|
|
if "disallowed_tags" in locale_problems:
|
|
for problem_string, tag_issues in locale_problems["disallowed_tags"].items():
|
|
if tag_issues > 0:
|
|
locale_disallowed_tags.add(problem_string)
|
|
if "improper_tags" in locale_problems:
|
|
for problem_string, tag_issues in locale_problems["improper_tags"].items():
|
|
if tag_issues > 0:
|
|
locale_improper_tags.add(problem_string)
|
|
|
|
locales_to_strings[locale] = {
|
|
"br": list(locale_missing_br_tags),
|
|
"b": list(locale_missing_b_tags),
|
|
"span": list(locale_missing_span_tags),
|
|
"disallowed_tags": list(locale_disallowed_tags),
|
|
"improper_tags": list(locale_improper_tags),
|
|
}
|
|
|
|
if locales_to_strings[locale]["br"] == []:
|
|
del locales_to_strings[locale]["br"]
|
|
if locales_to_strings[locale]["b"] == []:
|
|
del locales_to_strings[locale]["b"]
|
|
if locales_to_strings[locale]["span"] == []:
|
|
del locales_to_strings[locale]["span"]
|
|
if locales_to_strings[locale]["disallowed_tags"] == []:
|
|
del locales_to_strings[locale]["disallowed_tags"]
|
|
if locales_to_strings[locale]["improper_tags"] == []:
|
|
del locales_to_strings[locale]["improper_tags"]
|
|
|
|
console.info(f"Problem strings: {json.dumps(locales_to_strings, indent=2)}")
|
|
message += " See above for problem strings and which locales they are in."
|
|
for locale, locale_strings in locales_to_strings.items():
|
|
printed_locale = False
|
|
printed_problem_strings = set()
|
|
for tag_type, tag_strings in locale_strings.items():
|
|
if tag_strings:
|
|
if locale in ignored_strings_formatting and tag_strings == ignored_strings_formatting[locale]:
|
|
continue
|
|
if not printed_locale:
|
|
print(f"{locale}")
|
|
printed_locale = True
|
|
for tag_string in tag_strings:
|
|
if tag_string not in printed_problem_strings:
|
|
printed_problem_strings.add(tag_string)
|
|
number_of_tag_problems += 1
|
|
print(
|
|
f"- [{tag_string}](https://crowdin.com/editor/session-crossplatform-strings/300/en-{locale.replace('-','').replace('_','').lower()}?view=comfortable&filter=basic&value=3#q={tag_string})")
|
|
print(f"Total Problems: {number_of_tag_problems}")
|
|
|
|
if args.print_problems:
|
|
prettyPrintIssuesTable(problems)
|
|
message += " See above for details."
|
|
|
|
if args.write_problems:
|
|
writeFile(args.problems_file, json.dumps(problems, indent=2))
|
|
console.info(f"Problems written to {args.problems_file}")
|
|
message += f" Problems written to {args.problems_file}"
|
|
|
|
if not args.print_problems and not args.write_problems:
|
|
message += " Run the script with --print-problems or --write-problems to see the problems."
|
|
|
|
console.warn(message)
|
|
|
|
if found_old_dynamic_variables:
|
|
warning_message = (
|
|
"Old dynamic variables were found in the locales. Please update the locales to use the new dynamic variables. "
|
|
)
|
|
if args.print_old_dynamic_variables:
|
|
if args.print_problems:
|
|
warning_message += "See above for details (before the problems table)."
|
|
else:
|
|
warning_message += "See above for details."
|
|
else:
|
|
warning_message += "Run the script with --print-old-dynamic-variables to see the old dynamic variables."
|
|
console.warn(warning_message)
|
|
|
|
console.debug("Locales generation complete")
|
|
|
|
timer.stop()
|
|
|
|
if args.error_on_problems:
|
|
missing_keys_all = 0
|
|
additional_keys_all = 0
|
|
missing_variables_all = 0
|
|
additional_variables_all = 0
|
|
|
|
for locale_name, locale_issues in problems.items():
|
|
if locale_name == "en":
|
|
continue
|
|
|
|
missing_keys_all += len(locale_issues.get("missing_keys", []))
|
|
additional_keys_all += len(locale_issues.get("additional_keys", []))
|
|
missing_variables_all += sum(
|
|
len(v) for v in locale_issues.get("missing_variables", {}).values()
|
|
)
|
|
additional_variables_all += sum(
|
|
len(v) for v in locale_issues.get("additional_variables", {}).values()
|
|
)
|
|
|
|
EXIT_CODE = 0
|
|
|
|
if missing_keys_all > 0:
|
|
console.log(f"Missing keys: {missing_keys_all}")
|
|
|
|
if additional_keys_all > 0:
|
|
console.log(f"Additional keys: {additional_keys_all}")
|
|
|
|
if missing_variables_all > 0:
|
|
console.log(f"Missing variables: {missing_variables_all}")
|
|
EXIT_CODE = 1
|
|
|
|
if additional_variables_all > 0:
|
|
console.log(f"Additional variables: {additional_variables_all}")
|
|
EXIT_CODE = 1
|
|
|
|
if number_of_tag_problems > 0:
|
|
console.log(f"Formatting issues: {number_of_tag_problems}")
|
|
EXIT_CODE = 1
|
|
|
|
sys.exit(EXIT_CODE)
|
|
|
|
sys.exit(0)
|