mirror of https://github.com/usememos/memos
feat: update Chinese translations in zh.json and zh-Hant.json using locale_updater.py (#1506)
update zh.json and zh-Hant.json using locale_updater.pypull/1507/head
Peng Ding
2 years ago
committed by
GitHub
parent
66ed43cbcb
commit
0020498c10
@ -0,0 +1,202 @@
|
|||||||
|
# Author: Oaklight
|
||||||
|
# GitHub profile: https://github.com/Oaklight
|
||||||
|
# Date: April 9, 2023
|
||||||
|
# Description: This script is used to patch missing translations in a locale file.
|
||||||
|
|
||||||
|
# The script uses 'en.json' as the reference file to find missing keys in other locale files.
|
||||||
|
# It iterates through each field and their entries in 'en.json' and checks if the same field/entry exists in other files.
|
||||||
|
# If a field/entry is missing, the script prompts the source string, reference Google translation, and asks for confirmation or correction.
|
||||||
|
# The resulting file is saved as './*.proposed.json', and you should review it before merging and uploading.
|
||||||
|
|
||||||
|
# usage: locale_updater.py [-h] ref_locale tgt_locale
|
||||||
|
|
||||||
|
#TODO: add other NMT system for different preference and accuracy
|
||||||
|
|
||||||
|
import json
|
||||||
|
import requests
|
||||||
|
|
||||||
|
|
||||||
|
def flatten_json(nested_json, parent_key="", sep=":"):
|
||||||
|
flattened_dict = {}
|
||||||
|
for key, value in nested_json.items():
|
||||||
|
new_key = parent_key + sep + key if parent_key else key
|
||||||
|
if isinstance(value, dict):
|
||||||
|
flattened_dict.update(flatten_json(value, new_key, sep))
|
||||||
|
else:
|
||||||
|
flattened_dict[new_key] = value
|
||||||
|
return flattened_dict
|
||||||
|
|
||||||
|
|
||||||
|
def unflatten_json(flattened_dict, sep=":"):
|
||||||
|
nested_json = {}
|
||||||
|
for key, value in flattened_dict.items():
|
||||||
|
parts = key.split(sep)
|
||||||
|
current = nested_json
|
||||||
|
for part in parts[:-1]:
|
||||||
|
if part not in current:
|
||||||
|
current[part] = {}
|
||||||
|
current = current[part]
|
||||||
|
current[parts[-1]] = value
|
||||||
|
return nested_json
|
||||||
|
|
||||||
|
|
||||||
|
def sort_nested_json(nested_json):
|
||||||
|
if isinstance(nested_json, dict):
|
||||||
|
sorted_dict = {}
|
||||||
|
for key in sorted(nested_json.keys()):
|
||||||
|
sorted_dict[key] = sort_nested_json(nested_json[key])
|
||||||
|
return sorted_dict
|
||||||
|
elif isinstance(nested_json, list):
|
||||||
|
sorted_list = []
|
||||||
|
for item in nested_json:
|
||||||
|
sorted_list.append(sort_nested_json(item))
|
||||||
|
return sorted_list
|
||||||
|
else:
|
||||||
|
return nested_json
|
||||||
|
|
||||||
|
|
||||||
|
def google_translate(
|
||||||
|
source_text, source_language="en", target_language="zh-CN"
|
||||||
|
):
|
||||||
|
# Create post content
|
||||||
|
new_line = "\r\n"
|
||||||
|
post_content = "q=" + source_text.replace(new_line, " ")
|
||||||
|
|
||||||
|
# Send post request and get JSON response, using source_language and target_language
|
||||||
|
# url = "https://translate.googleapis.com/translate_a/single?client=gtx&sl=en&tl=zh-CN&dt=t"
|
||||||
|
url = f"https://translate.googleapis.com/translate_a/single?client=gtx&sl={source_language}&tl={target_language}&dt=t"
|
||||||
|
headers = {"Content-type": "application/x-www-form-urlencoded"}
|
||||||
|
response = requests.post(url, headers=headers, data=post_content.encode("utf-8"))
|
||||||
|
json_value = response.json()
|
||||||
|
|
||||||
|
# Extract translations from JSON
|
||||||
|
translations = [item[0] for item in json_value[0]]
|
||||||
|
translations = [t.replace(new_line, "") for t in translations]
|
||||||
|
target_text = translations[0]
|
||||||
|
|
||||||
|
return target_text
|
||||||
|
|
||||||
|
|
||||||
|
def get_code_name(json_filename):
|
||||||
|
# Remove extension and split language and country codes
|
||||||
|
file_parts = json_filename.split(".")[0].split("_")
|
||||||
|
lang_code = file_parts[0]
|
||||||
|
country_code = file_parts[1] if len(file_parts) > 1 else ""
|
||||||
|
|
||||||
|
# Map language code to code name
|
||||||
|
lang_map = {
|
||||||
|
"de": "de",
|
||||||
|
"en": "en",
|
||||||
|
"es": "es",
|
||||||
|
"fr": "fr",
|
||||||
|
"it": "it",
|
||||||
|
"ko": "ko",
|
||||||
|
"nl": "nl",
|
||||||
|
"pl": "pl",
|
||||||
|
"pt": "pt-BR",
|
||||||
|
"ru": "ru",
|
||||||
|
"sl": "sl",
|
||||||
|
"sv": "sv",
|
||||||
|
"tr": "tr",
|
||||||
|
"uk": "uk",
|
||||||
|
"vi": "vi",
|
||||||
|
"zh-Hant": "zh-TW",
|
||||||
|
"zh": "zh-CN",
|
||||||
|
}
|
||||||
|
code_name = lang_map.get(lang_code, "")
|
||||||
|
|
||||||
|
# Add country code if available
|
||||||
|
if country_code:
|
||||||
|
code_name += "-" + country_code.upper()
|
||||||
|
|
||||||
|
return code_name
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
# ref_locale = "./en.json"
|
||||||
|
# tgt_locale = "./zh.json"
|
||||||
|
# receive the reference locale and target locale from the command line using argparse
|
||||||
|
import argparse
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("ref_locale", help="reference locale file")
|
||||||
|
parser.add_argument("tgt_locale", help="target locale file")
|
||||||
|
args = parser.parse_args()
|
||||||
|
ref_locale = args.ref_locale
|
||||||
|
tgt_locale = args.tgt_locale
|
||||||
|
|
||||||
|
ref_codename = get_code_name(ref_locale)
|
||||||
|
tgt_codename = get_code_name(tgt_locale)
|
||||||
|
|
||||||
|
with open(ref_locale, "r") as f:
|
||||||
|
ref = json.load(f)
|
||||||
|
|
||||||
|
with open(tgt_locale, "r") as f:
|
||||||
|
tgt = json.load(f)
|
||||||
|
|
||||||
|
# using the flatten_json function, produce a temp json for each locale and save to the disk
|
||||||
|
ref_flat = flatten_json(ref)
|
||||||
|
tgt_flat = flatten_json(tgt)
|
||||||
|
|
||||||
|
# # save the flattened json to the disk
|
||||||
|
# with open("ref_flat.json", "w") as f:
|
||||||
|
# json.dump(ref_flat, f, indent=2, ensure_ascii=False)
|
||||||
|
# with open("tgt_flat.json", "w") as f:
|
||||||
|
# json.dump(tgt_flat, f, indent=2, ensure_ascii=False)
|
||||||
|
|
||||||
|
# first diff the keys to inform the user of the missing keys
|
||||||
|
missing_keys = set(ref_flat.keys()) - set(tgt_flat.keys())
|
||||||
|
# print total number of missing keys, in red color, number as default
|
||||||
|
if len(missing_keys) == 0:
|
||||||
|
print("\033[92m All keys are present in the target locale \033[0m")
|
||||||
|
exit()
|
||||||
|
else:
|
||||||
|
print(f"\033[91m Total missing keys: \033[0m {len(missing_keys)}")
|
||||||
|
|
||||||
|
|
||||||
|
# formatted print line by line, wrap the missing key in red color, and the English translation in green color
|
||||||
|
for key in missing_keys:
|
||||||
|
# print(f"Missing key: {key} | English: {ref_flat[key]}")
|
||||||
|
print(
|
||||||
|
"\033[91m"
|
||||||
|
+ f"Missing key: {key}"
|
||||||
|
+ "\033[0m"
|
||||||
|
+ " | "
|
||||||
|
+ "\033[92m"
|
||||||
|
+ f"English: {ref_flat[key]}"
|
||||||
|
+ "\033[0m"
|
||||||
|
)
|
||||||
|
print("=============================================")
|
||||||
|
print(f"\033[91m Total missing keys: \033[0m {len(missing_keys)}")
|
||||||
|
|
||||||
|
|
||||||
|
# now compare the tgt_flat with ref_flat to find all missing keys and prompt to terminal for translation. Then save back to the tgt_flat
|
||||||
|
|
||||||
|
# iterate over the missing key and their corresponding values in ref_flat, to get reference google translation using google_translate_to_chinese function
|
||||||
|
# then present the reference translation to the user in the terminal
|
||||||
|
# then present the user with a prompt to ask for translation
|
||||||
|
for i, key in enumerate(missing_keys):
|
||||||
|
print(
|
||||||
|
f"============================================= {i + 1}/{len(missing_keys)}"
|
||||||
|
)
|
||||||
|
# print wrap the missing key in red color, and the English translation in green color
|
||||||
|
print("\033[91m" + "Missing key: " + "\033[0m" + key)
|
||||||
|
print("\033[92m" + f"{ref_codename}: " + "\033[0m" + ref_flat[key])
|
||||||
|
# get reference translation from google translate, print in blue
|
||||||
|
proposal_google = google_translate(ref_flat[key], ref_codename, tgt_codename)
|
||||||
|
print("\033[94m" + f"Reference {tgt_codename} translation: " + "\033[0m" + proposal_google)
|
||||||
|
# prompt user for translation, or enter to use the reference translation, in green color
|
||||||
|
proposal = input("\033[92m" + "Enter translation: " + "\033[0m")
|
||||||
|
if proposal == "":
|
||||||
|
proposal = proposal_google
|
||||||
|
# save the translation to the tgt_flat
|
||||||
|
tgt_flat[key] = proposal
|
||||||
|
|
||||||
|
# unflatten the ref_flat.json and tgt_flat.json back to the original format. save to another file
|
||||||
|
ref_unflat = unflatten_json(ref_flat)
|
||||||
|
tgt_unflat = unflatten_json(tgt_flat)
|
||||||
|
# save the unflattened json to the disk, with original tgt file name with ".proposed" appended before .json
|
||||||
|
# by getting the file name from from the tgt_locale path
|
||||||
|
tgt_locale_name = tgt_locale.split("/")[-1].split(".")[0]
|
||||||
|
with open(f"{tgt_locale_name}.proposed.json", "w") as f:
|
||||||
|
json.dump(tgt_unflat, f, indent=2, ensure_ascii=False)
|
||||||
Loading…
Reference in New Issue