Copies for all (#1780)
* initial script * copies for all! * revert intl files * mistaken en to vi translation * improve translation * add vi translation, fix trnaalste script to respect existing metdata * revert translation files * fix translation to only add more without changing too much existing translations --------- Co-authored-by: ggurdin <46800240+ggurdin@users.noreply.github.com>pull/1688/head
Wilson
9 months ago
committed by
GitHub
parent
9a94045e5d
commit
76f6222e95
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,304 @@
|
||||
"""
|
||||
Prerequiresite:
|
||||
- Ensure you have an up-to-date `needed-translations.txt` file should you wish to translate only the missing translation keys. To generate an updated `needed-translations.txt` file, run `flutter gen-l10n`
|
||||
- Ensure you have python `openai` package installed. If not, run `pip install openai`.
|
||||
- Ensure you have an OpenAI API key set in your environment variable `OPENAI_API_KEY`. If not, you can set it by running `export OPENAI_API_KEY=your-api-key` on MacOS/Linux.
|
||||
|
||||
Usage:
|
||||
python scripts/translate.py
|
||||
"""
|
||||
|
||||
|
||||
def load_needed_translations() -> dict[str, list[str]]:
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
path_to_needed_translations = (
|
||||
Path(__file__).parent.parent / "needed-translations.txt"
|
||||
)
|
||||
if not path_to_needed_translations.exists():
|
||||
raise FileNotFoundError(
|
||||
f"File not found: {path_to_needed_translations}. Please run `flutter gen-l10n` to generate the file."
|
||||
)
|
||||
with open(path_to_needed_translations) as f:
|
||||
needed_translations = json.loads(f.read())
|
||||
|
||||
return needed_translations
|
||||
|
||||
|
||||
def load_translations(lang_code: str) -> dict[str, str]:
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
path_to_translations = (
|
||||
Path(__file__).parent.parent / "assets" / "l10n" / f"intl_{lang_code}.arb"
|
||||
)
|
||||
if not path_to_translations.exists():
|
||||
raise FileNotFoundError(
|
||||
f"File not found: {path_to_translations}. Please run `flutter gen-l10n` to generate the file."
|
||||
)
|
||||
|
||||
with open(path_to_translations) as f:
|
||||
translations = json.loads(f.read())
|
||||
|
||||
return translations
|
||||
|
||||
|
||||
def save_translations(lang_code: str, translations: dict[str, str]) -> None:
|
||||
import json
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from collections import OrderedDict
|
||||
|
||||
path_to_translations = (
|
||||
Path(__file__).parent.parent / "assets" / "l10n" / f"intl_{lang_code}.arb"
|
||||
)
|
||||
|
||||
translations["@@locale"] = lang_code
|
||||
translations["@@last_modified"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
|
||||
|
||||
# Load existing data to preserve order.
|
||||
if path_to_translations.exists():
|
||||
with open(path_to_translations, "r") as f:
|
||||
try:
|
||||
existing_data = json.load(f, object_pairs_hook=OrderedDict)
|
||||
except json.JSONDecodeError:
|
||||
existing_data = OrderedDict()
|
||||
else:
|
||||
existing_data = OrderedDict()
|
||||
|
||||
# Build final_ordered ensuring the metadata immediately follows its translation.
|
||||
final_ordered = OrderedDict()
|
||||
special_keys = ["@@locale", "@@last_modified"]
|
||||
final_ordered["@@locale"] = translations["@@locale"]
|
||||
final_ordered["@@last_modified"] = translations["@@last_modified"]
|
||||
|
||||
keys_added = set()
|
||||
|
||||
# Use preserved order from existing file.
|
||||
for key in existing_data:
|
||||
if key in special_keys:
|
||||
continue
|
||||
if key in translations:
|
||||
final_ordered[key] = translations[key]
|
||||
keys_added.add(key)
|
||||
meta_key = f"@{key}"
|
||||
if meta_key in translations:
|
||||
final_ordered[meta_key] = translations[meta_key]
|
||||
keys_added.add(meta_key)
|
||||
|
||||
# Append new translation keys (and their metadata immediately after) not in existing data.
|
||||
for key in translations:
|
||||
if key in special_keys or key.startswith("@") or key in keys_added:
|
||||
continue
|
||||
final_ordered[key] = translations[key]
|
||||
meta_key = f"@{key}"
|
||||
if meta_key in translations:
|
||||
final_ordered[meta_key] = translations[meta_key]
|
||||
keys_added.add(meta_key)
|
||||
keys_added.add(key)
|
||||
|
||||
# Append any leftover metadata keys.
|
||||
for key in translations:
|
||||
if key.startswith("@") and key not in keys_added:
|
||||
final_ordered[key] = translations[key]
|
||||
|
||||
with open(path_to_translations, "w") as f:
|
||||
f.write(json.dumps(final_ordered, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
def reconcile_metadata(lang_code: str, translation_keys: list[str]) -> None:
|
||||
"""
|
||||
For each translation key, update its metadata (the key prefixed with '@') by merging
|
||||
any existing metadata with computed metadata. For basic translations, if no metadata exists,
|
||||
add it; otherwise, leave it as is.
|
||||
"""
|
||||
translations = load_translations(lang_code)
|
||||
|
||||
for key in translation_keys:
|
||||
translation = translations[key]
|
||||
meta_key = f"@{key}"
|
||||
existing_meta = translations.get(meta_key, {})
|
||||
assert isinstance(translation, str)
|
||||
|
||||
# Case 1: Basic translations, no placeholders.
|
||||
if "{" not in translation:
|
||||
if not existing_meta:
|
||||
translations[meta_key] = {"type": "text", "placeholders": {}}
|
||||
# if metadata exists, leave it as is.
|
||||
|
||||
# Case 2: Translations with placeholders (no pluralization).
|
||||
elif (
|
||||
"{" in translation
|
||||
and "plural," not in translation
|
||||
and "other{" not in translation
|
||||
):
|
||||
# Compute placeholders.
|
||||
computed_placeholders = {}
|
||||
for placeholder in translation.split("{")[1:]:
|
||||
placeholder_name = placeholder.split("}")[0]
|
||||
computed_placeholders[placeholder_name] = {"type": "String"}
|
||||
if existing_meta:
|
||||
# Merge computed placeholders into existing metadata.
|
||||
existing_meta.setdefault("type", "text")
|
||||
existing_meta["placeholders"] = computed_placeholders
|
||||
translations[meta_key] = existing_meta
|
||||
else:
|
||||
translations[meta_key] = {
|
||||
"type": "text",
|
||||
"placeholders": computed_placeholders,
|
||||
}
|
||||
|
||||
# Case 3: Translations with pluralization.
|
||||
elif (
|
||||
"{" in translation and "plural," in translation and "other{" in translation
|
||||
):
|
||||
# Extract placeholders appearing before the plural part.
|
||||
prefix = translation.split("plural,")[0].split("{")[1]
|
||||
placeholders_list = [
|
||||
p.strip() for p in prefix.split(",") if p.strip() != ""
|
||||
]
|
||||
computed_placeholders = {ph: {} for ph in placeholders_list}
|
||||
if existing_meta:
|
||||
existing_meta.setdefault("type", "text")
|
||||
existing_meta["placeholders"] = computed_placeholders
|
||||
translations[meta_key] = existing_meta
|
||||
else:
|
||||
translations[meta_key] = {
|
||||
"type": "text",
|
||||
"placeholders": computed_placeholders,
|
||||
}
|
||||
|
||||
save_translations(lang_code, translations)
|
||||
|
||||
|
||||
def translate(lang_code: str, lang_display_name: str) -> None:
|
||||
"""
|
||||
Translate the needed translations from English to the target language.
|
||||
"""
|
||||
import json
|
||||
import random
|
||||
from openai import OpenAI
|
||||
|
||||
needed_translations = load_needed_translations()
|
||||
needed_translations = needed_translations.get(lang_code, [])
|
||||
english_translations_dict = load_translations("en")
|
||||
vietnamese_translations_dict = load_translations("vi")
|
||||
|
||||
# there are 3 types of translation keys: basic, with placeholders, with pluralization. Read more: TRANSLATORS_GUIDE.md
|
||||
|
||||
basic_translation_keys = [
|
||||
k
|
||||
for k in english_translations_dict.keys()
|
||||
if not k.startswith("@") and not english_translations_dict[k].startswith("{")
|
||||
]
|
||||
example_basic_translation_keys = (
|
||||
random.sample(basic_translation_keys, 2)
|
||||
if len(basic_translation_keys) > 2
|
||||
else basic_translation_keys
|
||||
)
|
||||
|
||||
placeholder_translation_keys = [
|
||||
k
|
||||
for k in english_translations_dict.keys()
|
||||
if not k.startswith("@")
|
||||
and "{" in english_translations_dict[k]
|
||||
and "plural," not in english_translations_dict[k]
|
||||
and "other{" not in english_translations_dict[k]
|
||||
]
|
||||
example_placeholder_translation_keys = (
|
||||
random.sample(placeholder_translation_keys, 2)
|
||||
if len(placeholder_translation_keys) > 2
|
||||
else placeholder_translation_keys
|
||||
)
|
||||
plural_translation_keys = [
|
||||
k
|
||||
for k in english_translations_dict.keys()
|
||||
if not k.startswith("@")
|
||||
and "{" in english_translations_dict[k]
|
||||
and "plural," in english_translations_dict[k]
|
||||
and "other{" in english_translations_dict[k]
|
||||
]
|
||||
example_plural_translation_keys = (
|
||||
random.sample(plural_translation_keys, 2)
|
||||
if len(plural_translation_keys) > 2
|
||||
else plural_translation_keys
|
||||
)
|
||||
|
||||
# build example translations
|
||||
example_english_translations = {}
|
||||
for key in example_basic_translation_keys:
|
||||
example_english_translations[key] = english_translations_dict[key]
|
||||
for key in example_placeholder_translation_keys:
|
||||
example_english_translations[key] = english_translations_dict[key]
|
||||
for key in example_plural_translation_keys:
|
||||
example_english_translations[key] = english_translations_dict[key]
|
||||
|
||||
example_vietnamese_translations = {}
|
||||
for key in example_basic_translation_keys:
|
||||
example_vietnamese_translations[key] = vietnamese_translations_dict[key]
|
||||
for key in example_placeholder_translation_keys:
|
||||
example_vietnamese_translations[key] = vietnamese_translations_dict[key]
|
||||
for key in example_plural_translation_keys:
|
||||
example_vietnamese_translations[key] = vietnamese_translations_dict[key]
|
||||
|
||||
new_translations = {}
|
||||
progress = 0
|
||||
for i in range(0, len(needed_translations), 20):
|
||||
chunk = needed_translations[i : i + 20]
|
||||
translation_requests = {}
|
||||
for key in chunk:
|
||||
translation_requests[key] = english_translations_dict[key]
|
||||
|
||||
prompt = f"""
|
||||
Please translate the following text from English to {lang_display_name}.
|
||||
Example:
|
||||
req: {json.dumps(example_english_translations, indent=2)}
|
||||
res: {json.dumps(example_vietnamese_translations, indent=2)}
|
||||
========================
|
||||
req: {json.dumps(translation_requests, indent=2)}
|
||||
res:
|
||||
"""
|
||||
|
||||
client = OpenAI()
|
||||
chat_completion = client.chat.completions.create(
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a translator that will only response to translation requests in json format without any additional information.",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": prompt,
|
||||
},
|
||||
],
|
||||
model="gpt-4o-mini",
|
||||
temperature=0.0,
|
||||
)
|
||||
response = chat_completion.choices[0].message.content
|
||||
_new_translations = json.loads(response)
|
||||
new_translations.update(_new_translations)
|
||||
print(f"Translated {progress + len(chunk)}/{len(needed_translations)}")
|
||||
progress += len(chunk)
|
||||
|
||||
# save translations
|
||||
current_translations = load_translations(lang_code)
|
||||
current_translations.update(new_translations)
|
||||
save_translations(lang_code, current_translations)
|
||||
|
||||
# reconcile metadata
|
||||
reconcile_metadata(lang_code, needed_translations)
|
||||
|
||||
|
||||
"""Example usage:
|
||||
python scripts/translate.py
|
||||
"""
|
||||
if __name__ == "__main__":
|
||||
lang_code = input("Enter the language code (e.g. vi, en): ").strip()
|
||||
lang_display_name = input(
|
||||
"Enter the language display name (e.g. Vietnamese, English): "
|
||||
)
|
||||
translate(
|
||||
lang_code=lang_code,
|
||||
lang_display_name=lang_display_name,
|
||||
)
|
||||
Loading…
Reference in New Issue