diff --git a/tools/translate_apps/apps_translations_to_apps.py b/tools/translate_apps/apps_translations_to_apps.py new file mode 100644 index 00000000..6d77c17d --- /dev/null +++ b/tools/translate_apps/apps_translations_to_apps.py @@ -0,0 +1,147 @@ +import time +import json + +from pathlib import Path + +import tomlkit + +from base import Repository, login, token, WORKING_BRANCH, get_repository_branches + + +def extract_strings_to_translate_from_apps(apps, translations_repository): + for app, infos in apps.items(): + repository_uri = infos["git"]["url"].replace("https://github.com/", "") + branch = infos["git"]["branch"] + + if "github.com" not in infos["git"]["url"]: + continue + + if app not in ( + "gotosocial", + "fluffychat", + "cinny", + "fittrackee", + "funkwhale", + "photoprism", + ): + continue + + print() + print(app) + print("=" * len(app)) + print(f"{repository_uri} -> branch '{branch}'") + + translations_path = Path(f"translations/apps/{app}/manifest/") + + if not translations_repository.file_exists(translations_path): + print(f"App {app} doesn't have translations on github.com/yunohost/apps_translations, skip") + continue + + translations_path = translations_repository.path / translations_path + + if "testing" in get_repository_branches(repository_uri, token): + branch = "testing" + + with Repository( + f"https://{login}:{token}@github.com/{repository_uri}", branch + ) as repository: + if not repository.file_exists("manifest.toml"): + continue + + repository.run_command( + [ + "git", + "checkout", + "-b", + WORKING_BRANCH, + "--track", + "origin/{branch}", + ] + ) + + manifest = tomlkit.loads(repository.read_file("manifest.toml")) + + for translation in translations_path.glob("*.json"): + language = translation.name[:-len(".json")] + + # english version is the base, never modify it + if language == "en": + continue + + translation = json.load(open(translation)) + + if translation.get("description", "").strip(): + manifest["description"][language] = translation["description"] + + for question in manifest.get("install", {}): + for strings_to_translate in ["ask", "help"]: + translation_key = f"install_{question}_{strings_to_translate}" + if not translation.get(translation_key, "").strip(): + continue + + if strings_to_translate not in manifest["install"][question]: + continue + + one_of_the_existing_languages = list( + manifest["install"][question][strings_to_translate].keys() + )[0] + current_identation = len( + manifest["install"][question][strings_to_translate][ + one_of_the_existing_languages + ].trivia.indent + ) + manifest["install"][question][strings_to_translate][ + language + ] = translation[translation_key] + manifest["install"][question][strings_to_translate][ + language + ].indent(current_identation) + + repository.write_file("manifest.toml", tomlkit.dumps(manifest)) + + if not repository.run_command("git status -s", capture_output=True).strip(): + continue + + # create or update merge request + repository.run_command("git diff") + repository.run_command("git add manifest.toml") + repository.run_command(["git", "commit", "-m", "feat(i18n): update translations for manifest.toml"]) + repository.run_command(["git", "push", "-f", "origin", f"{WORKING_BRANCH}:manifest_toml_i18n"]) + + if not repository.run_command( + "hub pr list -h manifest_toml_i18n", capture_output=True + ): + repository.run_command( + [ + "hub", + "pull-request", + "-m", + "Update translations for manifest.toml", + "-b", + branch, + "-h", + "manifest_toml_i18n", + "-p", + "-m", + "This pull request is automatically generated by scripts from the " + "[YunoHost/apps](https://github.com/YunoHost/apps) repository.\n\n" + "The translation is pull from weblate and is located here: " + f"https://translate.yunohost.org/projects/yunohost-apps/{app}/\n\n" + "If you wish to modify the translation (other than in english), please do " + "that directly on weblate since this is now the source of authority for it." + "\n\nDon't hesitate to reach the YunoHost team on " + "[matrix](https://matrix.to/#/#yunohost:matrix.org) if there is any " + "problem :heart:", + ] + ) + + time.sleep(2) + + +if __name__ == "__main__": + apps = json.load(open("../../builds/default/v3/apps.json"))["apps"] + + with Repository( + f"https://{login}:{token}@github.com/yunohost/apps_translations", "main" + ) as repository: + extract_strings_to_translate_from_apps(apps, repository) diff --git a/tools/translate_apps/base.py b/tools/translate_apps/base.py new file mode 100644 index 00000000..d739da8d --- /dev/null +++ b/tools/translate_apps/base.py @@ -0,0 +1,116 @@ +import os +import tempfile +import subprocess + +import requests + +from typing import Union +from pathlib import Path + +github_webhook_secret = open("github_webhook_secret", "r").read().strip() + +login = open("login").read().strip() +token = open("token").read().strip() + +weblate_token = open("weblate_token").read().strip() + +my_env = os.environ.copy() +my_env["GIT_TERMINAL_PROMPT"] = "0" +my_env["GIT_AUTHOR_NAME"] = "yunohost-bot" +my_env["GIT_AUTHOR_EMAIL"] = "yunohost@yunohost.org" +my_env["GIT_COMMITTER_NAME"] = "yunohost-bot" +my_env["GIT_COMMITTER_EMAIL"] = "yunohost@yunohost.org" +my_env["GITHUB_USER"] = login +my_env["GITHUB_TOKEN"] = token + +WORKING_BRANCH = "manifest_toml_i18n" + + +def get_repository_branches(repository, token): + branches = requests.get( + f"https://api.github.com/repos/{repository}/branches", + headers={ + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + "Accept": "application/vnd.github+json", + }, + ).json() + + return {x["name"] for x in branches} + + +class Repository: + def __init__(self, url, branch): + self.url = url + self.branch = branch + + def __enter__(self): + self.temporary_directory = tempfile.TemporaryDirectory() + self.path = Path(self.temporary_directory.name) + self.run_command( + [ + "git", + "clone", + self.url, + "--single-branch", + "--branch", + self.branch, + self.path, + ] + ) + + return self + + def run_command( + self, command: Union[str, list], capture_output=False + ) -> Union[str, int, subprocess.CompletedProcess]: + if isinstance(command, str): + kwargs = { + "args": f"cd {self.path} && {command}", + "shell": True, + "env": my_env, + } + + elif isinstance(command, list): + kwargs = {"args": command, "cwd": self.path, "env": my_env} + + if capture_output: + return subprocess.check_output(**kwargs).decode() + else: + print(f"\033[1;31m>>\033[0m \033[0;34m{command}\033[0m") + return subprocess.check_call(**kwargs) + + def run_command_as_if(self, command: Union[str, list]) -> bool: + if isinstance(command, str): + kwargs = { + "args": f"cd {self.path} && {command}", + "shell": True, + "env": my_env, + } + + elif isinstance(command, list): + kwargs = {"args": command, "cwd": self.path, "env": my_env} + + print(f"\033[1;31m>>\033[0m \033[0;34m{command}\033[0m") + return subprocess.run(**kwargs).returncode == 0 + + def file_exists(self, file_name: str) -> bool: + return (self.path / file_name).exists() + + def read_file(self, file_name: str) -> str: + return open((self.path / file_name).resolve(), "r").read() + + def write_file(self, file_name: str, content: str) -> None: + open((self.path / file_name).resolve(), "w").write(content) + + def remove_file(self, file_name: str) -> None: + os.remove(self.path / file_name) + + def append_to_file(self, file_name: str, content: str) -> None: + open((self.path / file_name).resolve(), "a").write(content) + + def __repr__(self): + return f'<__main__.Repository "{self.url.split("@")[1]}" path="{self.path}">' + + def __exit__(self, *args, **kwargs): + pass diff --git a/tools/translate_apps/push_or_update_apps_on_repository.py b/tools/translate_apps/push_or_update_apps_on_repository.py new file mode 100644 index 00000000..453d8939 --- /dev/null +++ b/tools/translate_apps/push_or_update_apps_on_repository.py @@ -0,0 +1,163 @@ +import time +import json + +from pathlib import Path +from collections import defaultdict + +import wlc +import tomlkit + +from base import Repository, login, token, weblate_token, get_repository_branches + + +def get_weblate_component(weblate, component_path): + try: + weblate.get_component(component_path) + except wlc.WeblateException: + return False + else: + return True + + +def extract_strings_to_translate_from_apps(apps, translations_repository): + weblate = wlc.Weblate(key=weblate_token, url="https://translate.yunohost.org/api/") + + # put all languages used on core by default for each component + core_languages_list = {x["language_code"] for x in weblate.get("components/yunohost/core/translations/")["results"]} + + for app, infos in apps.items(): + repository_uri = infos["git"]["url"].replace("https://github.com/", "") + branch = infos["git"]["branch"] + + if "github.com" not in infos["git"]["url"]: + continue + + if app not in ("gotosocial", "fluffychat", "cinny", "fittrackee", "funkwhale", "photoprism"): + continue + + print() + print(app) + print("=" * len(app)) + print(f"{repository_uri} -> branch '{branch}'") + + if "testing" in get_repository_branches(repository_uri, token): + branch = "testing" + + with Repository( + f"https://{login}:{token}@github.com/{repository_uri}", branch + ) as repository: + if not repository.file_exists("manifest.toml"): + continue + + manifest = tomlkit.loads(repository.read_file("manifest.toml")) + + translations_path = Path(f"translations/apps/{app}/manifest/") + + newly_created_translation = False + if not translations_repository.file_exists(translations_path): + (translations_repository.path / translations_path).mkdir(parents=True) + newly_created_translation = True + + translations = defaultdict(dict) + for language, strings_to_translate in manifest.get( + "description", {} + ).items(): + translations[language]["description"] = strings_to_translate + + for question in manifest.get("install", {}): + for strings_to_translate in ["ask", "help"]: + for language, message in ( + manifest["install"][question] + .get(strings_to_translate, {}) + .items() + ): + translations[language][ + f"install_{question}_{strings_to_translate}" + ] = message + + if newly_created_translation: + for language, translated_strings in translations.items(): + translations_repository.write_file( + translations_path / f"{language}.json", + json.dumps(translated_strings, indent=4, sort_keys=True, ensure_ascii=False) + "\n", + ) + else: + translations_repository.write_file( + translations_path / "en.json", + json.dumps(translations["en"], indent=4, sort_keys=True, ensure_ascii=False) + "\n", + ) + + # add strings that aren't already present but don't overwrite existing ones + for language, translated_strings in translations.items(): + if language == "en": + continue + + # if the translation file doesn't exist yet, dump it + if not translations_repository.file_exists(translations_path / f"{language}.json"): + translations_repository.write_file( + translations_path / f"{language}.json", + json.dumps(translated_strings, indent=4, sort_keys=True, ensure_ascii=False) + "\n", + ) + + else: # if it exists, only add keys that aren't already present + language_file = json.loads(translations_repository.read_file(translations_path / f"{language}.json")) + + if "description" in translated_strings and "description" not in language_file: + language_file["description"] = translated_strings["description"] + + for key, translated_string in translated_strings.items(): + if key not in language_file: + language_file[key] = translated_string + + translations_repository.write_file( + translations_path / f"{language}.json", + json.dumps(language_file, indent=4, sort_keys=True, ensure_ascii=False) + "\n", + ) + + # if something has been modified + if translations_repository.run_command("git status -s", capture_output=True).strip(): + translations_repository.run_command("git status -s") + translations_repository.run_command("git diff") + translations_repository.run_command(["git", "add", translations_path]) + translations_repository.run_command( + [ + "git", + "commit", + "-m", + f"feat(apps/i18n): extract strings to translate for application {app}", + ] + ) + translations_repository.run_command(["git", "push"]) + + if newly_created_translation or not get_weblate_component( + weblate, f"yunohost-apps/{app}" + ): + print("Creating component on weblate...") + weblate.create_component( + "yunohost-apps", + name=app, + slug=app, + file_format="json", + filemask=f"translations/apps/{app}/manifest/*.json", + repo="https://github.com/yunohost/apps_translations", + new_base=f"translations/apps/{app}/manifest/en.json", + template=f"translations/apps/{app}/manifest/en.json", + push="git@github.com:yunohost/apps_translations.git", + ) + print(f"Component created at https://translate.yunohost.org/projects/yunohost-apps/{app}/") + + component_existing_languages = {x["language_code"] for x in weblate.get(f"components/yunohost-apps/{app}/translations/")["results"]} + for language_code in sorted(core_languages_list - component_existing_languages): + print(f"Adding available language for translation: {language_code}") + weblate.post(f"components/yunohost-apps/{app}/translations/", **{"language_code": language_code}) + + time.sleep(2) + + +if __name__ == "__main__": + apps = json.load(open("../../builds/default/v3/apps.json"))["apps"] + + with Repository( + f"https://{login}:{token}@github.com/yunohost/apps_translations", "main" + ) as repository: + extract_strings_to_translate_from_apps(apps, repository) diff --git a/tools/translate_apps/requirements.txt b/tools/translate_apps/requirements.txt new file mode 100644 index 00000000..cf79a9fc --- /dev/null +++ b/tools/translate_apps/requirements.txt @@ -0,0 +1 @@ +wlc # weblate api