diff --git a/.github/workflows/added_dates.yml b/.github/workflows/added_dates.yml index 6a32fd44..397a7a5b 100644 --- a/.github/workflows/added_dates.yml +++ b/.github/workflows/added_dates.yml @@ -13,6 +13,7 @@ jobs: with: fetch-depth: 0 token: ${{ secrets.YUNOHOST_BOT_TOKEN }} + - name: Set up Python 3.11 uses: actions/setup-python@v5 with: diff --git a/.github/workflows/auto_messages_pot.yml b/.github/workflows/auto_messages_pot.yml index c290b6f8..6640e8e3 100644 --- a/.github/workflows/auto_messages_pot.yml +++ b/.github/workflows/auto_messages_pot.yml @@ -14,25 +14,29 @@ jobs: uses: actions/setup-python@v5 with: python-version: 3.11 + - name: Install dependencies run: | cd tools/readme_generator/ pip install -r requirements.txt + - name: Save old messges.pot for diffing later run: | cd tools/readme_generator/ cp messages.pot messages.pot.old + - name: Try to generate messages.pot run: | cd tools/readme_generator/ pybabel extract --ignore-dirs venv -F babel.cfg -o messages.pot . - - shell: pwsh - id: check_files_changed + + - name: Check if files changed run: | - $HasDiff = python .github/diff_pot_files.py tools/readme_generator/messages.pot.old tools/readme_generator/messages.pot - Write-Host "::set-output name=files_changed::$HasDiff" + HAS_DIFF=$(python .github/diff_pot_files.py tools/readme_generator/messages.pot.old tools/readme_generator/messages.pot) + echo "HAS_DIFF=$HAS_DIFF" >> $GITHUB_ENV + - name: Create Pull Request - if: steps.check_files_changed.outputs.files_changed == 'true' + if: ${{ env.HAS_DIFF == 'true' }} uses: peter-evans/create-pull-request@v6 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/autoblack.yml b/.github/workflows/autoblack.yml index 761222be..25c8e8b3 100644 --- a/.github/workflows/autoblack.yml +++ b/.github/workflows/autoblack.yml @@ -15,15 +15,8 @@ jobs: with: options: "." continue-on-error: true - - shell: pwsh - id: check_files_changed - run: | - # Diff HEAD with the previous commit - $diff = git diff - $HasDiff = $diff.Length -gt 0 - Write-Host "::set-output name=files_changed::$HasDiff" + - name: Create Pull Request - if: steps.check_files_changed.outputs.files_changed == 'true' uses: peter-evans/create-pull-request@v6 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test_toml.yaml b/.github/workflows/test_toml.yaml index 8cd0e4b4..6c67ec9f 100644 --- a/.github/workflows/test_toml.yaml +++ b/.github/workflows/test_toml.yaml @@ -11,15 +11,8 @@ jobs: - uses: actions/checkout@v4 - uses: uncenter/setup-taplo@v1 - run: taplo fmt - - shell: pwsh - id: check_files_changed - run: | - # Diff HEAD with the previous commit - $diff = git diff - $HasDiff = $diff.Length -gt 0 - Write-Host "::set-output name=files_changed::$HasDiff" + - name: Create Pull Request - if: steps.check_files_changed.outputs.files_changed == 'true' uses: peter-evans/create-pull-request@v6 with: token: ${{ secrets.GITHUB_TOKEN }} @@ -28,4 +21,4 @@ jobs: body: | This pull request uses the [Taplo](https://taplo.tamasfe.dev) formatter. base: ${{ github.head_ref }} # Creates pull request onto pull request or commit branch - branch: actions/toml \ No newline at end of file + branch: actions/toml diff --git a/apps.toml b/apps.toml index 9243af55..3975421f 100644 --- a/apps.toml +++ b/apps.toml @@ -43,7 +43,7 @@ added_date = 1674232499 # 2023/01/20 antifeatures = [ "deprecated-software" ] category = "social_media" deprecated_date = 1717071136 # 2024/05/30 -level = 8 +level = 7 state = "working" subtags = [ "microblogging", "pictures" ] url = "https://github.com/YunoHost-Apps/acropolis_ynh" @@ -70,7 +70,7 @@ added_date = 1674232499 # 2023/01/20 antifeatures = [ "deprecated-software" ] category = "system_tools" deprecated_date = 1717071136 # 2024/05/30 -level = 8 +level = 7 state = "working" subtags = [ "db" ] url = "https://github.com/YunoHost-Apps/adminer_ynh" @@ -165,7 +165,7 @@ url = "https://github.com/YunoHost-Apps/archivist_ynh" added_date = 1674232499 # 2023/01/20 antifeatures = [ "alpha-software" ] category = "small_utilities" -level = 7 +level = 4 state = "working" url = "https://github.com/YunoHost-Apps/armadietto_ynh" @@ -245,7 +245,7 @@ added_date = 1674232499 # 2023/01/20 antifeatures = [ "deprecated-software" ] category = "publishing" deprecated_date = 1717071136 # 2024/05/30 -level = 8 +level = 7 potential_alternative_to = [ "Blogger", "Coldfusion", "Wix" ] state = "working" subtags = [ "blog" ] @@ -511,7 +511,7 @@ added_date = 1674232499 # 2023/01/20 antifeatures = [ "deprecated-software" ] category = "publishing" deprecated_date = 1717071136 # 2024/05/30 -level = 8 +level = 7 potential_alternative_to = [ "Disqus" ] state = "working" subtags = [ "website" ] @@ -681,7 +681,7 @@ url = "https://github.com/YunoHost-Apps/dendrite_ynh" [dex] added_date = 1674232499 # 2023/01/20 category = "system_tools" -level = 6 +level = 0 state = "working" subtags = [ "network" ] url = "https://github.com/YunoHost-Apps/dex_ynh" @@ -851,7 +851,7 @@ url = "https://github.com/YunoHost-Apps/domoticz_ynh" added_date = 1674232499 # 2023/01/20 branch = "main" category = "dev" -level = 8 +level = 4 state = "working" subtags = [ "programming" ] url = "https://github.com/YunoHost-Apps/dont-code_ynh" @@ -988,7 +988,7 @@ added_date = 1674232499 # 2023/01/20 antifeatures = [ "deprecated-software" ] category = "small_utilities" deprecated_date = 1717071136 # 2024/05/30 -level = 8 +level = 7 state = "working" url = "https://github.com/YunoHost-Apps/encryptor-decryptor_ynh" @@ -1089,7 +1089,7 @@ url = "https://github.com/YunoHost-Apps/fastapi_ynh" [fider] added_date = 1674232499 # 2023/01/20 category = "productivity_and_management" -level = 3 +level = 0 state = "working" subtags = [ "poll" ] url = "https://github.com/YunoHost-Apps/fider_ynh" @@ -1115,7 +1115,7 @@ url = "https://github.com/YunoHost-Apps/filepizza_ynh" [firefish] added_date = 1691055044 # 2023/08/03 category = "social_media" -level = 0 +level = 7 potential_alternative_to = [ "Calckey", "Mastodon", "Misskey", "Pleroma", "Threads", "X" ] state = "working" url = "https://github.com/YunoHost-Apps/firefish_ynh" @@ -1247,7 +1247,7 @@ url = "https://github.com/YunoHost-Apps/freshrss_ynh" [friendica] added_date = 1674232499 # 2023/01/20 category = "social_media" -level = 8 +level = 6 potential_alternative_to = [ "Facebook" ] state = "working" subtags = [ "microblogging" ] @@ -1373,7 +1373,7 @@ url = "https://github.com/YunoHost-Apps/glance_ynh" [glitchsoc] added_date = 1674232499 # 2023/01/20 category = "social_media" -level = 6 +level = 8 state = "working" subtags = [ "microblogging" ] url = "https://github.com/YunoHost-Apps/glitchsoc_ynh" @@ -2130,7 +2130,7 @@ url = "https://github.com/YunoHost-Apps/loki_ynh" [lstu] added_date = 1674232499 # 2023/01/20 category = "small_utilities" -level = 8 +level = 6 potential_alternative_to = [ "bitly" ] state = "working" subtags = [ "url_shortener" ] @@ -2231,17 +2231,6 @@ state = "working" subtags = [ "chat" ] url = "https://github.com/YunoHost-Apps/matrix-appservice-irc_ynh" -[matrix-puppet-discord] -added_date = 1674232499 # 2023/01/20 -antifeatures = [ "deprecated-software" ] -category = "communication" -deprecated_date = 1711055457 # 2024/03/21 -level = 6 -potential_alternative_to = [ "Discord" ] -state = "working" -subtags = [ "chat" ] -url = "https://github.com/YunoHost-Apps/matrix-puppet-discord_ynh" - [matterbridge] added_date = 1674232499 # 2023/01/20 category = "communication" @@ -2623,7 +2612,7 @@ url = "https://github.com/YunoHost-Apps/nitter_ynh" [noalyss] added_date = 1674232499 # 2023/01/20 category = "productivity_and_management" -level = 8 +level = 0 potential_alternative_to = [ "Exact Online", "WinBooks", "Yooz" ] state = "working" subtags = [ "accounting", "business_and_ngos" ] @@ -2655,7 +2644,7 @@ url = "https://github.com/YunoHost-Apps/nodered_ynh" [nomad] added_date = 1674232499 # 2023/01/20 category = "dev" -level = 8 +level = 1 state = "working" url = "https://github.com/YunoHost-Apps/nomad_ynh" @@ -2944,7 +2933,7 @@ url = "https://github.com/YunoHost-Apps/petrolette_ynh" [pgadmin] added_date = 1674232499 # 2023/01/20 category = "system_tools" -level = 7 +level = 8 state = "working" subtags = [ "db" ] url = "https://github.com/YunoHost-Apps/pgadmin_ynh" @@ -3168,7 +3157,7 @@ added_date = 1674232499 # 2023/01/20 antifeatures = [ "deprecated-software" ] category = "publishing" deprecated_date = 1717071136 # 2024/05/30 -level = 8 +level = 7 potential_alternative_to = [ "Wix" ] state = "working" subtags = [ "website" ] @@ -3459,7 +3448,7 @@ url = "https://github.com/YunoHost-Apps/samba_ynh" [satdress] added_date = 1674232499 # 2023/01/20 category = "productivity_and_management" -level = 8 +level = 0 state = "working" url = "https://github.com/YunoHost-Apps/satdress_ynh" @@ -3484,7 +3473,7 @@ added_date = 1674232499 # 2023/01/20 antifeatures = [ "deprecated-software" ] category = "small_utilities" deprecated_date = 1717071136 # 2024/05/30 -level = 8 +level = 7 state = "working" url = "https://github.com/YunoHost-Apps/scrumblr_ynh" @@ -3772,7 +3761,7 @@ added_date = 1674232499 # 2023/01/20 antifeatures = [ "deprecated-software" ] category = "system_tools" deprecated_date = 1717071136 # 2024/05/30 -level = 8 +level = 7 state = "working" url = "https://github.com/YunoHost-Apps/ssh_chroot_dir_ynh" @@ -3896,7 +3885,7 @@ url = "https://github.com/YunoHost-Apps/teampass_ynh" [technitium-dns] added_date = 1674232499 # 2023/01/20 category = "system_tools" -level = 6 +level = 8 state = "working" subtags = [ "network" ] url = "https://github.com/YunoHost-Apps/technitium-dns_ynh" @@ -4001,7 +3990,7 @@ url = "https://github.com/YunoHost-Apps/traccar_ynh" [tracim] added_date = 1674232499 # 2023/01/20 category = "office" -level = 6 +level = 0 potential_alternative_to = [ "Dropbox", "Google Drive", "Slack", "Trello" ] state = "working" url = "https://github.com/YunoHost-Apps/tracim_ynh" @@ -4158,7 +4147,7 @@ url = "https://github.com/YunoHost-Apps/vikunja_ynh" [vore] added_date = 1690540859 # 2023/07/28 category = "reading" -level = 7 +level = 0 state = "working" subtags = [ "rssreader" ] url = "https://github.com/YunoHost-Apps/vore_ynh" @@ -4346,7 +4335,7 @@ url = "https://github.com/YunoHost-Apps/yacy_ynh" [yarr] added_date = 1715771420 # 2024/05/15 category = "reading" -level = 7 +level = 0 state = "working" subtags = [ "rssreader" ] url = "https://github.com/YunoHost-Apps/yarr_ynh" @@ -4423,7 +4412,7 @@ url = "https://github.com/YunoHost-Apps/zap_ynh" added_date = 1674232499 # 2023/01/20 antifeatures = [ "replaced-by-another-app" ] category = "small_utilities" -level = 7 +level = 4 potential_alternative_to = [ "Pastebin" ] state = "working" subtags = [ "pastebin" ] diff --git a/graveyard.toml b/graveyard.toml index 6fac92aa..bce03564 100644 --- a/graveyard.toml +++ b/graveyard.toml @@ -186,6 +186,16 @@ potential_alternative_to = [ "Google Groups" ] subtags = [ "email" ] url = "https://github.com/yunohost-apps/mailman_ynh" +[matrix-puppet-discord] +added_date = 1674232499 # 2023/01/20 +antifeatures = [ "deprecated-software" ] +category = "communication" +deprecated_date = 1711055457 # 2024/03/21 +killed_date = 1717576274 # 2024/06/05 +potential_alternative_to = [ "Discord" ] +subtags = [ "chat" ] +url = "https://github.com/YunoHost-Apps/matrix-puppet-discord_ynh" + [mediadrop] added_date = 1554588215 # 2019/04/07 category = "multimedia" diff --git a/store/translations/es/LC_MESSAGES/messages.mo b/store/translations/es/LC_MESSAGES/messages.mo index 55fc16dc..37c5ce85 100644 Binary files a/store/translations/es/LC_MESSAGES/messages.mo and b/store/translations/es/LC_MESSAGES/messages.mo differ diff --git a/store/translations/es/LC_MESSAGES/messages.po b/store/translations/es/LC_MESSAGES/messages.po index b2960f80..4925985f 100644 --- a/store/translations/es/LC_MESSAGES/messages.po +++ b/store/translations/es/LC_MESSAGES/messages.po @@ -8,8 +8,8 @@ msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" "POT-Creation-Date: 2024-05-09 23:14+0200\n" -"PO-Revision-Date: 2024-03-23 00:54+0000\n" -"Last-Translator: OniriCorpe \n" +"PO-Revision-Date: 2024-06-06 14:54+0000\n" +"Last-Translator: Éric Gaspar \n" "Language-Team: Spanish \n" "Language: es\n" @@ -17,6 +17,7 @@ msgstr "" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=n != 1;\n" +"X-Generator: Weblate 5.4.3\n" "Generated-By: Babel 2.14.0\n" #: app.py:162 @@ -26,7 +27,7 @@ msgstr "App %(app_id)s no encontrada" #: app.py:165 msgid "You must be logged in to be able to star an app" -msgstr "" +msgstr "Debes iniciar sesión para poder iniciar una aplicación" #: app.py:167 app.py:212 app.py:530 templates/wishlist_add.html:33 msgid "" @@ -36,14 +37,19 @@ msgid "" "least 5 topics, reading at least 30 posts, and spending at least 10 minutes " "reading posts." msgstr "" +"Tenga en cuenta que, debido a varios abusos, restringimos el inicio de " +"sesión en la tienda de aplicaciones a usuarios de 'nivel de confianza 1'.

El 'nivel de confianza 1' se obtiene después de interactuar un mínimo " +"con el foro, y más específicamente: ingresar al menos 5 temas, leer al menos " +"30 publicaciones y dedicar al menos 10 minutos a leer publicaciones." #: app.py:210 msgid "You must be logged in to submit an app to the wishlist" -msgstr "" +msgstr "Debes iniciar sesión para enviar una aplicación a la lista de deseos" #: app.py:225 msgid "Invalid CSRF token, please refresh the page and try again" -msgstr "" +msgstr "Token CSRF no válido. Actualice la página e inténtelo de nuevo" #: app.py:263 msgid "" diff --git a/store/translations/fr/LC_MESSAGES/messages.po b/store/translations/fr/LC_MESSAGES/messages.po index 3e36bfcc..8e68f05e 100644 --- a/store/translations/fr/LC_MESSAGES/messages.po +++ b/store/translations/fr/LC_MESSAGES/messages.po @@ -8,8 +8,8 @@ msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" "POT-Creation-Date: 2024-05-09 23:14+0200\n" -"PO-Revision-Date: 2024-05-20 21:19+0000\n" -"Last-Translator: ppr \n" +"PO-Revision-Date: 2024-06-06 14:54+0000\n" +"Last-Translator: Éric Gaspar \n" "Language-Team: French \n" "Language: fr\n" @@ -492,7 +492,7 @@ msgstr "Niveau 8" #: templates/charts.html:107 #, python-format msgid "Level %(level)s:" -msgstr "Niveau %(level)s:" +msgstr "Niveau %(level)s :" #: templates/charts.html:107 msgid "Total:" @@ -615,7 +615,6 @@ msgid "Inactive (%(days)s days ago)" msgstr "Inactif (%(days)s jours)" #: templates/dash.html:177 -#, fuzzy msgid "Packaging v1" msgstr "Packaging v1" diff --git a/store/translations/pt/LC_MESSAGES/messages.po b/store/translations/pt/LC_MESSAGES/messages.po index 1798b5fc..174f3223 100644 --- a/store/translations/pt/LC_MESSAGES/messages.po +++ b/store/translations/pt/LC_MESSAGES/messages.po @@ -8,14 +8,16 @@ msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" "POT-Creation-Date: 2024-05-09 23:14+0200\n" -"PO-Revision-Date: 2024-02-21 06:08+0100\n" -"Last-Translator: FULL NAME \n" -"Language-Team: pt \n" +"PO-Revision-Date: 2024-06-06 14:54+0000\n" +"Last-Translator: Éric Gaspar \n" +"Language-Team: Portuguese \n" "Language: pt\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n != 1);\n" +"X-Generator: Weblate 5.4.3\n" "Generated-By: Babel 2.14.0\n" #: app.py:162 @@ -25,7 +27,7 @@ msgstr "" #: app.py:165 msgid "You must be logged in to be able to star an app" -msgstr "" +msgstr "Você deve estar logado para poder marcar um aplicativo com estrela" #: app.py:167 app.py:212 app.py:530 templates/wishlist_add.html:33 msgid "" @@ -35,14 +37,20 @@ msgid "" "least 5 topics, reading at least 30 posts, and spending at least 10 minutes " "reading posts." msgstr "" +"Observe que, devido a vários abusos, restringimos o login na app store para " +"usuários de 'nível de confiança 1'.

O 'nível de confiança 1' é " +"obtido após uma interação mínima com o fórum e, mais especificamente: " +"inserir pelo menos 5 tópicos, ler pelo menos 30 postagens e passar pelo " +"menos 10 minutos lendo postagens." #: app.py:210 msgid "You must be logged in to submit an app to the wishlist" msgstr "" +"Você deve estar logado para enviar um aplicativo para a lista de desejos" #: app.py:225 msgid "Invalid CSRF token, please refresh the page and try again" -msgstr "" +msgstr "Token CSRF inválido. Atualize a página e tente novamente" #: app.py:263 msgid "" diff --git a/tools/app_caches.py b/tools/app_caches.py index 069f5f2c..af7b4a0f 100755 --- a/tools/app_caches.py +++ b/tools/app_caches.py @@ -3,6 +3,7 @@ import argparse import shutil import logging +from itertools import repeat from multiprocessing import Pool from pathlib import Path from typing import Any @@ -24,7 +25,9 @@ def app_cache_folder(app: str) -> Path: return APPS_CACHE_DIR / app -def app_cache_clone(app: str, infos: dict[str, str]) -> None: +def app_cache_clone( + app: str, infos: dict[str, str], all_branches: bool = False +) -> None: logging.info("Cloning %s...", app) git_depths = { "notworking": 5, @@ -37,18 +40,27 @@ def app_cache_clone(app: str, infos: dict[str, str]) -> None: infos["url"], to_path=app_cache_folder(app), depth=git_depths.get(infos["state"], git_depths["default"]), - single_branch=True, + single_branch=not all_branches, branch=infos.get("branch", "master"), ) -def app_cache_clone_or_update(app: str, infos: dict[str, str]) -> None: +def app_cache_clone_or_update( + app: str, + infos: dict[str, str], + ssh_clone: bool = False, + fetch_all_branches: bool = False, +) -> None: app_path = app_cache_folder(app) + # Patch url for ssh clone + if ssh_clone: + infos["url"] = infos["url"].replace("https://github.com/", "git@github.com:") + # Don't refresh if already refreshed during last hour age = git_repo_age(app_path) if age is False: - app_cache_clone(app, infos) + app_cache_clone(app, infos, fetch_all_branches) return # if age < 3600: @@ -60,29 +72,41 @@ def app_cache_clone_or_update(app: str, infos: dict[str, str]) -> None: repo.remote("origin").set_url(infos["url"]) branch = infos.get("branch", "master") - if repo.active_branch != branch: - all_branches = [str(b) for b in repo.branches] - if branch in all_branches: - repo.git.checkout(branch, "--force") - else: - repo.git.remote("set-branches", "--add", "origin", branch) - repo.remote("origin").fetch(f"{branch}:{branch}") + if fetch_all_branches: + repo.git.remote("set-branches", "origin", "*") + repo.remote("origin").fetch() + else: + if repo.active_branch != branch: + all_branches = [str(b) for b in repo.branches] + if branch in all_branches: + repo.git.checkout(branch, "--force") + else: + repo.git.remote("set-branches", "--add", "origin", branch) + repo.remote("origin").fetch(f"{branch}:{branch}") - repo.remote("origin").fetch(refspec=branch, force=True) - repo.git.reset("--hard", f"origin/{branch}") + repo.remote("origin").fetch(refspec=branch, force=True) + repo.git.reset("--hard", f"origin/{branch}") def __app_cache_clone_or_update_mapped(data): - name, info = data + name, info, ssh_clone, all_branches = data try: - app_cache_clone_or_update(name, info) + app_cache_clone_or_update(name, info, ssh_clone, all_branches) except Exception as err: logging.error("[App caches] Error while updating %s: %s", name, err) -def apps_cache_update_all(apps: dict[str, dict[str, Any]], parallel: int = 8) -> None: +def apps_cache_update_all( + apps: dict[str, dict[str, Any]], + parallel: int = 8, + ssh_clone: bool = False, + all_branches: bool = False, +) -> None: with Pool(processes=parallel) as pool: - tasks = pool.imap_unordered(__app_cache_clone_or_update_mapped, apps.items()) + tasks = pool.imap_unordered( + __app_cache_clone_or_update_mapped, + zip(apps.keys(), apps.values(), repeat(ssh_clone), repeat(all_branches)), + ) for _ in tqdm.tqdm(tasks, total=len(apps.keys()), ascii=" ·#"): pass @@ -101,6 +125,20 @@ def __run_for_catalog(): parser = argparse.ArgumentParser() parser.add_argument("-v", "--verbose", action="store_true") parser.add_argument("-j", "--processes", type=int, default=8) + parser.add_argument( + "-s", + "--ssh", + action=argparse.BooleanOptionalAction, + default=False, + help="Use ssh clones instead of https", + ) + parser.add_argument( + "-b", + "--all-branches", + action=argparse.BooleanOptionalAction, + default=False, + help="Download all branches from repo", + ) parser.add_argument( "-c", "--cleanup", @@ -114,7 +152,12 @@ def __run_for_catalog(): if args.cleanup: apps_cache_cleanup(get_catalog()) - apps_cache_update_all(get_catalog(), parallel=args.processes) + apps_cache_update_all( + get_catalog(), + parallel=args.processes, + ssh_clone=args.ssh, + all_branches=args.all_branches, + ) if __name__ == "__main__": diff --git a/tools/readme_generator/make_readme.py b/tools/readme_generator/make_readme.py index 06e3ca73..1d4f22d2 100755 --- a/tools/readme_generator/make_readme.py +++ b/tools/readme_generator/make_readme.py @@ -17,6 +17,7 @@ from langcodes import Language README_GEN_DIR = Path(__file__).resolve().parent APPS_REPO_ROOT = README_GEN_DIR.parent.parent +TRANSLATIONS_DIR = README_GEN_DIR / "translations" def value_for_lang(values: Dict, lang: str): if not isinstance(values, dict): @@ -54,13 +55,13 @@ def generate_READMEs(app_path: Path): return poparser = PoFileParser({}) - poparser.parse(open("messages.pot")) + poparser.parse((README_GEN_DIR / "messages.pot").open(encoding="utf-8")) # we only want to translate a README if all strings are translatables so we # do this loop to detect which language provides a full translation fully_translated_langs: List[str] = [] - for available_translations in os.listdir("translations"): - translations = Translations.load("translations", available_translations) + for available_translations in os.listdir(TRANSLATIONS_DIR): + translations = Translations.load(TRANSLATIONS_DIR, available_translations) is_fully_translated = True for sentence in poparser.catalog: @@ -111,7 +112,7 @@ def generate_READMEs(app_path: Path): loader=FileSystemLoader(README_GEN_DIR / "templates"), extensions=["jinja2.ext.i18n"], ) - translations = Translations.load("translations", [lang]) + translations = Translations.load(TRANSLATIONS_DIR, [lang]) env.install_gettext_translations(translations) template = env.get_template("README.md.j2") @@ -169,7 +170,7 @@ def generate_READMEs(app_path: Path): links_to_other_READMEs = [] for language in fully_translated_langs: - translations = Translations.load("translations", [language]) + translations = Translations.load(TRANSLATIONS_DIR, [language]) language_name_in_itself = Language.get(language).autonym() links_to_other_READMEs.append( ( diff --git a/tools/readme_generator/webhook.py b/tools/readme_generator/webhook.py deleted file mode 100755 index 3e668757..00000000 --- a/tools/readme_generator/webhook.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python3 - -import hashlib -import hmac -from functools import cache -import tempfile -from pathlib import Path - -from git import Actor, Repo -from sanic import HTTPResponse, Request, Sanic, response - -from make_readme import generate_READMEs - -TOOLS_DIR = Path(__file__).resolve().parent.parent - -app = Sanic(__name__) - - -@cache -def github_webhook_secret() -> str: - return ( - (TOOLS_DIR / ".github_webhook_secret") - .open("r", encoding="utf-8") - .read() - .strip() - ) - - -@cache -def github_login() -> str: - return (TOOLS_DIR / ".github_login").open("r", encoding="utf-8").read().strip() - - -@cache -def github_token() -> str: - return (TOOLS_DIR / ".github_token").open("r", encoding="utf-8").read().strip() - - -@app.route("/github", methods=["GET"]) -async def main_route(request: Request) -> HTTPResponse: - return response.text( - "You aren't supposed to go on this page using a browser, it's for webhooks push instead." - ) - - -@app.route("/github", methods=["POST"]) -async def on_push(request: Request) -> HTTPResponse: - header_signature = request.headers.get("X-Hub-Signature") - if header_signature is None: - print("no header X-Hub-Signature") - return response.json({"error": "No X-Hub-Signature"}, 403) - - sha_name, signature = header_signature.split("=") - if sha_name != "sha1": - print("signing algo isn't sha1, it's '%s'" % sha_name) - return response.json({"error": "Signing algorightm is not sha1 ?!"}, 501) - - # HMAC requires the key to be bytes, but data is string - mac = hmac.new( - github_webhook_secret().encode(), msg=request.body, digestmod=hashlib.sha1 - ) - - if not hmac.compare_digest(str(mac.hexdigest()), str(signature)): - return response.json({"error": "Bad signature ?!"}, 403) - - data = request.json - - repository = data["repository"]["full_name"] - branch = data["ref"].split("/", 2)[2] - - print(f"{repository} -> branch '{branch}'") - - with tempfile.TemporaryDirectory() as folder_str: - folder = Path(folder_str) - repo = Repo.clone_from( - f"https://{github_login()}:{github_token()}@github.com/{repository}", - to_path=folder, - single_branch=True, - branch=branch, - ) - - generate_READMEs(folder) - - repo.git.add("README*.md") - repo.git.add("ALL_README.md") - - diff_empty = len(repo.index.diff("HEAD")) == 0 - if diff_empty: - print("nothing to do") - return response.text("nothing to do") - - repo.index.commit( - "Auto-update READMEs", author=Actor("yunohost-bot", "yunohost@yunohost.org") - ) - repo.remote().push(quiet=False) - - return response.text("ok") - - -if __name__ == "__main__": - app.run(host="127.0.0.1", port=8123, debug=True) diff --git a/tools/readme_generator/webhook.service b/tools/readme_generator/webhook.service deleted file mode 100644 index e4783c74..00000000 --- a/tools/readme_generator/webhook.service +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=Auto-README webhook gunicorn daemon -After=network.target - -[Service] -PIDFile=/run/gunicorn/autoreadme_webhook-pid -User=autoreadme_webhook -Group=autoreadme_webhook -WorkingDirectory=__PATH_TO_README_GENERATOR__ -ExecStart=__PATH_TO_README_GENERATOR__/venv/bin/gunicorn -w 4 -b 127.0.0.1:8123 webhook:app -ExecReload=/bin/kill -s HUP $MAINPID -ExecStop=/bin/kill -s TERM $MAINPID -PrivateTmp=true - -[Install] -WantedBy=multi-user.target diff --git a/tools/webhooks/webhook.py b/tools/webhooks/webhook.py new file mode 100755 index 00000000..df30984c --- /dev/null +++ b/tools/webhooks/webhook.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 + +import sys +import hashlib +import argparse +import hmac +from functools import cache +import tempfile +import logging +from pathlib import Path + +from typing import Optional +from git import Actor, Repo, GitCommandError +from sanic import HTTPResponse, Request, Sanic, response + +# add apps/tools to sys.path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from readme_generator.make_readme import generate_READMEs + +TOOLS_DIR = Path(__file__).resolve().parent.parent + +DEBUG = False +UNSAFE = False + +APP = Sanic(__name__) + + +@cache +def github_webhook_secret() -> str: + return ( + (TOOLS_DIR / ".github_webhook_secret") + .open("r", encoding="utf-8") + .read() + .strip() + ) + + +@cache +def github_login() -> str: + return (TOOLS_DIR / ".github_login").open("r", encoding="utf-8").read().strip() + + +@cache +def github_token() -> str: + return (TOOLS_DIR / ".github_token").open("r", encoding="utf-8").read().strip() + + +@APP.route("/github", methods=["GET"]) +async def github_get(request: Request) -> HTTPResponse: + return response.text( + "You aren't supposed to go on this page using a browser, it's for webhooks push instead." + ) + + +@APP.route("/github", methods=["POST"]) +async def github_post(request: Request) -> HTTPResponse: + if UNSAFE and (signatures_reply := check_webhook_signatures(request)): + return signatures_reply + + event = request.headers.get("X-Github-Event") + if event == "push": + return on_push(request) + + return response.json({"error": f"Unknown event '{event}'"}, 422) + + +def check_webhook_signatures(request: Request) -> Optional[HTTPResponse]: + logging.warning("Unsafe webhook!") + header_signature = request.headers.get("X-Hub-Signature") + if header_signature is None: + logging.error("no header X-Hub-Signature") + return response.json({"error": "No X-Hub-Signature"}, 403) + + sha_name, signature = header_signature.split("=") + if sha_name != "sha1": + logging.error("signing algo isn't sha1, it's '%s'" % sha_name) + return response.json({"error": "Signing algorightm is not sha1 ?!"}, 501) + + # HMAC requires the key to be bytes, but data is string + mac = hmac.new( + github_webhook_secret().encode(), msg=request.body, digestmod=hashlib.sha1 + ) + + if not hmac.compare_digest(str(mac.hexdigest()), str(signature)): + return response.json({"error": "Bad signature ?!"}, 403) + return None + + +def on_push(request: Request) -> HTTPResponse: + data = request.json + repository = data["repository"]["full_name"] + branch = data["ref"].split("/", 2)[2] + + logging.info(f"{repository} -> branch '{branch}'") + + need_push = False + with tempfile.TemporaryDirectory() as folder_str: + folder = Path(folder_str) + repo = Repo.clone_from( + f"https://{github_login()}:{github_token()}@github.com/{repository}", + to_path=folder, + ) + + # First rebase the testing branch if possible + if branch in ["master", "testing"]: + result = git_repo_rebase_testing_fast_forward(repo) + need_push = need_push or result + + repo.git.checkout(branch) + result = generate_and_commit_readmes(repo) + need_push = need_push or result + + if not need_push: + logging.debug("nothing to do") + return response.text("nothing to do") + + logging.debug(f"Pushing {repository}") + repo.remote().push(quiet=False, all=True) + + return response.text("ok") + + +def generate_and_commit_readmes(repo: Repo) -> bool: + assert repo.working_tree_dir is not None + generate_READMEs(Path(repo.working_tree_dir)) + + repo.git.add("README*.md") + repo.git.add("ALL_README.md") + + diff_empty = len(repo.index.diff("HEAD")) == 0 + if diff_empty: + return False + + repo.index.commit( + "Auto-update READMEs", author=Actor("yunohost-bot", "yunohost@yunohost.org") + ) + return True + + +def git_repo_rebase_testing_fast_forward(repo: Repo) -> bool: + try: + repo.git.checkout("testing") + except GitCommandError: + return False + if not repo.is_ancestor("testing", "master"): + return False + repo.git.merge("master", ff_only=True) + return True + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("-d", "--debug", action="store_true") + parser.add_argument( + "-u", + "--unsafe", + action="store_true", + help="Disable Github signature checks on webhooks, for debug only.", + ) + args = parser.parse_args() + + if args.debug: + logging.getLogger().setLevel(logging.DEBUG) + + global DEBUG, UNSAFE + DEBUG = args.debug + UNSAFE = args.unsafe + + APP.run(host="127.0.0.1", port=8123, debug=args.debug) + + +if __name__ == "__main__": + main() diff --git a/tools/webhooks/webhook.service b/tools/webhooks/webhook.service new file mode 100644 index 00000000..bfd6ccca --- /dev/null +++ b/tools/webhooks/webhook.service @@ -0,0 +1,16 @@ +[Unit] +Description=Github webhooks for YunoHost-Apps management +After=network.target + +[Service] +PIDFile=/run/gunicorn/yunohost_apps_webhooks-pid +User=yunohost_apps_webhooks +Group=yunohost_apps_webhooks +WorkingDirectory=__PATH_TO_APPS_TOOLS__/webhooks +ExecStart=__PATH_TO_APPS_TOOLS__/webhooks/venv/bin/python3 webhook.py +ExecReload=/bin/kill -s HUP $MAINPID +ExecStop=/bin/kill -s TERM $MAINPID +PrivateTmp=true + +[Install] +WantedBy=multi-user.target