From 73296ab02d61827d4b686576fff3e8e3e1663cc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Wed, 7 Feb 2024 14:49:55 +0100 Subject: [PATCH 01/10] python: run isort, add shebangs on all files --- list_builder.py | 16 +++++++++------- tools/README-generator/__init__.py | 1 + tools/README-generator/make_readme.py | 3 ++- tools/README-generator/webhook.py | 13 +++++++------ tools/__init__.py | 1 + tools/autopatches/autopatch.py | 6 ++++-- tools/autopatches/patches/add-cpe/patch.sh | 3 ++- .../autoupdate_app_sources.py | 12 +++++++----- tools/autoupdate_app_sources/rest_api.py | 7 +++++-- .../autoupdater-upgrader/autoupdater-upgrader.py | 13 ++++++++----- tools/bot-repo-cleanup/cleanup.py | 2 +- tools/catalog_linter.py | 2 +- tools/packaging_v2/__init__.py | 1 + .../packaging_v2/convert_app_to_packaging_v2.py | 7 +++++-- .../convert_v1_manifest_to_v2_for_catalog.py | 2 ++ tools/update_app_levels/update_app_levels.py | 13 ++++++++----- 16 files changed, 64 insertions(+), 38 deletions(-) diff --git a/list_builder.py b/list_builder.py index 996dd05..2b12fd2 100755 --- a/list_builder.py +++ b/list_builder.py @@ -1,20 +1,22 @@ #!/usr/bin/python3 import copy -import sys +import json import os import re -import json -from shutil import which -import toml import subprocess +import sys import time -from typing import TextIO, Generator, Any +from collections import OrderedDict from pathlib import Path +from shutil import which +from typing import Any, Generator, TextIO + +import toml from git import Repo -from collections import OrderedDict -from tools.packaging_v2.convert_v1_manifest_to_v2_for_catalog import convert_v1_manifest_to_v2_for_catalog +from tools.packaging_v2.convert_v1_manifest_to_v2_for_catalog import \ + convert_v1_manifest_to_v2_for_catalog now = time.time() diff --git a/tools/README-generator/__init__.py b/tools/README-generator/__init__.py index e69de29..e5a0d9b 100644 --- a/tools/README-generator/__init__.py +++ b/tools/README-generator/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/tools/README-generator/make_readme.py b/tools/README-generator/make_readme.py index 5f004ea..d71f973 100755 --- a/tools/README-generator/make_readme.py +++ b/tools/README-generator/make_readme.py @@ -2,12 +2,13 @@ import argparse import json -import toml import os from pathlib import Path +import toml from jinja2 import Environment, FileSystemLoader + def value_for_lang(values, lang): if not isinstance(values, dict): return values diff --git a/tools/README-generator/webhook.py b/tools/README-generator/webhook.py index af220d8..7a76eab 100755 --- a/tools/README-generator/webhook.py +++ b/tools/README-generator/webhook.py @@ -1,15 +1,16 @@ -import os -import hmac -import shlex -import hashlib +#!/usr/bin/env python3 + import asyncio +import hashlib +import hmac +import os +import shlex import tempfile +from make_readme import generate_READMEs from sanic import Sanic, response from sanic.response import text -from make_readme import generate_READMEs - app = Sanic(__name__) github_webhook_secret = open("github_webhook_secret", "r").read().strip() diff --git a/tools/__init__.py b/tools/__init__.py index e69de29..e5a0d9b 100644 --- a/tools/__init__.py +++ b/tools/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/tools/autopatches/autopatch.py b/tools/autopatches/autopatch.py index 746ee51..6f52d30 100755 --- a/tools/autopatches/autopatch.py +++ b/tools/autopatches/autopatch.py @@ -1,9 +1,11 @@ #!/usr/bin/python3 + import json -import sys -import requests import os import subprocess +import sys + +import requests catalog = requests.get("https://raw.githubusercontent.com/YunoHost/apps/master/apps.json").json() diff --git a/tools/autopatches/patches/add-cpe/patch.sh b/tools/autopatches/patches/add-cpe/patch.sh index 02a697c..2b6b2d6 100644 --- a/tools/autopatches/patches/add-cpe/patch.sh +++ b/tools/autopatches/patches/add-cpe/patch.sh @@ -1,7 +1,8 @@ #!/usr/bin/python3 -import json import csv +import json + def find_cpe(app_id): with open("../../patches/add-cpe/cpe.csv", newline='') as f: diff --git a/tools/autoupdate_app_sources/autoupdate_app_sources.py b/tools/autoupdate_app_sources/autoupdate_app_sources.py index dfd15b7..b42dc78 100644 --- a/tools/autoupdate_app_sources/autoupdate_app_sources.py +++ b/tools/autoupdate_app_sources/autoupdate_app_sources.py @@ -1,13 +1,15 @@ -import time +#!/usr/bin/env python3 + +import glob import hashlib +import os import re import sys -import requests -import toml -import os -import glob +import time from datetime import datetime +import requests +import toml from rest_api import GithubAPI, GitlabAPI, RefType STRATEGIES = [ diff --git a/tools/autoupdate_app_sources/rest_api.py b/tools/autoupdate_app_sources/rest_api.py index a76821a..ccc6b4d 100644 --- a/tools/autoupdate_app_sources/rest_api.py +++ b/tools/autoupdate_app_sources/rest_api.py @@ -1,8 +1,11 @@ -from enum import Enum +#!/usr/bin/env python3 + import re -import requests +from enum import Enum from typing import List +import requests + class RefType(Enum): tags = 1 diff --git a/tools/autoupdater-upgrader/autoupdater-upgrader.py b/tools/autoupdater-upgrader/autoupdater-upgrader.py index 7252042..3e9c1a2 100755 --- a/tools/autoupdater-upgrader/autoupdater-upgrader.py +++ b/tools/autoupdater-upgrader/autoupdater-upgrader.py @@ -1,14 +1,17 @@ -#!venv/bin/python3 +#!/usr/bin/env python3 -import sys, os, time -import urllib.request, json +import json +import os import re +import sys +import time +import urllib.request -from github import Github import github - +from github import Github # Debug from rich.traceback import install + install(width=150, show_locals=True, locals_max_length=None, locals_max_string=None) ##### diff --git a/tools/bot-repo-cleanup/cleanup.py b/tools/bot-repo-cleanup/cleanup.py index f2275c6..2a9fa98 100644 --- a/tools/bot-repo-cleanup/cleanup.py +++ b/tools/bot-repo-cleanup/cleanup.py @@ -1,4 +1,4 @@ -#!venv/bin/python3 +#!/usr/bin/env python3 # Obtained with `pip install PyGithub`, better within a venv from github import Github diff --git a/tools/catalog_linter.py b/tools/catalog_linter.py index 4c383a9..6ce7ef9 100755 --- a/tools/catalog_linter.py +++ b/tools/catalog_linter.py @@ -2,10 +2,10 @@ import json import sys +from difflib import SequenceMatcher from functools import cache from pathlib import Path from typing import Any, Dict, Generator, List, Tuple -from difflib import SequenceMatcher import jsonschema import toml diff --git a/tools/packaging_v2/__init__.py b/tools/packaging_v2/__init__.py index e69de29..e5a0d9b 100644 --- a/tools/packaging_v2/__init__.py +++ b/tools/packaging_v2/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/tools/packaging_v2/convert_app_to_packaging_v2.py b/tools/packaging_v2/convert_app_to_packaging_v2.py index 7887cdb..2fa166c 100644 --- a/tools/packaging_v2/convert_app_to_packaging_v2.py +++ b/tools/packaging_v2/convert_app_to_packaging_v2.py @@ -1,7 +1,9 @@ +#!/usr/bin/env python3 + import argparse +import json import os import re -import json import subprocess from glob import glob @@ -226,7 +228,8 @@ def _convert_v1_manifest_to_v2(app_path): def _dump_v2_manifest_as_toml(manifest): import re - from tomlkit import document, nl, table, dumps, comment + + from tomlkit import comment, document, dumps, nl, table toml_manifest = document() toml_manifest.add("packaging_format", 2) diff --git a/tools/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py b/tools/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py index 0130c29..9fb3790 100644 --- a/tools/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py +++ b/tools/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + import copy diff --git a/tools/update_app_levels/update_app_levels.py b/tools/update_app_levels/update_app_levels.py index b8df4a1..221d4f2 100644 --- a/tools/update_app_levels/update_app_levels.py +++ b/tools/update_app_levels/update_app_levels.py @@ -1,12 +1,15 @@ -import time -import toml -import requests -import tempfile +#!/usr/bin/env python3 + +import json import os import sys -import json +import tempfile +import time from collections import OrderedDict +import requests +import toml + token = open(os.path.dirname(__file__) + "/../../.github_token").read().strip() tmpdir = tempfile.mkdtemp(prefix="update_app_levels_") From caf82f94cffd8fa6036520e34daac45bacc675ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Wed, 7 Feb 2024 15:02:49 +0100 Subject: [PATCH 02/10] Small markdown cleanup --- README.md | 60 ++++++++++++++++++++++++++++--------------------- store/README.md | 10 ++++----- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 7d69dfd..3536467 100644 --- a/README.md +++ b/README.md @@ -2,23 +2,23 @@ -Here you will find the repositories and versions of every apps available in YunoHost's default catalog. +This repository contains the default YunoHost app catalog, as well as tools +that can be run manually or automatically. -It is browsable here: https://yunohost.org/apps +The catalog is stored in [**`apps.toml`**](./apps.toml) and is browsable here: + -The main file of the catalog is [**apps.toml**](./apps.toml) which contains -references to the corresponding Git repositories for each application, along -with a few metadata about them such as its category or maintenance state. This -file regularly read by `list_builder.py` which publish the results on -https://app.yunohost.org/default/. +It contains refences to the apps' repositories, along with a few metadata about +them such as its category or maintenance state. This file is regularly read by +`list_builder.py` which publish the results on . -### Where can I learn about app packaging in YunoHost? +## Where can I learn about app packaging in YunoHost? -- You can browse the contributor documentation : https://yunohost.org/contributordoc +- You can browse [the contributor documentation](https://yunohost.org/contributordoc) - If you are not familiar with Git/GitHub, you can have a look at our [homemade guide](https://yunohost.org/#/packaging_apps_git) - Don't hesitate to reach for help on the dedicated [application packaging chatroom](https://yunohost.org/chat_rooms) ... we can even schedule an audio meeting to help you get started! -### How to add your app to the application catalog +## How to add your app to the application catalog > **Note** > The YunoHost project will **NOT** integrate in its catalog applications that are not @@ -30,16 +30,20 @@ https://app.yunohost.org/default/. > with keeping your app working and up to date with packaging evolutions on the long run. To add your application to the catalog: -* Fork this repository and edit the [apps.toml](https://github.com/YunoHost/apps/tree/master/apps.toml) file -* Add your app's ID and git information at the right alphabetical place -* Indicate the app's functioning state: `notworking`, `inprogress`, or `working` -* Indicate the app category, which you can pick from `categories.toml` -* Indicate any anti-feature that your app may be subject to, see `antifeatures.toml` (or remove the `antifeatures` key if there's none) -* Indicate if your app can be thought of as an alternative to popular proprietary services (or remove the `potential_alternative_to` key if there's none) -* *Do not* add the `level` entry by yourself. Our automatic test suite ("the CI") will handle it. + +* Fork [this repository](https://github.com/YunoHost/apps) +* Edit the [`apps.toml`](/apps.toml) file + * Add your app's ID and git information at the right alphabetical place + * Indicate the app's functioning state: `notworking`, `inprogress`, or `working` + * Indicate the app category, which you can pick from `categories.toml` + * Indicate any anti-feature that your app may be subject to, see `antifeatures.toml` (or remove the `antifeatures` key if there's none) + * Indicate if your app can be thought of as an alternative to popular proprietary services (or remove the `potential_alternative_to` key if there's none) + * *Do not* add the `level` entry by yourself. Our automatic test suite ("the CI") will handle it. +* Commit and push your modifications to your repository * Create a [Pull Request](https://github.com/YunoHost/apps/pulls/) App example addition: + ```toml [your_app] antifeatures = [ "deprecated-software" ] # Remove if no relevant antifeature applies @@ -58,17 +62,23 @@ url = "https://github.com/YunoHost-Apps/your_app_ynh" > obtain an access to the developer CI where you'll be able to test your app > easily. -### Updating apps levels in the catalog +## Updating apps levels in the catalog -App packagers should *not* manually set their apps' level. The levels of all the apps are automatically updated once per week on Friday, according to the results from the official app CI. +App packagers should *not* manually set their apps' level. The levels of all +the apps are automatically updated once per week on Friday, according to the +results from the official app CI. -### Apps flagged as not-maintained +## Apps flagged as not-maintained -Applications with no recent activity and no active sign from maintainer may be flagged in `apps.toml` with the `package-not-maintained` antifeature tag to signify that the app is inactive and may slowly become outdated with respect to the upstream, or with respect to good packaging practices. It does **not** mean that the app is not working anymore. +Applications with no recent activity and no active sign from maintainer may be +flagged in `apps.toml` with the `package-not-maintained` antifeature tag to +signify that the app is inactive and may slowly become outdated with respect to +the upstream, or with respect to good packaging practices. It does **not** mean +that the app is not working anymore. -Feel free to contact the app group if you feel like taking over the maintenance of a currently unmaintained app! +Feel free to contact the app group if you feel like taking over the maintenance +of a currently unmaintained app! -### `graveyard.toml` - -This file is for apps that are long-term not-working and unlikely to be ever revived +## `graveyard.toml` +This file is for apps that are long-term not-working and unlikely to be ever revived. diff --git a/store/README.md b/store/README.md index b60700b..c0c598d 100644 --- a/store/README.md +++ b/store/README.md @@ -4,7 +4,7 @@ This is a Flask app interfacing with YunoHost's app catalog for a cool browsing ## Developement -``` +```bash python3 -m venv venv source venv/bin/activate pip3 install -r requirements.txt @@ -19,22 +19,22 @@ curl https://app.yunohost.org/default/v3/apps.json > ../builds/default/v3/apps.j # You will also want to run list_builder.py to initialize the .apps_cache (at least for a few apps, you can Ctrl+C after a while) pushd .. - python3 list_builder.py + ./list_builder.py popd ``` And then start the dev server: -``` +```bash source venv/bin/activate FLASK_APP=app.py FLASK_ENV=development flask run ``` ## Translation -It's based on Flask-Babel : https://python-babel.github.io/ +It's based on Flask-Babel : -``` +```bash source venv/bin/activate pybabel extract --ignore-dirs venv -F babel.cfg -o messages.pot . From 1bc5b8862a8d67b964ebf0805ad536b1d99c2b66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Wed, 7 Feb 2024 15:04:55 +0100 Subject: [PATCH 03/10] Move list_builder.py to tools subdirectory --- README.md | 2 +- rebuild.sh | 2 +- store/README.md | 2 +- list_builder.py => tools/list_builder.py | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) rename list_builder.py => tools/list_builder.py (98%) diff --git a/README.md b/README.md index 3536467..7bfd625 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ The catalog is stored in [**`apps.toml`**](./apps.toml) and is browsable here: It contains refences to the apps' repositories, along with a few metadata about them such as its category or maintenance state. This file is regularly read by -`list_builder.py` which publish the results on . +`tools/list_builder.py` which publish the results on . ## Where can I learn about app packaging in YunoHost? diff --git a/rebuild.sh b/rebuild.sh index bcbfb86..7c4780f 100644 --- a/rebuild.sh +++ b/rebuild.sh @@ -7,4 +7,4 @@ cd $workdir date >> $log git pull &>/dev/null -./list_builder.py &>> $log || sendxmpppy "[listbuilder] Rebuilding the application list failed miserably" +./tools/list_builder.py &>> $log || sendxmpppy "[listbuilder] Rebuilding the application list failed miserably" diff --git a/store/README.md b/store/README.md index c0c598d..6222f4e 100644 --- a/store/README.md +++ b/store/README.md @@ -19,7 +19,7 @@ curl https://app.yunohost.org/default/v3/apps.json > ../builds/default/v3/apps.j # You will also want to run list_builder.py to initialize the .apps_cache (at least for a few apps, you can Ctrl+C after a while) pushd .. - ./list_builder.py + ./tools/list_builder.py popd ``` diff --git a/list_builder.py b/tools/list_builder.py similarity index 98% rename from list_builder.py rename to tools/list_builder.py index 2b12fd2..1994f01 100755 --- a/list_builder.py +++ b/tools/list_builder.py @@ -15,12 +15,12 @@ from typing import Any, Generator, TextIO import toml from git import Repo -from tools.packaging_v2.convert_v1_manifest_to_v2_for_catalog import \ - convert_v1_manifest_to_v2_for_catalog +from packaging_v2.convert_v1_manifest_to_v2_for_catalog import \ + convert_v1_manifest_to_v2_for_catalog # pylint: disable=import-error now = time.time() -REPO_APPS_PATH = Path(__file__).parent +REPO_APPS_PATH = Path(__file__).parent.parent # Load categories and reformat the structure to have a list with an "id" key categories = toml.load((REPO_APPS_PATH / "categories.toml").open("r", encoding="utf-8")) From 277c178301572da27a6a8fadd2ba4158020b8396 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Wed, 7 Feb 2024 17:09:30 +0100 Subject: [PATCH 04/10] Refactor update_appslevels.py: use argparse, gitpython, jinja2 for pr message --- tools/update_app_levels/update_app_levels.py | 275 +++++++++++++------ 1 file changed, 195 insertions(+), 80 deletions(-) mode change 100644 => 100755 tools/update_app_levels/update_app_levels.py diff --git a/tools/update_app_levels/update_app_levels.py b/tools/update_app_levels/update_app_levels.py old mode 100644 new mode 100755 index 221d4f2..c6a736f --- a/tools/update_app_levels/update_app_levels.py +++ b/tools/update_app_levels/update_app_levels.py @@ -1,109 +1,224 @@ #!/usr/bin/env python3 +""" +Update app catalog: commit, and create a merge request +""" +import argparse import json -import os -import sys +import logging import tempfile +import textwrap import time from collections import OrderedDict +from typing import Any +from pathlib import Path +import jinja2 import requests import toml +from git import Repo -token = open(os.path.dirname(__file__) + "/../../.github_token").read().strip() +# APPS_REPO = "YunoHost/apps" +APPS_REPO = "Salamandar/apps" -tmpdir = tempfile.mkdtemp(prefix="update_app_levels_") -os.system(f"git clone 'https://oauth2:{token}@github.com/yunohost/apps' {tmpdir}") -os.system(f"git -C {tmpdir} checkout -b update_app_levels") - -# Load the app catalog and filter out the non-working ones -catalog = toml.load(open(f"{tmpdir}/apps.toml")) - -# Fetch results from the CI CI_RESULTS_URL = "https://ci-apps.yunohost.org/ci/api/results" -ci_results = requests.get(CI_RESULTS_URL).json() -comment = { - "major_regressions": [], - "minor_regressions": [], - "improvements": [], - "outdated": [], - "missing": [], -} +REPO_APPS_ROOT = Path(Repo(__file__, search_parent_directories=True).working_dir) -for app, infos in catalog.items(): +VERBOSE = False - if infos.get("state") != "working": - continue - if app not in ci_results: - comment["missing"].append(app) - continue +def github_token() -> str | None: + github_token_path = REPO_APPS_ROOT.parent / ".github_token" + if github_token_path.exists(): + return github_token_path.open("r", encoding="utf-8").read().strip() + return None + +def get_ci_results() -> dict[str, dict[str, Any]]: + return requests.get(CI_RESULTS_URL, timeout=10).json() + + +def ci_result_is_outdated(result) -> bool: # 3600 * 24 * 60 = ~2 months - if (int(time.time()) - ci_results[app].get("timestamp", 0)) > 3600 * 24 * 60: - comment["outdated"].append(app) - continue + return (int(time.time()) - result.get("timestamp", 0)) > 3600 * 24 * 60 - ci_level = ci_results[app]["level"] - current_level = infos.get("level") - if ci_level == current_level: - continue - elif current_level is None or ci_level > current_level: - comment["improvements"].append((app, current_level, ci_level)) - elif ci_level < current_level: - if ci_level <= 4 and current_level > 4: - comment["major_regressions"].append((app, current_level, ci_level)) +def update_catalog(catalog, ci_results) -> dict: + """ + Actually change the catalog data + """ + # Re-sort the catalog keys / subkeys + for app, infos in catalog.items(): + catalog[app] = OrderedDict(sorted(infos.items())) + catalog = OrderedDict(sorted(catalog.items())) + + def app_level(app): + if app not in ci_results: + return 0 + if ci_result_is_outdated(ci_results[app]): + return 0 + return ci_results[app]["level"] + + for app, info in catalog.items(): + info["level"] = app_level(app) + + return catalog + + +def list_changes(catalog, ci_results) -> dict[str, list[tuple[str, int, int]]]: + """ + Lists changes for a pull request + """ + + changes = { + "major_regressions": [], + "minor_regressions": [], + "improvements": [], + "outdated": [], + "missing": [], + } + + for app, infos in catalog.items(): + if infos.get("state") != "working": + continue + + if app not in ci_results: + changes["missing"].append(app) + continue + + if ci_result_is_outdated(ci_results[app]): + changes["outdated"].append(app) + continue + + ci_level = ci_results[app]["level"] + current_level = infos.get("level") + + if ci_level == current_level: + continue + + if current_level is None or ci_level > current_level: + changes["improvements"].append((app, current_level, ci_level)) + continue + + if ci_level < current_level: + if ci_level <= 4 < current_level: + changes["major_regressions"].append((app, current_level, ci_level)) + else: + changes["minor_regressions"].append((app, current_level, ci_level)) + + return changes + + +def pretty_changes(changes: dict[str, list[tuple[str, int, int]]]) -> str: + pr_body_template = textwrap.dedent(""" + {%- if changes["major_regressions"] %} + ### Major regressions + {% for app in changes["major_regressions"] %} + - [ ] [{{app.0}}: {{app.1}} -> {{app.2}}](https://ci-apps.yunohost.org/ci/apps/{{app.0}}/latestjob) + {%- endfor %} + {% endif %} + {%- if changes["minor_regressions"] %} + ### Minor regressions + {% for app in changes["minor_regressions"] %} + - [ ] [{{app.0}}: {{app.1}} -> {{app.2}}](https://ci-apps.yunohost.org/ci/apps/{{app.0}}/latestjob) + {%- endfor %} + {% endif %} + {%- if changes["improvements"] %} + ### Improvements + {% for app in changes["improvements"] %} + - [{{app.0}}: {{app.1}} -> {{app.2}}](https://ci-apps.yunohost.org/ci/apps/{{app.0}}/latestjob) + {%- endfor %} + {% endif %} + {%- if changes["missing"] %} + ### Missing + {% for app in changes["missing"] %} + - [{{app}} (See latest job if it exists)](https://ci-apps.yunohost.org/ci/apps/{{app.0}}/latestjob) + {%- endfor %} + {% endif %} + {%- if changes["outdated"] %} + ### Outdated + {% for app in changes["outdated"] %} + - [ ] [{{app}} (See latest job if it exists)](https://ci-apps.yunohost.org/ci/apps/{{app.0}}/latestjob) + {%- endfor %} + {% endif %} + """) + + return jinja2.Environment().from_string(pr_body_template).render(changes=changes) + + +def make_pull_request(pr_body: str) -> None: + pr_data = { + "title": "Update app levels according to CI results", + "body": pr_body, + "head": "update_app_levels", + "base": "master" + } + + with requests.Session() as s: + s.headers.update({"Authorization": f"token {github_token()}"}) + response = s.post(f"https://api.github.com/repos/{APPS_REPO}/pulls", json.dumps(pr_data)) + + if response.status_code == 422: + response = s.get(f"https://api.github.com/repos/{APPS_REPO}/pulls", data={"head": "update_app_levels"}) + existing_url = response.json()[0]["html_url"] + logging.warning(f"A Pull Request already exists at {existing_url} !") else: - comment["minor_regressions"].append((app, current_level, ci_level)) + new_url = response.json()["html_url"] + logging.info(f"Opened a Pull Request at {new_url} !") - infos["level"] = ci_level + response.raise_for_status() -# Also re-sort the catalog keys / subkeys -for app, infos in catalog.items(): - catalog[app] = OrderedDict(sorted(infos.items())) -catalog = OrderedDict(sorted(catalog.items())) -updated_catalog = toml.dumps(catalog) -updated_catalog = updated_catalog.replace(",]", " ]") -open(f"{tmpdir}/apps.toml", "w").write(updated_catalog) +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--commit", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("--pr", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("-v", "--verbose", action=argparse.BooleanOptionalAction) + args = parser.parse_args() -os.system(f"git -C {tmpdir} commit apps.toml -m 'Update app levels according to CI results'") -os.system(f"git -C {tmpdir} push origin update_app_levels --force") -os.system(f"rm -rf {tmpdir}") + logging.getLogger().setLevel(logging.INFO) + global VERBOSE + if args.verbose: + VERBOSE = True + logging.getLogger().setLevel(logging.DEBUG) -PR_body = "" -if comment["major_regressions"]: - PR_body += "\n### Major regressions\n\n" - for app, current_level, new_level in comment['major_regressions']: - PR_body += f"- [ ] {app} | {current_level} -> {new_level} | https://ci-apps.yunohost.org/ci/apps/{app}/latestjob\n" -if comment["minor_regressions"]: - PR_body += "\n### Minor regressions\n\n" - for app, current_level, new_level in comment['minor_regressions']: - PR_body += f"- [ ] {app} | {current_level} -> {new_level} | https://ci-apps.yunohost.org/ci/apps/{app}/latestjob\n" -if comment["improvements"]: - PR_body += "\n### Improvements\n\n" - for app, current_level, new_level in comment['improvements']: - PR_body += f"- {app} | {current_level} -> {new_level} | https://ci-apps.yunohost.org/ci/apps/{app}/latestjob\n" -if comment["missing"]: - PR_body += "\n### Missing results\n\n" - for app in comment['missing']: - PR_body += f"- {app} | https://ci-apps.yunohost.org/ci/apps/{app}/latestjob\n" -if comment["outdated"]: - PR_body += "\n### Outdated results\n\n" - for app in comment['outdated']: - PR_body += f"- [ ] {app} | https://ci-apps.yunohost.org/ci/apps/{app}/latestjob\n" + with tempfile.TemporaryDirectory(prefix="update_app_levels_") as tmpdir: + logging.info("Cloning the repository...") + apps_repo = Repo.clone_from(f"git@github.com:{APPS_REPO}", to_path=tmpdir) -PR = {"title": "Update app levels according to CI results", - "body": PR_body, - "head": "update_app_levels", - "base": "master"} + # Load the app catalog and filter out the non-working ones + catalog = toml.load((Path(apps_repo.working_tree_dir) / "apps.toml").open("r", encoding="utf-8")) -with requests.Session() as s: - s.headers.update({"Authorization": f"token {token}"}) -r = s.post("https://api.github.com/repos/yunohost/apps/pulls", json.dumps(PR)) + new_branch = apps_repo.create_head("update_app_levels", apps_repo.refs.master) + apps_repo.head.reference = new_branch -if r.status_code != 200: - print(r.text) - sys.exit(1) + logging.info("Retrieving the CI results...") + ci_results = get_ci_results() + + # Now compute changes, then update the catalog + changes = list_changes(catalog, ci_results) + pr_body = pretty_changes(changes) + catalog = update_catalog(catalog, ci_results) + + # Save the new catalog + updated_catalog = toml.dumps(catalog) + updated_catalog = updated_catalog.replace(",]", " ]") + (Path(apps_repo.working_tree_dir) / "apps.toml").open("w", encoding="utf-8").write(updated_catalog) + + if args.commit: + logging.info("Committing and pushing the new catalog...") + apps_repo.index.add("apps.toml") + apps_repo.index.commit("Update app levels according to CI results") + apps_repo.remote().push(force=True) + + if VERBOSE: + print(pr_body) + + if args.pr: + logging.info("Opening a pull request...") + make_pull_request(pr_body) + + +if __name__ == "__main__": + main() From aae5c3444668f37f11f53627474d31e84c620d67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Wed, 7 Feb 2024 17:25:16 +0100 Subject: [PATCH 05/10] Handle updating pull requests --- tools/update_app_levels/update_app_levels.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tools/update_app_levels/update_app_levels.py b/tools/update_app_levels/update_app_levels.py index c6a736f..d9fb323 100755 --- a/tools/update_app_levels/update_app_levels.py +++ b/tools/update_app_levels/update_app_levels.py @@ -157,18 +157,25 @@ def make_pull_request(pr_body: str) -> None: with requests.Session() as s: s.headers.update({"Authorization": f"token {github_token()}"}) - response = s.post(f"https://api.github.com/repos/{APPS_REPO}/pulls", json.dumps(pr_data)) + response = s.post(f"https://api.github.com/repos/{APPS_REPO}/pulls", json=pr_data) if response.status_code == 422: response = s.get(f"https://api.github.com/repos/{APPS_REPO}/pulls", data={"head": "update_app_levels"}) - existing_url = response.json()[0]["html_url"] - logging.warning(f"A Pull Request already exists at {existing_url} !") + response.raise_for_status() + pr_number = response.json()[0]["number"] + + # head can't be updated + del pr_data["head"] + response = s.patch(f"https://api.github.com/repos/{APPS_REPO}/pulls/{pr_number}", json=pr_data) + response.raise_for_status() + existing_url = response.json()["html_url"] + logging.warning(f"An existing Pull Request has been updated at {existing_url} !") else: + response.raise_for_status() + new_url = response.json()["html_url"] logging.info(f"Opened a Pull Request at {new_url} !") - response.raise_for_status() - def main(): parser = argparse.ArgumentParser() From 181bda02eedb5a43a7a98226f6e99f07485900d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Wed, 7 Feb 2024 17:30:45 +0100 Subject: [PATCH 06/10] Add emojis to Pr results --- tools/update_app_levels/update_app_levels.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/update_app_levels/update_app_levels.py b/tools/update_app_levels/update_app_levels.py index d9fb323..eee8c35 100755 --- a/tools/update_app_levels/update_app_levels.py +++ b/tools/update_app_levels/update_app_levels.py @@ -113,31 +113,31 @@ def list_changes(catalog, ci_results) -> dict[str, list[tuple[str, int, int]]]: def pretty_changes(changes: dict[str, list[tuple[str, int, int]]]) -> str: pr_body_template = textwrap.dedent(""" {%- if changes["major_regressions"] %} - ### Major regressions + ### Major regressions 😭 {% for app in changes["major_regressions"] %} - - [ ] [{{app.0}}: {{app.1}} -> {{app.2}}](https://ci-apps.yunohost.org/ci/apps/{{app.0}}/latestjob) + - [ ] [{{app.0}}: {{app.1}} → {{app.2}}](https://ci-apps.yunohost.org/ci/apps/{{app.0}}/latestjob) {%- endfor %} {% endif %} {%- if changes["minor_regressions"] %} - ### Minor regressions + ### Minor regressions 😬 {% for app in changes["minor_regressions"] %} - - [ ] [{{app.0}}: {{app.1}} -> {{app.2}}](https://ci-apps.yunohost.org/ci/apps/{{app.0}}/latestjob) + - [ ] [{{app.0}}: {{app.1}} → {{app.2}}](https://ci-apps.yunohost.org/ci/apps/{{app.0}}/latestjob) {%- endfor %} {% endif %} {%- if changes["improvements"] %} - ### Improvements + ### Improvements 🥳 {% for app in changes["improvements"] %} - - [{{app.0}}: {{app.1}} -> {{app.2}}](https://ci-apps.yunohost.org/ci/apps/{{app.0}}/latestjob) + - [{{app.0}}: {{app.1}} → {{app.2}}](https://ci-apps.yunohost.org/ci/apps/{{app.0}}/latestjob) {%- endfor %} {% endif %} {%- if changes["missing"] %} - ### Missing + ### Missing 🫠 {% for app in changes["missing"] %} - [{{app}} (See latest job if it exists)](https://ci-apps.yunohost.org/ci/apps/{{app.0}}/latestjob) {%- endfor %} {% endif %} {%- if changes["outdated"] %} - ### Outdated + ### Outdated ⏰ {% for app in changes["outdated"] %} - [ ] [{{app}} (See latest job if it exists)](https://ci-apps.yunohost.org/ci/apps/{{app.0}}/latestjob) {%- endfor %} From 5d313598d6feacbf2e9bf1f9ecbe88e5ca8540e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Wed, 7 Feb 2024 17:33:01 +0100 Subject: [PATCH 07/10] Small cleanup --- tools/update_app_levels/update_app_levels.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tools/update_app_levels/update_app_levels.py b/tools/update_app_levels/update_app_levels.py index eee8c35..ee98a66 100755 --- a/tools/update_app_levels/update_app_levels.py +++ b/tools/update_app_levels/update_app_levels.py @@ -4,7 +4,6 @@ Update app catalog: commit, and create a merge request """ import argparse -import json import logging import tempfile import textwrap @@ -25,8 +24,6 @@ CI_RESULTS_URL = "https://ci-apps.yunohost.org/ci/api/results" REPO_APPS_ROOT = Path(Repo(__file__, search_parent_directories=True).working_dir) -VERBOSE = False - def github_token() -> str | None: github_token_path = REPO_APPS_ROOT.parent / ".github_token" @@ -185,9 +182,7 @@ def main(): args = parser.parse_args() logging.getLogger().setLevel(logging.INFO) - global VERBOSE if args.verbose: - VERBOSE = True logging.getLogger().setLevel(logging.DEBUG) with tempfile.TemporaryDirectory(prefix="update_app_levels_") as tmpdir: @@ -219,7 +214,7 @@ def main(): apps_repo.index.commit("Update app levels according to CI results") apps_repo.remote().push(force=True) - if VERBOSE: + if args.verbose: print(pr_body) if args.pr: From 09f790246fe8b5e38bad013d62bec166595f6f6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Wed, 7 Feb 2024 19:35:07 +0100 Subject: [PATCH 08/10] Add apps_caches.py script that handles the download of the apps cache for the other scripts --- tools/app_caches.py | 95 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 tools/app_caches.py diff --git a/tools/app_caches.py b/tools/app_caches.py new file mode 100644 index 0000000..099c75f --- /dev/null +++ b/tools/app_caches.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 + +import argparse +import shutil +import logging +from multiprocessing import Pool +from pathlib import Path +from typing import Any + +import tqdm + +from appslib.utils import (REPO_APPS_ROOT, # pylint: disable=import-error + get_catalog, git_repo_age, progressbar) +from git import Repo + + +def app_cache_folder(app: str) -> Path: + return REPO_APPS_ROOT / ".apps_cache" / app + + +def app_cache_clone(app: str, infos: dict[str, str]) -> None: + logging.info("Cloning %s...", app) + git_depths = { + "notworking": 5, + "inprogress": 20, + "default": 40, + } + if app_cache_folder(app).exists(): + shutil.rmtree(app_cache_folder(app)) + Repo.clone_from( + infos["url"], + to_path=app_cache_folder(app), + depth=git_depths.get(infos["state"], git_depths["default"]), + single_branch=True, branch=infos.get("branch", "master"), + ) + + +def app_cache_clone_or_update(app: str, infos: dict[str, str]) -> None: + app_path = app_cache_folder(app) + + # Don't refresh if already refreshed during last hour + age = git_repo_age(app_path) + if age is False: + app_cache_clone(app, infos) + return + + # if age < 3600: + # logging.info(f"Skipping {app}, it's been updated recently.") + # return + + logging.info("Updating %s...", app) + repo = Repo(app_path) + repo.remote("origin").set_url(infos["url"]) + + branch = infos.get("branch", "master") + if repo.active_branch != branch: + all_branches = [str(b) for b in repo.branches] + if branch in all_branches: + repo.git.checkout(branch, "--force") + else: + repo.git.remote("set-branches", "--add", "origin", branch) + repo.remote("origin").fetch(f"{branch}:{branch}") + + repo.remote("origin").fetch(refspec=branch, force=True) + repo.git.reset("--hard", f"origin/{branch}") + + +def __app_cache_clone_or_update_mapped(data): + name, info = data + try: + app_cache_clone_or_update(name, info) + except Exception as err: + logging.error("Error while updating %s: %s", name, err) + + +def apps_cache_update_all(apps: dict[str, dict[str, Any]], parallel: int = 8) -> None: + with Pool(processes=parallel) as pool: + tasks = pool.imap_unordered(__app_cache_clone_or_update_mapped, apps.items()) + for _ in tqdm.tqdm(tasks, total=len(apps.keys())): + pass + + +def __run_for_catalog(): + parser = argparse.ArgumentParser() + parser.add_argument("-v", "--verbose", action="store_true") + parser.add_argument("-j", "--processes", type=int, default=8) + args = parser.parse_args() + if args.verbose: + logging.getLogger().setLevel(logging.INFO) + + apps_cache_update_all(get_catalog(), parallel=args.processes) + + +if __name__ == "__main__": + __run_for_catalog() From 28a69ee43cc4b6255e742b89f59809102dde749e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Wed, 7 Feb 2024 19:37:45 +0100 Subject: [PATCH 09/10] add appslib --- tools/appslib/apps_cache.py | 68 +++++++++++++++++++++++++++++++++++ tools/appslib/utils.py | 72 +++++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 tools/appslib/apps_cache.py create mode 100644 tools/appslib/utils.py diff --git a/tools/appslib/apps_cache.py b/tools/appslib/apps_cache.py new file mode 100644 index 0000000..b8fd1e4 --- /dev/null +++ b/tools/appslib/apps_cache.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +import logging +from pathlib import Path + +import utils +from git import Repo + + +def apps_cache_path() -> Path: + path = apps_repo_root() / ".apps_cache" + path.mkdir() + return path + + +def app_cache_path(app: str) -> Path: + path = apps_cache_path() / app + path.mkdir() + return path + + +# def refresh_all_caches(catalog: dict[str, dict[str, str]]): +# for app, infos +# pass + + +def app_cache_clone(app: str, infos: dict[str, str]) -> None: + git_depths = { + "notworking": 5, + "inprogress": 20, + "default": 40, + } + + Repo.clone_from( + infos["url"], + to_path=app_cache_path(app), + depth=git_depths.get(infos["state"], git_depths["default"]), + single_branch=True, branch=infos.get("branch", "master"), + ) + + +def app_cache_update(app: str, infos: dict[str, str]) -> None: + app_path = app_cache_path(app) + age = utils.git_repo_age(app_path) + if age is False: + return app_cache_clone(app, infos) + + if age < 3600: + logging.info(f"Skipping {app}, it's been updated recently.") + return + + repo = Repo(app_path) + repo.remote("origin").set_url(infos["url"]) + + branch = infos.get("branch", "master") + if repo.active_branch != branch: + all_branches = [str(b) for b in repo.branches] + if branch in all_branches: + repo.git.checkout(branch, "--force") + else: + repo.git.remote("set-branches", "--add", "origin", branch) + repo.remote("origin").fetch(f"{branch}:{branch}") + + repo.remote("origin").fetch(refspec=branch, force=True) + repo.git.reset("--hard", f"origin/{branch}") + + +def cache_all_apps(catalog: dict[str, dict[str, str]]) -> None: diff --git a/tools/appslib/utils.py b/tools/appslib/utils.py new file mode 100644 index 0000000..f80650e --- /dev/null +++ b/tools/appslib/utils.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 + +import sys +import subprocess +from typing import Any, TextIO, Generator +import time +from functools import cache +from pathlib import Path +from git import Repo + +import toml + +REPO_APPS_ROOT = Path(Repo(__file__, search_parent_directories=True).working_dir) + + +@cache +def apps_repo_root() -> Path: + return Path(__file__).parent.parent.parent + + +def git(cmd: list[str], cwd: Path | None = None) -> str: + full_cmd = ["git"] + if cwd: + full_cmd.extend(["-C", str(cwd)]) + full_cmd.extend(cmd) + return subprocess.check_output( + full_cmd, + # env=my_env, + ).strip().decode("utf-8") + + +def git_repo_age(path: Path) -> bool | int: + for file in [path / ".git" / "FETCH_HEAD", path / ".git" / "HEAD"]: + if file.exists(): + return int(time.time() - file.stat().st_mtime) + return False + + +# Progress bar helper, stolen from https://stackoverflow.com/a/34482761 +def progressbar( + it: list[Any], + prefix: str = "", + size: int = 60, + file: TextIO = sys.stdout) -> Generator[Any, None, None]: + count = len(it) + + def show(j, name=""): + name += " " + x = int(size * j / count) + file.write( + "%s[%s%s] %i/%i %s\r" % (prefix, "#" * x, "." * (size - x), j, count, name) + ) + file.flush() + + show(0) + for i, item in enumerate(it): + yield item + show(i + 1, item[0]) + file.write("\n") + file.flush() + + +@cache +def get_catalog(working_only=False): + """Load the app catalog and filter out the non-working ones""" + catalog = toml.load((REPO_APPS_ROOT / "apps.toml").open("r", encoding="utf-8")) + if working_only: + catalog = { + app: infos for app, infos in catalog.items() + if infos.get("state") != "notworking" + } + return catalog From 6e8b92615898a989ab6849f65e38ee4388a1c98e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Wed, 7 Feb 2024 19:39:40 +0100 Subject: [PATCH 10/10] remove unused import, chmod +x --- tools/app_caches.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 tools/app_caches.py diff --git a/tools/app_caches.py b/tools/app_caches.py old mode 100644 new mode 100755 index 099c75f..f3e35fd --- a/tools/app_caches.py +++ b/tools/app_caches.py @@ -10,7 +10,7 @@ from typing import Any import tqdm from appslib.utils import (REPO_APPS_ROOT, # pylint: disable=import-error - get_catalog, git_repo_age, progressbar) + get_catalog, git_repo_age) from git import Repo