diff --git a/README.md b/README.md index 661cbb2..0b0f855 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Here you will find the repositories and versions of every apps available in Yuno It is browsable here: https://yunohost.org/apps -The main file of the catalog is [**apps.json**](./apps.json) which contains +The main file of the catalog is [**apps.toml**](./apps.toml) which contains references to the corresponding Git repositories for each application, along with a few metadata about them such as its category or maintenance state. This file regularly read by `list_builder.py` which publish the results on @@ -30,28 +30,23 @@ https://app.yunohost.org/default/. > with keeping your app working and up to date with packaging evolutions on the long run. To add your application to the catalog: -* Fork this repository and edit the [apps.json](https://github.com/YunoHost/apps/tree/master/apps.json) file +* Fork this repository and edit the [apps.toml](https://github.com/YunoHost/apps/tree/master/apps.toml) file * Add your app's ID and git information at the right alphabetical place * Indicate the app's functioning state: `notworking`, `inprogress`, or `working` -* Indicate the app category, which you can pick from `categories.yml` -* Indicate any anti-feature that your app may be subject to, see `antifeatures.yml` (or remove the `antifeatures` key if there's none) +* Indicate the app category, which you can pick from `categories.toml` +* Indicate any anti-feature that your app may be subject to, see `antifeatures.toml` (or remove the `antifeatures` key if there's none) * Indicate if your app can be thought of as an alternative to popular proprietary services (or remove the `potential_alternative_to` key if there's none) * *Do not* add the `level` entry by yourself. Our automatic test suite ("the CI") will handle it. * Create a [Pull Request](https://github.com/YunoHost/apps/pulls/) App example addition: -```json - "your_app": { - "antifeatures": [ - "deprecated-software" - ], - "potential_alternative_to": [ - "YouTube" - ], - "category": "pick_the_appropriate_category", - "state": "working", - "url": "https://github.com/YunoHost-Apps/your_app_ynh" - } +```toml +[your_app] +antifeatures = [ "deprecated-software" ] # Remove if no relevant antifeature applies +potential_alternative_to = [ "YouTube" ] # Indicate if your app can be thought of as an alternative to popular proprietary services (or remove if none applies) +category = "foobar" # Replace with the appropriate category id found in categories.toml +state = "working" +url = "https://github.com/YunoHost-Apps/your_app_ynh" ``` > **Warning** @@ -69,6 +64,6 @@ App packagers should *not* manually set their apps' level. The levels of all the ### Apps flagged as not-maintained -Applications with no recent activity and no active sign from maintainer may be flagged in `apps.json` with the `package-not-maintained` antifeature tag to signify that the app is inactive and may slowly become outdated with respect to the upstream, or with respect to good packaging practices. It does **not** mean that the app is not working anymore. +Applications with no recent activity and no active sign from maintainer may be flagged in `apps.toml` with the `package-not-maintained` antifeature tag to signify that the app is inactive and may slowly become outdated with respect to the upstream, or with respect to good packaging practices. It does **not** mean that the app is not working anymore. Feel free to contact the app group if you feel like taking over the maintenance of a currently unmaintained app! diff --git a/list_builder.py b/list_builder.py index f9921e0..8cbc00e 100755 --- a/list_builder.py +++ b/list_builder.py @@ -7,7 +7,6 @@ import re import json import toml import subprocess -import yaml import time from collections import OrderedDict @@ -15,7 +14,20 @@ from tools.packaging_v2.convert_v1_manifest_to_v2_for_catalog import convert_v1_ now = time.time() -catalog = json.load(open("apps.json")) +# Load categories and reformat the structure to have a list with an "id" key +categories = toml.load(open("categories.toml")) +for category_id, infos in categories.items(): + infos["id"] = category_id +categories = list(categories.values()) + +# (Same for antifeatures) +antifeatures = toml.load(open("antifeatures.toml")) +for antifeature_id, infos in antifeatures.items(): + infos["id"] = antifeature_id +antifeatures = list(antifeatures.values()) + +# Load the app catalog and filter out the non-working ones +catalog = toml.load(open("apps.toml")) catalog = { app: infos for app, infos in catalog.items() if infos.get("state") != "notworking" } @@ -181,8 +193,6 @@ def build_catalog(): result_dict_with_manifest_v1 = copy.deepcopy(result_dict) result_dict_with_manifest_v1 = {name: infos for name, infos in result_dict_with_manifest_v1.items() if float(str(infos["manifest"].get("packaging_format", "")).strip() or "0") < 2} - categories = yaml.load(open("categories.yml").read()) - antifeatures = yaml.load(open("antifeatures.yml").read()) os.system("mkdir -p ./builds/default/v2/") with open("builds/default/v2/apps.json", "w") as f: f.write( @@ -239,7 +249,6 @@ def build_catalog(): ############################## # Version for catalog in doc # ############################## - categories = yaml.load(open("categories.yml").read()) os.system("mkdir -p ./builds/default/doc_catalog") def infos_for_doc_catalog(infos): diff --git a/tools/README-generator/make_readme.py b/tools/README-generator/make_readme.py index 9788d89..5f004ea 100755 --- a/tools/README-generator/make_readme.py +++ b/tools/README-generator/make_readme.py @@ -4,7 +4,6 @@ import argparse import json import toml import os -import yaml from pathlib import Path from jinja2 import Environment, FileSystemLoader @@ -33,11 +32,10 @@ def generate_READMEs(app_path: str): upstream = manifest.get("upstream", {}) - catalog = json.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "apps.json")) + catalog = toml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "apps.toml")) from_catalog = catalog.get(manifest['id'], {}) - antifeatures_list = yaml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "antifeatures.yml"), Loader=yaml.SafeLoader) - antifeatures_list = { e['id']: e for e in antifeatures_list } + antifeatures_list = toml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "antifeatures.toml")) if not upstream and not (app_path / "doc" / "DISCLAIMER.md").exists(): print( diff --git a/tools/catalog_linter.py b/tools/catalog_linter.py new file mode 100644 index 0000000..6e17117 --- /dev/null +++ b/tools/catalog_linter.py @@ -0,0 +1,28 @@ +import toml +import sys + +catalog = toml.load(open('apps.toml')) +catalog = {app: infos for app, infos in catalog.items() if infos.get('state') == "working"} +categories = toml.load(open('categories.toml')).keys() + +def check_apps(): + + for app, infos in catalog.items(): + + repo_name = infos.get("url", "").split("/")[-1] + if repo_name != app + "_ynh": + yield f"{app}: repo name should be {app}_ynh, not in {repo_name}" + + category = infos.get("category") + if not category: + yield f"{app}: missing category" + if category not in categories: + yield f"{app}: category {category} is not defined in categories.toml" + +errors = list(check_apps()) + +for error in errors: + print(error) + +if errors: + sys.exit(1) diff --git a/tools/update_app_levels/update_app_levels.py b/tools/update_app_levels/update_app_levels.py new file mode 100644 index 0000000..12713de --- /dev/null +++ b/tools/update_app_levels/update_app_levels.py @@ -0,0 +1,106 @@ +import time +import toml +import requests +import tempfile +import os +import sys +import json +from collections import OrderedDict + +token = open(".github_token").read().strip() + +tmpdir = tempfile.mkdtemp(prefix="update_app_levels_") +os.system(f"git clone 'https://oauth2:{token}@github.com/yunohost/apps' {tmpdir}") +os.system(f"git -C {tmpdir} checkout -b update_app_levels") + +# Load the app catalog and filter out the non-working ones +catalog = toml.load(open(f"{tmpdir}/apps.toml")) + +# Fetch results from the CI +CI_RESULTS_URL = "https://ci-apps.yunohost.org/ci/logs/list_level_stable_amd64.json" +ci_results = requests.get(CI_RESULTS_URL).json() + +comment = { + "major_regressions": [], + "minor_regressions": [], + "improvements": [], + "outdated": [], + "missing": [], +} + +for app, infos in catalog.items(): + + if infos.get("state") != "working": + continue + + if app not in ci_results: + comment["missing"].append(app) + continue + + # 3600 * 24 * 60 = ~2 months + if (int(time.time()) - ci_results[app].get("timestamp", 0)) > 3600 * 24 * 60: + comment["outdated"].append(app) + continue + + ci_level = ci_results[app]["level"] + current_level = infos.get("level") + + if ci_level == current_level: + continue + elif current_level is None or ci_level > current_level: + comment["improvements"].append((app, current_level, ci_level)) + elif ci_level < current_level: + if ci_level < 4 and current_level >= 4: + comment["major_regressions"].append((app, current_level, ci_level)) + else: + comment["minor_regressions"].append((app, current_level, ci_level)) + + infos["level"] = ci_level + +# Also re-sort the catalog keys / subkeys +for app, infos in catalog.items(): + catalog[app] = OrderedDict(sorted(infos.items())) +catalog = OrderedDict(sorted(catalog.items())) + +updated_catalog = toml.dumps(catalog) +updated_catalog = updated_catalog.replace(",]", " ]") +open(f"{tmpdir}/apps.toml", "w").write(updated_catalog) + +os.system(f"git -C {tmpdir} commit apps.toml -m 'Update app levels according to CI results'") +os.system(f"git -C {tmpdir} push origin update_app_levels --force") +os.system(f"rm -rf {tmpdir}") + +PR_body = "" +if comment["major_regressions"]: + PR_body += "\n### Major regressions\n\n" + for app, current_level, new_level in comment['major_regressions']: + PR_body += f"- [ ] {app} | {current_level} -> {new_level} | https://ci-apps.yunohost.org/ci/apps/{app}/latestjob\n" +if comment["minor_regressions"]: + PR_body += "\n### Minor regressions\n\n" + for app, current_level, new_level in comment['minor_regressions']: + PR_body += f"- [ ] {app} | {current_level} -> {new_level} | https://ci-apps.yunohost.org/ci/apps/{app}/latestjob\n" +if comment["improvements"]: + PR_body += "\n### Improvements\n\n" + for app, current_level, new_level in comment['improvements']: + PR_body += f"- {app} | {current_level} -> {new_level} | https://ci-apps.yunohost.org/ci/apps/{app}/latestjob\n" +if comment["missing"]: + PR_body += "\n### Missing results\n\n" + for app in comment['missing']: + PR_body += f"- {app} | https://ci-apps.yunohost.org/ci/apps/{app}/latestjob\n" +if comment["outdated"]: + PR_body += "\n### Outdated results\n\n" + for app in comment['outdated']: + PR_body += f"- [ ] {app} | https://ci-apps.yunohost.org/ci/apps/{app}/latestjob\n" + +PR = {"title": "Update app levels accoring to CI results", + "body": PR_body, + "head": "update_app_levels", + "base": "master"} + +with requests.Session() as s: + s.headers.update({"Authorization": f"token {token}"}) +r = s.post("https://api.github.com/repos/yunohost/apps/pulls", json.dumps(PR)) + +if r.status_code != 200: + print(r.text) + sys.exit(1)