Merge pull request #1612 from YunoHost/toml-all-the-things

TOML all the things ?

README.md

@@ -6,7 +6,7 @@ Here you will find the repositories and versions of every apps available in Yuno
 
 It is browsable here: https://yunohost.org/apps
 
-The main file of the catalog is [**apps.json**](./apps.json) which contains
+The main file of the catalog is [**apps.toml**](./apps.toml) which contains
 references to the corresponding Git repositories for each application, along
 with a few metadata about them such as its category or maintenance state. This
 file regularly read by `list_builder.py` which publish the results on
@@ -30,28 +30,23 @@ https://app.yunohost.org/default/.
 > with keeping your app working and up to date with packaging evolutions on the long run.
 
 To add your application to the catalog:
-* Fork this repository and edit the [apps.json](https://github.com/YunoHost/apps/tree/master/apps.json) file
+* Fork this repository and edit the [apps.toml](https://github.com/YunoHost/apps/tree/master/apps.toml) file
 * Add your app's ID and git information at the right alphabetical place
 * Indicate the app's functioning state: `notworking`, `inprogress`, or `working`
-* Indicate the app category, which you can pick from `categories.yml`
-* Indicate any anti-feature that your app may be subject to, see `antifeatures.yml` (or remove the `antifeatures` key if there's none)
+* Indicate the app category, which you can pick from `categories.toml`
+* Indicate any anti-feature that your app may be subject to, see `antifeatures.toml` (or remove the `antifeatures` key if there's none)
 * Indicate if your app can be thought of as an alternative to popular proprietary services (or remove the `potential_alternative_to` key if there's none)
 * *Do not* add the `level` entry by yourself. Our automatic test suite ("the CI") will handle it.
 * Create a [Pull Request](https://github.com/YunoHost/apps/pulls/)
 
 App example addition:
-```json
-    "your_app": {
-        "antifeatures": [
-            "deprecated-software"
-        ],
-        "potential_alternative_to": [
-            "YouTube"
-        ],
-        "category": "pick_the_appropriate_category",
-        "state": "working",
-        "url": "https://github.com/YunoHost-Apps/your_app_ynh"
-    }
+```toml
+[your_app]
+antifeatures = [ "deprecated-software" ]   # Remove if no relevant antifeature applies
+potential_alternative_to = [ "YouTube" ]   # Indicate if your app can be thought of as an alternative to popular proprietary services (or remove if none applies)
+category = "foobar"                        # Replace with the appropriate category id found in categories.toml
+state = "working"
+url = "https://github.com/YunoHost-Apps/your_app_ynh"
 ```
 
 > **Warning**
@@ -69,6 +64,6 @@ App packagers should *not* manually set their apps' level. The levels of all the
 
 ### Apps flagged as not-maintained
 
-Applications with no recent activity and no active sign from maintainer may be flagged in `apps.json` with the `package-not-maintained` antifeature tag to signify that the app is inactive and may slowly become outdated with respect to the upstream, or with respect to good packaging practices. It does **not** mean that the app is not working anymore.
+Applications with no recent activity and no active sign from maintainer may be flagged in `apps.toml` with the `package-not-maintained` antifeature tag to signify that the app is inactive and may slowly become outdated with respect to the upstream, or with respect to good packaging practices. It does **not** mean that the app is not working anymore.
 
 Feel free to contact the app group if you feel like taking over the maintenance of a currently unmaintained app!

list_builder.py

@@ -7,7 +7,6 @@ import re
 import json
 import toml
 import subprocess
-import yaml
 import time
 
 from collections import OrderedDict
@@ -15,7 +14,20 @@ from tools.packaging_v2.convert_v1_manifest_to_v2_for_catalog import convert_v1_
 
 now = time.time()
 
-catalog = json.load(open("apps.json"))
+# Load categories and reformat the structure to have a list with an "id" key
+categories = toml.load(open("categories.toml"))
+for category_id, infos in categories.items():
+    infos["id"] = category_id
+categories = list(categories.values())
+
+# (Same for antifeatures)
+antifeatures = toml.load(open("antifeatures.toml"))
+for antifeature_id, infos in antifeatures.items():
+    infos["id"] = antifeature_id
+antifeatures = list(antifeatures.values())
+
+# Load the app catalog and filter out the non-working ones
+catalog = toml.load(open("apps.toml"))
 catalog = {
     app: infos for app, infos in catalog.items() if infos.get("state") != "notworking"
 }
@@ -181,8 +193,6 @@ def build_catalog():
     result_dict_with_manifest_v1 = copy.deepcopy(result_dict)
     result_dict_with_manifest_v1 = {name: infos for name, infos in result_dict_with_manifest_v1.items() if float(str(infos["manifest"].get("packaging_format", "")).strip() or "0") < 2}
 
-    categories = yaml.load(open("categories.yml").read())
-    antifeatures = yaml.load(open("antifeatures.yml").read())
     os.system("mkdir -p ./builds/default/v2/")
     with open("builds/default/v2/apps.json", "w") as f:
         f.write(
@@ -239,7 +249,6 @@ def build_catalog():
     ##############################
     # Version for catalog in doc #
     ##############################
-    categories = yaml.load(open("categories.yml").read())
     os.system("mkdir -p ./builds/default/doc_catalog")
 
     def infos_for_doc_catalog(infos):

tools/README-generator/make_readme.py

@@ -4,7 +4,6 @@ import argparse
 import json
 import toml
 import os
-import yaml
 from pathlib import Path
 
 from jinja2 import Environment, FileSystemLoader
@@ -33,11 +32,10 @@ def generate_READMEs(app_path: str):
 
     upstream = manifest.get("upstream", {})
 
-    catalog = json.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "apps.json"))
+    catalog = toml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "apps.toml"))
     from_catalog = catalog.get(manifest['id'], {})
 
-    antifeatures_list = yaml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "antifeatures.yml"), Loader=yaml.SafeLoader)
-    antifeatures_list = { e['id']: e for e in antifeatures_list }
+    antifeatures_list = toml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "antifeatures.toml"))
 
     if not upstream and not (app_path / "doc" / "DISCLAIMER.md").exists():
         print(

tools/catalog_linter.py

@@ -0,0 +1,28 @@
+import toml
+import sys
+
+catalog = toml.load(open('apps.toml'))
+catalog = {app: infos for app, infos in catalog.items() if infos.get('state') == "working"}
+categories = toml.load(open('categories.toml')).keys()
+
+def check_apps():
+
+    for app, infos in catalog.items():
+
+        repo_name = infos.get("url", "").split("/")[-1]
+        if repo_name != app + "_ynh":
+            yield f"{app}: repo name should be {app}_ynh, not in {repo_name}"
+
+        category = infos.get("category")
+        if not category:
+            yield f"{app}: missing category"
+        if category not in categories:
+            yield f"{app}: category {category} is not defined in categories.toml"
+
+errors = list(check_apps())
+
+for error in errors:
+    print(error)
+
+if errors:
+    sys.exit(1)

tools/update_app_levels/update_app_levels.py

@@ -0,0 +1,106 @@
+import time
+import toml
+import requests
+import tempfile
+import os
+import sys
+import json
+from collections import OrderedDict
+
+token = open(".github_token").read().strip()
+
+tmpdir = tempfile.mkdtemp(prefix="update_app_levels_")
+os.system(f"git clone 'https://oauth2:{token}@github.com/yunohost/apps' {tmpdir}")
+os.system(f"git -C {tmpdir} checkout -b update_app_levels")
+
+# Load the app catalog and filter out the non-working ones
+catalog = toml.load(open(f"{tmpdir}/apps.toml"))
+
+# Fetch results from the CI
+CI_RESULTS_URL = "https://ci-apps.yunohost.org/ci/logs/list_level_stable_amd64.json"
+ci_results = requests.get(CI_RESULTS_URL).json()
+
+comment = {
+    "major_regressions": [],
+    "minor_regressions": [],
+    "improvements": [],
+    "outdated": [],
+    "missing": [],
+}
+
+for app, infos in catalog.items():
+
+    if infos.get("state") != "working":
+        continue
+
+    if app not in ci_results:
+        comment["missing"].append(app)
+        continue
+
+    # 3600 * 24 * 60 = ~2 months
+    if (int(time.time()) - ci_results[app].get("timestamp", 0)) > 3600 * 24 * 60:
+        comment["outdated"].append(app)
+        continue
+
+    ci_level = ci_results[app]["level"]
+    current_level = infos.get("level")
+
+    if ci_level == current_level:
+        continue
+    elif current_level is None or ci_level > current_level:
+        comment["improvements"].append((app, current_level, ci_level))
+    elif ci_level < current_level:
+        if ci_level < 4 and current_level >= 4:
+            comment["major_regressions"].append((app, current_level, ci_level))
+        else:
+            comment["minor_regressions"].append((app, current_level, ci_level))
+
+    infos["level"] = ci_level
+
+# Also re-sort the catalog keys / subkeys
+for app, infos in catalog.items():
+    catalog[app] = OrderedDict(sorted(infos.items()))
+catalog = OrderedDict(sorted(catalog.items()))
+
+updated_catalog = toml.dumps(catalog)
+updated_catalog = updated_catalog.replace(",]", " ]")
+open(f"{tmpdir}/apps.toml", "w").write(updated_catalog)
+
+os.system(f"git -C {tmpdir} commit apps.toml -m 'Update app levels according to CI results'")
+os.system(f"git -C {tmpdir} push origin update_app_levels --force")
+os.system(f"rm -rf {tmpdir}")
+
+PR_body = ""
+if comment["major_regressions"]:
+    PR_body += "\n### Major regressions\n\n"
+    for app, current_level, new_level in comment['major_regressions']:
+        PR_body += f"- [ ] {app} | {current_level} -> {new_level} | https://ci-apps.yunohost.org/ci/apps/{app}/latestjob\n"
+if comment["minor_regressions"]:
+    PR_body += "\n### Minor regressions\n\n"
+    for app, current_level, new_level in comment['minor_regressions']:
+        PR_body += f"- [ ] {app} | {current_level} -> {new_level} | https://ci-apps.yunohost.org/ci/apps/{app}/latestjob\n"
+if comment["improvements"]:
+    PR_body += "\n### Improvements\n\n"
+    for app, current_level, new_level in comment['improvements']:
+        PR_body += f"- {app} | {current_level} -> {new_level} | https://ci-apps.yunohost.org/ci/apps/{app}/latestjob\n"
+if comment["missing"]:
+    PR_body += "\n### Missing results\n\n"
+    for app in comment['missing']:
+        PR_body += f"- {app} | https://ci-apps.yunohost.org/ci/apps/{app}/latestjob\n"
+if comment["outdated"]:
+    PR_body += "\n### Outdated results\n\n"
+    for app in comment['outdated']:
+        PR_body += f"- [ ] {app} | https://ci-apps.yunohost.org/ci/apps/{app}/latestjob\n"
+
+PR = {"title": "Update app levels accoring to CI results",
+      "body": PR_body,
+      "head": "update_app_levels",
+      "base": "master"}
+
+with requests.Session() as s:
+    s.headers.update({"Authorization": f"token {token}"})
+r = s.post("https://api.github.com/repos/yunohost/apps/pulls", json.dumps(PR))
+
+if r.status_code != 200:
+    print(r.text)
+    sys.exit(1)