From d8fcd4f4b0547caae41df60941be71e72d966983 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 13 Jun 2021 04:50:21 +0200 Subject: [PATCH 01/51] [mod] use with tempfile.TemporaryDirectory --- tools/README-generator/webhook.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tools/README-generator/webhook.py b/tools/README-generator/webhook.py index e33f994..5f52c7b 100755 --- a/tools/README-generator/webhook.py +++ b/tools/README-generator/webhook.py @@ -1,6 +1,6 @@ import subprocess import os -import shutil +import tempfile from github_webhook import Webhook from flask import Flask @@ -43,8 +43,7 @@ def on_push(data): repository = data["repository"]["full_name"] branch = data["ref"].split("/", 2)[2] - folder = subprocess.check_output(["mktemp", "-d"]).decode('utf-8').strip() - try: + with tempfile.TemporaryDirectory() as folder: git(["clone", f"https://{login}:{token}@github.com/{repository}", "--single-branch", "--branch", branch, folder]) generate_READMEs(folder) @@ -56,9 +55,7 @@ def on_push(data): git(["commit", "-a", "-m", "Auto-update README", "--author='Yunohost-Bot <>'"], in_folder=folder) git(["push", "origin", branch, "--quiet"], in_folder=folder) - finally: - if os.path.exists(folder): - shutil.rmtree(folder) + if __name__ == "__main__": app.run(host="0.0.0.0", port=8123) From d890d0b4f604faadf75771b177c28eb718cb7d5b Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 13 Jun 2021 05:12:34 +0200 Subject: [PATCH 02/51] [mod] port to sanic for perfs --- tools/README-generator/requirements.txt | 4 +- tools/README-generator/webhook.py | 64 ++++++++++++++++++------- 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/tools/README-generator/requirements.txt b/tools/README-generator/requirements.txt index 3e3ec9d..88cd6e2 100644 --- a/tools/README-generator/requirements.txt +++ b/tools/README-generator/requirements.txt @@ -1,4 +1,2 @@ -argparse jinja2 -github-webhook==1.0.4 -gunicorn==20.1.0 +sanic diff --git a/tools/README-generator/webhook.py b/tools/README-generator/webhook.py index 5f52c7b..b78b36d 100755 --- a/tools/README-generator/webhook.py +++ b/tools/README-generator/webhook.py @@ -1,15 +1,19 @@ -import subprocess import os +import hmac +import shlex +import hashlib +import asyncio import tempfile -from github_webhook import Webhook -from flask import Flask +from sanic import Sanic +from sanic.response import text +from sanic.exceptions import abort + from make_readme import generate_READMEs -app = Flask(__name__) +app = Sanic(__name__) github_webhook_secret = open("github_webhook_secret", "r").read().strip() -webhook = Webhook(app, endpoint="/github", secret=github_webhook_secret) login = open("login").read().strip() token = open("token").read().strip() @@ -22,39 +26,63 @@ my_env["GIT_COMMITTER_NAME"] = "Yunohost-Bot" my_env["GIT_COMMITTER_EMAIL"] = "yunohost@yunohost.org" -def git(cmd, in_folder=None): +async def git(cmd, in_folder=None): if not isinstance(cmd, list): cmd = cmd.split() if in_folder: cmd = ["-C", in_folder] + cmd cmd = ["git"] + cmd - return subprocess.check_output(cmd, env=my_env).strip().decode("utf-8") + cmd = " ".join(map(shlex.quote, cmd)) + command = await asyncio.create_subprocess_shell(cmd, env=my_env, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT) + data = await command.stdout.read() + return data.decode().strip() -@app.route("/github") -def main_route(): - return "You aren't supposed to go on this page using a browser, it's for webhooks push instead." +@app.route("/github", methods=["GET"]) +def main_route(request): + return text("You aren't supposed to go on this page using a browser, it's for webhooks push instead.") -@webhook.hook() -def on_push(data): +@app.route("/github", methods=["POST"]) +async def on_push(request): + header_signature = request.headers.get("X-Hub-Signature") + if header_signature is None: + print("no header X-Hub-Signature") + abort(403) + + sha_name, signature = header_signature.split("=") + if sha_name != "sha1": + print("signing algo isn't sha1, it's '%s'" % sha_name) + abort(501) + + # HMAC requires the key to be bytes, but data is string + mac = hmac.new(github_webhook_secret.encode(), msg=request.body, digestmod=hashlib.sha1) + + if not hmac.compare_digest(str(mac.hexdigest()), str(signature)): + abort(403) + + data = request.json repository = data["repository"]["full_name"] branch = data["ref"].split("/", 2)[2] with tempfile.TemporaryDirectory() as folder: - git(["clone", f"https://{login}:{token}@github.com/{repository}", "--single-branch", "--branch", branch, folder]) + await git(["clone", f"https://{login}:{token}@github.com/{repository}", "--single-branch", "--branch", branch, folder]) generate_READMEs(folder) - git(["add", "README*.md"], in_folder=folder) + await git(["add", "README*.md"], in_folder=folder) - diff_not_empty = bool(subprocess.check_output(["git", "diff", "HEAD", "--compact-summary"], cwd=folder).strip().decode("utf-8")) + diff_not_empty = await asyncio.create_subprocess_shell(" ".join(["git", "diff", "HEAD", "--compact-summary"]), cwd=folder, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT) + diff_not_empty = await diff_not_empty.stdout.read() + diff_not_empty = diff_not_empty.decode().strip() if not diff_not_empty: - return + return text("nothing to do") - git(["commit", "-a", "-m", "Auto-update README", "--author='Yunohost-Bot <>'"], in_folder=folder) - git(["push", "origin", branch, "--quiet"], in_folder=folder) + await git(["commit", "-a", "-m", "Auto-update README", "--author='Yunohost-Bot <>'"], in_folder=folder) + await git(["push", "origin", branch, "--quiet"], in_folder=folder) + + return text("ok") if __name__ == "__main__": From cfe373538a0a77194241a3863b7ab566bbdbc2dc Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 14 Jun 2021 16:34:26 +0200 Subject: [PATCH 03/51] [mod] add some debug logging --- tools/README-generator/webhook.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/README-generator/webhook.py b/tools/README-generator/webhook.py index b78b36d..a9165e6 100755 --- a/tools/README-generator/webhook.py +++ b/tools/README-generator/webhook.py @@ -34,6 +34,7 @@ async def git(cmd, in_folder=None): cmd = ["-C", in_folder] + cmd cmd = ["git"] + cmd cmd = " ".join(map(shlex.quote, cmd)) + print(cmd) command = await asyncio.create_subprocess_shell(cmd, env=my_env, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT) data = await command.stdout.read() return data.decode().strip() @@ -67,6 +68,8 @@ async def on_push(request): repository = data["repository"]["full_name"] branch = data["ref"].split("/", 2)[2] + print(f"{repository} -> branch '{branch}'") + with tempfile.TemporaryDirectory() as folder: await git(["clone", f"https://{login}:{token}@github.com/{repository}", "--single-branch", "--branch", branch, folder]) generate_READMEs(folder) @@ -77,6 +80,7 @@ async def on_push(request): diff_not_empty = await diff_not_empty.stdout.read() diff_not_empty = diff_not_empty.decode().strip() if not diff_not_empty: + print("nothing to do") return text("nothing to do") await git(["commit", "-a", "-m", "Auto-update README", "--author='Yunohost-Bot <>'"], in_folder=folder) From de62850ac565dcb76f1b5b8a6df90d0531eeb8e1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 4 Jul 2021 21:17:16 +0200 Subject: [PATCH 04/51] Don't overwrite translations from manifests + remove stale fr translations for descriptions --- list_builder.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/list_builder.py b/list_builder.py index 8af608b..6debc73 100755 --- a/list_builder.py +++ b/list_builder.py @@ -261,22 +261,21 @@ def include_translations_in_manifest(manifest): current_lang = locale.split(".")[0] translations = json.load(open(os.path.join("locales", locale), "r")) + # don't overwrite already existing translation in manifests for now key = "%s_manifest_description" % app_name - if translations.get(key, None): + if current_lang not in manifest["description"] and translations.get(key): manifest["description"][current_lang] = translations[key] for category, questions in manifest["arguments"].items(): for question in questions: key = "%s_manifest_arguments_%s_%s" % (app_name, category, question["name"]) # don't overwrite already existing translation in manifests for now - if translations.get(key) and "ask" in question and not current_lang not in question["ask"]: - #print("[ask]", current_lang, key) + if translations.get(key) and "ask" in question and current_lang not in question["ask"]: question["ask"][current_lang] = translations[key] key = "%s_manifest_arguments_%s_help_%s" % (app_name, category, question["name"]) # don't overwrite already existing translation in manifests for now - if translations.get(key) and not current_lang not in question.get("help", []): - #print("[help]", current_lang, key) + if translations.get(key) and current_lang not in question.get("help", []): question["help"][current_lang] = translations[key] return manifest From 92ecdf29cdf888a9859fedaf874d34f537d7e962 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 11 Jul 2021 16:06:59 +0200 Subject: [PATCH 05/51] Fix edge case where 'help' key doesn't exists yet there's a translation for it ... --- list_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/list_builder.py b/list_builder.py index 6debc73..0538090 100755 --- a/list_builder.py +++ b/list_builder.py @@ -275,7 +275,7 @@ def include_translations_in_manifest(manifest): key = "%s_manifest_arguments_%s_help_%s" % (app_name, category, question["name"]) # don't overwrite already existing translation in manifests for now - if translations.get(key) and current_lang not in question.get("help", []): + if translations.get(key) and "help" in question and current_lang not in question.get("help", []): question["help"][current_lang] = translations[key] return manifest From 55db4f6916324c4a2421f741ed3c474a53f3ad8c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 15 Aug 2021 16:38:07 +0200 Subject: [PATCH 06/51] Assume branch=master and revision=HEAD by default for all apps --- README.md | 11 +++++------ list_builder.py | 3 +++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b64506c..33e39a0 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,6 @@ To add your application to the catalog: App example addition: ```json "wallabag": { - "branch": "master", - "revision": "HEAD", "url": "https://github.com/abeudin/wallabag_ynh", "state": "working" } @@ -44,9 +42,10 @@ N.B. : We strongly encourage you to transfer the ownership of your repository to the Yunohost-Apps organization on Github, such that the community will help you with keeping your app working and up to date with packaging evolutions. -N.B.2 : If `"revision": "HEAD"` is used in `apps.json`, any commit to the -`master` branch on your app will automatically be published to the catalog. -Therefore we strongly encourage you to develop in separate branches, and only +N.B.2 : Implicitly, the catalog publishes the `HEAD` of branch `master` +(this can be overwritten by adding keys `branch` and `revision`). +Therefore, **be careful that any commit on the `master` branch will automatically be published**. +**We strongly encourage you to develop in separate branches**, and only merge changes that were carefully tested. Get in touch with the Apps group to obtain an access to the developer CI where you'll be able to test your app easily. @@ -72,7 +71,7 @@ instead of doing Pull Request for files in `locales` folder. ### How to make my app flagged as High Quality ? -A High Quality app will be highlighted in the app list and marked as a level 9 app. +A High Quality app will be highlighted in the app list and marked as a level 9 app. To become a High Quality app, a package has to follow the criterias listed [here](hq_validation_template.md). Once the app is validated is "high quality", the tag `"high_quality": true` diff --git a/list_builder.py b/list_builder.py index 0538090..be49b1f 100755 --- a/list_builder.py +++ b/list_builder.py @@ -212,6 +212,9 @@ def build_app_dict(app, infos): this_app_cache = app_cache_folder(app) assert os.path.exists(this_app_cache), "No cache yet for %s" % app + infos['branch'] = infos.get('branch', 'master') + infos['revision'] = infos.get('revision', 'HEAD') + # If using head, find the most recent meaningful commit in logs if infos["revision"] == "HEAD": relevant_files = ["manifest.json", "actions.json", "hooks/", "scripts/", "conf/", "sources/"] From 84b9af04e029d0fdf2e2665809530b3ce7517c88 Mon Sep 17 00:00:00 2001 From: ericgaspar Date: Sun, 22 Aug 2021 16:57:07 +0200 Subject: [PATCH 07/51] Add description --- tools/README-generator/make_readme.py | 9 +++++++++ tools/README-generator/templates/README.md.j2 | 2 +- tools/README-generator/templates/README_fr.md.j2 | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tools/README-generator/make_readme.py b/tools/README-generator/make_readme.py index 5491860..ddcc3d2 100755 --- a/tools/README-generator/make_readme.py +++ b/tools/README-generator/make_readme.py @@ -30,6 +30,14 @@ def generate_READMEs(app_path: str): template = env.get_template(f"README{lang_suffix}.md.j2") + if (app_path / "doc" / f"DESCRIPTION{lang_suffix}.md").exists(): + description = (app_path / "doc" / f"DESCRIPTION{lang_suffix}.md").read_text() + # Fallback to english if maintainer too lazy to translate the description + elif (app_path / "doc" / "DESCRIPTION.md").exists(): + description = (app_path / "doc" / "DESCRIPTION.md").read_text() + else: + description = None + if (app_path / "doc" / "screenshots").exists(): screenshots = os.listdir(os.path.join(app_path, "doc", "screenshots")) if ".gitkeep" in screenshots: @@ -48,6 +56,7 @@ def generate_READMEs(app_path: str): out = template.render( lang=lang, upstream=upstream, + description=description, screenshots=screenshots, disclaimer=disclaimer, manifest=manifest, diff --git a/tools/README-generator/templates/README.md.j2 b/tools/README-generator/templates/README.md.j2 index 9ceda16..6e58002 100644 --- a/tools/README-generator/templates/README.md.j2 +++ b/tools/README-generator/templates/README.md.j2 @@ -29,7 +29,7 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in ## Overview -{{manifest.description[lang]}} +{% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} **Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %} diff --git a/tools/README-generator/templates/README_fr.md.j2 b/tools/README-generator/templates/README_fr.md.j2 index 2fa3850..9169cbb 100644 --- a/tools/README-generator/templates/README_fr.md.j2 +++ b/tools/README-generator/templates/README_fr.md.j2 @@ -11,7 +11,7 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour ## Vue d'ensemble -{{manifest.description[lang]}} +{% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} **Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %} From 4cf979160492b443ad90ab0c5c9431c79e3c41be Mon Sep 17 00:00:00 2001 From: tituspijean Date: Sat, 4 Sep 2021 00:42:51 +0200 Subject: [PATCH 08/51] Fix author and committer name for README auto-update --- tools/README-generator/webhook.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/README-generator/webhook.py b/tools/README-generator/webhook.py index a9165e6..a2183df 100755 --- a/tools/README-generator/webhook.py +++ b/tools/README-generator/webhook.py @@ -20,9 +20,9 @@ token = open("token").read().strip() my_env = os.environ.copy() my_env["GIT_TERMINAL_PROMPT"] = "0" -my_env["GIT_AUTHOR_NAME"] = "Yunohost-Bot" +my_env["GIT_AUTHOR_NAME"] = "yunohost-bot" my_env["GIT_AUTHOR_EMAIL"] = "yunohost@yunohost.org" -my_env["GIT_COMMITTER_NAME"] = "Yunohost-Bot" +my_env["GIT_COMMITTER_NAME"] = "yunohost-bot" my_env["GIT_COMMITTER_EMAIL"] = "yunohost@yunohost.org" From 6dbe3b05bfcfba55914ee02f0756175e6e7d43ac Mon Sep 17 00:00:00 2001 From: yalh76 Date: Fri, 24 Sep 2021 21:30:14 +0200 Subject: [PATCH 09/51] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 33e39a0..73db579 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # YunoHost application catalog - + Here you will find the repositories and versions of every apps available in YunoHost's default catalog. From cff7381a7c094e9d781d23717edf9722e12bd6fd Mon Sep 17 00:00:00 2001 From: tituspijean Date: Tue, 26 Oct 2021 23:00:46 +0200 Subject: [PATCH 10/51] Homogenize bot's name/email b/w author/committer --- tools/README-generator/webhook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/README-generator/webhook.py b/tools/README-generator/webhook.py index a2183df..5f295de 100755 --- a/tools/README-generator/webhook.py +++ b/tools/README-generator/webhook.py @@ -83,7 +83,7 @@ async def on_push(request): print("nothing to do") return text("nothing to do") - await git(["commit", "-a", "-m", "Auto-update README", "--author='Yunohost-Bot <>'"], in_folder=folder) + await git(["commit", "-a", "-m", "Auto-update README", "--author='yunohost-bot '"], in_folder=folder) await git(["push", "origin", branch, "--quiet"], in_folder=folder) return text("ok") From 8477fa21670fc63a7b3ac3ecf61315649e74d6f8 Mon Sep 17 00:00:00 2001 From: tituspijean Date: Thu, 20 Jan 2022 14:21:52 +0100 Subject: [PATCH 11/51] App levels are automatically updated weekly --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 73db579..79e73ff 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,10 @@ merge changes that were carefully tested. Get in touch with the Apps group to obtain an access to the developer CI where you'll be able to test your app easily. +### Updating apps' level in the catalog + +App packagers should *not* manually set their apps' level. The levels of all the apps are automatically updated once per week on Friday. + #### Helper script You can use the add_or_update.py python script to add or update From 625c52c1b7131c7ef86a5260789fa83b061aa16b Mon Sep 17 00:00:00 2001 From: Tagada <36127788+Tagadda@users.noreply.github.com> Date: Fri, 21 Jan 2022 23:17:36 +0100 Subject: [PATCH 12/51] Anti-features draft --- list_builder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/list_builder.py b/list_builder.py index be49b1f..41c04d8 100755 --- a/list_builder.py +++ b/list_builder.py @@ -149,9 +149,10 @@ def build_catalog(): # Current version 2 # ##################### categories = yaml.load(open("categories.yml").read()) + antifeatures = yaml.load(open("antifeatures.yml").read()) os.system("mkdir -p ./builds/default/v2/") with open("builds/default/v2/apps.json", 'w') as f: - f.write(json.dumps({"apps": result_dict, "categories": categories}, sort_keys=True)) + f.write(json.dumps({"apps": result_dict, "categories": categories, "antifeatures": antifeatures}, sort_keys=True)) #################### # Legacy version 1 # @@ -199,6 +200,7 @@ def build_catalog(): "broken": level <= 0, "good_quality": level >= 8, "bad_quality": level <= 5, + "antifeatures": infos.get("antifeatures"), } result_dict_doc = {k: infos_for_doc_catalog(v) for k, v in result_dict.items() if v["state"] in ["working", "validated"]} with open("builds/default/doc_catalog/apps.json", 'w') as f: From b5dafd4d69540d564f1dceba3259e57d4978fdfb Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Wed, 9 Feb 2022 20:08:03 +0000 Subject: [PATCH 13/51] [fix] config_panel.toml is a relevant file --- list_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/list_builder.py b/list_builder.py index 41c04d8..9e04442 100755 --- a/list_builder.py +++ b/list_builder.py @@ -219,7 +219,7 @@ def build_app_dict(app, infos): # If using head, find the most recent meaningful commit in logs if infos["revision"] == "HEAD": - relevant_files = ["manifest.json", "actions.json", "hooks/", "scripts/", "conf/", "sources/"] + relevant_files = ["manifest.json", "config_panel.toml", "hooks/", "scripts/", "conf/", "sources/"] most_recent_relevant_commit = "rev-list --full-history --all -n 1 -- " + " ".join(relevant_files) infos["revision"] = git(most_recent_relevant_commit, in_folder=this_app_cache) assert re.match(r"^[0-9a-f]+$", infos["revision"]), "Output was not a commit? '%s'" % infos["revision"] From 46f44bd0db4e1e2884bc915d3d8389560573d9b8 Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Wed, 9 Feb 2022 20:09:00 +0000 Subject: [PATCH 14/51] build antifeature list from app manifest + catalog --- list_builder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/list_builder.py b/list_builder.py index 9e04442..59b9a26 100755 --- a/list_builder.py +++ b/list_builder.py @@ -200,7 +200,7 @@ def build_catalog(): "broken": level <= 0, "good_quality": level >= 8, "bad_quality": level <= 5, - "antifeatures": infos.get("antifeatures"), + "antifeatures": infos["antifeatures"], } result_dict_doc = {k: infos_for_doc_catalog(v) for k, v in result_dict.items() if v["state"] in ["working", "validated"]} with open("builds/default/doc_catalog/apps.json", 'w') as f: @@ -249,6 +249,7 @@ def build_app_dict(app, infos): 'featured': infos.get("featured", False), 'category': infos.get('category', None), 'subtags': infos.get('subtags', []), + 'antifeatures': list(set(manifest.get('antifeatures', []) + infos.get('antifeatures', []))) } From 88f8239b1652b0fea8a9f70c36a3f712a986fed0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Gaspar?= <46165813+ericgaspar@users.noreply.github.com> Date: Mon, 21 Feb 2022 19:05:04 +0100 Subject: [PATCH 15/51] minor typos --- README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 79e73ff..08bfda5 100644 --- a/README.md +++ b/README.md @@ -7,20 +7,20 @@ Here you will find the repositories and versions of every apps available in Yuno It is browsable here: https://yunohost.org/apps The main file of the catalog is [**apps.json**](./apps.json) which contains -references to the corresponding git repositories for each application, along +references to the corresponding Git repositories for each application, along with a few metadata about them such as its category or maintenance state. This file regularly read by `list_builder.py` which publish the results on https://app.yunohost.org/default/. -### Where can I learn about app packaging in Yunohost ? +### Where can I learn about app packaging in YunoHost? - You can browse the contributor documentation : https://yunohost.org/contributordoc -- If you are not familiar with Git/Github, you can have a look at our [homemade guide](https://yunohost.org/#/packaging_apps_git) -- Don't hesitate to reach for help on the dedicated [application packaging chatroom](https://yunohost.org/chat_rooms) ... we can even schedule an audio meeting to help you get started ! +- If you are not familiar with Git/GitHub, you can have a look at our [homemade guide](https://yunohost.org/#/packaging_apps_git) +- Don't hesitate to reach for help on the dedicated [application packaging chatroom](https://yunohost.org/chat_rooms) ... we can even schedule an audio meeting to help you get started! ### How to add your app to the application catalog -N.B. : the Yunohost project will **NOT** integrate in its catalog applications that are not +N.B.: The YunoHost project will **NOT** integrate in its catalog applications that are not based on free-software upstreams. To add your application to the catalog: @@ -38,11 +38,11 @@ App example addition: } ``` -N.B. : We strongly encourage you to transfer the ownership of your repository to -the Yunohost-Apps organization on Github, such that the community will help you +N.B.: We strongly encourage you to transfer the ownership of your repository to +the YunoHost-Apps organization on GitHub, such that the community will help you with keeping your app working and up to date with packaging evolutions. -N.B.2 : Implicitly, the catalog publishes the `HEAD` of branch `master` +N.B.2: Implicitly, the catalog publishes the `HEAD` of branch `master` (this can be overwritten by adding keys `branch` and `revision`). Therefore, **be careful that any commit on the `master` branch will automatically be published**. **We strongly encourage you to develop in separate branches**, and only @@ -56,8 +56,8 @@ App packagers should *not* manually set their apps' level. The levels of all the #### Helper script -You can use the add_or_update.py python script to add or update -your app from one of the 2 json files. +You can use the add_or_update.py Python script to add or update +your app from one of the 2 JSON files. Usage: @@ -67,13 +67,13 @@ Usage: ### How to help translating -Update on Nov. 2020 : this part is broken / not maintained anymore for the +Update on Nov. 2020: this part is broken / not maintained anymore for the moment... We invite you to use [translate.yunohost.org](https://translate.yunohost.org/) instead of doing Pull Request for files in `locales` folder. -### How to make my app flagged as High Quality ? +### How to make my app flagged as High Quality? A High Quality app will be highlighted in the app list and marked as a level 9 app. To become a High Quality app, a package has to follow the criterias listed [here](hq_validation_template.md). From db42e8323873bea04a4dba2f8f4ff1ebe313ca58 Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Tue, 22 Feb 2022 23:44:44 +0000 Subject: [PATCH 16/51] Add a warning when not on a master branch --- tools/README-generator/make_readme.py | 4 ++++ tools/README-generator/templates/README.md.j2 | 2 +- tools/README-generator/templates/README_fr.md.j2 | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/README-generator/make_readme.py b/tools/README-generator/make_readme.py index ddcc3d2..d00e1ed 100755 --- a/tools/README-generator/make_readme.py +++ b/tools/README-generator/make_readme.py @@ -53,6 +53,9 @@ def generate_READMEs(app_path: str): else: disclaimer = None + # Get the current branch using git inside the app path + branch = os.popen("git --git-dir=%s/.git --work-tree=%s rev-parse --abbrev-ref HEAD" % (app_path, app_path)).read().strip() + out = template.render( lang=lang, upstream=upstream, @@ -60,6 +63,7 @@ def generate_READMEs(app_path: str): screenshots=screenshots, disclaimer=disclaimer, manifest=manifest, + branch=branch, ) (app_path / f"README{lang_suffix}.md").write_text(out) diff --git a/tools/README-generator/templates/README.md.j2 b/tools/README-generator/templates/README.md.j2 index 6e58002..b0eb5c2 100644 --- a/tools/README-generator/templates/README.md.j2 +++ b/tools/README-generator/templates/README.md.j2 @@ -31,7 +31,7 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %} +**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if branch != "master" %} *(This is the `{{ branch }}` branch. See the [`master` branch](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/master) for the version in the catalog.)*{% endif %} {% if upstream.demo %}**Demo:** {{upstream.demo}}{% endif %} diff --git a/tools/README-generator/templates/README_fr.md.j2 b/tools/README-generator/templates/README_fr.md.j2 index 9169cbb..63ca8c4 100644 --- a/tools/README-generator/templates/README_fr.md.j2 +++ b/tools/README-generator/templates/README_fr.md.j2 @@ -13,7 +13,7 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %} +**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if branch != "master" %} *(Vous êtes sur la branche `{{ branch }}`. Voir la [branche `master`](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/master) pour la version du catalogue.)*{% endif %} {% if upstream.demo %}**Démo :** {{upstream.demo}}{% endif %} From be079a87126811df0d9da924aebc72a42a4c5698 Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Wed, 23 Feb 2022 06:36:19 +0000 Subject: [PATCH 17/51] Get the default branch from the apps.json --- tools/README-generator/make_readme.py | 4 ++++ tools/README-generator/templates/README.md.j2 | 2 +- tools/README-generator/templates/README_fr.md.j2 | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/README-generator/make_readme.py b/tools/README-generator/make_readme.py index d00e1ed..4b34ef5 100755 --- a/tools/README-generator/make_readme.py +++ b/tools/README-generator/make_readme.py @@ -18,6 +18,9 @@ def generate_READMEs(app_path: str): manifest = json.load(open(app_path / "manifest.json")) upstream = manifest.get("upstream", {}) + catalog = json.load(open(Path(__file__).parent.parent.parent / "apps.json")) + from_catalog = catalog.get(manifest['id'], {}) + if not upstream and not (app_path / "doc" / "DISCLAIMER.md").exists(): print( "There's no 'upstream' key in the manifest, and doc/DISCLAIMER.md doesn't exists - therefore assuming that we shall not auto-update the README.md for this app yet." @@ -64,6 +67,7 @@ def generate_READMEs(app_path: str): disclaimer=disclaimer, manifest=manifest, branch=branch, + default_branch=from_catalog.get('branch', 'master'), ) (app_path / f"README{lang_suffix}.md").write_text(out) diff --git a/tools/README-generator/templates/README.md.j2 b/tools/README-generator/templates/README.md.j2 index b0eb5c2..a0fdf07 100644 --- a/tools/README-generator/templates/README.md.j2 +++ b/tools/README-generator/templates/README.md.j2 @@ -31,7 +31,7 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if branch != "master" %} *(This is the `{{ branch }}` branch. See the [`master` branch](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/master) for the version in the catalog.)*{% endif %} +**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if branch != default_branch %} *(This is the `{{ branch }}` branch. See the [`{{ default_branch }}` branch](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/{{ default_branch }}) for the version in the catalog.)*{% endif %} {% if upstream.demo %}**Demo:** {{upstream.demo}}{% endif %} diff --git a/tools/README-generator/templates/README_fr.md.j2 b/tools/README-generator/templates/README_fr.md.j2 index 63ca8c4..14bd727 100644 --- a/tools/README-generator/templates/README_fr.md.j2 +++ b/tools/README-generator/templates/README_fr.md.j2 @@ -13,7 +13,7 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if branch != "master" %} *(Vous êtes sur la branche `{{ branch }}`. Voir la [branche `master`](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/master) pour la version du catalogue.)*{% endif %} +**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if branch != default_branch %} *(Vous êtes sur la branche `{{ branch }}`. Voir la [branche `{{ default_branch }}`](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/{{ default_branch }}) pour la version du catalogue.)*{% endif %} {% if upstream.demo %}**Démo :** {{upstream.demo}}{% endif %} From a02a2047eedc9bddab9076ebb3fd32646c500f72 Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Wed, 23 Feb 2022 07:07:45 +0000 Subject: [PATCH 18/51] fix --- tools/README-generator/make_readme.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/README-generator/make_readme.py b/tools/README-generator/make_readme.py index 4b34ef5..ac47a04 100755 --- a/tools/README-generator/make_readme.py +++ b/tools/README-generator/make_readme.py @@ -18,7 +18,7 @@ def generate_READMEs(app_path: str): manifest = json.load(open(app_path / "manifest.json")) upstream = manifest.get("upstream", {}) - catalog = json.load(open(Path(__file__).parent.parent.parent / "apps.json")) + catalog = json.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "apps.json")) from_catalog = catalog.get(manifest['id'], {}) if not upstream and not (app_path / "doc" / "DISCLAIMER.md").exists(): From 49fae19fc0f918eb8d754b7b34ae187ebe93f664 Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Wed, 23 Feb 2022 07:18:05 +0000 Subject: [PATCH 19/51] Add Anti-Features in READMEs --- tools/README-generator/make_readme.py | 12 ++++++++++++ tools/README-generator/requirements.txt | 1 + tools/README-generator/templates/README.md.j2 | 8 ++++++++ tools/README-generator/templates/README_fr.md.j2 | 8 ++++++++ 4 files changed, 29 insertions(+) diff --git a/tools/README-generator/make_readme.py b/tools/README-generator/make_readme.py index ddcc3d2..3e8be5a 100755 --- a/tools/README-generator/make_readme.py +++ b/tools/README-generator/make_readme.py @@ -3,6 +3,7 @@ import argparse import json import os +import yaml from pathlib import Path from jinja2 import Environment, FileSystemLoader @@ -18,6 +19,11 @@ def generate_READMEs(app_path: str): manifest = json.load(open(app_path / "manifest.json")) upstream = manifest.get("upstream", {}) + catalog = json.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "apps.json")) + from_catalog = catalog.get(manifest['id'], {}) + + antifeatures_list = yaml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "antifeatures.yaml")) + if not upstream and not (app_path / "doc" / "DISCLAIMER.md").exists(): print( "There's no 'upstream' key in the manifest, and doc/DISCLAIMER.md doesn't exists - therefore assuming that we shall not auto-update the README.md for this app yet." @@ -53,6 +59,12 @@ def generate_READMEs(app_path: str): else: disclaimer = None + # TODO: Add url to the documentation... and actually create that documentation :D + antifeatures = [antifeatures_list[a] for a in from_catalog.get('antifeatures', [])] + for antifeature in antifeatures: + antifeature['title'] = antifeature['title'].get(lang_suffix, 'en') + antifeature['description'] = antifeature['description'].get(lang_suffix, 'en') + out = template.render( lang=lang, upstream=upstream, diff --git a/tools/README-generator/requirements.txt b/tools/README-generator/requirements.txt index 88cd6e2..33fe25a 100644 --- a/tools/README-generator/requirements.txt +++ b/tools/README-generator/requirements.txt @@ -1,2 +1,3 @@ jinja2 sanic +pyyaml diff --git a/tools/README-generator/templates/README.md.j2 b/tools/README-generator/templates/README.md.j2 index 6e58002..8713a9a 100644 --- a/tools/README-generator/templates/README.md.j2 +++ b/tools/README-generator/templates/README.md.j2 @@ -49,6 +49,14 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in {{ disclaimer }} {% endif -%} +{% if antifeatures -%} +## Antifeatures + +{% for antifeature in antifeatures -%} + - **{{ antifeature.title }}**: {{ antifeature.description }} +{% endfor -%} +{% endif -%} + ## Documentation and resources {% if upstream.website -%}* Official app website: {{ upstream.website }} diff --git a/tools/README-generator/templates/README_fr.md.j2 b/tools/README-generator/templates/README_fr.md.j2 index 9169cbb..9e89c10 100644 --- a/tools/README-generator/templates/README_fr.md.j2 +++ b/tools/README-generator/templates/README_fr.md.j2 @@ -31,6 +31,14 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour {{ disclaimer }} {% endif -%} +{% if antifeatures -%} +## Fonctions indésirables + +{% for antifeature in antifeatures -%} + - **{{ antifeature.title }}**: {{ antifeature.description }} +{% endfor -%} +{% endif -%} + ## Documentations et ressources {% if upstream.website -%}* Site officiel de l'app : {{ upstream.website }} From 83b8e738cc3ff54489bc44c7d8c2825bbde5fab1 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Fri, 13 May 2022 00:13:27 +0200 Subject: [PATCH 20/51] Readmes improvements --- tools/README-generator/templates/README.md.j2 | 8 +++++--- tools/README-generator/templates/README_fr.md.j2 | 14 ++++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tools/README-generator/templates/README.md.j2 b/tools/README-generator/templates/README.md.j2 index 6e58002..19f87ef 100644 --- a/tools/README-generator/templates/README.md.j2 +++ b/tools/README-generator/templates/README.md.j2 @@ -19,7 +19,7 @@ It shall NOT be edited by hand. # {{manifest.name}} for YunoHost -[![Integration level](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) +[![Integration level](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Working status](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Maintenance status](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) [![Install {{manifest.name}} with YunoHost](https://install-app.yunohost.org/install-with-yunohost.svg)](https://install-app.yunohost.org/?app={{manifest.id}}) *[Lire ce readme en français.](./README_fr.md)* @@ -39,7 +39,7 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in ## Screenshots {% for screenshot in screenshots -%} - ![](./doc/screenshots/{{screenshot}}) + ![Screenshot of {{manifest.name}}](./doc/screenshots/{{screenshot}}) {% endfor %} {% endif -%} @@ -67,10 +67,12 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in Please send your pull request to the [testing branch](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing). To try the testing branch, please proceed like that. -``` + +``` bash sudo yunohost app install https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug or sudo yunohost app upgrade {{manifest.id}} -u https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug ``` **More info regarding app packaging:** https://yunohost.org/packaging_apps + diff --git a/tools/README-generator/templates/README_fr.md.j2 b/tools/README-generator/templates/README_fr.md.j2 index 9169cbb..f531abe 100644 --- a/tools/README-generator/templates/README_fr.md.j2 +++ b/tools/README-generator/templates/README_fr.md.j2 @@ -1,10 +1,14 @@ + + # {{manifest.name}} pour YunoHost -[![Niveau d'intégration](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) +[![Niveau d'intégration](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Status du fonctionnement](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Status de maintenance](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) [![Installer {{manifest.name}} avec YunoHost](https://install-app.yunohost.org/install-with-yunohost.svg)](https://install-app.yunohost.org/?app={{manifest.id}}) *[Read this readme in english.](./README.md)* -*[Lire ce readme en français.](./README_fr.md)* > *Ce package vous permet d'installer {{manifest.name}} rapidement et simplement sur un serveur YunoHost. Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour savoir comment l'installer et en profiter.* @@ -21,7 +25,7 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour ## Captures d'écran {% for screenshot in screenshots -%} - ![](./doc/screenshots/{{screenshot}}) + ![Capture d'écran de {{manifest.name}}](./doc/screenshots/{{screenshot}}) {% endfor %} {% endif -%} @@ -49,10 +53,12 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour Merci de faire vos pull request sur la [branche testing](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing). Pour essayer la branche testing, procédez comme suit. -``` + +``` bash sudo yunohost app install https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug ou sudo yunohost app upgrade {{manifest.id}} -u https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug ``` **Plus d'infos sur le packaging d'applications :** https://yunohost.org/packaging_apps + From 4fca389af9780b65d36e278be5c81bbbaa166bf5 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Fri, 13 May 2022 01:19:44 +0200 Subject: [PATCH 21/51] layout corrections (line breaks) --- tools/README-generator/templates/README.md.j2 | 9 ++++++--- tools/README-generator/templates/README_fr.md.j2 | 8 +++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tools/README-generator/templates/README.md.j2 b/tools/README-generator/templates/README.md.j2 index 19f87ef..e0f09d0 100644 --- a/tools/README-generator/templates/README.md.j2 +++ b/tools/README-generator/templates/README.md.j2 @@ -31,11 +31,14 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %} +**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}} +{% endif -%} -{% if upstream.demo %}**Demo:** {{upstream.demo}}{% endif %} +{% if upstream.demo %} +**Demo:** {{upstream.demo}} +{% endif -%} -{% if screenshots -%} +{% if screenshots %} ## Screenshots {% for screenshot in screenshots -%} diff --git a/tools/README-generator/templates/README_fr.md.j2 b/tools/README-generator/templates/README_fr.md.j2 index f531abe..a15ba8c 100644 --- a/tools/README-generator/templates/README_fr.md.j2 +++ b/tools/README-generator/templates/README_fr.md.j2 @@ -17,11 +17,13 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %} +**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}} +{% endif -%} -{% if upstream.demo %}**Démo :** {{upstream.demo}}{% endif %} +{% if upstream.demo %}**Démo :** {{upstream.demo}} +{% endif -%} -{% if screenshots -%} +{% if screenshots %} ## Captures d'écran {% for screenshot in screenshots -%} From b3dfdfc934efb2b78b66a72563f3dabbf24c1b24 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Fri, 13 May 2022 01:25:17 +0200 Subject: [PATCH 22/51] forgotten layout corrections (line breaks) --- tools/README-generator/templates/README_fr.md.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/README-generator/templates/README_fr.md.j2 b/tools/README-generator/templates/README_fr.md.j2 index a15ba8c..de77988 100644 --- a/tools/README-generator/templates/README_fr.md.j2 +++ b/tools/README-generator/templates/README_fr.md.j2 @@ -20,7 +20,8 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour **Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}} {% endif -%} -{% if upstream.demo %}**Démo :** {{upstream.demo}} +{% if upstream.demo %} +**Démo :** {{upstream.demo}} {% endif -%} {% if screenshots %} From 2cf6122d4281e6c4d10cd444d67fae0b3d9bbd2e Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Fri, 13 May 2022 18:15:34 +0200 Subject: [PATCH 23/51] add angle brackets around bare URLs (fix MD034) --- tools/README-generator/templates/README.md.j2 | 14 +++++++------- tools/README-generator/templates/README_fr.md.j2 | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tools/README-generator/templates/README.md.j2 b/tools/README-generator/templates/README.md.j2 index e0f09d0..4931754 100644 --- a/tools/README-generator/templates/README.md.j2 +++ b/tools/README-generator/templates/README.md.j2 @@ -54,16 +54,16 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in ## Documentation and resources -{% if upstream.website -%}* Official app website: {{ upstream.website }} +{% if upstream.website -%}* Official app website: <{{ upstream.website }}> {% endif -%} -{% if upstream.userdoc -%}* Official user documentation: {{ upstream.userdoc }} +{% if upstream.userdoc -%}* Official user documentation: <{{ upstream.userdoc }}> {% endif -%} -{% if upstream.admindoc -%}* Official admin documentation: {{ upstream.admindoc }} +{% if upstream.admindoc -%}* Official admin documentation: <{{ upstream.admindoc }}> {% endif -%} -{% if upstream.code -%}* Upstream app code repository: {{ upstream.code }} +{% if upstream.code -%}* Upstream app code repository: <{{ upstream.code }}> {% endif -%} -* YunoHost documentation for this app: https://yunohost.org/app_{{manifest.id}} -* Report a bug: https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/issues +* YunoHost documentation for this app: +* Report a bug: ## Developer info @@ -77,5 +77,5 @@ or sudo yunohost app upgrade {{manifest.id}} -u https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug ``` -**More info regarding app packaging:** https://yunohost.org/packaging_apps +**More info regarding app packaging:** diff --git a/tools/README-generator/templates/README_fr.md.j2 b/tools/README-generator/templates/README_fr.md.j2 index de77988..6f383ae 100644 --- a/tools/README-generator/templates/README_fr.md.j2 +++ b/tools/README-generator/templates/README_fr.md.j2 @@ -40,16 +40,16 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour ## Documentations et ressources -{% if upstream.website -%}* Site officiel de l'app : {{ upstream.website }} +{% if upstream.website -%}* Site officiel de l'app : <{{ upstream.website }}> {% endif -%} -{% if upstream.userdoc -%}* Documentation officielle utilisateur : {{ upstream.userdoc }} +{% if upstream.userdoc -%}* Documentation officielle utilisateur : <{{ upstream.userdoc }}> {% endif -%} -{% if upstream.admindoc -%}* Documentation officielle de l'admin : {{ upstream.admindoc }} +{% if upstream.admindoc -%}* Documentation officielle de l'admin : <{{ upstream.admindoc }}> {% endif -%} -{% if upstream.code -%}* Dépôt de code officiel de l'app : {{ upstream.code }} +{% if upstream.code -%}* Dépôt de code officiel de l'app : <{{ upstream.code }}> {% endif -%} -* Documentation YunoHost pour cette app : https://yunohost.org/app_{{manifest.id}} -* Signaler un bug : https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/issues +* Documentation YunoHost pour cette app : +* Signaler un bug : ## Informations pour les développeurs @@ -63,5 +63,5 @@ ou sudo yunohost app upgrade {{manifest.id}} -u https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug ``` -**Plus d'infos sur le packaging d'applications :** https://yunohost.org/packaging_apps +**Plus d'infos sur le packaging d'applications :** From ca986a16a3e14c997400be4da40375ccbf12fd97 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Sat, 21 May 2022 16:09:09 +0200 Subject: [PATCH 24/51] harmonising the style of lists (MD004) --- tools/README-generator/templates/README.md.j2 | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/README-generator/templates/README.md.j2 b/tools/README-generator/templates/README.md.j2 index 4931754..78a1139 100644 --- a/tools/README-generator/templates/README.md.j2 +++ b/tools/README-generator/templates/README.md.j2 @@ -1,13 +1,13 @@ {% if manifest.id == "example" -%} # Packaging your an app, starting from this example -- Copy this app before working on it, using the ['Use this template'](https://github.com/YunoHost/example_ynh/generate) button on the Github repo. -- Edit the `manifest.json` with app specific info. -- Edit the `install`, `upgrade`, `remove`, `backup`, and `restore` scripts, and any relevant conf files in `conf/`. - - Using the [script helpers documentation.](https://yunohost.org/packaging_apps_helpers) -- Add a `LICENSE` file for the package. -- Edit `doc/DISCLAIMER*.md` -- The `README.md` files are to be automatically generated by https://github.com/YunoHost/apps/tree/master/tools/README-generator +* Copy this app before working on it, using the ['Use this template'](https://github.com/YunoHost/example_ynh/generate) button on the Github repo. +* Edit the `manifest.json` with app specific info. +* Edit the `install`, `upgrade`, `remove`, `backup`, and `restore` scripts, and any relevant conf files in `conf/`. + * Using the [script helpers documentation.](https://yunohost.org/packaging_apps_helpers) +* Add a `LICENSE` file for the package. +* Edit `doc/DISCLAIMER*.md` +* The `README.md` files are to be automatically generated by https://github.com/YunoHost/apps/tree/master/tools/README-generator --- {% endif -%} From 10bf08156785b2785b9e22c89152d26fcc2b4839 Mon Sep 17 00:00:00 2001 From: tituspijean Date: Wed, 25 May 2022 09:26:35 +0200 Subject: [PATCH 25/51] =?UTF-8?q?Coquille=20dans=20le=20g=C3=A9n=C3=A9rate?= =?UTF-8?q?ur=20de=20README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: nicofrand --- tools/README-generator/templates/README_fr.md.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/README-generator/templates/README_fr.md.j2 b/tools/README-generator/templates/README_fr.md.j2 index 6f383ae..7c2bf63 100644 --- a/tools/README-generator/templates/README_fr.md.j2 +++ b/tools/README-generator/templates/README_fr.md.j2 @@ -5,7 +5,7 @@ It shall NOT be edited by hand. # {{manifest.name}} pour YunoHost -[![Niveau d'intégration](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Status du fonctionnement](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Status de maintenance](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) +[![Niveau d'intégration](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Statut du fonctionnement](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Status de maintenance](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) [![Installer {{manifest.name}} avec YunoHost](https://install-app.yunohost.org/install-with-yunohost.svg)](https://install-app.yunohost.org/?app={{manifest.id}}) *[Read this readme in english.](./README.md)* From 09a0bfd500d37a9b2abace79e4422063dca67f9d Mon Sep 17 00:00:00 2001 From: tituspijean Date: Wed, 25 May 2022 10:04:17 +0200 Subject: [PATCH 26/51] =?UTF-8?q?Coquille=20dans=20le=20g=C3=A9n=C3=A9rate?= =?UTF-8?q?ur=20de=20README,=20la=20suite?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: nicofrand --- tools/README-generator/templates/README_fr.md.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/README-generator/templates/README_fr.md.j2 b/tools/README-generator/templates/README_fr.md.j2 index 7c2bf63..ae51374 100644 --- a/tools/README-generator/templates/README_fr.md.j2 +++ b/tools/README-generator/templates/README_fr.md.j2 @@ -5,7 +5,7 @@ It shall NOT be edited by hand. # {{manifest.name}} pour YunoHost -[![Niveau d'intégration](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Statut du fonctionnement](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Status de maintenance](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) +[![Niveau d'intégration](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Statut du fonctionnement](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Statut de maintenance](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) [![Installer {{manifest.name}} avec YunoHost](https://install-app.yunohost.org/install-with-yunohost.svg)](https://install-app.yunohost.org/?app={{manifest.id}}) *[Read this readme in english.](./README.md)* From 2ba8910383ff2948b9e6c1ce88197b808c376100 Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Mon, 30 May 2022 11:59:54 +0000 Subject: [PATCH 27/51] fix --- tools/README-generator/make_readme.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tools/README-generator/make_readme.py b/tools/README-generator/make_readme.py index 3e8be5a..788b8a7 100755 --- a/tools/README-generator/make_readme.py +++ b/tools/README-generator/make_readme.py @@ -22,7 +22,8 @@ def generate_READMEs(app_path: str): catalog = json.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "apps.json")) from_catalog = catalog.get(manifest['id'], {}) - antifeatures_list = yaml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "antifeatures.yaml")) + antifeatures_list = yaml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "antifeatures.yml")) + antifeatures_list = { e['id']: e for e in antifeatures_list } if not upstream and not (app_path / "doc" / "DISCLAIMER.md").exists(): print( @@ -60,10 +61,13 @@ def generate_READMEs(app_path: str): disclaimer = None # TODO: Add url to the documentation... and actually create that documentation :D - antifeatures = [antifeatures_list[a] for a in from_catalog.get('antifeatures', [])] - for antifeature in antifeatures: - antifeature['title'] = antifeature['title'].get(lang_suffix, 'en') - antifeature['description'] = antifeature['description'].get(lang_suffix, 'en') + antifeatures = { a: antifeatures_list[a] for a in from_catalog.get('antifeatures', [])} + for k, v in antifeatures: + v['title'] = v['title'].get(lang_suffix, 'en') + if manifest.get("antifeatures", {}).get(k, 'en'): + v['description'] = manifest.get("antifeatures", {}).get(k, 'en') + else: + antifeature['description'] = antifeature['description'].get(lang_suffix, 'en') out = template.render( lang=lang, From 9db6bde4b01a9a62e362d6f2cca6abf6c77cfd9c Mon Sep 17 00:00:00 2001 From: yalh76 Date: Tue, 7 Jun 2022 22:48:50 +0200 Subject: [PATCH 28/51] Update README.md.j2 --- tools/README-generator/templates/README.md.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/README-generator/templates/README.md.j2 b/tools/README-generator/templates/README.md.j2 index 78a1139..d04393b 100644 --- a/tools/README-generator/templates/README.md.j2 +++ b/tools/README-generator/templates/README.md.j2 @@ -1,5 +1,5 @@ {% if manifest.id == "example" -%} -# Packaging your an app, starting from this example +# Packaging an app, starting from this example * Copy this app before working on it, using the ['Use this template'](https://github.com/YunoHost/example_ynh/generate) button on the Github repo. * Edit the `manifest.json` with app specific info. From fe52e9061034470124729c970322a8795a978d32 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 3 Aug 2022 15:41:11 +0200 Subject: [PATCH 29/51] black on list_builder.py --- list_builder.py | 219 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 163 insertions(+), 56 deletions(-) diff --git a/list_builder.py b/list_builder.py index 59b9a26..4f6d114 100755 --- a/list_builder.py +++ b/list_builder.py @@ -11,7 +11,9 @@ import time now = time.time() catalog = json.load(open("apps.json")) -catalog = {app: infos for app, infos in catalog.items() if infos.get('state') != 'notworking'} +catalog = { + app: infos for app, infos in catalog.items() if infos.get("state") != "notworking" +} my_env = os.environ.copy() my_env["GIT_TERMINAL_PROMPT"] = "0" @@ -19,31 +21,39 @@ my_env["GIT_TERMINAL_PROMPT"] = "0" os.makedirs(".apps_cache", exist_ok=True) os.makedirs("builds/", exist_ok=True) + def error(msg): msg = "[Applist builder error] " + msg if os.path.exists("/usr/bin/sendxmpppy"): - subprocess.call(["sendxmpppy", msg], stdout=open(os.devnull, 'wb')) + subprocess.call(["sendxmpppy", msg], stdout=open(os.devnull, "wb")) print(msg + "\n") + # Progress bar helper, stolen from https://stackoverflow.com/a/34482761 def progressbar(it, prefix="", size=60, file=sys.stdout): count = len(it) + def show(j, name=""): name += " " - x = int(size*j/count) - file.write("%s[%s%s] %i/%i %s\r" % (prefix, "#"*x, "."*(size-x), j, count, name)) + x = int(size * j / count) + file.write( + "%s[%s%s] %i/%i %s\r" % (prefix, "#" * x, "." * (size - x), j, count, name) + ) file.flush() + show(0) for i, item in enumerate(it): yield item - show(i+1, item[0]) + show(i + 1, item[0]) file.write("\n") file.flush() + ################################### # App git clones cache management # ################################### + def app_cache_folder(app): return os.path.join(".apps_cache", app) @@ -81,11 +91,13 @@ def init_cache(app, infos): else: depth = 40 - git("clone --quiet --depth {depth} --single-branch --branch {branch} {url} {folder}".format( - depth=depth, - url=infos["url"], - branch=infos.get("branch", "master"), - folder=app_cache_folder(app)) + git( + "clone --quiet --depth {depth} --single-branch --branch {branch} {url} {folder}".format( + depth=depth, + url=infos["url"], + branch=infos.get("branch", "master"), + folder=app_cache_folder(app), + ) ) @@ -102,24 +114,35 @@ def refresh_cache(app, infos): git("remote set-url origin " + infos["url"], in_folder=app_cache_folder(app)) # With git >= 2.22 # current_branch = git("branch --show-current", in_folder=app_cache_folder(app)) - current_branch = git("rev-parse --abbrev-ref HEAD", in_folder=app_cache_folder(app)) + current_branch = git( + "rev-parse --abbrev-ref HEAD", in_folder=app_cache_folder(app) + ) if current_branch != branch: # With git >= 2.13 # all_branches = git("branch --format=%(refname:short)", in_folder=app_cache_folder(app)).split() all_branches = git("branch", in_folder=app_cache_folder(app)).split() - all_branches.remove('*') + all_branches.remove("*") if branch not in all_branches: - git("remote set-branches --add origin %s" % branch, in_folder=app_cache_folder(app)) - git("fetch origin %s:%s" % (branch, branch), in_folder=app_cache_folder(app)) + git( + "remote set-branches --add origin %s" % branch, + in_folder=app_cache_folder(app), + ) + git( + "fetch origin %s:%s" % (branch, branch), + in_folder=app_cache_folder(app), + ) else: - git("checkout --force %s" % branch, in_folder=app_cache_folder(app)) + git("checkout --force %s" % branch, in_folder=app_cache_folder(app)) git("fetch --quiet origin %s --force" % branch, in_folder=app_cache_folder(app)) git("reset origin/%s --hard" % branch, in_folder=app_cache_folder(app)) except: # Sometimes there are tmp issue such that the refresh cache .. # we don't trigger an error unless the cache hasnt been updated since more than 24 hours - if os.path.exists(fetch_head) and now - os.path.getmtime(fetch_head) < 24*3600: + if ( + os.path.exists(fetch_head) + and now - os.path.getmtime(fetch_head) < 24 * 3600 + ): pass else: raise @@ -129,6 +152,7 @@ def refresh_cache(app, infos): # Actual list build management # ################################ + def build_catalog(): result_dict = {} @@ -151,33 +175,71 @@ def build_catalog(): categories = yaml.load(open("categories.yml").read()) antifeatures = yaml.load(open("antifeatures.yml").read()) os.system("mkdir -p ./builds/default/v2/") - with open("builds/default/v2/apps.json", 'w') as f: - f.write(json.dumps({"apps": result_dict, "categories": categories, "antifeatures": antifeatures}, sort_keys=True)) + with open("builds/default/v2/apps.json", "w") as f: + f.write( + json.dumps( + { + "apps": result_dict, + "categories": categories, + "antifeatures": antifeatures, + }, + sort_keys=True, + ) + ) #################### # Legacy version 1 # #################### os.system("mkdir -p ./builds/default/v1/") - with open("./builds/default/v1/apps.json", 'w') as f: + with open("./builds/default/v1/apps.json", "w") as f: f.write(json.dumps(result_dict, sort_keys=True)) #################### # Legacy version 0 # #################### - official_apps = set(["agendav", "ampache", "baikal", "dokuwiki", "etherpad_mypads", "hextris", "jirafeau", "kanboard", "my_webapp", "nextcloud", "opensondage", "phpmyadmin", "piwigo", "rainloop", "roundcube", "searx", "shellinabox", "strut", "synapse", "transmission", "ttrss", "wallabag2", "wordpress", "zerobin"]) + official_apps = set( + [ + "agendav", + "ampache", + "baikal", + "dokuwiki", + "etherpad_mypads", + "hextris", + "jirafeau", + "kanboard", + "my_webapp", + "nextcloud", + "opensondage", + "phpmyadmin", + "piwigo", + "rainloop", + "roundcube", + "searx", + "shellinabox", + "strut", + "synapse", + "transmission", + "ttrss", + "wallabag2", + "wordpress", + "zerobin", + ] + ) official_apps_dict = {k: v for k, v in result_dict.items() if k in official_apps} - community_apps_dict = {k: v for k, v in result_dict.items() if k not in official_apps} + community_apps_dict = { + k: v for k, v in result_dict.items() if k not in official_apps + } # We need the official apps to have "validated" as state to be recognized as official for app, infos in official_apps_dict.items(): infos["state"] = "validated" os.system("mkdir -p ./builds/default/v0/") - with open("./builds/default/v0/official.json", 'w') as f: + with open("./builds/default/v0/official.json", "w") as f: f.write(json.dumps(official_apps_dict, sort_keys=True)) - with open("./builds/default/v0/community.json", 'w') as f: + with open("./builds/default/v0/community.json", "w") as f: f.write(json.dumps(community_apps_dict, sort_keys=True)) ############################## @@ -185,6 +247,7 @@ def build_catalog(): ############################## categories = yaml.load(open("categories.yml").read()) os.system("mkdir -p ./builds/default/doc_catalog") + def infos_for_doc_catalog(infos): level = infos.get("level") if not isinstance(level, int): @@ -199,13 +262,21 @@ def build_catalog(): "level": level, "broken": level <= 0, "good_quality": level >= 8, - "bad_quality": level <= 5, + "bad_quality": level <= 5, "antifeatures": infos["antifeatures"], } - result_dict_doc = {k: infos_for_doc_catalog(v) for k, v in result_dict.items() if v["state"] in ["working", "validated"]} - with open("builds/default/doc_catalog/apps.json", 'w') as f: - f.write(json.dumps({"apps": result_dict_doc, "categories": categories}, sort_keys=True)) + result_dict_doc = { + k: infos_for_doc_catalog(v) + for k, v in result_dict.items() + if v["state"] in ["working", "validated"] + } + with open("builds/default/doc_catalog/apps.json", "w") as f: + f.write( + json.dumps( + {"apps": result_dict_doc, "categories": categories}, sort_keys=True + ) + ) def build_app_dict(app, infos): @@ -214,43 +285,63 @@ def build_app_dict(app, infos): this_app_cache = app_cache_folder(app) assert os.path.exists(this_app_cache), "No cache yet for %s" % app - infos['branch'] = infos.get('branch', 'master') - infos['revision'] = infos.get('revision', 'HEAD') + infos["branch"] = infos.get("branch", "master") + infos["revision"] = infos.get("revision", "HEAD") # If using head, find the most recent meaningful commit in logs if infos["revision"] == "HEAD": - relevant_files = ["manifest.json", "config_panel.toml", "hooks/", "scripts/", "conf/", "sources/"] - most_recent_relevant_commit = "rev-list --full-history --all -n 1 -- " + " ".join(relevant_files) + relevant_files = [ + "manifest.json", + "config_panel.toml", + "hooks/", + "scripts/", + "conf/", + "sources/", + ] + most_recent_relevant_commit = ( + "rev-list --full-history --all -n 1 -- " + " ".join(relevant_files) + ) infos["revision"] = git(most_recent_relevant_commit, in_folder=this_app_cache) - assert re.match(r"^[0-9a-f]+$", infos["revision"]), "Output was not a commit? '%s'" % infos["revision"] + assert re.match(r"^[0-9a-f]+$", infos["revision"]), ( + "Output was not a commit? '%s'" % infos["revision"] + ) # Otherwise, validate commit exists else: - assert infos["revision"] in git("rev-list --all", in_folder=this_app_cache).split("\n"), "Revision ain't in history ? %s" % infos["revision"] + assert infos["revision"] in git( + "rev-list --all", in_folder=this_app_cache + ).split("\n"), ("Revision ain't in history ? %s" % infos["revision"]) # Find timestamp corresponding to that commit - timestamp = git("show -s --format=%ct " + infos["revision"], in_folder=this_app_cache) - assert re.match(r"^[0-9]+$", timestamp), "Failed to get timestamp for revision ? '%s'" % timestamp + timestamp = git( + "show -s --format=%ct " + infos["revision"], in_folder=this_app_cache + ) + assert re.match(r"^[0-9]+$", timestamp), ( + "Failed to get timestamp for revision ? '%s'" % timestamp + ) timestamp = int(timestamp) # Build the dict with all the infos manifest = json.load(open(this_app_cache + "/manifest.json")) - return {'id':manifest["id"], - 'git': { - 'branch': infos['branch'], - 'revision': infos["revision"], - 'url': infos["url"] - }, - 'lastUpdate': timestamp, - 'manifest': include_translations_in_manifest(manifest), - 'state': infos['state'], - 'level': infos.get('level', '?'), - 'maintained': infos.get("maintained", True), - 'high_quality': infos.get("high_quality", False), - 'featured': infos.get("featured", False), - 'category': infos.get('category', None), - 'subtags': infos.get('subtags', []), - 'antifeatures': list(set(manifest.get('antifeatures', []) + infos.get('antifeatures', []))) - } + return { + "id": manifest["id"], + "git": { + "branch": infos["branch"], + "revision": infos["revision"], + "url": infos["url"], + }, + "lastUpdate": timestamp, + "manifest": include_translations_in_manifest(manifest), + "state": infos["state"], + "level": infos.get("level", "?"), + "maintained": infos.get("maintained", True), + "high_quality": infos.get("high_quality", False), + "featured": infos.get("featured", False), + "category": infos.get("category", None), + "subtags": infos.get("subtags", []), + "antifeatures": list( + set(manifest.get("antifeatures", []) + infos.get("antifeatures", [])) + ), + } def include_translations_in_manifest(manifest): @@ -274,14 +365,30 @@ def include_translations_in_manifest(manifest): for category, questions in manifest["arguments"].items(): for question in questions: - key = "%s_manifest_arguments_%s_%s" % (app_name, category, question["name"]) + key = "%s_manifest_arguments_%s_%s" % ( + app_name, + category, + question["name"], + ) # don't overwrite already existing translation in manifests for now - if translations.get(key) and "ask" in question and current_lang not in question["ask"]: + if ( + translations.get(key) + and "ask" in question + and current_lang not in question["ask"] + ): question["ask"][current_lang] = translations[key] - key = "%s_manifest_arguments_%s_help_%s" % (app_name, category, question["name"]) + key = "%s_manifest_arguments_%s_help_%s" % ( + app_name, + category, + question["name"], + ) # don't overwrite already existing translation in manifests for now - if translations.get(key) and "help" in question and current_lang not in question.get("help", []): + if ( + translations.get(key) + and "help" in question + and current_lang not in question.get("help", []) + ): question["help"][current_lang] = translations[key] return manifest From a3d1738e8e699c7305f4c3cdbcf74efa62106e1e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 3 Aug 2022 16:12:15 +0200 Subject: [PATCH 30/51] list_builder: Drop support for the superold v0 / v1 catalog API --- list_builder.py | 55 ------------------------------------------------- 1 file changed, 55 deletions(-) diff --git a/list_builder.py b/list_builder.py index 4f6d114..b96148f 100755 --- a/list_builder.py +++ b/list_builder.py @@ -187,61 +187,6 @@ def build_catalog(): ) ) - #################### - # Legacy version 1 # - #################### - os.system("mkdir -p ./builds/default/v1/") - with open("./builds/default/v1/apps.json", "w") as f: - f.write(json.dumps(result_dict, sort_keys=True)) - - #################### - # Legacy version 0 # - #################### - official_apps = set( - [ - "agendav", - "ampache", - "baikal", - "dokuwiki", - "etherpad_mypads", - "hextris", - "jirafeau", - "kanboard", - "my_webapp", - "nextcloud", - "opensondage", - "phpmyadmin", - "piwigo", - "rainloop", - "roundcube", - "searx", - "shellinabox", - "strut", - "synapse", - "transmission", - "ttrss", - "wallabag2", - "wordpress", - "zerobin", - ] - ) - - official_apps_dict = {k: v for k, v in result_dict.items() if k in official_apps} - community_apps_dict = { - k: v for k, v in result_dict.items() if k not in official_apps - } - - # We need the official apps to have "validated" as state to be recognized as official - for app, infos in official_apps_dict.items(): - infos["state"] = "validated" - - os.system("mkdir -p ./builds/default/v0/") - with open("./builds/default/v0/official.json", "w") as f: - f.write(json.dumps(official_apps_dict, sort_keys=True)) - - with open("./builds/default/v0/community.json", "w") as f: - f.write(json.dumps(community_apps_dict, sort_keys=True)) - ############################## # Version for catalog in doc # ############################## From 954fcc4addfe646353ab89cade3cbca6a29640ca Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 3 Aug 2022 21:02:49 +0200 Subject: [PATCH 31/51] Yoloadd an app catalog v3 + tools to convert apps from v1 to v2 --- list_builder.py | 24 ++ tools/__init__.py | 0 tools/packaging_v2/__init__.py | 0 .../convert_app_to_packaging_v2.py | 321 ++++++++++++++++++ .../convert_v1_manifest_to_v2_for_catalog.py | 55 +++ 5 files changed, 400 insertions(+) create mode 100644 tools/__init__.py create mode 100644 tools/packaging_v2/__init__.py create mode 100644 tools/packaging_v2/convert_app_to_packaging_v2.py create mode 100644 tools/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py diff --git a/list_builder.py b/list_builder.py index b96148f..1242244 100755 --- a/list_builder.py +++ b/list_builder.py @@ -1,5 +1,6 @@ #!/usr/bin/python3 +import copy import sys import os import re @@ -8,6 +9,8 @@ import subprocess import yaml import time +from tools.packaging_v2.convert_v1_manifest_to_v2_for_catalog import convert_v1_manifest_to_v2_for_catalog + now = time.time() catalog = json.load(open("apps.json")) @@ -187,6 +190,27 @@ def build_catalog(): ) ) + ################################### + # Catalog API v3 with manifest v2 # + ################################### + + result_dict_with_manifest_v2 = copy.deepcopy(result_dict) + for app in result_dict_with_manifest_v2: + app["manifest"] = convert_v1_manifest_to_v2_for_catalog(app["manifest"]) + + os.system("mkdir -p ./builds/default/v3/") + with open("builds/default/v3/apps.json", "w") as f: + f.write( + json.dumps( + { + "apps": result_dict_with_manifest_v2, + "categories": categories, + "antifeatures": antifeatures, + }, + sort_keys=True, + ) + ) + ############################## # Version for catalog in doc # ############################## diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/packaging_v2/__init__.py b/tools/packaging_v2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/packaging_v2/convert_app_to_packaging_v2.py b/tools/packaging_v2/convert_app_to_packaging_v2.py new file mode 100644 index 0000000..fd66325 --- /dev/null +++ b/tools/packaging_v2/convert_app_to_packaging_v2.py @@ -0,0 +1,321 @@ +import argparse +import os +import re +import json +import subprocess + + +def check_output(cmd): + return ( + subprocess.check_output(cmd, shell=True) + .decode("utf-8") + .strip() + ) + + +def _convert_v1_manifest_to_v2(app_path): + + manifest = json.load(open(app_path + "/manifest.json")) + + if "upstream" not in manifest: + manifest["upstream"] = {} + + if "license" in manifest and "license" not in manifest["upstream"]: + manifest["upstream"]["license"] = manifest["license"] + + if "url" in manifest and "website" not in manifest["upstream"]: + manifest["upstream"]["website"] = manifest["url"] + + manifest["integration"] = { + "yunohost": manifest.get("requirements", {}).get("yunohost"), + "architectures": "all", + "multi_instance": manifest.get("multi_instance", False), + "ldap": "?", + "sso": "?", + "disk": "50M", + "ram.build": "50M", + "ram.runtime": "50M" + } + + maintainer = manifest.get("maintainer", {}).get("name") + manifest["maintainers"] = [maintainer] if maintainer else [] + + install_questions = manifest["arguments"]["install"] + manifest["install"] = {} + for question in install_questions: + name = question.pop("name") + if "ask" in question and name in ["domain", "path", "admin", "is_public", "password"]: + question.pop("ask") + if question.get("example") and question.get("type") in ["domain", "path", "user", "boolean", "password"]: + question.pop("example") + + manifest["install"][name] = question + + # Rename is_public to init_main_permission + manifest["install"] = {(k if k != "is_public" else "init_main_permission"): v for k, v in manifest["install"].items()} + + if "init_main_permission" in manifest["install"]: + manifest["install"]["init_main_permission"]["type"] = "group" + if manifest["install"]["init_main_permission"].get("default") is True: + manifest["install"]["init_main_permission"]["default"] = "visitors" + elif manifest["install"]["init_main_permission"].get("default") is True: + manifest["install"]["init_main_permission"]["default"] = "all_users" + + if "domain" in manifest["install"] and "path" not in manifest["install"]: + manifest["install"]["domain"]["full_domain"] = True + + manifest["resources"] = {} + manifest["resources"]["system_user"] = {} + manifest["resources"]["install_dir"] = {} + + if os.system(f"grep -q 'datadir=' {app_path}/scripts/install") == 0: + manifest["resources"]["data_dir"] = {} + + manifest["resources"]["permissions"] = {} + + if os.system(f"grep -q 'ynh_webpath_register' '{app_path}/scripts/install'") == 0: + manifest["resources"]["permissions"]["main.url"] = "/" + + # FIXME: Parse ynh_permission_create --permission="admin" --url="/wp-login.php" --additional_urls="/wp-admin.php" --allowed=$admin_wordpress + + ports = check_output(f"sed -nr 's/(\\w+)=.*ynh_find_port[^0-9]*([0-9]+)\\)/\\1,\\2/p' '{app_path}/scripts/install'") + if ports: + manifest["resources"]["ports"] = {} + for port in ports.split("\n"): + name, default = port.split(",") + exposed = check_output(f"sed -nr 's/.*yunohost firewall allow .*(TCP|UDP|Both).*${name}/\\1/p' '{app_path}/scripts/install'") + if exposed == "Both": + exposed = True + + name = name.replace("_port", "").replace("port_", "") + if name == "port": + name = "main" + + manifest["resources"]["ports"][f"{name}.default"] = int(default) + if exposed: + manifest["resources"]["ports"][f"{name}.exposed"] = exposed + + maybequote = "[\"'\"'\"']?" + apt_dependencies = check_output(f"sed -nr 's/.*_dependencies={maybequote}(.*){maybequote}? *$/\\1/p' '{app_path}/scripts/_common.sh' | tr -d '\"' | sed 's@ @\\n@g'") + php_version = check_output(f"sed -nr 's/^ *YNH_PHP_VERSION={maybequote}(.*){maybequote}?$/\\1/p' '{app_path}/scripts/_common.sh' | tr -d \"\\\"'\"") + if apt_dependencies.strip(): + if php_version: + apt_dependencies = apt_dependencies.replace("${YNH_PHP_VERSION}", php_version) + apt_dependencies = ', '.join([d for d in apt_dependencies.split("\n") if d]) + manifest["resources"]["apt"] = {"packages": apt_dependencies} + + extra_apt_repos = check_output(r"sed -nr 's/.*_extra_app_dependencies.*repo=\"(.*)\".*package=\"(.*)\".*key=\"(.*)\"/\1,\2,\3/p' %s/scripts/install" % app_path) + if extra_apt_repos: + for i, extra_apt_repo in enumerate(extra_apt_repos.split("\n")): + repo, packages, key = extra_apt_repo.split(",") + packages = packages.replace('$', '#FIXME#$') + if "apt" not in manifest["resources"]: + manifest["resources"]["apt"] = {} + if "extras" not in manifest["resources"]["apt"]: + manifest["resources"]["apt"]["extras"] = [] + manifest["resources"]["apt"]["extras"].append({ + "repo": repo, + "key": key, + "packages": packages, + }) + + if os.system(f"grep -q 'ynh_mysql_setup_db' {app_path}/scripts/install") == 0: + manifest["resources"]["database"] = {"type": "mysql"} + elif os.system(f"grep -q 'ynh_psql_setup_db' {app_path}/scripts/install") == 0: + manifest["resources"]["database"] = {"type": "postgresql"} + + keys_to_keep = ["packaging_format", "id", "name", "description", "version", "maintainers", "upstream", "integration", "install", "resources"] + + keys_to_del = [key for key in manifest.keys() if key not in keys_to_keep] + for key in keys_to_del: + del manifest[key] + + return manifest + + +def _dump_v2_manifest_as_toml(manifest): + + import re + from tomlkit import document, nl, table, dumps, comment + + toml_manifest = document() + toml_manifest.add("packaging_format", 2) + toml_manifest.add(nl()) + toml_manifest.add("id", manifest["id"]) + toml_manifest.add("name", manifest["name"]) + for lang, value in manifest["description"].items(): + toml_manifest.add(f"description.{lang}", value) + toml_manifest.add(nl()) + toml_manifest.add("version", manifest["version"]) + toml_manifest.add(nl()) + toml_manifest.add("maintainers", manifest["maintainers"]) + + upstream = table() + for key, value in manifest["upstream"].items(): + upstream[key] = value + toml_manifest["upstream"] = upstream + + integration = table() + for key, value in manifest["integration"].items(): + integration.add(key, value) + integration["architectures"].comment('FIXME: can be replaced by a list of supported archs using the dpkg --print-architecture nomenclature (amd64/i386/armhf/arm64/armel), for example: ["amd64", "i386"]') + integration["ldap"].comment('FIXME: replace with true, false, or "not_relevant"') + integration["sso"].comment('FIXME: replace with true, false, or "not_relevant"') + integration["disk"].comment('FIXME: replace with an **estimate** minimum disk requirement. e.g. 20M, 400M, 1G, ...') + integration["ram.build"].comment('FIXME: replace with an **estimate** minimum ram requirement. e.g. 50M, 400M, 1G, ...') + integration["ram.runtime"].comment('FIXME: replace with an **estimate** minimum ram requirement. e.g. 50M, 400M, 1G, ...') + toml_manifest["integration"] = integration + + install = table() + for key, value in manifest["install"].items(): + install[key] = table() + install[key].indent(4) + + if key in ["domain", "path", "admin", "is_public", "password"]: + install[key].add(comment("this is a generic question - ask strings are automatically handled by Yunohost's core")) + + for lang, value2 in value.get("ask", {}).items(): + install[key].add(f"ask.{lang}", value2) + + for lang, value2 in value.get("help", {}).items(): + install[key].add(f"help.{lang}", value2) + + for key2, value2 in value.items(): + if key2 in ["ask", "help"]: + continue + install[key].add(key2, value2) + + toml_manifest["install"] = install + + resources = table() + for key, value in manifest["resources"].items(): + resources[key] = table() + resources[key].indent(4) + for key2, value2 in value.items(): + resources[key].add(key2, value2) + if key == "apt" and key2 == "extras": + for extra in resources[key][key2]: + extra.indent(8) + + toml_manifest["resources"] = resources + + toml_manifest_dump = dumps(toml_manifest) + + regex = re.compile(r'\"((description|ask|help)\.[a-z]{2})\"') + toml_manifest_dump = regex.sub(r'\1', toml_manifest_dump) + toml_manifest_dump = toml_manifest_dump.replace('"ram.build"', "ram.build") + toml_manifest_dump = toml_manifest_dump.replace('"ram.runtime"', "ram.runtime") + toml_manifest_dump = toml_manifest_dump.replace('"main.url"', "main.url") + toml_manifest_dump = toml_manifest_dump.replace('"main.default"', "main.default") + return toml_manifest_dump + + +def cleanup_scripts_and_conf(folder): + + patterns_to_remove_in_scripts = [ + "^.*ynh_abort_if_errors.*$", + "^.*YNH_APP_ARG.*$", + "^.*YNH_APP_INSTANCE_NAME.*$", + r"^ *final_path=", + r"^\s*final_path=", + "^.*test .*-(e|d) .*final_path.*$", + "^.*ynh_webpath_register.*$", + "^.*ynh_webpath_available.*$", + "^.*ynh_system_user_create.*$", + "^.*ynh_system_user_delete.*$", + "^.*ynh_permission_update.*$", + "^.*ynh_permission_create.*$", + "^.*if .*ynh_permission_exists.*$", + "^.*if .*ynh_legacy_permissions_exists.*$", + "^.*ynh_legacy_permissions_delete_all.*$", + "^.*ynh_app_setting_set .*(domain|path|final_path|admin|password|port|datadir|db_name|db_user|db_pwd).*$", + "^.*ynh_app_setting_.* is_public.*$", + r"^.*if.*\$is_public.*$", + "^.*_dependencies=.*$", + "^.*ynh_install_app_dependencies.*$", + "^.*ynh_install_extra_app_dependencies.*$", + "^.*ynh_remove_app_dependencies.*$", + r"^.*\$\(ynh_app_setting_get.*$", + r"^.*ynh_secure_remove .*\$final_path.*$", + r"^.*ynh_secure_remove .*\$datadir.*$", + "^.*ynh_backup_before_upgrade.*$", + "^.*ynh_clean_setup.*$", + "^.*ynh_restore_upgradebackup.*$", + "^db_name=.*$", + "^db_user=.*$", + "^db_pwd=.*$", + "^datadir=.*$", + "^.*ynh_psql_test_if_first_run.*$", + "^.*ynh_mysql_setup_db.*$", + "^.*ynh_psql_setup_db.*$", + "^.*ynh_mysql_remove_db.*$", + "^.*ynh_psql_remove_db.*$", + "^.*ynh_find_port.*$", + "^.*ynh_script_progression.*Finding an available port", + "^.*ynh_script_progression.*Backing up the app before upgrading", + "^.*ynh_script_progression.*Creating data directory", + "^.*ynh_script_progression.*system user", + "^.*ynh_script_progression.*installation settings", + "^.*ynh_print_info.*installation settings", + r"^.*ynh_script_progression.*\w+ dependencies", + "^.*ynh_script_progression.*Removing app main dir", + "^.*ynh_script_progression.*Validating.*parameters", + "^.*ynh_script_progression.*SQL database", + "^.*ynh_script_progression.*Configuring permissions", + ] + patterns_to_remove_in_scripts = [re.compile(f"({p})", re.MULTILINE) for p in patterns_to_remove_in_scripts] + + replaces = [ + ("path_url", "path"), + ("PATH_URL", "PATH"), + ("final_path", "install_dir"), + ("FINALPATH", "INSTALL_DIR"), + ("datadir", "data_dir"), + ("DATADIR", "DATA_DIR"), + ] + + for s in ["_common.sh", "install", "remove", "upgrade", "backup", "restore"]: + + script = f"{folder}/scripts/{s}" + + content = open(script).read() + + for pattern in patterns_to_remove_in_scripts: + content = pattern.sub(r"#REMOVEME? \1", content) + + for pattern, replace in replaces: + content = content.replace(pattern, replace) + + open(script, "w").write(content) + + for conf in os.listdir(f"{folder}/conf"): + + conf = f"{folder}/conf/{conf}" + + if not os.path.isfile(conf): + continue + + content = open(conf).read() + content_init = content + + for pattern, replace in replaces: + content = content.replace(pattern, replace) + + if content_init != content: + open(conf, "w").write(content) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Attempt to automatically convert a v1 YunoHost app to v2 (at least as much as possible) : parse the app scripts to auto-generate the manifest.toml, and remove now-useless lines from the app scripts" + ) + parser.add_argument( + "app_path", help="Path to the app to convert" + ) + + args = parser.parse_args() + + manifest = _convert_v1_manifest_to_v2(args.app_path) + open(args.app_path + "/manifest.toml", "w").write(_dump_v2_manifest_as_toml(manifest)) + + cleanup_scripts_and_conf(args.app_path) diff --git a/tools/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py b/tools/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py new file mode 100644 index 0000000..29f24ee --- /dev/null +++ b/tools/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py @@ -0,0 +1,55 @@ +import copy + + +def convert_v1_manifest_to_v2_for_catalog(manifest): + + manifest = copy.deepcopy(manifest) + + if "upstream" not in manifest: + manifest["upstream"] = {} + + if "license" in manifest and "license" not in manifest["upstream"]: + manifest["upstream"]["license"] = manifest["license"] + + if "url" in manifest and "website" not in manifest["upstream"]: + manifest["upstream"]["website"] = manifest["url"] + + manifest["integration"] = { + "yunohost": manifest.get("requirements", {}).get("yunohost", "").replace(">", "").replace("=", "").replace(" ", ""), + "architectures": "all", + "multi_instance": manifest.get("multi_instance", False), + "ldap": "?", + "sso": "?", + "disk": "50M", + "ram": {"build": "50M", "runtime": "10M"} + } + + maintainer = manifest.get("maintainer", {}).get("name") + manifest["maintainers"] = [maintainer] if maintainer else [] + + install_questions = manifest["arguments"]["install"] + + manifest["install"] = {} + for question in install_questions: + name = question.pop("name") + if "ask" in question and name in ["domain", "path", "admin", "is_public", "password"]: + question.pop("ask") + if question.get("example") and question.get("type") in ["domain", "path", "user", "boolean", "password"]: + question.pop("example") + + manifest["install"][name] = question + + manifest["resources"] = { + "system_user": {}, + "install_dir": { + "alias": "final_path" + } + } + + keys_to_keep = ["packaging_format", "id", "name", "description", "version", "maintainers", "upstream", "integration", "install", "resources"] + + keys_to_del = [key for key in manifest.keys() if key not in keys_to_keep] + for key in keys_to_del: + del manifest[key] + + return manifest From 8dabe03c00cfc6b33475b7fba3f9dc4772247e8b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 3 Aug 2022 21:31:17 +0200 Subject: [PATCH 32/51] Fix my previous commit /o\ --- list_builder.py | 2 +- tools/packaging_v2/convert_app_to_packaging_v2.py | 9 +++++++-- .../convert_v1_manifest_to_v2_for_catalog.py | 9 +++++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/list_builder.py b/list_builder.py index 1242244..2ebd800 100755 --- a/list_builder.py +++ b/list_builder.py @@ -195,7 +195,7 @@ def build_catalog(): ################################### result_dict_with_manifest_v2 = copy.deepcopy(result_dict) - for app in result_dict_with_manifest_v2: + for app in result_dict_with_manifest_v2.values(): app["manifest"] = convert_v1_manifest_to_v2_for_catalog(app["manifest"]) os.system("mkdir -p ./builds/default/v3/") diff --git a/tools/packaging_v2/convert_app_to_packaging_v2.py b/tools/packaging_v2/convert_app_to_packaging_v2.py index fd66325..21fe774 100644 --- a/tools/packaging_v2/convert_app_to_packaging_v2.py +++ b/tools/packaging_v2/convert_app_to_packaging_v2.py @@ -37,8 +37,13 @@ def _convert_v1_manifest_to_v2(app_path): "ram.runtime": "50M" } - maintainer = manifest.get("maintainer", {}).get("name") - manifest["maintainers"] = [maintainer] if maintainer else [] + maintainers = manifest.get("maintainer", {}) + if isinstance(maintainers, list): + maintainers = [m['name'] for m in maintainers] + else: + maintainers = [maintainers["name"]] if maintainers.get("name") else [] + + manifest["maintainers"] = maintainers install_questions = manifest["arguments"]["install"] manifest["install"] = {} diff --git a/tools/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py b/tools/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py index 29f24ee..0130c29 100644 --- a/tools/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py +++ b/tools/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py @@ -24,8 +24,13 @@ def convert_v1_manifest_to_v2_for_catalog(manifest): "ram": {"build": "50M", "runtime": "10M"} } - maintainer = manifest.get("maintainer", {}).get("name") - manifest["maintainers"] = [maintainer] if maintainer else [] + maintainers = manifest.get("maintainer", {}) + if isinstance(maintainers, list): + maintainers = [m['name'] for m in maintainers] + else: + maintainers = [maintainers["name"]] if maintainers.get("name") else [] + + manifest["maintainers"] = maintainers install_questions = manifest["arguments"]["install"] From 22dff664703f00c685a207945d377b6d0f1f8b6f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 Aug 2022 11:16:26 +0200 Subject: [PATCH 33/51] list_builder: drop the old i18n stuff ... this didnt get any update since 2.5 years and doesn't contain that many relevant translation anymore ... and it's gonna be a mess to adapt the code to cover both v1 and v2 manifest format ... --- list_builder.py | 54 +------------------------------------------------ 1 file changed, 1 insertion(+), 53 deletions(-) diff --git a/list_builder.py b/list_builder.py index 2ebd800..6c6b1f6 100755 --- a/list_builder.py +++ b/list_builder.py @@ -299,7 +299,7 @@ def build_app_dict(app, infos): "url": infos["url"], }, "lastUpdate": timestamp, - "manifest": include_translations_in_manifest(manifest), + "manifest": manifest, "state": infos["state"], "level": infos.get("level", "?"), "maintained": infos.get("maintained", True), @@ -313,58 +313,6 @@ def build_app_dict(app, infos): } -def include_translations_in_manifest(manifest): - - app_name = manifest["id"] - - for locale in os.listdir("locales"): - if not locale.endswith("json"): - continue - - if locale == "en.json": - continue - - current_lang = locale.split(".")[0] - translations = json.load(open(os.path.join("locales", locale), "r")) - - # don't overwrite already existing translation in manifests for now - key = "%s_manifest_description" % app_name - if current_lang not in manifest["description"] and translations.get(key): - manifest["description"][current_lang] = translations[key] - - for category, questions in manifest["arguments"].items(): - for question in questions: - key = "%s_manifest_arguments_%s_%s" % ( - app_name, - category, - question["name"], - ) - # don't overwrite already existing translation in manifests for now - if ( - translations.get(key) - and "ask" in question - and current_lang not in question["ask"] - ): - question["ask"][current_lang] = translations[key] - - key = "%s_manifest_arguments_%s_help_%s" % ( - app_name, - category, - question["name"], - ) - # don't overwrite already existing translation in manifests for now - if ( - translations.get(key) - and "help" in question - and current_lang not in question.get("help", []) - ): - question["help"][current_lang] = translations[key] - - return manifest - - -###################### - if __name__ == "__main__": refresh_all_caches() build_catalog() From 56046586071cab928e1104d96c1199bdd35f8daf Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 Aug 2022 11:17:46 +0200 Subject: [PATCH 34/51] move autopatches/ inside tools/ --- tools/autopatches/autopatch.py | 188 ++++++++++++++++++ .../patches/issue-and-pr-template/patch.sh | 14 ++ .../patches/issue-and-pr-template/pr_body.md | 2 + .../patches/issue-and-pr-template/pr_title.md | 1 + .../missing-seteu-in-change_url/patch.sh | 10 + .../missing-seteu-in-change_url/pr_body.md | 2 + .../missing-seteu-in-change_url/pr_title.md | 1 + .../patches/new-permission-system/patch.sh | 66 ++++++ .../patches/new-permission-system/pr_body.md | 11 + .../patches/new-permission-system/pr_title.md | 1 + 10 files changed, 296 insertions(+) create mode 100755 tools/autopatches/autopatch.py create mode 100644 tools/autopatches/patches/issue-and-pr-template/patch.sh create mode 100644 tools/autopatches/patches/issue-and-pr-template/pr_body.md create mode 100644 tools/autopatches/patches/issue-and-pr-template/pr_title.md create mode 100644 tools/autopatches/patches/missing-seteu-in-change_url/patch.sh create mode 100644 tools/autopatches/patches/missing-seteu-in-change_url/pr_body.md create mode 100644 tools/autopatches/patches/missing-seteu-in-change_url/pr_title.md create mode 100644 tools/autopatches/patches/new-permission-system/patch.sh create mode 100644 tools/autopatches/patches/new-permission-system/pr_body.md create mode 100644 tools/autopatches/patches/new-permission-system/pr_title.md diff --git a/tools/autopatches/autopatch.py b/tools/autopatches/autopatch.py new file mode 100755 index 0000000..746ee51 --- /dev/null +++ b/tools/autopatches/autopatch.py @@ -0,0 +1,188 @@ +#!/usr/bin/python3 +import json +import sys +import requests +import os +import subprocess + +catalog = requests.get("https://raw.githubusercontent.com/YunoHost/apps/master/apps.json").json() + +my_env = os.environ.copy() +my_env["GIT_TERMINAL_PROMPT"] = "0" +os.makedirs(".apps_cache", exist_ok=True) + +login = open("login").read().strip() +token = open("token").read().strip() +github_api = "https://api.github.com" + + +def apps(min_level=4): + + for app, infos in catalog.items(): + if infos.get("state") == "working" and infos.get("level", -1) > min_level: + infos["id"] = app + yield infos + + +def app_cache_folder(app): + return os.path.join(".apps_cache", app) + + +def git(cmd, in_folder=None): + + if not isinstance(cmd, list): + cmd = cmd.split() + if in_folder: + cmd = ["-C", in_folder] + cmd + cmd = ["git"] + cmd + return subprocess.check_output(cmd, env=my_env).strip().decode("utf-8") + + +# Progress bar helper, stolen from https://stackoverflow.com/a/34482761 +def progressbar(it, prefix="", size=60, file=sys.stdout): + it = list(it) + count = len(it) + def show(j, name=""): + name += " " + x = int(size*j/count) + file.write("%s[%s%s] %i/%i %s\r" % (prefix, "#"*x, "."*(size-x), j, count, name)) + file.flush() + show(0) + for i, item in enumerate(it): + yield item + show(i+1, item["id"]) + file.write("\n") + file.flush() + + +def build_cache(): + + for app in progressbar(apps(), "Git cloning: ", 40): + folder = os.path.join(".apps_cache", app["id"]) + reponame = app["url"].rsplit("/", 1)[-1] + git(f"clone --quiet --depth 1 --single-branch {app['url']} {folder}") + git(f"remote add fork https://{login}:{token}@github.com/{login}/{reponame}", in_folder=folder) + + +def apply(patch): + + patch_path = os.path.abspath(os.path.join("patches", patch, "patch.sh")) + + for app in progressbar(apps(), "Apply to: ", 40): + folder = os.path.join(".apps_cache", app["id"]) + current_branch = git(f"symbolic-ref --short HEAD", in_folder=folder) + git(f"reset --hard origin/{current_branch}", in_folder=folder) + os.system(f"cd {folder} && bash {patch_path}") + + +def diff(): + + for app in apps(): + folder = os.path.join(".apps_cache", app["id"]) + if bool(subprocess.check_output(f"cd {folder} && git diff", shell=True).strip().decode("utf-8")): + print("\n\n\n") + print("=================================") + print("Changes in : " + app["id"]) + print("=================================") + print("\n") + os.system(f"cd {folder} && git --no-pager diff") + + +def push(patch): + + title = "[autopatch] " + open(os.path.join("patches", patch, "pr_title.md")).read().strip() + + def diff_not_empty(app): + folder = os.path.join(".apps_cache", app["id"]) + return bool(subprocess.check_output(f"cd {folder} && git diff", shell=True).strip().decode("utf-8")) + + def app_is_on_github(app): + return "github.com" in app["url"] + + apps_to_push = [app for app in apps() if diff_not_empty(app) and app_is_on_github(app)] + + with requests.Session() as s: + s.headers.update({"Authorization": f"token {token}"}) + for app in progressbar(apps_to_push, "Forking: ", 40): + app["repo"] = app["url"][len("https://github.com/"):].strip("/") + fork_if_needed(app["repo"], s) + + for app in progressbar(apps_to_push, "Pushing: ", 40): + app["repo"] = app["url"][len("https://github.com/"):].strip("/") + app_repo_name = app["url"].rsplit("/", 1)[-1] + folder = os.path.join(".apps_cache", app["id"]) + current_branch = git(f"symbolic-ref --short HEAD", in_folder=folder) + git(f"reset origin/{current_branch}", in_folder=folder) + git(["commit", "-a", "-m", title, "--author='Yunohost-Bot <>'"], in_folder=folder) + try: + git(f"remote remove fork", in_folder=folder) + except Exception: + pass + git(f"remote add fork https://{login}:{token}@github.com/{login}/{app_repo_name}", in_folder=folder) + git(f"push fork {current_branch}:{patch} --quiet --force", in_folder=folder) + create_pull_request(app["repo"], patch, current_branch, s) + + +def fork_if_needed(repo, s): + + repo_name = repo.split("/")[-1] + r = s.get(github_api + f"/repos/{login}/{repo_name}") + + if r.status_code == 200: + return + + r = s.post(github_api + f"/repos/{repo}/forks") + + if r.status_code != 200: + print(r.text) + + +def create_pull_request(repo, patch, base_branch, s): + + PR = {"title": "[autopatch] " + open(os.path.join("patches", patch, "pr_title.md")).read().strip(), + "body": "This is an automatic PR\n\n" + open(os.path.join("patches", patch, "pr_body.md")).read().strip(), + "head": login + ":" + patch, + "base": base_branch, + "maintainer_can_modify": True} + + r = s.post(github_api + f"/repos/{repo}/pulls", json.dumps(PR)) + + if r.status_code != 200: + print(r.text) + else: + json.loads(r.text)["html_url"] + + +def main(): + + action = sys.argv[1] + if action == "--help": + print(""" + Example usage: + +# Init local git clone for all apps +./autopatch.py --build-cache + +# Apply patch in all local clones +./autopatch.py --apply explicit-php-version-in-deps + +# Inspect diff for all apps +./autopatch.py --diff + +# Push and create pull requests on all apps with non-empty diff +./autopatch.py --push explicit-php-version-in-deps +""") + + elif action == "--build-cache": + build_cache() + elif action == "--apply": + apply(sys.argv[2]) + elif action == "--diff": + diff() + elif action == "--push": + push(sys.argv[2]) + else: + print("Unknown action %s" % action) + + +main() diff --git a/tools/autopatches/patches/issue-and-pr-template/patch.sh b/tools/autopatches/patches/issue-and-pr-template/patch.sh new file mode 100644 index 0000000..d52d3b0 --- /dev/null +++ b/tools/autopatches/patches/issue-and-pr-template/patch.sh @@ -0,0 +1,14 @@ + +[ ! -e issue_template.md ] || git rm issue_template.md +[ ! -e pull_request_template.md ] || git rm pull_request_template.md + +[ ! -e .github ] || git rm -rf .github +mkdir -p .github + +# Sleep 1 to avoid too many requests on github (there's a rate limit anyway) +sleep 1 + +wget -O .github/ISSUE_TEMPLATE.md https://raw.githubusercontent.com/YunoHost/example_ynh/master/.github/ISSUE_TEMPLATE.md +wget -O .github/PULL_REQUEST_TEMPLATE.md https://raw.githubusercontent.com/YunoHost/example_ynh/master/.github/PULL_REQUEST_TEMPLATE.md + +git add .github diff --git a/tools/autopatches/patches/issue-and-pr-template/pr_body.md b/tools/autopatches/patches/issue-and-pr-template/pr_body.md new file mode 100644 index 0000000..471273e --- /dev/null +++ b/tools/autopatches/patches/issue-and-pr-template/pr_body.md @@ -0,0 +1,2 @@ + +This is an ***automated*** patch to sync the issue and PR template with the official templates in https://github.com/YunoHost/example_ynh/tree/master/.github diff --git a/tools/autopatches/patches/issue-and-pr-template/pr_title.md b/tools/autopatches/patches/issue-and-pr-template/pr_title.md new file mode 100644 index 0000000..f0bed1c --- /dev/null +++ b/tools/autopatches/patches/issue-and-pr-template/pr_title.md @@ -0,0 +1 @@ +Update issue and PR templates diff --git a/tools/autopatches/patches/missing-seteu-in-change_url/patch.sh b/tools/autopatches/patches/missing-seteu-in-change_url/patch.sh new file mode 100644 index 0000000..f930134 --- /dev/null +++ b/tools/autopatches/patches/missing-seteu-in-change_url/patch.sh @@ -0,0 +1,10 @@ + +cd scripts/ + +if [ ! -e change_url ] || grep -q 'ynh_abort_if_errors' change_url +then + # The app doesn't has any change url script or already has ynh_abort_if_error + exit 0 +fi + +sed 's@\(source /usr/share/yunohost/helpers\)@\1\nynh_abort_if_errors@g' -i change_url diff --git a/tools/autopatches/patches/missing-seteu-in-change_url/pr_body.md b/tools/autopatches/patches/missing-seteu-in-change_url/pr_body.md new file mode 100644 index 0000000..bb67bd0 --- /dev/null +++ b/tools/autopatches/patches/missing-seteu-in-change_url/pr_body.md @@ -0,0 +1,2 @@ + +This is an ***automated*** patch to fix the lack of `ynh_abort_if_errors` in change_url script diff --git a/tools/autopatches/patches/missing-seteu-in-change_url/pr_title.md b/tools/autopatches/patches/missing-seteu-in-change_url/pr_title.md new file mode 100644 index 0000000..efd0e73 --- /dev/null +++ b/tools/autopatches/patches/missing-seteu-in-change_url/pr_title.md @@ -0,0 +1 @@ +Missing ynh_abort_if_errors in change_url scripts diff --git a/tools/autopatches/patches/new-permission-system/patch.sh b/tools/autopatches/patches/new-permission-system/patch.sh new file mode 100644 index 0000000..f372a44 --- /dev/null +++ b/tools/autopatches/patches/new-permission-system/patch.sh @@ -0,0 +1,66 @@ + +cd scripts/ + +if grep -q 'ynh_legacy_permissions' upgrade || grep -q 'ynh_permission_' install +then + # App already using the new permission system - not patching anything + exit 0 +fi + +if ! grep -q "protected_\|skipped_" install +then + # App doesn't has any (un)protected / skipped setting ? + # Probably not a webapp or permission ain't relevant for it ? + exit 0 +fi + +CONFIGURE_PERMISSION_DURING_INSTALL=' +# Make app public if necessary +if [ \"\$is_public\" -eq 1 ] +then + ynh_permission_update --permission=\"main\" --add=\"visitors\" +fi +' + +MIGRATE_LEGACY_PERMISSIONS=' +#================================================= +# Migrate legacy permissions to new system +#================================================= +if ynh_legacy_permissions_exists +then + ynh_legacy_permissions_delete_all + + ynh_app_setting_delete --app=\$app --key=is_public +fi' + +for SCRIPT in "remove upgrade backup restore change_url" +do + [[ -e $SCRIPT ]] || continue + + perl -p0e 's@.*ynh_app_setting_.*protected_.*@@g' -i $SCRIPT + perl -p0e 's@.*ynh_app_setting_.*skipped_.*@@g' -i $SCRIPT + perl -p0e 's@\s*if.*-z.*is_public.*(.|\n)*?fi\s@\n@g' -i $SCRIPT + perl -p0e 's@\s*if.*is_public.*(-eq|=).*(.|\n)*?fi\s@\n@g' -i $SCRIPT + perl -p0e 's@is_public=.*\n@@g' -i $SCRIPT + perl -p0e 's@ynh_app_setting_.*is_public.*@@g' -i $SCRIPT + perl -p0e 's@.*# Make app .*@@g' -i $SCRIPT + perl -p0e 's@.*# Fix is_public as a boolean.*@@g' -i $SCRIPT + perl -p0e 's@.*# If app is public.*@@g' -i $SCRIPT + perl -p0e 's@.*# .*allow.*credentials.*anyway.*@@g' -i $SCRIPT + perl -p0e 's@.*ynh_script_progression.*SSOwat.*@@g' -i $SCRIPT + perl -p0e 's@#=*\s#.*SETUP SSOWAT.*\s#=*\s@@g' -i $SCRIPT +done + + +perl -p0e 's@.*domain_regex.*@@g' -i install +perl -p0e 's@.*# If app is public.*@@g' -i install +perl -p0e 's@.*# Make app .*@@g' -i install +perl -p0e 's@.*# .*allow.*credentials.*anyway.*@@g' -i install +perl -p0e "s@if.*is_public.*(-eq|=)(.|\n){0,100}setting(.|\n)*?fi\n@$CONFIGURE_PERMISSION_DURING_INSTALL@g" -i install +perl -p0e 's@.*ynh_app_setting_.*is_public.*\s@@g' -i install +perl -p0e 's@.*ynh_app_setting_.*protected_.*@@g' -i install +perl -p0e 's@.*ynh_app_setting_.*skipped_.*@@g' -i install + +grep -q 'is_public=' install || perl -p0e 's@(.*Configuring SSOwat.*)@\1\nynh_permission_update --permission=\"main\" --add=\"visitors\"@g' -i install + +perl -p0e "s@ynh_abort_if_errors@ynh_abort_if_errors\n$MIGRATE_LEGACY_PERMISSIONS@g" -i upgrade diff --git a/tools/autopatches/patches/new-permission-system/pr_body.md b/tools/autopatches/patches/new-permission-system/pr_body.md new file mode 100644 index 0000000..07f4e85 --- /dev/null +++ b/tools/autopatches/patches/new-permission-system/pr_body.md @@ -0,0 +1,11 @@ + +NB. : this is an ***automated*** attempt to migrate the app to the new permission system + +You should ***not*** blindly trust the proposed changes. In particular, the auto-patch will not handle: +- situations which are more complex than "if is_public is true, allow visitors" +- situations where the app needs to be temporarily public (then possible private) during initial configuration +- apps that need to define extra permission for specific section of the app (such as admin interface) +- apps using non-standard syntax +- other specific use cases + +***PLEASE*** carefully review, test and amend the proposed changes if you find that the autopatch did not do a proper job. diff --git a/tools/autopatches/patches/new-permission-system/pr_title.md b/tools/autopatches/patches/new-permission-system/pr_title.md new file mode 100644 index 0000000..0f9554c --- /dev/null +++ b/tools/autopatches/patches/new-permission-system/pr_title.md @@ -0,0 +1 @@ +Autopatch to migrate to new permission system From 81408ec1cad12078728869be3a7f139a937405cb Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 Aug 2022 16:29:26 +0200 Subject: [PATCH 35/51] Moar preparations for packaging v2 apps in catalog --- list_builder.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/list_builder.py b/list_builder.py index 6c6b1f6..6707370 100755 --- a/list_builder.py +++ b/list_builder.py @@ -172,9 +172,13 @@ def build_catalog(): result_dict[app_dict["id"]] = app_dict - ##################### - # Current version 2 # - ##################### + ############################# + # Current catalog API v2 # + ############################# + + result_dict_with_manifest_v1 = copy.deepcopy(result_dict) + result_dict_with_manifest_v1 = {name: infos for name, infos in result_dict_with_manifest_v1.items() if float(str(infos["manifest"].get("packaging_format", "")).strip() or "0") < 2} + categories = yaml.load(open("categories.yml").read()) antifeatures = yaml.load(open("antifeatures.yml").read()) os.system("mkdir -p ./builds/default/v2/") @@ -182,7 +186,7 @@ def build_catalog(): f.write( json.dumps( { - "apps": result_dict, + "apps": result_dict_with_manifest_v1, "categories": categories, "antifeatures": antifeatures, }, @@ -190,13 +194,15 @@ def build_catalog(): ) ) - ################################### - # Catalog API v3 with manifest v2 # - ################################### + ############################################# + # Catalog catalog API v3 (with manifest v2) # + ############################################# result_dict_with_manifest_v2 = copy.deepcopy(result_dict) for app in result_dict_with_manifest_v2.values(): - app["manifest"] = convert_v1_manifest_to_v2_for_catalog(app["manifest"]) + packaging_format = float(str(app["manifest"].get("packaging_format", "")).strip() or "0") + if packaging_format < 2: + app["manifest"] = convert_v1_manifest_to_v2_for_catalog(app["manifest"]) os.system("mkdir -p ./builds/default/v3/") with open("builds/default/v3/apps.json", "w") as f: @@ -238,7 +244,7 @@ def build_catalog(): result_dict_doc = { k: infos_for_doc_catalog(v) for k, v in result_dict.items() - if v["state"] in ["working", "validated"] + if v["state"] == "working" } with open("builds/default/doc_catalog/apps.json", "w") as f: f.write( @@ -261,6 +267,7 @@ def build_app_dict(app, infos): if infos["revision"] == "HEAD": relevant_files = [ "manifest.json", + "manifest.toml", "config_panel.toml", "hooks/", "scripts/", From 582e33e9d9da43ad52e629f6d4156fe6fe5108d9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 Aug 2022 17:05:46 +0200 Subject: [PATCH 36/51] Fix v2 packaging convert script (_common.sh may not exist) --- tools/packaging_v2/convert_app_to_packaging_v2.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/packaging_v2/convert_app_to_packaging_v2.py b/tools/packaging_v2/convert_app_to_packaging_v2.py index 21fe774..b2a11c3 100644 --- a/tools/packaging_v2/convert_app_to_packaging_v2.py +++ b/tools/packaging_v2/convert_app_to_packaging_v2.py @@ -101,8 +101,8 @@ def _convert_v1_manifest_to_v2(app_path): manifest["resources"]["ports"][f"{name}.exposed"] = exposed maybequote = "[\"'\"'\"']?" - apt_dependencies = check_output(f"sed -nr 's/.*_dependencies={maybequote}(.*){maybequote}? *$/\\1/p' '{app_path}/scripts/_common.sh' | tr -d '\"' | sed 's@ @\\n@g'") - php_version = check_output(f"sed -nr 's/^ *YNH_PHP_VERSION={maybequote}(.*){maybequote}?$/\\1/p' '{app_path}/scripts/_common.sh' | tr -d \"\\\"'\"") + apt_dependencies = check_output(f"sed -nr 's/.*_dependencies={maybequote}(.*){maybequote}? *$/\\1/p' '{app_path}/scripts/_common.sh' 2>/dev/null | tr -d '\"' | sed 's@ @\\n@g'") + php_version = check_output(f"sed -nr 's/^ *YNH_PHP_VERSION={maybequote}(.*){maybequote}?$/\\1/p' '{app_path}/scripts/_common.sh' 2>/dev/null | tr -d \"\\\"'\"") if apt_dependencies.strip(): if php_version: apt_dependencies = apt_dependencies.replace("${YNH_PHP_VERSION}", php_version) @@ -283,6 +283,9 @@ def cleanup_scripts_and_conf(folder): script = f"{folder}/scripts/{s}" + if not os.path.exists(script): + continue + content = open(script).read() for pattern in patterns_to_remove_in_scripts: From 08c3f20b5f0e912846cc6379aecf875c3f7f24a1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 Aug 2022 17:14:42 +0200 Subject: [PATCH 37/51] list_builder: gotta load manifest.toml, not json, for app v2 --- list_builder.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/list_builder.py b/list_builder.py index 6707370..2a4288b 100755 --- a/list_builder.py +++ b/list_builder.py @@ -5,10 +5,12 @@ import sys import os import re import json +import toml import subprocess import yaml import time +from collections import OrderedDict from tools.packaging_v2.convert_v1_manifest_to_v2_for_catalog import convert_v1_manifest_to_v2_for_catalog now = time.time() @@ -297,7 +299,11 @@ def build_app_dict(app, infos): timestamp = int(timestamp) # Build the dict with all the infos - manifest = json.load(open(this_app_cache + "/manifest.json")) + if os.path.exists(open(this_app_cache + "/manifest.toml")): + manifest = toml.load(open(this_app_cache + "/manifest.toml", _dict=OrderedDict)) + else: + manifest = json.load(open(this_app_cache + "/manifest.json")) + return { "id": manifest["id"], "git": { From 4f08ac96135a169d1a85aa16e1b785bb7abe9536 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 Aug 2022 17:19:23 +0200 Subject: [PATCH 38/51] Oopsies I commited nonsense --- list_builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/list_builder.py b/list_builder.py index 2a4288b..e41cdc1 100755 --- a/list_builder.py +++ b/list_builder.py @@ -299,8 +299,8 @@ def build_app_dict(app, infos): timestamp = int(timestamp) # Build the dict with all the infos - if os.path.exists(open(this_app_cache + "/manifest.toml")): - manifest = toml.load(open(this_app_cache + "/manifest.toml", _dict=OrderedDict)) + if os.path.exists(this_app_cache + "/manifest.toml"): + manifest = toml.load(open(this_app_cache + "/manifest.toml"), _dict=OrderedDict) else: manifest = json.load(open(this_app_cache + "/manifest.json")) From b1b61acffe958fcdd8a390036213029aecfab652 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 5 Aug 2022 14:56:52 +0200 Subject: [PATCH 39/51] packaging v2: add a 'CPE' field in upstream section --- tools/packaging_v2/convert_app_to_packaging_v2.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/packaging_v2/convert_app_to_packaging_v2.py b/tools/packaging_v2/convert_app_to_packaging_v2.py index b2a11c3..ba00153 100644 --- a/tools/packaging_v2/convert_app_to_packaging_v2.py +++ b/tools/packaging_v2/convert_app_to_packaging_v2.py @@ -26,6 +26,8 @@ def _convert_v1_manifest_to_v2(app_path): if "url" in manifest and "website" not in manifest["upstream"]: manifest["upstream"]["website"] = manifest["url"] + manifest["upstream"]["cpe"] = "???" + manifest["integration"] = { "yunohost": manifest.get("requirements", {}).get("yunohost"), "architectures": "all", @@ -158,6 +160,7 @@ def _dump_v2_manifest_as_toml(manifest): upstream = table() for key, value in manifest["upstream"].items(): upstream[key] = value + upstream["cpe"].comment("FIXME: optional but recommended if relevant, this is meant to contain the Common Platform Enumeration, which is sort of a standard id for applications defined by the NIST. In particular, Yunohost may use this is in the future to easily track CVE (=security reports) related to apps. The CPE may be obtained by searching here: https://nvd.nist.gov/products/cpe/search. For example, for Nextcloud, the CPE is 'cpe:2.3:a:nextcloud:nextcloud' (no need to include the version number)") toml_manifest["upstream"] = upstream integration = table() From 69ac8da1506afe9cec263f51df7fab27f40e331f Mon Sep 17 00:00:00 2001 From: tituspijean Date: Fri, 5 Aug 2022 16:39:38 +0200 Subject: [PATCH 40/51] Fix antifeatures list generation --- tools/README-generator/make_readme.py | 22 ++++++++++++++----- tools/README-generator/templates/README.md.j2 | 3 ++- .../templates/README_fr.md.j2 | 4 +++- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/tools/README-generator/make_readme.py b/tools/README-generator/make_readme.py index 788b8a7..d9c03db 100755 --- a/tools/README-generator/make_readme.py +++ b/tools/README-generator/make_readme.py @@ -8,6 +8,15 @@ from pathlib import Path from jinja2 import Environment, FileSystemLoader +def value_for_lang(values, lang): + if not isinstance(values, dict): + return values + if lang in values: + return values[lang] + elif "en" in values: + return values["en"] + else: + return list(values.values())[0] def generate_READMEs(app_path: str): @@ -22,7 +31,7 @@ def generate_READMEs(app_path: str): catalog = json.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "apps.json")) from_catalog = catalog.get(manifest['id'], {}) - antifeatures_list = yaml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "antifeatures.yml")) + antifeatures_list = yaml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "antifeatures.yml"), Loader=yaml.SafeLoader) antifeatures_list = { e['id']: e for e in antifeatures_list } if not upstream and not (app_path / "doc" / "DISCLAIMER.md").exists(): @@ -62,12 +71,12 @@ def generate_READMEs(app_path: str): # TODO: Add url to the documentation... and actually create that documentation :D antifeatures = { a: antifeatures_list[a] for a in from_catalog.get('antifeatures', [])} - for k, v in antifeatures: - v['title'] = v['title'].get(lang_suffix, 'en') - if manifest.get("antifeatures", {}).get(k, 'en'): - v['description'] = manifest.get("antifeatures", {}).get(k, 'en') + for k, v in antifeatures.items(): + antifeatures[k]['title'] = value_for_lang(v['title'], lang_suffix) + if manifest.get("antifeatures", {}).get(k, None): + antifeatures[k]['description'] = value_for_lang(manifest.get("antifeatures", {}).get(k, None), lang_suffix) else: - antifeature['description'] = antifeature['description'].get(lang_suffix, 'en') + antifeatures[k]['description'] = value_for_lang(antifeatures[k]['description'], lang_suffix) out = template.render( lang=lang, @@ -75,6 +84,7 @@ def generate_READMEs(app_path: str): description=description, screenshots=screenshots, disclaimer=disclaimer, + antifeatures=antifeatures, manifest=manifest, ) (app_path / f"README{lang_suffix}.md").write_text(out) diff --git a/tools/README-generator/templates/README.md.j2 b/tools/README-generator/templates/README.md.j2 index 8713a9a..990b5e0 100644 --- a/tools/README-generator/templates/README.md.j2 +++ b/tools/README-generator/templates/README.md.j2 @@ -52,8 +52,9 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in {% if antifeatures -%} ## Antifeatures -{% for antifeature in antifeatures -%} +{% for antifeature in antifeatures.values() -%} - **{{ antifeature.title }}**: {{ antifeature.description }} + {% endfor -%} {% endif -%} diff --git a/tools/README-generator/templates/README_fr.md.j2 b/tools/README-generator/templates/README_fr.md.j2 index 9e89c10..864dc60 100644 --- a/tools/README-generator/templates/README_fr.md.j2 +++ b/tools/README-generator/templates/README_fr.md.j2 @@ -34,11 +34,13 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour {% if antifeatures -%} ## Fonctions indésirables -{% for antifeature in antifeatures -%} +{% for antifeature in antifeatures.values() -%} - **{{ antifeature.title }}**: {{ antifeature.description }} + {% endfor -%} {% endif -%} + ## Documentations et ressources {% if upstream.website -%}* Site officiel de l'app : {{ upstream.website }} From 28a6a28f4df5035d2abd179196aff57917267768 Mon Sep 17 00:00:00 2001 From: tituspijean Date: Fri, 5 Aug 2022 16:41:27 +0200 Subject: [PATCH 41/51] Add red circle to antifeatures title Co-authored-by: Alexandre Aubin --- tools/README-generator/templates/README.md.j2 | 2 +- tools/README-generator/templates/README_fr.md.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/README-generator/templates/README.md.j2 b/tools/README-generator/templates/README.md.j2 index 990b5e0..91d6510 100644 --- a/tools/README-generator/templates/README.md.j2 +++ b/tools/README-generator/templates/README.md.j2 @@ -50,7 +50,7 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in {% endif -%} {% if antifeatures -%} -## Antifeatures +## :red_circle: Antifeatures {% for antifeature in antifeatures.values() -%} - **{{ antifeature.title }}**: {{ antifeature.description }} diff --git a/tools/README-generator/templates/README_fr.md.j2 b/tools/README-generator/templates/README_fr.md.j2 index 864dc60..22f4ebb 100644 --- a/tools/README-generator/templates/README_fr.md.j2 +++ b/tools/README-generator/templates/README_fr.md.j2 @@ -32,7 +32,7 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour {% endif -%} {% if antifeatures -%} -## Fonctions indésirables +## :red_circle: Fonctions indésirables {% for antifeature in antifeatures.values() -%} - **{{ antifeature.title }}**: {{ antifeature.description }} From c6805ade2625ffc5c64dffe3ed4929f0a79353a8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 5 Aug 2022 23:06:38 +0200 Subject: [PATCH 42/51] README-generator: improve mechanism to only add the 'specific branch' note when version is different from master + display master's version --- tools/README-generator/make_readme.py | 14 +++++++++++--- tools/README-generator/templates/README.md.j2 | 2 +- tools/README-generator/templates/README_fr.md.j2 | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/tools/README-generator/make_readme.py b/tools/README-generator/make_readme.py index ac47a04..049aa21 100755 --- a/tools/README-generator/make_readme.py +++ b/tools/README-generator/make_readme.py @@ -57,7 +57,14 @@ def generate_READMEs(app_path: str): disclaimer = None # Get the current branch using git inside the app path - branch = os.popen("git --git-dir=%s/.git --work-tree=%s rev-parse --abbrev-ref HEAD" % (app_path, app_path)).read().strip() + default_branch = from_catalog.get('branch', 'master') + current_branch = os.popen(f"git -C {app_path} rev-parse --abbrev-ref HEAD").read().strip() + + if default_branch != current_branch: + os.system(f"git -C {app_path} fetch origin {default_branch} 2>/dev/null") + default_branch_version = os.popen(f"git -C {app_path} show FETCH_HEAD:manifest.json | jq -r .version").read().strip() + else: + default_branch_version = None # we don't care in that case out = template.render( lang=lang, @@ -66,8 +73,9 @@ def generate_READMEs(app_path: str): screenshots=screenshots, disclaimer=disclaimer, manifest=manifest, - branch=branch, - default_branch=from_catalog.get('branch', 'master'), + current_branch=current_branch, + default_branch=default_branch, + default_branch_version=default_branch_version, ) (app_path / f"README{lang_suffix}.md").write_text(out) diff --git a/tools/README-generator/templates/README.md.j2 b/tools/README-generator/templates/README.md.j2 index 125833c..ec99391 100644 --- a/tools/README-generator/templates/README.md.j2 +++ b/tools/README-generator/templates/README.md.j2 @@ -31,7 +31,7 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if branch != default_branch %} *(This is the `{{ branch }}` branch. See the [`{{ default_branch }}` branch](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/{{ default_branch }}) for the version in the catalog.)*{% endif %} +**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if current_branch != default_branch and manifest.version != default_branch_version %} *(:warning: This is the `{{ current_branch }}` branch. The [`{{ default_branch }}` branch](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/{{ default_branch }}) used in the catalog is currently on version {{ default_branch_version.replace('~', '\~') }}.)*{% endif %} {% if upstream.demo %} **Demo:** {{upstream.demo}} diff --git a/tools/README-generator/templates/README_fr.md.j2 b/tools/README-generator/templates/README_fr.md.j2 index 7350947..d616280 100644 --- a/tools/README-generator/templates/README_fr.md.j2 +++ b/tools/README-generator/templates/README_fr.md.j2 @@ -17,7 +17,7 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if branch != default_branch %} *(Vous êtes sur la branche `{{ branch }}`. Voir la [branche `{{ default_branch }}`](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/{{ default_branch }}) pour la version du catalogue.)*{% endif %} +**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if current_branch != default_branch and manifest.version != default_branch_version %} *(:warning: Il s'agit de la branche `{{ current_branch }}`. La [branche `{{ default_branch }}`](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/{{ default_branch }}) utilisée dans le catalogue est actuellement en {{ default_branch_version.replace('~', '\~') }}.)*{% endif %} {% if upstream.demo %} **Démo :** {{upstream.demo}} From ebd5d0f7fb144e0ba149323e72fc7487eb237e44 Mon Sep 17 00:00:00 2001 From: Hadrien Date: Sun, 7 Aug 2022 15:15:40 +0200 Subject: [PATCH 43/51] Add a script to add cpe id to manifests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit indent + utf8 --- tools/autopatches/patches/add-cpe/cpe.csv | 507 +++++++++++++++++++++ tools/autopatches/patches/add-cpe/patch.sh | 19 + 2 files changed, 526 insertions(+) create mode 100644 tools/autopatches/patches/add-cpe/cpe.csv create mode 100644 tools/autopatches/patches/add-cpe/patch.sh diff --git a/tools/autopatches/patches/add-cpe/cpe.csv b/tools/autopatches/patches/add-cpe/cpe.csv new file mode 100644 index 0000000..f44e455 --- /dev/null +++ b/tools/autopatches/patches/add-cpe/cpe.csv @@ -0,0 +1,507 @@ +App YnH,Common Platform Enumeration +20euros, +243, +2fauth, +abantecart,cpe:2.3:a:abantecart:abantecart +acropolis, +adguardhome,cpe:2.3:a:adguard:adguardhome +adhocserver, +adminer,cpe:2.3:a:adminer:adminer +aeneria, +agendav, +agora, +airsonic,cpe:2.3:a:airsonic_project:airsonic +alltube, +ampache,cpe:2.3:a:ampache:ampache +anarchism, +anfora, +archivebox, +archivist,cpe:2.3:a:archivista:archivistabox +armadietto, +askbot,cpe:2.3:a:askbot:askbot +audiobookshelf, +automad,cpe:2.3:a:automad:automad +backdrop,cpe:2.3:a:backdropcms:backdrop +baikal, +bazarr, +beehive,cpe:2.3:a:beehive_forum:beehive_forum +bibliogram, +biboumi, +bicbucstriim, +blogotext,cpe:2.3:a:blogotext_project:blogotext +bludit,cpe:2.3:a:bludit:bludit +bolt, +bookstack,cpe:2.3:a:bookstackapp:bookstack +borg,cpe:2.3:a:borgbackup:borg +borgserver, +bozon, +cachet, +calibreweb, +castopod, +cesium, +cheky, +chtickynotes, +chuwiki, +cinny,cpe:2.3:a:cinny_project:cinny +civicrm_drupal7, +cockpit,cpe:2.3:a:agentejo:cockpit +code-server,cpe:2.3:a:coder:code-server +codimd,cpe:2.3:a:hackmd:codimd +coin, +collabora, +commento, +compteur_du_gase, +concrete5,cpe:2.3:a:concrete5:concrete5 +converse,cpe:2.3:a:conversejs:converse.js +cops, +coquelicot, +coturn,cpe:2.3:a:coturn_project:coturn +couchdb,cpe:2.3:a:apache:couchdb +couchpotato, +covoiturage, +cowyo, +cryptpad,cpe:2.3:a:xwiki:cryptpad +cubiks-2048, +cypht, +dato, +decidim, +democracyos, +dendrite, +dex, +diagramsnet, +diaspora, +digisteps, +digitools, +digiwords, +discourse,cpe:2.3:a:discourse:discourse +dispatch, +distbin, +django-fmd, +django-for-runners, +django-fritzconnection, +django_app, +docker-registry, +dockercontainer, +dockerrstudio, +dockerui, +documize,cpe:2.3:a:documize:documize +dokuwiki,cpe:2.3:a:dokuwiki:dokuwiki +dolibarr,cpe:2.3:a:dolibarr:dolibarr +domoticz,cpe:2.3:a:domoticz:domoticz +dotclear2, +droppy,cpe:2.3:a:droppy_project:droppy +drupal,cpe:2.3:a:drupal:drupal +drupal7, +duniter, +dynette, +easyappointments,cpe:2.3:a:easyappointments:easyappointments +ecko, +elabftw,cpe:2.3:a:elabftw:elabftw +element, +emailpoubelle, +emoncms,cpe:2.3:a:openenergymonitor:emoncms +encryptic, +encryptor-decryptor, +epicyon, +ergo, +ethercalc, +etherpad_mypads, +excalidraw, +fab-manager, +facette, +facilmap, +fallback, +ffsync, +filebrowser, +filepizza, +firefly-iii,cpe:2.3:a:firefly-iii:firefly_iii +flarum,cpe:2.3:a:flarum:flarum +flask,cpe:2.3:a:palletsprojects:flask +flood, +fluxbb,cpe:2.3:a:fluxbb:fluxbb +focalboard,cpe:2.3:a:mattermost:focalboard +foodsoft, +framaestro, +framaestro_hub, +framaforms, +framagames, +freeboard, +freepbx,cpe:2.3:a:freepbx:freepbx +freshrss,cpe:2.3:a:freshrss:freshrss +friendica,cpe:2.3:a:friendica:friendica +ftp_webapp, +ftssolr, +funkwhale, +galene, +galette,cpe:2.3:a:galette:galette +gamja, +garradin, +gateone,cpe:2.3:a:liftoffsoftware:gate_one +gekko, +gemserv, +getsimple,cpe:2.3:a:get-simple:getsimple_cms +ghost,cpe:2.3:a:ghost:ghost +gitea,cpe:2.3:a:gitea:gitea +gitlab,cpe:2.3:a:gitlab:gitlab +gitlab-runner,cpe:2.3:a:gitlab:runner +gitolite,cpe:2.3:a:gitolite:gitolite +gitrepositories, +gitweb, +glitchsoc, +glowingbear, +glpi,cpe:2.3:a:glpi-project:glpi +gnusocial, +gogs,cpe:2.3:a:gogs:gogs +gogswebhost, +gollum,cpe:2.3:a:gollum_project:gollum +gossa, +gotify, +gotosocial, +grafana,cpe:2.3:a:grafana:grafana +grammalecte, +grav, +grocy,cpe:2.3:a:grocy_project:grocy +grr,cpe:2.3:a:devome:grr +guacamole,cpe:2.3:a:apache:guacamole +h5ai,cpe:2.3:a:h5ai_project:h5ai +halcyon, +haste, +hat, +headphones, +hedgedoc,cpe:2.3:a:hedgedoc:hedgedoc +helloworld, +hextris, +homeassistant, +horde,cpe:2.3:a:horde:horde_application_framework +hotspot, +htmltool, +htpc-manager, +hubzilla,cpe:2.3:a:hubzilla:hubzilla +huginn, +humhub,cpe:2.3:a:humhub:humhub +hydrogen, +icecoder,cpe:2.3:a:icecoder:icecoder +ifconfig-io, +ifm, +ihatemoney,cpe:2.3:a:ihatemoney:i_hate_money +indexhibit,cpe:2.3:a:indexhibit:indexhibit +internetarchive, +invidious, +invoiceninja,cpe:2.3:a:invoiceninja:invoice_ninja +invoiceninja5, +jackett, +jappix,cpe:2.3:a:jappix_project:jappix +jappix_mini, +jeedom,cpe:2.3:a:jeedom:jeedom +jellyfin,cpe:2.3:a:jellyfin:jellyfin +jenkins,cpe:2.3:a:jenkins:jenkins +jirafeau,cpe:2.3:a:jirafeau:jirafeau +jitsi,cpe:2.3:a:jitsi:jitsi +joomla,cpe:2.3:a:joomla:joomla\! +jupyterlab,cpe:2.3:a:jupyter:nbdime-jupyterlab +kanboard,cpe:2.3:a:kanboard:kanboard +keeweb, +kimai2,cpe:2.3:a:kimai:kimai_2 +kiwiirc, +kiwix,cpe:2.3:a:kiwix:kiwix +kodi,cpe:2.3:a:kodi:kodi +komga, +kresus, +languagetool, +laverna, +lbcalerte, +leed, +lektor, +lemmy, +librarian, +libreddit, +libreerp, +librephotos, +librespeed, +libreto, +librex, +lidarr, +limesurvey,cpe:2.3:a:limesurvey:limesurvey +linuxdash, +lionwiki-t2t, +listmonk, +lstu, +luckysheet, +lufi, +lutim,cpe:2.3:a:lutim_project:lutim +lxd, +lxd-dashboard, +lychee,cpe:2.3:a:lycheeorganisation:lychee-v3 +mailman,cpe:2.3:a:gnu:mailman +mailman3, +mantis,cpe:2.3:a:mantisbt:mantisbt +mastodon,cpe:2.3:a:joinmastodon:mastodon +matomo,cpe:2.3:a:matomo:matomo +matrix-puppet-discord, +matterbridge, +mattermost,cpe:2.3:a:mattermost:mattermost +mautic,cpe:2.3:a:acquia:mautic +mautrix_facebook, +mautrix_signal, +mautrix_telegram, +mautrix_whatsapp, +mediadrop, +mediawiki,cpe:2.3:a:mediawiki:mediawiki +medusa, +meilisearch, +menu, +metabase,cpe:2.3:a:metabase:metabase +minchat, +mindmaps, +minetest,cpe:2.3:a:minetest:minetest +mineweb, +minidlna,cpe:2.3:a:readymedia_project:readymedia +miniflux, +minio,cpe:2.3:a:minio:minio +misskey,cpe:2.3:a:misskey:misskey +mobilizon, +modernpaste, +monica,cpe:2.3:a:monicahq:monica +monit, +monitorix,cpe:2.3:a:fibranet:monitorix +moodle,cpe:2.3:a:moodle:moodle +mopidy, +mosquitto,cpe:2.3:a:eclipse:mosquitto +movim,cpe:2.3:a:movim:movim +multi_webapp, +mumble-web, +mumble_admin_plugin, +mumbleserver, +munin,cpe:2.3:a:munin-monitoring:munin +my-mind, +my_capsule, +my_webapp, +mybb,cpe:2.3:a:mybb:mybb +mycryptochat, +mygpo, +mytinytodo,cpe:2.3:a:mytinytodo:mytinytodo +navidrome,cpe:2.3:a:navidrome:navidrome +netdata,cpe:2.3:a:netdata:netdata +neutrinet, +nextcloud,cpe:2.3:a:nextcloud:nextcloud +nexusoss, +nitter, +noalyss, +nocodb,cpe:2.3:a:xgenecloud:nocodb +nodebb,cpe:2.3:a:nodebb:nodebb +nodered,cpe:2.3:a:nodered:node-red-dashboard +nomad,cpe:2.3:a:jenkins:nomad +ntopng,cpe:2.3:a:ntop:ntopng +nullboard, +ofbiz,cpe:2.3:a:apache:ofbiz +omeka-s, +onlyoffice,cpe:2.3:a:onlyoffice:document_server +openidsimplesamlphp, +opennote, +openproject,cpe:2.3:a:openproject:openproject +opensondage, +opentracker, +osada, +osjs, +osmw, +osticket,cpe:2.3:a:osticket:osticket +outline, +overleaf, +owncast,cpe:2.3:a:owncast_project:owncast +owncloud,cpe:2.3:a:owncloud:owncloud +owntracks, +pagure,cpe:2.3:a:fedoraproject:389_directory_server +paperless-ngx, +peachpub, +peertube,cpe:2.3:a:framasoft:peertube +peertube-search-index, +pelican, +pepettes, +petitesannonces, +petrolette, +pgadmin,cpe:2.3:a:phppgadmin_project:phppgadmin +photonix, +photoprism, +photoview, +phpback, +phpbb,cpe:2.3:a:phpbb:phpbb +phpboost, +phpinfo, +phpipam,cpe:2.3:a:phpipam:phpipam +phpldapadmin,cpe:2.3:a:phpldapadmin_project:phpldapadmin +phplicensewatcher, +phpmyadmin,cpe:2.3:a:phpmyadmin:phpmyadmin +phpservermon,cpe:2.3:a:phpservermonitor:php_server_monitor +phpsysinfo, +pia, +pico, +pihole,cpe:2.3:a:pi-hole:pi-hole +piratebox, +piwigo,cpe:2.3:a:piwigo:piwigo +pixelfed, +plainpad, +pleroma, +plonecms, +plume,cpe:2.3:a:plume-cms:plume_cms +pluxml,cpe:2.3:a:pluxml:pluxml +pmwiki,cpe:2.3:a:pmwiki:pmwiki +portainer,cpe:2.3:a:portainer:portainer +prestashop,cpe:2.3:a:prestashop:prestashop +prettynoemiecms, +privatebin,cpe:2.3:a:privatebin:privatebin +proftpd,cpe:2.3:a:proftpd:proftpd +prometheus,cpe:2.3:a:prometheus:prometheus +prosody,cpe:2.3:a:prosody:prosody +prowlarr, +proxitok, +psitransfer, +pterodactyl,cpe:2.3:a:pterodactyl:panel +pufferpanel, +pydio,cpe:2.3:a:pydio:pydio +pyinventory, +pyload, +pytition, +qr, +question2answer,cpe:2.3:a:question2answer:question2answer +quizzes, +radarr, +radicale,cpe:2.3:a:radicale:radicale +rainloop,cpe:2.3:a:rainloop:webmail +redirect, +redmine,cpe:2.3:a:redmine:redmine +reel2bits, +remotestorage, +restic, +retroarch,cpe:2.3:a:libretro:retroarch +riot, +roadiz, +rocketchat,cpe:2.3:a:rocket.chat:rocket.chat +roundcube,cpe:2.3:a:roundcube:webmail +rportd, +rspamdui, +rss-bridge, +rutorrent, +samba,cpe:2.3:a:samba:samba +sat, +satdress, +scm, +scratch, +scrumblr, +seafile,cpe:2.3:a:seafile:seafile +searx, +seenthis, +selfoss, +send, +shaarli,cpe:2.3:a:shaarli_project:shaarli +shadowsocks,cpe:2.3:a:shadowsocks:shadowsocks-libev +shellinabox,cpe:2.3:a:shellinabox_project:shellinabox +shinken, +shiori, +shsd, +shuri, +sickbeard, +sickrage,cpe:2.3:a:sickrage:sickrage +signaturepdf, +simple-hash-generator, +simple-torrent, +sitemagiccms,cpe:2.3:a:sitemagic:sitemagic +slingcode, +snappymail, +snipeit,cpe:2.3:a:snipeitapp:snipe-it +snserver, +snweb, +soapbox,cpe:2.3:a:soapbox_project:soapbox +sogo,cpe:2.3:a:inverse:sogo +sonarr, +sonerezh, +spacedeck, +spftoolbox, +sphinx,cpe:2.3:a:sphinxsearch:sphinx +spip,cpe:2.3:a:spip:spip +squid3, +ssbroom, +ssh_chroot_dir, +staticwebapp, +streama, +strut, +subscribe, +subsonic,cpe:2.3:a:subsonic:subsonic +sutom, +svgedit, +sympa,cpe:2.3:a:sympa:sympa +synapse, +synapse-admin, +syncthing,cpe:2.3:a:syncthing:syncthing +tagspaces, +tailoredflow, +teampass,cpe:2.3:a:teampass:teampass +technitium-dns, +teddit, +telegram_chatbot, +tes3mp, +thelounge, +tiddlywiki, +tiki,cpe:2.3:a:tiki:tiki +timemachine, +timeoff, +tooljet, +torrelay, +tracim, +traggo, +transfersh, +transmission, +transpay, +transwhat, +trilium,cpe:2.3:a:trilium_project:trilium +trustyhash, +ttrss, +turtl,cpe:2.3:a:lyonbros:turtl +tutao, +tvheadend, +tyto, +ulogger, +umami, +umap, +ums, +unattended_upgrades, +unbound,cpe:2.3:a:nlnetlabs:unbound +uptime-kuma, +vaultwarden, +veloren, +vikunja, +vpnclient, +vpnserver, +wallabag2, +weblate,cpe:2.3:a:weblate:weblate +webmin,cpe:2.3:a:webmin:webmin +webogram, +webtrees,cpe:2.3:a:webtrees:webtrees +wekan,cpe:2.3:a:wekan_project:wekan +wemawema, +wetty, +whitebophir, +wifiwithme, +wikijs, +wildfly,cpe:2.3:a:redhat:jboss_wildfly_application_server +wireguard, +wisemapping, +wondercms,cpe:2.3:a:wondercms:wondercms +wordpress,cpe:2.3:a:wordpress:wordpress +writefreely, +yacy, +yellow, +yeswiki,cpe:2.3:a:yeswiki:yeswiki +yourls,cpe:2.3:a:yourls:yourls +youtube-dl-webui, +yunofav, +yunohost,cpe:2.3:o:yunohost:yunohost +yunomonitor, +yunorunner, +z-push, +zabbix,cpe:2.3:a:zabbix:zabbix +zap, +zerobin, +zeronet, +zomburl, +ztncui, +zusam, diff --git a/tools/autopatches/patches/add-cpe/patch.sh b/tools/autopatches/patches/add-cpe/patch.sh new file mode 100644 index 0000000..02a697c --- /dev/null +++ b/tools/autopatches/patches/add-cpe/patch.sh @@ -0,0 +1,19 @@ +#!/usr/bin/python3 + +import json +import csv + +def find_cpe(app_id): + with open("../../patches/add-cpe/cpe.csv", newline='') as f: + cpe_list = csv.reader(f) + for row in cpe_list: + if row[0] == app_id: + return row[1] + return False + +manifest = json.load(open("manifest.json")) +app_id = manifest['id'] +cpe = find_cpe(app_id) +if cpe: + manifest['upstream']['cpe'] = cpe + json.dump(manifest, open("manifest.json", "w"), indent=4, ensure_ascii=False) From b796cf292e0246cd22cfeaaec301834f7fbd7075 Mon Sep 17 00:00:00 2001 From: Hadrien Date: Sun, 7 Aug 2022 16:32:25 +0200 Subject: [PATCH 44/51] patch message --- tools/autopatches/patches/add-cpe/pr_body.md | 6 ++++++ tools/autopatches/patches/add-cpe/pr_title.md | 1 + 2 files changed, 7 insertions(+) create mode 100644 tools/autopatches/patches/add-cpe/pr_body.md create mode 100644 tools/autopatches/patches/add-cpe/pr_title.md diff --git a/tools/autopatches/patches/add-cpe/pr_body.md b/tools/autopatches/patches/add-cpe/pr_body.md new file mode 100644 index 0000000..5f08ae8 --- /dev/null +++ b/tools/autopatches/patches/add-cpe/pr_body.md @@ -0,0 +1,6 @@ + +This is an ***automated*** patch to add the (optional but recommended if relevant) Common Platform Enumeration (CPE) id, which is sort of a standard id for applications, defined by the NIST. + +In particular, Yunohost may use this is in the future to easily track CVE (=security reports) related to apps. + +The CPE may be obtained by searching here: https://nvd.nist.gov/products/cpe/search. For example, for Nextcloud, the CPE is 'cpe:2.3:a:nextcloud:nextcloud' (no need to include the version number)"). \ No newline at end of file diff --git a/tools/autopatches/patches/add-cpe/pr_title.md b/tools/autopatches/patches/add-cpe/pr_title.md new file mode 100644 index 0000000..5944c62 --- /dev/null +++ b/tools/autopatches/patches/add-cpe/pr_title.md @@ -0,0 +1 @@ +Add Common Platform Enumeration id to `manifest.json` From 6386d092b0d300139c3d79e85d7e383bf1d958c8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 14 Aug 2022 02:15:40 +0200 Subject: [PATCH 45/51] Packaging v2: add a 'fund' key in the upstream section --- tools/packaging_v2/convert_app_to_packaging_v2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/packaging_v2/convert_app_to_packaging_v2.py b/tools/packaging_v2/convert_app_to_packaging_v2.py index ba00153..4d6d97a 100644 --- a/tools/packaging_v2/convert_app_to_packaging_v2.py +++ b/tools/packaging_v2/convert_app_to_packaging_v2.py @@ -27,6 +27,7 @@ def _convert_v1_manifest_to_v2(app_path): manifest["upstream"]["website"] = manifest["url"] manifest["upstream"]["cpe"] = "???" + manifest["upstream"]["fund"] = "???" manifest["integration"] = { "yunohost": manifest.get("requirements", {}).get("yunohost"), @@ -161,6 +162,7 @@ def _dump_v2_manifest_as_toml(manifest): for key, value in manifest["upstream"].items(): upstream[key] = value upstream["cpe"].comment("FIXME: optional but recommended if relevant, this is meant to contain the Common Platform Enumeration, which is sort of a standard id for applications defined by the NIST. In particular, Yunohost may use this is in the future to easily track CVE (=security reports) related to apps. The CPE may be obtained by searching here: https://nvd.nist.gov/products/cpe/search. For example, for Nextcloud, the CPE is 'cpe:2.3:a:nextcloud:nextcloud' (no need to include the version number)") + upstream["fund"].comment("FIXME: optional but recommended (or remove if irrelevant / not applicable). This is meant to be an URL where people can financially support this app, especially when its development is based on volunteers and/or financed by its community. YunoHost may later advertise it in the webadmin.") toml_manifest["upstream"] = upstream integration = table() From f99b497ffdae515d2ce9676d3b7ef9480906b788 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 27 Aug 2022 16:12:13 +0200 Subject: [PATCH 46/51] Expose the 'potential_alternative_to' info in the app catalog --- list_builder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/list_builder.py b/list_builder.py index e41cdc1..93f77dd 100755 --- a/list_builder.py +++ b/list_builder.py @@ -241,6 +241,7 @@ def build_catalog(): "good_quality": level >= 8, "bad_quality": level <= 5, "antifeatures": infos["antifeatures"], + "potential_alternative_to": infos.get("potential_alternative_to", []), } result_dict_doc = { @@ -320,6 +321,7 @@ def build_app_dict(app, infos): "featured": infos.get("featured", False), "category": infos.get("category", None), "subtags": infos.get("subtags", []), + "potential_alternative_to": infos.get("potential_alternative_to", []), "antifeatures": list( set(manifest.get("antifeatures", []) + infos.get("antifeatures", [])) ), From b7c5e8958abcb5ed56bd9aab76c99e15b8023833 Mon Sep 17 00:00:00 2001 From: Tagada <36127788+Tagadda@users.noreply.github.com> Date: Tue, 13 Sep 2022 23:09:00 +0200 Subject: [PATCH 47/51] Fix a bug introduced with antifeatures --- list_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/list_builder.py b/list_builder.py index 93f77dd..2280e0d 100755 --- a/list_builder.py +++ b/list_builder.py @@ -323,7 +323,7 @@ def build_app_dict(app, infos): "subtags": infos.get("subtags", []), "potential_alternative_to": infos.get("potential_alternative_to", []), "antifeatures": list( - set(manifest.get("antifeatures", []) + infos.get("antifeatures", [])) + set(list(manifest.get("antifeatures", {}).keys()) + infos.get("antifeatures", [])) ), } From c345bd923751949b602ab7c80989ba5f537865f1 Mon Sep 17 00:00:00 2001 From: Tagada <36127788+Tagadda@users.noreply.github.com> Date: Thu, 6 Oct 2022 23:12:37 +0200 Subject: [PATCH 48/51] Revert "Add a warning on the README when not on the master branch" --- tools/README-generator/make_readme.py | 17 ++--------------- tools/README-generator/templates/README.md.j2 | 3 ++- .../README-generator/templates/README_fr.md.j2 | 3 ++- 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/tools/README-generator/make_readme.py b/tools/README-generator/make_readme.py index 592f009..d9c03db 100755 --- a/tools/README-generator/make_readme.py +++ b/tools/README-generator/make_readme.py @@ -32,7 +32,7 @@ def generate_READMEs(app_path: str): from_catalog = catalog.get(manifest['id'], {}) antifeatures_list = yaml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "antifeatures.yml"), Loader=yaml.SafeLoader) - antifeatures_list = {e['id']: e for e in antifeatures_list} + antifeatures_list = { e['id']: e for e in antifeatures_list } if not upstream and not (app_path / "doc" / "DISCLAIMER.md").exists(): print( @@ -69,18 +69,8 @@ def generate_READMEs(app_path: str): else: disclaimer = None - # Get the current branch using git inside the app path - default_branch = from_catalog.get('branch', 'master') - current_branch = os.popen(f"git -C {app_path} rev-parse --abbrev-ref HEAD").read().strip() - - if default_branch != current_branch: - os.system(f"git -C {app_path} fetch origin {default_branch} 2>/dev/null") - default_branch_version = os.popen(f"git -C {app_path} show FETCH_HEAD:manifest.json | jq -r .version").read().strip() - else: - default_branch_version = None # we don't care in that case - # TODO: Add url to the documentation... and actually create that documentation :D - antifeatures = {a: antifeatures_list[a] for a in from_catalog.get('antifeatures', [])} + antifeatures = { a: antifeatures_list[a] for a in from_catalog.get('antifeatures', [])} for k, v in antifeatures.items(): antifeatures[k]['title'] = value_for_lang(v['title'], lang_suffix) if manifest.get("antifeatures", {}).get(k, None): @@ -96,9 +86,6 @@ def generate_READMEs(app_path: str): disclaimer=disclaimer, antifeatures=antifeatures, manifest=manifest, - current_branch=current_branch, - default_branch=default_branch, - default_branch_version=default_branch_version, ) (app_path / f"README{lang_suffix}.md").write_text(out) diff --git a/tools/README-generator/templates/README.md.j2 b/tools/README-generator/templates/README.md.j2 index 0c5f9b0..e2d428f 100644 --- a/tools/README-generator/templates/README.md.j2 +++ b/tools/README-generator/templates/README.md.j2 @@ -31,7 +31,8 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if current_branch != default_branch and manifest.version != default_branch_version %} *(:warning: This is the `{{ current_branch }}` branch. The [`{{ default_branch }}` branch](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/{{ default_branch }}) used in the catalog is currently on version {{ default_branch_version.replace('~', '\~') }}.)*{% endif %} +**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}} +{% endif -%} {% if upstream.demo %} **Demo:** {{upstream.demo}} diff --git a/tools/README-generator/templates/README_fr.md.j2 b/tools/README-generator/templates/README_fr.md.j2 index 6171d44..95fb8f4 100644 --- a/tools/README-generator/templates/README_fr.md.j2 +++ b/tools/README-generator/templates/README_fr.md.j2 @@ -17,7 +17,8 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %}{% if current_branch != default_branch and manifest.version != default_branch_version %} *(:warning: Il s'agit de la branche `{{ current_branch }}`. La [branche `{{ default_branch }}`](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/{{ default_branch }}) utilisée dans le catalogue est actuellement en {{ default_branch_version.replace('~', '\~') }}.)*{% endif %} +**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}} +{% endif -%} {% if upstream.demo %} **Démo :** {{upstream.demo}} From c9a1acbd4bbd3bc4e1462382efb33532ce9c1b74 Mon Sep 17 00:00:00 2001 From: Tagada <36127788+Tagadda@users.noreply.github.com> Date: Thu, 6 Oct 2022 23:20:19 +0200 Subject: [PATCH 49/51] typo --- tools/README-generator/make_readme.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/README-generator/make_readme.py b/tools/README-generator/make_readme.py index d9c03db..cb65525 100755 --- a/tools/README-generator/make_readme.py +++ b/tools/README-generator/make_readme.py @@ -72,11 +72,11 @@ def generate_READMEs(app_path: str): # TODO: Add url to the documentation... and actually create that documentation :D antifeatures = { a: antifeatures_list[a] for a in from_catalog.get('antifeatures', [])} for k, v in antifeatures.items(): - antifeatures[k]['title'] = value_for_lang(v['title'], lang_suffix) + antifeatures[k]['title'] = value_for_lang(v['title'], lang) if manifest.get("antifeatures", {}).get(k, None): - antifeatures[k]['description'] = value_for_lang(manifest.get("antifeatures", {}).get(k, None), lang_suffix) + antifeatures[k]['description'] = value_for_lang(manifest.get("antifeatures", {}).get(k, None), lang) else: - antifeatures[k]['description'] = value_for_lang(antifeatures[k]['description'], lang_suffix) + antifeatures[k]['description'] = value_for_lang(antifeatures[k]['description'], lang) out = template.render( lang=lang, From 5ad77cd2b71ac46db72d180f3934de84740d9d64 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 11 Oct 2022 18:56:09 +0200 Subject: [PATCH 50/51] remove the app install question and resources parts which aint needed anymore by webadmin etc (or at least we think ;P) --- list_builder.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/list_builder.py b/list_builder.py index 2280e0d..8e3debe 100755 --- a/list_builder.py +++ b/list_builder.py @@ -206,6 +206,13 @@ def build_catalog(): if packaging_format < 2: app["manifest"] = convert_v1_manifest_to_v2_for_catalog(app["manifest"]) + # We also remove the app install question and resources parts which aint needed anymore by webadmin etc (or at least we think ;P) + for app in result_dict_with_manifest_v2.values(): + if "manifest" in app and "install" in app["manifest"]: + del app["manifest"]["install"] + if "manifest" in app and "resources" in app["manifest"]: + del app["manifest"]["resources"] + os.system("mkdir -p ./builds/default/v3/") with open("builds/default/v3/apps.json", "w") as f: f.write( From bcb696458c0d3d7440666f0465f392e1d0873d56 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 18 Oct 2022 18:35:20 +0200 Subject: [PATCH 51/51] =?UTF-8?q?Drop=20high=5Fquality=20flag=20=C3=A9=5F?= =?UTF-8?q?=C3=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/README.md b/README.md index 08bfda5..c185bd6 100644 --- a/README.md +++ b/README.md @@ -73,14 +73,6 @@ moment... We invite you to use [translate.yunohost.org](https://translate.yunohost.org/) instead of doing Pull Request for files in `locales` folder. -### How to make my app flagged as High Quality? - -A High Quality app will be highlighted in the app list and marked as a level 9 app. -To become a High Quality app, a package has to follow the criterias listed [here](hq_validation_template.md). - -Once the app is validated is "high quality", the tag `"high_quality": true` -shall be added to the app infos inside the catalog (`apps.json`). - ### Apps flagged as not-maintained Applications with no recent activity and no active sign from maintainer may be flagged in `apps.json` with `"maintained": false` to signify that the app is inactive and may slowly become outdated with respect to the upstream, or with respect to good packaging practices. It does **not** mean that the app is not working anymore.