From 88f8239b1652b0fea8a9f70c36a3f712a986fed0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Gaspar?= <46165813+ericgaspar@users.noreply.github.com> Date: Mon, 21 Feb 2022 19:05:04 +0100 Subject: [PATCH 01/19] minor typos --- README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 79e73ff..08bfda5 100644 --- a/README.md +++ b/README.md @@ -7,20 +7,20 @@ Here you will find the repositories and versions of every apps available in Yuno It is browsable here: https://yunohost.org/apps The main file of the catalog is [**apps.json**](./apps.json) which contains -references to the corresponding git repositories for each application, along +references to the corresponding Git repositories for each application, along with a few metadata about them such as its category or maintenance state. This file regularly read by `list_builder.py` which publish the results on https://app.yunohost.org/default/. -### Where can I learn about app packaging in Yunohost ? +### Where can I learn about app packaging in YunoHost? - You can browse the contributor documentation : https://yunohost.org/contributordoc -- If you are not familiar with Git/Github, you can have a look at our [homemade guide](https://yunohost.org/#/packaging_apps_git) -- Don't hesitate to reach for help on the dedicated [application packaging chatroom](https://yunohost.org/chat_rooms) ... we can even schedule an audio meeting to help you get started ! +- If you are not familiar with Git/GitHub, you can have a look at our [homemade guide](https://yunohost.org/#/packaging_apps_git) +- Don't hesitate to reach for help on the dedicated [application packaging chatroom](https://yunohost.org/chat_rooms) ... we can even schedule an audio meeting to help you get started! ### How to add your app to the application catalog -N.B. : the Yunohost project will **NOT** integrate in its catalog applications that are not +N.B.: The YunoHost project will **NOT** integrate in its catalog applications that are not based on free-software upstreams. To add your application to the catalog: @@ -38,11 +38,11 @@ App example addition: } ``` -N.B. : We strongly encourage you to transfer the ownership of your repository to -the Yunohost-Apps organization on Github, such that the community will help you +N.B.: We strongly encourage you to transfer the ownership of your repository to +the YunoHost-Apps organization on GitHub, such that the community will help you with keeping your app working and up to date with packaging evolutions. -N.B.2 : Implicitly, the catalog publishes the `HEAD` of branch `master` +N.B.2: Implicitly, the catalog publishes the `HEAD` of branch `master` (this can be overwritten by adding keys `branch` and `revision`). Therefore, **be careful that any commit on the `master` branch will automatically be published**. **We strongly encourage you to develop in separate branches**, and only @@ -56,8 +56,8 @@ App packagers should *not* manually set their apps' level. The levels of all the #### Helper script -You can use the add_or_update.py python script to add or update -your app from one of the 2 json files. +You can use the add_or_update.py Python script to add or update +your app from one of the 2 JSON files. Usage: @@ -67,13 +67,13 @@ Usage: ### How to help translating -Update on Nov. 2020 : this part is broken / not maintained anymore for the +Update on Nov. 2020: this part is broken / not maintained anymore for the moment... We invite you to use [translate.yunohost.org](https://translate.yunohost.org/) instead of doing Pull Request for files in `locales` folder. -### How to make my app flagged as High Quality ? +### How to make my app flagged as High Quality? A High Quality app will be highlighted in the app list and marked as a level 9 app. To become a High Quality app, a package has to follow the criterias listed [here](hq_validation_template.md). From 83b8e738cc3ff54489bc44c7d8c2825bbde5fab1 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Fri, 13 May 2022 00:13:27 +0200 Subject: [PATCH 02/19] Readmes improvements --- tools/README-generator/templates/README.md.j2 | 8 +++++--- tools/README-generator/templates/README_fr.md.j2 | 14 ++++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tools/README-generator/templates/README.md.j2 b/tools/README-generator/templates/README.md.j2 index 6e58002..19f87ef 100644 --- a/tools/README-generator/templates/README.md.j2 +++ b/tools/README-generator/templates/README.md.j2 @@ -19,7 +19,7 @@ It shall NOT be edited by hand. # {{manifest.name}} for YunoHost -[![Integration level](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) +[![Integration level](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Working status](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Maintenance status](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) [![Install {{manifest.name}} with YunoHost](https://install-app.yunohost.org/install-with-yunohost.svg)](https://install-app.yunohost.org/?app={{manifest.id}}) *[Lire ce readme en français.](./README_fr.md)* @@ -39,7 +39,7 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in ## Screenshots {% for screenshot in screenshots -%} - ![](./doc/screenshots/{{screenshot}}) + ![Screenshot of {{manifest.name}}](./doc/screenshots/{{screenshot}}) {% endfor %} {% endif -%} @@ -67,10 +67,12 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in Please send your pull request to the [testing branch](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing). To try the testing branch, please proceed like that. -``` + +``` bash sudo yunohost app install https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug or sudo yunohost app upgrade {{manifest.id}} -u https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug ``` **More info regarding app packaging:** https://yunohost.org/packaging_apps + diff --git a/tools/README-generator/templates/README_fr.md.j2 b/tools/README-generator/templates/README_fr.md.j2 index 9169cbb..f531abe 100644 --- a/tools/README-generator/templates/README_fr.md.j2 +++ b/tools/README-generator/templates/README_fr.md.j2 @@ -1,10 +1,14 @@ + + # {{manifest.name}} pour YunoHost -[![Niveau d'intégration](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) +[![Niveau d'intégration](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Status du fonctionnement](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Status de maintenance](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) [![Installer {{manifest.name}} avec YunoHost](https://install-app.yunohost.org/install-with-yunohost.svg)](https://install-app.yunohost.org/?app={{manifest.id}}) *[Read this readme in english.](./README.md)* -*[Lire ce readme en français.](./README_fr.md)* > *Ce package vous permet d'installer {{manifest.name}} rapidement et simplement sur un serveur YunoHost. Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour savoir comment l'installer et en profiter.* @@ -21,7 +25,7 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour ## Captures d'écran {% for screenshot in screenshots -%} - ![](./doc/screenshots/{{screenshot}}) + ![Capture d'écran de {{manifest.name}}](./doc/screenshots/{{screenshot}}) {% endfor %} {% endif -%} @@ -49,10 +53,12 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour Merci de faire vos pull request sur la [branche testing](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing). Pour essayer la branche testing, procédez comme suit. -``` + +``` bash sudo yunohost app install https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug ou sudo yunohost app upgrade {{manifest.id}} -u https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug ``` **Plus d'infos sur le packaging d'applications :** https://yunohost.org/packaging_apps + From 4fca389af9780b65d36e278be5c81bbbaa166bf5 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Fri, 13 May 2022 01:19:44 +0200 Subject: [PATCH 03/19] layout corrections (line breaks) --- tools/README-generator/templates/README.md.j2 | 9 ++++++--- tools/README-generator/templates/README_fr.md.j2 | 8 +++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tools/README-generator/templates/README.md.j2 b/tools/README-generator/templates/README.md.j2 index 19f87ef..e0f09d0 100644 --- a/tools/README-generator/templates/README.md.j2 +++ b/tools/README-generator/templates/README.md.j2 @@ -31,11 +31,14 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %} +**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}} +{% endif -%} -{% if upstream.demo %}**Demo:** {{upstream.demo}}{% endif %} +{% if upstream.demo %} +**Demo:** {{upstream.demo}} +{% endif -%} -{% if screenshots -%} +{% if screenshots %} ## Screenshots {% for screenshot in screenshots -%} diff --git a/tools/README-generator/templates/README_fr.md.j2 b/tools/README-generator/templates/README_fr.md.j2 index f531abe..a15ba8c 100644 --- a/tools/README-generator/templates/README_fr.md.j2 +++ b/tools/README-generator/templates/README_fr.md.j2 @@ -17,11 +17,13 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour {% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %} -**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}{% endif %} +**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}} +{% endif -%} -{% if upstream.demo %}**Démo :** {{upstream.demo}}{% endif %} +{% if upstream.demo %}**Démo :** {{upstream.demo}} +{% endif -%} -{% if screenshots -%} +{% if screenshots %} ## Captures d'écran {% for screenshot in screenshots -%} From b3dfdfc934efb2b78b66a72563f3dabbf24c1b24 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Fri, 13 May 2022 01:25:17 +0200 Subject: [PATCH 04/19] forgotten layout corrections (line breaks) --- tools/README-generator/templates/README_fr.md.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/README-generator/templates/README_fr.md.j2 b/tools/README-generator/templates/README_fr.md.j2 index a15ba8c..de77988 100644 --- a/tools/README-generator/templates/README_fr.md.j2 +++ b/tools/README-generator/templates/README_fr.md.j2 @@ -20,7 +20,8 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour **Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}} {% endif -%} -{% if upstream.demo %}**Démo :** {{upstream.demo}} +{% if upstream.demo %} +**Démo :** {{upstream.demo}} {% endif -%} {% if screenshots %} From 2cf6122d4281e6c4d10cd444d67fae0b3d9bbd2e Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Fri, 13 May 2022 18:15:34 +0200 Subject: [PATCH 05/19] add angle brackets around bare URLs (fix MD034) --- tools/README-generator/templates/README.md.j2 | 14 +++++++------- tools/README-generator/templates/README_fr.md.j2 | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tools/README-generator/templates/README.md.j2 b/tools/README-generator/templates/README.md.j2 index e0f09d0..4931754 100644 --- a/tools/README-generator/templates/README.md.j2 +++ b/tools/README-generator/templates/README.md.j2 @@ -54,16 +54,16 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in ## Documentation and resources -{% if upstream.website -%}* Official app website: {{ upstream.website }} +{% if upstream.website -%}* Official app website: <{{ upstream.website }}> {% endif -%} -{% if upstream.userdoc -%}* Official user documentation: {{ upstream.userdoc }} +{% if upstream.userdoc -%}* Official user documentation: <{{ upstream.userdoc }}> {% endif -%} -{% if upstream.admindoc -%}* Official admin documentation: {{ upstream.admindoc }} +{% if upstream.admindoc -%}* Official admin documentation: <{{ upstream.admindoc }}> {% endif -%} -{% if upstream.code -%}* Upstream app code repository: {{ upstream.code }} +{% if upstream.code -%}* Upstream app code repository: <{{ upstream.code }}> {% endif -%} -* YunoHost documentation for this app: https://yunohost.org/app_{{manifest.id}} -* Report a bug: https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/issues +* YunoHost documentation for this app: +* Report a bug: ## Developer info @@ -77,5 +77,5 @@ or sudo yunohost app upgrade {{manifest.id}} -u https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug ``` -**More info regarding app packaging:** https://yunohost.org/packaging_apps +**More info regarding app packaging:** diff --git a/tools/README-generator/templates/README_fr.md.j2 b/tools/README-generator/templates/README_fr.md.j2 index de77988..6f383ae 100644 --- a/tools/README-generator/templates/README_fr.md.j2 +++ b/tools/README-generator/templates/README_fr.md.j2 @@ -40,16 +40,16 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour ## Documentations et ressources -{% if upstream.website -%}* Site officiel de l'app : {{ upstream.website }} +{% if upstream.website -%}* Site officiel de l'app : <{{ upstream.website }}> {% endif -%} -{% if upstream.userdoc -%}* Documentation officielle utilisateur : {{ upstream.userdoc }} +{% if upstream.userdoc -%}* Documentation officielle utilisateur : <{{ upstream.userdoc }}> {% endif -%} -{% if upstream.admindoc -%}* Documentation officielle de l'admin : {{ upstream.admindoc }} +{% if upstream.admindoc -%}* Documentation officielle de l'admin : <{{ upstream.admindoc }}> {% endif -%} -{% if upstream.code -%}* Dépôt de code officiel de l'app : {{ upstream.code }} +{% if upstream.code -%}* Dépôt de code officiel de l'app : <{{ upstream.code }}> {% endif -%} -* Documentation YunoHost pour cette app : https://yunohost.org/app_{{manifest.id}} -* Signaler un bug : https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/issues +* Documentation YunoHost pour cette app : +* Signaler un bug : ## Informations pour les développeurs @@ -63,5 +63,5 @@ ou sudo yunohost app upgrade {{manifest.id}} -u https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug ``` -**Plus d'infos sur le packaging d'applications :** https://yunohost.org/packaging_apps +**Plus d'infos sur le packaging d'applications :** From ca986a16a3e14c997400be4da40375ccbf12fd97 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Sat, 21 May 2022 16:09:09 +0200 Subject: [PATCH 06/19] harmonising the style of lists (MD004) --- tools/README-generator/templates/README.md.j2 | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/README-generator/templates/README.md.j2 b/tools/README-generator/templates/README.md.j2 index 4931754..78a1139 100644 --- a/tools/README-generator/templates/README.md.j2 +++ b/tools/README-generator/templates/README.md.j2 @@ -1,13 +1,13 @@ {% if manifest.id == "example" -%} # Packaging your an app, starting from this example -- Copy this app before working on it, using the ['Use this template'](https://github.com/YunoHost/example_ynh/generate) button on the Github repo. -- Edit the `manifest.json` with app specific info. -- Edit the `install`, `upgrade`, `remove`, `backup`, and `restore` scripts, and any relevant conf files in `conf/`. - - Using the [script helpers documentation.](https://yunohost.org/packaging_apps_helpers) -- Add a `LICENSE` file for the package. -- Edit `doc/DISCLAIMER*.md` -- The `README.md` files are to be automatically generated by https://github.com/YunoHost/apps/tree/master/tools/README-generator +* Copy this app before working on it, using the ['Use this template'](https://github.com/YunoHost/example_ynh/generate) button on the Github repo. +* Edit the `manifest.json` with app specific info. +* Edit the `install`, `upgrade`, `remove`, `backup`, and `restore` scripts, and any relevant conf files in `conf/`. + * Using the [script helpers documentation.](https://yunohost.org/packaging_apps_helpers) +* Add a `LICENSE` file for the package. +* Edit `doc/DISCLAIMER*.md` +* The `README.md` files are to be automatically generated by https://github.com/YunoHost/apps/tree/master/tools/README-generator --- {% endif -%} From 10bf08156785b2785b9e22c89152d26fcc2b4839 Mon Sep 17 00:00:00 2001 From: tituspijean Date: Wed, 25 May 2022 09:26:35 +0200 Subject: [PATCH 07/19] =?UTF-8?q?Coquille=20dans=20le=20g=C3=A9n=C3=A9rate?= =?UTF-8?q?ur=20de=20README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: nicofrand --- tools/README-generator/templates/README_fr.md.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/README-generator/templates/README_fr.md.j2 b/tools/README-generator/templates/README_fr.md.j2 index 6f383ae..7c2bf63 100644 --- a/tools/README-generator/templates/README_fr.md.j2 +++ b/tools/README-generator/templates/README_fr.md.j2 @@ -5,7 +5,7 @@ It shall NOT be edited by hand. # {{manifest.name}} pour YunoHost -[![Niveau d'intégration](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Status du fonctionnement](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Status de maintenance](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) +[![Niveau d'intégration](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Statut du fonctionnement](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Status de maintenance](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) [![Installer {{manifest.name}} avec YunoHost](https://install-app.yunohost.org/install-with-yunohost.svg)](https://install-app.yunohost.org/?app={{manifest.id}}) *[Read this readme in english.](./README.md)* From 09a0bfd500d37a9b2abace79e4422063dca67f9d Mon Sep 17 00:00:00 2001 From: tituspijean Date: Wed, 25 May 2022 10:04:17 +0200 Subject: [PATCH 08/19] =?UTF-8?q?Coquille=20dans=20le=20g=C3=A9n=C3=A9rate?= =?UTF-8?q?ur=20de=20README,=20la=20suite?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: nicofrand --- tools/README-generator/templates/README_fr.md.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/README-generator/templates/README_fr.md.j2 b/tools/README-generator/templates/README_fr.md.j2 index 7c2bf63..ae51374 100644 --- a/tools/README-generator/templates/README_fr.md.j2 +++ b/tools/README-generator/templates/README_fr.md.j2 @@ -5,7 +5,7 @@ It shall NOT be edited by hand. # {{manifest.name}} pour YunoHost -[![Niveau d'intégration](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Statut du fonctionnement](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Status de maintenance](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) +[![Niveau d'intégration](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Statut du fonctionnement](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Statut de maintenance](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg) [![Installer {{manifest.name}} avec YunoHost](https://install-app.yunohost.org/install-with-yunohost.svg)](https://install-app.yunohost.org/?app={{manifest.id}}) *[Read this readme in english.](./README.md)* From 9db6bde4b01a9a62e362d6f2cca6abf6c77cfd9c Mon Sep 17 00:00:00 2001 From: yalh76 Date: Tue, 7 Jun 2022 22:48:50 +0200 Subject: [PATCH 09/19] Update README.md.j2 --- tools/README-generator/templates/README.md.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/README-generator/templates/README.md.j2 b/tools/README-generator/templates/README.md.j2 index 78a1139..d04393b 100644 --- a/tools/README-generator/templates/README.md.j2 +++ b/tools/README-generator/templates/README.md.j2 @@ -1,5 +1,5 @@ {% if manifest.id == "example" -%} -# Packaging your an app, starting from this example +# Packaging an app, starting from this example * Copy this app before working on it, using the ['Use this template'](https://github.com/YunoHost/example_ynh/generate) button on the Github repo. * Edit the `manifest.json` with app specific info. From fe52e9061034470124729c970322a8795a978d32 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 3 Aug 2022 15:41:11 +0200 Subject: [PATCH 10/19] black on list_builder.py --- list_builder.py | 219 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 163 insertions(+), 56 deletions(-) diff --git a/list_builder.py b/list_builder.py index 59b9a26..4f6d114 100755 --- a/list_builder.py +++ b/list_builder.py @@ -11,7 +11,9 @@ import time now = time.time() catalog = json.load(open("apps.json")) -catalog = {app: infos for app, infos in catalog.items() if infos.get('state') != 'notworking'} +catalog = { + app: infos for app, infos in catalog.items() if infos.get("state") != "notworking" +} my_env = os.environ.copy() my_env["GIT_TERMINAL_PROMPT"] = "0" @@ -19,31 +21,39 @@ my_env["GIT_TERMINAL_PROMPT"] = "0" os.makedirs(".apps_cache", exist_ok=True) os.makedirs("builds/", exist_ok=True) + def error(msg): msg = "[Applist builder error] " + msg if os.path.exists("/usr/bin/sendxmpppy"): - subprocess.call(["sendxmpppy", msg], stdout=open(os.devnull, 'wb')) + subprocess.call(["sendxmpppy", msg], stdout=open(os.devnull, "wb")) print(msg + "\n") + # Progress bar helper, stolen from https://stackoverflow.com/a/34482761 def progressbar(it, prefix="", size=60, file=sys.stdout): count = len(it) + def show(j, name=""): name += " " - x = int(size*j/count) - file.write("%s[%s%s] %i/%i %s\r" % (prefix, "#"*x, "."*(size-x), j, count, name)) + x = int(size * j / count) + file.write( + "%s[%s%s] %i/%i %s\r" % (prefix, "#" * x, "." * (size - x), j, count, name) + ) file.flush() + show(0) for i, item in enumerate(it): yield item - show(i+1, item[0]) + show(i + 1, item[0]) file.write("\n") file.flush() + ################################### # App git clones cache management # ################################### + def app_cache_folder(app): return os.path.join(".apps_cache", app) @@ -81,11 +91,13 @@ def init_cache(app, infos): else: depth = 40 - git("clone --quiet --depth {depth} --single-branch --branch {branch} {url} {folder}".format( - depth=depth, - url=infos["url"], - branch=infos.get("branch", "master"), - folder=app_cache_folder(app)) + git( + "clone --quiet --depth {depth} --single-branch --branch {branch} {url} {folder}".format( + depth=depth, + url=infos["url"], + branch=infos.get("branch", "master"), + folder=app_cache_folder(app), + ) ) @@ -102,24 +114,35 @@ def refresh_cache(app, infos): git("remote set-url origin " + infos["url"], in_folder=app_cache_folder(app)) # With git >= 2.22 # current_branch = git("branch --show-current", in_folder=app_cache_folder(app)) - current_branch = git("rev-parse --abbrev-ref HEAD", in_folder=app_cache_folder(app)) + current_branch = git( + "rev-parse --abbrev-ref HEAD", in_folder=app_cache_folder(app) + ) if current_branch != branch: # With git >= 2.13 # all_branches = git("branch --format=%(refname:short)", in_folder=app_cache_folder(app)).split() all_branches = git("branch", in_folder=app_cache_folder(app)).split() - all_branches.remove('*') + all_branches.remove("*") if branch not in all_branches: - git("remote set-branches --add origin %s" % branch, in_folder=app_cache_folder(app)) - git("fetch origin %s:%s" % (branch, branch), in_folder=app_cache_folder(app)) + git( + "remote set-branches --add origin %s" % branch, + in_folder=app_cache_folder(app), + ) + git( + "fetch origin %s:%s" % (branch, branch), + in_folder=app_cache_folder(app), + ) else: - git("checkout --force %s" % branch, in_folder=app_cache_folder(app)) + git("checkout --force %s" % branch, in_folder=app_cache_folder(app)) git("fetch --quiet origin %s --force" % branch, in_folder=app_cache_folder(app)) git("reset origin/%s --hard" % branch, in_folder=app_cache_folder(app)) except: # Sometimes there are tmp issue such that the refresh cache .. # we don't trigger an error unless the cache hasnt been updated since more than 24 hours - if os.path.exists(fetch_head) and now - os.path.getmtime(fetch_head) < 24*3600: + if ( + os.path.exists(fetch_head) + and now - os.path.getmtime(fetch_head) < 24 * 3600 + ): pass else: raise @@ -129,6 +152,7 @@ def refresh_cache(app, infos): # Actual list build management # ################################ + def build_catalog(): result_dict = {} @@ -151,33 +175,71 @@ def build_catalog(): categories = yaml.load(open("categories.yml").read()) antifeatures = yaml.load(open("antifeatures.yml").read()) os.system("mkdir -p ./builds/default/v2/") - with open("builds/default/v2/apps.json", 'w') as f: - f.write(json.dumps({"apps": result_dict, "categories": categories, "antifeatures": antifeatures}, sort_keys=True)) + with open("builds/default/v2/apps.json", "w") as f: + f.write( + json.dumps( + { + "apps": result_dict, + "categories": categories, + "antifeatures": antifeatures, + }, + sort_keys=True, + ) + ) #################### # Legacy version 1 # #################### os.system("mkdir -p ./builds/default/v1/") - with open("./builds/default/v1/apps.json", 'w') as f: + with open("./builds/default/v1/apps.json", "w") as f: f.write(json.dumps(result_dict, sort_keys=True)) #################### # Legacy version 0 # #################### - official_apps = set(["agendav", "ampache", "baikal", "dokuwiki", "etherpad_mypads", "hextris", "jirafeau", "kanboard", "my_webapp", "nextcloud", "opensondage", "phpmyadmin", "piwigo", "rainloop", "roundcube", "searx", "shellinabox", "strut", "synapse", "transmission", "ttrss", "wallabag2", "wordpress", "zerobin"]) + official_apps = set( + [ + "agendav", + "ampache", + "baikal", + "dokuwiki", + "etherpad_mypads", + "hextris", + "jirafeau", + "kanboard", + "my_webapp", + "nextcloud", + "opensondage", + "phpmyadmin", + "piwigo", + "rainloop", + "roundcube", + "searx", + "shellinabox", + "strut", + "synapse", + "transmission", + "ttrss", + "wallabag2", + "wordpress", + "zerobin", + ] + ) official_apps_dict = {k: v for k, v in result_dict.items() if k in official_apps} - community_apps_dict = {k: v for k, v in result_dict.items() if k not in official_apps} + community_apps_dict = { + k: v for k, v in result_dict.items() if k not in official_apps + } # We need the official apps to have "validated" as state to be recognized as official for app, infos in official_apps_dict.items(): infos["state"] = "validated" os.system("mkdir -p ./builds/default/v0/") - with open("./builds/default/v0/official.json", 'w') as f: + with open("./builds/default/v0/official.json", "w") as f: f.write(json.dumps(official_apps_dict, sort_keys=True)) - with open("./builds/default/v0/community.json", 'w') as f: + with open("./builds/default/v0/community.json", "w") as f: f.write(json.dumps(community_apps_dict, sort_keys=True)) ############################## @@ -185,6 +247,7 @@ def build_catalog(): ############################## categories = yaml.load(open("categories.yml").read()) os.system("mkdir -p ./builds/default/doc_catalog") + def infos_for_doc_catalog(infos): level = infos.get("level") if not isinstance(level, int): @@ -199,13 +262,21 @@ def build_catalog(): "level": level, "broken": level <= 0, "good_quality": level >= 8, - "bad_quality": level <= 5, + "bad_quality": level <= 5, "antifeatures": infos["antifeatures"], } - result_dict_doc = {k: infos_for_doc_catalog(v) for k, v in result_dict.items() if v["state"] in ["working", "validated"]} - with open("builds/default/doc_catalog/apps.json", 'w') as f: - f.write(json.dumps({"apps": result_dict_doc, "categories": categories}, sort_keys=True)) + result_dict_doc = { + k: infos_for_doc_catalog(v) + for k, v in result_dict.items() + if v["state"] in ["working", "validated"] + } + with open("builds/default/doc_catalog/apps.json", "w") as f: + f.write( + json.dumps( + {"apps": result_dict_doc, "categories": categories}, sort_keys=True + ) + ) def build_app_dict(app, infos): @@ -214,43 +285,63 @@ def build_app_dict(app, infos): this_app_cache = app_cache_folder(app) assert os.path.exists(this_app_cache), "No cache yet for %s" % app - infos['branch'] = infos.get('branch', 'master') - infos['revision'] = infos.get('revision', 'HEAD') + infos["branch"] = infos.get("branch", "master") + infos["revision"] = infos.get("revision", "HEAD") # If using head, find the most recent meaningful commit in logs if infos["revision"] == "HEAD": - relevant_files = ["manifest.json", "config_panel.toml", "hooks/", "scripts/", "conf/", "sources/"] - most_recent_relevant_commit = "rev-list --full-history --all -n 1 -- " + " ".join(relevant_files) + relevant_files = [ + "manifest.json", + "config_panel.toml", + "hooks/", + "scripts/", + "conf/", + "sources/", + ] + most_recent_relevant_commit = ( + "rev-list --full-history --all -n 1 -- " + " ".join(relevant_files) + ) infos["revision"] = git(most_recent_relevant_commit, in_folder=this_app_cache) - assert re.match(r"^[0-9a-f]+$", infos["revision"]), "Output was not a commit? '%s'" % infos["revision"] + assert re.match(r"^[0-9a-f]+$", infos["revision"]), ( + "Output was not a commit? '%s'" % infos["revision"] + ) # Otherwise, validate commit exists else: - assert infos["revision"] in git("rev-list --all", in_folder=this_app_cache).split("\n"), "Revision ain't in history ? %s" % infos["revision"] + assert infos["revision"] in git( + "rev-list --all", in_folder=this_app_cache + ).split("\n"), ("Revision ain't in history ? %s" % infos["revision"]) # Find timestamp corresponding to that commit - timestamp = git("show -s --format=%ct " + infos["revision"], in_folder=this_app_cache) - assert re.match(r"^[0-9]+$", timestamp), "Failed to get timestamp for revision ? '%s'" % timestamp + timestamp = git( + "show -s --format=%ct " + infos["revision"], in_folder=this_app_cache + ) + assert re.match(r"^[0-9]+$", timestamp), ( + "Failed to get timestamp for revision ? '%s'" % timestamp + ) timestamp = int(timestamp) # Build the dict with all the infos manifest = json.load(open(this_app_cache + "/manifest.json")) - return {'id':manifest["id"], - 'git': { - 'branch': infos['branch'], - 'revision': infos["revision"], - 'url': infos["url"] - }, - 'lastUpdate': timestamp, - 'manifest': include_translations_in_manifest(manifest), - 'state': infos['state'], - 'level': infos.get('level', '?'), - 'maintained': infos.get("maintained", True), - 'high_quality': infos.get("high_quality", False), - 'featured': infos.get("featured", False), - 'category': infos.get('category', None), - 'subtags': infos.get('subtags', []), - 'antifeatures': list(set(manifest.get('antifeatures', []) + infos.get('antifeatures', []))) - } + return { + "id": manifest["id"], + "git": { + "branch": infos["branch"], + "revision": infos["revision"], + "url": infos["url"], + }, + "lastUpdate": timestamp, + "manifest": include_translations_in_manifest(manifest), + "state": infos["state"], + "level": infos.get("level", "?"), + "maintained": infos.get("maintained", True), + "high_quality": infos.get("high_quality", False), + "featured": infos.get("featured", False), + "category": infos.get("category", None), + "subtags": infos.get("subtags", []), + "antifeatures": list( + set(manifest.get("antifeatures", []) + infos.get("antifeatures", [])) + ), + } def include_translations_in_manifest(manifest): @@ -274,14 +365,30 @@ def include_translations_in_manifest(manifest): for category, questions in manifest["arguments"].items(): for question in questions: - key = "%s_manifest_arguments_%s_%s" % (app_name, category, question["name"]) + key = "%s_manifest_arguments_%s_%s" % ( + app_name, + category, + question["name"], + ) # don't overwrite already existing translation in manifests for now - if translations.get(key) and "ask" in question and current_lang not in question["ask"]: + if ( + translations.get(key) + and "ask" in question + and current_lang not in question["ask"] + ): question["ask"][current_lang] = translations[key] - key = "%s_manifest_arguments_%s_help_%s" % (app_name, category, question["name"]) + key = "%s_manifest_arguments_%s_help_%s" % ( + app_name, + category, + question["name"], + ) # don't overwrite already existing translation in manifests for now - if translations.get(key) and "help" in question and current_lang not in question.get("help", []): + if ( + translations.get(key) + and "help" in question + and current_lang not in question.get("help", []) + ): question["help"][current_lang] = translations[key] return manifest From a3d1738e8e699c7305f4c3cdbcf74efa62106e1e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 3 Aug 2022 16:12:15 +0200 Subject: [PATCH 11/19] list_builder: Drop support for the superold v0 / v1 catalog API --- list_builder.py | 55 ------------------------------------------------- 1 file changed, 55 deletions(-) diff --git a/list_builder.py b/list_builder.py index 4f6d114..b96148f 100755 --- a/list_builder.py +++ b/list_builder.py @@ -187,61 +187,6 @@ def build_catalog(): ) ) - #################### - # Legacy version 1 # - #################### - os.system("mkdir -p ./builds/default/v1/") - with open("./builds/default/v1/apps.json", "w") as f: - f.write(json.dumps(result_dict, sort_keys=True)) - - #################### - # Legacy version 0 # - #################### - official_apps = set( - [ - "agendav", - "ampache", - "baikal", - "dokuwiki", - "etherpad_mypads", - "hextris", - "jirafeau", - "kanboard", - "my_webapp", - "nextcloud", - "opensondage", - "phpmyadmin", - "piwigo", - "rainloop", - "roundcube", - "searx", - "shellinabox", - "strut", - "synapse", - "transmission", - "ttrss", - "wallabag2", - "wordpress", - "zerobin", - ] - ) - - official_apps_dict = {k: v for k, v in result_dict.items() if k in official_apps} - community_apps_dict = { - k: v for k, v in result_dict.items() if k not in official_apps - } - - # We need the official apps to have "validated" as state to be recognized as official - for app, infos in official_apps_dict.items(): - infos["state"] = "validated" - - os.system("mkdir -p ./builds/default/v0/") - with open("./builds/default/v0/official.json", "w") as f: - f.write(json.dumps(official_apps_dict, sort_keys=True)) - - with open("./builds/default/v0/community.json", "w") as f: - f.write(json.dumps(community_apps_dict, sort_keys=True)) - ############################## # Version for catalog in doc # ############################## From 954fcc4addfe646353ab89cade3cbca6a29640ca Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 3 Aug 2022 21:02:49 +0200 Subject: [PATCH 12/19] Yoloadd an app catalog v3 + tools to convert apps from v1 to v2 --- list_builder.py | 24 ++ tools/__init__.py | 0 tools/packaging_v2/__init__.py | 0 .../convert_app_to_packaging_v2.py | 321 ++++++++++++++++++ .../convert_v1_manifest_to_v2_for_catalog.py | 55 +++ 5 files changed, 400 insertions(+) create mode 100644 tools/__init__.py create mode 100644 tools/packaging_v2/__init__.py create mode 100644 tools/packaging_v2/convert_app_to_packaging_v2.py create mode 100644 tools/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py diff --git a/list_builder.py b/list_builder.py index b96148f..1242244 100755 --- a/list_builder.py +++ b/list_builder.py @@ -1,5 +1,6 @@ #!/usr/bin/python3 +import copy import sys import os import re @@ -8,6 +9,8 @@ import subprocess import yaml import time +from tools.packaging_v2.convert_v1_manifest_to_v2_for_catalog import convert_v1_manifest_to_v2_for_catalog + now = time.time() catalog = json.load(open("apps.json")) @@ -187,6 +190,27 @@ def build_catalog(): ) ) + ################################### + # Catalog API v3 with manifest v2 # + ################################### + + result_dict_with_manifest_v2 = copy.deepcopy(result_dict) + for app in result_dict_with_manifest_v2: + app["manifest"] = convert_v1_manifest_to_v2_for_catalog(app["manifest"]) + + os.system("mkdir -p ./builds/default/v3/") + with open("builds/default/v3/apps.json", "w") as f: + f.write( + json.dumps( + { + "apps": result_dict_with_manifest_v2, + "categories": categories, + "antifeatures": antifeatures, + }, + sort_keys=True, + ) + ) + ############################## # Version for catalog in doc # ############################## diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/packaging_v2/__init__.py b/tools/packaging_v2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/packaging_v2/convert_app_to_packaging_v2.py b/tools/packaging_v2/convert_app_to_packaging_v2.py new file mode 100644 index 0000000..fd66325 --- /dev/null +++ b/tools/packaging_v2/convert_app_to_packaging_v2.py @@ -0,0 +1,321 @@ +import argparse +import os +import re +import json +import subprocess + + +def check_output(cmd): + return ( + subprocess.check_output(cmd, shell=True) + .decode("utf-8") + .strip() + ) + + +def _convert_v1_manifest_to_v2(app_path): + + manifest = json.load(open(app_path + "/manifest.json")) + + if "upstream" not in manifest: + manifest["upstream"] = {} + + if "license" in manifest and "license" not in manifest["upstream"]: + manifest["upstream"]["license"] = manifest["license"] + + if "url" in manifest and "website" not in manifest["upstream"]: + manifest["upstream"]["website"] = manifest["url"] + + manifest["integration"] = { + "yunohost": manifest.get("requirements", {}).get("yunohost"), + "architectures": "all", + "multi_instance": manifest.get("multi_instance", False), + "ldap": "?", + "sso": "?", + "disk": "50M", + "ram.build": "50M", + "ram.runtime": "50M" + } + + maintainer = manifest.get("maintainer", {}).get("name") + manifest["maintainers"] = [maintainer] if maintainer else [] + + install_questions = manifest["arguments"]["install"] + manifest["install"] = {} + for question in install_questions: + name = question.pop("name") + if "ask" in question and name in ["domain", "path", "admin", "is_public", "password"]: + question.pop("ask") + if question.get("example") and question.get("type") in ["domain", "path", "user", "boolean", "password"]: + question.pop("example") + + manifest["install"][name] = question + + # Rename is_public to init_main_permission + manifest["install"] = {(k if k != "is_public" else "init_main_permission"): v for k, v in manifest["install"].items()} + + if "init_main_permission" in manifest["install"]: + manifest["install"]["init_main_permission"]["type"] = "group" + if manifest["install"]["init_main_permission"].get("default") is True: + manifest["install"]["init_main_permission"]["default"] = "visitors" + elif manifest["install"]["init_main_permission"].get("default") is True: + manifest["install"]["init_main_permission"]["default"] = "all_users" + + if "domain" in manifest["install"] and "path" not in manifest["install"]: + manifest["install"]["domain"]["full_domain"] = True + + manifest["resources"] = {} + manifest["resources"]["system_user"] = {} + manifest["resources"]["install_dir"] = {} + + if os.system(f"grep -q 'datadir=' {app_path}/scripts/install") == 0: + manifest["resources"]["data_dir"] = {} + + manifest["resources"]["permissions"] = {} + + if os.system(f"grep -q 'ynh_webpath_register' '{app_path}/scripts/install'") == 0: + manifest["resources"]["permissions"]["main.url"] = "/" + + # FIXME: Parse ynh_permission_create --permission="admin" --url="/wp-login.php" --additional_urls="/wp-admin.php" --allowed=$admin_wordpress + + ports = check_output(f"sed -nr 's/(\\w+)=.*ynh_find_port[^0-9]*([0-9]+)\\)/\\1,\\2/p' '{app_path}/scripts/install'") + if ports: + manifest["resources"]["ports"] = {} + for port in ports.split("\n"): + name, default = port.split(",") + exposed = check_output(f"sed -nr 's/.*yunohost firewall allow .*(TCP|UDP|Both).*${name}/\\1/p' '{app_path}/scripts/install'") + if exposed == "Both": + exposed = True + + name = name.replace("_port", "").replace("port_", "") + if name == "port": + name = "main" + + manifest["resources"]["ports"][f"{name}.default"] = int(default) + if exposed: + manifest["resources"]["ports"][f"{name}.exposed"] = exposed + + maybequote = "[\"'\"'\"']?" + apt_dependencies = check_output(f"sed -nr 's/.*_dependencies={maybequote}(.*){maybequote}? *$/\\1/p' '{app_path}/scripts/_common.sh' | tr -d '\"' | sed 's@ @\\n@g'") + php_version = check_output(f"sed -nr 's/^ *YNH_PHP_VERSION={maybequote}(.*){maybequote}?$/\\1/p' '{app_path}/scripts/_common.sh' | tr -d \"\\\"'\"") + if apt_dependencies.strip(): + if php_version: + apt_dependencies = apt_dependencies.replace("${YNH_PHP_VERSION}", php_version) + apt_dependencies = ', '.join([d for d in apt_dependencies.split("\n") if d]) + manifest["resources"]["apt"] = {"packages": apt_dependencies} + + extra_apt_repos = check_output(r"sed -nr 's/.*_extra_app_dependencies.*repo=\"(.*)\".*package=\"(.*)\".*key=\"(.*)\"/\1,\2,\3/p' %s/scripts/install" % app_path) + if extra_apt_repos: + for i, extra_apt_repo in enumerate(extra_apt_repos.split("\n")): + repo, packages, key = extra_apt_repo.split(",") + packages = packages.replace('$', '#FIXME#$') + if "apt" not in manifest["resources"]: + manifest["resources"]["apt"] = {} + if "extras" not in manifest["resources"]["apt"]: + manifest["resources"]["apt"]["extras"] = [] + manifest["resources"]["apt"]["extras"].append({ + "repo": repo, + "key": key, + "packages": packages, + }) + + if os.system(f"grep -q 'ynh_mysql_setup_db' {app_path}/scripts/install") == 0: + manifest["resources"]["database"] = {"type": "mysql"} + elif os.system(f"grep -q 'ynh_psql_setup_db' {app_path}/scripts/install") == 0: + manifest["resources"]["database"] = {"type": "postgresql"} + + keys_to_keep = ["packaging_format", "id", "name", "description", "version", "maintainers", "upstream", "integration", "install", "resources"] + + keys_to_del = [key for key in manifest.keys() if key not in keys_to_keep] + for key in keys_to_del: + del manifest[key] + + return manifest + + +def _dump_v2_manifest_as_toml(manifest): + + import re + from tomlkit import document, nl, table, dumps, comment + + toml_manifest = document() + toml_manifest.add("packaging_format", 2) + toml_manifest.add(nl()) + toml_manifest.add("id", manifest["id"]) + toml_manifest.add("name", manifest["name"]) + for lang, value in manifest["description"].items(): + toml_manifest.add(f"description.{lang}", value) + toml_manifest.add(nl()) + toml_manifest.add("version", manifest["version"]) + toml_manifest.add(nl()) + toml_manifest.add("maintainers", manifest["maintainers"]) + + upstream = table() + for key, value in manifest["upstream"].items(): + upstream[key] = value + toml_manifest["upstream"] = upstream + + integration = table() + for key, value in manifest["integration"].items(): + integration.add(key, value) + integration["architectures"].comment('FIXME: can be replaced by a list of supported archs using the dpkg --print-architecture nomenclature (amd64/i386/armhf/arm64/armel), for example: ["amd64", "i386"]') + integration["ldap"].comment('FIXME: replace with true, false, or "not_relevant"') + integration["sso"].comment('FIXME: replace with true, false, or "not_relevant"') + integration["disk"].comment('FIXME: replace with an **estimate** minimum disk requirement. e.g. 20M, 400M, 1G, ...') + integration["ram.build"].comment('FIXME: replace with an **estimate** minimum ram requirement. e.g. 50M, 400M, 1G, ...') + integration["ram.runtime"].comment('FIXME: replace with an **estimate** minimum ram requirement. e.g. 50M, 400M, 1G, ...') + toml_manifest["integration"] = integration + + install = table() + for key, value in manifest["install"].items(): + install[key] = table() + install[key].indent(4) + + if key in ["domain", "path", "admin", "is_public", "password"]: + install[key].add(comment("this is a generic question - ask strings are automatically handled by Yunohost's core")) + + for lang, value2 in value.get("ask", {}).items(): + install[key].add(f"ask.{lang}", value2) + + for lang, value2 in value.get("help", {}).items(): + install[key].add(f"help.{lang}", value2) + + for key2, value2 in value.items(): + if key2 in ["ask", "help"]: + continue + install[key].add(key2, value2) + + toml_manifest["install"] = install + + resources = table() + for key, value in manifest["resources"].items(): + resources[key] = table() + resources[key].indent(4) + for key2, value2 in value.items(): + resources[key].add(key2, value2) + if key == "apt" and key2 == "extras": + for extra in resources[key][key2]: + extra.indent(8) + + toml_manifest["resources"] = resources + + toml_manifest_dump = dumps(toml_manifest) + + regex = re.compile(r'\"((description|ask|help)\.[a-z]{2})\"') + toml_manifest_dump = regex.sub(r'\1', toml_manifest_dump) + toml_manifest_dump = toml_manifest_dump.replace('"ram.build"', "ram.build") + toml_manifest_dump = toml_manifest_dump.replace('"ram.runtime"', "ram.runtime") + toml_manifest_dump = toml_manifest_dump.replace('"main.url"', "main.url") + toml_manifest_dump = toml_manifest_dump.replace('"main.default"', "main.default") + return toml_manifest_dump + + +def cleanup_scripts_and_conf(folder): + + patterns_to_remove_in_scripts = [ + "^.*ynh_abort_if_errors.*$", + "^.*YNH_APP_ARG.*$", + "^.*YNH_APP_INSTANCE_NAME.*$", + r"^ *final_path=", + r"^\s*final_path=", + "^.*test .*-(e|d) .*final_path.*$", + "^.*ynh_webpath_register.*$", + "^.*ynh_webpath_available.*$", + "^.*ynh_system_user_create.*$", + "^.*ynh_system_user_delete.*$", + "^.*ynh_permission_update.*$", + "^.*ynh_permission_create.*$", + "^.*if .*ynh_permission_exists.*$", + "^.*if .*ynh_legacy_permissions_exists.*$", + "^.*ynh_legacy_permissions_delete_all.*$", + "^.*ynh_app_setting_set .*(domain|path|final_path|admin|password|port|datadir|db_name|db_user|db_pwd).*$", + "^.*ynh_app_setting_.* is_public.*$", + r"^.*if.*\$is_public.*$", + "^.*_dependencies=.*$", + "^.*ynh_install_app_dependencies.*$", + "^.*ynh_install_extra_app_dependencies.*$", + "^.*ynh_remove_app_dependencies.*$", + r"^.*\$\(ynh_app_setting_get.*$", + r"^.*ynh_secure_remove .*\$final_path.*$", + r"^.*ynh_secure_remove .*\$datadir.*$", + "^.*ynh_backup_before_upgrade.*$", + "^.*ynh_clean_setup.*$", + "^.*ynh_restore_upgradebackup.*$", + "^db_name=.*$", + "^db_user=.*$", + "^db_pwd=.*$", + "^datadir=.*$", + "^.*ynh_psql_test_if_first_run.*$", + "^.*ynh_mysql_setup_db.*$", + "^.*ynh_psql_setup_db.*$", + "^.*ynh_mysql_remove_db.*$", + "^.*ynh_psql_remove_db.*$", + "^.*ynh_find_port.*$", + "^.*ynh_script_progression.*Finding an available port", + "^.*ynh_script_progression.*Backing up the app before upgrading", + "^.*ynh_script_progression.*Creating data directory", + "^.*ynh_script_progression.*system user", + "^.*ynh_script_progression.*installation settings", + "^.*ynh_print_info.*installation settings", + r"^.*ynh_script_progression.*\w+ dependencies", + "^.*ynh_script_progression.*Removing app main dir", + "^.*ynh_script_progression.*Validating.*parameters", + "^.*ynh_script_progression.*SQL database", + "^.*ynh_script_progression.*Configuring permissions", + ] + patterns_to_remove_in_scripts = [re.compile(f"({p})", re.MULTILINE) for p in patterns_to_remove_in_scripts] + + replaces = [ + ("path_url", "path"), + ("PATH_URL", "PATH"), + ("final_path", "install_dir"), + ("FINALPATH", "INSTALL_DIR"), + ("datadir", "data_dir"), + ("DATADIR", "DATA_DIR"), + ] + + for s in ["_common.sh", "install", "remove", "upgrade", "backup", "restore"]: + + script = f"{folder}/scripts/{s}" + + content = open(script).read() + + for pattern in patterns_to_remove_in_scripts: + content = pattern.sub(r"#REMOVEME? \1", content) + + for pattern, replace in replaces: + content = content.replace(pattern, replace) + + open(script, "w").write(content) + + for conf in os.listdir(f"{folder}/conf"): + + conf = f"{folder}/conf/{conf}" + + if not os.path.isfile(conf): + continue + + content = open(conf).read() + content_init = content + + for pattern, replace in replaces: + content = content.replace(pattern, replace) + + if content_init != content: + open(conf, "w").write(content) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Attempt to automatically convert a v1 YunoHost app to v2 (at least as much as possible) : parse the app scripts to auto-generate the manifest.toml, and remove now-useless lines from the app scripts" + ) + parser.add_argument( + "app_path", help="Path to the app to convert" + ) + + args = parser.parse_args() + + manifest = _convert_v1_manifest_to_v2(args.app_path) + open(args.app_path + "/manifest.toml", "w").write(_dump_v2_manifest_as_toml(manifest)) + + cleanup_scripts_and_conf(args.app_path) diff --git a/tools/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py b/tools/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py new file mode 100644 index 0000000..29f24ee --- /dev/null +++ b/tools/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py @@ -0,0 +1,55 @@ +import copy + + +def convert_v1_manifest_to_v2_for_catalog(manifest): + + manifest = copy.deepcopy(manifest) + + if "upstream" not in manifest: + manifest["upstream"] = {} + + if "license" in manifest and "license" not in manifest["upstream"]: + manifest["upstream"]["license"] = manifest["license"] + + if "url" in manifest and "website" not in manifest["upstream"]: + manifest["upstream"]["website"] = manifest["url"] + + manifest["integration"] = { + "yunohost": manifest.get("requirements", {}).get("yunohost", "").replace(">", "").replace("=", "").replace(" ", ""), + "architectures": "all", + "multi_instance": manifest.get("multi_instance", False), + "ldap": "?", + "sso": "?", + "disk": "50M", + "ram": {"build": "50M", "runtime": "10M"} + } + + maintainer = manifest.get("maintainer", {}).get("name") + manifest["maintainers"] = [maintainer] if maintainer else [] + + install_questions = manifest["arguments"]["install"] + + manifest["install"] = {} + for question in install_questions: + name = question.pop("name") + if "ask" in question and name in ["domain", "path", "admin", "is_public", "password"]: + question.pop("ask") + if question.get("example") and question.get("type") in ["domain", "path", "user", "boolean", "password"]: + question.pop("example") + + manifest["install"][name] = question + + manifest["resources"] = { + "system_user": {}, + "install_dir": { + "alias": "final_path" + } + } + + keys_to_keep = ["packaging_format", "id", "name", "description", "version", "maintainers", "upstream", "integration", "install", "resources"] + + keys_to_del = [key for key in manifest.keys() if key not in keys_to_keep] + for key in keys_to_del: + del manifest[key] + + return manifest From 8dabe03c00cfc6b33475b7fba3f9dc4772247e8b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 3 Aug 2022 21:31:17 +0200 Subject: [PATCH 13/19] Fix my previous commit /o\ --- list_builder.py | 2 +- tools/packaging_v2/convert_app_to_packaging_v2.py | 9 +++++++-- .../convert_v1_manifest_to_v2_for_catalog.py | 9 +++++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/list_builder.py b/list_builder.py index 1242244..2ebd800 100755 --- a/list_builder.py +++ b/list_builder.py @@ -195,7 +195,7 @@ def build_catalog(): ################################### result_dict_with_manifest_v2 = copy.deepcopy(result_dict) - for app in result_dict_with_manifest_v2: + for app in result_dict_with_manifest_v2.values(): app["manifest"] = convert_v1_manifest_to_v2_for_catalog(app["manifest"]) os.system("mkdir -p ./builds/default/v3/") diff --git a/tools/packaging_v2/convert_app_to_packaging_v2.py b/tools/packaging_v2/convert_app_to_packaging_v2.py index fd66325..21fe774 100644 --- a/tools/packaging_v2/convert_app_to_packaging_v2.py +++ b/tools/packaging_v2/convert_app_to_packaging_v2.py @@ -37,8 +37,13 @@ def _convert_v1_manifest_to_v2(app_path): "ram.runtime": "50M" } - maintainer = manifest.get("maintainer", {}).get("name") - manifest["maintainers"] = [maintainer] if maintainer else [] + maintainers = manifest.get("maintainer", {}) + if isinstance(maintainers, list): + maintainers = [m['name'] for m in maintainers] + else: + maintainers = [maintainers["name"]] if maintainers.get("name") else [] + + manifest["maintainers"] = maintainers install_questions = manifest["arguments"]["install"] manifest["install"] = {} diff --git a/tools/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py b/tools/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py index 29f24ee..0130c29 100644 --- a/tools/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py +++ b/tools/packaging_v2/convert_v1_manifest_to_v2_for_catalog.py @@ -24,8 +24,13 @@ def convert_v1_manifest_to_v2_for_catalog(manifest): "ram": {"build": "50M", "runtime": "10M"} } - maintainer = manifest.get("maintainer", {}).get("name") - manifest["maintainers"] = [maintainer] if maintainer else [] + maintainers = manifest.get("maintainer", {}) + if isinstance(maintainers, list): + maintainers = [m['name'] for m in maintainers] + else: + maintainers = [maintainers["name"]] if maintainers.get("name") else [] + + manifest["maintainers"] = maintainers install_questions = manifest["arguments"]["install"] From 22dff664703f00c685a207945d377b6d0f1f8b6f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 Aug 2022 11:16:26 +0200 Subject: [PATCH 14/19] list_builder: drop the old i18n stuff ... this didnt get any update since 2.5 years and doesn't contain that many relevant translation anymore ... and it's gonna be a mess to adapt the code to cover both v1 and v2 manifest format ... --- list_builder.py | 54 +------------------------------------------------ 1 file changed, 1 insertion(+), 53 deletions(-) diff --git a/list_builder.py b/list_builder.py index 2ebd800..6c6b1f6 100755 --- a/list_builder.py +++ b/list_builder.py @@ -299,7 +299,7 @@ def build_app_dict(app, infos): "url": infos["url"], }, "lastUpdate": timestamp, - "manifest": include_translations_in_manifest(manifest), + "manifest": manifest, "state": infos["state"], "level": infos.get("level", "?"), "maintained": infos.get("maintained", True), @@ -313,58 +313,6 @@ def build_app_dict(app, infos): } -def include_translations_in_manifest(manifest): - - app_name = manifest["id"] - - for locale in os.listdir("locales"): - if not locale.endswith("json"): - continue - - if locale == "en.json": - continue - - current_lang = locale.split(".")[0] - translations = json.load(open(os.path.join("locales", locale), "r")) - - # don't overwrite already existing translation in manifests for now - key = "%s_manifest_description" % app_name - if current_lang not in manifest["description"] and translations.get(key): - manifest["description"][current_lang] = translations[key] - - for category, questions in manifest["arguments"].items(): - for question in questions: - key = "%s_manifest_arguments_%s_%s" % ( - app_name, - category, - question["name"], - ) - # don't overwrite already existing translation in manifests for now - if ( - translations.get(key) - and "ask" in question - and current_lang not in question["ask"] - ): - question["ask"][current_lang] = translations[key] - - key = "%s_manifest_arguments_%s_help_%s" % ( - app_name, - category, - question["name"], - ) - # don't overwrite already existing translation in manifests for now - if ( - translations.get(key) - and "help" in question - and current_lang not in question.get("help", []) - ): - question["help"][current_lang] = translations[key] - - return manifest - - -###################### - if __name__ == "__main__": refresh_all_caches() build_catalog() From 56046586071cab928e1104d96c1199bdd35f8daf Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 Aug 2022 11:17:46 +0200 Subject: [PATCH 15/19] move autopatches/ inside tools/ --- tools/autopatches/autopatch.py | 188 ++++++++++++++++++ .../patches/issue-and-pr-template/patch.sh | 14 ++ .../patches/issue-and-pr-template/pr_body.md | 2 + .../patches/issue-and-pr-template/pr_title.md | 1 + .../missing-seteu-in-change_url/patch.sh | 10 + .../missing-seteu-in-change_url/pr_body.md | 2 + .../missing-seteu-in-change_url/pr_title.md | 1 + .../patches/new-permission-system/patch.sh | 66 ++++++ .../patches/new-permission-system/pr_body.md | 11 + .../patches/new-permission-system/pr_title.md | 1 + 10 files changed, 296 insertions(+) create mode 100755 tools/autopatches/autopatch.py create mode 100644 tools/autopatches/patches/issue-and-pr-template/patch.sh create mode 100644 tools/autopatches/patches/issue-and-pr-template/pr_body.md create mode 100644 tools/autopatches/patches/issue-and-pr-template/pr_title.md create mode 100644 tools/autopatches/patches/missing-seteu-in-change_url/patch.sh create mode 100644 tools/autopatches/patches/missing-seteu-in-change_url/pr_body.md create mode 100644 tools/autopatches/patches/missing-seteu-in-change_url/pr_title.md create mode 100644 tools/autopatches/patches/new-permission-system/patch.sh create mode 100644 tools/autopatches/patches/new-permission-system/pr_body.md create mode 100644 tools/autopatches/patches/new-permission-system/pr_title.md diff --git a/tools/autopatches/autopatch.py b/tools/autopatches/autopatch.py new file mode 100755 index 0000000..746ee51 --- /dev/null +++ b/tools/autopatches/autopatch.py @@ -0,0 +1,188 @@ +#!/usr/bin/python3 +import json +import sys +import requests +import os +import subprocess + +catalog = requests.get("https://raw.githubusercontent.com/YunoHost/apps/master/apps.json").json() + +my_env = os.environ.copy() +my_env["GIT_TERMINAL_PROMPT"] = "0" +os.makedirs(".apps_cache", exist_ok=True) + +login = open("login").read().strip() +token = open("token").read().strip() +github_api = "https://api.github.com" + + +def apps(min_level=4): + + for app, infos in catalog.items(): + if infos.get("state") == "working" and infos.get("level", -1) > min_level: + infos["id"] = app + yield infos + + +def app_cache_folder(app): + return os.path.join(".apps_cache", app) + + +def git(cmd, in_folder=None): + + if not isinstance(cmd, list): + cmd = cmd.split() + if in_folder: + cmd = ["-C", in_folder] + cmd + cmd = ["git"] + cmd + return subprocess.check_output(cmd, env=my_env).strip().decode("utf-8") + + +# Progress bar helper, stolen from https://stackoverflow.com/a/34482761 +def progressbar(it, prefix="", size=60, file=sys.stdout): + it = list(it) + count = len(it) + def show(j, name=""): + name += " " + x = int(size*j/count) + file.write("%s[%s%s] %i/%i %s\r" % (prefix, "#"*x, "."*(size-x), j, count, name)) + file.flush() + show(0) + for i, item in enumerate(it): + yield item + show(i+1, item["id"]) + file.write("\n") + file.flush() + + +def build_cache(): + + for app in progressbar(apps(), "Git cloning: ", 40): + folder = os.path.join(".apps_cache", app["id"]) + reponame = app["url"].rsplit("/", 1)[-1] + git(f"clone --quiet --depth 1 --single-branch {app['url']} {folder}") + git(f"remote add fork https://{login}:{token}@github.com/{login}/{reponame}", in_folder=folder) + + +def apply(patch): + + patch_path = os.path.abspath(os.path.join("patches", patch, "patch.sh")) + + for app in progressbar(apps(), "Apply to: ", 40): + folder = os.path.join(".apps_cache", app["id"]) + current_branch = git(f"symbolic-ref --short HEAD", in_folder=folder) + git(f"reset --hard origin/{current_branch}", in_folder=folder) + os.system(f"cd {folder} && bash {patch_path}") + + +def diff(): + + for app in apps(): + folder = os.path.join(".apps_cache", app["id"]) + if bool(subprocess.check_output(f"cd {folder} && git diff", shell=True).strip().decode("utf-8")): + print("\n\n\n") + print("=================================") + print("Changes in : " + app["id"]) + print("=================================") + print("\n") + os.system(f"cd {folder} && git --no-pager diff") + + +def push(patch): + + title = "[autopatch] " + open(os.path.join("patches", patch, "pr_title.md")).read().strip() + + def diff_not_empty(app): + folder = os.path.join(".apps_cache", app["id"]) + return bool(subprocess.check_output(f"cd {folder} && git diff", shell=True).strip().decode("utf-8")) + + def app_is_on_github(app): + return "github.com" in app["url"] + + apps_to_push = [app for app in apps() if diff_not_empty(app) and app_is_on_github(app)] + + with requests.Session() as s: + s.headers.update({"Authorization": f"token {token}"}) + for app in progressbar(apps_to_push, "Forking: ", 40): + app["repo"] = app["url"][len("https://github.com/"):].strip("/") + fork_if_needed(app["repo"], s) + + for app in progressbar(apps_to_push, "Pushing: ", 40): + app["repo"] = app["url"][len("https://github.com/"):].strip("/") + app_repo_name = app["url"].rsplit("/", 1)[-1] + folder = os.path.join(".apps_cache", app["id"]) + current_branch = git(f"symbolic-ref --short HEAD", in_folder=folder) + git(f"reset origin/{current_branch}", in_folder=folder) + git(["commit", "-a", "-m", title, "--author='Yunohost-Bot <>'"], in_folder=folder) + try: + git(f"remote remove fork", in_folder=folder) + except Exception: + pass + git(f"remote add fork https://{login}:{token}@github.com/{login}/{app_repo_name}", in_folder=folder) + git(f"push fork {current_branch}:{patch} --quiet --force", in_folder=folder) + create_pull_request(app["repo"], patch, current_branch, s) + + +def fork_if_needed(repo, s): + + repo_name = repo.split("/")[-1] + r = s.get(github_api + f"/repos/{login}/{repo_name}") + + if r.status_code == 200: + return + + r = s.post(github_api + f"/repos/{repo}/forks") + + if r.status_code != 200: + print(r.text) + + +def create_pull_request(repo, patch, base_branch, s): + + PR = {"title": "[autopatch] " + open(os.path.join("patches", patch, "pr_title.md")).read().strip(), + "body": "This is an automatic PR\n\n" + open(os.path.join("patches", patch, "pr_body.md")).read().strip(), + "head": login + ":" + patch, + "base": base_branch, + "maintainer_can_modify": True} + + r = s.post(github_api + f"/repos/{repo}/pulls", json.dumps(PR)) + + if r.status_code != 200: + print(r.text) + else: + json.loads(r.text)["html_url"] + + +def main(): + + action = sys.argv[1] + if action == "--help": + print(""" + Example usage: + +# Init local git clone for all apps +./autopatch.py --build-cache + +# Apply patch in all local clones +./autopatch.py --apply explicit-php-version-in-deps + +# Inspect diff for all apps +./autopatch.py --diff + +# Push and create pull requests on all apps with non-empty diff +./autopatch.py --push explicit-php-version-in-deps +""") + + elif action == "--build-cache": + build_cache() + elif action == "--apply": + apply(sys.argv[2]) + elif action == "--diff": + diff() + elif action == "--push": + push(sys.argv[2]) + else: + print("Unknown action %s" % action) + + +main() diff --git a/tools/autopatches/patches/issue-and-pr-template/patch.sh b/tools/autopatches/patches/issue-and-pr-template/patch.sh new file mode 100644 index 0000000..d52d3b0 --- /dev/null +++ b/tools/autopatches/patches/issue-and-pr-template/patch.sh @@ -0,0 +1,14 @@ + +[ ! -e issue_template.md ] || git rm issue_template.md +[ ! -e pull_request_template.md ] || git rm pull_request_template.md + +[ ! -e .github ] || git rm -rf .github +mkdir -p .github + +# Sleep 1 to avoid too many requests on github (there's a rate limit anyway) +sleep 1 + +wget -O .github/ISSUE_TEMPLATE.md https://raw.githubusercontent.com/YunoHost/example_ynh/master/.github/ISSUE_TEMPLATE.md +wget -O .github/PULL_REQUEST_TEMPLATE.md https://raw.githubusercontent.com/YunoHost/example_ynh/master/.github/PULL_REQUEST_TEMPLATE.md + +git add .github diff --git a/tools/autopatches/patches/issue-and-pr-template/pr_body.md b/tools/autopatches/patches/issue-and-pr-template/pr_body.md new file mode 100644 index 0000000..471273e --- /dev/null +++ b/tools/autopatches/patches/issue-and-pr-template/pr_body.md @@ -0,0 +1,2 @@ + +This is an ***automated*** patch to sync the issue and PR template with the official templates in https://github.com/YunoHost/example_ynh/tree/master/.github diff --git a/tools/autopatches/patches/issue-and-pr-template/pr_title.md b/tools/autopatches/patches/issue-and-pr-template/pr_title.md new file mode 100644 index 0000000..f0bed1c --- /dev/null +++ b/tools/autopatches/patches/issue-and-pr-template/pr_title.md @@ -0,0 +1 @@ +Update issue and PR templates diff --git a/tools/autopatches/patches/missing-seteu-in-change_url/patch.sh b/tools/autopatches/patches/missing-seteu-in-change_url/patch.sh new file mode 100644 index 0000000..f930134 --- /dev/null +++ b/tools/autopatches/patches/missing-seteu-in-change_url/patch.sh @@ -0,0 +1,10 @@ + +cd scripts/ + +if [ ! -e change_url ] || grep -q 'ynh_abort_if_errors' change_url +then + # The app doesn't has any change url script or already has ynh_abort_if_error + exit 0 +fi + +sed 's@\(source /usr/share/yunohost/helpers\)@\1\nynh_abort_if_errors@g' -i change_url diff --git a/tools/autopatches/patches/missing-seteu-in-change_url/pr_body.md b/tools/autopatches/patches/missing-seteu-in-change_url/pr_body.md new file mode 100644 index 0000000..bb67bd0 --- /dev/null +++ b/tools/autopatches/patches/missing-seteu-in-change_url/pr_body.md @@ -0,0 +1,2 @@ + +This is an ***automated*** patch to fix the lack of `ynh_abort_if_errors` in change_url script diff --git a/tools/autopatches/patches/missing-seteu-in-change_url/pr_title.md b/tools/autopatches/patches/missing-seteu-in-change_url/pr_title.md new file mode 100644 index 0000000..efd0e73 --- /dev/null +++ b/tools/autopatches/patches/missing-seteu-in-change_url/pr_title.md @@ -0,0 +1 @@ +Missing ynh_abort_if_errors in change_url scripts diff --git a/tools/autopatches/patches/new-permission-system/patch.sh b/tools/autopatches/patches/new-permission-system/patch.sh new file mode 100644 index 0000000..f372a44 --- /dev/null +++ b/tools/autopatches/patches/new-permission-system/patch.sh @@ -0,0 +1,66 @@ + +cd scripts/ + +if grep -q 'ynh_legacy_permissions' upgrade || grep -q 'ynh_permission_' install +then + # App already using the new permission system - not patching anything + exit 0 +fi + +if ! grep -q "protected_\|skipped_" install +then + # App doesn't has any (un)protected / skipped setting ? + # Probably not a webapp or permission ain't relevant for it ? + exit 0 +fi + +CONFIGURE_PERMISSION_DURING_INSTALL=' +# Make app public if necessary +if [ \"\$is_public\" -eq 1 ] +then + ynh_permission_update --permission=\"main\" --add=\"visitors\" +fi +' + +MIGRATE_LEGACY_PERMISSIONS=' +#================================================= +# Migrate legacy permissions to new system +#================================================= +if ynh_legacy_permissions_exists +then + ynh_legacy_permissions_delete_all + + ynh_app_setting_delete --app=\$app --key=is_public +fi' + +for SCRIPT in "remove upgrade backup restore change_url" +do + [[ -e $SCRIPT ]] || continue + + perl -p0e 's@.*ynh_app_setting_.*protected_.*@@g' -i $SCRIPT + perl -p0e 's@.*ynh_app_setting_.*skipped_.*@@g' -i $SCRIPT + perl -p0e 's@\s*if.*-z.*is_public.*(.|\n)*?fi\s@\n@g' -i $SCRIPT + perl -p0e 's@\s*if.*is_public.*(-eq|=).*(.|\n)*?fi\s@\n@g' -i $SCRIPT + perl -p0e 's@is_public=.*\n@@g' -i $SCRIPT + perl -p0e 's@ynh_app_setting_.*is_public.*@@g' -i $SCRIPT + perl -p0e 's@.*# Make app .*@@g' -i $SCRIPT + perl -p0e 's@.*# Fix is_public as a boolean.*@@g' -i $SCRIPT + perl -p0e 's@.*# If app is public.*@@g' -i $SCRIPT + perl -p0e 's@.*# .*allow.*credentials.*anyway.*@@g' -i $SCRIPT + perl -p0e 's@.*ynh_script_progression.*SSOwat.*@@g' -i $SCRIPT + perl -p0e 's@#=*\s#.*SETUP SSOWAT.*\s#=*\s@@g' -i $SCRIPT +done + + +perl -p0e 's@.*domain_regex.*@@g' -i install +perl -p0e 's@.*# If app is public.*@@g' -i install +perl -p0e 's@.*# Make app .*@@g' -i install +perl -p0e 's@.*# .*allow.*credentials.*anyway.*@@g' -i install +perl -p0e "s@if.*is_public.*(-eq|=)(.|\n){0,100}setting(.|\n)*?fi\n@$CONFIGURE_PERMISSION_DURING_INSTALL@g" -i install +perl -p0e 's@.*ynh_app_setting_.*is_public.*\s@@g' -i install +perl -p0e 's@.*ynh_app_setting_.*protected_.*@@g' -i install +perl -p0e 's@.*ynh_app_setting_.*skipped_.*@@g' -i install + +grep -q 'is_public=' install || perl -p0e 's@(.*Configuring SSOwat.*)@\1\nynh_permission_update --permission=\"main\" --add=\"visitors\"@g' -i install + +perl -p0e "s@ynh_abort_if_errors@ynh_abort_if_errors\n$MIGRATE_LEGACY_PERMISSIONS@g" -i upgrade diff --git a/tools/autopatches/patches/new-permission-system/pr_body.md b/tools/autopatches/patches/new-permission-system/pr_body.md new file mode 100644 index 0000000..07f4e85 --- /dev/null +++ b/tools/autopatches/patches/new-permission-system/pr_body.md @@ -0,0 +1,11 @@ + +NB. : this is an ***automated*** attempt to migrate the app to the new permission system + +You should ***not*** blindly trust the proposed changes. In particular, the auto-patch will not handle: +- situations which are more complex than "if is_public is true, allow visitors" +- situations where the app needs to be temporarily public (then possible private) during initial configuration +- apps that need to define extra permission for specific section of the app (such as admin interface) +- apps using non-standard syntax +- other specific use cases + +***PLEASE*** carefully review, test and amend the proposed changes if you find that the autopatch did not do a proper job. diff --git a/tools/autopatches/patches/new-permission-system/pr_title.md b/tools/autopatches/patches/new-permission-system/pr_title.md new file mode 100644 index 0000000..0f9554c --- /dev/null +++ b/tools/autopatches/patches/new-permission-system/pr_title.md @@ -0,0 +1 @@ +Autopatch to migrate to new permission system From 81408ec1cad12078728869be3a7f139a937405cb Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 Aug 2022 16:29:26 +0200 Subject: [PATCH 16/19] Moar preparations for packaging v2 apps in catalog --- list_builder.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/list_builder.py b/list_builder.py index 6c6b1f6..6707370 100755 --- a/list_builder.py +++ b/list_builder.py @@ -172,9 +172,13 @@ def build_catalog(): result_dict[app_dict["id"]] = app_dict - ##################### - # Current version 2 # - ##################### + ############################# + # Current catalog API v2 # + ############################# + + result_dict_with_manifest_v1 = copy.deepcopy(result_dict) + result_dict_with_manifest_v1 = {name: infos for name, infos in result_dict_with_manifest_v1.items() if float(str(infos["manifest"].get("packaging_format", "")).strip() or "0") < 2} + categories = yaml.load(open("categories.yml").read()) antifeatures = yaml.load(open("antifeatures.yml").read()) os.system("mkdir -p ./builds/default/v2/") @@ -182,7 +186,7 @@ def build_catalog(): f.write( json.dumps( { - "apps": result_dict, + "apps": result_dict_with_manifest_v1, "categories": categories, "antifeatures": antifeatures, }, @@ -190,13 +194,15 @@ def build_catalog(): ) ) - ################################### - # Catalog API v3 with manifest v2 # - ################################### + ############################################# + # Catalog catalog API v3 (with manifest v2) # + ############################################# result_dict_with_manifest_v2 = copy.deepcopy(result_dict) for app in result_dict_with_manifest_v2.values(): - app["manifest"] = convert_v1_manifest_to_v2_for_catalog(app["manifest"]) + packaging_format = float(str(app["manifest"].get("packaging_format", "")).strip() or "0") + if packaging_format < 2: + app["manifest"] = convert_v1_manifest_to_v2_for_catalog(app["manifest"]) os.system("mkdir -p ./builds/default/v3/") with open("builds/default/v3/apps.json", "w") as f: @@ -238,7 +244,7 @@ def build_catalog(): result_dict_doc = { k: infos_for_doc_catalog(v) for k, v in result_dict.items() - if v["state"] in ["working", "validated"] + if v["state"] == "working" } with open("builds/default/doc_catalog/apps.json", "w") as f: f.write( @@ -261,6 +267,7 @@ def build_app_dict(app, infos): if infos["revision"] == "HEAD": relevant_files = [ "manifest.json", + "manifest.toml", "config_panel.toml", "hooks/", "scripts/", From 582e33e9d9da43ad52e629f6d4156fe6fe5108d9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 Aug 2022 17:05:46 +0200 Subject: [PATCH 17/19] Fix v2 packaging convert script (_common.sh may not exist) --- tools/packaging_v2/convert_app_to_packaging_v2.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/packaging_v2/convert_app_to_packaging_v2.py b/tools/packaging_v2/convert_app_to_packaging_v2.py index 21fe774..b2a11c3 100644 --- a/tools/packaging_v2/convert_app_to_packaging_v2.py +++ b/tools/packaging_v2/convert_app_to_packaging_v2.py @@ -101,8 +101,8 @@ def _convert_v1_manifest_to_v2(app_path): manifest["resources"]["ports"][f"{name}.exposed"] = exposed maybequote = "[\"'\"'\"']?" - apt_dependencies = check_output(f"sed -nr 's/.*_dependencies={maybequote}(.*){maybequote}? *$/\\1/p' '{app_path}/scripts/_common.sh' | tr -d '\"' | sed 's@ @\\n@g'") - php_version = check_output(f"sed -nr 's/^ *YNH_PHP_VERSION={maybequote}(.*){maybequote}?$/\\1/p' '{app_path}/scripts/_common.sh' | tr -d \"\\\"'\"") + apt_dependencies = check_output(f"sed -nr 's/.*_dependencies={maybequote}(.*){maybequote}? *$/\\1/p' '{app_path}/scripts/_common.sh' 2>/dev/null | tr -d '\"' | sed 's@ @\\n@g'") + php_version = check_output(f"sed -nr 's/^ *YNH_PHP_VERSION={maybequote}(.*){maybequote}?$/\\1/p' '{app_path}/scripts/_common.sh' 2>/dev/null | tr -d \"\\\"'\"") if apt_dependencies.strip(): if php_version: apt_dependencies = apt_dependencies.replace("${YNH_PHP_VERSION}", php_version) @@ -283,6 +283,9 @@ def cleanup_scripts_and_conf(folder): script = f"{folder}/scripts/{s}" + if not os.path.exists(script): + continue + content = open(script).read() for pattern in patterns_to_remove_in_scripts: From 08c3f20b5f0e912846cc6379aecf875c3f7f24a1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 Aug 2022 17:14:42 +0200 Subject: [PATCH 18/19] list_builder: gotta load manifest.toml, not json, for app v2 --- list_builder.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/list_builder.py b/list_builder.py index 6707370..2a4288b 100755 --- a/list_builder.py +++ b/list_builder.py @@ -5,10 +5,12 @@ import sys import os import re import json +import toml import subprocess import yaml import time +from collections import OrderedDict from tools.packaging_v2.convert_v1_manifest_to_v2_for_catalog import convert_v1_manifest_to_v2_for_catalog now = time.time() @@ -297,7 +299,11 @@ def build_app_dict(app, infos): timestamp = int(timestamp) # Build the dict with all the infos - manifest = json.load(open(this_app_cache + "/manifest.json")) + if os.path.exists(open(this_app_cache + "/manifest.toml")): + manifest = toml.load(open(this_app_cache + "/manifest.toml", _dict=OrderedDict)) + else: + manifest = json.load(open(this_app_cache + "/manifest.json")) + return { "id": manifest["id"], "git": { From 4f08ac96135a169d1a85aa16e1b785bb7abe9536 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 Aug 2022 17:19:23 +0200 Subject: [PATCH 19/19] Oopsies I commited nonsense --- list_builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/list_builder.py b/list_builder.py index 2a4288b..e41cdc1 100755 --- a/list_builder.py +++ b/list_builder.py @@ -299,8 +299,8 @@ def build_app_dict(app, infos): timestamp = int(timestamp) # Build the dict with all the infos - if os.path.exists(open(this_app_cache + "/manifest.toml")): - manifest = toml.load(open(this_app_cache + "/manifest.toml", _dict=OrderedDict)) + if os.path.exists(this_app_cache + "/manifest.toml"): + manifest = toml.load(open(this_app_cache + "/manifest.toml"), _dict=OrderedDict) else: manifest = json.load(open(this_app_cache + "/manifest.json"))