Merge branch 'master' into support_gitlab_autoupgrade

README.md

@@ -67,3 +67,8 @@ App packagers should *not* manually set their apps' level. The levels of all the
 Applications with no recent activity and no active sign from maintainer may be flagged in `apps.toml` with the `package-not-maintained` antifeature tag to signify that the app is inactive and may slowly become outdated with respect to the upstream, or with respect to good packaging practices. It does **not** mean that the app is not working anymore.
 
 Feel free to contact the app group if you feel like taking over the maintenance of a currently unmaintained app!
+
+### `graveyard.toml`
+
+This file is for apps that are long-term not-working and unlikely to be ever revived
+

store/app.py

@@ -31,6 +31,8 @@ from utils import (
     get_wishlist,
     get_stars,
     get_app_md_and_screenshots,
+    save_wishlist_submit_for_ratelimit,
+    check_wishlist_submit_ratelimit,
 )
 
 app = Flask(__name__, static_url_path="/assets", static_folder="assets")
@@ -147,7 +149,7 @@ def star_app(app_id, action):
     if app_id not in get_catalog()["apps"] and app_id not in get_wishlist():
         return _("App %(app_id) not found", app_id=app_id), 404
     if not session.get("user", {}):
-        return _("You must be logged in to be able to star an app"), 401
+        return _("You must be logged in to be able to star an app") + "<br/><br/>" + _("Note that, due to various abuses, we restricted login on the app store to 'trust level 1' users.<br/><br/>'Trust level 1' is obtained after interacting a minimum with the forum, and more specifically: entering at least 5 topics, reading at least 30 posts, and spending at least 10 minutes reading posts."), 401
 
     app_star_folder = os.path.join(".stars", app_id)
     app_star_for_this_user = os.path.join(
@@ -190,7 +192,7 @@ def add_to_wishlist():
     if request.method == "POST":
         user = session.get("user", {})
         if not user:
-            errormsg = _("You must be logged in to submit an app to the wishlist")
+            errormsg = _("You must be logged in to submit an app to the wishlist") + "<br/><br/>" + _("Note that, due to various abuses, we restricted login on the app store to 'trust level 1' users.<br/><br/>'Trust level 1' is obtained after interacting a minimum with the forum, and more specifically: entering at least 5 topics, reading at least 30 posts, and spending at least 10 minutes reading posts.")
             return render_template(
                 "wishlist_add.html",
                 locale=get_locale(),
@@ -199,7 +201,6 @@ def add_to_wishlist():
                 successmsg=None,
                 errormsg=errormsg,
             )
-
         csrf_token = request.form["csrf_token"]
 
         if csrf_token != session.get("csrf_token"):
@@ -217,8 +218,15 @@ def add_to_wishlist():
         description = request.form["description"].strip().replace("\n", "")
         upstream = request.form["upstream"].strip().replace("\n", "")
         website = request.form["website"].strip().replace("\n", "")
+        license = request.form["license"].strip().replace("\n", "")
+
+        boring_keywords_to_check_for_people_not_reading_the_instructions = ["free", "open source", "open-source", "self-hosted", "simple", "lightweight", "light-weight", "best", "most", "fast", "flexible", "puissante", "powerful", "secure"]
 
         checks = [
+            (
+                check_wishlist_submit_ratelimit(session['user']['username']) is True,
+                _("Proposing wishlist additions is limited to once every 15 days per user.")
+            ),
             (len(name) >= 3, _("App name should be at least 3 characters")),
             (len(name) <= 30, _("App name should be less than 30 characters")),
             (
@@ -237,11 +245,27 @@ def add_to_wishlist():
                 len(upstream) <= 150,
                 _("Upstream code repo URL should be less than 150 characters"),
             ),
+            (
+                len(license) >= 10,
+                _("License URL should be at least 10 characters"),
+            ),
+            (
+                len(license) <= 250,
+                _("License URL should be less than 250 characters"),
+            ),
             (len(website) <= 150, _("Website URL should be less than 150 characters")),
             (
                 re.match(r"^[\w\.\-\(\)\ ]+$", name),
                 _("App name contains special characters"),
             ),
+            (
+                all(keyword not in description.lower() for keyword in boring_keywords_to_check_for_people_not_reading_the_instructions),
+                _("Please focus on what the app does, without using marketing, fuzzy terms, or repeating that the app is 'free' and 'self-hostable'.")
+            ),
+            (
+                description.lower().split()[0] != name and (len(description.split()) == 1 or description.lower().split()[1] not in ["is", "est"]),
+                _("No need to repeat '{app} is'. Focus on what the app does.")
+            )
         ]
 
         for check, errormsg in checks:
@@ -328,6 +352,7 @@ Proposed by **{session['user']['username']}**
 
 Website: {website}
 Upstream repo: {upstream}
+License: {license}
 Description: {description}
 
 - [ ] Confirm app is self-hostable and generally makes sense to possibly integrate in YunoHost
@@ -349,6 +374,9 @@ Description: {description}
             "Your proposed app has succesfully been submitted. It must now be validated by the YunoHost team. You can track progress here: <a href='%(url)s'>%(url)s</a>",
             url=url,
         )
+
+        save_wishlist_submit_for_ratelimit(session['user']['username'])
+
         return render_template(
             "wishlist_add.html",
             locale=get_locale(),
@@ -414,6 +442,9 @@ def sso_login_callback():
 
     uri_to_redirect_to_after_login = session.get("uri_to_redirect_to_after_login")
 
+    if "trust_level_1" not in user_data['groups'][0].split(','):
+        return _("Unfortunately, login was denied.") + "<br/><br/>" + _("Note that, due to various abuses, we restricted login on the app store to 'trust level 1' users.<br/><br/>'Trust level 1' is obtained after interacting a minimum with the forum, and more specifically: entering at least 5 topics, reading at least 30 posts, and spending at least 10 minutes reading posts."), 403
+
     session.clear()
     session["user"] = {
         "id": user_data["external_id"][0],

store/templates/wishlist_add.html

@@ -27,20 +27,24 @@
       <p class="mt-2 text-sm text-orange-700 font-bold">
         <i class="fa fa-exclamation-triangle fa-fw" aria-hidden="true"></i>
           {{ _("You must first login to be allowed to submit an app to the wishlist") }}
+          <br/><br/>
+      </p>
+      <p class="mt-2 text-sm text-orange-700">
+          {{ _("Note that, due to various abuses, we restricted login on the app store to 'trust level 1' users.<br/><br/>'Trust level 1' is obtained after interacting a minimum with the forum, and more specifically: entering at least 5 topics, reading at least 30 posts, and spending at least 10 minutes reading posts.") }}
+      </p>
+    </div>
+    {% else %}
+    <div role="alert" class="rounded-md border-s-4 border-orange-500 bg-orange-50 p-4 mb-5">
+      <p class="mt-2 text-sm text-orange-700 font-bold">
+        <i class="fa fa-exclamation-triangle fa-fw" aria-hidden="true"></i>
+          {{ _("Due to abuses, only one proposal every 15 days per user is allowed.") }}
+      </p>
+      <p class="mt-2 text-sm text-orange-700">
+          {{ _("Reviewing those proposals is tiring for volunteers, please don't yolo-send every random nerdy stuff you find on the Internet.") }}
       </p>
     </div>
     {% endif %}
 
-    <div role="alert" class="rounded-md border-s-4 border-sky-500 bg-sky-50 p-4">
-      <p class="mt-2 text-sm text-sky-700 font-bold">
-        <i class="fa fa-info-circle fa-fw" aria-hidden="true"></i>
-          {{ _("Please check the license of the app your are proposing") }}
-      </p>
-      <p class="mt-2 text-sm text-sky-700">
-          {{ _("The YunoHost project will only package free/open-source software (with possible case-by-case exceptions for apps which are not-totally-free)") }}
-      </p>
-    </div>
-
     {% if errormsg %}
     <div role="alert" class="rounded-md border-s-4 border-red-500 bg-red-50 p-4 my-5">
       <p class="mt-2 text-sm text-red-700 font-bold">
@@ -64,6 +68,10 @@
       <label for="upstream" class="mt-5 block font-bold text-gray-700">{{ _("Project code repository") }}</label>
       <input name="upstream" type="url" class="w-full mt-1 rounded-md border-gray-200 text-gray-700 shadow-sm"  maxlength="150" required >
 
+      <label for="license" class="mt-5 block font-bold text-gray-700">{{ _("Link to the project's LICENSE") }}</label>
+      <input name="license" type="url" class="w-full mt-1 rounded-md border-gray-200 text-gray-700 shadow-sm" required maxlength="250"></input>
+  <span class="text-xs text-gray-600 font-bold">{{ _("The YunoHost project will only package free/open-source software (with possible case-by-case exceptions for apps which are not-totally-free)") }}</span>
+
       <label for="website" class="mt-5 block font-bold text-gray-700">{{ _("Project website") }}</label>
       <input name="website" type="url" class="w-full mt-1 rounded-md border-gray-200 text-gray-700 shadow-sm"  maxlength="150" >
       <span class="text-xs text-gray-600">{{ _("Please *do not* just copy-paste the code repository URL. If the project has no proper website, then leave the field empty.") }}</span>

store/translations/fr/LC_MESSAGES/messages.po

@@ -7,7 +7,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2023-09-19 17:04+0200\n"
+"POT-Creation-Date: 2023-11-19 19:39+0100\n"
 "PO-Revision-Date: 2023-09-05 19:50+0200\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language: fr\n"
@@ -18,59 +18,73 @@ msgstr ""
 "Content-Transfer-Encoding: 8bit\n"
 "Generated-By: Babel 2.12.1\n"
 
-#: app.py:140
+#: app.py:148
 msgid "App %(app_id) not found"
 msgstr "L'app %(app_id) n'a pas été trouvée"
 
-#: app.py:142
+#: app.py:150
 msgid "You must be logged in to be able to star an app"
 msgstr "Vous devez être connecté·e pour mettre une app en favoris"
 
-#: app.py:185
+#: app.py:150 app.py:193 app.py:418 templates/wishlist_add.html:31
+msgid ""
+"Note that, due to various abuses, we restricted login on the app store to"
+" 'trust level 1' users.<br/><br/>'Trust level 1' is obtained after "
+"interacting a minimum with the forum, and more specifically: entering at "
+"least 5 topics, reading at least 30 posts, and spending at least 10 "
+"minutes reading posts."
+msgstr ""
+"Notez que, suite à divers abus, la connexion nécessite maintenant d'être"
+" 'trust level 1' sur le forum.<br/><br/>Le 'trust level 1' est obtenu après"
+" avoir intéragit un minimum avec le forum, et plus précisémment: ouvrir au"
+" moins 5 fils de discussion, lire au moins 30 messages, et passer au moins"
+" 10 minutes à lire des messages."
+
+#: app.py:193
 msgid "You must be logged in to submit an app to the wishlist"
 msgstr "Vous devez être connecté·e pour proposer une app pour la liste de souhaits"
 
-#: app.py:200
+#: app.py:206
 msgid "Invalid CSRF token, please refresh the form and try again"
 msgstr "Jeton CSRF invalide, prière de rafraîchir la page et retenter"
 
-#: app.py:216
+#: app.py:222
 msgid "App name should be at least 3 characters"
 msgstr "Le nom d'app devrait contenir au moins 3 caractères"
 
-#: app.py:217
+#: app.py:223
 msgid "App name should be less than 30 characters"
 msgstr "Le nom d'app devrait contenir moins de 30 caractères"
 
-#: app.py:220
+#: app.py:226
 msgid "App description should be at least 5 characters"
 msgstr "La description de l'app devrait contenir au moins 5 caractères"
 
-#: app.py:224
+#: app.py:230
 msgid "App description should be less than 100 characters"
 msgstr "La description de l'app devrait contenir moins de 100 caractères"
 
-#: app.py:228
+#: app.py:234
 msgid "Upstream code repo URL should be at least 10 characters"
 msgstr "L'URL du dépôt de code devrait contenir au moins 10 caractères"
 
-#: app.py:232
+#: app.py:238
 msgid "Upstream code repo URL should be less than 150 characters"
 msgstr "L'URL du dépôt de code devrait contenir moins de 150 caractères"
 
-#: app.py:234
+#: app.py:240
 msgid "Website URL should be less than 150 characters"
 msgstr "L'URL du site web devrait contenir moins de 150 caractères"
 
-#: app.py:237
+#: app.py:243
 msgid "App name contains special characters"
 msgstr "Le nom de l'app contiens des caractères spéciaux"
 
-#: app.py:270
+#: app.py:276
 msgid "An entry with the name %(slug) already exists in the wishlist"
 msgstr "Une entrée nommée $(slug) existe déjà dans la liste de souhaits"
 
-#: app.py:295
+#: app.py:299
 msgid ""
 "Failed to create the pull request to add the app to the wishlist ... "
 "please report the issue to the yunohost team"
@@ -78,15 +92,19 @@ msgstr ""
 "Échec de la création de la demande d'intégration de l'app dans la liste "
 "de souhaits ... merci de rapport le problème à l'équipe YunoHost"
 
-#: app.py:340
+#: app.py:348
-#, python-format
 msgid ""
 "Your proposed app has succesfully been submitted. It must now be "
-"validated by the YunoHost team. You can track progress here: %(url)s"
+"validated by the YunoHost team. You can track progress here: <a "
+"href='%(url)s'>%(url)s</a>"
 msgstr ""
 "Un demande d'intégration à la liste de souhaits a bien été créée pour "
 "cette app. Elle doit maintenant être validée par l'équipe YunoHost. Vous "
-"pouvez suivre cette demande ici: %(url)s"
+"pouvez suivre cette demande ici: <a href='%(url)s'>%(url)s</a>"
+
+#: app.py:418
+msgid "Unfortunately, login was denied."
+msgstr "Malheureusement, la connexion a été refusée."
 
 #: templates/app.html:10 templates/catalog.html:23
 #, python-format
@@ -116,7 +134,9 @@ msgstr ""
 msgid ""
 "This app has been good quality according to our automatic tests over at "
 "least one year."
-msgstr "Cette app est de bonne qualité d'après nos tests automatisés depuis au moins un an."
+msgstr ""
+"Cette app est de bonne qualité d'après nos tests automatisés depuis au "
+"moins un an."
 
 #: templates/app.html:81
 msgid "Try the demo"
@@ -204,34 +224,45 @@ msgstr "Dépôt de code du paquet YunoHost"
 msgid "YunoHost app store"
 msgstr "Store d'apps de YunoHost"
 
-#: templates/base.html:56 templates/base.html:149 templates/index.html:3
+#: templates/base.html:18 templates/base.html:113 templates/index.html:3
 msgid "Home"
 msgstr "Accueil"
 
-#: templates/base.html:65 templates/base.html:158
+#: templates/base.html:27 templates/base.html:122
 msgid "Catalog"
 msgstr "Catalogue"
 
-#: templates/base.html:71 templates/base.html:167
+#: templates/base.html:33 templates/base.html:131
 msgid "Wishlist"
 msgstr "Liste de souhaits"
 
-#: templates/base.html:84 templates/base.html:177
+#: templates/base.html:46 templates/base.html:141
 msgid "YunoHost documentation"
 msgstr "Documentation YunoHost"
 
-#: templates/base.html:92 templates/base.html:187
+#: templates/base.html:54 templates/base.html:151
 msgid "Login using YunoHost's forum"
 msgstr "Se connecter via le forum YunoHost"
 
-#: templates/base.html:122 templates/base.html:213
+#: templates/base.html:86 templates/base.html:179
 msgid "Logout"
 msgstr "Se déconnecter"
 
-#: templates/base.html:135
+#: templates/base.html:99
 msgid "Toggle menu"
 msgstr "Activer le menu"
 
+#: templates/base.html:197
+msgid ""
+"Made with <i class='text-red-500 fa fa-heart-o' aria-label='love'></i> "
+"using <a class='text-blue-800' "
+"href='https://flask.palletsprojects.com'>Flask</a> and <a class='text-"
+"blue-800' href='https://tailwindcss.com/'>TailwindCSS</a> - <a class"
+"='text-blue-800' "
+"href='https://github.com/YunoHost/apps/tree/master/store'><i class='fa "
+"fa-code fa-fw' aria-hidden='true'></i> Source</a>"
+msgstr ""
+
 #: templates/catalog.html:75 templates/catalog.html:80
 msgid "Application Catalog"
 msgstr "Catalogue d'applications"
@@ -253,17 +284,17 @@ msgid "Sort by"
 msgstr "Trier par"
 
 #: templates/catalog.html:123 templates/wishlist.html:45
-msgid "Alphabetical"
+#: templates/wishlist.html:78
-msgstr "Alphabétique"
+msgid "Popularity"
+msgstr "Popularité"
 
 #: templates/catalog.html:124
 msgid "Newest"
 msgstr "Nouveauté"
 
 #: templates/catalog.html:125 templates/wishlist.html:46
-#: templates/wishlist.html:78
+msgid "Alphabetical"
-msgid "Popularity"
+msgstr "Alphabétique"
-msgstr "Popularité"
 
 #: templates/catalog.html:128 templates/wishlist.html:49
 msgid "Requires to be logged-in"
@@ -274,7 +305,7 @@ msgstr "Nécessite d'être connecté·e"
 msgid "Show only apps you starred"
 msgstr "Montrer uniquement mes favoris"
 
-#: templates/catalog.html:155 templates/wishlist.html:152
+#: templates/catalog.html:155 templates/wishlist.html:154
 msgid "No results found."
 msgstr "Aucun résultat trouvé."
 
@@ -326,7 +357,7 @@ msgstr ""
 msgid "Suggest an app"
 msgstr "Suggérer une app"
 
-#: templates/wishlist.html:71 templates/wishlist_add.html:57
+#: templates/wishlist.html:71 templates/wishlist_add.html:59
 msgid "Name"
 msgstr "Nom"
 
@@ -338,11 +369,11 @@ msgstr "Description"
 msgid "Official website"
 msgstr "Site officiel"
 
-#: templates/wishlist.html:114 templates/wishlist.html:115
+#: templates/wishlist.html:115 templates/wishlist.html:116
 msgid "Code repository"
 msgstr "Dépôt de code officiel"
 
-#: templates/wishlist.html:127 templates/wishlist.html:128
+#: templates/wishlist.html:129 templates/wishlist.html:130
 msgid "Star this app"
 msgstr "Étoiler cette app"
 
@@ -354,11 +385,11 @@ msgstr "Suggérer une application à ajouter dans le catalogue de YunoHost"
 msgid "You must first login to be allowed to submit an app to the wishlist"
 msgstr "Vous devez être connecté·e pour proposer une app pour la liste de souhaits"
 
-#: templates/wishlist_add.html:37
+#: templates/wishlist_add.html:39
 msgid "Please check the license of the app your are proposing"
 msgstr "Merci de vérifier la licence de l'app que vous proposez"
 
-#: templates/wishlist_add.html:40
+#: templates/wishlist_add.html:42
 msgid ""
 "The YunoHost project will only package free/open-source software (with "
 "possible case-by-case exceptions for apps which are not-totally-free)"
@@ -367,15 +398,15 @@ msgstr ""
 "(avec quelques possibles exceptions au cas-par-cas pour des apps qui ne "
 "sont pas entièrement libres)"
 
-#: templates/wishlist_add.html:60
+#: templates/wishlist_add.html:62
 msgid "App's description"
 msgstr "Description de l'app"
 
-#: templates/wishlist_add.html:62
+#: templates/wishlist_add.html:64
 msgid "Please be concise and focus on what the app does."
 msgstr "Prière de rester concis et de se concentrer sur ce que l'app fait."
 
-#: templates/wishlist_add.html:62
+#: templates/wishlist_add.html:64
 msgid ""
 "No need to repeat '[App] is ...'. No need to state that it is free/open-"
 "source or self-hosted (otherwise it wouldn't be packaged for YunoHost). "
@@ -387,15 +418,15 @@ msgstr ""
 "Évitez les formulations marketing type 'le meilleur', ou les propriétés "
 "vagues telles que 'facile', 'simple', 'léger'."
 
-#: templates/wishlist_add.html:64
+#: templates/wishlist_add.html:66
 msgid "Project code repository"
 msgstr "Dépôt de code officiel"
 
-#: templates/wishlist_add.html:67
+#: templates/wishlist_add.html:69
 msgid "Project website"
 msgstr "Site officiel"
 
-#: templates/wishlist_add.html:69
+#: templates/wishlist_add.html:71
 msgid ""
 "Please *do not* just copy-paste the code repository URL. If the project "
 "has no proper website, then leave the field empty."
@@ -403,7 +434,7 @@ msgstr ""
 "Prière de *ne pas* juste copier-coller l'URL du dépôt de code. Si le "
 "projet n'a pas de vrai site web, laissez le champ vide."
 
-#: templates/wishlist_add.html:76
+#: templates/wishlist_add.html:78
 msgid "Submit"
 msgstr "Envoyer"
 

store/utils.py

@@ -1,3 +1,4 @@
+import time
 import base64
 import os
 import json
@@ -6,7 +7,7 @@ import subprocess
 import pycmarkgfm
 from emoji import emojize
 from flask import request
-
+from hashlib import md5
 
 AVAILABLE_LANGUAGES = ["en"] + os.listdir("translations")
 
@@ -92,6 +93,26 @@ def get_stars():
 get_stars.cache_checksum = None
 get_stars()
 
+def check_wishlist_submit_ratelimit(user):
+
+    dir_ = os.path.join(".wishlist_ratelimit")
+    if not os.path.exists(dir_):
+        os.mkdir(dir_)
+
+    f = os.path.join(dir_, md5(user.encode()).hexdigest())
+
+    return not os.path.exists(f) or (time.time() - os.path.getmtime(f)) > (15 * 24 * 3600) # 15 days
+
+def save_wishlist_submit_for_ratelimit(user):
+
+    dir_ = os.path.join(".wishlist_ratelimit")
+    if not os.path.exists(dir_):
+        os.mkdir(dir_)
+
+    f = os.path.join(dir_, md5(user.encode()).hexdigest())
+
+    open(f, "w").write("")
+
 
 def human_to_binary(size: str) -> int:
     symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y")

tools/autoupdate_app_sources/autoupdate_app_sources.py

@@ -47,7 +47,6 @@ else:
 
 
 def apps_to_run_auto_update_for():
-
     catalog = toml.load(open(os.path.dirname(__file__) + "/../../apps.toml"))
 
     apps_flagged_as_working_and_on_yunohost_apps_org = [
@@ -105,7 +104,6 @@ def filter_and_get_latest_tag(tags, app_id):
 
 
 def tag_to_int_tuple(tag):
-
     tag = tag.strip("v").strip(".")
     int_tuple = tag.split(".")
     assert all(i.isdigit() for i in int_tuple), f"Cant convert {tag} to int tuple :/"
@@ -127,7 +125,6 @@ def sha256_of_remote_file(url):
 
 class AppAutoUpdater:
     def __init__(self, app_id, app_id_is_local_app_dir=False):
-
         if app_id_is_local_app_dir:
             if not os.path.exists(app_id + "/manifest.toml"):
                 raise Exception("manifest.toml doesnt exists?")
@@ -158,11 +155,9 @@ class AppAutoUpdater:
         self.main_upstream = manifest.get("upstream", {}).get("code")
 
     def run(self):
-
         todos = {}
 
         for source, infos in self.sources.items():
-
             if "autoupdate" not in infos:
                 continue
 
@@ -176,9 +171,16 @@ class AppAutoUpdater:
 
             print(f"\n  Checking {source} ...")
 
-            new_version, new_asset_urls = self.get_latest_version_and_asset(
+            if strategy == "latest_github_release":
-                strategy, asset, infos, source
+                (
-            )
+                    new_version,
+                    new_asset_urls,
+                    changelog_url,
+                ) = self.get_latest_version_and_asset(strategy, asset, infos, source)
+            else:
+                (new_version, new_asset_urls) = self.get_latest_version_and_asset(
+                    strategy, asset, infos, source
+                )
 
             if source == "main":
                 print(f"Current version in manifest: {self.current_version}")
@@ -190,8 +192,12 @@ class AppAutoUpdater:
                 # Though we wrap this in a try/except pass, because don't want to miserably crash
                 # if the tag can't properly be converted to int tuple ...
                 try:
-                    if tag_to_int_tuple(self.current_version) > tag_to_int_tuple(new_version):
+                    if tag_to_int_tuple(self.current_version) > tag_to_int_tuple(
-                        print("Up to date (current version appears more recent than newest version found)")
+                        new_version
+                    ):
+                        print(
+                            "Up to date (current version appears more recent than newest version found)"
+                        )
                         continue
                 except:
                     pass
@@ -200,9 +206,15 @@ class AppAutoUpdater:
                     print("Up to date")
                     continue
 
-            if (isinstance(new_asset_urls, dict) and isinstance(infos.get("url"), str)) \
+            if (
-                or (isinstance(new_asset_urls, str) and not isinstance(infos.get("url"), str)):
+                isinstance(new_asset_urls, dict) and isinstance(infos.get("url"), str)
-                raise Exception(f"It looks like there's an inconsistency between the old asset list and the new ones ... one is arch-specific, the other is not ... Did you forget to define arch-specific regexes ? ... New asset url is/are : {new_asset_urls}")
+            ) or (
+                isinstance(new_asset_urls, str)
+                and not isinstance(infos.get("url"), str)
+            ):
+                raise Exception(
+                    f"It looks like there's an inconsistency between the old asset list and the new ones ... one is arch-specific, the other is not ... Did you forget to define arch-specific regexes ? ... New asset url is/are : {new_asset_urls}"
+                )
 
             if isinstance(new_asset_urls, str) and infos["url"] == new_asset_urls:
                 print(f"URL for asset {source} is up to date")
@@ -226,11 +238,15 @@ class AppAutoUpdater:
             return bool(todos)
 
         if "main" in todos:
+            if strategy == "latest_github_release":
+                title = f"Upgrade to v{new_version}"
+                message = f"Upgrade to v{new_version}\nChangelog: {changelog_url}"
+            else:
+                title = message = f"Upgrade to v{new_version}"
             new_version = todos["main"]["new_version"]
-            message = f"Upgrade to v{new_version}"
             new_branch = f"ci-auto-update-{new_version}"
         else:
-            message = "Upgrade sources"
+            title = message = "Upgrade sources"
             new_branch = "ci-auto-update-sources"
 
         try:
@@ -265,7 +281,7 @@ class AppAutoUpdater:
 
         # Open the PR
         pr = self.repo.create_pull(
-            title=message, body=message, head=new_branch, base=self.base_branch
+            title=title, body=message, head=new_branch, base=self.base_branch
         )
 
         print("Created PR " + self.repo.full_name + " updated with PR #" + str(pr.id))
@@ -273,12 +289,13 @@ class AppAutoUpdater:
         return bool(todos)
 
     def get_latest_version_and_asset(self, strategy, asset, infos, source):
-
+        upstream = (
-        upstream = infos.get("autoupdate", {}).get("upstream", self.main_upstream).strip("/")
+            infos.get("autoupdate", {}).get("upstream", self.main_upstream).strip("/")
+        )
 
         if "github" in strategy:
-            assert upstream and upstream.startswith(
+            assert (
-                "https://github.com/"
+                upstream and upstream.startswith("https://github.com/")
             ), f"When using strategy {strategy}, having a defined upstream code repo on github.com is required"
             api = GithubAPI(upstream, auth=auth)
         elif "gitlab" in strategy:
@@ -294,24 +311,24 @@ class AppAutoUpdater:
             latest_version_orig, latest_version = filter_and_get_latest_tag(
                 tags, self.app_id
             )
+            latest_release = [
+                release
+                for release in releases
+                if release["tag_name"] == latest_version_orig
+            ][0]
+            latest_assets = {
+                a["name"]: a["browser_download_url"]
+                for a in latest_release["assets"]
+                if not a["name"].endswith(".md5")
+            }
+            latest_release_html_url = latest_release["html_url"]
             if asset == "tarball":
                 latest_tarball = (
                     api.url_for_ref(latest_version_orig, RefType.tags)
                 )
-                return latest_version, latest_tarball
+                return latest_version, latest_tarball, latest_release_html_url
             # FIXME
             else:
-                latest_release = [
-                    release
-                    for release in releases
-                    if release["tag_name"] == latest_version_orig
-                ][0]
-                latest_assets = {
-                    a["name"]: a["browser_download_url"]
-                    for a in latest_release["assets"]
-                    if not a["name"].endswith(".md5")
-                }
-                latest_release_html_url = latest_release["html_url"]
                 if isinstance(asset, str):
                     matching_assets_urls = [
                         url
@@ -326,7 +343,11 @@ class AppAutoUpdater:
                         raise Exception(
                             f"Too many assets matching regex '{asset}' for release {latest_version} : {matching_assets_urls}. Full release details on {latest_release_html_url}"
                         )
-                    return latest_version, matching_assets_urls[0]
+                    return (
+                        latest_version,
+                        matching_assets_urls[0],
+                        latest_release_html_url,
+                    )
                 elif isinstance(asset, dict):
                     matching_assets_dicts = {}
                     for asset_name, asset_regex in asset.items():
@@ -344,7 +365,11 @@ class AppAutoUpdater:
                                 f"Too many assets matching regex '{asset}' for release {latest_version} : {matching_assets_urls}. Full release details on {latest_release_html_url}"
                             )
                         matching_assets_dicts[asset_name] = matching_assets_urls[0]
-                    return latest_version.strip("v"), matching_assets_dicts
+                    return (
+                        latest_version.strip("v"),
+                        matching_assets_dicts,
+                        latest_release_html_url,
+                    )
 
         elif strategy == "latest_github_tag" or strategy == "latest_gitlab_tag":
             if asset != "tarball":
@@ -367,8 +392,12 @@ class AppAutoUpdater:
             latest_commit = commits[0]
             latest_tarball = api.url_for_ref(latest_commit["sha"], RefType.commits)
             # Let's have the version as something like "2023.01.23"
-            latest_commit_date = datetime.strptime(latest_commit["commit"]["author"]["date"][:10], "%Y-%m-%d")
+            latest_commit_date = datetime.strptime(
-            version_format = infos.get("autoupdate", {}).get("force_version", "%Y.%m.%d")
+                latest_commit["commit"]["author"]["date"][:10], "%Y-%m-%d"
+            )
+            version_format = infos.get("autoupdate", {}).get(
+                "force_version", "%Y.%m.%d"
+            )
             latest_version = latest_commit_date.strftime(version_format)
 
             return latest_version, latest_tarball
@@ -376,7 +405,6 @@ class AppAutoUpdater:
     def replace_version_and_asset_in_manifest(
         self, content, new_version, new_assets_urls, current_assets, is_main
     ):
-
         if isinstance(new_assets_urls, str):
             sha256 = sha256_of_remote_file(new_assets_urls)
         elif isinstance(new_assets_urls, dict):
@@ -387,7 +415,7 @@ class AppAutoUpdater:
         if is_main:
 
             def repl(m):
-                return m.group(1) + new_version + "~ynh1\""
+                return m.group(1) + new_version + '~ynh1"'
 
             content = re.sub(
                 r"(\s*version\s*=\s*[\"\'])([\d\.]+)(\~ynh\d+[\"\'])", repl, content
@@ -412,7 +440,8 @@ def progressbar(it, prefix="", size=60, file=sys.stdout):
         name += "          "
         x = int(size * j / count)
         file.write(
-            "\n%s[%s%s] %i/%i %s\n" % (prefix, "#" * x, "." * (size - x), j, count, name)
+            "\n%s[%s%s] %i/%i %s\n"
+            % (prefix, "#" * x, "." * (size - x), j, count, name)
         )
         file.flush()
 
@@ -432,14 +461,15 @@ def paste_on_haste(data):
     TIMEOUT = 3
     try:
         url = SERVER_URL + "/documents"
-        response = requests.post(url, data=data.encode('utf-8'), timeout=TIMEOUT)
+        response = requests.post(url, data=data.encode("utf-8"), timeout=TIMEOUT)
         response.raise_for_status()
-        dockey = response.json()['key']
+        dockey = response.json()["key"]
         return SERVER_URL + "/raw/" + dockey
     except requests.exceptions.RequestException as e:
         print("\033[31mError: {}\033[0m".format(e))
         sys.exit(1)
 
+
 if __name__ == "__main__":
     args = [arg for arg in sys.argv[1:] if arg != "--commit-and-create-PR"]
 
@@ -455,6 +485,7 @@ if __name__ == "__main__":
             except Exception as e:
                 apps_failed.append(app)
                 import traceback
+
                 t = traceback.format_exc()
                 apps_failed_details[app] = t
                 print(t)
@@ -465,8 +496,15 @@ if __name__ == "__main__":
         if apps_failed:
             print(f"Apps failed: {', '.join(apps_failed)}")
             if os.path.exists("/usr/bin/sendxmpppy"):
-                 paste = '\n=========\n'.join([app + "\n-------\n" + trace + "\n\n" for app, trace in apps_failed_details.items()])
+                paste = "\n=========\n".join(
-                 paste_url = paste_on_haste(paste)
+                    [
-                 os.system(f"/usr/bin/sendxmpppy 'Failed to run the source auto-update for : {', '.join(apps_failed)}. Please run manually the `autoupdate_app_sources.py` script on these apps to debug what is happening! Debug log : {paste_url}'")
+                        app + "\n-------\n" + trace + "\n\n"
+                        for app, trace in apps_failed_details.items()
+                    ]
+                )
+                paste_url = paste_on_haste(paste)
+                os.system(
+                    f"/usr/bin/sendxmpppy 'Failed to run the source auto-update for : {', '.join(apps_failed)}. Please run manually the `autoupdate_app_sources.py` script on these apps to debug what is happening! Debug log : {paste_url}'"
+                )
         if apps_updated:
             print(f"Apps updated: {', '.join(apps_updated)}")

tools/catalog_linter.py

@@ -37,6 +37,12 @@ def get_wishlist() -> Dict[str, Dict[str, str]]:
     return toml.load(wishlist_path)
 
 
+@cache
+def get_graveyard() -> Dict[str, Dict[str, str]]:
+    wishlist_path = APPS_ROOT / "graveyard.toml"
+    return toml.load(wishlist_path)
+
+
 def validate_schema() -> Generator[str, None, None]:
     with open(APPS_ROOT / "schemas" / "apps.toml.schema.json", encoding="utf-8") as file:
         apps_catalog_schema = json.load(file)
@@ -50,9 +56,6 @@ def check_app(app: str, infos: Dict[str, Any]) -> Generator[Tuple[str, bool], No
         yield "state is missing", True
         return
 
-    if infos["state"] != "working":
-        return
-
     # validate that the app is not (anymore?) in the wishlist
     # we use fuzzy matching because the id in catalog may not be the same exact id as in the wishlist
     # some entries are ignore-hard-coded, because e.g. radarr an readarr are really different apps...
@@ -66,6 +69,16 @@ def check_app(app: str, infos: Dict[str, Any]) -> Generator[Tuple[str, bool], No
     if wishlist_matches:
         yield f"app seems to be listed in wishlist: {wishlist_matches}", True
 
+    ignored_graveyard_entries = ["mailman"]
+    graveyard_matches = [
+        grave
+        for grave in get_graveyard()
+        if grave not in ignored_graveyard_entries
+        and SequenceMatcher(None, app, grave).ratio() > 0.9
+    ]
+    if graveyard_matches:
+        yield f"app seems to be listed in graveyard: {graveyard_matches}", True
+
     repo_name = infos.get("url", "").split("/")[-1]
     if repo_name != f"{app}_ynh":
         yield f"repo name should be {app}_ynh, not in {repo_name}", True