Merge branch 'master' into support_gitlab_autoupgrade

This commit is contained in:
Alexandre Aubin 2024-01-24 00:14:59 +01:00 committed by GitHub
commit 5dc41f57ce
8 changed files with 253 additions and 106 deletions

View file

@ -67,3 +67,8 @@ App packagers should *not* manually set their apps' level. The levels of all the
Applications with no recent activity and no active sign from maintainer may be flagged in `apps.toml` with the `package-not-maintained` antifeature tag to signify that the app is inactive and may slowly become outdated with respect to the upstream, or with respect to good packaging practices. It does **not** mean that the app is not working anymore.
Feel free to contact the app group if you feel like taking over the maintenance of a currently unmaintained app!
### `graveyard.toml`
This file is for apps that are long-term not-working and unlikely to be ever revived

View file

@ -31,6 +31,8 @@ from utils import (
get_wishlist,
get_stars,
get_app_md_and_screenshots,
save_wishlist_submit_for_ratelimit,
check_wishlist_submit_ratelimit,
)
app = Flask(__name__, static_url_path="/assets", static_folder="assets")
@ -147,7 +149,7 @@ def star_app(app_id, action):
if app_id not in get_catalog()["apps"] and app_id not in get_wishlist():
return _("App %(app_id) not found", app_id=app_id), 404
if not session.get("user", {}):
return _("You must be logged in to be able to star an app"), 401
return _("You must be logged in to be able to star an app") + "<br/><br/>" + _("Note that, due to various abuses, we restricted login on the app store to 'trust level 1' users.<br/><br/>'Trust level 1' is obtained after interacting a minimum with the forum, and more specifically: entering at least 5 topics, reading at least 30 posts, and spending at least 10 minutes reading posts."), 401
app_star_folder = os.path.join(".stars", app_id)
app_star_for_this_user = os.path.join(
@ -190,7 +192,7 @@ def add_to_wishlist():
if request.method == "POST":
user = session.get("user", {})
if not user:
errormsg = _("You must be logged in to submit an app to the wishlist")
errormsg = _("You must be logged in to submit an app to the wishlist") + "<br/><br/>" + _("Note that, due to various abuses, we restricted login on the app store to 'trust level 1' users.<br/><br/>'Trust level 1' is obtained after interacting a minimum with the forum, and more specifically: entering at least 5 topics, reading at least 30 posts, and spending at least 10 minutes reading posts.")
return render_template(
"wishlist_add.html",
locale=get_locale(),
@ -199,7 +201,6 @@ def add_to_wishlist():
successmsg=None,
errormsg=errormsg,
)
csrf_token = request.form["csrf_token"]
if csrf_token != session.get("csrf_token"):
@ -217,8 +218,15 @@ def add_to_wishlist():
description = request.form["description"].strip().replace("\n", "")
upstream = request.form["upstream"].strip().replace("\n", "")
website = request.form["website"].strip().replace("\n", "")
license = request.form["license"].strip().replace("\n", "")
boring_keywords_to_check_for_people_not_reading_the_instructions = ["free", "open source", "open-source", "self-hosted", "simple", "lightweight", "light-weight", "best", "most", "fast", "flexible", "puissante", "powerful", "secure"]
checks = [
(
check_wishlist_submit_ratelimit(session['user']['username']) is True,
_("Proposing wishlist additions is limited to once every 15 days per user.")
),
(len(name) >= 3, _("App name should be at least 3 characters")),
(len(name) <= 30, _("App name should be less than 30 characters")),
(
@ -237,11 +245,27 @@ def add_to_wishlist():
len(upstream) <= 150,
_("Upstream code repo URL should be less than 150 characters"),
),
(
len(license) >= 10,
_("License URL should be at least 10 characters"),
),
(
len(license) <= 250,
_("License URL should be less than 250 characters"),
),
(len(website) <= 150, _("Website URL should be less than 150 characters")),
(
re.match(r"^[\w\.\-\(\)\ ]+$", name),
_("App name contains special characters"),
),
(
all(keyword not in description.lower() for keyword in boring_keywords_to_check_for_people_not_reading_the_instructions),
_("Please focus on what the app does, without using marketing, fuzzy terms, or repeating that the app is 'free' and 'self-hostable'.")
),
(
description.lower().split()[0] != name and (len(description.split()) == 1 or description.lower().split()[1] not in ["is", "est"]),
_("No need to repeat '{app} is'. Focus on what the app does.")
)
]
for check, errormsg in checks:
@ -328,6 +352,7 @@ Proposed by **{session['user']['username']}**
Website: {website}
Upstream repo: {upstream}
License: {license}
Description: {description}
- [ ] Confirm app is self-hostable and generally makes sense to possibly integrate in YunoHost
@ -349,6 +374,9 @@ Description: {description}
"Your proposed app has succesfully been submitted. It must now be validated by the YunoHost team. You can track progress here: <a href='%(url)s'>%(url)s</a>",
url=url,
)
save_wishlist_submit_for_ratelimit(session['user']['username'])
return render_template(
"wishlist_add.html",
locale=get_locale(),
@ -414,6 +442,9 @@ def sso_login_callback():
uri_to_redirect_to_after_login = session.get("uri_to_redirect_to_after_login")
if "trust_level_1" not in user_data['groups'][0].split(','):
return _("Unfortunately, login was denied.") + "<br/><br/>" + _("Note that, due to various abuses, we restricted login on the app store to 'trust level 1' users.<br/><br/>'Trust level 1' is obtained after interacting a minimum with the forum, and more specifically: entering at least 5 topics, reading at least 30 posts, and spending at least 10 minutes reading posts."), 403
session.clear()
session["user"] = {
"id": user_data["external_id"][0],

View file

@ -27,20 +27,24 @@
<p class="mt-2 text-sm text-orange-700 font-bold">
<i class="fa fa-exclamation-triangle fa-fw" aria-hidden="true"></i>
{{ _("You must first login to be allowed to submit an app to the wishlist") }}
<br/><br/>
</p>
<p class="mt-2 text-sm text-orange-700">
{{ _("Note that, due to various abuses, we restricted login on the app store to 'trust level 1' users.<br/><br/>'Trust level 1' is obtained after interacting a minimum with the forum, and more specifically: entering at least 5 topics, reading at least 30 posts, and spending at least 10 minutes reading posts.") }}
</p>
</div>
{% else %}
<div role="alert" class="rounded-md border-s-4 border-orange-500 bg-orange-50 p-4 mb-5">
<p class="mt-2 text-sm text-orange-700 font-bold">
<i class="fa fa-exclamation-triangle fa-fw" aria-hidden="true"></i>
{{ _("Due to abuses, only one proposal every 15 days per user is allowed.") }}
</p>
<p class="mt-2 text-sm text-orange-700">
{{ _("Reviewing those proposals is tiring for volunteers, please don't yolo-send every random nerdy stuff you find on the Internet.") }}
</p>
</div>
{% endif %}
<div role="alert" class="rounded-md border-s-4 border-sky-500 bg-sky-50 p-4">
<p class="mt-2 text-sm text-sky-700 font-bold">
<i class="fa fa-info-circle fa-fw" aria-hidden="true"></i>
{{ _("Please check the license of the app your are proposing") }}
</p>
<p class="mt-2 text-sm text-sky-700">
{{ _("The YunoHost project will only package free/open-source software (with possible case-by-case exceptions for apps which are not-totally-free)") }}
</p>
</div>
{% if errormsg %}
<div role="alert" class="rounded-md border-s-4 border-red-500 bg-red-50 p-4 my-5">
<p class="mt-2 text-sm text-red-700 font-bold">
@ -64,6 +68,10 @@
<label for="upstream" class="mt-5 block font-bold text-gray-700">{{ _("Project code repository") }}</label>
<input name="upstream" type="url" class="w-full mt-1 rounded-md border-gray-200 text-gray-700 shadow-sm" maxlength="150" required >
<label for="license" class="mt-5 block font-bold text-gray-700">{{ _("Link to the project's LICENSE") }}</label>
<input name="license" type="url" class="w-full mt-1 rounded-md border-gray-200 text-gray-700 shadow-sm" required maxlength="250"></input>
<span class="text-xs text-gray-600 font-bold">{{ _("The YunoHost project will only package free/open-source software (with possible case-by-case exceptions for apps which are not-totally-free)") }}</span>
<label for="website" class="mt-5 block font-bold text-gray-700">{{ _("Project website") }}</label>
<input name="website" type="url" class="w-full mt-1 rounded-md border-gray-200 text-gray-700 shadow-sm" maxlength="150" >
<span class="text-xs text-gray-600">{{ _("Please *do not* just copy-paste the code repository URL. If the project has no proper website, then leave the field empty.") }}</span>

View file

@ -7,7 +7,7 @@ msgid ""
msgstr ""
"Project-Id-Version: PROJECT VERSION\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2023-09-19 17:04+0200\n"
"POT-Creation-Date: 2023-11-19 19:39+0100\n"
"PO-Revision-Date: 2023-09-05 19:50+0200\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language: fr\n"
@ -18,59 +18,73 @@ msgstr ""
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: Babel 2.12.1\n"
#: app.py:140
#: app.py:148
msgid "App %(app_id) not found"
msgstr "L'app %(app_id) n'a pas été trouvée"
#: app.py:142
#: app.py:150
msgid "You must be logged in to be able to star an app"
msgstr "Vous devez être connecté·e pour mettre une app en favoris"
#: app.py:185
#: app.py:150 app.py:193 app.py:418 templates/wishlist_add.html:31
msgid ""
"Note that, due to various abuses, we restricted login on the app store to"
" 'trust level 1' users.<br/><br/>'Trust level 1' is obtained after "
"interacting a minimum with the forum, and more specifically: entering at "
"least 5 topics, reading at least 30 posts, and spending at least 10 "
"minutes reading posts."
msgstr ""
"Notez que, suite à divers abus, la connexion nécessite maintenant d'être"
" 'trust level 1' sur le forum.<br/><br/>Le 'trust level 1' est obtenu après"
" avoir intéragit un minimum avec le forum, et plus précisémment: ouvrir au"
" moins 5 fils de discussion, lire au moins 30 messages, et passer au moins"
" 10 minutes à lire des messages."
#: app.py:193
msgid "You must be logged in to submit an app to the wishlist"
msgstr "Vous devez être connecté·e pour proposer une app pour la liste de souhaits"
#: app.py:200
#: app.py:206
msgid "Invalid CSRF token, please refresh the form and try again"
msgstr "Jeton CSRF invalide, prière de rafraîchir la page et retenter"
#: app.py:216
#: app.py:222
msgid "App name should be at least 3 characters"
msgstr "Le nom d'app devrait contenir au moins 3 caractères"
#: app.py:217
#: app.py:223
msgid "App name should be less than 30 characters"
msgstr "Le nom d'app devrait contenir moins de 30 caractères"
#: app.py:220
#: app.py:226
msgid "App description should be at least 5 characters"
msgstr "La description de l'app devrait contenir au moins 5 caractères"
#: app.py:224
#: app.py:230
msgid "App description should be less than 100 characters"
msgstr "La description de l'app devrait contenir moins de 100 caractères"
#: app.py:228
#: app.py:234
msgid "Upstream code repo URL should be at least 10 characters"
msgstr "L'URL du dépôt de code devrait contenir au moins 10 caractères"
#: app.py:232
#: app.py:238
msgid "Upstream code repo URL should be less than 150 characters"
msgstr "L'URL du dépôt de code devrait contenir moins de 150 caractères"
#: app.py:234
#: app.py:240
msgid "Website URL should be less than 150 characters"
msgstr "L'URL du site web devrait contenir moins de 150 caractères"
#: app.py:237
#: app.py:243
msgid "App name contains special characters"
msgstr "Le nom de l'app contiens des caractères spéciaux"
#: app.py:270
#: app.py:276
msgid "An entry with the name %(slug) already exists in the wishlist"
msgstr "Une entrée nommée $(slug) existe déjà dans la liste de souhaits"
#: app.py:295
#: app.py:299
msgid ""
"Failed to create the pull request to add the app to the wishlist ... "
"please report the issue to the yunohost team"
@ -78,15 +92,19 @@ msgstr ""
"Échec de la création de la demande d'intégration de l'app dans la liste "
"de souhaits ... merci de rapport le problème à l'équipe YunoHost"
#: app.py:340
#, python-format
#: app.py:348
msgid ""
"Your proposed app has succesfully been submitted. It must now be "
"validated by the YunoHost team. You can track progress here: %(url)s"
"validated by the YunoHost team. You can track progress here: <a "
"href='%(url)s'>%(url)s</a>"
msgstr ""
"Un demande d'intégration à la liste de souhaits a bien été créée pour "
"cette app. Elle doit maintenant être validée par l'équipe YunoHost. Vous "
"pouvez suivre cette demande ici: %(url)s"
"pouvez suivre cette demande ici: <a href='%(url)s'>%(url)s</a>"
#: app.py:418
msgid "Unfortunately, login was denied."
msgstr "Malheureusement, la connexion a été refusée."
#: templates/app.html:10 templates/catalog.html:23
#, python-format
@ -116,7 +134,9 @@ msgstr ""
msgid ""
"This app has been good quality according to our automatic tests over at "
"least one year."
msgstr "Cette app est de bonne qualité d'après nos tests automatisés depuis au moins un an."
msgstr ""
"Cette app est de bonne qualité d'après nos tests automatisés depuis au "
"moins un an."
#: templates/app.html:81
msgid "Try the demo"
@ -204,34 +224,45 @@ msgstr "Dépôt de code du paquet YunoHost"
msgid "YunoHost app store"
msgstr "Store d'apps de YunoHost"
#: templates/base.html:56 templates/base.html:149 templates/index.html:3
#: templates/base.html:18 templates/base.html:113 templates/index.html:3
msgid "Home"
msgstr "Accueil"
#: templates/base.html:65 templates/base.html:158
#: templates/base.html:27 templates/base.html:122
msgid "Catalog"
msgstr "Catalogue"
#: templates/base.html:71 templates/base.html:167
#: templates/base.html:33 templates/base.html:131
msgid "Wishlist"
msgstr "Liste de souhaits"
#: templates/base.html:84 templates/base.html:177
#: templates/base.html:46 templates/base.html:141
msgid "YunoHost documentation"
msgstr "Documentation YunoHost"
#: templates/base.html:92 templates/base.html:187
#: templates/base.html:54 templates/base.html:151
msgid "Login using YunoHost's forum"
msgstr "Se connecter via le forum YunoHost"
#: templates/base.html:122 templates/base.html:213
#: templates/base.html:86 templates/base.html:179
msgid "Logout"
msgstr "Se déconnecter"
#: templates/base.html:135
#: templates/base.html:99
msgid "Toggle menu"
msgstr "Activer le menu"
#: templates/base.html:197
msgid ""
"Made with <i class='text-red-500 fa fa-heart-o' aria-label='love'></i> "
"using <a class='text-blue-800' "
"href='https://flask.palletsprojects.com'>Flask</a> and <a class='text-"
"blue-800' href='https://tailwindcss.com/'>TailwindCSS</a> - <a class"
"='text-blue-800' "
"href='https://github.com/YunoHost/apps/tree/master/store'><i class='fa "
"fa-code fa-fw' aria-hidden='true'></i> Source</a>"
msgstr ""
#: templates/catalog.html:75 templates/catalog.html:80
msgid "Application Catalog"
msgstr "Catalogue d'applications"
@ -253,17 +284,17 @@ msgid "Sort by"
msgstr "Trier par"
#: templates/catalog.html:123 templates/wishlist.html:45
msgid "Alphabetical"
msgstr "Alphabétique"
#: templates/wishlist.html:78
msgid "Popularity"
msgstr "Popularité"
#: templates/catalog.html:124
msgid "Newest"
msgstr "Nouveauté"
#: templates/catalog.html:125 templates/wishlist.html:46
#: templates/wishlist.html:78
msgid "Popularity"
msgstr "Popularité"
msgid "Alphabetical"
msgstr "Alphabétique"
#: templates/catalog.html:128 templates/wishlist.html:49
msgid "Requires to be logged-in"
@ -274,7 +305,7 @@ msgstr "Nécessite d'être connecté·e"
msgid "Show only apps you starred"
msgstr "Montrer uniquement mes favoris"
#: templates/catalog.html:155 templates/wishlist.html:152
#: templates/catalog.html:155 templates/wishlist.html:154
msgid "No results found."
msgstr "Aucun résultat trouvé."
@ -326,7 +357,7 @@ msgstr ""
msgid "Suggest an app"
msgstr "Suggérer une app"
#: templates/wishlist.html:71 templates/wishlist_add.html:57
#: templates/wishlist.html:71 templates/wishlist_add.html:59
msgid "Name"
msgstr "Nom"
@ -338,11 +369,11 @@ msgstr "Description"
msgid "Official website"
msgstr "Site officiel"
#: templates/wishlist.html:114 templates/wishlist.html:115
#: templates/wishlist.html:115 templates/wishlist.html:116
msgid "Code repository"
msgstr "Dépôt de code officiel"
#: templates/wishlist.html:127 templates/wishlist.html:128
#: templates/wishlist.html:129 templates/wishlist.html:130
msgid "Star this app"
msgstr "Étoiler cette app"
@ -354,11 +385,11 @@ msgstr "Suggérer une application à ajouter dans le catalogue de YunoHost"
msgid "You must first login to be allowed to submit an app to the wishlist"
msgstr "Vous devez être connecté·e pour proposer une app pour la liste de souhaits"
#: templates/wishlist_add.html:37
#: templates/wishlist_add.html:39
msgid "Please check the license of the app your are proposing"
msgstr "Merci de vérifier la licence de l'app que vous proposez"
#: templates/wishlist_add.html:40
#: templates/wishlist_add.html:42
msgid ""
"The YunoHost project will only package free/open-source software (with "
"possible case-by-case exceptions for apps which are not-totally-free)"
@ -367,15 +398,15 @@ msgstr ""
"(avec quelques possibles exceptions au cas-par-cas pour des apps qui ne "
"sont pas entièrement libres)"
#: templates/wishlist_add.html:60
#: templates/wishlist_add.html:62
msgid "App's description"
msgstr "Description de l'app"
#: templates/wishlist_add.html:62
#: templates/wishlist_add.html:64
msgid "Please be concise and focus on what the app does."
msgstr "Prière de rester concis et de se concentrer sur ce que l'app fait."
#: templates/wishlist_add.html:62
#: templates/wishlist_add.html:64
msgid ""
"No need to repeat '[App] is ...'. No need to state that it is free/open-"
"source or self-hosted (otherwise it wouldn't be packaged for YunoHost). "
@ -387,15 +418,15 @@ msgstr ""
"Évitez les formulations marketing type 'le meilleur', ou les propriétés "
"vagues telles que 'facile', 'simple', 'léger'."
#: templates/wishlist_add.html:64
#: templates/wishlist_add.html:66
msgid "Project code repository"
msgstr "Dépôt de code officiel"
#: templates/wishlist_add.html:67
#: templates/wishlist_add.html:69
msgid "Project website"
msgstr "Site officiel"
#: templates/wishlist_add.html:69
#: templates/wishlist_add.html:71
msgid ""
"Please *do not* just copy-paste the code repository URL. If the project "
"has no proper website, then leave the field empty."
@ -403,7 +434,7 @@ msgstr ""
"Prière de *ne pas* juste copier-coller l'URL du dépôt de code. Si le "
"projet n'a pas de vrai site web, laissez le champ vide."
#: templates/wishlist_add.html:76
#: templates/wishlist_add.html:78
msgid "Submit"
msgstr "Envoyer"

View file

@ -1,3 +1,4 @@
import time
import base64
import os
import json
@ -6,7 +7,7 @@ import subprocess
import pycmarkgfm
from emoji import emojize
from flask import request
from hashlib import md5
AVAILABLE_LANGUAGES = ["en"] + os.listdir("translations")
@ -92,6 +93,26 @@ def get_stars():
get_stars.cache_checksum = None
get_stars()
def check_wishlist_submit_ratelimit(user):
dir_ = os.path.join(".wishlist_ratelimit")
if not os.path.exists(dir_):
os.mkdir(dir_)
f = os.path.join(dir_, md5(user.encode()).hexdigest())
return not os.path.exists(f) or (time.time() - os.path.getmtime(f)) > (15 * 24 * 3600) # 15 days
def save_wishlist_submit_for_ratelimit(user):
dir_ = os.path.join(".wishlist_ratelimit")
if not os.path.exists(dir_):
os.mkdir(dir_)
f = os.path.join(dir_, md5(user.encode()).hexdigest())
open(f, "w").write("")
def human_to_binary(size: str) -> int:
symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y")

View file

@ -47,7 +47,6 @@ else:
def apps_to_run_auto_update_for():
catalog = toml.load(open(os.path.dirname(__file__) + "/../../apps.toml"))
apps_flagged_as_working_and_on_yunohost_apps_org = [
@ -105,7 +104,6 @@ def filter_and_get_latest_tag(tags, app_id):
def tag_to_int_tuple(tag):
tag = tag.strip("v").strip(".")
int_tuple = tag.split(".")
assert all(i.isdigit() for i in int_tuple), f"Cant convert {tag} to int tuple :/"
@ -127,7 +125,6 @@ def sha256_of_remote_file(url):
class AppAutoUpdater:
def __init__(self, app_id, app_id_is_local_app_dir=False):
if app_id_is_local_app_dir:
if not os.path.exists(app_id + "/manifest.toml"):
raise Exception("manifest.toml doesnt exists?")
@ -158,11 +155,9 @@ class AppAutoUpdater:
self.main_upstream = manifest.get("upstream", {}).get("code")
def run(self):
todos = {}
for source, infos in self.sources.items():
if "autoupdate" not in infos:
continue
@ -176,9 +171,16 @@ class AppAutoUpdater:
print(f"\n Checking {source} ...")
new_version, new_asset_urls = self.get_latest_version_and_asset(
strategy, asset, infos, source
)
if strategy == "latest_github_release":
(
new_version,
new_asset_urls,
changelog_url,
) = self.get_latest_version_and_asset(strategy, asset, infos, source)
else:
(new_version, new_asset_urls) = self.get_latest_version_and_asset(
strategy, asset, infos, source
)
if source == "main":
print(f"Current version in manifest: {self.current_version}")
@ -190,8 +192,12 @@ class AppAutoUpdater:
# Though we wrap this in a try/except pass, because don't want to miserably crash
# if the tag can't properly be converted to int tuple ...
try:
if tag_to_int_tuple(self.current_version) > tag_to_int_tuple(new_version):
print("Up to date (current version appears more recent than newest version found)")
if tag_to_int_tuple(self.current_version) > tag_to_int_tuple(
new_version
):
print(
"Up to date (current version appears more recent than newest version found)"
)
continue
except:
pass
@ -200,9 +206,15 @@ class AppAutoUpdater:
print("Up to date")
continue
if (isinstance(new_asset_urls, dict) and isinstance(infos.get("url"), str)) \
or (isinstance(new_asset_urls, str) and not isinstance(infos.get("url"), str)):
raise Exception(f"It looks like there's an inconsistency between the old asset list and the new ones ... one is arch-specific, the other is not ... Did you forget to define arch-specific regexes ? ... New asset url is/are : {new_asset_urls}")
if (
isinstance(new_asset_urls, dict) and isinstance(infos.get("url"), str)
) or (
isinstance(new_asset_urls, str)
and not isinstance(infos.get("url"), str)
):
raise Exception(
f"It looks like there's an inconsistency between the old asset list and the new ones ... one is arch-specific, the other is not ... Did you forget to define arch-specific regexes ? ... New asset url is/are : {new_asset_urls}"
)
if isinstance(new_asset_urls, str) and infos["url"] == new_asset_urls:
print(f"URL for asset {source} is up to date")
@ -226,11 +238,15 @@ class AppAutoUpdater:
return bool(todos)
if "main" in todos:
if strategy == "latest_github_release":
title = f"Upgrade to v{new_version}"
message = f"Upgrade to v{new_version}\nChangelog: {changelog_url}"
else:
title = message = f"Upgrade to v{new_version}"
new_version = todos["main"]["new_version"]
message = f"Upgrade to v{new_version}"
new_branch = f"ci-auto-update-{new_version}"
else:
message = "Upgrade sources"
title = message = "Upgrade sources"
new_branch = "ci-auto-update-sources"
try:
@ -265,7 +281,7 @@ class AppAutoUpdater:
# Open the PR
pr = self.repo.create_pull(
title=message, body=message, head=new_branch, base=self.base_branch
title=title, body=message, head=new_branch, base=self.base_branch
)
print("Created PR " + self.repo.full_name + " updated with PR #" + str(pr.id))
@ -273,12 +289,13 @@ class AppAutoUpdater:
return bool(todos)
def get_latest_version_and_asset(self, strategy, asset, infos, source):
upstream = infos.get("autoupdate", {}).get("upstream", self.main_upstream).strip("/")
upstream = (
infos.get("autoupdate", {}).get("upstream", self.main_upstream).strip("/")
)
if "github" in strategy:
assert upstream and upstream.startswith(
"https://github.com/"
assert (
upstream and upstream.startswith("https://github.com/")
), f"When using strategy {strategy}, having a defined upstream code repo on github.com is required"
api = GithubAPI(upstream, auth=auth)
elif "gitlab" in strategy:
@ -294,24 +311,24 @@ class AppAutoUpdater:
latest_version_orig, latest_version = filter_and_get_latest_tag(
tags, self.app_id
)
latest_release = [
release
for release in releases
if release["tag_name"] == latest_version_orig
][0]
latest_assets = {
a["name"]: a["browser_download_url"]
for a in latest_release["assets"]
if not a["name"].endswith(".md5")
}
latest_release_html_url = latest_release["html_url"]
if asset == "tarball":
latest_tarball = (
api.url_for_ref(latest_version_orig, RefType.tags)
)
return latest_version, latest_tarball
return latest_version, latest_tarball, latest_release_html_url
# FIXME
else:
latest_release = [
release
for release in releases
if release["tag_name"] == latest_version_orig
][0]
latest_assets = {
a["name"]: a["browser_download_url"]
for a in latest_release["assets"]
if not a["name"].endswith(".md5")
}
latest_release_html_url = latest_release["html_url"]
if isinstance(asset, str):
matching_assets_urls = [
url
@ -326,7 +343,11 @@ class AppAutoUpdater:
raise Exception(
f"Too many assets matching regex '{asset}' for release {latest_version} : {matching_assets_urls}. Full release details on {latest_release_html_url}"
)
return latest_version, matching_assets_urls[0]
return (
latest_version,
matching_assets_urls[0],
latest_release_html_url,
)
elif isinstance(asset, dict):
matching_assets_dicts = {}
for asset_name, asset_regex in asset.items():
@ -344,7 +365,11 @@ class AppAutoUpdater:
f"Too many assets matching regex '{asset}' for release {latest_version} : {matching_assets_urls}. Full release details on {latest_release_html_url}"
)
matching_assets_dicts[asset_name] = matching_assets_urls[0]
return latest_version.strip("v"), matching_assets_dicts
return (
latest_version.strip("v"),
matching_assets_dicts,
latest_release_html_url,
)
elif strategy == "latest_github_tag" or strategy == "latest_gitlab_tag":
if asset != "tarball":
@ -367,8 +392,12 @@ class AppAutoUpdater:
latest_commit = commits[0]
latest_tarball = api.url_for_ref(latest_commit["sha"], RefType.commits)
# Let's have the version as something like "2023.01.23"
latest_commit_date = datetime.strptime(latest_commit["commit"]["author"]["date"][:10], "%Y-%m-%d")
version_format = infos.get("autoupdate", {}).get("force_version", "%Y.%m.%d")
latest_commit_date = datetime.strptime(
latest_commit["commit"]["author"]["date"][:10], "%Y-%m-%d"
)
version_format = infos.get("autoupdate", {}).get(
"force_version", "%Y.%m.%d"
)
latest_version = latest_commit_date.strftime(version_format)
return latest_version, latest_tarball
@ -376,7 +405,6 @@ class AppAutoUpdater:
def replace_version_and_asset_in_manifest(
self, content, new_version, new_assets_urls, current_assets, is_main
):
if isinstance(new_assets_urls, str):
sha256 = sha256_of_remote_file(new_assets_urls)
elif isinstance(new_assets_urls, dict):
@ -387,7 +415,7 @@ class AppAutoUpdater:
if is_main:
def repl(m):
return m.group(1) + new_version + "~ynh1\""
return m.group(1) + new_version + '~ynh1"'
content = re.sub(
r"(\s*version\s*=\s*[\"\'])([\d\.]+)(\~ynh\d+[\"\'])", repl, content
@ -412,7 +440,8 @@ def progressbar(it, prefix="", size=60, file=sys.stdout):
name += " "
x = int(size * j / count)
file.write(
"\n%s[%s%s] %i/%i %s\n" % (prefix, "#" * x, "." * (size - x), j, count, name)
"\n%s[%s%s] %i/%i %s\n"
% (prefix, "#" * x, "." * (size - x), j, count, name)
)
file.flush()
@ -432,14 +461,15 @@ def paste_on_haste(data):
TIMEOUT = 3
try:
url = SERVER_URL + "/documents"
response = requests.post(url, data=data.encode('utf-8'), timeout=TIMEOUT)
response = requests.post(url, data=data.encode("utf-8"), timeout=TIMEOUT)
response.raise_for_status()
dockey = response.json()['key']
dockey = response.json()["key"]
return SERVER_URL + "/raw/" + dockey
except requests.exceptions.RequestException as e:
print("\033[31mError: {}\033[0m".format(e))
sys.exit(1)
if __name__ == "__main__":
args = [arg for arg in sys.argv[1:] if arg != "--commit-and-create-PR"]
@ -455,6 +485,7 @@ if __name__ == "__main__":
except Exception as e:
apps_failed.append(app)
import traceback
t = traceback.format_exc()
apps_failed_details[app] = t
print(t)
@ -465,8 +496,15 @@ if __name__ == "__main__":
if apps_failed:
print(f"Apps failed: {', '.join(apps_failed)}")
if os.path.exists("/usr/bin/sendxmpppy"):
paste = '\n=========\n'.join([app + "\n-------\n" + trace + "\n\n" for app, trace in apps_failed_details.items()])
paste_url = paste_on_haste(paste)
os.system(f"/usr/bin/sendxmpppy 'Failed to run the source auto-update for : {', '.join(apps_failed)}. Please run manually the `autoupdate_app_sources.py` script on these apps to debug what is happening! Debug log : {paste_url}'")
paste = "\n=========\n".join(
[
app + "\n-------\n" + trace + "\n\n"
for app, trace in apps_failed_details.items()
]
)
paste_url = paste_on_haste(paste)
os.system(
f"/usr/bin/sendxmpppy 'Failed to run the source auto-update for : {', '.join(apps_failed)}. Please run manually the `autoupdate_app_sources.py` script on these apps to debug what is happening! Debug log : {paste_url}'"
)
if apps_updated:
print(f"Apps updated: {', '.join(apps_updated)}")

View file

@ -37,6 +37,12 @@ def get_wishlist() -> Dict[str, Dict[str, str]]:
return toml.load(wishlist_path)
@cache
def get_graveyard() -> Dict[str, Dict[str, str]]:
wishlist_path = APPS_ROOT / "graveyard.toml"
return toml.load(wishlist_path)
def validate_schema() -> Generator[str, None, None]:
with open(APPS_ROOT / "schemas" / "apps.toml.schema.json", encoding="utf-8") as file:
apps_catalog_schema = json.load(file)
@ -50,9 +56,6 @@ def check_app(app: str, infos: Dict[str, Any]) -> Generator[Tuple[str, bool], No
yield "state is missing", True
return
if infos["state"] != "working":
return
# validate that the app is not (anymore?) in the wishlist
# we use fuzzy matching because the id in catalog may not be the same exact id as in the wishlist
# some entries are ignore-hard-coded, because e.g. radarr an readarr are really different apps...
@ -66,6 +69,16 @@ def check_app(app: str, infos: Dict[str, Any]) -> Generator[Tuple[str, bool], No
if wishlist_matches:
yield f"app seems to be listed in wishlist: {wishlist_matches}", True
ignored_graveyard_entries = ["mailman"]
graveyard_matches = [
grave
for grave in get_graveyard()
if grave not in ignored_graveyard_entries
and SequenceMatcher(None, app, grave).ratio() > 0.9
]
if graveyard_matches:
yield f"app seems to be listed in graveyard: {graveyard_matches}", True
repo_name = infos.get("url", "").split("/")[-1]
if repo_name != f"{app}_ynh":
yield f"repo name should be {app}_ynh, not in {repo_name}", True