diff --git a/.gitignore b/.gitignore index eae46b4c5..91b5b56e4 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,10 @@ src/locales # Test src/tests/apps + +# Tmp/local doc stuff +doc/bash-completion.sh +doc/bash_completion.d +doc/openapi.js +doc/openapi.json +doc/swagger diff --git a/bin/yunopaste b/bin/yunopaste index 679f13544..edf8d55c8 100755 --- a/bin/yunopaste +++ b/bin/yunopaste @@ -19,7 +19,7 @@ paste_data() { [[ -z "$json" ]] && _die "Unable to post the data to the server." key=$(echo "$json" \ - | python -c 'import json,sys;o=json.load(sys.stdin);print o["key"]' \ + | python3 -c 'import json,sys;o=json.load(sys.stdin);print(o["key"])' \ 2>/dev/null) [[ -z "$key" ]] && _die "Unable to parse the server response." diff --git a/conf/nginx/yunohost_admin.conf.inc b/conf/nginx/yunohost_admin.conf.inc index b5eff7a5e..84c49d30b 100644 --- a/conf/nginx/yunohost_admin.conf.inc +++ b/conf/nginx/yunohost_admin.conf.inc @@ -19,6 +19,10 @@ location /yunohost/admin/ { more_set_headers "Cache-Control: no-store, no-cache, must-revalidate"; } + location /yunohost/admin/applogos/ { + alias /usr/share/yunohost/applogos/; + } + more_set_headers "Content-Security-Policy: upgrade-insecure-requests; default-src 'self'; connect-src 'self' https://paste.yunohost.org wss://$host; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-eval'; object-src 'none'; img-src 'self' data:;"; more_set_headers "Content-Security-Policy-Report-Only:"; } diff --git a/debian/changelog b/debian/changelog index fb48eeed8..fbcddc9fb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,50 @@ +yunohost (11.1.2.1) testing; urgency=low + + - i18n: fix (un)defined string issues (dd33476f) + - doc: Revive the old auto documentation of API with swagger + - apps: don't clone 'master' branch by default, use git ls-remote to check what's the default branch instead (a6db52b7) + - ssowat: add use_remote_user_var_in_nginx_conf flag on permission (f258eab6) + + Thanks to all contributors <3 ! (ljf) + + -- Alexandre Aubin Mon, 09 Jan 2023 23:58:51 +0100 + +yunohost (11.1.2) testing; urgency=low + + - apps: Various fixes/improvements for appsv2, mostly related to webadmin integration ([#1526](https://github.com/yunohost/yunohost/pull/1526)) + - domains/regenconf: propagate mail/xmpp enable/disable toggle to actual system configs ([#1541](https://github.com/yunohost/yunohost/pull/1541)) + - settings: Add a virtual setting to enable passwordless sudo for admins (75cb3cb2) + - settings: Add a global setting to choose SSOwat's theme ([#1545](https://github.com/yunohost/yunohost/pull/1545)) + - certs: Improve trick to identify certs as self-signed (c38aba74) + - certs: be more resilient when mail cant be sent to root for some reason .. (d7ee1c23) + - certs/postfix: propagate postfix SNI stuff when renewing certificates (31794008) + - certs/xmpp: add to domain's certificate the alt subdomain muc ([#1163](https://github.com/yunohost/yunohost/pull/1163)) + - conf/ldap: fix issue where sudo doesn't work because sudo-ldap doesn't create /etc/sudo-ldap.conf :/ (d2417c33) + - configpanels: fix custom getter ([#1546](https://github.com/yunohost/yunohost/pull/1546)) + - configpanels: fix inconsistent return format for boolean, sometimes 1/0, sometimes True/False -> force normalization of values when calling get() for a single setting from a config panel (47b9b8b5) + - postfix/fail2ban: Add postfix SASL login failure to a fail2ban jail ([#1552](https://github.com/yunohost/yunohost/pull/1552)) + - mail: Fix flag case sensitivity in dovecot and rspamd sieve filter ([#1450](https://github.com/yunohost/yunohost/pull/1450)) + - misc: Don't disable avahi-daemon by force in conf_regen ([#1555](https://github.com/yunohost/yunohost/pull/1555)) + - misc: Fix yunopaste ([#1558](https://github.com/yunohost/yunohost/pull/1558)) + - misc: Don't take lock for read/GET operations (#1554) (0ac8e66a) + - i18n: Translations updated for Basque, French, Galician, Portuguese, Slovak, Spanish, Ukrainian + + Thanks to all contributors <3 ! (axolotle, DDATAA, Fabian Wilkens, Gabriel, José M, Jose Riha, ljf, Luis H. Porras, ppr, quiwy, Rafael Fontenelle, selfhoster1312, Tymofii-Lytvynenko, xabirequejo, Xavier Brochard) + + -- Alexandre Aubin Fri, 06 Jan 2023 00:12:53 +0100 + +yunohost (11.1.1.2) testing; urgency=low + + - group mailalias: the ldap class is in fact mailGroup, not mailAccount -_- (1cb5e43e) + + -- Alexandre Aubin Sat, 03 Dec 2022 15:57:22 +0100 + +yunohost (11.1.1.1) testing; urgency=low + + - Fix again the legacy patch for yunohost user create @_@ (46d6fab0) + + -- Alexandre Aubin Sat, 03 Dec 2022 14:13:09 +0100 + yunohost (11.1.1) testing; urgency=low - groups: add mail-aliases management (#1539) (0f9d9388) diff --git a/doc/api.html b/doc/api.html new file mode 100644 index 000000000..502d1247f --- /dev/null +++ b/doc/api.html @@ -0,0 +1,42 @@ + + + + + + Swagger UI + + + + + + +
+ + + + + + + diff --git a/doc/generate_api_doc.py b/doc/generate_api_doc.py new file mode 100644 index 000000000..fc44ffbcd --- /dev/null +++ b/doc/generate_api_doc.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" License + Copyright (C) 2013 YunoHost + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + You should have received a copy of the GNU Affero General Public License + along with this program; if not, see http://www.gnu.org/licenses +""" + +""" + Generate JSON specification files API +""" +import os +import sys +import yaml +import json +import requests + + +def main(): + """ """ + with open("../share/actionsmap.yml") as f: + action_map = yaml.safe_load(f) + + try: + with open("/etc/yunohost/current_host", "r") as f: + domain = f.readline().rstrip() + except IOError: + domain = requests.get("http://ip.yunohost.org").text + with open("../debian/changelog") as f: + top_changelog = f.readline() + api_version = top_changelog[top_changelog.find("(") + 1 : top_changelog.find(")")] + + csrf = { + "name": "X-Requested-With", + "in": "header", + "required": True, + "schema": {"type": "string", "default": "Swagger API"}, + } + + resource_list = { + "openapi": "3.0.3", + "info": { + "title": "YunoHost API", + "description": "This is the YunoHost API used on all YunoHost instances. This API is essentially used by YunoHost Webadmin.", + "version": api_version, + }, + "servers": [ + { + "url": "https://{domain}/yunohost/api", + "variables": { + "domain": { + "default": "demo.yunohost.org", + "description": "Your yunohost domain", + } + }, + } + ], + "tags": [{"name": "public", "description": "Public route"}], + "paths": { + "/login": { + "post": { + "tags": ["public"], + "summary": "Logs in and returns the authentication cookie", + "parameters": [csrf], + "requestBody": { + "required": True, + "content": { + "multipart/form-data": { + "schema": { + "type": "object", + "properties": { + "credentials": { + "type": "string", + "format": "password", + } + }, + "required": ["credentials"], + } + } + }, + }, + "security": [], + "responses": { + "200": { + "description": "Successfully login", + "headers": {"Set-Cookie": {"schema": {"type": "string"}}}, + } + }, + } + }, + "/installed": { + "get": { + "tags": ["public"], + "summary": "Test if the API is working", + "parameters": [], + "security": [], + "responses": { + "200": { + "description": "Successfully working", + } + }, + } + }, + }, + } + + def convert_categories(categories, parent_category=""): + for category, category_params in categories.items(): + if parent_category: + category = f"{parent_category} {category}" + if "subcategory_help" in category_params: + category_params["category_help"] = category_params["subcategory_help"] + + if "category_help" not in category_params: + category_params["category_help"] = "" + resource_list["tags"].append( + {"name": category, "description": category_params["category_help"]} + ) + + for action, action_params in category_params["actions"].items(): + if "action_help" not in action_params: + action_params["action_help"] = "" + if "api" not in action_params: + continue + if not isinstance(action_params["api"], list): + action_params["api"] = [action_params["api"]] + + for i, api in enumerate(action_params["api"]): + print(api) + method, path = api.split(" ") + method = method.lower() + key_param = "" + if "{" in path: + key_param = path[path.find("{") + 1 : path.find("}")] + resource_list["paths"].setdefault(path, {}) + + notes = "" + + operationId = f"{category}_{action}" + if i > 0: + operationId += f"_{i}" + operation = { + "tags": [category], + "operationId": operationId, + "summary": action_params["action_help"], + "description": notes, + "responses": {"200": {"description": "successful operation"}}, + } + if action_params.get("deprecated"): + operation["deprecated"] = True + + operation["parameters"] = [] + if method == "post": + operation["parameters"] = [csrf] + + if "arguments" in action_params: + if method in ["put", "post", "patch"]: + operation["requestBody"] = { + "required": True, + "content": { + "multipart/form-data": { + "schema": { + "type": "object", + "properties": {}, + "required": [], + } + } + }, + } + for arg_name, arg_params in action_params["arguments"].items(): + if "help" not in arg_params: + arg_params["help"] = "" + param_type = "query" + allow_multiple = False + required = True + allowable_values = None + name = str(arg_name).replace("-", "_") + if name[0] == "_": + required = False + if "full" in arg_params: + name = arg_params["full"][2:] + else: + name = name[2:] + name = name.replace("-", "_") + + if "choices" in arg_params: + allowable_values = arg_params["choices"] + _type = "string" + if "type" in arg_params: + types = {"open": "file", "int": "int"} + _type = types[arg_params["type"]] + if ( + "action" in arg_params + and arg_params["action"] == "store_true" + ): + _type = "boolean" + + if "nargs" in arg_params: + if arg_params["nargs"] == "*": + allow_multiple = True + required = False + _type = "array" + if arg_params["nargs"] == "+": + allow_multiple = True + required = True + _type = "array" + if arg_params["nargs"] == "?": + allow_multiple = False + required = False + else: + allow_multiple = False + + if name == key_param: + param_type = "path" + required = True + allow_multiple = False + + if method in ["put", "post", "patch"]: + schema = operation["requestBody"]["content"][ + "multipart/form-data" + ]["schema"] + schema["properties"][name] = { + "type": _type, + "description": arg_params["help"], + } + if required: + schema["required"].append(name) + prop_schema = schema["properties"][name] + else: + parameters = { + "name": name, + "in": param_type, + "description": arg_params["help"], + "required": required, + "schema": { + "type": _type, + }, + "explode": allow_multiple, + } + prop_schema = parameters["schema"] + operation["parameters"].append(parameters) + + if allowable_values is not None: + prop_schema["enum"] = allowable_values + if "default" in arg_params: + prop_schema["default"] = arg_params["default"] + if arg_params.get("metavar") == "PASSWORD": + prop_schema["format"] = "password" + if arg_params.get("metavar") == "MAIL": + prop_schema["format"] = "mail" + # Those lines seems to slow swagger ui too much + # if 'pattern' in arg_params.get('extra', {}): + # prop_schema['pattern'] = arg_params['extra']['pattern'][0] + + resource_list["paths"][path][method.lower()] = operation + + # Includes subcategories + if "subcategories" in category_params: + convert_categories(category_params["subcategories"], category) + + del action_map["_global"] + convert_categories(action_map) + + openapi_json = json.dumps(resource_list) + # Save the OpenAPI json + with open(os.getcwd() + "/openapi.json", "w") as f: + f.write(openapi_json) + + openapi_js = f"var openapiJSON = {openapi_json}" + with open(os.getcwd() + "/openapi.js", "w") as f: + f.write(openapi_js) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/helpers/config b/helpers/config index fce215a30..77f118c5f 100644 --- a/helpers/config +++ b/helpers/config @@ -22,7 +22,7 @@ _ynh_app_config_get_one() { if [[ "$bind" == "settings" ]]; then ynh_die --message="File '${short_setting}' can't be stored in settings" fi - old[$short_setting]="$(ls "$(echo $bind | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" 2>/dev/null || echo YNH_NULL)" + old[$short_setting]="$(ls "$(echo $bind | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" 2>/dev/null || echo YNH_NULL)" file_hash[$short_setting]="true" # Get multiline text from settings or from a full file @@ -32,7 +32,7 @@ _ynh_app_config_get_one() { elif [[ "$bind" == *":"* ]]; then ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" else - old[$short_setting]="$(cat $(echo $bind | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2>/dev/null || echo YNH_NULL)" + old[$short_setting]="$(cat $(echo $bind | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2>/dev/null || echo YNH_NULL)" fi # Get value from a kind of key/value file @@ -47,7 +47,7 @@ _ynh_app_config_get_one() { bind_after="$(echo "${bind_key_}" | cut -d'>' -f1)" bind_key_="$(echo "${bind_key_}" | cut -d'>' -f2)" fi - local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" old[$short_setting]="$(ynh_read_var_in_file --file="${bind_file}" --key="${bind_key_}" --after="${bind_after}")" fi @@ -73,7 +73,7 @@ _ynh_app_config_apply_one() { if [[ "$bind" == "settings" ]]; then ynh_die --message="File '${short_setting}' can't be stored in settings" fi - local bind_file="$(echo "$bind" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + local bind_file="$(echo "$bind" | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" if [[ "${!short_setting}" == "" ]]; then ynh_backup_if_checksum_is_different --file="$bind_file" ynh_secure_remove --file="$bind_file" @@ -98,7 +98,7 @@ _ynh_app_config_apply_one() { if [[ "$bind" == *":"* ]]; then ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically in a variable file, you have to create custom getter/setter" fi - local bind_file="$(echo "$bind" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + local bind_file="$(echo "$bind" | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" ynh_backup_if_checksum_is_different --file="$bind_file" echo "${!short_setting}" >"$bind_file" ynh_store_file_checksum --file="$bind_file" --update_only @@ -113,7 +113,7 @@ _ynh_app_config_apply_one() { bind_after="$(echo "${bind_key_}" | cut -d'>' -f1)" bind_key_="$(echo "${bind_key_}" | cut -d'>' -f2)" fi - local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" + local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__INSTALL_DIR__@$install_dir@ | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" ynh_backup_if_checksum_is_different --file="$bind_file" ynh_write_var_in_file --file="${bind_file}" --key="${bind_key_}" --value="${!short_setting}" --after="${bind_after}" diff --git a/helpers/php b/helpers/php index da833ae9e..6119c4870 100644 --- a/helpers/php +++ b/helpers/php @@ -474,9 +474,9 @@ YNH_COMPOSER_VERSION=${YNH_COMPOSER_VERSION:-$YNH_DEFAULT_COMPOSER_VERSION} # Execute a command with Composer # -# usage: ynh_composer_exec [--phpversion=phpversion] [--workdir=$final_path] --commands="commands" +# usage: ynh_composer_exec [--phpversion=phpversion] [--workdir=$install_dir] --commands="commands" # | arg: -v, --phpversion - PHP version to use with composer -# | arg: -w, --workdir - The directory from where the command will be executed. Default $final_path. +# | arg: -w, --workdir - The directory from where the command will be executed. Default $install_dir or $final_path # | arg: -c, --commands - Commands to execute. # # Requires YunoHost version 4.2 or higher. @@ -489,7 +489,7 @@ ynh_composer_exec() { local commands # Manage arguments with getopts ynh_handle_getopts_args "$@" - workdir="${workdir:-$final_path}" + workdir="${workdir:-${install_dir:-$final_path}}" phpversion="${phpversion:-$YNH_PHP_VERSION}" COMPOSER_HOME="$workdir/.composer" COMPOSER_MEMORY_LIMIT=-1 \ diff --git a/helpers/utils b/helpers/utils index 3b1e9c6bb..344493ff3 100644 --- a/helpers/utils +++ b/helpers/utils @@ -192,6 +192,9 @@ ynh_setup_source() { # Extract source into the app dir mkdir --parents "$dest_dir" + if [ -n "${install_dir:-}" ] && [ "$dest_dir" == "$install_dir" ]; then + _ynh_apply_default_permissions $dest_dir + fi if [ -n "${final_path:-}" ] && [ "$dest_dir" == "$final_path" ]; then _ynh_apply_default_permissions $dest_dir fi @@ -330,7 +333,7 @@ ynh_local_curl() { # | arg: -d, --destination= - Destination of the config file # # examples: -# ynh_add_config --template=".env" --destination="$final_path/.env" use the template file "../conf/.env" +# ynh_add_config --template=".env" --destination="$install_dir/.env" use the template file "../conf/.env" # ynh_add_config --template="/etc/nginx/sites-available/default" --destination="etc/nginx/sites-available/mydomain.conf" # # The template can be by default the name of a file in the conf directory @@ -444,8 +447,10 @@ ynh_replace_vars() { ynh_replace_string --match_string="__NAMETOCHANGE__" --replace_string="$app" --target_file="$file" ynh_replace_string --match_string="__USER__" --replace_string="$app" --target_file="$file" fi + # Legacy if test -n "${final_path:-}"; then ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$file" + ynh_replace_string --match_string="__INSTALL_DIR__" --replace_string="$final_path" --target_file="$file" fi if test -n "${YNH_PHP_VERSION:-}"; then ynh_replace_string --match_string="__PHPVERSION__" --replace_string="$YNH_PHP_VERSION" --target_file="$file" diff --git a/locales/en.json b/locales/en.json index 826092d88..3467132a8 100644 --- a/locales/en.json +++ b/locales/en.json @@ -13,6 +13,7 @@ "app_already_installed": "{app} is already installed", "app_already_installed_cant_change_url": "This app is already installed. The URL cannot be changed just by this function. Check in `app changeurl` if it's available.", "app_already_up_to_date": "{app} is already up-to-date", + "app_arch_not_supported": "This app can only be installed on architectures {', '.join(required)} but your server architecture is {current}", "app_argument_choice_invalid": "Pick a valid value for argument '{name}': '{value}' is not among the available choices ({choices})", "app_argument_invalid": "Pick a valid value for the argument '{name}': {error}", "app_argument_password_no_default": "Error while parsing password argument '{name}': password argument can't have a default value for security reasons", @@ -26,6 +27,7 @@ "app_full_domain_unavailable": "Sorry, this app must be installed on a domain of its own, but other apps are already installed on the domain '{domain}'. You could use a subdomain dedicated to this app instead.", "app_id_invalid": "Invalid app ID", "app_install_failed": "Unable to install {app}: {error}", + "app_resource_failed": "Provisioning, deprovisioning, or updating resources for {app} failed: {error}", "app_install_files_invalid": "These files cannot be installed", "app_install_script_failed": "An error occurred inside the app installation script", "app_label_deprecated": "This command is deprecated! Please use the new command 'yunohost user permission update' to manage the app label.", @@ -39,6 +41,8 @@ "app_manifest_install_ask_password": "Choose an administration password for this app", "app_manifest_install_ask_path": "Choose the URL path (after the domain) where this app should be installed", "app_not_correctly_installed": "{app} seems to be incorrectly installed", + "app_not_enough_disk": "This app requires {required} free space.", + "app_not_enough_ram": "This app requires {required} RAM to install/upgrade but only {current} is available right now.", "app_not_installed": "Could not find {app} in the list of installed apps: {all_apps}", "app_not_properly_removed": "{app} has not been properly removed", "app_not_upgraded": "The app '{failed_app}' failed to upgrade, and as a consequence the following apps' upgrades have been cancelled: {apps}", @@ -61,6 +65,7 @@ "app_upgrade_several_apps": "The following apps will be upgraded: {apps}", "app_upgrade_some_app_failed": "Some apps could not be upgraded", "app_upgraded": "{app} upgraded", + "app_yunohost_version_not_supported": "This app requires YunoHost >= {required} but current installed version is {current}", "apps_already_up_to_date": "All apps are already up-to-date", "apps_catalog_failed_to_download": "Unable to download the {apps_catalog} app catalog: {error}", "apps_catalog_init_success": "App catalog system initialized!", @@ -157,10 +162,11 @@ "config_validate_email": "Should be a valid email", "config_validate_time": "Should be a valid time like HH:MM", "config_validate_url": "Should be a valid web URL", - "config_version_not_supported": "Config panel versions '{version}' are not supported.", "confirm_app_install_danger": "DANGER! This app is known to be still experimental (if not explicitly not working)! You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or breaks your system... If you are willing to take that risk anyway, type '{answers}'", "confirm_app_install_thirdparty": "DANGER! This app is not part of YunoHost's app catalog. Installing third-party apps may compromise the integrity and security of your system. You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or breaks your system... If you are willing to take that risk anyway, type '{answers}'", "confirm_app_install_warning": "Warning: This app may work, but is not well-integrated into YunoHost. Some features such as single sign-on and backup/restore might not be available. Install anyway? [{answers}] ", + "confirm_app_insufficient_ram": "DANGER! This app requires {required} RAM to install/upgrade but only {current} is available right now. Even if this app could run, its installation/upgrade process requires a large amount of RAM so your server may freeze and fail miserably. If you are willing to take that risk anyway, type '{answers}'", + "confirm_notifications_read": "WARNING: You should check the app notifications above before continuing, there might be important stuff to know. [{answers}]", "custom_app_url_required": "You must provide a URL to upgrade your custom app {app}", "danger": "Danger:", "diagnosis_apps_allgood": "All installed apps respect basic packaging practices", diff --git a/locales/fr.json b/locales/fr.json index bbed04a4b..33949f1fd 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -737,9 +737,16 @@ "global_settings_setting_smtp_allow_ipv6": "Autoriser l'IPv6", "password_too_long": "Veuillez choisir un mot de passe de moins de 127 caractères", "domain_cannot_add_muc_upload": "Vous ne pouvez pas ajouter de domaines commençant par 'muc.'. Ce type de nom est réservé à la fonction de chat XMPP multi-utilisateurs intégrée à YunoHost.", - "group_update_aliases": "Mise à jour des alias du groupe '{group}'.", + "group_update_aliases": "Mise à jour des alias du groupe '{group}'", "group_no_change": "Rien à mettre à jour pour le groupe '{group}'", "global_settings_setting_portal_theme": "Thème du portail", "global_settings_setting_portal_theme_help": "Pour plus d'informations sur la création de thèmes de portail personnalisés, voir https://yunohost.org/theming", - "global_settings_setting_passwordless_sudo": "Permettre aux administrateurs d'utiliser 'sudo' sans retaper leur mot de passe" + "global_settings_setting_passwordless_sudo": "Permettre aux administrateurs d'utiliser 'sudo' sans retaper leur mot de passe", + "app_arch_not_supported": "Cette application ne peut être installée que sur les architectures {', '.join(required)}. L'architecture de votre serveur est {current}", + "app_resource_failed": "L'allocation automatique des ressources (provisioning), la suppression d'accès à ces ressources (déprovisioning) ou la mise à jour des ressources pour {app} a échoué : {error}", + "confirm_app_insufficient_ram": "ATTENTION ! Cette application requiert {required} de RAM pour l'installation/mise à niveau mais il n'y a que {current} de disponible actuellement. Même si cette application pouvait fonctionner, son processus d'installation/mise à niveau nécessite une grande quantité de RAM. Votre serveur pourrait donc geler (freezer) et planter lamentablement. Si vous êtes prêt à prendre ce risque, tapez '{answers}'", + "app_not_enough_disk": "Cette application nécessite {required} d'espace libre.", + "app_not_enough_ram": "Cette application nécessite {required} de mémoire vive (RAM) pour être installée/mise à niveau mais seule {current} de mémoire est disponible actuellement.", + "app_yunohost_version_not_supported": "Cette application nécessite une version de YunoHost >= {required}. La version installée est {current}", + "confirm_notifications_read": "AVERTISSEMENT : Vous devriez vérifier les notifications de l'application susmentionnée avant de continuer, il pourrait y avoir des éléments d'information importants à connaître. [{answers}]" } \ No newline at end of file diff --git a/maintenance/missing_i18n_keys.py b/maintenance/missing_i18n_keys.py index 26d46e658..f49fc923e 100644 --- a/maintenance/missing_i18n_keys.py +++ b/maintenance/missing_i18n_keys.py @@ -151,6 +151,7 @@ def find_expected_string_keys(): global_config = toml.load(open(ROOT + "share/config_global.toml")) # Boring hard-coding because there's no simple other way idk settings_without_help_key = [ + "passwordless_sudo", "smtp_relay_host", "smtp_relay_password", "smtp_relay_port", diff --git a/share/actionsmap.yml b/share/actionsmap.yml index f6a64f265..835bb5d43 100644 --- a/share/actionsmap.yml +++ b/share/actionsmap.yml @@ -545,9 +545,7 @@ domain: action_help: Check the current main domain, or change it deprecated_alias: - maindomain - api: - - GET /domains/main - - PUT /domains//main + api: PUT /domains//main arguments: -n: full: --new-main-domain @@ -780,6 +778,10 @@ app: full: --with-categories help: Also return a list of app categories action: store_true + -a: + full: --with-antifeatures + help: Also return a list of antifeatures categories + action: store_true ### app_search() search: @@ -795,6 +797,10 @@ app: arguments: app: help: Name, local path or git URL of the app to fetch the manifest of + -s: + full: --with-screenshot + help: Also return a base64 screenshot if any (API only) + action: store_true ### app_list() list: @@ -965,6 +971,17 @@ app: help: Undo redirection action: store_true + ### app_dismiss_notification + dismiss-notification: + hide_in_help: True + action_help: Dismiss post_install or post_upgrade notification + api: PUT /apps//dismiss_notification/ + arguments: + app: + help: App ID to dismiss notification for + name: + help: Notification name, either post_install or post_upgrade + ### app_ssowatconf() ssowatconf: action_help: Regenerate SSOwat configuration file diff --git a/src/app.py b/src/app.py index 0b8544292..7458808fc 100644 --- a/src/app.py +++ b/src/app.py @@ -29,7 +29,7 @@ import subprocess import tempfile import copy from collections import OrderedDict -from typing import List, Tuple, Dict, Any +from typing import List, Tuple, Dict, Any, Iterator, Optional from packaging import version from moulinette import Moulinette, m18n @@ -71,6 +71,7 @@ from yunohost.app_catalog import ( # noqa app_catalog, app_search, _load_apps_catalog, + APPS_CATALOG_LOGOS, ) logger = getActionLogger("yunohost.app") @@ -151,6 +152,13 @@ def app_info(app, full=False, upgradable=False): absolute_app_name, _ = _parse_app_instance_name(app) from_catalog = _load_apps_catalog()["apps"].get(absolute_app_name, {}) + # Check if $app.png exists in the app logo folder, this is a trick to be able to easily customize the logo + # of an app just by creating $app.png (instead of the hash.png) in the corresponding folder + ret["logo"] = ( + app + if os.path.exists(f"{APPS_CATALOG_LOGOS}/{app}.png") + else from_catalog.get("logo_hash") + ) ret["upgradable"] = _app_upgradable({**ret, "from_catalog": from_catalog}) if ret["upgradable"] == "yes": @@ -164,6 +172,8 @@ def app_info(app, full=False, upgradable=False): ret["current_version"] = f" ({current_revision})" ret["new_version"] = f" ({new_revision})" + ret["settings"] = settings + if not full: return ret @@ -175,7 +185,6 @@ def app_info(app, full=False, upgradable=False): ret["manifest"]["install"] = _set_default_ask_questions( ret["manifest"].get("install", {}) ) - ret["settings"] = settings ret["from_catalog"] = from_catalog @@ -185,6 +194,15 @@ def app_info(app, full=False, upgradable=False): ret["manifest"]["doc"][pagename][lang] = _hydrate_app_template( content, settings ) + + # Filter dismissed notification + ret["manifest"]["notifications"] = { + k: v + for k, v in ret["manifest"]["notifications"].items() + if not _notification_is_dismissed(k, settings) + } + + # Hydrate notifications (also filter uneeded post_upgrade notification based on version) for step, notifications in ret["manifest"]["notifications"].items(): for name, content_per_lang in notifications.items(): for lang, content in content_per_lang.items(): @@ -526,6 +544,8 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False if len(apps) > 1: logger.info(m18n.n("app_upgrade_several_apps", apps=", ".join(apps))) + notifications = {} + for number, app_instance_name in enumerate(apps): logger.info(m18n.n("app_upgrade_app_name", app=app_instance_name)) @@ -587,7 +607,30 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False upgrade_type = "UPGRADE_FULL" # Check requirements - _check_manifest_requirements(manifest, action="upgrade") + for name, passed, values, err in _check_manifest_requirements( + manifest, action="upgrade" + ): + if not passed: + if name == "ram": + # i18n: confirm_app_insufficient_ram + _ask_confirmation( + "confirm_app_insufficient_ram", params=values, force=force + ) + else: + raise YunohostValidationError(err, **values) + + # Display pre-upgrade notifications and ask for simple confirm + if ( + manifest["notifications"]["PRE_UPGRADE"] + and Moulinette.interface.type == "cli" + ): + settings = _get_app_settings(app_instance_name) + notifications = _filter_and_hydrate_notifications( + manifest["notifications"]["PRE_UPGRADE"], + current_version=app_current_version, + data=settings, + ) + _display_notifications(notifications, force=force) if manifest["packaging_format"] >= 2: if no_safety_backup: @@ -650,13 +693,12 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False if manifest["packaging_format"] >= 2: from yunohost.utils.resources import AppResourceManager - try: - AppResourceManager( - app_instance_name, wanted=manifest, current=app_dict["manifest"] - ).apply(rollback_if_failure=True) - except Exception: - # FIXME : improve error handling .... - raise + AppResourceManager( + app_instance_name, wanted=manifest, current=app_dict["manifest"] + ).apply( + rollback_and_raise_exception_if_failure=True, + operation_logger=operation_logger, + ) # Execute the app upgrade script upgrade_failed = True @@ -771,6 +813,24 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False # So much win logger.success(m18n.n("app_upgraded", app=app_instance_name)) + # Format post-upgrade notifications + if manifest["notifications"]["POST_UPGRADE"]: + # Get updated settings to hydrate notifications + settings = _get_app_settings(app_instance_name) + notifications = _filter_and_hydrate_notifications( + manifest["notifications"]["POST_UPGRADE"], + current_version=app_current_version, + data=settings, + ) + if Moulinette.interface.type == "cli": + # ask for simple confirm + _display_notifications(notifications, force=force) + + # Reset the dismiss flag for post upgrade notification + app_setting( + app_instance_name, "_dismiss_notification_post_upgrade", delete=True + ) + hook_callback("post_app_upgrade", env=env_dict) operation_logger.success() @@ -778,16 +838,50 @@ def app_upgrade(app=[], url=None, file=None, force=False, no_safety_backup=False logger.success(m18n.n("upgrade_complete")) + if Moulinette.interface.type == "api": + return {"notifications": {"POST_UPGRADE": notifications}} -def app_manifest(app): + +def app_manifest(app, with_screenshot=False): manifest, extracted_app_folder = _extract_app(app) - shutil.rmtree(extracted_app_folder) - raw_questions = manifest.get("install", {}).values() manifest["install"] = hydrate_questions_with_choices(raw_questions) + # Add a base64 image to be displayed in web-admin + if with_screenshot and Moulinette.interface.type == "api": + import base64 + + manifest["screenshot"] = None + screenshots_folder = os.path.join(extracted_app_folder, "doc", "screenshots") + + if os.path.exists(screenshots_folder): + with os.scandir(screenshots_folder) as it: + for entry in it: + ext = os.path.splitext(entry.name)[1].replace(".", "").lower() + if entry.is_file() and ext in ("png", "jpg", "jpeg", "webp", "gif"): + with open(entry.path, "rb") as img_file: + data = base64.b64encode(img_file.read()).decode("utf-8") + manifest[ + "screenshot" + ] = f"data:image/{ext};charset=utf-8;base64,{data}" + break + + shutil.rmtree(extracted_app_folder) + + manifest["requirements"] = {} + for name, passed, values, err in _check_manifest_requirements( + manifest, action="install" + ): + if Moulinette.interface.type == "api": + manifest["requirements"][name] = { + "pass": passed, + "values": values, + } + else: + manifest["requirements"][name] = "ok" if passed else m18n.n(err, **values) + return manifest @@ -807,19 +901,9 @@ def _confirm_app_install(app, force=False): # i18n: confirm_app_install_thirdparty if quality in ["danger", "thirdparty"]: - answer = Moulinette.prompt( - m18n.n("confirm_app_install_" + quality, answers="Yes, I understand"), - color="red", - ) - if answer != "Yes, I understand": - raise YunohostError("aborting") - + _ask_confirmation("confirm_app_install_" + quality, kind="hard") else: - answer = Moulinette.prompt( - m18n.n("confirm_app_install_" + quality, answers="Y/N"), color="yellow" - ) - if answer.upper() != "Y": - raise YunohostError("aborting") + _ask_confirmation("confirm_app_install_" + quality, kind="soft") @is_unit_operation() @@ -867,12 +951,11 @@ def app_install( manifest, extracted_app_folder = _extract_app(app) # Display pre_install notices in cli mode - if manifest["notifications"]["pre_install"] and Moulinette.interface.type == "cli": - for notice in manifest["notifications"]["pre_install"].values(): - # Should we render the markdown maybe? idk - print("==========") - print(_value_for_locale(notice)) - print("==========") + if manifest["notifications"]["PRE_INSTALL"] and Moulinette.interface.type == "cli": + notifications = _filter_and_hydrate_notifications( + manifest["notifications"]["PRE_INSTALL"] + ) + _display_notifications(notifications, force=force) packaging_format = manifest["packaging_format"] @@ -883,7 +966,17 @@ def app_install( app_id = manifest["id"] # Check requirements - _check_manifest_requirements(manifest, action="install") + for name, passed, values, err in _check_manifest_requirements( + manifest, action="install" + ): + if not passed: + if name == "ram": + _ask_confirmation( + "confirm_app_insufficient_ram", params=values, force=force + ) + else: + raise YunohostValidationError(err, **values) + _assert_system_is_sane_for_app(manifest, "pre") # Check if app can be forked @@ -961,16 +1054,18 @@ def app_install( recursive=True, ) + # Override manifest name by given label + # This info is also later picked-up by the 'permission' resource initialization + if label: + manifest["name"] = label + if packaging_format >= 2: from yunohost.utils.resources import AppResourceManager - try: - AppResourceManager(app_instance_name, wanted=manifest, current={}).apply( - rollback_if_failure=True - ) - except Exception: - # FIXME : improve error handling .... - raise + AppResourceManager(app_instance_name, wanted=manifest, current={}).apply( + rollback_and_raise_exception_if_failure=True, + operation_logger=operation_logger, + ) else: # Initialize the main permission for the app # The permission is initialized with no url associated, and with tile disabled @@ -980,7 +1075,7 @@ def app_install( permission_create( app_instance_name + ".main", allowed=["all_users"], - label=label if label else manifest["name"], + label=manifest["name"], show_tile=False, protected=False, ) @@ -1034,6 +1129,9 @@ def app_install( "Packagers /!\\ This app manually modified some system configuration files! This should not happen! If you need to do so, you should implement a proper conf_regen hook. Those configuration were affected:\n - " + "\n -".join(manually_modified_files_by_app) ) + # Actually forbid this for app packaging >= 2 + if packaging_format >= 2: + broke_the_system = True # If the install failed or broke the system, we remove it if install_failed or broke_the_system: @@ -1083,13 +1181,9 @@ def app_install( if packaging_format >= 2: from yunohost.utils.resources import AppResourceManager - try: - AppResourceManager( - app_instance_name, wanted={}, current=manifest - ).apply(rollback_if_failure=False) - except Exception: - # FIXME : improve error handling .... - raise + AppResourceManager( + app_instance_name, wanted={}, current=manifest + ).apply(rollback_and_raise_exception_if_failure=False) else: # Remove all permission in LDAP for permission_name in user_permission_list()["permissions"].keys(): @@ -1130,19 +1224,23 @@ def app_install( logger.success(m18n.n("installation_complete")) + # Get the generated settings to hydrate notifications + settings = _get_app_settings(app_instance_name) + notifications = _filter_and_hydrate_notifications( + manifest["notifications"]["POST_INSTALL"], data=settings + ) + # Display post_install notices in cli mode - if manifest["notifications"]["post_install"] and Moulinette.interface.type == "cli": - # (Call app_info to get the version hydrated with settings) - infos = app_info(app_instance_name, full=True) - for notice in infos["manifest"]["notifications"]["post_install"].values(): - # Should we render the markdown maybe? idk - print("==========") - print(_value_for_locale(notice)) - print("==========") + if notifications and Moulinette.interface.type == "cli": + _display_notifications(notifications, force=force) # Call postinstall hook hook_callback("post_app_install", env=env_dict) + # Return hydrated post install notif for API + if Moulinette.interface.type == "api": + return {"notifications": notifications} + @is_unit_operation() def app_remove(operation_logger, app, purge=False): @@ -1210,15 +1308,11 @@ def app_remove(operation_logger, app, purge=False): packaging_format = manifest["packaging_format"] if packaging_format >= 2: - try: - from yunohost.utils.resources import AppResourceManager + from yunohost.utils.resources import AppResourceManager - AppResourceManager(app, wanted={}, current=manifest).apply( - rollback_if_failure=False - ) - except Exception: - # FIXME : improve error handling .... - raise + AppResourceManager(app, wanted={}, current=manifest).apply( + rollback_and_raise_exception_if_failure=False, purge_data_dir=purge + ) else: # Remove all permission in LDAP for permission_name in user_permission_list(apps=[app])["permissions"].keys(): @@ -1501,6 +1595,8 @@ def app_ssowatconf(): } redirected_urls = {} + apps_using_remote_user_var_in_nginx = check_output('grep -nri \'$remote_user\' /etc/yunohost/apps/*/conf/*nginx*conf | awk -F/ \'{print $5}\' || true').strip().split("\n") + for app in _installed_apps(): app_settings = read_yaml(APPS_SETTING_PATH + app + "/settings.yml") or {} @@ -1539,7 +1635,10 @@ def app_ssowatconf(): if not uris: continue + app_id = perm_name.split(".")[0] + permissions[perm_name] = { + "use_remote_user_var_in_nginx_conf": app_id in apps_using_remote_user_var_in_nginx, "users": perm_info["corresponding_users"], "label": perm_info["label"], "show_tile": perm_info["show_tile"] @@ -1616,8 +1715,15 @@ def app_config_get(app, key="", full=False, export=False): else: mode = "classic" - config_ = AppConfigPanel(app) - return config_.get(key, mode) + try: + config_ = AppConfigPanel(app) + return config_.get(key, mode) + except YunohostValidationError as e: + if Moulinette.interface.type == "api" and e.key == "config_no_panel": + # Be more permissive when no config panel found + return {} + else: + raise @is_unit_operation() @@ -1690,6 +1796,7 @@ ynh_app_config_run $1 "app": app, "app_instance_nb": str(app_instance_nb), "final_path": settings.get("final_path", ""), + "install_dir": settings.get("install_dir", ""), "YNH_APP_BASEDIR": os.path.join(APPS_SETTING_PATH, app), } ) @@ -1924,6 +2031,7 @@ def _get_manifest_of_app(path): def _parse_app_doc_and_notifications(path): doc = {} + notification_names = ["PRE_INSTALL", "POST_INSTALL", "PRE_UPGRADE", "POST_UPGRADE"] for filepath in glob.glob(os.path.join(path, "doc") + "/*.md"): @@ -1934,7 +2042,12 @@ def _parse_app_doc_and_notifications(path): if not m: # FIXME: shall we display a warning ? idk continue + pagename, lang = m.groups() + + if pagename in notification_names: + continue + lang = lang.strip("_") if lang else "en" if pagename not in doc: @@ -1943,11 +2056,9 @@ def _parse_app_doc_and_notifications(path): notifications = {} - for step in ["pre_install", "post_install", "pre_upgrade", "post_upgrade"]: + for step in notification_names: notifications[step] = {} - for filepath in glob.glob( - os.path.join(path, "doc", "notifications", f"{step}*.md") - ): + for filepath in glob.glob(os.path.join(path, "doc", f"{step}*.md")): m = re.match(step + "(_[a-z]{2,3})?.md", filepath.split("/")[-1]) if not m: continue @@ -1957,9 +2068,7 @@ def _parse_app_doc_and_notifications(path): notifications[step][pagename] = {} notifications[step][pagename][lang] = read_file(filepath).strip() - for filepath in glob.glob( - os.path.join(path, "doc", "notifications", f"{step}.d") + "/*.md" - ): + for filepath in glob.glob(os.path.join(path, "doc", f"{step}.d") + "/*.md"): m = re.match( r"([A-Za-z0-9\.\~]*)(_[a-z]{2,3})?.md", filepath.split("/")[-1] ) @@ -2007,12 +2116,12 @@ def _convert_v1_manifest_to_v2(manifest): .replace(">", "") .replace("=", "") .replace(" ", ""), - "architectures": "all", + "architectures": "?", "multi_instance": manifest.get("multi_instance", False), "ldap": "?", "sso": "?", - "disk": "50M", - "ram": {"build": "50M", "runtime": "10M"}, + "disk": "?", + "ram": {"build": "?", "runtime": "?"}, } maintainers = manifest.get("maintainer", {}) @@ -2196,19 +2305,21 @@ def _extract_app(src: str) -> Tuple[Dict, str]: url = app_info["git"]["url"] branch = app_info["git"]["branch"] revision = str(app_info["git"]["revision"]) - return _extract_app_from_gitrepo(url, branch, revision, app_info) + return _extract_app_from_gitrepo( + url, branch=branch, revision=revision, app_info=app_info + ) # App is a git repo url elif _is_app_repo_url(src): url = src.strip().strip("/") - branch = "master" - revision = "HEAD" # gitlab urls may look like 'https://domain/org/group/repo/-/tree/testing' # compated to github urls looking like 'https://domain/org/repo/tree/testing' if "/-/" in url: url = url.replace("/-/", "/") if "/tree/" in url: url, branch = url.split("/tree/", 1) - return _extract_app_from_gitrepo(url, branch, revision, {}) + else: + branch = None + return _extract_app_from_gitrepo(url, branch=branch) # App is a local folder elif os.path.exists(src): return _extract_app_from_folder(src) @@ -2257,13 +2368,53 @@ def _extract_app_from_folder(path: str) -> Tuple[Dict, str]: logger.debug(m18n.n("done")) manifest["remote"] = {"type": "file", "path": path} + manifest["quality"] = {"level": -1, "state": "thirdparty"} + manifest["antifeatures"] = [] + manifest["potential_alternative_to"] = [] + return manifest, extracted_app_folder def _extract_app_from_gitrepo( - url: str, branch: str, revision: str, app_info: Dict = {} + url: str, branch: Optional[str] = None, revision: str = "HEAD", app_info: Dict = {} ) -> Tuple[Dict, str]: + logger.debug("Checking default branch") + + try: + git_ls_remote = check_output( + ["git", "ls-remote", "--symref", url, "HEAD"], + env={"GIT_TERMINAL_PROMPT": "0", "LC_ALL": "C"}, + shell=False, + ) + except Exception as e: + logger.error(str(e)) + raise YunohostError("app_sources_fetch_failed") + + if not branch: + default_branch = None + try: + for line in git_ls_remote.split("\n"): + # Look for the line formated like : + # ref: refs/heads/master HEAD + if "ref: refs/heads/" in line: + line = line.replace("/", " ").replace("\t", " ") + default_branch = line.split()[3] + except Exception: + pass + + if not default_branch: + logger.warning("Failed to parse default branch, trying 'main'") + branch = "main" + else: + if default_branch in ["testing", "dev"]: + logger.warning( + f"Trying 'master' branch instead of default '{default_branch}'" + ) + branch = "master" + else: + branch = default_branch + logger.debug(m18n.n("downloading")) extracted_app_folder = _make_tmp_workdir_for_app() @@ -2303,9 +2454,36 @@ def _extract_app_from_gitrepo( manifest["remote"]["revision"] = revision manifest["lastUpdate"] = app_info.get("lastUpdate") + manifest["quality"] = { + "level": app_info.get("level", -1), + "state": app_info.get("state", "thirdparty"), + } + manifest["antifeatures"] = app_info.get("antifeatures", []) + manifest["potential_alternative_to"] = app_info.get("potential_alternative_to", []) + return manifest, extracted_app_folder +def _list_upgradable_apps(): + upgradable_apps = list(app_list(upgradable=True)["apps"]) + + # Retrieve next manifest pre_upgrade notifications + for app in upgradable_apps: + absolute_app_name, _ = _parse_app_instance_name(app["id"]) + manifest, extracted_app_folder = _extract_app(absolute_app_name) + app["notifications"] = {} + if manifest["notifications"]["PRE_UPGRADE"]: + app["notifications"]["PRE_UPGRADE"] = _filter_and_hydrate_notifications( + manifest["notifications"]["PRE_UPGRADE"], + app["current_version"], + app["settings"], + ) + del app["settings"] + shutil.rmtree(extracted_app_folder) + + return upgradable_apps + + # # ############################### # # Small utilities # @@ -2354,74 +2532,100 @@ def _get_all_installed_apps_id(): return all_apps_ids_formatted -def _check_manifest_requirements(manifest: Dict, action: str): +def _check_manifest_requirements( + manifest: Dict, action: str = "" +) -> Iterator[Tuple[str, bool, object, str]]: """Check if required packages are met from the manifest""" + app_id = manifest["id"] + logger.debug(m18n.n("app_requirements_checking", app=app_id)) + + # Packaging format if manifest["packaging_format"] not in [1, 2]: raise YunohostValidationError("app_packaging_format_not_supported") - app_id = manifest["id"] - - logger.debug(m18n.n("app_requirements_checking", app=app_id)) - - # Yunohost version requirement - yunohost_requirement = version.parse( - manifest["integration"]["yunohost"].strip(">= ") or "4.3" + # Yunohost version + required_yunohost_version = ( + manifest["integration"].get("yunohost", "4.3").strip(">= ") ) - yunohost_installed_version = version.parse( - get_ynh_package_version("yunohost")["version"] + current_yunohost_version = get_ynh_package_version("yunohost")["version"] + + yield ( + "required_yunohost_version", + version.parse(required_yunohost_version) + <= version.parse(current_yunohost_version), + {"current": current_yunohost_version, "required": required_yunohost_version}, + "app_yunohost_version_not_supported", # i18n: app_yunohost_version_not_supported ) - if yunohost_requirement > yunohost_installed_version: - # FIXME : i18n - raise YunohostValidationError( - f"This app requires Yunohost >= {yunohost_requirement} but current installed version is {yunohost_installed_version}" - ) # Architectures arch_requirement = manifest["integration"]["architectures"] - if arch_requirement != "all": - arch = system_arch() - if arch not in arch_requirement: - # FIXME: i18n - raise YunohostValidationError( - f"This app can only be installed on architectures {', '.join(arch_requirement)} but your server architecture is {arch}" - ) + arch = system_arch() + + yield ( + "arch", + arch_requirement in ["all", "?"] or arch in arch_requirement, + {"current": arch, "required": arch_requirement}, + "app_arch_not_supported", # i18n: app_arch_not_supported + ) # Multi-instance - if action == "install" and manifest["integration"]["multi_instance"] is False: - apps = _installed_apps() - sibling_apps = [a for a in apps if a == app_id or a.startswith(f"{app_id}__")] - if len(sibling_apps) > 0: - raise YunohostValidationError("app_already_installed", app=app_id) + if action == "install": + multi_instance = manifest["integration"]["multi_instance"] is True + if not multi_instance: + apps = _installed_apps() + sibling_apps = [ + a for a in apps if a == app_id or a.startswith(f"{app_id}__") + ] + multi_instance = len(sibling_apps) == 0 + + yield ( + "install", + multi_instance, + {"app": app_id}, + "app_already_installed", # i18n: app_already_installed + ) # Disk if action == "install": - disk_requirement = manifest["integration"]["disk"] - - if free_space_in_directory("/") <= human_to_binary( - disk_requirement - ) or free_space_in_directory("/var") <= human_to_binary(disk_requirement): - # FIXME : i18m - raise YunohostValidationError( - f"This app requires {disk_requirement} free space." + root_free_space = free_space_in_directory("/") + var_free_space = free_space_in_directory("/var") + if manifest["integration"]["disk"] == "?": + has_enough_disk = True + else: + disk_req_bin = human_to_binary(manifest["integration"]["disk"]) + has_enough_disk = ( + root_free_space > disk_req_bin and var_free_space > disk_req_bin ) + free_space = binary_to_human(min(root_free_space, var_free_space)) - # Ram for build - ram_build_requirement = manifest["integration"]["ram"]["build"] - # Is "include_swap" really useful ? We should probably decide wether to always include it or not instead - ram_include_swap = manifest["integration"]["ram"].get("include_swap", False) - - ram, swap = ram_available() - if ram_include_swap: - ram += swap - - if ram < human_to_binary(ram_build_requirement): - # FIXME : i18n - ram_human = binary_to_human(ram) - raise YunohostValidationError( - f"This app requires {ram_build_requirement} RAM to install/upgrade but only {ram_human} is available right now." + yield ( + "disk", + has_enough_disk, + {"current": free_space, "required": manifest["integration"]["disk"]}, + "app_not_enough_disk", # i18n: app_not_enough_disk ) + # Ram + ram_requirement = manifest["integration"]["ram"] + ram, swap = ram_available() + # Is "include_swap" really useful ? We should probably decide wether to always include it or not instead + if ram_requirement.get("include_swap", False): + ram += swap + can_build = ram_requirement["build"] == "?" or ram > human_to_binary( + ram_requirement["build"] + ) + can_run = ram_requirement["runtime"] == "?" or ram > human_to_binary( + ram_requirement["runtime"] + ) + + yield ( + "ram", + can_build and can_run, + {"current": binary_to_human(ram), "required": ram_requirement["build"]}, + "app_not_enough_ram", # i18n: app_not_enough_ram + ) + def _guess_webapp_path_requirement(app_folder: str) -> str: @@ -2740,3 +2944,111 @@ def _assert_system_is_sane_for_app(manifest, when): raise YunohostValidationError("dpkg_is_broken") elif when == "post": raise YunohostError("this_action_broke_dpkg") + + +def app_dismiss_notification(app, name): + + assert isinstance(name, str) + name = name.lower() + assert name in ["post_install", "post_upgrade"] + _assert_is_installed(app) + + app_setting(app, f"_dismiss_notification_{name}", value="1") + + +def _notification_is_dismissed(name, settings): + # Check for _dismiss_notiication_$name setting and also auto-dismiss + # notifications after one week (otherwise people using mostly CLI would + # never really dismiss the notification and it would be displayed forever) + + if name == "POST_INSTALL": + return ( + settings.get("_dismiss_notification_post_install") + or (int(time.time()) - settings.get("install_time", 0)) / (24 * 3600) > 7 + ) + elif name == "POST_UPGRADE": + # Check on update_time also implicitly prevent the post_upgrade notification + # from being displayed after install, because update_time is only set during upgrade + return ( + settings.get("_dismiss_notification_post_upgrade") + or (int(time.time()) - settings.get("update_time", 0)) / (24 * 3600) > 7 + ) + else: + return False + + +def _filter_and_hydrate_notifications(notifications, current_version=None, data={}): + def is_version_more_recent_than_current_version(name): + # Boring code to handle the fact that "0.1 < 9999~ynh1" is False + + if "~" in name: + return version.parse(name) > version.parse(current_version) + else: + return version.parse(name) > version.parse(current_version.split("~")[0]) + + return { + # Should we render the markdown maybe? idk + name: _hydrate_app_template(_value_for_locale(content_per_lang), data) + for name, content_per_lang in notifications.items() + if current_version is None + or name == "main" + or is_version_more_recent_than_current_version(name) + } + + +def _display_notifications(notifications, force=False): + if not notifications: + return + + for name, content in notifications.items(): + print("==========") + print(content) + print("==========") + + # i18n: confirm_notifications_read + _ask_confirmation("confirm_notifications_read", kind="simple", force=force) + + +# FIXME: move this to Moulinette +def _ask_confirmation( + question: str, + params: dict = {}, + kind: str = "hard", + force: bool = False, +): + """ + Ask confirmation + + Keyword argument: + question -- m18n key or string + params -- dict of values passed to the string formating + kind -- "hard": ask with "Yes, I understand", "soft": "Y/N", "simple": "press enter" + force -- Will not ask for confirmation + + """ + if force or Moulinette.interface.type == "api": + return + + # If ran from the CLI in a non-interactive context, + # skip confirmation (except in hard mode) + if not os.isatty(1) and kind in ["simple", "soft"]: + return + if kind == "simple": + answer = Moulinette.prompt( + m18n.n(question, answers="Press enter to continue", **params), + color="yellow", + ) + answer = True + elif kind == "soft": + answer = Moulinette.prompt( + m18n.n(question, answers="Y/N", **params), color="yellow" + ) + answer = answer.upper() == "Y" + else: + answer = Moulinette.prompt( + m18n.n(question, answers="Yes, I understand", **params), color="red" + ) + answer = answer == "Yes, I understand" + + if not answer: + raise YunohostError("aborting") diff --git a/src/app_catalog.py b/src/app_catalog.py index 22599a5a5..5d4378544 100644 --- a/src/app_catalog.py +++ b/src/app_catalog.py @@ -18,6 +18,7 @@ # import os import re +import hashlib from moulinette import m18n from moulinette.utils.log import getActionLogger @@ -36,17 +37,18 @@ from yunohost.utils.error import YunohostError logger = getActionLogger("yunohost.app_catalog") APPS_CATALOG_CACHE = "/var/cache/yunohost/repo" +APPS_CATALOG_LOGOS = "/usr/share/yunohost/applogos" APPS_CATALOG_CONF = "/etc/yunohost/apps_catalog.yml" APPS_CATALOG_API_VERSION = 3 APPS_CATALOG_DEFAULT_URL = "https://app.yunohost.org/default" -def app_catalog(full=False, with_categories=False): +def app_catalog(full=False, with_categories=False, with_antifeatures=False): """ Return a dict of apps available to installation from Yunohost's app catalog """ - from yunohost.app import _installed_apps, _set_default_ask_questions + from yunohost.app import _installed_apps # Get app list from catalog cache catalog = _load_apps_catalog() @@ -65,28 +67,38 @@ def app_catalog(full=False, with_categories=False): "description": infos["manifest"]["description"], "level": infos["level"], } - else: - infos["manifest"]["install"] = _set_default_ask_questions( - infos["manifest"].get("install", {}) - ) - # Trim info for categories if not using --full - for category in catalog["categories"]: - category["title"] = _value_for_locale(category["title"]) - category["description"] = _value_for_locale(category["description"]) - for subtags in category.get("subtags", []): - subtags["title"] = _value_for_locale(subtags["title"]) + _catalog = {"apps": catalog["apps"]} - if not full: - catalog["categories"] = [ - {"id": c["id"], "description": c["description"]} - for c in catalog["categories"] - ] + if with_categories: + for category in catalog["categories"]: + category["title"] = _value_for_locale(category["title"]) + category["description"] = _value_for_locale(category["description"]) + for subtags in category.get("subtags", []): + subtags["title"] = _value_for_locale(subtags["title"]) - if not with_categories: - return {"apps": catalog["apps"]} - else: - return {"apps": catalog["apps"], "categories": catalog["categories"]} + if not full: + catalog["categories"] = [ + {"id": c["id"], "description": c["description"]} + for c in catalog["categories"] + ] + + _catalog["categories"] = catalog["categories"] + + if with_antifeatures: + for antifeature in catalog["antifeatures"]: + antifeature["title"] = _value_for_locale(antifeature["title"]) + antifeature["description"] = _value_for_locale(antifeature["description"]) + + if not full: + catalog["antifeatures"] = [ + {"id": a["id"], "description": a["description"]} + for a in catalog["antifeatures"] + ] + + _catalog["antifeatures"] = catalog["antifeatures"] + + return _catalog def app_search(string): @@ -172,6 +184,9 @@ def _update_apps_catalog(): logger.debug("Initialize folder for apps catalog cache") mkdir(APPS_CATALOG_CACHE, mode=0o750, parents=True, uid="root") + if not os.path.exists(APPS_CATALOG_LOGOS): + mkdir(APPS_CATALOG_LOGOS, mode=0o755, parents=True, uid="root") + for apps_catalog in apps_catalog_list: if apps_catalog["url"] is None: continue @@ -202,6 +217,46 @@ def _update_apps_catalog(): raw_msg=True, ) + # Download missing app logos + logos_to_download = [] + for app, infos in apps_catalog_content["apps"].items(): + logo_hash = infos.get("logo_hash") + if not logo_hash or os.path.exists(f"{APPS_CATALOG_LOGOS}/{logo_hash}.png"): + continue + logos_to_download.append(logo_hash) + + if len(logos_to_download) > 20: + logger.info( + f"(Will fetch {len(logos_to_download)} logos, this may take a couple minutes)" + ) + + import requests + from multiprocessing.pool import ThreadPool + + def fetch_logo(logo_hash): + try: + r = requests.get( + f"{apps_catalog['url']}/v{APPS_CATALOG_API_VERSION}/logos/{logo_hash}.png", + timeout=10, + ) + assert ( + r.status_code == 200 + ), f"Got status code {r.status_code}, expected 200" + if hashlib.sha256(r.content).hexdigest() != logo_hash: + raise Exception( + f"Found inconsistent hash while downloading logo {logo_hash}" + ) + open(f"{APPS_CATALOG_LOGOS}/{logo_hash}.png", "wb").write(r.content) + return True + except Exception as e: + logger.debug(f"Failed to download logo {logo_hash} : {e}") + return False + + results = ThreadPool(8).imap_unordered(fetch_logo, logos_to_download) + for result in results: + # Is this even needed to iterate on the results ? + pass + logger.success(m18n.n("apps_catalog_update_success")) @@ -211,7 +266,7 @@ def _load_apps_catalog(): corresponding to all known apps and categories """ - merged_catalog = {"apps": {}, "categories": []} + merged_catalog = {"apps": {}, "categories": [], "antifeatures": []} for apps_catalog_id in [L["id"] for L in _read_apps_catalog_list()]: @@ -261,7 +316,9 @@ def _load_apps_catalog(): info["repository"] = apps_catalog_id merged_catalog["apps"][app] = info - # Annnnd categories - merged_catalog["categories"] += apps_catalog_content["categories"] + # Annnnd categories + antifeatures + # (we use .get here, only because the dev catalog doesnt include the categories/antifeatures keys) + merged_catalog["categories"] += apps_catalog_content.get("categories", []) + merged_catalog["antifeatures"] += apps_catalog_content.get("antifeatures", []) return merged_catalog diff --git a/src/backup.py b/src/backup.py index 78d52210b..c3e47bddc 100644 --- a/src/backup.py +++ b/src/backup.py @@ -1518,13 +1518,10 @@ class RestoreManager: if manifest["packaging_format"] >= 2: from yunohost.utils.resources import AppResourceManager - try: - AppResourceManager( - app_instance_name, wanted=manifest, current={} - ).apply(rollback_if_failure=True) - except Exception: - # FIXME : improve error handling .... - raise + AppResourceManager(app_instance_name, wanted=manifest, current={}).apply( + rollback_and_raise_exception_if_failure=True, + operation_logger=operation_logger, + ) # Execute the app install script restore_failed = True diff --git a/src/firewall.py b/src/firewall.py index 8e0e70e99..6cf68f1f7 100644 --- a/src/firewall.py +++ b/src/firewall.py @@ -32,7 +32,13 @@ logger = getActionLogger("yunohost.firewall") def firewall_allow( - protocol, port, ipv4_only=False, ipv6_only=False, no_upnp=False, no_reload=False + protocol, + port, + ipv4_only=False, + ipv6_only=False, + no_upnp=False, + no_reload=False, + reload_only_if_change=False, ): """ Allow connections on a port @@ -70,14 +76,20 @@ def firewall_allow( "ipv6", ] + changed = False + for p in protocols: # Iterate over IP versions to add port for i in ipvs: if port not in firewall[i][p]: firewall[i][p].append(port) + changed = True else: ipv = "IPv%s" % i[3] - logger.warning(m18n.n("port_already_opened", port=port, ip_version=ipv)) + if not reload_only_if_change: + logger.warning( + m18n.n("port_already_opened", port=port, ip_version=ipv) + ) # Add port forwarding with UPnP if not no_upnp and port not in firewall["uPnP"][p]: firewall["uPnP"][p].append(port) @@ -89,12 +101,18 @@ def firewall_allow( # Update and reload firewall _update_firewall_file(firewall) - if not no_reload: + if not no_reload or (reload_only_if_change and changed): return firewall_reload() def firewall_disallow( - protocol, port, ipv4_only=False, ipv6_only=False, upnp_only=False, no_reload=False + protocol, + port, + ipv4_only=False, + ipv6_only=False, + upnp_only=False, + no_reload=False, + reload_only_if_change=False, ): """ Disallow connections on a port @@ -139,14 +157,20 @@ def firewall_disallow( elif upnp_only: ipvs = [] + changed = False + for p in protocols: # Iterate over IP versions to remove port for i in ipvs: if port in firewall[i][p]: firewall[i][p].remove(port) + changed = True else: ipv = "IPv%s" % i[3] - logger.warning(m18n.n("port_already_closed", port=port, ip_version=ipv)) + if not reload_only_if_change: + logger.warning( + m18n.n("port_already_closed", port=port, ip_version=ipv) + ) # Remove port forwarding with UPnP if upnp and port in firewall["uPnP"][p]: firewall["uPnP"][p].remove(port) @@ -156,7 +180,7 @@ def firewall_disallow( # Update and reload firewall _update_firewall_file(firewall) - if not no_reload: + if not no_reload or (reload_only_if_change and changed): return firewall_reload() diff --git a/src/permission.py b/src/permission.py index 801576afd..e451bb74c 100644 --- a/src/permission.py +++ b/src/permission.py @@ -479,6 +479,7 @@ def permission_url( url=None, add_url=None, remove_url=None, + set_url=None, auth_header=None, clear_urls=False, sync_perm=True, @@ -491,6 +492,7 @@ def permission_url( url -- (optional) URL for which access will be allowed/forbidden. add_url -- (optional) List of additional url to add for which access will be allowed/forbidden remove_url -- (optional) List of additional url to remove for which access will be allowed/forbidden + set_url -- (optional) List of additional url to set/replace for which access will be allowed/forbidden auth_header -- (optional) Define for the URL of this permission, if SSOwat pass the authentication header to the application clear_urls -- (optional) Clean all urls (url and additional_urls) """ @@ -556,6 +558,9 @@ def permission_url( new_additional_urls = [u for u in new_additional_urls if u not in remove_url] + if set_url: + new_additional_urls = set_url + if auth_header is None: auth_header = existing_permission["auth_header"] diff --git a/src/tests/test_app_config.py b/src/tests/test_app_config.py index 7c0d16f9d..b524a7a51 100644 --- a/src/tests/test_app_config.py +++ b/src/tests/test_app_config.py @@ -173,14 +173,14 @@ def test_app_config_bind_on_file(config_app): assert app_setting(config_app, "arg5") == "Foo Bar" -def test_app_config_custom_get(config_app): - - assert app_setting(config_app, "arg9") is None - assert ( - "Files in /var/www" - in app_config_get(config_app, "bind.function.arg9")["ask"]["en"] - ) - assert app_setting(config_app, "arg9") is None +# def test_app_config_custom_get(config_app): +# +# assert app_setting(config_app, "arg9") is None +# assert ( +# "Files in /var/www" +# in app_config_get(config_app, "bind.function.arg9")["ask"]["en"] +# ) +# assert app_setting(config_app, "arg9") is None def test_app_config_custom_validator(config_app): diff --git a/src/tests/test_app_resources.py b/src/tests/test_app_resources.py index cbfed78cb..879f6e29a 100644 --- a/src/tests/test_app_resources.py +++ b/src/tests/test_app_resources.py @@ -11,6 +11,7 @@ from yunohost.utils.resources import ( AppResourceClassesByType, ) from yunohost.permission import user_permission_list, permission_delete +from yunohost.firewall import firewall_list dummyfile = "/tmp/dummyappresource-testapp" @@ -75,7 +76,7 @@ def test_provision_dummy(): assert not os.path.exists(dummyfile) AppResourceManager("testapp", current=current, wanted=wanted).apply( - rollback_if_failure=False + rollback_and_raise_exception_if_failure=False ) assert open(dummyfile).read().strip() == "foo" @@ -89,7 +90,7 @@ def test_deprovision_dummy(): assert open(dummyfile).read().strip() == "foo" AppResourceManager("testapp", current=current, wanted=wanted).apply( - rollback_if_failure=False + rollback_and_raise_exception_if_failure=False ) assert not os.path.exists(dummyfile) @@ -101,7 +102,7 @@ def test_provision_dummy_nondefaultvalue(): assert not os.path.exists(dummyfile) AppResourceManager("testapp", current=current, wanted=wanted).apply( - rollback_if_failure=False + rollback_and_raise_exception_if_failure=False ) assert open(dummyfile).read().strip() == "bar" @@ -115,26 +116,11 @@ def test_update_dummy(): assert open(dummyfile).read().strip() == "foo" AppResourceManager("testapp", current=current, wanted=wanted).apply( - rollback_if_failure=False + rollback_and_raise_exception_if_failure=False ) assert open(dummyfile).read().strip() == "bar" -def test_update_dummy_fail(): - - current = {"resources": {"dummy": {}}} - wanted = {"resources": {"dummy": {"content": "forbiddenvalue"}}} - - open(dummyfile, "w").write("foo") - - assert open(dummyfile).read().strip() == "foo" - with pytest.raises(Exception): - AppResourceManager("testapp", current=current, wanted=wanted).apply( - rollback_if_failure=False - ) - assert open(dummyfile).read().strip() == "forbiddenvalue" - - def test_update_dummy_failwithrollback(): current = {"resources": {"dummy": {}}} @@ -145,7 +131,7 @@ def test_update_dummy_failwithrollback(): assert open(dummyfile).read().strip() == "foo" with pytest.raises(Exception): AppResourceManager("testapp", current=current, wanted=wanted).apply( - rollback_if_failure=True + rollback_and_raise_exception_if_failure=True ) assert open(dummyfile).read().strip() == "foo" @@ -276,6 +262,26 @@ def test_resource_ports_several(): assert not app_setting("testapp", "port_foobar") +def test_resource_ports_firewall(): + + r = AppResourceClassesByType["ports"] + conf = {"main": {"default": 12345}} + + r(conf, "testapp").provision_or_update() + + assert 12345 not in firewall_list()["opened_ports"] + + conf = {"main": {"default": 12345, "exposed": "TCP"}} + + r(conf, "testapp").provision_or_update() + + assert 12345 in firewall_list()["opened_ports"] + + r(conf, "testapp").deprovision() + + assert 12345 not in firewall_list()["opened_ports"] + + def test_resource_database(): r = AppResourceClassesByType["database"] @@ -397,9 +403,7 @@ def test_resource_permissions(): res = user_permission_list(full=True)["permissions"] - # FIXME FIXME FIXME : this is the current behavior but - # it is NOT okay. c.f. comment in the code - assert res["testapp.admin"]["url"] == "/admin" # should be '/adminpanel' + assert res["testapp.admin"]["url"] == "/adminpanel" r(conf, "testapp").deprovision() diff --git a/src/tests/test_apps.py b/src/tests/test_apps.py index 6cd52659d..6efdaa0b0 100644 --- a/src/tests/test_apps.py +++ b/src/tests/test_apps.py @@ -223,10 +223,10 @@ def test_legacy_app_manifest_preinstall(): assert "install" in m assert m["doc"] == {} assert m["notifications"] == { - "pre_install": {}, - "pre_upgrade": {}, - "post_install": {}, - "post_upgrade": {}, + "PRE_INSTALL": {}, + "PRE_UPGRADE": {}, + "POST_INSTALL": {}, + "POST_UPGRADE": {}, } @@ -249,11 +249,11 @@ def test_manifestv2_app_manifest_preinstall(): assert "notifications" in m assert ( "This is a dummy disclaimer to display prior to the install" - in m["notifications"]["pre_install"]["main"]["en"] + in m["notifications"]["PRE_INSTALL"]["main"]["en"] ) assert ( "Ceci est un faux disclaimer à présenter avant l'installation" - in m["notifications"]["pre_install"]["main"]["fr"] + in m["notifications"]["PRE_INSTALL"]["main"]["fr"] ) @@ -295,15 +295,15 @@ def test_manifestv2_app_info_postinstall(): assert "notifications" in m assert ( "The app install dir is /var/www/manifestv2_app" - in m["notifications"]["post_install"]["main"]["en"] + in m["notifications"]["POST_INSTALL"]["main"]["en"] ) assert ( "The app id is manifestv2_app" - in m["notifications"]["post_install"]["main"]["en"] + in m["notifications"]["POST_INSTALL"]["main"]["en"] ) assert ( f"The app url is {main_domain}/manifestv2" - in m["notifications"]["post_install"]["main"]["en"] + in m["notifications"]["POST_INSTALL"]["main"]["en"] ) @@ -341,7 +341,7 @@ def test_manifestv2_app_info_preupgrade(monkeypatch): # should parse the files in the original app repo, possibly with proper i18n etc assert ( "This is a dummy disclaimer to display prior to any upgrade" - in i["from_catalog"]["manifest"]["notifications"]["pre_upgrade"]["main"]["en"] + in i["from_catalog"]["manifest"]["notifications"]["PRE_UPGRADE"]["main"]["en"] ) diff --git a/src/tools.py b/src/tools.py index b4eef34cc..5c5b8077b 100644 --- a/src/tools.py +++ b/src/tools.py @@ -29,7 +29,11 @@ from moulinette.utils.log import getActionLogger from moulinette.utils.process import call_async_output from moulinette.utils.filesystem import read_yaml, write_to_yaml, cp, mkdir, rm, chown -from yunohost.app import app_upgrade, app_list +from yunohost.app import ( + app_upgrade, + app_list, + _list_upgradable_apps, +) from yunohost.app_catalog import ( _initialize_apps_catalog_system, _update_apps_catalog, @@ -363,7 +367,7 @@ def tools_update(target=None): except YunohostError as e: logger.error(str(e)) - upgradable_apps = list(app_list(upgradable=True)["apps"]) + upgradable_apps = _list_upgradable_apps() if len(upgradable_apps) == 0 and len(upgradable_system_packages) == 0: logger.info(m18n.n("already_up_to_date")) @@ -412,7 +416,8 @@ def tools_upgrade(operation_logger, target=None): if target not in ["apps", "system"]: raise YunohostValidationError( - "Uhoh ?! tools_upgrade should have 'apps' or 'system' value for argument target" + "Uhoh ?! tools_upgrade should have 'apps' or 'system' value for argument target", + raw_msg=True ) # @@ -506,7 +511,7 @@ def tools_upgrade(operation_logger, target=None): logger.warning( m18n.n( "tools_upgrade_failed", - packages_list=", ".join(upgradables), + packages_list=", ".join([p["name"] for p in upgradables]), ) ) diff --git a/src/user.py b/src/user.py index 2fcd8ab9f..deaebba5b 100644 --- a/src/user.py +++ b/src/user.py @@ -1242,11 +1242,13 @@ def user_group_update( logger.info(m18n.n("group_update_aliases", group=groupname)) new_attr_dict["mail"] = set(new_group_mail) - if new_attr_dict["mail"] and "mailAccount" not in group["objectClass"]: - new_attr_dict["objectClass"] = group["objectClass"] + ["mailAccount"] - elif not new_attr_dict["mail"] and "mailAccount" in group["objectClass"]: + if new_attr_dict["mail"] and "mailGroup" not in group["objectClass"]: + new_attr_dict["objectClass"] = group["objectClass"] + ["mailGroup"] + if not new_attr_dict["mail"] and "mailGroup" in group["objectClass"]: new_attr_dict["objectClass"] = [ - c for c in group["objectClass"] if c != "mailAccount" + c + for c in group["objectClass"] + if c != "mailGroup" and c != "mailAccount" ] if new_attr_dict: diff --git a/src/utils/config.py b/src/utils/config.py index 27e4b9509..da3c68ad8 100644 --- a/src/utils/config.py +++ b/src/utils/config.py @@ -479,9 +479,8 @@ class ConfigPanel: # Check TOML config panel is in a supported version if float(toml_config_panel["version"]) < CONFIG_PANEL_VERSION_SUPPORTED: - raise YunohostError( - "config_version_not_supported", version=toml_config_panel["version"] - ) + logger.error(f"Config panels version {toml_config_panel['version']} are not supported") + return None # Transform toml format into internal format format_description = { @@ -575,7 +574,7 @@ class ConfigPanel: subnode["name"] = key # legacy subnode.setdefault("optional", raw_infos.get("optional", True)) # If this section contains at least one button, it becomes an "action" section - if subnode["type"] == "button": + if subnode.get("type") == "button": out["is_action_section"] = True out.setdefault(sublevel, []).append(subnode) # Key/value are a property diff --git a/src/utils/resources.py b/src/utils/resources.py index f48722236..7b500ad3f 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -22,6 +22,7 @@ import shutil import random from typing import Dict, Any, List +from moulinette import m18n from moulinette.utils.process import check_output from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import mkdir, chown, chmod, write_to_file @@ -29,16 +30,12 @@ from moulinette.utils.filesystem import ( rm, ) -from yunohost.utils.error import YunohostError +from yunohost.utils.error import YunohostError, YunohostValidationError logger = getActionLogger("yunohost.app_resources") class AppResourceManager: - - # FIXME : add some sort of documentation mechanism - # to create a have a detailed description of each resource behavior - def __init__(self, app: str, current: Dict, wanted: Dict): self.app = app @@ -50,7 +47,9 @@ class AppResourceManager: if "resources" not in self.wanted: self.wanted["resources"] = {} - def apply(self, rollback_if_failure, **context): + def apply( + self, rollback_and_raise_exception_if_failure, operation_logger=None, **context + ): todos = list(self.compute_todos()) completed = [] @@ -69,12 +68,13 @@ class AppResourceManager: elif todo == "update": logger.info(f"Updating {name} ...") new.provision_or_update(context=context) - # FIXME FIXME FIXME : this exception doesnt catch Ctrl+C ?!?! - except Exception as e: + except (KeyboardInterrupt, Exception) as e: exception = e - # FIXME: better error handling ? display stacktrace ? - logger.warning(f"Failed to {todo} for {name} : {e}") - if rollback_if_failure: + if isinstance(e, KeyboardInterrupt): + logger.error(m18n.n("operation_interrupted")) + else: + logger.warning(f"Failed to {todo} {name} : {e}") + if rollback_and_raise_exception_if_failure: rollback = True completed.append((todo, name, old, new)) break @@ -97,12 +97,28 @@ class AppResourceManager: elif todo == "update": logger.info(f"Reverting {name} ...") old.provision_or_update(context=context) - except Exception as e: - # FIXME: better error handling ? display stacktrace ? - logger.error(f"Failed to rollback {name} : {e}") + except (KeyboardInterrupt, Exception) as e: + if isinstance(e, KeyboardInterrupt): + logger.error(m18n.n("operation_interrupted")) + else: + logger.error(f"Failed to rollback {name} : {e}") if exception: - raise exception + if rollback_and_raise_exception_if_failure: + logger.error( + m18n.n("app_resource_failed", app=self.app, error=exception) + ) + if operation_logger: + failure_message_with_debug_instructions = operation_logger.error( + str(exception) + ) + raise YunohostError( + failure_message_with_debug_instructions, raw_msg=True + ) + else: + raise YunohostError(str(exception), raw_msg=True) + else: + logger.error(exception) def compute_todos(self): @@ -248,7 +264,7 @@ class PermissionsResource(AppResource): ##### Provision/Update: - Delete any permissions that may exist and be related to this app yet is not declared anymore - - Loop over the declared permissions and create them if needed or update them with the new values (FIXME : update ain't implemented yet >_>) + - Loop over the declared permissions and create them if needed or update them with the new values ##### Deprovision: - Delete all permission related to this app @@ -302,7 +318,7 @@ class PermissionsResource(AppResource): from yunohost.permission import ( permission_create, - # permission_url, + permission_url, permission_delete, user_permission_list, user_permission_update, @@ -320,7 +336,8 @@ class PermissionsResource(AppResource): permission_delete(perm, force=True, sync_perm=False) for perm, infos in self.permissions.items(): - if f"{self.app}.{perm}" not in existing_perms: + perm_id = f"{self.app}.{perm}" + if perm_id not in existing_perms: # Use the 'allowed' key from the manifest, # or use the 'init_{perm}_permission' from the install questions # which is temporarily saved as a setting as an ugly hack to pass the info to this piece of code... @@ -330,7 +347,7 @@ class PermissionsResource(AppResource): or [] ) permission_create( - f"{self.app}.{perm}", + perm_id, allowed=init_allowed, # This is why the ugly hack with self.manager exists >_> label=self.manager.wanted["name"] if perm == "main" else perm, @@ -341,17 +358,19 @@ class PermissionsResource(AppResource): ) self.delete_setting(f"init_{perm}_permission") - user_permission_update( - f"{self.app}.{perm}", - show_tile=infos["show_tile"], - protected=infos["protected"], - sync_perm=False, - ) - else: - pass - # FIXME : current implementation of permission_url is hell for - # easy declarativeness of additional_urls >_> ... - # permission_url(f"{self.app}.{perm}", url=infos["url"], auth_header=infos["auth_header"], sync_perm=False) + user_permission_update( + perm_id, + show_tile=infos["show_tile"], + protected=infos["protected"], + sync_perm=False, + ) + permission_url( + perm_id, + url=infos["url"], + set_url=infos["additional_urls"], + auth_header=infos["auth_header"], + sync_perm=False, + ) permission_sync_to_user() @@ -523,6 +542,8 @@ class InstalldirAppResource(AppResource): if not current_install_dir and os.path.isdir(self.dir): rm(self.dir, recursive=True) + # isdir will be True if the path is a symlink pointing to a dir + # This should cover cases where people moved the data dir to another place via a symlink (ie we dont enter the if) if not os.path.isdir(self.dir): # Handle case where install location changed, in which case we shall move the existing install dir # FIXME: confirm that's what we wanna do @@ -551,8 +572,10 @@ class InstalldirAppResource(AppResource): perm_octal = 0o100 * owner_perm_octal + 0o010 * group_perm_octal - chmod(self.dir, perm_octal) - chown(self.dir, owner, group) + # NB: we use realpath here to cover cases where self.dir could actually be a symlink + # in which case we want to apply the perm to the pointed dir, not to the symlink + chmod(os.path.realpath(self.dir), perm_octal) + chown(os.path.realpath(self.dir), owner, group) # FIXME: shall we apply permissions recursively ? self.set_setting("install_dir", self.dir) @@ -592,9 +615,8 @@ class DatadirAppResource(AppResource): - save the value of `dir` as `data_dir` in the app's settings, which can be then used by the app scripts (`$data_dir`) and conf templates (`__DATA_DIR__`) ##### Deprovision: - - recursively deletes the directory if it exists - - FIXME: this should only be done if the PURGE option is set - - FIXME: this should also delete the corresponding setting + - (only if the purge option is chosen by the user) recursively deletes the directory if it exists + - also delete the corresponding setting ##### Legacy management: - In the past, the setting may have been called `datadir`. The code will automatically rename it as `data_dir`. @@ -628,11 +650,15 @@ class DatadirAppResource(AppResource): current_data_dir = self.get_setting("data_dir") or self.get_setting("datadir") + # isdir will be True if the path is a symlink pointing to a dir + # This should cover cases where people moved the data dir to another place via a symlink (ie we dont enter the if) if not os.path.isdir(self.dir): # Handle case where install location changed, in which case we shall move the existing install dir # FIXME: same as install_dir, is this what we want ? - # FIXME: What if people manually mved the data dir and changed the setting value and dont want the folder to be moved ? x_x if current_data_dir and os.path.isdir(current_data_dir): + logger.warning( + f"Moving {current_data_dir} to {self.dir} ... (this may take a while)" + ) shutil.move(current_data_dir, self.dir) else: mkdir(self.dir) @@ -651,8 +677,10 @@ class DatadirAppResource(AppResource): ) perm_octal = 0o100 * owner_perm_octal + 0o010 * group_perm_octal - chmod(self.dir, perm_octal) - chown(self.dir, owner, group) + # NB: we use realpath here to cover cases where self.dir could actually be a symlink + # in which case we want to apply the perm to the pointed dir, not to the symlink + chmod(os.path.realpath(self.dir), perm_octal) + chown(os.path.realpath(self.dir), owner, group) self.set_setting("data_dir", self.dir) self.delete_setting("datadir") # Legacy @@ -663,11 +691,10 @@ class DatadirAppResource(AppResource): assert self.owner.strip() assert self.group.strip() - # FIXME: This should rm the datadir only if purge is enabled - pass - # if os.path.isdir(self.dir): - # rm(self.dir, recursive=True) - # FIXME : in fact we should delete settings to be consistent + if context.get("purge_data_dir", False) and os.path.isdir(self.dir): + rm(self.dir, recursive=True) + + self.delete_setting("data_dir") class AptDependenciesAppResource(AppResource): @@ -756,16 +783,16 @@ class PortsResource(AppResource): ##### Properties (for every port name): - `default`: The prefered value for the port. If this port is already being used by another process right now, or is booked in another app's setting, the code will increment the value until it finds a free port and store that value as the setting. If no value is specified, a random value between 10000 and 60000 is used. - - `exposed`: (default: `false`) Wether this port should be opened on the firewall and be publicly reachable. This should be kept to `false` for the majority of apps than only need a port for internal reverse-proxying! Possible values: `false`, `true`(=`Both`), `Both`, `TCP`, `UDP`. This will result in the port being opened on the firewall, and the diagnosis checking that a program answers on that port. (FIXME: this is not implemented yet) - - `fixed`: (default: `false`) Tells that the app absolutely needs the specific value provided in `default`, typically because it's needed for a specific protocol (FIXME: this is not implemented yet) + - `exposed`: (default: `false`) Wether this port should be opened on the firewall and be publicly reachable. This should be kept to `false` for the majority of apps than only need a port for internal reverse-proxying! Possible values: `false`, `true`(=`Both`), `Both`, `TCP`, `UDP`. This will result in the port being opened on the firewall, and the diagnosis checking that a program answers on that port. + - `fixed`: (default: `false`) Tells that the app absolutely needs the specific value provided in `default`, typically because it's needed for a specific protocol ##### Provision/Update (for every port name): - If not already booked, look for a free port, starting with the `default` value (or a random value between 10000 and 60000 if no `default` set) - - (FIXME) If `exposed` is not `false`, open the port in the firewall accordingly - otherwise make sure it's closed. + - If `exposed` is not `false`, open the port in the firewall accordingly - otherwise make sure it's closed. - The value of the port is stored in the `$port` setting for the `main` port, or `$port_NAME` for other `NAME`s ##### Deprovision: - - (FIXME) Close the ports on the firewall + - Close the ports on the firewall if relevant - Deletes all the port settings ##### Legacy management: @@ -784,8 +811,8 @@ class PortsResource(AppResource): default_port_properties = { "default": None, - "exposed": False, # or True(="Both"), "TCP", "UDP" # FIXME : implement logic for exposed port (allow/disallow in firewall ?) - "fixed": False, # FIXME: implement logic. Corresponding to wether or not the port is "fixed" or any random port is ok + "exposed": False, # or True(="Both"), "TCP", "UDP" + "fixed": False, } ports: Dict[str, Dict[str, Any]] @@ -817,6 +844,8 @@ class PortsResource(AppResource): def provision_or_update(self, context: Dict = {}): + from yunohost.firewall import firewall_allow, firewall_disallow + for name, infos in self.ports.items(): setting_name = f"port_{name}" if name != "main" else "port" @@ -832,16 +861,37 @@ class PortsResource(AppResource): if not port_value: port_value = infos["default"] - while self._port_is_used(port_value): - port_value += 1 + + if infos["fixed"]: + if self._port_is_used(port_value): + raise YunohostValidationError( + f"Port {port_value} is already used by another process or app." + ) + else: + while self._port_is_used(port_value): + port_value += 1 self.set_setting(setting_name, port_value) + if infos["exposed"]: + firewall_allow(infos["exposed"], port_value, reload_only_if_change=True) + else: + firewall_disallow( + infos["exposed"], port_value, reload_only_if_change=True + ) + def deprovision(self, context: Dict = {}): + from yunohost.firewall import firewall_disallow + for name, infos in self.ports.items(): setting_name = f"port_{name}" if name != "main" else "port" + value = self.get_setting(setting_name) self.delete_setting(setting_name) + if value and str(value).strip(): + firewall_disallow( + infos["exposed"], int(value), reload_only_if_change=True + ) class DatabaseAppResource(AppResource): @@ -881,9 +931,10 @@ class DatabaseAppResource(AppResource): type = "database" priority = 90 + dbtype: str = "" default_properties: Dict[str, Any] = { - "type": None, # FIXME: eeeeeeeh is this really a good idea considering 'type' is supposed to be the resource type x_x + "dbtype": None, } def __init__(self, properties: Dict[str, Any], *args, **kwargs): @@ -893,16 +944,22 @@ class DatabaseAppResource(AppResource): "postgresql", ]: raise YunohostError( - "Specifying the type of db ('mysql' or 'postgresql') is mandatory for db resources" + "Specifying the type of db ('mysql' or 'postgresql') is mandatory for db resources", + raw_msg=True, ) + # Hack so that people can write type = "mysql/postgresql" in toml but it's loaded as dbtype + # to avoid conflicting with the generic self.type of the resource object ... + # dunno if that's really a good idea :| + properties = {"dbtype": properties["type"]} + super().__init__(properties, *args, **kwargs) def db_exists(self, db_name): - if self.type == "mysql": + if self.dbtype == "mysql": return os.system(f"mysqlshow '{db_name}' >/dev/null 2>/dev/null") == 0 - elif self.type == "postgresql": + elif self.dbtype == "postgresql": return ( os.system( f"sudo --login --user=postgres psql -c '' '{db_name}' >/dev/null 2>/dev/null" @@ -926,7 +983,7 @@ class DatabaseAppResource(AppResource): else: # Legacy setting migration legacypasswordsetting = ( - "psqlpwd" if self.type == "postgresql" else "mysqlpwd" + "psqlpwd" if self.dbtype == "postgresql" else "mysqlpwd" ) if self.get_setting(legacypasswordsetting): db_pwd = self.get_setting(legacypasswordsetting) @@ -941,12 +998,12 @@ class DatabaseAppResource(AppResource): if not self.db_exists(db_name): - if self.type == "mysql": + if self.dbtype == "mysql": self._run_script( "provision", f"ynh_mysql_create_db '{db_name}' '{db_user}' '{db_pwd}'", ) - elif self.type == "postgresql": + elif self.dbtype == "postgresql": self._run_script( "provision", f"ynh_psql_create_user '{db_user}' '{db_pwd}'; ynh_psql_create_db '{db_name}' '{db_user}'", @@ -957,11 +1014,11 @@ class DatabaseAppResource(AppResource): db_name = self.app.replace("-", "_").replace(".", "_") db_user = db_name - if self.type == "mysql": + if self.dbtype == "mysql": self._run_script( "deprovision", f"ynh_mysql_remove_db '{db_name}' '{db_user}'" ) - elif self.type == "postgresql": + elif self.dbtype == "postgresql": self._run_script( "deprovision", f"ynh_psql_remove_db '{db_name}' '{db_user}'" ) diff --git a/tests/test_helpers.d/ynhtest_setup_source.sh b/tests/test_helpers.d/ynhtest_setup_source.sh index fe61e7401..6a74a587c 100644 --- a/tests/test_helpers.d/ynhtest_setup_source.sh +++ b/tests/test_helpers.d/ynhtest_setup_source.sh @@ -18,51 +18,51 @@ _make_dummy_src() { } ynhtest_setup_source_nominal() { - final_path="$(mktemp -d -p $VAR_WWW)" + install_dir="$(mktemp -d -p $VAR_WWW)" _make_dummy_src > ../conf/dummy.src - ynh_setup_source --dest_dir="$final_path" --source_id="dummy" + ynh_setup_source --dest_dir="$install_dir" --source_id="dummy" - test -e "$final_path" - test -e "$final_path/index.html" + test -e "$install_dir" + test -e "$install_dir/index.html" } ynhtest_setup_source_nominal_upgrade() { - final_path="$(mktemp -d -p $VAR_WWW)" + install_dir="$(mktemp -d -p $VAR_WWW)" _make_dummy_src > ../conf/dummy.src - ynh_setup_source --dest_dir="$final_path" --source_id="dummy" + ynh_setup_source --dest_dir="$install_dir" --source_id="dummy" - test "$(cat $final_path/index.html)" == "Lorem Ipsum" + test "$(cat $install_dir/index.html)" == "Lorem Ipsum" # Except index.html to get overwritten during next ynh_setup_source - echo "IEditedYou!" > $final_path/index.html - test "$(cat $final_path/index.html)" == "IEditedYou!" + echo "IEditedYou!" > $install_dir/index.html + test "$(cat $install_dir/index.html)" == "IEditedYou!" - ynh_setup_source --dest_dir="$final_path" --source_id="dummy" + ynh_setup_source --dest_dir="$install_dir" --source_id="dummy" - test "$(cat $final_path/index.html)" == "Lorem Ipsum" + test "$(cat $install_dir/index.html)" == "Lorem Ipsum" } ynhtest_setup_source_with_keep() { - final_path="$(mktemp -d -p $VAR_WWW)" + install_dir="$(mktemp -d -p $VAR_WWW)" _make_dummy_src > ../conf/dummy.src - echo "IEditedYou!" > $final_path/index.html - echo "IEditedYou!" > $final_path/test.txt + echo "IEditedYou!" > $install_dir/index.html + echo "IEditedYou!" > $install_dir/test.txt - ynh_setup_source --dest_dir="$final_path" --source_id="dummy" --keep="index.html test.txt" + ynh_setup_source --dest_dir="$install_dir" --source_id="dummy" --keep="index.html test.txt" - test -e "$final_path" - test -e "$final_path/index.html" - test -e "$final_path/test.txt" - test "$(cat $final_path/index.html)" == "IEditedYou!" - test "$(cat $final_path/test.txt)" == "IEditedYou!" + test -e "$install_dir" + test -e "$install_dir/index.html" + test -e "$install_dir/test.txt" + test "$(cat $install_dir/index.html)" == "IEditedYou!" + test "$(cat $install_dir/test.txt)" == "IEditedYou!" } ynhtest_setup_source_with_patch() { - final_path="$(mktemp -d -p $VAR_WWW)" + install_dir="$(mktemp -d -p $VAR_WWW)" _make_dummy_src > ../conf/dummy.src mkdir -p ../sources/patches @@ -74,7 +74,7 @@ ynhtest_setup_source_with_patch() { +Lorem Ipsum dolor sit amet EOF - ynh_setup_source --dest_dir="$final_path" --source_id="dummy" + ynh_setup_source --dest_dir="$install_dir" --source_id="dummy" - test "$(cat $final_path/index.html)" == "Lorem Ipsum dolor sit amet" + test "$(cat $install_dir/index.html)" == "Lorem Ipsum dolor sit amet" }