From 22681a4f241c074feaa628b5c8d28716d65812e9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 28 Feb 2023 18:50:31 +0100 Subject: [PATCH 01/82] Kill the old 'unprotected/protected/skipped' setting hell --- helpers/setting | 18 ++------ src/app.py | 115 ------------------------------------------------ 2 files changed, 3 insertions(+), 130 deletions(-) diff --git a/helpers/setting b/helpers/setting index a2cf3a93d..a89f72091 100644 --- a/helpers/setting +++ b/helpers/setting @@ -18,11 +18,7 @@ ynh_app_setting_get() { ynh_handle_getopts_args "$@" app="${app:-$_globalapp}" - if [[ $key =~ (unprotected|protected|skipped)_ ]]; then - yunohost app setting $app $key - else - ynh_app_setting "get" "$app" "$key" - fi + ynh_app_setting "get" "$app" "$key" } # Set an application setting @@ -45,11 +41,7 @@ ynh_app_setting_set() { ynh_handle_getopts_args "$@" app="${app:-$_globalapp}" - if [[ $key =~ (unprotected|protected|skipped)_ ]]; then - yunohost app setting $app $key -v $value - else - ynh_app_setting "set" "$app" "$key" "$value" - fi + ynh_app_setting "set" "$app" "$key" "$value" } # Delete an application setting @@ -70,11 +62,7 @@ ynh_app_setting_delete() { ynh_handle_getopts_args "$@" app="${app:-$_globalapp}" - if [[ "$key" =~ (unprotected|skipped|protected)_ ]]; then - yunohost app setting $app $key -d - else - ynh_app_setting "delete" "$app" "$key" - fi + ynh_app_setting "delete" "$app" "$key" } # Small "hard-coded" interface to avoid calling "yunohost app" directly each diff --git a/src/app.py b/src/app.py index f17c46929..65402a024 100644 --- a/src/app.py +++ b/src/app.py @@ -1492,119 +1492,6 @@ def app_setting(app, key, value=None, delete=False): """ app_settings = _get_app_settings(app) or {} - # - # Legacy permission setting management - # (unprotected, protected, skipped_uri/regex) - # - - is_legacy_permission_setting = any( - key.startswith(word + "_") for word in ["unprotected", "protected", "skipped"] - ) - - if is_legacy_permission_setting: - from yunohost.permission import ( - user_permission_list, - user_permission_update, - permission_create, - permission_delete, - permission_url, - ) - - permissions = user_permission_list(full=True, apps=[app])["permissions"] - key_ = key.split("_")[0] - permission_name = f"{app}.legacy_{key_}_uris" - permission = permissions.get(permission_name) - - # GET - if value is None and not delete: - return ( - ",".join(permission.get("uris", []) + permission["additional_urls"]) - if permission - else None - ) - - # DELETE - if delete: - # If 'is_public' setting still exists, we interpret this as - # coming from a legacy app (because new apps shouldn't manage the - # is_public state themselves anymore...) - # - # In that case, we interpret the request for "deleting - # unprotected/skipped" setting as willing to make the app - # private - if ( - "is_public" in app_settings - and "visitors" in permissions[app + ".main"]["allowed"] - ): - if key.startswith("unprotected_") or key.startswith("skipped_"): - user_permission_update(app + ".main", remove="visitors") - - if permission: - permission_delete(permission_name) - - # SET - else: - urls = value - # If the request is about the root of the app (/), ( = the vast majority of cases) - # we interpret this as a change for the main permission - # (i.e. allowing/disallowing visitors) - if urls == "/": - if key.startswith("unprotected_") or key.startswith("skipped_"): - permission_url(app + ".main", url="/", sync_perm=False) - user_permission_update(app + ".main", add="visitors") - else: - user_permission_update(app + ".main", remove="visitors") - else: - urls = urls.split(",") - if key.endswith("_regex"): - urls = ["re:" + url for url in urls] - - if permission: - # In case of new regex, save the urls, to add a new time in the additional_urls - # In case of new urls, we do the same thing but inversed - if key.endswith("_regex"): - # List of urls to save - current_urls_or_regex = [ - url - for url in permission["additional_urls"] - if not url.startswith("re:") - ] - else: - # List of regex to save - current_urls_or_regex = [ - url - for url in permission["additional_urls"] - if url.startswith("re:") - ] - - new_urls = urls + current_urls_or_regex - # We need to clear urls because in the old setting the new setting override the old one and dont just add some urls - permission_url(permission_name, clear_urls=True, sync_perm=False) - permission_url(permission_name, add_url=new_urls) - else: - from yunohost.utils.legacy import legacy_permission_label - - # Let's create a "special" permission for the legacy settings - permission_create( - permission=permission_name, - # FIXME find a way to limit to only the user allowed to the main permission - allowed=["all_users"] - if key.startswith("protected_") - else ["all_users", "visitors"], - url=None, - additional_urls=urls, - auth_header=not key.startswith("skipped_"), - label=legacy_permission_label(app, key.split("_")[0]), - show_tile=False, - protected=True, - ) - - return - - # - # Regular setting management - # - # GET if value is None and not delete: return app_settings.get(key, None) @@ -1616,8 +1503,6 @@ def app_setting(app, key, value=None, delete=False): # SET else: - if key in ["redirected_urls", "redirected_regex"]: - value = yaml.safe_load(value) app_settings[key] = value _set_app_settings(app, app_settings) From 04eadd715c8723e43113461b92c7493a76760e02 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 28 Feb 2023 18:55:47 +0100 Subject: [PATCH 02/82] Kill old --package option and .ini for PHP FPM config --- helpers/php | 71 ++--------------------------------------------------- 1 file changed, 2 insertions(+), 69 deletions(-) diff --git a/helpers/php b/helpers/php index 417dbbc61..1b28b32f7 100644 --- a/helpers/php +++ b/helpers/php @@ -15,7 +15,7 @@ YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} # # ----------------------------------------------------------------------------- # -# usage 2: ynh_add_fpm_config [--phpversion=7.X] --usage=usage --footprint=footprint [--package=packages] [--dedicated_service] +# usage 2: ynh_add_fpm_config [--phpversion=7.X] --usage=usage --footprint=footprint [--dedicated_service] # | arg: -v, --phpversion= - Version of PHP to use. # | arg: -f, --footprint= - Memory footprint of the service (low/medium/high). # low - Less than 20 MB of RAM by pool. @@ -30,7 +30,6 @@ YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} # medium - Low usage, few people or/and publicly accessible. # high - High usage, frequently visited website. # -# | arg: -p, --package= - Additionnal PHP packages to install for a specific version of PHP # | arg: -d, --dedicated_service - Use a dedicated PHP-FPM service instead of the common one. # # @@ -60,16 +59,14 @@ ynh_add_fpm_config() { local _globalphpversion=${phpversion-:} # Declare an array to define the options of this helper. local legacy_args=vtufpd - local -A args_array=([v]=phpversion= [t]=use_template [u]=usage= [f]=footprint= [p]=package= [d]=dedicated_service) + local -A args_array=([v]=phpversion= [t]=use_template [u]=usage= [f]=footprint= [d]=dedicated_service) local phpversion local use_template local usage local footprint - local package local dedicated_service # Manage arguments with getopts ynh_handle_getopts_args "$@" - package=${package:-} # The default behaviour is to use the template. use_template="${use_template:-1}" @@ -103,13 +100,6 @@ ynh_add_fpm_config() { fi fi - # Legacy args (packager should just list their php dependency as regular apt dependencies... - if [ -n "$package" ]; then - # Install the additionnal packages from the default repository - ynh_print_warn --message "Argument --package of ynh_add_fpm_config is deprecated and to be removed in the future" - ynh_install_app_dependencies "$package" - fi - if [ $dedicated_service -eq 1 ]; then local fpm_service="${app}-phpfpm" local fpm_config_dir="/etc/php/$phpversion/dedicated-fpm" @@ -197,11 +187,6 @@ pm.process_idle_timeout = 10s local finalphpconf="$fpm_config_dir/pool.d/$app.conf" ynh_add_config --template="$phpfpm_path" --destination="$finalphpconf" - if [ -e "$YNH_APP_BASEDIR/conf/php-fpm.ini" ]; then - ynh_print_warn --message="Packagers ! Please do not use a separate php ini file, merge your directives in the pool file instead." - ynh_add_config --template="php-fpm.ini" --destination="$fpm_config_dir/conf.d/20-$app.ini" - fi - if [ $dedicated_service -eq 1 ]; then # Create a dedicated php-fpm.conf for the service local globalphpconf=$fpm_config_dir/php-fpm-$app.conf @@ -272,9 +257,6 @@ ynh_remove_fpm_config() { fi ynh_secure_remove --file="$fpm_config_dir/pool.d/$app.conf" - if [ -e $fpm_config_dir/conf.d/20-$app.ini ]; then - ynh_secure_remove --file="$fpm_config_dir/conf.d/20-$app.ini" - fi if [ $dedicated_service -eq 1 ]; then # Remove the dedicated service PHP-FPM service for the app @@ -286,55 +268,6 @@ ynh_remove_fpm_config() { elif ynh_package_is_installed --package="php${phpversion}-fpm"; then ynh_systemd_action --service_name=$fpm_service --action=reload fi - - # If the PHP version used is not the default version for YunoHost - # The second part with YNH_APP_PURGE is an ugly hack to guess that we're inside the remove script - # (we don't actually care about its value, we just check its not empty hence it exists) - if [ "$phpversion" != "$YNH_DEFAULT_PHP_VERSION" ] && [ -n "${YNH_APP_PURGE:-}" ] && dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then - # Remove app dependencies ... but ideally should happen via an explicit call from packager - ynh_remove_app_dependencies - fi -} - -# Install another version of PHP. -# -# [internal] -# -# Legacy, to be remove on bullseye -# -# usage: ynh_install_php --phpversion=phpversion [--package=packages] -# | arg: -v, --phpversion= - Version of PHP to install. -# | arg: -p, --package= - Additionnal PHP packages to install -# -# Requires YunoHost version 3.8.1 or higher. -ynh_install_php() { - # Declare an array to define the options of this helper. - local legacy_args=vp - local -A args_array=([v]=phpversion= [p]=package=) - local phpversion - local package - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - package=${package:-} - - if [ "$phpversion" == "$YNH_DEFAULT_PHP_VERSION" ]; then - ynh_die --message="Do not use ynh_install_php to install php$YNH_DEFAULT_PHP_VERSION" - fi - - ynh_install_app_dependencies "$package" -} - -# Remove the specific version of PHP used by the app. -# -# [internal] -# -# Legacy, to be remove on bullseye -# -# usage: ynh_remove_php -# -# Requires YunoHost version 3.8.1 or higher. -ynh_remove_php () { - ynh_remove_app_dependencies } # Define the values to configure PHP-FPM From 0ab7c952f160b4113fe8506a355e26523c2fd23b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 May 2023 20:11:10 +0200 Subject: [PATCH 03/82] bookworm/debian: adapt control file to bookworm --- debian/control | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/debian/control b/debian/control index 0258eaac7..3674a62a4 100644 --- a/debian/control +++ b/debian/control @@ -2,7 +2,7 @@ Source: yunohost Section: utils Priority: extra Maintainer: YunoHost Contributors -Build-Depends: debhelper (>=9), debhelper-compat (= 13), dh-python, python3-all (>= 3.7), python3-yaml, python3-jinja2 +Build-Depends: debhelper (>=9), debhelper-compat (= 13), dh-python, python3-all (>= 3.11), python3-yaml, python3-jinja2 Standards-Version: 3.9.6 Homepage: https://yunohost.org/ @@ -14,9 +14,9 @@ Depends: ${python3:Depends}, ${misc:Depends} , python3-psutil, python3-requests, python3-dnspython, python3-openssl , python3-miniupnpc, python3-dbus, python3-jinja2 , python3-toml, python3-packaging, python3-publicsuffix2 - , python3-ldap, python3-zeroconf (>= 0.36), python3-lexicon, + , python3-ldap, python3-zeroconf (>=0.47), python3-lexicon, , python-is-python3 - , nginx, nginx-extras (>=1.18) + , nginx, nginx-extras (>=1.22) , apt, apt-transport-https, apt-utils, dirmngr , openssh-server, iptables, fail2ban, bind9-dnsutils , openssl, ca-certificates, netcat-openbsd, iproute2 @@ -32,23 +32,18 @@ Depends: ${python3:Depends}, ${misc:Depends} Recommends: yunohost-admin , ntp, inetutils-ping | iputils-ping , bash-completion, rsyslog - , php7.4-common, php7.4-fpm, php7.4-ldap, php7.4-intl - , mariadb-server, php7.4-mysql - , php7.4-gd, php7.4-curl, php-php-gettext - , python3-pip , unattended-upgrades , libdbd-ldap-perl, libnet-dns-perl - , metronome (>=3.14.0) Conflicts: iptables-persistent , apache2 , bind9 - , nginx-extras (>= 1.19) - , openssl (>= 1.1.1o-0) - , slapd (>= 2.4.58) - , dovecot-core (>= 1:2.3.14) - , redis-server (>= 5:6.1) - , fail2ban (>= 0.11.3) - , iptables (>= 1.8.8) + , nginx-extras (>= 1.23) + , openssl (>= 3.1) + , slapd (>= 2.6) + , dovecot-core (>= 1:2.4) + , redis-server (>= 5:7.1) + , fail2ban (>= 1.1) + , iptables (>= 1.8.10) Description: manageable and configured self-hosting server YunoHost aims to make self-hosting accessible to everyone. It configures an email, Web and IM server alongside a LDAP base. It also provides From 37eac5e1214da02cb1994ba98c5d71c66e17fc4f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 May 2023 20:31:14 +0200 Subject: [PATCH 04/82] Update changelog for 12.0.0 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 64fc2ff23..9434b72fd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +yunohost (12.0.0) unstable; urgency=low + + - Tmp changelog to prepare Bookworm + + -- Alexandre Aubin Thu, 04 May 2023 20:30:19 +0200 + yunohost (11.1.18) stable; urgency=low - appsv2: always set an 'app' setting equal to app id to be able to use __APP__ in markdown templates ([#1645](https://github.com/yunohost/yunohost/pull/1645)) From 224f1b17306b031e9b9bc6fc551043c96ebb469c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 5 May 2023 00:09:09 +0200 Subject: [PATCH 05/82] firewall: fix upnpc.discover() behavior that somehow now trigger an exception when cant talk to upnp device --- src/firewall.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/firewall.py b/src/firewall.py index 310d263c6..d6e4b5317 100644 --- a/src/firewall.py +++ b/src/firewall.py @@ -402,7 +402,13 @@ def firewall_upnp(action="status", no_refresh=False): # Discover UPnP device(s) logger.debug("discovering UPnP devices...") - nb_dev = upnpc.discover() + try: + nb_dev = upnpc.discover() + except Exception as e: + logger.warning("Failed to find any UPnP device on the network") + nb_dev = -1 + enabled = False + logger.debug("found %d UPnP device(s)", int(nb_dev)) if nb_dev < 1: logger.error(m18n.n("upnp_dev_not_found")) From 859f9c05a5185883d02ed9a8105d08de16ce69b5 Mon Sep 17 00:00:00 2001 From: Kayou Date: Fri, 5 May 2023 16:37:33 +0200 Subject: [PATCH 06/82] Update test.gitlab-ci.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix tests for bookworm, don't try this at home² --- .gitlab/ci/test.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index b0ffd3db5..b59686bb3 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -1,7 +1,7 @@ .install_debs: &install_debs - apt-get update -o Acquire::Retries=3 - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb - - pip3 install -U mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 "packaging<22" + - pip3 install -U mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 "packaging<22 --break-system-packages" # for bookworm .test-stage: stage: test From 30bd0e05b2cdc259641e937c1c3f43eeb86ff9f5 Mon Sep 17 00:00:00 2001 From: Kayou Date: Fri, 5 May 2023 16:38:53 +0200 Subject: [PATCH 07/82] Update test.gitlab-ci.yml oopsie --- .gitlab/ci/test.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index b59686bb3..da048897c 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -1,7 +1,7 @@ .install_debs: &install_debs - apt-get update -o Acquire::Retries=3 - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb - - pip3 install -U mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 "packaging<22 --break-system-packages" # for bookworm + - pip3 install -U mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 "packaging<22" --break-system-packages # for bookworm .test-stage: stage: test From 191520ebddc7b0171fc36d552cc0aafcf3354e92 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 5 May 2023 19:22:37 +0200 Subject: [PATCH 08/82] mysql: dirty/ugly patch to autoinstall mariadb-server when calling ynhmysql_setup_db when it's not installed --- helpers/mysql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/helpers/mysql b/helpers/mysql index a5290f794..c9629537b 100644 --- a/helpers/mysql +++ b/helpers/mysql @@ -210,6 +210,9 @@ ynh_mysql_setup_db() { # If $db_pwd is not provided, use new_db_pwd instead for db_pwd db_pwd="${db_pwd:-$new_db_pwd}" + # Dirty patch for super-legacy apps + dpkg --list | grep -q "^ii mariadb-server" || { ynh_print_warn "Packager: you called ynh_mysql_setup_db without declaring a dependency to mariadb-server. Please add it to your apt dependencies !"; apt install mariadb-server --no-install-recommends -y; } + ynh_mysql_create_db "$db_name" "$db_user" "$db_pwd" ynh_app_setting_set --app=$app --key=mysqlpwd --value=$db_pwd } From 1fb3965e51a033a37d7abbb04386293009f98fa7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 5 May 2023 20:10:52 +0200 Subject: [PATCH 09/82] improve previous commit, use ynh_apt to have non-interactive apt etc --- helpers/mysql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/mysql b/helpers/mysql index c9629537b..eade5804e 100644 --- a/helpers/mysql +++ b/helpers/mysql @@ -211,7 +211,7 @@ ynh_mysql_setup_db() { db_pwd="${db_pwd:-$new_db_pwd}" # Dirty patch for super-legacy apps - dpkg --list | grep -q "^ii mariadb-server" || { ynh_print_warn "Packager: you called ynh_mysql_setup_db without declaring a dependency to mariadb-server. Please add it to your apt dependencies !"; apt install mariadb-server --no-install-recommends -y; } + dpkg --list | grep -q "^ii mariadb-server" || { ynh_print_warn "Packager: you called ynh_mysql_setup_db without declaring a dependency to mariadb-server. Please add it to your apt dependencies !"; ynh_apt install mariadb-server; } ynh_mysql_create_db "$db_name" "$db_user" "$db_pwd" ynh_app_setting_set --app=$app --key=mysqlpwd --value=$db_pwd From 7fc7d188ad59af01cff710326cdc94e852b1c9b5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 5 May 2023 20:11:16 +0200 Subject: [PATCH 10/82] Fix XMPP stuff that may not exists --- hooks/backup/27-data_xmpp | 4 ++-- hooks/restore/27-data_xmpp | 11 +++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/hooks/backup/27-data_xmpp b/hooks/backup/27-data_xmpp index 2cd93e02b..078184ed8 100644 --- a/hooks/backup/27-data_xmpp +++ b/hooks/backup/27-data_xmpp @@ -9,5 +9,5 @@ source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/data/xmpp" -ynh_backup /var/lib/metronome "${backup_dir}/var_lib_metronome" -ynh_backup /var/xmpp-upload/ "${backup_dir}/var_xmpp-upload" +ynh_backup /var/lib/metronome "${backup_dir}/var_lib_metronome" --not_mandatory +ynh_backup /var/xmpp-upload/ "${backup_dir}/var_xmpp-upload" --not_mandatory diff --git a/hooks/restore/27-data_xmpp b/hooks/restore/27-data_xmpp index 02a4c6703..f07ac6a33 100644 --- a/hooks/restore/27-data_xmpp +++ b/hooks/restore/27-data_xmpp @@ -1,4 +1,11 @@ backup_dir="$1/data/xmpp" -cp -a $backup_dir/var_lib_metronome/. /var/lib/metronome -cp -a $backup_dir/var_xmpp-upload/. /var/xmpp-upload +if [[ -e $backup_dir/var_lib_metronome/ ]] +then + cp -a $backup_dir/var_lib_metronome/. /var/lib/metronome +fi + +if [[ -e $backup_dir/var_xmpp-upload ]] +then + cp -a $backup_dir/var_xmpp-upload/. /var/xmpp-upload +fi From 09012989351bcd08b2fba77fabd6c79503467d18 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 5 May 2023 20:18:19 +0200 Subject: [PATCH 11/82] bookworm: add php7.x -> php8.2 autopatch --- src/app.py | 10 ++++----- src/backup.py | 2 +- src/utils/legacy.py | 52 ++++++++++++++++++++++++++++----------------- 3 files changed, 39 insertions(+), 25 deletions(-) diff --git a/src/app.py b/src/app.py index 59e58cde1..c19f654f3 100644 --- a/src/app.py +++ b/src/app.py @@ -2012,7 +2012,7 @@ def _get_app_settings(app): ): settings["path"] = "/" + settings["path"].strip("/") _set_app_settings(app, settings) - + # Make the app id available as $app too settings["app"] = app @@ -3044,10 +3044,10 @@ def _assert_system_is_sane_for_app(manifest, when): services = manifest.get("services", []) - # Some apps use php-fpm, php5-fpm or php7.x-fpm which is now php7.4-fpm + # Some apps use php-fpm, php5-fpm or php7.x-fpm which is now php8.2-fpm def replace_alias(service): - if service in ["php-fpm", "php5-fpm", "php7.0-fpm", "php7.3-fpm"]: - return "php7.4-fpm" + if service in ["php-fpm", "php5-fpm", "php7.0-fpm", "php7.3-fpm", "php7.4-fpm"]: + return "php8.2-fpm" else: return service @@ -3056,7 +3056,7 @@ def _assert_system_is_sane_for_app(manifest, when): # We only check those, mostly to ignore "custom" services # (added by apps) and because those are the most popular # services - service_filter = ["nginx", "php7.4-fpm", "mysql", "postfix"] + service_filter = ["nginx", "php8.2-fpm", "mysql", "postfix"] services = [str(s) for s in services if s in service_filter] if "nginx" not in services: diff --git a/src/backup.py b/src/backup.py index ce1e8ba2c..a23a8d8e0 100644 --- a/src/backup.py +++ b/src/backup.py @@ -1204,7 +1204,7 @@ class RestoreManager: def _patch_legacy_php_versions_in_csv_file(self): """ - Apply dirty patch to redirect php5 and php7.0 files to php7.4 + Apply dirty patch to redirect php5 and php7.x files to php8.2 """ from yunohost.utils.legacy import LEGACY_PHP_VERSION_REPLACEMENTS diff --git a/src/utils/legacy.py b/src/utils/legacy.py index 82507d64d..226a8c929 100644 --- a/src/utils/legacy.py +++ b/src/utils/legacy.py @@ -163,32 +163,45 @@ def translate_legacy_default_app_in_ssowant_conf_json_persistent(): LEGACY_PHP_VERSION_REPLACEMENTS = [ - ("/etc/php5", "/etc/php/7.4"), - ("/etc/php/7.0", "/etc/php/7.4"), - ("/etc/php/7.3", "/etc/php/7.4"), - ("/var/run/php5-fpm", "/var/run/php/php7.4-fpm"), - ("/var/run/php/php7.0-fpm", "/var/run/php/php7.4-fpm"), - ("/var/run/php/php7.3-fpm", "/var/run/php/php7.4-fpm"), - ("php5", "php7.4"), - ("php7.0", "php7.4"), - ("php7.3", "php7.4"), - ('YNH_PHP_VERSION="7.3"', 'YNH_PHP_VERSION="7.4"'), + ("/etc/php5", "/etc/php/8.2"), + ("/etc/php/7.0", "/etc/php/8.2"), + ("/etc/php/7.3", "/etc/php/8.2"), + ("/etc/php/7.4", "/etc/php/8.2"), + ("/var/run/php5-fpm", "/var/run/php/php8.2-fpm"), + ("/var/run/php/php7.0-fpm", "/var/run/php/php8.2-fpm"), + ("/var/run/php/php7.3-fpm", "/var/run/php/php8.2-fpm"), + ("/var/run/php/php7.4-fpm", "/var/run/php/php8.2-fpm"), + ("php5", "php8.2"), + ("php7.0", "php8.2"), + ("php7.3", "php8.2"), + ("php7.4", "php8.2"), + ('YNH_PHP_VERSION="7.3"', 'YNH_PHP_VERSION="8.2"'), + ('YNH_PHP_VERSION="7.4"', 'YNH_PHP_VERSION="8.2"'), ( 'phpversion="${phpversion:-7.0}"', - 'phpversion="${phpversion:-7.4}"', + 'phpversion="${phpversion:-8.2}"', ), # Many helpers like the composer ones use 7.0 by default ... ( 'phpversion="${phpversion:-7.3}"', + 'phpversion="${phpversion:-8.2}"', + ), # Many helpers like the composer ones use 7.0 by default ... + ( 'phpversion="${phpversion:-7.4}"', + 'phpversion="${phpversion:-8.2}"', ), # Many helpers like the composer ones use 7.0 by default ... ( '"$phpversion" == "7.0"', - '$(bc <<< "$phpversion >= 7.4") -eq 1', + '$(bc <<< "$phpversion >= 8.2") -eq 1', ), # patch ynh_install_php to refuse installing/removing php <= 7.3 ( '"$phpversion" == "7.3"', - '$(bc <<< "$phpversion >= 7.4") -eq 1', + '$(bc <<< "$phpversion >= 8.2") -eq 1', ), # patch ynh_install_php to refuse installing/removing php <= 7.3 + ( + '"$phpversion" == "7.4"', + '$(bc <<< "$phpversion >= 8.2") -eq 1', + ), # patch ynh_install_php to refuse installing/removing php <= 7.3 + ] @@ -217,15 +230,16 @@ def _patch_legacy_php_versions(app_folder): def _patch_legacy_php_versions_in_settings(app_folder): settings = read_yaml(os.path.join(app_folder, "settings.yml")) - if settings.get("fpm_config_dir") in ["/etc/php/7.0/fpm", "/etc/php/7.3/fpm"]: - settings["fpm_config_dir"] = "/etc/php/7.4/fpm" - if settings.get("fpm_service") in ["php7.0-fpm", "php7.3-fpm"]: - settings["fpm_service"] = "php7.4-fpm" - if settings.get("phpversion") in ["7.0", "7.3"]: - settings["phpversion"] = "7.4" + if settings.get("fpm_config_dir") in ["/etc/php/7.0/fpm", "/etc/php/7.3/fpm", "/etc/php/7.4/fpm"]: + settings["fpm_config_dir"] = "/etc/php/8.2/fpm" + if settings.get("fpm_service") in ["php7.0-fpm", "php7.3-fpm", "php7.4-fpm"]: + settings["fpm_service"] = "php8.2-fpm" + if settings.get("phpversion") in ["7.0", "7.3", "7.4"]: + settings["phpversion"] = "8.2" # We delete these checksums otherwise the file will appear as manually modified list_to_remove = [ + "checksum__etc_php_7.4_fpm_pool", "checksum__etc_php_7.3_fpm_pool", "checksum__etc_php_7.0_fpm_pool", "checksum__etc_nginx_conf.d", From 8fb225f3ad12d26d8caeef5182af74327f51b95c Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Fri, 5 May 2023 19:16:21 +0000 Subject: [PATCH 12/82] remove some legacy pre-bullseye workarounds --- bin/yunomdns | 8 ++------ src/app.py | 14 -------------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/bin/yunomdns b/bin/yunomdns index 1bdcf88ca..da9233045 100755 --- a/bin/yunomdns +++ b/bin/yunomdns @@ -132,12 +132,8 @@ def main() -> bool: ) continue - # Only broadcast IPv4 because IPv6 is buggy ... because we ain't using python3-ifaddr >= 0.1.7 - # Buster only ships 0.1.6 - # Bullseye ships 0.1.7 - # To be re-enabled once we're on bullseye... - # ips: List[str] = interfaces[interface]["ipv4"] + interfaces[interface]["ipv6"] - ips: List[str] = interfaces[interface]["ipv4"] + # Broadcast IPv4 and IPv6 + ips: List[str] = interfaces[interface]["ipv4"] + interfaces[interface]["ipv6"] # If at least one IP is listed if not ips: diff --git a/src/app.py b/src/app.py index c19f654f3..944c1b66f 100644 --- a/src/app.py +++ b/src/app.py @@ -1999,20 +1999,6 @@ def _get_app_settings(app): logger.error(m18n.n("app_not_correctly_installed", app=app)) return {} - # Stupid fix for legacy bullshit - # In the past, some setups did not have proper normalization for app domain/path - # Meaning some setups (as of January 2021) still have path=/foobar/ (with a trailing slash) - # resulting in stupid issue unless apps using ynh_app_normalize_path_stuff - # So we yolofix the settings if such an issue is found >_> - # A simple call to `yunohost app list` (which happens quite often) should be enough - # to migrate all app settings ... so this can probably be removed once we're past Bullseye... - if settings.get("path") != "/" and ( - settings.get("path", "").endswith("/") - or not settings.get("path", "/").startswith("/") - ): - settings["path"] = "/" + settings["path"].strip("/") - _set_app_settings(app, settings) - # Make the app id available as $app too settings["app"] = app From 1135cf1b62338a1371e1fd4d8ddd74ec165b9506 Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Fri, 5 May 2023 21:23:23 +0000 Subject: [PATCH 13/82] php-cli is needed for ynhtest_config.sh --- .gitlab/ci/test.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index da048897c..777174e48 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -1,6 +1,6 @@ .install_debs: &install_debs - apt-get update -o Acquire::Retries=3 - - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb + - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb php8.2-cli - pip3 install -U mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 "packaging<22" --break-system-packages # for bookworm .test-stage: From ea7bdb62ed3d37b3054abcf52b8e5ac10d159ee2 Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Fri, 5 May 2023 22:24:43 +0000 Subject: [PATCH 14/82] mariadb-client needed by tests/test_app_resources.py --- .gitlab/ci/test.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index 777174e48..013f45ee1 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -1,6 +1,6 @@ .install_debs: &install_debs - apt-get update -o Acquire::Retries=3 - - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb php8.2-cli + - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb php8.2-cli mariadb-client - pip3 install -U mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 "packaging<22" --break-system-packages # for bookworm .test-stage: From de3dd9436c26514d4edfa725c67c6514c486ee65 Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Fri, 5 May 2023 23:40:17 +0000 Subject: [PATCH 15/82] create migration : s/bullseye/bookworm s/buster/bullseye yolo, needs some more cleanup --- locales/en.json | 15 +- src/migrations/0027_migrate_to_bookworm.py | 546 +++++++++++++++++++++ 2 files changed, 560 insertions(+), 1 deletion(-) create mode 100644 src/migrations/0027_migrate_to_bookworm.py diff --git a/locales/en.json b/locales/en.json index 4dcb00ee6..ea51b4184 100644 --- a/locales/en.json +++ b/locales/en.json @@ -572,7 +572,20 @@ "migration_0024_rebuild_python_venv_disclaimer_rebuild": "Rebuilding the virtualenv will be attempted for the following apps (NB: the operation may take some time!): {rebuild_apps}", "migration_0024_rebuild_python_venv_failed": "Failed to rebuild the Python virtualenv for {app}. The app may not work as long as this is not resolved. You should fix the situation by forcing the upgrade of this app using `yunohost app upgrade --force {app}`.", "migration_0024_rebuild_python_venv_in_progress": "Now attempting to rebuild the Python virtualenv for `{app}`", - "migration_description_0021_migrate_to_bullseye": "Upgrade the system to Debian Bullseye and YunoHost 11.x", + "migration_0027_cleaning_up": "Cleaning up cache and packages not useful anymore...", + "migration_0027_hgjghjghjgeneral_warning": "Please note that this migration is a delicate operation. The YunoHost team did its best to review and test it, but the migration might still break parts of the system or its apps.\n\nTherefore, it is recommended to:\n - Perform a backup of any critical data or app. More info on https://yunohost.org/backup;\n - Be patient after launching the migration: Depending on your Internet connection and hardware, it might take up to a few hours for everything to upgrade.", + "migration_0027_main_upgrade": "Starting main upgrade...", + "migration_0027_modified_files": "Please note that the following files were found to be manually modified and might be overwritten following the upgrade: {manually_modified_files}", + "migration_0027_not_buster2": "The current Debian distribution is not Bullseye! If you already ran the Bullseye->Bookworm migration, then this error is symptomatic of the fact that the migration procedure was not 100% succesful (otherwise YunoHost would have flagged it as completed). It is recommended to investigate what happened with the support team, who will need the **full** log of the `migration, which can be found in Tools > Logs in the webadmin.", + "migration_0027_not_enough_free_space": "Free space is pretty low in /var/! You should have at least 1GB free to run this migration.", + "migration_0027_patch_yunohost_conflicts": "Applying patch to workaround conflict issue...", + "migration_0027_patching_sources_list": "Patching the sources.lists...", + "migration_0027_problematic_apps_warning": "Please note that the following possibly problematic installed apps were detected. It looks like those were not installed from the YunoHost app catalog, or are not flagged as 'working'. Consequently, it cannot be guaranteed that they will still work after the upgrade: {problematic_apps}", + "migration_0027_start": "Starting migration to Bookworm", + "migration_0027_still_on_buster_after_main_upgrade": "Something went wrong during the main upgrade, the system appears to still be on Debian Bullseye", + "migration_0027_system_not_fully_up_to_date": "Your system is not fully up-to-date. Please perform a regular upgrade before running the migration to Bookworm.", + "migration_0027_yunohost_upgrade": "Starting YunoHost core upgrade...", + "migration_description_0021_migrate_to_bullseye": "Upgrade the system to Debian Bookworm and YunoHost 12", "migration_description_0022_php73_to_php74_pools": "Migrate php7.3-fpm 'pool' conf files to php7.4", "migration_description_0023_postgresql_11_to_13": "Migrate databases from PostgreSQL 11 to 13", "migration_description_0024_rebuild_python_venv": "Repair Python app after bullseye migration", diff --git a/src/migrations/0027_migrate_to_bookworm.py b/src/migrations/0027_migrate_to_bookworm.py new file mode 100644 index 000000000..ef5510db3 --- /dev/null +++ b/src/migrations/0027_migrate_to_bookworm.py @@ -0,0 +1,546 @@ +import glob +import os + +from moulinette import m18n +from yunohost.utils.error import YunohostError +from moulinette.utils.log import getActionLogger +from moulinette.utils.process import check_output, call_async_output +from moulinette.utils.filesystem import read_file, rm, write_to_file + +from yunohost.tools import ( + Migration, + tools_update, + tools_upgrade, + _apt_log_line_is_relevant, +) +from yunohost.app import unstable_apps +from yunohost.regenconf import manually_modified_files, _force_clear_hashes +from yunohost.utils.system import ( + free_space_in_directory, + get_ynh_package_version, + _list_upgradable_apt_packages, +) +from yunohost.service import _get_services, _save_services + +logger = getActionLogger("yunohost.migration") + +N_CURRENT_DEBIAN = 10 +N_CURRENT_YUNOHOST = 4 + +VENV_REQUIREMENTS_SUFFIX = ".requirements_backup_for_bookworm_upgrade.txt" + + +def _get_all_venvs(dir, level=0, maxlevel=3): + """ + Returns the list of all python virtual env directories recursively + + Arguments: + dir - the directory to scan in + maxlevel - the depth of the recursion + level - do not edit this, used as an iterator + """ + if not os.path.exists(dir): + return [] + + result = [] + # Using os functions instead of glob, because glob doesn't support hidden folders, and we need recursion with a fixed depth + for file in os.listdir(dir): + path = os.path.join(dir, file) + if os.path.isdir(path): + activatepath = os.path.join(path, "bin", "activate") + if os.path.isfile(activatepath): + content = read_file(activatepath) + if ("VIRTUAL_ENV" in content) and ("PYTHONHOME" in content): + result.append(path) + continue + if level < maxlevel: + result += _get_all_venvs(path, level=level + 1) + return result + + +def _backup_pip_freeze_for_python_app_venvs(): + """ + Generate a requirements file for all python virtual env located inside /opt/ and /var/www/ + """ + + venvs = _get_all_venvs("/opt/") + _get_all_venvs("/var/www/") + for venv in venvs: + # Generate a requirements file from venv + os.system( + f"{venv}/bin/pip freeze > {venv}{VENV_REQUIREMENTS_SUFFIX} 2>/dev/null" + ) + + +class MyMigration(Migration): + "Upgrade the system to Debian Bookworm and Yunohost 11.x" + + mode = "manual" + + def run(self): + self.check_assertions() + + logger.info(m18n.n("migration_0021_start")) + + # + # Add new apt .deb signing key + # + + new_apt_key = "https://forge.yunohost.org/yunohost_bookworm.asc" + check_output(f"wget -O- {new_apt_key} -q | apt-key add -qq -") + + # + # Patch sources.list + # + logger.info(m18n.n("migration_0021_patching_sources_list")) + self.patch_apt_sources_list() + + # Stupid OVH has some repo configured which dont work with bookworm and break apt ... + os.system("sudo rm -f /etc/apt/sources.list.d/ovh-*.list") + + # Force add sury if it's not there yet + # This is to solve some weird issue with php-common breaking php7.3-common, + # hence breaking many php7.3-deps + # hence triggering some dependency conflict (or foobar-ynh-deps uninstall) + # Adding it there shouldnt be a big deal - Yunohost 11.x does add it + # through its regen conf anyway. + if not os.path.exists("/etc/apt/sources.list.d/extra_php_version.list"): + open("/etc/apt/sources.list.d/extra_php_version.list", "w").write( + "deb https://packages.sury.org/php/ bookworm main" + ) + + # Add Sury key even if extra_php_version.list was already there, + # because some old system may be using an outdated key not valid for Bookworm + # and that'll block the migration + os.system( + 'wget --timeout 900 --quiet "https://packages.sury.org/php/apt.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/extra_php_version.gpg"' + ) + + # Remove legacy, duplicated sury entry if it exists + if os.path.exists("/etc/apt/sources.list.d/sury.list"): + os.system("rm -rf /etc/apt/sources.list.d/sury.list") + + # + # Get requirements of the different venvs from python apps + # + + _backup_pip_freeze_for_python_app_venvs() + + # + # Run apt update + # + + tools_update(target="system") + + # Tell libc6 it's okay to restart system stuff during the upgrade + os.system( + "echo 'libc6 libraries/restart-without-asking boolean true' | debconf-set-selections" + ) + + # Do not restart nginx during the upgrade of nginx-common and nginx-extras ... + # c.f. https://manpages.debian.org/bookworm/init-system-helpers/deb-systemd-invoke.1p.en.html + # and zcat /usr/share/doc/init-system-helpers/README.policy-rc.d.gz + # and the code inside /usr/bin/deb-systemd-invoke to see how it calls /usr/sbin/policy-rc.d ... + # and also invoke-rc.d ... + write_to_file( + "/usr/sbin/policy-rc.d", + '#!/bin/bash\n[[ "$1" =~ "nginx" ]] && [[ "$2" == "restart" ]] && exit 101 || exit 0', + ) + os.system("chmod +x /usr/sbin/policy-rc.d") + + # Don't send an email to root about the postgresql migration. It should be handled automatically after. + os.system( + "echo 'postgresql-common postgresql-common/obsolete-major seen true' | debconf-set-selections" + ) + + # + # Patch yunohost conflicts + # + logger.info(m18n.n("migration_0021_patch_yunohost_conflicts")) + + self.patch_yunohost_conflicts() + + # + # Specific tweaking to get rid of custom my.cnf and use debian's default one + # (my.cnf is actually a symlink to mariadb.cnf) + # + + _force_clear_hashes(["/etc/mysql/my.cnf"]) + rm("/etc/mysql/mariadb.cnf", force=True) + rm("/etc/mysql/my.cnf", force=True) + ret = self.apt_install( + "mariadb-common --reinstall -o Dpkg::Options::='--force-confmiss'" + ) + if ret != 0: + raise YunohostError("Failed to reinstall mariadb-common ?", raw_msg=True) + + # + # /usr/share/yunohost/yunohost-config/ssl/yunoCA -> /usr/share/yunohost/ssl + # + if os.path.exists("/usr/share/yunohost/yunohost-config/ssl/yunoCA"): + os.system( + "mv /usr/share/yunohost/yunohost-config/ssl/yunoCA /usr/share/yunohost/ssl" + ) + rm("/usr/share/yunohost/yunohost-config", recursive=True, force=True) + + # + # /home/yunohost.conf -> /var/cache/yunohost/regenconf + # + if os.path.exists("/home/yunohost.conf"): + os.system("mv /home/yunohost.conf /var/cache/yunohost/regenconf") + rm("/home/yunohost.conf", recursive=True, force=True) + + # Remove legacy postgresql service record added by helpers, + # will now be dynamically handled by the core in bookworm + services = _get_services() + if "postgresql" in services: + del services["postgresql"] + _save_services(services) + + # + # Critical fix for RPI otherwise network is down after rebooting + # https://forum.yunohost.org/t/20652 + # + if os.system("systemctl | grep -q dhcpcd") == 0: + logger.info("Applying fix for DHCPCD ...") + os.system("mkdir -p /etc/systemd/system/dhcpcd.service.d") + write_to_file( + "/etc/systemd/system/dhcpcd.service.d/wait.conf", + "[Service]\nExecStart=\nExecStart=/usr/sbin/dhcpcd -w", + ) + + # + # Main upgrade + # + logger.info(m18n.n("migration_0021_main_upgrade")) + + apps_packages = self.get_apps_equivs_packages() + self.hold(apps_packages) + tools_upgrade(target="system", allow_yunohost_upgrade=False) + + if self.debian_major_version() == N_CURRENT_DEBIAN: + raise YunohostError("migration_0021_still_on_bullseye_after_main_upgrade") + + # Force explicit install of php8.2fpm and other old 'default' dependencies + # that are now only in Recommends + # + # Also, we need to install php8.2 equivalents of other php7.4 dependencies. + # For example, Nextcloud may depend on php7.3-zip, and after the php pool migration + # to autoupgrade Nextcloud to 8.2, it will need the php8.2-zip to work. + # The following list is based on an ad-hoc analysis of php deps found in the + # app ecosystem, with a known equivalent on php8.2. + # + # This is kinda a dirty hack as it doesnt properly update the *-ynh-deps virtual packages + # with the proper list of dependencies, and the dependencies install this way + # will get flagged as 'manually installed'. + # + # We'll probably want to do something during the Bookworm->Bookworm migration to re-flag + # these as 'auto' so they get autoremoved if not needed anymore. + # Also hopefully by then we'll have manifestv2 (maybe) and will be able to use + # the apt resource mecanism to regenerate the *-ynh-deps virtual packages ;) + + php74packages_suffixes = [ + "apcu", + "bcmath", + "bz2", + "dom", + "gmp", + "igbinary", + "imagick", + "imap", + "mbstring", + "memcached", + "mysqli", + "mysqlnd", + "pgsql", + "redis", + "simplexml", + "soap", + "sqlite3", + "ssh2", + "tidy", + "xml", + "xmlrpc", + "xsl", + "zip", + ] + + cmd = ( + "apt show '*-ynh-deps' 2>/dev/null" + " | grep Depends" + f" | grep -o -E \"php7.4-({'|'.join(php73packages_suffixes)})\"" + " | sort | uniq" + " | sed 's/php7.4/php8.2/g'" + " || true" + ) + + basephp82packages_to_install = [ + "php8.2-fpm", + "php8.2-common", + "php8.2-ldap", + "php8.2-intl", + "php8.2-mysql", + "php8.2-gd", + "php8.2-curl", + "php-php-gettext", + ] + + php74packages_to_install = basephp74packages_to_install + [ + f.strip() for f in check_output(cmd).split("\n") if f.strip() + ] + + ret = self.apt_install( + f"{' '.join(php74packages_to_install)} " + "$(dpkg --list | grep ynh-deps | awk '{print $2}') " + "-o Dpkg::Options::='--force-confmiss'" + ) + if ret != 0: + raise YunohostError( + "Failed to force the install of php dependencies ?", raw_msg=True + ) + + # Clean the mess + logger.info(m18n.n("migration_0021_cleaning_up")) + os.system( + "LC_ALL=C DEBIAN_FRONTEND=noninteractive APT_LISTCHANGES_FRONTEND=none apt autoremove --assume-yes" + ) + os.system("apt clean --assume-yes") + + # + # Stupid hack for stupid dnsmasq not picking up its new init.d script then breaking everything ... + # https://forum.yunohost.org/t/20676 + # + if os.path.exists("/etc/init.d/dnsmasq.dpkg-dist"): + logger.info("Copying new version for /etc/init.d/dnsmasq ...") + os.system("cp /etc/init.d/dnsmasq.dpkg-dist /etc/init.d/dnsmasq") + + # + # Yunohost upgrade + # + logger.info(m18n.n("migration_0021_yunohost_upgrade")) + + self.unhold(apps_packages) + + cmd = "LC_ALL=C" + cmd += " DEBIAN_FRONTEND=noninteractive" + cmd += " APT_LISTCHANGES_FRONTEND=none" + cmd += " apt dist-upgrade " + cmd += " --quiet -o=Dpkg::Use-Pty=0 --fix-broken --dry-run" + cmd += " | grep -q 'ynh-deps'" + + logger.info("Simulating upgrade...") + if os.system(cmd) == 0: + raise YunohostError( + "The upgrade cannot be completed, because some app dependencies would need to be removed?", + raw_msg=True, + ) + + postupgradecmds = f"apt-mark auto {' '.join(basephp74packages_to_install)}\n" + postupgradecmds += "rm -f /usr/sbin/policy-rc.d\n" + postupgradecmds += "echo 'Restarting nginx...' >&2\n" + postupgradecmds += "systemctl restart nginx\n" + + tools_upgrade(target="system", postupgradecmds=postupgradecmds) + + def debian_major_version(self): + # The python module "platform" and lsb_release are not reliable because + # on some setup, they may still return Release=9 even after upgrading to + # bullseye ... (Apparently this is related to OVH overriding some stuff + # with /etc/lsb-release for instance -_-) + # Instead, we rely on /etc/os-release which should be the raw info from + # the distribution... + return int( + check_output( + "grep VERSION_ID /etc/os-release | head -n 1 | tr '\"' ' ' | cut -d ' ' -f2" + ) + ) + + def yunohost_major_version(self): + return int(get_ynh_package_version("yunohost")["version"].split(".")[0]) + + def check_assertions(self): + # Be on bullseye (10.x) and yunohost 4.x + # NB : we do both check to cover situations where the upgrade crashed + # in the middle and debian version could be > 9.x but yunohost package + # would still be in 3.x... + if ( + not self.debian_major_version() == N_CURRENT_DEBIAN + and not self.yunohost_major_version() == N_CURRENT_YUNOHOST + ): + try: + # Here we try to find the previous migration log, which should be somewhat recent and be at least 10k (we keep the biggest one) + maybe_previous_migration_log_id = check_output( + "cd /var/log/yunohost/categories/operation && find -name '*migrate*.log' -size +10k -mtime -100 -exec ls -s {} \\; | sort -n | tr './' ' ' | awk '{print $2}' | tail -n 1" + ) + if maybe_previous_migration_log_id: + logger.info( + f"NB: the previous migration log id seems to be {maybe_previous_migration_log_id}. You can share it with the support team with : sudo yunohost log share {maybe_previous_migration_log_id}" + ) + except Exception: + # Yeah it's not that important ... it's to simplify support ... + pass + + raise YunohostError("migration_0021_not_bullseye2") + + # Have > 1 Go free space on /var/ ? + if free_space_in_directory("/var/") / (1024**3) < 1.0: + raise YunohostError("migration_0021_not_enough_free_space") + + # Have > 70 MB free space on /var/ ? + # FIXME: Create a way to ignore this check, on some system 70M is enough... + if free_space_in_directory("/boot/") / (1024**2) < 70.0: + raise YunohostError( + "/boot/ has less than 70MB available. This will probably trigger a crash during the upgrade because a new kernel needs to be installed. Please look for advice on the forum on how to remove old, unused kernels to free up some space in /boot/.", + raw_msg=True, + ) + + # Check system is up to date + # (but we don't if 'bookworm' is already in the sources.list ... + # which means maybe a previous upgrade crashed and we're re-running it) + if os.path.exists("/etc/apt/sources.list") and " bookworm " not in read_file( + "/etc/apt/sources.list" + ): + tools_update(target="system") + upgradable_system_packages = list(_list_upgradable_apt_packages()) + upgradable_system_packages = [ + package["name"] for package in upgradable_system_packages + ] + upgradable_system_packages = set(upgradable_system_packages) + # Lime2 have hold packages to avoid ethernet instability + # See https://github.com/YunoHost/arm-images/commit/b4ef8c99554fd1a122a306db7abacc4e2f2942df + lime2_hold_packages = set( + [ + "armbian-firmware", + "armbian-bsp-cli-lime2", + "linux-dtb-current-sunxi", + "linux-image-current-sunxi", + "linux-u-boot-lime2-current", + "linux-image-next-sunxi", + ] + ) + if upgradable_system_packages - lime2_hold_packages: + raise YunohostError("migration_0021_system_not_fully_up_to_date") + + @property + def disclaimer(self): + # Avoid having a super long disclaimer + uncessary check if we ain't + # on bullseye / yunohost 4.x anymore + # NB : we do both check to cover situations where the upgrade crashed + # in the middle and debian version could be >= 10.x but yunohost package + # would still be in 4.x... + if ( + not self.debian_major_version() == N_CURRENT_DEBIAN + and not self.yunohost_major_version() == N_CURRENT_YUNOHOST + ): + return None + + # Get list of problematic apps ? I.e. not official or community+working + problematic_apps = unstable_apps() + problematic_apps = "".join(["\n - " + app for app in problematic_apps]) + + # Manually modified files ? (c.f. yunohost service regen-conf) + modified_files = manually_modified_files() + modified_files = "".join(["\n - " + f for f in modified_files]) + + message = m18n.n("migration_0021_general_warning") + + message = ( + "N.B.: This migration has been tested by the community over the last few months but has only been declared stable recently. If your server hosts critical services and if you are not too confident with debugging possible issues, we recommend you to wait a little bit more while we gather more feedback and polish things up. If on the other hand you are relatively confident with debugging small issues that may arise, you are encouraged to run this migration ;)! You can read about remaining known issues and feedback from the community here: https://forum.yunohost.org/t/20590\n\n" + + message + ) + + if problematic_apps: + message += "\n\n" + m18n.n( + "migration_0021_problematic_apps_warning", + problematic_apps=problematic_apps, + ) + + if modified_files: + message += "\n\n" + m18n.n( + "migration_0021_modified_files", manually_modified_files=modified_files + ) + + return message + + def patch_apt_sources_list(self): + sources_list = glob.glob("/etc/apt/sources.list.d/*.list") + if os.path.exists("/etc/apt/sources.list"): + sources_list.append("/etc/apt/sources.list") + + # This : + # - replace single 'bullseye' occurence by 'bulleye' + # - comments lines containing "backports" + for f in sources_list: + command = ( + f"sed -i {f} " + "-e 's@ bullseye @ bookworm @g' " + "-e '/backports/ s@^#*@#@' " + "-e 's@ bullseye-@ bookworm-@g' " + ) + os.system(command) + + def get_apps_equivs_packages(self): + command = ( + "dpkg --get-selections" + " | grep -v deinstall" + " | awk '{print $1}'" + " | { grep 'ynh-deps$' || true; }" + ) + + output = check_output(command) + + return output.split("\n") if output else [] + + def hold(self, packages): + for package in packages: + os.system(f"apt-mark hold {package}") + + def unhold(self, packages): + for package in packages: + os.system(f"apt-mark unhold {package}") + + def apt_install(self, cmd): + def is_relevant(line): + return "Reading database ..." not in line.rstrip() + + callbacks = ( + lambda l: logger.info("+ " + l.rstrip() + "\r") + if _apt_log_line_is_relevant(l) + else logger.debug(l.rstrip() + "\r"), + lambda l: logger.warning(l.rstrip()) + if _apt_log_line_is_relevant(l) + else logger.debug(l.rstrip()), + ) + + cmd = ( + "LC_ALL=C DEBIAN_FRONTEND=noninteractive APT_LISTCHANGES_FRONTEND=none apt install --quiet -o=Dpkg::Use-Pty=0 --fix-broken --assume-yes " + + cmd + ) + + logger.debug("Running: %s" % cmd) + + return call_async_output(cmd, callbacks, shell=True) + + def patch_yunohost_conflicts(self): + # + # This is a super dirty hack to remove the conflicts from yunohost's debian/control file + # Those conflicts are there to prevent mistakenly upgrading critical packages + # such as dovecot, postfix, nginx, openssl, etc... usually related to mistakenly + # using backports etc. + # + # The hack consists in savagely removing the conflicts directly in /var/lib/dpkg/status + # + + # We only patch the conflict if we're on yunohost 4.x + if self.yunohost_major_version() != N_CURRENT_YUNOHOST: + return + + conflicts = check_output("dpkg-query -s yunohost | grep '^Conflicts:'").strip() + if conflicts: + # We want to keep conflicting with apache/bind9 tho + new_conflicts = "Conflicts: apache2, bind9" + + command = ( + f"sed -i /var/lib/dpkg/status -e 's@{conflicts}@{new_conflicts}@g'" + ) + logger.debug(f"Running: {command}") + os.system(command) From 2dbe34c0301a4bc666d3175d998ea68c70bb705c Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Fri, 5 May 2023 23:44:46 +0000 Subject: [PATCH 16/82] aaand mariadb-server ofc... --- .gitlab/ci/test.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index 013f45ee1..c5f1ee13c 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -1,6 +1,6 @@ .install_debs: &install_debs - apt-get update -o Acquire::Retries=3 - - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb php8.2-cli mariadb-client + - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb php8.2-cli mariadb-client mariadb-server - pip3 install -U mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 "packaging<22" --break-system-packages # for bookworm .test-stage: From f617287eb2e74103ef4f54dfe3d52a810e51514e Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Fri, 5 May 2023 23:58:30 +0000 Subject: [PATCH 17/82] woops thanks codeql i guess --- src/migrations/0027_migrate_to_bookworm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/migrations/0027_migrate_to_bookworm.py b/src/migrations/0027_migrate_to_bookworm.py index ef5510db3..85e2235af 100644 --- a/src/migrations/0027_migrate_to_bookworm.py +++ b/src/migrations/0027_migrate_to_bookworm.py @@ -267,7 +267,7 @@ class MyMigration(Migration): cmd = ( "apt show '*-ynh-deps' 2>/dev/null" " | grep Depends" - f" | grep -o -E \"php7.4-({'|'.join(php73packages_suffixes)})\"" + f" | grep -o -E \"php7.4-({'|'.join(php74packages_suffixes)})\"" " | sort | uniq" " | sed 's/php7.4/php8.2/g'" " || true" @@ -284,7 +284,7 @@ class MyMigration(Migration): "php-php-gettext", ] - php74packages_to_install = basephp74packages_to_install + [ + php74packages_to_install = basephp82packages_to_install + [ f.strip() for f in check_output(cmd).split("\n") if f.strip() ] From 691ce5eace8f8a0f107d607318d7fc02da5bcb56 Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Sat, 13 May 2023 14:37:32 +0000 Subject: [PATCH 18/82] fix: python3.11 now supports Possessive Quantifiers regex --- src/tests/test_appurl.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/tests/test_appurl.py b/src/tests/test_appurl.py index 351bb4e83..8e5b14d34 100644 --- a/src/tests/test_appurl.py +++ b/src/tests/test_appurl.py @@ -202,10 +202,6 @@ def test_normalize_permission_path_with_bad_regex(): ) # Full Regex - with pytest.raises(YunohostError): - _validate_and_sanitize_permission_url( - "re:" + maindomain + "/yolo?+/", maindomain + "/path", "test_permission" - ) with pytest.raises(YunohostError): _validate_and_sanitize_permission_url( "re:" + maindomain + "/yolo[1-9]**/", From 482f8f3443a37f5644a7be9c4743c453981659f9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 15 May 2023 17:37:11 +0200 Subject: [PATCH 19/82] backup: fix again backup hook for xmpp when data folder dont exist --- hooks/backup/27-data_xmpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/backup/27-data_xmpp b/hooks/backup/27-data_xmpp index 078184ed8..253aacdc2 100644 --- a/hooks/backup/27-data_xmpp +++ b/hooks/backup/27-data_xmpp @@ -9,5 +9,5 @@ source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/data/xmpp" -ynh_backup /var/lib/metronome "${backup_dir}/var_lib_metronome" --not_mandatory -ynh_backup /var/xmpp-upload/ "${backup_dir}/var_xmpp-upload" --not_mandatory +[[ ! -d /var/lib/metronome ]] || ynh_backup /var/lib/metronome "${backup_dir}/var_lib_metronome" --not_mandatory +[[ ! -d /var/xmpp-upload ]] || ynh_backup /var/xmpp-upload/ "${backup_dir}/var_xmpp-upload" --not_mandatory From 031c641b77cae7139c4be33f819e2c23cb0570ad Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 15 May 2023 18:51:18 +0200 Subject: [PATCH 20/82] tests: fix spy on m18n.n which in some cases doesnt work anymore ... not sure why, bit confused ... --- src/tests/conftest.py | 13 ++++-- src/tests/test_apps.py | 40 ++++++++--------- src/tests/test_backuprestore.py | 78 ++++++++++++++++---------------- src/tests/test_ldapauth.py | 2 +- src/tests/test_permission.py | 80 ++++++++++++++++----------------- src/tests/test_regenconf.py | 4 +- src/tests/test_user-group.py | 44 +++++++++--------- 7 files changed, 133 insertions(+), 128 deletions(-) diff --git a/src/tests/conftest.py b/src/tests/conftest.py index 393c33564..abba2ee19 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -1,5 +1,6 @@ import os import pytest +from unittest.mock import Mock import moulinette from moulinette import m18n, Moulinette @@ -23,11 +24,15 @@ def get_test_apps_dir(): @contextmanager -def message(mocker, key, **kwargs): - mocker.spy(m18n, "n") +def message(key, **kwargs): + m = Mock(wraps=m18n.n) + old_m18n = m18n.n + m18n.n = m yield - m18n.n.assert_any_call(key, **kwargs) - + try: + m.assert_any_call(key, **kwargs) + finally: + m18n.n = old_m18n @contextmanager def raiseYunohostError(mocker, key, **kwargs): diff --git a/src/tests/test_apps.py b/src/tests/test_apps.py index 747eb5dcd..a0c431531 100644 --- a/src/tests/test_apps.py +++ b/src/tests/test_apps.py @@ -392,9 +392,9 @@ def test_legacy_app_install_private(secondary_domain): assert app_is_not_installed(secondary_domain, "legacy_app") -def test_legacy_app_install_unknown_domain(mocker): +def test_legacy_app_install_unknown_domain(): with pytest.raises(YunohostError): - with message(mocker, "app_argument_invalid"): + with message("app_argument_invalid"): install_legacy_app("whatever.nope", "/legacy") assert app_is_not_installed("whatever.nope", "legacy_app") @@ -421,12 +421,12 @@ def test_legacy_app_install_multiple_instances(secondary_domain): assert app_is_not_installed(secondary_domain, "legacy_app__2") -def test_legacy_app_install_path_unavailable(mocker, secondary_domain): +def test_legacy_app_install_path_unavailable(secondary_domain): # These will be removed in teardown install_legacy_app(secondary_domain, "/legacy") with pytest.raises(YunohostError): - with message(mocker, "app_location_unavailable"): + with message("app_location_unavailable"): install_legacy_app(secondary_domain, "/") assert app_is_installed(secondary_domain, "legacy_app") @@ -442,19 +442,19 @@ def test_legacy_app_install_with_nginx_down(mocker, secondary_domain): install_legacy_app(secondary_domain, "/legacy") -def test_legacy_app_failed_install(mocker, secondary_domain): +def test_legacy_app_failed_install(secondary_domain): # This will conflict with the folder that the app # attempts to create, making the install fail mkdir("/var/www/legacy_app/", 0o750) with pytest.raises(YunohostError): - with message(mocker, "app_install_script_failed"): + with message("app_install_script_failed"): install_legacy_app(secondary_domain, "/legacy") assert app_is_not_installed(secondary_domain, "legacy_app") -def test_legacy_app_failed_remove(mocker, secondary_domain): +def test_legacy_app_failed_remove(secondary_domain): install_legacy_app(secondary_domain, "/legacy") # The remove script runs with set -eu and attempt to remove this @@ -486,52 +486,52 @@ def test_full_domain_app_with_conflicts(mocker, secondary_domain): install_full_domain_app(secondary_domain) -def test_systemfuckedup_during_app_install(mocker, secondary_domain): +def test_systemfuckedup_during_app_install(secondary_domain): with pytest.raises(YunohostError): - with message(mocker, "app_install_failed"): - with message(mocker, "app_action_broke_system"): + with message("app_install_failed"): + with message("app_action_broke_system"): install_break_yo_system(secondary_domain, breakwhat="install") assert app_is_not_installed(secondary_domain, "break_yo_system") -def test_systemfuckedup_during_app_remove(mocker, secondary_domain): +def test_systemfuckedup_during_app_remove(secondary_domain): install_break_yo_system(secondary_domain, breakwhat="remove") with pytest.raises(YunohostError): - with message(mocker, "app_action_broke_system"): - with message(mocker, "app_removed"): + with message("app_action_broke_system"): + with message("app_removed"): app_remove("break_yo_system") assert app_is_not_installed(secondary_domain, "break_yo_system") -def test_systemfuckedup_during_app_install_and_remove(mocker, secondary_domain): +def test_systemfuckedup_during_app_install_and_remove(secondary_domain): with pytest.raises(YunohostError): - with message(mocker, "app_install_failed"): - with message(mocker, "app_action_broke_system"): + with message("app_install_failed"): + with message("app_action_broke_system"): install_break_yo_system(secondary_domain, breakwhat="everything") assert app_is_not_installed(secondary_domain, "break_yo_system") -def test_systemfuckedup_during_app_upgrade(mocker, secondary_domain): +def test_systemfuckedup_during_app_upgrade(secondary_domain): install_break_yo_system(secondary_domain, breakwhat="upgrade") with pytest.raises(YunohostError): - with message(mocker, "app_action_broke_system"): + with message("app_action_broke_system"): app_upgrade( "break_yo_system", file=os.path.join(get_test_apps_dir(), "break_yo_system_ynh"), ) -def test_failed_multiple_app_upgrade(mocker, secondary_domain): +def test_failed_multiple_app_upgrade(secondary_domain): install_legacy_app(secondary_domain, "/legacy") install_break_yo_system(secondary_domain, breakwhat="upgrade") with pytest.raises(YunohostError): - with message(mocker, "app_not_upgraded"): + with message("app_not_upgraded"): app_upgrade( ["break_yo_system", "legacy_app"], file={ diff --git a/src/tests/test_backuprestore.py b/src/tests/test_backuprestore.py index 413d44470..a2dcfe8fb 100644 --- a/src/tests/test_backuprestore.py +++ b/src/tests/test_backuprestore.py @@ -236,10 +236,10 @@ def add_archive_system_from_4p2(): # -def test_backup_only_ldap(mocker): +def test_backup_only_ldap(): # Create the backup name = random_ascii(8) - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create(name=name, system=["conf_ldap"], apps=None) archives = backup_list()["archives"] @@ -253,7 +253,7 @@ def test_backup_only_ldap(mocker): def test_backup_system_part_that_does_not_exists(mocker): # Create the backup - with message(mocker, "backup_hook_unknown", hook="doesnt_exist"): + with message("backup_hook_unknown", hook="doesnt_exist"): with raiseYunohostError(mocker, "backup_nothings_done"): backup_create(system=["doesnt_exist"], apps=None) @@ -263,10 +263,10 @@ def test_backup_system_part_that_does_not_exists(mocker): # -def test_backup_and_restore_all_sys(mocker): +def test_backup_and_restore_all_sys(): name = random_ascii(8) # Create the backup - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create(name=name, system=[], apps=None) archives = backup_list()["archives"] @@ -284,7 +284,7 @@ def test_backup_and_restore_all_sys(mocker): assert not os.path.exists("/etc/ssowat/conf.json") # Restore the backup - with message(mocker, "restore_complete"): + with message("restore_complete"): backup_restore(name=archives[0], force=True, system=[], apps=None) # Check ssowat conf is back @@ -297,17 +297,17 @@ def test_backup_and_restore_all_sys(mocker): @pytest.mark.with_system_archive_from_4p2 -def test_restore_system_from_Ynh4p2(monkeypatch, mocker): +def test_restore_system_from_Ynh4p2(monkeypatch): name = random_ascii(8) # Backup current system - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create(name=name, system=[], apps=None) archives = backup_list()["archives"] assert len(archives) == 2 # Restore system archive from 3.8 try: - with message(mocker, "restore_complete"): + with message("restore_complete"): backup_restore( name=backup_list()["archives"][1], system=[], apps=None, force=True ) @@ -336,7 +336,7 @@ def test_backup_script_failure_handling(monkeypatch, mocker): # with the expected error message key monkeypatch.setattr("yunohost.backup.hook_exec", custom_hook_exec) - with message(mocker, "backup_app_failed", app="backup_recommended_app"): + with message("backup_app_failed", app="backup_recommended_app"): with raiseYunohostError(mocker, "backup_nothings_done"): backup_create(system=None, apps=["backup_recommended_app"]) @@ -363,7 +363,7 @@ def test_backup_not_enough_free_space(monkeypatch, mocker): def test_backup_app_not_installed(mocker): assert not _is_installed("wordpress") - with message(mocker, "unbackup_app", app="wordpress"): + with message("unbackup_app", app="wordpress"): with raiseYunohostError(mocker, "backup_nothings_done"): backup_create(system=None, apps=["wordpress"]) @@ -375,14 +375,14 @@ def test_backup_app_with_no_backup_script(mocker): assert not os.path.exists(backup_script) with message( - mocker, "backup_with_no_backup_script_for_app", app="backup_recommended_app" + "backup_with_no_backup_script_for_app", app="backup_recommended_app" ): with raiseYunohostError(mocker, "backup_nothings_done"): backup_create(system=None, apps=["backup_recommended_app"]) @pytest.mark.with_backup_recommended_app_installed -def test_backup_app_with_no_restore_script(mocker): +def test_backup_app_with_no_restore_script(): restore_script = "/etc/yunohost/apps/backup_recommended_app/scripts/restore" os.system("rm %s" % restore_script) assert not os.path.exists(restore_script) @@ -391,16 +391,16 @@ def test_backup_app_with_no_restore_script(mocker): # user... with message( - mocker, "backup_with_no_restore_script_for_app", app="backup_recommended_app" + "backup_with_no_restore_script_for_app", app="backup_recommended_app" ): backup_create(system=None, apps=["backup_recommended_app"]) @pytest.mark.clean_opt_dir -def test_backup_with_different_output_directory(mocker): +def test_backup_with_different_output_directory(): name = random_ascii(8) # Create the backup - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create( system=["conf_ynh_settings"], apps=None, @@ -420,10 +420,10 @@ def test_backup_with_different_output_directory(mocker): @pytest.mark.clean_opt_dir -def test_backup_using_copy_method(mocker): +def test_backup_using_copy_method(): # Create the backup name = random_ascii(8) - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create( system=["conf_ynh_settings"], apps=None, @@ -442,8 +442,8 @@ def test_backup_using_copy_method(mocker): @pytest.mark.with_wordpress_archive_from_4p2 @pytest.mark.with_custom_domain("yolo.test") -def test_restore_app_wordpress_from_Ynh4p2(mocker): - with message(mocker, "restore_complete"): +def test_restore_app_wordpress_from_Ynh4p2(): + with message("restore_complete"): backup_restore( system=None, name=backup_list()["archives"][0], apps=["wordpress"] ) @@ -461,7 +461,7 @@ def test_restore_app_script_failure_handling(monkeypatch, mocker): assert not _is_installed("wordpress") - with message(mocker, "app_restore_script_failed"): + with message("app_restore_script_failed"): with raiseYunohostError(mocker, "restore_nothings_done"): backup_restore( system=None, name=backup_list()["archives"][0], apps=["wordpress"] @@ -494,7 +494,7 @@ def test_restore_app_not_in_backup(mocker): assert not _is_installed("wordpress") assert not _is_installed("yoloswag") - with message(mocker, "backup_archive_app_not_found", app="yoloswag"): + with message("backup_archive_app_not_found", app="yoloswag"): with raiseYunohostError(mocker, "restore_nothings_done"): backup_restore( system=None, name=backup_list()["archives"][0], apps=["yoloswag"] @@ -509,7 +509,7 @@ def test_restore_app_not_in_backup(mocker): def test_restore_app_already_installed(mocker): assert not _is_installed("wordpress") - with message(mocker, "restore_complete"): + with message("restore_complete"): backup_restore( system=None, name=backup_list()["archives"][0], apps=["wordpress"] ) @@ -525,22 +525,22 @@ def test_restore_app_already_installed(mocker): @pytest.mark.with_legacy_app_installed -def test_backup_and_restore_legacy_app(mocker): - _test_backup_and_restore_app(mocker, "legacy_app") +def test_backup_and_restore_legacy_app(): + _test_backup_and_restore_app("legacy_app") @pytest.mark.with_backup_recommended_app_installed -def test_backup_and_restore_recommended_app(mocker): - _test_backup_and_restore_app(mocker, "backup_recommended_app") +def test_backup_and_restore_recommended_app(): + _test_backup_and_restore_app("backup_recommended_app") @pytest.mark.with_backup_recommended_app_installed_with_ynh_restore -def test_backup_and_restore_with_ynh_restore(mocker): - _test_backup_and_restore_app(mocker, "backup_recommended_app") +def test_backup_and_restore_with_ynh_restore(): + _test_backup_and_restore_app("backup_recommended_app") @pytest.mark.with_permission_app_installed -def test_backup_and_restore_permission_app(mocker): +def test_backup_and_restore_permission_app(): res = user_permission_list(full=True)["permissions"] assert "permissions_app.main" in res assert "permissions_app.admin" in res @@ -554,7 +554,7 @@ def test_backup_and_restore_permission_app(mocker): assert res["permissions_app.admin"]["allowed"] == ["alice"] assert res["permissions_app.dev"]["allowed"] == [] - _test_backup_and_restore_app(mocker, "permissions_app") + _test_backup_and_restore_app("permissions_app") res = user_permission_list(full=True)["permissions"] assert "permissions_app.main" in res @@ -570,10 +570,10 @@ def test_backup_and_restore_permission_app(mocker): assert res["permissions_app.dev"]["allowed"] == [] -def _test_backup_and_restore_app(mocker, app): +def _test_backup_and_restore_app(app): # Create a backup of this app name = random_ascii(8) - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create(name=name, system=None, apps=[app]) archives = backup_list()["archives"] @@ -590,7 +590,7 @@ def _test_backup_and_restore_app(mocker, app): assert app + ".main" not in user_permission_list()["permissions"] # Restore the app - with message(mocker, "restore_complete"): + with message("restore_complete"): backup_restore(system=None, name=archives[0], apps=[app]) assert app_is_installed(app) @@ -631,19 +631,19 @@ def test_restore_archive_with_bad_archive(mocker): clean_tmp_backup_directory() -def test_restore_archive_with_custom_hook(mocker): +def test_restore_archive_with_custom_hook(): custom_restore_hook_folder = os.path.join(CUSTOM_HOOK_FOLDER, "restore") os.system("touch %s/99-yolo" % custom_restore_hook_folder) # Backup with custom hook system name = random_ascii(8) - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create(name=name, system=[], apps=None) archives = backup_list()["archives"] assert len(archives) == 1 # Restore system with custom hook - with message(mocker, "restore_complete"): + with message("restore_complete"): backup_restore( name=backup_list()["archives"][0], system=[], apps=None, force=True ) @@ -651,7 +651,7 @@ def test_restore_archive_with_custom_hook(mocker): os.system("rm %s/99-yolo" % custom_restore_hook_folder) -def test_backup_binds_are_readonly(mocker, monkeypatch): +def test_backup_binds_are_readonly(monkeypatch): def custom_mount_and_backup(self): self._organize_files() @@ -676,5 +676,5 @@ def test_backup_binds_are_readonly(mocker, monkeypatch): # Create the backup name = random_ascii(8) - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create(name=name, system=[]) diff --git a/src/tests/test_ldapauth.py b/src/tests/test_ldapauth.py index 9e3ae36cc..73cb09d27 100644 --- a/src/tests/test_ldapauth.py +++ b/src/tests/test_ldapauth.py @@ -59,7 +59,7 @@ def test_authenticate_with_wrong_password(): assert expected_msg in str(exception) -def test_authenticate_server_down(mocker): +def test_authenticate_server_down(): os.system("systemctl stop slapd && sleep 5") LDAPAuth().authenticate_credentials(credentials="alice:Yunohost") diff --git a/src/tests/test_permission.py b/src/tests/test_permission.py index 10bd018d2..4ab333584 100644 --- a/src/tests/test_permission.py +++ b/src/tests/test_permission.py @@ -435,8 +435,8 @@ def test_permission_list(): # -def test_permission_create_main(mocker): - with message(mocker, "permission_created", permission="site.main"): +def test_permission_create_main(): + with message("permission_created", permission="site.main"): permission_create("site.main", allowed=["all_users"], protected=False) res = user_permission_list(full=True)["permissions"] @@ -446,8 +446,8 @@ def test_permission_create_main(mocker): assert res["site.main"]["protected"] is False -def test_permission_create_extra(mocker): - with message(mocker, "permission_created", permission="site.test"): +def test_permission_create_extra(): + with message("permission_created", permission="site.test"): permission_create("site.test") res = user_permission_list(full=True)["permissions"] @@ -466,8 +466,8 @@ def test_permission_create_with_specific_user(): assert res["site.test"]["allowed"] == ["alice"] -def test_permission_create_with_tile_management(mocker): - with message(mocker, "permission_created", permission="site.main"): +def test_permission_create_with_tile_management(): + with message("permission_created", permission="site.main"): _permission_create_with_dummy_app( "site.main", allowed=["all_users"], @@ -483,8 +483,8 @@ def test_permission_create_with_tile_management(mocker): assert res["site.main"]["show_tile"] is False -def test_permission_create_with_tile_management_with_main_default_value(mocker): - with message(mocker, "permission_created", permission="site.main"): +def test_permission_create_with_tile_management_with_main_default_value(): + with message("permission_created", permission="site.main"): _permission_create_with_dummy_app( "site.main", allowed=["all_users"], @@ -500,8 +500,8 @@ def test_permission_create_with_tile_management_with_main_default_value(mocker): assert res["site.main"]["show_tile"] is True -def test_permission_create_with_tile_management_with_not_main_default_value(mocker): - with message(mocker, "permission_created", permission="wiki.api"): +def test_permission_create_with_tile_management_with_not_main_default_value(): + with message("permission_created", permission="wiki.api"): _permission_create_with_dummy_app( "wiki.api", allowed=["all_users"], @@ -517,8 +517,8 @@ def test_permission_create_with_tile_management_with_not_main_default_value(mock assert res["wiki.api"]["show_tile"] is True -def test_permission_create_with_urls_management_without_url(mocker): - with message(mocker, "permission_created", permission="wiki.api"): +def test_permission_create_with_urls_management_without_url(): + with message("permission_created", permission="wiki.api"): _permission_create_with_dummy_app( "wiki.api", allowed=["all_users"], domain=maindomain, path="/site" ) @@ -530,8 +530,8 @@ def test_permission_create_with_urls_management_without_url(mocker): assert res["wiki.api"]["auth_header"] is True -def test_permission_create_with_urls_management_simple_domain(mocker): - with message(mocker, "permission_created", permission="site.main"): +def test_permission_create_with_urls_management_simple_domain(): + with message("permission_created", permission="site.main"): _permission_create_with_dummy_app( "site.main", allowed=["all_users"], @@ -553,8 +553,8 @@ def test_permission_create_with_urls_management_simple_domain(mocker): @pytest.mark.other_domains(number=2) -def test_permission_create_with_urls_management_multiple_domain(mocker): - with message(mocker, "permission_created", permission="site.main"): +def test_permission_create_with_urls_management_multiple_domain(): + with message("permission_created", permission="site.main"): _permission_create_with_dummy_app( "site.main", allowed=["all_users"], @@ -575,14 +575,14 @@ def test_permission_create_with_urls_management_multiple_domain(mocker): assert res["site.main"]["auth_header"] is True -def test_permission_delete(mocker): - with message(mocker, "permission_deleted", permission="wiki.main"): +def test_permission_delete(): + with message("permission_deleted", permission="wiki.main"): permission_delete("wiki.main", force=True) res = user_permission_list()["permissions"] assert "wiki.main" not in res - with message(mocker, "permission_deleted", permission="blog.api"): + with message("permission_deleted", permission="blog.api"): permission_delete("blog.api", force=False) res = user_permission_list()["permissions"] @@ -625,8 +625,8 @@ def test_permission_delete_main_without_force(mocker): # user side functions -def test_permission_add_group(mocker): - with message(mocker, "permission_updated", permission="wiki.main"): +def test_permission_add_group(): + with message("permission_updated", permission="wiki.main"): user_permission_update("wiki.main", add="alice") res = user_permission_list(full=True)["permissions"] @@ -634,8 +634,8 @@ def test_permission_add_group(mocker): assert set(res["wiki.main"]["corresponding_users"]) == {"alice", "bob"} -def test_permission_remove_group(mocker): - with message(mocker, "permission_updated", permission="blog.main"): +def test_permission_remove_group(): + with message("permission_updated", permission="blog.main"): user_permission_update("blog.main", remove="alice") res = user_permission_list(full=True)["permissions"] @@ -643,8 +643,8 @@ def test_permission_remove_group(mocker): assert res["blog.main"]["corresponding_users"] == [] -def test_permission_add_and_remove_group(mocker): - with message(mocker, "permission_updated", permission="wiki.main"): +def test_permission_add_and_remove_group(): + with message("permission_updated", permission="wiki.main"): user_permission_update("wiki.main", add="alice", remove="all_users") res = user_permission_list(full=True)["permissions"] @@ -652,9 +652,9 @@ def test_permission_add_and_remove_group(mocker): assert res["wiki.main"]["corresponding_users"] == ["alice"] -def test_permission_add_group_already_allowed(mocker): +def test_permission_add_group_already_allowed(): with message( - mocker, "permission_already_allowed", permission="blog.main", group="alice" + "permission_already_allowed", permission="blog.main", group="alice" ): user_permission_update("blog.main", add="alice") @@ -663,9 +663,9 @@ def test_permission_add_group_already_allowed(mocker): assert res["blog.main"]["corresponding_users"] == ["alice"] -def test_permission_remove_group_already_not_allowed(mocker): +def test_permission_remove_group_already_not_allowed(): with message( - mocker, "permission_already_disallowed", permission="blog.main", group="bob" + "permission_already_disallowed", permission="blog.main", group="bob" ): user_permission_update("blog.main", remove="bob") @@ -674,8 +674,8 @@ def test_permission_remove_group_already_not_allowed(mocker): assert res["blog.main"]["corresponding_users"] == ["alice"] -def test_permission_reset(mocker): - with message(mocker, "permission_updated", permission="blog.main"): +def test_permission_reset(): + with message("permission_updated", permission="blog.main"): user_permission_reset("blog.main") res = user_permission_list(full=True)["permissions"] @@ -693,42 +693,42 @@ def test_permission_reset_idempotency(): assert set(res["blog.main"]["corresponding_users"]) == {"alice", "bob"} -def test_permission_change_label(mocker): - with message(mocker, "permission_updated", permission="wiki.main"): +def test_permission_change_label(): + with message("permission_updated", permission="wiki.main"): user_permission_update("wiki.main", label="New Wiki") res = user_permission_list(full=True)["permissions"] assert res["wiki.main"]["label"] == "New Wiki" -def test_permission_change_label_with_same_value(mocker): - with message(mocker, "permission_updated", permission="wiki.main"): +def test_permission_change_label_with_same_value(): + with message("permission_updated", permission="wiki.main"): user_permission_update("wiki.main", label="Wiki") res = user_permission_list(full=True)["permissions"] assert res["wiki.main"]["label"] == "Wiki" -def test_permission_switch_show_tile(mocker): +def test_permission_switch_show_tile(): # Note that from the actionmap the value is passed as string, not as bool # Try with lowercase - with message(mocker, "permission_updated", permission="wiki.main"): + with message("permission_updated", permission="wiki.main"): user_permission_update("wiki.main", show_tile="false") res = user_permission_list(full=True)["permissions"] assert res["wiki.main"]["show_tile"] is False # Try with uppercase - with message(mocker, "permission_updated", permission="wiki.main"): + with message("permission_updated", permission="wiki.main"): user_permission_update("wiki.main", show_tile="TRUE") res = user_permission_list(full=True)["permissions"] assert res["wiki.main"]["show_tile"] is True -def test_permission_switch_show_tile_with_same_value(mocker): +def test_permission_switch_show_tile_with_same_value(): # Note that from the actionmap the value is passed as string, not as bool - with message(mocker, "permission_updated", permission="wiki.main"): + with message("permission_updated", permission="wiki.main"): user_permission_update("wiki.main", show_tile="True") res = user_permission_list(full=True)["permissions"] diff --git a/src/tests/test_regenconf.py b/src/tests/test_regenconf.py index 8dda1a7f2..3966ef291 100644 --- a/src/tests/test_regenconf.py +++ b/src/tests/test_regenconf.py @@ -87,7 +87,7 @@ def test_ssh_conf_unmanaged(): assert SSHD_CONFIG in _get_conf_hashes("ssh") -def test_ssh_conf_unmanaged_and_manually_modified(mocker): +def test_ssh_conf_unmanaged_and_manually_modified(): _force_clear_hashes([SSHD_CONFIG]) os.system("echo ' ' >> %s" % SSHD_CONFIG) @@ -98,7 +98,7 @@ def test_ssh_conf_unmanaged_and_manually_modified(mocker): assert SSHD_CONFIG in _get_conf_hashes("ssh") assert SSHD_CONFIG in manually_modified_files() - with message(mocker, "regenconf_need_to_explicitly_specify_ssh"): + with message("regenconf_need_to_explicitly_specify_ssh"): regen_conf(force=True) assert SSHD_CONFIG in _get_conf_hashes("ssh") diff --git a/src/tests/test_user-group.py b/src/tests/test_user-group.py index eececb827..57f9ffa3f 100644 --- a/src/tests/test_user-group.py +++ b/src/tests/test_user-group.py @@ -91,8 +91,8 @@ def test_list_groups(): # -def test_create_user(mocker): - with message(mocker, "user_created"): +def test_create_user(): + with message("user_created"): user_create("albert", maindomain, "test123Ynh", fullname="Albert Good") group_res = user_group_list()["groups"] @@ -102,8 +102,8 @@ def test_create_user(mocker): assert "albert" in group_res["all_users"]["members"] -def test_del_user(mocker): - with message(mocker, "user_deleted"): +def test_del_user(): + with message("user_deleted"): user_delete("alice") group_res = user_group_list()["groups"] @@ -112,7 +112,7 @@ def test_del_user(mocker): assert "alice" not in group_res["all_users"]["members"] -def test_import_user(mocker): +def test_import_user(): import csv from io import StringIO @@ -157,7 +157,7 @@ def test_import_user(mocker): } ) csv_io.seek(0) - with message(mocker, "user_import_success"): + with message("user_import_success"): user_import(csv_io, update=True, delete=True) group_res = user_group_list()["groups"] @@ -171,7 +171,7 @@ def test_import_user(mocker): assert "alice" not in group_res["dev"]["members"] -def test_export_user(mocker): +def test_export_user(): result = user_export() should_be = ( "username;firstname;lastname;password;mail;mail-alias;mail-forward;mailbox-quota;groups\r\n" @@ -182,8 +182,8 @@ def test_export_user(mocker): assert result == should_be -def test_create_group(mocker): - with message(mocker, "group_created", group="adminsys"): +def test_create_group(): + with message("group_created", group="adminsys"): user_group_create("adminsys") group_res = user_group_list()["groups"] @@ -192,8 +192,8 @@ def test_create_group(mocker): assert group_res["adminsys"]["members"] == [] -def test_del_group(mocker): - with message(mocker, "group_deleted", group="dev"): +def test_del_group(): + with message("group_deleted", group="dev"): user_group_delete("dev") group_res = user_group_list()["groups"] @@ -262,46 +262,46 @@ def test_del_group_that_does_not_exist(mocker): # -def test_update_user(mocker): - with message(mocker, "user_updated"): +def test_update_user(): + with message("user_updated"): user_update("alice", firstname="NewName", lastname="NewLast") info = user_info("alice") assert info["fullname"] == "NewName NewLast" - with message(mocker, "user_updated"): + with message("user_updated"): user_update("alice", fullname="New2Name New2Last") info = user_info("alice") assert info["fullname"] == "New2Name New2Last" -def test_update_group_add_user(mocker): - with message(mocker, "group_updated", group="dev"): +def test_update_group_add_user(): + with message("group_updated", group="dev"): user_group_update("dev", add=["bob"]) group_res = user_group_list()["groups"] assert set(group_res["dev"]["members"]) == {"alice", "bob"} -def test_update_group_add_user_already_in(mocker): - with message(mocker, "group_user_already_in_group", user="bob", group="apps"): +def test_update_group_add_user_already_in(): + with message("group_user_already_in_group", user="bob", group="apps"): user_group_update("apps", add=["bob"]) group_res = user_group_list()["groups"] assert group_res["apps"]["members"] == ["bob"] -def test_update_group_remove_user(mocker): - with message(mocker, "group_updated", group="apps"): +def test_update_group_remove_user(): + with message("group_updated", group="apps"): user_group_update("apps", remove=["bob"]) group_res = user_group_list()["groups"] assert group_res["apps"]["members"] == [] -def test_update_group_remove_user_not_already_in(mocker): - with message(mocker, "group_user_not_in_group", user="jack", group="apps"): +def test_update_group_remove_user_not_already_in(): + with message("group_user_not_in_group", user="jack", group="apps"): user_group_update("apps", remove=["jack"]) group_res = user_group_list()["groups"] From 1af88b0c55d5d3cadf8e758496254a3143abdf41 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 15 May 2023 19:04:32 +0200 Subject: [PATCH 21/82] ci: force tox install during lint tasks --- .gitlab/ci/lint.gitlab-ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.gitlab/ci/lint.gitlab-ci.yml b/.gitlab/ci/lint.gitlab-ci.yml index 7a8fbf1fb..6e33af408 100644 --- a/.gitlab/ci/lint.gitlab-ci.yml +++ b/.gitlab/ci/lint.gitlab-ci.yml @@ -1,3 +1,6 @@ +.install_tox: &install_tox + - pip3 install -U tox --break-system-packages + ######################################## # LINTER ######################################## @@ -8,6 +11,8 @@ lint39: image: "before-install" needs: [] allow_failure: true + before_script: + - *install_tox script: - tox -e py39-lint @@ -15,6 +20,8 @@ invalidcode39: stage: lint image: "before-install" needs: [] + before_script: + - *install_tox script: - tox -e py39-invalidcode @@ -22,6 +29,8 @@ mypy: stage: lint image: "before-install" needs: [] + before_script: + - *install_tox script: - tox -e py39-mypy @@ -29,6 +38,8 @@ black: stage: lint image: "before-install" needs: [] + before_script: + - *install_tox before_script: - apt-get update -y && apt-get install git hub -y - git config --global user.email "yunohost@yunohost.org" From bed9ecc09e057af2fca2788672aa426a2ca1dd24 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 15 May 2023 22:02:40 +0200 Subject: [PATCH 22/82] py39->py311 in tox --- .gitlab/ci/lint.gitlab-ci.yml | 12 ++++++------ tox.ini | 18 +++++++++--------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.gitlab/ci/lint.gitlab-ci.yml b/.gitlab/ci/lint.gitlab-ci.yml index 6e33af408..349819a90 100644 --- a/.gitlab/ci/lint.gitlab-ci.yml +++ b/.gitlab/ci/lint.gitlab-ci.yml @@ -6,7 +6,7 @@ ######################################## # later we must fix lint and format-check jobs and remove "allow_failure" -lint39: +lint311: stage: lint image: "before-install" needs: [] @@ -14,16 +14,16 @@ lint39: before_script: - *install_tox script: - - tox -e py39-lint + - tox -e py311-lint -invalidcode39: +invalidcode311: stage: lint image: "before-install" needs: [] before_script: - *install_tox script: - - tox -e py39-invalidcode + - tox -e py311-invalidcode mypy: stage: lint @@ -32,7 +32,7 @@ mypy: before_script: - *install_tox script: - - tox -e py39-mypy + - tox -e py311-mypy black: stage: lint @@ -49,7 +49,7 @@ black: script: # create a local branch that will overwrite distant one - git checkout -b "ci-format-${CI_COMMIT_REF_NAME}" --no-track - - tox -e py39-black-run + - tox -e py311-black-run - '[ $(git diff | wc -l) != 0 ] || exit 0' # stop if there is nothing to commit - git commit -am "[CI] Format code with Black" || true - git push -f origin "ci-format-${CI_COMMIT_REF_NAME}":"ci-format-${CI_COMMIT_REF_NAME}" diff --git a/tox.ini b/tox.ini index 49c78959d..58c443f25 100644 --- a/tox.ini +++ b/tox.ini @@ -1,15 +1,15 @@ [tox] -envlist = py39-{lint,invalidcode},py39-black-{run,check} +envlist = py311-{lint,invalidcode},py311-black-{run,check} [testenv] skip_install=True deps = - py39-{lint,invalidcode}: flake8 - py39-black-{run,check}: black - py39-mypy: mypy >= 0.900 + py311-{lint,invalidcode}: flake8 + py311-black-{run,check}: black + py311-mypy: mypy >= 0.900 commands = - py39-lint: flake8 src doc maintenance tests --ignore E402,E501,E203,W503,E741 --exclude src/vendor - py39-invalidcode: flake8 src bin maintenance --exclude src/tests,src/vendor --select F,E722,W605 - py39-black-check: black --check --diff bin src doc maintenance tests - py39-black-run: black bin src doc maintenance tests - py39-mypy: mypy --ignore-missing-import --install-types --non-interactive --follow-imports silent src/ --exclude (acme_tiny|migrations) + py311-lint: flake8 src doc maintenance tests --ignore E402,E501,E203,W503,E741 --exclude src/vendor + py311-invalidcode: flake8 src bin maintenance --exclude src/tests,src/vendor --select F,E722,W605 + py311-black-check: black --check --diff bin src doc maintenance tests + py311-black-run: black bin src doc maintenance tests + py311-mypy: mypy --ignore-missing-import --install-types --non-interactive --follow-imports silent src/ --exclude (acme_tiny|migrations) From 734db1994c30faac1d9d14ab40d32a275245f567 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Tue, 16 May 2023 11:30:56 +0200 Subject: [PATCH 23/82] ci: don't install any package with pip, it's supposed to be preinstalled --- .gitlab/ci/lint.gitlab-ci.yml | 11 ----------- .gitlab/ci/test.gitlab-ci.yml | 1 - 2 files changed, 12 deletions(-) diff --git a/.gitlab/ci/lint.gitlab-ci.yml b/.gitlab/ci/lint.gitlab-ci.yml index 349819a90..65b74ddca 100644 --- a/.gitlab/ci/lint.gitlab-ci.yml +++ b/.gitlab/ci/lint.gitlab-ci.yml @@ -1,6 +1,3 @@ -.install_tox: &install_tox - - pip3 install -U tox --break-system-packages - ######################################## # LINTER ######################################## @@ -11,8 +8,6 @@ lint311: image: "before-install" needs: [] allow_failure: true - before_script: - - *install_tox script: - tox -e py311-lint @@ -20,8 +15,6 @@ invalidcode311: stage: lint image: "before-install" needs: [] - before_script: - - *install_tox script: - tox -e py311-invalidcode @@ -29,8 +22,6 @@ mypy: stage: lint image: "before-install" needs: [] - before_script: - - *install_tox script: - tox -e py311-mypy @@ -38,8 +29,6 @@ black: stage: lint image: "before-install" needs: [] - before_script: - - *install_tox before_script: - apt-get update -y && apt-get install git hub -y - git config --global user.email "yunohost@yunohost.org" diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index c5f1ee13c..a49fc13b7 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -1,7 +1,6 @@ .install_debs: &install_debs - apt-get update -o Acquire::Retries=3 - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb php8.2-cli mariadb-client mariadb-server - - pip3 install -U mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 "packaging<22" --break-system-packages # for bookworm .test-stage: stage: test From 5564f7dc12fa5ac8036c683e2a3976d5f0327f14 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 16 May 2023 15:11:51 +0200 Subject: [PATCH 24/82] tests: fix remaining funky mocker.spy --- src/tests/test_app_catalog.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/tests/test_app_catalog.py b/src/tests/test_app_catalog.py index f7363dabe..40daf5873 100644 --- a/src/tests/test_app_catalog.py +++ b/src/tests/test_app_catalog.py @@ -5,6 +5,8 @@ import requests_mock import glob import shutil +from .conftest import message + from moulinette import m18n from moulinette.utils.filesystem import read_json, write_to_json, write_to_yaml @@ -258,13 +260,12 @@ def test_apps_catalog_load_with_conflicts_between_lists(mocker): assert "bar" in app_dict.keys() -def test_apps_catalog_load_with_oudated_api_version(mocker): +def test_apps_catalog_load_with_outdated_api_version(): # Initialize ... _initialize_apps_catalog_system() # Update with requests_mock.Mocker() as m: - mocker.spy(m18n, "n") m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG) _update_apps_catalog() @@ -282,10 +283,8 @@ def test_apps_catalog_load_with_oudated_api_version(mocker): with requests_mock.Mocker() as m: # Mock the server response with a dummy apps catalog m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG) - - mocker.spy(m18n, "n") - app_dict = _load_apps_catalog()["apps"] - m18n.n.assert_any_call("apps_catalog_update_success") + with message("apps_catalog_update_success"): + app_dict = _load_apps_catalog()["apps"] assert "foo" in app_dict.keys() assert "bar" in app_dict.keys() From 5b9721eb23c4086930dd16c1d2745688350731cd Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 16 May 2023 15:17:03 +0200 Subject: [PATCH 25/82] tests: fix bad regex permission test because python3.11 now accepts ++ quantifier, so change the 'bad regex' trick --- src/tests/test_permission.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/test_permission.py b/src/tests/test_permission.py index 4ab333584..8620e9611 100644 --- a/src/tests/test_permission.py +++ b/src/tests/test_permission.py @@ -806,7 +806,7 @@ def test_permission_main_url_regex(): def test_permission_main_url_bad_regex(mocker): with raiseYunohostError(mocker, "invalid_regex"): - permission_url("blog.main", url="re:/[a-z]++reboy/.*") + permission_url("blog.main", url="re:/[a-z]+++reboy/.*") @pytest.mark.other_domains(number=1) @@ -837,7 +837,7 @@ def test_permission_add_additional_regex(): def test_permission_add_additional_bad_regex(mocker): with raiseYunohostError(mocker, "invalid_regex"): - permission_url("blog.main", add_url=["re:/[a-z]++reboy/.*"]) + permission_url("blog.main", add_url=["re:/[a-z]+++reboy/.*"]) def test_permission_remove_additional_url(): From 3b754859230410f90967cb9ab454b1c8d1d363c5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 16 May 2023 15:21:27 +0200 Subject: [PATCH 26/82] tests: somehow using 'Domain' as http header aint supported anymore, gotta use Host --- src/tests/test_changeurl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/test_changeurl.py b/src/tests/test_changeurl.py index 04cb4a1a9..b8ca20355 100644 --- a/src/tests/test_changeurl.py +++ b/src/tests/test_changeurl.py @@ -39,7 +39,7 @@ def check_changeurl_app(path): assert appmap[maindomain][path]["id"] == "change_url_app" r = requests.get( - "https://127.0.0.1%s/" % path, headers={"domain": maindomain}, verify=False + "https://127.0.0.1%s/" % path, headers={"Host": maindomain}, verify=False ) assert r.status_code == 200 From 85b08e44c9ee03151cae1c35ef20902ffdb7ddd4 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Mon, 22 May 2023 15:29:21 +0200 Subject: [PATCH 27/82] ci: preinstall more package --- .gitlab/ci/doc.gitlab-ci.yml | 1 - .gitlab/ci/lint.gitlab-ci.yml | 1 - .gitlab/ci/test.gitlab-ci.yml | 2 +- .gitlab/ci/translation.gitlab-ci.yml | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitlab/ci/doc.gitlab-ci.yml b/.gitlab/ci/doc.gitlab-ci.yml index 4f6ea6ba1..183d153a4 100644 --- a/.gitlab/ci/doc.gitlab-ci.yml +++ b/.gitlab/ci/doc.gitlab-ci.yml @@ -7,7 +7,6 @@ generate-helpers-doc: image: "before-install" needs: [] before_script: - - apt-get update -y && apt-get install git hub -y - git config --global user.email "yunohost@yunohost.org" - git config --global user.name "$GITHUB_USER" script: diff --git a/.gitlab/ci/lint.gitlab-ci.yml b/.gitlab/ci/lint.gitlab-ci.yml index 65b74ddca..1eeb71eab 100644 --- a/.gitlab/ci/lint.gitlab-ci.yml +++ b/.gitlab/ci/lint.gitlab-ci.yml @@ -30,7 +30,6 @@ black: image: "before-install" needs: [] before_script: - - apt-get update -y && apt-get install git hub -y - git config --global user.email "yunohost@yunohost.org" - git config --global user.name "$GITHUB_USER" - hub clone --branch ${CI_COMMIT_REF_NAME} "https://$GITHUB_TOKEN:x-oauth-basic@github.com/YunoHost/yunohost.git" github_repo diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index a49fc13b7..2c6e1717d 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -1,6 +1,6 @@ .install_debs: &install_debs - apt-get update -o Acquire::Retries=3 - - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb php8.2-cli mariadb-client mariadb-server + - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb .test-stage: stage: test diff --git a/.gitlab/ci/translation.gitlab-ci.yml b/.gitlab/ci/translation.gitlab-ci.yml index 83db2b5a4..387860e40 100644 --- a/.gitlab/ci/translation.gitlab-ci.yml +++ b/.gitlab/ci/translation.gitlab-ci.yml @@ -16,7 +16,6 @@ autofix-translated-strings: image: "before-install" needs: [] before_script: - - apt-get update -y && apt-get install git hub -y - git config --global user.email "yunohost@yunohost.org" - git config --global user.name "$GITHUB_USER" - hub clone --branch ${CI_COMMIT_REF_NAME} "https://$GITHUB_TOKEN:x-oauth-basic@github.com/YunoHost/yunohost.git" github_repo From 23eaf609da112fda99e76d544d25a7634b685188 Mon Sep 17 00:00:00 2001 From: ElderTek Date: Thu, 25 May 2023 00:00:07 +0400 Subject: [PATCH 28/82] remove deprecated --- share/actionsmap.yml | 11 ----------- src/domain.py | 4 ---- 2 files changed, 15 deletions(-) diff --git a/share/actionsmap.yml b/share/actionsmap.yml index 58787790c..107853c33 100644 --- a/share/actionsmap.yml +++ b/share/actionsmap.yml @@ -537,17 +537,6 @@ domain: full: --force help: Do not ask confirmation to remove apps action: store_true - - - ### domain_dns_conf() - dns-conf: - deprecated: true - action_help: Generate sample DNS configuration for a domain - arguments: - domain: - help: Target domain - extra: - pattern: *pattern_domain ### domain_maindomain() main-domain: diff --git a/src/domain.py b/src/domain.py index 4f96d08c4..a2d570b4b 100644 --- a/src/domain.py +++ b/src/domain.py @@ -718,10 +718,6 @@ def domain_cert_renew(domain_list, force=False, no_checks=False, email=False): return certificate_renew(domain_list, force, no_checks, email) -def domain_dns_conf(domain): - return domain_dns_suggest(domain) - - def domain_dns_suggest(domain): from yunohost.dns import domain_dns_suggest From 78cd79ec480c5e4643b792ed9fcb8dd36fb882cd Mon Sep 17 00:00:00 2001 From: Kayou Date: Mon, 5 Jun 2023 10:11:50 +0200 Subject: [PATCH 29/82] Update debian/changelog --- debian/changelog | 1 - 1 file changed, 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 69418598c..bc06c7e42 100644 --- a/debian/changelog +++ b/debian/changelog @@ -15,7 +15,6 @@ yunohost (11.1.20) stable; urgency=low Thanks to all contributors <3 ! (axolotle, Éric Gaspar, Ilya, Jose Riha, Neko Nekowazarashi, Yann Autissier) -- Alexandre Aubin Sat, 20 May 2023 18:57:26 +0200 ->>>>>>> origin/dev yunohost (11.1.19) stable; urgency=low From 8728b2030cea9c83f3c65e77406cc2524ca4dfd8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 13 Jun 2023 14:55:28 +0200 Subject: [PATCH 30/82] Remove migrations/0027_migrate_to_bookworm for now because it's triggering errors on the CI, at least half of it should be reworked, and it should be in a separated PR to target dev(=bullseye) --- src/migrations/0027_migrate_to_bookworm.py | 546 --------------------- 1 file changed, 546 deletions(-) delete mode 100644 src/migrations/0027_migrate_to_bookworm.py diff --git a/src/migrations/0027_migrate_to_bookworm.py b/src/migrations/0027_migrate_to_bookworm.py deleted file mode 100644 index 85e2235af..000000000 --- a/src/migrations/0027_migrate_to_bookworm.py +++ /dev/null @@ -1,546 +0,0 @@ -import glob -import os - -from moulinette import m18n -from yunohost.utils.error import YunohostError -from moulinette.utils.log import getActionLogger -from moulinette.utils.process import check_output, call_async_output -from moulinette.utils.filesystem import read_file, rm, write_to_file - -from yunohost.tools import ( - Migration, - tools_update, - tools_upgrade, - _apt_log_line_is_relevant, -) -from yunohost.app import unstable_apps -from yunohost.regenconf import manually_modified_files, _force_clear_hashes -from yunohost.utils.system import ( - free_space_in_directory, - get_ynh_package_version, - _list_upgradable_apt_packages, -) -from yunohost.service import _get_services, _save_services - -logger = getActionLogger("yunohost.migration") - -N_CURRENT_DEBIAN = 10 -N_CURRENT_YUNOHOST = 4 - -VENV_REQUIREMENTS_SUFFIX = ".requirements_backup_for_bookworm_upgrade.txt" - - -def _get_all_venvs(dir, level=0, maxlevel=3): - """ - Returns the list of all python virtual env directories recursively - - Arguments: - dir - the directory to scan in - maxlevel - the depth of the recursion - level - do not edit this, used as an iterator - """ - if not os.path.exists(dir): - return [] - - result = [] - # Using os functions instead of glob, because glob doesn't support hidden folders, and we need recursion with a fixed depth - for file in os.listdir(dir): - path = os.path.join(dir, file) - if os.path.isdir(path): - activatepath = os.path.join(path, "bin", "activate") - if os.path.isfile(activatepath): - content = read_file(activatepath) - if ("VIRTUAL_ENV" in content) and ("PYTHONHOME" in content): - result.append(path) - continue - if level < maxlevel: - result += _get_all_venvs(path, level=level + 1) - return result - - -def _backup_pip_freeze_for_python_app_venvs(): - """ - Generate a requirements file for all python virtual env located inside /opt/ and /var/www/ - """ - - venvs = _get_all_venvs("/opt/") + _get_all_venvs("/var/www/") - for venv in venvs: - # Generate a requirements file from venv - os.system( - f"{venv}/bin/pip freeze > {venv}{VENV_REQUIREMENTS_SUFFIX} 2>/dev/null" - ) - - -class MyMigration(Migration): - "Upgrade the system to Debian Bookworm and Yunohost 11.x" - - mode = "manual" - - def run(self): - self.check_assertions() - - logger.info(m18n.n("migration_0021_start")) - - # - # Add new apt .deb signing key - # - - new_apt_key = "https://forge.yunohost.org/yunohost_bookworm.asc" - check_output(f"wget -O- {new_apt_key} -q | apt-key add -qq -") - - # - # Patch sources.list - # - logger.info(m18n.n("migration_0021_patching_sources_list")) - self.patch_apt_sources_list() - - # Stupid OVH has some repo configured which dont work with bookworm and break apt ... - os.system("sudo rm -f /etc/apt/sources.list.d/ovh-*.list") - - # Force add sury if it's not there yet - # This is to solve some weird issue with php-common breaking php7.3-common, - # hence breaking many php7.3-deps - # hence triggering some dependency conflict (or foobar-ynh-deps uninstall) - # Adding it there shouldnt be a big deal - Yunohost 11.x does add it - # through its regen conf anyway. - if not os.path.exists("/etc/apt/sources.list.d/extra_php_version.list"): - open("/etc/apt/sources.list.d/extra_php_version.list", "w").write( - "deb https://packages.sury.org/php/ bookworm main" - ) - - # Add Sury key even if extra_php_version.list was already there, - # because some old system may be using an outdated key not valid for Bookworm - # and that'll block the migration - os.system( - 'wget --timeout 900 --quiet "https://packages.sury.org/php/apt.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/extra_php_version.gpg"' - ) - - # Remove legacy, duplicated sury entry if it exists - if os.path.exists("/etc/apt/sources.list.d/sury.list"): - os.system("rm -rf /etc/apt/sources.list.d/sury.list") - - # - # Get requirements of the different venvs from python apps - # - - _backup_pip_freeze_for_python_app_venvs() - - # - # Run apt update - # - - tools_update(target="system") - - # Tell libc6 it's okay to restart system stuff during the upgrade - os.system( - "echo 'libc6 libraries/restart-without-asking boolean true' | debconf-set-selections" - ) - - # Do not restart nginx during the upgrade of nginx-common and nginx-extras ... - # c.f. https://manpages.debian.org/bookworm/init-system-helpers/deb-systemd-invoke.1p.en.html - # and zcat /usr/share/doc/init-system-helpers/README.policy-rc.d.gz - # and the code inside /usr/bin/deb-systemd-invoke to see how it calls /usr/sbin/policy-rc.d ... - # and also invoke-rc.d ... - write_to_file( - "/usr/sbin/policy-rc.d", - '#!/bin/bash\n[[ "$1" =~ "nginx" ]] && [[ "$2" == "restart" ]] && exit 101 || exit 0', - ) - os.system("chmod +x /usr/sbin/policy-rc.d") - - # Don't send an email to root about the postgresql migration. It should be handled automatically after. - os.system( - "echo 'postgresql-common postgresql-common/obsolete-major seen true' | debconf-set-selections" - ) - - # - # Patch yunohost conflicts - # - logger.info(m18n.n("migration_0021_patch_yunohost_conflicts")) - - self.patch_yunohost_conflicts() - - # - # Specific tweaking to get rid of custom my.cnf and use debian's default one - # (my.cnf is actually a symlink to mariadb.cnf) - # - - _force_clear_hashes(["/etc/mysql/my.cnf"]) - rm("/etc/mysql/mariadb.cnf", force=True) - rm("/etc/mysql/my.cnf", force=True) - ret = self.apt_install( - "mariadb-common --reinstall -o Dpkg::Options::='--force-confmiss'" - ) - if ret != 0: - raise YunohostError("Failed to reinstall mariadb-common ?", raw_msg=True) - - # - # /usr/share/yunohost/yunohost-config/ssl/yunoCA -> /usr/share/yunohost/ssl - # - if os.path.exists("/usr/share/yunohost/yunohost-config/ssl/yunoCA"): - os.system( - "mv /usr/share/yunohost/yunohost-config/ssl/yunoCA /usr/share/yunohost/ssl" - ) - rm("/usr/share/yunohost/yunohost-config", recursive=True, force=True) - - # - # /home/yunohost.conf -> /var/cache/yunohost/regenconf - # - if os.path.exists("/home/yunohost.conf"): - os.system("mv /home/yunohost.conf /var/cache/yunohost/regenconf") - rm("/home/yunohost.conf", recursive=True, force=True) - - # Remove legacy postgresql service record added by helpers, - # will now be dynamically handled by the core in bookworm - services = _get_services() - if "postgresql" in services: - del services["postgresql"] - _save_services(services) - - # - # Critical fix for RPI otherwise network is down after rebooting - # https://forum.yunohost.org/t/20652 - # - if os.system("systemctl | grep -q dhcpcd") == 0: - logger.info("Applying fix for DHCPCD ...") - os.system("mkdir -p /etc/systemd/system/dhcpcd.service.d") - write_to_file( - "/etc/systemd/system/dhcpcd.service.d/wait.conf", - "[Service]\nExecStart=\nExecStart=/usr/sbin/dhcpcd -w", - ) - - # - # Main upgrade - # - logger.info(m18n.n("migration_0021_main_upgrade")) - - apps_packages = self.get_apps_equivs_packages() - self.hold(apps_packages) - tools_upgrade(target="system", allow_yunohost_upgrade=False) - - if self.debian_major_version() == N_CURRENT_DEBIAN: - raise YunohostError("migration_0021_still_on_bullseye_after_main_upgrade") - - # Force explicit install of php8.2fpm and other old 'default' dependencies - # that are now only in Recommends - # - # Also, we need to install php8.2 equivalents of other php7.4 dependencies. - # For example, Nextcloud may depend on php7.3-zip, and after the php pool migration - # to autoupgrade Nextcloud to 8.2, it will need the php8.2-zip to work. - # The following list is based on an ad-hoc analysis of php deps found in the - # app ecosystem, with a known equivalent on php8.2. - # - # This is kinda a dirty hack as it doesnt properly update the *-ynh-deps virtual packages - # with the proper list of dependencies, and the dependencies install this way - # will get flagged as 'manually installed'. - # - # We'll probably want to do something during the Bookworm->Bookworm migration to re-flag - # these as 'auto' so they get autoremoved if not needed anymore. - # Also hopefully by then we'll have manifestv2 (maybe) and will be able to use - # the apt resource mecanism to regenerate the *-ynh-deps virtual packages ;) - - php74packages_suffixes = [ - "apcu", - "bcmath", - "bz2", - "dom", - "gmp", - "igbinary", - "imagick", - "imap", - "mbstring", - "memcached", - "mysqli", - "mysqlnd", - "pgsql", - "redis", - "simplexml", - "soap", - "sqlite3", - "ssh2", - "tidy", - "xml", - "xmlrpc", - "xsl", - "zip", - ] - - cmd = ( - "apt show '*-ynh-deps' 2>/dev/null" - " | grep Depends" - f" | grep -o -E \"php7.4-({'|'.join(php74packages_suffixes)})\"" - " | sort | uniq" - " | sed 's/php7.4/php8.2/g'" - " || true" - ) - - basephp82packages_to_install = [ - "php8.2-fpm", - "php8.2-common", - "php8.2-ldap", - "php8.2-intl", - "php8.2-mysql", - "php8.2-gd", - "php8.2-curl", - "php-php-gettext", - ] - - php74packages_to_install = basephp82packages_to_install + [ - f.strip() for f in check_output(cmd).split("\n") if f.strip() - ] - - ret = self.apt_install( - f"{' '.join(php74packages_to_install)} " - "$(dpkg --list | grep ynh-deps | awk '{print $2}') " - "-o Dpkg::Options::='--force-confmiss'" - ) - if ret != 0: - raise YunohostError( - "Failed to force the install of php dependencies ?", raw_msg=True - ) - - # Clean the mess - logger.info(m18n.n("migration_0021_cleaning_up")) - os.system( - "LC_ALL=C DEBIAN_FRONTEND=noninteractive APT_LISTCHANGES_FRONTEND=none apt autoremove --assume-yes" - ) - os.system("apt clean --assume-yes") - - # - # Stupid hack for stupid dnsmasq not picking up its new init.d script then breaking everything ... - # https://forum.yunohost.org/t/20676 - # - if os.path.exists("/etc/init.d/dnsmasq.dpkg-dist"): - logger.info("Copying new version for /etc/init.d/dnsmasq ...") - os.system("cp /etc/init.d/dnsmasq.dpkg-dist /etc/init.d/dnsmasq") - - # - # Yunohost upgrade - # - logger.info(m18n.n("migration_0021_yunohost_upgrade")) - - self.unhold(apps_packages) - - cmd = "LC_ALL=C" - cmd += " DEBIAN_FRONTEND=noninteractive" - cmd += " APT_LISTCHANGES_FRONTEND=none" - cmd += " apt dist-upgrade " - cmd += " --quiet -o=Dpkg::Use-Pty=0 --fix-broken --dry-run" - cmd += " | grep -q 'ynh-deps'" - - logger.info("Simulating upgrade...") - if os.system(cmd) == 0: - raise YunohostError( - "The upgrade cannot be completed, because some app dependencies would need to be removed?", - raw_msg=True, - ) - - postupgradecmds = f"apt-mark auto {' '.join(basephp74packages_to_install)}\n" - postupgradecmds += "rm -f /usr/sbin/policy-rc.d\n" - postupgradecmds += "echo 'Restarting nginx...' >&2\n" - postupgradecmds += "systemctl restart nginx\n" - - tools_upgrade(target="system", postupgradecmds=postupgradecmds) - - def debian_major_version(self): - # The python module "platform" and lsb_release are not reliable because - # on some setup, they may still return Release=9 even after upgrading to - # bullseye ... (Apparently this is related to OVH overriding some stuff - # with /etc/lsb-release for instance -_-) - # Instead, we rely on /etc/os-release which should be the raw info from - # the distribution... - return int( - check_output( - "grep VERSION_ID /etc/os-release | head -n 1 | tr '\"' ' ' | cut -d ' ' -f2" - ) - ) - - def yunohost_major_version(self): - return int(get_ynh_package_version("yunohost")["version"].split(".")[0]) - - def check_assertions(self): - # Be on bullseye (10.x) and yunohost 4.x - # NB : we do both check to cover situations where the upgrade crashed - # in the middle and debian version could be > 9.x but yunohost package - # would still be in 3.x... - if ( - not self.debian_major_version() == N_CURRENT_DEBIAN - and not self.yunohost_major_version() == N_CURRENT_YUNOHOST - ): - try: - # Here we try to find the previous migration log, which should be somewhat recent and be at least 10k (we keep the biggest one) - maybe_previous_migration_log_id = check_output( - "cd /var/log/yunohost/categories/operation && find -name '*migrate*.log' -size +10k -mtime -100 -exec ls -s {} \\; | sort -n | tr './' ' ' | awk '{print $2}' | tail -n 1" - ) - if maybe_previous_migration_log_id: - logger.info( - f"NB: the previous migration log id seems to be {maybe_previous_migration_log_id}. You can share it with the support team with : sudo yunohost log share {maybe_previous_migration_log_id}" - ) - except Exception: - # Yeah it's not that important ... it's to simplify support ... - pass - - raise YunohostError("migration_0021_not_bullseye2") - - # Have > 1 Go free space on /var/ ? - if free_space_in_directory("/var/") / (1024**3) < 1.0: - raise YunohostError("migration_0021_not_enough_free_space") - - # Have > 70 MB free space on /var/ ? - # FIXME: Create a way to ignore this check, on some system 70M is enough... - if free_space_in_directory("/boot/") / (1024**2) < 70.0: - raise YunohostError( - "/boot/ has less than 70MB available. This will probably trigger a crash during the upgrade because a new kernel needs to be installed. Please look for advice on the forum on how to remove old, unused kernels to free up some space in /boot/.", - raw_msg=True, - ) - - # Check system is up to date - # (but we don't if 'bookworm' is already in the sources.list ... - # which means maybe a previous upgrade crashed and we're re-running it) - if os.path.exists("/etc/apt/sources.list") and " bookworm " not in read_file( - "/etc/apt/sources.list" - ): - tools_update(target="system") - upgradable_system_packages = list(_list_upgradable_apt_packages()) - upgradable_system_packages = [ - package["name"] for package in upgradable_system_packages - ] - upgradable_system_packages = set(upgradable_system_packages) - # Lime2 have hold packages to avoid ethernet instability - # See https://github.com/YunoHost/arm-images/commit/b4ef8c99554fd1a122a306db7abacc4e2f2942df - lime2_hold_packages = set( - [ - "armbian-firmware", - "armbian-bsp-cli-lime2", - "linux-dtb-current-sunxi", - "linux-image-current-sunxi", - "linux-u-boot-lime2-current", - "linux-image-next-sunxi", - ] - ) - if upgradable_system_packages - lime2_hold_packages: - raise YunohostError("migration_0021_system_not_fully_up_to_date") - - @property - def disclaimer(self): - # Avoid having a super long disclaimer + uncessary check if we ain't - # on bullseye / yunohost 4.x anymore - # NB : we do both check to cover situations where the upgrade crashed - # in the middle and debian version could be >= 10.x but yunohost package - # would still be in 4.x... - if ( - not self.debian_major_version() == N_CURRENT_DEBIAN - and not self.yunohost_major_version() == N_CURRENT_YUNOHOST - ): - return None - - # Get list of problematic apps ? I.e. not official or community+working - problematic_apps = unstable_apps() - problematic_apps = "".join(["\n - " + app for app in problematic_apps]) - - # Manually modified files ? (c.f. yunohost service regen-conf) - modified_files = manually_modified_files() - modified_files = "".join(["\n - " + f for f in modified_files]) - - message = m18n.n("migration_0021_general_warning") - - message = ( - "N.B.: This migration has been tested by the community over the last few months but has only been declared stable recently. If your server hosts critical services and if you are not too confident with debugging possible issues, we recommend you to wait a little bit more while we gather more feedback and polish things up. If on the other hand you are relatively confident with debugging small issues that may arise, you are encouraged to run this migration ;)! You can read about remaining known issues and feedback from the community here: https://forum.yunohost.org/t/20590\n\n" - + message - ) - - if problematic_apps: - message += "\n\n" + m18n.n( - "migration_0021_problematic_apps_warning", - problematic_apps=problematic_apps, - ) - - if modified_files: - message += "\n\n" + m18n.n( - "migration_0021_modified_files", manually_modified_files=modified_files - ) - - return message - - def patch_apt_sources_list(self): - sources_list = glob.glob("/etc/apt/sources.list.d/*.list") - if os.path.exists("/etc/apt/sources.list"): - sources_list.append("/etc/apt/sources.list") - - # This : - # - replace single 'bullseye' occurence by 'bulleye' - # - comments lines containing "backports" - for f in sources_list: - command = ( - f"sed -i {f} " - "-e 's@ bullseye @ bookworm @g' " - "-e '/backports/ s@^#*@#@' " - "-e 's@ bullseye-@ bookworm-@g' " - ) - os.system(command) - - def get_apps_equivs_packages(self): - command = ( - "dpkg --get-selections" - " | grep -v deinstall" - " | awk '{print $1}'" - " | { grep 'ynh-deps$' || true; }" - ) - - output = check_output(command) - - return output.split("\n") if output else [] - - def hold(self, packages): - for package in packages: - os.system(f"apt-mark hold {package}") - - def unhold(self, packages): - for package in packages: - os.system(f"apt-mark unhold {package}") - - def apt_install(self, cmd): - def is_relevant(line): - return "Reading database ..." not in line.rstrip() - - callbacks = ( - lambda l: logger.info("+ " + l.rstrip() + "\r") - if _apt_log_line_is_relevant(l) - else logger.debug(l.rstrip() + "\r"), - lambda l: logger.warning(l.rstrip()) - if _apt_log_line_is_relevant(l) - else logger.debug(l.rstrip()), - ) - - cmd = ( - "LC_ALL=C DEBIAN_FRONTEND=noninteractive APT_LISTCHANGES_FRONTEND=none apt install --quiet -o=Dpkg::Use-Pty=0 --fix-broken --assume-yes " - + cmd - ) - - logger.debug("Running: %s" % cmd) - - return call_async_output(cmd, callbacks, shell=True) - - def patch_yunohost_conflicts(self): - # - # This is a super dirty hack to remove the conflicts from yunohost's debian/control file - # Those conflicts are there to prevent mistakenly upgrading critical packages - # such as dovecot, postfix, nginx, openssl, etc... usually related to mistakenly - # using backports etc. - # - # The hack consists in savagely removing the conflicts directly in /var/lib/dpkg/status - # - - # We only patch the conflict if we're on yunohost 4.x - if self.yunohost_major_version() != N_CURRENT_YUNOHOST: - return - - conflicts = check_output("dpkg-query -s yunohost | grep '^Conflicts:'").strip() - if conflicts: - # We want to keep conflicting with apache/bind9 tho - new_conflicts = "Conflicts: apache2, bind9" - - command = ( - f"sed -i /var/lib/dpkg/status -e 's@{conflicts}@{new_conflicts}@g'" - ) - logger.debug(f"Running: {command}") - os.system(command) From c4c353843c6fc147d1cbaa92e21edc3b09dda702 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 13 Jun 2023 14:56:21 +0200 Subject: [PATCH 31/82] Unused vars, unhappy linter gods --- src/firewall.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/firewall.py b/src/firewall.py index d6e4b5317..ccd7e6d88 100644 --- a/src/firewall.py +++ b/src/firewall.py @@ -404,7 +404,7 @@ def firewall_upnp(action="status", no_refresh=False): logger.debug("discovering UPnP devices...") try: nb_dev = upnpc.discover() - except Exception as e: + except Exception: logger.warning("Failed to find any UPnP device on the network") nb_dev = -1 enabled = False From 194eb9c6c7d6684a2802311bd6afcece7336d7d6 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 13 Jun 2023 20:14:46 +0200 Subject: [PATCH 32/82] conf: Update ciphers for nginx, postfix, dovecot --- conf/dovecot/dovecot.conf | 7 +++---- conf/nginx/security.conf.inc | 10 +++++----- conf/postfix/main.cf | 10 +++++----- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/conf/dovecot/dovecot.conf b/conf/dovecot/dovecot.conf index e614c3796..14d407563 100644 --- a/conf/dovecot/dovecot.conf +++ b/conf/dovecot/dovecot.conf @@ -13,9 +13,8 @@ protocols = imap sieve {% if pop3_enabled == "True" %}pop3{% endif %} mail_plugins = $mail_plugins quota notify push_notification ############################################################################### - -# generated 2020-08-18, Mozilla Guideline v5.6, Dovecot 2.3.4, OpenSSL 1.1.1d, intermediate configuration -# https://ssl-config.mozilla.org/#server=dovecot&version=2.3.4&config=intermediate&openssl=1.1.1d&guideline=5.6 +# generated 2023-06-13, Mozilla Guideline v5.7, Dovecot 2.3.19, OpenSSL 3.0.9, intermediate configuration +# https://ssl-config.mozilla.org/#server=dovecot&version=2.3.19&config=intermediate&openssl=3.0.9&guideline=5.7 ssl = required @@ -32,7 +31,7 @@ ssl_dh = = 1024 bits smtpd_tls_dh1024_param_file = /usr/share/yunohost/ffdhe2048.pem -tls_medium_cipherlist = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 +tls_medium_cipherlist = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305 {% else %} -# generated 2020-08-18, Mozilla Guideline v5.6, Postfix 3.4.14, OpenSSL 1.1.1d, modern configuration -# https://ssl-config.mozilla.org/#server=postfix&version=3.4.14&config=modern&openssl=1.1.1d&guideline=5.6 +# generated 2023-06-13, Mozilla Guideline v5.7, Postfix 3.7.5, OpenSSL 3.0.9, modern configuration +# https://ssl-config.mozilla.org/#server=postfix&version=3.7.5&config=modern&openssl=3.0.9&guideline=5.7 smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1, !TLSv1.2 smtpd_tls_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1, !TLSv1.2 From 81b96ad6d45c16eeae0bbcd4774bec139b63ca08 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 13 Jun 2023 21:30:20 +0200 Subject: [PATCH 33/82] tests: tmp tweaks to adapt for removed deprecated features --- src/tests/test_backuprestore.py | 5 ++++- src/tests/test_permission.py | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/tests/test_backuprestore.py b/src/tests/test_backuprestore.py index 873deec7d..eb59d4fea 100644 --- a/src/tests/test_backuprestore.py +++ b/src/tests/test_backuprestore.py @@ -439,7 +439,8 @@ def test_backup_using_copy_method(): # App restore # # - +# FIXME : switch to a backup from 11.x +@pytest.mark.skip @pytest.mark.with_wordpress_archive_from_4p2 @pytest.mark.with_custom_domain("yolo.test") def test_restore_app_wordpress_from_Ynh4p2(): @@ -504,6 +505,8 @@ def test_restore_app_not_in_backup(mocker): assert not _is_installed("yoloswag") +# FIXME : switch to a backup from 11.x +@pytest.mark.skip @pytest.mark.with_wordpress_archive_from_4p2 @pytest.mark.with_custom_domain("yolo.test") def test_restore_app_already_installed(mocker): diff --git a/src/tests/test_permission.py b/src/tests/test_permission.py index 8620e9611..dc9121745 100644 --- a/src/tests/test_permission.py +++ b/src/tests/test_permission.py @@ -1131,7 +1131,7 @@ def test_permission_app_propagation_on_ssowat(): def test_permission_legacy_app_propagation_on_ssowat(): app_install( os.path.join(get_test_apps_dir(), "legacy_app_ynh"), - args="domain=%s&domain_2=%s&path=%s&is_public=1" + args="domain=%s&domain_2=%s&path=%s&is_public=0" % (maindomain, other_domains[0], "/legacy"), force=True, ) @@ -1139,12 +1139,12 @@ def test_permission_legacy_app_propagation_on_ssowat(): # App is configured as public by default using the legacy unprotected_uri mechanics # It should automatically be migrated during the install res = user_permission_list(full=True)["permissions"] - assert "visitors" in res["legacy_app.main"]["allowed"] + assert "visitors" not in res["legacy_app.main"]["allowed"] assert "all_users" in res["legacy_app.main"]["allowed"] app_webroot = "https://%s/legacy" % maindomain - assert can_access_webpage(app_webroot, logged_as=None) + assert not can_access_webpage(app_webroot, logged_as=None) assert can_access_webpage(app_webroot, logged_as="alice") # Try to update the permission and check that permissions are still consistent From f6ab380730c7b687dcf8b82985c229d7b0c880ba Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 14 Jun 2023 01:34:42 +0200 Subject: [PATCH 34/82] helpers/php: Default PHP version in bookworm is now 8.2 --- helpers/php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/php b/helpers/php index 1b28b32f7..7e8d35d6e 100644 --- a/helpers/php +++ b/helpers/php @@ -1,6 +1,6 @@ #!/bin/bash -readonly YNH_DEFAULT_PHP_VERSION=7.4 +readonly YNH_DEFAULT_PHP_VERSION=8.2 # Declare the actual PHP version to use. # A packager willing to use another version of PHP can override the variable into its _common.sh. YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} From 8ac48ee09e7d3d77e3b636e6700c5b9a26dedc93 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 14 Jun 2023 08:04:16 +0200 Subject: [PATCH 35/82] Drop deprecated firstname/lastname in user_create/update + also drop old deprecated cert- commands --- share/actionsmap.yml | 76 +----------------------------------- src/domain.py | 3 +- src/tests/test_user-group.py | 8 +--- src/user.py | 45 ++++++--------------- 4 files changed, 17 insertions(+), 115 deletions(-) diff --git a/share/actionsmap.yml b/share/actionsmap.yml index e02f5c1d0..444533a1d 100644 --- a/share/actionsmap.yml +++ b/share/actionsmap.yml @@ -70,26 +70,10 @@ user: help: The full name of the user. For example 'Camille Dupont' extra: ask: ask_fullname - required: False + required: True pattern: &pattern_fullname - !!str ^([^\W_]{1,30}[ ,.'-]{0,3})+$ - "pattern_fullname" - -f: - full: --firstname - help: Deprecated. Use --fullname instead. - extra: - required: False - pattern: &pattern_firstname - - !!str ^([^\W\d_]{1,30}[ ,.'-]{0,3})+$ - - "pattern_firstname" - -l: - full: --lastname - help: Deprecated. Use --fullname instead. - extra: - required: False - pattern: &pattern_lastname - - !!str ^([^\W\d_]{1,30}[ ,.'-]{0,3})+$ - - "pattern_lastname" -p: full: --password help: User password @@ -147,16 +131,6 @@ user: help: The full name of the user. For example 'Camille Dupont' extra: pattern: *pattern_fullname - -f: - full: --firstname - help: Deprecated. Use --fullname instead. - extra: - pattern: *pattern_firstname - -l: - full: --lastname - help: Deprecated. Use --fullname instead. - extra: - pattern: *pattern_lastname -m: full: --mail extra: @@ -551,54 +525,6 @@ domain: extra: pattern: *pattern_domain - ### certificate_status() - cert-status: - deprecated: true - action_help: List status of current certificates (all by default). - arguments: - domain_list: - help: Domains to check - nargs: "*" - --full: - help: Show more details - action: store_true - - ### certificate_install() - cert-install: - deprecated: true - action_help: Install Let's Encrypt certificates for given domains (all by default). - arguments: - domain_list: - help: Domains for which to install the certificates - nargs: "*" - --force: - help: Install even if current certificate is not self-signed - action: store_true - --no-checks: - help: Does not perform any check that your domain seems correctly configured (DNS, reachability) before attempting to install. (Not recommended) - action: store_true - --self-signed: - help: Install self-signed certificate instead of Let's Encrypt - action: store_true - - ### certificate_renew() - cert-renew: - deprecated: true - action_help: Renew the Let's Encrypt certificates for given domains (all by default). - arguments: - domain_list: - help: Domains for which to renew the certificates - nargs: "*" - --force: - help: Ignore the validity threshold (30 days) - action: store_true - --email: - help: Send an email to root with logs if some renewing fails - action: store_true - --no-checks: - help: Does not perform any check that your domain seems correctly configured (DNS, reachability) before attempting to renew. (Not recommended) - action: store_true - ### domain_url_available() url-available: hide_in_help: True diff --git a/src/domain.py b/src/domain.py index a2d570b4b..834730890 100644 --- a/src/domain.py +++ b/src/domain.py @@ -154,11 +154,12 @@ def domain_info(domain): from yunohost.app import app_info from yunohost.dns import _get_registar_settings + from yunohost.certificate import certificate_status _assert_domain_exists(domain) registrar, _ = _get_registar_settings(domain) - certificate = domain_cert_status([domain], full=True)["certificates"][domain] + certificate = certificate_status([domain], full=True)["certificates"][domain] apps = [] for app in _installed_apps(): diff --git a/src/tests/test_user-group.py b/src/tests/test_user-group.py index 57f9ffa3f..f347fc9bc 100644 --- a/src/tests/test_user-group.py +++ b/src/tests/test_user-group.py @@ -263,12 +263,6 @@ def test_del_group_that_does_not_exist(mocker): def test_update_user(): - with message("user_updated"): - user_update("alice", firstname="NewName", lastname="NewLast") - - info = user_info("alice") - assert info["fullname"] == "NewName NewLast" - with message("user_updated"): user_update("alice", fullname="New2Name New2Last") @@ -315,7 +309,7 @@ def test_update_group_remove_user_not_already_in(): def test_update_user_that_doesnt_exist(mocker): with raiseYunohostError(mocker, "user_unknown"): - user_update("doesnt_exist", firstname="NewName", lastname="NewLast") + user_update("doesnt_exist", fullname="Foo Bar") def test_update_group_that_doesnt_exist(mocker): diff --git a/src/user.py b/src/user.py index 00876854e..9627a37cb 100644 --- a/src/user.py +++ b/src/user.py @@ -141,33 +141,20 @@ def user_create( domain, password, fullname=None, - firstname=None, - lastname=None, mailbox_quota="0", admin=False, from_import=False, loginShell=None, ): - if firstname or lastname: - logger.warning( - "Options --firstname / --lastname of 'yunohost user create' are deprecated. We recommend using --fullname instead." - ) - if not fullname or not fullname.strip(): - if not firstname.strip(): - raise YunohostValidationError( - "You should specify the fullname of the user using option -F" - ) - lastname = ( - lastname or " " - ) # Stupid hack because LDAP requires the sn/lastname attr, but it accepts a single whitespace... - fullname = f"{firstname} {lastname}".strip() - else: - fullname = fullname.strip() - firstname = fullname.split()[0] - lastname = ( - " ".join(fullname.split()[1:]) or " " - ) # Stupid hack because LDAP requires the sn/lastname attr, but it accepts a single whitespace... + raise YunohostValidationError( + "You should specify the fullname of the user using option -F" + ) + fullname = fullname.strip() + firstname = fullname.split()[0] + lastname = ( + " ".join(fullname.split()[1:]) or " " + ) # Stupid hack because LDAP requires the sn/lastname attr, but it accepts a single whitespace... from yunohost.domain import domain_list, _get_maindomain, _assert_domain_exists from yunohost.hook import hook_callback @@ -364,8 +351,6 @@ def user_delete(operation_logger, username, purge=False, from_import=False): def user_update( operation_logger, username, - firstname=None, - lastname=None, mail=None, change_password=None, add_mailforward=None, @@ -377,17 +362,15 @@ def user_update( fullname=None, loginShell=None, ): - if firstname or lastname: - logger.warning( - "Options --firstname / --lastname of 'yunohost user create' are deprecated. We recommend using --fullname instead." - ) - if fullname and fullname.strip(): fullname = fullname.strip() firstname = fullname.split()[0] lastname = ( " ".join(fullname.split()[1:]) or " " ) # Stupid hack because LDAP requires the sn/lastname attr, but it accepts a single whitespace... + else: + firstname = None + lastname = None from yunohost.domain import domain_list from yunohost.app import app_ssowatconf @@ -884,8 +867,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): user_update( new_infos["username"], - firstname=new_infos["firstname"], - lastname=new_infos["lastname"], + fullname=(new_infos["firstname"] + " " + new_infos["lastname"]).strip(), change_password=new_infos["password"], mailbox_quota=new_infos["mailbox-quota"], mail=new_infos["mail"], @@ -930,8 +912,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): user["password"], user["mailbox-quota"], from_import=True, - firstname=user["firstname"], - lastname=user["lastname"], + fullname=(user["firstname"] + " " + user["lastname"]).strip(), ) update(user) result["created"] += 1 From a673b3ed420723054c0aef6761cdf8c03fa77b6e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 14 Jun 2023 10:28:58 +0200 Subject: [PATCH 36/82] Postgresql is now version 15 --- helpers/postgresql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/postgresql b/helpers/postgresql index 796a36214..8ff63df43 100644 --- a/helpers/postgresql +++ b/helpers/postgresql @@ -1,7 +1,7 @@ #!/bin/bash PSQL_ROOT_PWD_FILE=/etc/yunohost/psql -PSQL_VERSION=13 +PSQL_VERSION=15 # Open a connection as a user # From ced6d5c975caa90fa51ff5506dc5a82ad992f0bf Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 18 Jun 2023 15:45:44 +0200 Subject: [PATCH 37/82] apps: fix version.parse now refusing to parse legacy version numbers --- src/app.py | 65 ++++++++++++++++++++++++------------------------------ 1 file changed, 29 insertions(+), 36 deletions(-) diff --git a/src/app.py b/src/app.py index 55d351a84..70c0657df 100644 --- a/src/app.py +++ b/src/app.py @@ -241,8 +241,8 @@ def _app_upgradable(app_infos): # Determine upgradability app_in_catalog = app_infos.get("from_catalog") - installed_version = version.parse(app_infos.get("version", "0~ynh0")) - version_in_catalog = version.parse( + installed_version = _parse_app_version(app_infos.get("version", "0~ynh0")) + version_in_catalog = _parse_app_version( app_infos.get("from_catalog", {}).get("manifest", {}).get("version", "0~ynh0") ) @@ -257,25 +257,7 @@ def _app_upgradable(app_infos): ): return "bad_quality" - # If the app uses the standard version scheme, use it to determine - # upgradability - if "~ynh" in str(installed_version) and "~ynh" in str(version_in_catalog): - if installed_version < version_in_catalog: - return "yes" - else: - return "no" - - # Legacy stuff for app with old / non-standard version numbers... - - # In case there is neither update_time nor install_time, we assume the app can/has to be upgraded - if not app_infos["from_catalog"].get("lastUpdate") or not app_infos[ - "from_catalog" - ].get("git"): - return "url_required" - - settings = app_infos["settings"] - local_update_time = settings.get("update_time", settings.get("install_time", 0)) - if app_infos["from_catalog"]["lastUpdate"] > local_update_time: + if installed_version < version_in_catalog: return "yes" else: return "no" @@ -620,9 +602,11 @@ def app_upgrade( # Manage upgrade type and avoid any upgrade if there is nothing to do upgrade_type = "UNKNOWN" # Get current_version and new version - app_new_version = version.parse(manifest.get("version", "?")) - app_current_version = version.parse(app_dict.get("version", "?")) - if "~ynh" in str(app_current_version) and "~ynh" in str(app_new_version): + app_new_version_raw = manifest.get("version", "?") + app_current_version_raw = app_dict.get("version", "?") + app_new_version = _parse_app_version(app_new_version_raw) + app_current_version = _parse_app_version(app_current_version_raw) + if "~ynh" in str(app_current_version_raw) and "~ynh" in str(app_new_version_raw): if app_current_version >= app_new_version and not force: # In case of upgrade from file or custom repository # No new version available @@ -642,10 +626,10 @@ def app_upgrade( upgrade_type = "UPGRADE_FORCED" else: app_current_version_upstream, app_current_version_pkg = str( - app_current_version + app_current_version_raw ).split("~ynh") app_new_version_upstream, app_new_version_pkg = str( - app_new_version + app_new_version_raw ).split("~ynh") if app_current_version_upstream == app_new_version_upstream: upgrade_type = "UPGRADE_PACKAGE" @@ -675,7 +659,7 @@ def app_upgrade( settings = _get_app_settings(app_instance_name) notifications = _filter_and_hydrate_notifications( manifest["notifications"]["PRE_UPGRADE"], - current_version=app_current_version, + current_version=app_current_version_raw, data=settings, ) _display_notifications(notifications, force=force) @@ -732,8 +716,8 @@ def app_upgrade( env_dict_more = { "YNH_APP_UPGRADE_TYPE": upgrade_type, - "YNH_APP_MANIFEST_VERSION": str(app_new_version), - "YNH_APP_CURRENT_VERSION": str(app_current_version), + "YNH_APP_MANIFEST_VERSION": str(app_new_version_raw), + "YNH_APP_CURRENT_VERSION": str(app_current_version_raw), } if manifest["packaging_format"] < 2: @@ -916,7 +900,7 @@ def app_upgrade( settings = _get_app_settings(app_instance_name) notifications = _filter_and_hydrate_notifications( manifest["notifications"]["POST_UPGRADE"], - current_version=app_current_version, + current_version=app_current_version_raw, data=settings, ) if Moulinette.interface.type == "cli": @@ -1924,6 +1908,20 @@ def _set_app_settings(app, settings): yaml.safe_dump(settings, f, default_flow_style=False) +def _parse_app_version(v): + + if v == "?": + return (0,0) + + try: + if "~" in v: + return (version.parse(v.split("~")[0]), int(v.split("~")[1].replace("ynh", ""))) + else: + return (version.parse(v), 0) + except Exception as e: + raise YunohostError(f"Failed to parse app version '{v}' : {e}", raw_msg=True) + + def _get_manifest_of_app(path): "Get app manifest stored in json or in toml" @@ -3020,12 +3018,7 @@ def _notification_is_dismissed(name, settings): def _filter_and_hydrate_notifications(notifications, current_version=None, data={}): def is_version_more_recent_than_current_version(name, current_version): current_version = str(current_version) - # Boring code to handle the fact that "0.1 < 9999~ynh1" is False - - if "~" in name: - return version.parse(name) > version.parse(current_version) - else: - return version.parse(name) > version.parse(current_version.split("~")[0]) + return _parse_app_version(name) > _parse_app_version(current_version) return { # Should we render the markdown maybe? idk From 8a865dadddbbe82a565a55524261f21c8510fa1b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 19 Jun 2023 16:04:31 +0200 Subject: [PATCH 38/82] apps: add YNH_DEFAULT_PHP_VERSION in app's dict as a boring workaround/fix for apps using YNH_DEFAULT_PHP_VERSION in _common.sh *before* sourcing helpers ... --- src/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app.py b/src/app.py index 70c0657df..4d11b47fb 100644 --- a/src/app.py +++ b/src/app.py @@ -2792,6 +2792,7 @@ def _make_environment_for_app_script( app_id, app_instance_nb = _parse_app_instance_name(app) env_dict = { + "YNH_DEFAULT_PHP_VERSION": "8.2", "YNH_APP_ID": app_id, "YNH_APP_INSTANCE_NAME": app, "YNH_APP_INSTANCE_NUMBER": str(app_instance_nb), From cab7667dcca4b44663c7ec6d5b939a00ba3bda4e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 4 Jul 2023 19:48:55 +0200 Subject: [PATCH 39/82] misc: more boring irrelevant postgresql warnings to filter out --- src/hook.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hook.py b/src/hook.py index 4b07d1c17..6c9c84a00 100644 --- a/src/hook.py +++ b/src/hook.py @@ -359,6 +359,7 @@ def hook_exec( r"Removing obsolete dictionary files", r"Creating new PostgreSQL cluster", r"/usr/lib/postgresql/13/bin/initdb", + r"/usr/lib/postgresql/15/bin/initdb", r"The files belonging to this database system will be owned by user", r"This user must also own the server process.", r"The database cluster will be initialized with locale", @@ -366,6 +367,7 @@ def hook_exec( r"The default text search configuration will be set to", r"Data page checksums are disabled.", r"fixing permissions on existing directory /var/lib/postgresql/13/main ... ok", + r"fixing permissions on existing directory /var/lib/postgresql/15/main ... ok", r"creating subdirectories \.\.\. ok", r"selecting dynamic .* \.\.\. ", r"selecting default .* \.\.\. ", From 2f2ff6eb190cd9a825e88d91b3101f2e3fba6c96 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 11 Jul 2023 15:58:59 +0200 Subject: [PATCH 40/82] Simplify fpm add config helper (Bookworm) (#1685) * Simplify ynh_add_fpm_config helper * helpers: drop dedicated_service option in ynh_add_fpm_config --- helpers/php | 97 ++++++----------------------------------------------- 1 file changed, 11 insertions(+), 86 deletions(-) diff --git a/helpers/php b/helpers/php index 5cfe521f1..21fb20e27 100644 --- a/helpers/php +++ b/helpers/php @@ -70,12 +70,11 @@ YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} ynh_add_fpm_config() { local _globalphpversion=${phpversion-:} # Declare an array to define the options of this helper. - local legacy_args=vufd - local -A args_array=([v]=phpversion= [u]=usage= [f]=footprint= [d]=dedicated_service) + local legacy_args=vuf + local -A args_array=([v]=phpversion= [u]=usage= [f]=footprint=) local phpversion local usage local footprint - local dedicated_service # Manage arguments with getopts ynh_handle_getopts_args "$@" @@ -103,8 +102,6 @@ ynh_add_fpm_config() { fi fi - # Do not use a dedicated service by default - dedicated_service=${dedicated_service:-0} # Set the default PHP-FPM version by default if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then @@ -128,38 +125,16 @@ ynh_add_fpm_config() { fi fi - if [ $dedicated_service -eq 1 ]; then - ynh_print_warn --message "Argument --dedicated_service of ynh_add_fpm_config is deprecated and to be removed in the future" - local fpm_service="${app}-phpfpm" - local fpm_config_dir="/etc/php/$phpversion/dedicated-fpm" - else - local fpm_service="php${phpversion}-fpm" - local fpm_config_dir="/etc/php/$phpversion/fpm" - fi + local fpm_service="php${phpversion}-fpm" + local fpm_config_dir="/etc/php/$phpversion/fpm" # Create the directory for FPM pools mkdir --parents "$fpm_config_dir/pool.d" ynh_app_setting_set --app=$app --key=fpm_config_dir --value="$fpm_config_dir" ynh_app_setting_set --app=$app --key=fpm_service --value="$fpm_service" - ynh_app_setting_set --app=$app --key=fpm_dedicated_service --value="$dedicated_service" ynh_app_setting_set --app=$app --key=phpversion --value=$phpversion - # Migrate from mutual PHP service to dedicated one. - if [ $dedicated_service -eq 1 ]; then - local old_fpm_config_dir="/etc/php/$phpversion/fpm" - # If a config file exist in the common pool, move it. - if [ -e "$old_fpm_config_dir/pool.d/$app.conf" ]; then - ynh_print_info --message="Migrate to a dedicated php-fpm service for $app." - # Create a backup of the old file before migration - ynh_backup_if_checksum_is_different --file="$old_fpm_config_dir/pool.d/$app.conf" - # Remove the old PHP config file - ynh_secure_remove --file="$old_fpm_config_dir/pool.d/$app.conf" - # Reload PHP to release the socket and allow the dedicated service to use it - ynh_systemd_action --service_name=php${phpversion}-fpm --action=reload - fi - fi - if [ $autogenconf == "false" ]; then # Usage 1, use the template in conf/php-fpm.conf local phpfpm_path="$YNH_APP_BASEDIR/conf/php-fpm.conf" @@ -212,51 +187,13 @@ pm.process_idle_timeout = 10s local finalphpconf="$fpm_config_dir/pool.d/$app.conf" ynh_add_config --template="$phpfpm_path" --destination="$finalphpconf" - if [ $dedicated_service -eq 1 ]; then - # Create a dedicated php-fpm.conf for the service - local globalphpconf=$fpm_config_dir/php-fpm-$app.conf - - echo "[global] -pid = /run/php/php__PHPVERSION__-fpm-__APP__.pid -error_log = /var/log/php/fpm-php.__APP__.log -syslog.ident = php-fpm-__APP__ -include = __FINALPHPCONF__ -" >$YNH_APP_BASEDIR/conf/php-fpm-$app.conf - - ynh_add_config --template="php-fpm-$app.conf" --destination="$globalphpconf" - - # Create a config for a dedicated PHP-FPM service for the app - echo "[Unit] -Description=PHP __PHPVERSION__ FastCGI Process Manager for __APP__ -After=network.target - -[Service] -Type=notify -PIDFile=/run/php/php__PHPVERSION__-fpm-__APP__.pid -ExecStart=/usr/sbin/php-fpm__PHPVERSION__ --nodaemonize --fpm-config __GLOBALPHPCONF__ -ExecReload=/bin/kill -USR2 \$MAINPID - -[Install] -WantedBy=multi-user.target -" >$YNH_APP_BASEDIR/conf/$fpm_service - - # Create this dedicated PHP-FPM service - ynh_add_systemd_config --service=$fpm_service --template=$fpm_service - # Integrate the service in YunoHost admin panel - yunohost service add $fpm_service --log /var/log/php/fpm-php.$app.log --description "Php-fpm dedicated to $app" - # Configure log rotate - ynh_use_logrotate --logfile=/var/log/php - # Restart the service, as this service is either stopped or only for this app - ynh_systemd_action --service_name=$fpm_service --action=restart - else - # Validate that the new php conf doesn't break php-fpm entirely - if ! php-fpm${phpversion} --test 2>/dev/null; then - php-fpm${phpversion} --test || true - ynh_secure_remove --file="$finalphpconf" - ynh_die --message="The new configuration broke php-fpm?" - fi - ynh_systemd_action --service_name=$fpm_service --action=reload + # Validate that the new php conf doesn't break php-fpm entirely + if ! php-fpm${phpversion} --test 2>/dev/null; then + php-fpm${phpversion} --test || true + ynh_secure_remove --file="$finalphpconf" + ynh_die --message="The new configuration broke php-fpm?" fi + ynh_systemd_action --service_name=$fpm_service --action=reload } # Remove the dedicated PHP-FPM config @@ -267,8 +204,6 @@ WantedBy=multi-user.target ynh_remove_fpm_config() { local fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir) local fpm_service=$(ynh_app_setting_get --app=$app --key=fpm_service) - local dedicated_service=$(ynh_app_setting_get --app=$app --key=fpm_dedicated_service) - dedicated_service=${dedicated_service:-0} # Get the version of PHP used by this app local phpversion=$(ynh_app_setting_get --app=$app --key=phpversion) @@ -282,17 +217,7 @@ ynh_remove_fpm_config() { fi ynh_secure_remove --file="$fpm_config_dir/pool.d/$app.conf" - - if [ $dedicated_service -eq 1 ]; then - # Remove the dedicated service PHP-FPM service for the app - ynh_remove_systemd_config --service=$fpm_service - # Remove the global PHP-FPM conf - ynh_secure_remove --file="$fpm_config_dir/php-fpm-$app.conf" - # Remove the service from the list of services known by YunoHost - yunohost service remove $fpm_service - elif ynh_package_is_installed --package="php${phpversion}-fpm"; then - ynh_systemd_action --service_name=$fpm_service --action=reload - fi + ynh_systemd_action --service_name=$fpm_service --action=reload } # Define the values to configure PHP-FPM From f1200b81dc605a5320bfd5124b43c0ee02946d14 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 11 Jul 2023 18:10:32 +0200 Subject: [PATCH 41/82] apt: always add yarn repo because it's annoying to have to deal with an extra repo in each nodejs app just to install a single package.. --- hooks/conf_regen/10-apt | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/hooks/conf_regen/10-apt b/hooks/conf_regen/10-apt index 93ff053b8..72c0773b9 100755 --- a/hooks/conf_regen/10-apt +++ b/hooks/conf_regen/10-apt @@ -23,19 +23,33 @@ Pin-Priority: 500" >>"${pending_dir}/etc/apt/preferences.d/extra_php_version" for package in $packages_to_refuse_from_sury; do echo " Package: $package -Pin: origin \"packages.sury.org\" +Pin: origin \"packages.sury.org\" Pin-Priority: -1" >>"${pending_dir}/etc/apt/preferences.d/extra_php_version" done + # Add yarn + echo "deb https://dl.yarnpkg.com/debian/ stable main" > "${pending_dir}/etc/apt/sources.list.d/yarn.list" + + # Ban everything from Yarn except Yarn + echo " +Package: * +Pin: origin \"dl.yarnpkg.com\" +Pin-Priority: -1 + +Package: yarn +Pin: origin \"dl.yarnpkg.com\" +Pin-Priority: 500" >>"${pending_dir}/etc/apt/preferences.d/yarn" + + # Ban apache2, bind9 echo " # PLEASE READ THIS WARNING AND DON'T EDIT THIS FILE -# You are probably reading this file because you tried to install apache2 or +# You are probably reading this file because you tried to install apache2 or # bind9. These 2 packages conflict with YunoHost. # Installing apache2 will break nginx and break the entire YunoHost ecosystem -# on your server, therefore don't remove those lines! +# on your server, therefore don't remove those lines! # You have been warned. @@ -69,6 +83,12 @@ do_post_regen() { wget --timeout 900 --quiet "https://packages.sury.org/php/apt.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/extra_php_version.gpg" fi + # Similar to Sury + if [[ ! -s /etc/apt/trusted.gpg.d/yarn.gpg ]] + then + wget --timeout 900 --quiet "https://dl.yarnpkg.com/debian/pubkey.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/yarn.gpg" + fi + # Make sure php7.4 is the default version when using php in cli if test -e /usr/bin/php$YNH_DEFAULT_PHP_VERSION then From 236e85eece3f4d6fa479c9d57de41710fc4c736c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 11 Jul 2023 18:12:08 +0200 Subject: [PATCH 42/82] apt: add signed-by clause to sury and yarn repo --- hooks/conf_regen/10-apt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/conf_regen/10-apt b/hooks/conf_regen/10-apt index 72c0773b9..725fdde82 100755 --- a/hooks/conf_regen/10-apt +++ b/hooks/conf_regen/10-apt @@ -11,7 +11,7 @@ do_pre_regen() { # Add sury mkdir -p ${pending_dir}/etc/apt/sources.list.d/ - echo "deb https://packages.sury.org/php/ $(lsb_release --codename --short) main" > "${pending_dir}/etc/apt/sources.list.d/extra_php_version.list" + echo "deb [signed-by=/etc/apt/trusted.gpg.d/extra_php_version.gpg] https://packages.sury.org/php/ $(lsb_release --codename --short) main" > "${pending_dir}/etc/apt/sources.list.d/extra_php_version.list" # Ban some packages from sury echo " @@ -28,7 +28,7 @@ Pin-Priority: -1" >>"${pending_dir}/etc/apt/preferences.d/extra_php_version" done # Add yarn - echo "deb https://dl.yarnpkg.com/debian/ stable main" > "${pending_dir}/etc/apt/sources.list.d/yarn.list" + echo "deb [signed-by=/etc/apt/trusted.gpg.d/yarn.gpg] https://dl.yarnpkg.com/debian/ stable main" > "${pending_dir}/etc/apt/sources.list.d/yarn.list" # Ban everything from Yarn except Yarn echo " From ced222eaa5505acae36c2a1e4ff9cddfa14657c7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 17 Jul 2023 19:09:04 +0200 Subject: [PATCH 43/82] doc: fix resource doc generation .. not sure why this line that removed legit indent was there --- doc/generate_resource_doc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/generate_resource_doc.py b/doc/generate_resource_doc.py index 201d25265..30a5273a1 100644 --- a/doc/generate_resource_doc.py +++ b/doc/generate_resource_doc.py @@ -62,7 +62,6 @@ for c in ResourceClasses: for resource_id, doc in sorted(ResourceDocString.items()): - doc = doc.replace("\n ", "\n") print("----------------") print("") From 781f924e30d8864e3908cdf402de8911aad42a41 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 17 Jul 2023 21:13:34 +0200 Subject: [PATCH 44/82] apps: hotfix for funky issue, apps getting named 'undefined' --- src/app.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/app.py b/src/app.py index 6f18d341f..75d336241 100644 --- a/src/app.py +++ b/src/app.py @@ -1157,6 +1157,10 @@ def app_install( recursive=True, ) + # Hotfix for bug in the webadmin while we fix the actual issue :D + if label == "undefined": + label = None + # Override manifest name by given label # This info is also later picked-up by the 'permission' resource initialization if label: From 1eb208db23dff63f35926635aa9ed457be885999 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 17 Jul 2023 21:19:50 +0200 Subject: [PATCH 45/82] Update changelog for 11.2.1 --- debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index ab5003b88..3a2747bec 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +yunohost (11.2.1) stable; urgency=low + + - doc: fix resource doc generation .. not sure why this line that removed legit indent was there (ced222ea) + - apps: hotfix for funky issue, apps getting named 'undefined' (781f924e) + + -- Alexandre Aubin Mon, 17 Jul 2023 21:13:54 +0200 + yunohost (11.2) stable; urgency=low - dyndns: add support for recovery passwords ([#1475](https://github.com/YunoHost/yunohost/pull/1475)) From dd93df45e6331a2188d09f8f6d7e2e49b0957403 Mon Sep 17 00:00:00 2001 From: yunohost-bot Date: Mon, 17 Jul 2023 20:34:49 +0000 Subject: [PATCH 46/82] [CI] Format code with Black --- doc/generate_resource_doc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/generate_resource_doc.py b/doc/generate_resource_doc.py index 30a5273a1..4c60d7950 100644 --- a/doc/generate_resource_doc.py +++ b/doc/generate_resource_doc.py @@ -62,7 +62,6 @@ for c in ResourceClasses: for resource_id, doc in sorted(ResourceDocString.items()): - print("----------------") print("") print(f"## {resource_id.replace('_', ' ').title()}") From 2ece3b65f6c6e51cdbb240bdb24cb74d8c21802b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 18 Jul 2023 00:19:16 +0200 Subject: [PATCH 47/82] Moulinette logging is an unecessarily complex mess, episode 57682 --- src/__init__.py | 13 +++---------- src/app.py | 4 ++-- src/app_catalog.py | 4 ++-- src/backup.py | 4 ++-- src/certificate.py | 5 ++--- src/diagnosis.py | 3 ++- src/dns.py | 5 ++--- src/domain.py | 4 ++-- src/dyndns.py | 4 ++-- src/firewall.py | 4 ++-- src/hook.py | 4 ++-- src/log.py | 3 +-- src/migrations/0021_migrate_to_bullseye.py | 4 ++-- src/migrations/0022_php73_to_php74_pools.py | 5 ++--- src/migrations/0023_postgresql_11_to_13.py | 4 ++-- src/migrations/0024_rebuild_python_venv.py | 4 ++-- .../0025_global_settings_to_configpanel.py | 4 ++-- src/migrations/0026_new_admins_group.py | 4 ++-- src/permission.py | 4 ++-- src/regenconf.py | 14 +++++++------- src/service.py | 5 ++--- src/settings.py | 4 ++-- src/tools.py | 4 ++-- src/user.py | 4 ++-- src/utils/configpanel.py | 4 ++-- src/utils/form.py | 4 ++-- src/utils/legacy.py | 5 +++-- src/utils/resources.py | 4 ++-- 28 files changed, 62 insertions(+), 72 deletions(-) diff --git a/src/__init__.py b/src/__init__.py index d13d61089..e23b62219 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -115,17 +115,11 @@ def init_logging(interface="cli", debug=False, quiet=False, logdir="/var/log/yun "version": 1, "disable_existing_loggers": True, "formatters": { - "console": { - "format": "%(relativeCreated)-5d %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s" + "tty-debug": { + "format": "%(relativeCreated)-4d %(level_with_color)s %(message)s" }, - "tty-debug": {"format": "%(relativeCreated)-4d %(fmessage)s"}, "precise": { - "format": "%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s" - }, - }, - "filters": { - "action": { - "()": "moulinette.utils.log.ActionFilter", + "format": "%(asctime)-15s %(levelname)-8s %(name)s.%(funcName)s - %(message)s" }, }, "handlers": { @@ -142,7 +136,6 @@ def init_logging(interface="cli", debug=False, quiet=False, logdir="/var/log/yun "class": "logging.FileHandler", "formatter": "precise", "filename": logfile, - "filters": ["action"], }, }, "loggers": { diff --git a/src/app.py b/src/app.py index 75d336241..483d34153 100644 --- a/src/app.py +++ b/src/app.py @@ -28,9 +28,9 @@ import tempfile import copy from typing import List, Tuple, Dict, Any, Iterator, Optional from packaging import version +from logging import getLogger from moulinette import Moulinette, m18n -from moulinette.utils.log import getActionLogger from moulinette.utils.process import run_commands, check_output from moulinette.utils.filesystem import ( read_file, @@ -71,7 +71,7 @@ from yunohost.app_catalog import ( # noqa APPS_CATALOG_LOGOS, ) -logger = getActionLogger("yunohost.app") +logger = getLogger("yunohost.app") APPS_SETTING_PATH = "/etc/yunohost/apps/" APP_TMP_WORKDIRS = "/var/cache/yunohost/app_tmp_work_dirs" diff --git a/src/app_catalog.py b/src/app_catalog.py index 9fb662845..2a50a0f82 100644 --- a/src/app_catalog.py +++ b/src/app_catalog.py @@ -19,9 +19,9 @@ import os import re import hashlib +from logging import getLogger from moulinette import m18n -from moulinette.utils.log import getActionLogger from moulinette.utils.network import download_json from moulinette.utils.filesystem import ( read_json, @@ -34,7 +34,7 @@ from moulinette.utils.filesystem import ( from yunohost.utils.i18n import _value_for_locale from yunohost.utils.error import YunohostError -logger = getActionLogger("yunohost.app_catalog") +logger = getLogger("yunohost.app_catalog") APPS_CATALOG_CACHE = "/var/cache/yunohost/repo" APPS_CATALOG_LOGOS = "/usr/share/yunohost/applogos" diff --git a/src/backup.py b/src/backup.py index ce1e8ba2c..505e071a1 100644 --- a/src/backup.py +++ b/src/backup.py @@ -30,10 +30,10 @@ from glob import glob from collections import OrderedDict from functools import reduce from packaging import version +from logging import getLogger from moulinette import Moulinette, m18n from moulinette.utils.text import random_ascii -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import ( read_file, mkdir, @@ -84,7 +84,7 @@ APP_MARGIN_SPACE_SIZE = 100 # In MB CONF_MARGIN_SPACE_SIZE = 10 # IN MB POSTINSTALL_ESTIMATE_SPACE_SIZE = 5 # In MB MB_ALLOWED_TO_ORGANIZE = 10 -logger = getActionLogger("yunohost.backup") +logger = getLogger("yunohost.backup") class BackupRestoreTargetsManager: diff --git a/src/certificate.py b/src/certificate.py index 76d3f32b7..c3aaacc87 100644 --- a/src/certificate.py +++ b/src/certificate.py @@ -21,11 +21,10 @@ import sys import shutil import subprocess from glob import glob - +from logging import getLogger from datetime import datetime from moulinette import m18n -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_file, chown, chmod from moulinette.utils.process import check_output @@ -38,7 +37,7 @@ from yunohost.service import _run_service_command from yunohost.regenconf import regen_conf from yunohost.log import OperationLogger -logger = getActionLogger("yunohost.certmanager") +logger = getLogger("yunohost.certmanager") CERT_FOLDER = "/etc/yunohost/certs/" TMP_FOLDER = "/var/www/.well-known/acme-challenge-private/" diff --git a/src/diagnosis.py b/src/diagnosis.py index 02047c001..be3208b02 100644 --- a/src/diagnosis.py +++ b/src/diagnosis.py @@ -21,6 +21,7 @@ import os import time import glob from importlib import import_module +from logging import getLogger from moulinette import m18n, Moulinette from moulinette.utils import log @@ -33,7 +34,7 @@ from moulinette.utils.filesystem import ( from yunohost.utils.error import YunohostError, YunohostValidationError -logger = log.getActionLogger("yunohost.diagnosis") +logger = getLogger("yunohost.diagnosis") DIAGNOSIS_CACHE = "/var/cache/yunohost/diagnosis/" DIAGNOSIS_CONFIG_FILE = "/etc/yunohost/diagnosis.yml" diff --git a/src/dns.py b/src/dns.py index e25d0f3ec..9a081e228 100644 --- a/src/dns.py +++ b/src/dns.py @@ -19,12 +19,11 @@ import os import re import time - +from logging import getLogger from difflib import SequenceMatcher from collections import OrderedDict from moulinette import m18n, Moulinette -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_file, write_to_file, read_toml, mkdir from yunohost.domain import ( @@ -42,7 +41,7 @@ from yunohost.settings import settings_get from yunohost.log import is_unit_operation from yunohost.hook import hook_callback -logger = getActionLogger("yunohost.domain") +logger = getLogger("yunohost.domain") DOMAIN_REGISTRAR_LIST_PATH = "/usr/share/yunohost/registrar_list.toml" diff --git a/src/domain.py b/src/domain.py index 8fc9799cd..2d36fdfa4 100644 --- a/src/domain.py +++ b/src/domain.py @@ -20,10 +20,10 @@ import os import time from typing import List, Optional from collections import OrderedDict +from logging import getLogger from moulinette import m18n, Moulinette from moulinette.core import MoulinetteError -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import write_to_file, read_yaml, write_to_yaml, rm from yunohost.app import ( @@ -39,7 +39,7 @@ from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.dns import is_yunohost_dyndns_domain from yunohost.log import is_unit_operation -logger = getActionLogger("yunohost.domain") +logger = getLogger("yunohost.domain") DOMAIN_SETTINGS_DIR = "/etc/yunohost/domains" diff --git a/src/dyndns.py b/src/dyndns.py index a3afd655f..5c9e2a36e 100644 --- a/src/dyndns.py +++ b/src/dyndns.py @@ -22,10 +22,10 @@ import glob import base64 import subprocess import hashlib +from logging import getLogger from moulinette import Moulinette, m18n from moulinette.core import MoulinetteError -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import write_to_file, rm, chown, chmod from moulinette.utils.network import download_json @@ -36,7 +36,7 @@ from yunohost.utils.dns import dig, is_yunohost_dyndns_domain from yunohost.log import is_unit_operation from yunohost.regenconf import regen_conf -logger = getActionLogger("yunohost.dyndns") +logger = getLogger("yunohost.dyndns") DYNDNS_PROVIDER = "dyndns.yunohost.org" DYNDNS_DNS_AUTH = ["ns0.yunohost.org", "ns1.yunohost.org"] diff --git a/src/firewall.py b/src/firewall.py index 392678fe1..9375aded0 100644 --- a/src/firewall.py +++ b/src/firewall.py @@ -19,16 +19,16 @@ import os import yaml import miniupnpc +from logging import getLogger from moulinette import m18n from yunohost.utils.error import YunohostError, YunohostValidationError from moulinette.utils import process -from moulinette.utils.log import getActionLogger FIREWALL_FILE = "/etc/yunohost/firewall.yml" UPNP_CRON_JOB = "/etc/cron.d/yunohost-firewall-upnp" -logger = getActionLogger("yunohost.firewall") +logger = getLogger("yunohost.firewall") def firewall_allow( diff --git a/src/hook.py b/src/hook.py index 4b07d1c17..60abb98d4 100644 --- a/src/hook.py +++ b/src/hook.py @@ -23,16 +23,16 @@ import tempfile import mimetypes from glob import iglob from importlib import import_module +from logging import getLogger from moulinette import m18n, Moulinette from yunohost.utils.error import YunohostError, YunohostValidationError -from moulinette.utils import log from moulinette.utils.filesystem import read_yaml, cp HOOK_FOLDER = "/usr/share/yunohost/hooks/" CUSTOM_HOOK_FOLDER = "/etc/yunohost/hooks.d/" -logger = log.getActionLogger("yunohost.hook") +logger = getLogger("yunohost.hook") def hook_add(app, file): diff --git a/src/log.py b/src/log.py index 5ab918e76..13683d8ef 100644 --- a/src/log.py +++ b/src/log.py @@ -32,10 +32,9 @@ from moulinette import m18n, Moulinette from moulinette.core import MoulinetteError from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.system import get_ynh_package_version -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_file, read_yaml -logger = getActionLogger("yunohost.log") +logger = getLogger("yunohost.log") CATEGORIES_PATH = "/var/log/yunohost/categories/" OPERATIONS_PATH = "/var/log/yunohost/categories/operation/" diff --git a/src/migrations/0021_migrate_to_bullseye.py b/src/migrations/0021_migrate_to_bullseye.py index f320577e1..eee8cc667 100644 --- a/src/migrations/0021_migrate_to_bullseye.py +++ b/src/migrations/0021_migrate_to_bullseye.py @@ -1,9 +1,9 @@ import glob import os +from logging import getLogger from moulinette import m18n from yunohost.utils.error import YunohostError -from moulinette.utils.log import getActionLogger from moulinette.utils.process import check_output, call_async_output from moulinette.utils.filesystem import read_file, rm, write_to_file @@ -22,7 +22,7 @@ from yunohost.utils.system import ( ) from yunohost.service import _get_services, _save_services -logger = getActionLogger("yunohost.migration") +logger = getLogger("yunohost.migration") N_CURRENT_DEBIAN = 10 N_CURRENT_YUNOHOST = 4 diff --git a/src/migrations/0022_php73_to_php74_pools.py b/src/migrations/0022_php73_to_php74_pools.py index dc428e504..23ecc5b89 100644 --- a/src/migrations/0022_php73_to_php74_pools.py +++ b/src/migrations/0022_php73_to_php74_pools.py @@ -1,15 +1,14 @@ import os import glob from shutil import copy2 - -from moulinette.utils.log import getActionLogger +from logging import getLogger from yunohost.app import _is_installed from yunohost.utils.legacy import _patch_legacy_php_versions_in_settings from yunohost.tools import Migration from yunohost.service import _run_service_command -logger = getActionLogger("yunohost.migration") +logger = getLogger("yunohost.migration") OLDPHP_POOLS = "/etc/php/7.3/fpm/pool.d" NEWPHP_POOLS = "/etc/php/7.4/fpm/pool.d" diff --git a/src/migrations/0023_postgresql_11_to_13.py b/src/migrations/0023_postgresql_11_to_13.py index 6d37ffa74..086b17af7 100644 --- a/src/migrations/0023_postgresql_11_to_13.py +++ b/src/migrations/0023_postgresql_11_to_13.py @@ -1,15 +1,15 @@ import subprocess import time import os +from logging import getLogger from moulinette import m18n from yunohost.utils.error import YunohostError, YunohostValidationError -from moulinette.utils.log import getActionLogger from yunohost.tools import Migration from yunohost.utils.system import free_space_in_directory, space_used_by_directory -logger = getActionLogger("yunohost.migration") +logger = getLogger("yunohost.migration") class MyMigration(Migration): diff --git a/src/migrations/0024_rebuild_python_venv.py b/src/migrations/0024_rebuild_python_venv.py index 01a229b87..5b77e69a0 100644 --- a/src/migrations/0024_rebuild_python_venv.py +++ b/src/migrations/0024_rebuild_python_venv.py @@ -1,14 +1,14 @@ import os +from logging import getLogger from moulinette import m18n -from moulinette.utils.log import getActionLogger from moulinette.utils.process import call_async_output from yunohost.tools import Migration, tools_migrations_state from moulinette.utils.filesystem import rm -logger = getActionLogger("yunohost.migration") +logger = getLogger("yunohost.migration") VENV_REQUIREMENTS_SUFFIX = ".requirements_backup_for_bullseye_upgrade.txt" diff --git a/src/migrations/0025_global_settings_to_configpanel.py b/src/migrations/0025_global_settings_to_configpanel.py index 3a8818461..76289e608 100644 --- a/src/migrations/0025_global_settings_to_configpanel.py +++ b/src/migrations/0025_global_settings_to_configpanel.py @@ -1,13 +1,13 @@ import os +from logging import getLogger from yunohost.utils.error import YunohostError -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_json, write_to_yaml from yunohost.tools import Migration from yunohost.utils.legacy import translate_legacy_settings_to_configpanel_settings -logger = getActionLogger("yunohost.migration") +logger = getLogger("yunohost.migration") SETTINGS_PATH = "/etc/yunohost/settings.yml" OLD_SETTINGS_PATH = "/etc/yunohost/settings.json" diff --git a/src/migrations/0026_new_admins_group.py b/src/migrations/0026_new_admins_group.py index 43f10a7b6..30237e720 100644 --- a/src/migrations/0026_new_admins_group.py +++ b/src/migrations/0026_new_admins_group.py @@ -1,8 +1,8 @@ -from moulinette.utils.log import getActionLogger +from logging import getLogger from yunohost.tools import Migration -logger = getActionLogger("yunohost.migration") +logger = getLogger("yunohost.migration") ################################################### # Tools used also for restoration diff --git a/src/permission.py b/src/permission.py index 72975561f..7ec6f17bc 100644 --- a/src/permission.py +++ b/src/permission.py @@ -20,13 +20,13 @@ import re import copy import grp import random +from logging import getLogger from moulinette import m18n -from moulinette.utils.log import getActionLogger from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.log import is_unit_operation -logger = getActionLogger("yunohost.user") +logger = getLogger("yunohost.user") SYSTEM_PERMS = ["mail", "xmpp", "sftp", "ssh"] diff --git a/src/regenconf.py b/src/regenconf.py index 74bbdb17c..f180368ad 100644 --- a/src/regenconf.py +++ b/src/regenconf.py @@ -20,12 +20,12 @@ import os import yaml import shutil import hashlib - +from logging import getLogger from difflib import unified_diff from datetime import datetime from moulinette import m18n -from moulinette.utils import log, filesystem +from moulinette.utils.filesystem import mkdir from moulinette.utils.process import check_output from yunohost.utils.error import YunohostError @@ -37,7 +37,7 @@ BACKUP_CONF_DIR = os.path.join(BASE_CONF_PATH, "backup") PENDING_CONF_DIR = os.path.join(BASE_CONF_PATH, "pending") REGEN_CONF_FILE = "/etc/yunohost/regenconf.yml" -logger = log.getActionLogger("yunohost.regenconf") +logger = getLogger("yunohost.regenconf") # FIXME : those ain't just services anymore ... what are we supposed to do with this ... @@ -102,7 +102,7 @@ def regen_conf( for name in names: shutil.rmtree(os.path.join(PENDING_CONF_DIR, name), ignore_errors=True) else: - filesystem.mkdir(PENDING_CONF_DIR, 0o755, True) + mkdir(PENDING_CONF_DIR, 0o755, True) # Execute hooks for pre-regen # element 2 and 3 with empty string is because of legacy... @@ -111,7 +111,7 @@ def regen_conf( def _pre_call(name, priority, path, args): # create the pending conf directory for the category category_pending_path = os.path.join(PENDING_CONF_DIR, name) - filesystem.mkdir(category_pending_path, 0o755, True, uid="root") + mkdir(category_pending_path, 0o755, True, uid="root") # return the arguments to pass to the script return pre_args + [ @@ -622,7 +622,7 @@ def _process_regen_conf(system_conf, new_conf=None, save=True): backup_dir = os.path.dirname(backup_path) if not os.path.isdir(backup_dir): - filesystem.mkdir(backup_dir, 0o755, True) + mkdir(backup_dir, 0o755, True) shutil.copy2(system_conf, backup_path) logger.debug( @@ -637,7 +637,7 @@ def _process_regen_conf(system_conf, new_conf=None, save=True): system_dir = os.path.dirname(system_conf) if not os.path.isdir(system_dir): - filesystem.mkdir(system_dir, 0o755, True) + mkdir(system_dir, 0o755, True) shutil.copyfile(new_conf, system_conf) logger.debug(m18n.n("regenconf_file_updated", conf=system_conf)) diff --git a/src/service.py b/src/service.py index 47bc1903a..6bb61d841 100644 --- a/src/service.py +++ b/src/service.py @@ -21,14 +21,13 @@ import os import time import yaml import subprocess - +from logging import getLogger from glob import glob from datetime import datetime from moulinette import m18n from yunohost.utils.error import YunohostError, YunohostValidationError from moulinette.utils.process import check_output -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import ( read_file, append_to_file, @@ -42,7 +41,7 @@ MOULINETTE_LOCK = "/var/run/moulinette_yunohost.lock" SERVICES_CONF = "/etc/yunohost/services.yml" SERVICES_CONF_BASE = "/usr/share/yunohost/conf/yunohost/services.yml" -logger = getActionLogger("yunohost.service") +logger = getLogger("yunohost.service") def service_add( diff --git a/src/settings.py b/src/settings.py index 6690ab3fd..e2f34bda9 100644 --- a/src/settings.py +++ b/src/settings.py @@ -18,18 +18,18 @@ # import os import subprocess +from logging import getLogger from moulinette import m18n from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.configpanel import ConfigPanel from yunohost.utils.form import BaseOption -from moulinette.utils.log import getActionLogger from yunohost.regenconf import regen_conf from yunohost.firewall import firewall_reload from yunohost.log import is_unit_operation from yunohost.utils.legacy import translate_legacy_settings_to_configpanel_settings -logger = getActionLogger("yunohost.settings") +logger = getLogger("yunohost.settings") SETTINGS_PATH = "/etc/yunohost/settings.yml" diff --git a/src/tools.py b/src/tools.py index cd48f00ee..23edf1004 100644 --- a/src/tools.py +++ b/src/tools.py @@ -24,9 +24,9 @@ import time from importlib import import_module from packaging import version from typing import List +from logging import getLogger from moulinette import Moulinette, m18n -from moulinette.utils.log import getActionLogger from moulinette.utils.process import call_async_output from moulinette.utils.filesystem import read_yaml, write_to_yaml, cp, mkdir, rm, chown @@ -55,7 +55,7 @@ from yunohost.log import is_unit_operation, OperationLogger MIGRATIONS_STATE_PATH = "/etc/yunohost/migrations.yaml" -logger = getActionLogger("yunohost.tools") +logger = getLogger("yunohost.tools") def tools_versions(): diff --git a/src/user.py b/src/user.py index 00876854e..780797a61 100644 --- a/src/user.py +++ b/src/user.py @@ -25,9 +25,9 @@ import random import string import subprocess import copy +from logging import getLogger from moulinette import Moulinette, m18n -from moulinette.utils.log import getActionLogger from moulinette.utils.process import check_output from yunohost.utils.error import YunohostError, YunohostValidationError @@ -35,7 +35,7 @@ from yunohost.service import service_status from yunohost.log import is_unit_operation from yunohost.utils.system import binary_to_human -logger = getActionLogger("yunohost.user") +logger = getLogger("yunohost.user") FIELDS_FOR_IMPORT = { "username": r"^[a-z0-9_]+$", diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index 42a030cbc..56e071956 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -22,11 +22,11 @@ import re import urllib.parse from collections import OrderedDict from typing import Union +from logging import getLogger from moulinette import Moulinette, m18n from moulinette.interfaces.cli import colorize from moulinette.utils.filesystem import mkdir, read_toml, read_yaml, write_to_yaml -from moulinette.utils.log import getActionLogger from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.form import ( OPTIONS, @@ -40,7 +40,7 @@ from yunohost.utils.form import ( ) from yunohost.utils.i18n import _value_for_locale -logger = getActionLogger("yunohost.configpanel") +logger = getLogger("yunohost.configpanel") CONFIG_PANEL_VERSION_SUPPORTED = 1.0 diff --git a/src/utils/form.py b/src/utils/form.py index 1ca03373e..e2e01ca12 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -25,16 +25,16 @@ import tempfile import urllib.parse from enum import Enum from typing import Any, Callable, Dict, List, Literal, Mapping, Optional, Union +from logging import getLogger from moulinette import Moulinette, m18n from moulinette.interfaces.cli import colorize from moulinette.utils.filesystem import read_file, write_to_file -from moulinette.utils.log import getActionLogger from yunohost.log import OperationLogger from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.i18n import _value_for_locale -logger = getActionLogger("yunohost.form") +logger = getLogger("yunohost.form") Context = dict[str, Any] diff --git a/src/utils/legacy.py b/src/utils/legacy.py index 82507d64d..8b44fb3fb 100644 --- a/src/utils/legacy.py +++ b/src/utils/legacy.py @@ -19,8 +19,9 @@ import os import re import glob +from logging import getLogger + from moulinette.core import MoulinetteError -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import ( read_file, write_to_file, @@ -32,7 +33,7 @@ from moulinette.utils.filesystem import ( from yunohost.utils.error import YunohostValidationError -logger = getActionLogger("yunohost.legacy") +logger = getLogger("yunohost.utils.legacy") LEGACY_PERMISSION_LABEL = { ("nextcloud", "skipped"): "api", # .well-known diff --git a/src/utils/resources.py b/src/utils/resources.py index 60a5f44f6..5e3f2c569 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -23,11 +23,11 @@ import random import tempfile import subprocess from typing import Dict, Any, List, Union +from logging import getLogger from moulinette import m18n from moulinette.utils.text import random_ascii from moulinette.utils.process import check_output -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import mkdir, chown, chmod, write_to_file from moulinette.utils.filesystem import ( rm, @@ -35,7 +35,7 @@ from moulinette.utils.filesystem import ( from yunohost.utils.system import system_arch from yunohost.utils.error import YunohostError, YunohostValidationError -logger = getActionLogger("yunohost.app_resources") +logger = getLogger("yunohost.utils.resources") class AppResourceManager: From 6b5c9a2a8bc0b628a9f45773e7375f51c4bc5782 Mon Sep 17 00:00:00 2001 From: tituspijean Date: Tue, 18 Jul 2023 11:25:08 +0200 Subject: [PATCH 48/82] Gandi's `api_protocol` field should be a `select` type --- share/registrar_list.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/registrar_list.toml b/share/registrar_list.toml index 3f478a03f..47218c9e3 100644 --- a/share/registrar_list.toml +++ b/share/registrar_list.toml @@ -227,7 +227,7 @@ redact = true [gandi.api_protocol] - type = "string" + type = "select" choices.rpc = "RPC" choices.rest = "REST" default = "rest" From e1ceb084c3df75e37be4fbde97c075ee75d775cf Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 18 Jul 2023 15:05:07 +0200 Subject: [PATCH 49/82] fix config panel readonly .value --- src/utils/configpanel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index 42a030cbc..ee9019303 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -609,7 +609,7 @@ class ConfigPanel: { question.id: question.value for question in questions - if question.value is not None + if not question.readonly and question.value is not None } ) From 2d54be6e8d64a3f19c215ed3553d928a1548b156 Mon Sep 17 00:00:00 2001 From: ppr Date: Tue, 18 Jul 2023 20:04:22 +0000 Subject: [PATCH 50/82] Translated using Weblate (French) Currently translated at 98.4% (768 of 780 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/fr/ --- locales/fr.json | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/locales/fr.json b/locales/fr.json index f98470c99..2cc0f612c 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -527,7 +527,7 @@ "pattern_email_forward": "L'adresse électronique doit être valide, le symbole '+' étant accepté (par exemple : johndoe+yunohost@exemple.com)", "global_settings_setting_smtp_relay_password": "Mot de passe du relais SMTP", "diagnosis_package_installed_from_sury": "Des paquets du système devraient être rétrogradé de version", - "additional_urls_already_added": "URL supplémentaire '{url}' déjà ajoutée pour la permission '{permission}'", + "additional_urls_already_added": "L'URL supplémentaire '{url}' a déjà été ajoutée pour la permission '{permission}'", "unknown_main_domain_path": "Domaine ou chemin inconnu pour '{app}'. Vous devez spécifier un domaine et un chemin pour pouvoir spécifier une URL pour l'autorisation.", "show_tile_cant_be_enabled_for_regex": "Vous ne pouvez pas activer 'show_tile' pour le moment, cela car l'URL de l'autorisation '{permission}' est une expression régulière", "show_tile_cant_be_enabled_for_url_not_defined": "Vous ne pouvez pas activer 'show_tile' pour le moment, car vous devez d'abord définir une URL pour l'autorisation '{permission}'", @@ -536,7 +536,7 @@ "permission_protected": "L'autorisation {permission} est protégée. Vous ne pouvez pas ajouter ou supprimer le groupe visiteurs à/de cette autorisation.", "invalid_regex": "Regex non valide : '{regex}'", "app_label_deprecated": "Cette commande est obsolète ! Veuillez utiliser la nouvelle commande 'yunohost user permission update' pour gérer l'étiquette de l'application.", - "additional_urls_already_removed": "URL supplémentaire '{url}' déjà supprimées pour la permission '{permission}'", + "additional_urls_already_removed": "L'URL supplémentaire '{url}' a déjà été supprimée pour la permission '{permission}'", "invalid_number": "Doit être un nombre", "diagnosis_basesystem_hardware_model": "Le modèle/architecture du serveur est {model}", "diagnosis_backports_in_sources_list": "Il semble que le gestionnaire de paquet APT soit configuré pour utiliser le dépôt des rétro-portages (backports). A moins que vous ne sachiez vraiment ce que vous faites, nous vous déconseillons fortement d'installer des paquets provenant du dépôt 'backports', car cela risque de créer des instabilités ou des conflits sur votre système.", @@ -766,5 +766,21 @@ "group_mailalias_add": "L'alias de courrier électronique '{mail}' sera ajouté au groupe '{group}'", "group_user_add": "L'utilisateur '{user}' sera ajouté au groupe '{group}'", "group_user_remove": "L'utilisateur '{user}' sera retiré du groupe '{group}'", - "group_mailalias_remove": "L'alias de courrier électronique '{mail}' sera supprimé du groupe '{group}'" + "group_mailalias_remove": "L'alias de courrier électronique '{mail}' sera supprimé du groupe '{group}'", + "ask_dyndns_recovery_password_explain": "Veuillez choisir un mot de passe de récupération pour votre domaine DynDNS, au cas où vous devriez le réinitialiser plus tard.", + "ask_dyndns_recovery_password": "Mot de passe de récupération DynDNS", + "ask_dyndns_recovery_password_explain_during_unsubscribe": "Veuillez saisir le mot de passe de récupération pour ce domaine DynDNS.", + "dyndns_no_recovery_password": "Aucun mot de passe de récupération n'a été spécifié ! Si vous perdez le contrôle de ce domaine, vous devrez contacter un administrateur de l'équipe YunoHost !", + "dyndns_subscribed": "Domaine DynDNS souscrit/enregistré", + "dyndns_subscribe_failed": "Impossible de souscrire/de s'enregistrer au domaine DynDNS : {erreur}", + "dyndns_unsubscribe_failed": "Impossible de se désinscrire du domaine DynDNS : {erreur}", + "dyndns_unsubscribed": "Désinscription du domaine DynDNS", + "dyndns_unsubscribe_denied": "Échec de la désinscription du domaine : informations d'identification non valides", + "dyndns_unsubscribe_already_unsubscribed": "Le domaine est déjà désabonné/retiré", + "dyndns_set_recovery_password_denied": "Échec de la définition du mot de passe de récupération : mot de passe non valide", + "dyndns_set_recovery_password_unknown_domain": "Échec de la définition du mot de passe de récupération : le domaine n'est pas enregistré", + "dyndns_set_recovery_password_invalid_password": "Échec de la définition du mot de passe de récupération : le mot de passe n'est pas assez fort/solide", + "dyndns_set_recovery_password_failed": "Échec de la définition du mot de passe de récupération : {erreur}", + "dyndns_set_recovery_password_success": "Mot de passe de récupération défini/configuré !", + "log_dyndns_unsubscribe": "Se désabonner d'un sous-domaine YunoHost '{}'" } From e4462556488444f8f9ed70385a1d711eda85be5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20M?= Date: Tue, 18 Jul 2023 03:49:58 +0000 Subject: [PATCH 51/82] Translated using Weblate (Galician) Currently translated at 100.0% (780 of 780 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/gl/ --- locales/gl.json | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/locales/gl.json b/locales/gl.json index 3aaacd9c9..1926f6148 100644 --- a/locales/gl.json +++ b/locales/gl.json @@ -766,5 +766,21 @@ "group_mailalias_add": "Vaise engadir o alias de correo '{mail}' ao grupo '{group}'", "group_mailalias_remove": "Vaise quitar o alias de email '{mail}' do grupo '{group}'", "group_user_add": "Vaise engadir a '{user}' ao grupo '{grupo}'", - "group_user_remove": "Vaise quitar a '{user}' do grupo '{grupo}'" + "group_user_remove": "Vaise quitar a '{user}' do grupo '{grupo}'", + "ask_dyndns_recovery_password_explain": "Elixe un contrasinal de recuperación para o teu dominio DynDNS, por se precisas restablecelo no futuro.", + "ask_dyndns_recovery_password": "Contrasinal de recuperación DynDNS", + "ask_dyndns_recovery_password_explain_during_unsubscribe": "Escribe o contrasinal de recuperación para este dominio DynDNS.", + "dyndns_no_recovery_password": "Non se estableceu un contrasinal de recuperación! Se perdes o control sobre dominio precisarás contactar coa administración do equipo YunoHost!", + "dyndns_subscribed": "Tes unha subscrición a un dominio DynDNS", + "dyndns_subscribe_failed": "Non te subscribiches ao dominio DynDNS: {error}", + "dyndns_unsubscribe_failed": "Non se retirou a subscrición ao dominio DynDNS: {error}", + "dyndns_unsubscribed": "Retirada a subscrición ao dominio DynDNS", + "dyndns_unsubscribe_denied": "Fallo ao intentar retirar subscrición: credenciais incorrectas", + "dyndns_unsubscribe_already_unsubscribed": "Non tes unha subscrición ao dominio", + "dyndns_set_recovery_password_denied": "Fallou o establecemento do contrasinal de recuperación: chave non válida", + "dyndns_set_recovery_password_unknown_domain": "Fallo ao establecer o contrasinal de recuperación: dominio non rexistrado", + "dyndns_set_recovery_password_invalid_password": "Fallo ao establecer contrasinal de recuperación: o contrasinal non é suficientemente forte", + "dyndns_set_recovery_password_failed": "Fallo ao establecer o contrasinal de recuperación: {error}", + "dyndns_set_recovery_password_success": "Estableceuse o contrasinal de recuperación!", + "log_dyndns_unsubscribe": "Retirar subscrición para o subdominio YunoHost '{}'" } From 53bc30b9fb1ca698b971ae10e579abce605b56b0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jul 2023 02:36:39 +0200 Subject: [PATCH 52/82] Update changelog for 11.2.2 --- debian/changelog | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/debian/changelog b/debian/changelog index 3a2747bec..103495d72 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,13 @@ +yunohost (11.2.2) stable; urgency=low + + - domains: Gandi's `api_protocol` field should be a `select` type ([#1693](https://github.com/yunohost/yunohost/pull/1693)) + - configpanel: fix .value call for readonly-type options (e1ceb084) + - i18n: Translations updated for French, Galician + + Thanks to all contributors <3 ! (axolotle, José M, ppr, tituspijean) + + -- Alexandre Aubin Wed, 19 Jul 2023 02:35:28 +0200 + yunohost (11.2.1) stable; urgency=low - doc: fix resource doc generation .. not sure why this line that removed legit indent was there (ced222ea) From 4fda8ed49fbf97c10343c15e794e74683a727785 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 21 Jul 2023 22:02:56 +0200 Subject: [PATCH 53/82] apps: fix another case of no attribute 'value' due to config panels/questions refactoring --- src/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.py b/src/app.py index 75d336241..7f4acdc1a 100644 --- a/src/app.py +++ b/src/app.py @@ -1098,7 +1098,7 @@ def app_install( args = { question.id: question.value for question in questions - if question.value is not None + if not question.readonly and question.value is not None } # Validate domain / path availability for webapps From 465f6da5cd4d716bbcb802dfd742114083034235 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 22 Jul 2023 16:48:48 +0200 Subject: [PATCH 54/82] Update changelog for 11.2.3 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 103495d72..586f8387b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +yunohost (11.2.3) stable; urgency=low + + - apps: fix another case of no attribute 'value' due to config panels/questions refactoring (4fda8ed49) + + -- Alexandre Aubin Sat, 22 Jul 2023 16:48:22 +0200 + yunohost (11.2.2) stable; urgency=low - domains: Gandi's `api_protocol` field should be a `select` type ([#1693](https://github.com/yunohost/yunohost/pull/1693)) From d2113b243ebc4f5ac3ffcce34e821f47bea27ba1 Mon Sep 17 00:00:00 2001 From: selfhoster1312 Date: Tue, 15 Aug 2023 15:35:36 +0200 Subject: [PATCH 55/82] Add information in yunohost app install --help --- share/actionsmap.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/share/actionsmap.yml b/share/actionsmap.yml index 48e12ba0f..0c3301874 100644 --- a/share/actionsmap.yml +++ b/share/actionsmap.yml @@ -934,14 +934,14 @@ app: help: Custom name for the app -a: full: --args - help: Serialized arguments for app script (i.e. "domain=domain.tld&path=/path") + help: Serialized arguments for app script (i.e. "domain=domain.tld&path=/path&init_main_permission=visitors") -n: full: --no-remove-on-failure help: Debug option to avoid removing the app on a failed installation action: store_true -f: full: --force - help: Do not ask confirmation if the app is not safe to use (low quality, experimental or 3rd party) + help: Do not ask confirmation if the app is not safe to use (low quality, experimental or 3rd party), or when the app displays a post-install notification action: store_true ### app_remove() From 0d88978c2a31e967570b04e14ab747956b2fb9b5 Mon Sep 17 00:00:00 2001 From: massyas Date: Wed, 16 Aug 2023 17:25:23 +0200 Subject: [PATCH 56/82] Fix typo in app_upgrade argument help --- share/actionsmap.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/actionsmap.yml b/share/actionsmap.yml index 0c3301874..e44a72125 100644 --- a/share/actionsmap.yml +++ b/share/actionsmap.yml @@ -980,7 +980,7 @@ app: action: store_true -c: full: --continue-on-failure - help: Continue to upgrade apps event if one or more upgrade failed + help: Continue to upgrade apps even if one or more upgrade failed action: store_true ### app_change_url() From d716746f281957fec6054744a13ce499420e76e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20M?= Date: Wed, 19 Jul 2023 12:09:22 +0000 Subject: [PATCH 57/82] Translated using Weblate (Galician) Currently translated at 100.0% (780 of 780 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/gl/ --- locales/gl.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/locales/gl.json b/locales/gl.json index 1926f6148..93f73b528 100644 --- a/locales/gl.json +++ b/locales/gl.json @@ -612,7 +612,7 @@ "domain_config_auth_consumer_key": "Chave consumidora", "log_domain_dns_push": "Enviar rexistros DNS para o dominio '{}'", "other_available_options": "... e outras {n} opcións dispoñibles non mostradas", - "domain_dns_registrar_yunohost": "Este dominio un dos de nohost.me / nohost.st / ynh.fr e a configuración DNS xestionaa directamente YunoHost se máis requisitos. (mira o comando 'yunohost dyndns update')", + "domain_dns_registrar_yunohost": "Este dominio é un dos de nohost.me / nohost.st / ynh.fr e a configuración DNS xestionaa directamente YunoHost sen máis requisitos. (mira o comando 'yunohost dyndns update')", "domain_dns_registrar_supported": "YunoHost detectou automáticamente que este dominio está xestionado pola rexistradora **{registrar}**. Se queres, YunoHost pode configurar automáticamente as súas zonas DNS, se proporcionas as credenciais de acceso á API. Podes ver a documentación sobre como obter as credenciais da API nesta páxina: https://yunohost.org/registrar_api_{registrar}. (Tamén podes configurar manualmente os rexistros DNS seguindo a documentación en https://yunohost.org/dns )", "domain_dns_push_partial_failure": "Actualización parcial dos rexistros DNS: informouse dalgúns avisos/erros.", "domain_config_auth_token": "Token de autenticación", @@ -654,7 +654,7 @@ "global_settings_setting_admin_strength": "Fortaleza do contrasinal de Admin", "global_settings_setting_user_strength": "Fortaleza do contrasinal da usuaria", "global_settings_setting_postfix_compatibility_help": "Compromiso entre compatibilidade e seguridade para o servidor Postfix. Aféctalle ao cifrado (e outros aspectos da seguridade)", - "global_settings_setting_ssh_compatibility_help": "Compromiso entre compatibilidade e seguridade para o servidor SSH. Aféctalle ao cifrado (e outros aspectos da seguridade)", + "global_settings_setting_ssh_compatibility_help": "Compromiso entre compatibilidade e seguridade para o servidor SSH. Aféctalle ao cifrado (e outros aspectos da seguridade). Le https://infosec.mozilla.org/guidelines/openssh for more info.", "global_settings_setting_ssh_password_authentication_help": "Permitir autenticación con contrasinal para SSH", "global_settings_setting_ssh_port": "Porto SSH", "global_settings_setting_webadmin_allowlist_help": "Enderezos IP con permiso para acceder á webadmin. Separados por vírgulas.", From 9fdbc5532f8f5144602f3430e2f9ef07f96dbc0e Mon Sep 17 00:00:00 2001 From: Suleyman Harmandar Date: Sat, 22 Jul 2023 05:45:21 +0000 Subject: [PATCH 58/82] Translated using Weblate (Turkish) Currently translated at 2.5% (20 of 780 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/tr/ --- locales/tr.json | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/locales/tr.json b/locales/tr.json index 1af0ffd54..3c15591f3 100644 --- a/locales/tr.json +++ b/locales/tr.json @@ -15,5 +15,10 @@ "additional_urls_already_added": "Ek URL '{url}' zaten '{permission}' izni için ek URL'ye eklendi", "additional_urls_already_removed": "Ek URL '{url}', '{permission}' izni için ek URL'de zaten kaldırıldı", "app_action_cannot_be_ran_because_required_services_down": "Bu eylemi gerçekleştirmek için şu servisler çalışıyor olmalıdır: {services}. Devam etmek için onları yeniden başlatın (ve muhtemelen neden çalışmadığını araştırın).", - "app_arch_not_supported": "Bu uygulama yalnızca {required} işlemci mimarisi üzerine kurulabilir ancak sunucunuzun işlemci mimarisi {current}." -} \ No newline at end of file + "app_arch_not_supported": "Bu uygulama yalnızca {required} işlemci mimarisi üzerine kurulabilir ancak sunucunuzun işlemci mimarisi {current}.", + "app_argument_choice_invalid": "'{name}'' için geçerli bir değer giriniz '{value}' mevcut seçimlerin arasında değil ({choices})", + "app_change_url_failed": "{app}: {error} için url değiştirilemedi", + "app_argument_required": "'{name}' değeri gerekli", + "app_argument_invalid": "'{name}': {error} için geçerli bir değer giriniz", + "app_argument_password_no_default": "'{name}': çözümlenirken bir hata meydana geldi. Parola argümanı güvenlik nedeniyle varsayılan değer alamaz" +} From 5eecfcae6755c517cdc73943a981544d11f3d094 Mon Sep 17 00:00:00 2001 From: Kuba Bazan Date: Sun, 23 Jul 2023 22:20:41 +0000 Subject: [PATCH 59/82] Translated using Weblate (Polish) Currently translated at 34.8% (272 of 780 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/pl/ --- locales/pl.json | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/locales/pl.json b/locales/pl.json index 52f2de3ca..78d295f61 100644 --- a/locales/pl.json +++ b/locales/pl.json @@ -103,7 +103,7 @@ "backup_applying_method_custom": "Wywołuję niestandardową metodę tworzenia kopii zapasowych '{method}'...", "app_remove_after_failed_install": "Usuwanie aplikacji po niepowodzeniu instalacji...", "app_upgrade_script_failed": "Wystąpił błąd w skrypcie aktualizacji aplikacji", - "apps_catalog_init_success": "Zainicjowano system katalogu aplikacji!", + "apps_catalog_init_success": "System katalogu aplikacji został zainicjowany!", "apps_catalog_obsolete_cache": "Pamięć podręczna katalogu aplikacji jest pusta lub przestarzała.", "app_extraction_failed": "Nie można wyodrębnić plików instalacyjnych", "app_packaging_format_not_supported": "Ta aplikacja nie może zostać zainstalowana, ponieważ jej format opakowania nie jest obsługiwany przez twoją wersję YunoHost. Prawdopodobnie powinieneś rozważyć aktualizację swojego systemu.", @@ -183,7 +183,7 @@ "app_failed_to_download_asset": "Nie udało się pobrać zasobu '{source_id}' ({url}) dla {app}: {out}", "backup_with_no_backup_script_for_app": "Aplikacja '{app}' nie posiada skryptu kopii zapasowej. Ignorowanie.", "backup_with_no_restore_script_for_app": "Aplikacja {app} nie posiada skryptu przywracania, co oznacza, że nie będzie można automatycznie przywrócić kopii zapasowej tej aplikacji.", - "certmanager_acme_not_configured_for_domain": "Wyzwanie ACME nie może zostać uruchomione dla domeny {domain}, ponieważ jej konfiguracja nginx nie zawiera odpowiedniego fragmentu kodu... Upewnij się, że konfiguracja nginx jest aktualna, używając polecenia yunohost tools regen-conf nginx --dry-run --with-diff.", + "certmanager_acme_not_configured_for_domain": "Wyzwanie ACME nie może być teraz uruchomione dla {domain}, ponieważ jego konfiguracja nginx nie zawiera odpowiedniego fragmentu kodu… Upewnij się, że twoja konfiguracja nginx jest aktualna, używając `yunohost tools regen-conf nginx --dry-run --with-diff`.", "certmanager_domain_dns_ip_differs_from_public_ip": "Rekordy DNS dla domeny '{domain}' różnią się od adresu IP tego serwera. Sprawdź kategorię 'Rekordy DNS' (podstawowe) w diagnozie, aby uzyskać więcej informacji. Jeśli niedawno dokonałeś zmiany rekordu A, poczekaj, aż zostanie on zaktualizowany (można skorzystać z narzędzi online do sprawdzania propagacji DNS). (Jeśli wiesz, co robisz, użyj opcji '--no-checks', aby wyłączyć te sprawdzania.)", "confirm_app_install_danger": "UWAGA! Ta aplikacja jest wciąż w fazie eksperymentalnej (jeśli nie działa jawnie)! Prawdopodobnie NIE powinieneś jej instalować, chyba że wiesz, co robisz. NIE ZOSTANIE udzielone wsparcie, jeśli ta aplikacja nie będzie działać poprawnie lub spowoduje uszkodzenie systemu... Jeśli mimo to jesteś gotów podjąć to ryzyko, wpisz '{answers}", "confirm_app_install_thirdparty": "UWAGA! Ta aplikacja nie jest częścią katalogu aplikacji YunoHost. Instalowanie aplikacji innych firm może naruszyć integralność i bezpieczeństwo systemu. Prawdopodobnie NIE powinieneś jej instalować, chyba że wiesz, co robisz. NIE ZOSTANIE udzielone wsparcie, jeśli ta aplikacja nie będzie działać poprawnie lub spowoduje uszkodzenie systemu... Jeśli mimo to jesteś gotów podjąć to ryzyko, wpisz '{answers}'", @@ -192,19 +192,19 @@ "config_no_panel": "Nie znaleziono panelu konfiguracji.", "config_unknown_filter_key": "Klucz filtru '{filter_key}' jest niepoprawny.", "config_validate_email": "Proszę podać poprawny adres e-mail", - "backup_hook_unknown": "Nieznany jest hook kopii zapasowej '{hook}'.", + "backup_hook_unknown": "Nieznany jest hook kopii zapasowej '{hook}'", "backup_no_uncompress_archive_dir": "Nie istnieje taki katalog nieskompresowanego archiwum.", - "backup_output_symlink_dir_broken": "Twój katalog archiwum '{path}' to uszkodzony dowiązanie symboliczne. Być może zapomniałeś o ponownym zamontowaniu lub podłączeniu nośnika przechowującego, do którego on wskazuje.", - "backup_system_part_failed": "Nie można wykonać kopii zapasowej części systemu '{part}'", + "backup_output_symlink_dir_broken": "Twój katalog archiwum ‘{path}’ to uszkodzony symlink. Być może zapomniałeś o ponownym zamontowaniu lub podłączeniu nośnika przechowującego, do którego on wskazuje.", + "backup_system_part_failed": "Nie udało się wykonać kopii zapasowej części systemu ‘{part}’", "config_validate_color": "Powinien być poprawnym szesnastkowym kodem koloru RGB.", "config_validate_date": "Data powinna być poprawna w formacie RRRR-MM-DD", "config_validate_time": "Podaj poprawny czas w formacie GG:MM", "certmanager_domain_not_diagnosed_yet": "Nie ma jeszcze wyników diagnozy dla domeny {domain}. Proszę ponownie uruchomić diagnozę dla kategorii 'Rekordy DNS' i 'Strona internetowa' w sekcji diagnozy, aby sprawdzić, czy domena jest gotowa do użycia Let's Encrypt. (Jeśli wiesz, co robisz, użyj opcji '--no-checks', aby wyłączyć te sprawdzania.)", "certmanager_cannot_read_cert": "Wystąpił problem podczas próby otwarcia bieżącego certyfikatu dla domeny {domain} (plik: {file}), przyczyna: {reason}", - "certmanager_no_cert_file": "Nie można odczytać pliku certyfikatu dla domeny {domain} (plik: {file}).", - "certmanager_self_ca_conf_file_not_found": "Nie można znaleźć pliku konfiguracyjnego dla autorytetu samopodpisującego (plik: {file})", - "backup_running_hooks": "Uruchamianie hooków kopii zapasowej...", - "backup_permission": "Uprawnienia kopii zapasowej dla aplikacji {app}", + "certmanager_no_cert_file": "Nie można odczytać pliku certyfikatu dla domeny {domain} (plik: {file})", + "certmanager_self_ca_conf_file_not_found": "Nie można znaleźć pliku konfiguracyjnego dla samodzielnie podpisanego upoważnienia do (file: {file})", + "backup_running_hooks": "Uruchamianie kopii zapasowej hooków...", + "backup_permission": "Uprawnienia do tworzenia kopii zapasowej dla aplikacji {app}", "certmanager_domain_cert_not_selfsigned": "Certyfikat dla domeny {domain} nie jest samopodpisany. Czy na pewno chcesz go zastąpić? (Użyj opcji '--force', aby to zrobić.)", "config_action_disabled": "Nie można uruchomić akcji '{action}', ponieważ jest ona wyłączona. Upewnij się, że spełnione są jej ograniczenia. Pomoc: {help}", "config_action_failed": "Nie udało się uruchomić akcji '{action}': {error}", @@ -214,7 +214,7 @@ "confirm_app_insufficient_ram": "UWAGA! Ta aplikacja wymaga {required} pamięci RAM do zainstalowania/aktualizacji, a obecnie dostępne jest tylko {current}. Nawet jeśli aplikacja mogłaby działać, proces instalacji/aktualizacji wymaga dużej ilości pamięci RAM, więc serwer może się zawiesić i niepowodzenie może być katastrofalne. Jeśli mimo to jesteś gotów podjąć to ryzyko, wpisz '{answers}'", "app_not_upgraded_broken_system": "Aplikacja '{failed_app}' nie powiodła się w procesie aktualizacji i spowodowała uszkodzenie systemu. W rezultacie anulowane zostały aktualizacje następujących aplikacji: {apps}", "app_not_upgraded_broken_system_continue": "Aplikacja '{failed_app}' nie powiodła się w procesie aktualizacji i spowodowała uszkodzenie systemu (parametr --continue-on-failure jest ignorowany). W rezultacie anulowane zostały aktualizacje następujących aplikacji: {apps}", - "certmanager_domain_http_not_working": "Domena {domain} wydaje się niedostępna przez HTTP. Sprawdź kategorię 'Strona internetowa' diagnostyki, aby uzyskać więcej informacji. (Jeśli wiesz, co robisz, użyj opcji '--no-checks', aby wyłączyć te sprawdzania.)", + "certmanager_domain_http_not_working": "Domena {domain} nie wydaje się być dostępna przez HTTP. Sprawdź kategorię 'Strona internetowa' diagnostyki, aby uzyskać więcej informacji. (Jeśli wiesz, co robisz, użyj opcji '--no-checks', aby wyłączyć te sprawdzania.)", "migration_0021_system_not_fully_up_to_date": "Twój system nie jest w pełni zaktualizowany! Proszę, wykonaj zwykłą aktualizację oprogramowania zanim rozpoczniesz migrację na system Bullseye.", "global_settings_setting_smtp_relay_port": "Port przekaźnika SMTP", "domain_config_cert_renew": "Odnów certyfikat Let's Encrypt", @@ -274,5 +274,12 @@ "global_settings_setting_smtp_allow_ipv6_help": "Zezwól na wykorzystywanie IPv7 do odbierania i wysyłania maili", "global_settings_setting_ssh_password_authentication": "Logowanie hasłem", "diagnosis_backports_in_sources_list": "Wygląda na to że apt (menedżer pakietów) został skonfigurowany tak, aby wykorzystywać repozytorium backported. Nie zalecamy wykorzystywania repozytorium backported, ponieważ może powodować problemy ze stabilnością i/lub konflikty z konfiguracją. No chyba, że wiesz co robisz.", - "domain_config_xmpp_help": "Uwaga: niektóre funkcje XMPP będą wymagały aktualizacji rekordów DNS i odnowienia certyfikatu Lets Encrypt w celu ich włączenia" + "domain_config_xmpp_help": "Uwaga: niektóre funkcje XMPP będą wymagały aktualizacji rekordów DNS i odnowienia certyfikatu Lets Encrypt w celu ich włączenia", + "ask_dyndns_recovery_password_explain": "Proszę wybrać hasło odzyskiwania dla swojej domeny DynDNS, na wypadek gdybyś musiał go później zresetować.", + "ask_dyndns_recovery_password_explain_during_unsubscribe": "Proszę wprowadzić hasło odzyskiwania dla tej domeny DynDNS.", + "certmanager_unable_to_parse_self_CA_name": "Nie można spasować nazwy organu samopodpisywanego (pliku: {file})", + "app_corrupt_source": "YunoHost był w stanie pobrać zasób ‘{source_id}’ ({url}) dla {app}, ale zasób nie pasuje do oczekiwanego sumy kontrolnej. Może to oznaczać, że na twoim serwerze wystąpiła tymczasowa awaria sieci, LUB zasób został jakoś zmieniony przez dostawcę usługi (lub złośliwego aktora?) i pakowacze YunoHost muszą zbadać sprawę i zaktualizować manifest aplikacji, aby odzwierciedlić tę zmianę. \nOczekiwana suma kontrolna sha256: {expected_sha256} \nPobrana suma kontrolna sha256: {computed_sha256} \nRozmiar pobranego pliku: {size}”", + "ask_dyndns_recovery_password": "Hasło odzyskiwania DynDNS", + "certmanager_hit_rate_limit": "Zbyt wiele certyfikatów zostało ostatnio wydanych dla tej dokładnej grupy domen {domain}. Spróbuj ponownie później. Zobacz https://letsencrypt.org/docs/rate-limits/ aby uzyskać więcej informacji", + "apps_failed_to_upgrade_line": "\n * {app_id} (aby zobaczyć odpowiedni dziennik, wykonaj ‘yunohost log show {operation_logger_name}’)" } From 927a17cf3046d49023e9cd8ac92be5a02978eb2a Mon Sep 17 00:00:00 2001 From: taco Date: Wed, 2 Aug 2023 18:07:56 +0000 Subject: [PATCH 60/82] Translated using Weblate (Spanish) Currently translated at 95.8% (748 of 780 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/es/ --- locales/es.json | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/locales/es.json b/locales/es.json index 85d7b1f43..5003c305c 100644 --- a/locales/es.json +++ b/locales/es.json @@ -522,8 +522,8 @@ "diagnosis_mail_outgoing_port_25_blocked_relay_vpn": "Algunos proveedores de internet no le permitirán desbloquear el puerto 25 porque no les importa la Neutralidad de la Red.
- Algunos proporcionan una alternativa usando un relay como servidor de correo lo que implica que el relay podrá espiar tu tráfico de correo.
- Una alternativa buena para la privacidad es utilizar una VPN *con una IP pública dedicada* para evitar estas limitaciones. Mira en https://yunohost.org/#/vpn_advantage
- Otra alternativa es cambiar de proveedor de internet a uno más amable con la Neutralidad de la Red", "diagnosis_backports_in_sources_list": "Parece que apt (el gestor de paquetes) está configurado para usar el repositorio backports. A menos que realmente sepas lo que estás haciendo, desaconsejamos absolutamente instalar paquetes desde backports, ya que pueden provocar comportamientos intestables o conflictos en el sistema.", "diagnosis_basesystem_hardware_model": "El modelo de servidor es {model}", - "additional_urls_already_removed": "La URL adicional '{url}' ya se ha eliminado para el permiso «{permission}»", - "additional_urls_already_added": "La URL adicional '{url}' ya se ha añadido para el permiso «{permission}»", + "additional_urls_already_removed": "URL adicional '{url}' ya eliminada en la URL adicional para permiso «{permission}»", + "additional_urls_already_added": "URL adicional '{url}' ya añadida en la URL adicional para permiso «{permission}»", "config_apply_failed": "Falló la aplicación de la nueva configuración: {error}", "app_restore_script_failed": "Ha ocurrido un error dentro del script de restauración de aplicaciones", "app_config_unable_to_apply": "No se pudieron aplicar los valores del panel configuración.", @@ -747,5 +747,10 @@ "global_settings_setting_smtp_relay_host": "Host de retransmisión SMTP", "migration_0024_rebuild_python_venv_disclaimer_rebuild": "Se intentará reconstruir el virtualenv para las siguientes apps (NB: ¡la operación puede llevar algún tiempo!): {rebuild_apps}", "migration_description_0025_global_settings_to_configpanel": "Migración de la nomenclatura de ajustes globales heredada a la nomenclatura nueva y moderna", - "registrar_infos": "Información sobre el registrador" -} \ No newline at end of file + "registrar_infos": "Información sobre el registrador", + "app_failed_to_download_asset": "Error al descargar el recurso '{source_id}' ({url}) para {app}: {out}", + "app_corrupt_source": "YunoHost ha podido descargar el recurso '{source_id}' ({url}) para {app}, pero no coincide con la suma de comprobación esperada. Esto puede significar que ocurrió un fallo de red en tu servidor, o que el recurso ha sido modificado por el responsable de la aplicación (¿o un actor malicioso?) y los responsables de empaquetar esta aplicación para YunoHost necesitan investigar y actualizar el manifesto de la aplicación para reflejar estos cambios. \n Suma de control sha256 esperada: {expected_sha256}\n Suma de control sha256 descargada: {computed_sha256}\n Tamaño del archivo descargado: {size}", + "app_change_url_failed": "No es possible cambiar la URL para {app}: {error}", + "app_change_url_require_full_domain": "{app} no se puede mover a esta nueva URL porque requiere un dominio completo (es decir, con una ruta = /)", + "app_change_url_script_failed": "Se ha producido un error en el script de modificación de la url" +} From f46dc30783e7b19295f3e5b36c7c85e34832cf5e Mon Sep 17 00:00:00 2001 From: Grzegorz Cichocki Date: Sat, 5 Aug 2023 22:19:12 +0000 Subject: [PATCH 61/82] Translated using Weblate (Polish) Currently translated at 35.0% (273 of 780 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/pl/ --- locales/pl.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locales/pl.json b/locales/pl.json index 78d295f61..17a9fa572 100644 --- a/locales/pl.json +++ b/locales/pl.json @@ -193,7 +193,7 @@ "config_unknown_filter_key": "Klucz filtru '{filter_key}' jest niepoprawny.", "config_validate_email": "Proszę podać poprawny adres e-mail", "backup_hook_unknown": "Nieznany jest hook kopii zapasowej '{hook}'", - "backup_no_uncompress_archive_dir": "Nie istnieje taki katalog nieskompresowanego archiwum.", + "backup_no_uncompress_archive_dir": "Nie istnieje taki katalog nieskompresowanego archiwum", "backup_output_symlink_dir_broken": "Twój katalog archiwum ‘{path}’ to uszkodzony symlink. Być może zapomniałeś o ponownym zamontowaniu lub podłączeniu nośnika przechowującego, do którego on wskazuje.", "backup_system_part_failed": "Nie udało się wykonać kopii zapasowej części systemu ‘{part}’", "config_validate_color": "Powinien być poprawnym szesnastkowym kodem koloru RGB.", From c4b3068d3abf3c4a1518dbd0d4df004441629673 Mon Sep 17 00:00:00 2001 From: Neko Nekowazarashi Date: Tue, 8 Aug 2023 15:12:19 +0000 Subject: [PATCH 62/82] Translated using Weblate (Indonesian) Currently translated at 52.9% (413 of 780 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/id/ --- locales/id.json | 51 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/locales/id.json b/locales/id.json index c6b023102..2cc7c56dc 100644 --- a/locales/id.json +++ b/locales/id.json @@ -391,5 +391,54 @@ "log_letsencrypt_cert_renew": "Memperbarui sertifikat Let's Encrypt '{}'", "log_selfsigned_cert_install": "Memasang sertifikat ditandai sendiri pada domain '{}'", "log_user_permission_reset": "Mengatur ulang izin '{}'", - "domain_config_xmpp": "Pesan Langsung (XMPP)" + "domain_config_xmpp": "Pesan Langsung (XMPP)", + "diagnosis_http_connection_error": "Masalah jaringan: tidak dapat terhubung dengan domain yang diminta, sangat mungkin terputus.", + "dyndns_ip_updated": "IP Anda diperbarui di DynDNS", + "ask_dyndns_recovery_password_explain": "Pilih kata sandi pemulihan untuk domain DynDNS Anda.", + "ask_dyndns_recovery_password": "Kata sandi pemulihan DynDNS", + "backup_output_directory_not_empty": "Anda harus memilih direktori yang kosong", + "service_reload_or_restart_failed": "Tidak dapat memuat atau memulai ulang layanan '{service}'\n\nLog layanan baru-baru ini:{logs}", + "service_reload_failed": "Tidak dapat memuat ulang layanan '{service}'\n\nLog layanan baru-baru ini:{logs}", + "service_start_failed": "Tidak dapat memulai layanan '{service}'\n\nLog layanan baru-baru ini: {logs}", + "diagnosis_apps_deprecated_practices": "Versi aplikasi yang dipasang ini masih menggunakan praktik pengemasan yang lama. Anda lebih baik untuk memperbarui aplikasi tersebut.", + "diagnosis_dns_bad_conf": "Beberapa rekaman DNS untuk domain {domain} ada yang tidak ada atau salah (kategori {category})", + "diagnosis_dns_good_conf": "Rekaman DNS untuk domain {domain} sudah diatur dengan benar (kategori {category})", + "dyndns_unavailable": "Domain '{domain}' tidak tersedia.", + "dyndns_set_recovery_password_denied": "Tidak dapat menyetel kata sandi pemulihan: tidak valid", + "dyndns_set_recovery_password_unknown_domain": "Tidak dapat menyetel kata sandi pemulihan: domain belum terdaftar", + "dyndns_set_recovery_password_invalid_password": "Tidak dapat menyetel kata sandi pemulihan: kata sandi tidak cukup kuat", + "dyndns_set_recovery_password_failed": "Tidak dapat menyetel kata sandi pemulihan: {error}", + "dyndns_set_recovery_password_success": "Kata sandi pemulihan berhasil disetel!", + "file_does_not_exist": "Berkas {path} tidak ada.", + "firewall_reload_failed": "Tidak dapat memuat ulang tembok api", + "firewall_reloaded": "Tembok api dimuat ulang", + "migration_description_0023_postgresql_11_to_13": "Migrasi basis data dari PostgreSQL 11 ke 13", + "service_enabled": "Layanan '{service}' akan secara mandiri dimulai saat pemulaian.", + "service_reloaded_or_restarted": "Layanan {service} dimuat atau dimulai ulang", + "service_stopped": "Layanan '{service}' diberhentikan", + "service_unknown": "Layanan yang tidak diketahui: '{service}'", + "updating_apt_cache": "Mengambil pembaruan yang tersedia untuk paket sistem...", + "group_mailalias_remove": "Alias surel '{mail}' akan dihapus dari kelompok '{group}'", + "migration_description_0021_migrate_to_bullseye": "Peningkatan sistem ke Debian Bullseye dan YunoHost 11.x", + "migration_description_0024_rebuild_python_venv": "Memperbaiki aplikasi Python setelah migrasi Bullseye", + "service_disable_failed": "Tidak dapat membuat layanan '{service}' dimulai saat pemulaian.\n\nLog layanan baru-baru ini:{logs}", + "service_disabled": "Layanan '{service}' tidak akan dimulai kembali saat pemulaian.", + "tools_upgrade_failed": "Tidak dapat memperbarui paket: {packages_list}", + "global_settings_setting_nginx_redirect_to_https": "Paksa HTTPS", + "backup_archive_system_part_not_available": "Segmen '{part}' tidak tersedia di cadangan ini", + "backup_output_directory_forbidden": "Pilih direktori yang berbeda. Cadangan tidak dapat dibuat di /bin, /boot, /dev, /etc, /lib, /root, /run, /sbin, /sys, /usr, /var, atau subfolder dari /home/yunohost.backup/archives", + "ask_dyndns_recovery_password_explain_during_unsubscribe": "Masukkan kata sandi pemulihan untuk domain DynDNS ini.", + "backup_output_symlink_dir_broken": "Direktori arsip Anda '{path}' rusak penautannya. Mungkin Anda lupa untuk menambatkan ulang atau memasukkan kembali penyimpanan tujuan penautan direktori arsip tersebut.", + "diagnosis_apps_not_in_app_catalog": "Aplikasi ini tidak ada di katalog aplikasi YunoHost. Jika aplikasi ini ada di sana sebelumnya dan dihapus, Anda disarankan untuk melepas aplikasi ini dikarenakan ini tidak akan menerima pembaruan dan mungkin bisa menghancurkan integritas dan keamanan sistem Anda.", + "dyndns_ip_update_failed": "Tidak dapat memperbarui IP Anda di DynDNS", + "service_restarted": "Layanan {service} dimulai ulang", + "service_started": "Layanan '{service}' dimulai", + "service_stop_failed": "Tidak dapat menghentikan layanan '{service}'\n\nLog layanan baru-baru ini: {logs}", + "apps_catalog_failed_to_download": "Tidak dapat mengunduh katalog aplikasi {apps_catalog}: {error}", + "backup_archive_corrupted": "Sepertinya arsip cadangan '{archive}' rusak: {error}", + "diagnosis_found_errors": "{errors} masalah signifikan ditemukan terkait dengan {category}!", + "restore_system_part_failed": "Tidak dapat memulihkan segmen '{part}'", + "service_enable_failed": "Tidak dapat membuat layanan '{service}' dimulai mandiri saat pemulaian.\n\nLog layanan baru-baru ini:{logs}", + "service_not_reloading_because_conf_broken": "Tidak memuat atau memulai ulang layanan '{name}' karena konfigurasinya rusak: {errors}", + "service_reloaded": "Layanan {service} dimuat ulang" } From 576992899cdb4785980dd5916295953593923879 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 16 Aug 2023 19:18:37 +0200 Subject: [PATCH 63/82] apps: allow to use jinja {% if foobar %} blocks in their notifications/doc pages --- src/app.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/src/app.py b/src/app.py index 7f4acdc1a..0b1be18a6 100644 --- a/src/app.py +++ b/src/app.py @@ -186,11 +186,17 @@ def app_info(app, full=False, upgradable=False): ret["from_catalog"] = from_catalog # Hydrate app notifications and doc + rendered_doc = {} for pagename, content_per_lang in ret["manifest"]["doc"].items(): for lang, content in content_per_lang.items(): - ret["manifest"]["doc"][pagename][lang] = _hydrate_app_template( - content, settings - ) + rendered_content = _hydrate_app_template(content, settings) + # Rendered content may be empty because of conditional blocks + if not rendered_content: + continue + if pagename not in rendered_doc: + rendered_doc[pagename] = {} + rendered_doc[pagename][lang] = rendered_content + ret["manifest"]["doc"] = rendered_doc # Filter dismissed notification ret["manifest"]["notifications"] = { @@ -201,9 +207,16 @@ def app_info(app, full=False, upgradable=False): # Hydrate notifications (also filter uneeded post_upgrade notification based on version) for step, notifications in ret["manifest"]["notifications"].items(): + rendered_notifications = {} for name, content_per_lang in notifications.items(): for lang, content in content_per_lang.items(): - notifications[name][lang] = _hydrate_app_template(content, settings) + rendered_content = _hydrate_app_template(content, settings) + if not rendered_content: + continue + if name not in rendered_notifications: + rendered_notifications[name] = {} + rendered_notifications[name][lang] = rendered_content + ret["manifest"]["notifications"][step] = rendered_notifications ret["is_webapp"] = "domain" in settings and "path" in settings @@ -2230,6 +2243,13 @@ def _parse_app_doc_and_notifications(path): def _hydrate_app_template(template, data): + + # Apply jinja for stuff like {% if .. %} blocks, + # but only if there's indeed an if block (to try to reduce overhead or idk) + if "{%" in template: + from jinja2 import Template + template = Template(template).render(**data) + stuff_to_replace = set(re.findall(r"__[A-Z0-9]+?[A-Z0-9_]*?[A-Z0-9]*?__", template)) for stuff in stuff_to_replace: @@ -2238,7 +2258,7 @@ def _hydrate_app_template(template, data): if varname in data: template = template.replace(stuff, str(data[varname])) - return template + return template.strip() def _convert_v1_manifest_to_v2(manifest): @@ -3145,7 +3165,7 @@ def _filter_and_hydrate_notifications(notifications, current_version=None, data= else: return version.parse(name) > version.parse(current_version.split("~")[0]) - return { + out = { # Should we render the markdown maybe? idk name: _hydrate_app_template(_value_for_locale(content_per_lang), data) for name, content_per_lang in notifications.items() @@ -3154,6 +3174,9 @@ def _filter_and_hydrate_notifications(notifications, current_version=None, data= or is_version_more_recent_than_current_version(name, current_version) } + # Filter out empty notifications (notifications may be empty because of if blocks) + return {name:content for name, content in out.items() if content and content.strip()} + def _display_notifications(notifications, force=False): if not notifications: From 32376cf18ffce2cffb54e4adf30856cab39e842c Mon Sep 17 00:00:00 2001 From: Tagada <36127788+Tagadda@users.noreply.github.com> Date: Mon, 21 Aug 2023 16:39:54 +0200 Subject: [PATCH 64/82] Use the existing db_name setting for database provising This should help upgrading an app from packaging v1 when db_name was different from `app` --- src/utils/resources.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index 60a5f44f6..ba02930c9 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -1337,8 +1337,8 @@ class DatabaseAppResource(AppResource): def provision_or_update(self, context: Dict = {}): # This is equivalent to ynh_sanitize_dbid - db_name = self.app.replace("-", "_").replace(".", "_") - db_user = db_name + db_user = self.app.replace("-", "_").replace(".", "_") + db_name = self.get_setting("db_name") || db_user self.set_setting("db_name", db_name) self.set_setting("db_user", db_user) @@ -1372,8 +1372,8 @@ class DatabaseAppResource(AppResource): ) def deprovision(self, context: Dict = {}): - db_name = self.app.replace("-", "_").replace(".", "_") - db_user = db_name + db_user = self.app.replace("-", "_").replace(".", "_") + db_name = self.get_setting("db_name") || db_user if self.dbtype == "mysql": self._run_script( From 07daa68770df82f92c1539228d5d003b9570fa29 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 22 Aug 2023 15:37:47 +0200 Subject: [PATCH 65/82] apps: BACKUP_CORE_ONLY was not set for pre-upgrade safety backups, resulting in unecessarily large pre-upgrade backups --- src/app.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/app.py b/src/app.py index 0b1be18a6..94485b176 100644 --- a/src/app.py +++ b/src/app.py @@ -707,9 +707,17 @@ def app_upgrade( safety_backup_name = f"{app_instance_name}-pre-upgrade2" other_safety_backup_name = f"{app_instance_name}-pre-upgrade1" - backup_create( - name=safety_backup_name, apps=[app_instance_name], system=None - ) + tweaked_backup_core_only = False + if "BACKUP_CORE_ONLY" not in os.environ: + tweaked_backup_core_only = True + os.environ["BACKUP_CORE_ONLY"] = "1" + try: + backup_create( + name=safety_backup_name, apps=[app_instance_name], system=None + ) + finally: + if tweaked_backup_core_only: + del os.environ["BACKUP_CORE_ONLY"] if safety_backup_name in backup_list()["archives"]: # if the backup suceeded, delete old safety backup to save space From ee4d94d3829192029e8ece8d87529a005280eea3 Mon Sep 17 00:00:00 2001 From: Tagada <36127788+Tagadda@users.noreply.github.com> Date: Tue, 22 Aug 2023 17:11:56 +0200 Subject: [PATCH 66/82] Update src/utils/resources.py Co-authored-by: Alexandre Aubin --- src/utils/resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index ba02930c9..fd476d9bd 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -1338,7 +1338,7 @@ class DatabaseAppResource(AppResource): def provision_or_update(self, context: Dict = {}): # This is equivalent to ynh_sanitize_dbid db_user = self.app.replace("-", "_").replace(".", "_") - db_name = self.get_setting("db_name") || db_user + db_name = self.get_setting("db_name") or db_user self.set_setting("db_name", db_name) self.set_setting("db_user", db_user) From 73a144fa4623f36f6f4b105801509cd5d183fec1 Mon Sep 17 00:00:00 2001 From: Tagada <36127788+Tagadda@users.noreply.github.com> Date: Tue, 22 Aug 2023 17:12:03 +0200 Subject: [PATCH 67/82] Update src/utils/resources.py Co-authored-by: Alexandre Aubin --- src/utils/resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index fd476d9bd..69b260334 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -1373,7 +1373,7 @@ class DatabaseAppResource(AppResource): def deprovision(self, context: Dict = {}): db_user = self.app.replace("-", "_").replace(".", "_") - db_name = self.get_setting("db_name") || db_user + db_name = self.get_setting("db_name") or db_user if self.dbtype == "mysql": self._run_script( From b0fe49ae8390299d022c84b5c824057fc28f7c31 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 22 Aug 2023 17:36:12 +0200 Subject: [PATCH 68/82] configpanels/forms : more edge cases with some questions not implementing some methods/attributes --- src/utils/configpanel.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index ee9019303..12024855a 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -160,11 +160,15 @@ class ConfigPanel: result[key] = {"ask": ask} if "current_value" in option: question_class = OPTIONS[option.get("type", OptionType.string)] - result[key]["value"] = question_class.humanize( - option["current_value"], option - ) + if hasattr(question_class, "humanize"): + result[key]["value"] = question_class.humanize( + option["current_value"], option + ) + else: + result[key]["value"] = option["current_value"] + # FIXME: semantics, technically here this is not about a prompt... - if question_class.hide_user_input_in_prompt: + if getattr(question_class, "hide_user_input_in_prompt", None): result[key][ "value" ] = "**************" # Prevent displaying password in `config get` From 0b0514374502bfa3093f71de7777a4a72598e43b Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Fri, 25 Aug 2023 03:23:33 +0200 Subject: [PATCH 69/82] [fix] Diagnosis: reverse DNS check should be case-insensitive #2235 --- src/diagnosers/24-mail.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/diagnosers/24-mail.py b/src/diagnosers/24-mail.py index df14222a5..c7fe9d04b 100644 --- a/src/diagnosers/24-mail.py +++ b/src/diagnosers/24-mail.py @@ -43,7 +43,7 @@ class MyDiagnoser(Diagnoser): dependencies: List[str] = ["ip"] def run(self): - self.ehlo_domain = _get_maindomain() + self.ehlo_domain = _get_maindomain().lower() self.mail_domains = domain_list()["domains"] self.ipversions, self.ips = self.get_ips_checked() @@ -132,7 +132,7 @@ class MyDiagnoser(Diagnoser): summary=summary, details=[summary + "_details"], ) - elif r["helo"] != self.ehlo_domain: + elif r["helo"].lower() != self.ehlo_domain: yield dict( meta={"test": "mail_ehlo", "ipversion": ipversion}, data={"wrong_ehlo": r["helo"], "right_ehlo": self.ehlo_domain}, @@ -185,7 +185,7 @@ class MyDiagnoser(Diagnoser): rdns_domain = "" if len(value) > 0: rdns_domain = value[0][:-1] if value[0].endswith(".") else value[0] - if rdns_domain != self.ehlo_domain: + if rdns_domain.lower() != self.ehlo_domain: details = [ "diagnosis_mail_fcrdns_different_from_ehlo_domain_details" ] + details @@ -194,7 +194,7 @@ class MyDiagnoser(Diagnoser): data={ "ip": ip, "ehlo_domain": self.ehlo_domain, - "rdns_domain": rdns_domain, + "rdns_domain": rdns_domain.lower(), }, status="ERROR", summary="diagnosis_mail_fcrdns_different_from_ehlo_domain", From 65d25710725b06d281630644b80d8d01dfba1bde Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 31 Aug 2023 17:23:21 +0200 Subject: [PATCH 70/82] helpers: add new --group option for ynh_add_fpm_config to customize the Group parameter --- helpers/php | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/helpers/php b/helpers/php index c9e5b1cb8..ee3d35cb1 100644 --- a/helpers/php +++ b/helpers/php @@ -70,8 +70,9 @@ YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} ynh_add_fpm_config() { local _globalphpversion=${phpversion-:} # Declare an array to define the options of this helper. - local legacy_args=vufpd - local -A args_array=([v]=phpversion= [u]=usage= [f]=footprint= [p]=package= [d]=dedicated_service) + local legacy_args=vufpdg + local -A args_array=([v]=phpversion= [u]=usage= [f]=footprint= [p]=package= [d]=dedicated_service [g]=group=) + local group local phpversion local usage local footprint @@ -80,6 +81,7 @@ ynh_add_fpm_config() { # Manage arguments with getopts ynh_handle_getopts_args "$@" package=${package:-} + group=${group:-} # The default behaviour is to use the template. local autogenconf=false @@ -180,12 +182,13 @@ ynh_add_fpm_config() { # Define the values to use for the configuration of PHP. ynh_get_scalable_phpfpm --usage=$usage --footprint=$footprint + local phpfpm_group=$([[ -n "$group" ]] && echo "$group" || echo "$app") local phpfpm_path="$YNH_APP_BASEDIR/conf/php-fpm.conf" echo " [__APP__] user = __APP__ -group = __APP__ +group = __PHPFPM_GROUP__ chdir = __INSTALL_DIR__ From 51d8608b40e2e2acd76a0ce34ceca9dab0cf13d0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 31 Aug 2023 17:33:08 +0200 Subject: [PATCH 71/82] Update changelog for 11.2.4 --- debian/changelog | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/debian/changelog b/debian/changelog index 586f8387b..610109fcd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,18 @@ +yunohost (11.2.4) stable; urgency=low + + - doc: Improve --help for 'yunohost app install' ([#1702](https://github.com/yunohost/yunohost/pull/1702)) + - helpers: add new --group option for ynh_add_fpm_config to customize the Group parameter (65d25710) + - apps: allow to use jinja {% if foobar %} blocks in their notifications/doc pages (57699289) + - apps: BACKUP_CORE_ONLY was not set for pre-upgrade safety backups, resulting in unecessarily large pre-upgrade backups (07daa687) + - apps: Use the existing db_name setting for database provising to ease v1->v2 transition with specific db_name ([#1704](https://github.com/yunohost/yunohost/pull/1704)) + - configpanels/forms: more edge cases with some questions not implementing some methods/attributes (b0fe49ae) + - diagnosis: reverse DNS check should be case-insensitive #2235 ([#1705](https://github.com/yunohost/yunohost/pull/1705)) + - i18n: Translations updated for Galician, Indonesian, Polish, Spanish, Turkish + + Thanks to all contributors <3 ! (Grzegorz Cichocki, José M, Kuba Bazan, ljf (zamentur), massyas, Neko Nekowazarashi, selfhoster1312, Suleyman Harmandar, taco, Tagada) + + -- Alexandre Aubin Thu, 31 Aug 2023 17:30:21 +0200 + yunohost (11.2.3) stable; urgency=low - apps: fix another case of no attribute 'value' due to config panels/questions refactoring (4fda8ed49) From 3dfab89c1f4c94115110b17b9a98efdfb0590a55 Mon Sep 17 00:00:00 2001 From: Kayou Date: Thu, 7 Sep 2023 14:59:08 +0200 Subject: [PATCH 72/82] check and re-download a prefetched file that doesn't match the checksum --- helpers/utils | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/helpers/utils b/helpers/utils index 52d7c734f..4694dd724 100644 --- a/helpers/utils +++ b/helpers/utils @@ -244,9 +244,11 @@ ynh_setup_source() { if [ "$src_format" = "docker" ]; then src_platform="${src_platform:-"linux/$YNH_ARCH"}" - elif test -e "$local_src"; then - cp $local_src $src_filename else + if test -e "$local_src"; then + cp $local_src $src_filename + fi + [ -n "$src_url" ] || ynh_die "Couldn't parse SOURCE_URL from $src_file_path ?" # If the file was prefetched but somehow doesn't match the sum, rm and redownload it From 79e41a1e4b9deed01181e8788bb8bec4493e2a32 Mon Sep 17 00:00:00 2001 From: Pierre de La Morinerie Date: Thu, 7 Sep 2023 16:22:16 +0200 Subject: [PATCH 73/82] app.py: fix typo in log statement --- src/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.py b/src/app.py index 94485b176..627909b70 100644 --- a/src/app.py +++ b/src/app.py @@ -809,7 +809,7 @@ def app_upgrade( and not no_safety_backup ): logger.warning( - "Upgrade failed ... attempting to restore the satefy backup (Yunohost first need to remove the app for this) ..." + "Upgrade failed ... attempting to restore the safety backup (Yunohost first need to remove the app for this) ..." ) app_remove(app_instance_name, force_workdir=extracted_app_folder) From 920fe527f4ef1ca5ffef72bc247497ac09bf5404 Mon Sep 17 00:00:00 2001 From: orhtej2 <2871798+orhtej2@users.noreply.github.com> Date: Thu, 7 Sep 2023 22:14:06 +0200 Subject: [PATCH 74/82] Allow system users to send mails from IPv6 localhost and in no-IP contexts. --- src/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.py b/src/app.py index 627909b70..df5eecae2 100644 --- a/src/app.py +++ b/src/app.py @@ -3261,7 +3261,7 @@ def regen_mail_app_user_config_for_dovecot_and_postfix(only=None): if dovecot: hashed_password = _hash_user_password(settings["mail_pwd"]) dovecot_passwd.append( - f"{app}:{hashed_password}::::::allow_nets=127.0.0.1/24" + f"{app}:{hashed_password}::::::allow_nets=::1,127.0.0.1/24,local" ) if postfix: mail_user = settings.get("mail_user", app) From 8eb2e72282ea6bcb2aab3901ffc412d42b273e4a Mon Sep 17 00:00:00 2001 From: Kay0u Date: Fri, 8 Sep 2023 15:13:20 +0200 Subject: [PATCH 75/82] Update Fail2ban jail.conf file from https://sources.debian.org/src/fail2ban/1.0.2-2/config/jail.conf/ --- conf/fail2ban/jail.conf | 301 ++++++++++++++++++++++++++++------------ 1 file changed, 213 insertions(+), 88 deletions(-) diff --git a/conf/fail2ban/jail.conf b/conf/fail2ban/jail.conf index bd522c4ba..fe8db527d 100644 --- a/conf/fail2ban/jail.conf +++ b/conf/fail2ban/jail.conf @@ -18,7 +18,7 @@ # See man 5 jail.conf for details. # # [DEFAULT] -# bantime = 3600 +# bantime = 1h # # [sshd] # enabled = true @@ -44,10 +44,52 @@ before = paths-debian.conf # MISCELLANEOUS OPTIONS # -# "ignoreip" can be an IP address, a CIDR mask or a DNS host. Fail2ban will not -# ban a host which matches an address in this list. Several addresses can be -# defined using space (and/or comma) separator. -ignoreip = 127.0.0.1/8 +# "bantime.increment" allows to use database for searching of previously banned ip's to increase a +# default ban time using special formula, default it is banTime * 1, 2, 4, 8, 16, 32... +#bantime.increment = true + +# "bantime.rndtime" is the max number of seconds using for mixing with random time +# to prevent "clever" botnets calculate exact time IP can be unbanned again: +#bantime.rndtime = + +# "bantime.maxtime" is the max number of seconds using the ban time can reach (doesn't grow further) +#bantime.maxtime = + +# "bantime.factor" is a coefficient to calculate exponent growing of the formula or common multiplier, +# default value of factor is 1 and with default value of formula, the ban time +# grows by 1, 2, 4, 8, 16 ... +#bantime.factor = 1 + +# "bantime.formula" used by default to calculate next value of ban time, default value below, +# the same ban time growing will be reached by multipliers 1, 2, 4, 8, 16, 32... +#bantime.formula = ban.Time * (1<<(ban.Count if ban.Count<20 else 20)) * banFactor +# +# more aggressive example of formula has the same values only for factor "2.0 / 2.885385" : +#bantime.formula = ban.Time * math.exp(float(ban.Count+1)*banFactor)/math.exp(1*banFactor) + +# "bantime.multipliers" used to calculate next value of ban time instead of formula, corresponding +# previously ban count and given "bantime.factor" (for multipliers default is 1); +# following example grows ban time by 1, 2, 4, 8, 16 ... and if last ban count greater as multipliers count, +# always used last multiplier (64 in example), for factor '1' and original ban time 600 - 10.6 hours +#bantime.multipliers = 1 2 4 8 16 32 64 +# following example can be used for small initial ban time (bantime=60) - it grows more aggressive at begin, +# for bantime=60 the multipliers are minutes and equal: 1 min, 5 min, 30 min, 1 hour, 5 hour, 12 hour, 1 day, 2 day +#bantime.multipliers = 1 5 30 60 300 720 1440 2880 + +# "bantime.overalljails" (if true) specifies the search of IP in the database will be executed +# cross over all jails, if false (default), only current jail of the ban IP will be searched +#bantime.overalljails = false + +# -------------------- + +# "ignoreself" specifies whether the local resp. own IP addresses should be ignored +# (default is true). Fail2ban will not ban a host which matches such addresses. +#ignoreself = true + +# "ignoreip" can be a list of IP addresses, CIDR masks or DNS hosts. Fail2ban +# will not ban a host which matches an address in this list. Several addresses +# can be defined using space (and/or comma) separator. +#ignoreip = 127.0.0.1/8 ::1 # External command that will take an tagged arguments to ignore, e.g. , # and return true if the IP is to be ignored. False otherwise. @@ -56,14 +98,17 @@ ignoreip = 127.0.0.1/8 ignorecommand = # "bantime" is the number of seconds that a host is banned. -bantime = 600 +bantime = 10m # A host is banned if it has generated "maxretry" during the last "findtime" # seconds. -findtime = 600 +findtime = 10m # "maxretry" is the number of failures before a host get banned. -maxretry = 10 +maxretry = 5 + +# "maxmatches" is the number of matches stored in ticket (resolvable via tag in actions). +maxmatches = %(maxretry)s # "backend" specifies the backend used to get files modification. # Available options are "pyinotify", "gamin", "polling", "systemd" and "auto". @@ -113,10 +158,13 @@ logencoding = auto enabled = false +# "mode" defines the mode of the filter (see corresponding filter implementation for more info). +mode = normal + # "filter" defines the filter to use by the jail. # By default jails have names matching their filter name # -filter = %(__name__)s +filter = %(__name__)s[mode=%(mode)s] # @@ -130,7 +178,7 @@ filter = %(__name__)s destemail = root@localhost # Sender email address used solely for some actions -sender = root@localhost +sender = root@ # E-mail action. Since 0.8.1 Fail2Ban uses sendmail MTA for the # mailing. Change mta configuration parameter to mail if you want to @@ -140,8 +188,8 @@ mta = sendmail # Default protocol protocol = tcp -# Specify chain where jumps would need to be added in iptables-* actions -chain = INPUT +# Specify chain where jumps would need to be added in ban-actions expecting parameter chain +chain = # Ports to be banned # Usually should be overridden in a particular jail @@ -161,51 +209,53 @@ banaction = iptables-multiport banaction_allports = iptables-allports # The simplest action to take: ban only -action_ = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] +action_ = %(banaction)s[port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report to the destemail. -action_mw = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - %(mta)s-whois[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] +action_mw = %(action_)s + %(mta)s-whois[sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report and relevant log lines # to the destemail. -action_mwl = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] +action_mwl = %(action_)s + %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] # See the IMPORTANT note in action.d/xarf-login-attack for when to use this action # # ban & send a xarf e-mail to abuse contact of IP address and include relevant log lines # to the destemail. -action_xarf = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - xarf-login-attack[service=%(__name__)s, sender="%(sender)s", logpath=%(logpath)s, port="%(port)s"] +action_xarf = %(action_)s + xarf-login-attack[service=%(__name__)s, sender="%(sender)s", logpath="%(logpath)s", port="%(port)s"] + +# ban & send a notification to one or more of the 50+ services supported by Apprise. +# See https://github.com/caronc/apprise/wiki for details on what is supported. +# +# You may optionally over-ride the default configuration line (containing the Apprise URLs) +# by using 'apprise[config="/alternate/path/to/apprise.cfg"]' otherwise +# /etc/fail2ban/apprise.conf is sourced for your supported notification configuration. +# action = %(action_)s +# apprise # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines # to the destemail. action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] - %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] + %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] # Report block via blocklist.de fail2ban reporting service API # -# See the IMPORTANT note in action.d/blocklist_de.conf for when to -# use this action. Create a file jail.d/blocklist_de.local containing -# [Init] -# blocklist_de_apikey = {api key from registration] +# See the IMPORTANT note in action.d/blocklist_de.conf for when to use this action. +# Specify expected parameters in file action.d/blocklist_de.local or if the interpolation +# `action_blocklist_de` used for the action, set value of `blocklist_de_apikey` +# in your `jail.local` globally (section [DEFAULT]) or per specific jail section (resp. in +# corresponding jail.d/my-jail.local file). # -action_blocklist_de = blocklist_de[email="%(sender)s", service=%(filter)s, apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"] +action_blocklist_de = blocklist_de[email="%(sender)s", service="%(__name__)s", apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"] -# Report ban via badips.com, and use as blacklist +# Report ban via abuseipdb.com. # -# See BadIPsAction docstring in config/action.d/badips.py for -# documentation for this action. +# See action.d/abuseipdb.conf for usage example and details. # -# NOTE: This action relies on banaction being present on start and therefore -# should be last action defined for a jail. -# -action_badips = badips.py[category="%(__name__)s", banaction="%(banaction)s", agent="%(fail2ban_agent)s"] -# -# Report ban via badips.com (uses action.d/badips.conf for reporting only) -# -action_badips_report = badips[category="%(__name__)s", agent="%(fail2ban_agent)s"] +action_abuseipdb = abuseipdb # Choose default action. To change, just override value of 'action' with the # interpolation to the chosen action shortcut (e.g. action_mw, action_mwl, etc) in jail.local @@ -223,15 +273,10 @@ action = %(action_)s [sshd] -port = ssh -logpath = %(sshd_log)s -backend = %(sshd_backend)s - - -[sshd-ddos] -# This jail corresponds to the standard configuration in Fail2ban. -# The mail-whois action send a notification e-mail with a whois request -# in the body. +# To use more aggressive sshd modes set filter parameter "mode" in jail.local: +# normal (default), ddos, extra or aggressive (combines all). +# See "tests/files/logs/sshd" or "filter.d/sshd.conf" for usage example and details. +#mode = normal port = ssh logpath = %(sshd_log)s backend = %(sshd_backend)s @@ -265,7 +310,7 @@ logpath = %(apache_error_log)s # for email addresses. The mail outputs are buffered. port = http,https logpath = %(apache_access_log)s -bantime = 172800 +bantime = 48h maxretry = 1 @@ -301,7 +346,7 @@ maxretry = 2 port = http,https logpath = %(apache_access_log)s maxretry = 1 -ignorecommand = %(ignorecommands_dir)s/apache-fakegooglebot +ignorecommand = %(fail2ban_confpath)s/filter.d/ignorecommands/apache-fakegooglebot [apache-modsecurity] @@ -321,12 +366,15 @@ maxretry = 1 [openhab-auth] filter = openhab -action = iptables-allports[name=NoAuthFailures] +banaction = %(banaction_allports)s logpath = /opt/openhab/logs/request.log +# To use more aggressive http-auth modes set filter parameter "mode" in jail.local: +# normal (default), aggressive (combines all), auth or fallback +# See "tests/files/logs/nginx-http-auth" or "filter.d/nginx-http-auth.conf" for usage example and details. [nginx-http-auth] - +# mode = normal port = http,https logpath = %(nginx_error_log)s @@ -342,8 +390,10 @@ logpath = %(nginx_error_log)s port = http,https logpath = %(nginx_error_log)s -maxretry = 2 +[nginx-bad-request] +port = http,https +logpath = %(nginx_access_log)s # Ban attackers that try to use PHP's URL-fopen() functionality # through GET/POST variables. - Experimental, with more than a year @@ -377,6 +427,8 @@ logpath = %(lighttpd_error_log)s port = http,https logpath = %(roundcube_errors_log)s +# Use following line in your jail.local if roundcube logs to journal. +#backend = %(syslog_backend)s [openwebmail] @@ -426,11 +478,13 @@ backend = %(syslog_backend)s port = http,https logpath = /var/log/tomcat*/catalina.out +#logpath = /var/log/guacamole.log [monit] #Ban clients brute-forcing the monit gui login port = 2812 logpath = /var/log/monit + /var/log/monit.log [webmin-auth] @@ -513,27 +567,29 @@ logpath = %(vsftpd_log)s # ASSP SMTP Proxy Jail [assp] -port = smtp,submission +port = smtp,465,submission logpath = /root/path/to/assp/logs/maillog.txt [courier-smtp] -port = smtp,submission +port = smtp,465,submission logpath = %(syslog_mail)s backend = %(syslog_backend)s [postfix] - -port = smtp,submission -logpath = %(postfix_log)s -backend = %(postfix_backend)s +# To use another modes set filter parameter "mode" in jail.local: +mode = more +port = smtp,465,submission +logpath = %(postfix_log)s +backend = %(postfix_backend)s [postfix-rbl] -port = smtp,submission +filter = postfix[mode=rbl] +port = smtp,465,submission logpath = %(postfix_log)s backend = %(postfix_backend)s maxretry = 1 @@ -541,14 +597,17 @@ maxretry = 1 [sendmail-auth] -port = submission,smtp +port = submission,465,smtp logpath = %(syslog_mail)s backend = %(syslog_backend)s [sendmail-reject] - -port = smtp,submission +# To use more aggressive modes set filter parameter "mode" in jail.local: +# normal (default), extra or aggressive +# See "tests/files/logs/sendmail-reject" or "filter.d/sendmail-reject.conf" for usage example and details. +#mode = normal +port = smtp,465,submission logpath = %(syslog_mail)s backend = %(syslog_backend)s @@ -556,7 +615,7 @@ backend = %(syslog_backend)s [qmail-rbl] filter = qmail -port = smtp,submission +port = smtp,465,submission logpath = /service/qmail/log/main/current @@ -564,14 +623,14 @@ logpath = /service/qmail/log/main/current # but can be set by syslog_facility in the dovecot configuration. [dovecot] -port = pop3,pop3s,imap,imaps,submission,sieve +port = pop3,pop3s,imap,imaps,submission,465,sieve logpath = %(dovecot_log)s backend = %(dovecot_backend)s [sieve] -port = smtp,submission +port = smtp,465,submission logpath = %(dovecot_log)s backend = %(dovecot_backend)s @@ -583,20 +642,21 @@ logpath = %(solidpop3d_log)s [exim] - -port = smtp,submission +# see filter.d/exim.conf for further modes supported from filter: +#mode = normal +port = smtp,465,submission logpath = %(exim_main_log)s [exim-spam] -port = smtp,submission +port = smtp,465,submission logpath = %(exim_main_log)s [kerio] -port = imap,smtp,imaps +port = imap,smtp,imaps,465 logpath = /opt/kerio/mailserver/store/logs/security.log @@ -607,14 +667,15 @@ logpath = /opt/kerio/mailserver/store/logs/security.log [courier-auth] -port = smtp,submission,imaps,pop3,pop3s +port = smtp,465,submission,imap,imaps,pop3,pop3s logpath = %(syslog_mail)s backend = %(syslog_backend)s [postfix-sasl] -port = smtp,submission,imap,imaps,pop3,pop3s +filter = postfix[mode=auth] +port = smtp,465,submission,imap,imaps,pop3,pop3s # You might consider monitoring /var/log/mail.warn instead if you are # running postfix since it would provide the same log lines at the # "warn" level but overall at the smaller filesize. @@ -631,7 +692,7 @@ backend = %(syslog_backend)s [squirrelmail] -port = smtp,submission,imap,imap2,imaps,pop3,pop3s,http,https,socks +port = smtp,465,submission,imap,imap2,imaps,pop3,pop3s,http,https,socks logpath = /var/lib/squirrelmail/prefs/squirrelmail_access_log @@ -684,8 +745,8 @@ logpath = /var/log/named/security.log [nsd] port = 53 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/nsd.log @@ -696,9 +757,8 @@ logpath = /var/log/nsd.log [asterisk] port = 5060,5061 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/asterisk/messages maxretry = 10 @@ -706,16 +766,22 @@ maxretry = 10 [freeswitch] port = 5060,5061 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/freeswitch.log maxretry = 10 +# enable adminlog; it will log to a file inside znc's directory by default. +[znc-adminlog] + +port = 6667 +logpath = /var/lib/znc/moddata/adminlog/znc.log + + # To log wrong MySQL access attempts add to /etc/my.cnf in [mysqld] or # equivalent section: -# log-warning = 2 +# log-warnings = 2 # # for syslog (daemon facility) # [mysqld_safe] @@ -731,6 +797,14 @@ logpath = %(mysql_log)s backend = %(mysql_backend)s +[mssql-auth] +# Default configuration for Microsoft SQL Server for Linux +# See the 'mssql-conf' manpage how to change logpath or port +logpath = /var/opt/mssql/log/errorlog +port = 1433 +filter = mssql-auth + + # Log wrong MongoDB auth (for details see filter 'filter.d/mongodb-auth.conf') [mongodb-auth] # change port when running with "--shardsvr" or "--configsvr" runtime operation @@ -749,8 +823,8 @@ logpath = /var/log/mongodb/mongodb.log logpath = /var/log/fail2ban.log banaction = %(banaction_allports)s -bantime = 604800 ; 1 week -findtime = 86400 ; 1 day +bantime = 1w +findtime = 1d # Generic filter for PAM. Has to be used with action which bans all @@ -786,11 +860,31 @@ logpath = /var/log/ejabberd/ejabberd.log [counter-strike] logpath = /opt/cstrike/logs/L[0-9]*.log -# Firewall: http://www.cstrike-planet.com/faq/6 tcpport = 27030,27031,27032,27033,27034,27035,27036,27037,27038,27039 udpport = 1200,27000,27001,27002,27003,27004,27005,27006,27007,27008,27009,27010,27011,27012,27013,27014,27015 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp"] + +[softethervpn] +port = 500,4500 +protocol = udp +logpath = /usr/local/vpnserver/security_log/*/sec.log + +[gitlab] +port = http,https +logpath = /var/log/gitlab/gitlab-rails/application.log + +[grafana] +port = http,https +logpath = /var/log/grafana/grafana.log + +[bitwarden] +port = http,https +logpath = /home/*/bwdata/logs/identity/Identity/log.txt + +[centreon] +port = http,https +logpath = /var/log/centreon/login.log # consider low maxretry and a long bantime # nobody except your own Nagios server should ever probe nrpe @@ -824,7 +918,9 @@ filter = apache-pass[knocking_url="%(knocking_url)s"] logpath = %(apache_access_log)s blocktype = RETURN returntype = DROP -bantime = 3600 +action = %(action_)s[blocktype=%(blocktype)s, returntype=%(returntype)s, + actionstart_on_demand=false, actionrepair_on_unban=true] +bantime = 1h maxretry = 1 findtime = 1 @@ -832,8 +928,8 @@ findtime = 1 [murmur] # AKA mumble-server port = 64738 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol=tcp, chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol=udp, chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/mumble-server/mumble-server.log @@ -851,5 +947,34 @@ logpath = /var/log/haproxy.log [slapd] port = ldap,ldaps -filter = slapd logpath = /var/log/slapd.log + +[domino-smtp] +port = smtp,ssmtp +logpath = /home/domino01/data/IBM_TECHNICAL_SUPPORT/console.log + +[phpmyadmin-syslog] +port = http,https +logpath = %(syslog_authpriv)s +backend = %(syslog_backend)s + + +[zoneminder] +# Zoneminder HTTP/HTTPS web interface auth +# Logs auth failures to apache2 error log +port = http,https +logpath = %(apache_error_log)s + +[traefik-auth] +# to use 'traefik-auth' filter you have to configure your Traefik instance, +# see `filter.d/traefik-auth.conf` for details and service example. +port = http,https +logpath = /var/log/traefik/access.log + +[scanlogd] +logpath = %(syslog_local0)s +banaction = %(banaction_allports)s + +[monitorix] +port = 8080 +logpath = /var/log/monitorix-httpd From d0b65d56614a04c44ae917a4872485f39ced2132 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Fri, 8 Sep 2023 15:17:25 +0200 Subject: [PATCH 76/82] revert important variables in fail2ban jail.conf --- conf/fail2ban/jail.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/fail2ban/jail.conf b/conf/fail2ban/jail.conf index fe8db527d..2ffc71c28 100644 --- a/conf/fail2ban/jail.conf +++ b/conf/fail2ban/jail.conf @@ -178,7 +178,7 @@ filter = %(__name__)s[mode=%(mode)s] destemail = root@localhost # Sender email address used solely for some actions -sender = root@ +sender = root@localhost # E-mail action. Since 0.8.1 Fail2Ban uses sendmail MTA for the # mailing. Change mta configuration parameter to mail if you want to @@ -189,7 +189,7 @@ mta = sendmail protocol = tcp # Specify chain where jumps would need to be added in ban-actions expecting parameter chain -chain = +chain = INPUT # Ports to be banned # Usually should be overridden in a particular jail From 2bd3dd2bba93ec414b368d308d7684ca1ca3591b Mon Sep 17 00:00:00 2001 From: Kayou Date: Fri, 8 Sep 2023 22:31:08 +0200 Subject: [PATCH 77/82] set maxretry to 10 --- conf/fail2ban/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/fail2ban/jail.conf b/conf/fail2ban/jail.conf index 2ffc71c28..8b6a2ec6d 100644 --- a/conf/fail2ban/jail.conf +++ b/conf/fail2ban/jail.conf @@ -105,7 +105,7 @@ bantime = 10m findtime = 10m # "maxretry" is the number of failures before a host get banned. -maxretry = 5 +maxretry = 10 # "maxmatches" is the number of matches stored in ticket (resolvable via tag in actions). maxmatches = %(maxretry)s From e77e9a0a9a804241159468b0024fbcf3e6801a40 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Fri, 8 Sep 2023 23:13:38 +0200 Subject: [PATCH 78/82] backup/restore tests from 11.2 --- src/tests/test_backuprestore.py | 52 ++++++++++++++++----------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/src/tests/test_backuprestore.py b/src/tests/test_backuprestore.py index eb59d4fea..4a59e7574 100644 --- a/src/tests/test_backuprestore.py +++ b/src/tests/test_backuprestore.py @@ -49,8 +49,8 @@ def setup_function(function): for m in function.__dict__.get("pytestmark", []) } - if "with_wordpress_archive_from_4p2" in markers: - add_archive_wordpress_from_4p2() + if "with_wordpress_archive_from_11p2" in markers: + add_archive_wordpress_from_11p2() assert len(backup_list()["archives"]) == 1 if "with_legacy_app_installed" in markers: @@ -72,8 +72,8 @@ def setup_function(function): ) assert app_is_installed("backup_recommended_app") - if "with_system_archive_from_4p2" in markers: - add_archive_system_from_4p2() + if "with_system_archive_from11p2" in markers: + add_archive_system_from_11p2() assert len(backup_list()["archives"]) == 1 if "with_permission_app_installed" in markers: @@ -148,7 +148,7 @@ def app_is_installed(app): def backup_test_dependencies_are_met(): # Dummy test apps (or backup archives) assert os.path.exists( - os.path.join(get_test_apps_dir(), "backup_wordpress_from_4p2") + os.path.join(get_test_apps_dir(), "backup_wordpress_from_11p2") ) assert os.path.exists(os.path.join(get_test_apps_dir(), "legacy_app_ynh")) assert os.path.exists( @@ -211,23 +211,23 @@ def install_app(app, path, additionnal_args=""): ) -def add_archive_wordpress_from_4p2(): +def add_archive_wordpress_from_11p2(): os.system("mkdir -p /home/yunohost.backup/archives") os.system( "cp " - + os.path.join(get_test_apps_dir(), "backup_wordpress_from_4p2/backup.tar") - + " /home/yunohost.backup/archives/backup_wordpress_from_4p2.tar" + + os.path.join(get_test_apps_dir(), "backup_wordpress_from_11p2/backup.tar") + + " /home/yunohost.backup/archives/backup_wordpress_from_11p2.tar" ) -def add_archive_system_from_4p2(): +def add_archive_system_from_11p2(): os.system("mkdir -p /home/yunohost.backup/archives") os.system( "cp " - + os.path.join(get_test_apps_dir(), "backup_system_from_4p2/backup.tar") - + " /home/yunohost.backup/archives/backup_system_from_4p2.tar" + + os.path.join(get_test_apps_dir(), "backup_system_from_11p2/backup.tar") + + " /home/yunohost.backup/archives/backup_system_from_11p2.tar" ) @@ -292,12 +292,12 @@ def test_backup_and_restore_all_sys(): # -# System restore from 3.8 # +# System restore from 11.2 # # -@pytest.mark.with_system_archive_from_4p2 -def test_restore_system_from_Ynh4p2(monkeypatch): +@pytest.mark.with_system_archive_from_11p2 +def test_restore_system_from_Ynh11p2(monkeypatch): name = random_ascii(8) # Backup current system with message("backup_created", name=name): @@ -305,7 +305,7 @@ def test_restore_system_from_Ynh4p2(monkeypatch): archives = backup_list()["archives"] assert len(archives) == 2 - # Restore system archive from 3.8 + # Restore system archive from 11.2 try: with message("restore_complete"): backup_restore( @@ -439,18 +439,17 @@ def test_backup_using_copy_method(): # App restore # # -# FIXME : switch to a backup from 11.x @pytest.mark.skip -@pytest.mark.with_wordpress_archive_from_4p2 +@pytest.mark.with_wordpress_archive_from_11p2 @pytest.mark.with_custom_domain("yolo.test") -def test_restore_app_wordpress_from_Ynh4p2(): +def test_restore_app_wordpress_from_Ynh11p2(): with message("restore_complete"): backup_restore( system=None, name=backup_list()["archives"][0], apps=["wordpress"] ) -@pytest.mark.with_wordpress_archive_from_4p2 +@pytest.mark.with_wordpress_archive_from_11p2 @pytest.mark.with_custom_domain("yolo.test") def test_restore_app_script_failure_handling(monkeypatch, mocker): def custom_hook_exec(name, *args, **kwargs): @@ -471,7 +470,7 @@ def test_restore_app_script_failure_handling(monkeypatch, mocker): assert not _is_installed("wordpress") -@pytest.mark.with_wordpress_archive_from_4p2 +@pytest.mark.with_wordpress_archive_from_11p2 def test_restore_app_not_enough_free_space(monkeypatch, mocker): def custom_free_space_in_directory(dirpath): return 0 @@ -490,7 +489,7 @@ def test_restore_app_not_enough_free_space(monkeypatch, mocker): assert not _is_installed("wordpress") -@pytest.mark.with_wordpress_archive_from_4p2 +@pytest.mark.with_wordpress_archive_from_11p2 def test_restore_app_not_in_backup(mocker): assert not _is_installed("wordpress") assert not _is_installed("yoloswag") @@ -505,9 +504,8 @@ def test_restore_app_not_in_backup(mocker): assert not _is_installed("yoloswag") -# FIXME : switch to a backup from 11.x @pytest.mark.skip -@pytest.mark.with_wordpress_archive_from_4p2 +@pytest.mark.with_wordpress_archive_from_11p2 @pytest.mark.with_custom_domain("yolo.test") def test_restore_app_already_installed(mocker): assert not _is_installed("wordpress") @@ -619,17 +617,17 @@ def test_restore_archive_with_no_json(mocker): backup_restore(name="badbackup", force=True) -@pytest.mark.with_wordpress_archive_from_4p2 +@pytest.mark.with_wordpress_archive_from_11p2 def test_restore_archive_with_bad_archive(mocker): # Break the archive os.system( - "head -n 1000 /home/yunohost.backup/archives/backup_wordpress_from_4p2.tar > /home/yunohost.backup/archives/backup_wordpress_from_4p2_bad.tar" + "head -n 1000 /home/yunohost.backup/archives/backup_wordpress_from_11p2.tar > /home/yunohost.backup/archives/backup_wordpress_from_11p2_bad.tar" ) - assert "backup_wordpress_from_4p2_bad" in backup_list()["archives"] + assert "backup_wordpress_from_11p2_bad" in backup_list()["archives"] with raiseYunohostError(mocker, "backup_archive_corrupted"): - backup_restore(name="backup_wordpress_from_4p2_bad", force=True) + backup_restore(name="backup_wordpress_from_11p2_bad", force=True) clean_tmp_backup_directory() From aed8ecb64594f71c0340a172b7bb8fb6ca2d82ed Mon Sep 17 00:00:00 2001 From: Kay0u Date: Fri, 8 Sep 2023 23:47:57 +0200 Subject: [PATCH 79/82] do not skip tests from 11.2 --- src/tests/test_backuprestore.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/tests/test_backuprestore.py b/src/tests/test_backuprestore.py index 4a59e7574..c0a55b65b 100644 --- a/src/tests/test_backuprestore.py +++ b/src/tests/test_backuprestore.py @@ -439,7 +439,6 @@ def test_backup_using_copy_method(): # App restore # # -@pytest.mark.skip @pytest.mark.with_wordpress_archive_from_11p2 @pytest.mark.with_custom_domain("yolo.test") def test_restore_app_wordpress_from_Ynh11p2(): @@ -504,7 +503,6 @@ def test_restore_app_not_in_backup(mocker): assert not _is_installed("yoloswag") -@pytest.mark.skip @pytest.mark.with_wordpress_archive_from_11p2 @pytest.mark.with_custom_domain("yolo.test") def test_restore_app_already_installed(mocker): From 142fad4b7898686f970d9bbae704e7da64203ed5 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Fri, 8 Sep 2023 23:50:30 +0200 Subject: [PATCH 80/82] typo --- src/tests/test_backuprestore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/test_backuprestore.py b/src/tests/test_backuprestore.py index c0a55b65b..ad9a1c1f2 100644 --- a/src/tests/test_backuprestore.py +++ b/src/tests/test_backuprestore.py @@ -72,7 +72,7 @@ def setup_function(function): ) assert app_is_installed("backup_recommended_app") - if "with_system_archive_from11p2" in markers: + if "with_system_archive_from_11p2" in markers: add_archive_system_from_11p2() assert len(backup_list()["archives"]) == 1 From df1f3149ea1d2182f212823c69e1ac07d4a3cf72 Mon Sep 17 00:00:00 2001 From: stanislas Date: Tue, 12 Sep 2023 00:37:47 +0200 Subject: [PATCH 81/82] The p value of a DMARC record can take the values none, quarantine or reject The validation is no more about the being similar to the expected config. Now wre check that the value of the p parameter of a DMARC record has the value none, quarantine or reject. No check for other parameters but it could be improved --- src/diagnosers/12-dnsrecords.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/diagnosers/12-dnsrecords.py b/src/diagnosers/12-dnsrecords.py index 196a2e1f9..19becb753 100644 --- a/src/diagnosers/12-dnsrecords.py +++ b/src/diagnosers/12-dnsrecords.py @@ -215,6 +215,11 @@ class MyDiagnoser(Diagnoser): for part in current if not part.startswith("ip4:") and not part.startswith("ip6:") } + if "v=DMARC1" in r["value"]: + for param in current: + key, value = param.split("=") + if key == "p": + return value in ["none", "quarantine", "reject"] return expected == current elif r["type"] == "MX": # For MX, we want to ignore the priority From b54a71b0cf365042e3d6e6f364b64d5608b238e4 Mon Sep 17 00:00:00 2001 From: Florian Date: Tue, 12 Sep 2023 09:02:04 +0300 Subject: [PATCH 82/82] Fix missleading example for ynh_setup_source There shouldn't be any trailing / for folders for ynh_setup_source --- helpers/utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/utils b/helpers/utils index 4694dd724..8b28000d6 100644 --- a/helpers/utils +++ b/helpers/utils @@ -75,7 +75,7 @@ fi # usage: ynh_setup_source --dest_dir=dest_dir [--source_id=source_id] [--keep="file1 file2"] [--full_replace] # | arg: -d, --dest_dir= - Directory where to setup sources # | arg: -s, --source_id= - Name of the source, defaults to `main` (when the sources resource exists in manifest.toml) or (legacy) `app` otherwise -# | arg: -k, --keep= - Space-separated list of files/folders that will be backup/restored in $dest_dir, such as a config file you don't want to overwrite. For example 'conf.json secrets.json logs/' +# | arg: -k, --keep= - Space-separated list of files/folders that will be backup/restored in $dest_dir, such as a config file you don't want to overwrite. For example 'conf.json secrets.json logs' (no trailing `/` for folders) # | arg: -r, --full_replace= - Remove previous sources before installing new sources # # #### New 'sources' resources