From 16391d7374def89fbe5fb336bf79b4c184ac3fb1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 28 Nov 2023 21:12:03 +0100 Subject: [PATCH] perf: lazyloading settings_get + pass all the global settings as an env variable to the regenconf to prevent having to 'yunohost settings get' like ten times ... reduces the regenconf runtime from 35s to 23s on my machine --- helpers/utils | 2 +- hooks/conf_regen/03-ssh | 15 +++++++-------- hooks/conf_regen/15-nginx | 12 ++++++------ hooks/conf_regen/19-postfix | 14 +++++++------- hooks/conf_regen/25-dovecot | 2 +- hooks/conf_regen/52-fail2ban | 2 +- src/backup.py | 2 +- src/dns.py | 3 ++- src/regenconf.py | 7 +++++++ 9 files changed, 33 insertions(+), 26 deletions(-) diff --git a/helpers/utils b/helpers/utils index 8b28000d6..447e3134b 100644 --- a/helpers/utils +++ b/helpers/utils @@ -1084,7 +1084,7 @@ _ynh_apply_default_permissions() { } int_to_bool() { - sed -e 's/^1$/True/g' -e 's/^0$/False/g' + sed -e 's/^1$/True/g' -e 's/^0$/False/g' -e 's/^true$/True/g' -e 's/^false$/False/g' } toml_to_json() { diff --git a/hooks/conf_regen/03-ssh b/hooks/conf_regen/03-ssh index d0351b4e5..747389727 100755 --- a/hooks/conf_regen/03-ssh +++ b/hooks/conf_regen/03-ssh @@ -9,17 +9,16 @@ do_pre_regen() { cd /usr/share/yunohost/conf/ssh + # Support different strategy for security configurations + export compatibility="$(jq -r '.ssh_compatibility' <<< "$YNH_SETTINGS")" + export port="$(jq -r '.ssh_port' <<< "$YNH_SETTINGS")" + export password_authentication="$(jq -r '.ssh_password_authentication' <<< "$YNH_SETTINGS" | int_to_bool)" + export ssh_keys=$(ls /etc/ssh/ssh_host_{ed25519,rsa,ecdsa}_key 2>/dev/null || true) + # do not listen to IPv6 if unavailable [[ -f /proc/net/if_inet6 ]] && ipv6_enabled=true || ipv6_enabled=false - - ssh_keys=$(ls /etc/ssh/ssh_host_{ed25519,rsa,ecdsa}_key 2>/dev/null || true) - - # Support different strategy for security configurations - export compatibility="$(yunohost settings get 'security.ssh.ssh_compatibility')" - export port="$(yunohost settings get 'security.ssh.ssh_port')" - export password_authentication="$(yunohost settings get 'security.ssh.ssh_password_authentication' | int_to_bool)" - export ssh_keys export ipv6_enabled + ynh_render_template "sshd_config" "${pending_dir}/etc/ssh/sshd_config" } diff --git a/hooks/conf_regen/15-nginx b/hooks/conf_regen/15-nginx index 9eabcd8b7..8f845984c 100755 --- a/hooks/conf_regen/15-nginx +++ b/hooks/conf_regen/15-nginx @@ -56,7 +56,7 @@ do_pre_regen() { # install / update plain conf files cp plain/* "$nginx_conf_dir" # remove the panel overlay if this is specified in settings - panel_overlay=$(yunohost settings get 'misc.portal.ssowat_panel_overlay_enabled' | int_to_bool) + panel_overlay="$(jq -r '.ssowat_panel_overlay_enabled' <<< "$YNH_SETTINGS" | int_to_bool)" if [ "$panel_overlay" == "False" ]; then echo "#" >"${nginx_conf_dir}/yunohost_panel.conf.inc" fi @@ -65,9 +65,9 @@ do_pre_regen() { main_domain=$(cat /etc/yunohost/current_host) # Support different strategy for security configurations - export redirect_to_https="$(yunohost settings get 'security.nginx.nginx_redirect_to_https' | int_to_bool)" - export compatibility="$(yunohost settings get 'security.nginx.nginx_compatibility')" - export experimental="$(yunohost settings get 'security.experimental.security_experimental_enabled' | int_to_bool)" + export redirect_to_https="$(jq -r '.nginx_redirect_to_https' <<< "$YNH_SETTINGS" | int_to_bool)" + export compatibility="$(jq -r '.nginx_compatibility' <<< "$YNH_SETTINGS" | int_to_bool)" + export experimental="$(jq -r '.security_experimental_enabled' <<< "$YNH_SETTINGS" | int_to_bool)" ynh_render_template "security.conf.inc" "${nginx_conf_dir}/security.conf.inc" cert_status=$(yunohost domain cert status --json) @@ -109,9 +109,9 @@ do_pre_regen() { done - export webadmin_allowlist_enabled=$(yunohost settings get security.webadmin.webadmin_allowlist_enabled | int_to_bool) + export webadmin_allowlist_enabled="$(jq -r '.webadmin_allowlist_enabled' <<< "$YNH_SETTINGS" | int_to_bool)" if [ "$webadmin_allowlist_enabled" == "True" ]; then - export webadmin_allowlist=$(yunohost settings get security.webadmin.webadmin_allowlist) + export webadmin_allowlist="$(jq -r '.webadmin_allowlist' <<< "$YNH_SETTINGS" | sed 's/^null$//g')" fi ynh_render_template "yunohost_admin.conf.inc" "${nginx_conf_dir}/yunohost_admin.conf.inc" ynh_render_template "yunohost_api.conf.inc" "${nginx_conf_dir}/yunohost_api.conf.inc" diff --git a/hooks/conf_regen/19-postfix b/hooks/conf_regen/19-postfix index d6ddcb5ee..694080302 100755 --- a/hooks/conf_regen/19-postfix +++ b/hooks/conf_regen/19-postfix @@ -22,19 +22,19 @@ do_pre_regen() { main_domain=$(cat /etc/yunohost/current_host) # Support different strategy for security configurations - export compatibility="$(yunohost settings get 'security.postfix.postfix_compatibility')" + export compatibility="$(jq -r '.postfix_compatibility' <<< "$YNH_SETTINGS")" # Add possibility to specify a relay # Could be useful with some isp with no 25 port open or more complex setup export relay_port="" export relay_user="" export relay_host="" - export relay_enabled="$(yunohost settings get 'email.smtp.smtp_relay_enabled' | int_to_bool)" + export relay_enabled="$(jq -r '.smtp_relay_enabled' <<< "$YNH_SETTINGS" | int_to_bool)" if [ "${relay_enabled}" == "True" ]; then - relay_host="$(yunohost settings get 'email.smtp.smtp_relay_host')" - relay_port="$(yunohost settings get 'email.smtp.smtp_relay_port')" - relay_user="$(yunohost settings get 'email.smtp.smtp_relay_user')" - relay_password="$(yunohost settings get 'email.smtp.smtp_relay_password')" + relay_host="$(jq -r '.smtp_relay_host' <<< "$YNH_SETTINGS")" + relay_port="$(jq -r '.smtp_relay_port' <<< "$YNH_SETTINGS")" + relay_user="$(jq -r '.smtp_relay_user' <<< "$YNH_SETTINGS")" + relay_password="$(jq -r '.smtp_relay_password' <<< "$YNH_SETTINGS")" # Avoid to display "Relay account paswword" to other users touch ${postfix_dir}/sasl_passwd @@ -56,7 +56,7 @@ do_pre_regen() { >"${default_dir}/postsrsd" # adapt it for IPv4-only hosts - ipv6="$(yunohost settings get 'email.smtp.smtp_allow_ipv6' | int_to_bool)" + ipv6="$(jq -r '.smtp_allow_ipv6' <<< "$YNH_SETTINGS" | int_to_bool)" if [ "$ipv6" == "False" ] || [ ! -f /proc/net/if_inet6 ]; then sed -i \ 's/ \[::ffff:127.0.0.0\]\/104 \[::1\]\/128//g' \ diff --git a/hooks/conf_regen/25-dovecot b/hooks/conf_regen/25-dovecot index 54b4e5d37..6185abe0e 100755 --- a/hooks/conf_regen/25-dovecot +++ b/hooks/conf_regen/25-dovecot @@ -16,7 +16,7 @@ do_pre_regen() { cp dovecot-ldap.conf "${dovecot_dir}/dovecot-ldap.conf" cp dovecot.sieve "${dovecot_dir}/global_script/dovecot.sieve" - export pop3_enabled="$(yunohost settings get 'email.pop3.pop3_enabled' | int_to_bool)" + export pop3_enabled="$(jq -r '.pop3_enabled' <<< "$YNH_SETTINGS" | int_to_bool)" export main_domain=$(cat /etc/yunohost/current_host) export domain_list="$(yunohost domain list --features mail_in mail_out --output-as json | jq -r ".domains[]" | tr '\n' ' ')" diff --git a/hooks/conf_regen/52-fail2ban b/hooks/conf_regen/52-fail2ban index 73f173410..0789556c4 100755 --- a/hooks/conf_regen/52-fail2ban +++ b/hooks/conf_regen/52-fail2ban @@ -18,7 +18,7 @@ do_pre_regen() { cp postfix-sasl.conf "${fail2ban_dir}/filter.d/postfix-sasl.conf" cp jail.conf "${fail2ban_dir}/jail.conf" - export ssh_port="$(yunohost settings get 'security.ssh.ssh_port')" + export ssh_port="$(jq -r '.ssh_port' <<< "$YNH_SETTINGS")" ynh_render_template "yunohost-jails.conf" "${fail2ban_dir}/jail.d/yunohost-jails.conf" } diff --git a/src/backup.py b/src/backup.py index 3e5e90634..236c31f68 100644 --- a/src/backup.py +++ b/src/backup.py @@ -76,7 +76,6 @@ from yunohost.utils.system import ( binary_to_human, space_used_by_directory, ) -from yunohost.settings import settings_get BACKUP_PATH = "/home/yunohost.backup" ARCHIVES_PATH = f"{BACKUP_PATH}/archives" @@ -1926,6 +1925,7 @@ class TarBackupMethod(BackupMethod): @property def _archive_file(self): + from yunohost.settings import settings_get if isinstance(self.manager, BackupManager) and settings_get( "misc.backup.backup_compress_tar_archives" ): diff --git a/src/dns.py b/src/dns.py index 6e5464092..f0438eb5c 100644 --- a/src/dns.py +++ b/src/dns.py @@ -37,7 +37,6 @@ from yunohost.domain import ( from yunohost.utils.dns import dig, is_yunohost_dyndns_domain, is_special_use_tld from yunohost.utils.error import YunohostValidationError, YunohostError from yunohost.utils.network import get_public_ip -from yunohost.settings import settings_get from yunohost.log import is_unit_operation from yunohost.hook import hook_callback @@ -145,6 +144,8 @@ def _build_dns_conf(base_domain, include_empty_AAAA_if_no_ipv6=False): } """ + from yunohost.settings import settings_get + basic = [] mail = [] xmpp = [] diff --git a/src/regenconf.py b/src/regenconf.py index f180368ad..432a824ac 100644 --- a/src/regenconf.py +++ b/src/regenconf.py @@ -20,6 +20,7 @@ import os import yaml import shutil import hashlib +import json from logging import getLogger from difflib import unified_diff from datetime import datetime @@ -63,6 +64,8 @@ def regen_conf( """ + from yunohost.settings import settings_get + if names is None: names = [] @@ -140,6 +143,10 @@ def regen_conf( domain_list(exclude_subdomains=True)["domains"] ) env["YNH_CONTEXT"] = "regenconf" + # perf: Export all global settings as a environment variable + # so that scripts dont have to call 'yunohost settings get' manually + # which is painful performance-wise + env["YNH_SETTINGS"] = json.dumps(settings_get("", export=True)) pre_result = hook_callback("conf_regen", names, pre_callback=_pre_call, env=env)