From cf32853f810adb4d4a0f674ab887bfbb117703de Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 25 Apr 2020 03:44:26 +0200 Subject: [PATCH 1/4] Fetch all cert-status at once because running a yunohost command takes ~3ish seconds per call --- data/hooks/conf_regen/15-nginx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data/hooks/conf_regen/15-nginx b/data/hooks/conf_regen/15-nginx index f8b7d8062..f1a278440 100755 --- a/data/hooks/conf_regen/15-nginx +++ b/data/hooks/conf_regen/15-nginx @@ -52,6 +52,8 @@ do_pre_regen() { export compatibility="$(yunohost settings get 'security.nginx.compatibility')" ynh_render_template "security.conf.inc" "${nginx_conf_dir}/security.conf.inc" + cert_status=$(yunohost domain cert-status --json) + # add domain conf files for domain in $domain_list; do domain_conf_dir="${nginx_conf_dir}/${domain}.d" @@ -61,7 +63,7 @@ do_pre_regen() { # NGINX server configuration export domain - export domain_cert_ca=$(yunohost domain cert-status $domain --json \ + export domain_cert_ca=$(echo $cert_status \ | jq ".certificates.\"$domain\".CA_type" \ | tr -d '"') From 319898baf7892582e90b5b9b24fccb8234939fe3 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 25 Apr 2020 03:49:30 +0200 Subject: [PATCH 2/4] Feed domain list to regen-conf hooks directly through env to avoid having to call 'yunohost domain list' --- data/hooks/conf_regen/12-metronome | 8 +++----- data/hooks/conf_regen/15-nginx | 14 +++++--------- data/hooks/conf_regen/19-postfix | 5 ++--- data/hooks/conf_regen/31-rspamd | 5 +---- data/hooks/conf_regen/43-dnsmasq | 5 ++--- src/yunohost/regenconf.py | 11 +++++++++-- 6 files changed, 22 insertions(+), 26 deletions(-) diff --git a/data/hooks/conf_regen/12-metronome b/data/hooks/conf_regen/12-metronome index 5a50b2b6e..8aee90212 100755 --- a/data/hooks/conf_regen/12-metronome +++ b/data/hooks/conf_regen/12-metronome @@ -14,7 +14,6 @@ do_pre_regen() { # retrieve variables main_domain=$(cat /etc/yunohost/current_host) - domain_list=$(yunohost domain list --output-as plain --quiet) # install main conf file cat metronome.cfg.lua \ @@ -22,7 +21,7 @@ do_pre_regen() { > "${metronome_dir}/metronome.cfg.lua" # add domain conf files - for domain in $domain_list; do + for domain in $YNH_DOMAINS; do cat domain.tpl.cfg.lua \ | sed "s/{{ domain }}/${domain}/g" \ > "${metronome_conf_dir}/${domain}.cfg.lua" @@ -33,7 +32,7 @@ do_pre_regen() { | awk '/^[^\.]+\.[^\.]+.*\.cfg\.lua$/ { print $1 }') for file in $conf_files; do domain=${file%.cfg.lua} - [[ $domain_list =~ $domain ]] \ + [[ $YNH_DOMAINS =~ $domain ]] \ || touch "${metronome_conf_dir}/${file}" done } @@ -43,10 +42,9 @@ do_post_regen() { # retrieve variables main_domain=$(cat /etc/yunohost/current_host) - domain_list=$(yunohost domain list --output-as plain --quiet) # create metronome directories for domains - for domain in $domain_list; do + for domain in $YNH_DOMAINS; do mkdir -p "/var/lib/metronome/${domain//./%2e}/pep" done # http_upload directory must be writable by metronome and readable by nginx diff --git a/data/hooks/conf_regen/15-nginx b/data/hooks/conf_regen/15-nginx index f1a278440..a43107599 100755 --- a/data/hooks/conf_regen/15-nginx +++ b/data/hooks/conf_regen/15-nginx @@ -46,7 +46,6 @@ do_pre_regen() { # retrieve variables main_domain=$(cat /etc/yunohost/current_host) - domain_list=$(yunohost domain list --output-as plain --quiet) # Support different strategy for security configurations export compatibility="$(yunohost settings get 'security.nginx.compatibility')" @@ -55,7 +54,7 @@ do_pre_regen() { cert_status=$(yunohost domain cert-status --json) # add domain conf files - for domain in $domain_list; do + for domain in $YNH_DOMAINS; do domain_conf_dir="${nginx_conf_dir}/${domain}.d" mkdir -p "$domain_conf_dir" mail_autoconfig_dir="${pending_dir}/var/www/.well-known/${domain}/autoconfig/mail/" @@ -83,7 +82,7 @@ do_pre_regen() { | awk '/^[^\.]+\.[^\.]+.*\.conf$/ { print $1 }') for file in $conf_files; do domain=${file%.conf} - [[ $domain_list =~ $domain ]] \ + [[ $YNH_DOMAINS =~ $domain ]] \ || touch "${nginx_conf_dir}/${file}" done @@ -91,7 +90,7 @@ do_pre_regen() { autoconfig_files=$(ls -1 /var/www/.well-known/*/autoconfig/mail/config-v1.1.xml 2>/dev/null || true) for file in $autoconfig_files; do domain=$(basename $(readlink -f $(dirname $file)/../..)) - [[ $domain_list =~ $domain ]] \ + [[ $YNH_DOMAINS =~ $domain ]] \ || (mkdir -p "$(dirname ${pending_dir}/${file})" && touch "${pending_dir}/${file}") done @@ -105,16 +104,13 @@ do_post_regen() { [ -z "$regen_conf_files" ] && exit 0 - # retrieve variables - domain_list=$(yunohost domain list --output-as plain --quiet) - # create NGINX conf directories for domains - for domain in $domain_list; do + for domain in $YNH_DOMAINS; do mkdir -p "/etc/nginx/conf.d/${domain}.d" done # Get rid of legacy lets encrypt snippets - for domain in $domain_list; do + for domain in $YNH_DOMAINS; do # If the legacy letsencrypt / acme-challenge domain-specific snippet is still there if [ -e /etc/nginx/conf.d/${domain}.d/000-acmechallenge.conf ] then diff --git a/data/hooks/conf_regen/19-postfix b/data/hooks/conf_regen/19-postfix index 10076b680..68afe4bc9 100755 --- a/data/hooks/conf_regen/19-postfix +++ b/data/hooks/conf_regen/19-postfix @@ -20,18 +20,17 @@ do_pre_regen() { # prepare main.cf conf file main_domain=$(cat /etc/yunohost/current_host) - domain_list=$(yunohost domain list --output-as plain --quiet | tr '\n' ' ') # Support different strategy for security configurations export compatibility="$(yunohost settings get 'security.postfix.compatibility')" export main_domain - export domain_list + export domain_list="$YNH_DOMAINS" ynh_render_template "main.cf" "${postfix_dir}/main.cf" cat postsrsd \ | sed "s/{{ main_domain }}/${main_domain}/g" \ - | sed "s/{{ domain_list }}/${domain_list}/g" \ + | sed "s/{{ domain_list }}/${YNH_DOMAINS}/g" \ > "${default_dir}/postsrsd" # adapt it for IPv4-only hosts diff --git a/data/hooks/conf_regen/31-rspamd b/data/hooks/conf_regen/31-rspamd index 26fea4336..861549e27 100755 --- a/data/hooks/conf_regen/31-rspamd +++ b/data/hooks/conf_regen/31-rspamd @@ -25,11 +25,8 @@ do_post_regen() { mkdir -p /etc/dkim chown _rspamd /etc/dkim - # retrieve domain list - domain_list=$(yunohost domain list --output-as plain --quiet) - # create DKIM key for domains - for domain in $domain_list; do + for domain in $YNH_DOMAINS; do domain_key="/etc/dkim/${domain}.mail.key" [ ! -f "$domain_key" ] && { # We use a 1024 bit size because nsupdate doesn't seem to be able to diff --git a/data/hooks/conf_regen/43-dnsmasq b/data/hooks/conf_regen/43-dnsmasq index 59a1f8a06..8a2985f34 100755 --- a/data/hooks/conf_regen/43-dnsmasq +++ b/data/hooks/conf_regen/43-dnsmasq @@ -26,10 +26,9 @@ do_pre_regen() { ynh_validate_ip4 "$ipv4" || ipv4='127.0.0.1' ipv6=$(curl -s -6 https://ip6.yunohost.org 2>/dev/null || true) ynh_validate_ip6 "$ipv6" || ipv6='' - domain_list=$(yunohost domain list --output-as plain --quiet) # add domain conf files - for domain in $domain_list; do + for domain in $YNH_DOMAINS; do cat domain.tpl \ | sed "s/{{ domain }}/${domain}/g" \ | sed "s/{{ ip }}/${ipv4}/g" \ @@ -42,7 +41,7 @@ do_pre_regen() { conf_files=$(ls -1 /etc/dnsmasq.d \ | awk '/^[^\.]+\.[^\.]+.*$/ { print $1 }') for domain in $conf_files; do - [[ $domain_list =~ $domain ]] \ + [[ $YNH_DOMAINS =~ $domain ]] \ || touch "${dnsmasq_dir}/${domain}" done } diff --git a/src/yunohost/regenconf.py b/src/yunohost/regenconf.py index ad84c8164..b81c043ce 100644 --- a/src/yunohost/regenconf.py +++ b/src/yunohost/regenconf.py @@ -141,7 +141,14 @@ def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run if "glances" in names: names.remove("glances") - pre_result = hook_callback('conf_regen', names, pre_callback=_pre_call) + # [Optimization] We compute and feed the domain list to the conf regen + # hooks to avoid having to call "yunohost domain list" so many times which + # ends up in wasted time (about 3~5 seconds per call on a RPi2) + from yunohost.domain import domain_list + env = {} + env["YNH_DOMAINS"] = " ".join(domain_list()["domains"]) + + pre_result = hook_callback('conf_regen', names, pre_callback=_pre_call, env=env) # Keep only the hook names with at least one success names = [hook for hook, infos in pre_result.items() @@ -310,7 +317,7 @@ def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run regen_conf_files = '' return post_args + [regen_conf_files, ] - hook_callback('conf_regen', names, pre_callback=_pre_call) + hook_callback('conf_regen', names, pre_callback=_pre_call, env=env) operation_logger.success() From b90155423d73614340c9aa9c85d902afd7b0f1e9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 25 Apr 2020 04:33:08 +0200 Subject: [PATCH 3/4] Add a caching mechanism to get_public_ip to avoid loading requests and calling ip.yunohost.org dozens of time per minute... --- src/yunohost/utils/network.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/yunohost/utils/network.py b/src/yunohost/utils/network.py index 3ae1ba910..92ca2a266 100644 --- a/src/yunohost/utils/network.py +++ b/src/yunohost/utils/network.py @@ -21,7 +21,9 @@ import os import re import logging +import time +from moulinette.utils.filesystem import read_file, write_to_file from moulinette.utils.network import download_text from moulinette.utils.process import check_output @@ -29,14 +31,24 @@ logger = logging.getLogger('yunohost.utils.network') def get_public_ip(protocol=4): - """Retrieve the public IP address from ip.yunohost.org""" - if protocol == 4: - url = 'https://ip.yunohost.org' - elif protocol == 6: - url = 'https://ip6.yunohost.org' + assert protocol in [4, 6], "Invalid protocol version for get_public_ip: %s, expected 4 or 6" % protocol + + cache_file = "/var/cache/yunohost/ipv%s" % protocol + cache_duration = 120 # 2 min + if os.path.exists(cache_file) and abs(os.path.getctime(cache_file) - time.time()) < cache_duration: + ip = read_file(cache_file).strip() + ip = ip if ip else None # Empty file (empty string) means there's no IP + logger.debug("Reusing IPv%s from cache: %s" % (protocol, ip)) else: - raise ValueError("invalid protocol version") + ip = get_public_ip_from_remote_server(protocol) + logger.debug("IP fetched: %s" % ip) + write_to_file(cache_file, ip or "") + return ip + + +def get_public_ip_from_remote_server(protocol=4): + """Retrieve the public IP address from ip.yunohost.org""" # We can know that ipv6 is not available directly if this file does not exists if protocol == 6 and not os.path.exists("/proc/net/if_inet6"): @@ -49,6 +61,9 @@ def get_public_ip(protocol=4): logger.debug("No default route for IPv%s, so assuming there's no IP address for that version" % protocol) return None + url = 'https://ip%s.yunohost.org' % (protocol if protocol != 4 else '') + logger.debug("Fetching IP from %s " % url) + try: return download_text(url, timeout=30).strip() except Exception as e: From 9ef2c60a90e828bd991f6d06e728b4537b9af358 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 25 Apr 2020 05:24:27 +0200 Subject: [PATCH 4/4] Uhoh we can't call domain_list if postinstall ain't done yet --- src/yunohost/regenconf.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/yunohost/regenconf.py b/src/yunohost/regenconf.py index b81c043ce..94883367b 100644 --- a/src/yunohost/regenconf.py +++ b/src/yunohost/regenconf.py @@ -146,7 +146,12 @@ def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run # ends up in wasted time (about 3~5 seconds per call on a RPi2) from yunohost.domain import domain_list env = {} - env["YNH_DOMAINS"] = " ".join(domain_list()["domains"]) + # Well we can only do domain_list() if postinstall is done ... + # ... but hooks that effectively need the domain list are only + # called only after the 'installed' flag is set so that's all good, + # though kinda tight-coupled to the postinstall logic :s + if os.path.exists("/etc/yunohost/installed"): + env["YNH_DOMAINS"] = " ".join(domain_list()["domains"]) pre_result = hook_callback('conf_regen', names, pre_callback=_pre_call, env=env)