Merge pull request #958 from YunoHost/misc-optimizations

Misc optimizations to speed up regen-conf and other things

data/hooks/conf_regen/12-metronome

@@ -14,7 +14,6 @@ do_pre_regen() {
 
   # retrieve variables
   main_domain=$(cat /etc/yunohost/current_host)
-  domain_list=$(yunohost domain list --output-as plain --quiet)
 
   # install main conf file
   cat metronome.cfg.lua \
@@ -22,7 +21,7 @@ do_pre_regen() {
     > "${metronome_dir}/metronome.cfg.lua"
 
   # add domain conf files
-  for domain in $domain_list; do
+  for domain in $YNH_DOMAINS; do
     cat domain.tpl.cfg.lua \
       | sed "s/{{ domain }}/${domain}/g" \
       > "${metronome_conf_dir}/${domain}.cfg.lua"
@@ -33,7 +32,7 @@ do_pre_regen() {
                  | awk '/^[^\.]+\.[^\.]+.*\.cfg\.lua$/ { print $1 }')
   for file in $conf_files; do
     domain=${file%.cfg.lua}
-    [[ $domain_list =~ $domain ]] \
+    [[ $YNH_DOMAINS =~ $domain ]] \
       || touch "${metronome_conf_dir}/${file}"
   done
 }
@@ -43,6 +42,9 @@ do_post_regen() {
 
   # retrieve variables
   main_domain=$(cat /etc/yunohost/current_host)
+  
+  # FIXME : small optimization to do to avoid calling a yunohost command ... 
+  # maybe another env variable like YNH_MAIN_DOMAINS idk
   domain_list=$(yunohost domain list --exclude-subdomains --output-as plain --quiet)
 
   # create metronome directories for domains

data/hooks/conf_regen/15-nginx

@@ -47,14 +47,15 @@ do_pre_regen() {
 
   # retrieve variables
   main_domain=$(cat /etc/yunohost/current_host)
-  domain_list=$(yunohost domain list --output-as plain --quiet)
 
   # Support different strategy for security configurations
   export compatibility="$(yunohost settings get 'security.nginx.compatibility')"
   ynh_render_template "security.conf.inc" "${nginx_conf_dir}/security.conf.inc"
 
+  cert_status=$(yunohost domain cert-status --json)
+
   # add domain conf files
-  for domain in $domain_list; do
+  for domain in $YNH_DOMAINS; do
     domain_conf_dir="${nginx_conf_dir}/${domain}.d"
     mkdir -p "$domain_conf_dir"
     mail_autoconfig_dir="${pending_dir}/var/www/.well-known/${domain}/autoconfig/mail/"
@@ -62,7 +63,7 @@ do_pre_regen() {
 
     # NGINX server configuration
     export domain
-    export domain_cert_ca=$(yunohost domain cert-status $domain --json \
+    export domain_cert_ca=$(echo $cert_status \
                             | jq ".certificates.\"$domain\".CA_type" \
                             | tr -d '"')
 
@@ -82,7 +83,7 @@ do_pre_regen() {
                  | awk '/^[^\.]+\.[^\.]+.*\.conf$/ { print $1 }')
   for file in $conf_files; do
     domain=${file%.conf}
-    [[ $domain_list =~ $domain ]] \
+    [[ $YNH_DOMAINS =~ $domain ]] \
       || touch "${nginx_conf_dir}/${file}"
   done
 
@@ -90,7 +91,7 @@ do_pre_regen() {
   autoconfig_files=$(ls -1 /var/www/.well-known/*/autoconfig/mail/config-v1.1.xml 2>/dev/null || true)
   for file in $autoconfig_files; do
     domain=$(basename $(readlink -f $(dirname $file)/../..))
-    [[ $domain_list =~ $domain ]] \
+    [[ $YNH_DOMAINS =~ $domain ]] \
       || (mkdir -p "$(dirname ${pending_dir}/${file})" && touch "${pending_dir}/${file}")
   done
 
@@ -104,16 +105,13 @@ do_post_regen() {
 
   [ -z "$regen_conf_files" ] && exit 0
 
-  # retrieve variables
-  domain_list=$(yunohost domain list --output-as plain --quiet)
-
   # create NGINX conf directories for domains
-  for domain in $domain_list; do
+  for domain in $YNH_DOMAINS; do
     mkdir -p "/etc/nginx/conf.d/${domain}.d"
   done
 
   # Get rid of legacy lets encrypt snippets
-  for domain in $domain_list; do
+  for domain in $YNH_DOMAINS; do
       # If the legacy letsencrypt / acme-challenge domain-specific snippet is still there
       if [ -e /etc/nginx/conf.d/${domain}.d/000-acmechallenge.conf ]
       then

data/hooks/conf_regen/19-postfix

@@ -20,18 +20,17 @@ do_pre_regen() {
 
   # prepare main.cf conf file
   main_domain=$(cat /etc/yunohost/current_host)
-  domain_list=$(yunohost domain list --output-as plain --quiet | tr '\n' ' ')
 
   # Support different strategy for security configurations
   export compatibility="$(yunohost settings get 'security.postfix.compatibility')"
 
   export main_domain
-  export domain_list
+  export domain_list="$YNH_DOMAINS"
   ynh_render_template "main.cf" "${postfix_dir}/main.cf"
 
   cat postsrsd \
     | sed "s/{{ main_domain }}/${main_domain}/g" \
-    | sed "s/{{ domain_list }}/${domain_list}/g" \
+    | sed "s/{{ domain_list }}/${YNH_DOMAINS}/g" \
     > "${default_dir}/postsrsd"
 
   # adapt it for IPv4-only hosts

data/hooks/conf_regen/31-rspamd

@@ -25,11 +25,8 @@ do_post_regen() {
   mkdir -p /etc/dkim
   chown _rspamd /etc/dkim
 
-  # retrieve domain list
-  domain_list=$(yunohost domain list --output-as plain --quiet)
-
   # create DKIM key for domains
-  for domain in $domain_list; do
+  for domain in $YNH_DOMAINS; do
     domain_key="/etc/dkim/${domain}.mail.key"
     [ ! -f "$domain_key" ] && {
       # We use a 1024 bit size because nsupdate doesn't seem to be able to

data/hooks/conf_regen/43-dnsmasq

@@ -26,10 +26,9 @@ do_pre_regen() {
   ynh_validate_ip4 "$ipv4" || ipv4='127.0.0.1'
   ipv6=$(curl -s -6 https://ip6.yunohost.org 2>/dev/null || true)
   ynh_validate_ip6 "$ipv6" || ipv6=''
-  domain_list=$(yunohost domain list --output-as plain --quiet)
 
   # add domain conf files
-  for domain in $domain_list; do
+  for domain in $YNH_DOMAINS; do
     cat domain.tpl \
       | sed "s/{{ domain }}/${domain}/g" \
       | sed "s/{{ ip }}/${ipv4}/g" \
@@ -42,7 +41,7 @@ do_pre_regen() {
   conf_files=$(ls -1 /etc/dnsmasq.d \
                  | awk '/^[^\.]+\.[^\.]+.*$/ { print $1 }')
   for domain in $conf_files; do
-    [[ $domain_list =~ $domain ]] \
+    [[ $YNH_DOMAINS =~ $domain ]] \
       || touch "${dnsmasq_dir}/${domain}"
   done
 }

src/yunohost/regenconf.py

@@ -141,7 +141,19 @@ def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run
     if "glances" in names:
         names.remove("glances")
 
-    pre_result = hook_callback('conf_regen', names, pre_callback=_pre_call)
+    # [Optimization] We compute and feed the domain list to the conf regen
+    # hooks to avoid having to call "yunohost domain list" so many times which
+    # ends up in wasted time (about 3~5 seconds per call on a RPi2)
+    from yunohost.domain import domain_list
+    env = {}
+    # Well we can only do domain_list() if postinstall is done ...
+    # ... but hooks that effectively need the domain list are only
+    # called only after the 'installed' flag is set so that's all good,
+    # though kinda tight-coupled to the postinstall logic :s
+    if os.path.exists("/etc/yunohost/installed"):
+        env["YNH_DOMAINS"] = " ".join(domain_list()["domains"])
+
+    pre_result = hook_callback('conf_regen', names, pre_callback=_pre_call, env=env)
 
     # Keep only the hook names with at least one success
     names = [hook for hook, infos in pre_result.items()
@@ -310,7 +322,7 @@ def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run
             regen_conf_files = ''
         return post_args + [regen_conf_files, ]
 
-    hook_callback('conf_regen', names, pre_callback=_pre_call)
+    hook_callback('conf_regen', names, pre_callback=_pre_call, env=env)
 
     operation_logger.success()
 

src/yunohost/utils/network.py

@@ -21,8 +21,10 @@
 import os
 import re
 import logging
+import time
 import dns.resolver
 
+from moulinette.utils.filesystem import read_file, write_to_file
 from moulinette.utils.network import download_text
 from moulinette.utils.process import check_output
 from moulinette.utils.filesystem import read_file
@@ -31,14 +33,24 @@ logger = logging.getLogger('yunohost.utils.network')
 
 
 def get_public_ip(protocol=4):
-    """Retrieve the public IP address from ip.yunohost.org"""
 
-    if protocol == 4:
+    assert protocol in [4, 6], "Invalid protocol version for get_public_ip: %s, expected 4 or 6" % protocol
-        url = 'https://ip.yunohost.org'
+
-    elif protocol == 6:
+    cache_file = "/var/cache/yunohost/ipv%s" % protocol
-        url = 'https://ip6.yunohost.org'
+    cache_duration = 120  # 2 min
+    if os.path.exists(cache_file) and abs(os.path.getctime(cache_file) - time.time()) < cache_duration:
+        ip = read_file(cache_file).strip()
+        ip = ip if ip else None  # Empty file (empty string) means there's no IP
+        logger.debug("Reusing IPv%s from cache: %s" % (protocol, ip))
     else:
-        raise ValueError("invalid protocol version")
+        ip = get_public_ip_from_remote_server(protocol)
+        logger.debug("IP fetched: %s" % ip)
+        write_to_file(cache_file, ip or "")
+    return ip
+
+
+def get_public_ip_from_remote_server(protocol=4):
+    """Retrieve the public IP address from ip.yunohost.org"""
 
     # We can know that ipv6 is not available directly if this file does not exists
     if protocol == 6 and not os.path.exists("/proc/net/if_inet6"):
@@ -51,6 +63,9 @@ def get_public_ip(protocol=4):
         logger.debug("No default route for IPv%s, so assuming there's no IP address for that version" % protocol)
         return None
 
+    url = 'https://ip%s.yunohost.org' % (protocol if protocol != 4 else '')
+    logger.debug("Fetching IP from %s " % url)
+
     try:
         return download_text(url, timeout=30).strip()
     except Exception as e: