Merge branch 'stretch-unstable' into clean_helpers

data/actionsmap/yunohost.yml

@@ -406,6 +406,10 @@ domain:
         list:
             action_help: List domains
             api: GET /domains
+            arguments:
+                --exclude-subdomains:
+                    help: Filter out domains that are obviously subdomains of other declared domains
+                    action: store_true
 
         ### domain_add()
         add:
@@ -563,6 +567,9 @@ app:
                     help: Also return a list of app categories
                     action: store_true
 
+        fetchlist:
+            deprecated: true
+
         ### app_list()
         list:
             action_help: List installed apps
@@ -572,6 +579,12 @@ app:
                     full: --full
                     help: Display all details, including the app manifest and various other infos
                     action: store_true
+                -i:
+                    full: --installed
+                    help: Dummy argument, does nothing anymore (still there only for backward compatibility)
+                    action: store_true
+                filter:
+                    nargs: '?'
 
         ### app_info()
         info:

data/helpers.d/postgresql

@@ -87,6 +87,7 @@ ynh_psql_create_db() {
 
     # grant all privilegies to user
     if [ -n "$user" ]; then
+        sql+="ALTER DATABASE ${db} OWNER TO ${user};"
         sql+="GRANT ALL PRIVILEGES ON DATABASE ${db} TO ${user} WITH GRANT OPTION;"
     fi
 

data/hooks/conf_regen/12-metronome

@@ -43,16 +43,16 @@ do_post_regen() {
 
   # retrieve variables
   main_domain=$(cat /etc/yunohost/current_host)
-  domain_list=$(yunohost domain list --output-as plain --quiet)
+  domain_list=$(yunohost domain list --exclude-subdomains --output-as plain --quiet)
 
   # create metronome directories for domains
   for domain in $domain_list; do
     mkdir -p "/var/lib/metronome/${domain//./%2e}/pep"
+    # http_upload directory must be writable by metronome and readable by nginx
+    mkdir -p  "/var/xmpp-upload/${domain}/upload"
+    chmod g+s "/var/xmpp-upload/${domain}/upload"
+    chown -R metronome:www-data "/var/xmpp-upload/${domain}"
   done
-  # http_upload directory must be writable by metronome and readable by nginx
-  mkdir -p  "/var/xmpp-upload/${main_domain}/upload"
-  chmod g+s "/var/xmpp-upload/${main_domain}/upload"
-  chown -R metronome:www-data "/var/xmpp-upload/${main_domain}"
 
   # fix some permissions
   chown -R metronome: /var/lib/metronome/

data/hooks/diagnosis/12-dnsrecords.py

@@ -2,9 +2,9 @@
 
 import os
 
-from moulinette.utils.process import check_output
 from moulinette.utils.filesystem import read_file
 
+from yunohost.utils.network import dig
 from yunohost.diagnosis import Diagnoser
 from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain
 
@@ -100,20 +100,14 @@ class DNSRecordsDiagnoser(Diagnoser):
             yield output
 
     def get_current_record(self, domain, name, type_):
-        if name == "@":
-            command = "dig +short @%s %s %s" % (self.resolver, type_, domain)
-        else:
-            command = "dig +short @%s %s %s.%s" % (self.resolver, type_, name, domain)
-        # FIXME : gotta handle case where this command fails ...
-        # e.g. no internet connectivity (dependency mechanism to good result from 'ip' diagosis ?)
-        # or the resolver is unavailable for some reason
-        output = check_output(command).strip().split("\n")
-        if len(output) == 0 or not output[0]:
+
+        query = "%s.%s" % (name, domain) if name != "@" else domain
+        success, answers = dig(query, type_, resolvers="force_external")
+
+        if success != "ok":
             return None
-        elif len(output) == 1:
-            return output[0]
         else:
-            return output
+            return answers[0] if len(answers) == 1 else answers
 
     def current_record_match_expected(self, r):
         if r["value"] is not None and r["current"] is None:

data/hooks/diagnosis/24-mail.py

@@ -13,6 +13,7 @@ from moulinette.utils.filesystem import read_yaml
 from yunohost.diagnosis import Diagnoser
 from yunohost.domain import _get_maindomain, domain_list
 from yunohost.settings import settings_get
+from yunohost.utils.network import dig
 
 DEFAULT_DNS_BLACKLIST = "/usr/share/yunohost/other/dnsbl_list.yml"
 
@@ -155,26 +156,25 @@ class MailDiagnoser(Diagnoser):
                 if not blacklist[item_type]:
                     continue
 
-                # Determine if we are listed on this RBL
-                try:
-                    subdomain = item
-                    if item_type != "domain":
-                        rev = dns.reversename.from_address(item)
-                        subdomain = str(rev.split(3)[0])
-                    query = subdomain + '.' + blacklist['dns_server']
-                    # TODO add timeout lifetime
-                    dns.resolver.query(query, "A")
-                except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers, dns.resolver.NoAnswer,
-                dns.exception.Timeout):
+                # Build the query for DNSBL
+                subdomain = item
+                if item_type != "domain":
+                    rev = dns.reversename.from_address(item)
+                    subdomain = str(rev.split(3)[0])
+                query = subdomain + '.' + blacklist['dns_server']
+
+                # Do the DNS Query
+                status, _ = dig(query, 'A')
+                if status != 'ok':
                     continue
 
                 # Try to get the reason
                 details = []
-                try:
-                    reason = str(dns.resolver.query(query, "TXT")[0])
+                status, answers = dig(query, 'TXT')
+                reason = "-"
+                if status == 'ok':
+                    reason = ', '.join(answers)
                     details.append("diagnosis_mail_blacklist_reason")
-                except Exception:
-                    reason = "-"
 
                 details.append("diagnosis_mail_blacklist_website")
 

data/templates/metronome/domain.tpl.cfg.lua

@@ -1,4 +1,5 @@
 VirtualHost "{{ domain }}"
+  enable = true
   ssl = {
         key = "/etc/yunohost/certs/{{ domain }}/key.pem";
         certificate = "/etc/yunohost/certs/{{ domain }}/crt.pem";
@@ -13,3 +14,58 @@ VirtualHost "{{ domain }}"
        namefield     = "cn",
        },
   }
+
+  -- Discovery items
+  disco_items = {
+    { "muc.{{ domain }}" },
+    { "pubsub.{{ domain }}" },
+    { "jabber.{{ domain }}" },
+    { "vjud.{{ domain }}" },
+    { "xmpp-upload.{{ domain }}" },
+  };
+
+--  contact_info = {
+--    abuse = { "mailto:abuse@{{ domain }}", "xmpp:admin@{{ domain }}" };
+--    admin = { "mailto:root@{{ domain }}", "xmpp:admin@{{ domain }}" };
+--  };
+
+------ Components ------
+-- You can specify components to add hosts that provide special services,
+-- like multi-user conferences, and transports.
+
+---Set up a MUC (multi-user chat) room server
+Component "muc.{{ domain }}" "muc"
+  name = "{{ domain }} Chatrooms"
+
+  modules_enabled = {
+    "muc_limits";
+    "muc_log";
+    "muc_log_mam";
+    "muc_log_http";
+    "muc_vcard";
+  }
+
+  muc_event_rate = 0.5
+  muc_burst_factor = 10
+
+---Set up a PubSub server
+Component "pubsub.{{ domain }}" "pubsub"
+  name = "{{ domain }} Publish/Subscribe"
+
+  unrestricted_node_creation = true -- Anyone can create a PubSub node (from any server)
+
+---Set up a HTTP Upload service
+Component "xmpp-upload.{{ domain }}" "http_upload"
+  name = "{{ domain }} Sharing Service"
+
+  http_file_path = "/var/xmpp-upload/{{ domain }}/upload"
+  http_external_url = "https://xmpp-upload.{{ domain }}:443"
+  http_file_base_path = "/upload"
+  http_file_size_limit = 6*1024*1024
+  http_file_quota = 60*1024*1024
+  http_upload_file_size_limit = 100 * 1024 * 1024 -- bytes
+  http_upload_quota = 10 * 1024 * 1024 * 1024 -- bytes
+
+---Set up a VJUD service
+Component "vjud.{{ domain }}" "vjud"
+  vjud_disco_name = "{{ domain }} User Directory"

data/templates/metronome/metronome.cfg.lua

@@ -81,14 +81,6 @@ http_interfaces = { "127.0.0.1", "::1" }
 -- Enable IPv6
 use_ipv6 = true
 
--- Discovery items
-disco_items = {
-	{ "muc.{{ main_domain }}" },
-	{ "pubsub.{{ main_domain }}" },
-	{ "xmpp-upload.{{ main_domain }}" },
-	{ "vjud.{{ main_domain }}" }
-};
-
 -- BOSH configuration (mod_bosh)
 consider_bosh_secure = true
 cross_domain_bosh = true
@@ -119,45 +111,6 @@ log = {
 Component "localhost" "http"
 	modules_enabled = { "bosh" }
 
----Set up a MUC (multi-user chat) room server
-Component "muc.{{ main_domain }}" "muc"
-	name = "{{ main_domain }} Chatrooms"
-
-	modules_enabled = {
-		"muc_limits";
-		"muc_log";
-		"muc_log_mam";
-		"muc_log_http";
-		"muc_vcard";
-    }
-
-	muc_event_rate = 0.5
-	muc_burst_factor = 10
-
----Set up a PubSub server
-Component "pubsub.{{ main_domain }}" "pubsub"
-	name = "{{ main_domain }} Publish/Subscribe"
-
-	unrestricted_node_creation = true -- Anyone can create a PubSub node (from any server)
-
----Set up a HTTP Upload service
-Component "xmpp-upload.{{ main_domain }}" "http_upload"
-	name = "{{ main_domain }} Sharing Service"
-
-	http_file_path = "/var/xmpp-upload/{{ main_domain }}/upload"
-	http_external_url = "https://xmpp-upload.{{ main_domain }}:443"
-	http_file_base_path = "/upload"
-	http_file_size_limit = 6*1024*1024
-	http_file_quota = 60*1024*1024
-	http_upload_file_size_limit = 100 * 1024 * 1024 -- bytes
-	http_upload_quota = 10 * 1024 * 1024 * 1024 -- bytes
-
-
----Set up a VJUD service
-Component "vjud.{{ main_domain }}" "vjud"
-	ud_disco_name = "{{ main_domain }} User Directory"
-
-
 ----------- Virtual hosts -----------
 -- You need to add a VirtualHost entry for each domain you wish Metronome to serve.
 -- Settings under each VirtualHost entry apply *only* to that host.

locales/en.json

@@ -110,7 +110,7 @@
     "backup_unable_to_organize_files": "Could not use the quick method to organize files in the archive",
     "backup_with_no_backup_script_for_app": "The app '{app:s}' has no backup script. Ignoring.",
     "backup_with_no_restore_script_for_app": "The '{app:s}' has no restoration script, you will not be able to automatically restore the backup of this app.",
-    "certmanager_acme_not_configured_for_domain": "Certificate for the domain '{domain:s}' does not appear to be correctly installed. Please run 'cert-install' for this domain first.",
+    "certmanager_acme_not_configured_for_domain": "The ACME challenge cannot be ran for {domain} right now because its nginx conf lacks the corresponding code snippet... Please make sure that your nginx configuration is up to date using `yunohost tools regen-conf nginx --dry-run --with-diff`.",
     "certmanager_attempt_to_renew_nonLE_cert": "The certificate for the domain '{domain:s}' is not issued by Let's Encrypt. Cannot renew it automatically!",
     "certmanager_attempt_to_renew_valid_cert": "The certificate for the domain '{domain:s}' is not about to expire! (You may use --force if you know what you're doing)",
     "certmanager_attempt_to_replace_valid_cert": "You are attempting to overwrite a good and valid certificate for domain {domain:s}! (Use --force to bypass)",

src/yunohost/app.py

@@ -110,12 +110,34 @@ def app_catalog(full=False, with_categories=False):
         return {"apps": catalog["apps"], "categories": catalog["categories"]}
 
 
-def app_list(full=False):
+
+# Old legacy function...
+def app_fetchlist():
+    logger.warning("'yunohost app fetchlist' is deprecated. Please use 'yunohost tools update --apps' instead")
+    from yunohost.tools import tools_update
+    tools_update(apps=True)
+
+
+def app_list(full=False, installed=False, filter=None):
     """
     List installed apps
     """
+
+    # Old legacy argument ... app_list was a combination of app_list and
+    # app_catalog before 3.8 ...
+    if installed:
+        logger.warning("Argument --installed ain't needed anymore when using 'yunohost app list'. It directly returns the list of installed apps..")
+
+    # Filter is a deprecated option...
+    if filter:
+        logger.warning("Using -f $appname in 'yunohost app list' is deprecated. Just use 'yunohost app list | grep -q 'id: $appname' to check a specific app is installed")
+
     out = []
     for app_id in sorted(_installed_apps()):
+
+        if filter and not app_id.startswith(filter):
+            continue
+
         try:
             app_info_dict = app_info(app_id, full=full)
         except Exception as e:

src/yunohost/certificate.py

@@ -34,15 +34,14 @@ import glob
 
 from datetime import datetime
 
-from yunohost.vendor.acme_tiny.acme_tiny import get_crt as sign_certificate
-
-from yunohost.utils.error import YunohostError
+from moulinette import m18n
 from moulinette.utils.log import getActionLogger
+from moulinette.utils.filesystem import read_file
 
+from yunohost.vendor.acme_tiny.acme_tiny import get_crt as sign_certificate
+from yunohost.utils.error import YunohostError
 from yunohost.utils.network import get_public_ip
 
-from moulinette import m18n
-from yunohost.app import app_ssowatconf
 from yunohost.service import _run_service_command
 from yunohost.regenconf import regen_conf
 from yunohost.log import OperationLogger
@@ -468,14 +467,15 @@ Subject: %s
 
 
 def _check_acme_challenge_configuration(domain):
-    # Check nginx conf file exists
-    nginx_conf_folder = "/etc/nginx/conf.d/%s.d" % domain
-    nginx_conf_file = "%s/000-acmechallenge.conf" % nginx_conf_folder
 
-    if not os.path.exists(nginx_conf_file):
-        return False
-    else:
+    domain_conf = "/etc/nginx/conf.d/%s.conf" % domain
+    if "include /etc/nginx/conf.d/acme-challenge.conf.inc" in read_file(domain_conf):
         return True
+    else:
+        # This is for legacy setups which haven't updated their domain conf to
+        # the new conf that include the acme snippet...
+        legacy_acme_conf = "/etc/nginx/conf.d/%s.d/000-acmechallenge.conf" % domain
+        return os.path.exists(legacy_acme_conf)
 
 
 def _fetch_and_enable_new_certificate(domain, staging=False, no_checks=False):
@@ -592,10 +592,10 @@ def _prepare_certificate_signing_request(domain, key_file, output_folder):
     # Set the domain
     csr.get_subject().CN = domain
 
-    from yunohost.domain import _get_maindomain
-    if domain == _get_maindomain():
-        # Include xmpp-upload subdomain in subject alternate names
-        subdomain="xmpp-upload." + domain
+    from yunohost.domain import domain_list
+    # For "parent" domains, include xmpp-upload subdomain in subject alternate names
+    if domain in domain_list(exclude_subdomains=True)["domains"]:
+        subdomain = "xmpp-upload." + domain
         try:
             _dns_ip_match_public_ip(get_public_ip(), subdomain)
             csr.add_extensions([crypto.X509Extension("subjectAltName", False, "DNS:" + subdomain)])

src/yunohost/domain.py

@@ -41,24 +41,26 @@ from yunohost.hook import hook_callback
 logger = getActionLogger('yunohost.domain')
 
 
-def domain_list():
+def domain_list(exclude_subdomains=False):
     """
     List domains
 
     Keyword argument:
-        filter -- LDAP filter used to search
-        offset -- Starting number for domain fetching
-        limit -- Maximum number of domain fetched
+        exclude_subdomains -- Filter out domains that are subdomains of other declared domains
 
     """
     from yunohost.utils.ldap import _get_ldap_interface
 
     ldap = _get_ldap_interface()
-    result = ldap.search('ou=domains,dc=yunohost,dc=org', 'virtualdomain=*', ['virtualdomain'])
+    result = [entry['virtualdomain'][0] for entry in ldap.search('ou=domains,dc=yunohost,dc=org', 'virtualdomain=*', ['virtualdomain'])]
 
     result_list = []
     for domain in result:
-        result_list.append(domain['virtualdomain'][0])
+        if exclude_subdomains:
+            parent_domain = domain.split(".", 1)[1]
+            if parent_domain in result:
+                continue
+        result_list.append(domain)
 
     return {'domains': result_list}
 

src/yunohost/utils/network.py

@@ -21,9 +21,11 @@
 import os
 import re
 import logging
+import dns.resolver
 
 from moulinette.utils.network import download_text
 from moulinette.utils.process import check_output
+from moulinette.utils.filesystem import read_file
 
 logger = logging.getLogger('yunohost.utils.network')
 
@@ -84,6 +86,52 @@ def get_gateway():
     return addr.popitem()[1] if len(addr) == 1 else None
 
 
+# Lazy dev caching to avoid re-reading the file multiple time when calling
+# dig() often during same yunohost operation
+external_resolvers_ = []
+
+
+def external_resolvers():
+
+    global external_resolvers_
+
+    if not external_resolvers_:
+        resolv_dnsmasq_conf = read_file("/etc/resolv.dnsmasq.conf").split("\n")
+        external_resolvers_ = [r.split(" ")[1] for r in resolv_dnsmasq_conf if r.startswith("nameserver")]
+
+    return external_resolvers_
+
+
+def dig(qname, rdtype="A", timeout=5, resolvers="local", edns_size=1500, full_answers=False):
+    """
+    Do a quick DNS request and avoid the "search" trap inside /etc/resolv.conf
+    """
+
+    if resolvers == "local":
+        resolvers = ["127.0.0.1"]
+    elif resolvers == "force_external":
+        resolvers = external_resolvers()
+    else:
+        assert isinstance(resolvers, list)
+
+    resolver = dns.resolver.Resolver(configure=False)
+    resolver.use_edns(0, 0, edns_size)
+    resolver.nameservers = resolvers
+    resolver.timeout = timeout
+    try:
+        answers = resolver.query(qname, rdtype)
+    except (dns.resolver.NXDOMAIN,
+            dns.resolver.NoNameservers,
+            dns.resolver.NoAnswer,
+            dns.exception.Timeout) as e:
+        return ("nok", (e.__class__.__name__, e))
+
+    if not full_answers:
+        answers = [answer.to_text() for answer in answers]
+
+    return ("ok", answers)
+
+
 def _extract_inet(string, skip_netmask=False, skip_loopback=True):
     """
     Extract IP addresses (v4 and/or v6) from a string limited to one