diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index e1229352c..0ad1268f2 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -406,6 +406,10 @@ domain: list: action_help: List domains api: GET /domains + arguments: + --exclude-subdomains: + help: Filter out domains that are obviously subdomains of other declared domains + action: store_true ### domain_add() add: @@ -563,6 +567,9 @@ app: help: Also return a list of app categories action: store_true + fetchlist: + deprecated: true + ### app_list() list: action_help: List installed apps @@ -572,6 +579,12 @@ app: full: --full help: Display all details, including the app manifest and various other infos action: store_true + -i: + full: --installed + help: Dummy argument, does nothing anymore (still there only for backward compatibility) + action: store_true + filter: + nargs: '?' ### app_info() info: diff --git a/data/helpers.d/postgresql b/data/helpers.d/postgresql index 4ac9fcbec..f7e9e00fc 100644 --- a/data/helpers.d/postgresql +++ b/data/helpers.d/postgresql @@ -87,6 +87,7 @@ ynh_psql_create_db() { # grant all privilegies to user if [ -n "$user" ]; then + sql+="ALTER DATABASE ${db} OWNER TO ${user};" sql+="GRANT ALL PRIVILEGES ON DATABASE ${db} TO ${user} WITH GRANT OPTION;" fi diff --git a/data/hooks/conf_regen/12-metronome b/data/hooks/conf_regen/12-metronome index 5a50b2b6e..55433e13c 100755 --- a/data/hooks/conf_regen/12-metronome +++ b/data/hooks/conf_regen/12-metronome @@ -43,16 +43,16 @@ do_post_regen() { # retrieve variables main_domain=$(cat /etc/yunohost/current_host) - domain_list=$(yunohost domain list --output-as plain --quiet) + domain_list=$(yunohost domain list --exclude-subdomains --output-as plain --quiet) # create metronome directories for domains for domain in $domain_list; do mkdir -p "/var/lib/metronome/${domain//./%2e}/pep" + # http_upload directory must be writable by metronome and readable by nginx + mkdir -p "/var/xmpp-upload/${domain}/upload" + chmod g+s "/var/xmpp-upload/${domain}/upload" + chown -R metronome:www-data "/var/xmpp-upload/${domain}" done - # http_upload directory must be writable by metronome and readable by nginx - mkdir -p "/var/xmpp-upload/${main_domain}/upload" - chmod g+s "/var/xmpp-upload/${main_domain}/upload" - chown -R metronome:www-data "/var/xmpp-upload/${main_domain}" # fix some permissions chown -R metronome: /var/lib/metronome/ diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 5ed7fc737..53afb2c2d 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -2,9 +2,9 @@ import os -from moulinette.utils.process import check_output from moulinette.utils.filesystem import read_file +from yunohost.utils.network import dig from yunohost.diagnosis import Diagnoser from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain @@ -100,20 +100,14 @@ class DNSRecordsDiagnoser(Diagnoser): yield output def get_current_record(self, domain, name, type_): - if name == "@": - command = "dig +short @%s %s %s" % (self.resolver, type_, domain) - else: - command = "dig +short @%s %s %s.%s" % (self.resolver, type_, name, domain) - # FIXME : gotta handle case where this command fails ... - # e.g. no internet connectivity (dependency mechanism to good result from 'ip' diagosis ?) - # or the resolver is unavailable for some reason - output = check_output(command).strip().split("\n") - if len(output) == 0 or not output[0]: + + query = "%s.%s" % (name, domain) if name != "@" else domain + success, answers = dig(query, type_, resolvers="force_external") + + if success != "ok": return None - elif len(output) == 1: - return output[0] else: - return output + return answers[0] if len(answers) == 1 else answers def current_record_match_expected(self, r): if r["value"] is not None and r["current"] is None: diff --git a/data/hooks/diagnosis/24-mail.py b/data/hooks/diagnosis/24-mail.py index 4ced72959..a60b4f0d4 100644 --- a/data/hooks/diagnosis/24-mail.py +++ b/data/hooks/diagnosis/24-mail.py @@ -13,6 +13,7 @@ from moulinette.utils.filesystem import read_yaml from yunohost.diagnosis import Diagnoser from yunohost.domain import _get_maindomain, domain_list from yunohost.settings import settings_get +from yunohost.utils.network import dig DEFAULT_DNS_BLACKLIST = "/usr/share/yunohost/other/dnsbl_list.yml" @@ -155,26 +156,25 @@ class MailDiagnoser(Diagnoser): if not blacklist[item_type]: continue - # Determine if we are listed on this RBL - try: - subdomain = item - if item_type != "domain": - rev = dns.reversename.from_address(item) - subdomain = str(rev.split(3)[0]) - query = subdomain + '.' + blacklist['dns_server'] - # TODO add timeout lifetime - dns.resolver.query(query, "A") - except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers, dns.resolver.NoAnswer, - dns.exception.Timeout): + # Build the query for DNSBL + subdomain = item + if item_type != "domain": + rev = dns.reversename.from_address(item) + subdomain = str(rev.split(3)[0]) + query = subdomain + '.' + blacklist['dns_server'] + + # Do the DNS Query + status, _ = dig(query, 'A') + if status != 'ok': continue # Try to get the reason details = [] - try: - reason = str(dns.resolver.query(query, "TXT")[0]) + status, answers = dig(query, 'TXT') + reason = "-" + if status == 'ok': + reason = ', '.join(answers) details.append("diagnosis_mail_blacklist_reason") - except Exception: - reason = "-" details.append("diagnosis_mail_blacklist_website") diff --git a/data/templates/metronome/domain.tpl.cfg.lua b/data/templates/metronome/domain.tpl.cfg.lua index e7f6bcef7..aa2f45e5a 100644 --- a/data/templates/metronome/domain.tpl.cfg.lua +++ b/data/templates/metronome/domain.tpl.cfg.lua @@ -1,4 +1,5 @@ VirtualHost "{{ domain }}" + enable = true ssl = { key = "/etc/yunohost/certs/{{ domain }}/key.pem"; certificate = "/etc/yunohost/certs/{{ domain }}/crt.pem"; @@ -13,3 +14,58 @@ VirtualHost "{{ domain }}" namefield = "cn", }, } + + -- Discovery items + disco_items = { + { "muc.{{ domain }}" }, + { "pubsub.{{ domain }}" }, + { "jabber.{{ domain }}" }, + { "vjud.{{ domain }}" }, + { "xmpp-upload.{{ domain }}" }, + }; + +-- contact_info = { +-- abuse = { "mailto:abuse@{{ domain }}", "xmpp:admin@{{ domain }}" }; +-- admin = { "mailto:root@{{ domain }}", "xmpp:admin@{{ domain }}" }; +-- }; + +------ Components ------ +-- You can specify components to add hosts that provide special services, +-- like multi-user conferences, and transports. + +---Set up a MUC (multi-user chat) room server +Component "muc.{{ domain }}" "muc" + name = "{{ domain }} Chatrooms" + + modules_enabled = { + "muc_limits"; + "muc_log"; + "muc_log_mam"; + "muc_log_http"; + "muc_vcard"; + } + + muc_event_rate = 0.5 + muc_burst_factor = 10 + +---Set up a PubSub server +Component "pubsub.{{ domain }}" "pubsub" + name = "{{ domain }} Publish/Subscribe" + + unrestricted_node_creation = true -- Anyone can create a PubSub node (from any server) + +---Set up a HTTP Upload service +Component "xmpp-upload.{{ domain }}" "http_upload" + name = "{{ domain }} Sharing Service" + + http_file_path = "/var/xmpp-upload/{{ domain }}/upload" + http_external_url = "https://xmpp-upload.{{ domain }}:443" + http_file_base_path = "/upload" + http_file_size_limit = 6*1024*1024 + http_file_quota = 60*1024*1024 + http_upload_file_size_limit = 100 * 1024 * 1024 -- bytes + http_upload_quota = 10 * 1024 * 1024 * 1024 -- bytes + +---Set up a VJUD service +Component "vjud.{{ domain }}" "vjud" + vjud_disco_name = "{{ domain }} User Directory" diff --git a/data/templates/metronome/metronome.cfg.lua b/data/templates/metronome/metronome.cfg.lua index b35684add..c1ea83281 100644 --- a/data/templates/metronome/metronome.cfg.lua +++ b/data/templates/metronome/metronome.cfg.lua @@ -81,14 +81,6 @@ http_interfaces = { "127.0.0.1", "::1" } -- Enable IPv6 use_ipv6 = true --- Discovery items -disco_items = { - { "muc.{{ main_domain }}" }, - { "pubsub.{{ main_domain }}" }, - { "xmpp-upload.{{ main_domain }}" }, - { "vjud.{{ main_domain }}" } -}; - -- BOSH configuration (mod_bosh) consider_bosh_secure = true cross_domain_bosh = true @@ -119,45 +111,6 @@ log = { Component "localhost" "http" modules_enabled = { "bosh" } ----Set up a MUC (multi-user chat) room server -Component "muc.{{ main_domain }}" "muc" - name = "{{ main_domain }} Chatrooms" - - modules_enabled = { - "muc_limits"; - "muc_log"; - "muc_log_mam"; - "muc_log_http"; - "muc_vcard"; - } - - muc_event_rate = 0.5 - muc_burst_factor = 10 - ----Set up a PubSub server -Component "pubsub.{{ main_domain }}" "pubsub" - name = "{{ main_domain }} Publish/Subscribe" - - unrestricted_node_creation = true -- Anyone can create a PubSub node (from any server) - ----Set up a HTTP Upload service -Component "xmpp-upload.{{ main_domain }}" "http_upload" - name = "{{ main_domain }} Sharing Service" - - http_file_path = "/var/xmpp-upload/{{ main_domain }}/upload" - http_external_url = "https://xmpp-upload.{{ main_domain }}:443" - http_file_base_path = "/upload" - http_file_size_limit = 6*1024*1024 - http_file_quota = 60*1024*1024 - http_upload_file_size_limit = 100 * 1024 * 1024 -- bytes - http_upload_quota = 10 * 1024 * 1024 * 1024 -- bytes - - ----Set up a VJUD service -Component "vjud.{{ main_domain }}" "vjud" - ud_disco_name = "{{ main_domain }} User Directory" - - ----------- Virtual hosts ----------- -- You need to add a VirtualHost entry for each domain you wish Metronome to serve. -- Settings under each VirtualHost entry apply *only* to that host. diff --git a/locales/en.json b/locales/en.json index c2c087031..aa1c4e4f2 100644 --- a/locales/en.json +++ b/locales/en.json @@ -110,7 +110,7 @@ "backup_unable_to_organize_files": "Could not use the quick method to organize files in the archive", "backup_with_no_backup_script_for_app": "The app '{app:s}' has no backup script. Ignoring.", "backup_with_no_restore_script_for_app": "The '{app:s}' has no restoration script, you will not be able to automatically restore the backup of this app.", - "certmanager_acme_not_configured_for_domain": "Certificate for the domain '{domain:s}' does not appear to be correctly installed. Please run 'cert-install' for this domain first.", + "certmanager_acme_not_configured_for_domain": "The ACME challenge cannot be ran for {domain} right now because its nginx conf lacks the corresponding code snippet... Please make sure that your nginx configuration is up to date using `yunohost tools regen-conf nginx --dry-run --with-diff`.", "certmanager_attempt_to_renew_nonLE_cert": "The certificate for the domain '{domain:s}' is not issued by Let's Encrypt. Cannot renew it automatically!", "certmanager_attempt_to_renew_valid_cert": "The certificate for the domain '{domain:s}' is not about to expire! (You may use --force if you know what you're doing)", "certmanager_attempt_to_replace_valid_cert": "You are attempting to overwrite a good and valid certificate for domain {domain:s}! (Use --force to bypass)", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index b94f57502..8dce2ff38 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -110,12 +110,34 @@ def app_catalog(full=False, with_categories=False): return {"apps": catalog["apps"], "categories": catalog["categories"]} -def app_list(full=False): + +# Old legacy function... +def app_fetchlist(): + logger.warning("'yunohost app fetchlist' is deprecated. Please use 'yunohost tools update --apps' instead") + from yunohost.tools import tools_update + tools_update(apps=True) + + +def app_list(full=False, installed=False, filter=None): """ List installed apps """ + + # Old legacy argument ... app_list was a combination of app_list and + # app_catalog before 3.8 ... + if installed: + logger.warning("Argument --installed ain't needed anymore when using 'yunohost app list'. It directly returns the list of installed apps..") + + # Filter is a deprecated option... + if filter: + logger.warning("Using -f $appname in 'yunohost app list' is deprecated. Just use 'yunohost app list | grep -q 'id: $appname' to check a specific app is installed") + out = [] for app_id in sorted(_installed_apps()): + + if filter and not app_id.startswith(filter): + continue + try: app_info_dict = app_info(app_id, full=full) except Exception as e: diff --git a/src/yunohost/certificate.py b/src/yunohost/certificate.py index fd792ccae..f3971be06 100644 --- a/src/yunohost/certificate.py +++ b/src/yunohost/certificate.py @@ -34,15 +34,14 @@ import glob from datetime import datetime -from yunohost.vendor.acme_tiny.acme_tiny import get_crt as sign_certificate - -from yunohost.utils.error import YunohostError +from moulinette import m18n from moulinette.utils.log import getActionLogger +from moulinette.utils.filesystem import read_file +from yunohost.vendor.acme_tiny.acme_tiny import get_crt as sign_certificate +from yunohost.utils.error import YunohostError from yunohost.utils.network import get_public_ip -from moulinette import m18n -from yunohost.app import app_ssowatconf from yunohost.service import _run_service_command from yunohost.regenconf import regen_conf from yunohost.log import OperationLogger @@ -468,14 +467,15 @@ Subject: %s def _check_acme_challenge_configuration(domain): - # Check nginx conf file exists - nginx_conf_folder = "/etc/nginx/conf.d/%s.d" % domain - nginx_conf_file = "%s/000-acmechallenge.conf" % nginx_conf_folder - if not os.path.exists(nginx_conf_file): - return False - else: + domain_conf = "/etc/nginx/conf.d/%s.conf" % domain + if "include /etc/nginx/conf.d/acme-challenge.conf.inc" in read_file(domain_conf): return True + else: + # This is for legacy setups which haven't updated their domain conf to + # the new conf that include the acme snippet... + legacy_acme_conf = "/etc/nginx/conf.d/%s.d/000-acmechallenge.conf" % domain + return os.path.exists(legacy_acme_conf) def _fetch_and_enable_new_certificate(domain, staging=False, no_checks=False): @@ -592,10 +592,10 @@ def _prepare_certificate_signing_request(domain, key_file, output_folder): # Set the domain csr.get_subject().CN = domain - from yunohost.domain import _get_maindomain - if domain == _get_maindomain(): - # Include xmpp-upload subdomain in subject alternate names - subdomain="xmpp-upload." + domain + from yunohost.domain import domain_list + # For "parent" domains, include xmpp-upload subdomain in subject alternate names + if domain in domain_list(exclude_subdomains=True)["domains"]: + subdomain = "xmpp-upload." + domain try: _dns_ip_match_public_ip(get_public_ip(), subdomain) csr.add_extensions([crypto.X509Extension("subjectAltName", False, "DNS:" + subdomain)]) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index c725b58c9..65320feeb 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -41,24 +41,26 @@ from yunohost.hook import hook_callback logger = getActionLogger('yunohost.domain') -def domain_list(): +def domain_list(exclude_subdomains=False): """ List domains Keyword argument: - filter -- LDAP filter used to search - offset -- Starting number for domain fetching - limit -- Maximum number of domain fetched + exclude_subdomains -- Filter out domains that are subdomains of other declared domains """ from yunohost.utils.ldap import _get_ldap_interface ldap = _get_ldap_interface() - result = ldap.search('ou=domains,dc=yunohost,dc=org', 'virtualdomain=*', ['virtualdomain']) + result = [entry['virtualdomain'][0] for entry in ldap.search('ou=domains,dc=yunohost,dc=org', 'virtualdomain=*', ['virtualdomain'])] result_list = [] for domain in result: - result_list.append(domain['virtualdomain'][0]) + if exclude_subdomains: + parent_domain = domain.split(".", 1)[1] + if parent_domain in result: + continue + result_list.append(domain) return {'domains': result_list} diff --git a/src/yunohost/utils/network.py b/src/yunohost/utils/network.py index 3ae1ba910..23b2310f8 100644 --- a/src/yunohost/utils/network.py +++ b/src/yunohost/utils/network.py @@ -21,9 +21,11 @@ import os import re import logging +import dns.resolver from moulinette.utils.network import download_text from moulinette.utils.process import check_output +from moulinette.utils.filesystem import read_file logger = logging.getLogger('yunohost.utils.network') @@ -84,6 +86,52 @@ def get_gateway(): return addr.popitem()[1] if len(addr) == 1 else None +# Lazy dev caching to avoid re-reading the file multiple time when calling +# dig() often during same yunohost operation +external_resolvers_ = [] + + +def external_resolvers(): + + global external_resolvers_ + + if not external_resolvers_: + resolv_dnsmasq_conf = read_file("/etc/resolv.dnsmasq.conf").split("\n") + external_resolvers_ = [r.split(" ")[1] for r in resolv_dnsmasq_conf if r.startswith("nameserver")] + + return external_resolvers_ + + +def dig(qname, rdtype="A", timeout=5, resolvers="local", edns_size=1500, full_answers=False): + """ + Do a quick DNS request and avoid the "search" trap inside /etc/resolv.conf + """ + + if resolvers == "local": + resolvers = ["127.0.0.1"] + elif resolvers == "force_external": + resolvers = external_resolvers() + else: + assert isinstance(resolvers, list) + + resolver = dns.resolver.Resolver(configure=False) + resolver.use_edns(0, 0, edns_size) + resolver.nameservers = resolvers + resolver.timeout = timeout + try: + answers = resolver.query(qname, rdtype) + except (dns.resolver.NXDOMAIN, + dns.resolver.NoNameservers, + dns.resolver.NoAnswer, + dns.exception.Timeout) as e: + return ("nok", (e.__class__.__name__, e)) + + if not full_answers: + answers = [answer.to_text() for answer in answers] + + return ("ok", answers) + + def _extract_inet(string, skip_netmask=False, skip_loopback=True): """ Extract IP addresses (v4 and/or v6) from a string limited to one