From 4500f56c32f93ca30698b905d384aca04e580a78 Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Fri, 27 Sep 2019 11:54:44 +0200 Subject: [PATCH 01/14] [fix] Psql user should own the database --- data/helpers.d/postgresql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/postgresql b/data/helpers.d/postgresql index d252ae2dc..e40553f9e 100644 --- a/data/helpers.d/postgresql +++ b/data/helpers.d/postgresql @@ -87,7 +87,7 @@ ynh_psql_create_db() { # grant all privilegies to user if [ -n "$user" ]; then - sql+="GRANT ALL PRIVILEGES ON DATABASE ${db} TO ${user} WITH GRANT OPTION;" + sql+="OWNER ${user} GRANT ALL PRIVILEGES ON DATABASE ${db} TO ${user} WITH GRANT OPTION;" fi ynh_psql_execute_as_root --sql="$sql" From b41d7b47a48dc3e3fb44284a43831da86e8efec6 Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Fri, 27 Sep 2019 11:59:11 +0200 Subject: [PATCH 02/14] [fix] SQL error --- data/helpers.d/postgresql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/postgresql b/data/helpers.d/postgresql index e40553f9e..1dac6715d 100644 --- a/data/helpers.d/postgresql +++ b/data/helpers.d/postgresql @@ -87,7 +87,7 @@ ynh_psql_create_db() { # grant all privilegies to user if [ -n "$user" ]; then - sql+="OWNER ${user} GRANT ALL PRIVILEGES ON DATABASE ${db} TO ${user} WITH GRANT OPTION;" + sql+="ALTER DATABASE ${db} OWNER TO ${user}; GRANT ALL PRIVILEGES ON DATABASE ${db} TO ${user} WITH GRANT OPTION;" fi ynh_psql_execute_as_root --sql="$sql" From 748dcfd8c5f869dd6d78fe471c9a1eb30d1a3334 Mon Sep 17 00:00:00 2001 From: pitchum Date: Sat, 4 Apr 2020 14:36:01 +0200 Subject: [PATCH 03/14] Setup XMPP components for each domain, not only the main domain. --- data/hooks/conf_regen/12-metronome | 8 +-- data/templates/metronome/domain.tpl.cfg.lua | 56 +++++++++++++++++++++ data/templates/metronome/metronome.cfg.lua | 47 ----------------- src/yunohost/certificate.py | 16 +++--- 4 files changed, 67 insertions(+), 60 deletions(-) diff --git a/data/hooks/conf_regen/12-metronome b/data/hooks/conf_regen/12-metronome index 5c9c67f11..903e9fb2e 100755 --- a/data/hooks/conf_regen/12-metronome +++ b/data/hooks/conf_regen/12-metronome @@ -48,11 +48,11 @@ do_post_regen() { # create metronome directories for domains for domain in $domain_list; do mkdir -p "/var/lib/metronome/${domain//./%2e}/pep" + # http_upload directory must be writable by metronome and readable by nginx + mkdir -p "/var/xmpp-upload/${domain}/upload" + chmod g+s "/var/xmpp-upload/${domain}/upload" + chown -R metronome:www-data "/var/xmpp-upload/${domain}" done - # http_upload directory must be writable by metronome and readable by nginx - mkdir -p "/var/xmpp-upload/${main_domain}/upload" - chmod g+s "/var/xmpp-upload/${main_domain}/upload" - chown -R metronome:www-data "/var/xmpp-upload/${main_domain}" # fix some permissions diff --git a/data/templates/metronome/domain.tpl.cfg.lua b/data/templates/metronome/domain.tpl.cfg.lua index e7f6bcef7..aa2f45e5a 100644 --- a/data/templates/metronome/domain.tpl.cfg.lua +++ b/data/templates/metronome/domain.tpl.cfg.lua @@ -1,4 +1,5 @@ VirtualHost "{{ domain }}" + enable = true ssl = { key = "/etc/yunohost/certs/{{ domain }}/key.pem"; certificate = "/etc/yunohost/certs/{{ domain }}/crt.pem"; @@ -13,3 +14,58 @@ VirtualHost "{{ domain }}" namefield = "cn", }, } + + -- Discovery items + disco_items = { + { "muc.{{ domain }}" }, + { "pubsub.{{ domain }}" }, + { "jabber.{{ domain }}" }, + { "vjud.{{ domain }}" }, + { "xmpp-upload.{{ domain }}" }, + }; + +-- contact_info = { +-- abuse = { "mailto:abuse@{{ domain }}", "xmpp:admin@{{ domain }}" }; +-- admin = { "mailto:root@{{ domain }}", "xmpp:admin@{{ domain }}" }; +-- }; + +------ Components ------ +-- You can specify components to add hosts that provide special services, +-- like multi-user conferences, and transports. + +---Set up a MUC (multi-user chat) room server +Component "muc.{{ domain }}" "muc" + name = "{{ domain }} Chatrooms" + + modules_enabled = { + "muc_limits"; + "muc_log"; + "muc_log_mam"; + "muc_log_http"; + "muc_vcard"; + } + + muc_event_rate = 0.5 + muc_burst_factor = 10 + +---Set up a PubSub server +Component "pubsub.{{ domain }}" "pubsub" + name = "{{ domain }} Publish/Subscribe" + + unrestricted_node_creation = true -- Anyone can create a PubSub node (from any server) + +---Set up a HTTP Upload service +Component "xmpp-upload.{{ domain }}" "http_upload" + name = "{{ domain }} Sharing Service" + + http_file_path = "/var/xmpp-upload/{{ domain }}/upload" + http_external_url = "https://xmpp-upload.{{ domain }}:443" + http_file_base_path = "/upload" + http_file_size_limit = 6*1024*1024 + http_file_quota = 60*1024*1024 + http_upload_file_size_limit = 100 * 1024 * 1024 -- bytes + http_upload_quota = 10 * 1024 * 1024 * 1024 -- bytes + +---Set up a VJUD service +Component "vjud.{{ domain }}" "vjud" + vjud_disco_name = "{{ domain }} User Directory" diff --git a/data/templates/metronome/metronome.cfg.lua b/data/templates/metronome/metronome.cfg.lua index b35684add..c1ea83281 100644 --- a/data/templates/metronome/metronome.cfg.lua +++ b/data/templates/metronome/metronome.cfg.lua @@ -81,14 +81,6 @@ http_interfaces = { "127.0.0.1", "::1" } -- Enable IPv6 use_ipv6 = true --- Discovery items -disco_items = { - { "muc.{{ main_domain }}" }, - { "pubsub.{{ main_domain }}" }, - { "xmpp-upload.{{ main_domain }}" }, - { "vjud.{{ main_domain }}" } -}; - -- BOSH configuration (mod_bosh) consider_bosh_secure = true cross_domain_bosh = true @@ -119,45 +111,6 @@ log = { Component "localhost" "http" modules_enabled = { "bosh" } ----Set up a MUC (multi-user chat) room server -Component "muc.{{ main_domain }}" "muc" - name = "{{ main_domain }} Chatrooms" - - modules_enabled = { - "muc_limits"; - "muc_log"; - "muc_log_mam"; - "muc_log_http"; - "muc_vcard"; - } - - muc_event_rate = 0.5 - muc_burst_factor = 10 - ----Set up a PubSub server -Component "pubsub.{{ main_domain }}" "pubsub" - name = "{{ main_domain }} Publish/Subscribe" - - unrestricted_node_creation = true -- Anyone can create a PubSub node (from any server) - ----Set up a HTTP Upload service -Component "xmpp-upload.{{ main_domain }}" "http_upload" - name = "{{ main_domain }} Sharing Service" - - http_file_path = "/var/xmpp-upload/{{ main_domain }}/upload" - http_external_url = "https://xmpp-upload.{{ main_domain }}:443" - http_file_base_path = "/upload" - http_file_size_limit = 6*1024*1024 - http_file_quota = 60*1024*1024 - http_upload_file_size_limit = 100 * 1024 * 1024 -- bytes - http_upload_quota = 10 * 1024 * 1024 * 1024 -- bytes - - ----Set up a VJUD service -Component "vjud.{{ main_domain }}" "vjud" - ud_disco_name = "{{ main_domain }} User Directory" - - ----------- Virtual hosts ----------- -- You need to add a VirtualHost entry for each domain you wish Metronome to serve. -- Settings under each VirtualHost entry apply *only* to that host. diff --git a/src/yunohost/certificate.py b/src/yunohost/certificate.py index 5fae59060..c6f520b4e 100644 --- a/src/yunohost/certificate.py +++ b/src/yunohost/certificate.py @@ -639,15 +639,13 @@ def _prepare_certificate_signing_request(domain, key_file, output_folder): # Set the domain csr.get_subject().CN = domain - from yunohost.domain import _get_maindomain - if domain == _get_maindomain(): - # Include xmpp-upload subdomain in subject alternate names - subdomain="xmpp-upload." + domain - try: - _dns_ip_match_public_ip(get_public_ip(), subdomain) - csr.add_extensions([crypto.X509Extension("subjectAltName", False, "DNS:" + subdomain)]) - except YunohostError: - logger.warning(m18n.n('certmanager_warning_subdomain_dns_record', subdomain=subdomain, domain=domain)) + # Include xmpp-upload subdomain in subject alternate names + subdomain="xmpp-upload." + domain + try: + _dns_ip_match_public_ip(get_public_ip(), subdomain) + csr.add_extensions([crypto.X509Extension("subjectAltName", False, "DNS:" + subdomain)]) + except YunohostError: + logger.warning(m18n.n('certmanager_warning_subdomain_dns_record', subdomain=subdomain, domain=domain)) # Set the key with open(key_file, 'rt') as f: From cf3b98b5237db1f67a28af7d9fd4f5852dc0a593 Mon Sep 17 00:00:00 2001 From: pitchum Date: Sat, 4 Apr 2020 15:06:44 +0200 Subject: [PATCH 04/14] Fix nginx config for xmpp-upload. --- data/templates/nginx/server.tpl.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/templates/nginx/server.tpl.conf b/data/templates/nginx/server.tpl.conf index 6316960c4..5a5176688 100644 --- a/data/templates/nginx/server.tpl.conf +++ b/data/templates/nginx/server.tpl.conf @@ -71,7 +71,7 @@ server { root /dev/null; location /upload/ { - alias /var/xmpp-upload/{{ domain }}/upload; + alias /var/xmpp-upload/{{ domain }}/upload/; # Pass all requests to metronome, except for GET and HEAD requests. limit_except GET HEAD { proxy_pass http://localhost:5290; From 22c88dc47e57980058265ae1083a5a8ef4284310 Mon Sep 17 00:00:00 2001 From: pitchum Date: Mon, 6 Apr 2020 20:38:42 +0200 Subject: [PATCH 05/14] Enable XMPP features only on "parent domains". --- data/actionsmap/yunohost.yml | 4 ++++ data/hooks/conf_regen/12-metronome | 2 +- src/yunohost/domain.py | 11 ++++++++--- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 3a4c9db97..cd1c4916f 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -399,6 +399,10 @@ domain: list: action_help: List domains api: GET /domains + arguments: + --exclude-subdomains: + help: Filter out domains that are obviously subdomains of other declared domains + action: store_true ### domain_add() add: diff --git a/data/hooks/conf_regen/12-metronome b/data/hooks/conf_regen/12-metronome index 903e9fb2e..25ccd40ac 100755 --- a/data/hooks/conf_regen/12-metronome +++ b/data/hooks/conf_regen/12-metronome @@ -43,7 +43,7 @@ do_post_regen() { # retrieve variables main_domain=$(cat /etc/yunohost/current_host) - domain_list=$(yunohost domain list --output-as plain --quiet) + domain_list=$(yunohost domain list --exclude-subdomains --output-as plain --quiet) # create metronome directories for domains for domain in $domain_list; do diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 23b5a4179..a1ac65b81 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -41,7 +41,7 @@ from yunohost.hook import hook_callback logger = getActionLogger('yunohost.domain') -def domain_list(): +def domain_list(exclude_subdomains=False): """ List domains @@ -49,16 +49,21 @@ def domain_list(): filter -- LDAP filter used to search offset -- Starting number for domain fetching limit -- Maximum number of domain fetched + exclude_subdomains -- Filter out domains that are subdomains of other declared domains """ from yunohost.utils.ldap import _get_ldap_interface ldap = _get_ldap_interface() - result = ldap.search('ou=domains,dc=yunohost,dc=org', 'virtualdomain=*', ['virtualdomain']) + result = [entry['virtualdomain'][0] for entry in ldap.search('ou=domains,dc=yunohost,dc=org', 'virtualdomain=*', ['virtualdomain'])] result_list = [] for domain in result: - result_list.append(domain['virtualdomain'][0]) + if exclude_subdomains: + parent_domain = domain.split(".", 1)[1] + if parent_domain in result: + continue + result_list.append(domain) return {'domains': result_list} From c42f7172f7a4ada26209cac392c844a2d57c6d01 Mon Sep 17 00:00:00 2001 From: pitchum Date: Wed, 22 Apr 2020 10:34:40 +0200 Subject: [PATCH 06/14] Do not include xmpp-upload in certificates of "child" domains Co-Authored-By: Alexandre Aubin --- src/yunohost/certificate.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/yunohost/certificate.py b/src/yunohost/certificate.py index c6f520b4e..aa137c784 100644 --- a/src/yunohost/certificate.py +++ b/src/yunohost/certificate.py @@ -639,13 +639,15 @@ def _prepare_certificate_signing_request(domain, key_file, output_folder): # Set the domain csr.get_subject().CN = domain - # Include xmpp-upload subdomain in subject alternate names - subdomain="xmpp-upload." + domain - try: - _dns_ip_match_public_ip(get_public_ip(), subdomain) - csr.add_extensions([crypto.X509Extension("subjectAltName", False, "DNS:" + subdomain)]) - except YunohostError: - logger.warning(m18n.n('certmanager_warning_subdomain_dns_record', subdomain=subdomain, domain=domain)) + from yunohost.domain import domain_list + # For "parent" domains, include xmpp-upload subdomain in subject alternate names + if domain in domain_list(exclude_subdomains=True)["domains"]: + subdomain="xmpp-upload." + domain + try: + _dns_ip_match_public_ip(get_public_ip(), subdomain) + csr.add_extensions([crypto.X509Extension("subjectAltName", False, "DNS:" + subdomain)]) + except YunohostError: + logger.warning(m18n.n('certmanager_warning_subdomain_dns_record', subdomain=subdomain, domain=domain)) # Set the key with open(key_file, 'rt') as f: From b9e226caed6d6fc9f775a9d3121a30ad258c0a70 Mon Sep 17 00:00:00 2001 From: pitchum Date: Fri, 24 Apr 2020 19:07:05 +0200 Subject: [PATCH 07/14] Remove deprecated docstrings. --- src/yunohost/domain.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index a1ac65b81..85d804584 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -46,9 +46,6 @@ def domain_list(exclude_subdomains=False): List domains Keyword argument: - filter -- LDAP filter used to search - offset -- Starting number for domain fetching - limit -- Maximum number of domain fetched exclude_subdomains -- Filter out domains that are subdomains of other declared domains """ From 69938c3feb50c2c72d9e7208b8b88c27d6f70174 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 26 Apr 2020 03:43:05 +0200 Subject: [PATCH 08/14] Re-add 'app fetchlist', 'app list -i', 'app list' filter for backward compatibility... --- data/actionsmap/yunohost.yml | 9 +++++++++ src/yunohost/app.py | 24 +++++++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index e1229352c..d55303d08 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -563,6 +563,9 @@ app: help: Also return a list of app categories action: store_true + fetchlist: + deprecated: true + ### app_list() list: action_help: List installed apps @@ -572,6 +575,12 @@ app: full: --full help: Display all details, including the app manifest and various other infos action: store_true + -i: + full: --installed + help: Dummy argument, does nothing anymore (still there only for backward compatibility) + action: store_true + filter: + nargs: '?' ### app_info() info: diff --git a/src/yunohost/app.py b/src/yunohost/app.py index b94f57502..8dce2ff38 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -110,12 +110,34 @@ def app_catalog(full=False, with_categories=False): return {"apps": catalog["apps"], "categories": catalog["categories"]} -def app_list(full=False): + +# Old legacy function... +def app_fetchlist(): + logger.warning("'yunohost app fetchlist' is deprecated. Please use 'yunohost tools update --apps' instead") + from yunohost.tools import tools_update + tools_update(apps=True) + + +def app_list(full=False, installed=False, filter=None): """ List installed apps """ + + # Old legacy argument ... app_list was a combination of app_list and + # app_catalog before 3.8 ... + if installed: + logger.warning("Argument --installed ain't needed anymore when using 'yunohost app list'. It directly returns the list of installed apps..") + + # Filter is a deprecated option... + if filter: + logger.warning("Using -f $appname in 'yunohost app list' is deprecated. Just use 'yunohost app list | grep -q 'id: $appname' to check a specific app is installed") + out = [] for app_id in sorted(_installed_apps()): + + if filter and not app_id.startswith(filter): + continue + try: app_info_dict = app_info(app_id, full=full) except Exception as e: From c6c85556ace4e720715ae38f46cc5b2f4f00de35 Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 19 Apr 2020 21:45:46 +0200 Subject: [PATCH 09/14] [fix] False positive on blacklist due to search in resovconf --- data/hooks/diagnosis/24-mail.py | 30 +++++++++++++++--------------- src/yunohost/utils/network.py | 19 +++++++++++++++++++ 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/data/hooks/diagnosis/24-mail.py b/data/hooks/diagnosis/24-mail.py index 4ced72959..afb88f7cf 100644 --- a/data/hooks/diagnosis/24-mail.py +++ b/data/hooks/diagnosis/24-mail.py @@ -13,6 +13,7 @@ from moulinette.utils.filesystem import read_yaml from yunohost.diagnosis import Diagnoser from yunohost.domain import _get_maindomain, domain_list from yunohost.settings import settings_get +from yunohost.utils.network import dig DEFAULT_DNS_BLACKLIST = "/usr/share/yunohost/other/dnsbl_list.yml" @@ -155,26 +156,25 @@ class MailDiagnoser(Diagnoser): if not blacklist[item_type]: continue - # Determine if we are listed on this RBL - try: - subdomain = item - if item_type != "domain": - rev = dns.reversename.from_address(item) - subdomain = str(rev.split(3)[0]) - query = subdomain + '.' + blacklist['dns_server'] - # TODO add timeout lifetime - dns.resolver.query(query, "A") - except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers, dns.resolver.NoAnswer, - dns.exception.Timeout): + # Build the query for DNSBL + subdomain = item + if item_type != "domain": + rev = dns.reversename.from_address(item) + subdomain = str(rev.split(3)[0]) + query = subdomain + '.' + blacklist['dns_server'] + + # Do the DNS Query + status, answers = dig(query, 'A') + if status != 'ok': continue # Try to get the reason details = [] - try: - reason = str(dns.resolver.query(query, "TXT")[0]) + status, answers = dig(query, 'TXT') + reason = "-" + if status == 'ok': + reason = ', '.join(answers) details.append("diagnosis_mail_blacklist_reason") - except Exception: - reason = "-" details.append("diagnosis_mail_blacklist_website") diff --git a/src/yunohost/utils/network.py b/src/yunohost/utils/network.py index 3ae1ba910..6dc4c22a0 100644 --- a/src/yunohost/utils/network.py +++ b/src/yunohost/utils/network.py @@ -21,6 +21,7 @@ import os import re import logging +import dns.resolver from moulinette.utils.network import download_text from moulinette.utils.process import check_output @@ -84,6 +85,24 @@ def get_gateway(): return addr.popitem()[1] if len(addr) == 1 else None +def dig(qname, rdtype="A", timeout=5, resolvers=["127.0.0.1"], edns_size=1500): + """ + Do a quick DNS request and avoid the "search" trap inside /etc/resolv.conf + """ + + resolver = dns.resolver.Resolver(configure=False) + resolver.use_edns(0, 0, edns_size) + resolver.nameservers = resolvers + resolver.timeout = timeout + try: + answers = resolver.query(qname, rdtype) + except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers, dns.resolver.NoAnswer, + dns.exception.Timeout) as e: + return ("nok", e.__class__.__name__, e) + + return ("ok", [(answer.to_text(), answer) for answer in answers]) + + def _extract_inet(string, skip_netmask=False, skip_loopback=True): """ Extract IP addresses (v4 and/or v6) from a string limited to one From 17d3ec5ad3e083df4920d3550151caee2c1ae7ca Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 Apr 2020 17:24:57 +0200 Subject: [PATCH 10/14] Improve new dig() helper, and use it in dnsrecords diagnosis as well --- data/hooks/diagnosis/12-dnsrecords.py | 20 +++++--------- data/hooks/diagnosis/24-mail.py | 2 +- src/yunohost/utils/network.py | 39 +++++++++++++++++++++++---- 3 files changed, 42 insertions(+), 19 deletions(-) diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 5ed7fc737..53afb2c2d 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -2,9 +2,9 @@ import os -from moulinette.utils.process import check_output from moulinette.utils.filesystem import read_file +from yunohost.utils.network import dig from yunohost.diagnosis import Diagnoser from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain @@ -100,20 +100,14 @@ class DNSRecordsDiagnoser(Diagnoser): yield output def get_current_record(self, domain, name, type_): - if name == "@": - command = "dig +short @%s %s %s" % (self.resolver, type_, domain) - else: - command = "dig +short @%s %s %s.%s" % (self.resolver, type_, name, domain) - # FIXME : gotta handle case where this command fails ... - # e.g. no internet connectivity (dependency mechanism to good result from 'ip' diagosis ?) - # or the resolver is unavailable for some reason - output = check_output(command).strip().split("\n") - if len(output) == 0 or not output[0]: + + query = "%s.%s" % (name, domain) if name != "@" else domain + success, answers = dig(query, type_, resolvers="force_external") + + if success != "ok": return None - elif len(output) == 1: - return output[0] else: - return output + return answers[0] if len(answers) == 1 else answers def current_record_match_expected(self, r): if r["value"] is not None and r["current"] is None: diff --git a/data/hooks/diagnosis/24-mail.py b/data/hooks/diagnosis/24-mail.py index afb88f7cf..a60b4f0d4 100644 --- a/data/hooks/diagnosis/24-mail.py +++ b/data/hooks/diagnosis/24-mail.py @@ -164,7 +164,7 @@ class MailDiagnoser(Diagnoser): query = subdomain + '.' + blacklist['dns_server'] # Do the DNS Query - status, answers = dig(query, 'A') + status, _ = dig(query, 'A') if status != 'ok': continue diff --git a/src/yunohost/utils/network.py b/src/yunohost/utils/network.py index 6dc4c22a0..23b2310f8 100644 --- a/src/yunohost/utils/network.py +++ b/src/yunohost/utils/network.py @@ -25,6 +25,7 @@ import dns.resolver from moulinette.utils.network import download_text from moulinette.utils.process import check_output +from moulinette.utils.filesystem import read_file logger = logging.getLogger('yunohost.utils.network') @@ -85,22 +86,50 @@ def get_gateway(): return addr.popitem()[1] if len(addr) == 1 else None -def dig(qname, rdtype="A", timeout=5, resolvers=["127.0.0.1"], edns_size=1500): +# Lazy dev caching to avoid re-reading the file multiple time when calling +# dig() often during same yunohost operation +external_resolvers_ = [] + + +def external_resolvers(): + + global external_resolvers_ + + if not external_resolvers_: + resolv_dnsmasq_conf = read_file("/etc/resolv.dnsmasq.conf").split("\n") + external_resolvers_ = [r.split(" ")[1] for r in resolv_dnsmasq_conf if r.startswith("nameserver")] + + return external_resolvers_ + + +def dig(qname, rdtype="A", timeout=5, resolvers="local", edns_size=1500, full_answers=False): """ Do a quick DNS request and avoid the "search" trap inside /etc/resolv.conf """ + if resolvers == "local": + resolvers = ["127.0.0.1"] + elif resolvers == "force_external": + resolvers = external_resolvers() + else: + assert isinstance(resolvers, list) + resolver = dns.resolver.Resolver(configure=False) resolver.use_edns(0, 0, edns_size) resolver.nameservers = resolvers resolver.timeout = timeout try: answers = resolver.query(qname, rdtype) - except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers, dns.resolver.NoAnswer, - dns.exception.Timeout) as e: - return ("nok", e.__class__.__name__, e) + except (dns.resolver.NXDOMAIN, + dns.resolver.NoNameservers, + dns.resolver.NoAnswer, + dns.exception.Timeout) as e: + return ("nok", (e.__class__.__name__, e)) - return ("ok", [(answer.to_text(), answer) for answer in answers]) + if not full_answers: + answers = [answer.to_text() for answer in answers] + + return ("ok", answers) def _extract_inet(string, skip_netmask=False, skip_loopback=True): From c1262ab9a93855e241d2c25197de05858547ab36 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 22 Apr 2020 03:09:28 +0200 Subject: [PATCH 11/14] Fix acme challenge code snippet detection for this domain --- locales/en.json | 2 +- src/yunohost/certificate.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/locales/en.json b/locales/en.json index c2c087031..b23d3b5c3 100644 --- a/locales/en.json +++ b/locales/en.json @@ -110,7 +110,7 @@ "backup_unable_to_organize_files": "Could not use the quick method to organize files in the archive", "backup_with_no_backup_script_for_app": "The app '{app:s}' has no backup script. Ignoring.", "backup_with_no_restore_script_for_app": "The '{app:s}' has no restoration script, you will not be able to automatically restore the backup of this app.", - "certmanager_acme_not_configured_for_domain": "Certificate for the domain '{domain:s}' does not appear to be correctly installed. Please run 'cert-install' for this domain first.", + "certmanager_acme_not_configured_for_domain": "The ACME challenge cannot be ran for this domain right now because you are missing a code snippet in nginx conf... Please make sure that your nginx configuration is up to date using `yunohost tools regen-conf nginx --dry-run --with-diff`.", "certmanager_attempt_to_renew_nonLE_cert": "The certificate for the domain '{domain:s}' is not issued by Let's Encrypt. Cannot renew it automatically!", "certmanager_attempt_to_renew_valid_cert": "The certificate for the domain '{domain:s}' is not about to expire! (You may use --force if you know what you're doing)", "certmanager_attempt_to_replace_valid_cert": "You are attempting to overwrite a good and valid certificate for domain {domain:s}! (Use --force to bypass)", diff --git a/src/yunohost/certificate.py b/src/yunohost/certificate.py index fd792ccae..89aadce99 100644 --- a/src/yunohost/certificate.py +++ b/src/yunohost/certificate.py @@ -38,6 +38,7 @@ from yunohost.vendor.acme_tiny.acme_tiny import get_crt as sign_certificate from yunohost.utils.error import YunohostError from moulinette.utils.log import getActionLogger +from moulinette.utils.filesystem import read_file from yunohost.utils.network import get_public_ip @@ -468,14 +469,15 @@ Subject: %s def _check_acme_challenge_configuration(domain): - # Check nginx conf file exists - nginx_conf_folder = "/etc/nginx/conf.d/%s.d" % domain - nginx_conf_file = "%s/000-acmechallenge.conf" % nginx_conf_folder - if not os.path.exists(nginx_conf_file): - return False - else: + domain_conf = "/etc/nginx/conf.d/%s.conf" % domain + if "include /etc/nginx/conf.d/acme-challenge.conf.inc" in read_file(domain_conf): return True + else: + # This is for legacy setups which haven't updated their domain conf to + # the new conf that include the acme snippet... + legacy_acme_conf = "/etc/nginx/conf.d/%s.d/000-acmechallenge.conf" % domain + return os.path.exists(legacy_acme_conf) def _fetch_and_enable_new_certificate(domain, staging=False, no_checks=False): From 32c300e62742da4645e15797da4eb317074a4da5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 22 Apr 2020 03:09:50 +0200 Subject: [PATCH 12/14] Reorganize import, make linter happier --- src/yunohost/certificate.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/yunohost/certificate.py b/src/yunohost/certificate.py index 89aadce99..5558caad5 100644 --- a/src/yunohost/certificate.py +++ b/src/yunohost/certificate.py @@ -34,16 +34,14 @@ import glob from datetime import datetime -from yunohost.vendor.acme_tiny.acme_tiny import get_crt as sign_certificate - -from yunohost.utils.error import YunohostError +from moulinette import m18n from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_file +from yunohost.vendor.acme_tiny.acme_tiny import get_crt as sign_certificate +from yunohost.utils.error import YunohostError from yunohost.utils.network import get_public_ip -from moulinette import m18n -from yunohost.app import app_ssowatconf from yunohost.service import _run_service_command from yunohost.regenconf import regen_conf from yunohost.log import OperationLogger @@ -597,7 +595,7 @@ def _prepare_certificate_signing_request(domain, key_file, output_folder): from yunohost.domain import _get_maindomain if domain == _get_maindomain(): # Include xmpp-upload subdomain in subject alternate names - subdomain="xmpp-upload." + domain + subdomain = "xmpp-upload." + domain try: _dns_ip_match_public_ip(get_public_ip(), subdomain) csr.add_extensions([crypto.X509Extension("subjectAltName", False, "DNS:" + subdomain)]) From f91eeff9dd3c09f8d8bfcf509541f484c536d340 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 22 Apr 2020 03:49:53 +0200 Subject: [PATCH 13/14] Uhoh we should use {domain}, fix wording.. --- locales/en.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locales/en.json b/locales/en.json index b23d3b5c3..aa1c4e4f2 100644 --- a/locales/en.json +++ b/locales/en.json @@ -110,7 +110,7 @@ "backup_unable_to_organize_files": "Could not use the quick method to organize files in the archive", "backup_with_no_backup_script_for_app": "The app '{app:s}' has no backup script. Ignoring.", "backup_with_no_restore_script_for_app": "The '{app:s}' has no restoration script, you will not be able to automatically restore the backup of this app.", - "certmanager_acme_not_configured_for_domain": "The ACME challenge cannot be ran for this domain right now because you are missing a code snippet in nginx conf... Please make sure that your nginx configuration is up to date using `yunohost tools regen-conf nginx --dry-run --with-diff`.", + "certmanager_acme_not_configured_for_domain": "The ACME challenge cannot be ran for {domain} right now because its nginx conf lacks the corresponding code snippet... Please make sure that your nginx configuration is up to date using `yunohost tools regen-conf nginx --dry-run --with-diff`.", "certmanager_attempt_to_renew_nonLE_cert": "The certificate for the domain '{domain:s}' is not issued by Let's Encrypt. Cannot renew it automatically!", "certmanager_attempt_to_renew_valid_cert": "The certificate for the domain '{domain:s}' is not about to expire! (You may use --force if you know what you're doing)", "certmanager_attempt_to_replace_valid_cert": "You are attempting to overwrite a good and valid certificate for domain {domain:s}! (Use --force to bypass)", From d6b2275b33a52f978c87770405f7acad44ab0471 Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Mon, 27 Apr 2020 18:30:33 +0200 Subject: [PATCH 14/14] [enh] On 2 lines it's better --- data/helpers.d/postgresql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data/helpers.d/postgresql b/data/helpers.d/postgresql index 1dac6715d..aac223214 100644 --- a/data/helpers.d/postgresql +++ b/data/helpers.d/postgresql @@ -87,7 +87,8 @@ ynh_psql_create_db() { # grant all privilegies to user if [ -n "$user" ]; then - sql+="ALTER DATABASE ${db} OWNER TO ${user}; GRANT ALL PRIVILEGES ON DATABASE ${db} TO ${user} WITH GRANT OPTION;" + sql+="ALTER DATABASE ${db} OWNER TO ${user};" + sql+="GRANT ALL PRIVILEGES ON DATABASE ${db} TO ${user} WITH GRANT OPTION;" fi ynh_psql_execute_as_root --sql="$sql"