From 9a8cbbd88369b959fed26d7de6ebca5858a863c0 Mon Sep 17 00:00:00 2001 From: Paco Date: Tue, 9 Mar 2021 21:34:30 +0100 Subject: [PATCH 001/130] sample _get_domain_and_subdomains_settings() --- src/yunohost/domain.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index cc9980549..1b1136a1b 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -45,6 +45,7 @@ from yunohost.hook import hook_callback logger = getActionLogger("yunohost.domain") +DOMAIN_SETTINGS_PATH = "/etc/yunohost/domains/" def domain_list(exclude_subdomains=False): """ @@ -657,3 +658,25 @@ def _get_DKIM(domain): p=dkim.group("p"), ), ) + + +def _get_domain_and_subdomains_settings(domain): + """ + Give data about a domain and its subdomains + """ + return { + "cmercier.fr" : { + "main": true, + "xmpp": true, + "mail": true, + "owned_dns_zone": true, + "ttl": 3600, + }, + "node.cmercier.fr" : { + "main": false, + "xmpp": false, + "mail": false, + "ttl": 3600, + }, + } + From c111b9c6c2b682856d23cfec1463fc82a5105d14 Mon Sep 17 00:00:00 2001 From: MercierCorentin Date: Tue, 9 Mar 2021 22:57:46 +0100 Subject: [PATCH 002/130] First implementation of configurable dns conf generation --- data/actionsmap/yunohost.yml | 7 -- src/yunohost/domain.py | 146 ++++++++++++++++++++--------------- 2 files changed, 83 insertions(+), 70 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 33b8b5cfe..ced353b75 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -467,13 +467,6 @@ domain: arguments: domain: help: Target domain - -t: - full: --ttl - help: Time To Live (TTL) in second before DNS servers update. Default is 3600 seconds (i.e. 1 hour). - extra: - pattern: - - !!str ^[0-9]+$ - - "pattern_positive_number" ### domain_maindomain() main-domain: diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 1b1136a1b..c62118a5a 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -25,6 +25,7 @@ """ import os import re +import sys from moulinette import m18n, msettings, msignals from moulinette.core import MoulinetteError @@ -275,22 +276,21 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False): logger.success(m18n.n("domain_deleted")) -def domain_dns_conf(domain, ttl=None): +def domain_dns_conf(domain): """ Generate DNS configuration for a domain Keyword argument: domain -- Domain name - ttl -- Time to live """ if domain not in domain_list()["domains"]: raise YunohostError("domain_name_unknown", domain=domain) - ttl = 3600 if ttl is None else ttl + domains_settings = _get_domain_and_subdomains_settings(domain) - dns_conf = _build_dns_conf(domain, ttl) + dns_conf = _build_dns_conf(domains_settings) result = "" @@ -411,7 +411,7 @@ def _get_maindomain(): return maindomain -def _build_dns_conf(domain, ttl=3600, include_empty_AAAA_if_no_ipv6=False): +def _build_dns_conf(domains): """ Internal function that will returns a data structure containing the needed information to generate/adapt the dns configuration @@ -451,72 +451,92 @@ def _build_dns_conf(domain, ttl=3600, include_empty_AAAA_if_no_ipv6=False): } """ + root = min(domains.keys(), key=(lambda k: len(k))) + + basic = [] + mail = [] + xmpp = [] + extra = [] ipv4 = get_public_ip() ipv6 = get_public_ip(6) - ########################### - # Basic ipv4/ipv6 records # - ########################### + name_prefix = root.partition(".")[0] - basic = [] - if ipv4: - basic.append(["@", ttl, "A", ipv4]) - if ipv6: - basic.append(["@", ttl, "AAAA", ipv6]) - elif include_empty_AAAA_if_no_ipv6: - basic.append(["@", ttl, "AAAA", None]) + for domain_name, domain in domains.items(): + print(domain_name) + ttl = domain["ttl"] - ######### - # Email # - ######### + owned_dns_zone = "owned_dns_zone" in domains[root] and domains[root]["owned_dns_zone"] == True + if domain_name == root: + name = name_prefix if not owned_dns_zone else "@" + else: + name = domain_name[0:-(1 + len(root))] + if not owned_dns_zone: + name += "." + name_prefix - mail = [ - ["@", ttl, "MX", "10 %s." % domain], - ["@", ttl, "TXT", '"v=spf1 a mx -all"'], - ] + ########################### + # Basic ipv4/ipv6 records # + ########################### + if ipv4: + basic.append([name, ttl, "A", ipv4]) - # DKIM/DMARC record - dkim_host, dkim_publickey = _get_DKIM(domain) + if ipv6: + basic.append([name, ttl, "AAAA", ipv6]) + # TODO + # elif include_empty_AAAA_if_no_ipv6: + # basic.append(["@", ttl, "AAAA", None]) - if dkim_host: - mail += [ - [dkim_host, ttl, "TXT", dkim_publickey], - ["_dmarc", ttl, "TXT", '"v=DMARC1; p=none"'], - ] + ######### + # Email # + ######### + if domain["mail"] == True: - ######## - # XMPP # - ######## + mail += [ + [name, ttl, "MX", "10 %s." % domain], + [name, ttl, "TXT", '"v=spf1 a mx -all"'], + ] - xmpp = [ - ["_xmpp-client._tcp", ttl, "SRV", "0 5 5222 %s." % domain], - ["_xmpp-server._tcp", ttl, "SRV", "0 5 5269 %s." % domain], - ["muc", ttl, "CNAME", "@"], - ["pubsub", ttl, "CNAME", "@"], - ["vjud", ttl, "CNAME", "@"], - ["xmpp-upload", ttl, "CNAME", "@"], - ] + # DKIM/DMARC record + dkim_host, dkim_publickey = _get_DKIM(domain) - ######### - # Extra # - ######### + if dkim_host: + mail += [ + [dkim_host, ttl, "TXT", dkim_publickey], + ["_dmarc", ttl, "TXT", '"v=DMARC1; p=none"'], + ] - extra = [] + ######## + # XMPP # + ######## + if domain["xmpp"] == True: + xmpp += [ + ["_xmpp-client._tcp", ttl, "SRV", "0 5 5222 %s." % domain_name], + ["_xmpp-server._tcp", ttl, "SRV", "0 5 5269 %s." % domain_name], + ["muc", ttl, "CNAME", name], + ["pubsub", ttl, "CNAME", name], + ["vjud", ttl, "CNAME", name], + ["xmpp-upload", ttl, "CNAME", name], + ] - if ipv4: - extra.append(["*", ttl, "A", ipv4]) + ######### + # Extra # + ######### - if ipv6: - extra.append(["*", ttl, "AAAA", ipv6]) - elif include_empty_AAAA_if_no_ipv6: - extra.append(["*", ttl, "AAAA", None]) - extra.append(["@", ttl, "CAA", '128 issue "letsencrypt.org"']) + if ipv4: + extra.append(["*", ttl, "A", ipv4]) - #################### - # Standard records # - #################### + if ipv6: + extra.append(["*", ttl, "AAAA", ipv6]) + elif include_empty_AAAA_if_no_ipv6: + extra.append(["*", ttl, "AAAA", None]) + + extra.append([name, ttl, "CAA", '128 issue "letsencrypt.org"']) + + #################### + # Standard records # + #################### records = { "basic": [ @@ -665,17 +685,17 @@ def _get_domain_and_subdomains_settings(domain): Give data about a domain and its subdomains """ return { - "cmercier.fr" : { - "main": true, - "xmpp": true, - "mail": true, - "owned_dns_zone": true, + "node.cmercier.fr" : { + "main": True, + "xmpp": True, + "mail": True, + "owned_dns_zone": True, "ttl": 3600, }, - "node.cmercier.fr" : { - "main": false, - "xmpp": false, - "mail": false, + "sub.node.cmercier.fr" : { + "main": False, + "xmpp": True, + "mail": False, "ttl": 3600, }, } From 1a4d02c9bf14c397b013890ff45f7648ccef8a97 Mon Sep 17 00:00:00 2001 From: Paco Date: Tue, 9 Mar 2021 23:53:50 +0100 Subject: [PATCH 003/130] Add loading of domain settings. Generate defaults for missing entries (and backward-compability) --- data/actionsmap/yunohost.yml | 6 +++ src/yunohost/domain.py | 73 +++++++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 33b8b5cfe..b2579be29 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -460,6 +460,12 @@ domain: help: Do not ask confirmation to remove apps action: store_true + settings: + action_help: Get settings for a domain + api: GET /domains//settings + arguments: + domain: + help: Target domain ### domain_dns_conf() dns-conf: action_help: Generate sample DNS configuration for a domain diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 1b1136a1b..d32b5954c 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -45,7 +45,7 @@ from yunohost.hook import hook_callback logger = getActionLogger("yunohost.domain") -DOMAIN_SETTINGS_PATH = "/etc/yunohost/domains/" +DOMAIN_SETTINGS_PATH = "/etc/yunohost/domains.yml" def domain_list(exclude_subdomains=False): """ @@ -661,7 +661,7 @@ def _get_DKIM(domain): def _get_domain_and_subdomains_settings(domain): - """ + """ Give data about a domain and its subdomains """ return { @@ -680,3 +680,72 @@ def _get_domain_and_subdomains_settings(domain): }, } +def _load_domain_settings(): + """ + Retrieve entries in domains.yml + And fill the holes if any + """ + # Retrieve entries in the YAML + if os.path.exists(DOMAIN_SETTINGS_PATH) and os.path.isfile(DOMAIN_SETTINGS_PATH): + old_domains = yaml.load(open(DOMAIN_SETTINGS_PATH, "r+")) + else: + old_domains = dict() + + # Create sanitized data + new_domains = dict() + + get_domain_list = domain_list() + + # Load main domain + maindomain = get_domain_list["main"] + + for domain in get_domain_list["domains"]: + # Update each setting if not present + new_domains[domain] = { + # Set "main" value + "main": True if domain == maindomain else False + } + # Set other values (default value if missing) + for setting, default in [ ("xmpp", True), ("mail", True), ("owned_dns_zone", True), ("ttl", 3600) ]: + if domain in old_domains.keys() and setting in old_domains[domain].keys(): + new_domains[domain][setting] = old_domains[domain][setting] + else: + new_domains[domain][setting] = default + + return new_domains + + +def domain_settings(domain): + """ + Get settings of a domain + + Keyword arguments: + domain -- The domain name + """ + return _get_domain_settings(domain, False) + +def _get_domain_settings(domain, subdomains): + """ + Get settings of a domain + + Keyword arguments: + domain -- The domain name + subdomains -- Do we include the subdomains? Default is False + + """ + domains = _load_domain_settings() + if not domain in domains.keys(): + return {} + + only_wanted_domains = dict() + for entry in domains.keys(): + if subdomains: + if domain in entry: + only_wanted_domains[entry] = domains[entry] + else: + if domain == entry: + only_wanted_domains[entry] = domains[entry] + + + return only_wanted_domains + From 8dd5859a46cba3e202a4c627f80edf85b348aa7c Mon Sep 17 00:00:00 2001 From: Paco Date: Tue, 9 Mar 2021 23:59:42 +0100 Subject: [PATCH 004/130] Integration of domain settings loading/generation with domains DNS entries generation --- src/yunohost/domain.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 986bcb826..b1052fdbb 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -288,7 +288,7 @@ def domain_dns_conf(domain): if domain not in domain_list()["domains"]: raise YunohostError("domain_name_unknown", domain=domain) - domains_settings = _get_domain_and_subdomains_settings(domain) + domains_settings = _get_domain_settings(domain, True) dns_conf = _build_dns_conf(domains_settings) @@ -464,7 +464,6 @@ def _build_dns_conf(domains): for domain_name, domain in domains.items(): - print(domain_name) ttl = domain["ttl"] owned_dns_zone = "owned_dns_zone" in domains[root] and domains[root]["owned_dns_zone"] == True @@ -493,7 +492,7 @@ def _build_dns_conf(domains): if domain["mail"] == True: mail += [ - [name, ttl, "MX", "10 %s." % domain], + [name, ttl, "MX", "10 %s." % domain_name], [name, ttl, "TXT", '"v=spf1 a mx -all"'], ] From cc3c073dc5b6402b387d848868cbbe7c4395bc36 Mon Sep 17 00:00:00 2001 From: Paco Date: Wed, 17 Mar 2021 21:24:13 +0100 Subject: [PATCH 005/130] Saving domain settings in /etc/yunohost/domains.yml --- data/actionsmap/yunohost.yml | 20 +++++++ src/yunohost/domain.py | 108 +++++++++++++++++++++++++---------- 2 files changed, 99 insertions(+), 29 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 052ace386..cb02ff781 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -466,6 +466,26 @@ domain: arguments: domain: help: Target domain + + set-settings: + action_help: Set settings of a domain + api: POST /domains//settings + arguments: + domain: + help: Target domain + -t: + full: --ttl + help: Time To Live of this domain's DNS records + -x: + full: --xmpp + help: Configure XMPP in this domain's DNS records? True or False + -m: + full: --mail + help: Configure mail in this domain's DNS records? True or False + -o: + full: --owned-dns-zone + help: Is this domain owned as a DNS zone? Is it a full domain, i.e not a subdomain? True or False + ### domain_dns_conf() dns-conf: action_help: Generate sample DNS configuration for a domain diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index b1052fdbb..0c8bade28 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -26,6 +26,7 @@ import os import re import sys +import yaml from moulinette import m18n, msettings, msignals from moulinette.core import MoulinetteError @@ -466,11 +467,11 @@ def _build_dns_conf(domains): for domain_name, domain in domains.items(): ttl = domain["ttl"] - owned_dns_zone = "owned_dns_zone" in domains[root] and domains[root]["owned_dns_zone"] == True + owned_dns_zone = "owned_dns_zone" in domains[root] and domains[root]["owned_dns_zone"] if domain_name == root: name = name_prefix if not owned_dns_zone else "@" else: - name = domain_name[0:-(1 + len(root))] + name = domain_name[0:-(1 + len(root))] if not owned_dns_zone: name += "." + name_prefix @@ -489,7 +490,7 @@ def _build_dns_conf(domains): ######### # Email # ######### - if domain["mail"] == True: + if domain["mail"]: mail += [ [name, ttl, "MX", "10 %s." % domain_name], @@ -508,7 +509,7 @@ def _build_dns_conf(domains): ######## # XMPP # ######## - if domain["xmpp"] == True: + if domain["xmpp"]: xmpp += [ ["_xmpp-client._tcp", ttl, "SRV", "0 5 5222 %s." % domain_name], ["_xmpp-server._tcp", ttl, "SRV", "0 5 5269 %s." % domain_name], @@ -679,26 +680,6 @@ def _get_DKIM(domain): ) -def _get_domain_and_subdomains_settings(domain): - """ - Give data about a domain and its subdomains - """ - return { - "node.cmercier.fr" : { - "main": True, - "xmpp": True, - "mail": True, - "owned_dns_zone": True, - "ttl": 3600, - }, - "sub.node.cmercier.fr" : { - "main": False, - "xmpp": True, - "mail": False, - "ttl": 3600, - }, - } - def _load_domain_settings(): """ Retrieve entries in domains.yml @@ -707,7 +688,8 @@ def _load_domain_settings(): # Retrieve entries in the YAML if os.path.exists(DOMAIN_SETTINGS_PATH) and os.path.isfile(DOMAIN_SETTINGS_PATH): old_domains = yaml.load(open(DOMAIN_SETTINGS_PATH, "r+")) - else: + + if old_domains is None: old_domains = dict() # Create sanitized data @@ -719,14 +701,16 @@ def _load_domain_settings(): maindomain = get_domain_list["main"] for domain in get_domain_list["domains"]: + is_maindomain = domain == maindomain + domain_in_old_domains = domain in old_domains.keys() # Update each setting if not present new_domains[domain] = { # Set "main" value - "main": True if domain == maindomain else False + "main": is_maindomain } # Set other values (default value if missing) - for setting, default in [ ("xmpp", True), ("mail", True), ("owned_dns_zone", True), ("ttl", 3600) ]: - if domain in old_domains.keys() and setting in old_domains[domain].keys(): + for setting, default in [ ("xmpp", is_maindomain), ("mail", is_maindomain), ("owned_dns_zone", True), ("ttl", 3600) ]: + if domain_in_old_domains and setting in old_domains[domain].keys(): new_domains[domain][setting] = old_domains[domain][setting] else: new_domains[domain][setting] = default @@ -743,6 +727,7 @@ def domain_settings(domain): """ return _get_domain_settings(domain, False) + def _get_domain_settings(domain, subdomains): """ Get settings of a domain @@ -765,6 +750,71 @@ def _get_domain_settings(domain, subdomains): if domain == entry: only_wanted_domains[entry] = domains[entry] - return only_wanted_domains + +def domain_set_settings(domain, ttl=None, xmpp=None, mail=None, owned_dns_zone=None): + """ + Set some settings of a domain, for DNS generation. + + Keyword arguments: + domain -- The domain name + --ttl -- the Time To Live for this domains DNS record + --xmpp -- configure XMPP DNS records for this domain + --mail -- configure mail DNS records for this domain + --owned_dns_zone -- is this domain DNS zone owned? (is it a full domain or a subdomain?) + """ + domains = _load_domain_settings() + + if not domain in domains.keys(): + raise YunohostError("domain_name_unknown", domain=domain) + + setting_set = False + + if ttl is not None: + try: + ttl = int(ttl) + except: + raise YunohostError("bad_value_type", value_type=type(ttl)) + + if ttl < 0: + raise YunohostError("must_be_positive", value_type=type(ttl)) + + domains[domain]["ttl"] = ttl + setting_set = True + + if xmpp is not None: + try: + xmpp = xmpp in ["True", "true", "1"] + except: + raise YunohostError("bad_value_type", value_type=type(xmpp)) + domains[domain]["xmpp"] = xmpp + setting_set = True + + if mail is not None: + try: + mail = mail in ["True", "true", "1"] + except: + raise YunohostError("bad_value_type", value_type=type(mail)) + + domains[domain]["mail"] = mail + setting_set = True + + if owned_dns_zone is not None: + try: + owned_dns_zone = owned_dns_zone in ["True", "true", "1"] + except: + raise YunohostError("bad_value_type", value_type=type(owned_dns_zone)) + + domains[domain]["owned_dns_zone"] = owned_dns_zone + setting_set = True + + if not setting_set: + raise YunohostError("no_setting_given") + + # Save the settings to the .yaml file + with open(DOMAIN_SETTINGS_PATH, 'w') as file: + yaml.dump(domains, file) + + return domains[domain] + From fa5b3198ccdaeeefa186f9638d92ef1407ebf845 Mon Sep 17 00:00:00 2001 From: Paco Date: Wed, 17 Mar 2021 21:26:45 +0100 Subject: [PATCH 006/130] Add TODOs for locales --- src/yunohost/domain.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 0c8bade28..04aa6b560 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -767,6 +767,7 @@ def domain_set_settings(domain, ttl=None, xmpp=None, mail=None, owned_dns_zone=N domains = _load_domain_settings() if not domain in domains.keys(): + # TODO add locales raise YunohostError("domain_name_unknown", domain=domain) setting_set = False @@ -775,9 +776,11 @@ def domain_set_settings(domain, ttl=None, xmpp=None, mail=None, owned_dns_zone=N try: ttl = int(ttl) except: + # TODO add locales raise YunohostError("bad_value_type", value_type=type(ttl)) if ttl < 0: + # TODO add locales raise YunohostError("must_be_positive", value_type=type(ttl)) domains[domain]["ttl"] = ttl @@ -787,6 +790,7 @@ def domain_set_settings(domain, ttl=None, xmpp=None, mail=None, owned_dns_zone=N try: xmpp = xmpp in ["True", "true", "1"] except: + # TODO add locales raise YunohostError("bad_value_type", value_type=type(xmpp)) domains[domain]["xmpp"] = xmpp setting_set = True @@ -795,6 +799,7 @@ def domain_set_settings(domain, ttl=None, xmpp=None, mail=None, owned_dns_zone=N try: mail = mail in ["True", "true", "1"] except: + # TODO add locales raise YunohostError("bad_value_type", value_type=type(mail)) domains[domain]["mail"] = mail @@ -804,12 +809,14 @@ def domain_set_settings(domain, ttl=None, xmpp=None, mail=None, owned_dns_zone=N try: owned_dns_zone = owned_dns_zone in ["True", "true", "1"] except: + # TODO add locales raise YunohostError("bad_value_type", value_type=type(owned_dns_zone)) domains[domain]["owned_dns_zone"] = owned_dns_zone setting_set = True if not setting_set: + # TODO add locales raise YunohostError("no_setting_given") # Save the settings to the .yaml file From f295dffd005e8af3e5f893191cdaaa9964b531dd Mon Sep 17 00:00:00 2001 From: Paco Date: Sun, 21 Mar 2021 22:27:57 +0100 Subject: [PATCH 007/130] Fix old_domains not assigned --- src/yunohost/domain.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 04aa6b560..6ed4eb281 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -686,6 +686,7 @@ def _load_domain_settings(): And fill the holes if any """ # Retrieve entries in the YAML + old_domains = None if os.path.exists(DOMAIN_SETTINGS_PATH) and os.path.isfile(DOMAIN_SETTINGS_PATH): old_domains = yaml.load(open(DOMAIN_SETTINGS_PATH, "r+")) From 3b6599ff0d26c2d744bf569bc8952cb141029221 Mon Sep 17 00:00:00 2001 From: MercierCorentin Date: Mon, 22 Mar 2021 00:00:03 +0100 Subject: [PATCH 008/130] Check if DNS zone is owned by user --- src/yunohost/domain.py | 4 +++- src/yunohost/utils/dns.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 src/yunohost/utils/dns.py diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 6ed4eb281..1909a0353 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -42,6 +42,7 @@ from yunohost.app import ( ) from yunohost.regenconf import regen_conf, _force_clear_hashes, _process_regen_conf from yunohost.utils.network import get_public_ip +from yunohost.utils.dns import get_public_suffix from yunohost.log import is_unit_operation from yunohost.hook import hook_callback @@ -703,6 +704,7 @@ def _load_domain_settings(): for domain in get_domain_list["domains"]: is_maindomain = domain == maindomain + default_owned_dns_zone = True if domain == get_public_suffix(domain) else False domain_in_old_domains = domain in old_domains.keys() # Update each setting if not present new_domains[domain] = { @@ -710,7 +712,7 @@ def _load_domain_settings(): "main": is_maindomain } # Set other values (default value if missing) - for setting, default in [ ("xmpp", is_maindomain), ("mail", is_maindomain), ("owned_dns_zone", True), ("ttl", 3600) ]: + for setting, default in [ ("xmpp", is_maindomain), ("mail", is_maindomain), ("owned_dns_zone", default_owned_dns_zone), ("ttl", 3600) ]: if domain_in_old_domains and setting in old_domains[domain].keys(): new_domains[domain][setting] = old_domains[domain][setting] else: diff --git a/src/yunohost/utils/dns.py b/src/yunohost/utils/dns.py new file mode 100644 index 000000000..3033743d1 --- /dev/null +++ b/src/yunohost/utils/dns.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +""" License + + Copyright (C) 2018 YUNOHOST.ORG + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program; if not, see http://www.gnu.org/licenses + +""" +from publicsuffix import PublicSuffixList + +YNH_DYNDNS_DOMAINS = ["nohost.me", "noho.st", "ynh.fr"] + +def get_public_suffix(domain): + """get_public_suffix("www.example.com") -> "example.com" + + Return the public suffix of a domain name based + """ + # Load domain public suffixes + psl = PublicSuffixList() + + public_suffix = psl.get_public_suffix(domain) + if public_suffix in YNH_DYNDNS_DOMAINS: + domain_prefix = domain_name[0:-(1 + len(public_suffix))] + public_suffix = domain_prefix.plit(".")[-1] + "." + public_suffix + + return public_suffix \ No newline at end of file From fa818a476a152a8b36c598049e947fa90d981ef6 Mon Sep 17 00:00:00 2001 From: Paco Date: Mon, 22 Mar 2021 01:57:20 +0100 Subject: [PATCH 009/130] Change API naming etc. Expect a new change to come! --- data/actionsmap/yunohost.yml | 83 +++++++++++++++++++++++++----------- src/yunohost/domain.py | 78 +++++++++++++++------------------ 2 files changed, 92 insertions(+), 69 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index cb02ff781..138f7c6cd 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -460,31 +460,6 @@ domain: help: Do not ask confirmation to remove apps action: store_true - settings: - action_help: Get settings for a domain - api: GET /domains//settings - arguments: - domain: - help: Target domain - - set-settings: - action_help: Set settings of a domain - api: POST /domains//settings - arguments: - domain: - help: Target domain - -t: - full: --ttl - help: Time To Live of this domain's DNS records - -x: - full: --xmpp - help: Configure XMPP in this domain's DNS records? True or False - -m: - full: --mail - help: Configure mail in this domain's DNS records? True or False - -o: - full: --owned-dns-zone - help: Is this domain owned as a DNS zone? Is it a full domain, i.e not a subdomain? True or False ### domain_dns_conf() dns-conf: @@ -493,6 +468,8 @@ domain: arguments: domain: help: Target domain + extra: + pattern: *pattern_domain ### domain_maindomain() main-domain: @@ -575,6 +552,62 @@ domain: path: help: The path to check (e.g. /coffee) + subcategories: + config: + subcategory_help: Domains DNS settings + actions: + # domain_config_list + list: + action_help: Get settings for all domains + api: GET /domains/list + arguments: + domain: + help: Target domain + extra: + pattern: *pattern_domain + + # domain_config_show + show: + action_help: Get settings for all domains + api: GET /domains//show + arguments: + domain: + help: Target domain + extra: + pattern: *pattern_domain + + # domain_config_get + get: + action_help: Get specific setting of a domain + api: GET /domains// + arguments: + domain: + help: Target domain + extra: + pattern: *pattern_domain + key: + help: Setting requested. One of ttl, xmpp, mail, owned_dns_zone + extra: + pattern: &pattern_domain_key + - !!str ^(ttl)|(xmpp)|(mail)|(owned_dns_zone)|$ + - "pattern_domain_key" + + # domain_config_set + set: + action_help: Set a setting of a domain + api: POST /domains// + arguments: + domain: + help: Target domain + extra: + pattern: *pattern_domain + key: + help: Setting requested. One of ttl (Time To Live of this domain's DNS records), xmpp (Configure XMPP in this domain's DNS records?), mail (Configure mail in this domain's DNS records?), owned_dns_zone (Is it a full domain, i.e not a subdomain?) + extra: + pattern: *pattern_domain_key + value: + help: Value of the setting. Must be a positive integer number for "ttl", or one of ("True", "False", "true", "false", "1", "0") for other settings + ### domain_info() # info: diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 6ed4eb281..41daeaa03 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -705,10 +705,8 @@ def _load_domain_settings(): is_maindomain = domain == maindomain domain_in_old_domains = domain in old_domains.keys() # Update each setting if not present - new_domains[domain] = { - # Set "main" value - "main": is_maindomain - } + new_domains[domain] = {} + # new_domains[domain] = { "main": is_maindomain } # Set other values (default value if missing) for setting, default in [ ("xmpp", is_maindomain), ("mail", is_maindomain), ("owned_dns_zone", True), ("ttl", 3600) ]: if domain_in_old_domains and setting in old_domains[domain].keys(): @@ -719,9 +717,9 @@ def _load_domain_settings(): return new_domains -def domain_settings(domain): +def domain_config_show(domain): """ - Get settings of a domain + Show settings of a domain Keyword arguments: domain -- The domain name @@ -729,6 +727,18 @@ def domain_settings(domain): return _get_domain_settings(domain, False) +def domain_config_get(domain, key): + """ + Show a setting of a domain + + Keyword arguments: + domain -- The domain name + key -- ttl, xmpp, mail, owned_dns_zone + """ + settings = _get_domain_settings(domain, False) + return settings[domain][key] + + def _get_domain_settings(domain, subdomains): """ Get settings of a domain @@ -740,7 +750,7 @@ def _get_domain_settings(domain, subdomains): """ domains = _load_domain_settings() if not domain in domains.keys(): - return {} + raise YunohostError("domain_name_unknown", domain=domain) only_wanted_domains = dict() for entry in domains.keys(): @@ -754,16 +764,19 @@ def _get_domain_settings(domain, subdomains): return only_wanted_domains -def domain_set_settings(domain, ttl=None, xmpp=None, mail=None, owned_dns_zone=None): +def domain_config_set(domain, key, value): + #(domain, ttl=None, xmpp=None, mail=None, owned_dns_zone=None): """ Set some settings of a domain, for DNS generation. Keyword arguments: domain -- The domain name - --ttl -- the Time To Live for this domains DNS record - --xmpp -- configure XMPP DNS records for this domain - --mail -- configure mail DNS records for this domain - --owned_dns_zone -- is this domain DNS zone owned? (is it a full domain or a subdomain?) + key must be one of this strings: + ttl -- the Time To Live for this domains DNS record + xmpp -- configure XMPP DNS records for this domain + mail -- configure mail DNS records for this domain + owned_dns_zone -- is this domain DNS zone owned? (is it a full domain or a subdomain?) + value must be set according to the key """ domains = _load_domain_settings() @@ -771,11 +784,9 @@ def domain_set_settings(domain, ttl=None, xmpp=None, mail=None, owned_dns_zone=N # TODO add locales raise YunohostError("domain_name_unknown", domain=domain) - setting_set = False - - if ttl is not None: + if "ttl" == key: try: - ttl = int(ttl) + ttl = int(value) except: # TODO add locales raise YunohostError("bad_value_type", value_type=type(ttl)) @@ -785,38 +796,17 @@ def domain_set_settings(domain, ttl=None, xmpp=None, mail=None, owned_dns_zone=N raise YunohostError("must_be_positive", value_type=type(ttl)) domains[domain]["ttl"] = ttl - setting_set = True - if xmpp is not None: - try: - xmpp = xmpp in ["True", "true", "1"] - except: - # TODO add locales - raise YunohostError("bad_value_type", value_type=type(xmpp)) - domains[domain]["xmpp"] = xmpp - setting_set = True + elif "xmpp" == key: + domains[domain]["xmpp"] = value in ["True", "true", "1"] - if mail is not None: - try: - mail = mail in ["True", "true", "1"] - except: - # TODO add locales - raise YunohostError("bad_value_type", value_type=type(mail)) + elif "mail" == key: + domains[domain]["mail"] = value in ["True", "true", "1"] - domains[domain]["mail"] = mail - setting_set = True + elif "owned_dns_zone" == key: + domains[domain]["owned_dns_zone"] = value in ["True", "true", "1"] - if owned_dns_zone is not None: - try: - owned_dns_zone = owned_dns_zone in ["True", "true", "1"] - except: - # TODO add locales - raise YunohostError("bad_value_type", value_type=type(owned_dns_zone)) - - domains[domain]["owned_dns_zone"] = owned_dns_zone - setting_set = True - - if not setting_set: + else: # TODO add locales raise YunohostError("no_setting_given") From b3675051570507643f218d14b35dcb0ab034f5cd Mon Sep 17 00:00:00 2001 From: MercierCorentin Date: Thu, 25 Mar 2021 20:52:14 +0100 Subject: [PATCH 010/130] XMPP configuration for subdomains (i.e. not owned zone dns) --- src/yunohost/domain.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 1909a0353..56b04e95b 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -461,20 +461,24 @@ def _build_dns_conf(domains): extra = [] ipv4 = get_public_ip() ipv6 = get_public_ip(6) + owned_dns_zone = "owned_dns_zone" in domains[root] and domains[root]["owned_dns_zone"] - name_prefix = root.partition(".")[0] - + root_prefix = root.partition(".")[0] + child_domain_suffix = "" for domain_name, domain in domains.items(): ttl = domain["ttl"] - owned_dns_zone = "owned_dns_zone" in domains[root] and domains[root]["owned_dns_zone"] if domain_name == root: - name = name_prefix if not owned_dns_zone else "@" + name = root_prefix if not owned_dns_zone else "@" else: name = domain_name[0:-(1 + len(root))] if not owned_dns_zone: - name += "." + name_prefix + name += "." + root_prefix + + if name != "@": + child_domain_suffix = "." + name + ########################### # Basic ipv4/ipv6 records # @@ -514,10 +518,10 @@ def _build_dns_conf(domains): xmpp += [ ["_xmpp-client._tcp", ttl, "SRV", "0 5 5222 %s." % domain_name], ["_xmpp-server._tcp", ttl, "SRV", "0 5 5269 %s." % domain_name], - ["muc", ttl, "CNAME", name], - ["pubsub", ttl, "CNAME", name], - ["vjud", ttl, "CNAME", name], - ["xmpp-upload", ttl, "CNAME", name], + ["muc" + child_domain_suffix, ttl, "CNAME", name], + ["pubsub" + child_domain_suffix, ttl, "CNAME", name], + ["vjud" + child_domain_suffix, ttl, "CNAME", name], + ["xmpp-upload" + child_domain_suffix, ttl, "CNAME", name], ] ######### From 503a5ed6d2562b9f7b9b39f9a4e290dffd790018 Mon Sep 17 00:00:00 2001 From: Paco Date: Thu, 25 Mar 2021 21:08:08 +0100 Subject: [PATCH 011/130] Add `yunohost domain config list` command --- data/actionsmap/yunohost.yml | 5 ----- src/yunohost/domain.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 138f7c6cd..c84e2c4b0 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -560,11 +560,6 @@ domain: list: action_help: Get settings for all domains api: GET /domains/list - arguments: - domain: - help: Target domain - extra: - pattern: *pattern_domain # domain_config_show show: diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index eec362bcb..e050ae6d6 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -719,6 +719,16 @@ def _load_domain_settings(): return new_domains +def domain_config_list(): + """ + Show settings of all domains + + Keyword arguments: + domain -- The domain name + """ + return _load_domain_settings() + + def domain_config_show(domain): """ Show settings of a domain From afe62877d3a87b8a93e65463002b140052aa1e45 Mon Sep 17 00:00:00 2001 From: MercierCorentin Date: Mon, 29 Mar 2021 14:46:03 +0200 Subject: [PATCH 012/130] Domain settings: transform cli to be like app settings --- data/actionsmap/yunohost.yml | 67 ++++++----------------- src/yunohost/domain.py | 103 ++++++++++++++--------------------- 2 files changed, 56 insertions(+), 114 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index c84e2c4b0..b363a218f 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -551,57 +551,22 @@ domain: pattern: *pattern_domain path: help: The path to check (e.g. /coffee) - - subcategories: - config: - subcategory_help: Domains DNS settings - actions: - # domain_config_list - list: - action_help: Get settings for all domains - api: GET /domains/list - - # domain_config_show - show: - action_help: Get settings for all domains - api: GET /domains//show - arguments: - domain: - help: Target domain - extra: - pattern: *pattern_domain - - # domain_config_get - get: - action_help: Get specific setting of a domain - api: GET /domains// - arguments: - domain: - help: Target domain - extra: - pattern: *pattern_domain - key: - help: Setting requested. One of ttl, xmpp, mail, owned_dns_zone - extra: - pattern: &pattern_domain_key - - !!str ^(ttl)|(xmpp)|(mail)|(owned_dns_zone)|$ - - "pattern_domain_key" - - # domain_config_set - set: - action_help: Set a setting of a domain - api: POST /domains// - arguments: - domain: - help: Target domain - extra: - pattern: *pattern_domain - key: - help: Setting requested. One of ttl (Time To Live of this domain's DNS records), xmpp (Configure XMPP in this domain's DNS records?), mail (Configure mail in this domain's DNS records?), owned_dns_zone (Is it a full domain, i.e not a subdomain?) - extra: - pattern: *pattern_domain_key - value: - help: Value of the setting. Must be a positive integer number for "ttl", or one of ("True", "False", "true", "false", "1", "0") for other settings + ### domain_setting() + setting: + action_help: Set or get an app setting value + api: GET /domains//settings + arguments: + domain: + help: Domaine name + key: + help: Key to get/set + -v: + full: --value + help: Value to set + -d: + full: --delete + help: Delete the key + action: store_true ### domain_info() diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index d88eb6b60..e799f52b2 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -722,38 +722,48 @@ def _load_domain_settings(): return new_domains - -def domain_config_list(): +def domain_setting(domain, key, value=None, delete=False): """ - Show settings of all domains + Set or get an app setting value + + Keyword argument: + value -- Value to set + app -- App ID + key -- Key to get/set + delete -- Delete the key - Keyword arguments: - domain -- The domain name """ - return _load_domain_settings() + domains = _load_domain_settings() + if not domain in domains.keys(): + # TODO add locales + raise YunohostError("domain_name_unknown", domain=domain) + + domain_settings = _get_domain_settings(domain, False) or {} -def domain_config_show(domain): - """ - Show settings of a domain + # GET + if value is None and not delete: + return domain_settings.get(key, None) - Keyword arguments: - domain -- The domain name - """ - return _get_domain_settings(domain, False) + # DELETE + if delete: + if key in domain_settings: + del domain_settings[key] + # SET + else: + + if "ttl" == key: + try: + ttl = int(value) + except: + # TODO add locales + raise YunohostError("bad_value_type", value_type=type(ttl)) -def domain_config_get(domain, key): - """ - Show a setting of a domain - - Keyword arguments: - domain -- The domain name - key -- ttl, xmpp, mail, owned_dns_zone - """ - settings = _get_domain_settings(domain, False) - return settings[domain][key] - + if ttl < 0: + # TODO add locales + raise YunohostError("must_be_positive", value_type=type(ttl)) + domain_settings[key] = value def _get_domain_settings(domain, subdomains): """ @@ -780,55 +790,22 @@ def _get_domain_settings(domain, subdomains): return only_wanted_domains -def domain_config_set(domain, key, value): - #(domain, ttl=None, xmpp=None, mail=None, owned_dns_zone=None): +def _set_domain_settings(domain, domain_settings): """ - Set some settings of a domain, for DNS generation. + Set settings of a domain Keyword arguments: domain -- The domain name - key must be one of this strings: - ttl -- the Time To Live for this domains DNS record - xmpp -- configure XMPP DNS records for this domain - mail -- configure mail DNS records for this domain - owned_dns_zone -- is this domain DNS zone owned? (is it a full domain or a subdomain?) - value must be set according to the key + settings -- Dict with doamin settings + """ domains = _load_domain_settings() - if not domain in domains.keys(): - # TODO add locales raise YunohostError("domain_name_unknown", domain=domain) - if "ttl" == key: - try: - ttl = int(value) - except: - # TODO add locales - raise YunohostError("bad_value_type", value_type=type(ttl)) - - if ttl < 0: - # TODO add locales - raise YunohostError("must_be_positive", value_type=type(ttl)) - - domains[domain]["ttl"] = ttl - - elif "xmpp" == key: - domains[domain]["xmpp"] = value in ["True", "true", "1"] - - elif "mail" == key: - domains[domain]["mail"] = value in ["True", "true", "1"] - - elif "owned_dns_zone" == key: - domains[domain]["owned_dns_zone"] = value in ["True", "true", "1"] - - else: - # TODO add locales - raise YunohostError("no_setting_given") + domains[domain] = domain_settings # Save the settings to the .yaml file with open(DOMAIN_SETTINGS_PATH, 'w') as file: - yaml.dump(domains, file) - - return domains[domain] + yaml.dump(domains, file, default_flow_style=False) From e16f14f794d1f3ad5764e59b1a24021fa0f2aefd Mon Sep 17 00:00:00 2001 From: Paco Date: Tue, 27 Apr 2021 21:42:18 +0200 Subject: [PATCH 013/130] fix . set operation still not working. --- data/actionsmap/yunohost.yml | 2 +- src/yunohost/domain.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 5df29422c..b61a9a26e 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -557,7 +557,7 @@ domain: api: GET /domains//settings arguments: domain: - help: Domaine name + help: Domain name key: help: Key to get/set -v: diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index cf0867134..59e445154 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -745,16 +745,20 @@ def domain_setting(domain, key, value=None, delete=False): # TODO add locales raise YunohostError("domain_name_unknown", domain=domain) - domain_settings = _get_domain_settings(domain, False) or {} + domain_settings = domains[domain] # GET if value is None and not delete: - return domain_settings.get(key, None) + if not key in domain_settings: + raise YunohostValidationError("This key doesn't exist!") + + return domain_settings[key] # DELETE if delete: if key in domain_settings: del domain_settings[key] + _set_domain_settings(domain, domain_settings) # SET else: @@ -770,6 +774,7 @@ def domain_setting(domain, key, value=None, delete=False): # TODO add locales raise YunohostError("must_be_positive", value_type=type(ttl)) domain_settings[key] = value + _set_domain_settings(domain, domain_settings) def _get_domain_settings(domain, subdomains): """ From eeab7cd1030c310dea1b8ef8bc9c3ce51d4e1343 Mon Sep 17 00:00:00 2001 From: Paco Date: Wed, 28 Apr 2021 00:26:19 +0200 Subject: [PATCH 014/130] ~ working push_config. Tested with Gandi. To be improved! --- data/actionsmap/yunohost.yml | 10 +++ src/yunohost/domain.py | 163 +++++++++++++++++++++++++++-------- 2 files changed, 136 insertions(+), 37 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 53d3e11c8..58a48c87f 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -447,6 +447,16 @@ domain: help: Subscribe to the DynDNS service action: store_true + ### domain_push_config() + push_config: + action_help: Push DNS records to registrar + api: GET /domains/push + arguments: + domain: + help: Domain name to add + extra: + pattern: *pattern_domain + ### domain_remove() remove: action_help: Delete domains diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index f878fe17a..05f2a16ae 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -27,6 +27,10 @@ import os import re import sys import yaml +import functools + +from lexicon.config import ConfigResolver +from lexicon.client import Client from moulinette import m18n, msettings, msignals from moulinette.core import MoulinetteError @@ -103,7 +107,6 @@ def domain_add(operation_logger, domain, dyndns=False): from yunohost.hook import hook_callback from yunohost.app import app_ssowatconf from yunohost.utils.ldap import _get_ldap_interface - from yunohost.certificate import _certificate_install_selfsigned if domain.startswith("xmpp-upload."): raise YunohostValidationError("domain_cannot_add_xmpp_upload") @@ -126,7 +129,7 @@ def domain_add(operation_logger, domain, dyndns=False): # Do not allow to subscribe to multiple dyndns domains... if _guess_current_dyndns_domain("dyndns.yunohost.org") != (None, None): - raise YunohostValidationError("domain_dyndns_already_subscribed") + raise YunohostValidationError('domain_dyndns_already_subscribed') # Check that this domain can effectively be provided by # dyndns.yunohost.org. (i.e. is it a nohost.me / noho.st) @@ -137,13 +140,14 @@ def domain_add(operation_logger, domain, dyndns=False): if dyndns: from yunohost.dyndns import dyndns_subscribe - # Actually subscribe dyndns_subscribe(domain=domain) - _certificate_install_selfsigned([domain], False) - try: + import yunohost.certificate + + yunohost.certificate._certificate_install_selfsigned([domain], False) + attr_dict = { "objectClass": ["mailDomain", "top"], "virtualdomain": domain, @@ -170,13 +174,13 @@ def domain_add(operation_logger, domain, dyndns=False): regen_conf(names=["nginx", "metronome", "dnsmasq", "postfix", "rspamd"]) app_ssowatconf() - except Exception as e: + except Exception: # Force domain removal silently try: domain_remove(domain, force=True) except Exception: pass - raise e + raise hook_callback("post_domain_add", args=[domain]) @@ -202,8 +206,8 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False): # the 'force' here is related to the exception happening in domain_add ... # we don't want to check the domain exists because the ldap add may have # failed - if not force and domain not in domain_list()["domains"]: - raise YunohostValidationError("domain_name_unknown", domain=domain) + if not force and domain not in domain_list()['domains']: + raise YunohostValidationError('domain_name_unknown', domain=domain) # Check domain is not the main domain if domain == _get_maindomain(): @@ -217,9 +221,7 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False): other_domains="\n * " + ("\n * ".join(other_domains)), ) else: - raise YunohostValidationError( - "domain_cannot_remove_main_add_new_one", domain=domain - ) + raise YunohostValidationError("domain_cannot_remove_main_add_new_one", domain=domain) # Check if apps are installed on the domain apps_on_that_domain = [] @@ -228,37 +230,21 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False): settings = _get_app_settings(app) label = app_info(app)["name"] if settings.get("domain") == domain: - apps_on_that_domain.append( - ( - app, - ' - %s "%s" on https://%s%s' - % (app, label, domain, settings["path"]) - if "path" in settings - else app, - ) - ) + apps_on_that_domain.append((app, " - %s \"%s\" on https://%s%s" % (app, label, domain, settings["path"]) if "path" in settings else app)) if apps_on_that_domain: if remove_apps: - if msettings.get("interface") == "cli" and not force: - answer = msignals.prompt( - m18n.n( - "domain_remove_confirm_apps_removal", - apps="\n".join([x[1] for x in apps_on_that_domain]), - answers="y/N", - ), - color="yellow", - ) + if msettings.get('interface') == "cli" and not force: + answer = msignals.prompt(m18n.n('domain_remove_confirm_apps_removal', + apps="\n".join([x[1] for x in apps_on_that_domain]), + answers='y/N'), color="yellow") if answer.upper() != "Y": raise YunohostError("aborting") for app, _ in apps_on_that_domain: app_remove(app) else: - raise YunohostValidationError( - "domain_uninstall_app_first", - apps="\n".join([x[1] for x in apps_on_that_domain]), - ) + raise YunohostValidationError('domain_uninstall_app_first', apps="\n".join([x[1] for x in apps_on_that_domain])) operation_logger.start() @@ -271,7 +257,7 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False): os.system("rm -rf /etc/yunohost/certs/%s" % domain) # Delete dyndns keys for this domain (if any) - os.system("rm -rf /etc/yunohost/dyndns/K%s.+*" % domain) + os.system('rm -rf /etc/yunohost/dyndns/K%s.+*' % domain) # Sometime we have weird issues with the regenconf where some files # appears as manually modified even though they weren't touched ... @@ -558,8 +544,9 @@ def _build_dns_conf(domains): if ipv6: extra.append(["*", ttl, "AAAA", ipv6]) - elif include_empty_AAAA_if_no_ipv6: - extra.append(["*", ttl, "AAAA", None]) + # TODO + # elif include_empty_AAAA_if_no_ipv6: + # extra.append(["*", ttl, "AAAA", None]) extra.append([name, ttl, "CAA", '128 issue "letsencrypt.org"']) @@ -838,3 +825,105 @@ def _set_domain_settings(domain, domain_settings): with open(DOMAIN_SETTINGS_PATH, 'w') as file: yaml.dump(domains, file, default_flow_style=False) + +def domain_push_config(domain): + """ + Send DNS records to the previously-configured registrar of the domain. + """ + # Generate the records + if domain not in domain_list()["domains"]: + raise YunohostValidationError("domain_name_unknown", domain=domain) + + domains_settings = _get_domain_settings(domain, True) + + dns_conf = _build_dns_conf(domains_settings) + + # Flatten the DNS conf + flatten_dns_conf = [] + for key in dns_conf: + list_of_records = dns_conf[key] + for record in list_of_records: + # FIXME Lexicon does not support CAA records + # See https://github.com/AnalogJ/lexicon/issues/282 and https://github.com/AnalogJ/lexicon/pull/371 + # They say it's trivial to implement it! + # And yet, it is still not done/merged + if record["type"] != "CAA": + # Add .domain.tdl to the name entry + record["name"] = "{}.{}".format(record["name"], domain) + flatten_dns_conf.append(record) + + # Get provider info + # TODO + provider = { + "name": "gandi", + "options": { + "api_protocol": "rest", + "auth_token": "vhcIALuRJKtoZiZyxfDYWLom" + } + } + + # Construct the base data structure to use lexicon's API. + base_config = { + "provider_name": provider["name"], + "domain": domain, # domain name + } + base_config[provider["name"]] = provider["options"] + + # Get types present in the generated records + types = set() + + for record in flatten_dns_conf: + types.add(record["type"]) + + # Fetch all types present in the generated records + distant_records = {} + + for key in types: + record_config = { + "action": "list", + "type": key, + } + final_lexicon = ConfigResolver().with_dict(dict_object=base_config).with_dict(dict_object=record_config) + # print('final_lexicon:', final_lexicon); + client = Client(final_lexicon) + distant_records[key] = client.execute() + + for key in types: + for distant_record in distant_records[key]: + print('distant_record:', distant_record); + for local_record in flatten_dns_conf: + print('local_record:', local_record); + + # Push the records + for record in flatten_dns_conf: + # For each record, first check if one record exists for the same (type, name) couple + it_exists = False + # TODO do not push if local and distant records are exactly the same ? + # is_the_same_record = False + + for distant_record in distant_records[record["type"]]: + if distant_record["type"] == record["type"] and distant_record["name"] == record["name"]: + it_exists = True + # previous TODO + # if distant_record["ttl"] = ... and distant_record["name"] ... + # is_the_same_record = True + + # Finally, push the new record or update the existing one + record_config = { + "action": "update" if it_exists else "create", # create, list, update, delete + "type": record["type"], # specify a type for record filtering, case sensitive in some cases. + "name": record["name"], + "content": record["value"], + # FIXME Delte TTL, doesn't work with Gandi. + # See https://github.com/AnalogJ/lexicon/issues/726 (similar issue) + # But I think there is another issue with Gandi. Or I'm misusing the API... + # "ttl": record["ttl"], + } + final_lexicon = ConfigResolver().with_dict(dict_object=base_config).with_dict(dict_object=record_config) + client = Client(final_lexicon) + print('pushed_record:', record_config, "→", end=' ') + results = client.execute() + print('results:', results); + # print("Failed" if results == False else "Ok") + +# def domain_config_fetch(domain, key, value): From bb140b2ba4353d7282504de573ec4f4b8d7d47ef Mon Sep 17 00:00:00 2001 From: MercierCorentin Date: Wed, 19 May 2021 03:02:13 +0200 Subject: [PATCH 015/130] Add providers parameter list --- data/other/providers.yml | 218 +++++++++++++++++++++++++++++++++++++++ debian/install | 1 + 2 files changed, 219 insertions(+) create mode 100644 data/other/providers.yml diff --git a/data/other/providers.yml b/data/other/providers.yml new file mode 100644 index 000000000..4ba69d97b --- /dev/null +++ b/data/other/providers.yml @@ -0,0 +1,218 @@ +- aliyun + - auth_key_id + - auth_secret +- aurora + - auth_api_key + - auth_secret_key +- azure + - auth_client_id + - auth_client_secret + - auth_tenant_id + - auth_subscription_id + - resource_group +- cloudflare + - auth_username + - auth_token + - zone_id +- cloudns + - auth_id + - auth_subid + - auth_subuser + - auth_password + - weight + - port +- cloudxns + - auth_username + - auth_token +- conoha + - auth_region + - auth_token + - auth_username + - auth_password + - auth_tenant_id +- constellix + - auth_username + - auth_token +- digitalocean + - auth_token +- dinahosting + - auth_username + - auth_password +- directadmin + - auth_password + - auth_username + - endpoint +- dnsimple + - auth_token + - auth_username + - auth_password + - auth_2fa +- dnsmadeeasy + - auth_username + - auth_token +- dnspark + - auth_username + - auth_token +- dnspod + - auth_username + - auth_token +- dreamhost + - auth_token +- dynu + - auth_token +- easydns + - auth_username + - auth_token +- easyname + - auth_username + - auth_password +- euserv + - auth_username + - auth_password +- exoscale + - auth_key + - auth_secret +- gandi + - auth_token + - api_protocol +- gehirn + - auth_token + - auth_secret +- glesys + - auth_username + - auth_token +- godaddy + - auth_key + - auth_secret +- googleclouddns + - auth_service_account_info +- gransy + - auth_username + - auth_password +- gratisdns + - auth_username + - auth_password +- henet + - auth_username + - auth_password +- hetzner + - auth_token +- hostingde + - auth_token +- hover + - auth_username + - auth_password +- infoblox + - auth_user + - auth_psw + - ib_view + - ib_host +- infomaniak + - auth_token +- internetbs + - auth_key + - auth_password +- inwx + - auth_username + - auth_password +- joker + - auth_token +- linode + - auth_token +- linode4 + - auth_token +- localzone + - filename +- luadns + - auth_username + - auth_token +- memset + - auth_token +- mythicbeasts + - auth_username + - auth_password + - auth_token +- namecheap + - auth_token + - auth_username + - auth_client_ip + - auth_sandbox +- namesilo + - auth_token +- netcup + - auth_customer_id + - auth_api_key + - auth_api_password +- nfsn + - auth_username + - auth_token +- njalla + - auth_token +- nsone + - auth_token +- onapp + - auth_username + - auth_token + - auth_server +- online + - auth_token +- ovh + - auth_entrypoint + - auth_application_key + - auth_application_secret + - auth_consumer_key +- plesk + - auth_username + - auth_password + - plesk_server +- pointhq + - auth_username + - auth_token +- powerdns + - auth_token + - pdns_server + - pdns_server_id + - pdns_disable_notify +- rackspace + - auth_account + - auth_username + - auth_api_key + - auth_token + - sleep_time +- rage4 + - auth_username + - auth_token +- rcodezero + - auth_token +- route53 + - auth_access_key + - auth_access_secret + - private_zone + - auth_username + - auth_token +- safedns + - auth_token +- sakuracloud + - auth_token + - auth_secret +- softlayer + - auth_username + - auth_api_key +- transip + - auth_username + - auth_api_key +- ultradns + - auth_token + - auth_username + - auth_password +- vultr + - auth_token +- yandex + - auth_token +- zeit + - auth_token +- zilore + - auth_key +- zonomi + - auth_token + - auth_entrypoint diff --git a/debian/install b/debian/install index 1691a4849..521f2d3af 100644 --- a/debian/install +++ b/debian/install @@ -8,6 +8,7 @@ data/other/yunoprompt.service /etc/systemd/system/ data/other/password/* /usr/share/yunohost/other/password/ data/other/dpkg-origins/yunohost /etc/dpkg/origins data/other/dnsbl_list.yml /usr/share/yunohost/other/ +data/other/providers_list.yml /usr/share/yunohost/other/ data/other/ffdhe2048.pem /usr/share/yunohost/other/ data/other/* /usr/share/yunohost/yunohost-config/moulinette/ data/templates/* /usr/share/yunohost/templates/ From 51d6d19810a230c38fd5f9134a12320d7e263d39 Mon Sep 17 00:00:00 2001 From: MercierCorentin Date: Thu, 20 May 2021 13:29:04 +0200 Subject: [PATCH 016/130] Add first version of set domain provider --- data/actionsmap/yunohost.yml | 24 ++ data/bash-completion.d/yunohost | 117 ++++++++- .../{providers.yml => providers_list.yml} | 138 +++++------ src/yunohost/app.py | 1 + src/yunohost/domain.py | 225 +++++++++++------- 5 files changed, 344 insertions(+), 161 deletions(-) rename data/other/{providers.yml => providers_list.yml} (79%) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 58a48c87f..87dfcf026 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -582,6 +582,30 @@ domain: full: --delete help: Delete the key action: store_true + subcategories: + registrar: + subcategory_help: Manage domains registrars + actions: + ### domain_registrar_set() + set: + action_help: Set domain registrar + api: POST /domains/registrar + arguments: + domain: + help: Domain name + registrar: + help: registrar_key, see yunohost domain registrar list + -a: + full: --args + help: Serialized arguments for app script (i.e. "domain=domain.tld&path=/path") + ### domain_registrar_set() + get: + action_help: Get domain registrar + api: GET /domains/registrar + ### domain_registrar_list() + list: + action_help: List available registrars + api: GET /domains/registrar/list ############################# # App # diff --git a/data/bash-completion.d/yunohost b/data/bash-completion.d/yunohost index 2572a391d..1be522db2 100644 --- a/data/bash-completion.d/yunohost +++ b/data/bash-completion.d/yunohost @@ -1,3 +1,114 @@ -# This file is automatically generated -# during Debian's package build by the script -# data/actionsmap/yunohost_completion.py +# +# completion for yunohost +# automatically generated from the actionsmap +# + +_yunohost() +{ + local cur prev opts narg + COMPREPLY=() + + # the number of words already typed + narg=${#COMP_WORDS[@]} + + # the current word being typed + cur="${COMP_WORDS[COMP_CWORD]}" + + # If one is currently typing a category, + # match with categorys + if [[ $narg == 2 ]]; then + opts="user domain app backup settings service firewall dyndns tools hook log diagnosis" + fi + + # If one already typed a category, + # match the actions or the subcategories of that category + if [[ $narg == 3 ]]; then + # the category typed + category="${COMP_WORDS[1]}" + + if [[ $category == "user" ]]; then + opts="list create delete update info group permission ssh" + fi + if [[ $category == "domain" ]]; then + opts="list add registrar push_config remove dns-conf main-domain cert-status cert-install cert-renew url-available setting " + fi + if [[ $category == "app" ]]; then + opts="catalog search manifest fetchlist list info map install remove upgrade change-url setting register-url makedefault ssowatconf change-label addaccess removeaccess clearaccess action config" + fi + if [[ $category == "backup" ]]; then + opts="create restore list info download delete " + fi + if [[ $category == "settings" ]]; then + opts="list get set reset-all reset " + fi + if [[ $category == "service" ]]; then + opts="add remove start stop reload restart reload_or_restart enable disable status log regen-conf " + fi + if [[ $category == "firewall" ]]; then + opts="list allow disallow upnp reload stop " + fi + if [[ $category == "dyndns" ]]; then + opts="subscribe update installcron removecron " + fi + if [[ $category == "tools" ]]; then + opts="adminpw maindomain postinstall update upgrade shell shutdown reboot regen-conf versions migrations" + fi + if [[ $category == "hook" ]]; then + opts="add remove info list callback exec " + fi + if [[ $category == "log" ]]; then + opts="list show share " + fi + if [[ $category == "diagnosis" ]]; then + opts="list show get run ignore unignore " + fi + fi + + # If one already typed an action or a subcategory, + # match the actions of that subcategory + if [[ $narg == 4 ]]; then + # the category typed + category="${COMP_WORDS[1]}" + + # the action or the subcategory typed + action_or_subcategory="${COMP_WORDS[2]}" + + if [[ $category == "user" ]]; then + if [[ $action_or_subcategory == "group" ]]; then + opts="list create delete info add remove" + fi + if [[ $action_or_subcategory == "permission" ]]; then + opts="list info update add remove reset" + fi + if [[ $action_or_subcategory == "ssh" ]]; then + opts="list-keys add-key remove-key" + fi + fi + if [[ $category == "app" ]]; then + if [[ $action_or_subcategory == "action" ]]; then + opts="list run" + fi + if [[ $action_or_subcategory == "config" ]]; then + opts="show-panel apply" + fi + fi + if [[ $category == "tools" ]]; then + if [[ $action_or_subcategory == "migrations" ]]; then + opts="list run state" + fi + fi + fi + + # If no options were found propose --help + if [ -z "$opts" ]; then + prev="${COMP_WORDS[COMP_CWORD-1]}" + + if [[ $prev != "--help" ]]; then + opts=( --help ) + fi + fi + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 +} + +complete -F _yunohost yunohost \ No newline at end of file diff --git a/data/other/providers.yml b/data/other/providers_list.yml similarity index 79% rename from data/other/providers.yml rename to data/other/providers_list.yml index 4ba69d97b..a006bd272 100644 --- a/data/other/providers.yml +++ b/data/other/providers_list.yml @@ -1,218 +1,218 @@ -- aliyun +aliyun: - auth_key_id - auth_secret -- aurora +aurora: - auth_api_key - auth_secret_key -- azure +azure: - auth_client_id - auth_client_secret - auth_tenant_id - auth_subscription_id - resource_group -- cloudflare +cloudflare: - auth_username - auth_token - zone_id -- cloudns +cloudns: - auth_id - auth_subid - auth_subuser - auth_password - weight - port -- cloudxns +cloudxns: - auth_username - auth_token -- conoha +conoha: - auth_region - auth_token - auth_username - auth_password - auth_tenant_id -- constellix +constellix: - auth_username - auth_token -- digitalocean +digitalocean: - auth_token -- dinahosting +dinahosting: - auth_username - auth_password -- directadmin +directadmin: - auth_password - auth_username - endpoint -- dnsimple +dnsimple: - auth_token - auth_username - auth_password - auth_2fa -- dnsmadeeasy +dnsmadeeasy: - auth_username - auth_token -- dnspark +dnspark: - auth_username - auth_token -- dnspod +dnspod: - auth_username - auth_token -- dreamhost +dreamhost: - auth_token -- dynu +dynu: - auth_token -- easydns +easydns: - auth_username - auth_token -- easyname +easyname: - auth_username - auth_password -- euserv +euserv: - auth_username - auth_password -- exoscale +exoscale: - auth_key - auth_secret -- gandi +gandi: - auth_token - api_protocol -- gehirn +gehirn: - auth_token - auth_secret -- glesys +glesys: - auth_username - auth_token -- godaddy +godaddy: - auth_key - auth_secret -- googleclouddns +googleclouddns: - auth_service_account_info -- gransy +gransy: - auth_username - auth_password -- gratisdns +gratisdns: - auth_username - auth_password -- henet +henet: - auth_username - auth_password -- hetzner +hetzner: - auth_token -- hostingde +hostingde: - auth_token -- hover +hover: - auth_username - auth_password -- infoblox +infoblox: - auth_user - auth_psw - ib_view - ib_host -- infomaniak +infomaniak: - auth_token -- internetbs +internetbs: - auth_key - auth_password -- inwx +inwx: - auth_username - auth_password -- joker +joker: - auth_token -- linode +linode: - auth_token -- linode4 +linode4: - auth_token -- localzone +localzone: - filename -- luadns +luadns: - auth_username - auth_token -- memset +memset: - auth_token -- mythicbeasts +mythicbeasts: - auth_username - auth_password - auth_token -- namecheap +namecheap: - auth_token - auth_username - auth_client_ip - auth_sandbox -- namesilo +namesilo: - auth_token -- netcup +netcup: - auth_customer_id - auth_api_key - auth_api_password -- nfsn +nfsn: - auth_username - auth_token -- njalla +njalla: - auth_token -- nsone +nsone: - auth_token -- onapp +onapp: - auth_username - auth_token - auth_server -- online +online: - auth_token -- ovh +ovh: - auth_entrypoint - auth_application_key - auth_application_secret - auth_consumer_key -- plesk +plesk: - auth_username - auth_password - plesk_server -- pointhq +pointhq: - auth_username - auth_token -- powerdns +powerdns: - auth_token - pdns_server - pdns_server_id - pdns_disable_notify -- rackspace +rackspace: - auth_account - auth_username - auth_api_key - auth_token - sleep_time -- rage4 +rage4: - auth_username - auth_token -- rcodezero +rcodezero: - auth_token -- route53 +route53: - auth_access_key - auth_access_secret - private_zone - auth_username - auth_token -- safedns +safedns: - auth_token -- sakuracloud +sakuracloud: - auth_token - auth_secret -- softlayer +softlayer: - auth_username - auth_api_key -- transip +transip: - auth_username - auth_api_key -- ultradns +ultradns: - auth_token - auth_username - auth_password -- vultr +vultr: - auth_token -- yandex +yandex: - auth_token -- zeit +zeit: - auth_token -- zilore +zilore: - auth_key -- zonomi +zonomi: - auth_token - auth_entrypoint diff --git a/src/yunohost/app.py b/src/yunohost/app.py index c048ca5ea..b9a7e634b 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2977,6 +2977,7 @@ ARGUMENTS_TYPE_PARSERS = { } + def _parse_args_in_yunohost_format(user_answers, argument_questions): """Parse arguments store in either manifest.json or actions.json or from a config panel against the user answers when they are present. diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 05f2a16ae..6180616bf 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -29,8 +29,8 @@ import sys import yaml import functools -from lexicon.config import ConfigResolver -from lexicon.client import Client +# from lexicon.config import ConfigResolver +# from lexicon.client import Client from moulinette import m18n, msettings, msignals from moulinette.core import MoulinetteError @@ -43,6 +43,7 @@ from yunohost.app import ( _installed_apps, _get_app_settings, _get_conflicting_apps, + _parse_args_in_yunohost_format ) from yunohost.regenconf import regen_conf, _force_clear_hashes, _process_regen_conf from yunohost.utils.network import get_public_ip @@ -53,6 +54,7 @@ from yunohost.hook import hook_callback logger = getActionLogger("yunohost.domain") DOMAIN_SETTINGS_PATH = "/etc/yunohost/domains.yml" +REGISTRAR_LIST_PATH = "/usr/share/yunohost/other/providers_list.yml" def domain_list(exclude_subdomains=False): """ @@ -825,105 +827,150 @@ def _set_domain_settings(domain, domain_settings): with open(DOMAIN_SETTINGS_PATH, 'w') as file: yaml.dump(domains, file, default_flow_style=False) +# def domain_get_registrar(): +def domain_registrar_set(domain, registrar, args): + + domains = _load_domain_settings() + if not domain in domains.keys(): + raise YunohostError("domain_name_unknown", domain=domain) -def domain_push_config(domain): - """ - Send DNS records to the previously-configured registrar of the domain. - """ - # Generate the records - if domain not in domain_list()["domains"]: - raise YunohostValidationError("domain_name_unknown", domain=domain) + registrars = yaml.load(open(REGISTRAR_LIST_PATH, "r+")) + if not registrar in registrars.keys(): + # FIXME créer l'erreur + raise YunohostError("registrar_unknown") + + parameters = registrars[registrar] + ask_args = [] + for parameter in parameters: + ask_args.append({ + 'name' : parameter, + 'type': 'string', + 'example': '', + 'default': '', + }) + args_dict = ( + {} if not args else dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) + ) + parsed_answer_dict = _parse_args_in_yunohost_format(args_dict, ask_args) - domains_settings = _get_domain_settings(domain, True) - - dns_conf = _build_dns_conf(domains_settings) - - # Flatten the DNS conf - flatten_dns_conf = [] - for key in dns_conf: - list_of_records = dns_conf[key] - for record in list_of_records: - # FIXME Lexicon does not support CAA records - # See https://github.com/AnalogJ/lexicon/issues/282 and https://github.com/AnalogJ/lexicon/pull/371 - # They say it's trivial to implement it! - # And yet, it is still not done/merged - if record["type"] != "CAA": - # Add .domain.tdl to the name entry - record["name"] = "{}.{}".format(record["name"], domain) - flatten_dns_conf.append(record) - - # Get provider info - # TODO - provider = { - "name": "gandi", - "options": { - "api_protocol": "rest", - "auth_token": "vhcIALuRJKtoZiZyxfDYWLom" + domain_provider = { + 'name': registrar, + 'options': { + } } + for arg_name, arg_value_and_type in parsed_answer_dict.items(): + domain_provider['options'][arg_name] = arg_value_and_type[0] + + domain_settings = domains[domain] + domain_settings["provider"] = domain_provider - # Construct the base data structure to use lexicon's API. - base_config = { - "provider_name": provider["name"], - "domain": domain, # domain name - } - base_config[provider["name"]] = provider["options"] + # Save the settings to the .yaml file + with open(DOMAIN_SETTINGS_PATH, 'w') as file: + yaml.dump(domains, file, default_flow_style=False) + - # Get types present in the generated records - types = set() - for record in flatten_dns_conf: - types.add(record["type"]) - # Fetch all types present in the generated records - distant_records = {} - for key in types: - record_config = { - "action": "list", - "type": key, - } - final_lexicon = ConfigResolver().with_dict(dict_object=base_config).with_dict(dict_object=record_config) - # print('final_lexicon:', final_lexicon); - client = Client(final_lexicon) - distant_records[key] = client.execute() +# def domain_push_config(domain): +# """ +# Send DNS records to the previously-configured registrar of the domain. +# """ +# # Generate the records +# if domain not in domain_list()["domains"]: +# raise YunohostValidationError("domain_name_unknown", domain=domain) - for key in types: - for distant_record in distant_records[key]: - print('distant_record:', distant_record); - for local_record in flatten_dns_conf: - print('local_record:', local_record); +# domains_settings = _get_domain_settings(domain, True) - # Push the records - for record in flatten_dns_conf: - # For each record, first check if one record exists for the same (type, name) couple - it_exists = False - # TODO do not push if local and distant records are exactly the same ? - # is_the_same_record = False +# dns_conf = _build_dns_conf(domains_settings) - for distant_record in distant_records[record["type"]]: - if distant_record["type"] == record["type"] and distant_record["name"] == record["name"]: - it_exists = True - # previous TODO - # if distant_record["ttl"] = ... and distant_record["name"] ... - # is_the_same_record = True +# # Flatten the DNS conf +# flatten_dns_conf = [] +# for key in dns_conf: +# list_of_records = dns_conf[key] +# for record in list_of_records: +# # FIXME Lexicon does not support CAA records +# # See https://github.com/AnalogJ/lexicon/issues/282 and https://github.com/AnalogJ/lexicon/pull/371 +# # They say it's trivial to implement it! +# # And yet, it is still not done/merged +# if record["type"] != "CAA": +# # Add .domain.tdl to the name entry +# record["name"] = "{}.{}".format(record["name"], domain) +# flatten_dns_conf.append(record) - # Finally, push the new record or update the existing one - record_config = { - "action": "update" if it_exists else "create", # create, list, update, delete - "type": record["type"], # specify a type for record filtering, case sensitive in some cases. - "name": record["name"], - "content": record["value"], - # FIXME Delte TTL, doesn't work with Gandi. - # See https://github.com/AnalogJ/lexicon/issues/726 (similar issue) - # But I think there is another issue with Gandi. Or I'm misusing the API... - # "ttl": record["ttl"], - } - final_lexicon = ConfigResolver().with_dict(dict_object=base_config).with_dict(dict_object=record_config) - client = Client(final_lexicon) - print('pushed_record:', record_config, "→", end=' ') - results = client.execute() - print('results:', results); - # print("Failed" if results == False else "Ok") +# # Get provider info +# # TODO +# provider = { +# "name": "gandi", +# "options": { +# "api_protocol": "rest", +# "auth_token": "vhcIALuRJKtoZiZyxfDYWLom" +# } +# } + +# # Construct the base data structure to use lexicon's API. +# base_config = { +# "provider_name": provider["name"], +# "domain": domain, # domain name +# } +# base_config[provider["name"]] = provider["options"] + +# # Get types present in the generated records +# types = set() + +# for record in flatten_dns_conf: +# types.add(record["type"]) + +# # Fetch all types present in the generated records +# distant_records = {} + +# for key in types: +# record_config = { +# "action": "list", +# "type": key, +# } +# final_lexicon = ConfigResolver().with_dict(dict_object=base_config).with_dict(dict_object=record_config) +# # print('final_lexicon:', final_lexicon); +# client = Client(final_lexicon) +# distant_records[key] = client.execute() + +# for key in types: +# for distant_record in distant_records[key]: +# print('distant_record:', distant_record); +# for local_record in flatten_dns_conf: +# print('local_record:', local_record); + +# # Push the records +# for record in flatten_dns_conf: +# # For each record, first check if one record exists for the same (type, name) couple +# it_exists = False +# # TODO do not push if local and distant records are exactly the same ? +# # is_the_same_record = False + +# for distant_record in distant_records[record["type"]]: +# if distant_record["type"] == record["type"] and distant_record["name"] == record["name"]: +# it_exists = True +# # previous TODO +# # if distant_record["ttl"] = ... and distant_record["name"] ... +# # is_the_same_record = True + +# # Finally, push the new record or update the existing one +# record_config = { +# "action": "update" if it_exists else "create", # create, list, update, delete +# "type": record["type"], # specify a type for record filtering, case sensitive in some cases. +# "name": record["name"], +# "content": record["value"], +# # FIXME Delte TTL, doesn't work with Gandi. +# # See https://github.com/AnalogJ/lexicon/issues/726 (similar issue) +# # But I think there is another issue with Gandi. Or I'm misusing the API... +# # "ttl": record["ttl"], +# } +# final_lexicon = ConfigResolver().with_dict(dict_object=base_config).with_dict(dict_object=record_config) +# client = Client(final_lexicon) +# print('pushed_record:', record_config, "→", end=' ') +# results = client.execute() +# print('results:', results); +# # print("Failed" if results == False else "Ok") # def domain_config_fetch(domain, key, value): From 5859028022bf63b949c7fcb16f022cf543115721 Mon Sep 17 00:00:00 2001 From: Paco Date: Fri, 21 May 2021 10:36:04 +0200 Subject: [PATCH 017/130] Uncomment push_config function --- src/yunohost/domain.py | 176 ++++++++++++++++++++--------------------- 1 file changed, 88 insertions(+), 88 deletions(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 6180616bf..594eea159 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -29,8 +29,8 @@ import sys import yaml import functools -# from lexicon.config import ConfigResolver -# from lexicon.client import Client +from lexicon.config import ConfigResolver +from lexicon.client import Client from moulinette import m18n, msettings, msignals from moulinette.core import MoulinetteError @@ -873,104 +873,104 @@ def domain_registrar_set(domain, registrar, args): -# def domain_push_config(domain): -# """ -# Send DNS records to the previously-configured registrar of the domain. -# """ -# # Generate the records -# if domain not in domain_list()["domains"]: -# raise YunohostValidationError("domain_name_unknown", domain=domain) +def domain_push_config(domain): + """ + Send DNS records to the previously-configured registrar of the domain. + """ + # Generate the records + if domain not in domain_list()["domains"]: + raise YunohostValidationError("domain_name_unknown", domain=domain) -# domains_settings = _get_domain_settings(domain, True) + domains_settings = _get_domain_settings(domain, True) -# dns_conf = _build_dns_conf(domains_settings) + dns_conf = _build_dns_conf(domains_settings) -# # Flatten the DNS conf -# flatten_dns_conf = [] -# for key in dns_conf: -# list_of_records = dns_conf[key] -# for record in list_of_records: -# # FIXME Lexicon does not support CAA records -# # See https://github.com/AnalogJ/lexicon/issues/282 and https://github.com/AnalogJ/lexicon/pull/371 -# # They say it's trivial to implement it! -# # And yet, it is still not done/merged -# if record["type"] != "CAA": -# # Add .domain.tdl to the name entry -# record["name"] = "{}.{}".format(record["name"], domain) -# flatten_dns_conf.append(record) + # Flatten the DNS conf + flatten_dns_conf = [] + for key in dns_conf: + list_of_records = dns_conf[key] + for record in list_of_records: + # FIXME Lexicon does not support CAA records + # See https://github.com/AnalogJ/lexicon/issues/282 and https://github.com/AnalogJ/lexicon/pull/371 + # They say it's trivial to implement it! + # And yet, it is still not done/merged + if record["type"] != "CAA": + # Add .domain.tdl to the name entry + record["name"] = "{}.{}".format(record["name"], domain) + flatten_dns_conf.append(record) -# # Get provider info -# # TODO -# provider = { -# "name": "gandi", -# "options": { -# "api_protocol": "rest", -# "auth_token": "vhcIALuRJKtoZiZyxfDYWLom" -# } -# } + # Get provider info + # TODO + provider = { + "name": "gandi", + "options": { + "api_protocol": "rest", + "auth_token": "vhcIALuRJKtoZiZyxfDYWLom" + } + } -# # Construct the base data structure to use lexicon's API. -# base_config = { -# "provider_name": provider["name"], -# "domain": domain, # domain name -# } -# base_config[provider["name"]] = provider["options"] + # Construct the base data structure to use lexicon's API. + base_config = { + "provider_name": provider["name"], + "domain": domain, # domain name + } + base_config[provider["name"]] = provider["options"] -# # Get types present in the generated records -# types = set() + # Get types present in the generated records + types = set() -# for record in flatten_dns_conf: -# types.add(record["type"]) + for record in flatten_dns_conf: + types.add(record["type"]) -# # Fetch all types present in the generated records -# distant_records = {} + # Fetch all types present in the generated records + distant_records = {} -# for key in types: -# record_config = { -# "action": "list", -# "type": key, -# } -# final_lexicon = ConfigResolver().with_dict(dict_object=base_config).with_dict(dict_object=record_config) -# # print('final_lexicon:', final_lexicon); -# client = Client(final_lexicon) -# distant_records[key] = client.execute() + for key in types: + record_config = { + "action": "list", + "type": key, + } + final_lexicon = ConfigResolver().with_dict(dict_object=base_config).with_dict(dict_object=record_config) + # print('final_lexicon:', final_lexicon); + client = Client(final_lexicon) + distant_records[key] = client.execute() -# for key in types: -# for distant_record in distant_records[key]: -# print('distant_record:', distant_record); -# for local_record in flatten_dns_conf: -# print('local_record:', local_record); + for key in types: + for distant_record in distant_records[key]: + print('distant_record:', distant_record); + for local_record in flatten_dns_conf: + print('local_record:', local_record); -# # Push the records -# for record in flatten_dns_conf: -# # For each record, first check if one record exists for the same (type, name) couple -# it_exists = False -# # TODO do not push if local and distant records are exactly the same ? -# # is_the_same_record = False + # Push the records + for record in flatten_dns_conf: + # For each record, first check if one record exists for the same (type, name) couple + it_exists = False + # TODO do not push if local and distant records are exactly the same ? + # is_the_same_record = False -# for distant_record in distant_records[record["type"]]: -# if distant_record["type"] == record["type"] and distant_record["name"] == record["name"]: -# it_exists = True -# # previous TODO -# # if distant_record["ttl"] = ... and distant_record["name"] ... -# # is_the_same_record = True + for distant_record in distant_records[record["type"]]: + if distant_record["type"] == record["type"] and distant_record["name"] == record["name"]: + it_exists = True + # previous TODO + # if distant_record["ttl"] = ... and distant_record["name"] ... + # is_the_same_record = True -# # Finally, push the new record or update the existing one -# record_config = { -# "action": "update" if it_exists else "create", # create, list, update, delete -# "type": record["type"], # specify a type for record filtering, case sensitive in some cases. -# "name": record["name"], -# "content": record["value"], -# # FIXME Delte TTL, doesn't work with Gandi. -# # See https://github.com/AnalogJ/lexicon/issues/726 (similar issue) -# # But I think there is another issue with Gandi. Or I'm misusing the API... -# # "ttl": record["ttl"], -# } -# final_lexicon = ConfigResolver().with_dict(dict_object=base_config).with_dict(dict_object=record_config) -# client = Client(final_lexicon) -# print('pushed_record:', record_config, "→", end=' ') -# results = client.execute() -# print('results:', results); -# # print("Failed" if results == False else "Ok") + # Finally, push the new record or update the existing one + record_config = { + "action": "update" if it_exists else "create", # create, list, update, delete + "type": record["type"], # specify a type for record filtering, case sensitive in some cases. + "name": record["name"], + "content": record["value"], + # FIXME Delte TTL, doesn't work with Gandi. + # See https://github.com/AnalogJ/lexicon/issues/726 (similar issue) + # But I think there is another issue with Gandi. Or I'm misusing the API... + # "ttl": record["ttl"], + } + final_lexicon = ConfigResolver().with_dict(dict_object=base_config).with_dict(dict_object=record_config) + client = Client(final_lexicon) + print('pushed_record:', record_config, "→", end=' ') + results = client.execute() + print('results:', results); + # print("Failed" if results == False else "Ok") # def domain_config_fetch(domain, key, value): From d4b40245321153d886838244bbcf045c325aa59e Mon Sep 17 00:00:00 2001 From: Paco Date: Fri, 21 May 2021 11:31:12 +0200 Subject: [PATCH 018/130] fix: do not delete provider options when loading them --- src/yunohost/domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 594eea159..801d2c916 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -727,7 +727,7 @@ def _load_domain_settings(): new_domains[domain] = {} # new_domains[domain] = { "main": is_maindomain } # Set other values (default value if missing) - for setting, default in [ ("xmpp", is_maindomain), ("mail", is_maindomain), ("owned_dns_zone", default_owned_dns_zone), ("ttl", 3600) ]: + for setting, default in [ ("xmpp", is_maindomain), ("mail", is_maindomain), ("owned_dns_zone", default_owned_dns_zone), ("ttl", 3600), ("provider", False)]: if domain_in_old_domains and setting in old_domains[domain].keys(): new_domains[domain][setting] = old_domains[domain][setting] else: From 914bd1f20aa260c31f21f86f0db0f33437c59b27 Mon Sep 17 00:00:00 2001 From: Paco Date: Fri, 21 May 2021 11:32:52 +0200 Subject: [PATCH 019/130] connect domain_push_config to the in-file provider options --- src/yunohost/domain.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 801d2c916..0f4702ec1 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -885,6 +885,13 @@ def domain_push_config(domain): dns_conf = _build_dns_conf(domains_settings) + provider = domains_settings[domain]["provider"] + + if provider == False: + # FIXME add locales + raise YunohostValidationError("registrar_is_not_set", domain=domain) + + # Flatten the DNS conf flatten_dns_conf = [] for key in dns_conf: @@ -899,16 +906,6 @@ def domain_push_config(domain): record["name"] = "{}.{}".format(record["name"], domain) flatten_dns_conf.append(record) - # Get provider info - # TODO - provider = { - "name": "gandi", - "options": { - "api_protocol": "rest", - "auth_token": "vhcIALuRJKtoZiZyxfDYWLom" - } - } - # Construct the base data structure to use lexicon's API. base_config = { "provider_name": provider["name"], @@ -951,7 +948,7 @@ def domain_push_config(domain): for distant_record in distant_records[record["type"]]: if distant_record["type"] == record["type"] and distant_record["name"] == record["name"]: it_exists = True - # previous TODO + # see previous TODO # if distant_record["ttl"] = ... and distant_record["name"] ... # is_the_same_record = True @@ -961,7 +958,7 @@ def domain_push_config(domain): "type": record["type"], # specify a type for record filtering, case sensitive in some cases. "name": record["name"], "content": record["value"], - # FIXME Delte TTL, doesn't work with Gandi. + # FIXME Removed TTL, because it doesn't work with Gandi. # See https://github.com/AnalogJ/lexicon/issues/726 (similar issue) # But I think there is another issue with Gandi. Or I'm misusing the API... # "ttl": record["ttl"], From e40f8fb861ca3c98726309e1a6424b2a2f0ad540 Mon Sep 17 00:00:00 2001 From: Corentin Mercier Date: Tue, 25 May 2021 13:46:09 +0200 Subject: [PATCH 020/130] Apply easy fixes from code review Co-authored-by: ljf (zamentur) --- data/actionsmap/yunohost.yml | 31 ++++++++++++++++++++++++------- src/yunohost/domain.py | 28 +++++++++++++++------------- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 87dfcf026..6a7050d61 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -450,7 +450,7 @@ domain: ### domain_push_config() push_config: action_help: Push DNS records to registrar - api: GET /domains/push + api: GET /domains//push arguments: domain: help: Domain name to add @@ -568,11 +568,13 @@ domain: help: The path to check (e.g. /coffee) ### domain_setting() setting: - action_help: Set or get an app setting value + action_help: Set or get a domain setting value api: GET /domains//settings arguments: domain: help: Domain name + extra: + pattern: *pattern_domain key: help: Key to get/set -v: @@ -589,23 +591,38 @@ domain: ### domain_registrar_set() set: action_help: Set domain registrar - api: POST /domains/registrar + api: POST /domains//registrar arguments: domain: help: Domain name + extra: + pattern: *pattern_domain registrar: help: registrar_key, see yunohost domain registrar list -a: full: --args - help: Serialized arguments for app script (i.e. "domain=domain.tld&path=/path") + help: Serialized arguments for registrar API (i.e. "auth_token=TOKEN&auth_username=USER"). ### domain_registrar_set() get: action_help: Get domain registrar - api: GET /domains/registrar + api: GET /domains//registrar + arguments: + domain: + help: Domain name + extra: + pattern: *pattern_domain ### domain_registrar_list() list: - action_help: List available registrars - api: GET /domains/registrar/list + action_help: List registrars configured by DNS zone + api: GET /domains/registrars + catalog: + action_help: List supported registrars API + api: GET /domains/registrars/catalog + arguments: + -f: + full: --full + help: Display all details, including info to create forms + action: store_true ############################# # App # diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 0f4702ec1..6eae65487 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -131,7 +131,7 @@ def domain_add(operation_logger, domain, dyndns=False): # Do not allow to subscribe to multiple dyndns domains... if _guess_current_dyndns_domain("dyndns.yunohost.org") != (None, None): - raise YunohostValidationError('domain_dyndns_already_subscribed') + raise YunohostValidationError("domain_dyndns_already_subscribed") # Check that this domain can effectively be provided by # dyndns.yunohost.org. (i.e. is it a nohost.me / noho.st) @@ -208,8 +208,8 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False): # the 'force' here is related to the exception happening in domain_add ... # we don't want to check the domain exists because the ldap add may have # failed - if not force and domain not in domain_list()['domains']: - raise YunohostValidationError('domain_name_unknown', domain=domain) + if not force and domain not in domain_list()["domains"]: + raise YunohostValidationError("domain_name_unknown", domain=domain) # Check domain is not the main domain if domain == _get_maindomain(): @@ -223,7 +223,9 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False): other_domains="\n * " + ("\n * ".join(other_domains)), ) else: - raise YunohostValidationError("domain_cannot_remove_main_add_new_one", domain=domain) + raise YunohostValidationError( + "domain_cannot_remove_main_add_new_one", domain=domain + ) # Check if apps are installed on the domain apps_on_that_domain = [] @@ -515,12 +517,12 @@ def _build_dns_conf(domains): ] # DKIM/DMARC record - dkim_host, dkim_publickey = _get_DKIM(domain) + dkim_host, dkim_publickey = _get_DKIM(domain_name) if dkim_host: mail += [ [dkim_host, ttl, "TXT", dkim_publickey], - ["_dmarc", ttl, "TXT", '"v=DMARC1; p=none"'], + [f"_dmarc{child_domain_suffix}", ttl, "TXT", '"v=DMARC1; p=none"'], ] ######## @@ -528,8 +530,8 @@ def _build_dns_conf(domains): ######## if domain["xmpp"]: xmpp += [ - ["_xmpp-client._tcp", ttl, "SRV", "0 5 5222 %s." % domain_name], - ["_xmpp-server._tcp", ttl, "SRV", "0 5 5269 %s." % domain_name], + [f"_xmpp-client._tcp{child_domain_suffix}", ttl, "SRV", f"0 5 5222 {domain_name}."], + [f"_xmpp-server._tcp{child_domain_suffix}", ttl, "SRV", f"0 5 5269 {domain_name}."], ["muc" + child_domain_suffix, ttl, "CNAME", name], ["pubsub" + child_domain_suffix, ttl, "CNAME", name], ["vjud" + child_domain_suffix, ttl, "CNAME", name], @@ -542,10 +544,10 @@ def _build_dns_conf(domains): if ipv4: - extra.append(["*", ttl, "A", ipv4]) + extra.append([f"*{child_domain_suffix}", ttl, "A", ipv4]) if ipv6: - extra.append(["*", ttl, "AAAA", ipv6]) + extra.append([f"*{child_domain_suffix}", ttl, "AAAA", ipv6]) # TODO # elif include_empty_AAAA_if_no_ipv6: # extra.append(["*", ttl, "AAAA", None]) @@ -727,7 +729,7 @@ def _load_domain_settings(): new_domains[domain] = {} # new_domains[domain] = { "main": is_maindomain } # Set other values (default value if missing) - for setting, default in [ ("xmpp", is_maindomain), ("mail", is_maindomain), ("owned_dns_zone", default_owned_dns_zone), ("ttl", 3600), ("provider", False)]: + for setting, default in [ ("xmpp", is_maindomain), ("mail", True), ("owned_dns_zone", default_owned_dns_zone), ("ttl", 3600), ("provider", False)]: if domain_in_old_domains and setting in old_domains[domain].keys(): new_domains[domain][setting] = old_domains[domain][setting] else: @@ -773,7 +775,7 @@ def domain_setting(domain, key, value=None, delete=False): if "ttl" == key: try: ttl = int(value) - except: + except ValueError: # TODO add locales raise YunohostError("bad_value_type", value_type=type(ttl)) @@ -934,7 +936,7 @@ def domain_push_config(domain): for key in types: for distant_record in distant_records[key]: - print('distant_record:', distant_record); + logger.debug(f"distant_record: {distant_record}"); for local_record in flatten_dns_conf: print('local_record:', local_record); From ced4da417121732cb9928cc3d017131219d9fc74 Mon Sep 17 00:00:00 2001 From: Paco Date: Tue, 25 May 2021 16:18:04 +0200 Subject: [PATCH 021/130] Run `black` & revert misguidedly cosmetic changes An obscur plugin must have done this... --- src/yunohost/domain.py | 159 ++++++++++++++++++++++++++--------------- 1 file changed, 102 insertions(+), 57 deletions(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 6eae65487..8677e1685 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -43,7 +43,7 @@ from yunohost.app import ( _installed_apps, _get_app_settings, _get_conflicting_apps, - _parse_args_in_yunohost_format + _parse_args_in_yunohost_format, ) from yunohost.regenconf import regen_conf, _force_clear_hashes, _process_regen_conf from yunohost.utils.network import get_public_ip @@ -56,6 +56,7 @@ logger = getActionLogger("yunohost.domain") DOMAIN_SETTINGS_PATH = "/etc/yunohost/domains.yml" REGISTRAR_LIST_PATH = "/usr/share/yunohost/other/providers_list.yml" + def domain_list(exclude_subdomains=False): """ List domains @@ -109,6 +110,7 @@ def domain_add(operation_logger, domain, dyndns=False): from yunohost.hook import hook_callback from yunohost.app import app_ssowatconf from yunohost.utils.ldap import _get_ldap_interface + from yunohost.certificate import _certificate_install_selfsigned if domain.startswith("xmpp-upload."): raise YunohostValidationError("domain_cannot_add_xmpp_upload") @@ -142,14 +144,13 @@ def domain_add(operation_logger, domain, dyndns=False): if dyndns: from yunohost.dyndns import dyndns_subscribe + # Actually subscribe dyndns_subscribe(domain=domain) + _certificate_install_selfsigned([domain], False) + try: - import yunohost.certificate - - yunohost.certificate._certificate_install_selfsigned([domain], False) - attr_dict = { "objectClass": ["mailDomain", "top"], "virtualdomain": domain, @@ -176,13 +177,13 @@ def domain_add(operation_logger, domain, dyndns=False): regen_conf(names=["nginx", "metronome", "dnsmasq", "postfix", "rspamd"]) app_ssowatconf() - except Exception: + except Exception as e: # Force domain removal silently try: domain_remove(domain, force=True) except Exception: pass - raise + raise e hook_callback("post_domain_add", args=[domain]) @@ -234,21 +235,37 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False): settings = _get_app_settings(app) label = app_info(app)["name"] if settings.get("domain") == domain: - apps_on_that_domain.append((app, " - %s \"%s\" on https://%s%s" % (app, label, domain, settings["path"]) if "path" in settings else app)) + apps_on_that_domain.append( + ( + app, + ' - %s "%s" on https://%s%s' + % (app, label, domain, settings["path"]) + if "path" in settings + else app, + ) + ) if apps_on_that_domain: if remove_apps: - if msettings.get('interface') == "cli" and not force: - answer = msignals.prompt(m18n.n('domain_remove_confirm_apps_removal', - apps="\n".join([x[1] for x in apps_on_that_domain]), - answers='y/N'), color="yellow") + if msettings.get("interface") == "cli" and not force: + answer = msignals.prompt( + m18n.n( + "domain_remove_confirm_apps_removal", + apps="\n".join([x[1] for x in apps_on_that_domain]), + answers="y/N", + ), + color="yellow", + ) if answer.upper() != "Y": raise YunohostError("aborting") for app, _ in apps_on_that_domain: app_remove(app) else: - raise YunohostValidationError('domain_uninstall_app_first', apps="\n".join([x[1] for x in apps_on_that_domain])) + raise YunohostValidationError( + "domain_uninstall_app_first", + apps="\n".join([x[1] for x in apps_on_that_domain]), + ) operation_logger.start() @@ -261,7 +278,7 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False): os.system("rm -rf /etc/yunohost/certs/%s" % domain) # Delete dyndns keys for this domain (if any) - os.system('rm -rf /etc/yunohost/dyndns/K%s.+*' % domain) + os.system("rm -rf /etc/yunohost/dyndns/K%s.+*" % domain) # Sometime we have weird issues with the regenconf where some files # appears as manually modified even though they weren't touched ... @@ -475,7 +492,9 @@ def _build_dns_conf(domains): extra = [] ipv4 = get_public_ip() ipv6 = get_public_ip(6) - owned_dns_zone = "owned_dns_zone" in domains[root] and domains[root]["owned_dns_zone"] + owned_dns_zone = ( + "owned_dns_zone" in domains[root] and domains[root]["owned_dns_zone"] + ) root_prefix = root.partition(".")[0] child_domain_suffix = "" @@ -484,15 +503,14 @@ def _build_dns_conf(domains): ttl = domain["ttl"] if domain_name == root: - name = root_prefix if not owned_dns_zone else "@" + name = root_prefix if not owned_dns_zone else "@" else: - name = domain_name[0:-(1 + len(root))] + name = domain_name[0 : -(1 + len(root))] if not owned_dns_zone: name += "." + root_prefix - + if name != "@": child_domain_suffix = "." + name - ########################### # Basic ipv4/ipv6 records # @@ -530,8 +548,18 @@ def _build_dns_conf(domains): ######## if domain["xmpp"]: xmpp += [ - [f"_xmpp-client._tcp{child_domain_suffix}", ttl, "SRV", f"0 5 5222 {domain_name}."], - [f"_xmpp-server._tcp{child_domain_suffix}", ttl, "SRV", f"0 5 5269 {domain_name}."], + [ + f"_xmpp-client._tcp{child_domain_suffix}", + ttl, + "SRV", + f"0 5 5222 {domain_name}.", + ], + [ + f"_xmpp-server._tcp{child_domain_suffix}", + ttl, + "SRV", + f"0 5 5269 {domain_name}.", + ], ["muc" + child_domain_suffix, ttl, "CNAME", name], ["pubsub" + child_domain_suffix, ttl, "CNAME", name], ["vjud" + child_domain_suffix, ttl, "CNAME", name], @@ -542,7 +570,6 @@ def _build_dns_conf(domains): # Extra # ######### - if ipv4: extra.append([f"*{child_domain_suffix}", ttl, "A", ipv4]) @@ -729,7 +756,13 @@ def _load_domain_settings(): new_domains[domain] = {} # new_domains[domain] = { "main": is_maindomain } # Set other values (default value if missing) - for setting, default in [ ("xmpp", is_maindomain), ("mail", True), ("owned_dns_zone", default_owned_dns_zone), ("ttl", 3600), ("provider", False)]: + for setting, default in [ + ("xmpp", is_maindomain), + ("mail", True), + ("owned_dns_zone", default_owned_dns_zone), + ("ttl", 3600), + ("provider", False), + ]: if domain_in_old_domains and setting in old_domains[domain].keys(): new_domains[domain][setting] = old_domains[domain][setting] else: @@ -737,6 +770,7 @@ def _load_domain_settings(): return new_domains + def domain_setting(domain, key, value=None, delete=False): """ Set or get an app setting value @@ -753,7 +787,7 @@ def domain_setting(domain, key, value=None, delete=False): if not domain in domains.keys(): # TODO add locales raise YunohostError("domain_name_unknown", domain=domain) - + domain_settings = domains[domain] # GET @@ -771,7 +805,7 @@ def domain_setting(domain, key, value=None, delete=False): # SET else: - + if "ttl" == key: try: ttl = int(value) @@ -785,6 +819,7 @@ def domain_setting(domain, key, value=None, delete=False): domain_settings[key] = value _set_domain_settings(domain, domain_settings) + def _get_domain_settings(domain, subdomains): """ Get settings of a domain @@ -826,12 +861,13 @@ def _set_domain_settings(domain, domain_settings): domains[domain] = domain_settings # Save the settings to the .yaml file - with open(DOMAIN_SETTINGS_PATH, 'w') as file: + with open(DOMAIN_SETTINGS_PATH, "w") as file: yaml.dump(domains, file, default_flow_style=False) + # def domain_get_registrar(): def domain_registrar_set(domain, registrar, args): - + domains = _load_domain_settings() if not domain in domains.keys(): raise YunohostError("domain_name_unknown", domain=domain) @@ -840,39 +876,33 @@ def domain_registrar_set(domain, registrar, args): if not registrar in registrars.keys(): # FIXME créer l'erreur raise YunohostError("registrar_unknown") - + parameters = registrars[registrar] ask_args = [] for parameter in parameters: - ask_args.append({ - 'name' : parameter, - 'type': 'string', - 'example': '', - 'default': '', - }) + ask_args.append( + { + "name": parameter, + "type": "string", + "example": "", + "default": "", + } + ) args_dict = ( {} if not args else dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) ) parsed_answer_dict = _parse_args_in_yunohost_format(args_dict, ask_args) - domain_provider = { - 'name': registrar, - 'options': { - - } - } + domain_provider = {"name": registrar, "options": {}} for arg_name, arg_value_and_type in parsed_answer_dict.items(): - domain_provider['options'][arg_name] = arg_value_and_type[0] - + domain_provider["options"][arg_name] = arg_value_and_type[0] + domain_settings = domains[domain] domain_settings["provider"] = domain_provider # Save the settings to the .yaml file - with open(DOMAIN_SETTINGS_PATH, 'w') as file: + with open(DOMAIN_SETTINGS_PATH, "w") as file: yaml.dump(domains, file, default_flow_style=False) - - - def domain_push_config(domain): @@ -893,7 +923,6 @@ def domain_push_config(domain): # FIXME add locales raise YunohostValidationError("registrar_is_not_set", domain=domain) - # Flatten the DNS conf flatten_dns_conf = [] for key in dns_conf: @@ -911,7 +940,7 @@ def domain_push_config(domain): # Construct the base data structure to use lexicon's API. base_config = { "provider_name": provider["name"], - "domain": domain, # domain name + "domain": domain, # domain name } base_config[provider["name"]] = provider["options"] @@ -929,16 +958,20 @@ def domain_push_config(domain): "action": "list", "type": key, } - final_lexicon = ConfigResolver().with_dict(dict_object=base_config).with_dict(dict_object=record_config) + final_lexicon = ( + ConfigResolver() + .with_dict(dict_object=base_config) + .with_dict(dict_object=record_config) + ) # print('final_lexicon:', final_lexicon); client = Client(final_lexicon) distant_records[key] = client.execute() for key in types: for distant_record in distant_records[key]: - logger.debug(f"distant_record: {distant_record}"); + logger.debug(f"distant_record: {distant_record}") for local_record in flatten_dns_conf: - print('local_record:', local_record); + print("local_record:", local_record) # Push the records for record in flatten_dns_conf: @@ -948,7 +981,10 @@ def domain_push_config(domain): # is_the_same_record = False for distant_record in distant_records[record["type"]]: - if distant_record["type"] == record["type"] and distant_record["name"] == record["name"]: + if ( + distant_record["type"] == record["type"] + and distant_record["name"] == record["name"] + ): it_exists = True # see previous TODO # if distant_record["ttl"] = ... and distant_record["name"] ... @@ -956,8 +992,12 @@ def domain_push_config(domain): # Finally, push the new record or update the existing one record_config = { - "action": "update" if it_exists else "create", # create, list, update, delete - "type": record["type"], # specify a type for record filtering, case sensitive in some cases. + "action": "update" + if it_exists + else "create", # create, list, update, delete + "type": record[ + "type" + ], # specify a type for record filtering, case sensitive in some cases. "name": record["name"], "content": record["value"], # FIXME Removed TTL, because it doesn't work with Gandi. @@ -965,11 +1005,16 @@ def domain_push_config(domain): # But I think there is another issue with Gandi. Or I'm misusing the API... # "ttl": record["ttl"], } - final_lexicon = ConfigResolver().with_dict(dict_object=base_config).with_dict(dict_object=record_config) + final_lexicon = ( + ConfigResolver() + .with_dict(dict_object=base_config) + .with_dict(dict_object=record_config) + ) client = Client(final_lexicon) - print('pushed_record:', record_config, "→", end=' ') + print("pushed_record:", record_config, "→", end=" ") results = client.execute() - print('results:', results); + print("results:", results) # print("Failed" if results == False else "Ok") + # def domain_config_fetch(domain, key, value): From 27a976f5a595e04721be2b17acc51f6629eade9a Mon Sep 17 00:00:00 2001 From: MercierCorentin Date: Fri, 28 May 2021 11:22:11 +0200 Subject: [PATCH 022/130] Delete file that shouldn't be committed --- data/bash-completion.d/yunohost | 114 -------------------------------- 1 file changed, 114 deletions(-) delete mode 100644 data/bash-completion.d/yunohost diff --git a/data/bash-completion.d/yunohost b/data/bash-completion.d/yunohost deleted file mode 100644 index 1be522db2..000000000 --- a/data/bash-completion.d/yunohost +++ /dev/null @@ -1,114 +0,0 @@ -# -# completion for yunohost -# automatically generated from the actionsmap -# - -_yunohost() -{ - local cur prev opts narg - COMPREPLY=() - - # the number of words already typed - narg=${#COMP_WORDS[@]} - - # the current word being typed - cur="${COMP_WORDS[COMP_CWORD]}" - - # If one is currently typing a category, - # match with categorys - if [[ $narg == 2 ]]; then - opts="user domain app backup settings service firewall dyndns tools hook log diagnosis" - fi - - # If one already typed a category, - # match the actions or the subcategories of that category - if [[ $narg == 3 ]]; then - # the category typed - category="${COMP_WORDS[1]}" - - if [[ $category == "user" ]]; then - opts="list create delete update info group permission ssh" - fi - if [[ $category == "domain" ]]; then - opts="list add registrar push_config remove dns-conf main-domain cert-status cert-install cert-renew url-available setting " - fi - if [[ $category == "app" ]]; then - opts="catalog search manifest fetchlist list info map install remove upgrade change-url setting register-url makedefault ssowatconf change-label addaccess removeaccess clearaccess action config" - fi - if [[ $category == "backup" ]]; then - opts="create restore list info download delete " - fi - if [[ $category == "settings" ]]; then - opts="list get set reset-all reset " - fi - if [[ $category == "service" ]]; then - opts="add remove start stop reload restart reload_or_restart enable disable status log regen-conf " - fi - if [[ $category == "firewall" ]]; then - opts="list allow disallow upnp reload stop " - fi - if [[ $category == "dyndns" ]]; then - opts="subscribe update installcron removecron " - fi - if [[ $category == "tools" ]]; then - opts="adminpw maindomain postinstall update upgrade shell shutdown reboot regen-conf versions migrations" - fi - if [[ $category == "hook" ]]; then - opts="add remove info list callback exec " - fi - if [[ $category == "log" ]]; then - opts="list show share " - fi - if [[ $category == "diagnosis" ]]; then - opts="list show get run ignore unignore " - fi - fi - - # If one already typed an action or a subcategory, - # match the actions of that subcategory - if [[ $narg == 4 ]]; then - # the category typed - category="${COMP_WORDS[1]}" - - # the action or the subcategory typed - action_or_subcategory="${COMP_WORDS[2]}" - - if [[ $category == "user" ]]; then - if [[ $action_or_subcategory == "group" ]]; then - opts="list create delete info add remove" - fi - if [[ $action_or_subcategory == "permission" ]]; then - opts="list info update add remove reset" - fi - if [[ $action_or_subcategory == "ssh" ]]; then - opts="list-keys add-key remove-key" - fi - fi - if [[ $category == "app" ]]; then - if [[ $action_or_subcategory == "action" ]]; then - opts="list run" - fi - if [[ $action_or_subcategory == "config" ]]; then - opts="show-panel apply" - fi - fi - if [[ $category == "tools" ]]; then - if [[ $action_or_subcategory == "migrations" ]]; then - opts="list run state" - fi - fi - fi - - # If no options were found propose --help - if [ -z "$opts" ]; then - prev="${COMP_WORDS[COMP_CWORD-1]}" - - if [[ $prev != "--help" ]]; then - opts=( --help ) - fi - fi - COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) - return 0 -} - -complete -F _yunohost yunohost \ No newline at end of file From 00098075fdb30812a5086456b9b7649d1ae24425 Mon Sep 17 00:00:00 2001 From: Paco Date: Sat, 29 May 2021 19:15:13 +0200 Subject: [PATCH 023/130] Split domains.yml into domains/{domain}.yml --- src/yunohost/domain.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 8677e1685..60711667a 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -53,7 +53,7 @@ from yunohost.hook import hook_callback logger = getActionLogger("yunohost.domain") -DOMAIN_SETTINGS_PATH = "/etc/yunohost/domains.yml" +DOMAIN_SETTINGS_DIR = "/etc/yunohost/domains" REGISTRAR_LIST_PATH = "/usr/share/yunohost/other/providers_list.yml" @@ -732,39 +732,38 @@ def _load_domain_settings(): Retrieve entries in domains.yml And fill the holes if any """ - # Retrieve entries in the YAML - old_domains = None - if os.path.exists(DOMAIN_SETTINGS_PATH) and os.path.isfile(DOMAIN_SETTINGS_PATH): - old_domains = yaml.load(open(DOMAIN_SETTINGS_PATH, "r+")) - - if old_domains is None: - old_domains = dict() + # Retrieve actual domain list + get_domain_list = domain_list() # Create sanitized data new_domains = dict() - get_domain_list = domain_list() - # Load main domain maindomain = get_domain_list["main"] for domain in get_domain_list["domains"]: + # Retrieve entries in the YAML + filepath = f"{DOMAIN_SETTINGS_DIR}/{domain}.yml" + old_domain = {} + if os.path.exists(filepath) and os.path.isfile(filepath): + old_domain = yaml.load(open(filepath, "r+")) + # If the file is empty or "corrupted" + if not type(old_domain) is set: + old_domain = {} is_maindomain = domain == maindomain default_owned_dns_zone = True if domain == get_public_suffix(domain) else False - domain_in_old_domains = domain in old_domains.keys() # Update each setting if not present new_domains[domain] = {} - # new_domains[domain] = { "main": is_maindomain } # Set other values (default value if missing) for setting, default in [ ("xmpp", is_maindomain), ("mail", True), ("owned_dns_zone", default_owned_dns_zone), ("ttl", 3600), - ("provider", False), + ("provider", {}), ]: - if domain_in_old_domains and setting in old_domains[domain].keys(): - new_domains[domain][setting] = old_domains[domain][setting] + if old_domain != {} and setting in old_domain.keys(): + new_domains[domain][setting] = old_domain[setting] else: new_domains[domain][setting] = default @@ -858,11 +857,13 @@ def _set_domain_settings(domain, domain_settings): if not domain in domains.keys(): raise YunohostError("domain_name_unknown", domain=domain) - domains[domain] = domain_settings - + # First create the DOMAIN_SETTINGS_DIR if it doesn't exist + if not os.path.exists(DOMAIN_SETTINGS_DIR): + os.mkdir(DOMAIN_SETTINGS_DIR) # Save the settings to the .yaml file - with open(DOMAIN_SETTINGS_PATH, "w") as file: - yaml.dump(domains, file, default_flow_style=False) + filepath = f"{DOMAIN_SETTINGS_DIR}/{domain}.yml" + with open(filepath, "w") as file: + yaml.dump(domain_settings, file, default_flow_style=False) # def domain_get_registrar(): @@ -901,8 +902,7 @@ def domain_registrar_set(domain, registrar, args): domain_settings["provider"] = domain_provider # Save the settings to the .yaml file - with open(DOMAIN_SETTINGS_PATH, "w") as file: - yaml.dump(domains, file, default_flow_style=False) + _set_domain_settings(domain, domain_settings) def domain_push_config(domain): From 3022a4756047f870233f20cb8de13731a113f47a Mon Sep 17 00:00:00 2001 From: Paco Date: Sat, 29 May 2021 19:39:59 +0200 Subject: [PATCH 024/130] Now using Dict.update() when loading settings Settings not anticipated will be loaded. They will not be removed on write. Original behavior: not anticipated keys are removed. --- src/yunohost/domain.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 60711667a..3c0fb479a 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -744,28 +744,26 @@ def _load_domain_settings(): for domain in get_domain_list["domains"]: # Retrieve entries in the YAML filepath = f"{DOMAIN_SETTINGS_DIR}/{domain}.yml" - old_domain = {} + on_disk_settings = {} if os.path.exists(filepath) and os.path.isfile(filepath): - old_domain = yaml.load(open(filepath, "r+")) + on_disk_settings = yaml.load(open(filepath, "r+")) # If the file is empty or "corrupted" - if not type(old_domain) is set: - old_domain = {} + if not type(on_disk_settings) is dict: + on_disk_settings = {} + # Generate defaults is_maindomain = domain == maindomain default_owned_dns_zone = True if domain == get_public_suffix(domain) else False + default_settings = { + "xmpp": is_maindomain, + "mail": True, + "owned_dns_zone": default_owned_dns_zone, + "ttl": 3600, + "provider": {}, + } # Update each setting if not present - new_domains[domain] = {} - # Set other values (default value if missing) - for setting, default in [ - ("xmpp", is_maindomain), - ("mail", True), - ("owned_dns_zone", default_owned_dns_zone), - ("ttl", 3600), - ("provider", {}), - ]: - if old_domain != {} and setting in old_domain.keys(): - new_domains[domain][setting] = old_domain[setting] - else: - new_domains[domain][setting] = default + default_settings.update(on_disk_settings) + # Add the domain to the list + new_domains[domain] = default_settings return new_domains From 2665c6235aba64a3fc17bc1992bf1a4908a21638 Mon Sep 17 00:00:00 2001 From: MercierCorentin Date: Thu, 3 Jun 2021 11:23:03 +0200 Subject: [PATCH 025/130] Rename provider_list.yml into dns_zone_hosters_list.yml --- .../other/{providers_list.yml => dns_zone_hosters_list.yml.yml} | 0 debian/install | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename data/other/{providers_list.yml => dns_zone_hosters_list.yml.yml} (100%) diff --git a/data/other/providers_list.yml b/data/other/dns_zone_hosters_list.yml.yml similarity index 100% rename from data/other/providers_list.yml rename to data/other/dns_zone_hosters_list.yml.yml diff --git a/debian/install b/debian/install index 521f2d3af..4ff0ed52d 100644 --- a/debian/install +++ b/debian/install @@ -8,7 +8,7 @@ data/other/yunoprompt.service /etc/systemd/system/ data/other/password/* /usr/share/yunohost/other/password/ data/other/dpkg-origins/yunohost /etc/dpkg/origins data/other/dnsbl_list.yml /usr/share/yunohost/other/ -data/other/providers_list.yml /usr/share/yunohost/other/ +data/other/dns_zone_hosters_list.yml /usr/share/yunohost/other/ data/other/ffdhe2048.pem /usr/share/yunohost/other/ data/other/* /usr/share/yunohost/yunohost-config/moulinette/ data/templates/* /usr/share/yunohost/templates/ From c66bfc84520567760389b5d6471472daab88bca9 Mon Sep 17 00:00:00 2001 From: MercierCorentin Date: Thu, 3 Jun 2021 11:25:41 +0200 Subject: [PATCH 026/130] Fix mistake in dns_zone_hosters_list.yml name --- .../{dns_zone_hosters_list.yml.yml => dns_zone_hosters_list.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename data/other/{dns_zone_hosters_list.yml.yml => dns_zone_hosters_list.yml} (100%) diff --git a/data/other/dns_zone_hosters_list.yml.yml b/data/other/dns_zone_hosters_list.yml similarity index 100% rename from data/other/dns_zone_hosters_list.yml.yml rename to data/other/dns_zone_hosters_list.yml From 97d68f85adaecac58eaf86f847b5b0b3d11faed4 Mon Sep 17 00:00:00 2001 From: MercierCorentin Date: Thu, 3 Jun 2021 15:01:40 +0200 Subject: [PATCH 027/130] Add domain registrar info --- data/actionsmap/yunohost.yml | 6 +++--- src/yunohost/domain.py | 17 ++++++++++++++++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 6a7050d61..48d9f6b7e 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -602,9 +602,9 @@ domain: -a: full: --args help: Serialized arguments for registrar API (i.e. "auth_token=TOKEN&auth_username=USER"). - ### domain_registrar_set() - get: - action_help: Get domain registrar + ### domain_registrar_info() + info: + action_help: Display info about registrar settings used for a domain api: GET /domains//registrar arguments: domain: diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 3c0fb479a..1b7235297 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -864,7 +864,22 @@ def _set_domain_settings(domain, domain_settings): yaml.dump(domain_settings, file, default_flow_style=False) -# def domain_get_registrar(): +def domain_registrar_info(domain): + + domains = _load_domain_settings() + if not domain in domains.keys(): + raise YunohostError("domain_name_unknown", domain=domain) + + provider = domains[domain]["provider"] + + if provider: + logger.info("Registrar name : " + provider['name']) + for option in provider['options']: + logger.info("Option " + option + " : "+provider['options'][option]) + else: + logger.info("Registrar settings are not set for " + domain) + + def domain_registrar_set(domain, registrar, args): domains = _load_domain_settings() From 008baf13509b1030ef4ea5742b99c74c31ce2b77 Mon Sep 17 00:00:00 2001 From: MercierCorentin Date: Fri, 4 Jun 2021 09:25:00 +0200 Subject: [PATCH 028/130] Add network util to get dns zone from domain name --- src/yunohost/utils/network.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/yunohost/utils/network.py b/src/yunohost/utils/network.py index d96151fa4..895a2fe5a 100644 --- a/src/yunohost/utils/network.py +++ b/src/yunohost/utils/network.py @@ -185,7 +185,27 @@ def dig( return ("ok", answers) +def get_dns_zone_from_domain(domain): + """ + Get the DNS zone of a domain + Keyword arguments: + domain -- The domain name + + """ + separator = "." + domain_subs = domain.split(separator) + for i in range(0, len(domain_subs)): + answer = dig(separator.join(domain_subs), rdtype="NS", full_answers=True) + if answer[0] == "ok" : + return separator.join(domain_subs) + elif answer[1][0] == "NXDOMAIN" : + return None + domain_subs.pop(0) + + # Should not be executed + return None + def _extract_inet(string, skip_netmask=False, skip_loopback=True): """ Extract IP addresses (v4 and/or v6) from a string limited to one From 4b9dbd92ebe2fce1d02873483491abc9a86f3584 Mon Sep 17 00:00:00 2001 From: MercierCorentin Date: Fri, 4 Jun 2021 09:42:04 +0200 Subject: [PATCH 029/130] Moved get_dns_zone_from_domain from utils/network to utils/dns --- src/yunohost/utils/dns.py | 23 ++++++++++++++++++++++- src/yunohost/utils/network.py | 21 --------------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/yunohost/utils/dns.py b/src/yunohost/utils/dns.py index 3033743d1..26347101f 100644 --- a/src/yunohost/utils/dns.py +++ b/src/yunohost/utils/dns.py @@ -35,4 +35,25 @@ def get_public_suffix(domain): domain_prefix = domain_name[0:-(1 + len(public_suffix))] public_suffix = domain_prefix.plit(".")[-1] + "." + public_suffix - return public_suffix \ No newline at end of file + return public_suffix + +def get_dns_zone_from_domain(domain): + """ + Get the DNS zone of a domain + + Keyword arguments: + domain -- The domain name + + """ + separator = "." + domain_subs = domain.split(separator) + for i in range(0, len(domain_subs)): + answer = dig(separator.join(domain_subs), rdtype="NS", full_answers=True) + if answer[0] == "ok" : + return separator.join(domain_subs) + elif answer[1][0] == "NXDOMAIN" : + return None + domain_subs.pop(0) + + # Should not be executed + return None \ No newline at end of file diff --git a/src/yunohost/utils/network.py b/src/yunohost/utils/network.py index 895a2fe5a..9423199bb 100644 --- a/src/yunohost/utils/network.py +++ b/src/yunohost/utils/network.py @@ -184,27 +184,6 @@ def dig( answers = [answer.to_text() for answer in answers] return ("ok", answers) - -def get_dns_zone_from_domain(domain): - """ - Get the DNS zone of a domain - - Keyword arguments: - domain -- The domain name - - """ - separator = "." - domain_subs = domain.split(separator) - for i in range(0, len(domain_subs)): - answer = dig(separator.join(domain_subs), rdtype="NS", full_answers=True) - if answer[0] == "ok" : - return separator.join(domain_subs) - elif answer[1][0] == "NXDOMAIN" : - return None - domain_subs.pop(0) - - # Should not be executed - return None def _extract_inet(string, skip_netmask=False, skip_loopback=True): """ From 06dcacbe8bc42c0b289247fa4c429d92b6a167a7 Mon Sep 17 00:00:00 2001 From: MercierCorentin Date: Fri, 4 Jun 2021 09:52:10 +0200 Subject: [PATCH 030/130] Fix import issue in utils/dns --- src/yunohost/utils/dns.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yunohost/utils/dns.py b/src/yunohost/utils/dns.py index 26347101f..00bbc4f62 100644 --- a/src/yunohost/utils/dns.py +++ b/src/yunohost/utils/dns.py @@ -19,6 +19,7 @@ """ from publicsuffix import PublicSuffixList +from yunohost.utils.network import dig YNH_DYNDNS_DOMAINS = ["nohost.me", "noho.st", "ynh.fr"] From 3812dcd7afc9e3b74d89b3ba12d554da32f4ae59 Mon Sep 17 00:00:00 2001 From: Paco Date: Fri, 4 Jun 2021 16:04:40 +0200 Subject: [PATCH 031/130] =?UTF-8?q?setting=20owned=5Fdns=5Fzone=20(Bool)?= =?UTF-8?q?=20=E2=86=92=20dns=5Fzone=20(String)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/yunohost/domain.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 1b7235297..51f7c9b82 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -47,7 +47,7 @@ from yunohost.app import ( ) from yunohost.regenconf import regen_conf, _force_clear_hashes, _process_regen_conf from yunohost.utils.network import get_public_ip -from yunohost.utils.dns import get_public_suffix +from yunohost.utils.dns import get_dns_zone_from_domain from yunohost.log import is_unit_operation from yunohost.hook import hook_callback @@ -354,6 +354,7 @@ def domain_dns_conf(domain): result += "\n{name} {ttl} IN {type} {value}".format(**record) if msettings.get("interface") == "cli": + # FIXME Update this to point to our "dns push" doc logger.info(m18n.n("domain_dns_conf_is_just_a_recommendation")) return result @@ -493,7 +494,8 @@ def _build_dns_conf(domains): ipv4 = get_public_ip() ipv6 = get_public_ip(6) owned_dns_zone = ( - "owned_dns_zone" in domains[root] and domains[root]["owned_dns_zone"] + # TODO test this + "dns_zone" in domains[root] and domains[root]["dns_zone"] == root ) root_prefix = root.partition(".")[0] @@ -727,9 +729,10 @@ def _get_DKIM(domain): ) +# FIXME split the loading of the domain settings → domain by domain (& file by file) def _load_domain_settings(): """ - Retrieve entries in domains.yml + Retrieve entries in domains/[domain].yml And fill the holes if any """ # Retrieve actual domain list @@ -752,11 +755,11 @@ def _load_domain_settings(): on_disk_settings = {} # Generate defaults is_maindomain = domain == maindomain - default_owned_dns_zone = True if domain == get_public_suffix(domain) else False + dns_zone = get_dns_zone_from_domain(domain) default_settings = { "xmpp": is_maindomain, "mail": True, - "owned_dns_zone": default_owned_dns_zone, + "dns_zone": dns_zone, "ttl": 3600, "provider": {}, } @@ -833,6 +836,8 @@ def _get_domain_settings(domain, subdomains): only_wanted_domains = dict() for entry in domains.keys(): if subdomains: + # FIXME does example.co is seen as a subdomain of example.com? + # TODO _is_subdomain_of_domain if domain in entry: only_wanted_domains[entry] = domains[entry] else: From b082f0314c227706e1fdf82249f0a80185661169 Mon Sep 17 00:00:00 2001 From: Paco Date: Fri, 11 Jun 2021 16:01:04 +0200 Subject: [PATCH 032/130] Split registrar settings to /etc/yunohost/registrars/{dns_zone}.yml --- src/yunohost/domain.py | 58 +++++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 51f7c9b82..e800798a9 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -54,6 +54,7 @@ from yunohost.hook import hook_callback logger = getActionLogger("yunohost.domain") DOMAIN_SETTINGS_DIR = "/etc/yunohost/domains" +REGISTRAR_SETTINGS_DIR = "/etc/yunohost/registrars" REGISTRAR_LIST_PATH = "/usr/share/yunohost/other/providers_list.yml" @@ -771,6 +772,22 @@ def _load_domain_settings(): return new_domains +def _load_registrar_setting(dns_zone): + """ + Retrieve entries in registrars/[dns_zone].yml + """ + + on_disk_settings = {} + filepath = f"{REGISTRAR_SETTINGS_DIR}/{dns_zone}.yml" + if os.path.exists(filepath) and os.path.isfile(filepath): + on_disk_settings = yaml.load(open(filepath, "r+")) + # If the file is empty or "corrupted" + if not type(on_disk_settings) is dict: + on_disk_settings = {} + + return on_disk_settings + + def domain_setting(domain, key, value=None, delete=False): """ Set or get an app setting value @@ -869,27 +886,32 @@ def _set_domain_settings(domain, domain_settings): yaml.dump(domain_settings, file, default_flow_style=False) -def domain_registrar_info(domain): - +def _load_zone_of_domain(domain): domains = _load_domain_settings() if not domain in domains.keys(): + # TODO add locales raise YunohostError("domain_name_unknown", domain=domain) + + return domains[domain]["dns_zone"] + + +def domain_registrar_info(domain): + + dns_zone = _load_zone_of_domain(domain) + registrar_info = _load_registrar_setting(dns_zone) + if not registrar_info: + # TODO add locales + raise YunohostError("no_registrar_set_for_this_dns_zone", dns_zone=dns_zone) - provider = domains[domain]["provider"] - - if provider: - logger.info("Registrar name : " + provider['name']) - for option in provider['options']: - logger.info("Option " + option + " : "+provider['options'][option]) - else: - logger.info("Registrar settings are not set for " + domain) + logger.info("Registrar name: " + registrar_info['name']) + for option_key, option_value in registrar_info['options'].items(): + logger.info("Option " + option_key + ": " + option_value) def domain_registrar_set(domain, registrar, args): - domains = _load_domain_settings() - if not domain in domains.keys(): - raise YunohostError("domain_name_unknown", domain=domain) + dns_zone = _load_zone_of_domain(domain) + registrar_info = _load_registrar_setting(dns_zone) registrars = yaml.load(open(REGISTRAR_LIST_PATH, "r+")) if not registrar in registrars.keys(): @@ -916,11 +938,13 @@ def domain_registrar_set(domain, registrar, args): for arg_name, arg_value_and_type in parsed_answer_dict.items(): domain_provider["options"][arg_name] = arg_value_and_type[0] - domain_settings = domains[domain] - domain_settings["provider"] = domain_provider - + # First create the REGISTRAR_SETTINGS_DIR if it doesn't exist + if not os.path.exists(REGISTRAR_SETTINGS_DIR): + os.mkdir(REGISTRAR_SETTINGS_DIR) # Save the settings to the .yaml file - _set_domain_settings(domain, domain_settings) + filepath = f"{REGISTRAR_SETTINGS_DIR}/{dns_zone}.yml" + with open(filepath, "w") as file: + yaml.dump(domain_provider, file, default_flow_style=False) def domain_push_config(domain): From 208df9601f7fb51d4d13ae4fce24617f9f07fe56 Mon Sep 17 00:00:00 2001 From: MercierCorentin Date: Fri, 18 Jun 2021 13:39:22 +0200 Subject: [PATCH 033/130] Add domain registrar catalog cli command --- data/actionsmap/yunohost.yml | 1 + src/yunohost/domain.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 2dd33c488..7c0272398 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -615,6 +615,7 @@ domain: list: action_help: List registrars configured by DNS zone api: GET /domains/registrars + ### domain_registrar_catalog() catalog: action_help: List supported registrars API api: GET /domains/registrars/catalog diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index e800798a9..920df934f 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -29,8 +29,7 @@ import sys import yaml import functools -from lexicon.config import ConfigResolver -from lexicon.client import Client +from lexicon import * from moulinette import m18n, msettings, msignals from moulinette.core import MoulinetteError @@ -907,6 +906,15 @@ def domain_registrar_info(domain): for option_key, option_value in registrar_info['options'].items(): logger.info("Option " + option_key + ": " + option_value) +def domain_registrar_catalog(full): + registrars = yaml.load(open(REGISTRAR_LIST_PATH, "r+")) + for registrar in registrars: + logger.info("Registrar : " + registrar) + if full : + logger.info("Options : ") + for option in registrars[registrar]: + logger.info("\t- " + option) + def domain_registrar_set(domain, registrar, args): From 7f76f0a613a457e82592be26b0296f42616877fa Mon Sep 17 00:00:00 2001 From: Paco Date: Fri, 18 Jun 2021 14:46:46 +0200 Subject: [PATCH 034/130] fix new locales --- locales/en.json | 3 +++ src/yunohost/domain.py | 11 +++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/locales/en.json b/locales/en.json index 84a01dfaa..75912cab5 100644 --- a/locales/en.json +++ b/locales/en.json @@ -285,7 +285,9 @@ "domain_dyndns_already_subscribed": "You have already subscribed to a DynDNS domain", "domain_dyndns_root_unknown": "Unknown DynDNS root domain", "domain_exists": "The domain already exists", + "domain_property_unknown": "The property {property} dooesn't exist", "domain_hostname_failed": "Unable to set new hostname. This might cause an issue later (it might be fine).", + "domain_registrar_unknown": "This registrar is unknown. Look for yours with the command `yunohost domain catalog`", "domain_remove_confirm_apps_removal": "Removing this domain will remove those applications:\n{apps}\n\nAre you sure you want to do that? [{answers}]", "domain_uninstall_app_first": "Those applications are still installed on your domain:\n{apps}\n\nPlease uninstall them using 'yunohost app remove the_app_id' or move them to another domain using 'yunohost app change-url the_app_id' before proceeding to domain removal", "domain_name_unknown": "Domain '{domain}' unknown", @@ -528,6 +530,7 @@ "regenconf_need_to_explicitly_specify_ssh": "The ssh configuration has been manually modified, but you need to explicitly specify category 'ssh' with --force to actually apply the changes.", "regex_incompatible_with_tile": "/!\\ Packagers! Permission '{permission}' has show_tile set to 'true' and you therefore cannot define a regex URL as the main URL", "regex_with_only_domain": "You can't use a regex for domain, only for path", + "registrar_is_not_set": "The registrar for this domain has not been configured", "restore_already_installed_app": "An app with the ID '{app:s}' is already installed", "restore_already_installed_apps": "The following apps can't be restored because they are already installed: {apps}", "restore_backup_too_old": "This backup archive can not be restored because it comes from a too-old YunoHost version.", diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 920df934f..6c90b533e 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -809,7 +809,7 @@ def domain_setting(domain, key, value=None, delete=False): # GET if value is None and not delete: if not key in domain_settings: - raise YunohostValidationError("This key doesn't exist!") + raise YunohostValidationError("domain_property_unknown", property=key) return domain_settings[key] @@ -827,11 +827,10 @@ def domain_setting(domain, key, value=None, delete=False): ttl = int(value) except ValueError: # TODO add locales - raise YunohostError("bad_value_type", value_type=type(ttl)) + raise YunohostError("invalid_number", value_type=type(ttl)) if ttl < 0: - # TODO add locales - raise YunohostError("must_be_positive", value_type=type(ttl)) + raise YunohostError("pattern_positive_number", value_type=type(ttl)) domain_settings[key] = value _set_domain_settings(domain, domain_settings) @@ -900,7 +899,7 @@ def domain_registrar_info(domain): registrar_info = _load_registrar_setting(dns_zone) if not registrar_info: # TODO add locales - raise YunohostError("no_registrar_set_for_this_dns_zone", dns_zone=dns_zone) + raise YunohostError("registrar_is_not_set", dns_zone=dns_zone) logger.info("Registrar name: " + registrar_info['name']) for option_key, option_value in registrar_info['options'].items(): @@ -924,7 +923,7 @@ def domain_registrar_set(domain, registrar, args): registrars = yaml.load(open(REGISTRAR_LIST_PATH, "r+")) if not registrar in registrars.keys(): # FIXME créer l'erreur - raise YunohostError("registrar_unknown") + raise YunohostError("domain_registrar_unknown") parameters = registrars[registrar] ask_args = [] From ab86d13b1ec1eb96f8b60332b65a439771c90b98 Mon Sep 17 00:00:00 2001 From: Paco Date: Fri, 18 Jun 2021 19:14:51 +0200 Subject: [PATCH 035/130] Update _load_domain_settings to load only requested domains --- locales/en.json | 2 +- src/yunohost/domain.py | 28 ++++++++++++++++++---------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/locales/en.json b/locales/en.json index 75912cab5..712ecb844 100644 --- a/locales/en.json +++ b/locales/en.json @@ -285,7 +285,7 @@ "domain_dyndns_already_subscribed": "You have already subscribed to a DynDNS domain", "domain_dyndns_root_unknown": "Unknown DynDNS root domain", "domain_exists": "The domain already exists", - "domain_property_unknown": "The property {property} dooesn't exist", + "domain_property_unknown": "The property {property} doesn't exist", "domain_hostname_failed": "Unable to set new hostname. This might cause an issue later (it might be fine).", "domain_registrar_unknown": "This registrar is unknown. Look for yours with the command `yunohost domain catalog`", "domain_remove_confirm_apps_removal": "Removing this domain will remove those applications:\n{apps}\n\nAre you sure you want to do that? [{answers}]", diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 6c90b533e..c13234d0b 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -729,14 +729,24 @@ def _get_DKIM(domain): ) -# FIXME split the loading of the domain settings → domain by domain (& file by file) -def _load_domain_settings(): +def _load_domain_settings(for_domains=[]): """ - Retrieve entries in domains/[domain].yml + Retrieve entries in /etc/yunohost/domains/[domain].yml And fill the holes if any """ # Retrieve actual domain list get_domain_list = domain_list() + domains = [] + + if for_domains: + # keep only the requested domains + domains = filter(lambda domain : domain in for_domains, get_domain_list["domains"]) + # check that all requested domains are there + unknown_domains = filter(lambda domain : not domain in domains, for_domains) + unknown_domain = next(unknown_domains, None) + if unknown_domain != None: + raise YunohostValidationError("domain_name_unknown", domain=unknown_domain) + # Create sanitized data new_domains = dict() @@ -744,7 +754,7 @@ def _load_domain_settings(): # Load main domain maindomain = get_domain_list["main"] - for domain in get_domain_list["domains"]: + for domain in domains: # Retrieve entries in the YAML filepath = f"{DOMAIN_SETTINGS_DIR}/{domain}.yml" on_disk_settings = {} @@ -799,7 +809,8 @@ def domain_setting(domain, key, value=None, delete=False): """ - domains = _load_domain_settings() + domains = _load_domain_settings([ domain ]) + if not domain in domains.keys(): # TODO add locales raise YunohostError("domain_name_unknown", domain=domain) @@ -871,8 +882,7 @@ def _set_domain_settings(domain, domain_settings): settings -- Dict with doamin settings """ - domains = _load_domain_settings() - if not domain in domains.keys(): + if domain not in domain_list()["domains"]: raise YunohostError("domain_name_unknown", domain=domain) # First create the DOMAIN_SETTINGS_DIR if it doesn't exist @@ -885,9 +895,7 @@ def _set_domain_settings(domain, domain_settings): def _load_zone_of_domain(domain): - domains = _load_domain_settings() - if not domain in domains.keys(): - # TODO add locales + if domain not in domain_list()["domains"]: raise YunohostError("domain_name_unknown", domain=domain) return domains[domain]["dns_zone"] From 22c9edb7d724baf5ef8a39c6792cc72f403fcd56 Mon Sep 17 00:00:00 2001 From: Paco Date: Sat, 19 Jun 2021 03:01:28 +0200 Subject: [PATCH 036/130] fix new _load_domain_settings bug --- src/yunohost/domain.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index c13234d0b..930d4841c 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -729,20 +729,20 @@ def _get_DKIM(domain): ) -def _load_domain_settings(for_domains=[]): +def _load_domain_settings(domains=[]): """ Retrieve entries in /etc/yunohost/domains/[domain].yml And fill the holes if any """ # Retrieve actual domain list get_domain_list = domain_list() - domains = [] - if for_domains: - # keep only the requested domains - domains = filter(lambda domain : domain in for_domains, get_domain_list["domains"]) - # check that all requested domains are there - unknown_domains = filter(lambda domain : not domain in domains, for_domains) + if domains: + # check existence of requested domains + all_known_domains = get_domain_list["domains"] + # filter inexisting domains + unknown_domains = filter(lambda domain : not domain in all_known_domains, domains) + # get first unknown domain unknown_domain = next(unknown_domains, None) if unknown_domain != None: raise YunohostValidationError("domain_name_unknown", domain=unknown_domain) @@ -812,7 +812,6 @@ def domain_setting(domain, key, value=None, delete=False): domains = _load_domain_settings([ domain ]) if not domain in domains.keys(): - # TODO add locales raise YunohostError("domain_name_unknown", domain=domain) domain_settings = domains[domain] From 31c31b1298cbedc7fe0d7df91211cb1f8a6c9397 Mon Sep 17 00:00:00 2001 From: Paco Date: Fri, 25 Jun 2021 10:48:33 +0200 Subject: [PATCH 037/130] Lexicon: ~ 4 bugfixes --- ...ne_hosters_list.yml => registrar_list.yml} | 0 debian/install | 2 +- src/yunohost/domain.py | 57 ++++++++++--------- 3 files changed, 32 insertions(+), 27 deletions(-) rename data/other/{dns_zone_hosters_list.yml => registrar_list.yml} (100%) diff --git a/data/other/dns_zone_hosters_list.yml b/data/other/registrar_list.yml similarity index 100% rename from data/other/dns_zone_hosters_list.yml rename to data/other/registrar_list.yml diff --git a/debian/install b/debian/install index 4ff0ed52d..55ddb34c6 100644 --- a/debian/install +++ b/debian/install @@ -8,7 +8,7 @@ data/other/yunoprompt.service /etc/systemd/system/ data/other/password/* /usr/share/yunohost/other/password/ data/other/dpkg-origins/yunohost /etc/dpkg/origins data/other/dnsbl_list.yml /usr/share/yunohost/other/ -data/other/dns_zone_hosters_list.yml /usr/share/yunohost/other/ +data/other/registrar_list.yml /usr/share/yunohost/other/ data/other/ffdhe2048.pem /usr/share/yunohost/other/ data/other/* /usr/share/yunohost/yunohost-config/moulinette/ data/templates/* /usr/share/yunohost/templates/ diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 930d4841c..9e79a13ae 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -29,7 +29,8 @@ import sys import yaml import functools -from lexicon import * +from lexicon.client import Client +from lexicon.config import ConfigResolver from moulinette import m18n, msettings, msignals from moulinette.core import MoulinetteError @@ -54,7 +55,7 @@ logger = getActionLogger("yunohost.domain") DOMAIN_SETTINGS_DIR = "/etc/yunohost/domains" REGISTRAR_SETTINGS_DIR = "/etc/yunohost/registrars" -REGISTRAR_LIST_PATH = "/usr/share/yunohost/other/providers_list.yml" +REGISTRAR_LIST_PATH = "/usr/share/yunohost/other/registrar_list.yml" def domain_list(exclude_subdomains=False): @@ -321,9 +322,7 @@ def domain_dns_conf(domain): if domain not in domain_list()["domains"]: raise YunohostValidationError("domain_name_unknown", domain=domain) - domains_settings = _get_domain_settings(domain, True) - - dns_conf = _build_dns_conf(domains_settings) + dns_conf = _build_dns_conf(domain) result = "" @@ -445,11 +444,14 @@ def _get_maindomain(): return maindomain -def _build_dns_conf(domains): +def _build_dns_conf(domain): """ Internal function that will returns a data structure containing the needed information to generate/adapt the dns configuration + Arguments: + domains -- List of a domain and its subdomains + The returned datastructure will have the following form: { "basic": [ @@ -485,7 +487,7 @@ def _build_dns_conf(domains): } """ - root = min(domains.keys(), key=(lambda k: len(k))) + domains = _get_domain_settings(domain, True) basic = [] mail = [] @@ -495,19 +497,19 @@ def _build_dns_conf(domains): ipv6 = get_public_ip(6) owned_dns_zone = ( # TODO test this - "dns_zone" in domains[root] and domains[root]["dns_zone"] == root + "dns_zone" in domains[domain] and domains[domain]["dns_zone"] == domain ) - root_prefix = root.partition(".")[0] + root_prefix = domain.partition(".")[0] child_domain_suffix = "" for domain_name, domain in domains.items(): ttl = domain["ttl"] - if domain_name == root: - name = root_prefix if not owned_dns_zone else "@" + if domain_name == domain: + name = "@" if owned_dns_zone else root_prefix else: - name = domain_name[0 : -(1 + len(root))] + name = domain_name[0 : -(1 + len(domain))] if not owned_dns_zone: name += "." + root_prefix @@ -746,6 +748,8 @@ def _load_domain_settings(domains=[]): unknown_domain = next(unknown_domains, None) if unknown_domain != None: raise YunohostValidationError("domain_name_unknown", domain=unknown_domain) + else: + domains = domain_list()["domains"] # Create sanitized data @@ -771,7 +775,6 @@ def _load_domain_settings(domains=[]): "mail": True, "dns_zone": dns_zone, "ttl": 3600, - "provider": {}, } # Update each setting if not present default_settings.update(on_disk_settings) @@ -845,6 +848,10 @@ def domain_setting(domain, key, value=None, delete=False): _set_domain_settings(domain, domain_settings) +def _is_subdomain_of(subdomain, domain): + return True if re.search("(^|\\.)" + domain + "$", subdomain) else False + + def _get_domain_settings(domain, subdomains): """ Get settings of a domain @@ -861,9 +868,7 @@ def _get_domain_settings(domain, subdomains): only_wanted_domains = dict() for entry in domains.keys(): if subdomains: - # FIXME does example.co is seen as a subdomain of example.com? - # TODO _is_subdomain_of_domain - if domain in entry: + if _is_subdomain_of(entry, domain): only_wanted_domains[entry] = domains[entry] else: if domain == entry: @@ -948,9 +953,9 @@ def domain_registrar_set(domain, registrar, args): ) parsed_answer_dict = _parse_args_in_yunohost_format(args_dict, ask_args) - domain_provider = {"name": registrar, "options": {}} + domain_registrar = {"name": registrar, "options": {}} for arg_name, arg_value_and_type in parsed_answer_dict.items(): - domain_provider["options"][arg_name] = arg_value_and_type[0] + domain_registrar["options"][arg_name] = arg_value_and_type[0] # First create the REGISTRAR_SETTINGS_DIR if it doesn't exist if not os.path.exists(REGISTRAR_SETTINGS_DIR): @@ -958,7 +963,7 @@ def domain_registrar_set(domain, registrar, args): # Save the settings to the .yaml file filepath = f"{REGISTRAR_SETTINGS_DIR}/{dns_zone}.yml" with open(filepath, "w") as file: - yaml.dump(domain_provider, file, default_flow_style=False) + yaml.dump(domain_registrar, file, default_flow_style=False) def domain_push_config(domain): @@ -969,13 +974,13 @@ def domain_push_config(domain): if domain not in domain_list()["domains"]: raise YunohostValidationError("domain_name_unknown", domain=domain) - domains_settings = _get_domain_settings(domain, True) + dns_conf = _build_dns_conf(domain) - dns_conf = _build_dns_conf(domains_settings) + domain_settings = _load_domain_settings([ domain ]) + dns_zone = domain_settings[domain]["dns_zone"] + registrar_setting = _load_registrar_setting(dns_zone) - provider = domains_settings[domain]["provider"] - - if provider == False: + if not registrar_setting: # FIXME add locales raise YunohostValidationError("registrar_is_not_set", domain=domain) @@ -995,10 +1000,10 @@ def domain_push_config(domain): # Construct the base data structure to use lexicon's API. base_config = { - "provider_name": provider["name"], + "provider_name": registrar_setting["name"], "domain": domain, # domain name } - base_config[provider["name"]] = provider["options"] + base_config[registrar_setting["name"]] = registrar_setting["options"] # Get types present in the generated records types = set() From d7c88cbf7813d9af6ecdc5933db0fe793c8a809f Mon Sep 17 00:00:00 2001 From: MercierCorentin Date: Tue, 13 Jul 2021 17:01:51 +0200 Subject: [PATCH 038/130] Add registrar name option to registrar catalog --- data/actionsmap/yunohost.yml | 3 +++ src/yunohost/domain.py | 23 +++++++++++++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 7c0272398..f0e7a5bd5 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -620,6 +620,9 @@ domain: action_help: List supported registrars API api: GET /domains/registrars/catalog arguments: + -r: + full: --registrar-name + help: Display given registrar info to create form -f: full: --full help: Display all details, including info to create forms diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 9e79a13ae..e87d1e118 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -917,15 +917,22 @@ def domain_registrar_info(domain): for option_key, option_value in registrar_info['options'].items(): logger.info("Option " + option_key + ": " + option_value) -def domain_registrar_catalog(full): - registrars = yaml.load(open(REGISTRAR_LIST_PATH, "r+")) - for registrar in registrars: - logger.info("Registrar : " + registrar) - if full : - logger.info("Options : ") - for option in registrars[registrar]: - logger.info("\t- " + option) +def _print_registrar_info(registrar_name, full, options): + logger.info("Registrar : " + registrar_name) + if full : + logger.info("Options : ") + for option in options: + logger.info("\t- " + option) +def domain_registrar_catalog(registrar_name, full): + registrars = yaml.load(open(REGISTRAR_LIST_PATH, "r+")) + + if registrar_name and registrar_name in registrars.keys() : + _print_registrar_info(registrar_name, True, registrars[registrar_name]) + else: + for registrar in registrars: + _print_registrar_info(registrar, full, registrars[registrar]) + def domain_registrar_set(domain, registrar, args): From 8c1d1dd99a5059f9cb0e153075af9cae3bb0215c Mon Sep 17 00:00:00 2001 From: MercierCorentin Date: Tue, 13 Jul 2021 18:17:23 +0200 Subject: [PATCH 039/130] Utils/dns get_dns_zone_from_domain improve --- src/yunohost/utils/dns.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/yunohost/utils/dns.py b/src/yunohost/utils/dns.py index 00bbc4f62..461fb0a4a 100644 --- a/src/yunohost/utils/dns.py +++ b/src/yunohost/utils/dns.py @@ -39,6 +39,7 @@ def get_public_suffix(domain): return public_suffix def get_dns_zone_from_domain(domain): + # TODO Check if this function is YNH_DYNDNS_DOMAINS compatible """ Get the DNS zone of a domain @@ -49,12 +50,15 @@ def get_dns_zone_from_domain(domain): separator = "." domain_subs = domain.split(separator) for i in range(0, len(domain_subs)): - answer = dig(separator.join(domain_subs), rdtype="NS", full_answers=True) + current_domain = separator.join(domain_subs) + answer = dig(current_domain, rdtype="NS", full_answers=True, resolvers="force_external") + print(answer) if answer[0] == "ok" : - return separator.join(domain_subs) - elif answer[1][0] == "NXDOMAIN" : - return None + # Domain is dns_zone + return current_domain + if separator.join(domain_subs[1:]) == get_public_suffix(current_domain): + # Couldn't check if domain is dns zone, + # returning private suffix + return current_domain domain_subs.pop(0) - - # Should not be executed return None \ No newline at end of file From 8104e48e4032f9bdca82103e581ddb7ac9a5af30 Mon Sep 17 00:00:00 2001 From: MercierCorentin Date: Tue, 13 Jul 2021 18:18:09 +0200 Subject: [PATCH 040/130] Fix _load_zone_of_domain --- src/yunohost/domain.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index e87d1e118..a33a4c425 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -899,7 +899,8 @@ def _set_domain_settings(domain, domain_settings): def _load_zone_of_domain(domain): - if domain not in domain_list()["domains"]: + domains = _load_domain_settings([domain]) + if domain not in domains.keys(): raise YunohostError("domain_name_unknown", domain=domain) return domains[domain]["dns_zone"] From d358452a03908948467289c9d9586e10d46db132 Mon Sep 17 00:00:00 2001 From: MercierCorentin Date: Thu, 15 Jul 2021 21:07:26 +0200 Subject: [PATCH 041/130] Bug fixe + cleaning --- data/actionsmap/yunohost.yml | 2 +- src/yunohost/domain.py | 2 +- src/yunohost/utils/dns.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index f0e7a5bd5..0349d2f28 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -448,7 +448,7 @@ domain: action: store_true ### domain_push_config() - push_config: + push-config: action_help: Push DNS records to registrar api: GET /domains//push arguments: diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index a33a4c425..6babe4f25 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -509,7 +509,7 @@ def _build_dns_conf(domain): if domain_name == domain: name = "@" if owned_dns_zone else root_prefix else: - name = domain_name[0 : -(1 + len(domain))] + name = domain_name if not owned_dns_zone: name += "." + root_prefix diff --git a/src/yunohost/utils/dns.py b/src/yunohost/utils/dns.py index 461fb0a4a..fd7cb1334 100644 --- a/src/yunohost/utils/dns.py +++ b/src/yunohost/utils/dns.py @@ -52,7 +52,6 @@ def get_dns_zone_from_domain(domain): for i in range(0, len(domain_subs)): current_domain = separator.join(domain_subs) answer = dig(current_domain, rdtype="NS", full_answers=True, resolvers="force_external") - print(answer) if answer[0] == "ok" : # Domain is dns_zone return current_domain From 4755c1c6d5a15217d398ae08f327e10d70607b34 Mon Sep 17 00:00:00 2001 From: MercierCorentin Date: Wed, 21 Jul 2021 23:14:23 +0200 Subject: [PATCH 042/130] Separate mail setting in mail_in and mail_out + fix domain settings types --- src/yunohost/domain.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 6babe4f25..a313cc28e 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -531,11 +531,14 @@ def _build_dns_conf(domain): ######### # Email # ######### - if domain["mail"]: - + if domain["mail_in"]: mail += [ - [name, ttl, "MX", "10 %s." % domain_name], - [name, ttl, "TXT", '"v=spf1 a mx -all"'], + [name, ttl, "MX", "10 %s." % domain_name] + ] + + if domain["mail_out"]: + mail += [ + [name, ttl, "TXT", '"v=spf1 a mx -all"'] ] # DKIM/DMARC record @@ -772,7 +775,8 @@ def _load_domain_settings(domains=[]): dns_zone = get_dns_zone_from_domain(domain) default_settings = { "xmpp": is_maindomain, - "mail": True, + "mail_in": True, + "mail_out": True, "dns_zone": dns_zone, "ttl": 3600, } @@ -805,13 +809,15 @@ def domain_setting(domain, key, value=None, delete=False): Set or get an app setting value Keyword argument: - value -- Value to set - app -- App ID + domain -- Domain Name key -- Key to get/set + value -- Value to set delete -- Delete the key """ + boolean_keys = ["mail_in", "mail_out", "xmpp"] + domains = _load_domain_settings([ domain ]) if not domain in domains.keys(): @@ -834,17 +840,22 @@ def domain_setting(domain, key, value=None, delete=False): # SET else: + if key in boolean_keys: + value = True if value.lower() in ['true', '1', 't', 'y', 'yes', "iloveynh"] else False if "ttl" == key: try: - ttl = int(value) + value = int(value) except ValueError: # TODO add locales - raise YunohostError("invalid_number", value_type=type(ttl)) + raise YunohostError("invalid_number", value_type=type(value)) - if ttl < 0: - raise YunohostError("pattern_positive_number", value_type=type(ttl)) + if value < 0: + raise YunohostError("pattern_positive_number", value_type=type(value)) + + # Set new value domain_settings[key] = value + # Save settings _set_domain_settings(domain, domain_settings) From 91a4dc49929086760f846071ce1ca8f18289edb3 Mon Sep 17 00:00:00 2001 From: MercierCorentin Date: Sun, 1 Aug 2021 19:02:51 +0200 Subject: [PATCH 043/130] Create domains test + Remove domain settings file when removing domain --- src/yunohost/domain.py | 5 +- src/yunohost/tests/test_domains.py | 165 +++++++++++++++++++++++++++++ 2 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 src/yunohost/tests/test_domains.py diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index a313cc28e..a45f70c25 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -281,6 +281,9 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False): # Delete dyndns keys for this domain (if any) os.system("rm -rf /etc/yunohost/dyndns/K%s.+*" % domain) + # Delete settings file for this domain + os.system(f"rm -rf {DOMAIN_SETTINGS_DIR}/{domain}.yml") + # Sometime we have weird issues with the regenconf where some files # appears as manually modified even though they weren't touched ... # There are a few ideas why this happens (like backup/restore nginx @@ -752,7 +755,7 @@ def _load_domain_settings(domains=[]): if unknown_domain != None: raise YunohostValidationError("domain_name_unknown", domain=unknown_domain) else: - domains = domain_list()["domains"] + domains = get_domain_list["domains"] # Create sanitized data diff --git a/src/yunohost/tests/test_domains.py b/src/yunohost/tests/test_domains.py new file mode 100644 index 000000000..c75954118 --- /dev/null +++ b/src/yunohost/tests/test_domains.py @@ -0,0 +1,165 @@ +import pytest + +import yaml +import os + +from moulinette.core import MoulinetteError + +from yunohost.utils.error import YunohostError, YunohostValidationError +from yunohost.utils.dns import get_dns_zone_from_domain +from yunohost.domain import ( + DOMAIN_SETTINGS_DIR, + REGISTRAR_LIST_PATH, + _get_maindomain, + domain_add, + domain_remove, + domain_list, + domain_main_domain, + domain_setting, + domain_dns_conf, + domain_registrar_set, + domain_registrar_catalog +) + +TEST_DOMAINS = [ + "example.tld", + "sub.example.tld", + "other-example.com" +] + +def setup_function(function): + + # Save domain list in variable to avoid multiple calls to domain_list() + domains = domain_list()["domains"] + + # First domain is main domain + if not TEST_DOMAINS[0] in domains: + domain_add(TEST_DOMAINS[0]) + else: + # Reset settings if any + os.system(f"rm -rf {DOMAIN_SETTINGS_DIR}/{TEST_DOMAINS[0]}.yml") + + if not _get_maindomain() == TEST_DOMAINS[0]: + domain_main_domain(TEST_DOMAINS[0]) + + # Clear other domains + for domain in domains: + if domain not in TEST_DOMAINS or domain == TEST_DOMAINS[2]: + # Clean domains not used for testing + domain_remove(domain) + elif domain in TEST_DOMAINS: + # Reset settings if any + os.system(f"rm -rf {DOMAIN_SETTINGS_DIR}/{domain}.yml") + + + # Create classical second domain of not exist + if TEST_DOMAINS[1] not in domains: + domain_add(TEST_DOMAINS[1]) + + # Third domain is not created + + clean() + + +def teardown_function(function): + + clean() + +def clean(): + pass + +# Domains management testing +def test_domain_add(): + assert TEST_DOMAINS[2] not in domain_list()["domains"] + domain_add(TEST_DOMAINS[2]) + assert TEST_DOMAINS[2] in domain_list()["domains"] + +def test_domain_add_existing_domain(): + with pytest.raises(MoulinetteError) as e_info: + assert TEST_DOMAINS[1] in domain_list()["domains"] + domain_add(TEST_DOMAINS[1]) + +def test_domain_remove(): + assert TEST_DOMAINS[1] in domain_list()["domains"] + domain_remove(TEST_DOMAINS[1]) + assert TEST_DOMAINS[1] not in domain_list()["domains"] + +def test_main_domain(): + current_main_domain = _get_maindomain() + assert domain_main_domain()["current_main_domain"] == current_main_domain + +def test_main_domain_change_unknown(): + with pytest.raises(YunohostValidationError) as e_info: + domain_main_domain(TEST_DOMAINS[2]) + +def test_change_main_domain(): + assert _get_maindomain() != TEST_DOMAINS[1] + domain_main_domain(TEST_DOMAINS[1]) + assert _get_maindomain() == TEST_DOMAINS[1] + +# Domain settings testing +def test_domain_setting_get_default_xmpp_main_domain(): + assert TEST_DOMAINS[0] in domain_list()["domains"] + assert domain_setting(TEST_DOMAINS[0], "xmpp") == True + +def test_domain_setting_get_default_xmpp(): + assert domain_setting(TEST_DOMAINS[1], "xmpp") == False + +def test_domain_setting_get_default_ttl(): + assert domain_setting(TEST_DOMAINS[1], "ttl") == 3600 + +def test_domain_setting_set_int(): + domain_setting(TEST_DOMAINS[1], "ttl", "10") + assert domain_setting(TEST_DOMAINS[1], "ttl") == 10 + +def test_domain_setting_set_bool_true(): + domain_setting(TEST_DOMAINS[1], "xmpp", "True") + assert domain_setting(TEST_DOMAINS[1], "xmpp") == True + domain_setting(TEST_DOMAINS[1], "xmpp", "true") + assert domain_setting(TEST_DOMAINS[1], "xmpp") == True + domain_setting(TEST_DOMAINS[1], "xmpp", "t") + assert domain_setting(TEST_DOMAINS[1], "xmpp") == True + domain_setting(TEST_DOMAINS[1], "xmpp", "1") + assert domain_setting(TEST_DOMAINS[1], "xmpp") == True + domain_setting(TEST_DOMAINS[1], "xmpp", "yes") + assert domain_setting(TEST_DOMAINS[1], "xmpp") == True + domain_setting(TEST_DOMAINS[1], "xmpp", "y") + assert domain_setting(TEST_DOMAINS[1], "xmpp") == True + +def test_domain_setting_set_bool_false(): + domain_setting(TEST_DOMAINS[1], "xmpp", "False") + assert domain_setting(TEST_DOMAINS[1], "xmpp") == False + domain_setting(TEST_DOMAINS[1], "xmpp", "false") + assert domain_setting(TEST_DOMAINS[1], "xmpp") == False + domain_setting(TEST_DOMAINS[1], "xmpp", "f") + assert domain_setting(TEST_DOMAINS[1], "xmpp") == False + domain_setting(TEST_DOMAINS[1], "xmpp", "0") + assert domain_setting(TEST_DOMAINS[1], "xmpp") == False + domain_setting(TEST_DOMAINS[1], "xmpp", "no") + assert domain_setting(TEST_DOMAINS[1], "xmpp") == False + domain_setting(TEST_DOMAINS[1], "xmpp", "n") + assert domain_setting(TEST_DOMAINS[1], "xmpp") == False + +def test_domain_settings_unknown(): + with pytest.raises(YunohostValidationError) as e_info: + domain_setting(TEST_DOMAINS[2], "xmpp", "False") + +# DNS utils testing +def test_get_dns_zone_from_domain_existing(): + assert get_dns_zone_from_domain("donate.yunohost.org") == "yunohost.org" + +def test_get_dns_zone_from_domain_not_existing(): + assert get_dns_zone_from_domain("non-existing-domain.yunohost.org") == "yunohost.org" + +# Domain registrar testing +def test_registrar_list_yaml_integrity(): + yaml.load(open(REGISTRAR_LIST_PATH, 'r')) + +def test_domain_registrar_catalog(): + domain_registrar_catalog() + +def test_domain_registrar_catalog_full(): + domain_registrar_catalog(None, True) + +def test_domain_registrar_catalog_registrar(): + domain_registrar_catalog("ovh") From 97bdedd72304aafd9e86f07f9476881a3ba06fa2 Mon Sep 17 00:00:00 2001 From: MercierCorentin Date: Sun, 1 Aug 2021 19:04:51 +0200 Subject: [PATCH 044/130] Migrate from public suffix to public suffix list --- data/hooks/diagnosis/12-dnsrecords.py | 4 ++-- src/yunohost/utils/dns.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 719ce4d6a..33789fd84 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -4,7 +4,7 @@ import os import re from datetime import datetime, timedelta -from publicsuffix import PublicSuffixList +from publicsuffixlist import PublicSuffixList from moulinette.utils.process import check_output @@ -37,7 +37,7 @@ class DNSRecordsDiagnoser(Diagnoser): # Check if a domain buy by the user will expire soon psl = PublicSuffixList() domains_from_registrar = [ - psl.get_public_suffix(domain) for domain in all_domains + psl.publicsuffix(domain) for domain in all_domains ] domains_from_registrar = [ domain for domain in domains_from_registrar if "." in domain diff --git a/src/yunohost/utils/dns.py b/src/yunohost/utils/dns.py index fd7cb1334..46b294602 100644 --- a/src/yunohost/utils/dns.py +++ b/src/yunohost/utils/dns.py @@ -18,7 +18,7 @@ along with this program; if not, see http://www.gnu.org/licenses """ -from publicsuffix import PublicSuffixList +from publicsuffixlist import PublicSuffixList from yunohost.utils.network import dig YNH_DYNDNS_DOMAINS = ["nohost.me", "noho.st", "ynh.fr"] @@ -31,7 +31,7 @@ def get_public_suffix(domain): # Load domain public suffixes psl = PublicSuffixList() - public_suffix = psl.get_public_suffix(domain) + public_suffix = psl.publicsuffix(domain) if public_suffix in YNH_DYNDNS_DOMAINS: domain_prefix = domain_name[0:-(1 + len(public_suffix))] public_suffix = domain_prefix.plit(".")[-1] + "." + public_suffix From 513a9f62c84d9d5c9413fde774bacb96803bbe86 Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Wed, 25 Aug 2021 12:13:45 +0200 Subject: [PATCH 045/130] [enh] SSH acronym MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Éric Gaspar <46165813+ericgaspar@users.noreply.github.com> --- locales/en.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locales/en.json b/locales/en.json index 712ecb844..6e6bb592b 100644 --- a/locales/en.json +++ b/locales/en.json @@ -527,7 +527,7 @@ "regenconf_dry_pending_applying": "Checking pending configuration which would have been applied for category '{category}'…", "regenconf_failed": "Could not regenerate the configuration for category(s): {categories}", "regenconf_pending_applying": "Applying pending configuration for category '{category}'...", - "regenconf_need_to_explicitly_specify_ssh": "The ssh configuration has been manually modified, but you need to explicitly specify category 'ssh' with --force to actually apply the changes.", + "regenconf_need_to_explicitly_specify_ssh": "The SSH configuration has been manually modified, but you need to explicitly specify category 'ssh' with --force to actually apply the changes.", "regex_incompatible_with_tile": "/!\\ Packagers! Permission '{permission}' has show_tile set to 'true' and you therefore cannot define a regex URL as the main URL", "regex_with_only_domain": "You can't use a regex for domain, only for path", "registrar_is_not_set": "The registrar for this domain has not been configured", From 808a69ca64611eb1b63cd5b2fae39febeadd33ec Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 28 Aug 2021 17:58:58 +0200 Subject: [PATCH 046/130] domain remove: Improve file/folder cleanup code --- src/yunohost/domain.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 533d0d7a5..990f5ce53 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -281,13 +281,14 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False): except Exception as e: raise YunohostError("domain_deletion_failed", domain=domain, error=e) - os.system("rm -rf /etc/yunohost/certs/%s" % domain) + stuff_to_delete = [ + f"/etc/yunohost/certs/{domain}", + f"/etc/yunohost/dyndns/K{domain}.+*", + f"{DOMAIN_SETTINGS_DIR}/{domain}.yml", + ] - # Delete dyndns keys for this domain (if any) - os.system("rm -rf /etc/yunohost/dyndns/K%s.+*" % domain) - - # Delete settings file for this domain - os.system(f"rm -rf {DOMAIN_SETTINGS_DIR}/{domain}.yml") + for stuff in stuff_to_delete: + os.system("rm -rf {stuff}") # Sometime we have weird issues with the regenconf where some files # appears as manually modified even though they weren't touched ... @@ -860,7 +861,7 @@ def domain_setting(domain, key, value=None, delete=False): if value < 0: raise YunohostError("pattern_positive_number", value_type=type(value)) - + # Set new value domain_settings[key] = value # Save settings @@ -932,18 +933,18 @@ def domain_registrar_info(domain): if not registrar_info: # TODO add locales raise YunohostError("registrar_is_not_set", dns_zone=dns_zone) - + logger.info("Registrar name: " + registrar_info['name']) for option_key, option_value in registrar_info['options'].items(): logger.info("Option " + option_key + ": " + option_value) def _print_registrar_info(registrar_name, full, options): logger.info("Registrar : " + registrar_name) - if full : + if full : logger.info("Options : ") for option in options: logger.info("\t- " + option) - + def domain_registrar_catalog(registrar_name, full): registrars = yaml.load(open(REGISTRAR_LIST_PATH, "r+")) From 5f338d3c547b4aaaadd0d2556cdd3d0d8d3438dd Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 28 Aug 2021 18:01:29 +0200 Subject: [PATCH 047/130] Semantics --- src/yunohost/domain.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 990f5ce53..24a19af55 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -496,7 +496,7 @@ def _build_dns_conf(domain): } """ - domains = _get_domain_settings(domain, True) + domains = _get_domain_settings(domain, include_subdomains=True) basic = [] mail = [] @@ -872,29 +872,29 @@ def _is_subdomain_of(subdomain, domain): return True if re.search("(^|\\.)" + domain + "$", subdomain) else False -def _get_domain_settings(domain, subdomains): +def _get_domain_settings(domain, include_subdomains=False): """ Get settings of a domain Keyword arguments: domain -- The domain name - subdomains -- Do we include the subdomains? Default is False + include_subdomains -- Do we include the subdomains? Default is False """ domains = _load_domain_settings() - if not domain in domains.keys(): + if domain not in domains.keys(): raise YunohostError("domain_name_unknown", domain=domain) - only_wanted_domains = dict() + out = dict() for entry in domains.keys(): - if subdomains: + if include_subdomains: if _is_subdomain_of(entry, domain): - only_wanted_domains[entry] = domains[entry] + out[entry] = domains[entry] else: if domain == entry: - only_wanted_domains[entry] = domains[entry] + out[entry] = domains[entry] - return only_wanted_domains + return out def _set_domain_settings(domain, domain_settings): From 13f2cabc46119a16a7d3bffa0e35ab68afaf277c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 28 Aug 2021 18:10:47 +0200 Subject: [PATCH 048/130] Use moulinette helpers to read/write yaml --- src/yunohost/domain.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 24a19af55..e21966697 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -34,9 +34,8 @@ from lexicon.config import ConfigResolver from moulinette import m18n, Moulinette from moulinette.core import MoulinetteError -from yunohost.utils.error import YunohostError, YunohostValidationError from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import write_to_file +from moulinette.utils.filesystem import write_to_file, read_yaml, write_to_yaml from yunohost.app import ( app_ssowatconf, @@ -46,6 +45,7 @@ from yunohost.app import ( _parse_args_in_yunohost_format, ) from yunohost.regenconf import regen_conf, _force_clear_hashes, _process_regen_conf +from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.network import get_public_ip from yunohost.utils.dns import get_dns_zone_from_domain from yunohost.log import is_unit_operation @@ -775,10 +775,7 @@ def _load_domain_settings(domains=[]): filepath = f"{DOMAIN_SETTINGS_DIR}/{domain}.yml" on_disk_settings = {} if os.path.exists(filepath) and os.path.isfile(filepath): - on_disk_settings = yaml.load(open(filepath, "r+")) - # If the file is empty or "corrupted" - if not type(on_disk_settings) is dict: - on_disk_settings = {} + on_disk_settings = read_yaml(filepath) or {} # Generate defaults is_maindomain = domain == maindomain dns_zone = get_dns_zone_from_domain(domain) @@ -805,10 +802,7 @@ def _load_registrar_setting(dns_zone): on_disk_settings = {} filepath = f"{REGISTRAR_SETTINGS_DIR}/{dns_zone}.yml" if os.path.exists(filepath) and os.path.isfile(filepath): - on_disk_settings = yaml.load(open(filepath, "r+")) - # If the file is empty or "corrupted" - if not type(on_disk_settings) is dict: - on_disk_settings = {} + on_disk_settings = read_yaml(filepath) or {} return on_disk_settings @@ -914,8 +908,7 @@ def _set_domain_settings(domain, domain_settings): os.mkdir(DOMAIN_SETTINGS_DIR) # Save the settings to the .yaml file filepath = f"{DOMAIN_SETTINGS_DIR}/{domain}.yml" - with open(filepath, "w") as file: - yaml.dump(domain_settings, file, default_flow_style=False) + write_to_yaml(filepath, domain_settings) def _load_zone_of_domain(domain): @@ -938,6 +931,7 @@ def domain_registrar_info(domain): for option_key, option_value in registrar_info['options'].items(): logger.info("Option " + option_key + ": " + option_value) + def _print_registrar_info(registrar_name, full, options): logger.info("Registrar : " + registrar_name) if full : @@ -945,8 +939,9 @@ def _print_registrar_info(registrar_name, full, options): for option in options: logger.info("\t- " + option) + def domain_registrar_catalog(registrar_name, full): - registrars = yaml.load(open(REGISTRAR_LIST_PATH, "r+")) + registrars = read_yaml(REGISTRAR_LIST_PATH) if registrar_name and registrar_name in registrars.keys() : _print_registrar_info(registrar_name, True, registrars[registrar_name]) @@ -990,8 +985,7 @@ def domain_registrar_set(domain, registrar, args): os.mkdir(REGISTRAR_SETTINGS_DIR) # Save the settings to the .yaml file filepath = f"{REGISTRAR_SETTINGS_DIR}/{dns_zone}.yml" - with open(filepath, "w") as file: - yaml.dump(domain_registrar, file, default_flow_style=False) + write_to_yaml(filepath, domain_registrar) def domain_push_config(domain): From 9f4ca2e8196871fbfc46d282939b69641042fd78 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 28 Aug 2021 18:11:04 +0200 Subject: [PATCH 049/130] Misc cleanup --- src/yunohost/domain.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index e21966697..b5a3273cf 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -750,25 +750,21 @@ def _load_domain_settings(domains=[]): """ # Retrieve actual domain list get_domain_list = domain_list() + all_known_domains = get_domain_list["domains"] + maindomain = get_domain_list["main"] if domains: - # check existence of requested domains - all_known_domains = get_domain_list["domains"] # filter inexisting domains - unknown_domains = filter(lambda domain : not domain in all_known_domains, domains) + unknown_domains = filter(lambda domain: domain not in all_known_domains, domains) # get first unknown domain unknown_domain = next(unknown_domains, None) - if unknown_domain != None: + if unknown_domain is None: raise YunohostValidationError("domain_name_unknown", domain=unknown_domain) else: - domains = get_domain_list["domains"] - + domains = all_known_domains # Create sanitized data - new_domains = dict() - - # Load main domain - maindomain = get_domain_list["main"] + out = dict() for domain in domains: # Retrieve entries in the YAML @@ -789,9 +785,9 @@ def _load_domain_settings(domains=[]): # Update each setting if not present default_settings.update(on_disk_settings) # Add the domain to the list - new_domains[domain] = default_settings + out[domain] = default_settings - return new_domains + return out def _load_registrar_setting(dns_zone): From 756e6041cb9e97de8dbb811eb7b6282477559994 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 28 Aug 2021 19:08:28 +0200 Subject: [PATCH 050/130] domains.py: Attempt to clarify build_dns_zone? --- src/yunohost/domain.py | 83 +++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 45 deletions(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index b5a3273cf..82f8861d4 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -453,7 +453,7 @@ def _get_maindomain(): return maindomain -def _build_dns_conf(domain): +def _build_dns_conf(base_domain): """ Internal function that will returns a data structure containing the needed information to generate/adapt the dns configuration @@ -496,43 +496,40 @@ def _build_dns_conf(domain): } """ - domains = _get_domain_settings(domain, include_subdomains=True) - basic = [] mail = [] xmpp = [] extra = [] ipv4 = get_public_ip() ipv6 = get_public_ip(6) - owned_dns_zone = ( - # TODO test this - "dns_zone" in domains[domain] and domains[domain]["dns_zone"] == domain - ) - root_prefix = domain.partition(".")[0] - child_domain_suffix = "" + domains_settings = _get_domain_settings(base_domain, include_subdomains=True) + base_dns_zone = domain_settings[base_domain].get("dns_zone") - for domain_name, domain in domains.items(): - ttl = domain["ttl"] + for domain, settings in domain_settings.items(): - if domain_name == domain: - name = "@" if owned_dns_zone else root_prefix - else: - name = domain_name - if not owned_dns_zone: - name += "." + root_prefix + # Domain # Base DNS zone # Basename # Suffix # + # ------------------ # ----------------- # --------- # -------- # + # domain.tld # domain.tld # @ # # + # sub.domain.tld # domain.tld # sub # .sub # + # foo.sub.domain.tld # domain.tld # foo.sub # .foo.sub # + # sub.domain.tld # sub.domain.tld # @ # # + # foo.sub.domain.tld # sub.domain.tld # foo # .foo # - if name != "@": - child_domain_suffix = "." + name + # FIXME: shouldn't the basename just be based on the dns_zone setting of this domain ? + basename = domain.replace(f"{base_dns_zone}", "").rstrip(".") or "@" + suffix = f".{basename}" if base_name != "@" else "" + + ttl = settings["ttl"] ########################### # Basic ipv4/ipv6 records # ########################### if ipv4: - basic.append([name, ttl, "A", ipv4]) + basic.append([basename, ttl, "A", ipv4]) if ipv6: - basic.append([name, ttl, "AAAA", ipv6]) + basic.append([basename, ttl, "AAAA", ipv6]) # TODO # elif include_empty_AAAA_if_no_ipv6: # basic.append(["@", ttl, "AAAA", None]) @@ -540,46 +537,42 @@ def _build_dns_conf(domain): ######### # Email # ######### - if domain["mail_in"]: - mail += [ - [name, ttl, "MX", "10 %s." % domain_name] - ] + if settings["mail_in"]: + mail.append([basename, ttl, "MX", f"10 {domain}."]) - if domain["mail_out"]: - mail += [ - [name, ttl, "TXT", '"v=spf1 a mx -all"'] - ] + if settings["mail_out"]: + mail.append([basename, ttl, "TXT", '"v=spf1 a mx -all"']) # DKIM/DMARC record - dkim_host, dkim_publickey = _get_DKIM(domain_name) + dkim_host, dkim_publickey = _get_DKIM(domain) if dkim_host: mail += [ - [dkim_host, ttl, "TXT", dkim_publickey], - [f"_dmarc{child_domain_suffix}", ttl, "TXT", '"v=DMARC1; p=none"'], + [f"{dkim_host}{suffix}", ttl, "TXT", dkim_publickey], + [f"_dmarc{suffix}", ttl, "TXT", '"v=DMARC1; p=none"'], ] ######## # XMPP # ######## - if domain["xmpp"]: + if settings["xmpp"]: xmpp += [ [ - f"_xmpp-client._tcp{child_domain_suffix}", + f"_xmpp-client._tcp{suffix}", ttl, "SRV", - f"0 5 5222 {domain_name}.", + f"0 5 5222 {domain}.", ], [ - f"_xmpp-server._tcp{child_domain_suffix}", + f"_xmpp-server._tcp{suffix}", ttl, "SRV", - f"0 5 5269 {domain_name}.", + f"0 5 5269 {domain}.", ], - ["muc" + child_domain_suffix, ttl, "CNAME", name], - ["pubsub" + child_domain_suffix, ttl, "CNAME", name], - ["vjud" + child_domain_suffix, ttl, "CNAME", name], - ["xmpp-upload" + child_domain_suffix, ttl, "CNAME", name], + [f"muc{suffix}", ttl, "CNAME", basename], + [f"pubsub{suffix}", ttl, "CNAME", basename], + [f"vjud{suffix}", ttl, "CNAME", basename], + [f"xmpp-upload{suffix}", ttl, "CNAME", basename], ] ######### @@ -587,15 +580,15 @@ def _build_dns_conf(domain): ######### if ipv4: - extra.append([f"*{child_domain_suffix}", ttl, "A", ipv4]) + extra.append([f"*{suffix}", ttl, "A", ipv4]) if ipv6: - extra.append([f"*{child_domain_suffix}", ttl, "AAAA", ipv6]) + extra.append([f"*{suffix}", ttl, "AAAA", ipv6]) # TODO # elif include_empty_AAAA_if_no_ipv6: # extra.append(["*", ttl, "AAAA", None]) - extra.append([name, ttl, "CAA", '128 issue "letsencrypt.org"']) + extra.append([basename, ttl, "CAA", '128 issue "letsencrypt.org"']) #################### # Standard records # @@ -626,7 +619,7 @@ def _build_dns_conf(domain): # Defined by custom hooks ships in apps for example ... - hook_results = hook_callback("custom_dns_rules", args=[domain]) + hook_results = hook_callback("custom_dns_rules", args=[base_domain]) for hook_name, results in hook_results.items(): # # There can be multiple results per hook name, so results look like From 2f3467dd17ade0575acc326fce886beae08891ad Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 28 Aug 2021 19:53:38 +0200 Subject: [PATCH 051/130] More cleanup / simplify / homogenize --- src/yunohost/domain.py | 194 +++++++++++++++-------------------------- 1 file changed, 72 insertions(+), 122 deletions(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 82f8861d4..582cb9bed 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -503,10 +503,13 @@ def _build_dns_conf(base_domain): ipv4 = get_public_ip() ipv6 = get_public_ip(6) - domains_settings = _get_domain_settings(base_domain, include_subdomains=True) - base_dns_zone = domain_settings[base_domain].get("dns_zone") + subdomains = _list_subdomains_of(base_domain) + domains_settings = {domain: _get_domain_settings(domain) + for domain in [base_domain] + subdomains} - for domain, settings in domain_settings.items(): + base_dns_zone = domains_settings[base_domain].get("dns_zone") + + for domain, settings in domains_settings.items(): # Domain # Base DNS zone # Basename # Suffix # # ------------------ # ----------------- # --------- # -------- # @@ -736,64 +739,39 @@ def _get_DKIM(domain): ) -def _load_domain_settings(domains=[]): +def _default_domain_settings(domain, is_main_domain): + return { + "xmpp": is_main_domain, + "mail_in": True, + "mail_out": True, + "dns_zone": get_dns_zone_from_domain(domain), + "ttl": 3600, + } + + +def _get_domain_settings(domain): """ Retrieve entries in /etc/yunohost/domains/[domain].yml - And fill the holes if any + And set default values if needed """ # Retrieve actual domain list - get_domain_list = domain_list() - all_known_domains = get_domain_list["domains"] - maindomain = get_domain_list["main"] + domain_list_ = domain_list() + known_domains = domain_list_["domains"] + maindomain = domain_list_["main"] - if domains: - # filter inexisting domains - unknown_domains = filter(lambda domain: domain not in all_known_domains, domains) - # get first unknown domain - unknown_domain = next(unknown_domains, None) - if unknown_domain is None: - raise YunohostValidationError("domain_name_unknown", domain=unknown_domain) - else: - domains = all_known_domains - - # Create sanitized data - out = dict() - - for domain in domains: - # Retrieve entries in the YAML - filepath = f"{DOMAIN_SETTINGS_DIR}/{domain}.yml" - on_disk_settings = {} - if os.path.exists(filepath) and os.path.isfile(filepath): - on_disk_settings = read_yaml(filepath) or {} - # Generate defaults - is_maindomain = domain == maindomain - dns_zone = get_dns_zone_from_domain(domain) - default_settings = { - "xmpp": is_maindomain, - "mail_in": True, - "mail_out": True, - "dns_zone": dns_zone, - "ttl": 3600, - } - # Update each setting if not present - default_settings.update(on_disk_settings) - # Add the domain to the list - out[domain] = default_settings - - return out - - -def _load_registrar_setting(dns_zone): - """ - Retrieve entries in registrars/[dns_zone].yml - """ + if domain not in known_domains: + raise YunohostValidationError("domain_name_unknown", domain=domain) + # Retrieve entries in the YAML + filepath = f"{DOMAIN_SETTINGS_DIR}/{domain}.yml" on_disk_settings = {} - filepath = f"{REGISTRAR_SETTINGS_DIR}/{dns_zone}.yml" if os.path.exists(filepath) and os.path.isfile(filepath): on_disk_settings = read_yaml(filepath) or {} - return on_disk_settings + # Inject defaults if needed (using the magic .update() ;)) + settings = _default_domain_settings(domain, domain == maindomain) + settings.update(on_disk_settings) + return settings def domain_setting(domain, key, value=None, delete=False): @@ -808,18 +786,11 @@ def domain_setting(domain, key, value=None, delete=False): """ - boolean_keys = ["mail_in", "mail_out", "xmpp"] - - domains = _load_domain_settings([ domain ]) - - if not domain in domains.keys(): - raise YunohostError("domain_name_unknown", domain=domain) - - domain_settings = domains[domain] + domain_settings = _get_domain_settings(domain) # GET if value is None and not delete: - if not key in domain_settings: + if key not in domain_settings: raise YunohostValidationError("domain_property_unknown", property=key) return domain_settings[key] @@ -832,7 +803,10 @@ def domain_setting(domain, key, value=None, delete=False): # SET else: - if key in boolean_keys: + # FIXME : in the future, implement proper setting types (+ defaults), + # maybe inspired from the global settings + + if key in ["mail_in", "mail_out", "xmpp"]: value = True if value.lower() in ['true', '1', 't', 'y', 'yes', "iloveynh"] else False if "ttl" == key: @@ -851,31 +825,17 @@ def domain_setting(domain, key, value=None, delete=False): _set_domain_settings(domain, domain_settings) -def _is_subdomain_of(subdomain, domain): - return True if re.search("(^|\\.)" + domain + "$", subdomain) else False +def _list_subdomains_of(parent_domain): + domain_list_ = domain_list()["domains"] -def _get_domain_settings(domain, include_subdomains=False): - """ - Get settings of a domain - - Keyword arguments: - domain -- The domain name - include_subdomains -- Do we include the subdomains? Default is False - - """ - domains = _load_domain_settings() - if domain not in domains.keys(): + if parent_domain not in domain_list_: raise YunohostError("domain_name_unknown", domain=domain) - out = dict() - for entry in domains.keys(): - if include_subdomains: - if _is_subdomain_of(entry, domain): - out[entry] = domains[entry] - else: - if domain == entry: - out[entry] = domains[entry] + out = [] + for domain in domain_list_: + if domain.endswith(f".{parent_domain}"): + out.append(domain) return out @@ -886,7 +846,7 @@ def _set_domain_settings(domain, domain_settings): Keyword arguments: domain -- The domain name - settings -- Dict with doamin settings + settings -- Dict with domain settings """ if domain not in domain_list()["domains"]: @@ -900,54 +860,49 @@ def _set_domain_settings(domain, domain_settings): write_to_yaml(filepath, domain_settings) -def _load_zone_of_domain(domain): - domains = _load_domain_settings([domain]) - if domain not in domains.keys(): - raise YunohostError("domain_name_unknown", domain=domain) +def _get_registrar_settings(dns_zone): + on_disk_settings = {} + filepath = f"{REGISTRAR_SETTINGS_DIR}/{dns_zone}.yml" + if os.path.exists(filepath) and os.path.isfile(filepath): + on_disk_settings = read_yaml(filepath) or {} - return domains[domain]["dns_zone"] + return on_disk_settings + + +def _set_registrar_settings(dns_zone): + if not os.path.exists(REGISTRAR_SETTINGS_DIR): + os.mkdir(REGISTRAR_SETTINGS_DIR) + filepath = f"{REGISTRAR_SETTINGS_DIR}/{dns_zone}.yml" + write_to_yaml(filepath, domain_registrar) def domain_registrar_info(domain): - dns_zone = _load_zone_of_domain(domain) - registrar_info = _load_registrar_setting(dns_zone) + dns_zone = _get_domain_settings(domain)["dns_zone"] + registrar_info = _get_registrar_settings(dns_zone) if not registrar_info: - # TODO add locales raise YunohostError("registrar_is_not_set", dns_zone=dns_zone) - logger.info("Registrar name: " + registrar_info['name']) - for option_key, option_value in registrar_info['options'].items(): - logger.info("Option " + option_key + ": " + option_value) - - -def _print_registrar_info(registrar_name, full, options): - logger.info("Registrar : " + registrar_name) - if full : - logger.info("Options : ") - for option in options: - logger.info("\t- " + option) + return registrar_info def domain_registrar_catalog(registrar_name, full): registrars = read_yaml(REGISTRAR_LIST_PATH) - if registrar_name and registrar_name in registrars.keys() : - _print_registrar_info(registrar_name, True, registrars[registrar_name]) + if registrar_name: + if registrar_name not in registrars.keys(): + raise YunohostError("domain_registrar_unknown", registrar=registrar_name) + else: + return registrars[registrar_name] else: - for registrar in registrars: - _print_registrar_info(registrar, full, registrars[registrar]) + return registrars def domain_registrar_set(domain, registrar, args): - dns_zone = _load_zone_of_domain(domain) - registrar_info = _load_registrar_setting(dns_zone) - - registrars = yaml.load(open(REGISTRAR_LIST_PATH, "r+")) - if not registrar in registrars.keys(): - # FIXME créer l'erreur - raise YunohostError("domain_registrar_unknown") + registrars = read_yaml(REGISTRAR_LIST_PATH) + if registrar not in registrars.keys(): + raise YunohostError("domain_registrar_unknown"i, registrar=registrar) parameters = registrars[registrar] ask_args = [] @@ -969,12 +924,8 @@ def domain_registrar_set(domain, registrar, args): for arg_name, arg_value_and_type in parsed_answer_dict.items(): domain_registrar["options"][arg_name] = arg_value_and_type[0] - # First create the REGISTRAR_SETTINGS_DIR if it doesn't exist - if not os.path.exists(REGISTRAR_SETTINGS_DIR): - os.mkdir(REGISTRAR_SETTINGS_DIR) - # Save the settings to the .yaml file - filepath = f"{REGISTRAR_SETTINGS_DIR}/{dns_zone}.yml" - write_to_yaml(filepath, domain_registrar) + dns_zone = _get_domain_settings(domain)["dns_zone"] + _set_registrar_settings(dns_zone, domain_registrar) def domain_push_config(domain): @@ -987,9 +938,8 @@ def domain_push_config(domain): dns_conf = _build_dns_conf(domain) - domain_settings = _load_domain_settings([ domain ]) - dns_zone = domain_settings[domain]["dns_zone"] - registrar_setting = _load_registrar_setting(dns_zone) + dns_zone = _get_domain_settings(domain)["dns_zone"] + registrar_setting = _get_registrar_settings(dns_zone) if not registrar_setting: # FIXME add locales From 22aa1f2538c1186eb466b761e12263d18b5b590c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 28 Aug 2021 20:13:58 +0200 Subject: [PATCH 052/130] Cleanup get_dns_zone_from_domain utils --- src/yunohost/utils/dns.py | 45 +++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/src/yunohost/utils/dns.py b/src/yunohost/utils/dns.py index 46b294602..1d67d73d0 100644 --- a/src/yunohost/utils/dns.py +++ b/src/yunohost/utils/dns.py @@ -26,15 +26,17 @@ YNH_DYNDNS_DOMAINS = ["nohost.me", "noho.st", "ynh.fr"] def get_public_suffix(domain): """get_public_suffix("www.example.com") -> "example.com" - Return the public suffix of a domain name based + Return the public suffix of a domain name based """ # Load domain public suffixes psl = PublicSuffixList() public_suffix = psl.publicsuffix(domain) + + # FIXME: wtf is this supposed to do ? :| if public_suffix in YNH_DYNDNS_DOMAINS: - domain_prefix = domain_name[0:-(1 + len(public_suffix))] - public_suffix = domain_prefix.plit(".")[-1] + "." + public_suffix + domain_prefix = domain[0:-(1 + len(public_suffix))] + public_suffix = domain_prefix.split(".")[-1] + "." + public_suffix return public_suffix @@ -45,19 +47,30 @@ def get_dns_zone_from_domain(domain): Keyword arguments: domain -- The domain name - + """ - separator = "." - domain_subs = domain.split(separator) - for i in range(0, len(domain_subs)): - current_domain = separator.join(domain_subs) - answer = dig(current_domain, rdtype="NS", full_answers=True, resolvers="force_external") - if answer[0] == "ok" : + + # For foo.bar.baz.gni we want to scan all the parent domains + # (including the domain itself) + # foo.bar.baz.gni + # bar.baz.gni + # baz.gni + # gni + parent_list = [domain.split(".", i)[-1] + for i, _ in enumerate(domain.split("."))] + + for parent in parent_list: + + # Check if there's a NS record for that domain + answer = dig(parent, rdtype="NS", full_answers=True, resolvers="force_external") + if answer[0] == "ok": # Domain is dns_zone - return current_domain - if separator.join(domain_subs[1:]) == get_public_suffix(current_domain): - # Couldn't check if domain is dns zone, + return parent + # Otherwise, check if the parent of this parent is in the public suffix list + if parent.split(".", 1)[-1] == get_public_suffix(parent): + # Couldn't check if domain is dns zone, # FIXME : why "couldn't" ...? # returning private suffix - return current_domain - domain_subs.pop(0) - return None \ No newline at end of file + return parent + + # FIXME: returning None will probably trigger bugs when this happens, code expects a domain string + return None From 8438efa6803ee120609adbdddf98b12f40b18f1e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 28 Aug 2021 20:20:22 +0200 Subject: [PATCH 053/130] Move dig() to utils.dns --- data/hooks/diagnosis/12-dnsrecords.py | 3 +- data/hooks/diagnosis/24-mail.py | 2 +- src/yunohost/dyndns.py | 3 +- src/yunohost/utils/dns.py | 74 ++++++++++++++++++++++++++- src/yunohost/utils/network.py | 70 ------------------------- 5 files changed, 77 insertions(+), 75 deletions(-) diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 247a3bf4d..4f1f89ef7 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -8,11 +8,10 @@ from publicsuffixlist import PublicSuffixList from moulinette.utils.process import check_output -from yunohost.utils.network import dig +from yunohost.utils.dns import dig, YNH_DYNDNS_DOMAINS from yunohost.diagnosis import Diagnoser from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain -YNH_DYNDNS_DOMAINS = ["nohost.me", "noho.st", "ynh.fr"] SPECIAL_USE_TLDS = ["local", "localhost", "onion", "test"] diff --git a/data/hooks/diagnosis/24-mail.py b/data/hooks/diagnosis/24-mail.py index 63f685a26..50b8dc12e 100644 --- a/data/hooks/diagnosis/24-mail.py +++ b/data/hooks/diagnosis/24-mail.py @@ -12,7 +12,7 @@ from moulinette.utils.filesystem import read_yaml from yunohost.diagnosis import Diagnoser from yunohost.domain import _get_maindomain, domain_list from yunohost.settings import settings_get -from yunohost.utils.network import dig +from yunohost.utils.dns import dig DEFAULT_DNS_BLACKLIST = "/usr/share/yunohost/other/dnsbl_list.yml" diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index c8249e439..071e93059 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -38,7 +38,8 @@ from moulinette.utils.network import download_json from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.domain import _get_maindomain, _build_dns_conf -from yunohost.utils.network import get_public_ip, dig +from yunohost.utils.network import get_public_ip +from yunohost.utils.dns import dig from yunohost.log import is_unit_operation from yunohost.regenconf import regen_conf diff --git a/src/yunohost/utils/dns.py b/src/yunohost/utils/dns.py index 1d67d73d0..ef89c35c5 100644 --- a/src/yunohost/utils/dns.py +++ b/src/yunohost/utils/dns.py @@ -18,11 +18,82 @@ along with this program; if not, see http://www.gnu.org/licenses """ +import dns.resolver from publicsuffixlist import PublicSuffixList -from yunohost.utils.network import dig +from moulinette.utils.filesystem import read_file YNH_DYNDNS_DOMAINS = ["nohost.me", "noho.st", "ynh.fr"] +# Lazy dev caching to avoid re-reading the file multiple time when calling +# dig() often during same yunohost operation +external_resolvers_ = [] + + +def external_resolvers(): + + global external_resolvers_ + + if not external_resolvers_: + resolv_dnsmasq_conf = read_file("/etc/resolv.dnsmasq.conf").split("\n") + external_resolvers_ = [ + r.split(" ")[1] for r in resolv_dnsmasq_conf if r.startswith("nameserver") + ] + # We keep only ipv4 resolvers, otherwise on IPv4-only instances, IPv6 + # will be tried anyway resulting in super-slow dig requests that'll wait + # until timeout... + external_resolvers_ = [r for r in external_resolvers_ if ":" not in r] + + return external_resolvers_ + + +def dig( + qname, rdtype="A", timeout=5, resolvers="local", edns_size=1500, full_answers=False +): + """ + Do a quick DNS request and avoid the "search" trap inside /etc/resolv.conf + """ + + # It's very important to do the request with a qname ended by . + # If we don't and the domain fail, dns resolver try a second request + # by concatenate the qname with the end of the "hostname" + if not qname.endswith("."): + qname += "." + + if resolvers == "local": + resolvers = ["127.0.0.1"] + elif resolvers == "force_external": + resolvers = external_resolvers() + else: + assert isinstance(resolvers, list) + + resolver = dns.resolver.Resolver(configure=False) + resolver.use_edns(0, 0, edns_size) + resolver.nameservers = resolvers + # resolver.timeout is used to trigger the next DNS query on resolvers list. + # In python-dns 1.16, this value is set to 2.0. However, this means that if + # the 3 first dns resolvers in list are down, we wait 6 seconds before to + # run the DNS query to a DNS resolvers up... + # In diagnosis dnsrecords, with 10 domains this means at least 12min, too long. + resolver.timeout = 1.0 + # resolver.lifetime is the timeout for resolver.query() + # By default set it to 5 seconds to allow 4 resolvers to be unreachable. + resolver.lifetime = timeout + try: + answers = resolver.query(qname, rdtype) + except ( + dns.resolver.NXDOMAIN, + dns.resolver.NoNameservers, + dns.resolver.NoAnswer, + dns.exception.Timeout, + ) as e: + return ("nok", (e.__class__.__name__, e)) + + if not full_answers: + answers = [answer.to_text() for answer in answers] + + return ("ok", answers) + + def get_public_suffix(domain): """get_public_suffix("www.example.com") -> "example.com" @@ -40,6 +111,7 @@ def get_public_suffix(domain): return public_suffix + def get_dns_zone_from_domain(domain): # TODO Check if this function is YNH_DYNDNS_DOMAINS compatible """ diff --git a/src/yunohost/utils/network.py b/src/yunohost/utils/network.py index 88ea5e5f6..4474af14f 100644 --- a/src/yunohost/utils/network.py +++ b/src/yunohost/utils/network.py @@ -22,7 +22,6 @@ import os import re import logging import time -import dns.resolver from moulinette.utils.filesystem import read_file, write_to_file from moulinette.utils.network import download_text @@ -124,75 +123,6 @@ def get_gateway(): return addr.popitem()[1] if len(addr) == 1 else None -# Lazy dev caching to avoid re-reading the file multiple time when calling -# dig() often during same yunohost operation -external_resolvers_ = [] - - -def external_resolvers(): - - global external_resolvers_ - - if not external_resolvers_: - resolv_dnsmasq_conf = read_file("/etc/resolv.dnsmasq.conf").split("\n") - external_resolvers_ = [ - r.split(" ")[1] for r in resolv_dnsmasq_conf if r.startswith("nameserver") - ] - # We keep only ipv4 resolvers, otherwise on IPv4-only instances, IPv6 - # will be tried anyway resulting in super-slow dig requests that'll wait - # until timeout... - external_resolvers_ = [r for r in external_resolvers_ if ":" not in r] - - return external_resolvers_ - - -def dig( - qname, rdtype="A", timeout=5, resolvers="local", edns_size=1500, full_answers=False -): - """ - Do a quick DNS request and avoid the "search" trap inside /etc/resolv.conf - """ - - # It's very important to do the request with a qname ended by . - # If we don't and the domain fail, dns resolver try a second request - # by concatenate the qname with the end of the "hostname" - if not qname.endswith("."): - qname += "." - - if resolvers == "local": - resolvers = ["127.0.0.1"] - elif resolvers == "force_external": - resolvers = external_resolvers() - else: - assert isinstance(resolvers, list) - - resolver = dns.resolver.Resolver(configure=False) - resolver.use_edns(0, 0, edns_size) - resolver.nameservers = resolvers - # resolver.timeout is used to trigger the next DNS query on resolvers list. - # In python-dns 1.16, this value is set to 2.0. However, this means that if - # the 3 first dns resolvers in list are down, we wait 6 seconds before to - # run the DNS query to a DNS resolvers up... - # In diagnosis dnsrecords, with 10 domains this means at least 12min, too long. - resolver.timeout = 1.0 - # resolver.lifetime is the timeout for resolver.query() - # By default set it to 5 seconds to allow 4 resolvers to be unreachable. - resolver.lifetime = timeout - try: - answers = resolver.query(qname, rdtype) - except ( - dns.resolver.NXDOMAIN, - dns.resolver.NoNameservers, - dns.resolver.NoAnswer, - dns.exception.Timeout, - ) as e: - return ("nok", (e.__class__.__name__, e)) - - if not full_answers: - answers = [answer.to_text() for answer in answers] - - return ("ok", answers) - def _extract_inet(string, skip_netmask=False, skip_loopback=True): """ Extract IP addresses (v4 and/or v6) from a string limited to one From 0ec1516f6ed125c7c5fc32cbb21c7c8f825e5869 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 28 Aug 2021 20:39:01 +0200 Subject: [PATCH 054/130] domains.py: Add cache for domain_list --- src/yunohost/domain.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 582cb9bed..e3e549e20 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -58,6 +58,10 @@ REGISTRAR_SETTINGS_DIR = "/etc/yunohost/registrars" REGISTRAR_LIST_PATH = "/usr/share/yunohost/other/registrar_list.yml" +# Lazy dev caching to avoid re-query ldap every time we need the domain list +domain_list_cache = {} + + def domain_list(exclude_subdomains=False): """ List domains @@ -66,6 +70,9 @@ def domain_list(exclude_subdomains=False): exclude_subdomains -- Filter out domains that are subdomains of other declared domains """ + if not exclude_subdomains and domain_list_cache: + return domain_list_cache + from yunohost.utils.ldap import _get_ldap_interface ldap = _get_ldap_interface() @@ -95,7 +102,8 @@ def domain_list(exclude_subdomains=False): result_list = sorted(result_list, key=cmp_domain) - return {"domains": result_list, "main": _get_maindomain()} + domain_list_cache = {"domains": result_list, "main": _get_maindomain()} + return domain_list_cache @is_unit_operation() @@ -164,6 +172,8 @@ def domain_add(operation_logger, domain, dyndns=False): ldap.add("virtualdomain=%s,ou=domains" % domain, attr_dict) except Exception as e: raise YunohostError("domain_creation_failed", domain=domain, error=e) + finally: + domain_list_cache = {} # Don't regen these conf if we're still in postinstall if os.path.exists("/etc/yunohost/installed"): @@ -280,6 +290,8 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False): ldap.remove("virtualdomain=" + domain + ",ou=domains") except Exception as e: raise YunohostError("domain_deletion_failed", domain=domain, error=e) + finally: + domain_list_cache = {} stuff_to_delete = [ f"/etc/yunohost/certs/{domain}", @@ -393,7 +405,7 @@ def domain_main_domain(operation_logger, new_main_domain=None): # Apply changes to ssl certs try: write_to_file("/etc/yunohost/current_host", new_main_domain) - + domain_list_cache = {} _set_hostname(new_main_domain) except Exception as e: logger.warning("%s" % e, exc_info=1) @@ -755,9 +767,8 @@ def _get_domain_settings(domain): And set default values if needed """ # Retrieve actual domain list - domain_list_ = domain_list() - known_domains = domain_list_["domains"] - maindomain = domain_list_["main"] + known_domains = domain_list()["domains"] + maindomain = domain_list()["main"] if domain not in known_domains: raise YunohostValidationError("domain_name_unknown", domain=domain) From 1eb059931d8be499e4102d0c9692a8cebfbd4041 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 28 Aug 2021 21:02:13 +0200 Subject: [PATCH 055/130] Savagely split the dns/registrar stuff in a new dns.py --- data/hooks/diagnosis/12-dnsrecords.py | 3 +- src/yunohost/dns.py | 563 ++++++++++++++++++++++++ src/yunohost/domain.py | 587 ++------------------------ src/yunohost/dyndns.py | 5 +- 4 files changed, 614 insertions(+), 544 deletions(-) create mode 100644 src/yunohost/dns.py diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 4f1f89ef7..727fd2e13 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -10,7 +10,8 @@ from moulinette.utils.process import check_output from yunohost.utils.dns import dig, YNH_DYNDNS_DOMAINS from yunohost.diagnosis import Diagnoser -from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain +from yunohost.domain import domain_list, _get_maindomain +from yunohost.dns import _build_dns_conf SPECIAL_USE_TLDS = ["local", "localhost", "onion", "test"] diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py new file mode 100644 index 000000000..590d50876 --- /dev/null +++ b/src/yunohost/dns.py @@ -0,0 +1,563 @@ +# -*- coding: utf-8 -*- + +""" License + + Copyright (C) 2013 YunoHost + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program; if not, see http://www.gnu.org/licenses + +""" + +""" yunohost_domain.py + + Manage domains +""" +import os +import re + +from lexicon.client import Client +from lexicon.config import ConfigResolver + +from moulinette import m18n, Moulinette +from moulinette.utils.log import getActionLogger +from moulinette.utils.filesystem import read_yaml, write_to_yaml + +from yunohost.domain import domain_list, _get_domain_settings +from yunohost.app import _parse_args_in_yunohost_format +from yunohost.utils.error import YunohostValidationError +from yunohost.utils.network import get_public_ip +from yunohost.log import is_unit_operation +from yunohost.hook import hook_callback + +logger = getActionLogger("yunohost.domain") + +REGISTRAR_SETTINGS_DIR = "/etc/yunohost/registrars" +REGISTRAR_LIST_PATH = "/usr/share/yunohost/other/registrar_list.yml" + + +def domain_dns_conf(domain): + """ + Generate DNS configuration for a domain + + Keyword argument: + domain -- Domain name + + """ + + if domain not in domain_list()["domains"]: + raise YunohostValidationError("domain_name_unknown", domain=domain) + + dns_conf = _build_dns_conf(domain) + + result = "" + + result += "; Basic ipv4/ipv6 records" + for record in dns_conf["basic"]: + result += "\n{name} {ttl} IN {type} {value}".format(**record) + + result += "\n\n" + result += "; XMPP" + for record in dns_conf["xmpp"]: + result += "\n{name} {ttl} IN {type} {value}".format(**record) + + result += "\n\n" + result += "; Mail" + for record in dns_conf["mail"]: + result += "\n{name} {ttl} IN {type} {value}".format(**record) + result += "\n\n" + + result += "; Extra" + for record in dns_conf["extra"]: + result += "\n{name} {ttl} IN {type} {value}".format(**record) + + for name, record_list in dns_conf.items(): + if name not in ("basic", "xmpp", "mail", "extra") and record_list: + result += "\n\n" + result += "; " + name + for record in record_list: + result += "\n{name} {ttl} IN {type} {value}".format(**record) + + if Moulinette.interface.type == "cli": + # FIXME Update this to point to our "dns push" doc + logger.info(m18n.n("domain_dns_conf_is_just_a_recommendation")) + + return result + + +def _build_dns_conf(base_domain): + """ + Internal function that will returns a data structure containing the needed + information to generate/adapt the dns configuration + + Arguments: + domains -- List of a domain and its subdomains + + The returned datastructure will have the following form: + { + "basic": [ + # if ipv4 available + {"type": "A", "name": "@", "value": "123.123.123.123", "ttl": 3600}, + # if ipv6 available + {"type": "AAAA", "name": "@", "value": "valid-ipv6", "ttl": 3600}, + ], + "xmpp": [ + {"type": "SRV", "name": "_xmpp-client._tcp", "value": "0 5 5222 domain.tld.", "ttl": 3600}, + {"type": "SRV", "name": "_xmpp-server._tcp", "value": "0 5 5269 domain.tld.", "ttl": 3600}, + {"type": "CNAME", "name": "muc", "value": "@", "ttl": 3600}, + {"type": "CNAME", "name": "pubsub", "value": "@", "ttl": 3600}, + {"type": "CNAME", "name": "vjud", "value": "@", "ttl": 3600} + {"type": "CNAME", "name": "xmpp-upload", "value": "@", "ttl": 3600} + ], + "mail": [ + {"type": "MX", "name": "@", "value": "10 domain.tld.", "ttl": 3600}, + {"type": "TXT", "name": "@", "value": "\"v=spf1 a mx ip4:123.123.123.123 ipv6:valid-ipv6 -all\"", "ttl": 3600 }, + {"type": "TXT", "name": "mail._domainkey", "value": "\"v=DKIM1; k=rsa; p=some-super-long-key\"", "ttl": 3600}, + {"type": "TXT", "name": "_dmarc", "value": "\"v=DMARC1; p=none\"", "ttl": 3600} + ], + "extra": [ + # if ipv4 available + {"type": "A", "name": "*", "value": "123.123.123.123", "ttl": 3600}, + # if ipv6 available + {"type": "AAAA", "name": "*", "value": "valid-ipv6", "ttl": 3600}, + {"type": "CAA", "name": "@", "value": "128 issue \"letsencrypt.org\"", "ttl": 3600}, + ], + "example_of_a_custom_rule": [ + {"type": "SRV", "name": "_matrix", "value": "domain.tld.", "ttl": 3600} + ], + } + """ + + basic = [] + mail = [] + xmpp = [] + extra = [] + ipv4 = get_public_ip() + ipv6 = get_public_ip(6) + + subdomains = _list_subdomains_of(base_domain) + domains_settings = {domain: _get_domain_settings(domain) + for domain in [base_domain] + subdomains} + + base_dns_zone = domains_settings[base_domain].get("dns_zone") + + for domain, settings in domains_settings.items(): + + # Domain # Base DNS zone # Basename # Suffix # + # ------------------ # ----------------- # --------- # -------- # + # domain.tld # domain.tld # @ # # + # sub.domain.tld # domain.tld # sub # .sub # + # foo.sub.domain.tld # domain.tld # foo.sub # .foo.sub # + # sub.domain.tld # sub.domain.tld # @ # # + # foo.sub.domain.tld # sub.domain.tld # foo # .foo # + + # FIXME: shouldn't the basename just be based on the dns_zone setting of this domain ? + basename = domain.replace(f"{base_dns_zone}", "").rstrip(".") or "@" + suffix = f".{basename}" if base_name != "@" else "" + + ttl = settings["ttl"] + + ########################### + # Basic ipv4/ipv6 records # + ########################### + if ipv4: + basic.append([basename, ttl, "A", ipv4]) + + if ipv6: + basic.append([basename, ttl, "AAAA", ipv6]) + # TODO + # elif include_empty_AAAA_if_no_ipv6: + # basic.append(["@", ttl, "AAAA", None]) + + ######### + # Email # + ######### + if settings["mail_in"]: + mail.append([basename, ttl, "MX", f"10 {domain}."]) + + if settings["mail_out"]: + mail.append([basename, ttl, "TXT", '"v=spf1 a mx -all"']) + + # DKIM/DMARC record + dkim_host, dkim_publickey = _get_DKIM(domain) + + if dkim_host: + mail += [ + [f"{dkim_host}{suffix}", ttl, "TXT", dkim_publickey], + [f"_dmarc{suffix}", ttl, "TXT", '"v=DMARC1; p=none"'], + ] + + ######## + # XMPP # + ######## + if settings["xmpp"]: + xmpp += [ + [ + f"_xmpp-client._tcp{suffix}", + ttl, + "SRV", + f"0 5 5222 {domain}.", + ], + [ + f"_xmpp-server._tcp{suffix}", + ttl, + "SRV", + f"0 5 5269 {domain}.", + ], + [f"muc{suffix}", ttl, "CNAME", basename], + [f"pubsub{suffix}", ttl, "CNAME", basename], + [f"vjud{suffix}", ttl, "CNAME", basename], + [f"xmpp-upload{suffix}", ttl, "CNAME", basename], + ] + + ######### + # Extra # + ######### + + if ipv4: + extra.append([f"*{suffix}", ttl, "A", ipv4]) + + if ipv6: + extra.append([f"*{suffix}", ttl, "AAAA", ipv6]) + # TODO + # elif include_empty_AAAA_if_no_ipv6: + # extra.append(["*", ttl, "AAAA", None]) + + extra.append([basename, ttl, "CAA", '128 issue "letsencrypt.org"']) + + #################### + # Standard records # + #################### + + records = { + "basic": [ + {"name": name, "ttl": ttl_, "type": type_, "value": value} + for name, ttl_, type_, value in basic + ], + "xmpp": [ + {"name": name, "ttl": ttl_, "type": type_, "value": value} + for name, ttl_, type_, value in xmpp + ], + "mail": [ + {"name": name, "ttl": ttl_, "type": type_, "value": value} + for name, ttl_, type_, value in mail + ], + "extra": [ + {"name": name, "ttl": ttl_, "type": type_, "value": value} + for name, ttl_, type_, value in extra + ], + } + + ################## + # Custom records # + ################## + + # Defined by custom hooks ships in apps for example ... + + hook_results = hook_callback("custom_dns_rules", args=[base_domain]) + for hook_name, results in hook_results.items(): + # + # There can be multiple results per hook name, so results look like + # {'/some/path/to/hook1': + # { 'state': 'succeed', + # 'stdreturn': [{'type': 'SRV', + # 'name': 'stuff.foo.bar.', + # 'value': 'yoloswag', + # 'ttl': 3600}] + # }, + # '/some/path/to/hook2': + # { ... }, + # [...] + # + # Loop over the sub-results + custom_records = [ + v["stdreturn"] for v in results.values() if v and v["stdreturn"] + ] + + records[hook_name] = [] + for record_list in custom_records: + # Check that record_list is indeed a list of dict + # with the required keys + if ( + not isinstance(record_list, list) + or any(not isinstance(record, dict) for record in record_list) + or any( + key not in record + for record in record_list + for key in ["name", "ttl", "type", "value"] + ) + ): + # Display an error, mainly for app packagers trying to implement a hook + logger.warning( + "Ignored custom record from hook '%s' because the data is not a *list* of dict with keys name, ttl, type and value. Raw data : %s" + % (hook_name, record_list) + ) + continue + + records[hook_name].extend(record_list) + + return records + + +def _get_DKIM(domain): + DKIM_file = "/etc/dkim/{domain}.mail.txt".format(domain=domain) + + if not os.path.isfile(DKIM_file): + return (None, None) + + with open(DKIM_file) as f: + dkim_content = f.read() + + # Gotta manage two formats : + # + # Legacy + # ----- + # + # mail._domainkey IN TXT ( "v=DKIM1; k=rsa; " + # "p=" ) + # + # New + # ------ + # + # mail._domainkey IN TXT ( "v=DKIM1; h=sha256; k=rsa; " + # "p=" ) + + is_legacy_format = " h=sha256; " not in dkim_content + + # Legacy DKIM format + if is_legacy_format: + dkim = re.match( + ( + r"^(?P[a-z_\-\.]+)[\s]+([0-9]+[\s]+)?IN[\s]+TXT[\s]+" + r'[^"]*"v=(?P[^";]+);' + r'[\s"]*k=(?P[^";]+);' + r'[\s"]*p=(?P

[^";]+)' + ), + dkim_content, + re.M | re.S, + ) + else: + dkim = re.match( + ( + r"^(?P[a-z_\-\.]+)[\s]+([0-9]+[\s]+)?IN[\s]+TXT[\s]+" + r'[^"]*"v=(?P[^";]+);' + r'[\s"]*h=(?P[^";]+);' + r'[\s"]*k=(?P[^";]+);' + r'[\s"]*p=(?P

[^";]+)' + ), + dkim_content, + re.M | re.S, + ) + + if not dkim: + return (None, None) + + if is_legacy_format: + return ( + dkim.group("host"), + '"v={v}; k={k}; p={p}"'.format( + v=dkim.group("v"), k=dkim.group("k"), p=dkim.group("p") + ), + ) + else: + return ( + dkim.group("host"), + '"v={v}; h={h}; k={k}; p={p}"'.format( + v=dkim.group("v"), + h=dkim.group("h"), + k=dkim.group("k"), + p=dkim.group("p"), + ), + ) + + +def _get_registrar_settings(dns_zone): + on_disk_settings = {} + filepath = f"{REGISTRAR_SETTINGS_DIR}/{dns_zone}.yml" + if os.path.exists(filepath) and os.path.isfile(filepath): + on_disk_settings = read_yaml(filepath) or {} + + return on_disk_settings + + +def _set_registrar_settings(dns_zone, domain_registrar): + if not os.path.exists(REGISTRAR_SETTINGS_DIR): + os.mkdir(REGISTRAR_SETTINGS_DIR) + filepath = f"{REGISTRAR_SETTINGS_DIR}/{dns_zone}.yml" + write_to_yaml(filepath, domain_registrar) + + +def domain_registrar_info(domain): + + dns_zone = _get_domain_settings(domain)["dns_zone"] + registrar_info = _get_registrar_settings(dns_zone) + if not registrar_info: + raise YunohostValidationError("registrar_is_not_set", dns_zone=dns_zone) + + return registrar_info + + +def domain_registrar_catalog(registrar_name, full): + registrars = read_yaml(REGISTRAR_LIST_PATH) + + if registrar_name: + if registrar_name not in registrars.keys(): + raise YunohostValidationError("domain_registrar_unknown", registrar=registrar_name) + else: + return registrars[registrar_name] + else: + return registrars + + +def domain_registrar_set(domain, registrar, args): + + registrars = read_yaml(REGISTRAR_LIST_PATH) + if registrar not in registrars.keys(): + raise YunohostValidationError("domain_registrar_unknown"i, registrar=registrar) + + parameters = registrars[registrar] + ask_args = [] + for parameter in parameters: + ask_args.append( + { + "name": parameter, + "type": "string", + "example": "", + "default": "", + } + ) + args_dict = ( + {} if not args else dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) + ) + parsed_answer_dict = _parse_args_in_yunohost_format(args_dict, ask_args) + + domain_registrar = {"name": registrar, "options": {}} + for arg_name, arg_value_and_type in parsed_answer_dict.items(): + domain_registrar["options"][arg_name] = arg_value_and_type[0] + + dns_zone = _get_domain_settings(domain)["dns_zone"] + _set_registrar_settings(dns_zone, domain_registrar) + + +def domain_push_config(domain): + """ + Send DNS records to the previously-configured registrar of the domain. + """ + # Generate the records + if domain not in domain_list()["domains"]: + raise YunohostValidationError("domain_name_unknown", domain=domain) + + dns_conf = _build_dns_conf(domain) + + dns_zone = _get_domain_settings(domain)["dns_zone"] + registrar_setting = _get_registrar_settings(dns_zone) + + if not registrar_setting: + # FIXME add locales + raise YunohostValidationError("registrar_is_not_set", domain=domain) + + # Flatten the DNS conf + flatten_dns_conf = [] + for key in dns_conf: + list_of_records = dns_conf[key] + for record in list_of_records: + # FIXME Lexicon does not support CAA records + # See https://github.com/AnalogJ/lexicon/issues/282 and https://github.com/AnalogJ/lexicon/pull/371 + # They say it's trivial to implement it! + # And yet, it is still not done/merged + if record["type"] != "CAA": + # Add .domain.tdl to the name entry + record["name"] = "{}.{}".format(record["name"], domain) + flatten_dns_conf.append(record) + + # Construct the base data structure to use lexicon's API. + base_config = { + "provider_name": registrar_setting["name"], + "domain": domain, # domain name + } + base_config[registrar_setting["name"]] = registrar_setting["options"] + + # Get types present in the generated records + types = set() + + for record in flatten_dns_conf: + types.add(record["type"]) + + # Fetch all types present in the generated records + distant_records = {} + + for key in types: + record_config = { + "action": "list", + "type": key, + } + final_lexicon = ( + ConfigResolver() + .with_dict(dict_object=base_config) + .with_dict(dict_object=record_config) + ) + # print('final_lexicon:', final_lexicon); + client = Client(final_lexicon) + distant_records[key] = client.execute() + + for key in types: + for distant_record in distant_records[key]: + logger.debug(f"distant_record: {distant_record}") + for local_record in flatten_dns_conf: + print("local_record:", local_record) + + # Push the records + for record in flatten_dns_conf: + # For each record, first check if one record exists for the same (type, name) couple + it_exists = False + # TODO do not push if local and distant records are exactly the same ? + # is_the_same_record = False + + for distant_record in distant_records[record["type"]]: + if ( + distant_record["type"] == record["type"] + and distant_record["name"] == record["name"] + ): + it_exists = True + # see previous TODO + # if distant_record["ttl"] = ... and distant_record["name"] ... + # is_the_same_record = True + + # Finally, push the new record or update the existing one + record_config = { + "action": "update" + if it_exists + else "create", # create, list, update, delete + "type": record[ + "type" + ], # specify a type for record filtering, case sensitive in some cases. + "name": record["name"], + "content": record["value"], + # FIXME Removed TTL, because it doesn't work with Gandi. + # See https://github.com/AnalogJ/lexicon/issues/726 (similar issue) + # But I think there is another issue with Gandi. Or I'm misusing the API... + # "ttl": record["ttl"], + } + final_lexicon = ( + ConfigResolver() + .with_dict(dict_object=base_config) + .with_dict(dict_object=record_config) + ) + client = Client(final_lexicon) + print("pushed_record:", record_config, "→", end=" ") + results = client.execute() + print("results:", results) + # print("Failed" if results == False else "Ok") + + +# def domain_config_fetch(domain, key, value): diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index e3e549e20..c18d5f665 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -24,13 +24,6 @@ Manage domains """ import os -import re -import sys -import yaml -import functools - -from lexicon.client import Client -from lexicon.config import ConfigResolver from moulinette import m18n, Moulinette from moulinette.core import MoulinetteError @@ -42,20 +35,15 @@ from yunohost.app import ( _installed_apps, _get_app_settings, _get_conflicting_apps, - _parse_args_in_yunohost_format, ) from yunohost.regenconf import regen_conf, _force_clear_hashes, _process_regen_conf from yunohost.utils.error import YunohostError, YunohostValidationError -from yunohost.utils.network import get_public_ip -from yunohost.utils.dns import get_dns_zone_from_domain from yunohost.log import is_unit_operation from yunohost.hook import hook_callback logger = getActionLogger("yunohost.domain") DOMAIN_SETTINGS_DIR = "/etc/yunohost/domains" -REGISTRAR_SETTINGS_DIR = "/etc/yunohost/registrars" -REGISTRAR_LIST_PATH = "/usr/share/yunohost/other/registrar_list.yml" # Lazy dev caching to avoid re-query ldap every time we need the domain list @@ -331,55 +319,6 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False): logger.success(m18n.n("domain_deleted")) -def domain_dns_conf(domain): - """ - Generate DNS configuration for a domain - - Keyword argument: - domain -- Domain name - - """ - - if domain not in domain_list()["domains"]: - raise YunohostValidationError("domain_name_unknown", domain=domain) - - dns_conf = _build_dns_conf(domain) - - result = "" - - result += "; Basic ipv4/ipv6 records" - for record in dns_conf["basic"]: - result += "\n{name} {ttl} IN {type} {value}".format(**record) - - result += "\n\n" - result += "; XMPP" - for record in dns_conf["xmpp"]: - result += "\n{name} {ttl} IN {type} {value}".format(**record) - - result += "\n\n" - result += "; Mail" - for record in dns_conf["mail"]: - result += "\n{name} {ttl} IN {type} {value}".format(**record) - result += "\n\n" - - result += "; Extra" - for record in dns_conf["extra"]: - result += "\n{name} {ttl} IN {type} {value}".format(**record) - - for name, record_list in dns_conf.items(): - if name not in ("basic", "xmpp", "mail", "extra") and record_list: - result += "\n\n" - result += "; " + name - for record in record_list: - result += "\n{name} {ttl} IN {type} {value}".format(**record) - - if Moulinette.interface.type == "cli": - # FIXME Update this to point to our "dns push" doc - logger.info(m18n.n("domain_dns_conf_is_just_a_recommendation")) - - return result - - @is_unit_operation() def domain_main_domain(operation_logger, new_main_domain=None): """ @@ -421,32 +360,6 @@ def domain_main_domain(operation_logger, new_main_domain=None): logger.success(m18n.n("main_domain_changed")) -def domain_cert_status(domain_list, full=False): - import yunohost.certificate - - return yunohost.certificate.certificate_status(domain_list, full) - - -def domain_cert_install( - domain_list, force=False, no_checks=False, self_signed=False, staging=False -): - import yunohost.certificate - - return yunohost.certificate.certificate_install( - domain_list, force, no_checks, self_signed, staging - ) - - -def domain_cert_renew( - domain_list, force=False, no_checks=False, email=False, staging=False -): - import yunohost.certificate - - return yunohost.certificate.certificate_renew( - domain_list, force, no_checks, email, staging - ) - - def domain_url_available(domain, path): """ Check availability of a web path @@ -465,293 +378,8 @@ def _get_maindomain(): return maindomain -def _build_dns_conf(base_domain): - """ - Internal function that will returns a data structure containing the needed - information to generate/adapt the dns configuration - - Arguments: - domains -- List of a domain and its subdomains - - The returned datastructure will have the following form: - { - "basic": [ - # if ipv4 available - {"type": "A", "name": "@", "value": "123.123.123.123", "ttl": 3600}, - # if ipv6 available - {"type": "AAAA", "name": "@", "value": "valid-ipv6", "ttl": 3600}, - ], - "xmpp": [ - {"type": "SRV", "name": "_xmpp-client._tcp", "value": "0 5 5222 domain.tld.", "ttl": 3600}, - {"type": "SRV", "name": "_xmpp-server._tcp", "value": "0 5 5269 domain.tld.", "ttl": 3600}, - {"type": "CNAME", "name": "muc", "value": "@", "ttl": 3600}, - {"type": "CNAME", "name": "pubsub", "value": "@", "ttl": 3600}, - {"type": "CNAME", "name": "vjud", "value": "@", "ttl": 3600} - {"type": "CNAME", "name": "xmpp-upload", "value": "@", "ttl": 3600} - ], - "mail": [ - {"type": "MX", "name": "@", "value": "10 domain.tld.", "ttl": 3600}, - {"type": "TXT", "name": "@", "value": "\"v=spf1 a mx ip4:123.123.123.123 ipv6:valid-ipv6 -all\"", "ttl": 3600 }, - {"type": "TXT", "name": "mail._domainkey", "value": "\"v=DKIM1; k=rsa; p=some-super-long-key\"", "ttl": 3600}, - {"type": "TXT", "name": "_dmarc", "value": "\"v=DMARC1; p=none\"", "ttl": 3600} - ], - "extra": [ - # if ipv4 available - {"type": "A", "name": "*", "value": "123.123.123.123", "ttl": 3600}, - # if ipv6 available - {"type": "AAAA", "name": "*", "value": "valid-ipv6", "ttl": 3600}, - {"type": "CAA", "name": "@", "value": "128 issue \"letsencrypt.org\"", "ttl": 3600}, - ], - "example_of_a_custom_rule": [ - {"type": "SRV", "name": "_matrix", "value": "domain.tld.", "ttl": 3600} - ], - } - """ - - basic = [] - mail = [] - xmpp = [] - extra = [] - ipv4 = get_public_ip() - ipv6 = get_public_ip(6) - - subdomains = _list_subdomains_of(base_domain) - domains_settings = {domain: _get_domain_settings(domain) - for domain in [base_domain] + subdomains} - - base_dns_zone = domains_settings[base_domain].get("dns_zone") - - for domain, settings in domains_settings.items(): - - # Domain # Base DNS zone # Basename # Suffix # - # ------------------ # ----------------- # --------- # -------- # - # domain.tld # domain.tld # @ # # - # sub.domain.tld # domain.tld # sub # .sub # - # foo.sub.domain.tld # domain.tld # foo.sub # .foo.sub # - # sub.domain.tld # sub.domain.tld # @ # # - # foo.sub.domain.tld # sub.domain.tld # foo # .foo # - - # FIXME: shouldn't the basename just be based on the dns_zone setting of this domain ? - basename = domain.replace(f"{base_dns_zone}", "").rstrip(".") or "@" - suffix = f".{basename}" if base_name != "@" else "" - - ttl = settings["ttl"] - - ########################### - # Basic ipv4/ipv6 records # - ########################### - if ipv4: - basic.append([basename, ttl, "A", ipv4]) - - if ipv6: - basic.append([basename, ttl, "AAAA", ipv6]) - # TODO - # elif include_empty_AAAA_if_no_ipv6: - # basic.append(["@", ttl, "AAAA", None]) - - ######### - # Email # - ######### - if settings["mail_in"]: - mail.append([basename, ttl, "MX", f"10 {domain}."]) - - if settings["mail_out"]: - mail.append([basename, ttl, "TXT", '"v=spf1 a mx -all"']) - - # DKIM/DMARC record - dkim_host, dkim_publickey = _get_DKIM(domain) - - if dkim_host: - mail += [ - [f"{dkim_host}{suffix}", ttl, "TXT", dkim_publickey], - [f"_dmarc{suffix}", ttl, "TXT", '"v=DMARC1; p=none"'], - ] - - ######## - # XMPP # - ######## - if settings["xmpp"]: - xmpp += [ - [ - f"_xmpp-client._tcp{suffix}", - ttl, - "SRV", - f"0 5 5222 {domain}.", - ], - [ - f"_xmpp-server._tcp{suffix}", - ttl, - "SRV", - f"0 5 5269 {domain}.", - ], - [f"muc{suffix}", ttl, "CNAME", basename], - [f"pubsub{suffix}", ttl, "CNAME", basename], - [f"vjud{suffix}", ttl, "CNAME", basename], - [f"xmpp-upload{suffix}", ttl, "CNAME", basename], - ] - - ######### - # Extra # - ######### - - if ipv4: - extra.append([f"*{suffix}", ttl, "A", ipv4]) - - if ipv6: - extra.append([f"*{suffix}", ttl, "AAAA", ipv6]) - # TODO - # elif include_empty_AAAA_if_no_ipv6: - # extra.append(["*", ttl, "AAAA", None]) - - extra.append([basename, ttl, "CAA", '128 issue "letsencrypt.org"']) - - #################### - # Standard records # - #################### - - records = { - "basic": [ - {"name": name, "ttl": ttl_, "type": type_, "value": value} - for name, ttl_, type_, value in basic - ], - "xmpp": [ - {"name": name, "ttl": ttl_, "type": type_, "value": value} - for name, ttl_, type_, value in xmpp - ], - "mail": [ - {"name": name, "ttl": ttl_, "type": type_, "value": value} - for name, ttl_, type_, value in mail - ], - "extra": [ - {"name": name, "ttl": ttl_, "type": type_, "value": value} - for name, ttl_, type_, value in extra - ], - } - - ################## - # Custom records # - ################## - - # Defined by custom hooks ships in apps for example ... - - hook_results = hook_callback("custom_dns_rules", args=[base_domain]) - for hook_name, results in hook_results.items(): - # - # There can be multiple results per hook name, so results look like - # {'/some/path/to/hook1': - # { 'state': 'succeed', - # 'stdreturn': [{'type': 'SRV', - # 'name': 'stuff.foo.bar.', - # 'value': 'yoloswag', - # 'ttl': 3600}] - # }, - # '/some/path/to/hook2': - # { ... }, - # [...] - # - # Loop over the sub-results - custom_records = [ - v["stdreturn"] for v in results.values() if v and v["stdreturn"] - ] - - records[hook_name] = [] - for record_list in custom_records: - # Check that record_list is indeed a list of dict - # with the required keys - if ( - not isinstance(record_list, list) - or any(not isinstance(record, dict) for record in record_list) - or any( - key not in record - for record in record_list - for key in ["name", "ttl", "type", "value"] - ) - ): - # Display an error, mainly for app packagers trying to implement a hook - logger.warning( - "Ignored custom record from hook '%s' because the data is not a *list* of dict with keys name, ttl, type and value. Raw data : %s" - % (hook_name, record_list) - ) - continue - - records[hook_name].extend(record_list) - - return records - - -def _get_DKIM(domain): - DKIM_file = "/etc/dkim/{domain}.mail.txt".format(domain=domain) - - if not os.path.isfile(DKIM_file): - return (None, None) - - with open(DKIM_file) as f: - dkim_content = f.read() - - # Gotta manage two formats : - # - # Legacy - # ----- - # - # mail._domainkey IN TXT ( "v=DKIM1; k=rsa; " - # "p=" ) - # - # New - # ------ - # - # mail._domainkey IN TXT ( "v=DKIM1; h=sha256; k=rsa; " - # "p=" ) - - is_legacy_format = " h=sha256; " not in dkim_content - - # Legacy DKIM format - if is_legacy_format: - dkim = re.match( - ( - r"^(?P[a-z_\-\.]+)[\s]+([0-9]+[\s]+)?IN[\s]+TXT[\s]+" - r'[^"]*"v=(?P[^";]+);' - r'[\s"]*k=(?P[^";]+);' - r'[\s"]*p=(?P

[^";]+)' - ), - dkim_content, - re.M | re.S, - ) - else: - dkim = re.match( - ( - r"^(?P[a-z_\-\.]+)[\s]+([0-9]+[\s]+)?IN[\s]+TXT[\s]+" - r'[^"]*"v=(?P[^";]+);' - r'[\s"]*h=(?P[^";]+);' - r'[\s"]*k=(?P[^";]+);' - r'[\s"]*p=(?P

[^";]+)' - ), - dkim_content, - re.M | re.S, - ) - - if not dkim: - return (None, None) - - if is_legacy_format: - return ( - dkim.group("host"), - '"v={v}; k={k}; p={p}"'.format( - v=dkim.group("v"), k=dkim.group("k"), p=dkim.group("p") - ), - ) - else: - return ( - dkim.group("host"), - '"v={v}; h={h}; k={k}; p={p}"'.format( - v=dkim.group("v"), - h=dkim.group("h"), - k=dkim.group("k"), - p=dkim.group("p"), - ), - ) - - def _default_domain_settings(domain, is_main_domain): + from yunohost.utils.dns import get_dns_zone_from_domain return { "xmpp": is_main_domain, "mail_in": True, @@ -825,10 +453,10 @@ def domain_setting(domain, key, value=None, delete=False): value = int(value) except ValueError: # TODO add locales - raise YunohostError("invalid_number", value_type=type(value)) + raise YunohostValidationError("invalid_number", value_type=type(value)) if value < 0: - raise YunohostError("pattern_positive_number", value_type=type(value)) + raise YunohostValidationError("pattern_positive_number", value_type=type(value)) # Set new value domain_settings[key] = value @@ -870,184 +498,59 @@ def _set_domain_settings(domain, domain_settings): filepath = f"{DOMAIN_SETTINGS_DIR}/{domain}.yml" write_to_yaml(filepath, domain_settings) - -def _get_registrar_settings(dns_zone): - on_disk_settings = {} - filepath = f"{REGISTRAR_SETTINGS_DIR}/{dns_zone}.yml" - if os.path.exists(filepath) and os.path.isfile(filepath): - on_disk_settings = read_yaml(filepath) or {} - - return on_disk_settings +# +# +# Stuff managed in other files +# +# -def _set_registrar_settings(dns_zone): - if not os.path.exists(REGISTRAR_SETTINGS_DIR): - os.mkdir(REGISTRAR_SETTINGS_DIR) - filepath = f"{REGISTRAR_SETTINGS_DIR}/{dns_zone}.yml" - write_to_yaml(filepath, domain_registrar) +def domain_cert_status(domain_list, full=False): + import yunohost.certificate + + return yunohost.certificate.certificate_status(domain_list, full) + + +def domain_cert_install( + domain_list, force=False, no_checks=False, self_signed=False, staging=False +): + import yunohost.certificate + + return yunohost.certificate.certificate_install( + domain_list, force, no_checks, self_signed, staging + ) + + +def domain_cert_renew( + domain_list, force=False, no_checks=False, email=False, staging=False +): + import yunohost.certificate + + return yunohost.certificate.certificate_renew( + domain_list, force, no_checks, email, staging + ) + + +def domain_dns_conf(domain): + import yunohost.dns + return yunohost.dns.domain_dns_conf(domain) def domain_registrar_info(domain): - - dns_zone = _get_domain_settings(domain)["dns_zone"] - registrar_info = _get_registrar_settings(dns_zone) - if not registrar_info: - raise YunohostError("registrar_is_not_set", dns_zone=dns_zone) - - return registrar_info + import yunohost.dns + return yunohost.dns.domain_registrar_info(domain) def domain_registrar_catalog(registrar_name, full): - registrars = read_yaml(REGISTRAR_LIST_PATH) - - if registrar_name: - if registrar_name not in registrars.keys(): - raise YunohostError("domain_registrar_unknown", registrar=registrar_name) - else: - return registrars[registrar_name] - else: - return registrars + import yunohost.dns + return yunohost.dns.domain_registrar_catalog(registrar_name, full) def domain_registrar_set(domain, registrar, args): - - registrars = read_yaml(REGISTRAR_LIST_PATH) - if registrar not in registrars.keys(): - raise YunohostError("domain_registrar_unknown"i, registrar=registrar) - - parameters = registrars[registrar] - ask_args = [] - for parameter in parameters: - ask_args.append( - { - "name": parameter, - "type": "string", - "example": "", - "default": "", - } - ) - args_dict = ( - {} if not args else dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) - ) - parsed_answer_dict = _parse_args_in_yunohost_format(args_dict, ask_args) - - domain_registrar = {"name": registrar, "options": {}} - for arg_name, arg_value_and_type in parsed_answer_dict.items(): - domain_registrar["options"][arg_name] = arg_value_and_type[0] - - dns_zone = _get_domain_settings(domain)["dns_zone"] - _set_registrar_settings(dns_zone, domain_registrar) + import yunohost.dns + return yunohost.dns.domain_registrar_set(domain, registrar, args) def domain_push_config(domain): - """ - Send DNS records to the previously-configured registrar of the domain. - """ - # Generate the records - if domain not in domain_list()["domains"]: - raise YunohostValidationError("domain_name_unknown", domain=domain) - - dns_conf = _build_dns_conf(domain) - - dns_zone = _get_domain_settings(domain)["dns_zone"] - registrar_setting = _get_registrar_settings(dns_zone) - - if not registrar_setting: - # FIXME add locales - raise YunohostValidationError("registrar_is_not_set", domain=domain) - - # Flatten the DNS conf - flatten_dns_conf = [] - for key in dns_conf: - list_of_records = dns_conf[key] - for record in list_of_records: - # FIXME Lexicon does not support CAA records - # See https://github.com/AnalogJ/lexicon/issues/282 and https://github.com/AnalogJ/lexicon/pull/371 - # They say it's trivial to implement it! - # And yet, it is still not done/merged - if record["type"] != "CAA": - # Add .domain.tdl to the name entry - record["name"] = "{}.{}".format(record["name"], domain) - flatten_dns_conf.append(record) - - # Construct the base data structure to use lexicon's API. - base_config = { - "provider_name": registrar_setting["name"], - "domain": domain, # domain name - } - base_config[registrar_setting["name"]] = registrar_setting["options"] - - # Get types present in the generated records - types = set() - - for record in flatten_dns_conf: - types.add(record["type"]) - - # Fetch all types present in the generated records - distant_records = {} - - for key in types: - record_config = { - "action": "list", - "type": key, - } - final_lexicon = ( - ConfigResolver() - .with_dict(dict_object=base_config) - .with_dict(dict_object=record_config) - ) - # print('final_lexicon:', final_lexicon); - client = Client(final_lexicon) - distant_records[key] = client.execute() - - for key in types: - for distant_record in distant_records[key]: - logger.debug(f"distant_record: {distant_record}") - for local_record in flatten_dns_conf: - print("local_record:", local_record) - - # Push the records - for record in flatten_dns_conf: - # For each record, first check if one record exists for the same (type, name) couple - it_exists = False - # TODO do not push if local and distant records are exactly the same ? - # is_the_same_record = False - - for distant_record in distant_records[record["type"]]: - if ( - distant_record["type"] == record["type"] - and distant_record["name"] == record["name"] - ): - it_exists = True - # see previous TODO - # if distant_record["ttl"] = ... and distant_record["name"] ... - # is_the_same_record = True - - # Finally, push the new record or update the existing one - record_config = { - "action": "update" - if it_exists - else "create", # create, list, update, delete - "type": record[ - "type" - ], # specify a type for record filtering, case sensitive in some cases. - "name": record["name"], - "content": record["value"], - # FIXME Removed TTL, because it doesn't work with Gandi. - # See https://github.com/AnalogJ/lexicon/issues/726 (similar issue) - # But I think there is another issue with Gandi. Or I'm misusing the API... - # "ttl": record["ttl"], - } - final_lexicon = ( - ConfigResolver() - .with_dict(dict_object=base_config) - .with_dict(dict_object=record_config) - ) - client = Client(final_lexicon) - print("pushed_record:", record_config, "→", end=" ") - results = client.execute() - print("results:", results) - # print("Failed" if results == False else "Ok") - - -# def domain_config_fetch(domain, key, value): + import yunohost.dns + return yunohost.dns.domain_push_config(domain) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index 071e93059..9cb6dc567 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -37,7 +37,7 @@ from moulinette.utils.filesystem import write_to_file, read_file from moulinette.utils.network import download_json from yunohost.utils.error import YunohostError, YunohostValidationError -from yunohost.domain import _get_maindomain, _build_dns_conf +from yunohost.domain import _get_maindomain from yunohost.utils.network import get_public_ip from yunohost.utils.dns import dig from yunohost.log import is_unit_operation @@ -225,6 +225,9 @@ def dyndns_update( ipv6 -- IPv6 address to send """ + + from yunohost.dns import _build_dns_conf + # Get old ipv4/v6 old_ipv4, old_ipv6 = (None, None) # (default values) From 7e048b85b9e25d900404962852239da63532d9a5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 28 Aug 2021 21:32:11 +0200 Subject: [PATCH 056/130] Misc fixes --- src/yunohost/dns.py | 4 ++-- src/yunohost/domain.py | 22 +++++++++++++++++----- src/yunohost/settings.py | 9 +++++++-- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 590d50876..ce0fc7a7d 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -31,7 +31,7 @@ from lexicon.config import ConfigResolver from moulinette import m18n, Moulinette from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import read_yaml, write_to_yaml +from moulinette.utils.filesystem import mkdir, read_yaml, write_to_yaml from yunohost.domain import domain_list, _get_domain_settings from yunohost.app import _parse_args_in_yunohost_format @@ -392,7 +392,7 @@ def _get_registrar_settings(dns_zone): def _set_registrar_settings(dns_zone, domain_registrar): if not os.path.exists(REGISTRAR_SETTINGS_DIR): - os.mkdir(REGISTRAR_SETTINGS_DIR) + mkdir(REGISTRAR_SETTINGS_DIR, mode=0o700) filepath = f"{REGISTRAR_SETTINGS_DIR}/{dns_zone}.yml" write_to_yaml(filepath, domain_registrar) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index c18d5f665..71b30451e 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -28,8 +28,9 @@ import os from moulinette import m18n, Moulinette from moulinette.core import MoulinetteError from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import write_to_file, read_yaml, write_to_yaml +from moulinette.utils.filesystem import mkdir, write_to_file, read_yaml, write_to_yaml +from yunohost.settings import is_boolean from yunohost.app import ( app_ssowatconf, _installed_apps, @@ -58,6 +59,7 @@ def domain_list(exclude_subdomains=False): exclude_subdomains -- Filter out domains that are subdomains of other declared domains """ + global domain_list_cache if not exclude_subdomains and domain_list_cache: return domain_list_cache @@ -161,6 +163,7 @@ def domain_add(operation_logger, domain, dyndns=False): except Exception as e: raise YunohostError("domain_creation_failed", domain=domain, error=e) finally: + global domain_list_cache domain_list_cache = {} # Don't regen these conf if we're still in postinstall @@ -279,6 +282,7 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False): except Exception as e: raise YunohostError("domain_deletion_failed", domain=domain, error=e) finally: + global domain_list_cache domain_list_cache = {} stuff_to_delete = [ @@ -344,6 +348,7 @@ def domain_main_domain(operation_logger, new_main_domain=None): # Apply changes to ssl certs try: write_to_file("/etc/yunohost/current_host", new_main_domain) + global domain_list_cache domain_list_cache = {} _set_hostname(new_main_domain) except Exception as e: @@ -378,10 +383,10 @@ def _get_maindomain(): return maindomain -def _default_domain_settings(domain, is_main_domain): +def _default_domain_settings(domain): from yunohost.utils.dns import get_dns_zone_from_domain return { - "xmpp": is_main_domain, + "xmpp": domain == domain_list()["main"], "mail_in": True, "mail_out": True, "dns_zone": get_dns_zone_from_domain(domain), @@ -408,7 +413,7 @@ def _get_domain_settings(domain): on_disk_settings = read_yaml(filepath) or {} # Inject defaults if needed (using the magic .update() ;)) - settings = _default_domain_settings(domain, domain == maindomain) + settings = _default_domain_settings(domain) settings.update(on_disk_settings) return settings @@ -446,7 +451,14 @@ def domain_setting(domain, key, value=None, delete=False): # maybe inspired from the global settings if key in ["mail_in", "mail_out", "xmpp"]: - value = True if value.lower() in ['true', '1', 't', 'y', 'yes', "iloveynh"] else False + _is_boolean, value = is_boolean(value) + if not _is_boolean: + raise YunohostValidationError( + "global_settings_bad_type_for_setting", + setting=key, + received_type="not boolean", + expected_type="boolean", + ) if "ttl" == key: try: diff --git a/src/yunohost/settings.py b/src/yunohost/settings.py index fe072cddb..bc3a56d89 100644 --- a/src/yunohost/settings.py +++ b/src/yunohost/settings.py @@ -18,6 +18,9 @@ SETTINGS_PATH_OTHER_LOCATION = "/etc/yunohost/settings-%s.json" def is_boolean(value): + TRUE = ["true", "on", "yes", "y", "1"] + FALSE = ["false", "off", "no", "n", "0"] + """ Ensure a string value is intended as a boolean @@ -30,9 +33,11 @@ def is_boolean(value): """ if isinstance(value, bool): return True, value + if value in [0, 1]: + return True, bool(value) elif isinstance(value, str): - if str(value).lower() in ["true", "on", "yes", "false", "off", "no"]: - return True, str(value).lower() in ["true", "on", "yes"] + if str(value).lower() in TRUE + FALSE: + return True, str(value).lower() in TRUE else: return False, None else: From 951c6695b869cb3bed331e293d38770145718ad4 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 28 Aug 2021 21:32:40 +0200 Subject: [PATCH 057/130] domain settings: Only store the diff with respect to defaults, should make possible migrations easier idk --- src/yunohost/domain.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 71b30451e..ad6cc99dc 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -503,12 +503,15 @@ def _set_domain_settings(domain, domain_settings): if domain not in domain_list()["domains"]: raise YunohostError("domain_name_unknown", domain=domain) + defaults = _default_domain_settings(domain) + diff_with_defaults = {k: v for k, v in domain_settings.items() if defaults.get(k) != v} + # First create the DOMAIN_SETTINGS_DIR if it doesn't exist if not os.path.exists(DOMAIN_SETTINGS_DIR): - os.mkdir(DOMAIN_SETTINGS_DIR) + mkdir(DOMAIN_SETTINGS_DIR, mode=0o700) # Save the settings to the .yaml file filepath = f"{DOMAIN_SETTINGS_DIR}/{domain}.yml" - write_to_yaml(filepath, domain_settings) + write_to_yaml(filepath, diff_with_defaults) # # From dded1cb7754ad6422e79110cc6c8beb6b5e546a9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 28 Aug 2021 21:49:35 +0200 Subject: [PATCH 058/130] Moar fikses --- src/yunohost/dns.py | 19 +++++++++++++++++-- src/yunohost/domain.py | 15 --------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index ce0fc7a7d..3943f7ed3 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -95,6 +95,21 @@ def domain_dns_conf(domain): return result +def _list_subdomains_of(parent_domain): + + domain_list_ = domain_list()["domains"] + + if parent_domain not in domain_list_: + raise YunohostError("domain_name_unknown", domain=domain) + + out = [] + for domain in domain_list_: + if domain.endswith(f".{parent_domain}"): + out.append(domain) + + return out + + def _build_dns_conf(base_domain): """ Internal function that will returns a data structure containing the needed @@ -163,7 +178,7 @@ def _build_dns_conf(base_domain): # FIXME: shouldn't the basename just be based on the dns_zone setting of this domain ? basename = domain.replace(f"{base_dns_zone}", "").rstrip(".") or "@" - suffix = f".{basename}" if base_name != "@" else "" + suffix = f".{basename}" if basename != "@" else "" ttl = settings["ttl"] @@ -423,7 +438,7 @@ def domain_registrar_set(domain, registrar, args): registrars = read_yaml(REGISTRAR_LIST_PATH) if registrar not in registrars.keys(): - raise YunohostValidationError("domain_registrar_unknown"i, registrar=registrar) + raise YunohostValidationError("domain_registrar_unknown", registrar=registrar) parameters = registrars[registrar] ask_args = [] diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index ad6cc99dc..8bb6f5a34 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -476,21 +476,6 @@ def domain_setting(domain, key, value=None, delete=False): _set_domain_settings(domain, domain_settings) -def _list_subdomains_of(parent_domain): - - domain_list_ = domain_list()["domains"] - - if parent_domain not in domain_list_: - raise YunohostError("domain_name_unknown", domain=domain) - - out = [] - for domain in domain_list_: - if domain.endswith(f".{parent_domain}"): - out.append(domain) - - return out - - def _set_domain_settings(domain, domain_settings): """ Set settings of a domain From 5a93e0640d3372054517411a49650e72606a6880 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 28 Aug 2021 21:53:19 +0200 Subject: [PATCH 059/130] domain_push_config -> domain_registrar_push --- data/actionsmap/yunohost.yml | 30 ++++++++++++++++++------------ src/yunohost/dns.py | 2 +- src/yunohost/domain.py | 4 ++-- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index c4323f166..686502b2c 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -439,16 +439,6 @@ domain: help: Subscribe to the DynDNS service action: store_true - ### domain_push_config() - push-config: - action_help: Push DNS records to registrar - api: GET /domains//push - arguments: - domain: - help: Domain name to add - extra: - pattern: *pattern_domain - ### domain_remove() remove: action_help: Delete domains @@ -558,6 +548,7 @@ domain: pattern: *pattern_domain path: help: The path to check (e.g. /coffee) + ### domain_setting() setting: action_help: Set or get a domain setting value @@ -576,12 +567,13 @@ domain: full: --delete help: Delete the key action: store_true + subcategories: registrar: subcategory_help: Manage domains registrars - actions: + actions: ### domain_registrar_set() - set: + set: action_help: Set domain registrar api: POST /domains//registrar arguments: @@ -594,6 +586,7 @@ domain: -a: full: --args help: Serialized arguments for registrar API (i.e. "auth_token=TOKEN&auth_username=USER"). + ### domain_registrar_info() info: action_help: Display info about registrar settings used for a domain @@ -603,10 +596,12 @@ domain: help: Domain name extra: pattern: *pattern_domain + ### domain_registrar_list() list: action_help: List registrars configured by DNS zone api: GET /domains/registrars + ### domain_registrar_catalog() catalog: action_help: List supported registrars API @@ -620,6 +615,17 @@ domain: help: Display all details, including info to create forms action: store_true + ### domain_registrar_push() + push: + action_help: Push DNS records to registrar + api: PUT /domains//registrar/push + arguments: + domain: + help: Domain name to push DNS conf for + extra: + pattern: *pattern_domain + + ############################# # App # ############################# diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 3943f7ed3..8d8a6c735 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -464,7 +464,7 @@ def domain_registrar_set(domain, registrar, args): _set_registrar_settings(dns_zone, domain_registrar) -def domain_push_config(domain): +def domain_registrar_push(domain): """ Send DNS records to the previously-configured registrar of the domain. """ diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 8bb6f5a34..e1b247d7d 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -551,6 +551,6 @@ def domain_registrar_set(domain, registrar, args): return yunohost.dns.domain_registrar_set(domain, registrar, args) -def domain_push_config(domain): +def domain_registrar_push(domain): import yunohost.dns - return yunohost.dns.domain_push_config(domain) + return yunohost.dns.domain_registrar_push(domain) From 4089c34685a33ca5f8276516e0340ea769d0f656 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 28 Aug 2021 21:55:44 +0200 Subject: [PATCH 060/130] Add logging to domain_registrar_push --- src/yunohost/dns.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 8d8a6c735..41e6dc374 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -464,7 +464,8 @@ def domain_registrar_set(domain, registrar, args): _set_registrar_settings(dns_zone, domain_registrar) -def domain_registrar_push(domain): +@is_unit_operation() +def domain_registrar_push(operation_logger, domain): """ Send DNS records to the previously-configured registrar of the domain. """ @@ -508,6 +509,8 @@ def domain_registrar_push(domain): for record in flatten_dns_conf: types.add(record["type"]) + operation_logger.start() + # Fetch all types present in the generated records distant_records = {} From 1c2fff750d4942901a3e3776a49c57d92dfbf8a4 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 31 Aug 2021 18:33:15 +0200 Subject: [PATCH 061/130] dns/lexicon: Tidying up the push function --- src/yunohost/dns.py | 117 +++++++++++++++++++------------------------- 1 file changed, 50 insertions(+), 67 deletions(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 41e6dc374..e3131bcdd 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -26,9 +26,6 @@ import os import re -from lexicon.client import Client -from lexicon.config import ConfigResolver - from moulinette import m18n, Moulinette from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import mkdir, read_yaml, write_to_yaml @@ -469,96 +466,81 @@ def domain_registrar_push(operation_logger, domain): """ Send DNS records to the previously-configured registrar of the domain. """ - # Generate the records + + from lexicon.client import Client as LexiconClient + from lexicon.config import ConfigResolver as LexiconConfigResolver + if domain not in domain_list()["domains"]: raise YunohostValidationError("domain_name_unknown", domain=domain) - dns_conf = _build_dns_conf(domain) - dns_zone = _get_domain_settings(domain)["dns_zone"] - registrar_setting = _get_registrar_settings(dns_zone) + registrar_settings = _get_registrar_settingss(dns_zone) - if not registrar_setting: - # FIXME add locales + if not registrar_settings: raise YunohostValidationError("registrar_is_not_set", domain=domain) + # Generate the records + dns_conf = _build_dns_conf(domain) + # Flatten the DNS conf - flatten_dns_conf = [] - for key in dns_conf: - list_of_records = dns_conf[key] - for record in list_of_records: - # FIXME Lexicon does not support CAA records - # See https://github.com/AnalogJ/lexicon/issues/282 and https://github.com/AnalogJ/lexicon/pull/371 - # They say it's trivial to implement it! - # And yet, it is still not done/merged - if record["type"] != "CAA": - # Add .domain.tdl to the name entry - record["name"] = "{}.{}".format(record["name"], domain) - flatten_dns_conf.append(record) + dns_conf = [record for record in records_for_category for records_for_category in dns_conf.values()] + + # FIXME Lexicon does not support CAA records + # See https://github.com/AnalogJ/lexicon/issues/282 and https://github.com/AnalogJ/lexicon/pull/371 + # They say it's trivial to implement it! + # And yet, it is still not done/merged + dns_conf = [record for record in dns_conf if record["type"] != "CAA"] + + # We need absolute names? FIXME: should we add a trailing dot needed here ? + for record in dns_conf: + record["name"] = f"{record['name']}.{domain}" # Construct the base data structure to use lexicon's API. base_config = { - "provider_name": registrar_setting["name"], - "domain": domain, # domain name + "provider_name": registrar_settings["name"], + "domain": domain, + registrar_settings["name"]: registrar_settings["options"] } - base_config[registrar_setting["name"]] = registrar_setting["options"] - - # Get types present in the generated records - types = set() - - for record in flatten_dns_conf: - types.add(record["type"]) operation_logger.start() # Fetch all types present in the generated records - distant_records = {} + current_remote_records = {} + + # Get unique types present in the generated records + types = {record["type"] for record in dns_conf} for key in types: - record_config = { + fetch_records_for_type = { "action": "list", "type": key, } - final_lexicon = ( - ConfigResolver() + query = ( + LexiconConfigResolver() .with_dict(dict_object=base_config) - .with_dict(dict_object=record_config) + .with_dict(dict_object=fetch_records_for_type) ) - # print('final_lexicon:', final_lexicon); - client = Client(final_lexicon) - distant_records[key] = client.execute() + current_remote_records[key] = LexiconClient(query).execute() for key in types: - for distant_record in distant_records[key]: - logger.debug(f"distant_record: {distant_record}") - for local_record in flatten_dns_conf: + for current_remote_record in current_remote_records[key]: + logger.debug(f"current_remote_record: {current_remote_record}") + for local_record in dns_conf: print("local_record:", local_record) # Push the records - for record in flatten_dns_conf: - # For each record, first check if one record exists for the same (type, name) couple - it_exists = False - # TODO do not push if local and distant records are exactly the same ? - # is_the_same_record = False + for record in dns_conf: - for distant_record in distant_records[record["type"]]: - if ( - distant_record["type"] == record["type"] - and distant_record["name"] == record["name"] - ): - it_exists = True - # see previous TODO - # if distant_record["ttl"] = ... and distant_record["name"] ... - # is_the_same_record = True + # For each record, first check if one record exists for the same (type, name) couple + # TODO do not push if local and distant records are exactly the same ? + type_and_name = (record["type"], record["name"]) + already_exists = any((r["type"], r["name"]) == type_and_name + for r in current_remote_records[record["type"]]) # Finally, push the new record or update the existing one - record_config = { - "action": "update" - if it_exists - else "create", # create, list, update, delete - "type": record[ - "type" - ], # specify a type for record filtering, case sensitive in some cases. + record_to_push = { + "action": "update" if already_exists else "create" + "type": record["type"] "name": record["name"], "content": record["value"], # FIXME Removed TTL, because it doesn't work with Gandi. @@ -566,14 +548,15 @@ def domain_registrar_push(operation_logger, domain): # But I think there is another issue with Gandi. Or I'm misusing the API... # "ttl": record["ttl"], } - final_lexicon = ( + + print("pushed_record:", record_to_push, "→", end=" ") + + query = ( ConfigResolver() .with_dict(dict_object=base_config) - .with_dict(dict_object=record_config) + .with_dict(dict_object=record_to_push) ) - client = Client(final_lexicon) - print("pushed_record:", record_config, "→", end=" ") - results = client.execute() + results = LexiconClient(query).execute() print("results:", results) # print("Failed" if results == False else "Ok") From 01bc6762aac99d7fcd85439ba69863f6a6ce0cd0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 1 Sep 2021 16:36:38 +0200 Subject: [PATCH 062/130] registrars: remove unimplemented or unecessary stuff --- data/actionsmap/yunohost.yml | 24 ++++++------------------ src/yunohost/dns.py | 16 ++++------------ src/yunohost/domain.py | 14 +++++++------- 3 files changed, 17 insertions(+), 37 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 686502b2c..50b93342b 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -572,6 +572,12 @@ domain: registrar: subcategory_help: Manage domains registrars actions: + ### domain_registrar_catalog() + catalog: + action_help: List supported registrars API + api: GET /domains/registrars/catalog + + ### domain_registrar_set() set: action_help: Set domain registrar @@ -597,24 +603,6 @@ domain: extra: pattern: *pattern_domain - ### domain_registrar_list() - list: - action_help: List registrars configured by DNS zone - api: GET /domains/registrars - - ### domain_registrar_catalog() - catalog: - action_help: List supported registrars API - api: GET /domains/registrars/catalog - arguments: - -r: - full: --registrar-name - help: Display given registrar info to create form - -f: - full: --full - help: Display all details, including info to create forms - action: store_true - ### domain_registrar_push() push: action_help: Push DNS records to registrar diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index e3131bcdd..4e68203be 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -419,16 +419,8 @@ def domain_registrar_info(domain): return registrar_info -def domain_registrar_catalog(registrar_name, full): - registrars = read_yaml(REGISTRAR_LIST_PATH) - - if registrar_name: - if registrar_name not in registrars.keys(): - raise YunohostValidationError("domain_registrar_unknown", registrar=registrar_name) - else: - return registrars[registrar_name] - else: - return registrars +def domain_registrar_catalog(): + return read_yaml(REGISTRAR_LIST_PATH) def domain_registrar_set(domain, registrar, args): @@ -539,8 +531,8 @@ def domain_registrar_push(operation_logger, domain): # Finally, push the new record or update the existing one record_to_push = { - "action": "update" if already_exists else "create" - "type": record["type"] + "action": "update" if already_exists else "create", + "type": record["type"], "name": record["name"], "content": record["value"], # FIXME Removed TTL, because it doesn't work with Gandi. diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index e1b247d7d..bc2e6d7af 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -536,14 +536,9 @@ def domain_dns_conf(domain): return yunohost.dns.domain_dns_conf(domain) -def domain_registrar_info(domain): +def domain_registrar_catalog(): import yunohost.dns - return yunohost.dns.domain_registrar_info(domain) - - -def domain_registrar_catalog(registrar_name, full): - import yunohost.dns - return yunohost.dns.domain_registrar_catalog(registrar_name, full) + return yunohost.dns.domain_registrar_catalog() def domain_registrar_set(domain, registrar, args): @@ -551,6 +546,11 @@ def domain_registrar_set(domain, registrar, args): return yunohost.dns.domain_registrar_set(domain, registrar, args) +def domain_registrar_info(domain): + import yunohost.dns + return yunohost.dns.domain_registrar_info(domain) + + def domain_registrar_push(domain): import yunohost.dns return yunohost.dns.domain_registrar_push(domain) From d5b1eecd0754fd7cbd9a4bddcd222389b01b2c26 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 1 Sep 2021 16:53:28 +0200 Subject: [PATCH 063/130] Add a small _assert_domain_exists to avoid always repeating the same code snippet --- src/yunohost/app.py | 14 +++++++------- src/yunohost/certificate.py | 14 ++++---------- src/yunohost/dns.py | 15 +++++---------- src/yunohost/domain.py | 23 +++++++++++------------ src/yunohost/permission.py | 11 ++++------- src/yunohost/user.py | 5 ++--- 6 files changed, 33 insertions(+), 49 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index f4acb198f..09136ef48 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1340,7 +1340,7 @@ def app_makedefault(operation_logger, app, domain=None): domain """ - from yunohost.domain import domain_list + from yunohost.domain import _assert_domain_exists app_settings = _get_app_settings(app) app_domain = app_settings["domain"] @@ -1348,9 +1348,10 @@ def app_makedefault(operation_logger, app, domain=None): if domain is None: domain = app_domain - operation_logger.related_to.append(("domain", domain)) - elif domain not in domain_list()["domains"]: - raise YunohostValidationError("domain_name_unknown", domain=domain) + + _assert_domain_exists(domain) + + operation_logger.related_to.append(("domain", domain)) if "/" in app_map(raw=True)[domain]: raise YunohostValidationError( @@ -3078,13 +3079,12 @@ def _get_conflicting_apps(domain, path, ignore_app=None): ignore_app -- An optional app id to ignore (c.f. the change_url usecase) """ - from yunohost.domain import domain_list + from yunohost.domain import _assert_domain_exists domain, path = _normalize_domain_path(domain, path) # Abort if domain is unknown - if domain not in domain_list()["domains"]: - raise YunohostValidationError("domain_name_unknown", domain=domain) + _assert_domain_exists(domain) # Fetch apps map apps_map = app_map(raw=True) diff --git a/src/yunohost/certificate.py b/src/yunohost/certificate.py index 52d58777b..817f9d57a 100644 --- a/src/yunohost/certificate.py +++ b/src/yunohost/certificate.py @@ -86,11 +86,8 @@ def certificate_status(domain_list, full=False): domain_list = yunohost.domain.domain_list()["domains"] # Else, validate that yunohost knows the domains given else: - yunohost_domains_list = yunohost.domain.domain_list()["domains"] for domain in domain_list: - # Is it in Yunohost domain list? - if domain not in yunohost_domains_list: - raise YunohostValidationError("domain_name_unknown", domain=domain) + yunohost.domain._assert_domain_exists(domain) certificates = {} @@ -267,9 +264,7 @@ def _certificate_install_letsencrypt( # Else, validate that yunohost knows the domains given else: for domain in domain_list: - yunohost_domains_list = yunohost.domain.domain_list()["domains"] - if domain not in yunohost_domains_list: - raise YunohostValidationError("domain_name_unknown", domain=domain) + yunohost.domain._assert_domain_exists(domain) # Is it self-signed? status = _get_status(domain) @@ -368,9 +363,8 @@ def certificate_renew( else: for domain in domain_list: - # Is it in Yunohost dmomain list? - if domain not in yunohost.domain.domain_list()["domains"]: - raise YunohostValidationError("domain_name_unknown", domain=domain) + # Is it in Yunohost domain list? + yunohost.domain._assert_domain_exists(domain) status = _get_status(domain) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 4e68203be..8399c5a4c 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -30,7 +30,7 @@ from moulinette import m18n, Moulinette from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import mkdir, read_yaml, write_to_yaml -from yunohost.domain import domain_list, _get_domain_settings +from yunohost.domain import domain_list, _get_domain_settings, _assert_domain_exists from yunohost.app import _parse_args_in_yunohost_format from yunohost.utils.error import YunohostValidationError from yunohost.utils.network import get_public_ip @@ -52,8 +52,7 @@ def domain_dns_conf(domain): """ - if domain not in domain_list()["domains"]: - raise YunohostValidationError("domain_name_unknown", domain=domain) + _assert_domain_exists(domain) dns_conf = _build_dns_conf(domain) @@ -94,13 +93,10 @@ def domain_dns_conf(domain): def _list_subdomains_of(parent_domain): - domain_list_ = domain_list()["domains"] - - if parent_domain not in domain_list_: - raise YunohostError("domain_name_unknown", domain=domain) + _assert_domain_exists(parent_domain) out = [] - for domain in domain_list_: + for domain in domain_list()["domains"]: if domain.endswith(f".{parent_domain}"): out.append(domain) @@ -462,8 +458,7 @@ def domain_registrar_push(operation_logger, domain): from lexicon.client import Client as LexiconClient from lexicon.config import ConfigResolver as LexiconConfigResolver - if domain not in domain_list()["domains"]: - raise YunohostValidationError("domain_name_unknown", domain=domain) + _assert_domain_exists(domain) dns_zone = _get_domain_settings(domain)["dns_zone"] registrar_settings = _get_registrar_settingss(dns_zone) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index bc2e6d7af..d7863a0e1 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -96,6 +96,11 @@ def domain_list(exclude_subdomains=False): return domain_list_cache +def _assert_domain_exists(domain): + if domain not in domain_list()["domains"]: + raise YunohostValidationError("domain_name_unknown", domain=domain) + + @is_unit_operation() def domain_add(operation_logger, domain, dyndns=False): """ @@ -216,8 +221,8 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False): # the 'force' here is related to the exception happening in domain_add ... # we don't want to check the domain exists because the ldap add may have # failed - if not force and domain not in domain_list()["domains"]: - raise YunohostValidationError("domain_name_unknown", domain=domain) + if not force: + _assert_domain_exists(domain) # Check domain is not the main domain if domain == _get_maindomain(): @@ -339,8 +344,7 @@ def domain_main_domain(operation_logger, new_main_domain=None): return {"current_main_domain": _get_maindomain()} # Check domain exists - if new_main_domain not in domain_list()["domains"]: - raise YunohostValidationError("domain_name_unknown", domain=new_main_domain) + _assert_domain_exists(new_main_domain) operation_logger.related_to.append(("domain", new_main_domain)) operation_logger.start() @@ -399,12 +403,7 @@ def _get_domain_settings(domain): Retrieve entries in /etc/yunohost/domains/[domain].yml And set default values if needed """ - # Retrieve actual domain list - known_domains = domain_list()["domains"] - maindomain = domain_list()["main"] - - if domain not in known_domains: - raise YunohostValidationError("domain_name_unknown", domain=domain) + _assert_domain_exists(domain) # Retrieve entries in the YAML filepath = f"{DOMAIN_SETTINGS_DIR}/{domain}.yml" @@ -485,8 +484,8 @@ def _set_domain_settings(domain, domain_settings): settings -- Dict with domain settings """ - if domain not in domain_list()["domains"]: - raise YunohostError("domain_name_unknown", domain=domain) + + _assert_domain_exists(domain) defaults = _default_domain_settings(domain) diff_with_defaults = {k: v for k, v in domain_settings.items() if defaults.get(k) != v} diff --git a/src/yunohost/permission.py b/src/yunohost/permission.py index 01330ad7f..d579ff47a 100644 --- a/src/yunohost/permission.py +++ b/src/yunohost/permission.py @@ -860,11 +860,9 @@ def _validate_and_sanitize_permission_url(url, app_base_path, app): re:^/api/.*|/scripts/api.js$ """ - from yunohost.domain import domain_list + from yunohost.domain import _assert_domain_exists from yunohost.app import _assert_no_conflicting_apps - domains = domain_list()["domains"] - # # Regexes # @@ -896,8 +894,8 @@ def _validate_and_sanitize_permission_url(url, app_base_path, app): domain, path = url[3:].split("/", 1) path = "/" + path - if domain.replace("%", "").replace("\\", "") not in domains: - raise YunohostValidationError("domain_name_unknown", domain=domain) + domain_with_no_regex = domain.replace("%", "").replace("\\", "") + _assert_domain_exists(domain_with_no_regex) validate_regex(path) @@ -931,8 +929,7 @@ def _validate_and_sanitize_permission_url(url, app_base_path, app): domain, path = split_domain_path(url) sanitized_url = domain + path - if domain not in domains: - raise YunohostValidationError("domain_name_unknown", domain=domain) + _assert_domain_exists(domain) _assert_no_conflicting_apps(domain, path, ignore_app=app) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 01513f3bd..4863afea9 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -101,7 +101,7 @@ def user_create( mail=None, ): - from yunohost.domain import domain_list, _get_maindomain + from yunohost.domain import domain_list, _get_maindomain, _assert_domain_exists from yunohost.hook import hook_callback from yunohost.utils.password import assert_password_is_strong_enough from yunohost.utils.ldap import _get_ldap_interface @@ -135,8 +135,7 @@ def user_create( domain = maindomain # Check that the domain exists - if domain not in domain_list()["domains"]: - raise YunohostValidationError("domain_name_unknown", domain=domain) + _assert_domain_exists(domain) mail = username + "@" + domain ldap = _get_ldap_interface() From e4783aa00a47b5ba88cda0117c1ed59ef3a75b1d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 1 Sep 2021 21:07:41 +0200 Subject: [PATCH 064/130] Various fixes after tests on OVH --- data/actionsmap/yunohost.yml | 4 +++ src/yunohost/dns.py | 59 +++++++++++++++++++++++++----------- src/yunohost/domain.py | 4 +-- 3 files changed, 47 insertions(+), 20 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 50b93342b..58564b9f7 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -612,6 +612,10 @@ domain: help: Domain name to push DNS conf for extra: pattern: *pattern_domain + -d: + full: --dry-run + help: Only display what's to be pushed + action: store_true ############################# diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 8399c5a4c..0866c3662 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -421,6 +421,8 @@ def domain_registrar_catalog(): def domain_registrar_set(domain, registrar, args): + _assert_domain_exists(domain) + registrars = read_yaml(REGISTRAR_LIST_PATH) if registrar not in registrars.keys(): raise YunohostValidationError("domain_registrar_unknown", registrar=registrar) @@ -450,7 +452,7 @@ def domain_registrar_set(domain, registrar, args): @is_unit_operation() -def domain_registrar_push(operation_logger, domain): +def domain_registrar_push(operation_logger, domain, dry_run=False): """ Send DNS records to the previously-configured registrar of the domain. """ @@ -461,7 +463,7 @@ def domain_registrar_push(operation_logger, domain): _assert_domain_exists(domain) dns_zone = _get_domain_settings(domain)["dns_zone"] - registrar_settings = _get_registrar_settingss(dns_zone) + registrar_settings = _get_registrar_settings(dns_zone) if not registrar_settings: raise YunohostValidationError("registrar_is_not_set", domain=domain) @@ -470,7 +472,7 @@ def domain_registrar_push(operation_logger, domain): dns_conf = _build_dns_conf(domain) # Flatten the DNS conf - dns_conf = [record for record in records_for_category for records_for_category in dns_conf.values()] + dns_conf = [record for records_for_category in dns_conf.values() for record in records_for_category] # FIXME Lexicon does not support CAA records # See https://github.com/AnalogJ/lexicon/issues/282 and https://github.com/AnalogJ/lexicon/pull/371 @@ -479,8 +481,16 @@ def domain_registrar_push(operation_logger, domain): dns_conf = [record for record in dns_conf if record["type"] != "CAA"] # We need absolute names? FIXME: should we add a trailing dot needed here ? + # Seems related to the fact that when fetching current records, they'll contain '.domain.tld' instead of @ + # and we want to check if it already exists or not (c.f. create/update) for record in dns_conf: - record["name"] = f"{record['name']}.{domain}" + if record["name"] == "@": + record["name"] = f".{domain}" + else: + record["name"] = f"{record['name']}.{domain}" + + if record["type"] == "CNAME" and record["value"] == "@": + record["value"] = domain + "." # Construct the base data structure to use lexicon's API. base_config = { @@ -492,12 +502,13 @@ def domain_registrar_push(operation_logger, domain): operation_logger.start() # Fetch all types present in the generated records - current_remote_records = {} + current_remote_records = [] # Get unique types present in the generated records types = {record["type"] for record in dns_conf} for key in types: + print("fetcing type: " + key) fetch_records_for_type = { "action": "list", "type": key, @@ -507,13 +518,12 @@ def domain_registrar_push(operation_logger, domain): .with_dict(dict_object=base_config) .with_dict(dict_object=fetch_records_for_type) ) - current_remote_records[key] = LexiconClient(query).execute() + current_remote_records.extend(LexiconClient(query).execute()) - for key in types: - for current_remote_record in current_remote_records[key]: - logger.debug(f"current_remote_record: {current_remote_record}") - for local_record in dns_conf: - print("local_record:", local_record) + changes = {} + + if dry_run: + return {"current_records": current_remote_records, "dns_conf": dns_conf, "changes": changes} # Push the records for record in dns_conf: @@ -522,7 +532,7 @@ def domain_registrar_push(operation_logger, domain): # TODO do not push if local and distant records are exactly the same ? type_and_name = (record["type"], record["name"]) already_exists = any((r["type"], r["name"]) == type_and_name - for r in current_remote_records[record["type"]]) + for r in current_remote_records) # Finally, push the new record or update the existing one record_to_push = { @@ -530,22 +540,35 @@ def domain_registrar_push(operation_logger, domain): "type": record["type"], "name": record["name"], "content": record["value"], - # FIXME Removed TTL, because it doesn't work with Gandi. - # See https://github.com/AnalogJ/lexicon/issues/726 (similar issue) - # But I think there is another issue with Gandi. Or I'm misusing the API... - # "ttl": record["ttl"], + "ttl": record["ttl"], } - print("pushed_record:", record_to_push, "→", end=" ") + # FIXME Removed TTL, because it doesn't work with Gandi. + # See https://github.com/AnalogJ/lexicon/issues/726 (similar issue) + # But I think there is another issue with Gandi. Or I'm misusing the API... + if base_config["provider_name"] == "gandi": + del record_to_push["ttle"] + print("pushed_record:", record_to_push) + + + # FIXME FIXME FIXME: if a matching record already exists multiple time, + # the current code crashes (at least on OVH) ... we need to provide a specific identifier to update query = ( - ConfigResolver() + LexiconConfigResolver() .with_dict(dict_object=base_config) .with_dict(dict_object=record_to_push) ) + + print(query) + print(query.__dict__) results = LexiconClient(query).execute() print("results:", results) # print("Failed" if results == False else "Ok") + # FIXME FIXME FIXME : if one create / update crash, it shouldn't block everything + + # FIXME : is it possible to push multiple create/update request at once ? + # def domain_config_fetch(domain, key, value): diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index d7863a0e1..d05e31f17 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -550,6 +550,6 @@ def domain_registrar_info(domain): return yunohost.dns.domain_registrar_info(domain) -def domain_registrar_push(domain): +def domain_registrar_push(domain, dry_run): import yunohost.dns - return yunohost.dns.domain_registrar_push(domain) + return yunohost.dns.domain_registrar_push(domain, dry_run) From d2dea2e94ef60ebba49bc262edc5091395ec28e1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 2 Sep 2021 02:24:25 +0200 Subject: [PATCH 065/130] Various changes to try to implement a proper dry-run + proper list of stuff to create/update/delete --- src/yunohost/dns.py | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 0866c3662..41c6e73f1 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -468,11 +468,26 @@ def domain_registrar_push(operation_logger, domain, dry_run=False): if not registrar_settings: raise YunohostValidationError("registrar_is_not_set", domain=domain) - # Generate the records - dns_conf = _build_dns_conf(domain) + # Convert the generated conf into a format that matches what we'll fetch using the API + # Makes it easier to compare "wanted records" with "current records on remote" + dns_conf = [] + for records in _build_dns_conf(domain).values(): + for record in records: - # Flatten the DNS conf - dns_conf = [record for records_for_category in dns_conf.values() for record in records_for_category] + # Make sure we got "absolute" values instead of @ + name = f"{record['name']}.{domain}" if record["name"] != "@" else f".{domain}" + type_ = record["type"] + content = record["value"] + + if content == "@" and record["type"] == "CNAME": + content = domain + "." + + dns_conf.append({ + "name": name, + "type": type_, + "ttl": record["ttl"], + "content": content + }) # FIXME Lexicon does not support CAA records # See https://github.com/AnalogJ/lexicon/issues/282 and https://github.com/AnalogJ/lexicon/pull/371 @@ -480,18 +495,6 @@ def domain_registrar_push(operation_logger, domain, dry_run=False): # And yet, it is still not done/merged dns_conf = [record for record in dns_conf if record["type"] != "CAA"] - # We need absolute names? FIXME: should we add a trailing dot needed here ? - # Seems related to the fact that when fetching current records, they'll contain '.domain.tld' instead of @ - # and we want to check if it already exists or not (c.f. create/update) - for record in dns_conf: - if record["name"] == "@": - record["name"] = f".{domain}" - else: - record["name"] = f"{record['name']}.{domain}" - - if record["type"] == "CNAME" and record["value"] == "@": - record["value"] = domain + "." - # Construct the base data structure to use lexicon's API. base_config = { "provider_name": registrar_settings["name"], @@ -499,13 +502,11 @@ def domain_registrar_push(operation_logger, domain, dry_run=False): registrar_settings["name"]: registrar_settings["options"] } - operation_logger.start() - # Fetch all types present in the generated records current_remote_records = [] # Get unique types present in the generated records - types = {record["type"] for record in dns_conf} + types = ["A", "AAAA", "MX", "TXT", "CNAME", "SRV"] for key in types: print("fetcing type: " + key) @@ -525,6 +526,8 @@ def domain_registrar_push(operation_logger, domain, dry_run=False): if dry_run: return {"current_records": current_remote_records, "dns_conf": dns_conf, "changes": changes} + operation_logger.start() + # Push the records for record in dns_conf: @@ -547,7 +550,7 @@ def domain_registrar_push(operation_logger, domain, dry_run=False): # See https://github.com/AnalogJ/lexicon/issues/726 (similar issue) # But I think there is another issue with Gandi. Or I'm misusing the API... if base_config["provider_name"] == "gandi": - del record_to_push["ttle"] + del record_to_push["ttl"] print("pushed_record:", record_to_push) From 0874e9a646cb861a15d201214d1666378fe9ad99 Mon Sep 17 00:00:00 2001 From: ljf Date: Fri, 3 Sep 2021 17:07:34 +0200 Subject: [PATCH 066/130] [wip] Config Panel for domain --- data/actionsmap/yunohost.yml | 63 +++- data/other/config_domain.toml | 38 ++ data/other/registrar_list.toml | 636 +++++++++++++++++++++++++++++++++ src/yunohost/domain.py | 129 ++----- src/yunohost/utils/config.py | 30 +- 5 files changed, 783 insertions(+), 113 deletions(-) create mode 100644 data/other/config_domain.toml create mode 100644 data/other/registrar_list.toml diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 5d1b757bd..ea4a1f577 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -467,6 +467,17 @@ domain: help: Target domain extra: pattern: *pattern_domain + + ### domain_push_config() + push-config: + action_help: Push DNS records to registrar + api: GET /domains//push + arguments: + domain: + help: Domain name to add + extra: + pattern: *pattern_domain + ### domain_maindomain() main-domain: @@ -549,26 +560,40 @@ domain: path: help: The path to check (e.g. /coffee) - ### domain_setting() - setting: - action_help: Set or get a domain setting value - api: GET /domains//settings - arguments: - domain: - help: Domain name - extra: - pattern: *pattern_domain - key: - help: Key to get/set - -v: - full: --value - help: Value to set - -d: - full: --delete - help: Delete the key - action: store_true - subcategories: + + config: + subcategory_help: Domain settings + actions: + + ### domain_config_get() + get: + action_help: Display a domain configuration + api: GET /domains//config + arguments: + domain: + help: Domain name + key: + help: A question or form key + nargs: '?' + + ### domain_config_set() + set: + action_help: Apply a new configuration + api: PUT /domains//config + arguments: + app: + help: Domain name + key: + help: The question or form key + nargs: '?' + -v: + full: --value + help: new value + -a: + full: --args + help: Serialized arguments for new configuration (i.e. "mail_in=0&mail_out=0") + registrar: subcategory_help: Manage domains registrars actions: diff --git a/data/other/config_domain.toml b/data/other/config_domain.toml new file mode 100644 index 000000000..4821aa53b --- /dev/null +++ b/data/other/config_domain.toml @@ -0,0 +1,38 @@ +version = "1.0" +i18n = "domain_config" + +[feature] + [feature.mail] + [feature.mail.mail_out] + type = "boolean" + default = true + + [feature.mail.mail_in] + type = "boolean" + default = true + + [feature.mail.backup_mx] + type = "tags" + pattern.regexp = "^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$" + pattern.error = "pattern_error" + default = [] + + [feature.xmpp] + [feature.mail.xmpp] + type = "boolean" + default = false + +[dns] + [dns.registrar] + [dns.registrar.unsupported] + ask = "DNS zone of this domain can't be auto-configured, you should do it manually." + type = "alert" + style = "info" + helpLink.href = "https://yunohost.org/dns_config" + helpLink.text = "How to configure manually my DNS zone" + + [dns.advanced] + [dns.advanced.ttl] + type = "number" + min = 0 + default = 3600 diff --git a/data/other/registrar_list.toml b/data/other/registrar_list.toml new file mode 100644 index 000000000..33d115075 --- /dev/null +++ b/data/other/registrar_list.toml @@ -0,0 +1,636 @@ +[aliyun] + [aliyun.auth_key_id] + type = "string" + redact = True + + [aliyun.auth_secret] + type = "password" + +[aurora] + [aurora.auth_api_key] + type = "string" + redact = True + + [aurora.auth_secret_key] + type = "password" + +[azure] + [azure.auth_client_id] + type = "string" + redact = True + + [azure.auth_client_secret] + type = "password" + + [azure.auth_tenant_id] + type = "string" + redact = True + + [azure.auth_subscription_id] + type = "string" + redact = True + + [azure.resource_group] + type = "string" + redact = True + +[cloudflare] + [cloudflare.auth_username] + type = "string" + redact = True + + [cloudflare.auth_token] + type = "string" + redact = True + + [cloudflare.zone_id] + type = "string" + redact = True + +[cloudns] + [cloudns.auth_id] + type = "string" + redact = True + + [cloudns.auth_subid] + type = "string" + redact = True + + [cloudns.auth_subuser] + type = "string" + redact = True + + [cloudns.auth_password] + type = "password" + + [cloudns.weight] + type = "number" + + [cloudns.port] + type = "number" +[cloudxns] + [cloudxns.auth_username] + type = "string" + redact = True + + [cloudxns.auth_token] + type = "string" + redact = True + +[conoha] + [conoha.auth_region] + type = "string" + redact = True + + [conoha.auth_token] + type = "string" + redact = True + + [conoha.auth_username] + type = "string" + redact = True + + [conoha.auth_password] + type = "password" + + [conoha.auth_tenant_id] + type = "string" + redact = True + +[constellix] + [constellix.auth_username] + type = "string" + redact = True + + [constellix.auth_token] + type = "string" + redact = True + +[digitalocean] + [digitalocean.auth_token] + type = "string" + redact = True + +[dinahosting] + [dinahosting.auth_username] + type = "string" + redact = True + + [dinahosting.auth_password] + type = "password" + +[directadmin] + [directadmin.auth_password] + type = "password" + + [directadmin.auth_username] + type = "string" + redact = True + + [directadmin.endpoint] + type = "string" + redact = True + +[dnsimple] + [dnsimple.auth_token] + type = "string" + redact = True + + [dnsimple.auth_username] + type = "string" + redact = True + + [dnsimple.auth_password] + type = "password" + + [dnsimple.auth_2fa] + type = "string" + redact = True + +[dnsmadeeasy] + [dnsmadeeasy.auth_username] + type = "string" + redact = True + + [dnsmadeeasy.auth_token] + type = "string" + redact = True + +[dnspark] + [dnspark.auth_username] + type = "string" + redact = True + + [dnspark.auth_token] + type = "string" + redact = True + +[dnspod] + [dnspod.auth_username] + type = "string" + redact = True + + [dnspod.auth_token] + type = "string" + redact = True + +[dreamhost] + [dreamhost.auth_token] + type = "string" + redact = True + +[dynu] + [dynu.auth_token] + type = "string" + redact = True + +[easydns] + [easydns.auth_username] + type = "string" + redact = True + + [easydns.auth_token] + type = "string" + redact = True + +[easyname] + [easyname.auth_username] + type = "string" + redact = True + + [easyname.auth_password] + type = "password" + +[euserv] + [euserv.auth_username] + type = "string" + redact = True + + [euserv.auth_password] + type = "password" + +[exoscale] + [exoscale.auth_key] + type = "string" + redact = True + + [exoscale.auth_secret] + type = "password" + +[gandi] + [gandi.auth_token] + type = "string" + redact = True + + [gandi.api_protocol] + type = "string" + choices.rpc = "RPC" + choices.rest = "REST" + +[gehirn] + [gehirn.auth_token] + type = "string" + redact = True + + [gehirn.auth_secret] + type = "password" + +[glesys] + [glesys.auth_username] + type = "string" + redact = True + + [glesys.auth_token] + type = "string" + redact = True + +[godaddy] + [godaddy.auth_key] + type = "string" + redact = True + + [godaddy.auth_secret] + type = "password" + +[googleclouddns] + [goggleclouddns.auth_service_account_info] + type = "string" + redact = True + +[gransy] + [gransy.auth_username] + type = "string" + redact = True + + [gransy.auth_password] + type = "password" + +[gratisdns] + [gratisdns.auth_username] + type = "string" + redact = True + + [gratisdns.auth_password] + type = "password" + +[henet] + [henet.auth_username] + type = "string" + redact = True + + [henet.auth_password] + type = "password" + +[hetzner] + [hetzner.auth_token] + type = "string" + redact = True + +[hostingde] + [hostingde.auth_token] + type = "string" + redact = True + +[hover] + [hover.auth_username] + type = "string" + redact = True + + [hover.auth_password] + type = "password" + +[infoblox] + [infoblox.auth_user] + type = "string" + redact = True + + [infoblox.auth_psw] + type = "password" + + [infoblox.ib_view] + type = "string" + redact = True + + [infoblox.ib_host] + type = "string" + redact = True + +[infomaniak] + [infomaniak.auth_token] + type = "string" + redact = True + +[internetbs] + [internetbs.auth_key] + type = "string" + redact = True + + [internetbs.auth_password] + type = "string" + redact = True + +[inwx] + [inwx.auth_username] + type = "string" + redact = True + + [inwx.auth_password] + type = "password" + +[joker] + [joker.auth_token] + type = "string" + redact = True + +[linode] + [linode.auth_token] + type = "string" + redact = True + +[linode4] + [linode4.auth_token] + type = "string" + redact = True + +[localzone] + [localzone.filename] + type = "string" + redact = True + +[luadns] + [luadns.auth_username] + type = "string" + redact = True + + [luadns.auth_token] + type = "string" + redact = True + +[memset] + [memset.auth_token] + type = "string" + redact = True + +[mythicbeasts] + [mythicbeasts.auth_username] + type = "string" + redact = True + + [mythicbeasts.auth_password] + type = "password" + + [mythicbeasts.auth_token] + type = "string" + redact = True + +[namecheap] + [namecheap.auth_token] + type = "string" + redact = True + + [namecheap.auth_username] + type = "string" + redact = True + + [namecheap.auth_client_ip] + type = "string" + redact = True + + [namecheap.auth_sandbox] + type = "string" + redact = True + +[namesilo] + [namesilo.auth_token] + type = "string" + redact = True + +[netcup] + [netcup.auth_customer_id] + type = "string" + redact = True + + [netcup.auth_api_key] + type = "string" + redact = True + + [netcup.auth_api_password] + type = "password" + +[nfsn] + [nfsn.auth_username] + type = "string" + redact = True + + [nfsn.auth_token] + type = "string" + redact = True + +[njalla] + [njalla.auth_token] + type = "string" + redact = True + +[nsone] + [nsone.auth_token] + type = "string" + redact = True + +[onapp] + [onapp.auth_username] + type = "string" + redact = True + + [onapp.auth_token] + type = "string" + redact = True + + [onapp.auth_server] + type = "string" + redact = True + +[online] + [online.auth_token] + type = "string" + redact = True + +[ovh] + [ovh.auth_entrypoint] + type = "string" + redact = True + + [ovh.auth_application_key] + type = "string" + redact = True + + [ovh.auth_application_secret] + type = "password" + + [ovh.auth_consumer_key] + type = "string" + redact = True + +[plesk] + [plesk.auth_username] + type = "string" + redact = True + + [plesk.auth_password] + type = "password" + + [plesk.plesk_server] + type = "string" + redact = True + +[pointhq] + [pointhq.auth_username] + type = "string" + redact = True + + [pointhq.auth_token] + type = "string" + redact = True + +[powerdns] + [powerdns.auth_token] + type = "string" + redact = True + + [powerdns.pdns_server] + type = "string" + redact = True + + [powerdns.pdns_server_id] + type = "string" + redact = True + + [powerdns.pdns_disable_notify] + type = "boolean" + +[rackspace] + [rackspace.auth_account] + type = "string" + redact = True + + [rackspace.auth_username] + type = "string" + redact = True + + [rackspace.auth_api_key] + type = "string" + redact = True + + [rackspace.auth_token] + type = "string" + redact = True + + [rackspace.sleep_time] + type = "string" + redact = True + +[rage4] + [rage4.auth_username] + type = "string" + redact = True + + [rage4.auth_token] + type = "string" + redact = True + +[rcodezero] + [rcodezero.auth_token] + type = "string" + redact = True + +[route53] + [route53.auth_access_key] + type = "string" + redact = True + + [route53.auth_access_secret] + type = "password" + + [route53.private_zone] + type = "string" + redact = True + + [route53.auth_username] + type = "string" + redact = True + + [route53.auth_token] + type = "string" + redact = True + +[safedns] + [safedns.auth_token] + type = "string" + redact = True + +[sakuracloud] + [sakuracloud.auth_token] + type = "string" + redact = True + + [sakuracloud.auth_secret] + type = "password" + +[softlayer] + [softlayer.auth_username] + type = "string" + redact = True + + [softlayer.auth_api_key] + type = "string" + redact = True + +[transip] + [transip.auth_username] + type = "string" + redact = True + + [transip.auth_api_key] + type = "string" + redact = True + +[ultradns] + [ultradns.auth_token] + type = "string" + redact = True + + [ultradns.auth_username] + type = "string" + redact = True + + [ultradns.auth_password] + type = "password" + +[vultr] + [vultr.auth_token] + type = "string" + redact = True + +[yandex] + [yandex.auth_token] + type = "string" + redact = True + +[zeit] + [zeit.auth_token] + type = "string" + redact = True + +[zilore] + [zilore.auth_key] + type = "string" + redact = True + +[zonomi] + [zonomy.auth_token] + type = "string" + redact = True + + [zonomy.auth_entrypoint] + type = "string" + redact = True + diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index d05e31f17..59ad68979 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -387,115 +387,58 @@ def _get_maindomain(): return maindomain -def _default_domain_settings(domain): - from yunohost.utils.dns import get_dns_zone_from_domain - return { - "xmpp": domain == domain_list()["main"], - "mail_in": True, - "mail_out": True, - "dns_zone": get_dns_zone_from_domain(domain), - "ttl": 3600, - } - - def _get_domain_settings(domain): """ Retrieve entries in /etc/yunohost/domains/[domain].yml And set default values if needed """ - _assert_domain_exists(domain) - - # Retrieve entries in the YAML - filepath = f"{DOMAIN_SETTINGS_DIR}/{domain}.yml" - on_disk_settings = {} - if os.path.exists(filepath) and os.path.isfile(filepath): - on_disk_settings = read_yaml(filepath) or {} - - # Inject defaults if needed (using the magic .update() ;)) - settings = _default_domain_settings(domain) - settings.update(on_disk_settings) - return settings + config = DomainConfigPanel(domain) + return config.get(mode='export') -def domain_setting(domain, key, value=None, delete=False): +def domain_config_get(domain, key='', mode='classic'): """ - Set or get an app setting value - - Keyword argument: - domain -- Domain Name - key -- Key to get/set - value -- Value to set - delete -- Delete the key - + Display a domain configuration """ - domain_settings = _get_domain_settings(domain) + config = DomainConfigPanel(domain) + return config.get(key, mode) - # GET - if value is None and not delete: - if key not in domain_settings: - raise YunohostValidationError("domain_property_unknown", property=key) - - return domain_settings[key] - - # DELETE - if delete: - if key in domain_settings: - del domain_settings[key] - _set_domain_settings(domain, domain_settings) - - # SET - else: - # FIXME : in the future, implement proper setting types (+ defaults), - # maybe inspired from the global settings - - if key in ["mail_in", "mail_out", "xmpp"]: - _is_boolean, value = is_boolean(value) - if not _is_boolean: - raise YunohostValidationError( - "global_settings_bad_type_for_setting", - setting=key, - received_type="not boolean", - expected_type="boolean", - ) - - if "ttl" == key: - try: - value = int(value) - except ValueError: - # TODO add locales - raise YunohostValidationError("invalid_number", value_type=type(value)) - - if value < 0: - raise YunohostValidationError("pattern_positive_number", value_type=type(value)) - - # Set new value - domain_settings[key] = value - # Save settings - _set_domain_settings(domain, domain_settings) - - -def _set_domain_settings(domain, domain_settings): +@is_unit_operation() +def domain_config_set(operation_logger, app, key=None, value=None, args=None, args_file=None): """ - Set settings of a domain - - Keyword arguments: - domain -- The domain name - settings -- Dict with domain settings - + Apply a new domain configuration """ - _assert_domain_exists(domain) + config = DomainConfigPanel(domain) + return config.set(key, value, args, args_file) - defaults = _default_domain_settings(domain) - diff_with_defaults = {k: v for k, v in domain_settings.items() if defaults.get(k) != v} +class DomainConfigPanel(ConfigPanel): + def __init__(domain): + _assert_domain_exist(domain) + self.domain = domain + super().__init( + config_path=DOMAIN_CONFIG_PATH.format(domain=domain), + save_path=DOMAIN_SETTINGS_PATH.format(domain=domain) + ) - # First create the DOMAIN_SETTINGS_DIR if it doesn't exist - if not os.path.exists(DOMAIN_SETTINGS_DIR): - mkdir(DOMAIN_SETTINGS_DIR, mode=0o700) - # Save the settings to the .yaml file - filepath = f"{DOMAIN_SETTINGS_DIR}/{domain}.yml" - write_to_yaml(filepath, diff_with_defaults) + def _get_toml(self): + from yunohost.utils.dns import get_dns_zone_from_domain + toml = super()._get_toml() + self.dns_zone = get_dns_zone_from_domain(self.domain) + + try: + registrar = _relevant_provider_for_domain(self.dns_zone) + except ValueError: + return toml + + registrar_list = read_toml("/usr/share/yunohost/other/registrar_list.toml") + toml['dns']['registrar'] = registrar_list[registrar] + return toml + + def _load_current_values(): + # TODO add mechanism to share some settings with other domains on the same zone + super()._load_current_values() # # diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 34883dcf7..b3ef34c17 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -46,6 +46,7 @@ logger = getActionLogger("yunohost.config") CONFIG_PANEL_VERSION_SUPPORTED = 1.0 class ConfigPanel: + save_mode = "diff" def __init__(self, config_path, save_path=None): self.config_path = config_path @@ -56,6 +57,7 @@ class ConfigPanel: def get(self, key='', mode='classic'): self.filter_key = key or '' + self.mode = mode # Read config panel toml self._get_config_panel() @@ -273,13 +275,39 @@ class ConfigPanel: )) self.new_values = {key: str(value[0]) for key, value in self.new_values.items() if not value[0] is None} + def _get_default_values(self): + return { key: option['default'] + for _, _, option in self._iterate() if 'default' in option } + + def _load_current_values(self): + """ + Retrieve entries in YAML file + And set default values if needed + """ + + # Retrieve entries in the YAML + on_disk_settings = {} + if os.path.exists(self.save_path) and os.path.isfile(self.save_path): + on_disk_settings = read_yaml(self.save_path) or {} + + # Inject defaults if needed (using the magic .update() ;)) + self.values = self._get_default_values(self) + self.values.update(on_disk_settings) + def _apply(self): logger.info("Running config script...") dir_path = os.path.dirname(os.path.realpath(self.save_path)) if not os.path.exists(dir_path): mkdir(dir_path, mode=0o700) + + if self.save_mode == 'diff': + defaults = self._get_default_values() + values_to_save = {k: v for k, v in values.items() if defaults.get(k) != v} + else: + values_to_save = {**self.values, **self.new_values} + # Save the settings to the .yaml file - write_to_yaml(self.save_path, self.new_values) + write_to_yaml(self.save_path, values_to_save) def _reload_services(self): From a0f471065ce2d7637aa118c261dd1c8988c99e7c Mon Sep 17 00:00:00 2001 From: ljf Date: Fri, 3 Sep 2021 19:05:58 +0200 Subject: [PATCH 067/130] [fix] Fix yunohost domain config get --- data/actionsmap/yunohost.yml | 173 ++++++++++++++++++++-------------- data/other/config_domain.toml | 7 +- src/yunohost/dns.py | 65 ------------- src/yunohost/domain.py | 37 +++----- src/yunohost/utils/config.py | 8 +- 5 files changed, 124 insertions(+), 166 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index ea4a1f577..0da811522 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -460,6 +460,7 @@ domain: ### domain_dns_conf() dns-conf: + deprecated: true action_help: Generate sample DNS configuration for a domain api: GET /domains//dns arguments: @@ -468,17 +469,6 @@ domain: extra: pattern: *pattern_domain - ### domain_push_config() - push-config: - action_help: Push DNS records to registrar - api: GET /domains//push - arguments: - domain: - help: Domain name to add - extra: - pattern: *pattern_domain - - ### domain_maindomain() main-domain: action_help: Check the current main domain, or change it @@ -496,6 +486,7 @@ domain: ### certificate_status() cert-status: + deprecated: true action_help: List status of current certificates (all by default). api: GET /domains//cert arguments: @@ -508,6 +499,7 @@ domain: ### certificate_install() cert-install: + deprecated: true action_help: Install Let's Encrypt certificates for given domains (all by default). api: PUT /domains//cert arguments: @@ -529,6 +521,7 @@ domain: ### certificate_renew() cert-renew: + deprecated: true action_help: Renew the Let's Encrypt certificates for given domains (all by default). api: PUT /domains//cert/renew arguments: @@ -562,76 +555,57 @@ domain: subcategories: - config: - subcategory_help: Domain settings - actions: - - ### domain_config_get() - get: - action_help: Display a domain configuration - api: GET /domains//config - arguments: - domain: - help: Domain name - key: - help: A question or form key - nargs: '?' - - ### domain_config_set() - set: - action_help: Apply a new configuration - api: PUT /domains//config - arguments: - app: - help: Domain name - key: - help: The question or form key - nargs: '?' - -v: - full: --value - help: new value - -a: - full: --args - help: Serialized arguments for new configuration (i.e. "mail_in=0&mail_out=0") - - registrar: - subcategory_help: Manage domains registrars + config: + subcategory_help: Domain settings actions: - ### domain_registrar_catalog() - catalog: - action_help: List supported registrars API - api: GET /domains/registrars/catalog + ### domain_config_get() + get: + action_help: Display a domain configuration + api: GET /domains//config + arguments: + domain: + help: Domain name + key: + help: A question or form key + nargs: '?' - ### domain_registrar_set() + ### domain_config_set() set: - action_help: Set domain registrar - api: POST /domains//registrar + action_help: Apply a new configuration + api: PUT /domains//config + arguments: + app: + help: Domain name + key: + help: The question or form key + nargs: '?' + -v: + full: --value + help: new value + -a: + full: --args + help: Serialized arguments for new configuration (i.e. "mail_in=0&mail_out=0") + + dns: + subcategory_help: Manage domains DNS + actions: + ### domain_dns_conf() + suggest: + action_help: Generate sample DNS configuration for a domain + api: + - GET /domains//dns + - GET /domains//dns/suggest arguments: domain: - help: Domain name + help: Target domain extra: pattern: *pattern_domain - registrar: - help: registrar_key, see yunohost domain registrar list - -a: - full: --args - help: Serialized arguments for registrar API (i.e. "auth_token=TOKEN&auth_username=USER"). - - ### domain_registrar_info() - info: - action_help: Display info about registrar settings used for a domain - api: GET /domains//registrar - arguments: - domain: - help: Domain name - extra: - pattern: *pattern_domain - - ### domain_registrar_push() + + ### domain_dns_push() push: action_help: Push DNS records to registrar - api: PUT /domains//registrar/push + api: POST /domains//dns/push arguments: domain: help: Domain name to push DNS conf for @@ -642,6 +616,63 @@ domain: help: Only display what's to be pushed action: store_true + cert: + subcategory_help: Manage domains DNS + actions: + ### certificate_status() + status: + action_help: List status of current certificates (all by default). + api: GET /domains//cert + arguments: + domain_list: + help: Domains to check + nargs: "*" + --full: + help: Show more details + action: store_true + + ### certificate_install() + install: + action_help: Install Let's Encrypt certificates for given domains (all by default). + api: PUT /domains//cert + arguments: + domain_list: + help: Domains for which to install the certificates + nargs: "*" + --force: + help: Install even if current certificate is not self-signed + action: store_true + --no-checks: + help: Does not perform any check that your domain seems correctly configured (DNS, reachability) before attempting to install. (Not recommended) + action: store_true + --self-signed: + help: Install self-signed certificate instead of Let's Encrypt + action: store_true + --staging: + help: Use the fake/staging Let's Encrypt certification authority. The new certificate won't actually be enabled - it is only intended to test the main steps of the procedure. + action: store_true + + ### certificate_renew() + renew: + action_help: Renew the Let's Encrypt certificates for given domains (all by default). + api: PUT /domains//cert/renew + arguments: + domain_list: + help: Domains for which to renew the certificates + nargs: "*" + --force: + help: Ignore the validity threshold (30 days) + action: store_true + --email: + help: Send an email to root with logs if some renewing fails + action: store_true + --no-checks: + help: Does not perform any check that your domain seems correctly configured (DNS, reachability) before attempting to renew. (Not recommended) + action: store_true + --staging: + help: Use the fake/staging Let's Encrypt certification authority. The new certificate won't actually be enabled - it is only intended to test the main steps of the procedure. + action: store_true + ############################# # App # diff --git a/data/other/config_domain.toml b/data/other/config_domain.toml index 4821aa53b..b2211417d 100644 --- a/data/other/config_domain.toml +++ b/data/other/config_domain.toml @@ -13,12 +13,13 @@ i18n = "domain_config" [feature.mail.backup_mx] type = "tags" - pattern.regexp = "^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$" - pattern.error = "pattern_error" default = [] + pattern.regexp = '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$' + pattern.error = "pattern_error" [feature.xmpp] - [feature.mail.xmpp] + + [feature.xmpp.xmpp] type = "boolean" default = false diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 41c6e73f1..df3b578db 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -39,9 +39,6 @@ from yunohost.hook import hook_callback logger = getActionLogger("yunohost.domain") -REGISTRAR_SETTINGS_DIR = "/etc/yunohost/registrars" -REGISTRAR_LIST_PATH = "/usr/share/yunohost/other/registrar_list.yml" - def domain_dns_conf(domain): """ @@ -389,68 +386,6 @@ def _get_DKIM(domain): ) -def _get_registrar_settings(dns_zone): - on_disk_settings = {} - filepath = f"{REGISTRAR_SETTINGS_DIR}/{dns_zone}.yml" - if os.path.exists(filepath) and os.path.isfile(filepath): - on_disk_settings = read_yaml(filepath) or {} - - return on_disk_settings - - -def _set_registrar_settings(dns_zone, domain_registrar): - if not os.path.exists(REGISTRAR_SETTINGS_DIR): - mkdir(REGISTRAR_SETTINGS_DIR, mode=0o700) - filepath = f"{REGISTRAR_SETTINGS_DIR}/{dns_zone}.yml" - write_to_yaml(filepath, domain_registrar) - - -def domain_registrar_info(domain): - - dns_zone = _get_domain_settings(domain)["dns_zone"] - registrar_info = _get_registrar_settings(dns_zone) - if not registrar_info: - raise YunohostValidationError("registrar_is_not_set", dns_zone=dns_zone) - - return registrar_info - - -def domain_registrar_catalog(): - return read_yaml(REGISTRAR_LIST_PATH) - - -def domain_registrar_set(domain, registrar, args): - - _assert_domain_exists(domain) - - registrars = read_yaml(REGISTRAR_LIST_PATH) - if registrar not in registrars.keys(): - raise YunohostValidationError("domain_registrar_unknown", registrar=registrar) - - parameters = registrars[registrar] - ask_args = [] - for parameter in parameters: - ask_args.append( - { - "name": parameter, - "type": "string", - "example": "", - "default": "", - } - ) - args_dict = ( - {} if not args else dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) - ) - parsed_answer_dict = _parse_args_in_yunohost_format(args_dict, ask_args) - - domain_registrar = {"name": registrar, "options": {}} - for arg_name, arg_value_and_type in parsed_answer_dict.items(): - domain_registrar["options"][arg_name] = arg_value_and_type[0] - - dns_zone = _get_domain_settings(domain)["dns_zone"] - _set_registrar_settings(dns_zone, domain_registrar) - - @is_unit_operation() def domain_registrar_push(operation_logger, domain, dry_run=False): """ diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 59ad68979..064311a49 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -38,14 +38,16 @@ from yunohost.app import ( _get_conflicting_apps, ) from yunohost.regenconf import regen_conf, _force_clear_hashes, _process_regen_conf +from yunohost.utils.config import ConfigPanel from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.log import is_unit_operation from yunohost.hook import hook_callback logger = getActionLogger("yunohost.domain") +DOMAIN_CONFIG_PATH = "/usr/share/yunohost/other/config_domain.toml" DOMAIN_SETTINGS_DIR = "/etc/yunohost/domains" - +DOMAIN_REGISTRAR_LIST_PATH = "/usr/share/yunohost/other/registrar_list.toml" # Lazy dev caching to avoid re-query ldap every time we need the domain list domain_list_cache = {} @@ -413,16 +415,18 @@ def domain_config_set(operation_logger, app, key=None, value=None, args=None, ar config = DomainConfigPanel(domain) return config.set(key, value, args, args_file) + class DomainConfigPanel(ConfigPanel): - def __init__(domain): - _assert_domain_exist(domain) + def __init__(self, domain): + _assert_domain_exists(domain) self.domain = domain - super().__init( - config_path=DOMAIN_CONFIG_PATH.format(domain=domain), - save_path=DOMAIN_SETTINGS_PATH.format(domain=domain) + super().__init__( + config_path=DOMAIN_CONFIG_PATH, + save_path=f"{DOMAIN_SETTINGS_DIR}/{domain}.yml" ) def _get_toml(self): + from lexicon.providers.auto import _relevant_provider_for_domain from yunohost.utils.dns import get_dns_zone_from_domain toml = super()._get_toml() self.dns_zone = get_dns_zone_from_domain(self.domain) @@ -432,11 +436,11 @@ class DomainConfigPanel(ConfigPanel): except ValueError: return toml - registrar_list = read_toml("/usr/share/yunohost/other/registrar_list.toml") + registrar_list = read_toml(DOMAIN_REGISTRAR_LIST_PATH) toml['dns']['registrar'] = registrar_list[registrar] return toml - def _load_current_values(): + def _load_current_values(self): # TODO add mechanism to share some settings with other domains on the same zone super()._load_current_values() @@ -478,21 +482,6 @@ def domain_dns_conf(domain): return yunohost.dns.domain_dns_conf(domain) -def domain_registrar_catalog(): - import yunohost.dns - return yunohost.dns.domain_registrar_catalog() - - -def domain_registrar_set(domain, registrar, args): - import yunohost.dns - return yunohost.dns.domain_registrar_set(domain, registrar, args) - - -def domain_registrar_info(domain): - import yunohost.dns - return yunohost.dns.domain_registrar_info(domain) - - -def domain_registrar_push(domain, dry_run): +def domain_dns_push(domain, dry_run): import yunohost.dns return yunohost.dns.domain_registrar_push(domain, dry_run) diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index b3ef34c17..b8ba489e6 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -85,8 +85,10 @@ class ConfigPanel: key = f"{panel['id']}.{section['id']}.{option['id']}" if mode == 'export': result[option['id']] = option.get('current_value') - else: + elif 'ask' in option: result[key] = { 'ask': _value_for_locale(option['ask']) } + elif 'i18n' in self.config: + result[key] = { 'ask': m18n.n(self.config['i18n'] + '_' + option['id']) } if 'current_value' in option: result[key]['value'] = option['current_value'] @@ -276,7 +278,7 @@ class ConfigPanel: self.new_values = {key: str(value[0]) for key, value in self.new_values.items() if not value[0] is None} def _get_default_values(self): - return { key: option['default'] + return { option['id']: option['default'] for _, _, option in self._iterate() if 'default' in option } def _load_current_values(self): @@ -291,7 +293,7 @@ class ConfigPanel: on_disk_settings = read_yaml(self.save_path) or {} # Inject defaults if needed (using the magic .update() ;)) - self.values = self._get_default_values(self) + self.values = self._get_default_values() self.values.update(on_disk_settings) def _apply(self): From 666ebdd8d43dbf6cb4d8130555cc051abf68bdd4 Mon Sep 17 00:00:00 2001 From: ljf Date: Sat, 4 Sep 2021 19:03:07 +0200 Subject: [PATCH 068/130] [enh] yes/no args on boolean aquestion + semantic --- src/yunohost/utils/config.py | 540 +++++++++++++++++++---------------- 1 file changed, 287 insertions(+), 253 deletions(-) diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index b8ba489e6..380da1027 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -90,7 +90,8 @@ class ConfigPanel: elif 'i18n' in self.config: result[key] = { 'ask': m18n.n(self.config['i18n'] + '_' + option['id']) } if 'current_value' in option: - result[key]['value'] = option['current_value'] + question_class = ARGUMENTS_TYPE_PARSERS[option.get("type", "string")] + result[key]['value'] = question_class.humanize(option['current_value'], option) return result @@ -144,7 +145,7 @@ class ConfigPanel: raise finally: # Delete files uploaded from API - FileArgumentParser.clean_upload_dirs() + FileQuestion.clean_upload_dirs() if self.errors: return { @@ -275,7 +276,8 @@ class ConfigPanel: self.new_values.update(parse_args_in_yunohost_format( self.args, section['options'] )) - self.new_values = {key: str(value[0]) for key, value in self.new_values.items() if not value[0] is None} + self.new_values = {key: value[0] for key, value in self.new_values.items() if not value[0] is None} + self.errors = None def _get_default_values(self): return { option['id']: option['default'] @@ -297,31 +299,31 @@ class ConfigPanel: self.values.update(on_disk_settings) def _apply(self): - logger.info("Running config script...") + logger.info("Saving the new configuration...") dir_path = os.path.dirname(os.path.realpath(self.save_path)) if not os.path.exists(dir_path): mkdir(dir_path, mode=0o700) + values_to_save = {**self.values, **self.new_values} if self.save_mode == 'diff': defaults = self._get_default_values() - values_to_save = {k: v for k, v in values.items() if defaults.get(k) != v} - else: - values_to_save = {**self.values, **self.new_values} + values_to_save = {k: v for k, v in values_to_save.items() if defaults.get(k) != v} # Save the settings to the .yaml file write_to_yaml(self.save_path, values_to_save) def _reload_services(self): - logger.info("Reloading services...") services_to_reload = set() for panel, section, obj in self._iterate(['panel', 'section', 'option']): services_to_reload |= set(obj.get('services', [])) services_to_reload = list(services_to_reload) services_to_reload.sort(key = 'nginx'.__eq__) + if services_to_reload: + logger.info("Reloading services...") for service in services_to_reload: - if '__APP__': + if '__APP__' in service: service = service.replace('__APP__', self.app) logger.debug(f"Reloading {service}") if not _run_service_command('reload-or-restart', service): @@ -345,141 +347,140 @@ class ConfigPanel: yield (panel, section, option) -class Question: - "empty class to store questions information" - - -class YunoHostArgumentFormatParser(object): +class Question(object): hide_user_input_in_prompt = False operation_logger = None - def parse_question(self, question, user_answers): - parsed_question = Question() - - parsed_question.name = question["name"] - parsed_question.type = question.get("type", 'string') - parsed_question.default = question.get("default", None) - parsed_question.current_value = question.get("current_value") - parsed_question.optional = question.get("optional", False) - parsed_question.choices = question.get("choices", []) - parsed_question.pattern = question.get("pattern") - parsed_question.ask = question.get("ask", {'en': f"{parsed_question.name}"}) - parsed_question.help = question.get("help") - parsed_question.helpLink = question.get("helpLink") - parsed_question.value = user_answers.get(parsed_question.name) - parsed_question.redact = question.get('redact', False) + def __init__(self, question, user_answers): + self.name = question["name"] + self.type = question.get("type", 'string') + self.default = question.get("default", None) + self.current_value = question.get("current_value") + self.optional = question.get("optional", False) + self.choices = question.get("choices", []) + self.pattern = question.get("pattern") + self.ask = question.get("ask", {'en': f"{self.name}"}) + self.help = question.get("help") + self.helpLink = question.get("helpLink") + self.value = user_answers.get(self.name) + self.redact = question.get('redact', False) # Empty value is parsed as empty string - if parsed_question.default == "": - parsed_question.default = None + if self.default == "": + self.default = None - return parsed_question + @staticmethod + def humanize(value, option={}): + return str(value) - def parse(self, question, user_answers): - question = self.parse_question(question, user_answers) + @staticmethod + def normalize(value, option={}): + return value + + def ask_if_needed(self): while True: # Display question if no value filled or if it's a readonly message if Moulinette.interface.type== 'cli': - text_for_user_input_in_cli = self._format_text_for_user_input_in_cli( - question - ) + text_for_user_input_in_cli = self._format_text_for_user_input_in_cli() if getattr(self, "readonly", False): Moulinette.display(text_for_user_input_in_cli) - elif question.value is None: + elif self.value is None: prefill = "" - if question.current_value is not None: - prefill = question.current_value - elif question.default is not None: - prefill = question.default - question.value = Moulinette.prompt( + if self.current_value is not None: + prefill = self.humanize(self.current_value, self) + elif self.default is not None: + prefill = self.default + self.value = Moulinette.prompt( message=text_for_user_input_in_cli, is_password=self.hide_user_input_in_prompt, confirm=self.hide_user_input_in_prompt, prefill=prefill, - is_multiline=(question.type == "text") + is_multiline=(self.type == "text") ) + # Normalization + # This is done to enforce a certain formating like for boolean + self.value = self.normalize(self.value, self) # Apply default value - if question.value in [None, ""] and question.default is not None: - question.value = ( + if self.value in [None, ""] and self.default is not None: + self.value = ( getattr(self, "default_value", None) - if question.default is None - else question.default + if self.default is None + else self.default ) # Prevalidation try: - self._prevalidate(question) + self._prevalidate() except YunohostValidationError as e: if Moulinette.interface.type== 'api': raise Moulinette.display(str(e), 'error') - question.value = None + self.value = None continue break - # this is done to enforce a certain formating like for boolean - # by default it doesn't do anything - question.value = self._post_parse_value(question) + self.value = self._post_parse_value() - return (question.value, self.argument_type) + return (self.value, self.argument_type) - def _prevalidate(self, question): - if question.value in [None, ""] and not question.optional: + + def _prevalidate(self): + if self.value in [None, ""] and not self.optional: raise YunohostValidationError( - "app_argument_required", name=question.name + "app_argument_required", name=self.name ) # we have an answer, do some post checks - if question.value is not None: - if question.choices and question.value not in question.choices: - self._raise_invalid_answer(question) - if question.pattern and not re.match(question.pattern['regexp'], str(question.value)): + if self.value is not None: + if self.choices and self.value not in self.choices: + self._raise_invalid_answer(self) + if self.pattern and not re.match(self.pattern['regexp'], str(self.value)): raise YunohostValidationError( - question.pattern['error'], - name=question.name, - value=question.value, + self.pattern['error'], + name=self.name, + value=self.value, ) - def _raise_invalid_answer(self, question): + def _raise_invalid_answer(self): raise YunohostValidationError( "app_argument_choice_invalid", - name=question.name, - value=question.value, - choices=", ".join(question.choices), + name=self.name, + value=self.value, + choices=", ".join(self.choices), ) - def _format_text_for_user_input_in_cli(self, question): - text_for_user_input_in_cli = _value_for_locale(question.ask) + def _format_text_for_user_input_in_cli(self): + text_for_user_input_in_cli = _value_for_locale(self.ask) - if question.choices: - text_for_user_input_in_cli += " [{0}]".format(" | ".join(question.choices)) + if self.choices: + text_for_user_input_in_cli += " [{0}]".format(" | ".join(self.choices)) - if question.help or question.helpLink: + if self.help or self.helpLink: text_for_user_input_in_cli += ":\033[m" - if question.help: + if self.help: text_for_user_input_in_cli += "\n - " - text_for_user_input_in_cli += _value_for_locale(question.help) - if question.helpLink: - if not isinstance(question.helpLink, dict): - question.helpLink = {'href': question.helpLink} - text_for_user_input_in_cli += f"\n - See {question.helpLink['href']}" + text_for_user_input_in_cli += _value_for_locale(self.help) + if self.helpLink: + if not isinstance(self.helpLink, dict): + self.helpLink = {'href': self.helpLink} + text_for_user_input_in_cli += f"\n - See {self.helpLink['href']}" return text_for_user_input_in_cli - def _post_parse_value(self, question): - if not question.redact: - return question.value + def _post_parse_value(self): + if not self.redact: + return self.value # Tell the operation_logger to redact all password-type / secret args # Also redact the % escaped version of the password that might appear in # the 'args' section of metadata (relevant for password with non-alphanumeric char) data_to_redact = [] - if question.value and isinstance(question.value, str): - data_to_redact.append(question.value) - if question.current_value and isinstance(question.current_value, str): - data_to_redact.append(question.current_value) + if self.value and isinstance(self.value, str): + data_to_redact.append(self.value) + if self.current_value and isinstance(self.current_value, str): + data_to_redact.append(self.current_value) data_to_redact += [ urllib.parse.quote(data) for data in data_to_redact @@ -488,50 +489,60 @@ class YunoHostArgumentFormatParser(object): if self.operation_logger: self.operation_logger.data_to_redact.extend(data_to_redact) elif data_to_redact: - raise YunohostError("app_argument_cant_redact", arg=question.name) + raise YunohostError("app_argument_cant_redact", arg=self.name) - return question.value + return self.value -class StringArgumentParser(YunoHostArgumentFormatParser): +class StringQuestion(Question): argument_type = "string" default_value = "" -class TagsArgumentParser(YunoHostArgumentFormatParser): +class TagsQuestion(Question): argument_type = "tags" - def _prevalidate(self, question): - values = question.value - for value in values.split(','): - question.value = value - super()._prevalidate(question) - question.value = values + @staticmethod + def humanize(value, option={}): + if isinstance(value, list): + return ','.join(value) + return value + + def _prevalidate(self): + values = self.value + if isinstance(values, str): + values = values.split(',') + for value in values: + self.value = value + super()._prevalidate() + self.value = values -class PasswordArgumentParser(YunoHostArgumentFormatParser): +class PasswordQuestion(Question): hide_user_input_in_prompt = True argument_type = "password" default_value = "" forbidden_chars = "{}" - def parse_question(self, question, user_answers): - question = super(PasswordArgumentParser, self).parse_question( - question, user_answers - ) - question.redact = True - if question.default is not None: + def __init__(self, question, user_answers): + super().__init__(question, user_answers) + self.redact = True + if self.default is not None: raise YunohostValidationError( - "app_argument_password_no_default", name=question.name + "app_argument_password_no_default", name=self.name ) - return question + @staticmethod + def humanize(value, option={}): + if value: + return '***' # Avoid to display the password on screen + return "" - def _prevalidate(self, question): - super()._prevalidate(question) + def _prevalidate(self): + super()._prevalidate() - if question.value is not None: - if any(char in question.value for char in self.forbidden_chars): + if self.value is not None: + if any(char in self.value for char in self.forbidden_chars): raise YunohostValidationError( "pattern_password_app", forbidden_chars=self.forbidden_chars ) @@ -539,181 +550,206 @@ class PasswordArgumentParser(YunoHostArgumentFormatParser): # If it's an optional argument the value should be empty or strong enough from yunohost.utils.password import assert_password_is_strong_enough - assert_password_is_strong_enough("user", question.value) + assert_password_is_strong_enough("user", self.value) -class PathArgumentParser(YunoHostArgumentFormatParser): +class PathQuestion(Question): argument_type = "path" default_value = "" -class BooleanArgumentParser(YunoHostArgumentFormatParser): +class BooleanQuestion(Question): argument_type = "boolean" default_value = False + yes_answers = ["1", "yes", "y", "true", "t", "on"] + no_answers = ["0", "no", "n", "false", "f", "off"] - def parse_question(self, question, user_answers): - question = super().parse_question( - question, user_answers + @staticmethod + def humanize(value, option={}): + yes = option.get('yes', 1) + no = option.get('no', 0) + value = str(value).lower() + if value == str(yes).lower(): + return 'yes' + if value == str(no).lower(): + return 'no' + if value in BooleanQuestion.yes_answers: + return 'yes' + if value in BooleanQuestion.no_answers: + return 'no' + + if value in ['none', ""]: + return '' + + raise YunohostValidationError( + "app_argument_choice_invalid", + name=self.name, + value=self.value, + choices="yes, no, y, n, 1, 0", ) - if question.default is None: - question.default = False + @staticmethod + def normalize(value, option={}): + yes = option.get('yes', 1) + no = option.get('no', 0) - return question + if str(value).lower() in BooleanQuestion.yes_answers: + return yes - def _format_text_for_user_input_in_cli(self, question): - text_for_user_input_in_cli = _value_for_locale(question.ask) + if str(value).lower() in BooleanQuestion.no_answers: + return no + + if value in [None, ""]: + return None + raise YunohostValidationError( + "app_argument_choice_invalid", + name=self.name, + value=self.value, + choices="yes, no, y, n, 1, 0", + ) + + def __init__(self, question, user_answers): + super().__init__(question, user_answers) + self.yes = question.get('yes', 1) + self.no = question.get('no', 0) + if self.default is None: + self.default = False + + + def _format_text_for_user_input_in_cli(self): + text_for_user_input_in_cli = _value_for_locale(self.ask) text_for_user_input_in_cli += " [yes | no]" - if question.default is not None: - formatted_default = "yes" if question.default else "no" + if self.default is not None: + formatted_default = self.humanize(self.default) text_for_user_input_in_cli += " (default: {0})".format(formatted_default) return text_for_user_input_in_cli - def _post_parse_value(self, question): - if isinstance(question.value, bool): - return 1 if question.value else 0 - - if str(question.value).lower() in ["1", "yes", "y", "true"]: - return 1 - - if str(question.value).lower() in ["0", "no", "n", "false"]: - return 0 - - raise YunohostValidationError( - "app_argument_choice_invalid", - name=question.name, - value=question.value, - choices="yes, no, y, n, 1, 0", - ) + def get(self, key, default=None): + try: + return getattr(self, key) + except AttributeError: + return default -class DomainArgumentParser(YunoHostArgumentFormatParser): + +class DomainQuestion(Question): argument_type = "domain" - def parse_question(self, question, user_answers): + def __init__(self, question, user_answers): from yunohost.domain import domain_list, _get_maindomain - question = super(DomainArgumentParser, self).parse_question( - question, user_answers - ) + super().__init__(question, user_answers) - if question.default is None: - question.default = _get_maindomain() + if self.default is None: + self.default = _get_maindomain() - question.choices = domain_list()["domains"] + self.choices = domain_list()["domains"] - return question - def _raise_invalid_answer(self, question): + def _raise_invalid_answer(self): raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("domain_unknown") + "app_argument_invalid", field=self.name, error=m18n.n("domain_unknown") ) -class UserArgumentParser(YunoHostArgumentFormatParser): +class UserQuestion(Question): argument_type = "user" - def parse_question(self, question, user_answers): + def __init__(self, question, user_answers): from yunohost.user import user_list, user_info from yunohost.domain import _get_maindomain - question = super(UserArgumentParser, self).parse_question( - question, user_answers - ) - question.choices = user_list()["users"] - if question.default is None: + super().__init__(question, user_answers) + self.choices = user_list()["users"] + if self.default is None: root_mail = "root@%s" % _get_maindomain() - for user in question.choices.keys(): + for user in self.choices.keys(): if root_mail in user_info(user).get("mail-aliases", []): - question.default = user + self.default = user break - return question - def _raise_invalid_answer(self, question): + def _raise_invalid_answer(self): raise YunohostValidationError( "app_argument_invalid", - field=question.name, - error=m18n.n("user_unknown", user=question.value), + field=self.name, + error=m18n.n("user_unknown", user=self.value), ) -class NumberArgumentParser(YunoHostArgumentFormatParser): +class NumberQuestion(Question): argument_type = "number" default_value = "" - def parse_question(self, question, user_answers): - question_parsed = super().parse_question( - question, user_answers - ) - question_parsed.min = question.get('min', None) - question_parsed.max = question.get('max', None) - if question_parsed.default is None: - question_parsed.default = 0 + @staticmethod + def humanize(value, option={}): + return str(value) - return question_parsed + def __init__(self, question, user_answers): + super().__init__(question, user_answers) + self.min = question.get('min', None) + self.max = question.get('max', None) + self.step = question.get('step', None) - def _prevalidate(self, question): - super()._prevalidate(question) - if not isinstance(question.value, int) and not (isinstance(question.value, str) and question.value.isdigit()): + + def _prevalidate(self): + super()._prevalidate() + if not isinstance(self.value, int) and not (isinstance(self.value, str) and self.value.isdigit()): raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("invalid_number") + "app_argument_invalid", field=self.name, error=m18n.n("invalid_number") ) - if question.min is not None and int(question.value) < question.min: + if self.min is not None and int(self.value) < self.min: raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("invalid_number") + "app_argument_invalid", field=self.name, error=m18n.n("invalid_number") ) - if question.max is not None and int(question.value) > question.max: + if self.max is not None and int(self.value) > self.max: raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("invalid_number") + "app_argument_invalid", field=self.name, error=m18n.n("invalid_number") ) - def _post_parse_value(self, question): - if isinstance(question.value, int): - return super()._post_parse_value(question) + def _post_parse_value(self): + if isinstance(self.value, int): + return super()._post_parse_value() - if isinstance(question.value, str) and question.value.isdigit(): - return int(question.value) + if isinstance(self.value, str) and self.value.isdigit(): + return int(self.value) raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("invalid_number") + "app_argument_invalid", field=self.name, error=m18n.n("invalid_number") ) -class DisplayTextArgumentParser(YunoHostArgumentFormatParser): +class DisplayTextQuestion(Question): argument_type = "display_text" readonly = True - def parse_question(self, question, user_answers): - question_parsed = super().parse_question( - question, user_answers - ) + def __init__(self, question, user_answers): + super().__init__(question, user_answers) - question_parsed.optional = True - question_parsed.style = question.get('style', 'info') + self.optional = True + self.style = question.get('style', 'info') - return question_parsed - def _format_text_for_user_input_in_cli(self, question): - text = question.ask['en'] + def _format_text_for_user_input_in_cli(self): + text = self.ask['en'] - if question.style in ['success', 'info', 'warning', 'danger']: + if self.style in ['success', 'info', 'warning', 'danger']: color = { 'success': 'green', 'info': 'cyan', 'warning': 'yellow', 'danger': 'red' } - return colorize(m18n.g(question.style), color[question.style]) + f" {text}" + return colorize(m18n.g(self.style), color[self.style]) + f" {text}" else: return text -class FileArgumentParser(YunoHostArgumentFormatParser): +class FileQuestion(Question): argument_type = "file" upload_dirs = [] @@ -725,55 +761,52 @@ class FileArgumentParser(YunoHostArgumentFormatParser): if os.path.exists(upload_dir): shutil.rmtree(upload_dir) - def parse_question(self, question, user_answers): - question_parsed = super().parse_question( - question, user_answers - ) - if question.get('accept'): - question_parsed.accept = question.get('accept').replace(' ', '').split(',') + def __init__(self, question, user_answers): + super().__init__(question, user_answers) + if self.get('accept'): + self.accept = question.get('accept').replace(' ', '').split(',') else: - question_parsed.accept = [] + self.accept = [] if Moulinette.interface.type== 'api': - if user_answers.get(f"{question_parsed.name}[name]"): - question_parsed.value = { - 'content': question_parsed.value, - 'filename': user_answers.get(f"{question_parsed.name}[name]", question_parsed.name), + if user_answers.get(f"{self.name}[name]"): + self.value = { + 'content': self.value, + 'filename': user_answers.get(f"{self.name}[name]", self.name), } # If path file are the same - if question_parsed.value and str(question_parsed.value) == question_parsed.current_value: - question_parsed.value = None + if self.value and str(self.value) == self.current_value: + self.value = None - return question_parsed - def _prevalidate(self, question): - super()._prevalidate(question) - if isinstance(question.value, str) and question.value and not os.path.exists(question.value): + def _prevalidate(self): + super()._prevalidate() + if isinstance(self.value, str) and self.value and not os.path.exists(self.value): raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("invalid_number1") + "app_argument_invalid", field=self.name, error=m18n.n("invalid_number1") ) - if question.value in [None, ''] or not question.accept: + if self.value in [None, ''] or not self.accept: return - filename = question.value if isinstance(question.value, str) else question.value['filename'] - if '.' not in filename or '.' + filename.split('.')[-1] not in question.accept: + filename = self.value if isinstance(self.value, str) else self.value['filename'] + if '.' not in filename or '.' + filename.split('.')[-1] not in self.accept: raise YunohostValidationError( - "app_argument_invalid", field=question.name, error=m18n.n("invalid_number2") + "app_argument_invalid", field=self.name, error=m18n.n("invalid_number2") ) - def _post_parse_value(self, question): + def _post_parse_value(self): from base64 import b64decode # Upload files from API # A file arg contains a string with "FILENAME:BASE64_CONTENT" - if not question.value: - return question.value + if not self.value: + return self.value if Moulinette.interface.type== 'api': upload_dir = tempfile.mkdtemp(prefix='tmp_configpanel_') - FileArgumentParser.upload_dirs += [upload_dir] - filename = question.value['filename'] - logger.debug(f"Save uploaded file {question.value['filename']} from API into {upload_dir}") + FileQuestion.upload_dirs += [upload_dir] + filename = self.value['filename'] + logger.debug(f"Save uploaded file {self.value['filename']} from API into {upload_dir}") # Filename is given by user of the API. For security reason, we have replaced # os.path.join to avoid the user to be able to rewrite a file in filesystem @@ -785,7 +818,7 @@ class FileArgumentParser(YunoHostArgumentFormatParser): while os.path.exists(file_path): file_path = os.path.normpath(upload_dir + "/" + filename + (".%d" % i)) i += 1 - content = question.value['content'] + content = self.value['content'] try: with open(file_path, 'wb') as f: f.write(b64decode(content)) @@ -793,31 +826,31 @@ class FileArgumentParser(YunoHostArgumentFormatParser): raise YunohostError("cannot_write_file", file=file_path, error=str(e)) except Exception as e: raise YunohostError("error_writing_file", file=file_path, error=str(e)) - question.value = file_path - return question.value + self.value = file_path + return self.value ARGUMENTS_TYPE_PARSERS = { - "string": StringArgumentParser, - "text": StringArgumentParser, - "select": StringArgumentParser, - "tags": TagsArgumentParser, - "email": StringArgumentParser, - "url": StringArgumentParser, - "date": StringArgumentParser, - "time": StringArgumentParser, - "color": StringArgumentParser, - "password": PasswordArgumentParser, - "path": PathArgumentParser, - "boolean": BooleanArgumentParser, - "domain": DomainArgumentParser, - "user": UserArgumentParser, - "number": NumberArgumentParser, - "range": NumberArgumentParser, - "display_text": DisplayTextArgumentParser, - "alert": DisplayTextArgumentParser, - "markdown": DisplayTextArgumentParser, - "file": FileArgumentParser, + "string": StringQuestion, + "text": StringQuestion, + "select": StringQuestion, + "tags": TagsQuestion, + "email": StringQuestion, + "url": StringQuestion, + "date": StringQuestion, + "time": StringQuestion, + "color": StringQuestion, + "password": PasswordQuestion, + "path": PathQuestion, + "boolean": BooleanQuestion, + "domain": DomainQuestion, + "user": UserQuestion, + "number": NumberQuestion, + "range": NumberQuestion, + "display_text": DisplayTextQuestion, + "alert": DisplayTextQuestion, + "markdown": DisplayTextQuestion, + "file": FileQuestion, } def parse_args_in_yunohost_format(user_answers, argument_questions): @@ -834,11 +867,12 @@ def parse_args_in_yunohost_format(user_answers, argument_questions): parsed_answers_dict = OrderedDict() for question in argument_questions: - parser = ARGUMENTS_TYPE_PARSERS[question.get("type", "string")]() + question_class = ARGUMENTS_TYPE_PARSERS[question.get("type", "string")] + question = question_class(question, user_answers) - answer = parser.parse(question=question, user_answers=user_answers) + answer = question.ask_if_needed() if answer is not None: - parsed_answers_dict[question["name"]] = answer + parsed_answers_dict[question.name] = answer return parsed_answers_dict From 97c0a74f8f52fcfcd2ded683115aa543a5a9730e Mon Sep 17 00:00:00 2001 From: ljf Date: Sat, 4 Sep 2021 19:03:11 +0200 Subject: [PATCH 069/130] [enh] yes/no args on boolean aquestion + semantic --- src/yunohost/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 215f91d48..d3717c6e7 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -55,7 +55,7 @@ from moulinette.utils.filesystem import ( from yunohost.service import service_status, _run_service_command from yunohost.utils import packages, config -from yunohost.utils.config import ConfigPanel, parse_args_in_yunohost_format, YunoHostArgumentFormatParser +from yunohost.utils.config import ConfigPanel, parse_args_in_yunohost_format, Question from yunohost.utils.i18n import _value_for_locale from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.filesystem import free_space_in_directory @@ -1773,7 +1773,7 @@ def app_config_set(operation_logger, app, key=None, value=None, args=None, args_ config = AppConfigPanel(app) - YunoHostArgumentFormatParser.operation_logger = operation_logger + Question.operation_logger = operation_logger operation_logger.start() result = config.set(key, value, args, args_file) From fd5b2e8953624b5686552c00630c4ba24b4f074f Mon Sep 17 00:00:00 2001 From: ljf Date: Sat, 4 Sep 2021 19:05:13 +0200 Subject: [PATCH 070/130] [enh] Several fixes in domain config --- data/actionsmap/yunohost.yml | 2 +- data/other/config_domain.toml | 8 +- data/other/registrar_list.toml | 238 ++++++++++++++++----------------- src/yunohost/domain.py | 16 ++- 4 files changed, 135 insertions(+), 129 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 0da811522..759dd1f22 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -575,7 +575,7 @@ domain: action_help: Apply a new configuration api: PUT /domains//config arguments: - app: + domain: help: Domain name key: help: The question or form key diff --git a/data/other/config_domain.toml b/data/other/config_domain.toml index b2211417d..07aaf085e 100644 --- a/data/other/config_domain.toml +++ b/data/other/config_domain.toml @@ -3,13 +3,14 @@ i18n = "domain_config" [feature] [feature.mail] + services = ['postfix', 'dovecot'] [feature.mail.mail_out] type = "boolean" - default = true + default = 1 [feature.mail.mail_in] type = "boolean" - default = true + default = 1 [feature.mail.backup_mx] type = "tags" @@ -21,10 +22,11 @@ i18n = "domain_config" [feature.xmpp.xmpp] type = "boolean" - default = false + default = 0 [dns] [dns.registrar] + optional = true [dns.registrar.unsupported] ask = "DNS zone of this domain can't be auto-configured, you should do it manually." type = "alert" diff --git a/data/other/registrar_list.toml b/data/other/registrar_list.toml index 33d115075..1c2e73111 100644 --- a/data/other/registrar_list.toml +++ b/data/other/registrar_list.toml @@ -1,7 +1,7 @@ [aliyun] [aliyun.auth_key_id] type = "string" - redact = True + redact = true [aliyun.auth_secret] type = "password" @@ -9,7 +9,7 @@ [aurora] [aurora.auth_api_key] type = "string" - redact = True + redact = true [aurora.auth_secret_key] type = "password" @@ -17,48 +17,48 @@ [azure] [azure.auth_client_id] type = "string" - redact = True + redact = true [azure.auth_client_secret] type = "password" [azure.auth_tenant_id] type = "string" - redact = True + redact = true [azure.auth_subscription_id] type = "string" - redact = True + redact = true [azure.resource_group] type = "string" - redact = True + redact = true [cloudflare] [cloudflare.auth_username] type = "string" - redact = True + redact = true [cloudflare.auth_token] type = "string" - redact = True + redact = true [cloudflare.zone_id] type = "string" - redact = True + redact = true [cloudns] [cloudns.auth_id] type = "string" - redact = True + redact = true [cloudns.auth_subid] type = "string" - redact = True + redact = true [cloudns.auth_subuser] type = "string" - redact = True + redact = true [cloudns.auth_password] type = "password" @@ -71,50 +71,50 @@ [cloudxns] [cloudxns.auth_username] type = "string" - redact = True + redact = true [cloudxns.auth_token] type = "string" - redact = True + redact = true [conoha] [conoha.auth_region] type = "string" - redact = True + redact = true [conoha.auth_token] type = "string" - redact = True + redact = true [conoha.auth_username] type = "string" - redact = True + redact = true [conoha.auth_password] type = "password" [conoha.auth_tenant_id] type = "string" - redact = True + redact = true [constellix] [constellix.auth_username] type = "string" - redact = True + redact = true [constellix.auth_token] type = "string" - redact = True + redact = true [digitalocean] [digitalocean.auth_token] type = "string" - redact = True + redact = true [dinahosting] [dinahosting.auth_username] type = "string" - redact = True + redact = true [dinahosting.auth_password] type = "password" @@ -125,78 +125,78 @@ [directadmin.auth_username] type = "string" - redact = True + redact = true [directadmin.endpoint] type = "string" - redact = True + redact = true [dnsimple] [dnsimple.auth_token] type = "string" - redact = True + redact = true [dnsimple.auth_username] type = "string" - redact = True + redact = true [dnsimple.auth_password] type = "password" [dnsimple.auth_2fa] type = "string" - redact = True + redact = true [dnsmadeeasy] [dnsmadeeasy.auth_username] type = "string" - redact = True + redact = true [dnsmadeeasy.auth_token] type = "string" - redact = True + redact = true [dnspark] [dnspark.auth_username] type = "string" - redact = True + redact = true [dnspark.auth_token] type = "string" - redact = True + redact = true [dnspod] [dnspod.auth_username] type = "string" - redact = True + redact = true [dnspod.auth_token] type = "string" - redact = True + redact = true [dreamhost] [dreamhost.auth_token] type = "string" - redact = True + redact = true [dynu] [dynu.auth_token] type = "string" - redact = True + redact = true [easydns] [easydns.auth_username] type = "string" - redact = True + redact = true [easydns.auth_token] type = "string" - redact = True + redact = true [easyname] [easyname.auth_username] type = "string" - redact = True + redact = true [easyname.auth_password] type = "password" @@ -204,7 +204,7 @@ [euserv] [euserv.auth_username] type = "string" - redact = True + redact = true [euserv.auth_password] type = "password" @@ -212,7 +212,7 @@ [exoscale] [exoscale.auth_key] type = "string" - redact = True + redact = true [exoscale.auth_secret] type = "password" @@ -220,7 +220,7 @@ [gandi] [gandi.auth_token] type = "string" - redact = True + redact = true [gandi.api_protocol] type = "string" @@ -230,7 +230,7 @@ [gehirn] [gehirn.auth_token] type = "string" - redact = True + redact = true [gehirn.auth_secret] type = "password" @@ -238,16 +238,16 @@ [glesys] [glesys.auth_username] type = "string" - redact = True + redact = true [glesys.auth_token] type = "string" - redact = True + redact = true [godaddy] [godaddy.auth_key] type = "string" - redact = True + redact = true [godaddy.auth_secret] type = "password" @@ -255,12 +255,12 @@ [googleclouddns] [goggleclouddns.auth_service_account_info] type = "string" - redact = True + redact = true [gransy] [gransy.auth_username] type = "string" - redact = True + redact = true [gransy.auth_password] type = "password" @@ -268,7 +268,7 @@ [gratisdns] [gratisdns.auth_username] type = "string" - redact = True + redact = true [gratisdns.auth_password] type = "password" @@ -276,7 +276,7 @@ [henet] [henet.auth_username] type = "string" - redact = True + redact = true [henet.auth_password] type = "password" @@ -284,17 +284,17 @@ [hetzner] [hetzner.auth_token] type = "string" - redact = True + redact = true [hostingde] [hostingde.auth_token] type = "string" - redact = True + redact = true [hover] [hover.auth_username] type = "string" - redact = True + redact = true [hover.auth_password] type = "password" @@ -302,37 +302,37 @@ [infoblox] [infoblox.auth_user] type = "string" - redact = True + redact = true [infoblox.auth_psw] type = "password" [infoblox.ib_view] type = "string" - redact = True + redact = true [infoblox.ib_host] type = "string" - redact = True + redact = true [infomaniak] [infomaniak.auth_token] type = "string" - redact = True + redact = true [internetbs] [internetbs.auth_key] type = "string" - redact = True + redact = true [internetbs.auth_password] type = "string" - redact = True + redact = true [inwx] [inwx.auth_username] type = "string" - redact = True + redact = true [inwx.auth_password] type = "password" @@ -340,79 +340,79 @@ [joker] [joker.auth_token] type = "string" - redact = True + redact = true [linode] [linode.auth_token] type = "string" - redact = True + redact = true [linode4] [linode4.auth_token] type = "string" - redact = True + redact = true [localzone] [localzone.filename] type = "string" - redact = True + redact = true [luadns] [luadns.auth_username] type = "string" - redact = True + redact = true [luadns.auth_token] type = "string" - redact = True + redact = true [memset] [memset.auth_token] type = "string" - redact = True + redact = true [mythicbeasts] [mythicbeasts.auth_username] type = "string" - redact = True + redact = true [mythicbeasts.auth_password] type = "password" [mythicbeasts.auth_token] type = "string" - redact = True + redact = true [namecheap] [namecheap.auth_token] type = "string" - redact = True + redact = true [namecheap.auth_username] type = "string" - redact = True + redact = true [namecheap.auth_client_ip] type = "string" - redact = True + redact = true [namecheap.auth_sandbox] type = "string" - redact = True + redact = true [namesilo] [namesilo.auth_token] type = "string" - redact = True + redact = true [netcup] [netcup.auth_customer_id] type = "string" - redact = True + redact = true [netcup.auth_api_key] type = "string" - redact = True + redact = true [netcup.auth_api_password] type = "password" @@ -420,89 +420,89 @@ [nfsn] [nfsn.auth_username] type = "string" - redact = True + redact = true [nfsn.auth_token] type = "string" - redact = True + redact = true [njalla] [njalla.auth_token] type = "string" - redact = True + redact = true [nsone] [nsone.auth_token] type = "string" - redact = True + redact = true [onapp] [onapp.auth_username] type = "string" - redact = True + redact = true [onapp.auth_token] type = "string" - redact = True + redact = true [onapp.auth_server] type = "string" - redact = True + redact = true [online] [online.auth_token] type = "string" - redact = True + redact = true [ovh] [ovh.auth_entrypoint] type = "string" - redact = True + redact = true [ovh.auth_application_key] type = "string" - redact = True + redact = true [ovh.auth_application_secret] type = "password" [ovh.auth_consumer_key] type = "string" - redact = True + redact = true [plesk] [plesk.auth_username] type = "string" - redact = True + redact = true [plesk.auth_password] type = "password" [plesk.plesk_server] type = "string" - redact = True + redact = true [pointhq] [pointhq.auth_username] type = "string" - redact = True + redact = true [pointhq.auth_token] type = "string" - redact = True + redact = true [powerdns] [powerdns.auth_token] type = "string" - redact = True + redact = true [powerdns.pdns_server] type = "string" - redact = True + redact = true [powerdns.pdns_server_id] type = "string" - redact = True + redact = true [powerdns.pdns_disable_notify] type = "boolean" @@ -510,67 +510,67 @@ [rackspace] [rackspace.auth_account] type = "string" - redact = True + redact = true [rackspace.auth_username] type = "string" - redact = True + redact = true [rackspace.auth_api_key] type = "string" - redact = True + redact = true [rackspace.auth_token] type = "string" - redact = True + redact = true [rackspace.sleep_time] type = "string" - redact = True + redact = true [rage4] [rage4.auth_username] type = "string" - redact = True + redact = true [rage4.auth_token] type = "string" - redact = True + redact = true [rcodezero] [rcodezero.auth_token] type = "string" - redact = True + redact = true [route53] [route53.auth_access_key] type = "string" - redact = True + redact = true [route53.auth_access_secret] type = "password" [route53.private_zone] type = "string" - redact = True + redact = true [route53.auth_username] type = "string" - redact = True + redact = true [route53.auth_token] type = "string" - redact = True + redact = true [safedns] [safedns.auth_token] type = "string" - redact = True + redact = true [sakuracloud] [sakuracloud.auth_token] type = "string" - redact = True + redact = true [sakuracloud.auth_secret] type = "password" @@ -578,29 +578,29 @@ [softlayer] [softlayer.auth_username] type = "string" - redact = True + redact = true [softlayer.auth_api_key] type = "string" - redact = True + redact = true [transip] [transip.auth_username] type = "string" - redact = True + redact = true [transip.auth_api_key] type = "string" - redact = True + redact = true [ultradns] [ultradns.auth_token] type = "string" - redact = True + redact = true [ultradns.auth_username] type = "string" - redact = True + redact = true [ultradns.auth_password] type = "password" @@ -608,29 +608,29 @@ [vultr] [vultr.auth_token] type = "string" - redact = True + redact = true [yandex] [yandex.auth_token] type = "string" - redact = True + redact = true [zeit] [zeit.auth_token] type = "string" - redact = True + redact = true [zilore] [zilore.auth_key] type = "string" - redact = True + redact = true [zonomi] [zonomy.auth_token] type = "string" - redact = True + redact = true [zonomy.auth_entrypoint] type = "string" - redact = True + redact = true diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 064311a49..217a787d9 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -28,7 +28,9 @@ import os from moulinette import m18n, Moulinette from moulinette.core import MoulinetteError from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import mkdir, write_to_file, read_yaml, write_to_yaml +from moulinette.utils.filesystem import ( + mkdir, write_to_file, read_yaml, write_to_yaml, read_toml +) from yunohost.settings import is_boolean from yunohost.app import ( @@ -37,8 +39,10 @@ from yunohost.app import ( _get_app_settings, _get_conflicting_apps, ) -from yunohost.regenconf import regen_conf, _force_clear_hashes, _process_regen_conf -from yunohost.utils.config import ConfigPanel +from yunohost.regenconf import ( + regen_conf, _force_clear_hashes, _process_regen_conf +) +from yunohost.utils.config import ConfigPanel, Question from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.log import is_unit_operation from yunohost.hook import hook_callback @@ -407,11 +411,11 @@ def domain_config_get(domain, key='', mode='classic'): return config.get(key, mode) @is_unit_operation() -def domain_config_set(operation_logger, app, key=None, value=None, args=None, args_file=None): +def domain_config_set(operation_logger, domain, key=None, value=None, args=None, args_file=None): """ Apply a new domain configuration """ - + Question.operation_logger = operation_logger config = DomainConfigPanel(domain) return config.set(key, value, args, args_file) @@ -432,7 +436,7 @@ class DomainConfigPanel(ConfigPanel): self.dns_zone = get_dns_zone_from_domain(self.domain) try: - registrar = _relevant_provider_for_domain(self.dns_zone) + registrar = _relevant_provider_for_domain(self.dns_zone)[0] except ValueError: return toml From 2413c378df18a4f830aed96a49ffc2e55b58d6e5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 5 Sep 2021 01:04:22 +0200 Subject: [PATCH 071/130] Gotta mkdir data/bash-completion.d at build time --- data/actionsmap/yunohost_completion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data/actionsmap/yunohost_completion.py b/data/actionsmap/yunohost_completion.py index 3891aee9c..c801e2f3c 100644 --- a/data/actionsmap/yunohost_completion.py +++ b/data/actionsmap/yunohost_completion.py @@ -13,6 +13,7 @@ import yaml THIS_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) ACTIONSMAP_FILE = THIS_SCRIPT_DIR + "/yunohost.yml" +os.system(f"mkdir {THIS_SCRIPT_DIR}/../bash-completion.d") BASH_COMPLETION_FILE = THIS_SCRIPT_DIR + "/../bash-completion.d/yunohost" From bd384d094f05d40cbe3da63100ead332771b598e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 5 Sep 2021 01:08:40 +0200 Subject: [PATCH 072/130] Unused imports --- src/yunohost/dns.py | 2 -- src/yunohost/domain.py | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index df3b578db..045a33e05 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -28,10 +28,8 @@ import re from moulinette import m18n, Moulinette from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import mkdir, read_yaml, write_to_yaml from yunohost.domain import domain_list, _get_domain_settings, _assert_domain_exists -from yunohost.app import _parse_args_in_yunohost_format from yunohost.utils.error import YunohostValidationError from yunohost.utils.network import get_public_ip from yunohost.log import is_unit_operation diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 064311a49..28f6037da 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -28,9 +28,8 @@ import os from moulinette import m18n, Moulinette from moulinette.core import MoulinetteError from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import mkdir, write_to_file, read_yaml, write_to_yaml +from moulinette.utils.filesystem import write_to_file -from yunohost.settings import is_boolean from yunohost.app import ( app_ssowatconf, _installed_apps, @@ -41,7 +40,6 @@ from yunohost.regenconf import regen_conf, _force_clear_hashes, _process_regen_c from yunohost.utils.config import ConfigPanel from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.log import is_unit_operation -from yunohost.hook import hook_callback logger = getActionLogger("yunohost.domain") From 6aead55c1c87d76237f4175c4052a1203119edbe Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 5 Sep 2021 16:34:18 +0200 Subject: [PATCH 073/130] [enh] Add a comment --- data/other/config_domain.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/data/other/config_domain.toml b/data/other/config_domain.toml index 07aaf085e..44766e2d0 100644 --- a/data/other/config_domain.toml +++ b/data/other/config_domain.toml @@ -27,6 +27,7 @@ i18n = "domain_config" [dns] [dns.registrar] optional = true + # This part is replace dynamically by DomainConfigPanel [dns.registrar.unsupported] ask = "DNS zone of this domain can't be auto-configured, you should do it manually." type = "alert" From 41cf266e252c69feeafb5286a6b57f8249f4db77 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 12 Sep 2021 15:40:17 +0200 Subject: [PATCH 074/130] dns-autoconf: remove dependency to public suffix list, not sure what scenario it was supposed to cover, probably goodenough without it --- src/yunohost/utils/dns.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/src/yunohost/utils/dns.py b/src/yunohost/utils/dns.py index ef89c35c5..848cafeba 100644 --- a/src/yunohost/utils/dns.py +++ b/src/yunohost/utils/dns.py @@ -19,11 +19,8 @@ """ import dns.resolver -from publicsuffixlist import PublicSuffixList from moulinette.utils.filesystem import read_file -YNH_DYNDNS_DOMAINS = ["nohost.me", "noho.st", "ynh.fr"] - # Lazy dev caching to avoid re-reading the file multiple time when calling # dig() often during same yunohost operation external_resolvers_ = [] @@ -94,24 +91,6 @@ def dig( return ("ok", answers) -def get_public_suffix(domain): - """get_public_suffix("www.example.com") -> "example.com" - - Return the public suffix of a domain name based - """ - # Load domain public suffixes - psl = PublicSuffixList() - - public_suffix = psl.publicsuffix(domain) - - # FIXME: wtf is this supposed to do ? :| - if public_suffix in YNH_DYNDNS_DOMAINS: - domain_prefix = domain[0:-(1 + len(public_suffix))] - public_suffix = domain_prefix.split(".")[-1] + "." + public_suffix - - return public_suffix - - def get_dns_zone_from_domain(domain): # TODO Check if this function is YNH_DYNDNS_DOMAINS compatible """ @@ -138,11 +117,6 @@ def get_dns_zone_from_domain(domain): if answer[0] == "ok": # Domain is dns_zone return parent - # Otherwise, check if the parent of this parent is in the public suffix list - if parent.split(".", 1)[-1] == get_public_suffix(parent): - # Couldn't check if domain is dns zone, # FIXME : why "couldn't" ...? - # returning private suffix - return parent # FIXME: returning None will probably trigger bugs when this happens, code expects a domain string return None From 88a624ddbcdf991f78d3c4f9e8be6ec13ffdd2ab Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 12 Sep 2021 15:44:51 +0200 Subject: [PATCH 075/130] Revert publicsuffix changes in dnsrecord diagnoser --- data/hooks/diagnosis/12-dnsrecords.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 727fd2e13..90c42c0d7 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -4,15 +4,16 @@ import os import re from datetime import datetime, timedelta -from publicsuffixlist import PublicSuffixList +from publicsuffix import PublicSuffixList from moulinette.utils.process import check_output -from yunohost.utils.dns import dig, YNH_DYNDNS_DOMAINS +from yunohost.utils.dns import dig from yunohost.diagnosis import Diagnoser from yunohost.domain import domain_list, _get_maindomain from yunohost.dns import _build_dns_conf +YNH_DYNDNS_DOMAINS = ["nohost.me", "noho.st", "ynh.fr"] SPECIAL_USE_TLDS = ["local", "localhost", "onion", "test"] @@ -44,7 +45,7 @@ class DNSRecordsDiagnoser(Diagnoser): # Check if a domain buy by the user will expire soon psl = PublicSuffixList() domains_from_registrar = [ - psl.publicsuffix(domain) for domain in all_domains + psl.get_public_suffix(domain) for domain in all_domains ] domains_from_registrar = [ domain for domain in domains_from_registrar if "." in domain From d7b79154ff2eee6db876e329656d2dc9f67b0cd7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 12 Sep 2021 16:07:10 +0200 Subject: [PATCH 076/130] debian: Add python3-lexicon dependency --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 2e101dca3..90bac0a0d 100644 --- a/debian/control +++ b/debian/control @@ -14,7 +14,7 @@ Depends: ${python3:Depends}, ${misc:Depends} , python3-psutil, python3-requests, python3-dnspython, python3-openssl , python3-miniupnpc, python3-dbus, python3-jinja2 , python3-toml, python3-packaging, python3-publicsuffix, - , python3-ldap, python3-zeroconf, + , python3-ldap, python3-zeroconf, python3-lexicon, , apt, apt-transport-https, apt-utils, dirmngr , php7.3-common, php7.3-fpm, php7.3-ldap, php7.3-intl , mariadb-server, php7.3-mysql From 4533b74d6ced0288bddcd9fbf3dfb7474170f611 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 12 Sep 2021 21:32:43 +0200 Subject: [PATCH 077/130] autodns: Various tweaks and refactorings to make test pass --- .gitlab/ci/test.gitlab-ci.yml | 19 ++++ data/actionsmap/yunohost.yml | 10 +- data/hooks/conf_regen/15-nginx | 2 +- data/hooks/diagnosis/12-dnsrecords.py | 3 +- data/other/config_domain.toml | 17 ++-- locales/en.json | 2 + src/yunohost/dns.py | 137 ++++++++++++++++++++++++-- src/yunohost/domain.py | 48 ++++----- src/yunohost/tests/test_dns.py | 66 +++++++++++++ src/yunohost/tests/test_domains.py | 105 ++++++-------------- src/yunohost/utils/config.py | 2 + src/yunohost/utils/dns.py | 32 +----- src/yunohost/utils/ldap.py | 5 +- 13 files changed, 299 insertions(+), 149 deletions(-) create mode 100644 src/yunohost/tests/test_dns.py diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index 2dc45171b..f270ba982 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -85,6 +85,25 @@ test-helpers: changes: - data/helpers.d/* +test-domains: + extends: .test-stage + script: + - cd src/yunohost + - python3 -m pytest tests/test_domains.py + only: + changes: + - src/yunohost/domain.py + +test-dns: + extends: .test-stage + script: + - cd src/yunohost + - python3 -m pytest tests/test_dns.py + only: + changes: + - src/yunohost/dns.py + - src/yunohost/utils/dns.py + test-apps: extends: .test-stage script: diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index f9fcaffc0..c118c90a2 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -589,8 +589,16 @@ domain: domain: help: Domain name key: - help: A question or form key + help: A specific panel, section or a question identifier nargs: '?' + -f: + full: --full + help: Display all details (meant to be used by the API) + action: store_true + -e: + full: --export + help: Only export key/values, meant to be reimported using "config set --args-file" + action: store_true ### domain_config_set() set: diff --git a/data/hooks/conf_regen/15-nginx b/data/hooks/conf_regen/15-nginx index 040ed090d..0c41ea50b 100755 --- a/data/hooks/conf_regen/15-nginx +++ b/data/hooks/conf_regen/15-nginx @@ -65,7 +65,7 @@ do_pre_regen() { export experimental="$(yunohost settings get 'security.experimental.enabled')" ynh_render_template "security.conf.inc" "${nginx_conf_dir}/security.conf.inc" - cert_status=$(yunohost domain cert-status --json) + cert_status=$(yunohost domain cert status --json) # add domain conf files for domain in $YNH_DOMAINS; do diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 90c42c0d7..854f348f5 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -8,12 +8,11 @@ from publicsuffix import PublicSuffixList from moulinette.utils.process import check_output -from yunohost.utils.dns import dig +from yunohost.utils.dns import dig, YNH_DYNDNS_DOMAINS from yunohost.diagnosis import Diagnoser from yunohost.domain import domain_list, _get_maindomain from yunohost.dns import _build_dns_conf -YNH_DYNDNS_DOMAINS = ["nohost.me", "noho.st", "ynh.fr"] SPECIAL_USE_TLDS = ["local", "localhost", "onion", "test"] diff --git a/data/other/config_domain.toml b/data/other/config_domain.toml index 44766e2d0..20963764b 100644 --- a/data/other/config_domain.toml +++ b/data/other/config_domain.toml @@ -2,8 +2,10 @@ version = "1.0" i18n = "domain_config" [feature] + [feature.mail] - services = ['postfix', 'dovecot'] + services = ['postfix', 'dovecot'] + [feature.mail.mail_out] type = "boolean" default = 1 @@ -25,17 +27,14 @@ i18n = "domain_config" default = 0 [dns] + [dns.registrar] - optional = true - # This part is replace dynamically by DomainConfigPanel - [dns.registrar.unsupported] - ask = "DNS zone of this domain can't be auto-configured, you should do it manually." - type = "alert" - style = "info" - helpLink.href = "https://yunohost.org/dns_config" - helpLink.text = "How to configure manually my DNS zone" + optional = true + + # This part is automatically generated in DomainConfigPanel [dns.advanced] + [dns.advanced.ttl] type = "number" min = 0 diff --git a/locales/en.json b/locales/en.json index 1d51f59d3..e39c765c0 100644 --- a/locales/en.json +++ b/locales/en.json @@ -395,6 +395,7 @@ "iptables_unavailable": "You cannot play with iptables here. You are either in a container or your kernel does not support it", "ldap_server_down": "Unable to reach LDAP server", "ldap_server_is_down_restart_it": "The LDAP service is down, attempt to restart it...", + "ldap_attribute_already_exists": "LDAP attribute '{attribute}' already exists with value '{value}'", "log_app_action_run": "Run action of the '{}' app", "log_app_change_url": "Change the URL of the '{}' app", "log_app_config_set": "Apply config to the '{}' app", @@ -409,6 +410,7 @@ "log_corrupted_md_file": "The YAML metadata file associated with logs is damaged: '{md_file}\nError: {error}'", "log_does_exists": "There is no operation log with the name '{log}', use 'yunohost log list' to see all available operation logs", "log_domain_add": "Add '{}' domain into system configuration", + "log_domain_config_set": "Update configuration for domain '{}'", "log_domain_main_domain": "Make '{}' the main domain", "log_domain_remove": "Remove '{}' domain from system configuration", "log_dyndns_subscribe": "Subscribe to a YunoHost subdomain '{}'", diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 045a33e05..aa5e79c82 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -25,11 +25,15 @@ """ import os import re +import time +from collections import OrderedDict from moulinette import m18n, Moulinette from moulinette.utils.log import getActionLogger +from moulinette.utils.filesystem import read_file, write_to_file, read_toml -from yunohost.domain import domain_list, _get_domain_settings, _assert_domain_exists +from yunohost.domain import domain_list, _assert_domain_exists, domain_config_get +from yunohost.utils.dns import dig, YNH_DYNDNS_DOMAINS from yunohost.utils.error import YunohostValidationError from yunohost.utils.network import get_public_ip from yunohost.log import is_unit_operation @@ -37,8 +41,10 @@ from yunohost.hook import hook_callback logger = getActionLogger("yunohost.domain") +DOMAIN_REGISTRAR_LIST_PATH = "/usr/share/yunohost/other/registrar_list.toml" -def domain_dns_conf(domain): + +def domain_dns_suggest(domain): """ Generate DNS configuration for a domain @@ -149,10 +155,10 @@ def _build_dns_conf(base_domain): ipv6 = get_public_ip(6) subdomains = _list_subdomains_of(base_domain) - domains_settings = {domain: _get_domain_settings(domain) + domains_settings = {domain: domain_config_get(domain) for domain in [base_domain] + subdomains} - base_dns_zone = domains_settings[base_domain].get("dns_zone") + base_dns_zone = _get_dns_zone_for_domain(base_domain) for domain, settings in domains_settings.items(): @@ -384,6 +390,126 @@ def _get_DKIM(domain): ) +def _get_dns_zone_for_domain(domain): + """ + Get the DNS zone of a domain + + Keyword arguments: + domain -- The domain name + + """ + + # First, check if domain is a nohost.me / noho.st / ynh.fr + # This is mainly meant to speed up things for "dyndns update" + # ... otherwise we end up constantly doing a bunch of dig requests + for ynh_dyndns_domain in YNH_DYNDNS_DOMAINS: + if domain.endswith('.' + ynh_dyndns_domain): + return ynh_dyndns_domain + + # Check cache + cache_folder = "/var/cache/yunohost/dns_zones" + cache_file = f"{cache_folder}/{domain}" + cache_duration = 3600 # one hour + if ( + os.path.exists(cache_file) + and abs(os.path.getctime(cache_file) - time.time()) < cache_duration + ): + dns_zone = read_file(cache_file).strip() + if dns_zone: + return dns_zone + + # Check cache for parent domain + # This is another strick to try to prevent this function from being + # a bottleneck on system with 1 main domain + 10ish subdomains + # when building the dns conf for the main domain (which will call domain_config_get, etc...) + parent_domain = domain.split(".", 1)[1] + if parent_domain in domain_list()["domains"]: + parent_cache_file = f"{cache_folder}/{parent_domain}" + if ( + os.path.exists(parent_cache_file) + and abs(os.path.getctime(parent_cache_file) - time.time()) < cache_duration + ): + dns_zone = read_file(parent_cache_file).strip() + if dns_zone: + return dns_zone + + # For foo.bar.baz.gni we want to scan all the parent domains + # (including the domain itself) + # foo.bar.baz.gni + # bar.baz.gni + # baz.gni + # gni + # Until we find the first one that has a NS record + parent_list = [domain.split(".", i)[-1] + for i, _ in enumerate(domain.split("."))] + + for parent in parent_list: + + # Check if there's a NS record for that domain + answer = dig(parent, rdtype="NS", full_answers=True, resolvers="force_external") + if answer[0] == "ok": + os.system(f"mkdir -p {cache_folder}") + write_to_file(cache_file, parent) + return parent + + logger.warning(f"Could not identify the dns_zone for domain {domain}, returning {parent_list[-1]}") + return parent_list[-1] + + +def _get_registrar_config_section(domain): + + from lexicon.providers.auto import _relevant_provider_for_domain + + registrar_infos = {} + + dns_zone = _get_dns_zone_for_domain(domain) + + # If parent domain exists in yunohost + parent_domain = domain.split(".", 1)[1] + if parent_domain in domain_list()["domains"]: + registrar_infos["explanation"] = OrderedDict({ + "type": "alert", + "style": "info", + "ask": f"This domain is a subdomain of {parent_domain}. DNS registrar configuration should be managed in {parent_domain}'s configuration panel.", # FIXME: i18n + "value": None + }) + return OrderedDict(registrar_infos) + + # TODO big project, integrate yunohost's dynette as a registrar-like provider + # TODO big project, integrate other dyndns providers such as netlib.re, or cf the list of dyndns providers supported by cloudron... + if dns_zone in YNH_DYNDNS_DOMAINS: + registrar_infos["explanation"] = OrderedDict({ + "type": "alert", + "style": "success", + "ask": "This domain is a nohost.me / nohost.st / ynh.fr and its DNS configuration is therefore automatically handled by Yunohost.", # FIXME: i18n + "value": "yunohost" + }) + return OrderedDict(registrar_infos) + + try: + registrar = _relevant_provider_for_domain(dns_zone)[0] + except ValueError: + registrar_infos["explanation"] = OrderedDict({ + "type": "alert", + "style": "warning", + "ask": "YunoHost could not automatically detect the registrar handling this domain. You should manually configure your DNS records following the documentation at https://yunohost.org/dns.", # FIXME : i18n + "value": None + }) + else: + + registrar_infos["explanation"] = OrderedDict({ + "type": "alert", + "style": "info", + "ask": f"YunoHost automatically detected that this domain is handled by the registrar **{registrar}**. If you want, YunoHost will automatically configure this DNS zone, if you provide it with the following informations. You can also manually configure your DNS records following the documentation as https://yunohost.org/dns.", # FIXME: i18n + "value": registrar + }) + # TODO : add a help tip with the link to the registar's API doc (c.f. Lexicon's README) + registrar_list = read_toml(DOMAIN_REGISTRAR_LIST_PATH) + registrar_infos.update(registrar_list[registrar]) + + return OrderedDict(registrar_infos) + + @is_unit_operation() def domain_registrar_push(operation_logger, domain, dry_run=False): """ @@ -395,8 +521,7 @@ def domain_registrar_push(operation_logger, domain, dry_run=False): _assert_domain_exists(domain) - dns_zone = _get_domain_settings(domain)["dns_zone"] - registrar_settings = _get_registrar_settings(dns_zone) + registrar_settings = domain_config_get(domain, key='', full=True) if not registrar_settings: raise YunohostValidationError("registrar_is_not_set", domain=domain) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 4cf223510..0bdede11d 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -29,7 +29,7 @@ from moulinette import m18n, Moulinette from moulinette.core import MoulinetteError from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import ( - mkdir, write_to_file, read_yaml, write_to_yaml, read_toml + mkdir, write_to_file, read_yaml, write_to_yaml ) from yunohost.app import ( @@ -49,7 +49,6 @@ logger = getActionLogger("yunohost.domain") DOMAIN_CONFIG_PATH = "/usr/share/yunohost/other/config_domain.toml" DOMAIN_SETTINGS_DIR = "/etc/yunohost/domains" -DOMAIN_REGISTRAR_LIST_PATH = "/usr/share/yunohost/other/registrar_list.toml" # Lazy dev caching to avoid re-query ldap every time we need the domain list domain_list_cache = {} @@ -391,23 +390,25 @@ def _get_maindomain(): return maindomain -def _get_domain_settings(domain): - """ - Retrieve entries in /etc/yunohost/domains/[domain].yml - And set default values if needed - """ - config = DomainConfigPanel(domain) - return config.get(mode='export') - - -def domain_config_get(domain, key='', mode='classic'): +def domain_config_get(domain, key='', full=False, export=False): """ Display a domain configuration """ + if full and export: + raise YunohostValidationError("You can't use --full and --export together.", raw_msg=True) + + if full: + mode = "full" + elif export: + mode = "export" + else: + mode = "classic" + config = DomainConfigPanel(domain) return config.get(key, mode) + @is_unit_operation() def domain_config_set(operation_logger, domain, key=None, value=None, args=None, args_file=None): """ @@ -415,31 +416,28 @@ def domain_config_set(operation_logger, domain, key=None, value=None, args=None, """ Question.operation_logger = operation_logger config = DomainConfigPanel(domain) - return config.set(key, value, args, args_file) + return config.set(key, value, args, args_file, operation_logger=operation_logger) class DomainConfigPanel(ConfigPanel): + def __init__(self, domain): _assert_domain_exists(domain) self.domain = domain + self.save_mode = "diff" super().__init__( config_path=DOMAIN_CONFIG_PATH, save_path=f"{DOMAIN_SETTINGS_DIR}/{domain}.yml" ) def _get_toml(self): - from lexicon.providers.auto import _relevant_provider_for_domain - from yunohost.utils.dns import get_dns_zone_from_domain + from yunohost.dns import _get_registrar_config_section + toml = super()._get_toml() - self.dns_zone = get_dns_zone_from_domain(self.domain) - try: - registrar = _relevant_provider_for_domain(self.dns_zone)[0] - except ValueError: - return toml + toml['feature']['xmpp']['xmpp']['default'] = 1 if self.domain == _get_maindomain() else 0 + toml['dns']['registrar'] = _get_registrar_config_section(self.domain) - registrar_list = read_toml(DOMAIN_REGISTRAR_LIST_PATH) - toml['dns']['registrar'] = registrar_list[registrar] return toml def _load_current_values(self): @@ -480,8 +478,12 @@ def domain_cert_renew( def domain_dns_conf(domain): + return domain_dns_suggest(domain) + + +def domain_dns_suggest(domain): import yunohost.dns - return yunohost.dns.domain_dns_conf(domain) + return yunohost.dns.domain_dns_suggest(domain) def domain_dns_push(domain, dry_run): diff --git a/src/yunohost/tests/test_dns.py b/src/yunohost/tests/test_dns.py new file mode 100644 index 000000000..7adae84fd --- /dev/null +++ b/src/yunohost/tests/test_dns.py @@ -0,0 +1,66 @@ +import pytest + +import yaml +import os + +from moulinette.utils.filesystem import read_toml + +from yunohost.utils.error import YunohostError, YunohostValidationError +from yunohost.dns import ( + DOMAIN_REGISTRAR_LIST_PATH, + _get_dns_zone_for_domain, + _get_registrar_config_section +) + + +def setup_function(function): + + clean() + + +def teardown_function(function): + + clean() + + +def clean(): + pass + + +# DNS utils testing +def test_get_dns_zone_from_domain_existing(): + assert _get_dns_zone_for_domain("yunohost.org") == "yunohost.org" + assert _get_dns_zone_for_domain("donate.yunohost.org") == "yunohost.org" + assert _get_dns_zone_for_domain("fr.wikipedia.org") == "wikipedia.org" + assert _get_dns_zone_for_domain("www.fr.wikipedia.org") == "wikipedia.org" + assert _get_dns_zone_for_domain("non-existing-domain.yunohost.org") == "yunohost.org" + assert _get_dns_zone_for_domain("yolo.nohost.me") == "nohost.me" + assert _get_dns_zone_for_domain("foo.yolo.nohost.me") == "nohost.me" + assert _get_dns_zone_for_domain("yolo.test") == "test" + assert _get_dns_zone_for_domain("foo.yolo.test") == "test" + + +# Domain registrar testing +def test_registrar_list_integrity(): + assert read_toml(DOMAIN_REGISTRAR_LIST_PATH) + + +def test_magic_guess_registrar_weird_domain(): + assert _get_registrar_config_section("yolo.test")["explanation"]["value"] is None + + +def test_magic_guess_registrar_ovh(): + assert _get_registrar_config_section("yolo.yunohost.org")["explanation"]["value"] == "ovh" + + +def test_magic_guess_registrar_yunodyndns(): + assert _get_registrar_config_section("yolo.nohost.me")["explanation"]["value"] == "yunohost" + + +#def domain_dns_suggest(domain): +# return yunohost.dns.domain_dns_conf(domain) +# +# +#def domain_dns_push(domain, dry_run): +# import yunohost.dns +# return yunohost.dns.domain_registrar_push(domain, dry_run) diff --git a/src/yunohost/tests/test_domains.py b/src/yunohost/tests/test_domains.py index c75954118..04f434b6c 100644 --- a/src/yunohost/tests/test_domains.py +++ b/src/yunohost/tests/test_domains.py @@ -1,24 +1,18 @@ import pytest - -import yaml import os from moulinette.core import MoulinetteError -from yunohost.utils.error import YunohostError, YunohostValidationError -from yunohost.utils.dns import get_dns_zone_from_domain +from yunohost.utils.error import YunohostValidationError from yunohost.domain import ( DOMAIN_SETTINGS_DIR, - REGISTRAR_LIST_PATH, _get_maindomain, domain_add, domain_remove, domain_list, domain_main_domain, - domain_setting, - domain_dns_conf, - domain_registrar_set, - domain_registrar_catalog + domain_config_get, + domain_config_set, ) TEST_DOMAINS = [ @@ -27,6 +21,7 @@ TEST_DOMAINS = [ "other-example.com" ] + def setup_function(function): # Save domain list in variable to avoid multiple calls to domain_list() @@ -40,8 +35,8 @@ def setup_function(function): os.system(f"rm -rf {DOMAIN_SETTINGS_DIR}/{TEST_DOMAINS[0]}.yml") if not _get_maindomain() == TEST_DOMAINS[0]: - domain_main_domain(TEST_DOMAINS[0]) - + domain_main_domain(TEST_DOMAINS[0]) + # Clear other domains for domain in domains: if domain not in TEST_DOMAINS or domain == TEST_DOMAINS[2]: @@ -51,7 +46,6 @@ def setup_function(function): # Reset settings if any os.system(f"rm -rf {DOMAIN_SETTINGS_DIR}/{domain}.yml") - # Create classical second domain of not exist if TEST_DOMAINS[1] not in domains: domain_add(TEST_DOMAINS[1]) @@ -65,101 +59,62 @@ def teardown_function(function): clean() + def clean(): pass + # Domains management testing def test_domain_add(): assert TEST_DOMAINS[2] not in domain_list()["domains"] domain_add(TEST_DOMAINS[2]) assert TEST_DOMAINS[2] in domain_list()["domains"] + def test_domain_add_existing_domain(): - with pytest.raises(MoulinetteError) as e_info: + with pytest.raises(MoulinetteError): assert TEST_DOMAINS[1] in domain_list()["domains"] domain_add(TEST_DOMAINS[1]) + def test_domain_remove(): assert TEST_DOMAINS[1] in domain_list()["domains"] domain_remove(TEST_DOMAINS[1]) assert TEST_DOMAINS[1] not in domain_list()["domains"] + def test_main_domain(): current_main_domain = _get_maindomain() assert domain_main_domain()["current_main_domain"] == current_main_domain + def test_main_domain_change_unknown(): - with pytest.raises(YunohostValidationError) as e_info: + with pytest.raises(YunohostValidationError): domain_main_domain(TEST_DOMAINS[2]) + def test_change_main_domain(): assert _get_maindomain() != TEST_DOMAINS[1] domain_main_domain(TEST_DOMAINS[1]) - assert _get_maindomain() == TEST_DOMAINS[1] + assert _get_maindomain() == TEST_DOMAINS[1] + # Domain settings testing -def test_domain_setting_get_default_xmpp_main_domain(): - assert TEST_DOMAINS[0] in domain_list()["domains"] - assert domain_setting(TEST_DOMAINS[0], "xmpp") == True +def test_domain_config_get_default(): + assert domain_config_get(TEST_DOMAINS[0], "feature.xmpp.xmpp") == 1 + assert domain_config_get(TEST_DOMAINS[1], "feature.xmpp.xmpp") == 0 + assert domain_config_get(TEST_DOMAINS[1], "dns.advanced.ttl") == 3600 -def test_domain_setting_get_default_xmpp(): - assert domain_setting(TEST_DOMAINS[1], "xmpp") == False -def test_domain_setting_get_default_ttl(): - assert domain_setting(TEST_DOMAINS[1], "ttl") == 3600 +def test_domain_config_set(): + assert domain_config_get(TEST_DOMAINS[1], "feature.xmpp.xmpp") == 0 + domain_config_set(TEST_DOMAINS[1], "feature.xmpp.xmpp", "yes") + assert domain_config_get(TEST_DOMAINS[1], "feature.xmpp.xmpp") == 1 -def test_domain_setting_set_int(): - domain_setting(TEST_DOMAINS[1], "ttl", "10") - assert domain_setting(TEST_DOMAINS[1], "ttl") == 10 + domain_config_set(TEST_DOMAINS[1], "dns.advanced.ttl", 10) + assert domain_config_get(TEST_DOMAINS[1], "dns.advanced.ttl") == 10 -def test_domain_setting_set_bool_true(): - domain_setting(TEST_DOMAINS[1], "xmpp", "True") - assert domain_setting(TEST_DOMAINS[1], "xmpp") == True - domain_setting(TEST_DOMAINS[1], "xmpp", "true") - assert domain_setting(TEST_DOMAINS[1], "xmpp") == True - domain_setting(TEST_DOMAINS[1], "xmpp", "t") - assert domain_setting(TEST_DOMAINS[1], "xmpp") == True - domain_setting(TEST_DOMAINS[1], "xmpp", "1") - assert domain_setting(TEST_DOMAINS[1], "xmpp") == True - domain_setting(TEST_DOMAINS[1], "xmpp", "yes") - assert domain_setting(TEST_DOMAINS[1], "xmpp") == True - domain_setting(TEST_DOMAINS[1], "xmpp", "y") - assert domain_setting(TEST_DOMAINS[1], "xmpp") == True -def test_domain_setting_set_bool_false(): - domain_setting(TEST_DOMAINS[1], "xmpp", "False") - assert domain_setting(TEST_DOMAINS[1], "xmpp") == False - domain_setting(TEST_DOMAINS[1], "xmpp", "false") - assert domain_setting(TEST_DOMAINS[1], "xmpp") == False - domain_setting(TEST_DOMAINS[1], "xmpp", "f") - assert domain_setting(TEST_DOMAINS[1], "xmpp") == False - domain_setting(TEST_DOMAINS[1], "xmpp", "0") - assert domain_setting(TEST_DOMAINS[1], "xmpp") == False - domain_setting(TEST_DOMAINS[1], "xmpp", "no") - assert domain_setting(TEST_DOMAINS[1], "xmpp") == False - domain_setting(TEST_DOMAINS[1], "xmpp", "n") - assert domain_setting(TEST_DOMAINS[1], "xmpp") == False - -def test_domain_settings_unknown(): - with pytest.raises(YunohostValidationError) as e_info: - domain_setting(TEST_DOMAINS[2], "xmpp", "False") - -# DNS utils testing -def test_get_dns_zone_from_domain_existing(): - assert get_dns_zone_from_domain("donate.yunohost.org") == "yunohost.org" - -def test_get_dns_zone_from_domain_not_existing(): - assert get_dns_zone_from_domain("non-existing-domain.yunohost.org") == "yunohost.org" - -# Domain registrar testing -def test_registrar_list_yaml_integrity(): - yaml.load(open(REGISTRAR_LIST_PATH, 'r')) - -def test_domain_registrar_catalog(): - domain_registrar_catalog() - -def test_domain_registrar_catalog_full(): - domain_registrar_catalog(None, True) - -def test_domain_registrar_catalog_registrar(): - domain_registrar_catalog("ovh") +def test_domain_configs_unknown(): + with pytest.raises(YunohostValidationError): + domain_config_get(TEST_DOMAINS[2], "feature.xmpp.xmpp.xmpp") diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 6bc8384bf..7c839e359 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -170,7 +170,9 @@ class ConfigPanel: raise YunohostError(f"The filter key {filter_key} has too many sub-levels, the max is 3.", raw_msg=True) if not os.path.exists(self.config_path): + logger.debug(f"Config panel {self.config_path} doesn't exists") return None + toml_config_panel = self._get_toml() # Check TOML config panel is in a supported version diff --git a/src/yunohost/utils/dns.py b/src/yunohost/utils/dns.py index 848cafeba..9af6df8d6 100644 --- a/src/yunohost/utils/dns.py +++ b/src/yunohost/utils/dns.py @@ -21,6 +21,8 @@ import dns.resolver from moulinette.utils.filesystem import read_file +YNH_DYNDNS_DOMAINS = ["nohost.me", "noho.st", "ynh.fr"] + # Lazy dev caching to avoid re-reading the file multiple time when calling # dig() often during same yunohost operation external_resolvers_ = [] @@ -90,33 +92,3 @@ def dig( return ("ok", answers) - -def get_dns_zone_from_domain(domain): - # TODO Check if this function is YNH_DYNDNS_DOMAINS compatible - """ - Get the DNS zone of a domain - - Keyword arguments: - domain -- The domain name - - """ - - # For foo.bar.baz.gni we want to scan all the parent domains - # (including the domain itself) - # foo.bar.baz.gni - # bar.baz.gni - # baz.gni - # gni - parent_list = [domain.split(".", i)[-1] - for i, _ in enumerate(domain.split("."))] - - for parent in parent_list: - - # Check if there's a NS record for that domain - answer = dig(parent, rdtype="NS", full_answers=True, resolvers="force_external") - if answer[0] == "ok": - # Domain is dns_zone - return parent - - # FIXME: returning None will probably trigger bugs when this happens, code expects a domain string - return None diff --git a/src/yunohost/utils/ldap.py b/src/yunohost/utils/ldap.py index 4f571ce6f..9edb2960b 100644 --- a/src/yunohost/utils/ldap.py +++ b/src/yunohost/utils/ldap.py @@ -101,7 +101,8 @@ class LDAPInterface: except ldap.SERVER_DOWN: raise YunohostError( "Service slapd is not running but is required to perform this action ... " - "You can try to investigate what's happening with 'systemctl status slapd'" + "You can try to investigate what's happening with 'systemctl status slapd'", + raw_msg=True ) # Check that we are indeed logged in with the right identity @@ -289,7 +290,7 @@ class LDAPInterface: attr_found[0], attr_found[1], ) - raise MoulinetteError( + raise YunohostError( "ldap_attribute_already_exists", attribute=attr_found[0], value=attr_found[1], From af0c12a62b81a08be798f475d6c1098623fafb84 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 13 Sep 2021 03:11:23 +0200 Subject: [PATCH 078/130] Unused imports --- src/yunohost/domain.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 0bdede11d..f16358558 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -28,9 +28,7 @@ import os from moulinette import m18n, Moulinette from moulinette.core import MoulinetteError from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import ( - mkdir, write_to_file, read_yaml, write_to_yaml -) +from moulinette.utils.filesystem import write_to_file from yunohost.app import ( app_ssowatconf, From 6031cc784699aa5fbf933971d560bf6eb3afe248 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 13 Sep 2021 17:16:22 +0200 Subject: [PATCH 079/130] Misc fixes for cert commands --- data/actionsmap/yunohost.yml | 2 +- data/hooks/conf_regen/01-yunohost | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index c118c90a2..b961460d0 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -647,7 +647,7 @@ domain: action: store_true cert: - subcategory_help: Manage domains DNS + subcategory_help: Manage domain certificates actions: ### certificate_status() status: diff --git a/data/hooks/conf_regen/01-yunohost b/data/hooks/conf_regen/01-yunohost index dd018e8f1..e9c0fc4aa 100755 --- a/data/hooks/conf_regen/01-yunohost +++ b/data/hooks/conf_regen/01-yunohost @@ -82,7 +82,7 @@ EOF # Cron job that renew lets encrypt certificates if there's any that needs renewal cat > $pending_dir/etc/cron.daily/yunohost-certificate-renew << EOF #!/bin/bash -yunohost domain cert-renew --email +yunohost domain cert renew --email EOF # If we subscribed to a dyndns domain, add the corresponding cron From 2782a89a645b56c462717616093d43f53f95e1c3 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 13 Sep 2021 20:55:54 +0200 Subject: [PATCH 080/130] domain config: Add notes about other config stuff we may want to implement in the future --- data/other/config_domain.toml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/data/other/config_domain.toml b/data/other/config_domain.toml index 20963764b..af23b5e04 100644 --- a/data/other/config_domain.toml +++ b/data/other/config_domain.toml @@ -1,6 +1,15 @@ version = "1.0" i18n = "domain_config" +# +# Other things we may want to implement in the future: +# +# - maindomain handling +# - default app +# - autoredirect www in nginx conf +# - ? +# + [feature] [feature.mail] From fc07abf871dc3a0f1fc31c356ba1eb805392ee2d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 13 Sep 2021 20:56:43 +0200 Subject: [PATCH 081/130] autodns: Misc refactoring to have dns push dry-run somewhat working + implement diff strategy --- data/other/registrar_list.toml | 10 ++- src/yunohost/dns.py | 131 ++++++++++++++++++++++++----- src/yunohost/domain.py | 7 ++ src/yunohost/tests/test_dns.py | 29 ++++--- src/yunohost/tests/test_domains.py | 7 ++ 5 files changed, 148 insertions(+), 36 deletions(-) diff --git a/data/other/registrar_list.toml b/data/other/registrar_list.toml index 1c2e73111..8407fce42 100644 --- a/data/other/registrar_list.toml +++ b/data/other/registrar_list.toml @@ -456,15 +456,17 @@ [ovh] [ovh.auth_entrypoint] - type = "string" - redact = true - + type = "select" + choices = ["ovh-eu", "ovh-ca", "soyoustart-eu", "soyoustart-ca", "kimsufi-eu", "kimsufi-ca"] + default = "ovh-eu" + [ovh.auth_application_key] type = "string" redact = true [ovh.auth_application_secret] - type = "password" + type = "string" + redact = true [ovh.auth_consumer_key] type = "string" diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index aa5e79c82..a3dd0fd33 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -155,7 +155,7 @@ def _build_dns_conf(base_domain): ipv6 = get_public_ip(6) subdomains = _list_subdomains_of(base_domain) - domains_settings = {domain: domain_config_get(domain) + domains_settings = {domain: domain_config_get(domain, export=True) for domain in [base_domain] + subdomains} base_dns_zone = _get_dns_zone_for_domain(base_domain) @@ -467,7 +467,7 @@ def _get_registrar_config_section(domain): # If parent domain exists in yunohost parent_domain = domain.split(".", 1)[1] if parent_domain in domain_list()["domains"]: - registrar_infos["explanation"] = OrderedDict({ + registrar_infos["registrar"] = OrderedDict({ "type": "alert", "style": "info", "ask": f"This domain is a subdomain of {parent_domain}. DNS registrar configuration should be managed in {parent_domain}'s configuration panel.", # FIXME: i18n @@ -478,7 +478,7 @@ def _get_registrar_config_section(domain): # TODO big project, integrate yunohost's dynette as a registrar-like provider # TODO big project, integrate other dyndns providers such as netlib.re, or cf the list of dyndns providers supported by cloudron... if dns_zone in YNH_DYNDNS_DOMAINS: - registrar_infos["explanation"] = OrderedDict({ + registrar_infos["registrar"] = OrderedDict({ "type": "alert", "style": "success", "ask": "This domain is a nohost.me / nohost.st / ynh.fr and its DNS configuration is therefore automatically handled by Yunohost.", # FIXME: i18n @@ -489,7 +489,7 @@ def _get_registrar_config_section(domain): try: registrar = _relevant_provider_for_domain(dns_zone)[0] except ValueError: - registrar_infos["explanation"] = OrderedDict({ + registrar_infos["registrar"] = OrderedDict({ "type": "alert", "style": "warning", "ask": "YunoHost could not automatically detect the registrar handling this domain. You should manually configure your DNS records following the documentation at https://yunohost.org/dns.", # FIXME : i18n @@ -497,15 +497,19 @@ def _get_registrar_config_section(domain): }) else: - registrar_infos["explanation"] = OrderedDict({ + registrar_infos["registrar"] = OrderedDict({ "type": "alert", "style": "info", - "ask": f"YunoHost automatically detected that this domain is handled by the registrar **{registrar}**. If you want, YunoHost will automatically configure this DNS zone, if you provide it with the following informations. You can also manually configure your DNS records following the documentation as https://yunohost.org/dns.", # FIXME: i18n + "ask": f"YunoHost automatically detected that this domain is handled by the registrar **{registrar}**. If you want, YunoHost will automatically configure this DNS zone, if you provide it with the appropriate API credentials. You can also manually configure your DNS records following the documentation as https://yunohost.org/dns.", # FIXME: i18n "value": registrar }) # TODO : add a help tip with the link to the registar's API doc (c.f. Lexicon's README) registrar_list = read_toml(DOMAIN_REGISTRAR_LIST_PATH) - registrar_infos.update(registrar_list[registrar]) + registrar_credentials = registrar_list[registrar] + for credential, infos in registrar_credentials.items(): + infos["default"] = infos.get("default", "") + infos["optional"] = infos.get("optional", "False") + registrar_infos.update(registrar_credentials) return OrderedDict(registrar_infos) @@ -521,14 +525,25 @@ def domain_registrar_push(operation_logger, domain, dry_run=False): _assert_domain_exists(domain) - registrar_settings = domain_config_get(domain, key='', full=True) + settings = domain_config_get(domain, key='dns.registrar') - if not registrar_settings: - raise YunohostValidationError("registrar_is_not_set", domain=domain) + registrar_id = settings["dns.registrar.registrar"].get("value") + + if not registrar_id or registrar_id == "yunohost": + raise YunohostValidationError("registrar_push_not_applicable", domain=domain) + + registrar_credentials = { + k.split('.')[-1]: v["value"] + for k, v in settings.items() + if k != "dns.registrar.registar" + } + + if not all(registrar_credentials.values()): + raise YunohostValidationError("registrar_is_not_configured", domain=domain) # Convert the generated conf into a format that matches what we'll fetch using the API # Makes it easier to compare "wanted records" with "current records on remote" - dns_conf = [] + wanted_records = [] for records in _build_dns_conf(domain).values(): for record in records: @@ -540,7 +555,7 @@ def domain_registrar_push(operation_logger, domain, dry_run=False): if content == "@" and record["type"] == "CNAME": content = domain + "." - dns_conf.append({ + wanted_records.append({ "name": name, "type": type_, "ttl": record["ttl"], @@ -551,23 +566,24 @@ def domain_registrar_push(operation_logger, domain, dry_run=False): # See https://github.com/AnalogJ/lexicon/issues/282 and https://github.com/AnalogJ/lexicon/pull/371 # They say it's trivial to implement it! # And yet, it is still not done/merged - dns_conf = [record for record in dns_conf if record["type"] != "CAA"] + wanted_records = [record for record in wanted_records if record["type"] != "CAA"] # Construct the base data structure to use lexicon's API. + base_config = { - "provider_name": registrar_settings["name"], + "provider_name": registrar_id, "domain": domain, - registrar_settings["name"]: registrar_settings["options"] + registrar_id: registrar_credentials } # Fetch all types present in the generated records - current_remote_records = [] + current_records = [] # Get unique types present in the generated records types = ["A", "AAAA", "MX", "TXT", "CNAME", "SRV"] for key in types: - print("fetcing type: " + key) + print("fetching type: " + key) fetch_records_for_type = { "action": "list", "type": key, @@ -577,12 +593,87 @@ def domain_registrar_push(operation_logger, domain, dry_run=False): .with_dict(dict_object=base_config) .with_dict(dict_object=fetch_records_for_type) ) - current_remote_records.extend(LexiconClient(query).execute()) + current_records.extend(LexiconClient(query).execute()) + + # Ignore records which are for a higher-level domain + # i.e. we don't care about the records for domain.tld when pushing yuno.domain.tld + current_records = [r for r in current_records if r['name'].endswith(f'.{domain}')] + + # Step 0 : Get the list of unique (type, name) + # And compare the current and wanted records + # + # i.e. we want this kind of stuff: + # wanted current + # (A, .domain.tld) 1.2.3.4 1.2.3.4 + # (A, www.domain.tld) 1.2.3.4 5.6.7.8 + # (A, foobar.domain.tld) 1.2.3.4 + # (AAAA, .domain.tld) 2001::abcd + # (MX, .domain.tld) 10 domain.tld [10 mx1.ovh.net, 20 mx2.ovh.net] + # (TXT, .domain.tld) "v=spf1 ..." ["v=spf1", "foobar"] + # (SRV, .domain.tld) 0 5 5269 domain.tld + changes = {"delete": [], "update": [], "create": []} + type_and_names = set([(r["type"], r["name"]) for r in current_records + wanted_records]) + comparison = {type_and_name: {"current": [], "wanted": []} for type_and_name in type_and_names} + + for record in current_records: + comparison[(record["type"], record["name"])]["current"].append(record) + + for record in wanted_records: + comparison[(record["type"], record["name"])]["wanted"].append(record) + + for type_and_name, records in comparison.items(): + # + # Step 1 : compute a first "diff" where we remove records which are the same on both sides + # NB / FIXME? : in all this we ignore the TTL value for now... + # + diff = {"current": [], "wanted": []} + current_contents = [r["content"] for r in records["current"]] + wanted_contents = [r["content"] for r in records["wanted"]] + + print("--------") + print(type_and_name) + print(current_contents) + print(wanted_contents) + + for record in records["current"]: + if record["content"] not in wanted_contents: + diff["current"].append(record) + for record in records["wanted"]: + if record["content"] not in current_contents: + diff["wanted"].append(record) + + # + # Step 2 : simple case: 0 or 1 record on one side, 0 or 1 on the other + # -> either nothing do (0/0) or a creation (0/1) or a deletion (1/0), or an update (1/1) + # + if len(diff["current"]) == 0 and len(diff["wanted"]) == 0: + # No diff, nothing to do + continue + + if len(diff["current"]) == 1 and len(diff["wanted"]) == 0: + changes["delete"].append(diff["current"][0]) + continue + + if len(diff["current"]) == 0 and len(diff["wanted"]) == 1: + changes["create"].append(diff["wanted"][0]) + continue + # + if len(diff["current"]) == 1 and len(diff["wanted"]) == 1: + diff["current"][0]["content"] = diff["wanted"][0]["content"] + changes["update"].append(diff["current"][0]) + continue + + # + # Step 3 : N record on one side, M on the other, watdo # FIXME + # + for record in diff["wanted"]: + print(f"Dunno watdo with {type_and_name} : {record['content']}") + for record in diff["current"]: + print(f"Dunno watdo with {type_and_name} : {record['content']}") - changes = {} if dry_run: - return {"current_records": current_remote_records, "dns_conf": dns_conf, "changes": changes} + return {"changes": changes} operation_logger.start() diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index f16358558..f0a42a2d5 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -436,12 +436,19 @@ class DomainConfigPanel(ConfigPanel): toml['feature']['xmpp']['xmpp']['default'] = 1 if self.domain == _get_maindomain() else 0 toml['dns']['registrar'] = _get_registrar_config_section(self.domain) + # FIXME: Ugly hack to save the registar id/value and reinject it in _load_current_values ... + self.registar_id = toml['dns']['registrar']['registrar']['value'] + return toml def _load_current_values(self): + # TODO add mechanism to share some settings with other domains on the same zone super()._load_current_values() + # FIXME: Ugly hack to save the registar id/value and reinject it in _load_current_values ... + self.values["registrar"] = self.registar_id + # # # Stuff managed in other files diff --git a/src/yunohost/tests/test_dns.py b/src/yunohost/tests/test_dns.py index 7adae84fd..58c2be3a5 100644 --- a/src/yunohost/tests/test_dns.py +++ b/src/yunohost/tests/test_dns.py @@ -1,15 +1,13 @@ import pytest -import yaml -import os - from moulinette.utils.filesystem import read_toml -from yunohost.utils.error import YunohostError, YunohostValidationError +from yunohost.domain import domain_add, domain_remove from yunohost.dns import ( DOMAIN_REGISTRAR_LIST_PATH, _get_dns_zone_for_domain, - _get_registrar_config_section + _get_registrar_config_section, + _build_dns_conf, ) @@ -46,21 +44,28 @@ def test_registrar_list_integrity(): def test_magic_guess_registrar_weird_domain(): - assert _get_registrar_config_section("yolo.test")["explanation"]["value"] is None + assert _get_registrar_config_section("yolo.test")["registrar"]["value"] is None def test_magic_guess_registrar_ovh(): - assert _get_registrar_config_section("yolo.yunohost.org")["explanation"]["value"] == "ovh" + assert _get_registrar_config_section("yolo.yunohost.org")["registrar"]["value"] == "ovh" def test_magic_guess_registrar_yunodyndns(): - assert _get_registrar_config_section("yolo.nohost.me")["explanation"]["value"] == "yunohost" + assert _get_registrar_config_section("yolo.nohost.me")["registrar"]["value"] == "yunohost" -#def domain_dns_suggest(domain): -# return yunohost.dns.domain_dns_conf(domain) -# -# +@pytest.fixture +def example_domain(): + domain_add("example.tld") + yield "example_tld" + domain_remove("example.tld") + + +def test_domain_dns_suggest(example_domain): + + assert _build_dns_conf(example_domain) + #def domain_dns_push(domain, dry_run): # import yunohost.dns # return yunohost.dns.domain_registrar_push(domain, dry_run) diff --git a/src/yunohost/tests/test_domains.py b/src/yunohost/tests/test_domains.py index 04f434b6c..b964d2ab6 100644 --- a/src/yunohost/tests/test_domains.py +++ b/src/yunohost/tests/test_domains.py @@ -106,6 +106,13 @@ def test_domain_config_get_default(): assert domain_config_get(TEST_DOMAINS[1], "dns.advanced.ttl") == 3600 +def test_domain_config_get_export(): + + assert domain_config_get(TEST_DOMAINS[0], export=True)["xmpp"] == 1 + assert domain_config_get(TEST_DOMAINS[1], export=True)["xmpp"] == 0 + assert domain_config_get(TEST_DOMAINS[1], export=True)["ttl"] == 3600 + + def test_domain_config_set(): assert domain_config_get(TEST_DOMAINS[1], "feature.xmpp.xmpp") == 0 domain_config_set(TEST_DOMAINS[1], "feature.xmpp.xmpp", "yes") From 0a21be694c41f4c06fe012447478bc4fc7c3d57e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 13 Sep 2021 22:35:38 +0200 Subject: [PATCH 082/130] We don't need to add .. to pythonpath during tests? --- src/yunohost/tests/conftest.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/yunohost/tests/conftest.py b/src/yunohost/tests/conftest.py index 8c00693c0..d87ef445e 100644 --- a/src/yunohost/tests/conftest.py +++ b/src/yunohost/tests/conftest.py @@ -1,14 +1,11 @@ import os import pytest -import sys import moulinette from moulinette import m18n, Moulinette from yunohost.utils.error import YunohostError from contextlib import contextmanager -sys.path.append("..") - @pytest.fixture(scope="session", autouse=True) def clone_test_app(request): @@ -77,6 +74,7 @@ moulinette.core.Moulinette18n.n = new_m18nn def pytest_cmdline_main(config): + import sys sys.path.insert(0, "/usr/lib/moulinette/") import yunohost From d5e366511af8f644362fb49b90e60346845d0394 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 14 Sep 2021 18:04:02 +0200 Subject: [PATCH 083/130] autodns: Moar fixes and stuff after tests on the battlefield --- data/actionsmap/yunohost.yml | 6 + data/other/registrar_list.toml | 27 ++-- src/yunohost/dns.py | 233 ++++++++++++++++++++------------- src/yunohost/domain.py | 4 +- 4 files changed, 169 insertions(+), 101 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index b961460d0..93391a81b 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -645,6 +645,12 @@ domain: full: --dry-run help: Only display what's to be pushed action: store_true + --autoremove: + help: Also autoremove records which are stale or not part of the recommended configuration + action: store_true + --purge: + help: Delete all records + action: store_true cert: subcategory_help: Manage domain certificates diff --git a/data/other/registrar_list.toml b/data/other/registrar_list.toml index 8407fce42..406066dc9 100644 --- a/data/other/registrar_list.toml +++ b/data/other/registrar_list.toml @@ -4,7 +4,8 @@ redact = true [aliyun.auth_secret] - type = "password" + type = "string" + redact = true [aurora] [aurora.auth_api_key] @@ -12,7 +13,8 @@ redact = true [aurora.auth_secret_key] - type = "password" + type = "string" + redact = true [azure] [azure.auth_client_id] @@ -20,7 +22,8 @@ redact = true [azure.auth_client_secret] - type = "password" + type = "string" + redact = true [azure.auth_tenant_id] type = "string" @@ -215,7 +218,8 @@ redact = true [exoscale.auth_secret] - type = "password" + type = "string" + redact = true [gandi] [gandi.auth_token] @@ -233,7 +237,8 @@ redact = true [gehirn.auth_secret] - type = "password" + type = "string" + redact = true [glesys] [glesys.auth_username] @@ -250,7 +255,8 @@ redact = true [godaddy.auth_secret] - type = "password" + type = "string" + redact = true [googleclouddns] [goggleclouddns.auth_service_account_info] @@ -415,7 +421,8 @@ redact = true [netcup.auth_api_password] - type = "password" + type = "string" + redact = true [nfsn] [nfsn.auth_username] @@ -550,7 +557,8 @@ redact = true [route53.auth_access_secret] - type = "password" + type = "string" + redact = true [route53.private_zone] type = "string" @@ -575,7 +583,8 @@ redact = true [sakuracloud.auth_secret] - type = "password" + type = "string" + redact = true [softlayer] [softlayer.auth_username] diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index a3dd0fd33..83319e541 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -26,6 +26,7 @@ import os import re import time +from difflib import SequenceMatcher from collections import OrderedDict from moulinette import m18n, Moulinette @@ -515,7 +516,7 @@ def _get_registrar_config_section(domain): @is_unit_operation() -def domain_registrar_push(operation_logger, domain, dry_run=False): +def domain_registrar_push(operation_logger, domain, dry_run=False, autoremove=False, purge=False): """ Send DNS records to the previously-configured registrar of the domain. """ @@ -527,15 +528,15 @@ def domain_registrar_push(operation_logger, domain, dry_run=False): settings = domain_config_get(domain, key='dns.registrar') - registrar_id = settings["dns.registrar.registrar"].get("value") + registrar = settings["dns.registrar.registrar"].get("value") - if not registrar_id or registrar_id == "yunohost": + if not registrar or registrar in ["None", "yunohost"]: raise YunohostValidationError("registrar_push_not_applicable", domain=domain) registrar_credentials = { - k.split('.')[-1]: v["value"] - for k, v in settings.items() - if k != "dns.registrar.registar" + k.split('.')[-1]: v["value"] + for k, v in settings.items() + if k != "dns.registrar.registar" } if not all(registrar_credentials.values()): @@ -547,11 +548,12 @@ def domain_registrar_push(operation_logger, domain, dry_run=False): for records in _build_dns_conf(domain).values(): for record in records: - # Make sure we got "absolute" values instead of @ - name = f"{record['name']}.{domain}" if record["name"] != "@" else f".{domain}" + # Make sure the name is a FQDN + name = f"{record['name']}.{domain}" if record["name"] != "@" else f"{domain}" type_ = record["type"] content = record["value"] + # Make sure the content is also a FQDN (with trailing . ?) if content == "@" and record["type"] == "CNAME": content = domain + "." @@ -568,37 +570,48 @@ def domain_registrar_push(operation_logger, domain, dry_run=False): # And yet, it is still not done/merged wanted_records = [record for record in wanted_records if record["type"] != "CAA"] + if purge: + wanted_records = [] + autoremove = True + # Construct the base data structure to use lexicon's API. base_config = { - "provider_name": registrar_id, + "provider_name": registrar, "domain": domain, - registrar_id: registrar_credentials + registrar: registrar_credentials } - # Fetch all types present in the generated records - current_records = [] - - # Get unique types present in the generated records - types = ["A", "AAAA", "MX", "TXT", "CNAME", "SRV"] - - for key in types: - print("fetching type: " + key) - fetch_records_for_type = { - "action": "list", - "type": key, - } - query = ( + # Ugly hack to be able to fetch all record types at once: + # we initialize a LexiconClient with type: dummytype, + # then trigger ourselves the authentication + list_records + # instead of calling .execute() + query = ( LexiconConfigResolver() .with_dict(dict_object=base_config) - .with_dict(dict_object=fetch_records_for_type) - ) - current_records.extend(LexiconClient(query).execute()) + .with_dict(dict_object={"action": "list", "type": "dummytype"}) + ) + # current_records.extend( + client = LexiconClient(query) + client.provider.authenticate() + current_records = client.provider.list_records() + + # Keep only records for relevant types: A, AAAA, MX, TXT, CNAME, SRV + relevant_types = ["A", "AAAA", "MX", "TXT", "CNAME", "SRV"] + current_records = [r for r in current_records if r["type"] in relevant_types] # Ignore records which are for a higher-level domain # i.e. we don't care about the records for domain.tld when pushing yuno.domain.tld current_records = [r for r in current_records if r['name'].endswith(f'.{domain}')] + for record in current_records: + + # Try to get rid of weird stuff like ".domain.tld" or "@.domain.tld" + record["name"] = record["name"].strip("@").strip(".") + + # Some API return '@' in content and we shall convert it to absolute/fqdn + record["content"] = record["content"].replace('@.', domain + ".").replace('@', domain + ".") + # Step 0 : Get the list of unique (type, name) # And compare the current and wanted records # @@ -622,105 +635,145 @@ def domain_registrar_push(operation_logger, domain, dry_run=False): comparison[(record["type"], record["name"])]["wanted"].append(record) for type_and_name, records in comparison.items(): + # # Step 1 : compute a first "diff" where we remove records which are the same on both sides # NB / FIXME? : in all this we ignore the TTL value for now... # - diff = {"current": [], "wanted": []} - current_contents = [r["content"] for r in records["current"]] wanted_contents = [r["content"] for r in records["wanted"]] + current_contents = [r["content"] for r in records["current"]] - print("--------") - print(type_and_name) - print(current_contents) - print(wanted_contents) - - for record in records["current"]: - if record["content"] not in wanted_contents: - diff["current"].append(record) - for record in records["wanted"]: - if record["content"] not in current_contents: - diff["wanted"].append(record) + current = [r for r in records["current"] if r["content"] not in wanted_contents] + wanted = [r for r in records["wanted"] if r["content"] not in current_contents] # # Step 2 : simple case: 0 or 1 record on one side, 0 or 1 on the other # -> either nothing do (0/0) or a creation (0/1) or a deletion (1/0), or an update (1/1) # - if len(diff["current"]) == 0 and len(diff["wanted"]) == 0: + if len(current) == 0 and len(wanted) == 0: # No diff, nothing to do continue - if len(diff["current"]) == 1 and len(diff["wanted"]) == 0: - changes["delete"].append(diff["current"][0]) + if len(current) == 1 and len(wanted) == 0: + changes["delete"].append(current[0]) continue - if len(diff["current"]) == 0 and len(diff["wanted"]) == 1: - changes["create"].append(diff["wanted"][0]) + if len(current) == 0 and len(wanted) == 1: + changes["create"].append(wanted[0]) continue - # - if len(diff["current"]) == 1 and len(diff["wanted"]) == 1: - diff["current"][0]["content"] = diff["wanted"][0]["content"] - changes["update"].append(diff["current"][0]) + + if len(current) == 1 and len(wanted) == 1: + current[0]["old_content"] = current[0]["content"] + current[0]["content"] = wanted[0]["content"] + changes["update"].append(current[0]) continue # - # Step 3 : N record on one side, M on the other, watdo # FIXME + # Step 3 : N record on one side, M on the other # - for record in diff["wanted"]: - print(f"Dunno watdo with {type_and_name} : {record['content']}") - for record in diff["current"]: - print(f"Dunno watdo with {type_and_name} : {record['content']}") + # Fuzzy matching strategy: + # For each wanted record, try to find a current record which looks like the wanted one + # -> if found, trigger an update + # -> if no match found, trigger a create + # + for record in wanted: + def likeliness(r): + # We compute this only on the first 100 chars, to have a high value even for completely different DKIM keys + return SequenceMatcher(None, r["content"][:100], record["content"][:100]).ratio() + + matches = sorted(current, key=lambda r: likeliness(r), reverse=True) + if matches and likeliness(matches[0]) > 0.50: + match = matches[0] + # Remove the match from 'current' so that it's not added to the removed stuff later + current.remove(match) + match["old_content"] = match["content"] + match["content"] = record["content"] + changes["update"].append(match) + else: + changes["create"].append(record) + + # + # For all other remaining current records: + # -> trigger deletions + # + for record in current: + changes["delete"].append(record) + + def human_readable_record(action, record): + name = record["name"] + name = name.strip(".") + name = name.replace('.' + domain, "") + name = name.replace(domain, "@") + name = name[:20] + t = record["type"] + if action in ["create", "update"]: + old_content = record.get("old_content", "(None)")[:30] + new_content = record.get("content", "(None)")[:30] + else: + new_content = record.get("old_content", "(None)")[:30] + old_content = record.get("content", "(None)")[:30] + + return f'{name:>20} [{t:^5}] {old_content:^30} -> {new_content:^30}' if dry_run: - return {"changes": changes} + out = [] + for action in ["delete", "create", "update"]: + out.append("\n" + action + ":\n") + for record in changes[action]: + out.append(human_readable_record(action, record)) + + return '\n'.join(out) operation_logger.start() # Push the records - for record in dns_conf: + for action in ["delete", "create", "update"]: + if action == "delete" and not autoremove: + continue - # For each record, first check if one record exists for the same (type, name) couple - # TODO do not push if local and distant records are exactly the same ? - type_and_name = (record["type"], record["name"]) - already_exists = any((r["type"], r["name"]) == type_and_name - for r in current_remote_records) + for record in changes[action]: - # Finally, push the new record or update the existing one - record_to_push = { - "action": "update" if already_exists else "create", - "type": record["type"], - "name": record["name"], - "content": record["value"], - "ttl": record["ttl"], - } + record["action"] = action - # FIXME Removed TTL, because it doesn't work with Gandi. - # See https://github.com/AnalogJ/lexicon/issues/726 (similar issue) - # But I think there is another issue with Gandi. Or I'm misusing the API... - if base_config["provider_name"] == "gandi": - del record_to_push["ttl"] + # Apparently Lexicon yields us some 'id' during fetch + # But wants 'identifier' during push ... + if "id" in record: + record["identifier"] = record["id"] + del record["id"] - print("pushed_record:", record_to_push) + if "old_content" in record: + del record["old_content"] + if registrar == "godaddy": + if record["name"] == domain: + record["name"] = "@." + record["name"] + if record["type"] in ["MX", "SRV"]: + logger.warning(f"Pushing {record['type']} records is not properly supported by Lexicon/Godaddy.") + continue - # FIXME FIXME FIXME: if a matching record already exists multiple time, - # the current code crashes (at least on OVH) ... we need to provide a specific identifier to update - query = ( - LexiconConfigResolver() - .with_dict(dict_object=base_config) - .with_dict(dict_object=record_to_push) - ) + # FIXME Removed TTL, because it doesn't work with Gandi. + # See https://github.com/AnalogJ/lexicon/issues/726 (similar issue) + # But I think there is another issue with Gandi. Or I'm misusing the API... + if registrar == "gandi": + del record["ttl"] - print(query) - print(query.__dict__) - results = LexiconClient(query).execute() - print("results:", results) - # print("Failed" if results == False else "Ok") + logger.info(action + " : " + human_readable_record(action, record)) - # FIXME FIXME FIXME : if one create / update crash, it shouldn't block everything + query = ( + LexiconConfigResolver() + .with_dict(dict_object=base_config) + .with_dict(dict_object=record) + ) - # FIXME : is it possible to push multiple create/update request at once ? + try: + result = LexiconClient(query).execute() + except Exception as e: + logger.error(f"Failed to {action} record {record['type']}/{record['name']} : {e}") + else: + if result: + logger.success("Done!") + else: + logger.error("Uhoh!?") - -# def domain_config_fetch(domain, key, value): + # FIXME : implement a system to properly report what worked and what did not at the end of the command.. diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index f0a42a2d5..9c9ad2d5e 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -491,6 +491,6 @@ def domain_dns_suggest(domain): return yunohost.dns.domain_dns_suggest(domain) -def domain_dns_push(domain, dry_run): +def domain_dns_push(domain, dry_run, autoremove, purge): import yunohost.dns - return yunohost.dns.domain_registrar_push(domain, dry_run) + return yunohost.dns.domain_registrar_push(domain, dry_run, autoremove, purge) From fa271db569d299ceb126a470717dd6e7a85ef23a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 14 Sep 2021 19:14:33 +0200 Subject: [PATCH 084/130] ci: Don't cd to src/yunohost to run pytest (to prevent ambiguous 'import dns.resolver' trying to import our own dns.py :s) --- .gitlab/ci/test.gitlab-ci.yml | 45 ++++++++++++----------------------- 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index f270ba982..b3aea606f 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -88,8 +88,7 @@ test-helpers: test-domains: extends: .test-stage script: - - cd src/yunohost - - python3 -m pytest tests/test_domains.py + - python3 -m pytest src/yunohost/tests/test_domains.py only: changes: - src/yunohost/domain.py @@ -97,8 +96,7 @@ test-domains: test-dns: extends: .test-stage script: - - cd src/yunohost - - python3 -m pytest tests/test_dns.py + - python3 -m pytest src/yunohost/tests/test_dns.py only: changes: - src/yunohost/dns.py @@ -107,8 +105,7 @@ test-dns: test-apps: extends: .test-stage script: - - cd src/yunohost - - python3 -m pytest tests/test_apps.py + - python3 -m pytest src/yunohost/tests/test_apps.py only: changes: - src/yunohost/app.py @@ -116,8 +113,7 @@ test-apps: test-appscatalog: extends: .test-stage script: - - cd src/yunohost - - python3 -m pytest tests/test_appscatalog.py + - python3 -m pytest src/yunohost/tests/test_appscatalog.py only: changes: - src/yunohost/app.py @@ -125,8 +121,7 @@ test-appscatalog: test-appurl: extends: .test-stage script: - - cd src/yunohost - - python3 -m pytest tests/test_appurl.py + - python3 -m pytest src/yunohost/tests/test_appurl.py only: changes: - src/yunohost/app.py @@ -134,8 +129,7 @@ test-appurl: test-questions: extends: .test-stage script: - - cd src/yunohost - - python3 -m pytest tests/test_questions.py + - python3 -m pytest src/yunohost/tests/test_questions.py only: changes: - src/yunohost/utils/config.py @@ -143,8 +137,7 @@ test-questions: test-app-config: extends: .test-stage script: - - cd src/yunohost - - python3 -m pytest tests/test_app_config.py + - python3 -m pytest src/yunohost/tests/test_app_config.py only: changes: - src/yunohost/app.py @@ -153,8 +146,7 @@ test-app-config: test-changeurl: extends: .test-stage script: - - cd src/yunohost - - python3 -m pytest tests/test_changeurl.py + - python3 -m pytest src/yunohost/tests/test_changeurl.py only: changes: - src/yunohost/app.py @@ -162,8 +154,7 @@ test-changeurl: test-backuprestore: extends: .test-stage script: - - cd src/yunohost - - python3 -m pytest tests/test_backuprestore.py + - python3 -m pytest src/yunohost/tests/test_backuprestore.py only: changes: - src/yunohost/backup.py @@ -171,8 +162,7 @@ test-backuprestore: test-permission: extends: .test-stage script: - - cd src/yunohost - - python3 -m pytest tests/test_permission.py + - python3 -m pytest src/yunohost/tests/test_permission.py only: changes: - src/yunohost/permission.py @@ -180,8 +170,7 @@ test-permission: test-settings: extends: .test-stage script: - - cd src/yunohost - - python3 -m pytest tests/test_settings.py + - python3 -m pytest src/yunohost/tests/test_settings.py only: changes: - src/yunohost/settings.py @@ -189,8 +178,7 @@ test-settings: test-user-group: extends: .test-stage script: - - cd src/yunohost - - python3 -m pytest tests/test_user-group.py + - python3 -m pytest src/yunohost/tests/test_user-group.py only: changes: - src/yunohost/user.py @@ -198,8 +186,7 @@ test-user-group: test-regenconf: extends: .test-stage script: - - cd src/yunohost - - python3 -m pytest tests/test_regenconf.py + - python3 -m pytest src/yunohost/tests/test_regenconf.py only: changes: - src/yunohost/regenconf.py @@ -207,8 +194,7 @@ test-regenconf: test-service: extends: .test-stage script: - - cd src/yunohost - - python3 -m pytest tests/test_service.py + - python3 -m pytest src/yunohost/tests/test_service.py only: changes: - src/yunohost/service.py @@ -216,8 +202,7 @@ test-service: test-ldapauth: extends: .test-stage script: - - cd src/yunohost - - python3 -m pytest tests/test_ldapauth.py + - python3 -m pytest src/yunohost/tests/test_ldapauth.py only: changes: - src/yunohost/authenticators/*.py From 57f00190b0b6b272429530d94f4c3131d4113b10 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 14 Sep 2021 22:58:43 +0200 Subject: [PATCH 085/130] debian: install the new domain config panel and registrar list toml to /usr/share --- data/other/registrar_list.yml | 218 ---------------------------------- debian/install | 3 +- 2 files changed, 2 insertions(+), 219 deletions(-) delete mode 100644 data/other/registrar_list.yml diff --git a/data/other/registrar_list.yml b/data/other/registrar_list.yml deleted file mode 100644 index a006bd272..000000000 --- a/data/other/registrar_list.yml +++ /dev/null @@ -1,218 +0,0 @@ -aliyun: - - auth_key_id - - auth_secret -aurora: - - auth_api_key - - auth_secret_key -azure: - - auth_client_id - - auth_client_secret - - auth_tenant_id - - auth_subscription_id - - resource_group -cloudflare: - - auth_username - - auth_token - - zone_id -cloudns: - - auth_id - - auth_subid - - auth_subuser - - auth_password - - weight - - port -cloudxns: - - auth_username - - auth_token -conoha: - - auth_region - - auth_token - - auth_username - - auth_password - - auth_tenant_id -constellix: - - auth_username - - auth_token -digitalocean: - - auth_token -dinahosting: - - auth_username - - auth_password -directadmin: - - auth_password - - auth_username - - endpoint -dnsimple: - - auth_token - - auth_username - - auth_password - - auth_2fa -dnsmadeeasy: - - auth_username - - auth_token -dnspark: - - auth_username - - auth_token -dnspod: - - auth_username - - auth_token -dreamhost: - - auth_token -dynu: - - auth_token -easydns: - - auth_username - - auth_token -easyname: - - auth_username - - auth_password -euserv: - - auth_username - - auth_password -exoscale: - - auth_key - - auth_secret -gandi: - - auth_token - - api_protocol -gehirn: - - auth_token - - auth_secret -glesys: - - auth_username - - auth_token -godaddy: - - auth_key - - auth_secret -googleclouddns: - - auth_service_account_info -gransy: - - auth_username - - auth_password -gratisdns: - - auth_username - - auth_password -henet: - - auth_username - - auth_password -hetzner: - - auth_token -hostingde: - - auth_token -hover: - - auth_username - - auth_password -infoblox: - - auth_user - - auth_psw - - ib_view - - ib_host -infomaniak: - - auth_token -internetbs: - - auth_key - - auth_password -inwx: - - auth_username - - auth_password -joker: - - auth_token -linode: - - auth_token -linode4: - - auth_token -localzone: - - filename -luadns: - - auth_username - - auth_token -memset: - - auth_token -mythicbeasts: - - auth_username - - auth_password - - auth_token -namecheap: - - auth_token - - auth_username - - auth_client_ip - - auth_sandbox -namesilo: - - auth_token -netcup: - - auth_customer_id - - auth_api_key - - auth_api_password -nfsn: - - auth_username - - auth_token -njalla: - - auth_token -nsone: - - auth_token -onapp: - - auth_username - - auth_token - - auth_server -online: - - auth_token -ovh: - - auth_entrypoint - - auth_application_key - - auth_application_secret - - auth_consumer_key -plesk: - - auth_username - - auth_password - - plesk_server -pointhq: - - auth_username - - auth_token -powerdns: - - auth_token - - pdns_server - - pdns_server_id - - pdns_disable_notify -rackspace: - - auth_account - - auth_username - - auth_api_key - - auth_token - - sleep_time -rage4: - - auth_username - - auth_token -rcodezero: - - auth_token -route53: - - auth_access_key - - auth_access_secret - - private_zone - - auth_username - - auth_token -safedns: - - auth_token -sakuracloud: - - auth_token - - auth_secret -softlayer: - - auth_username - - auth_api_key -transip: - - auth_username - - auth_api_key -ultradns: - - auth_token - - auth_username - - auth_password -vultr: - - auth_token -yandex: - - auth_token -zeit: - - auth_token -zilore: - - auth_key -zonomi: - - auth_token - - auth_entrypoint diff --git a/debian/install b/debian/install index 55ddb34c6..a653a40ba 100644 --- a/debian/install +++ b/debian/install @@ -8,7 +8,8 @@ data/other/yunoprompt.service /etc/systemd/system/ data/other/password/* /usr/share/yunohost/other/password/ data/other/dpkg-origins/yunohost /etc/dpkg/origins data/other/dnsbl_list.yml /usr/share/yunohost/other/ -data/other/registrar_list.yml /usr/share/yunohost/other/ +data/other/config_domain.toml /usr/share/yunohost/other/ +data/other/registrar_list.toml /usr/share/yunohost/other/ data/other/ffdhe2048.pem /usr/share/yunohost/other/ data/other/* /usr/share/yunohost/yunohost-config/moulinette/ data/templates/* /usr/share/yunohost/templates/ From 217ae87bb3e425fd315ce1a7dd6aea58ad9eb74c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 15 Sep 2021 17:26:39 +0200 Subject: [PATCH 086/130] Fix duplicate cert routes --- data/actionsmap/yunohost.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 93391a81b..6a5e594d3 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -484,7 +484,6 @@ domain: dns-conf: deprecated: true action_help: Generate sample DNS configuration for a domain - api: GET /domains//dns arguments: domain: help: Target domain @@ -510,7 +509,6 @@ domain: cert-status: deprecated: true action_help: List status of current certificates (all by default). - api: GET /domains//cert arguments: domain_list: help: Domains to check @@ -523,7 +521,6 @@ domain: cert-install: deprecated: true action_help: Install Let's Encrypt certificates for given domains (all by default). - api: PUT /domains//cert arguments: domain_list: help: Domains for which to install the certificates @@ -545,7 +542,6 @@ domain: cert-renew: deprecated: true action_help: Renew the Let's Encrypt certificates for given domains (all by default). - api: PUT /domains//cert/renew arguments: domain_list: help: Domains for which to renew the certificates From 96b112ac7f397b966d3a67cd9b80d8ee5776ab82 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 16 Sep 2021 17:49:58 +0200 Subject: [PATCH 087/130] config get: Also inject ask strings in full mode --- src/yunohost/utils/config.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index b11d95138..abb901e84 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -65,10 +65,6 @@ class ConfigPanel: self._load_current_values() self._hydrate() - # Format result in full mode - if mode == "full": - return self.config - # In 'classic' mode, we display the current value if key refer to an option if self.filter_key.count(".") == 2 and mode == "classic": option = self.filter_key.split(".")[-1] @@ -81,13 +77,19 @@ class ConfigPanel: key = f"{panel['id']}.{section['id']}.{option['id']}" if mode == "export": result[option["id"]] = option.get("current_value") + continue + + ask = None + if "ask" in option: + ask = _value_for_locale(option["ask"]) + elif "i18n" in self.config: + ask = m18n.n(self.config["i18n"] + "_" + option["id"]) + + if mode == "full": + # edit self.config directly + option["ask"] = ask else: - if "ask" in option: - result[key] = {"ask": _value_for_locale(option["ask"])} - elif "i18n" in self.config: - result[key] = { - "ask": m18n.n(self.config["i18n"] + "_" + option["id"]) - } + result[key] = {"ask": ask} if "current_value" in option: question_class = ARGUMENTS_TYPE_PARSERS[ option.get("type", "string") @@ -96,7 +98,10 @@ class ConfigPanel: option["current_value"], option ) - return result + if mode == "full": + return self.config + else: + return result def set( self, key=None, value=None, args=None, args_file=None, operation_logger=None From 86a74903db339a669b7c40063f142fc1019d464e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 16 Sep 2021 20:21:41 +0200 Subject: [PATCH 088/130] autodns dryrun: return a proper dict/list structure instead of a raw string --- src/yunohost/dns.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 83319e541..7df6f3aff 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -717,13 +717,12 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, autoremove=Fa return f'{name:>20} [{t:^5}] {old_content:^30} -> {new_content:^30}' if dry_run: - out = [] + out = {"delete": [], "create": [], "update": [], "ignored": []} for action in ["delete", "create", "update"]: - out.append("\n" + action + ":\n") for record in changes[action]: - out.append(human_readable_record(action, record)) + out[action].append(human_readable_record(action, record)) - return '\n'.join(out) + return out operation_logger.start() From bc39788da9da85e8f28b694fec37ef01382276da Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 16 Sep 2021 21:50:44 +0200 Subject: [PATCH 089/130] autodns: minor simplification in create/delete/update strategy --- src/yunohost/dns.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 7df6f3aff..6d589a10e 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -647,25 +647,21 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, autoremove=Fa wanted = [r for r in records["wanted"] if r["content"] not in current_contents] # - # Step 2 : simple case: 0 or 1 record on one side, 0 or 1 on the other - # -> either nothing do (0/0) or a creation (0/1) or a deletion (1/0), or an update (1/1) + # Step 2 : simple case: 0 record on one side, 0 on the other + # -> either nothing do (0/0) or creations (0/N) or deletions (N/0) # if len(current) == 0 and len(wanted) == 0: # No diff, nothing to do continue - if len(current) == 1 and len(wanted) == 0: - changes["delete"].append(current[0]) + elif len(wanted) == 0: + for r in current: + changes["delete"].append(r) continue - if len(current) == 0 and len(wanted) == 1: - changes["create"].append(wanted[0]) - continue - - if len(current) == 1 and len(wanted) == 1: - current[0]["old_content"] = current[0]["content"] - current[0]["content"] = wanted[0]["content"] - changes["update"].append(current[0]) + elif len(current) == 0: + for r in current: + changes["create"].append(r) continue # From 0a404f6d56afabd26bd73226404a61187e9fe7dd Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 17 Sep 2021 00:30:47 +0200 Subject: [PATCH 090/130] autodns: Improve the push system to save managed dns record hashes, similar to the regenconf mecanism --- data/actionsmap/yunohost.yml | 4 +- src/yunohost/dns.py | 89 +++++++++++++++++++++++++----------- src/yunohost/domain.py | 24 ++++++++-- 3 files changed, 85 insertions(+), 32 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 6a5e594d3..b845ded21 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -641,8 +641,8 @@ domain: full: --dry-run help: Only display what's to be pushed action: store_true - --autoremove: - help: Also autoremove records which are stale or not part of the recommended configuration + --force: + help: Also update/remove records which were not originally set by Yunohost, or which have been manually modified action: store_true --purge: help: Delete all records diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 6d589a10e..fdb6d4647 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -26,6 +26,8 @@ import os import re import time +import hashlib + from difflib import SequenceMatcher from collections import OrderedDict @@ -33,7 +35,7 @@ from moulinette import m18n, Moulinette from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_file, write_to_file, read_toml -from yunohost.domain import domain_list, _assert_domain_exists, domain_config_get +from yunohost.domain import domain_list, _assert_domain_exists, domain_config_get, _get_domain_settings, _set_domain_settings from yunohost.utils.dns import dig, YNH_DYNDNS_DOMAINS from yunohost.utils.error import YunohostValidationError from yunohost.utils.network import get_public_ip @@ -516,7 +518,7 @@ def _get_registrar_config_section(domain): @is_unit_operation() -def domain_registrar_push(operation_logger, domain, dry_run=False, autoremove=False, purge=False): +def domain_registrar_push(operation_logger, domain, dry_run=False, force=False, purge=False): """ Send DNS records to the previously-configured registrar of the domain. """ @@ -533,6 +535,8 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, autoremove=Fa if not registrar or registrar in ["None", "yunohost"]: raise YunohostValidationError("registrar_push_not_applicable", domain=domain) + base_dns_zone = _get_dns_zone_for_domain(domain) + registrar_credentials = { k.split('.')[-1]: v["value"] for k, v in settings.items() @@ -549,13 +553,13 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, autoremove=Fa for record in records: # Make sure the name is a FQDN - name = f"{record['name']}.{domain}" if record["name"] != "@" else f"{domain}" + name = f"{record['name']}.{base_dns_zone}" if record["name"] != "@" else base_dns_zone type_ = record["type"] content = record["value"] # Make sure the content is also a FQDN (with trailing . ?) if content == "@" and record["type"] == "CNAME": - content = domain + "." + content = base_dns_zone + "." wanted_records.append({ "name": name, @@ -572,29 +576,31 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, autoremove=Fa if purge: wanted_records = [] - autoremove = True + force = True # Construct the base data structure to use lexicon's API. base_config = { "provider_name": registrar, - "domain": domain, + "domain": base_dns_zone, registrar: registrar_credentials } # Ugly hack to be able to fetch all record types at once: - # we initialize a LexiconClient with type: dummytype, + # we initialize a LexiconClient with a dummy type "all" + # (which lexicon doesnt actually understands) # then trigger ourselves the authentication + list_records # instead of calling .execute() query = ( LexiconConfigResolver() .with_dict(dict_object=base_config) - .with_dict(dict_object={"action": "list", "type": "dummytype"}) + .with_dict(dict_object={"action": "list", "type": "all"}) ) # current_records.extend( client = LexiconClient(query) client.provider.authenticate() current_records = client.provider.list_records() + managed_dns_records_hashes = _get_managed_dns_records_hashes(domain) # Keep only records for relevant types: A, AAAA, MX, TXT, CNAME, SRV relevant_types = ["A", "AAAA", "MX", "TXT", "CNAME", "SRV"] @@ -602,7 +608,7 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, autoremove=Fa # Ignore records which are for a higher-level domain # i.e. we don't care about the records for domain.tld when pushing yuno.domain.tld - current_records = [r for r in current_records if r['name'].endswith(f'.{domain}')] + current_records = [r for r in current_records if r['name'].endswith(f'.{domain}') or r['name'] == domain] for record in current_records: @@ -610,7 +616,10 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, autoremove=Fa record["name"] = record["name"].strip("@").strip(".") # Some API return '@' in content and we shall convert it to absolute/fqdn - record["content"] = record["content"].replace('@.', domain + ".").replace('@', domain + ".") + record["content"] = record["content"].replace('@.', base_dns_zone + ".").replace('@', base_dns_zone + ".") + + # Check if this record was previously set by YunoHost + record["managed_by_yunohost"] = _hash_dns_record(record) in managed_dns_records_hashes # Step 0 : Get the list of unique (type, name) # And compare the current and wanted records @@ -624,7 +633,7 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, autoremove=Fa # (MX, .domain.tld) 10 domain.tld [10 mx1.ovh.net, 20 mx2.ovh.net] # (TXT, .domain.tld) "v=spf1 ..." ["v=spf1", "foobar"] # (SRV, .domain.tld) 0 5 5269 domain.tld - changes = {"delete": [], "update": [], "create": []} + changes = {"delete": [], "update": [], "create": [], "unchanged": []} type_and_names = set([(r["type"], r["name"]) for r in current_records + wanted_records]) comparison = {type_and_name: {"current": [], "wanted": []} for type_and_name in type_and_names} @@ -652,16 +661,15 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, autoremove=Fa # if len(current) == 0 and len(wanted) == 0: # No diff, nothing to do + changes["unchanged"].extend(records["current"]) continue elif len(wanted) == 0: - for r in current: - changes["delete"].append(r) + changes["delete"].extend(current) continue elif len(current) == 0: - for r in current: - changes["create"].append(r) + changes["create"].extend(wanted) continue # @@ -699,8 +707,8 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, autoremove=Fa def human_readable_record(action, record): name = record["name"] name = name.strip(".") - name = name.replace('.' + domain, "") - name = name.replace(domain, "@") + name = name.replace('.' + base_dns_zone, "") + name = name.replace(base_dns_zone, "@") name = name[:20] t = record["type"] if action in ["create", "update"]: @@ -710,10 +718,15 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, autoremove=Fa new_content = record.get("old_content", "(None)")[:30] old_content = record.get("content", "(None)")[:30] - return f'{name:>20} [{t:^5}] {old_content:^30} -> {new_content:^30}' + if not force and action in ["update", "delete"]: + ignored = "" if record["managed_by_yunohost"] else "(ignored, won't be changed by Yunohost unless forced)" + else: + ignored = "" + + return f'{name:>20} [{t:^5}] {old_content:^30} -> {new_content:^30} {ignored}' if dry_run: - out = {"delete": [], "create": [], "update": [], "ignored": []} + out = {"delete": [], "create": [], "update": []} for action in ["delete", "create", "update"]: for record in changes[action]: out[action].append(human_readable_record(action, record)) @@ -722,13 +735,17 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, autoremove=Fa operation_logger.start() + new_managed_dns_records_hashes = [_hash_dns_record(r) for r in changes["unchanged"]] + # Push the records for action in ["delete", "create", "update"]: - if action == "delete" and not autoremove: - continue for record in changes[action]: + if not force and action in ["update", "delete"] and not record["managed_by_yunohost"]: + # Don't overwrite manually-set or manually-modified records + continue + record["action"] = action # Apparently Lexicon yields us some 'id' during fetch @@ -737,11 +754,10 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, autoremove=Fa record["identifier"] = record["id"] del record["id"] - if "old_content" in record: - del record["old_content"] + logger.info(action + " : " + human_readable_record(action, record)) if registrar == "godaddy": - if record["name"] == domain: + if record["name"] == base_dns_zone: record["name"] = "@." + record["name"] if record["type"] in ["MX", "SRV"]: logger.warning(f"Pushing {record['type']} records is not properly supported by Lexicon/Godaddy.") @@ -753,8 +769,6 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, autoremove=Fa if registrar == "gandi": del record["ttl"] - logger.info(action + " : " + human_readable_record(action, record)) - query = ( LexiconConfigResolver() .with_dict(dict_object=base_config) @@ -767,8 +781,29 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, autoremove=Fa logger.error(f"Failed to {action} record {record['type']}/{record['name']} : {e}") else: if result: + new_managed_dns_records_hashes.append(_hash_dns_record(record)) logger.success("Done!") else: logger.error("Uhoh!?") - # FIXME : implement a system to properly report what worked and what did not at the end of the command.. + _set_managed_dns_records_hashes(domain, new_managed_dns_records_hashes) + + # FIXME : implement a system to properly report what worked and what did not at the end of the command.. + + +def _get_managed_dns_records_hashes(domain: str) -> list: + return _get_domain_settings(domain).get("managed_dns_records_hashes", []) + + +def _set_managed_dns_records_hashes(domain: str, hashes: list) -> None: + settings = _get_domain_settings(domain) + settings["managed_dns_records_hashes"] = hashes or [] + _set_domain_settings(domain, settings) + + +def _hash_dns_record(record: dict) -> int: + + fields = ["name", "type", "content"] + record_ = {f: record.get(f) for f in fields} + + return hash(frozenset(record_.items())) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 9c9ad2d5e..a76faafc0 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -28,7 +28,7 @@ import os from moulinette import m18n, Moulinette from moulinette.core import MoulinetteError from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import write_to_file +from moulinette.utils.filesystem import write_to_file, read_yaml, write_to_yaml from yunohost.app import ( app_ssowatconf, @@ -449,6 +449,24 @@ class DomainConfigPanel(ConfigPanel): # FIXME: Ugly hack to save the registar id/value and reinject it in _load_current_values ... self.values["registrar"] = self.registar_id + +def _get_domain_settings(domain: str) -> dict: + + _assert_domain_exists(domain) + + if os.path.exists(f"{DOMAIN_SETTINGS_DIR}/{domain}.yml"): + return read_yaml(f"{DOMAIN_SETTINGS_DIR}/{domain}.yml") or {} + else: + return {} + + +def _set_domain_settings(domain: str, settings: dict) -> None: + + _assert_domain_exists(domain) + + write_to_yaml(f"{DOMAIN_SETTINGS_DIR}/{domain}.yml", settings) + + # # # Stuff managed in other files @@ -491,6 +509,6 @@ def domain_dns_suggest(domain): return yunohost.dns.domain_dns_suggest(domain) -def domain_dns_push(domain, dry_run, autoremove, purge): +def domain_dns_push(domain, dry_run, force, purge): import yunohost.dns - return yunohost.dns.domain_registrar_push(domain, dry_run, autoremove, purge) + return yunohost.dns.domain_registrar_push(domain, dry_run, force, purge) From cfceca581f0e873fef2910da1c181397e8b8a895 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 17 Sep 2021 00:34:44 +0200 Subject: [PATCH 091/130] domain config: prevent a warning being raised because of unexpected value key --- src/yunohost/domain.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index a76faafc0..2c826bb51 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -438,6 +438,7 @@ class DomainConfigPanel(ConfigPanel): # FIXME: Ugly hack to save the registar id/value and reinject it in _load_current_values ... self.registar_id = toml['dns']['registrar']['registrar']['value'] + del toml['dns']['registrar']['registrar']['value'] return toml From 7655b7b7c38dacbc0258418585b1d8daf0a125c6 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 17 Sep 2021 00:47:16 +0200 Subject: [PATCH 092/130] dns: fix typo in tests --- src/yunohost/tests/test_dns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/tests/test_dns.py b/src/yunohost/tests/test_dns.py index 58c2be3a5..154d1dc9a 100644 --- a/src/yunohost/tests/test_dns.py +++ b/src/yunohost/tests/test_dns.py @@ -58,7 +58,7 @@ def test_magic_guess_registrar_yunodyndns(): @pytest.fixture def example_domain(): domain_add("example.tld") - yield "example_tld" + yield "example.tld" domain_remove("example.tld") From 71b09ae0e7c9553c8ad9b00e9e8455268511c4aa Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 17 Sep 2021 01:15:36 +0200 Subject: [PATCH 093/130] autodns dry-run: return a proper datastructure to the api instead of ~humanfriendly string --- src/yunohost/dns.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index fdb6d4647..09c8b978a 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -726,12 +726,15 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, force=False, return f'{name:>20} [{t:^5}] {old_content:^30} -> {new_content:^30} {ignored}' if dry_run: - out = {"delete": [], "create": [], "update": []} - for action in ["delete", "create", "update"]: - for record in changes[action]: - out[action].append(human_readable_record(action, record)) + if Moulinette.interface.type == "api": + return changes + else: + out = {"delete": [], "create": [], "update": []} + for action in ["delete", "create", "update"]: + for record in changes[action]: + out[action].append(human_readable_record(action, record)) - return out + return out operation_logger.start() From 00cc672b8962bcec530b700ca2a9fad48b3217b5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 17 Sep 2021 02:14:40 +0200 Subject: [PATCH 094/130] autodns: small fix for Gandi which returns TXT record not prefixed/suffixed with quotes --- src/yunohost/dns.py | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 09c8b978a..cae9037c2 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -618,6 +618,12 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, force=False, # Some API return '@' in content and we shall convert it to absolute/fqdn record["content"] = record["content"].replace('@.', base_dns_zone + ".").replace('@', base_dns_zone + ".") + if record["type"] == "TXT": + if not record["content"].startswith('"'): + record["content"] = '"' + record["content"] + if not record["content"].endswith('"'): + record["content"] = record["content"] + '"' + # Check if this record was previously set by YunoHost record["managed_by_yunohost"] = _hash_dns_record(record) in managed_dns_records_hashes @@ -711,26 +717,33 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, force=False, name = name.replace(base_dns_zone, "@") name = name[:20] t = record["type"] - if action in ["create", "update"]: - old_content = record.get("old_content", "(None)")[:30] - new_content = record.get("content", "(None)")[:30] - else: - new_content = record.get("old_content", "(None)")[:30] - old_content = record.get("content", "(None)")[:30] if not force and action in ["update", "delete"]: ignored = "" if record["managed_by_yunohost"] else "(ignored, won't be changed by Yunohost unless forced)" else: ignored = "" - return f'{name:>20} [{t:^5}] {old_content:^30} -> {new_content:^30} {ignored}' + if action == "create": + old_content = record.get("old_content", "(None)")[:30] + new_content = record.get("content", "(None)")[:30] + return f'{name:>20} [{t:^5}] {new_content:^30} {ignored}' + elif action == "update": + old_content = record.get("old_content", "(None)")[:30] + new_content = record.get("content", "(None)")[:30] + return f'{name:>20} [{t:^5}] {old_content:^30} -> {new_content:^30} {ignored}' + elif action == "unchanged": + old_content = new_content = record.get("content", "(None)")[:30] + return f'{name:>20} [{t:^5}] {old_content:^30}' + else: + old_content = record.get("content", "(None)")[:30] + return f'{name:>20} [{t:^5}] {old_content:^30} {ignored}' if dry_run: if Moulinette.interface.type == "api": return changes else: - out = {"delete": [], "create": [], "update": []} - for action in ["delete", "create", "update"]: + out = {"delete": [], "create": [], "update": [], "unchanged": []} + for action in ["delete", "create", "update", "unchanged"]: for record in changes[action]: out[action].append(human_readable_record(action, record)) From 6ea538a43bb13f1f4d539e566487d6595de8a312 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 17 Sep 2021 04:16:30 +0200 Subject: [PATCH 095/130] autodns: better error management + cli ux --- src/yunohost/dns.py | 82 ++++++++++++++++++++++++++++++++------------- 1 file changed, 58 insertions(+), 24 deletions(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index cae9037c2..c9fe6bb62 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -37,7 +37,7 @@ from moulinette.utils.filesystem import read_file, write_to_file, read_toml from yunohost.domain import domain_list, _assert_domain_exists, domain_config_get, _get_domain_settings, _set_domain_settings from yunohost.utils.dns import dig, YNH_DYNDNS_DOMAINS -from yunohost.utils.error import YunohostValidationError +from yunohost.utils.error import YunohostValidationError, YunohostError from yunohost.utils.network import get_public_ip from yunohost.log import is_unit_operation from yunohost.hook import hook_callback @@ -572,7 +572,7 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, force=False, # See https://github.com/AnalogJ/lexicon/issues/282 and https://github.com/AnalogJ/lexicon/pull/371 # They say it's trivial to implement it! # And yet, it is still not done/merged - wanted_records = [record for record in wanted_records if record["type"] != "CAA"] + #wanted_records = [record for record in wanted_records if record["type"] != "CAA"] if purge: wanted_records = [] @@ -596,14 +596,17 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, force=False, .with_dict(dict_object=base_config) .with_dict(dict_object={"action": "list", "type": "all"}) ) - # current_records.extend( client = LexiconClient(query) client.provider.authenticate() - current_records = client.provider.list_records() + try: + current_records = client.provider.list_records() + except Exception as e: + raise YunohostError("Failed to list current records using the registrar's API: %s" % str(e), raw_msg=True) # FIXME: i18n + managed_dns_records_hashes = _get_managed_dns_records_hashes(domain) # Keep only records for relevant types: A, AAAA, MX, TXT, CNAME, SRV - relevant_types = ["A", "AAAA", "MX", "TXT", "CNAME", "SRV"] + relevant_types = ["A", "AAAA", "MX", "TXT", "CNAME", "SRV", "CAA"] current_records = [r for r in current_records if r["type"] in relevant_types] # Ignore records which are for a higher-level domain @@ -640,7 +643,8 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, force=False, # (TXT, .domain.tld) "v=spf1 ..." ["v=spf1", "foobar"] # (SRV, .domain.tld) 0 5 5269 domain.tld changes = {"delete": [], "update": [], "create": [], "unchanged": []} - type_and_names = set([(r["type"], r["name"]) for r in current_records + wanted_records]) + + type_and_names = sorted(set([(r["type"], r["name"]) for r in current_records + wanted_records])) comparison = {type_and_name: {"current": [], "wanted": []} for type_and_name in type_and_names} for record in current_records: @@ -749,20 +753,47 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, force=False, return out + # If --force ain't used, we won't delete/update records not managed by yunohost + if not force: + for action in ["delete", "update"]: + changes[action] = [r for r in changes[action] if not r["managed_by_yunohost"]] + + def progress(info=""): + progress.nb += 1 + width = 20 + bar = int(progress.nb * width / progress.total) + bar = "[" + "#" * bar + "." * (width - bar) + "]" + if info: + bar += " > " + info + if progress.old == bar: + return + progress.old = bar + logger.info(bar) + + progress.nb = 0 + progress.old = "" + progress.total = len(changes["delete"] + changes["create"] + changes["update"]) + + if progress.total == 0: + logger.success("Records already up to date, nothing to do.") # FIXME : i18n + return {} + + # + # Actually push the records + # + operation_logger.start() + logger.info("Pushing DNS records...") new_managed_dns_records_hashes = [_hash_dns_record(r) for r in changes["unchanged"]] + results = {"warnings": [], "errors": []} - # Push the records for action in ["delete", "create", "update"]: for record in changes[action]: - if not force and action in ["update", "delete"] and not record["managed_by_yunohost"]: - # Don't overwrite manually-set or manually-modified records - continue - - record["action"] = action + relative_name = record['name'].replace(base_dns_zone, '').rstrip('.') or '@' + progress(f"{action} {record['type']:^5} / {relative_name}") # FIXME: i18n # Apparently Lexicon yields us some 'id' during fetch # But wants 'identifier' during push ... @@ -770,21 +801,15 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, force=False, record["identifier"] = record["id"] del record["id"] - logger.info(action + " : " + human_readable_record(action, record)) - if registrar == "godaddy": if record["name"] == base_dns_zone: record["name"] = "@." + record["name"] if record["type"] in ["MX", "SRV"]: logger.warning(f"Pushing {record['type']} records is not properly supported by Lexicon/Godaddy.") + results["warning"].append(f"Pushing {record['type']} records is not properly supported by Lexicon/Godaddy.") continue - # FIXME Removed TTL, because it doesn't work with Gandi. - # See https://github.com/AnalogJ/lexicon/issues/726 (similar issue) - # But I think there is another issue with Gandi. Or I'm misusing the API... - if registrar == "gandi": - del record["ttl"] - + record["action"] = action query = ( LexiconConfigResolver() .with_dict(dict_object=base_config) @@ -794,18 +819,27 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, force=False, try: result = LexiconClient(query).execute() except Exception as e: - logger.error(f"Failed to {action} record {record['type']}/{record['name']} : {e}") + logger.error(f"Failed to {action} record {record['type']}/{record['name']} : {e}") # i18n? + results["errors"].append(f"Failed to {action} record {record['type']}/{record['name']} : {e}") else: if result: new_managed_dns_records_hashes.append(_hash_dns_record(record)) - logger.success("Done!") else: - logger.error("Uhoh!?") + results["errors"].append(f"Failed to {action} record {record['type']}/{record['name']} : unknown error?") _set_managed_dns_records_hashes(domain, new_managed_dns_records_hashes) - # FIXME : implement a system to properly report what worked and what did not at the end of the command.. + # Everything succeeded + if len(results["errors"]) == 0: + logger.success("DNS records updated!") # FIXME: i18n + return {} + # Everything failed + elif len(results["errors"]) + len(results["warnings"]) == progress.total: + logger.error("Updating the DNS records failed miserably") # FIXME: i18n + else: + logger.warning("DNS records partially updated: some warnings/errors were reported.") # FIXME: i18n + return results def _get_managed_dns_records_hashes(domain: str) -> list: return _get_domain_settings(domain).get("managed_dns_records_hashes", []) From 7ab9889521cb48f9e3c514f7903070d226218604 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 17 Sep 2021 04:55:07 +0200 Subject: [PATCH 096/130] autodns: i18n --- locales/en.json | 13 +++++++++---- src/yunohost/dns.py | 6 +++--- src/yunohost/domain.py | 2 +- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/locales/en.json b/locales/en.json index 282576ee8..4578075b4 100644 --- a/locales/en.json +++ b/locales/en.json @@ -313,12 +313,18 @@ "domain_dyndns_already_subscribed": "You have already subscribed to a DynDNS domain", "domain_dyndns_root_unknown": "Unknown DynDNS root domain", "domain_exists": "The domain already exists", - "domain_property_unknown": "The property {property} doesn't exist", "domain_hostname_failed": "Unable to set new hostname. This might cause an issue later (it might be fine).", "domain_name_unknown": "Domain '{domain}' unknown", - "domain_registrar_unknown": "This registrar is unknown. Look for yours with the command `yunohost domain catalog`", "domain_remove_confirm_apps_removal": "Removing this domain will remove those applications:\n{apps}\n\nAre you sure you want to do that? [{answers}]", "domain_uninstall_app_first": "Those applications are still installed on your domain:\n{apps}\n\nPlease uninstall them using 'yunohost app remove the_app_id' or move them to another domain using 'yunohost app change-url the_app_id' before proceeding to domain removal", + "domain_registrar_is_not_configured": "The registrar is not yet configured for domain {domain}.", + "domain_dns_push_not_applicable": "The DNS push feature is not applicable to domain {domain}", + "domain_config_auth_token": "Authentication token", + "domain_config_api_protocol": "API protocol", + "domain_config_auth_entrypoint": "API entry point", + "domain_config_auth_application_key": "Application key", + "domain_config_auth_application_secret": "Application secret key", + "domain_config_auth_consumer_key": "Consumer key", "domains_available": "Available domains:", "done": "Done", "downloading": "Downloading...", @@ -423,6 +429,7 @@ "log_domain_config_set": "Update configuration for domain '{}'", "log_domain_main_domain": "Make '{}' the main domain", "log_domain_remove": "Remove '{}' domain from system configuration", + "log_domain_dns_push": "Push DNS records for domain '{}'", "log_dyndns_subscribe": "Subscribe to a YunoHost subdomain '{}'", "log_dyndns_update": "Update the IP associated with your YunoHost subdomain '{}'", "log_help_to_get_failed_log": "The operation '{desc}' could not be completed. Please share the full log of this operation using the command 'yunohost log share {name}' to get help", @@ -530,7 +537,6 @@ "pattern_password": "Must be at least 3 characters long", "pattern_password_app": "Sorry, passwords can not contain the following characters: {forbidden_chars}", "pattern_port_or_range": "Must be a valid port number (i.e. 0-65535) or range of ports (e.g. 100:200)", - "pattern_positive_number": "Must be a positive number", "pattern_username": "Must be lower-case alphanumeric and underscore characters only", "permission_already_allowed": "Group '{group}' already has permission '{permission}' enabled", "permission_already_disallowed": "Group '{group}' already has permission '{permission}' disabled", @@ -569,7 +575,6 @@ "regenconf_would_be_updated": "The configuration would have been updated for category '{category}'", "regex_incompatible_with_tile": "/!\\ Packagers! Permission '{permission}' has show_tile set to 'true' and you therefore cannot define a regex URL as the main URL", "regex_with_only_domain": "You can't use a regex for domain, only for path", - "registrar_is_not_set": "The registrar for this domain has not been configured", "restore_already_installed_app": "An app with the ID '{app}' is already installed", "restore_already_installed_apps": "The following apps can't be restored because they are already installed: {apps}", "restore_backup_too_old": "This backup archive can not be restored because it comes from a too-old YunoHost version.", diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index c9fe6bb62..1e2037ce5 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -518,7 +518,7 @@ def _get_registrar_config_section(domain): @is_unit_operation() -def domain_registrar_push(operation_logger, domain, dry_run=False, force=False, purge=False): +def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge=False): """ Send DNS records to the previously-configured registrar of the domain. """ @@ -533,7 +533,7 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, force=False, registrar = settings["dns.registrar.registrar"].get("value") if not registrar or registrar in ["None", "yunohost"]: - raise YunohostValidationError("registrar_push_not_applicable", domain=domain) + raise YunohostValidationError("domain_dns_push_not_applicable", domain=domain) base_dns_zone = _get_dns_zone_for_domain(domain) @@ -544,7 +544,7 @@ def domain_registrar_push(operation_logger, domain, dry_run=False, force=False, } if not all(registrar_credentials.values()): - raise YunohostValidationError("registrar_is_not_configured", domain=domain) + raise YunohostValidationError("domain_registrar_is_not_configured", domain=domain) # Convert the generated conf into a format that matches what we'll fetch using the API # Makes it easier to compare "wanted records" with "current records on remote" diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 2c826bb51..d13900224 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -512,4 +512,4 @@ def domain_dns_suggest(domain): def domain_dns_push(domain, dry_run, force, purge): import yunohost.dns - return yunohost.dns.domain_registrar_push(domain, dry_run, force, purge) + return yunohost.dns.domain_dns_push(domain, dry_run, force, purge) From c5a835c3918cd1c81960e42fd1c155f820c3c66a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 17 Sep 2021 04:55:14 +0200 Subject: [PATCH 097/130] autodns: misc fixes/enh --- data/other/config_domain.toml | 12 ++++++------ data/other/registrar_list.toml | 2 ++ src/yunohost/dns.py | 13 +++++++++++-- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/data/other/config_domain.toml b/data/other/config_domain.toml index af23b5e04..095e561a1 100644 --- a/data/other/config_domain.toml +++ b/data/other/config_domain.toml @@ -13,7 +13,7 @@ i18n = "domain_config" [feature] [feature.mail] - services = ['postfix', 'dovecot'] + #services = ['postfix', 'dovecot'] [feature.mail.mail_out] type = "boolean" @@ -23,11 +23,11 @@ i18n = "domain_config" type = "boolean" default = 1 - [feature.mail.backup_mx] - type = "tags" - default = [] - pattern.regexp = '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$' - pattern.error = "pattern_error" + #[feature.mail.backup_mx] + #type = "tags" + #default = [] + #pattern.regexp = '^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$' + #pattern.error = "pattern_error" [feature.xmpp] diff --git a/data/other/registrar_list.toml b/data/other/registrar_list.toml index 406066dc9..afb213aa1 100644 --- a/data/other/registrar_list.toml +++ b/data/other/registrar_list.toml @@ -230,6 +230,8 @@ type = "string" choices.rpc = "RPC" choices.rest = "REST" + default = "rest" + visible = "false" [gehirn] [gehirn.auth_token] diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 1e2037ce5..07e5297d5 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -26,7 +26,6 @@ import os import re import time -import hashlib from difflib import SequenceMatcher from collections import OrderedDict @@ -506,6 +505,15 @@ def _get_registrar_config_section(domain): "ask": f"YunoHost automatically detected that this domain is handled by the registrar **{registrar}**. If you want, YunoHost will automatically configure this DNS zone, if you provide it with the appropriate API credentials. You can also manually configure your DNS records following the documentation as https://yunohost.org/dns.", # FIXME: i18n "value": registrar }) + + TESTED_REGISTRARS = ["ovh", "gandi"] + if registrar not in TESTED_REGISTRARS: + registrar_infos["experimental_disclaimer"] = OrderedDict({ + "type": "alert", + "style": "danger", + "ask": f"So far, the interface with **{registrar}**'s API has not been properly tested and reviewed by the YunoHost's community. Support is **very experimental** - be careful!", # FIXME: i18n + }) + # TODO : add a help tip with the link to the registar's API doc (c.f. Lexicon's README) registrar_list = read_toml(DOMAIN_REGISTRAR_LIST_PATH) registrar_credentials = registrar_list[registrar] @@ -572,6 +580,7 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= # See https://github.com/AnalogJ/lexicon/issues/282 and https://github.com/AnalogJ/lexicon/pull/371 # They say it's trivial to implement it! # And yet, it is still not done/merged + # Update by Aleks: it works - at least with Gandi ?! #wanted_records = [record for record in wanted_records if record["type"] != "CAA"] if purge: @@ -756,7 +765,7 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= # If --force ain't used, we won't delete/update records not managed by yunohost if not force: for action in ["delete", "update"]: - changes[action] = [r for r in changes[action] if not r["managed_by_yunohost"]] + changes[action] = [r for r in changes[action] if r["managed_by_yunohost"]] def progress(info=""): progress.nb += 1 From dbff4316d3b00e09e526f938e4d0e5563c9ebfee Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 17 Sep 2021 05:12:22 +0200 Subject: [PATCH 098/130] config: fix rendering of core-defined strings during config set --- src/yunohost/utils/config.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/yunohost/utils/config.py b/src/yunohost/utils/config.py index 3102f28f2..02e657a73 100644 --- a/src/yunohost/utils/config.py +++ b/src/yunohost/utils/config.py @@ -354,6 +354,11 @@ class ConfigPanel: def _ask(self): logger.debug("Ask unanswered question and prevalidate data") + if "i18n" in self.config: + for panel, section, option in self._iterate(): + if "ask" not in option: + option["ask"] = m18n.n(self.config["i18n"] + "_" + option["id"]) + def display_header(message): """CLI panel/section header display""" if Moulinette.interface.type == "cli" and self.filter_key.count(".") < 2: From b15004135abbebcf96ad76b833dc5adaf0db5481 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 17 Sep 2021 05:17:50 +0200 Subject: [PATCH 099/130] domain config: Moar i18n --- locales/en.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/locales/en.json b/locales/en.json index 4578075b4..30ec91f63 100644 --- a/locales/en.json +++ b/locales/en.json @@ -319,6 +319,9 @@ "domain_uninstall_app_first": "Those applications are still installed on your domain:\n{apps}\n\nPlease uninstall them using 'yunohost app remove the_app_id' or move them to another domain using 'yunohost app change-url the_app_id' before proceeding to domain removal", "domain_registrar_is_not_configured": "The registrar is not yet configured for domain {domain}.", "domain_dns_push_not_applicable": "The DNS push feature is not applicable to domain {domain}", + "domain_config_mail_in": "Incoming emails", + "domain_config_mail_out": "Outgoing emails", + "domain_config_xmpp": "XMPP", "domain_config_auth_token": "Authentication token", "domain_config_api_protocol": "API protocol", "domain_config_auth_entrypoint": "API entry point", From a75d9aff34e426cb791af5b4c0ce1aea7c0c2514 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 17 Sep 2021 05:22:37 +0200 Subject: [PATCH 100/130] Moaaar i18n --- locales/en.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/locales/en.json b/locales/en.json index 30ec91f63..af099b42f 100644 --- a/locales/en.json +++ b/locales/en.json @@ -323,6 +323,8 @@ "domain_config_mail_out": "Outgoing emails", "domain_config_xmpp": "XMPP", "domain_config_auth_token": "Authentication token", + "domain_config_auth_key": "Authentication key", + "domain_config_auth_secret": "Authentication secret", "domain_config_api_protocol": "API protocol", "domain_config_auth_entrypoint": "API entry point", "domain_config_auth_application_key": "Application key", From d4d576c492ff9c89f5f0163bce2961c7c5efa417 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 17 Sep 2021 14:36:46 +0200 Subject: [PATCH 101/130] autodns: return relative name in dry run output --- src/yunohost/dns.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 07e5297d5..20ee8bbf5 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -723,11 +723,14 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= for record in current: changes["delete"].append(record) - def human_readable_record(action, record): - name = record["name"] + def relative_name(name): name = name.strip(".") name = name.replace('.' + base_dns_zone, "") name = name.replace(base_dns_zone, "@") + return name + + def human_readable_record(action, record): + name = relative_name(record["name"]) name = name[:20] t = record["type"] @@ -753,6 +756,9 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= if dry_run: if Moulinette.interface.type == "api": + for records in changes.values(): + for record in records: + record["name"] = relative_name(record["name"]) return changes else: out = {"delete": [], "create": [], "update": [], "unchanged": []} From 93b99635b7566fabec4c7d1646daa5fb31889080 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 17 Sep 2021 15:41:57 +0200 Subject: [PATCH 102/130] autodns: fix/simplify registrar settings fetching --- src/yunohost/dns.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 20ee8bbf5..3c73974ee 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -536,20 +536,17 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= _assert_domain_exists(domain) - settings = domain_config_get(domain, key='dns.registrar') + settings = domain_config_get(domain, key='dns.registrar', export=True) - registrar = settings["dns.registrar.registrar"].get("value") + registrar = settings.get("registrar") if not registrar or registrar in ["None", "yunohost"]: raise YunohostValidationError("domain_dns_push_not_applicable", domain=domain) base_dns_zone = _get_dns_zone_for_domain(domain) - registrar_credentials = { - k.split('.')[-1]: v["value"] - for k, v in settings.items() - if k != "dns.registrar.registar" - } + registrar_credentials = settings + registrar_credentials.pop("registrar") if not all(registrar_credentials.values()): raise YunohostValidationError("domain_registrar_is_not_configured", domain=domain) From c8caabf8f819cd4d443a4a71f2097fefc2c3a0d2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 17 Sep 2021 16:12:25 +0200 Subject: [PATCH 103/130] autodns: pop experimental_disclaimer because it's not an actual credential --- src/yunohost/dns.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 3c73974ee..341a66ad7 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -547,6 +547,8 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= registrar_credentials = settings registrar_credentials.pop("registrar") + if "experimental_disclaimer" in registrar_credentials: + registrar_credentials.pop("experimental_disclaimer") if not all(registrar_credentials.values()): raise YunohostValidationError("domain_registrar_is_not_configured", domain=domain) From 68f2eea0ae13d2fb8ba59af648e0437932e9771e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 18 Sep 2021 13:58:41 +0200 Subject: [PATCH 104/130] autodns: proper error management for authentication error --- locales/en.json | 1 + src/yunohost/dns.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/locales/en.json b/locales/en.json index af099b42f..07a46b8df 100644 --- a/locales/en.json +++ b/locales/en.json @@ -319,6 +319,7 @@ "domain_uninstall_app_first": "Those applications are still installed on your domain:\n{apps}\n\nPlease uninstall them using 'yunohost app remove the_app_id' or move them to another domain using 'yunohost app change-url the_app_id' before proceeding to domain removal", "domain_registrar_is_not_configured": "The registrar is not yet configured for domain {domain}.", "domain_dns_push_not_applicable": "The DNS push feature is not applicable to domain {domain}", + "domain_dns_push_failed_to_authenticate": "Failed to authenticate on registrar's API. Most probably the credentials are incorrect? (Error: {error})", "domain_config_mail_in": "Incoming emails", "domain_config_mail_out": "Outgoing emails", "domain_config_xmpp": "XMPP", diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 341a66ad7..8bc48a5ac 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -605,7 +605,11 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= .with_dict(dict_object={"action": "list", "type": "all"}) ) client = LexiconClient(query) - client.provider.authenticate() + try: + client.provider.authenticate() + except Exception as e: + raise YunohostValidationError("domain_dns_push_failed_to_authenticate", error=str(e)) + try: current_records = client.provider.list_records() except Exception as e: From 5812c8f1ae77d63f2ab9f96051d1ce007fae33d1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 18 Sep 2021 22:47:06 +0200 Subject: [PATCH 105/130] autodns: Disable ttl setting for now because meh --- data/other/config_domain.toml | 12 ++++++------ src/yunohost/dns.py | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/data/other/config_domain.toml b/data/other/config_domain.toml index 095e561a1..10ff5ce5c 100644 --- a/data/other/config_domain.toml +++ b/data/other/config_domain.toml @@ -42,9 +42,9 @@ i18n = "domain_config" # This part is automatically generated in DomainConfigPanel - [dns.advanced] - - [dns.advanced.ttl] - type = "number" - min = 0 - default = 3600 +# [dns.advanced] +# +# [dns.advanced.ttl] +# type = "number" +# min = 0 +# default = 3600 diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 8bc48a5ac..a006a19a2 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -176,7 +176,8 @@ def _build_dns_conf(base_domain): basename = domain.replace(f"{base_dns_zone}", "").rstrip(".") or "@" suffix = f".{basename}" if basename != "@" else "" - ttl = settings["ttl"] + #ttl = settings["ttl"] + ttl = "3600" ########################### # Basic ipv4/ipv6 records # From fa31d49bf9493c4b70fb5e46e6a5bd95bb6f487f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 18 Sep 2021 23:10:17 +0200 Subject: [PATCH 106/130] autodns: Improve handling of the subdomain case --- locales/en.json | 3 ++- src/yunohost/dns.py | 13 +++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/locales/en.json b/locales/en.json index 07a46b8df..c4413380a 100644 --- a/locales/en.json +++ b/locales/en.json @@ -318,7 +318,8 @@ "domain_remove_confirm_apps_removal": "Removing this domain will remove those applications:\n{apps}\n\nAre you sure you want to do that? [{answers}]", "domain_uninstall_app_first": "Those applications are still installed on your domain:\n{apps}\n\nPlease uninstall them using 'yunohost app remove the_app_id' or move them to another domain using 'yunohost app change-url the_app_id' before proceeding to domain removal", "domain_registrar_is_not_configured": "The registrar is not yet configured for domain {domain}.", - "domain_dns_push_not_applicable": "The DNS push feature is not applicable to domain {domain}", + "domain_dns_push_not_applicable": "The DNS push feature is not applicable to domain {domain}. You should manually configure your DNS records following the documentation at https://yunohost.org/dns_config.", + "domain_dns_push_managed_in_parent_domain": "The DNS push feature is managed in the parent domain {parent_domain}.", "domain_dns_push_failed_to_authenticate": "Failed to authenticate on registrar's API. Most probably the credentials are incorrect? (Error: {error})", "domain_config_mail_in": "Incoming emails", "domain_config_mail_out": "Outgoing emails", diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index a006a19a2..98933ba75 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -470,11 +470,17 @@ def _get_registrar_config_section(domain): # If parent domain exists in yunohost parent_domain = domain.split(".", 1)[1] if parent_domain in domain_list()["domains"]: + + if Moulinette.interface.type = "api": + parent_domain_link = "[{parent_domain}](#/domains/{parent_domain}/config)" + else: + parent_domain_link = parent_domain + registrar_infos["registrar"] = OrderedDict({ "type": "alert", "style": "info", - "ask": f"This domain is a subdomain of {parent_domain}. DNS registrar configuration should be managed in {parent_domain}'s configuration panel.", # FIXME: i18n - "value": None + "ask": f"This domain is a subdomain of {parent_domain_link}. DNS registrar configuration should be managed in {parent_domain}'s configuration panel.", # FIXME: i18n + "value": "parent_domain" }) return OrderedDict(registrar_infos) @@ -544,6 +550,9 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= if not registrar or registrar in ["None", "yunohost"]: raise YunohostValidationError("domain_dns_push_not_applicable", domain=domain) + if registrar == "parent_domain": + raise YunohostValidationError("domain_dns_push_managed_in_parent_domain", domain=domain, parent_domain=registrar) + base_dns_zone = _get_dns_zone_for_domain(domain) registrar_credentials = settings From 2c997d43e1ecc43ccf805b6e9bcb078e827817de Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 18 Sep 2021 23:15:36 +0200 Subject: [PATCH 107/130] autodns: typo --- src/yunohost/dns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 98933ba75..f4d89820a 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -834,7 +834,7 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= record["name"] = "@." + record["name"] if record["type"] in ["MX", "SRV"]: logger.warning(f"Pushing {record['type']} records is not properly supported by Lexicon/Godaddy.") - results["warning"].append(f"Pushing {record['type']} records is not properly supported by Lexicon/Godaddy.") + results["warnings"].append(f"Pushing {record['type']} records is not properly supported by Lexicon/Godaddy.") continue record["action"] = action From b3e9cf19db1a0bfa8c0b21ed17ba3e86fac06ef2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 18 Sep 2021 23:18:03 +0200 Subject: [PATCH 108/130] =?UTF-8?q?autodns:=20typo=C2=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/yunohost/dns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index f4d89820a..686a016c0 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -471,7 +471,7 @@ def _get_registrar_config_section(domain): parent_domain = domain.split(".", 1)[1] if parent_domain in domain_list()["domains"]: - if Moulinette.interface.type = "api": + if Moulinette.interface.type == "api": parent_domain_link = "[{parent_domain}](#/domains/{parent_domain}/config)" else: parent_domain_link = parent_domain From 002d25452231b629d43d41a9e96ba09d84bd69c7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 18 Sep 2021 23:28:34 +0200 Subject: [PATCH 109/130] =?UTF-8?q?autodns:=20typo=C2=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/yunohost/dns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 686a016c0..ab97841a7 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -472,7 +472,7 @@ def _get_registrar_config_section(domain): if parent_domain in domain_list()["domains"]: if Moulinette.interface.type == "api": - parent_domain_link = "[{parent_domain}](#/domains/{parent_domain}/config)" + parent_domain_link = f"[{parent_domain}](#/domains/{parent_domain}/config)" else: parent_domain_link = parent_domain From c762cd98581c935ecfc3033f3c6b39ca00707722 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 18 Sep 2021 23:35:37 +0200 Subject: [PATCH 110/130] =?UTF-8?q?autodns:=20typo=E2=81=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/yunohost/dns.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index ab97841a7..dfec7ceb9 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -551,7 +551,8 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= raise YunohostValidationError("domain_dns_push_not_applicable", domain=domain) if registrar == "parent_domain": - raise YunohostValidationError("domain_dns_push_managed_in_parent_domain", domain=domain, parent_domain=registrar) + parent_domain = domain.split(".", 1)[1] + raise YunohostValidationError("domain_dns_push_managed_in_parent_domain", domain=domain, parent_domain=parent_domain) base_dns_zone = _get_dns_zone_for_domain(domain) From b5e20cadf4df595d1217668fdd6d38b817ad51e2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 19 Sep 2021 00:32:42 +0200 Subject: [PATCH 111/130] autodns: Various fixes, improvement for edge case handling --- locales/en.json | 6 +++--- src/yunohost/dns.py | 46 ++++++++++++++++++++++++++++++--------------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/locales/en.json b/locales/en.json index c4413380a..fb755c08b 100644 --- a/locales/en.json +++ b/locales/en.json @@ -318,9 +318,9 @@ "domain_remove_confirm_apps_removal": "Removing this domain will remove those applications:\n{apps}\n\nAre you sure you want to do that? [{answers}]", "domain_uninstall_app_first": "Those applications are still installed on your domain:\n{apps}\n\nPlease uninstall them using 'yunohost app remove the_app_id' or move them to another domain using 'yunohost app change-url the_app_id' before proceeding to domain removal", "domain_registrar_is_not_configured": "The registrar is not yet configured for domain {domain}.", - "domain_dns_push_not_applicable": "The DNS push feature is not applicable to domain {domain}. You should manually configure your DNS records following the documentation at https://yunohost.org/dns_config.", - "domain_dns_push_managed_in_parent_domain": "The DNS push feature is managed in the parent domain {parent_domain}.", - "domain_dns_push_failed_to_authenticate": "Failed to authenticate on registrar's API. Most probably the credentials are incorrect? (Error: {error})", + "domain_dns_push_not_applicable": "The automatic DNS configuration feature is not applicable to domain {domain}. You should manually configure your DNS records following the documentation at https://yunohost.org/dns_config.", + "domain_dns_push_managed_in_parent_domain": "The automatic DNS configuration feature is managed in the parent domain {parent_domain}.", + "domain_dns_push_failed_to_authenticate": "Failed to authenticate on registrar's API for domain '{domain}'. Most probably the credentials are incorrect? (Error: {error})", "domain_config_mail_in": "Incoming emails", "domain_config_mail_out": "Outgoing emails", "domain_config_xmpp": "XMPP", diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index dfec7ceb9..daeae0c7f 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -490,7 +490,7 @@ def _get_registrar_config_section(domain): registrar_infos["registrar"] = OrderedDict({ "type": "alert", "style": "success", - "ask": "This domain is a nohost.me / nohost.st / ynh.fr and its DNS configuration is therefore automatically handled by Yunohost.", # FIXME: i18n + "ask": "This domain is a nohost.me / nohost.st / ynh.fr and its DNS configuration is therefore automatically handled by Yunohost without any further configuration.", # FIXME: i18n "value": "yunohost" }) return OrderedDict(registrar_infos) @@ -532,6 +532,20 @@ def _get_registrar_config_section(domain): return OrderedDict(registrar_infos) +def _get_registar_settings(domain): + + _assert_domain_exists(domain) + + settings = domain_config_get(domain, key='dns.registrar', export=True) + + registrar = settings.pop("registrar") + + if "experimental_disclaimer" in settings: + settings.pop("experimental_disclaimer") + + return registrar, settings + + @is_unit_operation() def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge=False): """ @@ -541,29 +555,31 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= from lexicon.client import Client as LexiconClient from lexicon.config import ConfigResolver as LexiconConfigResolver + registrar, registrar_credentials = _get_registar_settings(domain) + _assert_domain_exists(domain) - settings = domain_config_get(domain, key='dns.registrar', export=True) - - registrar = settings.get("registrar") - - if not registrar or registrar in ["None", "yunohost"]: + if not registrar or registrar == "None": # yes it's None as a string raise YunohostValidationError("domain_dns_push_not_applicable", domain=domain) + # FIXME: in the future, properly unify this with yunohost dyndns update + if registrar == "yunohost": + logger.info("This domain is a nohost.me / nohost.st / ynh.fr and its DNS configuration is therefore already automatically handled by Yunohost without any further configuration.") # FIXME: i18n + return {} + if registrar == "parent_domain": parent_domain = domain.split(".", 1)[1] - raise YunohostValidationError("domain_dns_push_managed_in_parent_domain", domain=domain, parent_domain=parent_domain) - - base_dns_zone = _get_dns_zone_for_domain(domain) - - registrar_credentials = settings - registrar_credentials.pop("registrar") - if "experimental_disclaimer" in registrar_credentials: - registrar_credentials.pop("experimental_disclaimer") + registar, registrar_credentials = _get_registar_settings(parent_domain) + if any(registrar_credentials.values()): + raise YunohostValidationError("domain_dns_push_managed_in_parent_domain", domain=domain, parent_domain=parent_domain) + else: + raise YunohostValidationError("domain_registrar_is_not_configured", domain=domain) if not all(registrar_credentials.values()): raise YunohostValidationError("domain_registrar_is_not_configured", domain=domain) + base_dns_zone = _get_dns_zone_for_domain(domain) + # Convert the generated conf into a format that matches what we'll fetch using the API # Makes it easier to compare "wanted records" with "current records on remote" wanted_records = [] @@ -619,7 +635,7 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= try: client.provider.authenticate() except Exception as e: - raise YunohostValidationError("domain_dns_push_failed_to_authenticate", error=str(e)) + raise YunohostValidationError("domain_dns_push_failed_to_authenticate", domain=domain, error=str(e)) try: current_records = client.provider.list_records() From 8faad01263aad74d497c8967d5346f9955c5f39a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 19 Sep 2021 01:23:33 +0200 Subject: [PATCH 112/130] Misc fixes --- src/yunohost/dns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index daeae0c7f..c2eb463f7 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -177,7 +177,7 @@ def _build_dns_conf(base_domain): suffix = f".{basename}" if basename != "@" else "" #ttl = settings["ttl"] - ttl = "3600" + ttl = 3600 ########################### # Basic ipv4/ipv6 records # @@ -573,7 +573,7 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= if any(registrar_credentials.values()): raise YunohostValidationError("domain_dns_push_managed_in_parent_domain", domain=domain, parent_domain=parent_domain) else: - raise YunohostValidationError("domain_registrar_is_not_configured", domain=domain) + raise YunohostValidationError("domain_registrar_is_not_configured", domain=parent_domain) if not all(registrar_credentials.values()): raise YunohostValidationError("domain_registrar_is_not_configured", domain=domain) From f90854809c43eb1e428f0a09714332d634333e30 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 19 Sep 2021 01:23:51 +0200 Subject: [PATCH 113/130] domain config: add disclaimer that enabling/disabling features only impact dns configuration for now --- data/other/config_domain.toml | 5 +++++ locales/en.json | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/data/other/config_domain.toml b/data/other/config_domain.toml index 10ff5ce5c..93551458b 100644 --- a/data/other/config_domain.toml +++ b/data/other/config_domain.toml @@ -15,6 +15,11 @@ i18n = "domain_config" [feature.mail] #services = ['postfix', 'dovecot'] + [feature.mail.features_disclaimer] + type = "alert" + style = "warning" + icon = "warning" + [feature.mail.mail_out] type = "boolean" default = 1 diff --git a/locales/en.json b/locales/en.json index fb755c08b..ab8b32127 100644 --- a/locales/en.json +++ b/locales/en.json @@ -321,9 +321,10 @@ "domain_dns_push_not_applicable": "The automatic DNS configuration feature is not applicable to domain {domain}. You should manually configure your DNS records following the documentation at https://yunohost.org/dns_config.", "domain_dns_push_managed_in_parent_domain": "The automatic DNS configuration feature is managed in the parent domain {parent_domain}.", "domain_dns_push_failed_to_authenticate": "Failed to authenticate on registrar's API for domain '{domain}'. Most probably the credentials are incorrect? (Error: {error})", + "domain_config_features_disclaimer": "So far, enabling/disabling mail or XMPP features only impact the recommended and automatic DNS configuration, not system configurations!", "domain_config_mail_in": "Incoming emails", "domain_config_mail_out": "Outgoing emails", - "domain_config_xmpp": "XMPP", + "domain_config_xmpp": "Instant messaging (XMPP)", "domain_config_auth_token": "Authentication token", "domain_config_auth_key": "Authentication key", "domain_config_auth_secret": "Authentication secret", From 76bd3a6f83798412672ee9b68bcb404c078b1da6 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 19 Sep 2021 01:46:48 +0200 Subject: [PATCH 114/130] autodns: Add link to registrar api doc --- src/yunohost/dns.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index c2eb463f7..7dba8c0d8 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -471,6 +471,7 @@ def _get_registrar_config_section(domain): parent_domain = domain.split(".", 1)[1] if parent_domain in domain_list()["domains"]: + # Dirty hack to have a link on the webadmin if Moulinette.interface.type == "api": parent_domain_link = f"[{parent_domain}](#/domains/{parent_domain}/config)" else: @@ -509,7 +510,7 @@ def _get_registrar_config_section(domain): registrar_infos["registrar"] = OrderedDict({ "type": "alert", "style": "info", - "ask": f"YunoHost automatically detected that this domain is handled by the registrar **{registrar}**. If you want, YunoHost will automatically configure this DNS zone, if you provide it with the appropriate API credentials. You can also manually configure your DNS records following the documentation as https://yunohost.org/dns.", # FIXME: i18n + "ask": f"YunoHost automatically detected that this domain is handled by the registrar **{registrar}**. If you want, YunoHost will automatically configure this DNS zone, if you provide it with the appropriate API credentials. You read documentation on how to get your API credentials on this page: https://yunohost.org/registar_api_{registrar}. (You can also manually configure your DNS records following the documentation as https://yunohost.org/dns )", # FIXME: i18n "value": registrar }) From 52b3cb5622993f5bad7e93a7b9d2aaac5060a28f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 19 Sep 2021 01:56:53 +0200 Subject: [PATCH 115/130] app config panel: Add supports_config_panel in app_info for webadmin --- src/yunohost/app.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 871e3dc00..2d9ecce22 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -235,6 +235,9 @@ def app_info(app, full=False): ret["supports_multi_instance"] = is_true( local_manifest.get("multi_instance", False) ) + ret["supports_config_panel"] = os.path.exists( + os.path.join(setting_path, "config_panel.toml") + ) ret["permissions"] = permissions ret["label"] = permissions.get(app + ".main", {}).get("label") From e3ce03ac85c2fc9dba853b9c421063a99fe201f2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 19 Sep 2021 16:18:07 +0200 Subject: [PATCH 116/130] autodns: i18n --- locales/en.json | 12 ++++++++++++ src/yunohost/dns.py | 40 +++++++++++++++++++++------------------- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/locales/en.json b/locales/en.json index ab8b32127..e49228db3 100644 --- a/locales/en.json +++ b/locales/en.json @@ -320,7 +320,19 @@ "domain_registrar_is_not_configured": "The registrar is not yet configured for domain {domain}.", "domain_dns_push_not_applicable": "The automatic DNS configuration feature is not applicable to domain {domain}. You should manually configure your DNS records following the documentation at https://yunohost.org/dns_config.", "domain_dns_push_managed_in_parent_domain": "The automatic DNS configuration feature is managed in the parent domain {parent_domain}.", + "domain_dns_registrar_managed_in_parent_domain": "This domain is a subdomain of {parent_domain_link}. DNS registrar configuration should be managed in {parent_domain}'s configuration panel.", + "domain_dns_registrar_yunohost": "This domain is a nohost.me / nohost.st / ynh.fr and its DNS configuration is therefore automatically handled by Yunohost without any further configuration. (see the 'yunohost dyndns update' command)", + "domain_dns_registrar_not_supported": "YunoHost could not automatically detect the registrar handling this domain. You should manually configure your DNS records following the documentation at https://yunohost.org/dns.", + "domain_dns_registrar_supported": "YunoHost automatically detected that this domain is handled by the registrar **{registrar}**. If you want, YunoHost will automatically configure this DNS zone, if you provide it with the appropriate API credentials. You can find documentation on how to obtain your API credentials on this page: https://yunohost.org/registar_api_{registrar}. (You can also manually configure your DNS records following the documentation at https://yunohost.org/dns )", + "domain_dns_registrar_experimental": "So far, the interface with **{registrar}**'s API has not been properly tested and reviewed by the YunoHost community. Support is **very experimental** - be careful!", "domain_dns_push_failed_to_authenticate": "Failed to authenticate on registrar's API for domain '{domain}'. Most probably the credentials are incorrect? (Error: {error})", + "domain_dns_push_failed_to_list": "Failed to list current records using the registrar's API: {error}", + "domain_dns_push_already_up_to_date": "Records already up to date, nothing to do.", + "domain_dns_pushing": "Pushing DNS records...", + "domain_dns_push_record_failed": "Failed to {action} record {type}/{name} : {error}", + "domain_dns_push_success": "DNS records updated!", + "domain_dns_push_failed": "Updating the DNS records failed miserably.", + "domain_dns_push_partial_failure": "DNS records partially updated: some warnings/errors were reported.", "domain_config_features_disclaimer": "So far, enabling/disabling mail or XMPP features only impact the recommended and automatic DNS configuration, not system configurations!", "domain_config_mail_in": "Incoming emails", "domain_config_mail_out": "Outgoing emails", diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 7dba8c0d8..8d44804f3 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -172,8 +172,7 @@ def _build_dns_conf(base_domain): # sub.domain.tld # sub.domain.tld # @ # # # foo.sub.domain.tld # sub.domain.tld # foo # .foo # - # FIXME: shouldn't the basename just be based on the dns_zone setting of this domain ? - basename = domain.replace(f"{base_dns_zone}", "").rstrip(".") or "@" + basename = domain.replace(base_dns_zone, "").rstrip(".") or "@" suffix = f".{basename}" if basename != "@" else "" #ttl = settings["ttl"] @@ -480,7 +479,7 @@ def _get_registrar_config_section(domain): registrar_infos["registrar"] = OrderedDict({ "type": "alert", "style": "info", - "ask": f"This domain is a subdomain of {parent_domain_link}. DNS registrar configuration should be managed in {parent_domain}'s configuration panel.", # FIXME: i18n + "ask": m18n.n("domain_dns_registrar_managed_in_parent_domain", parent_domain=domain, parent_domain_link=parent_domain_link), "value": "parent_domain" }) return OrderedDict(registrar_infos) @@ -491,7 +490,7 @@ def _get_registrar_config_section(domain): registrar_infos["registrar"] = OrderedDict({ "type": "alert", "style": "success", - "ask": "This domain is a nohost.me / nohost.st / ynh.fr and its DNS configuration is therefore automatically handled by Yunohost without any further configuration.", # FIXME: i18n + "ask": m18n.n("domain_dns_registrar_yunohost"), "value": "yunohost" }) return OrderedDict(registrar_infos) @@ -502,7 +501,7 @@ def _get_registrar_config_section(domain): registrar_infos["registrar"] = OrderedDict({ "type": "alert", "style": "warning", - "ask": "YunoHost could not automatically detect the registrar handling this domain. You should manually configure your DNS records following the documentation at https://yunohost.org/dns.", # FIXME : i18n + "ask": m18n.n("domain_dns_registrar_not_supported"), "value": None }) else: @@ -510,7 +509,7 @@ def _get_registrar_config_section(domain): registrar_infos["registrar"] = OrderedDict({ "type": "alert", "style": "info", - "ask": f"YunoHost automatically detected that this domain is handled by the registrar **{registrar}**. If you want, YunoHost will automatically configure this DNS zone, if you provide it with the appropriate API credentials. You read documentation on how to get your API credentials on this page: https://yunohost.org/registar_api_{registrar}. (You can also manually configure your DNS records following the documentation as https://yunohost.org/dns )", # FIXME: i18n + "ask": m18n.n("domain_dns_registrar_supported", registrar=registrar), "value": registrar }) @@ -519,7 +518,7 @@ def _get_registrar_config_section(domain): registrar_infos["experimental_disclaimer"] = OrderedDict({ "type": "alert", "style": "danger", - "ask": f"So far, the interface with **{registrar}**'s API has not been properly tested and reviewed by the YunoHost's community. Support is **very experimental** - be careful!", # FIXME: i18n + "ask": m18n.n("domain_dns_registrar_experimental", registrar=registrar), }) # TODO : add a help tip with the link to the registar's API doc (c.f. Lexicon's README) @@ -565,7 +564,7 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= # FIXME: in the future, properly unify this with yunohost dyndns update if registrar == "yunohost": - logger.info("This domain is a nohost.me / nohost.st / ynh.fr and its DNS configuration is therefore already automatically handled by Yunohost without any further configuration.") # FIXME: i18n + logger.info(m18n.n("domain_dns_registrar_yunohost")) return {} if registrar == "parent_domain": @@ -641,7 +640,7 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= try: current_records = client.provider.list_records() except Exception as e: - raise YunohostError("Failed to list current records using the registrar's API: %s" % str(e), raw_msg=True) # FIXME: i18n + raise YunohostValidationError("domain_dns_push_failed_to_list", error=str(e)) managed_dns_records_hashes = _get_managed_dns_records_hashes(domain) @@ -697,7 +696,6 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= # # Step 1 : compute a first "diff" where we remove records which are the same on both sides - # NB / FIXME? : in all this we ignore the TTL value for now... # wanted_contents = [r["content"] for r in records["wanted"]] current_contents = [r["content"] for r in records["current"]] @@ -821,7 +819,7 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= progress.total = len(changes["delete"] + changes["create"] + changes["update"]) if progress.total == 0: - logger.success("Records already up to date, nothing to do.") # FIXME : i18n + logger.success(m18n.n("domain_dns_push_already_up_to_date")) return {} # @@ -829,7 +827,7 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= # operation_logger.start() - logger.info("Pushing DNS records...") + logger.info(m18n.n("domain_dns_puhsing")) new_managed_dns_records_hashes = [_hash_dns_record(r) for r in changes["unchanged"]] results = {"warnings": [], "errors": []} @@ -839,7 +837,7 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= for record in changes[action]: relative_name = record['name'].replace(base_dns_zone, '').rstrip('.') or '@' - progress(f"{action} {record['type']:^5} / {relative_name}") # FIXME: i18n + progress(f"{action} {record['type']:^5} / {relative_name}") # FIXME: i18n but meh # Apparently Lexicon yields us some 'id' during fetch # But wants 'identifier' during push ... @@ -865,28 +863,32 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= try: result = LexiconClient(query).execute() except Exception as e: - logger.error(f"Failed to {action} record {record['type']}/{record['name']} : {e}") # i18n? - results["errors"].append(f"Failed to {action} record {record['type']}/{record['name']} : {e}") + msg = m18n.n("domain_dns_push_record_failed", action=action, type=record['type'], name=record['name'], error=str(e)) + logger.error(msg) + results["errors"].append(msg) else: if result: new_managed_dns_records_hashes.append(_hash_dns_record(record)) else: - results["errors"].append(f"Failed to {action} record {record['type']}/{record['name']} : unknown error?") + msg = m18n.n("domain_dns_push_record_failed", action=action, type=record['type'], name=record['name'], error="unkonwn error?") + logger.error(msg) + results["errors"].append(msg) _set_managed_dns_records_hashes(domain, new_managed_dns_records_hashes) # Everything succeeded if len(results["errors"]) == 0: - logger.success("DNS records updated!") # FIXME: i18n + logger.success(m18n.("domain_dns_push_success")) return {} # Everything failed elif len(results["errors"]) + len(results["warnings"]) == progress.total: - logger.error("Updating the DNS records failed miserably") # FIXME: i18n + logger.error(m18n.("domain_dns_push_failed")) else: - logger.warning("DNS records partially updated: some warnings/errors were reported.") # FIXME: i18n + logger.warning(m18n.("domain_dns_push_partial_failure")) return results + def _get_managed_dns_records_hashes(domain: str) -> list: return _get_domain_settings(domain).get("managed_dns_records_hashes", []) From d161d0744a7f3e45b7ec922d57977f3b22f7e128 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 19 Sep 2021 16:50:57 +0200 Subject: [PATCH 117/130] dns: don't have empty sections in recommended conf --- src/yunohost/dns.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 8d44804f3..112478251 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -61,24 +61,28 @@ def domain_dns_suggest(domain): result = "" - result += "; Basic ipv4/ipv6 records" - for record in dns_conf["basic"]: - result += "\n{name} {ttl} IN {type} {value}".format(**record) + if dns_conf["basic"]: + result += "; Basic ipv4/ipv6 records" + for record in dns_conf["basic"]: + result += "\n{name} {ttl} IN {type} {value}".format(**record) - result += "\n\n" - result += "; XMPP" - for record in dns_conf["xmpp"]: - result += "\n{name} {ttl} IN {type} {value}".format(**record) + if dns_conf["mail"]: + result += "\n\n" + result += "; Mail" + for record in dns_conf["mail"]: + result += "\n{name} {ttl} IN {type} {value}".format(**record) + result += "\n\n" - result += "\n\n" - result += "; Mail" - for record in dns_conf["mail"]: - result += "\n{name} {ttl} IN {type} {value}".format(**record) - result += "\n\n" + if dns_conf["xmpp"]: + result += "\n\n" + result += "; XMPP" + for record in dns_conf["xmpp"]: + result += "\n{name} {ttl} IN {type} {value}".format(**record) - result += "; Extra" - for record in dns_conf["extra"]: - result += "\n{name} {ttl} IN {type} {value}".format(**record) + if dns_conf["extra"]: + result += "; Extra" + for record in dns_conf["extra"]: + result += "\n{name} {ttl} IN {type} {value}".format(**record) for name, record_list in dns_conf.items(): if name not in ("basic", "xmpp", "mail", "extra") and record_list: From 1543984a29c12d85aab837650eb3c690b9017796 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 19 Sep 2021 16:51:17 +0200 Subject: [PATCH 118/130] Fix i18n tests --- src/yunohost/dns.py | 8 ++++---- src/yunohost/tests/test_domains.py | 5 ----- tests/test_i18n_keys.py | 22 ++++++++++++++++++++++ 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 112478251..d26cc3ca4 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -831,7 +831,7 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= # operation_logger.start() - logger.info(m18n.n("domain_dns_puhsing")) + logger.info(m18n.n("domain_dns_pushing")) new_managed_dns_records_hashes = [_hash_dns_record(r) for r in changes["unchanged"]] results = {"warnings": [], "errors": []} @@ -882,13 +882,13 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= # Everything succeeded if len(results["errors"]) == 0: - logger.success(m18n.("domain_dns_push_success")) + logger.success(m18n.n("domain_dns_push_success")) return {} # Everything failed elif len(results["errors"]) + len(results["warnings"]) == progress.total: - logger.error(m18n.("domain_dns_push_failed")) + logger.error(m18n.n("domain_dns_push_failed")) else: - logger.warning(m18n.("domain_dns_push_partial_failure")) + logger.warning(m18n.n("domain_dns_push_partial_failure")) return results diff --git a/src/yunohost/tests/test_domains.py b/src/yunohost/tests/test_domains.py index b964d2ab6..bdb1b8a96 100644 --- a/src/yunohost/tests/test_domains.py +++ b/src/yunohost/tests/test_domains.py @@ -103,14 +103,12 @@ def test_change_main_domain(): def test_domain_config_get_default(): assert domain_config_get(TEST_DOMAINS[0], "feature.xmpp.xmpp") == 1 assert domain_config_get(TEST_DOMAINS[1], "feature.xmpp.xmpp") == 0 - assert domain_config_get(TEST_DOMAINS[1], "dns.advanced.ttl") == 3600 def test_domain_config_get_export(): assert domain_config_get(TEST_DOMAINS[0], export=True)["xmpp"] == 1 assert domain_config_get(TEST_DOMAINS[1], export=True)["xmpp"] == 0 - assert domain_config_get(TEST_DOMAINS[1], export=True)["ttl"] == 3600 def test_domain_config_set(): @@ -118,9 +116,6 @@ def test_domain_config_set(): domain_config_set(TEST_DOMAINS[1], "feature.xmpp.xmpp", "yes") assert domain_config_get(TEST_DOMAINS[1], "feature.xmpp.xmpp") == 1 - domain_config_set(TEST_DOMAINS[1], "dns.advanced.ttl", 10) - assert domain_config_get(TEST_DOMAINS[1], "dns.advanced.ttl") == 10 - def test_domain_configs_unknown(): with pytest.raises(YunohostValidationError): diff --git a/tests/test_i18n_keys.py b/tests/test_i18n_keys.py index 33c1f7b65..e14ac88a8 100644 --- a/tests/test_i18n_keys.py +++ b/tests/test_i18n_keys.py @@ -6,6 +6,7 @@ import glob import json import yaml import subprocess +import toml ignore = [ "password_too_simple_", @@ -165,6 +166,24 @@ def find_expected_string_keys(): for check in checks: yield "diagnosis_mail_%s" % check + registrars = toml.load(open('data/other/registrar_list.toml')) + supported_registrars = ["ovh", "gandi", "godaddy"] + for registrar in supported_registrars: + for key in registrars[registrar].keys(): + yield f"domain_config_{key}" + + domain_config = toml.load(open('data/other/config_domain.toml')) + for panel in domain_config.values(): + if not isinstance(panel, dict): + continue + for section in panel.values(): + if not isinstance(section, dict): + continue + for key, values in section.items(): + if not isinstance(values, dict): + continue + yield f"domain_config_{key}" + ############################################################################### # Load en locale json keys # @@ -204,3 +223,6 @@ def test_unused_i18n_keys(): raise Exception( "Those i18n keys appears unused:\n" " - " + "\n - ".join(unused_keys) ) + +test_undefined_i18n_keys() +test_unused_i18n_keys() From ae03be3bad73afa47e371ae02bf30d6ee1ee7a43 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 19 Sep 2021 17:10:14 +0200 Subject: [PATCH 119/130] i18n tests: simplify hardcoded stuff --- data/hooks/diagnosis/12-dnsrecords.py | 7 ++++++ data/hooks/diagnosis/21-web.py | 4 +++ data/hooks/diagnosis/24-mail.py | 14 +++++++---- src/yunohost/app.py | 4 +++ src/yunohost/utils/password.py | 5 ++++ tests/test_i18n_keys.py | 36 --------------------------- 6 files changed, 29 insertions(+), 41 deletions(-) diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 854f348f5..36180781f 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -199,6 +199,7 @@ class DNSRecordsDiagnoser(Diagnoser): status_ns, _ = dig(domain, "NS", resolvers="force_external") status_a, _ = dig(domain, "A", resolvers="force_external") if "ok" not in [status_ns, status_a]: + # i18n: diagnosis_domain_not_found_details details["not_found"].append( ( "diagnosis_domain_%s_details" % (expire_date), @@ -233,6 +234,12 @@ class DNSRecordsDiagnoser(Diagnoser): # Allow to ignore specifically a single domain if len(details[alert_type]) == 1: meta["domain"] = details[alert_type][0][1]["domain"] + + # i18n: diagnosis_domain_expiration_not_found + # i18n: diagnosis_domain_expiration_error + # i18n: diagnosis_domain_expiration_warning + # i18n: diagnosis_domain_expiration_success + # i18n: diagnosis_domain_expiration_not_found_details yield dict( meta=meta, data={}, diff --git a/data/hooks/diagnosis/21-web.py b/data/hooks/diagnosis/21-web.py index 40a6c26b4..2072937e5 100644 --- a/data/hooks/diagnosis/21-web.py +++ b/data/hooks/diagnosis/21-web.py @@ -121,6 +121,10 @@ class WebDiagnoser(Diagnoser): for domain in domains: + # i18n: diagnosis_http_bad_status_code + # i18n: diagnosis_http_connection_error + # i18n: diagnosis_http_timeout + # If both IPv4 and IPv6 (if applicable) are good if all( results[ipversion][domain]["status"] == "ok" for ipversion in ipversions diff --git a/data/hooks/diagnosis/24-mail.py b/data/hooks/diagnosis/24-mail.py index 50b8dc12e..266678557 100644 --- a/data/hooks/diagnosis/24-mail.py +++ b/data/hooks/diagnosis/24-mail.py @@ -35,11 +35,11 @@ class MailDiagnoser(Diagnoser): # TODO check that the recent mail logs are not filled with thousand of email sending (unusual number of mail sent) # TODO check for unusual failed sending attempt being refused in the logs ? checks = [ - "check_outgoing_port_25", - "check_ehlo", - "check_fcrdns", - "check_blacklist", - "check_queue", + "check_outgoing_port_25", # i18n: diagnosis_mail_outgoing_port_25_ok + "check_ehlo", # i18n: diagnosis_mail_ehlo_ok + "check_fcrdns", # i18n: diagnosis_mail_fcrdns_ok + "check_blacklist", # i18n: diagnosis_mail_blacklist_ok + "check_queue", # i18n: diagnosis_mail_queue_ok ] for check in checks: self.logger_debug("Running " + check) @@ -102,6 +102,10 @@ class MailDiagnoser(Diagnoser): continue if r["status"] != "ok": + # i18n: diagnosis_mail_ehlo_bad_answer + # i18n: diagnosis_mail_ehlo_bad_answer_details + # i18n: diagnosis_mail_ehlo_unreachable + # i18n: diagnosis_mail_ehlo_unreachable_details summary = r["status"].replace("error_smtp_", "diagnosis_mail_ehlo_") yield dict( meta={"test": "mail_ehlo", "ipversion": ipversion}, diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 2d9ecce22..78428595d 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -820,6 +820,10 @@ def app_install( if confirm is None or force or Moulinette.interface.type == "api": return + # i18n: confirm_app_install_warning + # i18n: confirm_app_install_danger + # i18n: confirm_app_install_thirdparty + if confirm in ["danger", "thirdparty"]: answer = Moulinette.prompt( m18n.n("confirm_app_install_" + confirm, answers="Yes, I understand"), diff --git a/src/yunohost/utils/password.py b/src/yunohost/utils/password.py index 9e693d8cd..188850183 100644 --- a/src/yunohost/utils/password.py +++ b/src/yunohost/utils/password.py @@ -111,8 +111,13 @@ class PasswordValidator(object): listed = password in SMALL_PWD_LIST or self.is_in_most_used_list(password) strength_level = self.strength_level(password) if listed: + # i18n: password_listed return ("error", "password_listed") if strength_level < self.validation_strength: + # i18n: password_too_simple_1 + # i18n: password_too_simple_2 + # i18n: password_too_simple_3 + # i18n: password_too_simple_4 return ("error", "password_too_simple_%s" % self.validation_strength) return ("success", "") diff --git a/tests/test_i18n_keys.py b/tests/test_i18n_keys.py index e14ac88a8..119ba85f1 100644 --- a/tests/test_i18n_keys.py +++ b/tests/test_i18n_keys.py @@ -8,14 +8,6 @@ import yaml import subprocess import toml -ignore = [ - "password_too_simple_", - "password_listed", - "backup_method_", - "backup_applying_method_", - "confirm_app_install_", -] - ############################################################################### # Find used keys in python code # ############################################################################### @@ -138,34 +130,6 @@ def find_expected_string_keys(): yield "backup_applying_method_%s" % method yield "backup_method_%s_finished" % method - for level in ["danger", "thirdparty", "warning"]: - yield "confirm_app_install_%s" % level - - for errortype in ["not_found", "error", "warning", "success", "not_found_details"]: - yield "diagnosis_domain_expiration_%s" % errortype - yield "diagnosis_domain_not_found_details" - - for errortype in ["bad_status_code", "connection_error", "timeout"]: - yield "diagnosis_http_%s" % errortype - - yield "password_listed" - for i in [1, 2, 3, 4]: - yield "password_too_simple_%s" % i - - checks = [ - "outgoing_port_25_ok", - "ehlo_ok", - "fcrdns_ok", - "blacklist_ok", - "queue_ok", - "ehlo_bad_answer", - "ehlo_unreachable", - "ehlo_bad_answer_details", - "ehlo_unreachable_details", - ] - for check in checks: - yield "diagnosis_mail_%s" % check - registrars = toml.load(open('data/other/registrar_list.toml')) supported_registrars = ["ovh", "gandi", "godaddy"] for registrar in supported_registrars: From 04487ed6bf3a0d7e4119e75fa093071b49906bbd Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 19 Sep 2021 17:30:52 +0200 Subject: [PATCH 120/130] dns: Don't include subdomains stuff in dyndns update, because this probably ain't gonna work --- src/yunohost/dns.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index d26cc3ca4..4ac62fb46 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -160,7 +160,14 @@ def _build_dns_conf(base_domain): ipv4 = get_public_ip() ipv6 = get_public_ip(6) - subdomains = _list_subdomains_of(base_domain) + # If this is a ynh_dyndns_domain, we're not gonna include all the subdomains in the conf + # Because dynette only accept a specific list of name/type + # And the wildcard */A already covers the bulk of use cases + if any(base_domain.endswith("." + ynh_dyndns_domain) for ynh_dyndns_domain in YNH_DYNDNS_DOMAINS): + subdomains = [] + else: + subdomains = _list_subdomains_of(base_domain) + domains_settings = {domain: domain_config_get(domain, export=True) for domain in [base_domain] + subdomains} From 7fd76a688479ace4d9742439efc46cbb7b1b5b16 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 19 Sep 2021 17:32:35 +0200 Subject: [PATCH 121/130] dns: Reintroduce include_empty_AAAA_if_no_ipv6 option, needed for diagnosis --- src/yunohost/dns.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 4ac62fb46..a4fcbc3fa 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -110,7 +110,7 @@ def _list_subdomains_of(parent_domain): return out -def _build_dns_conf(base_domain): +def _build_dns_conf(base_domain, include_empty_AAAA_if_no_ipv6=False): """ Internal function that will returns a data structure containing the needed information to generate/adapt the dns configuration @@ -197,9 +197,8 @@ def _build_dns_conf(base_domain): if ipv6: basic.append([basename, ttl, "AAAA", ipv6]) - # TODO - # elif include_empty_AAAA_if_no_ipv6: - # basic.append(["@", ttl, "AAAA", None]) + elif include_empty_aaaa_if_no_ipv6: + basic.append(["@", ttl, "AAAA", None]) ######### # Email # @@ -251,9 +250,8 @@ def _build_dns_conf(base_domain): if ipv6: extra.append([f"*{suffix}", ttl, "AAAA", ipv6]) - # TODO - # elif include_empty_AAAA_if_no_ipv6: - # extra.append(["*", ttl, "AAAA", None]) + elif include_empty_AAAA_if_no_ipv6: + extra.append(["*", ttl, "AAAA", None]) extra.append([basename, ttl, "CAA", '128 issue "letsencrypt.org"']) From 69c756918f908b61c05c61b4b71cf5912aa931d2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 19 Sep 2021 17:37:44 +0200 Subject: [PATCH 122/130] dyndns: Don't try to push anything if no ipv4/ipv6 --- src/yunohost/dyndns.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index 9cb6dc567..4297a3408 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -228,10 +228,6 @@ def dyndns_update( from yunohost.dns import _build_dns_conf - # Get old ipv4/v6 - - old_ipv4, old_ipv6 = (None, None) # (default values) - # If domain is not given, try to guess it from keys available... if domain is None: (domain, key) = _guess_current_dyndns_domain(dyn_host) @@ -311,6 +307,10 @@ def dyndns_update( logger.debug("Old IPv4/v6 are (%s, %s)" % (old_ipv4, old_ipv6)) logger.debug("Requested IPv4/v6 are (%s, %s)" % (ipv4, ipv6)) + if ipv4 is None and ipv6 is None: + logger.debug("No ipv4 nor ipv6 ?! Sounds like the server is not connected to the internet, or the ip.yunohost.org infrastructure is down somehow") + return + # no need to update if (not force and not dry_run) and (old_ipv4 == ipv4 and old_ipv6 == ipv6): logger.info("No updated needed.") From c12f9b64ea21a0c5b524981d72e685230a6b3e65 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 19 Sep 2021 17:40:21 +0200 Subject: [PATCH 123/130] Typo --- src/yunohost/dns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index a4fcbc3fa..d7ed3d724 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -197,7 +197,7 @@ def _build_dns_conf(base_domain, include_empty_AAAA_if_no_ipv6=False): if ipv6: basic.append([basename, ttl, "AAAA", ipv6]) - elif include_empty_aaaa_if_no_ipv6: + elif include_empty_AAAA_if_no_ipv6: basic.append(["@", ttl, "AAAA", None]) ######### From 499f06f5f6cf40dfffc2d32b42878011d2096587 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 19 Sep 2021 17:41:19 +0200 Subject: [PATCH 124/130] autodns: Godaddy doesn't supports CAA records --- src/yunohost/dns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index d7ed3d724..02ed31b21 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -857,7 +857,7 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= if registrar == "godaddy": if record["name"] == base_dns_zone: record["name"] = "@." + record["name"] - if record["type"] in ["MX", "SRV"]: + if record["type"] in ["MX", "SRV", "CAA"]: logger.warning(f"Pushing {record['type']} records is not properly supported by Lexicon/Godaddy.") results["warnings"].append(f"Pushing {record['type']} records is not properly supported by Lexicon/Godaddy.") continue From 065ebec8210c23142c611e837edd08a10ce1b536 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 19 Sep 2021 17:41:52 +0200 Subject: [PATCH 125/130] autodns: Minor fix for error handling --- src/yunohost/dns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index 02ed31b21..cf1e2d636 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -886,7 +886,7 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= _set_managed_dns_records_hashes(domain, new_managed_dns_records_hashes) # Everything succeeded - if len(results["errors"]) == 0: + if len(results["errors"]) + len(results["warnings"]) == 0: logger.success(m18n.n("domain_dns_push_success")) return {} # Everything failed From 8f8b6eae7cafd6dfc2885b4938026e0d153edccf Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 19 Sep 2021 18:02:45 +0200 Subject: [PATCH 126/130] domains: Make sure domain setting folder exists with appropriate perms --- data/hooks/conf_regen/01-yunohost | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data/hooks/conf_regen/01-yunohost b/data/hooks/conf_regen/01-yunohost index e9c0fc4aa..445faa5a4 100755 --- a/data/hooks/conf_regen/01-yunohost +++ b/data/hooks/conf_regen/01-yunohost @@ -35,6 +35,10 @@ do_init_regen() { mkdir -p /home/yunohost.app chmod 755 /home/yunohost.app + # Domain settings + mkdir -p /etc/yunohost/domains + chmod 700 /etc/yunohost/domains + # Backup folders mkdir -p /home/yunohost.backup/archives chmod 750 /home/yunohost.backup/archives @@ -179,6 +183,8 @@ do_post_regen() { [ ! -e "/home/$USER" ] || setfacl -m g:all_users:--- /home/$USER done + # Domain settings + mkdir -p /etc/yunohost/domains # Misc configuration / state files chown root:root $(ls /etc/yunohost/{*.yml,*.yaml,*.json,mysql,psql} 2>/dev/null) @@ -187,6 +193,7 @@ do_post_regen() { # Apps folder, custom hooks folder [[ ! -e /etc/yunohost/hooks.d ]] || (chown root /etc/yunohost/hooks.d && chmod 700 /etc/yunohost/hooks.d) [[ ! -e /etc/yunohost/apps ]] || (chown root /etc/yunohost/apps && chmod 700 /etc/yunohost/apps) + [[ ! -e /etc/yunohost/domains ]] || (chown root /etc/yunohost/domains && chmod 700 /etc/yunohost/domains) # Create ssh.app and sftp.app groups if they don't exist yet grep -q '^ssh.app:' /etc/group || groupadd ssh.app From 2422a25f55a3211979032bb77024d2b6f2cbf6be Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 19 Sep 2021 18:04:32 +0200 Subject: [PATCH 127/130] domains: make sure to backup/restore domain settings --- data/hooks/backup/20-conf_ynh_settings | 1 + data/hooks/restore/20-conf_ynh_settings | 1 + 2 files changed, 2 insertions(+) diff --git a/data/hooks/backup/20-conf_ynh_settings b/data/hooks/backup/20-conf_ynh_settings index 77148c4d9..9b56f1579 100644 --- a/data/hooks/backup/20-conf_ynh_settings +++ b/data/hooks/backup/20-conf_ynh_settings @@ -12,6 +12,7 @@ backup_dir="${1}/conf/ynh" # Backup the configuration ynh_backup "/etc/yunohost/firewall.yml" "${backup_dir}/firewall.yml" ynh_backup "/etc/yunohost/current_host" "${backup_dir}/current_host" +ynh_backup "/etc/yunohost/domains" "${backup_dir}/domains" [ ! -e "/etc/yunohost/settings.json" ] || ynh_backup "/etc/yunohost/settings.json" "${backup_dir}/settings.json" [ ! -d "/etc/yunohost/dyndns" ] || ynh_backup "/etc/yunohost/dyndns" "${backup_dir}/dyndns" [ ! -d "/etc/dkim" ] || ynh_backup "/etc/dkim" "${backup_dir}/dkim" diff --git a/data/hooks/restore/20-conf_ynh_settings b/data/hooks/restore/20-conf_ynh_settings index 4de29a4aa..4c4c6ed5e 100644 --- a/data/hooks/restore/20-conf_ynh_settings +++ b/data/hooks/restore/20-conf_ynh_settings @@ -2,6 +2,7 @@ backup_dir="$1/conf/ynh" cp -a "${backup_dir}/current_host" /etc/yunohost/current_host cp -a "${backup_dir}/firewall.yml" /etc/yunohost/firewall.yml +cp -a "${backup_dir}/domains" /etc/yunohost/domains [ ! -e "${backup_dir}/settings.json" ] || cp -a "${backup_dir}/settings.json" "/etc/yunohost/settings.json" [ ! -d "${backup_dir}/dyndns" ] || cp -raT "${backup_dir}/dyndns" "/etc/yunohost/dyndns" [ ! -d "${backup_dir}/dkim" ] || cp -raT "${backup_dir}/dkim" "/etc/dkim" From cdabfc12cc47bce14856b61b25d095d3ca07b3a7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 19 Sep 2021 19:34:45 +0200 Subject: [PATCH 128/130] dns: Repair diagnosis ugh --- data/hooks/diagnosis/12-dnsrecords.py | 34 +++++++++++++++------------ src/yunohost/dns.py | 29 ++++++++++++++--------- src/yunohost/domain.py | 4 ++++ 3 files changed, 41 insertions(+), 26 deletions(-) diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 36180781f..e3cbe7078 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -11,7 +11,7 @@ from moulinette.utils.process import check_output from yunohost.utils.dns import dig, YNH_DYNDNS_DOMAINS from yunohost.diagnosis import Diagnoser from yunohost.domain import domain_list, _get_maindomain -from yunohost.dns import _build_dns_conf +from yunohost.dns import _build_dns_conf, _get_dns_zone_for_domain SPECIAL_USE_TLDS = ["local", "localhost", "onion", "test"] @@ -26,17 +26,15 @@ class DNSRecordsDiagnoser(Diagnoser): main_domain = _get_maindomain() - all_domains = domain_list()["domains"] + all_domains = domain_list(exclude_subdomains=True)["domains"] for domain in all_domains: self.logger_debug("Diagnosing DNS conf for %s" % domain) - is_subdomain = domain.split(".", 1)[1] in all_domains is_specialusedomain = any( domain.endswith("." + tld) for tld in SPECIAL_USE_TLDS ) for report in self.check_domain( domain, domain == main_domain, - is_subdomain=is_subdomain, is_specialusedomain=is_specialusedomain, ): yield report @@ -55,16 +53,16 @@ class DNSRecordsDiagnoser(Diagnoser): for report in self.check_expiration_date(domains_from_registrar): yield report - def check_domain(self, domain, is_main_domain, is_subdomain, is_specialusedomain): + def check_domain(self, domain, is_main_domain, is_specialusedomain): + + base_dns_zone = _get_dns_zone_for_domain(domain) + basename = domain.replace(base_dns_zone, "").rstrip(".") or "@" expected_configuration = _build_dns_conf( domain, include_empty_AAAA_if_no_ipv6=True ) categories = ["basic", "mail", "xmpp", "extra"] - # For subdomains, we only diagnosis A and AAAA records - if is_subdomain: - categories = ["basic"] if is_specialusedomain: categories = [] @@ -82,8 +80,16 @@ class DNSRecordsDiagnoser(Diagnoser): results = {} for r in records: + id_ = r["type"] + ":" + r["name"] - r["current"] = self.get_current_record(domain, r["name"], r["type"]) + fqdn = r["name"] + "." + base_dns_zone if r["name"] != "@" else domain + + # Ugly hack to not check mail records for subdomains stuff, otherwise will end up in a shitstorm of errors for people with many subdomains... + # Should find a cleaner solution in the suggested conf... + if r["type"] in ["MX", "TXT"] and fqdn not in [domain, f'mail._domainkey.{domain}', f'_dmarc.{domain}']: + continue + + r["current"] = self.get_current_record(fqdn, r["type"]) if r["value"] == "@": r["value"] = domain + "." @@ -106,7 +112,7 @@ class DNSRecordsDiagnoser(Diagnoser): # A bad or missing A record is critical ... # And so is a wrong AAAA record # (However, a missing AAAA record is acceptable) - if results["A:@"] != "OK" or results["AAAA:@"] == "WRONG": + if results[f"A:{basename}"] != "OK" or results[f"AAAA:{basename}"] == "WRONG": return True return False @@ -139,10 +145,9 @@ class DNSRecordsDiagnoser(Diagnoser): yield output - def get_current_record(self, domain, name, type_): + def get_current_record(self, fqdn, type_): - query = "%s.%s" % (name, domain) if name != "@" else domain - success, answers = dig(query, type_, resolvers="force_external") + success, answers = dig(fqdn, type_, resolvers="force_external") if success != "ok": return None @@ -170,7 +175,7 @@ class DNSRecordsDiagnoser(Diagnoser): ) # For SPF, ignore parts starting by ip4: or ip6: - if r["name"] == "@": + if 'v=spf1' in r["value"]: current = { part for part in current @@ -189,7 +194,6 @@ class DNSRecordsDiagnoser(Diagnoser): """ Alert if expiration date of a domain is soon """ - details = {"not_found": [], "error": [], "warning": [], "success": []} for domain in domains: diff --git a/src/yunohost/dns.py b/src/yunohost/dns.py index cf1e2d636..745578806 100644 --- a/src/yunohost/dns.py +++ b/src/yunohost/dns.py @@ -198,7 +198,7 @@ def _build_dns_conf(base_domain, include_empty_AAAA_if_no_ipv6=False): if ipv6: basic.append([basename, ttl, "AAAA", ipv6]) elif include_empty_AAAA_if_no_ipv6: - basic.append(["@", ttl, "AAAA", None]) + basic.append([basename, ttl, "AAAA", None]) ######### # Email # @@ -245,15 +245,17 @@ def _build_dns_conf(base_domain, include_empty_AAAA_if_no_ipv6=False): # Extra # ######### - if ipv4: - extra.append([f"*{suffix}", ttl, "A", ipv4]) + # Only recommend wildcard and CAA for the top level + if domain == base_domain: + if ipv4: + extra.append([f"*{suffix}", ttl, "A", ipv4]) - if ipv6: - extra.append([f"*{suffix}", ttl, "AAAA", ipv6]) - elif include_empty_AAAA_if_no_ipv6: - extra.append(["*", ttl, "AAAA", None]) + if ipv6: + extra.append([f"*{suffix}", ttl, "AAAA", ipv6]) + elif include_empty_AAAA_if_no_ipv6: + extra.append([f"*{suffix}", ttl, "AAAA", None]) - extra.append([basename, ttl, "CAA", '128 issue "letsencrypt.org"']) + extra.append([basename, ttl, "CAA", '128 issue "letsencrypt.org"']) #################### # Standard records # @@ -463,8 +465,13 @@ def _get_dns_zone_for_domain(domain): write_to_file(cache_file, parent) return parent - logger.warning(f"Could not identify the dns_zone for domain {domain}, returning {parent_list[-1]}") - return parent_list[-1] + if len(parent_list) >= 2: + zone = parent_list[-2] + else: + zone = parent_list[-1] + + logger.warning(f"Could not identify the dns zone for domain {domain}, returning {zone}") + return zone def _get_registrar_config_section(domain): @@ -649,7 +656,7 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= try: current_records = client.provider.list_records() except Exception as e: - raise YunohostValidationError("domain_dns_push_failed_to_list", error=str(e)) + raise YunohostError("domain_dns_push_failed_to_list", error=str(e)) managed_dns_records_hashes = _get_managed_dns_records_hashes(domain) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index d13900224..a5db7c7ab 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -93,6 +93,10 @@ def domain_list(exclude_subdomains=False): result_list = sorted(result_list, key=cmp_domain) + # Don't cache answer if using exclude_subdomains + if exclude_subdomains: + return {"domains": result_list, "main": _get_maindomain()} + domain_list_cache = {"domains": result_list, "main": _get_maindomain()} return domain_list_cache From 18e2fb14c4c9dc592223855c82dbb7a8402f117b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 19 Sep 2021 19:35:23 +0200 Subject: [PATCH 129/130] tests: uhoh forgot to remove some tmp stuff --- tests/test_i18n_keys.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_i18n_keys.py b/tests/test_i18n_keys.py index 119ba85f1..90e14848d 100644 --- a/tests/test_i18n_keys.py +++ b/tests/test_i18n_keys.py @@ -187,6 +187,3 @@ def test_unused_i18n_keys(): raise Exception( "Those i18n keys appears unused:\n" " - " + "\n - ".join(unused_keys) ) - -test_undefined_i18n_keys() -test_unused_i18n_keys() From be8c6f2c35c55ef142fd346ba05b170ce022c5a9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 19 Sep 2021 22:59:15 +0200 Subject: [PATCH 130/130] Fix tests --- src/yunohost/backup.py | 3 +++ src/yunohost/tests/test_backuprestore.py | 4 +++- src/yunohost/tests/test_dns.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 80f01fd35..7b580e424 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -44,6 +44,7 @@ from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_file, mkdir, write_to_yaml, read_yaml from moulinette.utils.process import check_output +from yunohost.domain import domain_list_cache from yunohost.app import ( app_info, _is_installed, @@ -1284,6 +1285,8 @@ class RestoreManager: else: operation_logger.success() + domain_list_cache = {} + regen_conf() _tools_migrations_run_after_system_restore( diff --git a/src/yunohost/tests/test_backuprestore.py b/src/yunohost/tests/test_backuprestore.py index b24d3442d..6e2c3b514 100644 --- a/src/yunohost/tests/test_backuprestore.py +++ b/src/yunohost/tests/test_backuprestore.py @@ -2,6 +2,7 @@ import pytest import os import shutil import subprocess +from mock import patch from .conftest import message, raiseYunohostError, get_test_apps_dir @@ -77,7 +78,8 @@ def setup_function(function): if "with_permission_app_installed" in markers: assert not app_is_installed("permissions_app") user_create("alice", "Alice", "White", maindomain, "test123Ynh") - install_app("permissions_app_ynh", "/urlpermissionapp" "&admin=alice") + with patch.object(os, "isatty", return_value=False): + install_app("permissions_app_ynh", "/urlpermissionapp" "&admin=alice") assert app_is_installed("permissions_app") if "with_custom_domain" in markers: diff --git a/src/yunohost/tests/test_dns.py b/src/yunohost/tests/test_dns.py index 154d1dc9a..35940764c 100644 --- a/src/yunohost/tests/test_dns.py +++ b/src/yunohost/tests/test_dns.py @@ -34,7 +34,7 @@ def test_get_dns_zone_from_domain_existing(): assert _get_dns_zone_for_domain("non-existing-domain.yunohost.org") == "yunohost.org" assert _get_dns_zone_for_domain("yolo.nohost.me") == "nohost.me" assert _get_dns_zone_for_domain("foo.yolo.nohost.me") == "nohost.me" - assert _get_dns_zone_for_domain("yolo.test") == "test" + assert _get_dns_zone_for_domain("yolo.test") == "yolo.test" assert _get_dns_zone_for_domain("foo.yolo.test") == "test"