From d72156b91f95b5c0ee5283bf3c3ee2c9507406ea Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 20 Apr 2020 01:40:37 +0200 Subject: [PATCH 01/15] [enh] Check domain expiration date --- data/hooks/diagnosis/12-dnsrecords.py | 79 +++++++++++++++++++++++++-- debian/control | 2 +- locales/en.json | 5 ++ 3 files changed, 81 insertions(+), 5 deletions(-) diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 53afb2c2d..402f5f994 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -1,6 +1,10 @@ #!/usr/bin/env python import os +import re + +from datetime import datetime, timedelta +from subprocess import Popen, PIPE from moulinette.utils.filesystem import read_file @@ -8,6 +12,7 @@ from yunohost.utils.network import dig from yunohost.diagnosis import Diagnoser from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain +SMALL_SUFFIX_LIST = ['noho.st', 'nohost.me', 'ynh.fr', 'netlib.re'] class DNSRecordsDiagnoser(Diagnoser): @@ -31,10 +36,12 @@ class DNSRecordsDiagnoser(Diagnoser): is_subdomain = domain.split(".",1)[1] in all_domains for report in self.check_domain(domain, domain == main_domain, is_subdomain=is_subdomain): yield report - - # FIXME : somewhere, should implement a check for reverse DNS ... - - # FIXME / TODO : somewhere, could also implement a check for domain expiring soon + + # Check if a domain buy by the user will expire soon + domains_from_registrar = ['.'.join(domain.split('.')[-2:]) for domain in all_domains] + domains_from_registrar = set(domains_from_registrar) - set(SMALL_SUFFIX_LIST) + for report in self.check_expiration_date(domains_from_registrar): + yield report def check_domain(self, domain, is_main_domain, is_subdomain): @@ -137,5 +144,69 @@ class DNSRecordsDiagnoser(Diagnoser): return r["current"] == r["value"] + def check_expiration_date(self, domains): + """ + Alert if expiration date of a domain is soon + """ + + # FIXME find a way to ignore a specific domain without + # create a report by domain each time. We need something small + details = { + "not_found": [], + "error": [], + "warning": [], + "info": [] + } + + for domain in domains: + expire_date = self.get_domain_expiration(domain) + + if not expire_date: + details["not_found"].append(( + "diagnosis_domain_expiration_date_not_found", + {"domain": domain})) + continue + + expire_in = expire_date - datetime.now() + + alert_type = "info" + if expire_in <= timedelta(7): + alert_type = "error" + elif expire_in <= timedelta(30): + alert_type = "warning" + + args = { + "domain": domain, + "days": expire_in.days - 1, + "expire_date": str(expire_date) + } + details[alert_type].append(("diagnosis_domain_expires_in", args)) + + for alert_type in ["error", "warning", "not_found", "info"]: + if details[alert_type]: + yield dict(meta={"category": "expiration"}, + data={}, + status=alert_type.upper() if alert_type != "not_found" else "INFO", + summary="diagnosis_domain_expiration_" + alert_type, + details=details[alert_type]) + + def get_domain_expiration(self, domain): + """ + Return the expiration datetime of a domain or None + """ + + p1 = Popen(['whois', domain], stdout=PIPE) + p2 = Popen(['grep', 'Expir'], stdin=p1.stdout, stdout=PIPE) + out, err = p2.communicate() + out = out.decode("utf-8").split('\n') + p1.terminate() + #p2.terminate() + + for line in out: + match = re.search(r'\d{4}-\d{2}-\d{2}', line) + if match is not None: + return datetime.strptime(match.group(), '%Y-%m-%d') + return None + def main(args, env, loggers): return DNSRecordsDiagnoser(args, env, loggers).diagnose() diff --git a/debian/control b/debian/control index 5bcd78491..fcffa87f6 100644 --- a/debian/control +++ b/debian/control @@ -29,7 +29,7 @@ Depends: ${python:Depends}, ${misc:Depends} , redis-server , metronome , git, curl, wget, cron, unzip, jq - , lsb-release, haveged, fake-hwclock, equivs, lsof + , lsb-release, haveged, fake-hwclock, equivs, lsof, whois Recommends: yunohost-admin , ntp, inetutils-ping | iputils-ping , bash-completion, rsyslog diff --git a/locales/en.json b/locales/en.json index 3607052e3..7f6d9ce34 100644 --- a/locales/en.json +++ b/locales/en.json @@ -172,6 +172,11 @@ "diagnosis_dns_missing_record": "According to the recommended DNS configuration, you should add a DNS record with the following info.
Type: {type}
Name: {name}
Value: {value}", "diagnosis_dns_discrepancy": "The following DNS record does not seem to follow the recommended configuration:
Type: {type}
Name: {name}
Current value: {current}
Excepted value: {value}", "diagnosis_dns_point_to_doc": "Please check the documentation at https://yunohost.org/dns_config if you need help about configuring DNS records.", + "diagnosis_domain_expiration_not_found": "Unable to check the expiration date of some domains", + "diagnosis_domain_expiration_info": "Domains expiration dates", + "diagnosis_domain_expiration_warning": "Some domains expire in less than a month", + "diagnosis_domain_expiration_error": "Some domains expire in less than a week", + "diagnosis_domain_expires_in": "{domain} expires in {days} days.", "diagnosis_services_running": "Service {service} is running!", "diagnosis_services_conf_broken": "Configuration is broken for service {service}!", "diagnosis_services_bad_status": "Service {service} is {status} :(", From c0f27c02353d7007f0ae32823213866c6321fa35 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 20 Apr 2020 01:47:16 +0200 Subject: [PATCH 02/15] [fix] i18n checks --- tests/test_i18n_keys.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_i18n_keys.py b/tests/test_i18n_keys.py index 9125c5d52..e3edc4d48 100644 --- a/tests/test_i18n_keys.py +++ b/tests/test_i18n_keys.py @@ -119,13 +119,16 @@ def find_expected_string_keys(): for level in ["danger", "thirdparty", "warning"]: yield "confirm_app_install_%s" % level + for errortype in ["not_found", "error", "warning", "info"]: + yield "diagnosis_domain_expiration_" % errortype + for errortype in ["bad_status_code", "connection_error", "timeout"]: yield "diagnosis_http_%s" % errortype yield "password_listed" for i in [1, 2, 3, 4]: yield "password_too_simple_%s" % i - + checks = ["outgoing_port_25_ok", "ehlo_ok", "fcrdns_ok", "blacklist_ok", "queue_ok", "ehlo_bad_answer", "ehlo_unreachable", "ehlo_bad_answer_details", From 6954ca9002c808dab7ad3d3b2b3c27f31661850b Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 20 Apr 2020 01:50:04 +0200 Subject: [PATCH 03/15] [fix] i18n checks --- tests/test_i18n_keys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_i18n_keys.py b/tests/test_i18n_keys.py index e3edc4d48..6dcfd5c70 100644 --- a/tests/test_i18n_keys.py +++ b/tests/test_i18n_keys.py @@ -120,7 +120,7 @@ def find_expected_string_keys(): yield "confirm_app_install_%s" % level for errortype in ["not_found", "error", "warning", "info"]: - yield "diagnosis_domain_expiration_" % errortype + yield "diagnosis_domain_expiration_%s" % errortype for errortype in ["bad_status_code", "connection_error", "timeout"]: yield "diagnosis_http_%s" % errortype From d98d753f521def498bd8300294a3ff7b21b0f16a Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 27 Apr 2020 17:07:38 +0200 Subject: [PATCH 04/15] [fix] Bad i18n key --- data/hooks/diagnosis/12-dnsrecords.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 402f5f994..b6d87aed0 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -163,7 +163,7 @@ class DNSRecordsDiagnoser(Diagnoser): if not expire_date: details["not_found"].append(( - "diagnosis_domain_expiration_date_not_found", + "diagnosis_domain_expiration_not_found", {"domain": domain})) continue From cdb917e5652bc3782ef861272be9ae3acf32d0db Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 27 Apr 2020 18:09:07 +0200 Subject: [PATCH 05/15] [enh] Explain why domain expiration not found --- data/hooks/diagnosis/12-dnsrecords.py | 47 +++++++++++++++------------ locales/en.json | 2 ++ tests/test_i18n_keys.py | 3 +- 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index b6d87aed0..f03e92df3 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -14,6 +14,7 @@ from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain SMALL_SUFFIX_LIST = ['noho.st', 'nohost.me', 'ynh.fr', 'netlib.re'] + class DNSRecordsDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] @@ -33,12 +34,13 @@ class DNSRecordsDiagnoser(Diagnoser): all_domains = domain_list()["domains"] for domain in all_domains: self.logger_debug("Diagnosing DNS conf for %s" % domain) - is_subdomain = domain.split(".",1)[1] in all_domains + is_subdomain = domain.split(".", 1)[1] in all_domains for report in self.check_domain(domain, domain == main_domain, is_subdomain=is_subdomain): yield report - + # Check if a domain buy by the user will expire soon domains_from_registrar = ['.'.join(domain.split('.')[-2:]) for domain in all_domains] + domains_from_registrar = ['ynh.local', 'grimaud.me', 'netlib.re', 'arn-fai.net'] domains_from_registrar = set(domains_from_registrar) - set(SMALL_SUFFIX_LIST) for report in self.check_expiration_date(domains_from_registrar): yield report @@ -74,7 +76,6 @@ class DNSRecordsDiagnoser(Diagnoser): results[id_] = "WRONG" discrepancies.append(("diagnosis_dns_discrepancy", r)) - def its_important(): # Every mail DNS records are important for main domain # For other domain, we only report it as a warning for now... @@ -135,7 +136,7 @@ class DNSRecordsDiagnoser(Diagnoser): if r["name"] == "@": current = {part for part in current if not part.startswith("ip4:") and not part.startswith("ip6:")} return expected == current - elif r["type"] == "MX": + elif r["type"] == "MX": # For MX, we want to ignore the priority expected = r["value"].split()[-1] current = r["current"].split()[-1] @@ -143,14 +144,11 @@ class DNSRecordsDiagnoser(Diagnoser): else: return r["current"] == r["value"] - def check_expiration_date(self, domains): """ Alert if expiration date of a domain is soon """ - # FIXME find a way to ignore a specific domain without - # create a report by domain each time. We need something small details = { "not_found": [], "error": [], @@ -161,9 +159,9 @@ class DNSRecordsDiagnoser(Diagnoser): for domain in domains: expire_date = self.get_domain_expiration(domain) - if not expire_date: + if isinstance(expire_date, str): details["not_found"].append(( - "diagnosis_domain_expiration_not_found", + "diagnosis_%s_details" % (expire_date), {"domain": domain})) continue @@ -184,9 +182,17 @@ class DNSRecordsDiagnoser(Diagnoser): for alert_type in ["error", "warning", "not_found", "info"]: if details[alert_type]: - yield dict(meta={"category": "expiration"}, + if alert_type == "not_found": + meta = {"test": "domain_not_found"} + else: + meta = {"test": "domain_expiration"} + # Allow to ignor specifically a single domain + if len(details[alert_type]) == 1: + meta["domain"] = details[alert_type][0][1]["domain"] + meta["domain"] = details[alert_type][0][1]["domain"] + yield dict(meta=meta, data={}, - status=alert_type.upper() if alert_type != "not_found" else "INFO", + status=alert_type.upper() if alert_type != "not_found" else "WARNING", summary="diagnosis_domain_expiration_" + alert_type, details=details[alert_type]) @@ -194,19 +200,20 @@ class DNSRecordsDiagnoser(Diagnoser): """ Return the expiration datetime of a domain or None """ + # "echo failed" avoid to trigger CalledProcessError + command = "whois -H %s || echo failed" % (domain) + out = check_output(command).strip().split("\n") - p1 = Popen(['whois', domain], stdout=PIPE) - p2 = Popen(['grep', 'Expir'], stdin=p1.stdout, stdout=PIPE) - out, err = p2.communicate() - out = out.decode("utf-8").split('\n') - p1.terminate() - #p2.terminate() + # If there is less 5 lines, it's NOT FOUND response + if len(out) <= 4: + return "domain_not_found" for line in out: - match = re.search(r'\d{4}-\d{2}-\d{2}', line) + match = re.search(r'Expir.+(\d{4}-\d{2}-\d{2})', line) if match is not None: - return datetime.strptime(match.group(), '%Y-%m-%d') - return None + return datetime.strptime(match.group(1), '%Y-%m-%d') + return "domain_expiration_not_found" + def main(args, env, loggers): return DNSRecordsDiagnoser(args, env, loggers).diagnose() diff --git a/locales/en.json b/locales/en.json index 7f6d9ce34..a0e7454f1 100644 --- a/locales/en.json +++ b/locales/en.json @@ -173,6 +173,8 @@ "diagnosis_dns_discrepancy": "The following DNS record does not seem to follow the recommended configuration:
Type: {type}
Name: {name}
Current value: {current}
Excepted value: {value}", "diagnosis_dns_point_to_doc": "Please check the documentation at https://yunohost.org/dns_config if you need help about configuring DNS records.", "diagnosis_domain_expiration_not_found": "Unable to check the expiration date of some domains", + "diagnosis_domain_not_found_details": "The domain {domain} doesn't exist in WHOIS database !", + "diagnosis_domain_expiration_not_found_details": "The WHOIS returns some info about the domain {domain} but we are not able to found the expiration date inside those info.", "diagnosis_domain_expiration_info": "Domains expiration dates", "diagnosis_domain_expiration_warning": "Some domains expire in less than a month", "diagnosis_domain_expiration_error": "Some domains expire in less than a week", diff --git a/tests/test_i18n_keys.py b/tests/test_i18n_keys.py index 6dcfd5c70..6ddfa9c4a 100644 --- a/tests/test_i18n_keys.py +++ b/tests/test_i18n_keys.py @@ -119,8 +119,9 @@ def find_expected_string_keys(): for level in ["danger", "thirdparty", "warning"]: yield "confirm_app_install_%s" % level - for errortype in ["not_found", "error", "warning", "info"]: + for errortype in ["not_found", "error", "warning", "info", "not_found_details"]: yield "diagnosis_domain_expiration_%s" % errortype + yield "diagnosis_domain_not_found_details" for errortype in ["bad_status_code", "connection_error", "timeout"]: yield "diagnosis_http_%s" % errortype From c347e368fc8913e2042ae62589a6775746c6e3e9 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 27 Apr 2020 18:22:22 +0200 Subject: [PATCH 06/15] [fix] Remove this damn test --- data/hooks/diagnosis/12-dnsrecords.py | 1 - 1 file changed, 1 deletion(-) diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index f03e92df3..003c19cfb 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -40,7 +40,6 @@ class DNSRecordsDiagnoser(Diagnoser): # Check if a domain buy by the user will expire soon domains_from_registrar = ['.'.join(domain.split('.')[-2:]) for domain in all_domains] - domains_from_registrar = ['ynh.local', 'grimaud.me', 'netlib.re', 'arn-fai.net'] domains_from_registrar = set(domains_from_registrar) - set(SMALL_SUFFIX_LIST) for report in self.check_expiration_date(domains_from_registrar): yield report From d1b694447a15e799cd514eb106623e33e86db87b Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 27 Apr 2020 23:37:45 +0200 Subject: [PATCH 07/15] [enh] Use publicsuffix list to avoid alert on dyndns domain --- data/hooks/diagnosis/12-dnsrecords.py | 40 ++++++++++++++++++--------- debian/control | 2 +- locales/en.json | 2 +- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 003c19cfb..c92c2648e 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -4,15 +4,16 @@ import os import re from datetime import datetime, timedelta -from subprocess import Popen, PIPE +from publicsuffix import PublicSuffixList from moulinette.utils.filesystem import read_file from yunohost.utils.network import dig from yunohost.diagnosis import Diagnoser from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain +from yunohost.utils.network import dig -SMALL_SUFFIX_LIST = ['noho.st', 'nohost.me', 'ynh.fr', 'netlib.re'] +PENDING_SUFFIX_LIST = ['ynh.fr', 'netlib.re'] class DNSRecordsDiagnoser(Diagnoser): @@ -39,8 +40,11 @@ class DNSRecordsDiagnoser(Diagnoser): yield report # Check if a domain buy by the user will expire soon - domains_from_registrar = ['.'.join(domain.split('.')[-2:]) for domain in all_domains] - domains_from_registrar = set(domains_from_registrar) - set(SMALL_SUFFIX_LIST) + psl = PublicSuffixList() + all_domains = ["grimaud.me", "reflexlibre.net", "netlib.re", "noho.st", "nohost.me", "ynh.fr", "test.noho.st", "hub.netlib.re", "sans-nuage.fr", "yunohost.org", "yunohost.local", "free.fr"] + domains_from_registrar = [psl.get_public_suffix(domain) for domain in all_domains] + domains_from_registrar = [domain for domain in domains_from_registrar if "." in domain] + domains_from_registrar = set(domains_from_registrar) - set(PENDING_SUFFIX_LIST) for report in self.check_expiration_date(domains_from_registrar): yield report @@ -159,9 +163,12 @@ class DNSRecordsDiagnoser(Diagnoser): expire_date = self.get_domain_expiration(domain) if isinstance(expire_date, str): - details["not_found"].append(( - "diagnosis_%s_details" % (expire_date), - {"domain": domain})) + status_ns, _ = dig(domain, "NS", resolvers="force_external") + status_a, _ = dig(domain, "A", resolvers="force_external") + if "ok" not in [status_ns, status_a]: + details["not_found"].append(( + "diagnosis_domain_%s_details" % (expire_date), + {"domain": domain})) continue expire_in = expire_date - datetime.now() @@ -199,19 +206,26 @@ class DNSRecordsDiagnoser(Diagnoser): """ Return the expiration datetime of a domain or None """ - # "echo failed" avoid to trigger CalledProcessError - command = "whois -H %s || echo failed" % (domain) + command = "whois -H %s" % (domain) + + # Reduce output to determine if whois answer is equivalent to NOT FOUND out = check_output(command).strip().split("\n") + filtered_out = [line for line in out + if re.search(r'^\w{4,25}:', line, re.IGNORECASE) and + not re.match(r'>>> Last update of whois', line, re.IGNORECASE) and + not re.match(r'^NOTICE:', line, re.IGNORECASE) and + not re.match(r'^%%', line, re.IGNORECASE) and + not re.match(r'"https?:"', line, re.IGNORECASE)] # If there is less 5 lines, it's NOT FOUND response - if len(out) <= 4: - return "domain_not_found" + if len(filtered_out) <= 6: + return "not_found" for line in out: - match = re.search(r'Expir.+(\d{4}-\d{2}-\d{2})', line) + match = re.search(r'Expir.+(\d{4}-\d{2}-\d{2})', line, re.IGNORECASE) if match is not None: return datetime.strptime(match.group(1), '%Y-%m-%d') - return "domain_expiration_not_found" + return "expiration_not_found" def main(args, env, loggers): diff --git a/debian/control b/debian/control index fcffa87f6..5061ad4f2 100644 --- a/debian/control +++ b/debian/control @@ -29,7 +29,7 @@ Depends: ${python:Depends}, ${misc:Depends} , redis-server , metronome , git, curl, wget, cron, unzip, jq - , lsb-release, haveged, fake-hwclock, equivs, lsof, whois + , lsb-release, haveged, fake-hwclock, equivs, lsof, whois, python-publicsuffix Recommends: yunohost-admin , ntp, inetutils-ping | iputils-ping , bash-completion, rsyslog diff --git a/locales/en.json b/locales/en.json index a0e7454f1..3f957c702 100644 --- a/locales/en.json +++ b/locales/en.json @@ -173,7 +173,7 @@ "diagnosis_dns_discrepancy": "The following DNS record does not seem to follow the recommended configuration:
Type: {type}
Name: {name}
Current value: {current}
Excepted value: {value}", "diagnosis_dns_point_to_doc": "Please check the documentation at https://yunohost.org/dns_config if you need help about configuring DNS records.", "diagnosis_domain_expiration_not_found": "Unable to check the expiration date of some domains", - "diagnosis_domain_not_found_details": "The domain {domain} doesn't exist in WHOIS database !", + "diagnosis_domain_not_found_details": "The domain {domain} doesn't exist in WHOIS database or is expired !", "diagnosis_domain_expiration_not_found_details": "The WHOIS returns some info about the domain {domain} but we are not able to found the expiration date inside those info.", "diagnosis_domain_expiration_info": "Domains expiration dates", "diagnosis_domain_expiration_warning": "Some domains expire in less than a month", From 575aa674015d3a5c343a102d31d807fb4f04a5dd Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 28 Apr 2020 00:30:38 +0200 Subject: [PATCH 08/15] [fix] whois on co.uk --- data/hooks/diagnosis/12-dnsrecords.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index c92c2648e..d47e33d33 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -7,11 +7,11 @@ from datetime import datetime, timedelta from publicsuffix import PublicSuffixList from moulinette.utils.filesystem import read_file +from moulinette.utils.process import check_output from yunohost.utils.network import dig from yunohost.diagnosis import Diagnoser from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain -from yunohost.utils.network import dig PENDING_SUFFIX_LIST = ['ynh.fr', 'netlib.re'] @@ -41,7 +41,6 @@ class DNSRecordsDiagnoser(Diagnoser): # Check if a domain buy by the user will expire soon psl = PublicSuffixList() - all_domains = ["grimaud.me", "reflexlibre.net", "netlib.re", "noho.st", "nohost.me", "ynh.fr", "test.noho.st", "hub.netlib.re", "sans-nuage.fr", "yunohost.org", "yunohost.local", "free.fr"] domains_from_registrar = [psl.get_public_suffix(domain) for domain in all_domains] domains_from_registrar = [domain for domain in domains_from_registrar if "." in domain] domains_from_registrar = set(domains_from_registrar) - set(PENDING_SUFFIX_LIST) @@ -211,7 +210,7 @@ class DNSRecordsDiagnoser(Diagnoser): # Reduce output to determine if whois answer is equivalent to NOT FOUND out = check_output(command).strip().split("\n") filtered_out = [line for line in out - if re.search(r'^\w{4,25}:', line, re.IGNORECASE) and + if re.search(r'^[a-zA-Z0-9 ]{4,25}:', line, re.IGNORECASE) and not re.match(r'>>> Last update of whois', line, re.IGNORECASE) and not re.match(r'^NOTICE:', line, re.IGNORECASE) and not re.match(r'^%%', line, re.IGNORECASE) and @@ -225,6 +224,11 @@ class DNSRecordsDiagnoser(Diagnoser): match = re.search(r'Expir.+(\d{4}-\d{2}-\d{2})', line, re.IGNORECASE) if match is not None: return datetime.strptime(match.group(1), '%Y-%m-%d') + + match = re.search(r'Expir.+(\d{2}-\w{3}-\d{4})', line, re.IGNORECASE) + if match is not None: + return datetime.strptime(match.group(1), '%d-%b-%Y') + return "expiration_not_found" From b241c2fa1d552c719e83181940893092bef53316 Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 28 Apr 2020 00:53:23 +0200 Subject: [PATCH 09/15] [enh] Whois not working --- data/hooks/diagnosis/12-dnsrecords.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index d47e33d33..7ff791823 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -3,6 +3,7 @@ import os import re +from subprocess import CalledProcessError from datetime import datetime, timedelta from publicsuffix import PublicSuffixList @@ -161,13 +162,15 @@ class DNSRecordsDiagnoser(Diagnoser): for domain in domains: expire_date = self.get_domain_expiration(domain) - if isinstance(expire_date, str): + if isinstance(expire_date, str) and expire_date != "not_working": status_ns, _ = dig(domain, "NS", resolvers="force_external") status_a, _ = dig(domain, "A", resolvers="force_external") if "ok" not in [status_ns, status_a]: details["not_found"].append(( "diagnosis_domain_%s_details" % (expire_date), {"domain": domain})) + else: + self.logger_debug("Dyndns domain: %s" % (domain)) continue expire_in = expire_date - datetime.now() @@ -207,8 +210,13 @@ class DNSRecordsDiagnoser(Diagnoser): """ command = "whois -H %s" % (domain) + try: + out = check_output(command).strip().split("\n") + except CalledProcessError as e: + self.logger_warning("Unable to get whois data for %s . Could be due to a rate limit on whois. Error: %s" % (domain, str(e))) + return "not_working" + # Reduce output to determine if whois answer is equivalent to NOT FOUND - out = check_output(command).strip().split("\n") filtered_out = [line for line in out if re.search(r'^[a-zA-Z0-9 ]{4,25}:', line, re.IGNORECASE) and not re.match(r'>>> Last update of whois', line, re.IGNORECASE) and @@ -216,7 +224,7 @@ class DNSRecordsDiagnoser(Diagnoser): not re.match(r'^%%', line, re.IGNORECASE) and not re.match(r'"https?:"', line, re.IGNORECASE)] - # If there is less 5 lines, it's NOT FOUND response + # If there is less than 7 lines, it's NOT FOUND response if len(filtered_out) <= 6: return "not_found" From 2c7a059f1903aeb5cf623af2c83883bc9c12c010 Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Tue, 28 Apr 2020 17:01:11 +0200 Subject: [PATCH 10/15] [enh] Add a small comments to explain the pending suffix list --- data/hooks/diagnosis/12-dnsrecords.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 7ff791823..8d59538dc 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -14,6 +14,8 @@ from yunohost.utils.network import dig from yunohost.diagnosis import Diagnoser from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain +# We put here domains we know has dyndns provider, but that are not yet +# registered in the public suffix list PENDING_SUFFIX_LIST = ['ynh.fr', 'netlib.re'] From 4e84b6368851e0bb03632378a05abec78ca56792 Mon Sep 17 00:00:00 2001 From: ljf Date: Wed, 29 Apr 2020 00:23:33 +0200 Subject: [PATCH 11/15] [fix] Who is the creator of whois ? #consistency --- data/hooks/diagnosis/12-dnsrecords.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 7ff791823..90dc2e04b 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -162,7 +162,7 @@ class DNSRecordsDiagnoser(Diagnoser): for domain in domains: expire_date = self.get_domain_expiration(domain) - if isinstance(expire_date, str) and expire_date != "not_working": + if isinstance(expire_date, str): status_ns, _ = dig(domain, "NS", resolvers="force_external") status_a, _ = dig(domain, "A", resolvers="force_external") if "ok" not in [status_ns, status_a]: @@ -208,13 +208,8 @@ class DNSRecordsDiagnoser(Diagnoser): """ Return the expiration datetime of a domain or None """ - command = "whois -H %s" % (domain) - - try: - out = check_output(command).strip().split("\n") - except CalledProcessError as e: - self.logger_warning("Unable to get whois data for %s . Could be due to a rate limit on whois. Error: %s" % (domain, str(e))) - return "not_working" + command = "whois -H %s || echo failed" % (domain) + out = check_output(command).strip().split("\n") # Reduce output to determine if whois answer is equivalent to NOT FOUND filtered_out = [line for line in out From 76de0bb2e9bfd8b0a26ba0123eb6ce754fac0439 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 29 Apr 2020 02:42:23 +0200 Subject: [PATCH 12/15] Remove stale code --- data/hooks/diagnosis/12-dnsrecords.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 9db6f88bc..ef720928b 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -3,18 +3,16 @@ import os import re -from subprocess import CalledProcessError from datetime import datetime, timedelta from publicsuffix import PublicSuffixList -from moulinette.utils.filesystem import read_file from moulinette.utils.process import check_output from yunohost.utils.network import dig from yunohost.diagnosis import Diagnoser from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain -# We put here domains we know has dyndns provider, but that are not yet +# We put here domains we know has dyndns provider, but that are not yet # registered in the public suffix list PENDING_SUFFIX_LIST = ['ynh.fr', 'netlib.re'] @@ -27,12 +25,6 @@ class DNSRecordsDiagnoser(Diagnoser): def run(self): - resolvers = read_file("/etc/resolv.dnsmasq.conf").split("\n") - ipv4_resolvers = [r.split(" ")[1] for r in resolvers if r.startswith("nameserver") and ":" not in r] - # FIXME some day ... handle ipv4-only and ipv6-only servers. For now we assume we have at least ipv4 - assert ipv4_resolvers != [], "Uhoh, need at least one IPv4 DNS resolver ..." - - self.resolver = ipv4_resolvers[0] main_domain = _get_maindomain() all_domains = domain_list()["domains"] From f22ac67468ea35a29c42328e16b4298610370d6a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 29 Apr 2020 02:43:37 +0200 Subject: [PATCH 13/15] Success for domains not about to expire --- data/hooks/diagnosis/12-dnsrecords.py | 9 ++++----- tests/test_i18n_keys.py | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index ef720928b..60efcdf9f 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -150,7 +150,7 @@ class DNSRecordsDiagnoser(Diagnoser): "not_found": [], "error": [], "warning": [], - "info": [] + "success": [] } for domain in domains: @@ -169,7 +169,7 @@ class DNSRecordsDiagnoser(Diagnoser): expire_in = expire_date - datetime.now() - alert_type = "info" + alert_type = "success" if expire_in <= timedelta(7): alert_type = "error" elif expire_in <= timedelta(30): @@ -182,16 +182,15 @@ class DNSRecordsDiagnoser(Diagnoser): } details[alert_type].append(("diagnosis_domain_expires_in", args)) - for alert_type in ["error", "warning", "not_found", "info"]: + for alert_type in ["success", "error", "warning", "not_found"]: if details[alert_type]: if alert_type == "not_found": meta = {"test": "domain_not_found"} else: meta = {"test": "domain_expiration"} - # Allow to ignor specifically a single domain + # Allow to ignore specifically a single domain if len(details[alert_type]) == 1: meta["domain"] = details[alert_type][0][1]["domain"] - meta["domain"] = details[alert_type][0][1]["domain"] yield dict(meta=meta, data={}, status=alert_type.upper() if alert_type != "not_found" else "WARNING", diff --git a/tests/test_i18n_keys.py b/tests/test_i18n_keys.py index 6ddfa9c4a..874794e11 100644 --- a/tests/test_i18n_keys.py +++ b/tests/test_i18n_keys.py @@ -119,7 +119,7 @@ def find_expected_string_keys(): for level in ["danger", "thirdparty", "warning"]: yield "confirm_app_install_%s" % level - for errortype in ["not_found", "error", "warning", "info", "not_found_details"]: + for errortype in ["not_found", "error", "warning", "success", "not_found_details"]: yield "diagnosis_domain_expiration_%s" % errortype yield "diagnosis_domain_not_found_details" From b528336fd05f0490a0409940afc186d2ddc68c4a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 29 Apr 2020 02:43:58 +0200 Subject: [PATCH 14/15] Update / improve strings --- locales/en.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/locales/en.json b/locales/en.json index 3f957c702..da37f144c 100644 --- a/locales/en.json +++ b/locales/en.json @@ -172,12 +172,12 @@ "diagnosis_dns_missing_record": "According to the recommended DNS configuration, you should add a DNS record with the following info.
Type: {type}
Name: {name}
Value: {value}", "diagnosis_dns_discrepancy": "The following DNS record does not seem to follow the recommended configuration:
Type: {type}
Name: {name}
Current value: {current}
Excepted value: {value}", "diagnosis_dns_point_to_doc": "Please check the documentation at https://yunohost.org/dns_config if you need help about configuring DNS records.", - "diagnosis_domain_expiration_not_found": "Unable to check the expiration date of some domains", - "diagnosis_domain_not_found_details": "The domain {domain} doesn't exist in WHOIS database or is expired !", - "diagnosis_domain_expiration_not_found_details": "The WHOIS returns some info about the domain {domain} but we are not able to found the expiration date inside those info.", - "diagnosis_domain_expiration_info": "Domains expiration dates", - "diagnosis_domain_expiration_warning": "Some domains expire in less than a month", - "diagnosis_domain_expiration_error": "Some domains expire in less than a week", + "diagnosis_domain_expiration_not_found": "Unable to check the expiration date for some domains", + "diagnosis_domain_not_found_details": "The domain {domain} doesn't exist in WHOIS database or is expired!", + "diagnosis_domain_expiration_not_found_details": "The WHOIS information for domain {domain} doesn't seem to contain the information about the expiration date?", + "diagnosis_domain_expiration_success": "Your domains are registered and not going to expire anytime soon.", + "diagnosis_domain_expiration_warning": "Some domains will expire soon!", + "diagnosis_domain_expiration_error": "Some domains will expire VERY SOON!", "diagnosis_domain_expires_in": "{domain} expires in {days} days.", "diagnosis_services_running": "Service {service} is running!", "diagnosis_services_conf_broken": "Configuration is broken for service {service}!", From 415e805f74cc312ca9752c6075f858a64fe00831 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 29 Apr 2020 02:44:39 +0200 Subject: [PATCH 15/15] Change threshold to warn earlier about soon-to-expire domain --- data/hooks/diagnosis/12-dnsrecords.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 60efcdf9f..560127bb0 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -170,9 +170,9 @@ class DNSRecordsDiagnoser(Diagnoser): expire_in = expire_date - datetime.now() alert_type = "success" - if expire_in <= timedelta(7): + if expire_in <= timedelta(15): alert_type = "error" - elif expire_in <= timedelta(30): + elif expire_in <= timedelta(45): alert_type = "warning" args = {