From d8feb1b72ae605100e8656f39e874209fa43172f Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 7 Apr 2020 01:53:05 +0200 Subject: [PATCH 1/9] [enh] Add RBL check --- data/hooks/diagnosis/24-mail.py | 89 ++++++++++++++++++++++++++++++++- locales/en.json | 3 ++ 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/data/hooks/diagnosis/24-mail.py b/data/hooks/diagnosis/24-mail.py index 3f9517bb0..731267593 100644 --- a/data/hooks/diagnosis/24-mail.py +++ b/data/hooks/diagnosis/24-mail.py @@ -1,9 +1,42 @@ #!/usr/bin/env python import os +import dns.resolver + +from moulinette.utils.network import download_text from yunohost.diagnosis import Diagnoser +DEFAULT_BLACKLIST = [ + ('zen.spamhaus.org' , 'Spamhaus SBL, XBL and PBL' ), + ('dnsbl.sorbs.net' , 'SORBS aggregated' ), + ('safe.dnsbl.sorbs.net' , "'safe' subset of SORBS aggregated"), + ('ix.dnsbl.manitu.net' , 'Heise iX NiX Spam' ), + ('babl.rbl.webiron.net' , 'Bad Abuse' ), + ('cabl.rbl.webiron.net' , 'Chronicly Bad Abuse' ), + ('truncate.gbudb.net' , 'Exclusively Spam/Malware' ), + ('dnsbl-1.uceprotect.net' , 'Trapserver Cluster' ), + ('cbl.abuseat.org' , 'Net of traps' ), + ('dnsbl.cobion.com' , 'used in IBM products' ), + ('psbl.surriel.com' , 'passive list, easy to unlist' ), + ('dnsrbl.org' , 'Real-time black list' ), + ('db.wpbl.info' , 'Weighted private' ), + ('bl.spamcop.net' , 'Based on spamcop users' ), + ('dyna.spamrats.com' , 'Dynamic IP addresses' ), + ('spam.spamrats.com' , 'Manual submissions' ), + ('auth.spamrats.com' , 'Suspicious authentications' ), + ('dnsbl.inps.de' , 'automated and reported' ), + ('bl.blocklist.de' , 'fail2ban reports etc.' ), + ('srnblack.surgate.net' , 'feeders' ), + ('all.s5h.net' , 'traps' ), + ('rbl.realtimeblacklist.com' , 'lists ip ranges' ), + ('b.barracudacentral.org' , 'traps' ), + ('hostkarma.junkemailfilter.com', 'Autotected Virus Senders' ), + ('rbl.megarbl.net' , 'Curated Spamtraps' ), + ('ubl.unsubscore.com' , 'Collected Opt-Out Addresses' ), + ('0spam.fusionzero.com' , 'Spam Trap' ), +] + class MailDiagnoser(Diagnoser): @@ -14,6 +47,7 @@ class MailDiagnoser(Diagnoser): def run(self): # Is outgoing port 25 filtered somehow ? + self.logger_debug("Running outgoing 25 port check") if os.system('/bin/nc -z -w2 yunohost.org 25') == 0: yield dict(meta={"test": "ougoing_port_25"}, status="SUCCESS", @@ -23,9 +57,22 @@ class MailDiagnoser(Diagnoser): status="ERROR", summary="diagnosis_mail_ougoing_port_25_blocked") + # Is Reverse DNS well configured ? - # Mail blacklist using dig requests (c.f. ljf's code) + # Are IPs blacklisted ? + self.logger_debug("Running RBL detection") + blacklisted_details = tuple(self.check_blacklisted(self.get_public_ip(4))) + blacklisted_details += tuple(self.check_blacklisted(self.get_public_ip(6))) + if blacklisted_details: + yield dict(meta={}, + status="ERROR", + summary=("diagnosis_mail_blacklist_nok", {}), + details=blacklisted_details) + else: + yield dict(meta={}, + status="SUCCESS", + summary=("diagnosis_mail_blacklist_ok", {})) # SMTP reachability (c.f. check-smtp to be implemented on yunohost's remote diagnoser) @@ -37,6 +84,46 @@ class MailDiagnoser(Diagnoser): # check for unusual failed sending attempt being refused in the logs ? + def check_blacklisted(self, ip): + """ Check with dig onto blacklist DNS server + """ + if ip is None: + return + + for blacklist, description in DEFAULT_BLACKLIST: + + # Determine if we are listed on this RBL + try: + rev = dns.reversename.from_address(ip) + query = str(rev.split(3)[0]) + '.' + blacklist + # TODO add timeout lifetime + dns.resolver.query(query, "A") + except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers, dns.resolver.NoAnswer, + dns.exception.Timeout): + continue + + # Try to get the reason + reason = "not explained" + try: + reason = str(dns.resolver.query(query, "TXT")[0]) + except Exception: + pass + + yield ('diagnosis_mail_blacklisted_by', + (ip, blacklist, reason)) + + def get_public_ip(self, protocol=4): + # TODO we might call this function from another side + assert protocol in [4, 6], "Invalid protocol version, it should be either 4 or 6 and was '%s'" % repr(protocol) + + url = 'https://ip%s.yunohost.org' % ('6' if protocol == 6 else '') + + try: + return download_text(url, timeout=30).strip() + except Exception as e: + self.logger_debug("Could not get public IPv%s : %s" % (str(protocol), str(e))) + return None + def main(args, env, loggers): return MailDiagnoser(args, env, loggers).diagnose() diff --git a/locales/en.json b/locales/en.json index 800a1d696..dbce8f367 100644 --- a/locales/en.json +++ b/locales/en.json @@ -186,6 +186,9 @@ "diagnosis_swap_ok": "The system has {total} of swap!", "diagnosis_mail_ougoing_port_25_ok": "Outgoing port 25 is not blocked and email can be sent to other servers.", "diagnosis_mail_ougoing_port_25_blocked": "Outgoing port 25 appears to be blocked. You should try to unblock it in your internet service provider (or hosting provider) configuration panel. Meanwhile, the server won't be able to send emails to other servers.", + "diagnosis_mail_blacklist_ok": "Your server public IP are not listed on email blacklist.", + "diagnosis_mail_blacklist_nok": "Your server public IPs are listed on email blacklist.", + "diagnosis_mail_blacklisted_by": "{0} is listed on {1}. Reason: {2}", "diagnosis_regenconf_allgood": "All configurations files are in line with the recommended configuration!", "diagnosis_regenconf_manually_modified": "Configuration file {file} appears to have been manually modified.", "diagnosis_regenconf_manually_modified_details": "This is probably OK if you know what you're doing! YunoHost will stop updating this file automatically... But beware that YunoHost upgrades could contain important recommended changes. If you want to, you can inspect the differences with yunohost tools regen-conf {category} --dry-run --with-diff and force the reset to the recommended configuration with yunohost tools regen-conf {category} --force", From bb162662c6007d729c6105c7e40352fba8500015 Mon Sep 17 00:00:00 2001 From: ljf Date: Sat, 11 Apr 2020 19:34:34 +0200 Subject: [PATCH 2/9] [enh] Use named var in i18n --- data/hooks/diagnosis/24-mail.py | 20 ++++++++++++-------- locales/en.json | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/data/hooks/diagnosis/24-mail.py b/data/hooks/diagnosis/24-mail.py index 731267593..25d0ff984 100644 --- a/data/hooks/diagnosis/24-mail.py +++ b/data/hooks/diagnosis/24-mail.py @@ -62,17 +62,21 @@ class MailDiagnoser(Diagnoser): # Are IPs blacklisted ? self.logger_debug("Running RBL detection") - blacklisted_details = tuple(self.check_blacklisted(self.get_public_ip(4))) - blacklisted_details += tuple(self.check_blacklisted(self.get_public_ip(6))) + ipv4 = Diagnoser.get_cached_report_item("ip", {"test": "ipv4"}) + global_ipv4 = ipv4.get("data", {}).get("global", {}) + ipv6 = Diagnoser.get_cached_report_item("ip", {"test": "ipv6"}) + global_ipv6 = ipv6.get("data", {}).get("global", {}) + blacklisted_details = tuple(self.check_blacklisted(global_ipv4)) + blacklisted_details += tuple(self.check_blacklisted(global_ipv6)) if blacklisted_details: - yield dict(meta={}, + yield dict(meta={"test": "mail_blacklist"}, status="ERROR", - summary=("diagnosis_mail_blacklist_nok", {}), - details=blacklisted_details) + summary="diagnosis_mail_blacklist_nok", + details=list(blacklisted_details)) else: - yield dict(meta={}, + yield dict(meta={"test": "mail_blacklist"}, status="SUCCESS", - summary=("diagnosis_mail_blacklist_ok", {})) + summary="diagnosis_mail_blacklist_ok") # SMTP reachability (c.f. check-smtp to be implemented on yunohost's remote diagnoser) @@ -110,7 +114,7 @@ class MailDiagnoser(Diagnoser): pass yield ('diagnosis_mail_blacklisted_by', - (ip, blacklist, reason)) + {'ip': ip, 'blacklist': blacklist, 'reason': reason}) def get_public_ip(self, protocol=4): # TODO we might call this function from another side diff --git a/locales/en.json b/locales/en.json index dbce8f367..26c51d253 100644 --- a/locales/en.json +++ b/locales/en.json @@ -188,7 +188,7 @@ "diagnosis_mail_ougoing_port_25_blocked": "Outgoing port 25 appears to be blocked. You should try to unblock it in your internet service provider (or hosting provider) configuration panel. Meanwhile, the server won't be able to send emails to other servers.", "diagnosis_mail_blacklist_ok": "Your server public IP are not listed on email blacklist.", "diagnosis_mail_blacklist_nok": "Your server public IPs are listed on email blacklist.", - "diagnosis_mail_blacklisted_by": "{0} is listed on {1}. Reason: {2}", + "diagnosis_mail_blacklisted_by": "{ip} is listed on {blacklist}. Reason: {reason}", "diagnosis_regenconf_allgood": "All configurations files are in line with the recommended configuration!", "diagnosis_regenconf_manually_modified": "Configuration file {file} appears to have been manually modified.", "diagnosis_regenconf_manually_modified_details": "This is probably OK if you know what you're doing! YunoHost will stop updating this file automatically... But beware that YunoHost upgrades could contain important recommended changes. If you want to, you can inspect the differences with yunohost tools regen-conf {category} --dry-run --with-diff and force the reset to the recommended configuration with yunohost tools regen-conf {category} --force", From 0b7984adf117a413b63d8604d6b54cea22bc3c87 Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 12 Apr 2020 04:14:49 +0200 Subject: [PATCH 3/9] [enh] Improve DNSBL check --- data/hooks/diagnosis/24-mail.py | 120 +++++++++------------ data/other/dnsbl_list.yml | 184 ++++++++++++++++++++++++++++++++ debian/install | 1 + locales/en.json | 6 +- 4 files changed, 237 insertions(+), 74 deletions(-) create mode 100644 data/other/dnsbl_list.yml diff --git a/data/hooks/diagnosis/24-mail.py b/data/hooks/diagnosis/24-mail.py index 25d0ff984..333d98c8a 100644 --- a/data/hooks/diagnosis/24-mail.py +++ b/data/hooks/diagnosis/24-mail.py @@ -4,38 +4,11 @@ import os import dns.resolver from moulinette.utils.network import download_text +from moulinette.utils.filesystem import read_yaml from yunohost.diagnosis import Diagnoser -DEFAULT_BLACKLIST = [ - ('zen.spamhaus.org' , 'Spamhaus SBL, XBL and PBL' ), - ('dnsbl.sorbs.net' , 'SORBS aggregated' ), - ('safe.dnsbl.sorbs.net' , "'safe' subset of SORBS aggregated"), - ('ix.dnsbl.manitu.net' , 'Heise iX NiX Spam' ), - ('babl.rbl.webiron.net' , 'Bad Abuse' ), - ('cabl.rbl.webiron.net' , 'Chronicly Bad Abuse' ), - ('truncate.gbudb.net' , 'Exclusively Spam/Malware' ), - ('dnsbl-1.uceprotect.net' , 'Trapserver Cluster' ), - ('cbl.abuseat.org' , 'Net of traps' ), - ('dnsbl.cobion.com' , 'used in IBM products' ), - ('psbl.surriel.com' , 'passive list, easy to unlist' ), - ('dnsrbl.org' , 'Real-time black list' ), - ('db.wpbl.info' , 'Weighted private' ), - ('bl.spamcop.net' , 'Based on spamcop users' ), - ('dyna.spamrats.com' , 'Dynamic IP addresses' ), - ('spam.spamrats.com' , 'Manual submissions' ), - ('auth.spamrats.com' , 'Suspicious authentications' ), - ('dnsbl.inps.de' , 'automated and reported' ), - ('bl.blocklist.de' , 'fail2ban reports etc.' ), - ('srnblack.surgate.net' , 'feeders' ), - ('all.s5h.net' , 'traps' ), - ('rbl.realtimeblacklist.com' , 'lists ip ranges' ), - ('b.barracudacentral.org' , 'traps' ), - ('hostkarma.junkemailfilter.com', 'Autotected Virus Senders' ), - ('rbl.megarbl.net' , 'Curated Spamtraps' ), - ('ubl.unsubscore.com' , 'Collected Opt-Out Addresses' ), - ('0spam.fusionzero.com' , 'Spam Trap' ), -] +DEFAULT_DNS_BLACKLIST = "/usr/share/yunohost/other/dnsbl_list.yml" class MailDiagnoser(Diagnoser): @@ -57,17 +30,13 @@ class MailDiagnoser(Diagnoser): status="ERROR", summary="diagnosis_mail_ougoing_port_25_blocked") - # Is Reverse DNS well configured ? + # Forward-confirmed reverse DNS (FCrDNS) verification - # Are IPs blacklisted ? - self.logger_debug("Running RBL detection") - ipv4 = Diagnoser.get_cached_report_item("ip", {"test": "ipv4"}) - global_ipv4 = ipv4.get("data", {}).get("global", {}) - ipv6 = Diagnoser.get_cached_report_item("ip", {"test": "ipv6"}) - global_ipv6 = ipv6.get("data", {}).get("global", {}) - blacklisted_details = tuple(self.check_blacklisted(global_ipv4)) - blacklisted_details += tuple(self.check_blacklisted(global_ipv6)) + # Are IPs listed on a DNSBL ? + self.logger_debug("Running DNSBL detection") + + blacklisted_details = self.check_ip_dnsbl() if blacklisted_details: yield dict(meta={"test": "mail_blacklist"}, status="ERROR", @@ -88,45 +57,54 @@ class MailDiagnoser(Diagnoser): # check for unusual failed sending attempt being refused in the logs ? - def check_blacklisted(self, ip): + def check_blacklisted(self): """ Check with dig onto blacklist DNS server """ - if ip is None: - return + dns_blacklists = read_yaml(DEFAULT_DNS_BLACKLIST) + for ip in self.get_public_ips(): + for blacklist in dns_blacklists: + + if "." in ip and not blacklist.ipv4: + continue - for blacklist, description in DEFAULT_BLACKLIST: + if ":" in ip and not blacklist.ipv6: + continue + + # Determine if we are listed on this RBL + try: + rev = dns.reversename.from_address(ip) + query = str(rev.split(3)[0]) + '.' + blacklist.dns_server + # TODO add timeout lifetime + dns.resolver.query(query, "A") + except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers, dns.resolver.NoAnswer, + dns.exception.Timeout): + continue - # Determine if we are listed on this RBL - try: - rev = dns.reversename.from_address(ip) - query = str(rev.split(3)[0]) + '.' + blacklist - # TODO add timeout lifetime - dns.resolver.query(query, "A") - except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers, dns.resolver.NoAnswer, - dns.exception.Timeout): - continue + # Try to get the reason + reason = "not explained" + try: + reason = str(dns.resolver.query(query, "TXT")[0]) + except Exception: + pass - # Try to get the reason - reason = "not explained" - try: - reason = str(dns.resolver.query(query, "TXT")[0]) - except Exception: - pass + yield ('diagnosis_mail_blacklisted_by', { + 'ip': ip, + 'blacklist': blacklist, + 'reason': reason}) - yield ('diagnosis_mail_blacklisted_by', - {'ip': ip, 'blacklist': blacklist, 'reason': reason}) - - def get_public_ip(self, protocol=4): - # TODO we might call this function from another side - assert protocol in [4, 6], "Invalid protocol version, it should be either 4 or 6 and was '%s'" % repr(protocol) - - url = 'https://ip%s.yunohost.org' % ('6' if protocol == 6 else '') - - try: - return download_text(url, timeout=30).strip() - except Exception as e: - self.logger_debug("Could not get public IPv%s : %s" % (str(protocol), str(e))) - return None + def get_public_ips(self): + # Todo code a better way to access a data + ipv4 = Diagnoser.get_cached_report("ip", {"test": "ipv4"}) + if ipv4: + global_ipv4 = ipv4.get("data", {}).get("global", {}) + if global_ipv4: + yield global_ipv4 + + ipv6 = Diagnoser.get_cached_report("ip", {"test": "ipv6"}) + if ipv6: + global_ipv6 = ipv6.get("data", {}).get("global", {}) + if global_ipv6: + yield global_ipv6 def main(args, env, loggers): diff --git a/data/other/dnsbl_list.yml b/data/other/dnsbl_list.yml new file mode 100644 index 000000000..839aeaab6 --- /dev/null +++ b/data/other/dnsbl_list.yml @@ -0,0 +1,184 @@ +# Used by GAFAM +- name: Spamhaus ZEN + dns_server: zen.spamhaus.org + website: https://www.spamhaus.org/zen/ + ipv4: true + ipv6: true + domain: false +- name: Barracuda Reputation Block List + dns_server: b.barracudacentral.org + website: https://barracudacentral.org/rbl/ + ipv4: true + ipv6: false + domain: false +- name: Hostkarma + dns_server: hostkarma.junkemailfilter.com + website: https://ipadmin.junkemailfilter.com/remove.php + ipv4: true + ipv6: false + domain: false +- name: ImproWare IP based spamlist + dns_server: spamrbl.imp.ch + website: https://antispam.imp.ch/ + ipv4: true + ipv6: false + domain: false +- name: ImproWare IP based wormlist + dns_server: wormrbl.imp.ch + website: https://antispam.imp.ch/ + ipv4: true + ipv6: false + domain: false +- name: Backscatterer.org + dns_server: ips.backscatterer.org + website: http://www.backscatterer.org/ + ipv4: true + ipv6: false + domain: false +- name: inps.de + dns_server: dnsbl.inps.de + website: http://dnsbl.inps.de/ + ipv4: true + ipv6: false + domain: false +- name: LASHBACK + dns_server: ubl.unsubscore.com + website: https://blacklist.lashback.com/ + ipv4: true + ipv6: false + domain: false +- name: Mailspike.org + dns_server: bl.mailspike.net + website: http://www.mailspike.net/ + ipv4: true + ipv6: false + domain: false +- name: NiX Spam + dns_server: ix.dnsbl.manitu.net + website: http://www.dnsbl.manitu.net/ + ipv4: true + ipv6: false + domain: false +- name: REDHAWK + dns_server: access.redhawk.org + website: https://www.redhawk.org/SpamHawk/query.php + ipv4: true + ipv6: false + domain: false +- name: SORBS Open SMTP relays + dns_server: smtp.dnsbl.sorbs.net + website: http://www.sorbs.net/ + ipv4: true + ipv6: false + domain: false +- name: SORBS Spamhost (last 28 days) + dns_server: recent.spam.dnsbl.sorbs.net + website: http://www.sorbs.net/ + ipv4: true + ipv6: false + domain: false +- name: SORBS Spamhost (last 48 hours) + dns_server: new.spam.dnsbl.sorbs.net + website: http://www.sorbs.net/ + ipv4: true + ipv6: false + domain: false +- name: SpamCop Blocking List + dns_server: bl.spamcop.net + website: https://www.spamcop.net/bl.shtml + ipv4: true + ipv6: false + domain: false +- name: Spam Eating Monkey SEM-BACKSCATTER + dns_server: backscatter.spameatingmonkey.net + website: https://spameatingmonkey.com/services + ipv4: true + ipv6: false + domain: false +- name: Spam Eating Monkey SEM-BLACK + dns_server: bl.spameatingmonkey.net + website: https://spameatingmonkey.com/services + ipv4: true + ipv6: false + domain: false +- name: Spam Eating Monkey SEM-IPV6BL + dns_server: bl.ipv6.spameatingmonkey.net + website: https://spameatingmonkey.com/services + ipv4: false + ipv6: true + domain: false +- name: SpamRATS! all + dns_server: all.spamrats.com + website: http://www.spamrats.com/ + ipv4: true + ipv6: false + domain: false +- name: PSBL (Passive Spam Block List) + dns_server: psbl.surriel.com + website: http://psbl.surriel.com/ + ipv4: true + ipv6: false + domain: false +- name: SWINOG + dns_server: dnsrbl.swinog.ch + website: https://antispam.imp.ch/ + ipv4: true + ipv6: false + domain: false +- name: GBUdb Truncate + dns_server: truncate.gbudb.net + website: http://www.gbudb.com/truncate/index.jsp + ipv4: true + ipv6: false + domain: false +- name: Weighted Private Block List + dns_server: db.wpbl.info + website: http://www.wpbl.info/ + ipv4: true + ipv6: false + domain: false +# Used by GAFAM +- name: Composite Blocking List + dns_server: cbl.abuseat.org + website: cbl.abuseat.org + ipv4: true + ipv6: false + domain: false +# Used by GAFAM +- name: SenderScore Blacklist + dns_server: bl.score.senderscore.com + website: https://senderscore.com + ipv4: true + ipv6: false + domain: false +- name: Invaluement + dns_server: sip.invaluement.com + website: https://www.invaluement.com/ + ipv4: true + ipv6: false + domain: false +# Added cause it supports IPv6 +- name: AntiCaptcha.NET IPv6 + dns_server: dnsbl6.anticaptcha.net + website: http://anticaptcha.net/ + ipv4: false + ipv6: true + domain: false +- name: SPFBL.net RBL + dns_server: dnsbl.spfbl.net + website: https://spfbl.net/en/dnsbl/ + ipv4: true + ipv6: true + domain: true +- name: Suomispam Blacklist + dns_server: bl.suomispam.net + website: http://suomispam.net/ + ipv4: true + ipv6: true + domain: false +- name: NordSpam + dns_server: bl.nordspam.com + website: https://www.nordspam.com/ + ipv4: true + ipv6: true + domain: false diff --git a/debian/install b/debian/install index e0743cdd1..cf682d958 100644 --- a/debian/install +++ b/debian/install @@ -7,6 +7,7 @@ data/hooks/* /usr/share/yunohost/hooks/ data/other/yunoprompt.service /etc/systemd/system/ data/other/password/* /usr/share/yunohost/other/password/ data/other/dpkg-origins/yunohost /etc/dpkg/origins +data/other/dnsbl_list.yml /usr/share/yunohost/other/dnsbl_list.yml data/other/* /usr/share/yunohost/yunohost-config/moulinette/ data/templates/* /usr/share/yunohost/templates/ data/helpers /usr/share/yunohost/ diff --git a/locales/en.json b/locales/en.json index 26c51d253..37ae2a34f 100644 --- a/locales/en.json +++ b/locales/en.json @@ -186,9 +186,9 @@ "diagnosis_swap_ok": "The system has {total} of swap!", "diagnosis_mail_ougoing_port_25_ok": "Outgoing port 25 is not blocked and email can be sent to other servers.", "diagnosis_mail_ougoing_port_25_blocked": "Outgoing port 25 appears to be blocked. You should try to unblock it in your internet service provider (or hosting provider) configuration panel. Meanwhile, the server won't be able to send emails to other servers.", - "diagnosis_mail_blacklist_ok": "Your server public IP are not listed on email blacklist.", - "diagnosis_mail_blacklist_nok": "Your server public IPs are listed on email blacklist.", - "diagnosis_mail_blacklisted_by": "{ip} is listed on {blacklist}. Reason: {reason}", + "diagnosis_mail_blacklist_ok": "Your server public IP are not listed on email blacklists.", + "diagnosis_mail_blacklist_nok": "Your server public IPs are listed on email blacklists.", + "diagnosis_mail_blacklisted_by": "{ip} is listed on {blacklist.name}. Reason: {reason}. See {blacklist.website}", "diagnosis_regenconf_allgood": "All configurations files are in line with the recommended configuration!", "diagnosis_regenconf_manually_modified": "Configuration file {file} appears to have been manually modified.", "diagnosis_regenconf_manually_modified_details": "This is probably OK if you know what you're doing! YunoHost will stop updating this file automatically... But beware that YunoHost upgrades could contain important recommended changes. If you want to, you can inspect the differences with yunohost tools regen-conf {category} --dry-run --with-diff and force the reset to the recommended configuration with yunohost tools regen-conf {category} --force", From 5b0698e798421c1a3d71147c279b326b4b2726a6 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 13 Apr 2020 16:41:27 +0200 Subject: [PATCH 4/9] [fix] Bad call to dict --- data/hooks/diagnosis/24-mail.py | 19 ++++++++++--------- locales/en.json | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/data/hooks/diagnosis/24-mail.py b/data/hooks/diagnosis/24-mail.py index 333d98c8a..f4f897e28 100644 --- a/data/hooks/diagnosis/24-mail.py +++ b/data/hooks/diagnosis/24-mail.py @@ -36,12 +36,13 @@ class MailDiagnoser(Diagnoser): # Are IPs listed on a DNSBL ? self.logger_debug("Running DNSBL detection") - blacklisted_details = self.check_ip_dnsbl() + blacklisted_details = list(self.check_dnsbl(self.get_public_ips())) + print(blacklisted_details) if blacklisted_details: yield dict(meta={"test": "mail_blacklist"}, status="ERROR", summary="diagnosis_mail_blacklist_nok", - details=list(blacklisted_details)) + details=blacklisted_details) else: yield dict(meta={"test": "mail_blacklist"}, status="SUCCESS", @@ -57,23 +58,22 @@ class MailDiagnoser(Diagnoser): # check for unusual failed sending attempt being refused in the logs ? - def check_blacklisted(self): + def check_dnsbl(self, ips): """ Check with dig onto blacklist DNS server """ dns_blacklists = read_yaml(DEFAULT_DNS_BLACKLIST) - for ip in self.get_public_ips(): + for ip in ips: for blacklist in dns_blacklists: - - if "." in ip and not blacklist.ipv4: + if "." in ip and not blacklist['ipv4']: continue - if ":" in ip and not blacklist.ipv6: + if ":" in ip and not blacklist['ipv6']: continue # Determine if we are listed on this RBL try: rev = dns.reversename.from_address(ip) - query = str(rev.split(3)[0]) + '.' + blacklist.dns_server + query = str(rev.split(3)[0]) + '.' + blacklist['dns_server'] # TODO add timeout lifetime dns.resolver.query(query, "A") except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers, dns.resolver.NoAnswer, @@ -89,7 +89,8 @@ class MailDiagnoser(Diagnoser): yield ('diagnosis_mail_blacklisted_by', { 'ip': ip, - 'blacklist': blacklist, + 'blacklist_name': blacklist['name'], + 'blacklist_website': blacklist['website'], 'reason': reason}) def get_public_ips(self): diff --git a/locales/en.json b/locales/en.json index 37ae2a34f..93f7680bf 100644 --- a/locales/en.json +++ b/locales/en.json @@ -188,7 +188,7 @@ "diagnosis_mail_ougoing_port_25_blocked": "Outgoing port 25 appears to be blocked. You should try to unblock it in your internet service provider (or hosting provider) configuration panel. Meanwhile, the server won't be able to send emails to other servers.", "diagnosis_mail_blacklist_ok": "Your server public IP are not listed on email blacklists.", "diagnosis_mail_blacklist_nok": "Your server public IPs are listed on email blacklists.", - "diagnosis_mail_blacklisted_by": "{ip} is listed on {blacklist.name}. Reason: {reason}. See {blacklist.website}", + "diagnosis_mail_blacklisted_by": "{ip} is listed on {blacklist_name}. Reason: {reason}. See {blacklist_website}", "diagnosis_regenconf_allgood": "All configurations files are in line with the recommended configuration!", "diagnosis_regenconf_manually_modified": "Configuration file {file} appears to have been manually modified.", "diagnosis_regenconf_manually_modified_details": "This is probably OK if you know what you're doing! YunoHost will stop updating this file automatically... But beware that YunoHost upgrades could contain important recommended changes. If you want to, you can inspect the differences with yunohost tools regen-conf {category} --dry-run --with-diff and force the reset to the recommended configuration with yunohost tools regen-conf {category} --force", From 027a0ed73c9281fd35582d9e683348483f12f7bd Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 14 Apr 2020 03:56:35 +0200 Subject: [PATCH 5/9] [wip] Add rDNS and mailqueue check --- data/hooks/diagnosis/24-mail.py | 78 ++++++++++++++++++++++++++++++--- locales/en.json | 12 ++++- 2 files changed, 82 insertions(+), 8 deletions(-) diff --git a/data/hooks/diagnosis/24-mail.py b/data/hooks/diagnosis/24-mail.py index f4f897e28..b91bfec85 100644 --- a/data/hooks/diagnosis/24-mail.py +++ b/data/hooks/diagnosis/24-mail.py @@ -2,11 +2,15 @@ import os import dns.resolver +import smtplib +import socket +from moulinette.utils.process import check_output from moulinette.utils.network import download_text from moulinette.utils.filesystem import read_yaml from yunohost.diagnosis import Diagnoser +from yunohost.domain import _get_maindomain DEFAULT_DNS_BLACKLIST = "/usr/share/yunohost/other/dnsbl_list.yml" @@ -18,6 +22,8 @@ class MailDiagnoser(Diagnoser): dependencies = ["ip"] def run(self): + + ips = self.get_public_ips() # Is outgoing port 25 filtered somehow ? self.logger_debug("Running outgoing 25 port check") @@ -30,14 +36,56 @@ class MailDiagnoser(Diagnoser): status="ERROR", summary="diagnosis_mail_ougoing_port_25_blocked") - # Forward-confirmed reverse DNS (FCrDNS) verification + # Get HELO and be sure postfix is running + # TODO SMTP reachability (c.f. check-smtp to be implemented on yunohost's remote diagnoser) + server = None + result = dict(meta={"test": "mail_ehlo"}, + status="SUCCESS", + summary="diagnosis_mail_service_working") + try: + server = smtplib.SMTP("127.0.0.1", 25, timeout=10) + ehlo = server.ehlo() + ehlo_domain = ehlo[1].decode("utf-8").split("\n")[0] + except OSError: + result = dict(meta={"test": "mail_ehlo"}, + status="ERROR", + summary="diagnosis_mail_service_not_working") + ehlo_domain = _get_maindomain() + if server: + server.quit() + yield result + # Forward-confirmed reverse DNS (FCrDNS) verification + self.logger_debug("Running Forward-confirmed reverse DNS check") + for ip in ips: + try: + rdns_domain, _, _ = socket.gethostbyaddr(ip) + except socket.herror as e: + yield dict(meta={"test": "mail_fcrdns"}, + data={"ip": ip, "ehlo_domain": ehlo_domain}, + status="ERROR", + summary="diagnosis_mail_reverse_dns_missing") + continue + else: + if rdns_domain != ehlo_domain: + yield dict(meta={"test": "mail_fcrdns"}, + data={"ip": ip, "ehlo_domain": ehlo_domain, + "rdns_domain": rdns_domain}, + status="ERROR", + summary="diagnosis_mail_rdns_different_from_ehlo_domain") + else: + yield dict(meta={"test": "mail_fcrdns"}, + data={"ip": ip, "ehlo_domain": ehlo_domain}, + status="SUCCESS", + summary="diagnosis_mail_rdns_equal_to_ehlo_domain") + + # TODO Is a A/AAAA and MX Record ? # Are IPs listed on a DNSBL ? - self.logger_debug("Running DNSBL detection") + self.logger_debug("Running DNS Blacklist detection") + # TODO Test if domain are blacklisted too blacklisted_details = list(self.check_dnsbl(self.get_public_ips())) - print(blacklisted_details) if blacklisted_details: yield dict(meta={"test": "mail_blacklist"}, status="ERROR", @@ -48,11 +96,29 @@ class MailDiagnoser(Diagnoser): status="SUCCESS", summary="diagnosis_mail_blacklist_ok") - # SMTP reachability (c.f. check-smtp to be implemented on yunohost's remote diagnoser) + # TODO Are outgoing public IPs authorized to send mail by SPF ? + + # TODO Validate DKIM and dmarc ? - # ideally, SPF / DMARC / DKIM validation ... (c.f. https://github.com/alexAubin/yunoScripts/blob/master/yunoDKIM.py possibly though that looks horrible) - # check that the mail queue is not filled with hundreds of email pending + # Is mail queue filled with hundreds of email pending ? + command = 'postqueue -p | grep -c "^[A-Z0-9]"' + output = check_output(command).strip() + try: + pending_emails = int(output) + except ValueError: + yield dict(meta={"test": "mail_queue"}, + status="ERROR", + summary="diagnosis_mail_cannot_get_queue") + else: + if pending_emails > 300: + yield dict(meta={"test": "mail_queue"}, + status="WARNING", + summary="diagnosis_mail_queue_too_many_pending_emails") + else: + yield dict(meta={"test": "mail_queue"}, + status="INFO", + summary="diagnosis_mail_queue_ok") # check that the recent mail logs are not filled with thousand of email sending (unusual number of mail sent) diff --git a/locales/en.json b/locales/en.json index 93f7680bf..978ceb831 100644 --- a/locales/en.json +++ b/locales/en.json @@ -186,9 +186,17 @@ "diagnosis_swap_ok": "The system has {total} of swap!", "diagnosis_mail_ougoing_port_25_ok": "Outgoing port 25 is not blocked and email can be sent to other servers.", "diagnosis_mail_ougoing_port_25_blocked": "Outgoing port 25 appears to be blocked. You should try to unblock it in your internet service provider (or hosting provider) configuration panel. Meanwhile, the server won't be able to send emails to other servers.", - "diagnosis_mail_blacklist_ok": "Your server public IP are not listed on email blacklists.", - "diagnosis_mail_blacklist_nok": "Your server public IPs are listed on email blacklists.", + "diagnosis_mail_blacklist_ok": "The public IPs of this instance are not listed on email blacklists.", + "diagnosis_mail_blacklist_nok": "Some of the public IPs of this instance are listed on email blacklists.", "diagnosis_mail_blacklisted_by": "{ip} is listed on {blacklist_name}. Reason: {reason}. See {blacklist_website}", + "diagnosis_mail_service_working": "Postfix mail service answer correctly.", + "diagnosis_mail_service_not_working": "Postfix mail service don't answer to EHLO request.", + "diagnosis_mail_reverse_dns_missing": "No reverse DNS defined for the ip {ip}.", + "diagnosis_mail_rdns_different_from_ehlo_domain": "Your reverse DNS {rdns_domain} is different from your EHLO domain {ehlo_domain} on {ip}.", + "diagnosis_mail_rdns_equal_to_ehlo_domain": "Your reverse DNS is equal to your EHLO domain {ehlo_domain} on {ip}.", + "diagnosis_mail_queue_unavailable": "Can not consult number of pending emails in queue", + "diagnosis_mail_queue_too_big": "The mail queue has {nb_pending} pending emails in the mail queue. It seems abnormal.", + "diagnosis_mail_queue_unavailable": "The mail queue has {nb_pending} pending emails in the mail queue.", "diagnosis_regenconf_allgood": "All configurations files are in line with the recommended configuration!", "diagnosis_regenconf_manually_modified": "Configuration file {file} appears to have been manually modified.", "diagnosis_regenconf_manually_modified_details": "This is probably OK if you know what you're doing! YunoHost will stop updating this file automatically... But beware that YunoHost upgrades could contain important recommended changes. If you want to, you can inspect the differences with yunohost tools regen-conf {category} --dry-run --with-diff and force the reset to the recommended configuration with yunohost tools regen-conf {category} --force", From da6ae405dd426fabb72d9673bfc2b5ac02accbe2 Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 14 Apr 2020 03:59:33 +0200 Subject: [PATCH 6/9] [fix] Missing pending number args --- data/hooks/diagnosis/24-mail.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data/hooks/diagnosis/24-mail.py b/data/hooks/diagnosis/24-mail.py index b91bfec85..f1a267641 100644 --- a/data/hooks/diagnosis/24-mail.py +++ b/data/hooks/diagnosis/24-mail.py @@ -113,10 +113,12 @@ class MailDiagnoser(Diagnoser): else: if pending_emails > 300: yield dict(meta={"test": "mail_queue"}, + data={'nb_pending': pending_emails}, status="WARNING", summary="diagnosis_mail_queue_too_many_pending_emails") else: yield dict(meta={"test": "mail_queue"}, + data={'nb_pending': pending_emails}, status="INFO", summary="diagnosis_mail_queue_ok") From a17adc274c90517b42bdcdf31d9a12b58f43d7d9 Mon Sep 17 00:00:00 2001 From: ljf Date: Sat, 18 Apr 2020 17:08:09 +0200 Subject: [PATCH 7/9] [wip] Small refactoring for mail diagnoser --- data/hooks/diagnosis/24-mail.py | 279 ++++++++++++++++++-------------- locales/en.json | 15 +- 2 files changed, 165 insertions(+), 129 deletions(-) diff --git a/data/hooks/diagnosis/24-mail.py b/data/hooks/diagnosis/24-mail.py index f1a267641..1336e8c2b 100644 --- a/data/hooks/diagnosis/24-mail.py +++ b/data/hooks/diagnosis/24-mail.py @@ -2,15 +2,21 @@ import os import dns.resolver -import smtplib import socket +import re + +from subprocess import CalledProcessError +from types import FunctionType from moulinette.utils.process import check_output from moulinette.utils.network import download_text from moulinette.utils.filesystem import read_yaml from yunohost.diagnosis import Diagnoser -from yunohost.domain import _get_maindomain +from yunohost.domain import _get_maindomain, domain_list +from yunohost.utils.error import YunohostError + +DIAGNOSIS_SERVER = "diagnosis.yunohost.org" DEFAULT_DNS_BLACKLIST = "/usr/share/yunohost/other/dnsbl_list.yml" @@ -22,126 +28,124 @@ class MailDiagnoser(Diagnoser): dependencies = ["ip"] def run(self): - - ips = self.get_public_ips() - # Is outgoing port 25 filtered somehow ? - self.logger_debug("Running outgoing 25 port check") - if os.system('/bin/nc -z -w2 yunohost.org 25') == 0: - yield dict(meta={"test": "ougoing_port_25"}, - status="SUCCESS", - summary="diagnosis_mail_ougoing_port_25_ok") - else: - yield dict(meta={"test": "outgoing_port_25"}, - status="ERROR", - summary="diagnosis_mail_ougoing_port_25_blocked") + self.ehlo_domain = _get_maindomain() + self.mail_domains = domain_list()["domains"] + self.ipversions, self.ips = self.get_ips_checked() - # Get HELO and be sure postfix is running - # TODO SMTP reachability (c.f. check-smtp to be implemented on yunohost's remote diagnoser) - server = None - result = dict(meta={"test": "mail_ehlo"}, - status="SUCCESS", - summary="diagnosis_mail_service_working") - try: - server = smtplib.SMTP("127.0.0.1", 25, timeout=10) - ehlo = server.ehlo() - ehlo_domain = ehlo[1].decode("utf-8").split("\n")[0] - except OSError: - result = dict(meta={"test": "mail_ehlo"}, - status="ERROR", - summary="diagnosis_mail_service_not_working") - ehlo_domain = _get_maindomain() - if server: - server.quit() - yield result + # TODO Is a A/AAAA and MX Record ? + # TODO Are outgoing public IPs authorized to send mail by SPF ? + # TODO Validate DKIM and dmarc ? + # TODO check that the recent mail logs are not filled with thousand of email sending (unusual number of mail sent) + # TODO check for unusual failed sending attempt being refused in the logs ? + checks = [name for name, value in MailDiagnoser.__dict__.items() + if type(value) == FunctionType and name.startswith("check_")] + for check in checks: + self.logger_debug("Running " + check) + for report in getattr(self, check): + yield report + else: + name = checks[6:] + yield dict(meta={"test": "mail_" + name}, + status="SUCCESS", + summary="diagnosis_mail_" + name + "_ok") - # Forward-confirmed reverse DNS (FCrDNS) verification - self.logger_debug("Running Forward-confirmed reverse DNS check") - for ip in ips: + + def check_outgoing_port_25(self): + """ + Check outgoing port 25 is open and not blocked by router + This check is ran on IPs we could used to send mail. + """ + + for ipversion in self.ipversions: + cmd = '/bin/nc -{ipversion} -z -w2 yunohost.org 25'.format({ + 'ipversion': ipversion}) + if os.system(cmd) != 0: + yield dict(meta={"test": "outgoing_port_25", "ipversion": ipversion}, + data={}, + status="ERROR", + summary="diagnosis_mail_ougoing_port_25_blocked") + + + def check_ehlo(self): + """ + Check the server is reachable from outside and it's the good one + This check is ran on IPs we could used to send mail. + """ + + for ipversion in self.ipversions: + try: + r = Diagnoser.remote_diagnosis('check-smtp', + data={}, + ipversion=ipversion) + except Exception as e: + yield dict(meta={"test": "mail_ehlo", "ipversion": ipversion}, + data={"error": e}, + status="WARNING", + summary="diagnosis_mail_ehlo_could_not_diagnose") + continue + + if r["status"] == "error_smtp_unreachable": + yield dict(meta={"test": "mail_ehlo", "ipversion": ipversion}, + data={}, + status="ERROR", + summary="diagnosis_mail_ehlo_unavailable") + elif r["helo"] != self.ehlo_domain: + yield dict(meta={"test": "mail_ehlo", "ipversion": ipversion}, + data={"wrong_ehlo": r["helo"], "right_ehlo": self.ehlo_domain}, + status="ERROR", + summary="diagnosis_mail_ehlo_wrong") + + + def check_fcrdns(self): + """ + Check the reverse DNS is well defined by doing a Forward-confirmed + reverse DNS check + This check is ran on IPs we could used to send mail. + """ + + for ip in self.ips: try: rdns_domain, _, _ = socket.gethostbyaddr(ip) - except socket.herror as e: - yield dict(meta={"test": "mail_fcrdns"}, - data={"ip": ip, "ehlo_domain": ehlo_domain}, + except socket.herror: + yield dict(meta={"test": "mail_fcrdns", "ip": ip}, + data={"ehlo_domain": self.ehlo_domain}, status="ERROR", summary="diagnosis_mail_reverse_dns_missing") continue - else: - if rdns_domain != ehlo_domain: - yield dict(meta={"test": "mail_fcrdns"}, - data={"ip": ip, "ehlo_domain": ehlo_domain, - "rdns_domain": rdns_domain}, - status="ERROR", - summary="diagnosis_mail_rdns_different_from_ehlo_domain") - else: - yield dict(meta={"test": "mail_fcrdns"}, - data={"ip": ip, "ehlo_domain": ehlo_domain}, - status="SUCCESS", - summary="diagnosis_mail_rdns_equal_to_ehlo_domain") - - # TODO Is a A/AAAA and MX Record ? - - # Are IPs listed on a DNSBL ? - self.logger_debug("Running DNS Blacklist detection") - # TODO Test if domain are blacklisted too - - blacklisted_details = list(self.check_dnsbl(self.get_public_ips())) - if blacklisted_details: - yield dict(meta={"test": "mail_blacklist"}, - status="ERROR", - summary="diagnosis_mail_blacklist_nok", - details=blacklisted_details) - else: - yield dict(meta={"test": "mail_blacklist"}, - status="SUCCESS", - summary="diagnosis_mail_blacklist_ok") - - # TODO Are outgoing public IPs authorized to send mail by SPF ? - - # TODO Validate DKIM and dmarc ? + if rdns_domain != self.ehlo_domain: + yield dict(meta={"test": "mail_fcrdns", "ip": ip}, + data={"ehlo_domain": self.ehlo_domain, + "rdns_domain": rdns_domain}, + status="ERROR", + summary="diagnosis_mail_rdns_different_from_ehlo_domain") - # Is mail queue filled with hundreds of email pending ? - command = 'postqueue -p | grep -c "^[A-Z0-9]"' - output = check_output(command).strip() - try: - pending_emails = int(output) - except ValueError: - yield dict(meta={"test": "mail_queue"}, - status="ERROR", - summary="diagnosis_mail_cannot_get_queue") - else: - if pending_emails > 300: - yield dict(meta={"test": "mail_queue"}, - data={'nb_pending': pending_emails}, - status="WARNING", - summary="diagnosis_mail_queue_too_many_pending_emails") - else: - yield dict(meta={"test": "mail_queue"}, - data={'nb_pending': pending_emails}, - status="INFO", - summary="diagnosis_mail_queue_ok") - - # check that the recent mail logs are not filled with thousand of email sending (unusual number of mail sent) - - # check for unusual failed sending attempt being refused in the logs ? - - def check_dnsbl(self, ips): - """ Check with dig onto blacklist DNS server + def check_blacklist(self): """ + Check with dig onto blacklist DNS server + This check is ran on IPs and domains we could used to send mail. + """ + dns_blacklists = read_yaml(DEFAULT_DNS_BLACKLIST) - for ip in ips: + for item in self.ips + self.mail_domains: for blacklist in dns_blacklists: - if "." in ip and not blacklist['ipv4']: + item_type = "domain" + if ":" in item: + item_type = 'ipv6' + elif re.match(r'^\d+\.\d+\.\d+\.\d+$', item): + item_type = 'ipv4' + + if not blacklist[item_type]: continue - if ":" in ip and not blacklist['ipv6']: - continue - # Determine if we are listed on this RBL try: - rev = dns.reversename.from_address(ip) - query = str(rev.split(3)[0]) + '.' + blacklist['dns_server'] + subdomain = item + if item_type != "domain": + rev = dns.reversename.from_address(item) + subdomain = str(rev.split(3)[0]) + query = subdomain + '.' + blacklist['dns_server'] # TODO add timeout lifetime dns.resolver.query(query, "A") except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers, dns.resolver.NoAnswer, @@ -149,32 +153,63 @@ class MailDiagnoser(Diagnoser): continue # Try to get the reason - reason = "not explained" try: reason = str(dns.resolver.query(query, "TXT")[0]) except Exception: - pass + reason = "-" - yield ('diagnosis_mail_blacklisted_by', { - 'ip': ip, - 'blacklist_name': blacklist['name'], - 'blacklist_website': blacklist['website'], - 'reason': reason}) + yield dict(meta={"test": "mail_blacklist", "item": item, + "blacklist": blacklist["dns_server"]}, + data={'blacklist_name': blacklist['name'], + 'blacklist_website': blacklist['website'], + 'reason': reason}, + status="ERROR", + summary='diagnosis_mail_blacklist_listed_by') - def get_public_ips(self): - # Todo code a better way to access a data - ipv4 = Diagnoser.get_cached_report("ip", {"test": "ipv4"}) - if ipv4: + def check_queue(self): + """ + Check mail queue is not filled with hundreds of email pending + """ + + command = 'postqueue -p | grep -v "Mail queue is empty" | grep -c "^[A-Z0-9]"' + try: + output = check_output(command).strip() + pending_emails = int(output) + except (ValueError, CalledProcessError) as e: + yield dict(meta={"test": "mail_queue"}, + data={"error": e}, + status="ERROR", + summary="diagnosis_mail_cannot_get_queue") + else: + if pending_emails > 100: + yield dict(meta={"test": "mail_queue"}, + data={'nb_pending': pending_emails}, + status="WARNING", + summary="diagnosis_mail_queue_too_many_pending_emails") + else: + yield dict(meta={"test": "mail_queue"}, + data={'nb_pending': pending_emails}, + status="SUCCESS", + summary="diagnosis_mail_queue_ok") + + + def get_ips_checked(self): + outgoing_ipversions = [] + outgoing_ips = [] + ipv4 = Diagnoser.get_cached_report("ip", {"test": "ipv4"}) or {} + if ipv4.get("status") == "SUCCESS": + outgoing_ipversions.append(4) global_ipv4 = ipv4.get("data", {}).get("global", {}) if global_ipv4: - yield global_ipv4 - - ipv6 = Diagnoser.get_cached_report("ip", {"test": "ipv6"}) - if ipv6: + outgoing_ips.append(global_ipv4) + + ipv6 = Diagnoser.get_cached_report("ip", {"test": "ipv6"}) or {} + if ipv6.get("status") == "SUCCESS": + outgoing_ipversions.append(6) global_ipv6 = ipv6.get("data", {}).get("global", {}) if global_ipv6: - yield global_ipv6 - + outgoing_ips.append(global_ipv6) + return (outgoing_ipversions, outgoing_ips) def main(args, env, loggers): return MailDiagnoser(args, env, loggers).diagnose() diff --git a/locales/en.json b/locales/en.json index 978ceb831..1a17c484f 100644 --- a/locales/en.json +++ b/locales/en.json @@ -185,18 +185,19 @@ "diagnosis_swap_notsomuch": "The system has only {total} swap. You should consider having at least 256 MB to avoid situations where the system runs out of memory.", "diagnosis_swap_ok": "The system has {total} of swap!", "diagnosis_mail_ougoing_port_25_ok": "Outgoing port 25 is not blocked and email can be sent to other servers.", - "diagnosis_mail_ougoing_port_25_blocked": "Outgoing port 25 appears to be blocked. You should try to unblock it in your internet service provider (or hosting provider) configuration panel. Meanwhile, the server won't be able to send emails to other servers.", - "diagnosis_mail_blacklist_ok": "The public IPs of this instance are not listed on email blacklists.", - "diagnosis_mail_blacklist_nok": "Some of the public IPs of this instance are listed on email blacklists.", - "diagnosis_mail_blacklisted_by": "{ip} is listed on {blacklist_name}. Reason: {reason}. See {blacklist_website}", - "diagnosis_mail_service_working": "Postfix mail service answer correctly.", - "diagnosis_mail_service_not_working": "Postfix mail service don't answer to EHLO request.", + "diagnosis_mail_ougoing_port_25_blocked": "Outgoing port 25 appears to be blocked in IPv{ipversion}. You should try to unblock it in your internet service provider (or hosting provider) configuration panel. Meanwhile, the server won't be able to send emails to other servers.", + "diagnosis_mail_ehlo_ok": "Postfix mail service answer correctly from outside", + "diagnosis_mail_ehlo_unavailable": "Postfix mail service don't answer to EHLO request on IPv{ipversion}.", + "diagnosis_mail_ehlo_wrong": "A mail server answer {wrong_ehlo} instead {right_ehlo} on IPv{ipversion}.", + "diagnosis_mail_ehlo_could_not_diagnose": "Could not diagnose if postfix mail server is reachable from outside. Error: {error}", "diagnosis_mail_reverse_dns_missing": "No reverse DNS defined for the ip {ip}.", "diagnosis_mail_rdns_different_from_ehlo_domain": "Your reverse DNS {rdns_domain} is different from your EHLO domain {ehlo_domain} on {ip}.", "diagnosis_mail_rdns_equal_to_ehlo_domain": "Your reverse DNS is equal to your EHLO domain {ehlo_domain} on {ip}.", + "diagnosis_mail_blacklist_ok": "The public IPs of this instance are not listed on email blacklists.", + "diagnosis_mail_blacklist_listed_by": "{item} is blacklisted on {blacklist_name}. Reason: {reason}. See {blacklist_website}", "diagnosis_mail_queue_unavailable": "Can not consult number of pending emails in queue", "diagnosis_mail_queue_too_big": "The mail queue has {nb_pending} pending emails in the mail queue. It seems abnormal.", - "diagnosis_mail_queue_unavailable": "The mail queue has {nb_pending} pending emails in the mail queue.", + "diagnosis_mail_queue_ok": "The mail queue has {nb_pending} pending emails in the mail queue.", "diagnosis_regenconf_allgood": "All configurations files are in line with the recommended configuration!", "diagnosis_regenconf_manually_modified": "Configuration file {file} appears to have been manually modified.", "diagnosis_regenconf_manually_modified_details": "This is probably OK if you know what you're doing! YunoHost will stop updating this file automatically... But beware that YunoHost upgrades could contain important recommended changes. If you want to, you can inspect the differences with yunohost tools regen-conf {category} --dry-run --with-diff and force the reset to the recommended configuration with yunohost tools regen-conf {category} --force", From b1124b7080aae3c1750503b430cfc4c067184f7c Mon Sep 17 00:00:00 2001 From: ljf Date: Sat, 18 Apr 2020 19:06:45 +0200 Subject: [PATCH 8/9] [fix] Maildiagnoser typo --- data/hooks/diagnosis/24-mail.py | 22 +++++++++++----------- locales/en.json | 10 +++++----- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/data/hooks/diagnosis/24-mail.py b/data/hooks/diagnosis/24-mail.py index 1336e8c2b..4c36d7ca0 100644 --- a/data/hooks/diagnosis/24-mail.py +++ b/data/hooks/diagnosis/24-mail.py @@ -24,7 +24,7 @@ DEFAULT_DNS_BLACKLIST = "/usr/share/yunohost/other/dnsbl_list.yml" class MailDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] - cache_duration = 600 + cache_duration = 0 dependencies = ["ip"] def run(self): @@ -42,10 +42,11 @@ class MailDiagnoser(Diagnoser): if type(value) == FunctionType and name.startswith("check_")] for check in checks: self.logger_debug("Running " + check) - for report in getattr(self, check): + reports = list(getattr(self, check)()) + for report in reports: yield report - else: - name = checks[6:] + if not reports: + name = check[6:] yield dict(meta={"test": "mail_" + name}, status="SUCCESS", summary="diagnosis_mail_" + name + "_ok") @@ -58,8 +59,7 @@ class MailDiagnoser(Diagnoser): """ for ipversion in self.ipversions: - cmd = '/bin/nc -{ipversion} -z -w2 yunohost.org 25'.format({ - 'ipversion': ipversion}) + cmd = '/bin/nc -{ipversion} -z -w2 yunohost.org 25'.format(ipversion=ipversion) if os.system(cmd) != 0: yield dict(meta={"test": "outgoing_port_25", "ipversion": ipversion}, data={}, @@ -80,7 +80,7 @@ class MailDiagnoser(Diagnoser): ipversion=ipversion) except Exception as e: yield dict(meta={"test": "mail_ehlo", "ipversion": ipversion}, - data={"error": e}, + data={"error": str(e)}, status="WARNING", summary="diagnosis_mail_ehlo_could_not_diagnose") continue @@ -111,14 +111,14 @@ class MailDiagnoser(Diagnoser): yield dict(meta={"test": "mail_fcrdns", "ip": ip}, data={"ehlo_domain": self.ehlo_domain}, status="ERROR", - summary="diagnosis_mail_reverse_dns_missing") + summary="diagnosis_mail_fcrdns_dns_missing") continue if rdns_domain != self.ehlo_domain: yield dict(meta={"test": "mail_fcrdns", "ip": ip}, data={"ehlo_domain": self.ehlo_domain, "rdns_domain": rdns_domain}, status="ERROR", - summary="diagnosis_mail_rdns_different_from_ehlo_domain") + summary="diagnosis_mail_fcrdns_different_from_ehlo_domain") def check_blacklist(self): @@ -177,9 +177,9 @@ class MailDiagnoser(Diagnoser): pending_emails = int(output) except (ValueError, CalledProcessError) as e: yield dict(meta={"test": "mail_queue"}, - data={"error": e}, + data={"error": str(e)}, status="ERROR", - summary="diagnosis_mail_cannot_get_queue") + summary="diagnosis_mail_queue_unavailable") else: if pending_emails > 100: yield dict(meta={"test": "mail_queue"}, diff --git a/locales/en.json b/locales/en.json index 1a17c484f..327dba2a9 100644 --- a/locales/en.json +++ b/locales/en.json @@ -184,15 +184,15 @@ "diagnosis_swap_none": "The system has no swap at all. You should consider adding at least 256 MB of swap to avoid situations where the system runs out of memory.", "diagnosis_swap_notsomuch": "The system has only {total} swap. You should consider having at least 256 MB to avoid situations where the system runs out of memory.", "diagnosis_swap_ok": "The system has {total} of swap!", - "diagnosis_mail_ougoing_port_25_ok": "Outgoing port 25 is not blocked and email can be sent to other servers.", - "diagnosis_mail_ougoing_port_25_blocked": "Outgoing port 25 appears to be blocked in IPv{ipversion}. You should try to unblock it in your internet service provider (or hosting provider) configuration panel. Meanwhile, the server won't be able to send emails to other servers.", + "diagnosis_mail_outgoing_port_25_ok": "Outgoing port 25 is not blocked and email can be sent to other servers.", + "diagnosis_mail_outgoing_port_25_blocked": "Outgoing port 25 appears to be blocked in IPv{ipversion}. You should try to unblock it in your internet service provider (or hosting provider) configuration panel. Meanwhile, the server won't be able to send emails to other servers.", "diagnosis_mail_ehlo_ok": "Postfix mail service answer correctly from outside", "diagnosis_mail_ehlo_unavailable": "Postfix mail service don't answer to EHLO request on IPv{ipversion}.", "diagnosis_mail_ehlo_wrong": "A mail server answer {wrong_ehlo} instead {right_ehlo} on IPv{ipversion}.", "diagnosis_mail_ehlo_could_not_diagnose": "Could not diagnose if postfix mail server is reachable from outside. Error: {error}", - "diagnosis_mail_reverse_dns_missing": "No reverse DNS defined for the ip {ip}.", - "diagnosis_mail_rdns_different_from_ehlo_domain": "Your reverse DNS {rdns_domain} is different from your EHLO domain {ehlo_domain} on {ip}.", - "diagnosis_mail_rdns_equal_to_ehlo_domain": "Your reverse DNS is equal to your EHLO domain {ehlo_domain} on {ip}.", + "diagnosis_mail_fcrdns_dns_missing": "No reverse DNS defined for the ip {ip}.", + "diagnosis_mail_fcrdns_different_from_ehlo_domain": "Your reverse DNS {rdns_domain} is different from your EHLO domain {ehlo_domain} on {ip}.", + "diagnosis_mail_fcrdns_ok": "Your reverse DNS is well configured.", "diagnosis_mail_blacklist_ok": "The public IPs of this instance are not listed on email blacklists.", "diagnosis_mail_blacklist_listed_by": "{item} is blacklisted on {blacklist_name}. Reason: {reason}. See {blacklist_website}", "diagnosis_mail_queue_unavailable": "Can not consult number of pending emails in queue", From 0014fe29033c6eeb2e4238b7283ea342ff72fc34 Mon Sep 17 00:00:00 2001 From: ljf Date: Sat, 18 Apr 2020 20:40:18 +0200 Subject: [PATCH 9/9] [fix] Order of mail checks and mail queue --- data/hooks/diagnosis/24-mail.py | 26 ++++++++++++++++---------- locales/en.json | 30 ++++++++++++++++++------------ 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/data/hooks/diagnosis/24-mail.py b/data/hooks/diagnosis/24-mail.py index 4c36d7ca0..b122e876a 100644 --- a/data/hooks/diagnosis/24-mail.py +++ b/data/hooks/diagnosis/24-mail.py @@ -6,15 +6,12 @@ import socket import re from subprocess import CalledProcessError -from types import FunctionType from moulinette.utils.process import check_output -from moulinette.utils.network import download_text from moulinette.utils.filesystem import read_yaml from yunohost.diagnosis import Diagnoser from yunohost.domain import _get_maindomain, domain_list -from yunohost.utils.error import YunohostError DIAGNOSIS_SERVER = "diagnosis.yunohost.org" @@ -38,8 +35,8 @@ class MailDiagnoser(Diagnoser): # TODO Validate DKIM and dmarc ? # TODO check that the recent mail logs are not filled with thousand of email sending (unusual number of mail sent) # TODO check for unusual failed sending attempt being refused in the logs ? - checks = [name for name, value in MailDiagnoser.__dict__.items() - if type(value) == FunctionType and name.startswith("check_")] + checks = ["check_outgoing_port_25", "check_ehlo", "check_fcrdns", + "check_blacklist", "check_queue"] for check in checks: self.logger_debug("Running " + check) reports = list(getattr(self, check)()) @@ -64,7 +61,9 @@ class MailDiagnoser(Diagnoser): yield dict(meta={"test": "outgoing_port_25", "ipversion": ipversion}, data={}, status="ERROR", - summary="diagnosis_mail_ougoing_port_25_blocked") + summary="diagnosis_mail_ougoing_port_25_blocked", + details=["diagnosis_mail_ougoing_port_25_blocked_details", + "diagnosis_mail_outgoing_port_25_blocked_relay_vpn"]) def check_ehlo(self): @@ -82,7 +81,8 @@ class MailDiagnoser(Diagnoser): yield dict(meta={"test": "mail_ehlo", "ipversion": ipversion}, data={"error": str(e)}, status="WARNING", - summary="diagnosis_mail_ehlo_could_not_diagnose") + summary="diagnosis_mail_ehlo_could_not_diagnose", + details=["diagnosis_mail_ehlo_could_not_diagnose_details"]) continue if r["status"] == "error_smtp_unreachable": @@ -153,25 +153,30 @@ class MailDiagnoser(Diagnoser): continue # Try to get the reason + details = [] try: reason = str(dns.resolver.query(query, "TXT")[0]) + details.append("diagnosis_mail_blacklist_reason") except Exception: reason = "-" + details.append("diagnosis_mail_blacklist_website") + yield dict(meta={"test": "mail_blacklist", "item": item, "blacklist": blacklist["dns_server"]}, data={'blacklist_name': blacklist['name'], 'blacklist_website': blacklist['website'], 'reason': reason}, status="ERROR", - summary='diagnosis_mail_blacklist_listed_by') + summary='diagnosis_mail_blacklist_listed_by', + details=details) def check_queue(self): """ Check mail queue is not filled with hundreds of email pending """ - command = 'postqueue -p | grep -v "Mail queue is empty" | grep -c "^[A-Z0-9]"' + command = 'postqueue -p | grep -v "Mail queue is empty" | grep -c "^[A-Z0-9]" || true' try: output = check_output(command).strip() pending_emails = int(output) @@ -179,7 +184,8 @@ class MailDiagnoser(Diagnoser): yield dict(meta={"test": "mail_queue"}, data={"error": str(e)}, status="ERROR", - summary="diagnosis_mail_queue_unavailable") + summary="diagnosis_mail_queue_unavailable", + details="diagnosis_mail_queue_unavailable_details") else: if pending_emails > 100: yield dict(meta={"test": "mail_queue"}, diff --git a/locales/en.json b/locales/en.json index 327dba2a9..d2f4a925b 100644 --- a/locales/en.json +++ b/locales/en.json @@ -184,20 +184,26 @@ "diagnosis_swap_none": "The system has no swap at all. You should consider adding at least 256 MB of swap to avoid situations where the system runs out of memory.", "diagnosis_swap_notsomuch": "The system has only {total} swap. You should consider having at least 256 MB to avoid situations where the system runs out of memory.", "diagnosis_swap_ok": "The system has {total} of swap!", - "diagnosis_mail_outgoing_port_25_ok": "Outgoing port 25 is not blocked and email can be sent to other servers.", - "diagnosis_mail_outgoing_port_25_blocked": "Outgoing port 25 appears to be blocked in IPv{ipversion}. You should try to unblock it in your internet service provider (or hosting provider) configuration panel. Meanwhile, the server won't be able to send emails to other servers.", + "diagnosis_mail_outgoing_port_25_ok": "Outgoing port 25 is open, emails can be sent", + "diagnosis_mail_outgoing_port_25_blocked": "Outgoing port 25 appears to be bloecked in IPv{ipversion}", + "diagnosis_mail_outgoing_port_25_blocked_details": "You should first try to unblock it in your internet service provider (or hosting provider) configuration panel or by sending a ticket to your hosting provider. Meanwhile, the server won't be able to send emails to other servers.", + "diagnosis_mail_outgoing_port_25_blocked_relay_vpn": "Some providers won't let you unblock outgoing port 25 because they don't care about Net Neutrality.
- Some of them provide the alternative of using a mail server relay though it implies that the relay will be able to spy on your email traffic.
- A privacy-friendly alternative is to use a VPN *with a dedicated public IP* to bypass this kind of limits. See https://yunohost.org/#/vpn_advantage
- Finally, it's also possible to change of provider", "diagnosis_mail_ehlo_ok": "Postfix mail service answer correctly from outside", - "diagnosis_mail_ehlo_unavailable": "Postfix mail service don't answer to EHLO request on IPv{ipversion}.", - "diagnosis_mail_ehlo_wrong": "A mail server answer {wrong_ehlo} instead {right_ehlo} on IPv{ipversion}.", - "diagnosis_mail_ehlo_could_not_diagnose": "Could not diagnose if postfix mail server is reachable from outside. Error: {error}", - "diagnosis_mail_fcrdns_dns_missing": "No reverse DNS defined for the ip {ip}.", - "diagnosis_mail_fcrdns_different_from_ehlo_domain": "Your reverse DNS {rdns_domain} is different from your EHLO domain {ehlo_domain} on {ip}.", - "diagnosis_mail_fcrdns_ok": "Your reverse DNS is well configured.", - "diagnosis_mail_blacklist_ok": "The public IPs of this instance are not listed on email blacklists.", - "diagnosis_mail_blacklist_listed_by": "{item} is blacklisted on {blacklist_name}. Reason: {reason}. See {blacklist_website}", + "diagnosis_mail_ehlo_unavailable": "Postfix mail service don't answer to EHLO request on IPv{ipversion}", + "diagnosis_mail_ehlo_wrong": "A mail server answers {wrong_ehlo} instead {right_ehlo} on IPv{ipversion}", + "diagnosis_mail_ehlo_could_not_diagnose": "Could not diagnose if postfix mail server is reachable from outside", + "diagnosis_mail_ehlo_could_not_diagnose_details": "Error: {error}", + "diagnosis_mail_fcrdns_ok": "Your reverse DNS is well configured", + "diagnosis_mail_fcrdns_dns_missing": "No reverse DNS defined for the ip {ip}", + "diagnosis_mail_fcrdns_different_from_ehlo_domain": "Your reverse DNS {rdns_domain} is different from your EHLO domain {ehlo_domain} on {ip}", + "diagnosis_mail_blacklist_ok": "IPs and domains used by this server to send mail are not on most used email blacklists", + "diagnosis_mail_blacklist_listed_by": "{item} is blacklisted on {blacklist_name}", + "diagnosis_mail_blacklist_reason": "The blacklist explains: {reason}", + "diagnosis_mail_blacklist_website": "After identifying why you are listed and fixed it, feel free to ask for delisting on {blacklist_website}", + "diagnosis_mail_queue_ok": "{nb_pending} pending emails in the mail queues", "diagnosis_mail_queue_unavailable": "Can not consult number of pending emails in queue", - "diagnosis_mail_queue_too_big": "The mail queue has {nb_pending} pending emails in the mail queue. It seems abnormal.", - "diagnosis_mail_queue_ok": "The mail queue has {nb_pending} pending emails in the mail queue.", + "diagnosis_mail_queue_unavailable_details": "Error: {error}", + "diagnosis_mail_queue_too_big": "Too many pending emails in mail queue ({nb_pending} emails)", "diagnosis_regenconf_allgood": "All configurations files are in line with the recommended configuration!", "diagnosis_regenconf_manually_modified": "Configuration file {file} appears to have been manually modified.", "diagnosis_regenconf_manually_modified_details": "This is probably OK if you know what you're doing! YunoHost will stop updating this file automatically... But beware that YunoHost upgrades could contain important recommended changes. If you want to, you can inspect the differences with yunohost tools regen-conf {category} --dry-run --with-diff and force the reset to the recommended configuration with yunohost tools regen-conf {category} --force",