From 0b7984adf117a413b63d8604d6b54cea22bc3c87 Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 12 Apr 2020 04:14:49 +0200 Subject: [PATCH] [enh] Improve DNSBL check --- data/hooks/diagnosis/24-mail.py | 120 +++++++++------------ data/other/dnsbl_list.yml | 184 ++++++++++++++++++++++++++++++++ debian/install | 1 + locales/en.json | 6 +- 4 files changed, 237 insertions(+), 74 deletions(-) create mode 100644 data/other/dnsbl_list.yml diff --git a/data/hooks/diagnosis/24-mail.py b/data/hooks/diagnosis/24-mail.py index 25d0ff984..333d98c8a 100644 --- a/data/hooks/diagnosis/24-mail.py +++ b/data/hooks/diagnosis/24-mail.py @@ -4,38 +4,11 @@ import os import dns.resolver from moulinette.utils.network import download_text +from moulinette.utils.filesystem import read_yaml from yunohost.diagnosis import Diagnoser -DEFAULT_BLACKLIST = [ - ('zen.spamhaus.org' , 'Spamhaus SBL, XBL and PBL' ), - ('dnsbl.sorbs.net' , 'SORBS aggregated' ), - ('safe.dnsbl.sorbs.net' , "'safe' subset of SORBS aggregated"), - ('ix.dnsbl.manitu.net' , 'Heise iX NiX Spam' ), - ('babl.rbl.webiron.net' , 'Bad Abuse' ), - ('cabl.rbl.webiron.net' , 'Chronicly Bad Abuse' ), - ('truncate.gbudb.net' , 'Exclusively Spam/Malware' ), - ('dnsbl-1.uceprotect.net' , 'Trapserver Cluster' ), - ('cbl.abuseat.org' , 'Net of traps' ), - ('dnsbl.cobion.com' , 'used in IBM products' ), - ('psbl.surriel.com' , 'passive list, easy to unlist' ), - ('dnsrbl.org' , 'Real-time black list' ), - ('db.wpbl.info' , 'Weighted private' ), - ('bl.spamcop.net' , 'Based on spamcop users' ), - ('dyna.spamrats.com' , 'Dynamic IP addresses' ), - ('spam.spamrats.com' , 'Manual submissions' ), - ('auth.spamrats.com' , 'Suspicious authentications' ), - ('dnsbl.inps.de' , 'automated and reported' ), - ('bl.blocklist.de' , 'fail2ban reports etc.' ), - ('srnblack.surgate.net' , 'feeders' ), - ('all.s5h.net' , 'traps' ), - ('rbl.realtimeblacklist.com' , 'lists ip ranges' ), - ('b.barracudacentral.org' , 'traps' ), - ('hostkarma.junkemailfilter.com', 'Autotected Virus Senders' ), - ('rbl.megarbl.net' , 'Curated Spamtraps' ), - ('ubl.unsubscore.com' , 'Collected Opt-Out Addresses' ), - ('0spam.fusionzero.com' , 'Spam Trap' ), -] +DEFAULT_DNS_BLACKLIST = "/usr/share/yunohost/other/dnsbl_list.yml" class MailDiagnoser(Diagnoser): @@ -57,17 +30,13 @@ class MailDiagnoser(Diagnoser): status="ERROR", summary="diagnosis_mail_ougoing_port_25_blocked") - # Is Reverse DNS well configured ? + # Forward-confirmed reverse DNS (FCrDNS) verification - # Are IPs blacklisted ? - self.logger_debug("Running RBL detection") - ipv4 = Diagnoser.get_cached_report_item("ip", {"test": "ipv4"}) - global_ipv4 = ipv4.get("data", {}).get("global", {}) - ipv6 = Diagnoser.get_cached_report_item("ip", {"test": "ipv6"}) - global_ipv6 = ipv6.get("data", {}).get("global", {}) - blacklisted_details = tuple(self.check_blacklisted(global_ipv4)) - blacklisted_details += tuple(self.check_blacklisted(global_ipv6)) + # Are IPs listed on a DNSBL ? + self.logger_debug("Running DNSBL detection") + + blacklisted_details = self.check_ip_dnsbl() if blacklisted_details: yield dict(meta={"test": "mail_blacklist"}, status="ERROR", @@ -88,45 +57,54 @@ class MailDiagnoser(Diagnoser): # check for unusual failed sending attempt being refused in the logs ? - def check_blacklisted(self, ip): + def check_blacklisted(self): """ Check with dig onto blacklist DNS server """ - if ip is None: - return + dns_blacklists = read_yaml(DEFAULT_DNS_BLACKLIST) + for ip in self.get_public_ips(): + for blacklist in dns_blacklists: + + if "." in ip and not blacklist.ipv4: + continue - for blacklist, description in DEFAULT_BLACKLIST: + if ":" in ip and not blacklist.ipv6: + continue + + # Determine if we are listed on this RBL + try: + rev = dns.reversename.from_address(ip) + query = str(rev.split(3)[0]) + '.' + blacklist.dns_server + # TODO add timeout lifetime + dns.resolver.query(query, "A") + except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers, dns.resolver.NoAnswer, + dns.exception.Timeout): + continue - # Determine if we are listed on this RBL - try: - rev = dns.reversename.from_address(ip) - query = str(rev.split(3)[0]) + '.' + blacklist - # TODO add timeout lifetime - dns.resolver.query(query, "A") - except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers, dns.resolver.NoAnswer, - dns.exception.Timeout): - continue + # Try to get the reason + reason = "not explained" + try: + reason = str(dns.resolver.query(query, "TXT")[0]) + except Exception: + pass - # Try to get the reason - reason = "not explained" - try: - reason = str(dns.resolver.query(query, "TXT")[0]) - except Exception: - pass + yield ('diagnosis_mail_blacklisted_by', { + 'ip': ip, + 'blacklist': blacklist, + 'reason': reason}) - yield ('diagnosis_mail_blacklisted_by', - {'ip': ip, 'blacklist': blacklist, 'reason': reason}) - - def get_public_ip(self, protocol=4): - # TODO we might call this function from another side - assert protocol in [4, 6], "Invalid protocol version, it should be either 4 or 6 and was '%s'" % repr(protocol) - - url = 'https://ip%s.yunohost.org' % ('6' if protocol == 6 else '') - - try: - return download_text(url, timeout=30).strip() - except Exception as e: - self.logger_debug("Could not get public IPv%s : %s" % (str(protocol), str(e))) - return None + def get_public_ips(self): + # Todo code a better way to access a data + ipv4 = Diagnoser.get_cached_report("ip", {"test": "ipv4"}) + if ipv4: + global_ipv4 = ipv4.get("data", {}).get("global", {}) + if global_ipv4: + yield global_ipv4 + + ipv6 = Diagnoser.get_cached_report("ip", {"test": "ipv6"}) + if ipv6: + global_ipv6 = ipv6.get("data", {}).get("global", {}) + if global_ipv6: + yield global_ipv6 def main(args, env, loggers): diff --git a/data/other/dnsbl_list.yml b/data/other/dnsbl_list.yml new file mode 100644 index 000000000..839aeaab6 --- /dev/null +++ b/data/other/dnsbl_list.yml @@ -0,0 +1,184 @@ +# Used by GAFAM +- name: Spamhaus ZEN + dns_server: zen.spamhaus.org + website: https://www.spamhaus.org/zen/ + ipv4: true + ipv6: true + domain: false +- name: Barracuda Reputation Block List + dns_server: b.barracudacentral.org + website: https://barracudacentral.org/rbl/ + ipv4: true + ipv6: false + domain: false +- name: Hostkarma + dns_server: hostkarma.junkemailfilter.com + website: https://ipadmin.junkemailfilter.com/remove.php + ipv4: true + ipv6: false + domain: false +- name: ImproWare IP based spamlist + dns_server: spamrbl.imp.ch + website: https://antispam.imp.ch/ + ipv4: true + ipv6: false + domain: false +- name: ImproWare IP based wormlist + dns_server: wormrbl.imp.ch + website: https://antispam.imp.ch/ + ipv4: true + ipv6: false + domain: false +- name: Backscatterer.org + dns_server: ips.backscatterer.org + website: http://www.backscatterer.org/ + ipv4: true + ipv6: false + domain: false +- name: inps.de + dns_server: dnsbl.inps.de + website: http://dnsbl.inps.de/ + ipv4: true + ipv6: false + domain: false +- name: LASHBACK + dns_server: ubl.unsubscore.com + website: https://blacklist.lashback.com/ + ipv4: true + ipv6: false + domain: false +- name: Mailspike.org + dns_server: bl.mailspike.net + website: http://www.mailspike.net/ + ipv4: true + ipv6: false + domain: false +- name: NiX Spam + dns_server: ix.dnsbl.manitu.net + website: http://www.dnsbl.manitu.net/ + ipv4: true + ipv6: false + domain: false +- name: REDHAWK + dns_server: access.redhawk.org + website: https://www.redhawk.org/SpamHawk/query.php + ipv4: true + ipv6: false + domain: false +- name: SORBS Open SMTP relays + dns_server: smtp.dnsbl.sorbs.net + website: http://www.sorbs.net/ + ipv4: true + ipv6: false + domain: false +- name: SORBS Spamhost (last 28 days) + dns_server: recent.spam.dnsbl.sorbs.net + website: http://www.sorbs.net/ + ipv4: true + ipv6: false + domain: false +- name: SORBS Spamhost (last 48 hours) + dns_server: new.spam.dnsbl.sorbs.net + website: http://www.sorbs.net/ + ipv4: true + ipv6: false + domain: false +- name: SpamCop Blocking List + dns_server: bl.spamcop.net + website: https://www.spamcop.net/bl.shtml + ipv4: true + ipv6: false + domain: false +- name: Spam Eating Monkey SEM-BACKSCATTER + dns_server: backscatter.spameatingmonkey.net + website: https://spameatingmonkey.com/services + ipv4: true + ipv6: false + domain: false +- name: Spam Eating Monkey SEM-BLACK + dns_server: bl.spameatingmonkey.net + website: https://spameatingmonkey.com/services + ipv4: true + ipv6: false + domain: false +- name: Spam Eating Monkey SEM-IPV6BL + dns_server: bl.ipv6.spameatingmonkey.net + website: https://spameatingmonkey.com/services + ipv4: false + ipv6: true + domain: false +- name: SpamRATS! all + dns_server: all.spamrats.com + website: http://www.spamrats.com/ + ipv4: true + ipv6: false + domain: false +- name: PSBL (Passive Spam Block List) + dns_server: psbl.surriel.com + website: http://psbl.surriel.com/ + ipv4: true + ipv6: false + domain: false +- name: SWINOG + dns_server: dnsrbl.swinog.ch + website: https://antispam.imp.ch/ + ipv4: true + ipv6: false + domain: false +- name: GBUdb Truncate + dns_server: truncate.gbudb.net + website: http://www.gbudb.com/truncate/index.jsp + ipv4: true + ipv6: false + domain: false +- name: Weighted Private Block List + dns_server: db.wpbl.info + website: http://www.wpbl.info/ + ipv4: true + ipv6: false + domain: false +# Used by GAFAM +- name: Composite Blocking List + dns_server: cbl.abuseat.org + website: cbl.abuseat.org + ipv4: true + ipv6: false + domain: false +# Used by GAFAM +- name: SenderScore Blacklist + dns_server: bl.score.senderscore.com + website: https://senderscore.com + ipv4: true + ipv6: false + domain: false +- name: Invaluement + dns_server: sip.invaluement.com + website: https://www.invaluement.com/ + ipv4: true + ipv6: false + domain: false +# Added cause it supports IPv6 +- name: AntiCaptcha.NET IPv6 + dns_server: dnsbl6.anticaptcha.net + website: http://anticaptcha.net/ + ipv4: false + ipv6: true + domain: false +- name: SPFBL.net RBL + dns_server: dnsbl.spfbl.net + website: https://spfbl.net/en/dnsbl/ + ipv4: true + ipv6: true + domain: true +- name: Suomispam Blacklist + dns_server: bl.suomispam.net + website: http://suomispam.net/ + ipv4: true + ipv6: true + domain: false +- name: NordSpam + dns_server: bl.nordspam.com + website: https://www.nordspam.com/ + ipv4: true + ipv6: true + domain: false diff --git a/debian/install b/debian/install index e0743cdd1..cf682d958 100644 --- a/debian/install +++ b/debian/install @@ -7,6 +7,7 @@ data/hooks/* /usr/share/yunohost/hooks/ data/other/yunoprompt.service /etc/systemd/system/ data/other/password/* /usr/share/yunohost/other/password/ data/other/dpkg-origins/yunohost /etc/dpkg/origins +data/other/dnsbl_list.yml /usr/share/yunohost/other/dnsbl_list.yml data/other/* /usr/share/yunohost/yunohost-config/moulinette/ data/templates/* /usr/share/yunohost/templates/ data/helpers /usr/share/yunohost/ diff --git a/locales/en.json b/locales/en.json index 26c51d253..37ae2a34f 100644 --- a/locales/en.json +++ b/locales/en.json @@ -186,9 +186,9 @@ "diagnosis_swap_ok": "The system has {total} of swap!", "diagnosis_mail_ougoing_port_25_ok": "Outgoing port 25 is not blocked and email can be sent to other servers.", "diagnosis_mail_ougoing_port_25_blocked": "Outgoing port 25 appears to be blocked. You should try to unblock it in your internet service provider (or hosting provider) configuration panel. Meanwhile, the server won't be able to send emails to other servers.", - "diagnosis_mail_blacklist_ok": "Your server public IP are not listed on email blacklist.", - "diagnosis_mail_blacklist_nok": "Your server public IPs are listed on email blacklist.", - "diagnosis_mail_blacklisted_by": "{ip} is listed on {blacklist}. Reason: {reason}", + "diagnosis_mail_blacklist_ok": "Your server public IP are not listed on email blacklists.", + "diagnosis_mail_blacklist_nok": "Your server public IPs are listed on email blacklists.", + "diagnosis_mail_blacklisted_by": "{ip} is listed on {blacklist.name}. Reason: {reason}. See {blacklist.website}", "diagnosis_regenconf_allgood": "All configurations files are in line with the recommended configuration!", "diagnosis_regenconf_manually_modified": "Configuration file {file} appears to have been manually modified.", "diagnosis_regenconf_manually_modified_details": "This is probably OK if you know what you're doing! YunoHost will stop updating this file automatically... But beware that YunoHost upgrades could contain important recommended changes. If you want to, you can inspect the differences with yunohost tools regen-conf {category} --dry-run --with-diff and force the reset to the recommended configuration with yunohost tools regen-conf {category} --force",