[enh] Improve DNSBL check

data/hooks/diagnosis/24-mail.py

@@ -4,38 +4,11 @@ import os
 import dns.resolver
 
 from moulinette.utils.network import download_text
+from moulinette.utils.filesystem import read_yaml
 
 from yunohost.diagnosis import Diagnoser
 
-DEFAULT_BLACKLIST = [
+DEFAULT_DNS_BLACKLIST = "/usr/share/yunohost/other/dnsbl_list.yml"
-    ('zen.spamhaus.org'             , 'Spamhaus SBL, XBL and PBL'        ),
-    ('dnsbl.sorbs.net'              , 'SORBS aggregated'                 ),
-    ('safe.dnsbl.sorbs.net'         , "'safe' subset of SORBS aggregated"),
-    ('ix.dnsbl.manitu.net'          , 'Heise iX NiX Spam'                ),
-    ('babl.rbl.webiron.net'         , 'Bad Abuse'                        ),
-    ('cabl.rbl.webiron.net'         , 'Chronicly Bad Abuse'              ),
-    ('truncate.gbudb.net'           , 'Exclusively Spam/Malware'         ),
-    ('dnsbl-1.uceprotect.net'       , 'Trapserver Cluster'               ),
-    ('cbl.abuseat.org'              , 'Net of traps'                     ),
-    ('dnsbl.cobion.com'             , 'used in IBM products'             ),
-    ('psbl.surriel.com'             , 'passive list, easy to unlist'     ),
-    ('dnsrbl.org'                   , 'Real-time black list'             ),
-    ('db.wpbl.info'                 , 'Weighted private'                 ),
-    ('bl.spamcop.net'               , 'Based on spamcop users'           ),
-    ('dyna.spamrats.com'            , 'Dynamic IP addresses'             ),
-    ('spam.spamrats.com'            , 'Manual submissions'               ),
-    ('auth.spamrats.com'            , 'Suspicious authentications'       ),
-    ('dnsbl.inps.de'                , 'automated and reported'           ),
-    ('bl.blocklist.de'              , 'fail2ban reports etc.'            ),
-    ('srnblack.surgate.net'         , 'feeders'                          ),
-    ('all.s5h.net'                  , 'traps'                            ),
-    ('rbl.realtimeblacklist.com'    , 'lists ip ranges'                  ),
-    ('b.barracudacentral.org'       , 'traps'                            ),
-    ('hostkarma.junkemailfilter.com', 'Autotected Virus Senders'         ),
-    ('rbl.megarbl.net'              , 'Curated Spamtraps'                ),
-    ('ubl.unsubscore.com'           , 'Collected Opt-Out Addresses'      ),
-    ('0spam.fusionzero.com'         , 'Spam Trap'                        ),
-]
 
 
 class MailDiagnoser(Diagnoser):
@@ -57,17 +30,13 @@ class MailDiagnoser(Diagnoser):
                        status="ERROR",
                        summary="diagnosis_mail_ougoing_port_25_blocked")
 
-        # Is Reverse DNS well configured ?
+        # Forward-confirmed reverse DNS (FCrDNS) verification
 
 
-        # Are IPs blacklisted ?
+        # Are IPs listed on a DNSBL ?
-        self.logger_debug("Running RBL detection")
+        self.logger_debug("Running DNSBL detection")
-        ipv4 = Diagnoser.get_cached_report_item("ip", {"test": "ipv4"})
+
-        global_ipv4 = ipv4.get("data", {}).get("global", {})
+        blacklisted_details = self.check_ip_dnsbl()
-        ipv6 = Diagnoser.get_cached_report_item("ip", {"test": "ipv6"})
-        global_ipv6 = ipv6.get("data", {}).get("global", {})
-        blacklisted_details = tuple(self.check_blacklisted(global_ipv4))
-        blacklisted_details += tuple(self.check_blacklisted(global_ipv6))
         if blacklisted_details:
             yield dict(meta={"test": "mail_blacklist"},
                        status="ERROR",
@@ -88,18 +57,23 @@ class MailDiagnoser(Diagnoser):
 
         # check for unusual failed sending attempt being refused in the logs ?
 
-    def check_blacklisted(self, ip):
+    def check_blacklisted(self):
         """ Check with dig onto blacklist DNS server
         """
-        if ip is None:
+        dns_blacklists = read_yaml(DEFAULT_DNS_BLACKLIST)
-            return
+        for ip in self.get_public_ips():
+            for blacklist in dns_blacklists:
                 
-        for blacklist, description in DEFAULT_BLACKLIST:
+                if "." in ip and not blacklist.ipv4:
+                    continue
+
+                if ":" in ip and not blacklist.ipv6:
+                    continue
                 
                 # Determine if we are listed on this RBL
                 try:
                     rev = dns.reversename.from_address(ip)
-                query = str(rev.split(3)[0]) + '.' + blacklist
+                    query = str(rev.split(3)[0]) + '.' + blacklist.dns_server
                     # TODO add timeout lifetime
                     dns.resolver.query(query, "A")
                 except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers, dns.resolver.NoAnswer,
@@ -113,20 +87,24 @@ class MailDiagnoser(Diagnoser):
                 except Exception:
                     pass
 
-            yield ('diagnosis_mail_blacklisted_by',
+                yield ('diagnosis_mail_blacklisted_by', {
-                   {'ip': ip, 'blacklist': blacklist, 'reason': reason})
+                    'ip': ip,
+                    'blacklist': blacklist,
+                    'reason': reason})
 
-    def get_public_ip(self, protocol=4):
+    def get_public_ips(self):
-        # TODO we might call this function from another side
+        # Todo code a better way to access a data
-        assert protocol in [4, 6], "Invalid protocol version, it should be either 4 or 6 and was '%s'" % repr(protocol)
+        ipv4 = Diagnoser.get_cached_report("ip", {"test": "ipv4"})
+        if ipv4:
+            global_ipv4 = ipv4.get("data", {}).get("global", {})
+            if global_ipv4:
+                yield global_ipv4
         
-        url = 'https://ip%s.yunohost.org' % ('6' if protocol == 6 else '')
+        ipv6 = Diagnoser.get_cached_report("ip", {"test": "ipv6"})
-
+        if ipv6:
-        try:
+            global_ipv6 = ipv6.get("data", {}).get("global", {})
-            return download_text(url, timeout=30).strip()
+            if global_ipv6:
-        except Exception as e:
+                yield global_ipv6
-            self.logger_debug("Could not get public IPv%s : %s" % (str(protocol), str(e)))
-            return None
 
 
 def main(args, env, loggers):

data/other/dnsbl_list.yml

@@ -0,0 +1,184 @@
+# Used by GAFAM
+- name: Spamhaus ZEN
+  dns_server: zen.spamhaus.org
+  website: https://www.spamhaus.org/zen/
+  ipv4: true
+  ipv6: true
+  domain: false
+- name: Barracuda Reputation Block List
+  dns_server: b.barracudacentral.org
+  website: https://barracudacentral.org/rbl/
+  ipv4: true
+  ipv6: false
+  domain: false
+- name: Hostkarma
+  dns_server: hostkarma.junkemailfilter.com
+  website: https://ipadmin.junkemailfilter.com/remove.php
+  ipv4: true
+  ipv6: false
+  domain: false
+- name: ImproWare IP based spamlist
+  dns_server: spamrbl.imp.ch
+  website: https://antispam.imp.ch/
+  ipv4: true
+  ipv6: false
+  domain: false
+- name: ImproWare IP based wormlist
+  dns_server: wormrbl.imp.ch
+  website: https://antispam.imp.ch/
+  ipv4: true
+  ipv6: false
+  domain: false
+- name: Backscatterer.org 
+  dns_server: ips.backscatterer.org 
+  website: http://www.backscatterer.org/ 
+  ipv4: true
+  ipv6: false
+  domain: false
+- name: inps.de
+  dns_server: dnsbl.inps.de
+  website: http://dnsbl.inps.de/ 
+  ipv4: true
+  ipv6: false
+  domain: false
+- name: LASHBACK 
+  dns_server: ubl.unsubscore.com
+  website: https://blacklist.lashback.com/
+  ipv4: true
+  ipv6: false
+  domain: false
+- name: Mailspike.org
+  dns_server: bl.mailspike.net 
+  website: http://www.mailspike.net/
+  ipv4: true
+  ipv6: false
+  domain: false
+- name: NiX Spam 
+  dns_server: ix.dnsbl.manitu.net
+  website: http://www.dnsbl.manitu.net/ 
+  ipv4: true
+  ipv6: false
+  domain: false
+- name: REDHAWK 
+  dns_server: access.redhawk.org
+  website: https://www.redhawk.org/SpamHawk/query.php 
+  ipv4: true
+  ipv6: false
+  domain: false
+- name: SORBS Open SMTP relays 
+  dns_server: smtp.dnsbl.sorbs.net
+  website: http://www.sorbs.net/ 
+  ipv4: true
+  ipv6: false
+  domain: false
+- name: SORBS Spamhost (last 28 days) 
+  dns_server: recent.spam.dnsbl.sorbs.net
+  website: http://www.sorbs.net/ 
+  ipv4: true
+  ipv6: false
+  domain: false
+- name: SORBS Spamhost (last 48 hours) 
+  dns_server: new.spam.dnsbl.sorbs.net 
+  website: http://www.sorbs.net/
+  ipv4: true
+  ipv6: false
+  domain: false
+- name: SpamCop Blocking List 
+  dns_server: bl.spamcop.net 
+  website: https://www.spamcop.net/bl.shtml 
+  ipv4: true
+  ipv6: false
+  domain: false
+- name: Spam Eating Monkey SEM-BACKSCATTER 
+  dns_server: backscatter.spameatingmonkey.net
+  website: https://spameatingmonkey.com/services 
+  ipv4: true
+  ipv6: false
+  domain: false
+- name: Spam Eating Monkey SEM-BLACK
+  dns_server: bl.spameatingmonkey.net
+  website: https://spameatingmonkey.com/services
+  ipv4: true
+  ipv6: false
+  domain: false
+- name: Spam Eating Monkey SEM-IPV6BL 
+  dns_server: bl.ipv6.spameatingmonkey.net 
+  website: https://spameatingmonkey.com/services 
+  ipv4: false
+  ipv6: true
+  domain: false
+- name: SpamRATS! all 
+  dns_server: all.spamrats.com 
+  website: http://www.spamrats.com/ 
+  ipv4: true
+  ipv6: false
+  domain: false
+- name: PSBL (Passive Spam Block List) 
+  dns_server: psbl.surriel.com
+  website: http://psbl.surriel.com/
+  ipv4: true
+  ipv6: false
+  domain: false
+- name: SWINOG 
+  dns_server: dnsrbl.swinog.ch
+  website: https://antispam.imp.ch/ 
+  ipv4: true
+  ipv6: false
+  domain: false
+- name: GBUdb Truncate
+  dns_server: truncate.gbudb.net
+  website: http://www.gbudb.com/truncate/index.jsp
+  ipv4: true
+  ipv6: false
+  domain: false
+- name: Weighted Private Block List 
+  dns_server: db.wpbl.info 
+  website: http://www.wpbl.info/ 
+  ipv4: true
+  ipv6: false
+  domain: false
+# Used by GAFAM
+- name: Composite Blocking List 
+  dns_server: cbl.abuseat.org
+  website: cbl.abuseat.org 
+  ipv4: true
+  ipv6: false
+  domain: false
+# Used by GAFAM
+- name: SenderScore Blacklist 
+  dns_server: bl.score.senderscore.com
+  website: https://senderscore.com 
+  ipv4: true
+  ipv6: false
+  domain: false
+- name: Invaluement 
+  dns_server: sip.invaluement.com
+  website: https://www.invaluement.com/ 
+  ipv4: true
+  ipv6: false
+  domain: false
+# Added cause it supports IPv6
+- name: AntiCaptcha.NET IPv6
+  dns_server: dnsbl6.anticaptcha.net
+  website: http://anticaptcha.net/ 
+  ipv4: false
+  ipv6: true
+  domain: false
+- name: SPFBL.net RBL 
+  dns_server: dnsbl.spfbl.net
+  website: https://spfbl.net/en/dnsbl/
+  ipv4: true
+  ipv6: true
+  domain: true
+- name: Suomispam Blacklist 
+  dns_server: bl.suomispam.net 
+  website: http://suomispam.net/ 
+  ipv4: true
+  ipv6: true
+  domain: false
+- name: NordSpam 
+  dns_server: bl.nordspam.com
+  website: https://www.nordspam.com/ 
+  ipv4: true
+  ipv6: true
+  domain: false

debian/install

@@ -7,6 +7,7 @@ data/hooks/* /usr/share/yunohost/hooks/
 data/other/yunoprompt.service /etc/systemd/system/
 data/other/password/* /usr/share/yunohost/other/password/
 data/other/dpkg-origins/yunohost /etc/dpkg/origins
+data/other/dnsbl_list.yml /usr/share/yunohost/other/dnsbl_list.yml
 data/other/* /usr/share/yunohost/yunohost-config/moulinette/
 data/templates/* /usr/share/yunohost/templates/
 data/helpers /usr/share/yunohost/

locales/en.json

@@ -186,9 +186,9 @@
     "diagnosis_swap_ok": "The system has {total} of swap!",
     "diagnosis_mail_ougoing_port_25_ok": "Outgoing port 25 is not blocked and email can be sent to other servers.",
     "diagnosis_mail_ougoing_port_25_blocked": "Outgoing port 25 appears to be blocked. You should try to unblock it in your internet service provider (or hosting provider) configuration panel. Meanwhile, the server won't be able to send emails to other servers.",
-    "diagnosis_mail_blacklist_ok": "Your server public IP are not listed on email blacklist.",
+    "diagnosis_mail_blacklist_ok": "Your server public IP are not listed on email blacklists.",
-    "diagnosis_mail_blacklist_nok": "Your server public IPs are listed on email blacklist.",
+    "diagnosis_mail_blacklist_nok": "Your server public IPs are listed on email blacklists.",
-    "diagnosis_mail_blacklisted_by": "{ip} is listed on {blacklist}. Reason: {reason}",
+    "diagnosis_mail_blacklisted_by": "{ip} is listed on {blacklist.name}. Reason: {reason}. See {blacklist.website}",
     "diagnosis_regenconf_allgood": "All configurations files are in line with the recommended configuration!",
     "diagnosis_regenconf_manually_modified": "Configuration file <code>{file}</code> appears to have been manually modified.",
     "diagnosis_regenconf_manually_modified_details": "This is probably OK if you know what you're doing! YunoHost will stop updating this file automatically... But beware that YunoHost upgrades could contain important recommended changes. If you want to, you can inspect the differences with <cmd>yunohost tools regen-conf {category} --dry-run --with-diff</cmd> and force the reset to the recommended configuration with <cmd>yunohost tools regen-conf {category} --force</cmd>",