mirror of
https://github.com/YunoHost/yunohost.git
synced 2024-09-03 20:06:10 +02:00
Merge remote-tracking branch 'origin/enh-diagnosis-mail' into misc-enh-diagnosis
This commit is contained in:
Alexandre Aubin
commit
ecb984a6b7
4 changed files with 405 additions and 23 deletions
|
|
@ -1,42 +1,221 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
import os
|
||||
import dns.resolver
|
||||
import socket
|
||||
import re
|
||||
|
||||
from subprocess import CalledProcessError
|
||||
|
||||
from moulinette.utils.process import check_output
|
||||
from moulinette.utils.filesystem import read_yaml
|
||||
|
||||
from yunohost.diagnosis import Diagnoser
|
||||
from yunohost.domain import _get_maindomain, domain_list
|
||||
|
||||
DIAGNOSIS_SERVER = "diagnosis.yunohost.org"
|
||||
|
||||
DEFAULT_DNS_BLACKLIST = "/usr/share/yunohost/other/dnsbl_list.yml"
|
||||
|
||||
|
||||
class MailDiagnoser(Diagnoser):
|
||||
|
||||
id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
|
||||
cache_duration = 600
|
||||
cache_duration = 0
|
||||
dependencies = ["ip"]
|
||||
|
||||
def run(self):
|
||||
|
||||
# Is outgoing port 25 filtered somehow ?
|
||||
if os.system('/bin/nc -z -w2 yunohost.org 25') == 0:
|
||||
yield dict(meta={"test": "ougoing_port_25"},
|
||||
status="SUCCESS",
|
||||
summary="diagnosis_mail_ougoing_port_25_ok")
|
||||
else:
|
||||
yield dict(meta={"test": "outgoing_port_25"},
|
||||
self.ehlo_domain = _get_maindomain()
|
||||
self.mail_domains = domain_list()["domains"]
|
||||
self.ipversions, self.ips = self.get_ips_checked()
|
||||
|
||||
# TODO Is a A/AAAA and MX Record ?
|
||||
# TODO Are outgoing public IPs authorized to send mail by SPF ?
|
||||
# TODO Validate DKIM and dmarc ?
|
||||
# TODO check that the recent mail logs are not filled with thousand of email sending (unusual number of mail sent)
|
||||
# TODO check for unusual failed sending attempt being refused in the logs ?
|
||||
checks = ["check_outgoing_port_25", "check_ehlo", "check_fcrdns",
|
||||
"check_blacklist", "check_queue"]
|
||||
for check in checks:
|
||||
self.logger_debug("Running " + check)
|
||||
reports = list(getattr(self, check)())
|
||||
for report in reports:
|
||||
yield report
|
||||
if not reports:
|
||||
name = check[6:]
|
||||
yield dict(meta={"test": "mail_" + name},
|
||||
status="SUCCESS",
|
||||
summary="diagnosis_mail_" + name + "_ok")
|
||||
|
||||
|
||||
def check_outgoing_port_25(self):
|
||||
"""
|
||||
Check outgoing port 25 is open and not blocked by router
|
||||
This check is ran on IPs we could used to send mail.
|
||||
"""
|
||||
|
||||
for ipversion in self.ipversions:
|
||||
cmd = '/bin/nc -{ipversion} -z -w2 yunohost.org 25'.format(ipversion=ipversion)
|
||||
if os.system(cmd) != 0:
|
||||
yield dict(meta={"test": "outgoing_port_25", "ipversion": ipversion},
|
||||
data={},
|
||||
status="ERROR",
|
||||
summary="diagnosis_mail_ougoing_port_25_blocked",
|
||||
details=["diagnosis_mail_ougoing_port_25_blocked_details",
|
||||
"diagnosis_mail_outgoing_port_25_blocked_relay_vpn"])
|
||||
|
||||
|
||||
def check_ehlo(self):
|
||||
"""
|
||||
Check the server is reachable from outside and it's the good one
|
||||
This check is ran on IPs we could used to send mail.
|
||||
"""
|
||||
|
||||
for ipversion in self.ipversions:
|
||||
try:
|
||||
r = Diagnoser.remote_diagnosis('check-smtp',
|
||||
data={},
|
||||
ipversion=ipversion)
|
||||
except Exception as e:
|
||||
yield dict(meta={"test": "mail_ehlo", "ipversion": ipversion},
|
||||
data={"error": str(e)},
|
||||
status="WARNING",
|
||||
summary="diagnosis_mail_ehlo_could_not_diagnose",
|
||||
details=["diagnosis_mail_ehlo_could_not_diagnose_details"])
|
||||
continue
|
||||
|
||||
if r["status"] == "error_smtp_unreachable":
|
||||
yield dict(meta={"test": "mail_ehlo", "ipversion": ipversion},
|
||||
data={},
|
||||
status="ERROR",
|
||||
summary="diagnosis_mail_ehlo_unavailable")
|
||||
elif r["helo"] != self.ehlo_domain:
|
||||
yield dict(meta={"test": "mail_ehlo", "ipversion": ipversion},
|
||||
data={"wrong_ehlo": r["helo"], "right_ehlo": self.ehlo_domain},
|
||||
status="ERROR",
|
||||
summary="diagnosis_mail_ehlo_wrong")
|
||||
|
||||
|
||||
def check_fcrdns(self):
|
||||
"""
|
||||
Check the reverse DNS is well defined by doing a Forward-confirmed
|
||||
reverse DNS check
|
||||
This check is ran on IPs we could used to send mail.
|
||||
"""
|
||||
|
||||
for ip in self.ips:
|
||||
try:
|
||||
rdns_domain, _, _ = socket.gethostbyaddr(ip)
|
||||
except socket.herror:
|
||||
yield dict(meta={"test": "mail_fcrdns", "ip": ip},
|
||||
data={"ehlo_domain": self.ehlo_domain},
|
||||
status="ERROR",
|
||||
summary="diagnosis_mail_fcrdns_dns_missing")
|
||||
continue
|
||||
if rdns_domain != self.ehlo_domain:
|
||||
yield dict(meta={"test": "mail_fcrdns", "ip": ip},
|
||||
data={"ehlo_domain": self.ehlo_domain,
|
||||
"rdns_domain": rdns_domain},
|
||||
status="ERROR",
|
||||
summary="diagnosis_mail_fcrdns_different_from_ehlo_domain")
|
||||
|
||||
|
||||
def check_blacklist(self):
|
||||
"""
|
||||
Check with dig onto blacklist DNS server
|
||||
This check is ran on IPs and domains we could used to send mail.
|
||||
"""
|
||||
|
||||
dns_blacklists = read_yaml(DEFAULT_DNS_BLACKLIST)
|
||||
for item in self.ips + self.mail_domains:
|
||||
for blacklist in dns_blacklists:
|
||||
item_type = "domain"
|
||||
if ":" in item:
|
||||
item_type = 'ipv6'
|
||||
elif re.match(r'^\d+\.\d+\.\d+\.\d+$', item):
|
||||
item_type = 'ipv4'
|
||||
|
||||
if not blacklist[item_type]:
|
||||
continue
|
||||
|
||||
# Determine if we are listed on this RBL
|
||||
try:
|
||||
subdomain = item
|
||||
if item_type != "domain":
|
||||
rev = dns.reversename.from_address(item)
|
||||
subdomain = str(rev.split(3)[0])
|
||||
query = subdomain + '.' + blacklist['dns_server']
|
||||
# TODO add timeout lifetime
|
||||
dns.resolver.query(query, "A")
|
||||
except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers, dns.resolver.NoAnswer,
|
||||
dns.exception.Timeout):
|
||||
continue
|
||||
|
||||
# Try to get the reason
|
||||
details = []
|
||||
try:
|
||||
reason = str(dns.resolver.query(query, "TXT")[0])
|
||||
details.append("diagnosis_mail_blacklist_reason")
|
||||
except Exception:
|
||||
reason = "-"
|
||||
|
||||
details.append("diagnosis_mail_blacklist_website")
|
||||
|
||||
yield dict(meta={"test": "mail_blacklist", "item": item,
|
||||
"blacklist": blacklist["dns_server"]},
|
||||
data={'blacklist_name': blacklist['name'],
|
||||
'blacklist_website': blacklist['website'],
|
||||
'reason': reason},
|
||||
status="ERROR",
|
||||
summary='diagnosis_mail_blacklist_listed_by',
|
||||
details=details)
|
||||
|
||||
def check_queue(self):
|
||||
"""
|
||||
Check mail queue is not filled with hundreds of email pending
|
||||
"""
|
||||
|
||||
command = 'postqueue -p | grep -v "Mail queue is empty" | grep -c "^[A-Z0-9]" || true'
|
||||
try:
|
||||
output = check_output(command).strip()
|
||||
pending_emails = int(output)
|
||||
except (ValueError, CalledProcessError) as e:
|
||||
yield dict(meta={"test": "mail_queue"},
|
||||
data={"error": str(e)},
|
||||
status="ERROR",
|
||||
summary="diagnosis_mail_ougoing_port_25_blocked")
|
||||
summary="diagnosis_mail_queue_unavailable",
|
||||
details="diagnosis_mail_queue_unavailable_details")
|
||||
else:
|
||||
if pending_emails > 100:
|
||||
yield dict(meta={"test": "mail_queue"},
|
||||
data={'nb_pending': pending_emails},
|
||||
status="WARNING",
|
||||
summary="diagnosis_mail_queue_too_many_pending_emails")
|
||||
else:
|
||||
yield dict(meta={"test": "mail_queue"},
|
||||
data={'nb_pending': pending_emails},
|
||||
status="SUCCESS",
|
||||
summary="diagnosis_mail_queue_ok")
|
||||
|
||||
|
||||
def get_ips_checked(self):
|
||||
outgoing_ipversions = []
|
||||
outgoing_ips = []
|
||||
ipv4 = Diagnoser.get_cached_report("ip", {"test": "ipv4"}) or {}
|
||||
if ipv4.get("status") == "SUCCESS":
|
||||
outgoing_ipversions.append(4)
|
||||
global_ipv4 = ipv4.get("data", {}).get("global", {})
|
||||
if global_ipv4:
|
||||
outgoing_ips.append(global_ipv4)
|
||||
|
||||
# Mail blacklist using dig requests (c.f. ljf's code)
|
||||
|
||||
# SMTP reachability (c.f. check-smtp to be implemented on yunohost's remote diagnoser)
|
||||
|
||||
# ideally, SPF / DMARC / DKIM validation ... (c.f. https://github.com/alexAubin/yunoScripts/blob/master/yunoDKIM.py possibly though that looks horrible)
|
||||
|
||||
# check that the mail queue is not filled with hundreds of email pending
|
||||
|
||||
# check that the recent mail logs are not filled with thousand of email sending (unusual number of mail sent)
|
||||
|
||||
# check for unusual failed sending attempt being refused in the logs ?
|
||||
|
||||
ipv6 = Diagnoser.get_cached_report("ip", {"test": "ipv6"}) or {}
|
||||
if ipv6.get("status") == "SUCCESS":
|
||||
outgoing_ipversions.append(6)
|
||||
global_ipv6 = ipv6.get("data", {}).get("global", {})
|
||||
if global_ipv6:
|
||||
outgoing_ips.append(global_ipv6)
|
||||
return (outgoing_ipversions, outgoing_ips)
|
||||
|
||||
def main(args, env, loggers):
|
||||
return MailDiagnoser(args, env, loggers).diagnose()
|
||||
|
|
|
|||
184
data/other/dnsbl_list.yml
Normal file
184
data/other/dnsbl_list.yml
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
# Used by GAFAM
|
||||
- name: Spamhaus ZEN
|
||||
dns_server: zen.spamhaus.org
|
||||
website: https://www.spamhaus.org/zen/
|
||||
ipv4: true
|
||||
ipv6: true
|
||||
domain: false
|
||||
- name: Barracuda Reputation Block List
|
||||
dns_server: b.barracudacentral.org
|
||||
website: https://barracudacentral.org/rbl/
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
- name: Hostkarma
|
||||
dns_server: hostkarma.junkemailfilter.com
|
||||
website: https://ipadmin.junkemailfilter.com/remove.php
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
- name: ImproWare IP based spamlist
|
||||
dns_server: spamrbl.imp.ch
|
||||
website: https://antispam.imp.ch/
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
- name: ImproWare IP based wormlist
|
||||
dns_server: wormrbl.imp.ch
|
||||
website: https://antispam.imp.ch/
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
- name: Backscatterer.org
|
||||
dns_server: ips.backscatterer.org
|
||||
website: http://www.backscatterer.org/
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
- name: inps.de
|
||||
dns_server: dnsbl.inps.de
|
||||
website: http://dnsbl.inps.de/
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
- name: LASHBACK
|
||||
dns_server: ubl.unsubscore.com
|
||||
website: https://blacklist.lashback.com/
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
- name: Mailspike.org
|
||||
dns_server: bl.mailspike.net
|
||||
website: http://www.mailspike.net/
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
- name: NiX Spam
|
||||
dns_server: ix.dnsbl.manitu.net
|
||||
website: http://www.dnsbl.manitu.net/
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
- name: REDHAWK
|
||||
dns_server: access.redhawk.org
|
||||
website: https://www.redhawk.org/SpamHawk/query.php
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
- name: SORBS Open SMTP relays
|
||||
dns_server: smtp.dnsbl.sorbs.net
|
||||
website: http://www.sorbs.net/
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
- name: SORBS Spamhost (last 28 days)
|
||||
dns_server: recent.spam.dnsbl.sorbs.net
|
||||
website: http://www.sorbs.net/
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
- name: SORBS Spamhost (last 48 hours)
|
||||
dns_server: new.spam.dnsbl.sorbs.net
|
||||
website: http://www.sorbs.net/
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
- name: SpamCop Blocking List
|
||||
dns_server: bl.spamcop.net
|
||||
website: https://www.spamcop.net/bl.shtml
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
- name: Spam Eating Monkey SEM-BACKSCATTER
|
||||
dns_server: backscatter.spameatingmonkey.net
|
||||
website: https://spameatingmonkey.com/services
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
- name: Spam Eating Monkey SEM-BLACK
|
||||
dns_server: bl.spameatingmonkey.net
|
||||
website: https://spameatingmonkey.com/services
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
- name: Spam Eating Monkey SEM-IPV6BL
|
||||
dns_server: bl.ipv6.spameatingmonkey.net
|
||||
website: https://spameatingmonkey.com/services
|
||||
ipv4: false
|
||||
ipv6: true
|
||||
domain: false
|
||||
- name: SpamRATS! all
|
||||
dns_server: all.spamrats.com
|
||||
website: http://www.spamrats.com/
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
- name: PSBL (Passive Spam Block List)
|
||||
dns_server: psbl.surriel.com
|
||||
website: http://psbl.surriel.com/
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
- name: SWINOG
|
||||
dns_server: dnsrbl.swinog.ch
|
||||
website: https://antispam.imp.ch/
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
- name: GBUdb Truncate
|
||||
dns_server: truncate.gbudb.net
|
||||
website: http://www.gbudb.com/truncate/index.jsp
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
- name: Weighted Private Block List
|
||||
dns_server: db.wpbl.info
|
||||
website: http://www.wpbl.info/
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
# Used by GAFAM
|
||||
- name: Composite Blocking List
|
||||
dns_server: cbl.abuseat.org
|
||||
website: cbl.abuseat.org
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
# Used by GAFAM
|
||||
- name: SenderScore Blacklist
|
||||
dns_server: bl.score.senderscore.com
|
||||
website: https://senderscore.com
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
- name: Invaluement
|
||||
dns_server: sip.invaluement.com
|
||||
website: https://www.invaluement.com/
|
||||
ipv4: true
|
||||
ipv6: false
|
||||
domain: false
|
||||
# Added cause it supports IPv6
|
||||
- name: AntiCaptcha.NET IPv6
|
||||
dns_server: dnsbl6.anticaptcha.net
|
||||
website: http://anticaptcha.net/
|
||||
ipv4: false
|
||||
ipv6: true
|
||||
domain: false
|
||||
- name: SPFBL.net RBL
|
||||
dns_server: dnsbl.spfbl.net
|
||||
website: https://spfbl.net/en/dnsbl/
|
||||
ipv4: true
|
||||
ipv6: true
|
||||
domain: true
|
||||
- name: Suomispam Blacklist
|
||||
dns_server: bl.suomispam.net
|
||||
website: http://suomispam.net/
|
||||
ipv4: true
|
||||
ipv6: true
|
||||
domain: false
|
||||
- name: NordSpam
|
||||
dns_server: bl.nordspam.com
|
||||
website: https://www.nordspam.com/
|
||||
ipv4: true
|
||||
ipv6: true
|
||||
domain: false
|
||||
1
debian/install
vendored
1
debian/install
vendored
|
|
@ -7,6 +7,7 @@ data/hooks/* /usr/share/yunohost/hooks/
|
|||
data/other/yunoprompt.service /etc/systemd/system/
|
||||
data/other/password/* /usr/share/yunohost/other/password/
|
||||
data/other/dpkg-origins/yunohost /etc/dpkg/origins
|
||||
data/other/dnsbl_list.yml /usr/share/yunohost/other/dnsbl_list.yml
|
||||
data/other/* /usr/share/yunohost/yunohost-config/moulinette/
|
||||
data/templates/* /usr/share/yunohost/templates/
|
||||
data/helpers /usr/share/yunohost/
|
||||
|
|
|
|||
|
|
@ -184,8 +184,26 @@
|
|||
"diagnosis_swap_none": "The system has no swap at all. You should consider adding at least 256 MB of swap to avoid situations where the system runs out of memory.",
|
||||
"diagnosis_swap_notsomuch": "The system has only {total} swap. You should consider having at least 256 MB to avoid situations where the system runs out of memory.",
|
||||
"diagnosis_swap_ok": "The system has {total} of swap!",
|
||||
"diagnosis_mail_ougoing_port_25_ok": "Outgoing port 25 is not blocked and email can be sent to other servers.",
|
||||
"diagnosis_mail_ougoing_port_25_blocked": "Outgoing port 25 appears to be blocked. You should try to unblock it in your internet service provider (or hosting provider) configuration panel. Meanwhile, the server won't be able to send emails to other servers.",
|
||||
"diagnosis_mail_outgoing_port_25_ok": "Outgoing port 25 is open, emails can be sent",
|
||||
"diagnosis_mail_outgoing_port_25_blocked": "Outgoing port 25 appears to be bloecked in IPv{ipversion}",
|
||||
"diagnosis_mail_outgoing_port_25_blocked_details": "You should first try to unblock it in your internet service provider (or hosting provider) configuration panel or by sending a ticket to your hosting provider. Meanwhile, the server won't be able to send emails to other servers.",
|
||||
"diagnosis_mail_outgoing_port_25_blocked_relay_vpn": "Some providers won't let you unblock outgoing port 25 because they don't care about Net Neutrality.<br> - Some of them provide the alternative of <a href='https://yunohost.org/#/smtp_relay'>using a mail server relay</a> though it implies that the relay will be able to spy on your email traffic.<br>- A privacy-friendly alternative is to use a VPN *with a dedicated public IP* to bypass this kind of limits. See <a href='https://yunohost.org/#/vpn_advantage'>https://yunohost.org/#/vpn_advantage</a><br>- Finally, it's also possible to <a href='https://yunohost.org/#/isp'>change of provider</a>",
|
||||
"diagnosis_mail_ehlo_ok": "Postfix mail service answer correctly from outside",
|
||||
"diagnosis_mail_ehlo_unavailable": "Postfix mail service don't answer to EHLO request on IPv{ipversion}",
|
||||
"diagnosis_mail_ehlo_wrong": "A mail server answers {wrong_ehlo} instead {right_ehlo} on IPv{ipversion}",
|
||||
"diagnosis_mail_ehlo_could_not_diagnose": "Could not diagnose if postfix mail server is reachable from outside",
|
||||
"diagnosis_mail_ehlo_could_not_diagnose_details": "Error: {error}",
|
||||
"diagnosis_mail_fcrdns_ok": "Your reverse DNS is well configured",
|
||||
"diagnosis_mail_fcrdns_dns_missing": "No reverse DNS defined for the ip {ip}",
|
||||
"diagnosis_mail_fcrdns_different_from_ehlo_domain": "Your reverse DNS {rdns_domain} is different from your EHLO domain {ehlo_domain} on {ip}",
|
||||
"diagnosis_mail_blacklist_ok": "IPs and domains used by this server to send mail are not on most used email blacklists",
|
||||
"diagnosis_mail_blacklist_listed_by": "{item} is blacklisted on {blacklist_name}",
|
||||
"diagnosis_mail_blacklist_reason": "The blacklist explains: {reason}",
|
||||
"diagnosis_mail_blacklist_website": "After identifying why you are listed and fixed it, feel free to ask for delisting on {blacklist_website}",
|
||||
"diagnosis_mail_queue_ok": "{nb_pending} pending emails in the mail queues",
|
||||
"diagnosis_mail_queue_unavailable": "Can not consult number of pending emails in queue",
|
||||
"diagnosis_mail_queue_unavailable_details": "Error: {error}",
|
||||
"diagnosis_mail_queue_too_big": "Too many pending emails in mail queue ({nb_pending} emails)",
|
||||
"diagnosis_regenconf_allgood": "All configurations files are in line with the recommended configuration!",
|
||||
"diagnosis_regenconf_manually_modified": "Configuration file <code>{file}</code> appears to have been manually modified.",
|
||||
"diagnosis_regenconf_manually_modified_details": "This is probably OK if you know what you're doing! YunoHost will stop updating this file automatically... But beware that YunoHost upgrades could contain important recommended changes. If you want to, you can inspect the differences with <cmd>yunohost tools regen-conf {category} --dry-run --with-diff</cmd> and force the reset to the recommended configuration with <cmd>yunohost tools regen-conf {category} --force</cmd>",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue