Merge pull request #534 from YunoHost/new-diagnosis-system

[enh] New diagnosis system

data/actionsmap/yunohost.yml

@@ -1469,16 +1469,6 @@ tools:
                     help: Upgrade only the system packages
                     action: store_true
 
-        ### tools_diagnosis()
-        diagnosis:
-            action_help: YunoHost diagnosis
-            api: GET /diagnosis
-            arguments:
-                -p:
-                    full: --private
-                    help: Show private data (domain, IP)
-                    action: store_true
-
         ### tools_port_available()
         port-available:
             action_help: Check availability of a local port
@@ -1724,3 +1714,59 @@ log:
                 --share:
                     help: Share the full log using yunopaste
                     action: store_true
+
+
+#############################
+#          Diagnosis        #
+#############################
+diagnosis:
+    category_help: Look for possible issues on the server
+    actions:
+
+        list:
+            action_help: List diagnosis categories
+            api: GET /diagnosis/list
+
+        show:
+            action_help: Show most recents diagnosis results
+            api: GET /diagnosis/show
+            arguments:
+                categories:
+                    help: Diagnosis categories to display (all by default)
+                    nargs: "*"
+                --full:
+                    help: Display additional information
+                    action: store_true
+                --issues:
+                    help: Only display issues
+                    action: store_true
+                --share:
+                    help: Share the logs using yunopaste
+                    action: store_true
+
+        run:
+            action_help: Show most recents diagnosis results
+            api: POST /diagnosis/run
+            arguments:
+                categories:
+                    help: Diagnosis categories to run (all by default)
+                    nargs: "*"
+                --force:
+                    help: Ignore the cached report even if it is still 'fresh'
+                    action: store_true
+
+        ignore:
+            action_help: Configure some diagnosis results to be ignored and therefore not considered as actual issues
+            api: POST /diagnosis/ignore
+            arguments:
+                --add-filter:
+                    help: "Add a filter. The first element should be a diagnosis category, and other criterias can be provided using the infos from the 'meta' sections in 'yunohost diagnosis show'. For example: 'dnsrecords domain=yolo.test category=xmpp'"
+                    nargs: "*"
+                    metavar: CRITERIA
+                --remove-filter:
+                    help: Remove a filter (it should be an existing filter as listed with --list)
+                    nargs: "*"
+                    metavar: CRITERIA
+                --list:
+                    help: List active ignore filters
+                    action: store_true

data/hooks/diagnosis/00-basesystem.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+
+import os
+
+from moulinette.utils.filesystem import read_file
+from yunohost.diagnosis import Diagnoser
+from yunohost.utils.packages import ynh_packages_version
+
+
+class BaseSystemDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 3600 * 24
+    dependencies = []
+
+    def run(self):
+
+        # Kernel version
+        kernel_version = read_file('/proc/sys/kernel/osrelease').strip()
+        yield dict(meta={"test": "kernel"},
+                   status="INFO",
+                   summary=("diagnosis_basesystem_kernel", {"kernel_version": kernel_version}))
+
+        # FIXME / TODO : add virt/vm technology using systemd-detect-virt and/or machine arch
+
+        # Debian release
+        debian_version = read_file("/etc/debian_version").strip()
+        yield dict(meta={"test": "host"},
+                   status="INFO",
+                   summary=("diagnosis_basesystem_host", {"debian_version": debian_version}))
+
+        # Yunohost packages versions
+        ynh_packages = ynh_packages_version()
+        # We check if versions are consistent (e.g. all 3.6 and not 3 packages with 3.6 and the other with 3.5)
+        # This is a classical issue for upgrades that failed in the middle
+        # (or people upgrading half of the package because they did 'apt upgrade' instead of 'dist-upgrade')
+        # Here, ynh_core_version is for example "3.5.4.12", so [:3] is "3.5" and we check it's the same for all packages
+        ynh_core_version = ynh_packages["yunohost"]["version"]
+        consistent_versions = all(infos["version"][:3] == ynh_core_version[:3] for infos in ynh_packages.values())
+        ynh_version_details = [("diagnosis_basesystem_ynh_single_version", (package, infos["version"]))
+                               for package, infos in ynh_packages.items()]
+
+        if consistent_versions:
+            yield dict(meta={"test": "ynh_versions"},
+                       status="INFO",
+                       summary=("diagnosis_basesystem_ynh_main_version", {"main_version": ynh_core_version[:3]}),
+                       details=ynh_version_details)
+        else:
+            yield dict(meta={"test": "ynh_versions"},
+                       status="ERROR",
+                       summary=("diagnosis_basesystem_ynh_inconsistent_versions", {}),
+                       details=ynh_version_details)
+
+
+def main(args, env, loggers):
+    return BaseSystemDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/10-ip.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python
+
+import os
+import random
+
+from moulinette.utils.network import download_text
+from moulinette.utils.process import check_output
+from moulinette.utils.filesystem import read_file
+
+from yunohost.diagnosis import Diagnoser
+
+
+class IPDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 60
+    dependencies = []
+
+    def run(self):
+
+        # ############################################################ #
+        # PING : Check that we can ping outside at least in ipv4 or v6 #
+        # ############################################################ #
+
+        can_ping_ipv4 = self.can_ping_outside(4)
+        can_ping_ipv6 = self.can_ping_outside(6)
+
+        if not can_ping_ipv4 and not can_ping_ipv6:
+            yield dict(meta={"test": "ping"},
+                       status="ERROR",
+                       summary=("diagnosis_ip_not_connected_at_all", {}))
+            # Not much else we can do if there's no internet at all
+            return
+
+        # ###################################################### #
+        # DNS RESOLUTION : Check that we can resolve domain name #
+        # (later needed to talk to ip. and ip6.yunohost.org)     #
+        # ###################################################### #
+
+        can_resolve_dns = self.can_resolve_dns()
+
+        # In every case, we can check that resolvconf seems to be okay
+        # (symlink managed by resolvconf service + pointing to dnsmasq)
+        good_resolvconf = self.resolvconf_is_symlink() and self.resolvconf_points_to_localhost()
+
+        # If we can't resolve domain names at all, that's a pretty big issue ...
+        # If it turns out that at the same time, resolvconf is bad, that's probably
+        # the cause of this, so we use a different message in that case
+        if not can_resolve_dns:
+            yield dict(meta={"test": "dnsresolv"},
+                       status="ERROR",
+                       summary=("diagnosis_ip_broken_dnsresolution", {}) if good_resolvconf
+                          else ("diagnosis_ip_broken_resolvconf", {}))
+            return
+        # Otherwise, if the resolv conf is bad but we were able to resolve domain name,
+        # still warn that we're using a weird resolv conf ...
+        elif not good_resolvconf:
+            yield dict(meta={"test": "dnsresolv"},
+                       status="WARNING",
+                       summary=("diagnosis_ip_weird_resolvconf", {}),
+                       details=[("diagnosis_ip_weird_resolvconf_details", ())])
+        else:
+            yield dict(meta={"test": "dnsresolv"},
+                       status="SUCCESS",
+                       summary=("diagnosis_ip_dnsresolution_working", {}))
+
+        # ##################################################### #
+        # IP DIAGNOSIS : Check that we're actually able to talk #
+        # to a web server to fetch current IPv4 and v6          #
+        # ##################################################### #
+
+        ipv4 = self.get_public_ip(4) if can_ping_ipv4 else None
+        ipv6 = self.get_public_ip(6) if can_ping_ipv6 else None
+
+        yield dict(meta={"test": "ip", "version": 4},
+                   data=ipv4,
+                   status="SUCCESS" if ipv4 else "ERROR",
+                   summary=("diagnosis_ip_connected_ipv4", {}) if ipv4
+                      else ("diagnosis_ip_no_ipv4", {}))
+
+        yield dict(meta={"test": "ip", "version": 6},
+                   data=ipv6,
+                   status="SUCCESS" if ipv6 else "WARNING",
+                   summary=("diagnosis_ip_connected_ipv6", {}) if ipv6
+                      else ("diagnosis_ip_no_ipv6", {}))
+
+        # TODO / FIXME : add some attempt to detect ISP (using whois ?) ?
+
+    def can_ping_outside(self, protocol=4):
+
+        assert protocol in [4, 6], "Invalid protocol version, it should be either 4 or 6 and was '%s'" % repr(protocol)
+
+        # We can know that ipv6 is not available directly if this file does not exists
+        if protocol == 6 and not os.path.exists("/proc/net/if_inet6"):
+            return False
+
+        # If we are indeed connected in ipv4 or ipv6, we should find a default route
+        routes = check_output("ip -%s route" % protocol).split("\n")
+        if not [r for r in routes if r.startswith("default")]:
+            return False
+
+        # We use the resolver file as a list of well-known, trustable (ie not google ;)) IPs that we can ping
+        resolver_file = "/usr/share/yunohost/templates/dnsmasq/plain/resolv.dnsmasq.conf"
+        resolvers = [r.split(" ")[1] for r in read_file(resolver_file).split("\n") if r.startswith("nameserver")]
+
+        if protocol == 4:
+            resolvers = [r for r in resolvers if ":" not in r]
+        if protocol == 6:
+            resolvers = [r for r in resolvers if ":" in r]
+
+        assert resolvers != [], "Uhoh, need at least one IPv%s DNS resolver in %s ..." % (protocol, resolver_file)
+
+        # So let's try to ping the first 4~5 resolvers (shuffled)
+        # If we succesfully ping any of them, we conclude that we are indeed connected
+        def ping(protocol, target):
+            return os.system("ping%s -c1 -W 3 %s >/dev/null 2>/dev/null" % ("" if protocol == 4 else "6", target)) == 0
+
+        random.shuffle(resolvers)
+        return any(ping(protocol, resolver) for resolver in resolvers[:5])
+
+    def can_resolve_dns(self):
+        return os.system("dig +short ip.yunohost.org >/dev/null 2>/dev/null") == 0
+
+    def resolvconf_is_symlink(self):
+        return os.path.realpath("/etc/resolv.conf") == "/run/resolvconf/resolv.conf"
+
+    def resolvconf_points_to_localhost(self):
+        file_ = "/etc/resolv.conf"
+        resolvers = [r.split(" ")[1] for r in read_file(file_).split("\n") if r.startswith("nameserver")]
+        return resolvers == ["127.0.0.1"]
+
+    def get_public_ip(self, protocol=4):
+
+        # FIXME - TODO : here we assume that DNS resolution for ip.yunohost.org is working
+        # but if we want to be able to diagnose DNS resolution issues independently from
+        # internet connectivity, we gotta rely on fixed IPs first....
+
+        assert protocol in [4, 6], "Invalid protocol version, it should be either 4 or 6 and was '%s'" % repr(protocol)
+
+        url = 'https://ip%s.yunohost.org' % ('6' if protocol == 6 else '')
+
+        try:
+            return download_text(url, timeout=30).strip()
+        except Exception as e:
+            self.logger_debug("Could not get public IPv%s : %s" % (str(protocol), str(e)))
+            return None
+
+
+def main(args, env, loggers):
+    return IPDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/12-dnsrecords.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+
+import os
+
+from moulinette.utils.process import check_output
+from moulinette.utils.filesystem import read_file
+
+from yunohost.diagnosis import Diagnoser
+from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain
+
+
+class DNSRecordsDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 3600 * 24
+    dependencies = ["ip"]
+
+    def run(self):
+
+        resolvers = read_file("/etc/resolv.dnsmasq.conf").split("\n")
+        ipv4_resolvers = [r.split(" ")[1] for r in resolvers if r.startswith("nameserver") and ":" not in r]
+        # FIXME some day ... handle ipv4-only and ipv6-only servers. For now we assume we have at least ipv4
+        assert ipv4_resolvers != [], "Uhoh, need at least one IPv4 DNS resolver ..."
+
+        self.resolver = ipv4_resolvers[0]
+        main_domain = _get_maindomain()
+
+        all_domains = domain_list()["domains"]
+        for domain in all_domains:
+            self.logger_debug("Diagnosing DNS conf for %s" % domain)
+            for report in self.check_domain(domain, domain == main_domain):
+                yield report
+
+        # FIXME : somewhere, should implement a check for reverse DNS ...
+
+        # FIXME / TODO : somewhere, could also implement a check for domain expiring soon
+
+    def check_domain(self, domain, is_main_domain):
+
+        expected_configuration = _build_dns_conf(domain)
+
+        # Here if there are no AAAA record, we should add something to expect "no" AAAA record
+        # to properly diagnose situations where people have a AAAA record but no IPv6
+
+        for category, records in expected_configuration.items():
+
+            discrepancies = []
+
+            for r in records:
+                current_value = self.get_current_record(domain, r["name"], r["type"]) or "None"
+                expected_value = r["value"] if r["value"] != "@" else domain + "."
+
+                if current_value == "None":
+                    discrepancies.append(("diagnosis_dns_missing_record", (r["type"], r["name"], expected_value)))
+                elif current_value != expected_value:
+                    discrepancies.append(("diagnosis_dns_discrepancy", (r["type"], r["name"], expected_value, current_value)))
+
+            if discrepancies:
+                status = "ERROR" if (category == "basic" or (is_main_domain and category != "extra")) else "WARNING"
+                summary = ("diagnosis_dns_bad_conf", {"domain": domain, "category": category})
+            else:
+                status = "SUCCESS"
+                summary = ("diagnosis_dns_good_conf", {"domain": domain, "category": category})
+
+            output = dict(meta={"domain": domain, "category": category},
+                          status=status,
+                          summary=summary)
+
+            if discrepancies:
+                output["details"] = discrepancies
+
+            yield output
+
+    def get_current_record(self, domain, name, type_):
+        if name == "@":
+            command = "dig +short @%s %s %s" % (self.resolver, type_, domain)
+        else:
+            command = "dig +short @%s %s %s.%s" % (self.resolver, type_, name, domain)
+        # FIXME : gotta handle case where this command fails ...
+        # e.g. no internet connectivity (dependency mechanism to good result from 'ip' diagosis ?)
+        # or the resolver is unavailable for some reason
+        output = check_output(command).strip()
+        if output.startswith('"') and output.endswith('"'):
+            output = '"' + ' '.join(output.replace('"', ' ').split()) + '"'
+        return output
+
+
+def main(args, env, loggers):
+    return DNSRecordsDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/14-ports.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+
+import os
+import requests
+
+from yunohost.diagnosis import Diagnoser
+from yunohost.utils.error import YunohostError
+
+
+class PortsDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 3600
+    dependencies = ["ip"]
+
+    def run(self):
+
+        # FIXME / TODO : in the future, maybe we want to report different
+        # things per port depending on how important they are
+        # (e.g. XMPP sounds to me much less important than other ports)
+        # Ideally, a port could be related to a service...
+        # FIXME / TODO : for now this list of port is hardcoded, might want
+        # to fetch this from the firewall.yml in /etc/yunohost/
+        ports = [22, 25, 53, 80, 443, 587, 993, 5222, 5269]
+
+        try:
+            r = requests.post('https://ynhdiagnoser.netlib.re/check-ports', json={'ports': ports}, timeout=30).json()
+            if "status" not in r.keys():
+                raise Exception("Bad syntax for response ? Raw json: %s" % str(r))
+            elif r["status"] == "error":
+                if "content" in r.keys():
+                    raise Exception(r["content"])
+                else:
+                    raise Exception("Bad syntax for response ? Raw json: %s" % str(r))
+            elif r["status"] != "ok" or "ports" not in r.keys() or not isinstance(r["ports"], dict):
+                raise Exception("Bad syntax for response ? Raw json: %s" % str(r))
+        except Exception as e:
+            raise YunohostError("diagnosis_ports_could_not_diagnose", error=e)
+
+        for port in ports:
+            if r["ports"].get(str(port), None) is not True:
+                yield dict(meta={"port": port},
+                           status="ERROR",
+                           summary=("diagnosis_ports_unreachable", {"port": port}))
+            else:
+                yield dict(meta={},
+                           status="SUCCESS",
+                           summary=("diagnosis_ports_ok", {"port": port}))
+
+
+def main(args, env, loggers):
+    return PortsDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/16-http.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+
+import os
+import random
+import requests
+
+from yunohost.diagnosis import Diagnoser
+from yunohost.domain import domain_list
+from yunohost.utils.error import YunohostError
+
+
+class HttpDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 3600
+    dependencies = ["ip"]
+
+    def run(self):
+
+        nonce_digits = "0123456789abcedf"
+
+        all_domains = domain_list()["domains"]
+        for domain in all_domains:
+
+            nonce = ''.join(random.choice(nonce_digits) for i in range(16))
+            os.system("rm -rf /tmp/.well-known/ynh-diagnosis/")
+            os.system("mkdir -p /tmp/.well-known/ynh-diagnosis/")
+            os.system("touch /tmp/.well-known/ynh-diagnosis/%s" % nonce)
+
+            try:
+                r = requests.post('https://ynhdiagnoser.netlib.re/check-http', json={'domain': domain, "nonce": nonce}, timeout=30).json()
+                if "status" not in r.keys():
+                    raise Exception("Bad syntax for response ? Raw json: %s" % str(r))
+                elif r["status"] == "error" and ("code" not in r.keys() or r["code"] not in ["error_http_check_connection_error", "error_http_check_unknown_error"]):
+                    if "content" in r.keys():
+                        raise Exception(r["content"])
+                    else:
+                        raise Exception("Bad syntax for response ? Raw json: %s" % str(r))
+            except Exception as e:
+                raise YunohostError("diagnosis_http_could_not_diagnose", error=e)
+
+            if r["status"] == "ok":
+                yield dict(meta={"domain": domain},
+                           status="SUCCESS",
+                           summary=("diagnosis_http_ok", {"domain": domain}))
+            else:
+                yield dict(meta={"domain": domain},
+                           status="ERROR",
+                           summary=("diagnosis_http_unreachable", {"domain": domain}))
+
+        # In there or idk where else ...
+        # try to diagnose hairpinning situation by crafting a request for the
+        # global ip (from within local network) and seeing if we're getting the right page ?
+
+
+def main(args, env, loggers):
+    return HttpDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/18-mail.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+import os
+
+from yunohost.diagnosis import Diagnoser
+
+
+class MailDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 3600
+    dependencies = ["ip"]
+
+    def run(self):
+
+         # TODO / FIXME TO BE IMPLEMETED in the future ...
+
+        yield dict(meta={},
+                   status="WARNING",
+                   summary=("nothing_implemented_yet", {}))
+
+        # Mail blacklist using dig requests (c.f. ljf's code)
+
+        # Outgoing port 25 (c.f. code in monitor.py, a simple 'nc -zv yunohost.org 25' IIRC)
+
+        # SMTP reachability (c.f. check-smtp to be implemented on yunohost's remote diagnoser)
+
+        # ideally, SPF / DMARC / DKIM validation ... (c.f. https://github.com/alexAubin/yunoScripts/blob/master/yunoDKIM.py possibly though that looks horrible)
+
+        # check that the mail queue is not filled with hundreds of email pending
+
+        # check that the recent mail logs are not filled with thousand of email sending (unusual number of mail sent)
+
+        # check for unusual failed sending attempt being refused in the logs ?
+
+
+def main(args, env, loggers):
+    return MailDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/30-services.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+
+import os
+
+from yunohost.diagnosis import Diagnoser
+from yunohost.service import service_status
+
+# TODO : all these are arbitrary, should be collectively validated
+services_ignored = {"glances"}
+services_critical = {"dnsmasq", "fail2ban", "yunohost-firewall", "nginx", "slapd", "ssh"}
+# TODO / FIXME : we should do something about this postfix thing
+# The nominal value is to be "exited" ... some daemon is actually running
+# in a different thread that the thing started by systemd, which is fine
+# but somehow sometimes it gets killed and there's no easy way to detect it
+# Just randomly restarting it will fix ths issue. We should find some trick
+# to identify the PID of the process and check it's still up or idk
+services_expected_to_be_exited = {"postfix", "yunohost-firewall"}
+
+class ServicesDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 300
+    dependencies = []
+
+    def run(self):
+
+        all_result = service_status()
+
+        for service, result in all_result.items():
+
+            if service in services_ignored:
+                continue
+
+            item = dict(meta={"service": service})
+            expected_status = "running" if service not in services_expected_to_be_exited else "exited"
+
+            # TODO / FIXME : might also want to check that services are enabled
+
+            if result["active"] != "active" or result["status"] != expected_status:
+                item["status"] = "WARNING" if service not in services_critical else "ERROR"
+                item["summary"] = ("diagnosis_services_bad_status", {"service": service, "status": result["active"] + "/" + result["status"]})
+
+                # TODO : could try to append the tail of the service log to the "details" key ...
+            else:
+                item["status"] = "SUCCESS"
+                item["summary"] = ("diagnosis_services_good_status", {"service": service, "status": result["active"] + "/" + result["status"]})
+
+            yield item
+
+def main(args, env, loggers):
+    return ServicesDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/50-systemresources.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+import os
+import psutil
+
+from yunohost.diagnosis import Diagnoser
+
+class SystemResourcesDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 3600 * 24
+    dependencies = []
+
+    def run(self):
+
+        #
+        # RAM
+        #
+
+        ram = psutil.virtual_memory()
+        ram_total_abs_MB = ram.total / (1024**2)
+        ram_available_abs_MB = ram.available / (1024**2)
+        ram_available_percent = round(100 * ram.available / ram.total)
+        item = dict(meta={"test": "ram"})
+        infos = {"total_abs_MB": ram_total_abs_MB, "available_abs_MB": ram_available_abs_MB, "available_percent": ram_available_percent}
+        if ram_available_abs_MB < 100 or ram_available_percent < 5:
+            item["status"] = "ERROR"
+            item["summary"] = ("diagnosis_ram_verylow", infos)
+        elif ram_available_abs_MB < 200 or ram_available_percent < 10:
+            item["status"] = "WARNING"
+            item["summary"] = ("diagnosis_ram_low", infos)
+        else:
+            item["status"] = "SUCCESS"
+            item["summary"] = ("diagnosis_ram_ok", infos)
+        yield item
+
+        #
+        # Swap
+        #
+
+        swap = psutil.swap_memory()
+        swap_total_abs_MB = swap.total / (1024*1024)
+        item = dict(meta={"test": "swap"})
+        infos = {"total_MB": swap_total_abs_MB}
+        if swap_total_abs_MB <= 0:
+            item["status"] = "ERROR"
+            item["summary"] = ("diagnosis_swap_none", infos)
+        elif swap_total_abs_MB <= 256:
+            item["status"] = "WARNING"
+            item["summary"] = ("diagnosis_swap_notsomuch", infos)
+        else:
+            item["status"] = "SUCCESS"
+            item["summary"] = ("diagnosis_swap_ok", infos)
+        yield item
+
+        #
+        # Disks usage
+        #
+
+        disk_partitions = psutil.disk_partitions()
+
+        for disk_partition in disk_partitions:
+            device = disk_partition.device
+            mountpoint = disk_partition.mountpoint
+
+            usage = psutil.disk_usage(mountpoint)
+            free_abs_GB = usage.free / (1024 ** 3)
+            free_percent = 100 - usage.percent
+
+            item = dict(meta={"test": "diskusage", "mountpoint": mountpoint})
+            infos = {"mountpoint": mountpoint, "device": device, "free_abs_GB": free_abs_GB, "free_percent": free_percent}
+            if free_abs_GB < 1 or free_percent < 5:
+                item["status"] = "ERROR"
+                item["summary"] = ("diagnosis_diskusage_verylow", infos)
+            elif free_abs_GB < 2 or free_percent < 10:
+                item["status"] = "WARNING"
+                item["summary"] = ("diagnosis_diskusage_low", infos)
+            else:
+                item["status"] = "SUCCESS"
+                item["summary"] = ("diagnosis_diskusage_ok", infos)
+
+            yield item
+
+
+def main(args, env, loggers):
+    return SystemResourcesDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/70-regenconf.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+
+import os
+
+import subprocess
+from yunohost.diagnosis import Diagnoser
+from yunohost.regenconf import manually_modified_files, manually_modified_files_compared_to_debian_default
+
+
+class RegenconfDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 300
+    dependencies = []
+
+    def run(self):
+
+        # nginx -t
+        p = subprocess.Popen("nginx -t".split(),
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.STDOUT)
+        out, _ = p.communicate()
+
+        if p.returncode != 0:
+            yield dict(meta={"test": "nginx-t"},
+                       status="ERROR",
+                       summary=("diagnosis_regenconf_nginx_conf_broken", {}),
+                       details=[(out, ())]
+                       )
+
+        regenconf_modified_files = manually_modified_files()
+        debian_modified_files = manually_modified_files_compared_to_debian_default(ignore_handled_by_regenconf=True)
+
+        if regenconf_modified_files == []:
+            yield dict(meta={"test": "regenconf"},
+                       status="SUCCESS",
+                       summary=("diagnosis_regenconf_allgood", {})
+                       )
+        else:
+            for f in regenconf_modified_files:
+                yield dict(meta={"test": "regenconf", "file": f},
+                           status="WARNING",
+                           summary=("diagnosis_regenconf_manually_modified", {"file": f}),
+                           details=[("diagnosis_regenconf_manually_modified_details", {})]
+                           )
+
+        for f in debian_modified_files:
+            yield dict(meta={"test": "debian", "file": f},
+                       status="WARNING",
+                       summary=("diagnosis_regenconf_manually_modified_debian", {"file": f}),
+                       details=[("diagnosis_regenconf_manually_modified_debian_details", {})]
+                       )
+
+
+def main(args, env, loggers):
+    return RegenconfDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/90-security.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+
+import os
+import json
+import subprocess
+
+from yunohost.diagnosis import Diagnoser
+from moulinette.utils.filesystem import read_json, write_to_json
+
+
+class SecurityDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 3600
+    dependencies = []
+
+    def run(self):
+
+        "CVE-2017-5754"
+
+        if self.is_vulnerable_to_meltdown():
+            yield dict(meta={"test": "meltdown"},
+                       status="ERROR",
+                       summary=("diagnosis_security_vulnerable_to_meltdown", {}),
+                       details=[("diagnosis_security_vulnerable_to_meltdown_details", ())]
+                       )
+        else:
+            yield dict(meta={},
+                       status="SUCCESS",
+                       summary=("diagnosis_security_all_good", {})
+                       )
+
+
+    def is_vulnerable_to_meltdown(self):
+        # meltdown CVE: https://security-tracker.debian.org/tracker/CVE-2017-5754
+
+        # We use a cache file to avoid re-running the script so many times,
+        # which can be expensive (up to around 5 seconds on ARM)
+        # and make the admin appear to be slow (c.f. the calls to diagnosis
+        # from the webadmin)
+        #
+        # The cache is in /tmp and shall disappear upon reboot
+        # *or* we compare it to dpkg.log modification time
+        # such that it's re-ran if there was package upgrades
+        # (e.g. from yunohost)
+        cache_file = "/tmp/yunohost-meltdown-diagnosis"
+        dpkg_log = "/var/log/dpkg.log"
+        if os.path.exists(cache_file):
+            if not os.path.exists(dpkg_log) or os.path.getmtime(cache_file) > os.path.getmtime(dpkg_log):
+                self.logger_debug("Using cached results for meltdown checker, from %s" % cache_file)
+                return read_json(cache_file)[0]["VULNERABLE"]
+
+        # script taken from https://github.com/speed47/spectre-meltdown-checker
+        # script commit id is store directly in the script
+        SCRIPT_PATH = "/usr/lib/moulinette/yunohost/vendor/spectre-meltdown-checker/spectre-meltdown-checker.sh"
+
+        # '--variant 3' corresponds to Meltdown
+        # example output from the script:
+        # [{"NAME":"MELTDOWN","CVE":"CVE-2017-5754","VULNERABLE":false,"INFOS":"PTI mitigates the vulnerability"}]
+        try:
+            self.logger_debug("Running meltdown vulnerability checker")
+            call = subprocess.Popen("bash %s --batch json --variant 3" %
+                                    SCRIPT_PATH, shell=True,
+                                    stdout=subprocess.PIPE,
+                                    stderr=subprocess.PIPE)
+
+            # TODO / FIXME : here we are ignoring error messages ...
+            # in particular on RPi2 and other hardware, the script complains about
+            # "missing some kernel info (see -v), accuracy might be reduced"
+            # Dunno what to do about that but we probably don't want to harass
+            # users with this warning ...
+            output, err = call.communicate()
+            assert call.returncode in (0, 2, 3), "Return code: %s" % call.returncode
+
+            # If there are multiple lines, sounds like there was some messages
+            # in stdout that are not json >.> ... Try to get the actual json
+            # stuff which should be the last line
+            output = output.strip()
+            if "\n" in output:
+                self.logger_debug("Original meltdown checker output : %s" % output)
+                output = output.split("\n")[-1]
+
+            CVEs = json.loads(output)
+            assert len(CVEs) == 1
+            assert CVEs[0]["NAME"] == "MELTDOWN"
+        except Exception as e:
+            import traceback
+            traceback.print_exc()
+            self.logger_warning("Something wrong happened when trying to diagnose Meltdown vunerability, exception: %s" % e)
+            raise Exception("Command output for failed meltdown check: '%s'" % output)
+
+        self.logger_debug("Writing results from meltdown checker to cache file, %s" % cache_file)
+        write_to_json(cache_file, CVEs)
+        return CVEs[0]["VULNERABLE"]
+
+
+def main(args, env, loggers):
+    return SecurityDiagnoser(args, env, loggers).diagnose()

data/templates/nginx/server.tpl.conf

@@ -16,6 +16,10 @@ server {
         return 301 https://$http_host$request_uri;
     }
 
+    location /.well-known/ynh-diagnosis/ {
+        alias /tmp/.well-known/ynh-diagnosis/;
+    }
+
     location /.well-known/autoconfig/mail/ {
         alias /var/www/.well-known/{{ domain }}/autoconfig/mail/;
     }

locales/en.json

@@ -150,13 +150,72 @@
     "confirm_app_install_thirdparty": "DANGER! This app is not part of Yunohost's app catalog. Installing third-party apps may compromise the integrity and security of your system. You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or break your system… If you are willing to take that risk anyway, type '{answers:s}'",
     "custom_app_url_required": "You must provide a URL to upgrade your custom app {app:s}",
     "custom_appslist_name_required": "You must provide a name for your custom app list",
-    "diagnosis_debian_version_error": "Could not retrieve the Debian version: {error}",
-    "diagnosis_kernel_version_error": "Could not retrieve kernel version: {error}",
-    "diagnosis_monitor_disk_error": "Could not monitor disks: {error}",
-    "diagnosis_monitor_system_error": "Could not monitor system: {error}",
-    "diagnosis_no_apps": "No such installed app",
-    "dpkg_is_broken": "You cannot do this right now because dpkg/APT (the system package managers) seems to be in a broken state… You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.",
-    "dpkg_lock_not_available": "This command can't be ran right now because another program seems to be using the lock of dpkg (the system package manager)",
+    "diagnosis_basesystem_host": "Server is running Debian {debian_version}.",
+    "diagnosis_basesystem_kernel": "Server is running Linux kernel {kernel_version}",
+    "diagnosis_basesystem_ynh_single_version": "{0} version: {1}",
+    "diagnosis_basesystem_ynh_main_version": "Server is running YunoHost {main_version}",
+    "diagnosis_basesystem_ynh_inconsistent_versions": "You are running inconsistents versions of the YunoHost packages ... most probably because of a failed or partial upgrade.",
+    "diagnosis_display_tip_web": "You can go to the Diagnosis section (in the home screen) to see the issues found.",
+    "diagnosis_display_tip_cli": "You can run 'yunohost diagnosis show --issues' to display the issues found.",
+    "diagnosis_failed_for_category": "Diagnosis failed for category '{category}' : {error}",
+    "diagnosis_cache_still_valid": "(Cache still valid for {category} diagnosis. Not re-diagnosing yet!)",
+    "diagnosis_cant_run_because_of_dep": "Can't run diagnosis for {category} while there are important issues related to {dep}.",
+    "diagnosis_ignored_issues": "(+ {nb_ignored} ignored issue(s))",
+    "diagnosis_found_errors": "Found {errors} significant issue(s) related to {category}!",
+    "diagnosis_found_errors_and_warnings": "Found {errors} significant issue(s) (and {warnings} warning(s)) related to {category}!",
+    "diagnosis_found_warnings": "Found {warnings} item(s) that could be improved for {category}.",
+    "diagnosis_everything_ok": "Everything looks good for {category}!",
+    "diagnosis_failed": "Failed to fetch diagnosis result for category '{category}' : {error}",
+    "diagnosis_ip_connected_ipv4": "The server is connected to the Internet through IPv4 !",
+    "diagnosis_ip_no_ipv4": "The server does not have a working IPv4.",
+    "diagnosis_ip_connected_ipv6": "The server is connected to the Internet through IPv6 !",
+    "diagnosis_ip_no_ipv6": "The server does not have a working IPv6.",
+    "diagnosis_ip_not_connected_at_all": "The server does not seem to be connected to the Internet at all!?",
+    "diagnosis_ip_dnsresolution_working": "Domain name resolution is working!",
+    "diagnosis_ip_broken_dnsresolution": "Domain name resolution seems to be broken for some reason ... Is a firewall blocking DNS requests ?",
+    "diagnosis_ip_broken_resolvconf": "Domain name resolution seems to be broken on your server, which seems related to /etc/resolv.conf not pointing to 127.0.0.1.",
+    "diagnosis_ip_weird_resolvconf": "DNS resolution seems to be working, but be careful that you seem to be using a custom /etc/resolv.conf.",
+    "diagnosis_ip_weird_resolvconf_details": "Instead, this file should be a symlink to /etc/resolvconf/run/resolv.conf itself pointing to 127.0.0.1 (dnsmasq). The actual resolvers should be configured via /etc/resolv.dnsmasq.conf.",
+    "diagnosis_dns_good_conf": "Good DNS configuration for domain {domain} (category {category})",
+    "diagnosis_dns_bad_conf": "Bad / missing DNS configuration for domain {domain} (category {category})",
+    "diagnosis_dns_missing_record": "According to the recommended DNS configuration, you should add a DNS record with type {0}, name {1} and value {2}",
+    "diagnosis_dns_discrepancy": "According to the recommended DNS configuration, the value for the DNS record with type {0} and name {1} should be {2}, not {3}.",
+    "diagnosis_services_good_status": "Service {service} is {status} as expected!",
+    "diagnosis_services_bad_status": "Service {service} is {status} :/",
+    "diagnosis_diskusage_verylow": "Storage {mountpoint} (on device {device}) has only {free_abs_GB} GB ({free_percent}%) space remaining. You should really consider cleaning up some space.",
+    "diagnosis_diskusage_low": "Storage {mountpoint} (on device {device}) has only {free_abs_GB} GB ({free_percent}%) space remaining. Be careful.",
+    "diagnosis_diskusage_ok": "Storage {mountpoint} (on device {device}) still has {free_abs_GB} GB ({free_percent}%) space left!",
+    "diagnosis_ram_verylow": "The system has only {available_abs_MB} MB ({available_percent}%) RAM left! (out of {total_abs_MB} MB)",
+    "diagnosis_ram_low": "The system has {available_abs_MB} MB ({available_percent}%) RAM left out of {total_abs_MB} MB. Be careful.",
+    "diagnosis_ram_ok": "The system still has {available_abs_MB} MB ({available_percent}%) RAM left out of {total_abs_MB} MB.",
+    "diagnosis_swap_none": "The system has no swap at all. You should consider adding at least 256 MB of swap to avoid situations where the system runs out of memory.",
+    "diagnosis_swap_notsomuch": "The system has only {total_MB} MB swap. You should consider having at least 256 MB to avoid situations where the system runs out of memory.",
+    "diagnosis_swap_ok": "The system has {total_MB} MB of swap!",
+    "diagnosis_regenconf_allgood": "All configurations files are in line with the recommended configuration!",
+    "diagnosis_regenconf_manually_modified": "Configuration file {file} was manually modified.",
+    "diagnosis_regenconf_manually_modified_details": "This is probably OK as long as you know what you're doing ;) !",
+    "diagnosis_regenconf_manually_modified_debian": "Configuration file {file} was manually modified compared to Debian's default.",
+    "diagnosis_regenconf_manually_modified_debian_details": "This may probably be OK, but gotta keep an eye on it...",
+    "diagnosis_regenconf_nginx_conf_broken": "The nginx configuration appears to be broken!",
+    "diagnosis_security_all_good": "No critical security vulnerability was found.",
+    "diagnosis_security_vulnerable_to_meltdown": "You appear vulnerable to the Meltdown criticial security vulnerability",
+    "diagnosis_security_vulnerable_to_meltdown_details": "To fix this, you should upgrade your system and reboot to load the new linux kernel (or contact your server provider if this doesn't work). See https://meltdownattack.com/ for more infos.",
+    "diagnosis_description_basesystem": "Base system",
+    "diagnosis_description_ip": "Internet connectivity",
+    "diagnosis_description_dnsrecords": "DNS records",
+    "diagnosis_description_services": "Services status check",
+    "diagnosis_description_systemresources": "System resources",
+    "diagnosis_description_ports": "Ports exposure",
+    "diagnosis_description_http": "HTTP exposure",
+    "diagnosis_description_regenconf": "System configurations",
+    "diagnosis_description_security": "Security checks",
+    "diagnosis_ports_could_not_diagnose": "Could not diagnose if ports are reachable from outside. Error: {error}",
+    "diagnosis_ports_unreachable": "Port {port} is not reachable from outside.",
+    "diagnosis_ports_ok": "Port {port} is reachable from outside.",
+    "diagnosis_http_could_not_diagnose": "Could not diagnose if domain is reachable from outside. Error: {error}",
+    "diagnosis_http_ok": "Domain {domain} is reachable from outside.",
+    "diagnosis_http_unreachable": "Domain {domain} is unreachable through HTTP from outside.",
+    "diagnosis_unknown_categories": "The following categories are unknown : {categories}",
     "domain_cannot_remove_main": "You cannot remove '{domain:s}' since it's the main domain, you need first to set another domain as the main domain using 'yunohost domain main-domain -n <another-domain>', here is the list of candidate domains: {other_domains:s}",
     "domain_cannot_remove_main_add_new_one": "You cannot remove '{domain:s}' since it's the main domain and your only domain, you need to first add another domain using 'yunohost domain add <another-domain.com>', then set is as the main domain using 'yunohost domain main-domain -n <another-domain.com>' and then you can remove the domain '{domain:s}' using 'yunohost domain remove {domain:s}'.'",
     "domain_cert_gen_failed": "Could not generate certificate",
@@ -174,6 +233,8 @@
     "domains_available": "Available domains:",
     "done": "Done",
     "downloading": "Downloading…",
+    "dpkg_is_broken": "You cannot do this right now because dpkg/APT (the system package managers) seems to be in a broken state… You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.",
+    "dpkg_lock_not_available": "This command can't be ran right now because another program seems to be using the lock of dpkg (the system package manager)",
     "dyndns_could_not_check_provide": "Could not check if {provider:s} can provide {domain:s}.",
     "dyndns_could_not_check_available": "Could not check if {domain:s} is available on {provider:s}.",
     "dyndns_cron_installed": "DynDNS cron job created",

src/yunohost/app.py

@@ -1619,7 +1619,8 @@ def app_ssowatconf():
     for domain in domains:
         skipped_urls.extend([domain + '/yunohost/admin', domain + '/yunohost/api'])
 
-    # Authorize ACME challenge url
+    # Authorize ynh remote diagnosis, ACME challenge and mail autoconfig urls
+    skipped_regex.append("^[^/]*/%.well%-known/ynh%-diagnosis/.*$")
     skipped_regex.append("^[^/]*/%.well%-known/acme%-challenge/.*$")
     skipped_regex.append("^[^/]*/%.well%-known/autoconfig/mail/config%-v1%.1%.xml.*$")
 

src/yunohost/diagnosis.py

@@ -0,0 +1,426 @@
+# -*- coding: utf-8 -*-
+
+""" License
+
+    Copyright (C) 2018 YunoHost
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program; if not, see http://www.gnu.org/licenses
+
+"""
+
+""" diagnosis.py
+
+    Look for possible issues on the server
+"""
+
+import os
+import time
+
+from moulinette import m18n, msettings
+from moulinette.utils import log
+from moulinette.utils.filesystem import read_json, write_to_json, read_yaml, write_to_yaml
+
+from yunohost.utils.error import YunohostError
+from yunohost.hook import hook_list, hook_exec
+
+logger = log.getActionLogger('yunohost.diagnosis')
+
+DIAGNOSIS_CACHE = "/var/cache/yunohost/diagnosis/"
+DIAGNOSIS_CONFIG_FILE = '/etc/yunohost/diagnosis.yml'
+
+def diagnosis_list():
+    all_categories_names = [h for h, _ in _list_diagnosis_categories()]
+    return {"categories": all_categories_names}
+
+
+def diagnosis_show(categories=[], issues=False, full=False, share=False):
+
+    # Get all the categories
+    all_categories = _list_diagnosis_categories()
+    all_categories_names = [category for category, _ in all_categories]
+
+    # Check the requested category makes sense
+    if categories == []:
+        categories = all_categories_names
+    else:
+        unknown_categories = [c for c in categories if c not in all_categories_names]
+        if unknown_categories:
+            raise YunohostError('diagnosis_unknown_categories', categories=", ".join(categories))
+
+    # Fetch all reports
+    all_reports = []
+    for category in categories:
+        try:
+            report = Diagnoser.get_cached_report(category)
+        except Exception as e:
+            logger.error(m18n.n("diagnosis_failed", category=category, error=str(e)))
+        else:
+            add_ignore_flag_to_issues(report)
+            if not full:
+                del report["timestamp"]
+                del report["cached_for"]
+                report["items"] = [item for item in report["items"] if not item["ignored"]]
+                for item in report["items"]:
+                    del item["meta"]
+                    del item["ignored"]
+                    if "data" in item:
+                        del item["data"]
+            if issues:
+                report["items"] = [item for item in report["items"] if item["status"] in ["WARNING", "ERROR"]]
+                # Ignore this category if no issue was found
+                if not report["items"]:
+                    continue
+
+            all_reports.append(report)
+
+    if share:
+        from yunohost.utils.yunopaste import yunopaste
+        content = _dump_human_readable_reports(all_reports)
+        url = yunopaste(content)
+
+        logger.info(m18n.n("log_available_on_yunopaste", url=url))
+        if msettings.get('interface') == 'api':
+            return {"url": url}
+        else:
+            return
+    else:
+        return {"reports": all_reports}
+
+def _dump_human_readable_reports(reports):
+
+    output = ""
+
+    for report in reports:
+        output += "=================================\n"
+        output += "{description} ({id})\n".format(**report)
+        output += "=================================\n\n"
+        for item in report["items"]:
+            output += "[{status}] {summary}\n".format(**item)
+            for detail in item.get("details", []):
+                output += "  - " + detail + "\n"
+            output += "\n"
+        output += "\n\n"
+
+    return(output)
+
+
+def diagnosis_run(categories=[], force=False):
+
+    # Get all the categories
+    all_categories = _list_diagnosis_categories()
+    all_categories_names = [category for category, _ in all_categories]
+
+    # Check the requested category makes sense
+    if categories == []:
+        categories = all_categories_names
+    else:
+        unknown_categories = [c for c in categories if c not in all_categories_names]
+        if unknown_categories:
+            raise YunohostError('diagnosis_unknown_categories', categories=", ".join(unknown_categories))
+
+    issues = []
+    # Call the hook ...
+    diagnosed_categories = []
+    for category in categories:
+        logger.debug("Running diagnosis for %s ..." % category)
+        path = [p for n, p in all_categories if n == category][0]
+
+        try:
+            code, report = hook_exec(path, args={"force": force}, env=None)
+        except Exception as e:
+            logger.error(m18n.n("diagnosis_failed_for_category", category=category, error=str(e)), exc_info=True)
+        else:
+            diagnosed_categories.append(category)
+            if report != {}:
+                issues.extend([item for item in report["items"] if item["status"] in ["WARNING", "ERROR"]])
+
+    if issues:
+        if msettings.get("interface") == "api":
+            logger.info(m18n.n("diagnosis_display_tip_web"))
+        else:
+            logger.info(m18n.n("diagnosis_display_tip_cli"))
+
+    return
+
+
+def diagnosis_ignore(add_filter=None, remove_filter=None, list=False):
+    """
+    This action is meant for the admin to ignore issues reported by the
+    diagnosis system if they are known and understood by the admin.  For
+    example, the lack of ipv6 on an instance, or badly configured XMPP dns
+    records if the admin doesn't care so much about XMPP. The point being that
+    the diagnosis shouldn't keep complaining about those known and "expected"
+    issues, and instead focus on new unexpected issues that could arise.
+
+    For example, to ignore badly XMPP dnsrecords for domain yolo.test:
+
+        yunohost diagnosis ignore --add-filter dnsrecords domain=yolo.test category=xmpp
+                                                  ^              ^             ^
+                                            the general    additional       other
+                                            diagnosis       criterias       criteria
+                                            category to    to target        to target
+                                            act on           specific       specific
+                                                             reports        reports
+    Or to ignore all dnsrecords issues:
+
+        yunohost diagnosis ignore --add-filter dnsrecords
+
+    The filters are stored in the diagnosis configuration in a data structure like:
+
+    ignore_filters: {
+        "ip": [
+           {"version": 6}     # Ignore all issues related to ipv6
+        ],
+        "dnsrecords": [
+           {"domain": "yolo.test", "category": "xmpp"}, # Ignore all issues related to DNS xmpp records for yolo.test
+           {}                                           # Ignore all issues about dnsrecords
+        ]
+    }
+    """
+
+    # Ignore filters are stored in
+    configuration = _diagnosis_read_configuration()
+
+    if list:
+        return {"ignore_filters": configuration.get("ignore_filters", {})}
+
+    def validate_filter_criterias(filter_):
+
+        # Get all the categories
+        all_categories = _list_diagnosis_categories()
+        all_categories_names = [category for category, _ in all_categories]
+
+        # Sanity checks for the provided arguments
+        if len(filter_) == 0:
+            raise YunohostError("You should provide at least one criteria being the diagnosis category to ignore")
+        category = filter_[0]
+        if category not in all_categories_names:
+            raise YunohostError("%s is not a diagnosis category" % category)
+        if any("=" not in criteria for criteria in filter_[1:]):
+            raise YunohostError("Extra criterias should be of the form key=value (e.g. domain=yolo.test)")
+
+        # Convert the provided criteria into a nice dict
+        criterias = {c.split("=")[0]: c.split("=")[1] for c in filter_[1:]}
+
+        return category, criterias
+
+    if add_filter:
+
+        category, criterias = validate_filter_criterias(add_filter)
+
+        # Fetch current issues for the requested category
+        current_issues_for_this_category = diagnosis_show(categories=[category], issues=True, full=True)
+        current_issues_for_this_category = current_issues_for_this_category["reports"][0].get("items", {})
+
+        # Accept the given filter only if the criteria effectively match an existing issue
+        if not any(issue_matches_criterias(i, criterias) for i in current_issues_for_this_category):
+            raise YunohostError("No issues was found matching the given criteria.")
+
+        # Make sure the subdicts/lists exists
+        if "ignore_filters" not in configuration:
+            configuration["ignore_filters"] = {}
+        if category not in configuration["ignore_filters"]:
+            configuration["ignore_filters"][category] = []
+
+        if criterias in configuration["ignore_filters"][category]:
+            logger.warning("This filter already exists.")
+            return
+
+        configuration["ignore_filters"][category].append(criterias)
+        _diagnosis_write_configuration(configuration)
+        logger.success("Filter added")
+        return
+
+    if remove_filter:
+
+        category, criterias = validate_filter_criterias(remove_filter)
+
+        # Make sure the subdicts/lists exists
+        if "ignore_filters" not in configuration:
+            configuration["ignore_filters"] = {}
+        if category not in configuration["ignore_filters"]:
+            configuration["ignore_filters"][category] = []
+
+        if criterias not in configuration["ignore_filters"][category]:
+            raise YunohostError("This filter does not exists.")
+
+        configuration["ignore_filters"][category].remove(criterias)
+        _diagnosis_write_configuration(configuration)
+        logger.success("Filter removed")
+        return
+
+
+def _diagnosis_read_configuration():
+    if not os.path.exists(DIAGNOSIS_CONFIG_FILE):
+        return {}
+
+    return read_yaml(DIAGNOSIS_CONFIG_FILE)
+
+
+def _diagnosis_write_configuration(conf):
+    write_to_yaml(DIAGNOSIS_CONFIG_FILE, conf)
+
+
+def issue_matches_criterias(issue, criterias):
+    """
+    e.g. an issue with:
+       meta:
+          domain: yolo.test
+          category: xmpp
+
+    matches the criterias {"domain": "yolo.test"}
+    """
+    for key, value in criterias.items():
+        if key not in issue["meta"]:
+            return False
+        if str(issue["meta"][key]) != value:
+            return False
+    return True
+
+def add_ignore_flag_to_issues(report):
+    """
+    Iterate over issues in a report, and flag them as ignored if they match an
+    ignored filter from the configuration
+
+    N.B. : for convenience. we want to make sure the "ignored" key is set for
+    every item in the report
+    """
+
+    ignore_filters = _diagnosis_read_configuration().get("ignore_filters", {}).get(report["id"], [])
+
+    for report_item in report["items"]:
+        report_item["ignored"] = False
+        if report_item["status"] not in ["WARNING", "ERROR"]:
+            continue
+        for criterias in ignore_filters:
+            if issue_matches_criterias(report_item, criterias):
+                report_item["ignored"] = True
+                break
+
+
+############################################################
+
+
+class Diagnoser():
+
+    def __init__(self, args, env, loggers):
+
+        # FIXME ? That stuff with custom loggers is weird ... (mainly inherited from the bash hooks, idk)
+        self.logger_debug, self.logger_warning, self.logger_info = loggers
+        self.env = env
+        self.args = args or {}
+        self.cache_file = Diagnoser.cache_file(self.id_)
+        self.description = Diagnoser.get_description(self.id_)
+
+    def cached_time_ago(self):
+
+        if not os.path.exists(self.cache_file):
+            return 99999999
+        return time.time() - os.path.getmtime(self.cache_file)
+
+    def write_cache(self, report):
+        if not os.path.exists(DIAGNOSIS_CACHE):
+            os.makedirs(DIAGNOSIS_CACHE)
+        return write_to_json(self.cache_file, report)
+
+    def diagnose(self):
+
+        if not self.args.get("force", False) and self.cached_time_ago() < self.cache_duration:
+            self.logger_debug("Cache still valid : %s" % self.cache_file)
+            logger.info(m18n.n("diagnosis_cache_still_valid", category=self.description))
+            return 0, {}
+
+        for dependency in self.dependencies:
+            dep_report = Diagnoser.get_cached_report(dependency)
+            dep_errors = [item for item in dep_report["items"] if item["status"] == "ERROR"]
+            if dep_errors:
+                logger.error(m18n.n("diagnosis_cant_run_because_of_dep", category=self.description, dep=Diagnoser.get_description(dependency)))
+                return 1, {}
+
+        self.logger_debug("Running diagnostic for %s" % self.id_)
+
+        items = list(self.run())
+
+        new_report = {"id": self.id_,
+                      "cached_for": self.cache_duration,
+                      "items": items}
+
+        self.logger_debug("Updating cache %s" % self.cache_file)
+        self.write_cache(new_report)
+        Diagnoser.i18n(new_report)
+        add_ignore_flag_to_issues(new_report)
+
+        errors   = [item for item in new_report["items"] if item["status"] == "ERROR" and not item["ignored"]]
+        warnings = [item for item in new_report["items"] if item["status"] == "WARNING" and not item["ignored"]]
+        errors_ignored = [item for item in new_report["items"] if item["status"] == "ERROR" and item["ignored"]]
+        warning_ignored = [item for item in new_report["items"] if item["status"] == "WARNING" and item["ignored"]]
+        ignored_msg = " " + m18n.n("diagnosis_ignored_issues", nb_ignored=len(errors_ignored+warning_ignored)) if errors_ignored or warning_ignored else ""
+
+        if errors and warnings:
+            logger.error(m18n.n("diagnosis_found_errors_and_warnings", errors=len(errors), warnings=len(warnings), category=new_report["description"]) + ignored_msg)
+        elif errors:
+            logger.error(m18n.n("diagnosis_found_errors", errors=len(errors), category=new_report["description"]) + ignored_msg)
+        elif warnings:
+            logger.warning(m18n.n("diagnosis_found_warnings", warnings=len(warnings), category=new_report["description"]) + ignored_msg)
+        else:
+            logger.success(m18n.n("diagnosis_everything_ok", category=new_report["description"]) + ignored_msg)
+
+        return 0, new_report
+
+    @staticmethod
+    def cache_file(id_):
+        return os.path.join(DIAGNOSIS_CACHE, "%s.json" % id_)
+
+    @staticmethod
+    def get_cached_report(id_):
+        filename = Diagnoser.cache_file(id_)
+        report = read_json(filename)
+        report["timestamp"] = int(os.path.getmtime(filename))
+        Diagnoser.i18n(report)
+        return report
+
+    @staticmethod
+    def get_description(id_):
+        key = "diagnosis_description_" + id_
+        descr = m18n.n(key)
+        # If no description available, fallback to id
+        return descr if descr != key else id_
+
+    @staticmethod
+    def i18n(report):
+
+        # "Render" the strings with m18n.n
+        # N.B. : we do those m18n.n right now instead of saving the already-translated report
+        # because we can't be sure we'll redisplay the infos with the same locale as it
+        # was generated ... e.g. if the diagnosing happened inside a cron job with locale EN
+        # instead of FR used by the actual admin...
+
+        report["description"] = Diagnoser.get_description(report["id"])
+
+        for item in report["items"]:
+            summary_key, summary_args = item["summary"]
+            item["summary"] = m18n.n(summary_key, **summary_args)
+
+            if "details" in item:
+                item["details"] = [m18n.n(key, *values) for key, values in item["details"]]
+
+
+def _list_diagnosis_categories():
+    hooks_raw = hook_list("diagnosis", list_by="priority", show_info=True)["hooks"]
+    hooks = []
+    for _, some_hooks in sorted(hooks_raw.items(), key=lambda h: int(h[0])):
+        for name, info in some_hooks.items():
+            hooks.append((name, info["path"]))
+
+    return hooks

src/yunohost/regenconf.py

@@ -534,31 +534,32 @@ def _process_regen_conf(system_conf, new_conf=None, save=True):
 
 def manually_modified_files():
 
-    # We do this to have --quiet, i.e. don't throw a whole bunch of logs
-    # just to fetch this...
-    # Might be able to optimize this by looking at what the regen conf does
-    # and only do the part that checks file hashes...
-    cmd = "yunohost tools regen-conf --dry-run --output-as json --quiet"
-    j = json.loads(subprocess.check_output(cmd.split()))
-
-    # j is something like :
-    # {"postfix": {"applied": {}, "pending": {"/etc/postfix/main.cf": {"status": "modified"}}}
-
     output = []
-    for app, actions in j.items():
-        for action, files in actions.items():
-            for filename, infos in files.items():
-                if infos["status"] == "modified":
-                    output.append(filename)
+    regenconf_categories = _get_regenconf_infos()
+    for category, infos in regenconf_categories.items():
+        conffiles = infos["conffiles"]
+        for path, hash_ in conffiles.items():
+            if hash_ != _calculate_hash(path):
+                output.append(path)
 
     return output
 
 
-def manually_modified_files_compared_to_debian_default():
+def manually_modified_files_compared_to_debian_default(ignore_handled_by_regenconf=False):
 
     # from https://serverfault.com/a/90401
-    r = subprocess.check_output("dpkg-query -W -f='${Conffiles}\n' '*' \
+    files = subprocess.check_output("dpkg-query -W -f='${Conffiles}\n' '*' \
                                    | awk 'OFS=\"  \"{print $2,$1}' \
                                    | md5sum -c 2>/dev/null \
                                    | awk -F': ' '$2 !~ /OK/{print $1}'", shell=True)
-    return r.strip().split("\n")
+    files = files.strip().split("\n")
+
+    if ignore_handled_by_regenconf:
+        regenconf_categories = _get_regenconf_infos()
+        regenconf_files = []
+        for infos in regenconf_categories.values():
+            regenconf_files.extend(infos["conffiles"].keys())
+
+        files = [f for f in files if f not in regenconf_files]
+
+    return files

src/yunohost/tools.py

@@ -30,23 +30,19 @@ import json
 import subprocess
 import pwd
 import socket
-from xmlrpclib import Fault
 from importlib import import_module
-from collections import OrderedDict
 
 from moulinette import msignals, m18n
 from moulinette.utils.log import getActionLogger
 from moulinette.utils.process import check_output, call_async_output
 from moulinette.utils.filesystem import read_json, write_to_json, read_yaml, write_to_yaml
-from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, app_list, _install_appslist_fetch_cron
+from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, _install_appslist_fetch_cron
 from yunohost.domain import domain_add, domain_list
 from yunohost.dyndns import _dyndns_available, _dyndns_provides
 from yunohost.firewall import firewall_upnp
-from yunohost.service import service_status, service_start, service_enable
+from yunohost.service import service_start, service_enable
 from yunohost.regenconf import regen_conf
-from yunohost.monitor import monitor_disk, monitor_system
-from yunohost.utils.packages import ynh_packages_version, _dump_sources_list, _list_upgradable_apt_packages
-from yunohost.utils.network import get_public_ip
+from yunohost.utils.packages import _dump_sources_list, _list_upgradable_apt_packages
 from yunohost.utils.error import YunohostError
 from yunohost.log import is_unit_operation, OperationLogger
 
@@ -672,184 +668,6 @@ def tools_upgrade(operation_logger, apps=None, system=False):
             operation_logger.success()
 
 
-def tools_diagnosis(private=False):
-    """
-    Return global info about current yunohost instance to help debugging
-
-    """
-    diagnosis = OrderedDict()
-
-    # Debian release
-    try:
-        with open('/etc/debian_version', 'r') as f:
-            debian_version = f.read().rstrip()
-    except IOError as e:
-        logger.warning(m18n.n('diagnosis_debian_version_error', error=format(e)), exc_info=1)
-    else:
-        diagnosis['host'] = "Debian %s" % debian_version
-
-    # Kernel version
-    try:
-        with open('/proc/sys/kernel/osrelease', 'r') as f:
-            kernel_version = f.read().rstrip()
-    except IOError as e:
-        logger.warning(m18n.n('diagnosis_kernel_version_error', error=format(e)), exc_info=1)
-    else:
-        diagnosis['kernel'] = kernel_version
-
-    # Packages version
-    diagnosis['packages'] = ynh_packages_version()
-
-    diagnosis["backports"] = check_output("dpkg -l |awk '/^ii/ && $3 ~ /bpo[6-8]/ {print $2}'").split()
-
-    # Server basic monitoring
-    diagnosis['system'] = OrderedDict()
-    try:
-        disks = monitor_disk(units=['filesystem'], human_readable=True)
-    except (YunohostError, Fault) as e:
-        logger.warning(m18n.n('diagnosis_monitor_disk_error', error=format(e)), exc_info=1)
-    else:
-        diagnosis['system']['disks'] = {}
-        for disk in disks:
-            if isinstance(disks[disk], str):
-                diagnosis['system']['disks'][disk] = disks[disk]
-            else:
-                diagnosis['system']['disks'][disk] = 'Mounted on %s, %s (%s free)' % (
-                    disks[disk]['mnt_point'],
-                    disks[disk]['size'],
-                    disks[disk]['avail']
-                )
-
-    try:
-        system = monitor_system(units=['cpu', 'memory'], human_readable=True)
-    except YunohostError as e:
-        logger.warning(m18n.n('diagnosis_monitor_system_error', error=format(e)), exc_info=1)
-    else:
-        diagnosis['system']['memory'] = {
-            'ram': '%s (%s free)' % (system['memory']['ram']['total'], system['memory']['ram']['free']),
-            'swap': '%s (%s free)' % (system['memory']['swap']['total'], system['memory']['swap']['free']),
-        }
-
-    # nginx -t
-    p = subprocess.Popen("nginx -t".split(),
-                         stdout=subprocess.PIPE,
-                         stderr=subprocess.STDOUT)
-    out, _ = p.communicate()
-    diagnosis["nginx"] = out.strip().split("\n")
-    if p.returncode != 0:
-        logger.error(out)
-
-    # Services status
-    services = service_status()
-    diagnosis['services'] = {}
-
-    for service in services:
-        diagnosis['services'][service] = "%s (%s)" % (services[service]['status'], services[service]['loaded'])
-
-    # YNH Applications
-    try:
-        applications = app_list()['apps']
-    except YunohostError as e:
-        diagnosis['applications'] = m18n.n('diagnosis_no_apps')
-    else:
-        diagnosis['applications'] = {}
-        for application in applications:
-            if application['installed']:
-                diagnosis['applications'][application['id']] = application['label'] if application['label'] else application['name']
-
-    # Private data
-    if private:
-        diagnosis['private'] = OrderedDict()
-
-        # Public IP
-        diagnosis['private']['public_ip'] = {}
-        diagnosis['private']['public_ip']['IPv4'] = get_public_ip(4)
-        diagnosis['private']['public_ip']['IPv6'] = get_public_ip(6)
-
-        # Domains
-        diagnosis['private']['domains'] = domain_list()['domains']
-
-        diagnosis['private']['regen_conf'] = regen_conf(with_diff=True, dry_run=True)
-
-    try:
-        diagnosis['security'] = {
-            "CVE-2017-5754": {
-                "name": "meltdown",
-                "vulnerable": _check_if_vulnerable_to_meltdown(),
-            }
-        }
-    except Exception as e:
-        import traceback
-        traceback.print_exc()
-        logger.warning("Unable to check for meltdown vulnerability: %s" % e)
-
-    return diagnosis
-
-
-def _check_if_vulnerable_to_meltdown():
-    # meltdown CVE: https://security-tracker.debian.org/tracker/CVE-2017-5754
-
-    # We use a cache file to avoid re-running the script so many times,
-    # which can be expensive (up to around 5 seconds on ARM)
-    # and make the admin appear to be slow (c.f. the calls to diagnosis
-    # from the webadmin)
-    #
-    # The cache is in /tmp and shall disappear upon reboot
-    # *or* we compare it to dpkg.log modification time
-    # such that it's re-ran if there was package upgrades
-    # (e.g. from yunohost)
-    cache_file = "/tmp/yunohost-meltdown-diagnosis"
-    dpkg_log = "/var/log/dpkg.log"
-    if os.path.exists(cache_file):
-        if not os.path.exists(dpkg_log) or os.path.getmtime(cache_file) > os.path.getmtime(dpkg_log):
-            logger.debug("Using cached results for meltdown checker, from %s" % cache_file)
-            return read_json(cache_file)[0]["VULNERABLE"]
-
-    # script taken from https://github.com/speed47/spectre-meltdown-checker
-    # script commit id is store directly in the script
-    file_dir = os.path.split(__file__)[0]
-    SCRIPT_PATH = os.path.join(file_dir, "./vendor/spectre-meltdown-checker/spectre-meltdown-checker.sh")
-
-    # '--variant 3' corresponds to Meltdown
-    # example output from the script:
-    # [{"NAME":"MELTDOWN","CVE":"CVE-2017-5754","VULNERABLE":false,"INFOS":"PTI mitigates the vulnerability"}]
-    try:
-        logger.debug("Running meltdown vulnerability checker")
-        call = subprocess.Popen("bash %s --batch json --variant 3" %
-                                SCRIPT_PATH, shell=True,
-                                stdout=subprocess.PIPE,
-                                stderr=subprocess.PIPE)
-
-        # TODO / FIXME : here we are ignoring error messages ...
-        # in particular on RPi2 and other hardware, the script complains about
-        # "missing some kernel info (see -v), accuracy might be reduced"
-        # Dunno what to do about that but we probably don't want to harass
-        # users with this warning ...
-        output, err = call.communicate()
-        assert call.returncode in (0, 2, 3), "Return code: %s" % call.returncode
-
-        # If there are multiple lines, sounds like there was some messages
-        # in stdout that are not json >.> ... Try to get the actual json
-        # stuff which should be the last line
-        output = output.strip()
-        if "\n" in output:
-            logger.debug("Original meltdown checker output : %s" % output)
-            output = output.split("\n")[-1]
-
-        CVEs = json.loads(output)
-        assert len(CVEs) == 1
-        assert CVEs[0]["NAME"] == "MELTDOWN"
-    except Exception as e:
-        import traceback
-        traceback.print_exc()
-        logger.warning("Something wrong happened when trying to diagnose Meltdown vunerability, exception: %s" % e)
-        raise Exception("Command output for failed meltdown check: '%s'" % output)
-
-    logger.debug("Writing results from meltdown checker to cache file, %s" % cache_file)
-    write_to_json(cache_file, CVEs)
-    return CVEs[0]["VULNERABLE"]
-
-
 def tools_port_available(port):
     """
     Check availability of a local port