From f987e7872c9f09bc7320bed2aa16360a72a1ccb1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 28 Aug 2018 23:33:22 +0000 Subject: [PATCH 01/61] Skeleton / draft of API --- data/actionsmap/yunohost.yml | 50 ++++++++++++++++++++++++++++++++++++ src/yunohost/diagnosis.py | 44 +++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 src/yunohost/diagnosis.py diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 22037f05f..4f849160f 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1861,3 +1861,53 @@ log: --share: help: Share the full log using yunopaste action: store_true + + +############################# +# Diagnosis # +############################# +diagnosis: + category_help: Look for possible issues on the server + actions: + + list: + action_help: List diagnosis categories + api: GET /diagnosis/list + + report: + action_help: Show most recents diagnosis results + api: GET /diagnosis/report + arguments: + categories: + help: Diagnosis categories to display (all by default) + nargs: "*" + --full: + help: Display additional information + action: store_true + + run: + action_help: Show most recents diagnosis results + api: POST /diagnosis/run + arguments: + categories: + help: Diagnosis categories to run (all by default) + nargs: "*" + --force: + help: Display additional information + action: store_true + -a: + help: Serialized arguments for diagnosis scripts (e.g. "domain=domain.tld") + full: --args + + ignore: + action_help: Configure some diagnosis results to be ignored + api: PUT /diagnosis/ignore + arguments: + category: + help: Diagnosis category to be affected + -a: + help: Arguments, to be used to ignore only some parts of a report (e.g. "domain=domain.tld") + full: --args + --unignore: + help: Unignore a previously ignored report + action: store_true diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py new file mode 100644 index 000000000..aafbdcec3 --- /dev/null +++ b/src/yunohost/diagnosis.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +""" License + + Copyright (C) 2018 YunoHost + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program; if not, see http://www.gnu.org/licenses + +""" + +""" diagnosis.py + + Look for possible issues on the server +""" + +from moulinette import m18n +from moulinette.core import MoulinetteError +from moulinette.utils import log + +logger = log.getActionLogger('yunohost.diagnosis') + +def diagnosis_list(): + pass + +def diagnosis_report(categories=[], full=False): + pass + +def diagnosis_run(categories=[], force=False, args=""): + pass + +def diagnosis_ignore(category, args="", unignore=False): + pass + From 1d946ad073ed298185da49a6a5f111332c3daf25 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 29 Aug 2018 00:33:02 +0000 Subject: [PATCH 02/61] Implement diagnosis categories listing --- src/yunohost/diagnosis.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index aafbdcec3..0d312a7c1 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -28,10 +28,13 @@ from moulinette import m18n from moulinette.core import MoulinetteError from moulinette.utils import log +from yunohost.hook import hook_list + logger = log.getActionLogger('yunohost.diagnosis') def diagnosis_list(): - pass + all_categories_names = [ h for h, _ in _list_diagnosis_categories() ] + return { "categories": all_categories_names } def diagnosis_report(categories=[], full=False): pass @@ -42,3 +45,13 @@ def diagnosis_run(categories=[], force=False, args=""): def diagnosis_ignore(category, args="", unignore=False): pass +############################################################ + +def _list_diagnosis_categories(): + hooks_raw = hook_list("diagnosis", list_by="priority", show_info=True)["hooks"] + hooks = [] + for _, some_hooks in sorted(hooks_raw.items(), key=lambda h:int(h[0])): + for name, info in some_hooks.items(): + hooks.append((name, info["path"])) + + return hooks From b42bd20311797f59feb9ff7476ed19e49d20f8e5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 29 Aug 2018 01:34:15 +0000 Subject: [PATCH 03/61] First draft for diagnosis_run --- data/actionsmap/yunohost.yml | 2 +- locales/en.json | 1 + src/yunohost/diagnosis.py | 38 +++++++++++++++++++++++++++++++++--- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 4f849160f..aa85fdf70 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1893,7 +1893,7 @@ diagnosis: help: Diagnosis categories to run (all by default) nargs: "*" --force: - help: Display additional information + help: Ignore the cached report even if it is still 'fresh' action: store_true -a: help: Serialized arguments for diagnosis scripts (e.g. "domain=domain.tld") diff --git a/locales/en.json b/locales/en.json index f681fc4ea..a91da4fe9 100644 --- a/locales/en.json +++ b/locales/en.json @@ -547,6 +547,7 @@ "user_update_failed": "Could not update user {user}: {error}", "user_updated": "User info changed", "users_available": "Available users:", + "unknown_categories": "The following categories are unknown : {categories}", "yunohost_already_installed": "YunoHost is already installed", "yunohost_ca_creation_failed": "Could not create certificate authority", "yunohost_ca_creation_success": "Local certification authority created.", diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 0d312a7c1..10f09a576 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -24,14 +24,18 @@ Look for possible issues on the server """ +import errno + from moulinette import m18n from moulinette.core import MoulinetteError from moulinette.utils import log -from yunohost.hook import hook_list +from yunohost.hook import hook_list, hook_exec logger = log.getActionLogger('yunohost.diagnosis') +DIAGNOSIS_CACHE = "/var/cache/yunohost/diagnosis/" + def diagnosis_list(): all_categories_names = [ h for h, _ in _list_diagnosis_categories() ] return { "categories": all_categories_names } @@ -39,8 +43,36 @@ def diagnosis_list(): def diagnosis_report(categories=[], full=False): pass -def diagnosis_run(categories=[], force=False, args=""): - pass +def diagnosis_run(categories=[], force=False, args=None): + + # Get all the categories + all_categories = _list_diagnosis_categories() + all_categories_names = [ category for category, _ in all_categories ] + + # Check the requested category makes sense + if categories == []: + categories = all_categories_names + else: + unknown_categories = [ c for c in categories if c not in all_categories_names ] + if unknown_categories: + raise MoulinetteError(m18n.n('unknown_categories', categories=", ".join(categories))) + + # Transform "arg1=val1&arg2=val2" to { "arg1": "val1", "arg2": "val2" } + if args is not None: + args = { arg.split("=")[0]: arg.split("=")[1] for arg in args.split("&") } + else: + args = {} + args["force"] = force + + + # Call the hook ... + for category in categories: + logger.debug("Running diagnosis for %s ..." % category) + path = [p for n, p in all_categories if n == category ][0] + + # TODO : get the return value and do something with it + hook_exec(path, args=args, env=None) + def diagnosis_ignore(category, args="", unignore=False): pass From 7fb694dbccb62bcac206ffdf0688f7e1a52bd251 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 29 Aug 2018 23:48:16 +0000 Subject: [PATCH 04/61] Add diagnoser example for ip --- data/hooks/diagnosis/10-ip.py | 55 +++++++++++++++++++++++++++++++++++ src/yunohost/diagnosis.py | 15 +++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 data/hooks/diagnosis/10-ip.py diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py new file mode 100644 index 000000000..d8ab53c56 --- /dev/null +++ b/data/hooks/diagnosis/10-ip.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +from moulinette import m18n +from moulinette.utils.network import download_text +from yunohost.diagnosis import Diagnoser + +class IPDiagnoser(Diagnoser): + + def validate_args(self, args): + if "version" not in args.keys(): + return { "versions" : [4, 6] } + else: + if str(args["version"]) not in ["4", "6"]: + raise MoulinetteError(1, "Invalid version, should be 4 or 6.") + return { "versions" : [int(args["version"])] } + + def run(self): + + versions = self.args["versions"] + + if 4 in versions: + ipv4 = self.get_public_ip(4) + yield dict(meta = {"version": "4"}, + result = ipv4, + report = ("SUCCESS", m18n.n("diagnosis_network_connected_ipv4")) if ipv4 \ + else ("ERROR", m18n.n("diagnosis_network_no_ipv4"))) + + if 6 in versions: + ipv6 = self.get_public_ip(6) + yield dict(meta = {"version": "6"}, + result = ipv6, + report = ("SUCCESS", m18n.n("diagnosis_network_connected_ipv6")) if ipv6 \ + else ("WARNING", m18n.n("diagnosis_network_no_ipv6"))) + + + def get_public_ip(self, protocol=4): + + if protocol == 4: + url = 'https://ip.yunohost.org' + elif protocol == 6: + url = 'https://ip6.yunohost.org' + else: + raise ValueError("invalid protocol version") + + try: + return download_text(url, timeout=30).strip() + except Exception as e: + self.logger_debug("Could not get public IPv%s : %s" % (str(protocol), str(e))) + return None + + +def main(args, env, loggers): + + return IPDiagnoser(args, env, loggers).report() + diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 10f09a576..b5e0fa05a 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -71,7 +71,7 @@ def diagnosis_run(categories=[], force=False, args=None): path = [p for n, p in all_categories if n == category ][0] # TODO : get the return value and do something with it - hook_exec(path, args=args, env=None) + return {"report": hook_exec(path, args=args, env=None) } def diagnosis_ignore(category, args="", unignore=False): @@ -79,6 +79,19 @@ def diagnosis_ignore(category, args="", unignore=False): ############################################################ +class Diagnoser(): + + def __init__(self, args, env, loggers): + + self.logger_debug, self.logger_warning, self.logger_info = loggers + self.env = env + self.args = self.validate_args(args) + + def report(self): + + # TODO : implement some caching mecanism in there + return list(self.run()) + def _list_diagnosis_categories(): hooks_raw = hook_list("diagnosis", list_by="priority", show_info=True)["hooks"] hooks = [] From d34ddcaaf2964200b0a1edef167bb792b22cde26 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 30 Aug 2018 13:36:43 +0000 Subject: [PATCH 05/61] Implement caching mechanism --- data/hooks/diagnosis/10-ip.py | 6 ++++++ src/yunohost/diagnosis.py | 38 +++++++++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index d8ab53c56..a9485d7f4 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -1,11 +1,17 @@ #!/usr/bin/env python +import os + from moulinette import m18n from moulinette.utils.network import download_text from yunohost.diagnosis import Diagnoser class IPDiagnoser(Diagnoser): + id_ = os.path.splitext(os.path.basename(__file__))[0] + description = m18n.n("internet_connectivity") + cache_duration = 60 + def validate_args(self, args): if "version" not in args.keys(): return { "versions" : [4, 6] } diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index b5e0fa05a..f0ffcd619 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -25,10 +25,13 @@ """ import errno +import os +import time from moulinette import m18n from moulinette.core import MoulinetteError from moulinette.utils import log +from moulinette.utils.filesystem import read_json, write_to_json from yunohost.hook import hook_list, hook_exec @@ -87,10 +90,41 @@ class Diagnoser(): self.env = env self.args = self.validate_args(args) + @property + def cache_file(self): + return os.path.join(DIAGNOSIS_CACHE, "%s.json" % self.id_) + + def cached_time_ago(self): + + if not os.path.exists(self.cache_file): + return 99999999 + return time.time() - os.path.getmtime(self.cache_file) + + def get_cached_report(self): + return read_json(self.cache_file) + + def write_cache(self, report): + if not os.path.exists(DIAGNOSIS_CACHE): + os.makedirs(DIAGNOSIS_CACHE) + return write_to_json(self.cache_file, report) + def report(self): - # TODO : implement some caching mecanism in there - return list(self.run()) + print(self.cached_time_ago()) + + if self.args.get("force", False) or self.cached_time_ago() < self.cache_duration: + self.logger_debug("Using cached report from %s" % self.cache_file) + return self.get_cached_report() + + new_report = list(self.run()) + + # TODO / FIXME : should handle the case where we only did a partial diagnosis + self.logger_debug("Updating cache %s" % self.cache_file) + self.write_cache(new_report) + + return new_report + + def _list_diagnosis_categories(): hooks_raw = hook_list("diagnosis", list_by="priority", show_info=True)["hooks"] From f11206c0fa16a77aaabb441f33ab77f5c3446f2a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 30 Aug 2018 13:48:14 +0000 Subject: [PATCH 06/61] Turns out it's not really a good idea to do the internationalization right here as the strings will be kept already translated in the cache --- data/hooks/diagnosis/10-ip.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index a9485d7f4..eefa7798d 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -9,7 +9,7 @@ from yunohost.diagnosis import Diagnoser class IPDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0] - description = m18n.n("internet_connectivity") + description = "internet_connectivity" cache_duration = 60 def validate_args(self, args): @@ -26,17 +26,17 @@ class IPDiagnoser(Diagnoser): if 4 in versions: ipv4 = self.get_public_ip(4) - yield dict(meta = {"version": "4"}, + yield dict(meta = {"version": 4}, result = ipv4, - report = ("SUCCESS", m18n.n("diagnosis_network_connected_ipv4")) if ipv4 \ - else ("ERROR", m18n.n("diagnosis_network_no_ipv4"))) + report = ("SUCCESS", "diagnosis_network_connected_ipv4", {}) if ipv4 \ + else ("ERROR", "diagnosis_network_no_ipv4", {})) if 6 in versions: ipv6 = self.get_public_ip(6) - yield dict(meta = {"version": "6"}, + yield dict(meta = {"version": 6}, result = ipv6, - report = ("SUCCESS", m18n.n("diagnosis_network_connected_ipv6")) if ipv6 \ - else ("WARNING", m18n.n("diagnosis_network_no_ipv6"))) + report = ("SUCCESS", "diagnosis_network_connected_ipv6", {}) if ipv6 \ + else ("WARNING", "diagnosis_network_no_ipv6", {})) def get_public_ip(self, protocol=4): From 12df96f33e24404ea578e99e6c6dc370e7cc51b8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 30 Aug 2018 14:05:48 +0000 Subject: [PATCH 07/61] Wrap the report with meta infos --- data/hooks/diagnosis/10-ip.py | 1 - src/yunohost/diagnosis.py | 7 ++++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index eefa7798d..bde96b22e 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -38,7 +38,6 @@ class IPDiagnoser(Diagnoser): report = ("SUCCESS", "diagnosis_network_connected_ipv6", {}) if ipv6 \ else ("WARNING", "diagnosis_network_no_ipv6", {})) - def get_public_ip(self, protocol=4): if protocol == 4: diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index f0ffcd619..8fc46967b 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -110,13 +110,14 @@ class Diagnoser(): def report(self): - print(self.cached_time_ago()) - if self.args.get("force", False) or self.cached_time_ago() < self.cache_duration: self.logger_debug("Using cached report from %s" % self.cache_file) return self.get_cached_report() - new_report = list(self.run()) + new_report = { "id": self.id_, + "cached_for": self.cache_duration, + "reports": list(self.run()) + } # TODO / FIXME : should handle the case where we only did a partial diagnosis self.logger_debug("Updating cache %s" % self.cache_file) From cb6f53fc2bac2f215a6fbd0d43db08ad8d65a76d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 30 Aug 2018 14:18:44 +0000 Subject: [PATCH 08/61] Fix --force mechanism --- src/yunohost/diagnosis.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 8fc46967b..93c2dfc67 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -88,7 +88,8 @@ class Diagnoser(): self.logger_debug, self.logger_warning, self.logger_info = loggers self.env = env - self.args = self.validate_args(args) + self.args = args + self.args.update(self.validate_args(args)) @property def cache_file(self): @@ -110,10 +111,12 @@ class Diagnoser(): def report(self): - if self.args.get("force", False) or self.cached_time_ago() < self.cache_duration: + if not self.args.get("force", False) and self.cached_time_ago() < self.cache_duration: self.logger_debug("Using cached report from %s" % self.cache_file) return self.get_cached_report() + self.logger_debug("Running diagnostic for %s" % self.id_) + new_report = { "id": self.id_, "cached_for": self.cache_duration, "reports": list(self.run()) From faa4682d77cb512f21b12e901a78c2789ab28b3c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 30 Aug 2018 14:53:36 +0000 Subject: [PATCH 09/61] Forgot to keep the description --- src/yunohost/diagnosis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 93c2dfc67..09fcd8dcd 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -118,6 +118,7 @@ class Diagnoser(): self.logger_debug("Running diagnostic for %s" % self.id_) new_report = { "id": self.id_, + "description": self.description, "cached_for": self.cache_duration, "reports": list(self.run()) } From abffba960747378f1f714514ba932a9f0222a5f5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 30 Aug 2018 15:08:46 +0000 Subject: [PATCH 10/61] Remove the priority in the id of the diagnoser --- data/hooks/diagnosis/10-ip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index bde96b22e..b0a3ca1e9 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -8,7 +8,7 @@ from yunohost.diagnosis import Diagnoser class IPDiagnoser(Diagnoser): - id_ = os.path.splitext(os.path.basename(__file__))[0] + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] description = "internet_connectivity" cache_duration = 60 From 8a415579bf54d1515040fbe6f8d84f1d889effc4 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 30 Aug 2018 16:01:01 +0000 Subject: [PATCH 11/61] Implement diagnosis_show --- data/actionsmap/yunohost.yml | 4 +-- src/yunohost/diagnosis.py | 51 +++++++++++++++++++++++++++++------- 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index aa85fdf70..47a858b27 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1874,9 +1874,9 @@ diagnosis: action_help: List diagnosis categories api: GET /diagnosis/list - report: + show: action_help: Show most recents diagnosis results - api: GET /diagnosis/report + api: GET /diagnosis/show arguments: categories: help: Diagnosis categories to display (all by default) diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 09fcd8dcd..5144e9c06 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -43,8 +43,33 @@ def diagnosis_list(): all_categories_names = [ h for h, _ in _list_diagnosis_categories() ] return { "categories": all_categories_names } -def diagnosis_report(categories=[], full=False): - pass +def diagnosis_show(categories=[], full=False): + + # Get all the categories + all_categories = _list_diagnosis_categories() + all_categories_names = [ category for category, _ in all_categories ] + + # Check the requested category makes sense + if categories == []: + categories = all_categories_names + else: + unknown_categories = [ c for c in categories if c not in all_categories_names ] + if unknown_categories: + raise MoulinetteError(m18n.n('unknown_categories', categories=", ".join(categories))) + + # Fetch all reports + all_reports = [ Diagnoser.get_cached_report(c) for c in categories ] + + # "Render" the strings with m18n.n + for report in all_reports: + + report["description"] = m18n.n(report["description"]) + + for r in report["reports"]: + type_, message_key, message_args = r["report"] + r["report"] = (type_, m18n.n(message_key, **message_args)) + + return {"reports": all_reports} def diagnosis_run(categories=[], force=False, args=None): @@ -82,6 +107,7 @@ def diagnosis_ignore(category, args="", unignore=False): ############################################################ + class Diagnoser(): def __init__(self, args, env, loggers): @@ -90,10 +116,7 @@ class Diagnoser(): self.env = env self.args = args self.args.update(self.validate_args(args)) - - @property - def cache_file(self): - return os.path.join(DIAGNOSIS_CACHE, "%s.json" % self.id_) + self.cache_file = Diagnoser.cache_file(self.id_) def cached_time_ago(self): @@ -101,9 +124,6 @@ class Diagnoser(): return 99999999 return time.time() - os.path.getmtime(self.cache_file) - def get_cached_report(self): - return read_json(self.cache_file) - def write_cache(self, report): if not os.path.exists(DIAGNOSIS_CACHE): os.makedirs(DIAGNOSIS_CACHE) @@ -113,7 +133,7 @@ class Diagnoser(): if not self.args.get("force", False) and self.cached_time_ago() < self.cache_duration: self.logger_debug("Using cached report from %s" % self.cache_file) - return self.get_cached_report() + return Diagnoser.get_cached_report(self.id_) self.logger_debug("Running diagnostic for %s" % self.id_) @@ -129,6 +149,17 @@ class Diagnoser(): return new_report + @staticmethod + def cache_file(id_): + return os.path.join(DIAGNOSIS_CACHE, "%s.json" % id_) + + @staticmethod + def get_cached_report(id_): + filename = Diagnoser.cache_file(id_) + report = read_json(filename) + report["timestamp"] = int(os.path.getmtime(filename)) + return report + def _list_diagnosis_categories(): From b03e3a487e8c54d116be839f46217ab8044371fe Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 30 Aug 2018 17:49:42 +0000 Subject: [PATCH 12/61] Handle cases where some category might fail for some reason --- data/hooks/diagnosis/10-ip.py | 4 ++-- src/yunohost/diagnosis.py | 26 +++++++++++++++++--------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index b0a3ca1e9..898a6ac0f 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -55,6 +55,6 @@ class IPDiagnoser(Diagnoser): def main(args, env, loggers): - - return IPDiagnoser(args, env, loggers).report() + IPDiagnoser(args, env, loggers).diagnose() + return 0 diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 5144e9c06..805ac0b97 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -58,7 +58,12 @@ def diagnosis_show(categories=[], full=False): raise MoulinetteError(m18n.n('unknown_categories', categories=", ".join(categories))) # Fetch all reports - all_reports = [ Diagnoser.get_cached_report(c) for c in categories ] + all_reports = [] + for category in categories: + try: + all_reports.append(Diagnoser.get_cached_report(category)) + except Exception as e: + logger.error("Failed to fetch diagnosis result for category '%s' : %s" % (category, str(e))) # FIXME : i18n # "Render" the strings with m18n.n for report in all_reports: @@ -83,7 +88,7 @@ def diagnosis_run(categories=[], force=False, args=None): else: unknown_categories = [ c for c in categories if c not in all_categories_names ] if unknown_categories: - raise MoulinetteError(m18n.n('unknown_categories', categories=", ".join(categories))) + raise MoulinetteError(m18n.n('unknown_categories', categories=", ".join(unknown_categories))) # Transform "arg1=val1&arg2=val2" to { "arg1": "val1", "arg2": "val2" } if args is not None: @@ -92,15 +97,20 @@ def diagnosis_run(categories=[], force=False, args=None): args = {} args["force"] = force - # Call the hook ... + successes = [] for category in categories: logger.debug("Running diagnosis for %s ..." % category) path = [p for n, p in all_categories if n == category ][0] - # TODO : get the return value and do something with it - return {"report": hook_exec(path, args=args, env=None) } + try: + hook_exec(path, args=args, env=None) + successes.append(category) + except Exception as e: + # FIXME / TODO : add stacktrace here ? + logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e))) # FIXME : i18n + return diagnosis_show(successes) def diagnosis_ignore(category, args="", unignore=False): pass @@ -132,8 +142,8 @@ class Diagnoser(): def report(self): if not self.args.get("force", False) and self.cached_time_ago() < self.cache_duration: - self.logger_debug("Using cached report from %s" % self.cache_file) - return Diagnoser.get_cached_report(self.id_) + self.logger_debug("Cache still valid : %s" % self.cache_file) + return self.logger_debug("Running diagnostic for %s" % self.id_) @@ -147,8 +157,6 @@ class Diagnoser(): self.logger_debug("Updating cache %s" % self.cache_file) self.write_cache(new_report) - return new_report - @staticmethod def cache_file(id_): return os.path.join(DIAGNOSIS_CACHE, "%s.json" % id_) From 77b0920dac4d15b6ba70405f63ba023f229be0ae Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 31 Aug 2018 01:35:12 +0000 Subject: [PATCH 13/61] Forgot to change this --- src/yunohost/diagnosis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 805ac0b97..48e9977c1 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -139,7 +139,7 @@ class Diagnoser(): os.makedirs(DIAGNOSIS_CACHE) return write_to_json(self.cache_file, report) - def report(self): + def diagnose(self): if not self.args.get("force", False) and self.cached_time_ago() < self.cache_duration: self.logger_debug("Cache still valid : %s" % self.cache_file) From 85930163a09653fec43336f965d5fa9a2bc20497 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 31 Aug 2018 16:36:10 +0000 Subject: [PATCH 14/61] First draft of DNS diagnoser --- data/hooks/diagnosis/12-dns.py | 93 ++++++++++++++++++++++++++++++++++ locales/en.json | 4 +- 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 data/hooks/diagnosis/12-dns.py diff --git a/data/hooks/diagnosis/12-dns.py b/data/hooks/diagnosis/12-dns.py new file mode 100644 index 000000000..b4cedebad --- /dev/null +++ b/data/hooks/diagnosis/12-dns.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python + +import os + +from moulinette import m18n +from moulinette.utils.network import download_text +from moulinette.core import MoulinetteError, init_authenticator +from moulinette.utils.process import check_output + +from yunohost.diagnosis import Diagnoser +from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain + +# Instantiate LDAP Authenticator +auth_identifier = ('ldap', 'ldap-anonymous') +auth_parameters = {'uri': 'ldap://localhost:389', 'base_dn': 'dc=yunohost,dc=org'} +auth = init_authenticator(auth_identifier, auth_parameters) + +class DNSDiagnoser(Diagnoser): + + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] + description = "dns_configurations" + cache_duration = 3600*24 + + def validate_args(self, args): + all_domains = domain_list(auth)["domains"] + if "domain" not in args.keys(): + return { "domains" : all_domains } + else: + if args["domain"] not in all_domains: + raise MoulinetteError(errno.EINVAL, m18n.n('domain_unknown')) + return { "domains" : [ args["domain"] ] } + + def run(self): + + self.resolver = check_output('grep "$nameserver" /etc/resolv.dnsmasq.conf').split("\n")[0].split(" ")[1] + + main_domain = _get_maindomain() + + for domain in self.args["domains"]: + self.logger_info("Diagnosing DNS conf for %s" % domain) + for report in self.check_domain(domain, domain==main_domain): + yield report + + def check_domain(self, domain, is_main_domain): + + expected_configuration = _build_dns_conf(domain) + + # Here if there are no AAAA record, we should add something to expect "no" AAAA record + # to properly diagnose situations where people have a AAAA record but no IPv6 + + for category, records in expected_configuration.items(): + + discrepancies = [] + + for r in records: + current_value = self.get_current_record(domain, r["name"], r["type"]) or "None" + expected_value = r["value"] if r["value"] != "@" else domain+"." + + if current_value != expected_value: + discrepancies.append((r, expected_value, current_value)) + + if discrepancies: + if category == "basic" or is_main_domain: + level = "ERROR" + else: + level = "WARNING" + report = (level, "diagnosis_dns_bad_conf", {"domain": domain, "category": category}) + else: + level = "SUCCESS" + report = ("SUCCESS", "diagnosis_dns_good_conf", {"domain": domain, "category": category}) + + # FIXME : add management of details of what's wrong if there are discrepancies + yield dict(meta = {"domain": domain, "category": category}, + result = level, report = report ) + + + + def get_current_record(self, domain, name, type_): + if name == "@": + command = "dig +short @%s %s %s" % (self.resolver, type_, domain) + else: + command = "dig +short @%s %s %s.%s" % (self.resolver, type_, name, domain) + output = check_output(command).strip() + output = output.replace("\;",";") + if output.startswith('"') and output.endswith('"'): + output = '"' + ' '.join(output.replace('"',' ').split()) + '"' + return output + + +def main(args, env, loggers): + DNSDiagnoser(args, env, loggers).diagnose() + return 0 + diff --git a/locales/en.json b/locales/en.json index a91da4fe9..77701478c 100644 --- a/locales/en.json +++ b/locales/en.json @@ -155,7 +155,9 @@ "diagnosis_no_apps": "No installed application", "dpkg_is_broken": "You cannot do this right now because dpkg/APT (the system package managers) seems to be in a broken state… You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.", "dpkg_lock_not_available": "This command can't be ran right now because another program seems to be using the lock of dpkg (the system package manager)", - "domain_cannot_remove_main": "Cannot remove main domain. Set one first", + "diagnosis_dns_good_conf": "Good DNS configuration for {domain} : {category}.", + "diagnosis_dns_bad_conf": "Bad DNS configuration for {domain} : {category}.", + "domain_cannot_remove_main": "Cannot remove main domain. Set a new main domain first", "domain_cert_gen_failed": "Could not generate certificate", "domain_created": "Domain created", "domain_creation_failed": "Could not create domain {domain}: {error}", From ded4895b7e590ab60b5349f6a22e983782191adb Mon Sep 17 00:00:00 2001 From: Bram Date: Sat, 1 Sep 2018 02:50:22 +0200 Subject: [PATCH 15/61] [mod] misc, better error message I'm using repr to be able to detect if it's a string or a number since it's an error I'm expecting --- data/hooks/diagnosis/10-ip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index 898a6ac0f..574741da9 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -45,7 +45,7 @@ class IPDiagnoser(Diagnoser): elif protocol == 6: url = 'https://ip6.yunohost.org' else: - raise ValueError("invalid protocol version") + raise ValueError("invalid protocol version, it should be either 4 or 6 and was '%s'" % repr(protocol)) try: return download_text(url, timeout=30).strip() From 2b2ff08f08e8e62761e3a126090ebebd3bce7877 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 2 May 2019 17:59:35 +0200 Subject: [PATCH 16/61] Fix error handling (Yunohost / Moulinette / Asserts) --- data/hooks/diagnosis/10-ip.py | 6 +++--- data/hooks/diagnosis/12-dns.py | 6 ++---- src/yunohost/diagnosis.py | 9 ++++----- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index 574741da9..d229eea8f 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -4,6 +4,7 @@ import os from moulinette import m18n from moulinette.utils.network import download_text + from yunohost.diagnosis import Diagnoser class IPDiagnoser(Diagnoser): @@ -16,8 +17,7 @@ class IPDiagnoser(Diagnoser): if "version" not in args.keys(): return { "versions" : [4, 6] } else: - if str(args["version"]) not in ["4", "6"]: - raise MoulinetteError(1, "Invalid version, should be 4 or 6.") + assert str(args["version"]) in ["4", "6"], "Invalid version, should be 4 or 6." return { "versions" : [int(args["version"])] } def run(self): @@ -30,7 +30,7 @@ class IPDiagnoser(Diagnoser): result = ipv4, report = ("SUCCESS", "diagnosis_network_connected_ipv4", {}) if ipv4 \ else ("ERROR", "diagnosis_network_no_ipv4", {})) - + if 6 in versions: ipv6 = self.get_public_ip(6) yield dict(meta = {"version": 6}, diff --git a/data/hooks/diagnosis/12-dns.py b/data/hooks/diagnosis/12-dns.py index b4cedebad..9bf6a13a3 100644 --- a/data/hooks/diagnosis/12-dns.py +++ b/data/hooks/diagnosis/12-dns.py @@ -2,9 +2,8 @@ import os -from moulinette import m18n from moulinette.utils.network import download_text -from moulinette.core import MoulinetteError, init_authenticator +from moulinette.core import init_authenticator from moulinette.utils.process import check_output from yunohost.diagnosis import Diagnoser @@ -26,8 +25,7 @@ class DNSDiagnoser(Diagnoser): if "domain" not in args.keys(): return { "domains" : all_domains } else: - if args["domain"] not in all_domains: - raise MoulinetteError(errno.EINVAL, m18n.n('domain_unknown')) + assert args["domain"] in all_domains, "Unknown domain" return { "domains" : [ args["domain"] ] } def run(self): diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 48e9977c1..22770ce87 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -24,15 +24,14 @@ Look for possible issues on the server """ -import errno import os import time from moulinette import m18n -from moulinette.core import MoulinetteError from moulinette.utils import log from moulinette.utils.filesystem import read_json, write_to_json +from yunohost.utils.error import YunohostError from yunohost.hook import hook_list, hook_exec logger = log.getActionLogger('yunohost.diagnosis') @@ -55,7 +54,7 @@ def diagnosis_show(categories=[], full=False): else: unknown_categories = [ c for c in categories if c not in all_categories_names ] if unknown_categories: - raise MoulinetteError(m18n.n('unknown_categories', categories=", ".join(categories))) + raise YunohostError('unknown_categories', categories=", ".join(categories)) # Fetch all reports all_reports = [] @@ -88,7 +87,7 @@ def diagnosis_run(categories=[], force=False, args=None): else: unknown_categories = [ c for c in categories if c not in all_categories_names ] if unknown_categories: - raise MoulinetteError(m18n.n('unknown_categories', categories=", ".join(unknown_categories))) + raise YunohostError('unknown_categories', categories=", ".join(unknown_categories)) # Transform "arg1=val1&arg2=val2" to { "arg1": "val1", "arg2": "val2" } if args is not None: @@ -108,7 +107,7 @@ def diagnosis_run(categories=[], force=False, args=None): successes.append(category) except Exception as e: # FIXME / TODO : add stacktrace here ? - logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e))) # FIXME : i18n + logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e))) # FIXME : i18n return diagnosis_show(successes) From 3200fef39c3fb5966031d62153b02baa82f38dfb Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 21 May 2019 20:08:09 +0200 Subject: [PATCH 17/61] Implement detail mechanism for DNS category --- data/hooks/diagnosis/12-dns.py | 17 ++++++++++++----- src/yunohost/diagnosis.py | 3 +++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/data/hooks/diagnosis/12-dns.py b/data/hooks/diagnosis/12-dns.py index 9bf6a13a3..e6370ba05 100644 --- a/data/hooks/diagnosis/12-dns.py +++ b/data/hooks/diagnosis/12-dns.py @@ -54,8 +54,10 @@ class DNSDiagnoser(Diagnoser): current_value = self.get_current_record(domain, r["name"], r["type"]) or "None" expected_value = r["value"] if r["value"] != "@" else domain+"." - if current_value != expected_value: - discrepancies.append((r, expected_value, current_value)) + if current_value == "None": + discrepancies.append(("diagnosis_dns_missing_record", (r["type"], r["name"], expected_value))) + elif current_value != expected_value: + discrepancies.append(("diagnosis_dns_discrepancy", (r["type"], r["name"], expected_value, current_value))) if discrepancies: if category == "basic" or is_main_domain: @@ -66,11 +68,16 @@ class DNSDiagnoser(Diagnoser): else: level = "SUCCESS" report = ("SUCCESS", "diagnosis_dns_good_conf", {"domain": domain, "category": category}) + details = None - # FIXME : add management of details of what's wrong if there are discrepancies - yield dict(meta = {"domain": domain, "category": category}, - result = level, report = report ) + output = dict(meta = {"domain": domain, "category": category}, + result = level, + report = report ) + if discrepancies: + output["details"] = discrepancies + + yield output def get_current_record(self, domain, name, type_): diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 22770ce87..a8fae4124 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -73,6 +73,9 @@ def diagnosis_show(categories=[], full=False): type_, message_key, message_args = r["report"] r["report"] = (type_, m18n.n(message_key, **message_args)) + if "details" in r: + r["details"] = [ m18n.n(key, *values) for key, values in r["details"] ] + return {"reports": all_reports} def diagnosis_run(categories=[], force=False, args=None): From aafef0a8efad86a1c0b223bc7111133d388ce971 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 21 May 2019 20:08:31 +0200 Subject: [PATCH 18/61] Add i18n messages --- locales/en.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/locales/en.json b/locales/en.json index 77701478c..0245bf49d 100644 --- a/locales/en.json +++ b/locales/en.json @@ -155,8 +155,15 @@ "diagnosis_no_apps": "No installed application", "dpkg_is_broken": "You cannot do this right now because dpkg/APT (the system package managers) seems to be in a broken state… You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.", "dpkg_lock_not_available": "This command can't be ran right now because another program seems to be using the lock of dpkg (the system package manager)", + "diagnosis_network_connected_ipv4": "The server is connected to the Internet through IPv4 !", + "diagnosis_network_no_ipv4": "The server does not have a working IPv4.", + "diagnosis_network_connected_ipv6": "The server is connect to the Internet through IPv6 !", + "diagnosis_network_no_ipv6": "The server does not have a working IPv6.", "diagnosis_dns_good_conf": "Good DNS configuration for {domain} : {category}.", "diagnosis_dns_bad_conf": "Bad DNS configuration for {domain} : {category}.", + "diagnosis_dns_missing_record": "According to the recommended DNS configuration, you should add a DNS record with type {0}, name {1} and value {2}", + "diagnosis_dns_discrepancy": "According to the recommended DNS configuration, the value for the DNS record with type {0} and name {1} should be {2}, not {3}.", + "dns_configurations": "Domain name configuration (DNS)", "domain_cannot_remove_main": "Cannot remove main domain. Set a new main domain first", "domain_cert_gen_failed": "Could not generate certificate", "domain_created": "Domain created", @@ -238,6 +245,7 @@ "hook_name_unknown": "Unknown hook name '{name:s}'", "installation_complete": "Installation complete", "installation_failed": "Something went wrong with the installation", + "internet_connectivity": "Internet connectivity", "ip6tables_unavailable": "You cannot play with ip6tables here. You are either in a container or your kernel does not support it", "iptables_unavailable": "You cannot play with iptables here. You are either in a container or your kernel does not support it", "log_corrupted_md_file": "The YAML metadata file associated with logs is damaged: '{md_file}\nError: {error}'", From 06e02de548f4735ab9a9944ac784918700df9bb5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 13 Jul 2019 18:19:16 +0200 Subject: [PATCH 19/61] Add traceback for easier debugging --- src/yunohost/diagnosis.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index a8fae4124..99767e1b8 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -109,8 +109,7 @@ def diagnosis_run(categories=[], force=False, args=None): hook_exec(path, args=args, env=None) successes.append(category) except Exception as e: - # FIXME / TODO : add stacktrace here ? - logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e))) # FIXME : i18n + logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e)), exc_info=True) # FIXME : i18n return diagnosis_show(successes) From 1105b7d943d20dbe2939e5b4310803e9da744ec2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 13 Jul 2019 18:19:50 +0200 Subject: [PATCH 20/61] We don't need this auth madness anymore --- data/hooks/diagnosis/12-dns.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/data/hooks/diagnosis/12-dns.py b/data/hooks/diagnosis/12-dns.py index e6370ba05..3a61b0503 100644 --- a/data/hooks/diagnosis/12-dns.py +++ b/data/hooks/diagnosis/12-dns.py @@ -3,17 +3,11 @@ import os from moulinette.utils.network import download_text -from moulinette.core import init_authenticator from moulinette.utils.process import check_output from yunohost.diagnosis import Diagnoser from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain -# Instantiate LDAP Authenticator -auth_identifier = ('ldap', 'ldap-anonymous') -auth_parameters = {'uri': 'ldap://localhost:389', 'base_dn': 'dc=yunohost,dc=org'} -auth = init_authenticator(auth_identifier, auth_parameters) - class DNSDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] @@ -21,7 +15,7 @@ class DNSDiagnoser(Diagnoser): cache_duration = 3600*24 def validate_args(self, args): - all_domains = domain_list(auth)["domains"] + all_domains = domain_list()["domains"] if "domain" not in args.keys(): return { "domains" : all_domains } else: From bd3a378d285edd907858963d671ebd63cf76c210 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 13 Jul 2019 18:22:29 +0200 Subject: [PATCH 21/61] Use only ipv4 resolver for DNS records diagnosis --- data/hooks/diagnosis/12-dns.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/data/hooks/diagnosis/12-dns.py b/data/hooks/diagnosis/12-dns.py index 3a61b0503..90f52c82d 100644 --- a/data/hooks/diagnosis/12-dns.py +++ b/data/hooks/diagnosis/12-dns.py @@ -4,6 +4,7 @@ import os from moulinette.utils.network import download_text from moulinette.utils.process import check_output +from moulinette.utils.filesystem import read_file from yunohost.diagnosis import Diagnoser from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain @@ -24,8 +25,12 @@ class DNSDiagnoser(Diagnoser): def run(self): - self.resolver = check_output('grep "$nameserver" /etc/resolv.dnsmasq.conf').split("\n")[0].split(" ")[1] + resolvers = read_file("/etc/resolv.dnsmasq.conf").split("\n") + ipv4_resolvers = [r.split(" ")[1] for r in resolvers if r.startswith("nameserver") and ":" not in r] + # FIXME some day ... handle ipv4-only and ipv6-only servers. For now we assume we have at least ipv4 + assert ipv4_resolvers != [], "Uhoh, need at least one IPv4 DNS resolver ..." + self.resolver = ipv4_resolvers[0] main_domain = _get_maindomain() for domain in self.args["domains"]: From 0ce4eb0a27a8c6e128835bc9787d14d7ac294f04 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 13 Jul 2019 19:23:49 +0200 Subject: [PATCH 22/61] Fix the return interface of diagnosis hooks --- data/hooks/diagnosis/10-ip.py | 3 +-- data/hooks/diagnosis/12-dns.py | 3 +-- src/yunohost/diagnosis.py | 13 ++++++++----- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index d229eea8f..19e4806f6 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -55,6 +55,5 @@ class IPDiagnoser(Diagnoser): def main(args, env, loggers): - IPDiagnoser(args, env, loggers).diagnose() - return 0 + return IPDiagnoser(args, env, loggers).diagnose() diff --git a/data/hooks/diagnosis/12-dns.py b/data/hooks/diagnosis/12-dns.py index 90f52c82d..09f8cd4bf 100644 --- a/data/hooks/diagnosis/12-dns.py +++ b/data/hooks/diagnosis/12-dns.py @@ -92,6 +92,5 @@ class DNSDiagnoser(Diagnoser): def main(args, env, loggers): - DNSDiagnoser(args, env, loggers).diagnose() - return 0 + return DNSDiagnoser(args, env, loggers).diagnose() diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 99767e1b8..38c59793f 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -100,18 +100,19 @@ def diagnosis_run(categories=[], force=False, args=None): args["force"] = force # Call the hook ... - successes = [] + diagnosed_categories = [] for category in categories: logger.debug("Running diagnosis for %s ..." % category) path = [p for n, p in all_categories if n == category ][0] try: hook_exec(path, args=args, env=None) - successes.append(category) except Exception as e: logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e)), exc_info=True) # FIXME : i18n + else: + diagnosed_categories.append(category) - return diagnosis_show(successes) + return diagnosis_show(diagnosed_categories) def diagnosis_ignore(category, args="", unignore=False): pass @@ -125,8 +126,8 @@ class Diagnoser(): self.logger_debug, self.logger_warning, self.logger_info = loggers self.env = env - self.args = args - self.args.update(self.validate_args(args)) + self.args = args or {} + self.args.update(self.validate_args(self.args)) self.cache_file = Diagnoser.cache_file(self.id_) def cached_time_ago(self): @@ -158,6 +159,8 @@ class Diagnoser(): self.logger_debug("Updating cache %s" % self.cache_file) self.write_cache(new_report) + return 0, new_report + @staticmethod def cache_file(id_): return os.path.join(DIAGNOSIS_CACHE, "%s.json" % id_) From af23f53d8295affdc54cd3c7fc435e2d1c6bb205 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 13 Jul 2019 19:52:04 +0200 Subject: [PATCH 23/61] Simplify / reorganize i18n management for report and description --- data/hooks/diagnosis/10-ip.py | 1 - .../diagnosis/{12-dns.py => 12-dnsrecords.py} | 7 ++- locales/en.json | 4 +- src/yunohost/diagnosis.py | 44 +++++++++++++------ 4 files changed, 36 insertions(+), 20 deletions(-) rename data/hooks/diagnosis/{12-dns.py => 12-dnsrecords.py} (94%) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index 19e4806f6..665c0ff0d 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -10,7 +10,6 @@ from yunohost.diagnosis import Diagnoser class IPDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] - description = "internet_connectivity" cache_duration = 60 def validate_args(self, args): diff --git a/data/hooks/diagnosis/12-dns.py b/data/hooks/diagnosis/12-dnsrecords.py similarity index 94% rename from data/hooks/diagnosis/12-dns.py rename to data/hooks/diagnosis/12-dnsrecords.py index 09f8cd4bf..5edfc2d41 100644 --- a/data/hooks/diagnosis/12-dns.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -9,10 +9,9 @@ from moulinette.utils.filesystem import read_file from yunohost.diagnosis import Diagnoser from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain -class DNSDiagnoser(Diagnoser): +class DNSRecordsDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] - description = "dns_configurations" cache_duration = 3600*24 def validate_args(self, args): @@ -34,7 +33,7 @@ class DNSDiagnoser(Diagnoser): main_domain = _get_maindomain() for domain in self.args["domains"]: - self.logger_info("Diagnosing DNS conf for %s" % domain) + self.logger_debug("Diagnosing DNS conf for %s" % domain) for report in self.check_domain(domain, domain==main_domain): yield report @@ -92,5 +91,5 @@ class DNSDiagnoser(Diagnoser): def main(args, env, loggers): - return DNSDiagnoser(args, env, loggers).diagnose() + return DNSRecordsDiagnoser(args, env, loggers).diagnose() diff --git a/locales/en.json b/locales/en.json index 0245bf49d..0bb6d7275 100644 --- a/locales/en.json +++ b/locales/en.json @@ -163,7 +163,8 @@ "diagnosis_dns_bad_conf": "Bad DNS configuration for {domain} : {category}.", "diagnosis_dns_missing_record": "According to the recommended DNS configuration, you should add a DNS record with type {0}, name {1} and value {2}", "diagnosis_dns_discrepancy": "According to the recommended DNS configuration, the value for the DNS record with type {0} and name {1} should be {2}, not {3}.", - "dns_configurations": "Domain name configuration (DNS)", + "diagnosis_description_ip": "Internet connectivity", + "diagnosis_description_dnsrecords": "DNS records", "domain_cannot_remove_main": "Cannot remove main domain. Set a new main domain first", "domain_cert_gen_failed": "Could not generate certificate", "domain_created": "Domain created", @@ -245,7 +246,6 @@ "hook_name_unknown": "Unknown hook name '{name:s}'", "installation_complete": "Installation complete", "installation_failed": "Something went wrong with the installation", - "internet_connectivity": "Internet connectivity", "ip6tables_unavailable": "You cannot play with ip6tables here. You are either in a container or your kernel does not support it", "iptables_unavailable": "You cannot play with iptables here. You are either in a container or your kernel does not support it", "log_corrupted_md_file": "The YAML metadata file associated with logs is damaged: '{md_file}\nError: {error}'", diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 38c59793f..fb5220679 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -64,18 +64,6 @@ def diagnosis_show(categories=[], full=False): except Exception as e: logger.error("Failed to fetch diagnosis result for category '%s' : %s" % (category, str(e))) # FIXME : i18n - # "Render" the strings with m18n.n - for report in all_reports: - - report["description"] = m18n.n(report["description"]) - - for r in report["reports"]: - type_, message_key, message_args = r["report"] - r["report"] = (type_, m18n.n(message_key, **message_args)) - - if "details" in r: - r["details"] = [ m18n.n(key, *values) for key, values in r["details"] ] - return {"reports": all_reports} def diagnosis_run(categories=[], force=False, args=None): @@ -130,6 +118,13 @@ class Diagnoser(): self.args.update(self.validate_args(self.args)) self.cache_file = Diagnoser.cache_file(self.id_) + descr_key = "diagnosis_description_" + self.id_ + self.description = m18n.n(descr_key) + # If no description available, fallback to id + if self.description == descr_key: + self.description = report["id"] + + def cached_time_ago(self): if not os.path.exists(self.cache_file): @@ -145,12 +140,12 @@ class Diagnoser(): if not self.args.get("force", False) and self.cached_time_ago() < self.cache_duration: self.logger_debug("Cache still valid : %s" % self.cache_file) + # FIXME uhoh that's not consistent with the other return later return self.logger_debug("Running diagnostic for %s" % self.id_) new_report = { "id": self.id_, - "description": self.description, "cached_for": self.cache_duration, "reports": list(self.run()) } @@ -158,6 +153,7 @@ class Diagnoser(): # TODO / FIXME : should handle the case where we only did a partial diagnosis self.logger_debug("Updating cache %s" % self.cache_file) self.write_cache(new_report) + Diagnoser.i18n(new_report) return 0, new_report @@ -170,8 +166,30 @@ class Diagnoser(): filename = Diagnoser.cache_file(id_) report = read_json(filename) report["timestamp"] = int(os.path.getmtime(filename)) + Diagnoser.i18n(report) return report + @staticmethod + def i18n(report): + + # "Render" the strings with m18n.n + # N.B. : we do those m18n.n right now instead of saving the already-translated report + # because we can't be sure we'll redisplay the infos with the same locale as it + # was generated ... e.g. if the diagnosing happened inside a cron job with locale EN + # instead of FR used by the actual admin... + + descr_key = "diagnosis_description_" + report["id"] + report["description"] = m18n.n(descr_key) + # If no description available, fallback to id + if report["description"] == descr_key: + report["description"] = report["id"] + + for r in report["reports"]: + type_, message_key, message_args = r["report"] + r["report"] = (type_, m18n.n(message_key, **message_args)) + + if "details" in r: + r["details"] = [ m18n.n(key, *values) for key, values in r["details"] ] def _list_diagnosis_categories(): From 9405362caff317f5d90423c21803b0a00e94d66a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 13 Jul 2019 21:00:00 +0200 Subject: [PATCH 24/61] Cooler messages summarizing what's found, instead of displaying a huge unreadable wall of json/yaml --- src/yunohost/diagnosis.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index fb5220679..7297e6d4b 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -27,7 +27,7 @@ import os import time -from moulinette import m18n +from moulinette import m18n, msettings from moulinette.utils import log from moulinette.utils.filesystem import read_json, write_to_json @@ -87,6 +87,7 @@ def diagnosis_run(categories=[], force=False, args=None): args = {} args["force"] = force + found_issues = False # Call the hook ... diagnosed_categories = [] for category in categories: @@ -94,13 +95,23 @@ def diagnosis_run(categories=[], force=False, args=None): path = [p for n, p in all_categories if n == category ][0] try: - hook_exec(path, args=args, env=None) + code, report = hook_exec(path, args=args, env=None) except Exception as e: logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e)), exc_info=True) # FIXME : i18n else: diagnosed_categories.append(category) + if report != {}: + issues = [r for r in report["reports"] if r["report"][0] in ["ERROR", "WARNING"]] + if issues: + found_issues = True - return diagnosis_show(diagnosed_categories) + if found_issues: + if msettings.get("interface") == "api": + logger.info("You can go to the Diagnosis section (in the home screen) to see the issues found.") + else: + logger.info("You can run 'yunohost diagnosis show --issues' to display the issues found.") + + return def diagnosis_ignore(category, args="", unignore=False): pass @@ -140,8 +151,8 @@ class Diagnoser(): if not self.args.get("force", False) and self.cached_time_ago() < self.cache_duration: self.logger_debug("Cache still valid : %s" % self.cache_file) - # FIXME uhoh that's not consistent with the other return later - return + logger.info("(Cache still valid for %s diagnosis. Not re-diagnosing yet!)" % self.description) + return 0, {} self.logger_debug("Running diagnostic for %s" % self.id_) @@ -155,6 +166,17 @@ class Diagnoser(): self.write_cache(new_report) Diagnoser.i18n(new_report) + errors = [r for r in new_report["reports"] if r["report"][0] == "ERROR"] + warnings = [r for r in new_report["reports"] if r["report"][0] == "WARNING"] + + # FIXME : i18n + if errors: + logger.error("Found %s significant issue(s) related to %s!" % (len(errors), new_report["description"])) + elif warnings: + logger.warning("Found %s item(s) that could be improved for %s." % (len(warnings), new_report["description"])) + else: + logger.success("Everything looks good for %s!" % new_report["description"]) + return 0, new_report @staticmethod From 1d8ba7fa95305cf440d3a3888813bde13d5cc564 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 13 Jul 2019 21:38:35 +0200 Subject: [PATCH 25/61] Implement diagnosis show --full and --issues --- data/actionsmap/yunohost.yml | 3 +++ src/yunohost/diagnosis.py | 18 ++++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 47a858b27..6b89a819b 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1884,6 +1884,9 @@ diagnosis: --full: help: Display additional information action: store_true + --issues: + help: Only display issues + action: store_true run: action_help: Show most recents diagnosis results diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 7297e6d4b..de73bd680 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -42,7 +42,7 @@ def diagnosis_list(): all_categories_names = [ h for h, _ in _list_diagnosis_categories() ] return { "categories": all_categories_names } -def diagnosis_show(categories=[], full=False): +def diagnosis_show(categories=[], issues=False, full=False): # Get all the categories all_categories = _list_diagnosis_categories() @@ -60,9 +60,23 @@ def diagnosis_show(categories=[], full=False): all_reports = [] for category in categories: try: - all_reports.append(Diagnoser.get_cached_report(category)) + cat_report = Diagnoser.get_cached_report(category) except Exception as e: logger.error("Failed to fetch diagnosis result for category '%s' : %s" % (category, str(e))) # FIXME : i18n + else: + if not full: + del cat_report["timestamp"] + del cat_report["cached_for"] + for report in cat_report["reports"]: + del report["meta"] + del report["result"] + if issues: + cat_report["reports"] = [ r for r in cat_report["reports"] if r["report"][0] != "SUCCESS" ] + if not cat_report["reports"]: + continue + + all_reports.append(cat_report) + return {"reports": all_reports} From 41c3b054baf7640d5f97164643e9fe779885c843 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 13 Jul 2019 22:07:07 +0200 Subject: [PATCH 26/61] Fix semantic, way too many things called 'report' ... --- data/hooks/diagnosis/10-ip.py | 14 ++++---- data/hooks/diagnosis/12-dnsrecords.py | 16 ++++----- src/yunohost/diagnosis.py | 48 ++++++++++++++------------- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index 665c0ff0d..f38d1fadf 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -26,16 +26,18 @@ class IPDiagnoser(Diagnoser): if 4 in versions: ipv4 = self.get_public_ip(4) yield dict(meta = {"version": 4}, - result = ipv4, - report = ("SUCCESS", "diagnosis_network_connected_ipv4", {}) if ipv4 \ - else ("ERROR", "diagnosis_network_no_ipv4", {})) + data = ipv4, + status = "SUCCESS" if ipv4 else "ERROR", + summary = ("diagnosis_network_connected_ipv4", {}) if ipv4 \ + else ("diagnosis_network_no_ipv4", {})) if 6 in versions: ipv6 = self.get_public_ip(6) yield dict(meta = {"version": 6}, - result = ipv6, - report = ("SUCCESS", "diagnosis_network_connected_ipv6", {}) if ipv6 \ - else ("WARNING", "diagnosis_network_no_ipv6", {})) + data = ipv6, + status = "SUCCESS" if ipv6 else "WARNING", + summary = ("diagnosis_network_connected_ipv6", {}) if ipv6 \ + else ("diagnosis_network_no_ipv6", {})) def get_public_ip(self, protocol=4): diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 5edfc2d41..3ba64445d 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -58,19 +58,15 @@ class DNSRecordsDiagnoser(Diagnoser): discrepancies.append(("diagnosis_dns_discrepancy", (r["type"], r["name"], expected_value, current_value))) if discrepancies: - if category == "basic" or is_main_domain: - level = "ERROR" - else: - level = "WARNING" - report = (level, "diagnosis_dns_bad_conf", {"domain": domain, "category": category}) + status = "ERROR" if (category == "basic" or is_main_domain) else "WARNING" + summary = ("diagnosis_dns_bad_conf", {"domain": domain, "category": category}) else: - level = "SUCCESS" - report = ("SUCCESS", "diagnosis_dns_good_conf", {"domain": domain, "category": category}) - details = None + status = "SUCCESS" + summary = ("diagnosis_dns_good_conf", {"domain": domain, "category": category}) output = dict(meta = {"domain": domain, "category": category}, - result = level, - report = report ) + status = status, + summary = summary) if discrepancies: output["details"] = discrepancies diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index de73bd680..523a5c891 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -60,22 +60,24 @@ def diagnosis_show(categories=[], issues=False, full=False): all_reports = [] for category in categories: try: - cat_report = Diagnoser.get_cached_report(category) + report = Diagnoser.get_cached_report(category) except Exception as e: logger.error("Failed to fetch diagnosis result for category '%s' : %s" % (category, str(e))) # FIXME : i18n else: if not full: - del cat_report["timestamp"] - del cat_report["cached_for"] - for report in cat_report["reports"]: - del report["meta"] - del report["result"] + del report["timestamp"] + del report["cached_for"] + for item in report["items"]: + del item["meta"] + if "data" in item: + del item["data"] if issues: - cat_report["reports"] = [ r for r in cat_report["reports"] if r["report"][0] != "SUCCESS" ] - if not cat_report["reports"]: + report["items"] = [ item for item in report["items"] if item["status"] != "SUCCESS" ] + # Ignore this category if no issue was found + if not report["items"]: continue - all_reports.append(cat_report) + all_reports.append(report) return {"reports": all_reports} @@ -101,7 +103,7 @@ def diagnosis_run(categories=[], force=False, args=None): args = {} args["force"] = force - found_issues = False + issues = [] # Call the hook ... diagnosed_categories = [] for category in categories: @@ -115,11 +117,9 @@ def diagnosis_run(categories=[], force=False, args=None): else: diagnosed_categories.append(category) if report != {}: - issues = [r for r in report["reports"] if r["report"][0] in ["ERROR", "WARNING"]] - if issues: - found_issues = True + issues.extend([item for item in report["items"] if item["status"] != "SUCCESS"]) - if found_issues: + if issues: if msettings.get("interface") == "api": logger.info("You can go to the Diagnosis section (in the home screen) to see the issues found.") else: @@ -147,7 +147,7 @@ class Diagnoser(): self.description = m18n.n(descr_key) # If no description available, fallback to id if self.description == descr_key: - self.description = report["id"] + self.description = self.id_ def cached_time_ago(self): @@ -170,9 +170,11 @@ class Diagnoser(): self.logger_debug("Running diagnostic for %s" % self.id_) + items = list(self.run()) + new_report = { "id": self.id_, "cached_for": self.cache_duration, - "reports": list(self.run()) + "items": items } # TODO / FIXME : should handle the case where we only did a partial diagnosis @@ -180,8 +182,8 @@ class Diagnoser(): self.write_cache(new_report) Diagnoser.i18n(new_report) - errors = [r for r in new_report["reports"] if r["report"][0] == "ERROR"] - warnings = [r for r in new_report["reports"] if r["report"][0] == "WARNING"] + errors = [item for item in new_report["items"] if item["status"] == "ERROR"] + warnings = [item for item in new_report["items"] if item["status"] == "WARNING"] # FIXME : i18n if errors: @@ -220,12 +222,12 @@ class Diagnoser(): if report["description"] == descr_key: report["description"] = report["id"] - for r in report["reports"]: - type_, message_key, message_args = r["report"] - r["report"] = (type_, m18n.n(message_key, **message_args)) + for item in report["items"]: + summary_key, summary_args = item["summary"] + item["summary"] = m18n.n(summary_key, **summary_args) - if "details" in r: - r["details"] = [ m18n.n(key, *values) for key, values in r["details"] ] + if "details" in item: + item["details"] = [ m18n.n(key, *values) for key, values in item["details"] ] def _list_diagnosis_categories(): From 4d5ace06dbc3466013838c61c2f4efc8c15bfa69 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 13 Jul 2019 23:11:28 +0200 Subject: [PATCH 27/61] Add test that we can ping outside before talking to ip.yunohost.org --- data/hooks/diagnosis/10-ip.py | 54 ++++++++++++++++++++++++++- data/hooks/diagnosis/12-dnsrecords.py | 3 ++ locales/en.json | 4 +- 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index f38d1fadf..3259c6a4a 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -1,9 +1,12 @@ #!/usr/bin/env python import os +import random from moulinette import m18n from moulinette.utils.network import download_text +from moulinette.utils.process import check_output +from moulinette.utils.filesystem import read_file from yunohost.diagnosis import Diagnoser @@ -24,7 +27,12 @@ class IPDiagnoser(Diagnoser): versions = self.args["versions"] if 4 in versions: - ipv4 = self.get_public_ip(4) + + if not self.can_ping_outside(4): + ipv4 = None + else: + ipv4 = self.get_public_ip(4) + yield dict(meta = {"version": 4}, data = ipv4, status = "SUCCESS" if ipv4 else "ERROR", @@ -32,15 +40,57 @@ class IPDiagnoser(Diagnoser): else ("diagnosis_network_no_ipv4", {})) if 6 in versions: - ipv6 = self.get_public_ip(6) + + if not self.can_ping_outside(4): + ipv6 = None + else: + ipv6 = self.get_public_ip(6) + yield dict(meta = {"version": 6}, data = ipv6, status = "SUCCESS" if ipv6 else "WARNING", summary = ("diagnosis_network_connected_ipv6", {}) if ipv6 \ else ("diagnosis_network_no_ipv6", {})) + + def can_ping_outside(self, protocol=4): + + assert protocol in [4, 6], "Invalid protocol version, it should be either 4 or 6 and was '%s'" % repr(protocol) + + # We can know that ipv6 is not available directly if this file does not exists + if protocol == 6 and not os.path.exists("/proc/net/if_inet6"): + return False + + # If we are indeed connected in ipv4 or ipv6, we should find a default route + routes = check_output("ip -%s route" % protocol).split("\n") + if not [r for r in routes if r.startswith("default")]: + return False + + # We use the resolver file as a list of well-known, trustable (ie not google ;)) IPs that we can ping + resolver_file = "/usr/share/yunohost/templates/dnsmasq/plain/resolv.dnsmasq.conf" + resolvers = [r.split(" ")[1] for r in read_file(resolver_file).split("\n") if r.startswith("nameserver")] + + if protocol == 4: + resolvers = [r for r in resolvers if ":" not in r] + if protocol == 6: + resolvers = [r for r in resolvers if ":" in r] + + assert resolvers != [], "Uhoh, need at least one IPv%s DNS resolver in %s ..." % (protocol, resolver_file) + + # So let's try to ping the first 4~5 resolvers (shuffled) + # If we succesfully ping any of them, we conclude that we are indeed connected + def ping(protocol, target): + return os.system("ping -c1 -%s -W 3 %s >/dev/null 2>/dev/null" % (protocol, target)) == 0 + + random.shuffle(resolvers) + return any(ping(protocol, resolver) for resolver in resolvers[:5]) + def get_public_ip(self, protocol=4): + # FIXME - TODO : here we assume that DNS resolution for ip.yunohost.org is working + # but if we want to be able to diagnose DNS resolution issues independently from + # internet connectivity, we gotta rely on fixed IPs first.... + if protocol == 4: url = 'https://ip.yunohost.org' elif protocol == 6: diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 3ba64445d..c8b81fd2c 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -79,6 +79,9 @@ class DNSRecordsDiagnoser(Diagnoser): command = "dig +short @%s %s %s" % (self.resolver, type_, domain) else: command = "dig +short @%s %s %s.%s" % (self.resolver, type_, name, domain) + # FIXME : gotta handle case where this command fails ... + # e.g. no internet connectivity (dependency mechanism to good result from 'ip' diagosis ?) + # or the resolver is unavailable for some reason output = check_output(command).strip() output = output.replace("\;",";") if output.startswith('"') and output.endswith('"'): diff --git a/locales/en.json b/locales/en.json index 0bb6d7275..ae5e4dc53 100644 --- a/locales/en.json +++ b/locales/en.json @@ -159,8 +159,8 @@ "diagnosis_network_no_ipv4": "The server does not have a working IPv4.", "diagnosis_network_connected_ipv6": "The server is connect to the Internet through IPv6 !", "diagnosis_network_no_ipv6": "The server does not have a working IPv6.", - "diagnosis_dns_good_conf": "Good DNS configuration for {domain} : {category}.", - "diagnosis_dns_bad_conf": "Bad DNS configuration for {domain} : {category}.", + "diagnosis_dns_good_conf": "Good DNS configuration for domain {domain} (category {category})", + "diagnosis_dns_bad_conf": "Bad DNS configuration for domain {domain} (category {category})", "diagnosis_dns_missing_record": "According to the recommended DNS configuration, you should add a DNS record with type {0}, name {1} and value {2}", "diagnosis_dns_discrepancy": "According to the recommended DNS configuration, the value for the DNS record with type {0} and name {1} should be {2}, not {3}.", "diagnosis_description_ip": "Internet connectivity", From 5f4450ab87f4a0985a877539cba5fc40231c8555 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 14 Jul 2019 00:35:42 +0200 Subject: [PATCH 28/61] Add DNS resolution tests --- data/hooks/diagnosis/10-ip.py | 52 +++++++++++++++++++++++++++++++---- locales/en.json | 11 +++++--- 2 files changed, 54 insertions(+), 9 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index 3259c6a4a..1835927a2 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -28,16 +28,44 @@ class IPDiagnoser(Diagnoser): if 4 in versions: + # If we can't ping, there's not much else we can do if not self.can_ping_outside(4): ipv4 = None + # If we do ping, check that we can resolv domain name else: - ipv4 = self.get_public_ip(4) + can_resolve_dns = self.can_resolve_dns() + # And if we do, then we can fetch the public ip + if can_resolve_dns: + ipv4 = self.get_public_ip(4) + # In every case, we can check that resolvconf seems to be okay + # (symlink managed by resolvconf service + pointing to dnsmasq) + good_resolvconf = self.resolvconf_is_symlink() and self.resolvconf_points_to_localhost() + + # If we can't resolve domain names at all, that's a pretty big issue ... + # If it turns out that at the same time, resolvconf is bad, that's probably + # the cause of this, so we use a different message in that case + if not can_resolve_dns: + yield dict(meta = {"name": "dnsresolution"}, + status = "ERROR", + summary = ("diagnosis_ip_broken_dnsresolution", {}) if good_resolvconf + else ("diagnosis_ip_broken_resolvconf", {})) + # Otherwise, if the resolv conf is bad but we were able to resolve domain name, + # still warn that we're using a weird resolv conf ... + elif not good_resolvconf: + yield dict(meta = {"name": "dnsresolution"}, + status = "WARNING", + summary = ("diagnosis_ip_weird_resolvconf", {})) + else: + # Well, maybe we could report a "success", "dns resolution is working", idk if it's worth it + pass + + # And finally, we actually report the ipv4 connectivity stuff yield dict(meta = {"version": 4}, data = ipv4, status = "SUCCESS" if ipv4 else "ERROR", - summary = ("diagnosis_network_connected_ipv4", {}) if ipv4 \ - else ("diagnosis_network_no_ipv4", {})) + summary = ("diagnosis_ip_connected_ipv4", {}) if ipv4 \ + else ("diagnosis_ip_no_ipv4", {})) if 6 in versions: @@ -49,8 +77,8 @@ class IPDiagnoser(Diagnoser): yield dict(meta = {"version": 6}, data = ipv6, status = "SUCCESS" if ipv6 else "WARNING", - summary = ("diagnosis_network_connected_ipv6", {}) if ipv6 \ - else ("diagnosis_network_no_ipv6", {})) + summary = ("diagnosis_ip_connected_ipv6", {}) if ipv6 \ + else ("diagnosis_ip_no_ipv6", {})) def can_ping_outside(self, protocol=4): @@ -85,6 +113,20 @@ class IPDiagnoser(Diagnoser): random.shuffle(resolvers) return any(ping(protocol, resolver) for resolver in resolvers[:5]) + + def can_resolve_dns(self): + return os.system("dig +short ip.yunohost.org >/dev/null 2>/dev/null") == 0 + + + def resolvconf_is_symlink(self): + return os.path.realpath("/etc/resolv.conf") == "/run/resolvconf/resolv.conf" + + def resolvconf_points_to_localhost(self): + file_ = "/etc/resolv.conf" + resolvers = [r.split(" ")[1] for r in read_file(file_).split("\n") if r.startswith("nameserver")] + return resolvers == ["127.0.0.1"] + + def get_public_ip(self, protocol=4): # FIXME - TODO : here we assume that DNS resolution for ip.yunohost.org is working diff --git a/locales/en.json b/locales/en.json index ae5e4dc53..515993884 100644 --- a/locales/en.json +++ b/locales/en.json @@ -155,10 +155,13 @@ "diagnosis_no_apps": "No installed application", "dpkg_is_broken": "You cannot do this right now because dpkg/APT (the system package managers) seems to be in a broken state… You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.", "dpkg_lock_not_available": "This command can't be ran right now because another program seems to be using the lock of dpkg (the system package manager)", - "diagnosis_network_connected_ipv4": "The server is connected to the Internet through IPv4 !", - "diagnosis_network_no_ipv4": "The server does not have a working IPv4.", - "diagnosis_network_connected_ipv6": "The server is connect to the Internet through IPv6 !", - "diagnosis_network_no_ipv6": "The server does not have a working IPv6.", + "diagnosis_ip_connected_ipv4": "The server is connected to the Internet through IPv4 !", + "diagnosis_ip_no_ipv4": "The server does not have a working IPv4.", + "diagnosis_ip_connected_ipv6": "The server is connected to the Internet through IPv6 !", + "diagnosis_ip_no_ipv6": "The server does not have a working IPv6.", + "diagnosis_ip_broken_dnsresolution": "Domain name resolution seems to be broken for some reason ... Is a firewall blocking DNS requests ?", + "diagnosis_ip_broken_resolvconf": "Domain name resolution seems to be broken on your server, which seems related to /etc/resolv.conf not pointing to 127.0.0.1.", + "diagnosis_ip_weird_resolvconf": "Be careful that you seem to be using a custom /etc/resolv.conf. Instead, this file should be a symlink to /etc/resolvconf/run/resolv.conf itself pointing to 127.0.0.1 (dnsmasq).", "diagnosis_dns_good_conf": "Good DNS configuration for domain {domain} (category {category})", "diagnosis_dns_bad_conf": "Bad DNS configuration for domain {domain} (category {category})", "diagnosis_dns_missing_record": "According to the recommended DNS configuration, you should add a DNS record with type {0}, name {1} and value {2}", From aed53786f2f6e3cac78af79205d956a366b4efc3 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 14 Jul 2019 00:45:09 +0200 Subject: [PATCH 29/61] Make the PEP8 gods less angry --- data/hooks/diagnosis/10-ip.py | 45 ++++++++++++--------------- data/hooks/diagnosis/12-dnsrecords.py | 25 +++++++-------- src/yunohost/diagnosis.py | 39 ++++++++++++----------- 3 files changed, 51 insertions(+), 58 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index 1835927a2..1f6c31f50 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -3,13 +3,13 @@ import os import random -from moulinette import m18n from moulinette.utils.network import download_text from moulinette.utils.process import check_output from moulinette.utils.filesystem import read_file from yunohost.diagnosis import Diagnoser + class IPDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] @@ -17,10 +17,10 @@ class IPDiagnoser(Diagnoser): def validate_args(self, args): if "version" not in args.keys(): - return { "versions" : [4, 6] } + return {"versions": [4, 6]} else: assert str(args["version"]) in ["4", "6"], "Invalid version, should be 4 or 6." - return { "versions" : [int(args["version"])] } + return {"versions": [int(args["version"])]} def run(self): @@ -46,26 +46,26 @@ class IPDiagnoser(Diagnoser): # If it turns out that at the same time, resolvconf is bad, that's probably # the cause of this, so we use a different message in that case if not can_resolve_dns: - yield dict(meta = {"name": "dnsresolution"}, - status = "ERROR", - summary = ("diagnosis_ip_broken_dnsresolution", {}) if good_resolvconf - else ("diagnosis_ip_broken_resolvconf", {})) + yield dict(meta={"name": "dnsresolution"}, + status="ERROR", + summary=("diagnosis_ip_broken_dnsresolution", {}) if good_resolvconf + else ("diagnosis_ip_broken_resolvconf", {})) # Otherwise, if the resolv conf is bad but we were able to resolve domain name, # still warn that we're using a weird resolv conf ... elif not good_resolvconf: - yield dict(meta = {"name": "dnsresolution"}, - status = "WARNING", - summary = ("diagnosis_ip_weird_resolvconf", {})) + yield dict(meta={"name": "dnsresolution"}, + status="WARNING", + summary=("diagnosis_ip_weird_resolvconf", {})) else: # Well, maybe we could report a "success", "dns resolution is working", idk if it's worth it pass # And finally, we actually report the ipv4 connectivity stuff - yield dict(meta = {"version": 4}, - data = ipv4, - status = "SUCCESS" if ipv4 else "ERROR", - summary = ("diagnosis_ip_connected_ipv4", {}) if ipv4 \ - else ("diagnosis_ip_no_ipv4", {})) + yield dict(meta={"version": 4}, + data=ipv4, + status="SUCCESS" if ipv4 else "ERROR", + summary=("diagnosis_ip_connected_ipv4", {}) if ipv4 + else ("diagnosis_ip_no_ipv4", {})) if 6 in versions: @@ -74,12 +74,11 @@ class IPDiagnoser(Diagnoser): else: ipv6 = self.get_public_ip(6) - yield dict(meta = {"version": 6}, - data = ipv6, - status = "SUCCESS" if ipv6 else "WARNING", - summary = ("diagnosis_ip_connected_ipv6", {}) if ipv6 \ - else ("diagnosis_ip_no_ipv6", {})) - + yield dict(meta={"version": 6}, + data=ipv6, + status="SUCCESS" if ipv6 else "WARNING", + summary=("diagnosis_ip_connected_ipv6", {}) if ipv6 + else ("diagnosis_ip_no_ipv6", {})) def can_ping_outside(self, protocol=4): @@ -113,11 +112,9 @@ class IPDiagnoser(Diagnoser): random.shuffle(resolvers) return any(ping(protocol, resolver) for resolver in resolvers[:5]) - def can_resolve_dns(self): return os.system("dig +short ip.yunohost.org >/dev/null 2>/dev/null") == 0 - def resolvconf_is_symlink(self): return os.path.realpath("/etc/resolv.conf") == "/run/resolvconf/resolv.conf" @@ -126,7 +123,6 @@ class IPDiagnoser(Diagnoser): resolvers = [r.split(" ")[1] for r in read_file(file_).split("\n") if r.startswith("nameserver")] return resolvers == ["127.0.0.1"] - def get_public_ip(self, protocol=4): # FIXME - TODO : here we assume that DNS resolution for ip.yunohost.org is working @@ -149,4 +145,3 @@ class IPDiagnoser(Diagnoser): def main(args, env, loggers): return IPDiagnoser(args, env, loggers).diagnose() - diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index c8b81fd2c..493010c59 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -2,25 +2,25 @@ import os -from moulinette.utils.network import download_text from moulinette.utils.process import check_output from moulinette.utils.filesystem import read_file from yunohost.diagnosis import Diagnoser from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain + class DNSRecordsDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] - cache_duration = 3600*24 + cache_duration = 3600 * 24 def validate_args(self, args): all_domains = domain_list()["domains"] if "domain" not in args.keys(): - return { "domains" : all_domains } + return {"domains": all_domains} else: assert args["domain"] in all_domains, "Unknown domain" - return { "domains" : [ args["domain"] ] } + return {"domains": [args["domain"]]} def run(self): @@ -34,7 +34,7 @@ class DNSRecordsDiagnoser(Diagnoser): for domain in self.args["domains"]: self.logger_debug("Diagnosing DNS conf for %s" % domain) - for report in self.check_domain(domain, domain==main_domain): + for report in self.check_domain(domain, domain == main_domain): yield report def check_domain(self, domain, is_main_domain): @@ -44,13 +44,13 @@ class DNSRecordsDiagnoser(Diagnoser): # Here if there are no AAAA record, we should add something to expect "no" AAAA record # to properly diagnose situations where people have a AAAA record but no IPv6 - for category, records in expected_configuration.items(): + for category, records in expected_configuration.items(): discrepancies = [] for r in records: current_value = self.get_current_record(domain, r["name"], r["type"]) or "None" - expected_value = r["value"] if r["value"] != "@" else domain+"." + expected_value = r["value"] if r["value"] != "@" else domain + "." if current_value == "None": discrepancies.append(("diagnosis_dns_missing_record", (r["type"], r["name"], expected_value))) @@ -64,16 +64,15 @@ class DNSRecordsDiagnoser(Diagnoser): status = "SUCCESS" summary = ("diagnosis_dns_good_conf", {"domain": domain, "category": category}) - output = dict(meta = {"domain": domain, "category": category}, - status = status, - summary = summary) + output = dict(meta={"domain": domain, "category": category}, + status=status, + summary=summary) if discrepancies: output["details"] = discrepancies yield output - def get_current_record(self, domain, name, type_): if name == "@": command = "dig +short @%s %s %s" % (self.resolver, type_, domain) @@ -83,12 +82,10 @@ class DNSRecordsDiagnoser(Diagnoser): # e.g. no internet connectivity (dependency mechanism to good result from 'ip' diagosis ?) # or the resolver is unavailable for some reason output = check_output(command).strip() - output = output.replace("\;",";") if output.startswith('"') and output.endswith('"'): - output = '"' + ' '.join(output.replace('"',' ').split()) + '"' + output = '"' + ' '.join(output.replace('"', ' ').split()) + '"' return output def main(args, env, loggers): return DNSRecordsDiagnoser(args, env, loggers).diagnose() - diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 523a5c891..9b17a7457 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -38,21 +38,23 @@ logger = log.getActionLogger('yunohost.diagnosis') DIAGNOSIS_CACHE = "/var/cache/yunohost/diagnosis/" + def diagnosis_list(): - all_categories_names = [ h for h, _ in _list_diagnosis_categories() ] - return { "categories": all_categories_names } + all_categories_names = [h for h, _ in _list_diagnosis_categories()] + return {"categories": all_categories_names} + def diagnosis_show(categories=[], issues=False, full=False): # Get all the categories all_categories = _list_diagnosis_categories() - all_categories_names = [ category for category, _ in all_categories ] + all_categories_names = [category for category, _ in all_categories] # Check the requested category makes sense if categories == []: categories = all_categories_names else: - unknown_categories = [ c for c in categories if c not in all_categories_names ] + unknown_categories = [c for c in categories if c not in all_categories_names] if unknown_categories: raise YunohostError('unknown_categories', categories=", ".join(categories)) @@ -62,7 +64,7 @@ def diagnosis_show(categories=[], issues=False, full=False): try: report = Diagnoser.get_cached_report(category) except Exception as e: - logger.error("Failed to fetch diagnosis result for category '%s' : %s" % (category, str(e))) # FIXME : i18n + logger.error("Failed to fetch diagnosis result for category '%s' : %s" % (category, str(e))) # FIXME : i18n else: if not full: del report["timestamp"] @@ -72,33 +74,33 @@ def diagnosis_show(categories=[], issues=False, full=False): if "data" in item: del item["data"] if issues: - report["items"] = [ item for item in report["items"] if item["status"] != "SUCCESS" ] + report["items"] = [item for item in report["items"] if item["status"] != "SUCCESS"] # Ignore this category if no issue was found if not report["items"]: continue all_reports.append(report) - return {"reports": all_reports} + def diagnosis_run(categories=[], force=False, args=None): # Get all the categories all_categories = _list_diagnosis_categories() - all_categories_names = [ category for category, _ in all_categories ] + all_categories_names = [category for category, _ in all_categories] # Check the requested category makes sense if categories == []: categories = all_categories_names else: - unknown_categories = [ c for c in categories if c not in all_categories_names ] + unknown_categories = [c for c in categories if c not in all_categories_names] if unknown_categories: raise YunohostError('unknown_categories', categories=", ".join(unknown_categories)) # Transform "arg1=val1&arg2=val2" to { "arg1": "val1", "arg2": "val2" } if args is not None: - args = { arg.split("=")[0]: arg.split("=")[1] for arg in args.split("&") } + args = {arg.split("=")[0]: arg.split("=")[1] for arg in args.split("&")} else: args = {} args["force"] = force @@ -108,12 +110,12 @@ def diagnosis_run(categories=[], force=False, args=None): diagnosed_categories = [] for category in categories: logger.debug("Running diagnosis for %s ..." % category) - path = [p for n, p in all_categories if n == category ][0] + path = [p for n, p in all_categories if n == category][0] try: code, report = hook_exec(path, args=args, env=None) except Exception as e: - logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e)), exc_info=True) # FIXME : i18n + logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e)), exc_info=True) # FIXME : i18n else: diagnosed_categories.append(category) if report != {}: @@ -127,6 +129,7 @@ def diagnosis_run(categories=[], force=False, args=None): return + def diagnosis_ignore(category, args="", unignore=False): pass @@ -149,7 +152,6 @@ class Diagnoser(): if self.description == descr_key: self.description = self.id_ - def cached_time_ago(self): if not os.path.exists(self.cache_file): @@ -172,10 +174,9 @@ class Diagnoser(): items = list(self.run()) - new_report = { "id": self.id_, - "cached_for": self.cache_duration, - "items": items - } + new_report = {"id": self.id_, + "cached_for": self.cache_duration, + "items": items} # TODO / FIXME : should handle the case where we only did a partial diagnosis self.logger_debug("Updating cache %s" % self.cache_file) @@ -227,13 +228,13 @@ class Diagnoser(): item["summary"] = m18n.n(summary_key, **summary_args) if "details" in item: - item["details"] = [ m18n.n(key, *values) for key, values in item["details"] ] + item["details"] = [m18n.n(key, *values) for key, values in item["details"]] def _list_diagnosis_categories(): hooks_raw = hook_list("diagnosis", list_by="priority", show_info=True)["hooks"] hooks = [] - for _, some_hooks in sorted(hooks_raw.items(), key=lambda h:int(h[0])): + for _, some_hooks in sorted(hooks_raw.items(), key=lambda h: int(h[0])): for name, info in some_hooks.items(): hooks.append((name, info["path"])) From 1019e95b1d6d4d094c884d2c902a66a3d3deafa4 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 20 Jul 2019 18:44:32 +0200 Subject: [PATCH 30/61] Implement a first version for services status check --- data/hooks/diagnosis/30-services.py | 54 +++++++++++++++++++++++++++++ locales/en.json | 3 ++ 2 files changed, 57 insertions(+) create mode 100644 data/hooks/diagnosis/30-services.py diff --git a/data/hooks/diagnosis/30-services.py b/data/hooks/diagnosis/30-services.py new file mode 100644 index 000000000..4f08247f1 --- /dev/null +++ b/data/hooks/diagnosis/30-services.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python + +import os + +from yunohost.diagnosis import Diagnoser +from yunohost.service import service_status + +# TODO : all these are arbitrary, should be collectively validated +services_ignored = {"glances"} +services_critical = {"dnsmasq", "fail2ban", "yunohost-firewall", "nginx", "slapd", "ssh"} +# TODO / FIXME : we should do something about this postfix thing +# The nominal value is to be "exited" ... some daemon is actually running +# in a different thread that the thing started by systemd, which is fine +# but somehow sometimes it gets killed and there's no easy way to detect it +# Just randomly restarting it will fix ths issue. We should find some trick +# to identify the PID of the process and check it's still up or idk +services_expected_to_be_exited = {"postfix", "yunohost-firewall"} + +class ServicesDiagnoser(Diagnoser): + + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] + cache_duration = 300 + + def validate_args(self, args): + # TODO / FIXME Ugh do we really need this arg system + return {} + + def run(self): + + all_result = service_status() + + for service, result in all_result.items(): + + if service in services_ignored: + continue + + item = dict(meta={"service": service}) + expected_status = "running" if service not in services_expected_to_be_exited else "exited" + + # TODO / FIXME : might also want to check that services are enabled + + if result["active"] != "active" or result["status"] != expected_status: + item["status"] = "WARNING" if service not in services_critical else "ERROR" + item["summary"] = ("diagnosis_services_bad_status", {"service": service, "status": result["active"] + "/" + result["status"]}) + + # TODO : could try to append the tail of the service log to the "details" key ... + else: + item["status"] = "SUCCESS" + item["summary"] = ("diagnosis_services_good_status", {"service": service, "status": result["active"] + "/" + result["status"]}) + + yield item + +def main(args, env, loggers): + return ServicesDiagnoser(args, env, loggers).diagnose() diff --git a/locales/en.json b/locales/en.json index 515993884..8fcb0e773 100644 --- a/locales/en.json +++ b/locales/en.json @@ -166,8 +166,11 @@ "diagnosis_dns_bad_conf": "Bad DNS configuration for domain {domain} (category {category})", "diagnosis_dns_missing_record": "According to the recommended DNS configuration, you should add a DNS record with type {0}, name {1} and value {2}", "diagnosis_dns_discrepancy": "According to the recommended DNS configuration, the value for the DNS record with type {0} and name {1} should be {2}, not {3}.", + "diagnosis_services_good_status": "Service {service} is {status} as expected!", + "diagnosis_services_bad_status": "Service {service} is {status} :/", "diagnosis_description_ip": "Internet connectivity", "diagnosis_description_dnsrecords": "DNS records", + "diagnosis_description_services": "Services status check", "domain_cannot_remove_main": "Cannot remove main domain. Set a new main domain first", "domain_cert_gen_failed": "Could not generate certificate", "domain_created": "Domain created", From 24f9d475b8d79fbf5c57034cba49d4bee013fea5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 20 Jul 2019 18:44:53 +0200 Subject: [PATCH 31/61] Implement a first version for disk usage check --- data/hooks/diagnosis/50-diskusage.py | 42 ++++++++++++++++++++++++++++ locales/en.json | 4 +++ 2 files changed, 46 insertions(+) create mode 100644 data/hooks/diagnosis/50-diskusage.py diff --git a/data/hooks/diagnosis/50-diskusage.py b/data/hooks/diagnosis/50-diskusage.py new file mode 100644 index 000000000..84ce3845c --- /dev/null +++ b/data/hooks/diagnosis/50-diskusage.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +import os +import psutil + +from yunohost.diagnosis import Diagnoser + +class DiskUsageDiagnoser(Diagnoser): + + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] + cache_duration = 3600 * 24 + + def validate_args(self, args): + # TODO / FIXME Ugh do we really need this arg system + return {} + + def run(self): + + disk_partitions = psutil.disk_partitions() + + for disk_partition in disk_partitions: + device = disk_partition.device + mountpoint = disk_partition.mountpoint + + usage = psutil.disk_usage(mountpoint) + free_Go = usage.free / (1024 ** 3) + free_percent = 100 - usage.percent + + item = dict(meta={"mountpoint": mountpoint, "device": device}) + if free_Go < 1 or free_percent < 5: + item["status"] = "ERROR" + item["summary"] = ("diagnosis_diskusage_verylow", {"mountpoint": mountpoint, "device": device, "free_percent": free_percent}) + elif free_Go < 2 or free_percent < 10: + item["status"] = "WARNING" + item["summary"] = ("diagnosis_diskusage_low", {"mountpoint": mountpoint, "device": device, "free_percent": free_percent}) + else: + item["status"] = "SUCCESS" + item["summary"] = ("diagnosis_diskusage_ok", {"mountpoint": mountpoint, "device": device, "free_percent": free_percent}) + + yield item + +def main(args, env, loggers): + return DiskUsageDiagnoser(args, env, loggers).diagnose() diff --git a/locales/en.json b/locales/en.json index 8fcb0e773..2e93e367f 100644 --- a/locales/en.json +++ b/locales/en.json @@ -168,9 +168,13 @@ "diagnosis_dns_discrepancy": "According to the recommended DNS configuration, the value for the DNS record with type {0} and name {1} should be {2}, not {3}.", "diagnosis_services_good_status": "Service {service} is {status} as expected!", "diagnosis_services_bad_status": "Service {service} is {status} :/", + "diagnosis_diskusage_verylow": "Storage {mountpoint} (on device {device}) has only {free_percent}% space remaining. You should really consider cleaning up some space.", + "diagnosis_diskusage_low": "Storage {mountpoint} (on device {device}) has only {free_percent}% space remaining. Be careful", + "diagnosis_diskusage_ok": "Storage {mountpoint} (on device {device}) still has {free_percent}% space left!", "diagnosis_description_ip": "Internet connectivity", "diagnosis_description_dnsrecords": "DNS records", "diagnosis_description_services": "Services status check", + "diagnosis_description_diskusage": "Disk usage", "domain_cannot_remove_main": "Cannot remove main domain. Set a new main domain first", "domain_cert_gen_failed": "Could not generate certificate", "domain_created": "Domain created", From d2bbb5a2b31718054365a0ee5d63c2298776f32e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 20 Jul 2019 19:02:11 +0200 Subject: [PATCH 32/61] This 'args' things sounds like a big YAGNI after all --- data/actionsmap/yunohost.yml | 3 - data/hooks/diagnosis/10-ip.py | 105 ++++++++++++-------------- data/hooks/diagnosis/12-dnsrecords.py | 11 +-- data/hooks/diagnosis/30-services.py | 4 - data/hooks/diagnosis/50-diskusage.py | 4 - src/yunohost/diagnosis.py | 12 +-- 6 files changed, 54 insertions(+), 85 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 6b89a819b..3d72bb57a 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1898,9 +1898,6 @@ diagnosis: --force: help: Ignore the cached report even if it is still 'fresh' action: store_true - -a: - help: Serialized arguments for diagnosis scripts (e.g. "domain=domain.tld") - full: --args ignore: action_help: Configure some diagnosis results to be ignored diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index 1f6c31f50..4a6ee75ce 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -15,70 +15,65 @@ class IPDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] cache_duration = 60 - def validate_args(self, args): - if "version" not in args.keys(): - return {"versions": [4, 6]} - else: - assert str(args["version"]) in ["4", "6"], "Invalid version, should be 4 or 6." - return {"versions": [int(args["version"])]} - def run(self): - versions = self.args["versions"] + # + # IPv4 Diagnosis + # - if 4 in versions: + # If we can't ping, there's not much else we can do + if not self.can_ping_outside(4): + ipv4 = None + # If we do ping, check that we can resolv domain name + else: + can_resolve_dns = self.can_resolve_dns() + # And if we do, then we can fetch the public ip + if can_resolve_dns: + ipv4 = self.get_public_ip(4) - # If we can't ping, there's not much else we can do - if not self.can_ping_outside(4): - ipv4 = None - # If we do ping, check that we can resolv domain name - else: - can_resolve_dns = self.can_resolve_dns() - # And if we do, then we can fetch the public ip - if can_resolve_dns: - ipv4 = self.get_public_ip(4) + # In every case, we can check that resolvconf seems to be okay + # (symlink managed by resolvconf service + pointing to dnsmasq) + good_resolvconf = self.resolvconf_is_symlink() and self.resolvconf_points_to_localhost() - # In every case, we can check that resolvconf seems to be okay - # (symlink managed by resolvconf service + pointing to dnsmasq) - good_resolvconf = self.resolvconf_is_symlink() and self.resolvconf_points_to_localhost() + # If we can't resolve domain names at all, that's a pretty big issue ... + # If it turns out that at the same time, resolvconf is bad, that's probably + # the cause of this, so we use a different message in that case + if not can_resolve_dns: + yield dict(meta={"name": "dnsresolution"}, + status="ERROR", + summary=("diagnosis_ip_broken_dnsresolution", {}) if good_resolvconf + else ("diagnosis_ip_broken_resolvconf", {})) + # Otherwise, if the resolv conf is bad but we were able to resolve domain name, + # still warn that we're using a weird resolv conf ... + elif not good_resolvconf: + yield dict(meta={"name": "dnsresolution"}, + status="WARNING", + summary=("diagnosis_ip_weird_resolvconf", {})) + else: + # Well, maybe we could report a "success", "dns resolution is working", idk if it's worth it + pass - # If we can't resolve domain names at all, that's a pretty big issue ... - # If it turns out that at the same time, resolvconf is bad, that's probably - # the cause of this, so we use a different message in that case - if not can_resolve_dns: - yield dict(meta={"name": "dnsresolution"}, - status="ERROR", - summary=("diagnosis_ip_broken_dnsresolution", {}) if good_resolvconf - else ("diagnosis_ip_broken_resolvconf", {})) - # Otherwise, if the resolv conf is bad but we were able to resolve domain name, - # still warn that we're using a weird resolv conf ... - elif not good_resolvconf: - yield dict(meta={"name": "dnsresolution"}, - status="WARNING", - summary=("diagnosis_ip_weird_resolvconf", {})) - else: - # Well, maybe we could report a "success", "dns resolution is working", idk if it's worth it - pass + # And finally, we actually report the ipv4 connectivity stuff + yield dict(meta={"version": 4}, + data=ipv4, + status="SUCCESS" if ipv4 else "ERROR", + summary=("diagnosis_ip_connected_ipv4", {}) if ipv4 + else ("diagnosis_ip_no_ipv4", {})) - # And finally, we actually report the ipv4 connectivity stuff - yield dict(meta={"version": 4}, - data=ipv4, - status="SUCCESS" if ipv4 else "ERROR", - summary=("diagnosis_ip_connected_ipv4", {}) if ipv4 - else ("diagnosis_ip_no_ipv4", {})) + # + # IPv6 Diagnosis + # - if 6 in versions: + if not self.can_ping_outside(4): + ipv6 = None + else: + ipv6 = self.get_public_ip(6) - if not self.can_ping_outside(4): - ipv6 = None - else: - ipv6 = self.get_public_ip(6) - - yield dict(meta={"version": 6}, - data=ipv6, - status="SUCCESS" if ipv6 else "WARNING", - summary=("diagnosis_ip_connected_ipv6", {}) if ipv6 - else ("diagnosis_ip_no_ipv6", {})) + yield dict(meta={"version": 6}, + data=ipv6, + status="SUCCESS" if ipv6 else "WARNING", + summary=("diagnosis_ip_connected_ipv6", {}) if ipv6 + else ("diagnosis_ip_no_ipv6", {})) def can_ping_outside(self, protocol=4): diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 493010c59..0f47ff136 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -14,14 +14,6 @@ class DNSRecordsDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] cache_duration = 3600 * 24 - def validate_args(self, args): - all_domains = domain_list()["domains"] - if "domain" not in args.keys(): - return {"domains": all_domains} - else: - assert args["domain"] in all_domains, "Unknown domain" - return {"domains": [args["domain"]]} - def run(self): resolvers = read_file("/etc/resolv.dnsmasq.conf").split("\n") @@ -32,7 +24,8 @@ class DNSRecordsDiagnoser(Diagnoser): self.resolver = ipv4_resolvers[0] main_domain = _get_maindomain() - for domain in self.args["domains"]: + all_domains = domain_list()["domains"] + for domain in all_domains: self.logger_debug("Diagnosing DNS conf for %s" % domain) for report in self.check_domain(domain, domain == main_domain): yield report diff --git a/data/hooks/diagnosis/30-services.py b/data/hooks/diagnosis/30-services.py index 4f08247f1..5029e0a5d 100644 --- a/data/hooks/diagnosis/30-services.py +++ b/data/hooks/diagnosis/30-services.py @@ -21,10 +21,6 @@ class ServicesDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] cache_duration = 300 - def validate_args(self, args): - # TODO / FIXME Ugh do we really need this arg system - return {} - def run(self): all_result = service_status() diff --git a/data/hooks/diagnosis/50-diskusage.py b/data/hooks/diagnosis/50-diskusage.py index 84ce3845c..2c6fe387b 100644 --- a/data/hooks/diagnosis/50-diskusage.py +++ b/data/hooks/diagnosis/50-diskusage.py @@ -9,10 +9,6 @@ class DiskUsageDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] cache_duration = 3600 * 24 - def validate_args(self, args): - # TODO / FIXME Ugh do we really need this arg system - return {} - def run(self): disk_partitions = psutil.disk_partitions() diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 9b17a7457..e7aca585f 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -84,7 +84,7 @@ def diagnosis_show(categories=[], issues=False, full=False): return {"reports": all_reports} -def diagnosis_run(categories=[], force=False, args=None): +def diagnosis_run(categories=[], force=False): # Get all the categories all_categories = _list_diagnosis_categories() @@ -98,13 +98,6 @@ def diagnosis_run(categories=[], force=False, args=None): if unknown_categories: raise YunohostError('unknown_categories', categories=", ".join(unknown_categories)) - # Transform "arg1=val1&arg2=val2" to { "arg1": "val1", "arg2": "val2" } - if args is not None: - args = {arg.split("=")[0]: arg.split("=")[1] for arg in args.split("&")} - else: - args = {} - args["force"] = force - issues = [] # Call the hook ... diagnosed_categories = [] @@ -113,7 +106,7 @@ def diagnosis_run(categories=[], force=False, args=None): path = [p for n, p in all_categories if n == category][0] try: - code, report = hook_exec(path, args=args, env=None) + code, report = hook_exec(path, args={"force": force}, env=None) except Exception as e: logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e)), exc_info=True) # FIXME : i18n else: @@ -143,7 +136,6 @@ class Diagnoser(): self.logger_debug, self.logger_warning, self.logger_info = loggers self.env = env self.args = args or {} - self.args.update(self.validate_args(self.args)) self.cache_file = Diagnoser.cache_file(self.id_) descr_key = "diagnosis_description_" + self.id_ From 35f6b778956b3755fec10718be0091d416cfdadc Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 20 Jul 2019 19:30:09 +0200 Subject: [PATCH 33/61] Reclarify ip diagnoser --- data/hooks/diagnosis/10-ip.py | 72 ++++++++++++++++++----------------- locales/en.json | 2 + 2 files changed, 40 insertions(+), 34 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index 4a6ee75ce..a4cfc0a48 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -17,19 +17,26 @@ class IPDiagnoser(Diagnoser): def run(self): - # - # IPv4 Diagnosis - # + # ############################################################ # + # PING : Check that we can ping outside at least in ipv4 or v6 # + # ############################################################ # - # If we can't ping, there's not much else we can do - if not self.can_ping_outside(4): - ipv4 = None - # If we do ping, check that we can resolv domain name - else: - can_resolve_dns = self.can_resolve_dns() - # And if we do, then we can fetch the public ip - if can_resolve_dns: - ipv4 = self.get_public_ip(4) + can_ping_ipv4 = self.can_ping_outside(4) + can_ping_ipv6 = self.can_ping_outside(6) + + if not can_ping_ipv4 and not can_ping_ipv6: + yield dict(meta={"test": "ping"}, + status="ERROR", + summary=("diagnosis_ip_not_connected_at_all", {})) + # Not much else we can do if there's no internet at all + return + + # ###################################################### # + # DNS RESOLUTION : Check that we can resolve domain name # + # (later needed to talk to ip. and ip6.yunohost.org) # + # ###################################################### # + + can_resolve_dns = self.can_resolve_dns() # In every case, we can check that resolvconf seems to be okay # (symlink managed by resolvconf service + pointing to dnsmasq) @@ -39,37 +46,37 @@ class IPDiagnoser(Diagnoser): # If it turns out that at the same time, resolvconf is bad, that's probably # the cause of this, so we use a different message in that case if not can_resolve_dns: - yield dict(meta={"name": "dnsresolution"}, + yield dict(meta={"test": "dnsresolv"}, status="ERROR", summary=("diagnosis_ip_broken_dnsresolution", {}) if good_resolvconf else ("diagnosis_ip_broken_resolvconf", {})) + return # Otherwise, if the resolv conf is bad but we were able to resolve domain name, # still warn that we're using a weird resolv conf ... elif not good_resolvconf: - yield dict(meta={"name": "dnsresolution"}, + yield dict(meta={"test": "dnsresolv"}, status="WARNING", summary=("diagnosis_ip_weird_resolvconf", {})) else: - # Well, maybe we could report a "success", "dns resolution is working", idk if it's worth it - pass + yield dict(meta={"test": "dnsresolv"}, + status="SUCCESS", + summary=("diagnosis_ip_dnsresolution_working", {})) - # And finally, we actually report the ipv4 connectivity stuff - yield dict(meta={"version": 4}, + # ##################################################### # + # IP DIAGNOSIS : Check that we're actually able to talk # + # to a web server to fetch current IPv4 and v6 # + # ##################################################### # + + ipv4 = self.get_public_ip(4) if can_ping_ipv4 else None + ipv6 = self.get_public_ip(6) if can_ping_ipv6 else None + + yield dict(meta={"test": "ip", "version": 4}, data=ipv4, status="SUCCESS" if ipv4 else "ERROR", summary=("diagnosis_ip_connected_ipv4", {}) if ipv4 else ("diagnosis_ip_no_ipv4", {})) - # - # IPv6 Diagnosis - # - - if not self.can_ping_outside(4): - ipv6 = None - else: - ipv6 = self.get_public_ip(6) - - yield dict(meta={"version": 6}, + yield dict(meta={"test": "ip", "version": 6}, data=ipv6, status="SUCCESS" if ipv6 else "WARNING", summary=("diagnosis_ip_connected_ipv6", {}) if ipv6 @@ -124,12 +131,9 @@ class IPDiagnoser(Diagnoser): # but if we want to be able to diagnose DNS resolution issues independently from # internet connectivity, we gotta rely on fixed IPs first.... - if protocol == 4: - url = 'https://ip.yunohost.org' - elif protocol == 6: - url = 'https://ip6.yunohost.org' - else: - raise ValueError("invalid protocol version, it should be either 4 or 6 and was '%s'" % repr(protocol)) + assert protocol in [4, 6], "Invalid protocol version, it should be either 4 or 6 and was '%s'" % repr(protocol) + + url = 'https://ip%s.yunohost.org' % ('6' if protocol == 6 else '') try: return download_text(url, timeout=30).strip() diff --git a/locales/en.json b/locales/en.json index 2e93e367f..8d6828979 100644 --- a/locales/en.json +++ b/locales/en.json @@ -159,6 +159,8 @@ "diagnosis_ip_no_ipv4": "The server does not have a working IPv4.", "diagnosis_ip_connected_ipv6": "The server is connected to the Internet through IPv6 !", "diagnosis_ip_no_ipv6": "The server does not have a working IPv6.", + "diagnosis_ip_not_connected_at_all": "The server does not seem to be connected to the Internet at all!?", + "diagnosis_ip_dnsresolution_working": "Domain name resolution is working!", "diagnosis_ip_broken_dnsresolution": "Domain name resolution seems to be broken for some reason ... Is a firewall blocking DNS requests ?", "diagnosis_ip_broken_resolvconf": "Domain name resolution seems to be broken on your server, which seems related to /etc/resolv.conf not pointing to 127.0.0.1.", "diagnosis_ip_weird_resolvconf": "Be careful that you seem to be using a custom /etc/resolv.conf. Instead, this file should be a symlink to /etc/resolvconf/run/resolv.conf itself pointing to 127.0.0.1 (dnsmasq).", From f690ff6e1e5ed6f80e185f8b0d5af0248ff025d8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 30 Jul 2019 18:53:17 +0200 Subject: [PATCH 34/61] First version of port exposure diagnosis --- data/hooks/diagnosis/14-ports.py | 53 ++++++++++++++++++++++++++++++++ locales/en.json | 4 +++ 2 files changed, 57 insertions(+) create mode 100644 data/hooks/diagnosis/14-ports.py diff --git a/data/hooks/diagnosis/14-ports.py b/data/hooks/diagnosis/14-ports.py new file mode 100644 index 000000000..6b260f3e0 --- /dev/null +++ b/data/hooks/diagnosis/14-ports.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +import os +import requests + +from yunohost.diagnosis import Diagnoser + + +class PortsDiagnoser(Diagnoser): + + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] + cache_duration = 3600 + + def run(self): + + # FIXME / TODO : in the future, maybe we want to report different + # things per port depending on how important they are + # (e.g. XMPP sounds to me much less important than other ports) + # Ideally, a port could be related to a service... + # FIXME / TODO : for now this list of port is hardcoded, might want + # to fetch this from the firewall.yml in /etc/yunohost/ + ports = [ 22, 25, 53, 80, 443, 587, 993, 5222, 5269 ] + + try: + r = requests.post('https://ynhdiagnoser.netlib.re/check-ports', json={'ports': ports}).json() + if not "status" in r.keys(): + raise Exception("Bad syntax for response ? Raw json: %s" % str(r)) + elif r["status"] == "error": + if "content" in r.keys(): + raise Exception(r["content"]) + else: + raise Exception("Bad syntax for response ? Raw json: %s" % str(r)) + elif r["status"] != "ok" or "ports" not in r.keys() or not isinstance(r["ports"], dict): + raise Exception("Bad syntax for response ? Raw json: %s" % str(r)) + except Exception as e: + raise YunohostError("diagnosis_ports_could_not_diagnose", error=e) + + found_issues = False + for port in ports: + if r["ports"].get(str(port), None) != True: + found_issues = True + yield dict(meta={"port": port}, + status="ERROR", + summary=("diagnosis_ports_unreachable", {"port":port})) + + if not found_issues: + yield dict(meta={}, + status="SUCCESS", + summary=("diagnosis_ports_ok",{})) + + +def main(args, env, loggers): + return PortsDiagnoser(args, env, loggers).diagnose() diff --git a/locales/en.json b/locales/en.json index 8d6828979..0a2204725 100644 --- a/locales/en.json +++ b/locales/en.json @@ -177,6 +177,10 @@ "diagnosis_description_dnsrecords": "DNS records", "diagnosis_description_services": "Services status check", "diagnosis_description_diskusage": "Disk usage", + "diagnosis_description_ports": "Ports exposure", + "diagnosis_ports_could_not_diagnose": "Could not diagnose if ports are reachable from outside. Error: {error}", + "diagnosis_ports_unreachable": "Port {port} is not reachable from outside.", + "diagnosis_ports_ok": "Relevant ports are reachable from outside!", "domain_cannot_remove_main": "Cannot remove main domain. Set a new main domain first", "domain_cert_gen_failed": "Could not generate certificate", "domain_created": "Domain created", From 6c48c131a8cb12b56566a09c68d2e59de68182ef Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 31 Jul 2019 01:02:31 +0200 Subject: [PATCH 35/61] Fix small issues in port diagnoser --- data/hooks/diagnosis/14-ports.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/data/hooks/diagnosis/14-ports.py b/data/hooks/diagnosis/14-ports.py index 6b260f3e0..8206474f8 100644 --- a/data/hooks/diagnosis/14-ports.py +++ b/data/hooks/diagnosis/14-ports.py @@ -4,6 +4,7 @@ import os import requests from yunohost.diagnosis import Diagnoser +from yunohost.utils.error import YunohostError class PortsDiagnoser(Diagnoser): @@ -19,11 +20,11 @@ class PortsDiagnoser(Diagnoser): # Ideally, a port could be related to a service... # FIXME / TODO : for now this list of port is hardcoded, might want # to fetch this from the firewall.yml in /etc/yunohost/ - ports = [ 22, 25, 53, 80, 443, 587, 993, 5222, 5269 ] + ports = [22, 25, 53, 80, 443, 587, 993, 5222, 5269] try: - r = requests.post('https://ynhdiagnoser.netlib.re/check-ports', json={'ports': ports}).json() - if not "status" in r.keys(): + r = requests.post('https://ynhdiagnoser.netlib.re/check-ports', json={'ports': ports}, timeout=30).json() + if "status" not in r.keys(): raise Exception("Bad syntax for response ? Raw json: %s" % str(r)) elif r["status"] == "error": if "content" in r.keys(): @@ -37,16 +38,16 @@ class PortsDiagnoser(Diagnoser): found_issues = False for port in ports: - if r["ports"].get(str(port), None) != True: + if r["ports"].get(str(port), None) is not True: found_issues = True yield dict(meta={"port": port}, status="ERROR", - summary=("diagnosis_ports_unreachable", {"port":port})) + summary=("diagnosis_ports_unreachable", {"port": port})) if not found_issues: yield dict(meta={}, status="SUCCESS", - summary=("diagnosis_ports_ok",{})) + summary=("diagnosis_ports_ok", {})) def main(args, env, loggers): From f050b3c5b86bf6c844fc67597d6949324d75be3d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 31 Jul 2019 01:08:21 +0200 Subject: [PATCH 36/61] First version of http exposure diagnosis --- data/hooks/diagnosis/16-http.py | 54 ++++++++++++++++++++++++++++ data/templates/nginx/server.tpl.conf | 4 +++ locales/en.json | 4 +++ src/yunohost/app.py | 3 +- 4 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 data/hooks/diagnosis/16-http.py diff --git a/data/hooks/diagnosis/16-http.py b/data/hooks/diagnosis/16-http.py new file mode 100644 index 000000000..b6b92fc77 --- /dev/null +++ b/data/hooks/diagnosis/16-http.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python + +import os +import random +import requests + +from yunohost.diagnosis import Diagnoser +from yunohost.domain import domain_list +from yunohost.utils.error import YunohostError + + +class HttpDiagnoser(Diagnoser): + + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] + cache_duration = 3600 + + def run(self): + + nonce_digits = "0123456789abcedf" + + all_domains = domain_list()["domains"] + for domain in all_domains: + + nonce = ''.join(random.choice(nonce_digits) for i in range(16)) + os.system("rm -rf /tmp/.well-known/ynh-diagnosis/") + os.system("mkdir -p /tmp/.well-known/ynh-diagnosis/") + os.system("touch /tmp/.well-known/ynh-diagnosis/%s" % nonce) + + try: + r = requests.post('https://ynhdiagnoser.netlib.re/check-http', json={'domain': domain, "nonce": nonce}, timeout=30).json() + print(r) + if "status" not in r.keys(): + raise Exception("Bad syntax for response ? Raw json: %s" % str(r)) + elif r["status"] == "error" and ("code" not in r.keys() or r["code"] not in ["error_http_check_connection_error", "error_http_check_unknown_error"]): + if "content" in r.keys(): + raise Exception(r["content"]) + else: + raise Exception("Bad syntax for response ? Raw json: %s" % str(r)) + except Exception as e: + print(e) + raise YunohostError("diagnosis_http_could_not_diagnose", error=e) + + if r["status"] == "ok": + yield dict(meta={"domain": domain}, + status="SUCCESS", + summary=("diagnosis_http_ok", {"domain": domain})) + else: + yield dict(meta={"domain": domain}, + status="ERROR", + summary=("diagnosis_http_unreachable", {"domain": domain})) + + +def main(args, env, loggers): + return HttpDiagnoser(args, env, loggers).diagnose() diff --git a/data/templates/nginx/server.tpl.conf b/data/templates/nginx/server.tpl.conf index 4a5e91557..9acc6c0fd 100644 --- a/data/templates/nginx/server.tpl.conf +++ b/data/templates/nginx/server.tpl.conf @@ -16,6 +16,10 @@ server { return 301 https://$http_host$request_uri; } + location /.well-known/ynh-diagnosis/ { + alias /tmp/.well-known/ynh-diagnosis/; + } + location /.well-known/autoconfig/mail/ { alias /var/www/.well-known/{{ domain }}/autoconfig/mail/; } diff --git a/locales/en.json b/locales/en.json index 0a2204725..ac44122fe 100644 --- a/locales/en.json +++ b/locales/en.json @@ -178,9 +178,13 @@ "diagnosis_description_services": "Services status check", "diagnosis_description_diskusage": "Disk usage", "diagnosis_description_ports": "Ports exposure", + "diagnosis_description_http": "HTTP exposure", "diagnosis_ports_could_not_diagnose": "Could not diagnose if ports are reachable from outside. Error: {error}", "diagnosis_ports_unreachable": "Port {port} is not reachable from outside.", "diagnosis_ports_ok": "Relevant ports are reachable from outside!", + "diagnosis_http_could_not_diagnose": "Could not diagnose if domain is reachable from outside. Error: {error}", + "diagnosis_http_ok": "Domain {domain} is reachable from outside.", + "diagnosis_http_unreachable": "Domain {domain} is unreachable through HTTP from outside.", "domain_cannot_remove_main": "Cannot remove main domain. Set a new main domain first", "domain_cert_gen_failed": "Could not generate certificate", "domain_created": "Domain created", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index e9e6ce14e..b4962d5f6 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1463,7 +1463,8 @@ def app_ssowatconf(): for domain in domains: skipped_urls.extend([domain + '/yunohost/admin', domain + '/yunohost/api']) - # Authorize ACME challenge url + # Authorize ynh remote diagnosis, ACME challenge and mail autoconfig urls + skipped_regex.append("^[^/]*/%.well%-known/ynh%-diagnosis/.*$") skipped_regex.append("^[^/]*/%.well%-known/acme%-challenge/.*$") skipped_regex.append("^[^/]*/%.well%-known/autoconfig/mail/config%-v1%.1%.xml.*$") From 91ec775ebb695b7a4e3a58951b51a6bd343dfc20 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 31 Jul 2019 16:54:25 +0200 Subject: [PATCH 37/61] Implement basic dependency system between diagnoser --- data/hooks/diagnosis/10-ip.py | 1 + data/hooks/diagnosis/12-dnsrecords.py | 1 + data/hooks/diagnosis/14-ports.py | 1 + data/hooks/diagnosis/16-http.py | 3 +-- data/hooks/diagnosis/30-services.py | 1 + data/hooks/diagnosis/50-diskusage.py | 1 + src/yunohost/diagnosis.py | 29 +++++++++++++++++---------- 7 files changed, 24 insertions(+), 13 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index a4cfc0a48..b29076467 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -14,6 +14,7 @@ class IPDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] cache_duration = 60 + dependencies = [] def run(self): diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 0f47ff136..0e8aaa07e 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -13,6 +13,7 @@ class DNSRecordsDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] cache_duration = 3600 * 24 + dependencies = ["ip"] def run(self): diff --git a/data/hooks/diagnosis/14-ports.py b/data/hooks/diagnosis/14-ports.py index 8206474f8..82a44384a 100644 --- a/data/hooks/diagnosis/14-ports.py +++ b/data/hooks/diagnosis/14-ports.py @@ -11,6 +11,7 @@ class PortsDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] cache_duration = 3600 + dependencies = ["ip"] def run(self): diff --git a/data/hooks/diagnosis/16-http.py b/data/hooks/diagnosis/16-http.py index b6b92fc77..cc335df8b 100644 --- a/data/hooks/diagnosis/16-http.py +++ b/data/hooks/diagnosis/16-http.py @@ -13,6 +13,7 @@ class HttpDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] cache_duration = 3600 + dependencies = ["ip"] def run(self): @@ -28,7 +29,6 @@ class HttpDiagnoser(Diagnoser): try: r = requests.post('https://ynhdiagnoser.netlib.re/check-http', json={'domain': domain, "nonce": nonce}, timeout=30).json() - print(r) if "status" not in r.keys(): raise Exception("Bad syntax for response ? Raw json: %s" % str(r)) elif r["status"] == "error" and ("code" not in r.keys() or r["code"] not in ["error_http_check_connection_error", "error_http_check_unknown_error"]): @@ -37,7 +37,6 @@ class HttpDiagnoser(Diagnoser): else: raise Exception("Bad syntax for response ? Raw json: %s" % str(r)) except Exception as e: - print(e) raise YunohostError("diagnosis_http_could_not_diagnose", error=e) if r["status"] == "ok": diff --git a/data/hooks/diagnosis/30-services.py b/data/hooks/diagnosis/30-services.py index 5029e0a5d..6589d83f2 100644 --- a/data/hooks/diagnosis/30-services.py +++ b/data/hooks/diagnosis/30-services.py @@ -20,6 +20,7 @@ class ServicesDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] cache_duration = 300 + dependencies = [] def run(self): diff --git a/data/hooks/diagnosis/50-diskusage.py b/data/hooks/diagnosis/50-diskusage.py index 2c6fe387b..74b8eb4b9 100644 --- a/data/hooks/diagnosis/50-diskusage.py +++ b/data/hooks/diagnosis/50-diskusage.py @@ -8,6 +8,7 @@ class DiskUsageDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] cache_duration = 3600 * 24 + dependencies = [] def run(self): diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index e7aca585f..14b332fe3 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -137,12 +137,7 @@ class Diagnoser(): self.env = env self.args = args or {} self.cache_file = Diagnoser.cache_file(self.id_) - - descr_key = "diagnosis_description_" + self.id_ - self.description = m18n.n(descr_key) - # If no description available, fallback to id - if self.description == descr_key: - self.description = self.id_ + self.description = Diagnoser.get_description(self.id_) def cached_time_ago(self): @@ -159,9 +154,18 @@ class Diagnoser(): if not self.args.get("force", False) and self.cached_time_ago() < self.cache_duration: self.logger_debug("Cache still valid : %s" % self.cache_file) + # FIXME : i18n logger.info("(Cache still valid for %s diagnosis. Not re-diagnosing yet!)" % self.description) return 0, {} + for dependency in self.dependencies: + dep_report = Diagnoser.get_cached_report(dependency) + dep_errors = [item for item in dep_report["items"] if item["status"] == "ERROR"] + if dep_errors: + # FIXME : i18n + logger.error("Can't run diagnosis for %s while there are important issues related to %s." % (self.description, Diagnoser.get_description(dependency))) + return 1, {} + self.logger_debug("Running diagnostic for %s" % self.id_) items = list(self.run()) @@ -200,6 +204,13 @@ class Diagnoser(): Diagnoser.i18n(report) return report + @staticmethod + def get_description(id_): + key = "diagnosis_description_" + id_ + descr = m18n.n(key) + # If no description available, fallback to id + return descr if descr != key else id_ + @staticmethod def i18n(report): @@ -209,11 +220,7 @@ class Diagnoser(): # was generated ... e.g. if the diagnosing happened inside a cron job with locale EN # instead of FR used by the actual admin... - descr_key = "diagnosis_description_" + report["id"] - report["description"] = m18n.n(descr_key) - # If no description available, fallback to id - if report["description"] == descr_key: - report["description"] = report["id"] + report["description"] = Diagnoser.get_description(report["id"]) for item in report["items"]: summary_key, summary_args = item["summary"] From 612a96e1e2410eabf677aec2c75194b477fe3cc0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 31 Jul 2019 17:36:51 +0200 Subject: [PATCH 38/61] Yield one item per port open to be consistent with other diagnosers --- data/hooks/diagnosis/14-ports.py | 11 ++++------- locales/en.json | 2 +- src/yunohost/diagnosis.py | 2 ++ 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/data/hooks/diagnosis/14-ports.py b/data/hooks/diagnosis/14-ports.py index 82a44384a..b953f35a9 100644 --- a/data/hooks/diagnosis/14-ports.py +++ b/data/hooks/diagnosis/14-ports.py @@ -37,18 +37,15 @@ class PortsDiagnoser(Diagnoser): except Exception as e: raise YunohostError("diagnosis_ports_could_not_diagnose", error=e) - found_issues = False for port in ports: if r["ports"].get(str(port), None) is not True: - found_issues = True yield dict(meta={"port": port}, status="ERROR", summary=("diagnosis_ports_unreachable", {"port": port})) - - if not found_issues: - yield dict(meta={}, - status="SUCCESS", - summary=("diagnosis_ports_ok", {})) + else: + yield dict(meta={}, + status="SUCCESS", + summary=("diagnosis_ports_ok", {"port": port})) def main(args, env, loggers): diff --git a/locales/en.json b/locales/en.json index ac44122fe..6dce769f1 100644 --- a/locales/en.json +++ b/locales/en.json @@ -181,7 +181,7 @@ "diagnosis_description_http": "HTTP exposure", "diagnosis_ports_could_not_diagnose": "Could not diagnose if ports are reachable from outside. Error: {error}", "diagnosis_ports_unreachable": "Port {port} is not reachable from outside.", - "diagnosis_ports_ok": "Relevant ports are reachable from outside!", + "diagnosis_ports_ok": "Port {port} is reachable from outside.", "diagnosis_http_could_not_diagnose": "Could not diagnose if domain is reachable from outside. Error: {error}", "diagnosis_http_ok": "Domain {domain} is reachable from outside.", "diagnosis_http_unreachable": "Domain {domain} is unreachable through HTTP from outside.", diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 14b332fe3..88316a15f 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -116,8 +116,10 @@ def diagnosis_run(categories=[], force=False): if issues: if msettings.get("interface") == "api": + # FIXME: i18n logger.info("You can go to the Diagnosis section (in the home screen) to see the issues found.") else: + # FIXME: i18n logger.info("You can run 'yunohost diagnosis show --issues' to display the issues found.") return From 0dc1909c68b6ef594b992764711c81f0ec169ad1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 31 Jul 2019 20:16:22 +0200 Subject: [PATCH 39/61] Misc small UX stuff --- data/hooks/diagnosis/10-ip.py | 3 ++- data/hooks/diagnosis/12-dnsrecords.py | 2 +- locales/en.json | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index b29076467..8c8dbe95b 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -57,7 +57,8 @@ class IPDiagnoser(Diagnoser): elif not good_resolvconf: yield dict(meta={"test": "dnsresolv"}, status="WARNING", - summary=("diagnosis_ip_weird_resolvconf", {})) + summary=("diagnosis_ip_weird_resolvconf", {}), + details=[("diagnosis_ip_weird_resolvconf_details", ())]) else: yield dict(meta={"test": "dnsresolv"}, status="SUCCESS", diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 0e8aaa07e..b59ffbd54 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -52,7 +52,7 @@ class DNSRecordsDiagnoser(Diagnoser): discrepancies.append(("diagnosis_dns_discrepancy", (r["type"], r["name"], expected_value, current_value))) if discrepancies: - status = "ERROR" if (category == "basic" or is_main_domain) else "WARNING" + status = "ERROR" if (category == "basic" or (is_main_domain and category != "extra")) else "WARNING" summary = ("diagnosis_dns_bad_conf", {"domain": domain, "category": category}) else: status = "SUCCESS" diff --git a/locales/en.json b/locales/en.json index 6dce769f1..65b3ef64d 100644 --- a/locales/en.json +++ b/locales/en.json @@ -163,9 +163,10 @@ "diagnosis_ip_dnsresolution_working": "Domain name resolution is working!", "diagnosis_ip_broken_dnsresolution": "Domain name resolution seems to be broken for some reason ... Is a firewall blocking DNS requests ?", "diagnosis_ip_broken_resolvconf": "Domain name resolution seems to be broken on your server, which seems related to /etc/resolv.conf not pointing to 127.0.0.1.", - "diagnosis_ip_weird_resolvconf": "Be careful that you seem to be using a custom /etc/resolv.conf. Instead, this file should be a symlink to /etc/resolvconf/run/resolv.conf itself pointing to 127.0.0.1 (dnsmasq).", + "diagnosis_ip_weird_resolvconf": "DNS resolution seems to be working, but be careful that you seem to be using a custom /etc/resolv.conf.", + "diagnosis_ip_weird_resolvconf_details": "Instead, this file should be a symlink to /etc/resolvconf/run/resolv.conf itself pointing to 127.0.0.1 (dnsmasq). The actual resolvers should be configured via /etc/resolv.dnsmasq.conf.", "diagnosis_dns_good_conf": "Good DNS configuration for domain {domain} (category {category})", - "diagnosis_dns_bad_conf": "Bad DNS configuration for domain {domain} (category {category})", + "diagnosis_dns_bad_conf": "Bad / missing DNS configuration for domain {domain} (category {category})", "diagnosis_dns_missing_record": "According to the recommended DNS configuration, you should add a DNS record with type {0}, name {1} and value {2}", "diagnosis_dns_discrepancy": "According to the recommended DNS configuration, the value for the DNS record with type {0} and name {1} should be {2}, not {3}.", "diagnosis_services_good_status": "Service {service} is {status} as expected!", From 4cbd1b06c2d572c908a3d7a6d4e82b6738f7a3da Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 31 Jul 2019 21:25:44 +0200 Subject: [PATCH 40/61] Add a regenconf diagnoser to report manually modified files... --- data/hooks/diagnosis/70-regenconf.py | 42 +++++++++++++++++++++++++++ locales/en.json | 5 ++++ src/yunohost/regenconf.py | 43 ++++++++++++++-------------- 3 files changed, 69 insertions(+), 21 deletions(-) create mode 100644 data/hooks/diagnosis/70-regenconf.py diff --git a/data/hooks/diagnosis/70-regenconf.py b/data/hooks/diagnosis/70-regenconf.py new file mode 100644 index 000000000..94c41feb5 --- /dev/null +++ b/data/hooks/diagnosis/70-regenconf.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +import os + +from yunohost.diagnosis import Diagnoser +from yunohost.regenconf import manually_modified_files, manually_modified_files_compared_to_debian_default + + +class RegenconfDiagnoser(Diagnoser): + + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] + cache_duration = 300 + dependencies = [] + + def run(self): + + regenconf_modified_files = manually_modified_files() + debian_modified_files = manually_modified_files_compared_to_debian_default(ignore_handled_by_regenconf=True) + + if regenconf_modified_files == []: + yield dict(meta={"test": "regenconf"}, + status="SUCCESS", + summary=("diagnosis_regenconf_allgood", {}) + ) + else: + for f in regenconf_modified_files: + yield dict(meta={"test": "regenconf", "file": f}, + status="WARNING", + summary=("diagnosis_regenconf_manually_modified", {"file": f}), + details=[("diagnosis_regenconf_manually_modified_details", {})] + ) + + for f in debian_modified_files: + yield dict(meta={"test": "debian", "file": f}, + status="WARNING", + summary=("diagnosis_regenconf_manually_modified_debian", {"file": f}), + details=[("diagnosis_regenconf_manually_modified_debian_details", {})] + ) + + +def main(args, env, loggers): + return RegenconfDiagnoser(args, env, loggers).diagnose() diff --git a/locales/en.json b/locales/en.json index 65b3ef64d..105891571 100644 --- a/locales/en.json +++ b/locales/en.json @@ -174,6 +174,11 @@ "diagnosis_diskusage_verylow": "Storage {mountpoint} (on device {device}) has only {free_percent}% space remaining. You should really consider cleaning up some space.", "diagnosis_diskusage_low": "Storage {mountpoint} (on device {device}) has only {free_percent}% space remaining. Be careful", "diagnosis_diskusage_ok": "Storage {mountpoint} (on device {device}) still has {free_percent}% space left!", + "diagnosis_regenconf_allgood": "All configurations files are in line with the recommended configuration!", + "diagnosis_regenconf_manually_modified": "Configuration file {file} was manually modified.", + "diagnosis_regenconf_manually_modified_details": "This is probably OK as long as you know what you're doing ;) !", + "diagnosis_regenconf_manually_modified_debian": "Configuration file {file} was manually modified compared to Debian's default.", + "diagnosis_regenconf_manually_modified_debian_details": "This may probably be OK, but gotta keep an eye on it...", "diagnosis_description_ip": "Internet connectivity", "diagnosis_description_dnsrecords": "DNS records", "diagnosis_description_services": "Services status check", diff --git a/src/yunohost/regenconf.py b/src/yunohost/regenconf.py index b7a42dd9d..b09824d58 100644 --- a/src/yunohost/regenconf.py +++ b/src/yunohost/regenconf.py @@ -525,31 +525,32 @@ def _process_regen_conf(system_conf, new_conf=None, save=True): def manually_modified_files(): - # We do this to have --quiet, i.e. don't throw a whole bunch of logs - # just to fetch this... - # Might be able to optimize this by looking at what the regen conf does - # and only do the part that checks file hashes... - cmd = "yunohost tools regen-conf --dry-run --output-as json --quiet" - j = json.loads(subprocess.check_output(cmd.split())) - - # j is something like : - # {"postfix": {"applied": {}, "pending": {"/etc/postfix/main.cf": {"status": "modified"}}} - output = [] - for app, actions in j.items(): - for action, files in actions.items(): - for filename, infos in files.items(): - if infos["status"] == "modified": - output.append(filename) + regenconf_categories = _get_regenconf_infos() + for category, infos in regenconf_categories.items(): + conffiles = infos["conffiles"] + for path, hash_ in conffiles.items(): + if hash_ != _calculate_hash(path): + output.append(path) return output -def manually_modified_files_compared_to_debian_default(): +def manually_modified_files_compared_to_debian_default(ignore_handled_by_regenconf=False): # from https://serverfault.com/a/90401 - r = subprocess.check_output("dpkg-query -W -f='${Conffiles}\n' '*' \ - | awk 'OFS=\" \"{print $2,$1}' \ - | md5sum -c 2>/dev/null \ - | awk -F': ' '$2 !~ /OK/{print $1}'", shell=True) - return r.strip().split("\n") + files = subprocess.check_output("dpkg-query -W -f='${Conffiles}\n' '*' \ + | awk 'OFS=\" \"{print $2,$1}' \ + | md5sum -c 2>/dev/null \ + | awk -F': ' '$2 !~ /OK/{print $1}'", shell=True) + files = files.strip().split("\n") + + if ignore_handled_by_regenconf: + regenconf_categories = _get_regenconf_infos() + regenconf_files = [] + for infos in regenconf_categories.values(): + regenconf_files.extend(infos["conffiles"].keys()) + + files = [f for f in files if f not in regenconf_files] + + return files From cee3b4de27dd03fb58f3d4400592c4bf0ec3e017 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 31 Jul 2019 22:04:55 +0200 Subject: [PATCH 41/61] Add nginx -t check to regenconf diagnoser --- data/hooks/diagnosis/70-regenconf.py | 14 ++++++++++++++ locales/en.json | 2 ++ 2 files changed, 16 insertions(+) diff --git a/data/hooks/diagnosis/70-regenconf.py b/data/hooks/diagnosis/70-regenconf.py index 94c41feb5..105d43fa3 100644 --- a/data/hooks/diagnosis/70-regenconf.py +++ b/data/hooks/diagnosis/70-regenconf.py @@ -2,6 +2,7 @@ import os +import subprocess from yunohost.diagnosis import Diagnoser from yunohost.regenconf import manually_modified_files, manually_modified_files_compared_to_debian_default @@ -14,6 +15,19 @@ class RegenconfDiagnoser(Diagnoser): def run(self): + # nginx -t + p = subprocess.Popen("nginx -t".split(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + out, _ = p.communicate() + + if p.returncode != 0: + yield dict(meta={"test": "nginx-t"}, + status="ERROR", + summary=("diagnosis_regenconf_nginx_conf_broken", {}), + details=[(out, ())] + ) + regenconf_modified_files = manually_modified_files() debian_modified_files = manually_modified_files_compared_to_debian_default(ignore_handled_by_regenconf=True) diff --git a/locales/en.json b/locales/en.json index 105891571..f06310679 100644 --- a/locales/en.json +++ b/locales/en.json @@ -179,12 +179,14 @@ "diagnosis_regenconf_manually_modified_details": "This is probably OK as long as you know what you're doing ;) !", "diagnosis_regenconf_manually_modified_debian": "Configuration file {file} was manually modified compared to Debian's default.", "diagnosis_regenconf_manually_modified_debian_details": "This may probably be OK, but gotta keep an eye on it...", + "diagnosis_regenconf_nginx_conf_broken": "The nginx configuration appears to be broken!", "diagnosis_description_ip": "Internet connectivity", "diagnosis_description_dnsrecords": "DNS records", "diagnosis_description_services": "Services status check", "diagnosis_description_diskusage": "Disk usage", "diagnosis_description_ports": "Ports exposure", "diagnosis_description_http": "HTTP exposure", + "diagnosis_description_regenconf": "System configurations", "diagnosis_ports_could_not_diagnose": "Could not diagnose if ports are reachable from outside. Error: {error}", "diagnosis_ports_unreachable": "Port {port} is not reachable from outside.", "diagnosis_ports_ok": "Port {port} is reachable from outside.", From b81cd4fc68ceb0ed63ff9526958bef60c558e1ab Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 31 Jul 2019 22:10:00 +0200 Subject: [PATCH 42/61] Add security diagnoser with meltdown checks --- data/hooks/diagnosis/90-security.py | 98 +++++++++++++++++++++++++++++ locales/en.json | 4 ++ 2 files changed, 102 insertions(+) create mode 100644 data/hooks/diagnosis/90-security.py diff --git a/data/hooks/diagnosis/90-security.py b/data/hooks/diagnosis/90-security.py new file mode 100644 index 000000000..0b1b61226 --- /dev/null +++ b/data/hooks/diagnosis/90-security.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python + +import os +import json +import subprocess + +from yunohost.diagnosis import Diagnoser +from moulinette.utils.filesystem import read_json, write_to_json + + +class SecurityDiagnoser(Diagnoser): + + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] + cache_duration = 3600 + dependencies = [] + + def run(self): + + "CVE-2017-5754" + + if self.is_vulnerable_to_meltdown(): + yield dict(meta={"test": "meltdown"}, + status="ERROR", + summary=("diagnosis_security_vulnerable_to_meltdown", {}), + details=[("diagnosis_security_vulnerable_to_meltdown_details", ())] + ) + else: + yield dict(meta={}, + status="SUCCESS", + summary=("diagnosis_security_all_good", {}) + ) + + + def is_vulnerable_to_meltdown(self): + # meltdown CVE: https://security-tracker.debian.org/tracker/CVE-2017-5754 + + # We use a cache file to avoid re-running the script so many times, + # which can be expensive (up to around 5 seconds on ARM) + # and make the admin appear to be slow (c.f. the calls to diagnosis + # from the webadmin) + # + # The cache is in /tmp and shall disappear upon reboot + # *or* we compare it to dpkg.log modification time + # such that it's re-ran if there was package upgrades + # (e.g. from yunohost) + cache_file = "/tmp/yunohost-meltdown-diagnosis" + dpkg_log = "/var/log/dpkg.log" + if os.path.exists(cache_file): + if not os.path.exists(dpkg_log) or os.path.getmtime(cache_file) > os.path.getmtime(dpkg_log): + self.logger_debug("Using cached results for meltdown checker, from %s" % cache_file) + return read_json(cache_file)[0]["VULNERABLE"] + + # script taken from https://github.com/speed47/spectre-meltdown-checker + # script commit id is store directly in the script + SCRIPT_PATH = "/usr/lib/moulinette/yunohost/vendor/spectre-meltdown-checker/spectre-meltdown-checker.sh" + + # '--variant 3' corresponds to Meltdown + # example output from the script: + # [{"NAME":"MELTDOWN","CVE":"CVE-2017-5754","VULNERABLE":false,"INFOS":"PTI mitigates the vulnerability"}] + try: + self.logger_debug("Running meltdown vulnerability checker") + call = subprocess.Popen("bash %s --batch json --variant 3" % + SCRIPT_PATH, shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + # TODO / FIXME : here we are ignoring error messages ... + # in particular on RPi2 and other hardware, the script complains about + # "missing some kernel info (see -v), accuracy might be reduced" + # Dunno what to do about that but we probably don't want to harass + # users with this warning ... + output, err = call.communicate() + assert call.returncode in (0, 2, 3), "Return code: %s" % call.returncode + + # If there are multiple lines, sounds like there was some messages + # in stdout that are not json >.> ... Try to get the actual json + # stuff which should be the last line + output = output.strip() + if "\n" in output: + self.logger_debug("Original meltdown checker output : %s" % output) + output = output.split("\n")[-1] + + CVEs = json.loads(output) + assert len(CVEs) == 1 + assert CVEs[0]["NAME"] == "MELTDOWN" + except Exception as e: + import traceback + traceback.print_exc() + self.logger_warning("Something wrong happened when trying to diagnose Meltdown vunerability, exception: %s" % e) + raise Exception("Command output for failed meltdown check: '%s'" % output) + + self.logger_debug("Writing results from meltdown checker to cache file, %s" % cache_file) + write_to_json(cache_file, CVEs) + return CVEs[0]["VULNERABLE"] + + +def main(args, env, loggers): + return SecurityDiagnoser(args, env, loggers).diagnose() diff --git a/locales/en.json b/locales/en.json index f06310679..c4330b08a 100644 --- a/locales/en.json +++ b/locales/en.json @@ -180,6 +180,9 @@ "diagnosis_regenconf_manually_modified_debian": "Configuration file {file} was manually modified compared to Debian's default.", "diagnosis_regenconf_manually_modified_debian_details": "This may probably be OK, but gotta keep an eye on it...", "diagnosis_regenconf_nginx_conf_broken": "The nginx configuration appears to be broken!", + "diagnosis_security_all_good": "No critical security vulnerability was found.", + "diagnosis_security_vulnerable_to_meltdown": "You appear vulnerable to the Meltdown criticial security vulnerability", + "diagnosis_security_vulnerable_to_meltdown_details": "To fix this, you should upgrade your system and reboot to load the new linux kernel (or contact your server provider if this doesn't work). See https://meltdownattack.com/ for more infos.", "diagnosis_description_ip": "Internet connectivity", "diagnosis_description_dnsrecords": "DNS records", "diagnosis_description_services": "Services status check", @@ -187,6 +190,7 @@ "diagnosis_description_ports": "Ports exposure", "diagnosis_description_http": "HTTP exposure", "diagnosis_description_regenconf": "System configurations", + "diagnosis_description_security": "Security checks", "diagnosis_ports_could_not_diagnose": "Could not diagnose if ports are reachable from outside. Error: {error}", "diagnosis_ports_unreachable": "Port {port} is not reachable from outside.", "diagnosis_ports_ok": "Port {port} is reachable from outside.", From 0c232b6cb5eb397f09d4d5024218b687a5cfcf46 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 1 Aug 2019 19:22:01 +0200 Subject: [PATCH 43/61] Implement diagnosis show --share --- data/actionsmap/yunohost.yml | 3 +++ src/yunohost/diagnosis.py | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 3d72bb57a..20eb8a0f8 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1887,6 +1887,9 @@ diagnosis: --issues: help: Only display issues action: store_true + --share: + help: Share the logs using yunopaste + action: store_true run: action_help: Show most recents diagnosis results diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 88316a15f..99a798b91 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -44,7 +44,7 @@ def diagnosis_list(): return {"categories": all_categories_names} -def diagnosis_show(categories=[], issues=False, full=False): +def diagnosis_show(categories=[], issues=False, full=False, share=False): # Get all the categories all_categories = _list_diagnosis_categories() @@ -81,7 +81,35 @@ def diagnosis_show(categories=[], issues=False, full=False): all_reports.append(report) - return {"reports": all_reports} + if share: + from yunohost.utils.yunopaste import yunopaste + content = _dump_human_readable_reports(all_reports) + url = yunopaste(content) + + logger.info(m18n.n("log_available_on_yunopaste", url=url)) + if msettings.get('interface') == 'api': + return {"url": url} + else: + return + else: + return {"reports": all_reports} + +def _dump_human_readable_reports(reports): + + output = "" + + for report in reports: + output += "=================================\n" + output += "{description} ({id})\n".format(**report) + output += "=================================\n\n" + for item in report["items"]: + output += "[{status}] {summary}\n".format(**item) + for detail in item.get("details", []): + output += " - " + detail + "\n" + output += "\n" + output += "\n\n" + + return(output) def diagnosis_run(categories=[], force=False): From c4ba8534c5dbc7b214afaedbae5b704a6bcf4339 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 1 Aug 2019 19:54:46 +0200 Subject: [PATCH 44/61] Implement i18n stuff --- locales/en.json | 9 +++++++++ src/yunohost/diagnosis.py | 25 ++++++++++--------------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/locales/en.json b/locales/en.json index c4330b08a..979edbbef 100644 --- a/locales/en.json +++ b/locales/en.json @@ -155,6 +155,15 @@ "diagnosis_no_apps": "No installed application", "dpkg_is_broken": "You cannot do this right now because dpkg/APT (the system package managers) seems to be in a broken state… You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.", "dpkg_lock_not_available": "This command can't be ran right now because another program seems to be using the lock of dpkg (the system package manager)", + "diagnosis_display_tip_web": "You can go to the Diagnosis section (in the home screen) to see the issues found.", + "diagnosis_display_tip_cli": "You can run 'yunohost diagnosis show --issues' to display the issues found.", + "diagnosis_failed_for_category": "Diagnosis failed for category '{category}' : {error}", + "diagnosis_cache_still_valid": "(Cache still valid for {category} diagnosis. Not re-diagnosing yet!)", + "diagnosis_cant_run_because_of_dep": "Can't run diagnosis for {category} while there are important issues related to {dep}.", + "diagnosis_found_issues": "Found {errors} significant issue(s) related to {category}!", + "diagnosis_found_warnings": "Found {warnings} item(s) that could be improved for {category}.", + "diagnosis_everything_ok": "Everything looks good for {category}!", + "diagnosis_failed": "Failed to fetch diagnosis result for category '{category}' : {error}", "diagnosis_ip_connected_ipv4": "The server is connected to the Internet through IPv4 !", "diagnosis_ip_no_ipv4": "The server does not have a working IPv4.", "diagnosis_ip_connected_ipv6": "The server is connected to the Internet through IPv6 !", diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 99a798b91..6cf207282 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -64,7 +64,7 @@ def diagnosis_show(categories=[], issues=False, full=False, share=False): try: report = Diagnoser.get_cached_report(category) except Exception as e: - logger.error("Failed to fetch diagnosis result for category '%s' : %s" % (category, str(e))) # FIXME : i18n + logger.error(m18n.n("diagnosis_failed", category=category, error=str(e))) else: if not full: del report["timestamp"] @@ -136,7 +136,7 @@ def diagnosis_run(categories=[], force=False): try: code, report = hook_exec(path, args={"force": force}, env=None) except Exception as e: - logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e)), exc_info=True) # FIXME : i18n + logger.error(m18n.n("diagnosis_failed_for_category", category=category, error=str(e)), exc_info=True) else: diagnosed_categories.append(category) if report != {}: @@ -144,11 +144,9 @@ def diagnosis_run(categories=[], force=False): if issues: if msettings.get("interface") == "api": - # FIXME: i18n - logger.info("You can go to the Diagnosis section (in the home screen) to see the issues found.") + logger.info(m18n.n("diagnosis_display_tip_web")) else: - # FIXME: i18n - logger.info("You can run 'yunohost diagnosis show --issues' to display the issues found.") + logger.info(m18n.n("diagnosis_display_tip_cli")) return @@ -163,6 +161,7 @@ class Diagnoser(): def __init__(self, args, env, loggers): + # FIXME ? That stuff with custom loggers is weird ... (mainly inherited from the bash hooks, idk) self.logger_debug, self.logger_warning, self.logger_info = loggers self.env = env self.args = args or {} @@ -184,16 +183,14 @@ class Diagnoser(): if not self.args.get("force", False) and self.cached_time_ago() < self.cache_duration: self.logger_debug("Cache still valid : %s" % self.cache_file) - # FIXME : i18n - logger.info("(Cache still valid for %s diagnosis. Not re-diagnosing yet!)" % self.description) + logger.info(m18n.n("diagnosis_cache_still_valid", category=self.description)) return 0, {} for dependency in self.dependencies: dep_report = Diagnoser.get_cached_report(dependency) dep_errors = [item for item in dep_report["items"] if item["status"] == "ERROR"] if dep_errors: - # FIXME : i18n - logger.error("Can't run diagnosis for %s while there are important issues related to %s." % (self.description, Diagnoser.get_description(dependency))) + logger.error(m18n.n("diagnosis_cant_run_because_of_dep", category=self.description, dep=Diagnoser.get_description(dependency))) return 1, {} self.logger_debug("Running diagnostic for %s" % self.id_) @@ -204,7 +201,6 @@ class Diagnoser(): "cached_for": self.cache_duration, "items": items} - # TODO / FIXME : should handle the case where we only did a partial diagnosis self.logger_debug("Updating cache %s" % self.cache_file) self.write_cache(new_report) Diagnoser.i18n(new_report) @@ -212,13 +208,12 @@ class Diagnoser(): errors = [item for item in new_report["items"] if item["status"] == "ERROR"] warnings = [item for item in new_report["items"] if item["status"] == "WARNING"] - # FIXME : i18n if errors: - logger.error("Found %s significant issue(s) related to %s!" % (len(errors), new_report["description"])) + logger.error(m18n.n("diagnosis_found_issues", errors=len(errors), category=new_report["description"])) elif warnings: - logger.warning("Found %s item(s) that could be improved for %s." % (len(warnings), new_report["description"])) + logger.warning(m18n.n("diagnosis_found_warnings", warnings=len(warnings), category=new_report["description"])) else: - logger.success("Everything looks good for %s!" % new_report["description"]) + logger.success(m18n.n("diagnosis_everything_ok", category=new_report["description"])) return 0, new_report From 33180d0947118c53f7e6c96da2882483a9be6df9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 1 Aug 2019 21:11:13 +0200 Subject: [PATCH 45/61] Add base system diagnostic --- data/hooks/diagnosis/00-basesystem.py | 54 +++++++++++++++++++++++++++ locales/en.json | 6 +++ src/yunohost/diagnosis.py | 4 +- 3 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 data/hooks/diagnosis/00-basesystem.py diff --git a/data/hooks/diagnosis/00-basesystem.py b/data/hooks/diagnosis/00-basesystem.py new file mode 100644 index 000000000..8fa90e65e --- /dev/null +++ b/data/hooks/diagnosis/00-basesystem.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python + +import os + +from moulinette.utils.filesystem import read_file +from yunohost.diagnosis import Diagnoser +from yunohost.utils.packages import ynh_packages_version + + +class BaseSystemDiagnoser(Diagnoser): + + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] + cache_duration = 3600 * 24 + dependencies = [] + + def run(self): + + # Kernel version + kernel_version = read_file('/proc/sys/kernel/osrelease').strip() + yield dict(meta={"test": "kernel"}, + status="INFO", + summary=("diagnosis_basesystem_kernel", {"kernel_version": kernel_version})) + + # Debian release + debian_version = read_file("/etc/debian_version").strip() + yield dict(meta={"test": "host"}, + status="INFO", + summary=("diagnosis_basesystem_host", {"debian_version": debian_version})) + + # Yunohost packages versions + ynh_packages = ynh_packages_version() + # We check if versions are consistent (e.g. all 3.6 and not 3 packages with 3.6 and the other with 3.5) + # This is a classical issue for upgrades that failed in the middle + # (or people upgrading half of the package because they did 'apt upgrade' instead of 'dist-upgrade') + # Here, ynh_core_version is for example "3.5.4.12", so [:3] is "3.5" and we check it's the same for all packages + ynh_core_version = ynh_packages["yunohost"]["version"] + consistent_versions = all(infos["version"][:3] == ynh_core_version[:3] for infos in ynh_packages.values()) + ynh_version_details = [("diagnosis_basesystem_ynh_single_version", (package, infos["version"])) + for package, infos in ynh_packages.items()] + + if consistent_versions: + yield dict(meta={"test": "ynh_versions"}, + status="INFO", + summary=("diagnosis_basesystem_ynh_main_version", {"main_version": ynh_core_version[:3]}), + details=ynh_version_details) + else: + yield dict(meta={"test": "ynh_versions"}, + status="ERROR", + summary=("diagnosis_basesystem_ynh_inconsistent_versions", {}), + details=ynh_version_details) + + +def main(args, env, loggers): + return BaseSystemDiagnoser(args, env, loggers).diagnose() diff --git a/locales/en.json b/locales/en.json index 979edbbef..f942d3dc4 100644 --- a/locales/en.json +++ b/locales/en.json @@ -150,6 +150,11 @@ "custom_appslist_name_required": "You must provide a name for your custom app list", "diagnosis_debian_version_error": "Could not retrieve the Debian version: {error}", "diagnosis_kernel_version_error": "Could not retrieve kernel version: {error}", + "diagnosis_basesystem_host": "Server is running Debian {debian_version}.", + "diagnosis_basesystem_kernel": "Server is running Linux kernel {kernel_version}", + "diagnosis_basesystem_ynh_single_version": "{0} version: {1}", + "diagnosis_basesystem_ynh_main_version": "Server is running YunoHost {main_version}", + "diagnosis_basesystem_ynh_inconsistent_versions": "You are running inconsistents versions of the YunoHost packages ... most probably because of a failed or partial upgrade.", "diagnosis_monitor_disk_error": "Could not monitor disks: {error}", "diagnosis_monitor_system_error": "Could not monitor system: {error}", "diagnosis_no_apps": "No installed application", @@ -192,6 +197,7 @@ "diagnosis_security_all_good": "No critical security vulnerability was found.", "diagnosis_security_vulnerable_to_meltdown": "You appear vulnerable to the Meltdown criticial security vulnerability", "diagnosis_security_vulnerable_to_meltdown_details": "To fix this, you should upgrade your system and reboot to load the new linux kernel (or contact your server provider if this doesn't work). See https://meltdownattack.com/ for more infos.", + "diagnosis_description_basesystem": "Base system", "diagnosis_description_ip": "Internet connectivity", "diagnosis_description_dnsrecords": "DNS records", "diagnosis_description_services": "Services status check", diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 6cf207282..b9fe111ed 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -74,7 +74,7 @@ def diagnosis_show(categories=[], issues=False, full=False, share=False): if "data" in item: del item["data"] if issues: - report["items"] = [item for item in report["items"] if item["status"] != "SUCCESS"] + report["items"] = [item for item in report["items"] if item["status"] in ["WARNING", "ERROR"]] # Ignore this category if no issue was found if not report["items"]: continue @@ -140,7 +140,7 @@ def diagnosis_run(categories=[], force=False): else: diagnosed_categories.append(category) if report != {}: - issues.extend([item for item in report["items"] if item["status"] != "SUCCESS"]) + issues.extend([item for item in report["items"] if item["status"] in ["WARNING", "ERROR"]]) if issues: if msettings.get("interface") == "api": From 47c7c72455bbcfaded9a5da0d753ce961511c628 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 1 Aug 2019 21:53:31 +0200 Subject: [PATCH 46/61] Add RAM and swap diagnosis + improve message for disk usage --- data/hooks/diagnosis/50-diskusage.py | 39 ---------- data/hooks/diagnosis/50-systemresources.py | 87 ++++++++++++++++++++++ locales/en.json | 14 +++- 3 files changed, 97 insertions(+), 43 deletions(-) delete mode 100644 data/hooks/diagnosis/50-diskusage.py create mode 100644 data/hooks/diagnosis/50-systemresources.py diff --git a/data/hooks/diagnosis/50-diskusage.py b/data/hooks/diagnosis/50-diskusage.py deleted file mode 100644 index 74b8eb4b9..000000000 --- a/data/hooks/diagnosis/50-diskusage.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python -import os -import psutil - -from yunohost.diagnosis import Diagnoser - -class DiskUsageDiagnoser(Diagnoser): - - id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] - cache_duration = 3600 * 24 - dependencies = [] - - def run(self): - - disk_partitions = psutil.disk_partitions() - - for disk_partition in disk_partitions: - device = disk_partition.device - mountpoint = disk_partition.mountpoint - - usage = psutil.disk_usage(mountpoint) - free_Go = usage.free / (1024 ** 3) - free_percent = 100 - usage.percent - - item = dict(meta={"mountpoint": mountpoint, "device": device}) - if free_Go < 1 or free_percent < 5: - item["status"] = "ERROR" - item["summary"] = ("diagnosis_diskusage_verylow", {"mountpoint": mountpoint, "device": device, "free_percent": free_percent}) - elif free_Go < 2 or free_percent < 10: - item["status"] = "WARNING" - item["summary"] = ("diagnosis_diskusage_low", {"mountpoint": mountpoint, "device": device, "free_percent": free_percent}) - else: - item["status"] = "SUCCESS" - item["summary"] = ("diagnosis_diskusage_ok", {"mountpoint": mountpoint, "device": device, "free_percent": free_percent}) - - yield item - -def main(args, env, loggers): - return DiskUsageDiagnoser(args, env, loggers).diagnose() diff --git a/data/hooks/diagnosis/50-systemresources.py b/data/hooks/diagnosis/50-systemresources.py new file mode 100644 index 000000000..3399c4682 --- /dev/null +++ b/data/hooks/diagnosis/50-systemresources.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +import os +import psutil + +from yunohost.diagnosis import Diagnoser + +class SystemResourcesDiagnoser(Diagnoser): + + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] + cache_duration = 3600 * 24 + dependencies = [] + + def run(self): + + # + # RAM + # + + ram = psutil.virtual_memory() + ram_total_abs_MB = ram.total / (1024**2) + ram_available_abs_MB = ram.available / (1024**2) + ram_available_percent = round(100 * ram.available / ram.total) + item = dict(meta={"test": "ram"}) + infos = {"total_abs_MB": ram_total_abs_MB, "available_abs_MB": ram_available_abs_MB, "available_percent": ram_available_percent} + if ram_available_abs_MB < 100 or ram_available_percent < 5: + item["status"] = "ERROR" + item["summary"] = ("diagnosis_ram_verylow", infos) + elif ram_available_abs_MB < 200 or ram_available_percent < 10: + item["status"] = "WARNING" + item["summary"] = ("diagnosis_ram_low", infos) + else: + item["status"] = "SUCCESS" + item["summary"] = ("diagnosis_ram_ok", infos) + print(item) + yield item + + # + # Swap + # + + swap = psutil.swap_memory() + swap_total_abs_MB = swap.total / (1024*1024) + item = dict(meta={"test": "swap"}) + infos = {"total_MB": swap_total_abs_MB} + if swap_total_abs_MB <= 0: + item["status"] = "ERROR" + item["summary"] = ("diagnosis_swap_none", infos) + elif swap_total_abs_MB <= 256: + item["status"] = "WARNING" + item["summary"] = ("diagnosis_swap_notsomuch", infos) + else: + item["status"] = "SUCCESS" + item["summary"] = ("diagnosis_swap_ok", infos) + print(item) + yield item + + # + # Disks usage + # + + disk_partitions = psutil.disk_partitions() + + for disk_partition in disk_partitions: + device = disk_partition.device + mountpoint = disk_partition.mountpoint + + usage = psutil.disk_usage(mountpoint) + free_abs_GB = usage.free / (1024 ** 3) + free_percent = 100 - usage.percent + + item = dict(meta={"mountpoint": mountpoint, "device": device}) + infos = {"mountpoint": mountpoint, "device": device, "free_abs_GB": free_abs_GB, "free_percent": free_percent} + if free_abs_GB < 1 or free_percent < 5: + item["status"] = "ERROR" + item["summary"] = ("diagnosis_diskusage_verylow", infos) + elif free_abs_GB < 2 or free_percent < 10: + item["status"] = "WARNING" + item["summary"] = ("diagnosis_diskusage_low", infos) + else: + item["status"] = "SUCCESS" + item["summary"] = ("diagnosis_diskusage_ok", infos) + + yield item + + +def main(args, env, loggers): + return SystemResourcesDiagnoser(args, env, loggers).diagnose() diff --git a/locales/en.json b/locales/en.json index f942d3dc4..40edb1425 100644 --- a/locales/en.json +++ b/locales/en.json @@ -185,9 +185,15 @@ "diagnosis_dns_discrepancy": "According to the recommended DNS configuration, the value for the DNS record with type {0} and name {1} should be {2}, not {3}.", "diagnosis_services_good_status": "Service {service} is {status} as expected!", "diagnosis_services_bad_status": "Service {service} is {status} :/", - "diagnosis_diskusage_verylow": "Storage {mountpoint} (on device {device}) has only {free_percent}% space remaining. You should really consider cleaning up some space.", - "diagnosis_diskusage_low": "Storage {mountpoint} (on device {device}) has only {free_percent}% space remaining. Be careful", - "diagnosis_diskusage_ok": "Storage {mountpoint} (on device {device}) still has {free_percent}% space left!", + "diagnosis_diskusage_verylow": "Storage {mountpoint} (on device {device}) has only {free_abs_GB} GB ({free_percent}%) space remaining. You should really consider cleaning up some space.", + "diagnosis_diskusage_low": "Storage {mountpoint} (on device {device}) has only {free_abs_GB} GB ({free_percent}%) space remaining. Be careful.", + "diagnosis_diskusage_ok": "Storage {mountpoint} (on device {device}) still has {free_abs_GB} GB ({free_percent}%) space left!", + "diagnosis_ram_verylow": "The system has only {available_abs_MB} MB ({available_percent}%) RAM left! (out of {total_abs_MB} MB)", + "diagnosis_ram_low": "The system has {available_abs_MB} MB ({available_percent}%) RAM left out of {total_abs_MB} MB. Be careful.", + "diagnosis_ram_ok": "The system still has {available_abs_MB} MB ({available_percent}%) RAM left out of {total_abs_MB} MB.", + "diagnosis_swap_none": "The system has no swap at all. You should consider adding at least 256 MB of swap to avoid situations where the system runs out of memory.", + "diagnosis_swap_notsomuch": "The system has only {total_MB} MB swap. You should consider having at least 256 MB to avoid situations where the system runs out of memory.", + "diagnosis_swap_ok": "The system has {total_MB} MB of swap!", "diagnosis_regenconf_allgood": "All configurations files are in line with the recommended configuration!", "diagnosis_regenconf_manually_modified": "Configuration file {file} was manually modified.", "diagnosis_regenconf_manually_modified_details": "This is probably OK as long as you know what you're doing ;) !", @@ -201,7 +207,7 @@ "diagnosis_description_ip": "Internet connectivity", "diagnosis_description_dnsrecords": "DNS records", "diagnosis_description_services": "Services status check", - "diagnosis_description_diskusage": "Disk usage", + "diagnosis_description_systemresources": "System resources", "diagnosis_description_ports": "Ports exposure", "diagnosis_description_http": "HTTP exposure", "diagnosis_description_regenconf": "System configurations", From 94f3557aeb3c6d2c95728692d59f027a2f7fe793 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 1 Aug 2019 22:02:08 +0200 Subject: [PATCH 47/61] Remove old 'tools diagnosis', superseded by the new diagnosis system --- data/actionsmap/yunohost.yml | 10 -- src/yunohost/tools.py | 188 +---------------------------------- 2 files changed, 3 insertions(+), 195 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 20eb8a0f8..1c96ce3e8 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1606,16 +1606,6 @@ tools: help: Upgrade only the system packages action: store_true - ### tools_diagnosis() - diagnosis: - action_help: YunoHost diagnosis - api: GET /diagnosis - arguments: - -p: - full: --private - help: Show private data (domain, IP) - action: store_true - ### tools_port_available() port-available: action_help: Check availability of a local port diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 64689fe0c..034157e3a 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -30,23 +30,19 @@ import json import subprocess import pwd import socket -from xmlrpclib import Fault from importlib import import_module -from collections import OrderedDict from moulinette import msignals, m18n from moulinette.utils.log import getActionLogger from moulinette.utils.process import check_output, call_async_output from moulinette.utils.filesystem import read_json, write_to_json, read_yaml, write_to_yaml -from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, app_list, _install_appslist_fetch_cron +from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, _install_appslist_fetch_cron from yunohost.domain import domain_add, domain_list, _get_maindomain, _set_maindomain from yunohost.dyndns import _dyndns_available, _dyndns_provides from yunohost.firewall import firewall_upnp -from yunohost.service import service_status, service_start, service_enable +from yunohost.service import service_start, service_enable from yunohost.regenconf import regen_conf -from yunohost.monitor import monitor_disk, monitor_system -from yunohost.utils.packages import ynh_packages_version, _dump_sources_list, _list_upgradable_apt_packages -from yunohost.utils.network import get_public_ip +from yunohost.utils.packages import _dump_sources_list, _list_upgradable_apt_packages from yunohost.utils.error import YunohostError from yunohost.log import is_unit_operation, OperationLogger @@ -726,184 +722,6 @@ def tools_upgrade(operation_logger, apps=None, system=False): operation_logger.success() -def tools_diagnosis(private=False): - """ - Return global info about current yunohost instance to help debugging - - """ - diagnosis = OrderedDict() - - # Debian release - try: - with open('/etc/debian_version', 'r') as f: - debian_version = f.read().rstrip() - except IOError as e: - logger.warning(m18n.n('diagnosis_debian_version_error', error=format(e)), exc_info=1) - else: - diagnosis['host'] = "Debian %s" % debian_version - - # Kernel version - try: - with open('/proc/sys/kernel/osrelease', 'r') as f: - kernel_version = f.read().rstrip() - except IOError as e: - logger.warning(m18n.n('diagnosis_kernel_version_error', error=format(e)), exc_info=1) - else: - diagnosis['kernel'] = kernel_version - - # Packages version - diagnosis['packages'] = ynh_packages_version() - - diagnosis["backports"] = check_output("dpkg -l |awk '/^ii/ && $3 ~ /bpo[6-8]/ {print $2}'").split() - - # Server basic monitoring - diagnosis['system'] = OrderedDict() - try: - disks = monitor_disk(units=['filesystem'], human_readable=True) - except (YunohostError, Fault) as e: - logger.warning(m18n.n('diagnosis_monitor_disk_error', error=format(e)), exc_info=1) - else: - diagnosis['system']['disks'] = {} - for disk in disks: - if isinstance(disks[disk], str): - diagnosis['system']['disks'][disk] = disks[disk] - else: - diagnosis['system']['disks'][disk] = 'Mounted on %s, %s (%s free)' % ( - disks[disk]['mnt_point'], - disks[disk]['size'], - disks[disk]['avail'] - ) - - try: - system = monitor_system(units=['cpu', 'memory'], human_readable=True) - except YunohostError as e: - logger.warning(m18n.n('diagnosis_monitor_system_error', error=format(e)), exc_info=1) - else: - diagnosis['system']['memory'] = { - 'ram': '%s (%s free)' % (system['memory']['ram']['total'], system['memory']['ram']['free']), - 'swap': '%s (%s free)' % (system['memory']['swap']['total'], system['memory']['swap']['free']), - } - - # nginx -t - p = subprocess.Popen("nginx -t".split(), - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - out, _ = p.communicate() - diagnosis["nginx"] = out.strip().split("\n") - if p.returncode != 0: - logger.error(out) - - # Services status - services = service_status() - diagnosis['services'] = {} - - for service in services: - diagnosis['services'][service] = "%s (%s)" % (services[service]['status'], services[service]['loaded']) - - # YNH Applications - try: - applications = app_list()['apps'] - except YunohostError as e: - diagnosis['applications'] = m18n.n('diagnosis_no_apps') - else: - diagnosis['applications'] = {} - for application in applications: - if application['installed']: - diagnosis['applications'][application['id']] = application['label'] if application['label'] else application['name'] - - # Private data - if private: - diagnosis['private'] = OrderedDict() - - # Public IP - diagnosis['private']['public_ip'] = {} - diagnosis['private']['public_ip']['IPv4'] = get_public_ip(4) - diagnosis['private']['public_ip']['IPv6'] = get_public_ip(6) - - # Domains - diagnosis['private']['domains'] = domain_list()['domains'] - - diagnosis['private']['regen_conf'] = regen_conf(with_diff=True, dry_run=True) - - try: - diagnosis['security'] = { - "CVE-2017-5754": { - "name": "meltdown", - "vulnerable": _check_if_vulnerable_to_meltdown(), - } - } - except Exception as e: - import traceback - traceback.print_exc() - logger.warning("Unable to check for meltdown vulnerability: %s" % e) - - return diagnosis - - -def _check_if_vulnerable_to_meltdown(): - # meltdown CVE: https://security-tracker.debian.org/tracker/CVE-2017-5754 - - # We use a cache file to avoid re-running the script so many times, - # which can be expensive (up to around 5 seconds on ARM) - # and make the admin appear to be slow (c.f. the calls to diagnosis - # from the webadmin) - # - # The cache is in /tmp and shall disappear upon reboot - # *or* we compare it to dpkg.log modification time - # such that it's re-ran if there was package upgrades - # (e.g. from yunohost) - cache_file = "/tmp/yunohost-meltdown-diagnosis" - dpkg_log = "/var/log/dpkg.log" - if os.path.exists(cache_file): - if not os.path.exists(dpkg_log) or os.path.getmtime(cache_file) > os.path.getmtime(dpkg_log): - logger.debug("Using cached results for meltdown checker, from %s" % cache_file) - return read_json(cache_file)[0]["VULNERABLE"] - - # script taken from https://github.com/speed47/spectre-meltdown-checker - # script commit id is store directly in the script - file_dir = os.path.split(__file__)[0] - SCRIPT_PATH = os.path.join(file_dir, "./vendor/spectre-meltdown-checker/spectre-meltdown-checker.sh") - - # '--variant 3' corresponds to Meltdown - # example output from the script: - # [{"NAME":"MELTDOWN","CVE":"CVE-2017-5754","VULNERABLE":false,"INFOS":"PTI mitigates the vulnerability"}] - try: - logger.debug("Running meltdown vulnerability checker") - call = subprocess.Popen("bash %s --batch json --variant 3" % - SCRIPT_PATH, shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - # TODO / FIXME : here we are ignoring error messages ... - # in particular on RPi2 and other hardware, the script complains about - # "missing some kernel info (see -v), accuracy might be reduced" - # Dunno what to do about that but we probably don't want to harass - # users with this warning ... - output, err = call.communicate() - assert call.returncode in (0, 2, 3), "Return code: %s" % call.returncode - - # If there are multiple lines, sounds like there was some messages - # in stdout that are not json >.> ... Try to get the actual json - # stuff which should be the last line - output = output.strip() - if "\n" in output: - logger.debug("Original meltdown checker output : %s" % output) - output = output.split("\n")[-1] - - CVEs = json.loads(output) - assert len(CVEs) == 1 - assert CVEs[0]["NAME"] == "MELTDOWN" - except Exception as e: - import traceback - traceback.print_exc() - logger.warning("Something wrong happened when trying to diagnose Meltdown vunerability, exception: %s" % e) - raise Exception("Command output for failed meltdown check: '%s'" % output) - - logger.debug("Writing results from meltdown checker to cache file, %s" % cache_file) - write_to_json(cache_file, CVEs) - return CVEs[0]["VULNERABLE"] - - def tools_port_available(port): """ Check availability of a local port From d113b6a53f126c289d4148f5a210c1bbf21ae118 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 5 Aug 2019 23:53:32 +0200 Subject: [PATCH 48/61] Adding some notes about diagnosis items to be implemented --- data/hooks/diagnosis/12-dnsrecords.py | 2 ++ data/hooks/diagnosis/18-mail.py | 28 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 data/hooks/diagnosis/18-mail.py diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index b59ffbd54..8c6565da9 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -31,6 +31,8 @@ class DNSRecordsDiagnoser(Diagnoser): for report in self.check_domain(domain, domain == main_domain): yield report + # FIXME : somewhere, should implement a check for reverse DNS ... + def check_domain(self, domain, is_main_domain): expected_configuration = _build_dns_conf(domain) diff --git a/data/hooks/diagnosis/18-mail.py b/data/hooks/diagnosis/18-mail.py new file mode 100644 index 000000000..5cf897e72 --- /dev/null +++ b/data/hooks/diagnosis/18-mail.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python + +import os + +from yunohost.diagnosis import Diagnoser + + +class MailDiagnoser(Diagnoser): + + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] + cache_duration = 3600 + dependencies = ["ip"] + + def run(self): + + return # TODO / FIXME TO BE IMPLEMETED in the future ... + + # Mail blacklist using dig requests (c.f. ljf's code) + + # Outgoing port 25 (c.f. code in monitor.py, a simple 'nc -zv yunohost.org 25' IIRC) + + # SMTP reachability (c.f. check-smtp to be implemented on yunohost's remote diagnoser) + + # ideally, SPF / DMARC / DKIM validation ... (c.f. https://github.com/alexAubin/yunoScripts/blob/master/yunoDKIM.py possibly though that looks horrible) + + +def main(args, env, loggers): + return MailDiagnoser(args, env, loggers).diagnose() From 339b6d9cbe2c97ffe249b99fb6b4c7f8c06437d7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 11 Aug 2019 16:23:47 +0200 Subject: [PATCH 49/61] Moar notes about what could be implemented for mail diagnoser --- data/hooks/diagnosis/18-mail.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/data/hooks/diagnosis/18-mail.py b/data/hooks/diagnosis/18-mail.py index 5cf897e72..100ace22f 100644 --- a/data/hooks/diagnosis/18-mail.py +++ b/data/hooks/diagnosis/18-mail.py @@ -23,6 +23,12 @@ class MailDiagnoser(Diagnoser): # ideally, SPF / DMARC / DKIM validation ... (c.f. https://github.com/alexAubin/yunoScripts/blob/master/yunoDKIM.py possibly though that looks horrible) + # check that the mail queue is not filled with hundreds of email pending + + # check that the recent mail logs are not filled with thousand of email sending (unusual number of mail sent) + + # check for unusual failed sending attempt being refused in the logs ? + def main(args, env, loggers): return MailDiagnoser(args, env, loggers).diagnose() From e0fa87cb364cd6d76bf4f80fd5da2c3ae51c9ca5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 18 Aug 2019 04:53:32 +0200 Subject: [PATCH 50/61] Note for the future about trying to diagnose hairpinning --- data/hooks/diagnosis/16-http.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/data/hooks/diagnosis/16-http.py b/data/hooks/diagnosis/16-http.py index cc335df8b..7ca258628 100644 --- a/data/hooks/diagnosis/16-http.py +++ b/data/hooks/diagnosis/16-http.py @@ -48,6 +48,10 @@ class HttpDiagnoser(Diagnoser): status="ERROR", summary=("diagnosis_http_unreachable", {"domain": domain})) + # In there or idk where else ... + # try to diagnose hairpinning situation by crafting a request for the + # global ip (from within local network) and seeing if we're getting the right page ? + def main(args, env, loggers): return HttpDiagnoser(args, env, loggers).diagnose() From 356f2b9ec1362b33b03ca8254d9d25c2bfcc22f1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 22 Aug 2019 12:42:25 +0200 Subject: [PATCH 51/61] Moar ideas --- data/hooks/diagnosis/00-basesystem.py | 2 ++ data/hooks/diagnosis/10-ip.py | 2 ++ data/hooks/diagnosis/12-dnsrecords.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/data/hooks/diagnosis/00-basesystem.py b/data/hooks/diagnosis/00-basesystem.py index 8fa90e65e..8bd522ee7 100644 --- a/data/hooks/diagnosis/00-basesystem.py +++ b/data/hooks/diagnosis/00-basesystem.py @@ -21,6 +21,8 @@ class BaseSystemDiagnoser(Diagnoser): status="INFO", summary=("diagnosis_basesystem_kernel", {"kernel_version": kernel_version})) + # FIXME / TODO : add virt/vm technology using systemd-detect-virt and/or machine arch + # Debian release debian_version = read_file("/etc/debian_version").strip() yield dict(meta={"test": "host"}, diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index 8c8dbe95b..e09dd343b 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -84,6 +84,8 @@ class IPDiagnoser(Diagnoser): summary=("diagnosis_ip_connected_ipv6", {}) if ipv6 else ("diagnosis_ip_no_ipv6", {})) + # TODO / FIXME : add some attempt to detect ISP (using whois ?) ? + def can_ping_outside(self, protocol=4): assert protocol in [4, 6], "Invalid protocol version, it should be either 4 or 6 and was '%s'" % repr(protocol) diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 8c6565da9..e2f7bcc2d 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -33,6 +33,8 @@ class DNSRecordsDiagnoser(Diagnoser): # FIXME : somewhere, should implement a check for reverse DNS ... + # FIXME / TODO : somewhere, could also implement a check for domain expiring soon + def check_domain(self, domain, is_main_domain): expected_configuration = _build_dns_conf(domain) From 3d7f37176cd8839e0455d414e1eb1aeb31274f2e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 10 Oct 2019 16:23:38 +0200 Subject: [PATCH 52/61] Remove debug prints --- data/hooks/diagnosis/50-systemresources.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/data/hooks/diagnosis/50-systemresources.py b/data/hooks/diagnosis/50-systemresources.py index 3399c4682..7e93a9ec0 100644 --- a/data/hooks/diagnosis/50-systemresources.py +++ b/data/hooks/diagnosis/50-systemresources.py @@ -31,7 +31,6 @@ class SystemResourcesDiagnoser(Diagnoser): else: item["status"] = "SUCCESS" item["summary"] = ("diagnosis_ram_ok", infos) - print(item) yield item # @@ -51,7 +50,6 @@ class SystemResourcesDiagnoser(Diagnoser): else: item["status"] = "SUCCESS" item["summary"] = ("diagnosis_swap_ok", infos) - print(item) yield item # From 02d6a0212f508fcd1df78bf2656447f9bd544f6d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 10 Oct 2019 16:40:06 +0200 Subject: [PATCH 53/61] Remove debug prints --- data/hooks/diagnosis/10-ip.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index e09dd343b..9c4257306 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -113,8 +113,10 @@ class IPDiagnoser(Diagnoser): # So let's try to ping the first 4~5 resolvers (shuffled) # If we succesfully ping any of them, we conclude that we are indeed connected def ping(protocol, target): + print("ping -c1 -%s -W 3 %s >/dev/null 2>/dev/null" % (protocol, target)) return os.system("ping -c1 -%s -W 3 %s >/dev/null 2>/dev/null" % (protocol, target)) == 0 + random.shuffle(resolvers) return any(ping(protocol, resolver) for resolver in resolvers[:5]) From e67e9e27ba281913cd0e6b518a252e7c4c536feb Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 10 Oct 2019 16:48:58 +0200 Subject: [PATCH 54/61] Hmm somehow there seem to be different version of ping supporting or not the -4 / -6 ... let's see if this workaroud works in all contexts --- data/hooks/diagnosis/10-ip.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index 9c4257306..552092fe3 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -113,9 +113,7 @@ class IPDiagnoser(Diagnoser): # So let's try to ping the first 4~5 resolvers (shuffled) # If we succesfully ping any of them, we conclude that we are indeed connected def ping(protocol, target): - print("ping -c1 -%s -W 3 %s >/dev/null 2>/dev/null" % (protocol, target)) - return os.system("ping -c1 -%s -W 3 %s >/dev/null 2>/dev/null" % (protocol, target)) == 0 - + return os.system("ping%s -c1 -W 3 %s >/dev/null 2>/dev/null" % ("" if protocol == 4 else "6", target)) == 0 random.shuffle(resolvers) return any(ping(protocol, resolver) for resolver in resolvers[:5]) From d6eb55d2a2107e217935256667d4aef52bd64593 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 11 Oct 2019 20:04:53 +0200 Subject: [PATCH 55/61] Add tmp dummy mail report so that the diagnoser kinda works instead of failing miserably --- data/hooks/diagnosis/18-mail.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/data/hooks/diagnosis/18-mail.py b/data/hooks/diagnosis/18-mail.py index 100ace22f..c12c15cff 100644 --- a/data/hooks/diagnosis/18-mail.py +++ b/data/hooks/diagnosis/18-mail.py @@ -13,7 +13,11 @@ class MailDiagnoser(Diagnoser): def run(self): - return # TODO / FIXME TO BE IMPLEMETED in the future ... + # TODO / FIXME TO BE IMPLEMETED in the future ... + + yield dict(meta={}, + status="WARNING", + summary=("nothing_implemented_yet", {})) # Mail blacklist using dig requests (c.f. ljf's code) From f75cd82593cc0feaab0f86f0d57b2f53c895ab5e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 11 Oct 2019 20:05:46 +0200 Subject: [PATCH 56/61] First part of implementing the ignore mechanism --- data/actionsmap/yunohost.yml | 19 ++--- src/yunohost/diagnosis.py | 132 +++++++++++++++++++++++++++++++++-- 2 files changed, 139 insertions(+), 12 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 1c96ce3e8..9b694c853 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1893,14 +1893,17 @@ diagnosis: action: store_true ignore: - action_help: Configure some diagnosis results to be ignored + action_help: Configure some diagnosis results to be ignored and therefore not considered as actual issues api: PUT /diagnosis/ignore arguments: - category: - help: Diagnosis category to be affected - -a: - help: Arguments, to be used to ignore only some parts of a report (e.g. "domain=domain.tld") - full: --args - --unignore: - help: Unignore a previously ignored report + --add-filter: + help: "Add a filter. The first element should be a diagnosis category, and other criterias can be provided using the infos from the 'meta' sections in 'yunohost diagnosis show'. For example: 'dnsrecords domain=yolo.test category=xmpp'" + nargs: "*" + metavar: CRITERIA + --remove-filter: + help: Remove a filter (it should be an existing filter as listed with --list) + nargs: "*" + metavar: CRITERIA + --list: + help: List active ignore filters action: store_true diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index b9fe111ed..da69e5d5e 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -29,7 +29,7 @@ import time from moulinette import m18n, msettings from moulinette.utils import log -from moulinette.utils.filesystem import read_json, write_to_json +from moulinette.utils.filesystem import read_json, write_to_json, read_yaml, write_to_yaml from yunohost.utils.error import YunohostError from yunohost.hook import hook_list, hook_exec @@ -37,7 +37,7 @@ from yunohost.hook import hook_list, hook_exec logger = log.getActionLogger('yunohost.diagnosis') DIAGNOSIS_CACHE = "/var/cache/yunohost/diagnosis/" - +DIAGNOSIS_CONFIG_FILE = '/etc/yunohost/diagnosis.yml' def diagnosis_list(): all_categories_names = [h for h, _ in _list_diagnosis_categories()] @@ -151,8 +151,132 @@ def diagnosis_run(categories=[], force=False): return -def diagnosis_ignore(category, args="", unignore=False): - pass +def diagnosis_ignore(add_filter=None, remove_filter=None, list=False): + """ + This action is meant for the admin to ignore issues reported by the + diagnosis system if they are known and understood by the admin. For + example, the lack of ipv6 on an instance, or badly configured XMPP dns + records if the admin doesn't care so much about XMPP. The point being that + the diagnosis shouldn't keep complaining about those known and "expected" + issues, and instead focus on new unexpected issues that could arise. + + For example, to ignore badly XMPP dnsrecords for domain yolo.test: + + yunohost diagnosis ignore --add-filter dnsrecords domain=yolo.test category=xmpp + ^ ^ ^ + the general additional other + diagnosis criterias criteria + category to to target to target + act on specific specific + reports reports + Or to ignore all dnsrecords issues: + + yunohost diagnosis ignore --add-filter dnsrecords + + The filters are stored in the diagnosis configuration in a data structure like: + + ignore_filters: { + "ip": [ + {"version": 6} # Ignore all issues related to ipv6 + ], + "dnsrecords": [ + {"domain": "yolo.test", "category": "xmpp"}, # Ignore all issues related to DNS xmpp records for yolo.test + {} # Ignore all issues about dnsrecords + ] + } + """ + + # Ignore filters are stored in + configuration = _diagnosis_read_configuration() + + if list: + return {"ignore_filters": configuration.get("ignore_filters", {})} + + def validate_filter_criterias(filter_): + + # Get all the categories + all_categories = _list_diagnosis_categories() + all_categories_names = [category for category, _ in all_categories] + + # Sanity checks for the provided arguments + if len(filter_) == 0: + raise YunohostError("You should provide at least one criteria being the diagnosis category to ignore") + category = filter_[0] + if category not in all_categories_names: + raise YunohostError("%s is not a diagnosis category" % category) + if any("=" not in criteria for criteria in filter_[1:]): + raise YunohostError("Extra criterias should be of the form key=value (e.g. domain=yolo.test)") + + # Convert the provided criteria into a nice dict + criterias = {c.split("=")[0]: c.split("=")[1] for c in filter_[1:]} + + return category, criterias + + if add_filter: + + category, criterias = validate_filter_criterias(add_filter) + + # Fetch current issues for the requested category + current_issues_for_this_category = diagnosis_show(categories=[category], issues=True, full=True) + current_issues_for_this_category = current_issues_for_this_category["reports"][0].get("items", {}) + + # Accept the given filter only if the criteria effectively match an existing issue + if not any(issue_matches_criterias(i, criterias) for i in current_issues_for_this_category): + raise YunohostError("No issues was found matching the given criteria.") + + # Make sure the subdicts/lists exists + if "ignore_filters" not in configuration: + configuration["ignore_filters"] = {} + if category not in configuration["ignore_filters"]: + configuration["ignore_filters"][category] = [] + + if criterias in configuration["ignore_filters"][category]: + logger.warning("This filter already exists.") + return + + configuration["ignore_filters"][category].append(criterias) + _diagnosis_write_configuration(configuration) + logger.success("Filter added") + return + + if remove_filter: + + category, criterias = validate_filter_criterias(remove_filter) + + # Make sure the subdicts/lists exists + if "ignore_filters" not in configuration: + configuration["ignore_filters"] = {} + if category not in configuration["ignore_filters"]: + configuration["ignore_filters"][category] = [] + + if criterias not in configuration["ignore_filters"][category]: + raise YunohostError("This filter does not exists.") + + configuration["ignore_filters"][category].remove(criterias) + _diagnosis_write_configuration(configuration) + logger.success("Filter removed") + return + + +def _diagnosis_read_configuration(): + if not os.path.exists(DIAGNOSIS_CONFIG_FILE): + return {} + + return read_yaml(DIAGNOSIS_CONFIG_FILE) + + +def _diagnosis_write_configuration(conf): + write_to_yaml(DIAGNOSIS_CONFIG_FILE, conf) + + +def issue_matches_criterias(issues, criterias): + for key, value in criterias.items(): + if key not in issues["meta"]: + return False + if str(issues["meta"][key]) != value: + return False + return True + ############################################################ From 97f9d3ea3753db40622b20df967e1644ac678c04 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 11 Oct 2019 22:42:21 +0200 Subject: [PATCH 57/61] Integrate the ignore mechanism with the rest of the code --- locales/en.json | 4 ++- src/yunohost/diagnosis.py | 55 ++++++++++++++++++++++++++++++++------- 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/locales/en.json b/locales/en.json index 40edb1425..afcb44edb 100644 --- a/locales/en.json +++ b/locales/en.json @@ -165,7 +165,9 @@ "diagnosis_failed_for_category": "Diagnosis failed for category '{category}' : {error}", "diagnosis_cache_still_valid": "(Cache still valid for {category} diagnosis. Not re-diagnosing yet!)", "diagnosis_cant_run_because_of_dep": "Can't run diagnosis for {category} while there are important issues related to {dep}.", - "diagnosis_found_issues": "Found {errors} significant issue(s) related to {category}!", + "diagnosis_ignored_issues": "(+ {nb_ignored} ignored issue(s))", + "diagnosis_found_errors": "Found {errors} significant issue(s) related to {category}!", + "diagnosis_found_errors_and_warnings": "Found {errors} significant issue(s) (and {warnings} warning(s)) related to {category}!", "diagnosis_found_warnings": "Found {warnings} item(s) that could be improved for {category}.", "diagnosis_everything_ok": "Everything looks good for {category}!", "diagnosis_failed": "Failed to fetch diagnosis result for category '{category}' : {error}", diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index da69e5d5e..19dd03042 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -66,11 +66,14 @@ def diagnosis_show(categories=[], issues=False, full=False, share=False): except Exception as e: logger.error(m18n.n("diagnosis_failed", category=category, error=str(e))) else: + add_ignore_flag_to_issues(report) if not full: del report["timestamp"] del report["cached_for"] + report["items"] = [item for item in report["items"] if not item["ignored"]] for item in report["items"]: del item["meta"] + del item["ignored"] if "data" in item: del item["data"] if issues: @@ -269,14 +272,42 @@ def _diagnosis_write_configuration(conf): write_to_yaml(DIAGNOSIS_CONFIG_FILE, conf) -def issue_matches_criterias(issues, criterias): +def issue_matches_criterias(issue, criterias): + """ + e.g. an issue with: + meta: + domain: yolo.test + category: xmpp + + matches the criterias {"domain": "yolo.test"} + """ for key, value in criterias.items(): - if key not in issues["meta"]: + if key not in issue["meta"]: return False - if str(issues["meta"][key]) != value: + if str(issue["meta"][key]) != value: return False return True +def add_ignore_flag_to_issues(report): + """ + Iterate over issues in a report, and flag them as ignored if they match an + ignored filter from the configuration + + N.B. : for convenience. we want to make sure the "ignored" key is set for + every item in the report + """ + + ignore_filters = _diagnosis_read_configuration().get("ignore_filters", {}).get(report["id"], []) + + for report_item in report["items"]: + report_item["ignored"] = False + if report_item["status"] not in ["WARNING", "ERROR"]: + continue + for criterias in ignore_filters: + if issue_matches_criterias(report_item, criterias): + report_item["ignored"] = True + break + ############################################################ @@ -328,16 +359,22 @@ class Diagnoser(): self.logger_debug("Updating cache %s" % self.cache_file) self.write_cache(new_report) Diagnoser.i18n(new_report) + add_ignore_flag_to_issues(new_report) - errors = [item for item in new_report["items"] if item["status"] == "ERROR"] - warnings = [item for item in new_report["items"] if item["status"] == "WARNING"] + errors = [item for item in new_report["items"] if item["status"] == "ERROR" and not item["ignored"]] + warnings = [item for item in new_report["items"] if item["status"] == "WARNING" and not item["ignored"]] + errors_ignored = [item for item in new_report["items"] if item["status"] == "ERROR" and item["ignored"]] + warning_ignored = [item for item in new_report["items"] if item["status"] == "WARNING" and item["ignored"]] + ignored_msg = " " + m18n.n("diagnosis_ignored_issues", nb_ignored=len(errors_ignored+warning_ignored)) if errors_ignored or warning_ignored else "" - if errors: - logger.error(m18n.n("diagnosis_found_issues", errors=len(errors), category=new_report["description"])) + if errors and warnings: + logger.error(m18n.n("diagnosis_found_errors_and_warnings", errors=len(errors), warnings=len(warnings), category=new_report["description"]) + ignored_msg) + elif errors: + logger.error(m18n.n("diagnosis_found_errors", errors=len(errors), category=new_report["description"]) + ignored_msg) elif warnings: - logger.warning(m18n.n("diagnosis_found_warnings", warnings=len(warnings), category=new_report["description"])) + logger.warning(m18n.n("diagnosis_found_warnings", warnings=len(warnings), category=new_report["description"]) + ignored_msg) else: - logger.success(m18n.n("diagnosis_everything_ok", category=new_report["description"])) + logger.success(m18n.n("diagnosis_everything_ok", category=new_report["description"]) + ignored_msg) return 0, new_report From 51e7a56522e49edc498677aeb5ec08fa174d713c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 13 Oct 2019 18:42:45 +0200 Subject: [PATCH 58/61] Improve metadata for diskusage tests --- data/hooks/diagnosis/50-systemresources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/hooks/diagnosis/50-systemresources.py b/data/hooks/diagnosis/50-systemresources.py index 7e93a9ec0..95f58ddb7 100644 --- a/data/hooks/diagnosis/50-systemresources.py +++ b/data/hooks/diagnosis/50-systemresources.py @@ -66,7 +66,7 @@ class SystemResourcesDiagnoser(Diagnoser): free_abs_GB = usage.free / (1024 ** 3) free_percent = 100 - usage.percent - item = dict(meta={"mountpoint": mountpoint, "device": device}) + item = dict(meta={"test": "diskusage", "mountpoint": mountpoint}) infos = {"mountpoint": mountpoint, "device": device, "free_abs_GB": free_abs_GB, "free_percent": free_percent} if free_abs_GB < 1 or free_percent < 5: item["status"] = "ERROR" From 0839de2d6a3684056e0d0bf692c4391b1ac153ef Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 13 Oct 2019 23:02:46 +0200 Subject: [PATCH 59/61] Switching to POST method because it's more practical than PUT, idk what im doing --- data/actionsmap/yunohost.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 9b694c853..e7a1d1ad2 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1894,7 +1894,7 @@ diagnosis: ignore: action_help: Configure some diagnosis results to be ignored and therefore not considered as actual issues - api: PUT /diagnosis/ignore + api: POST /diagnosis/ignore arguments: --add-filter: help: "Add a filter. The first element should be a diagnosis category, and other criterias can be provided using the infos from the 'meta' sections in 'yunohost diagnosis show'. For example: 'dnsrecords domain=yolo.test category=xmpp'" From 5818de3a824a1c869c12cc4760b4511d53baa83d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 14 Oct 2019 04:48:56 +0200 Subject: [PATCH 60/61] Remove the whole monitoring / glances stuff --- data/actionsmap/yunohost.yml | 141 ----- data/templates/glances/glances.default | 5 - data/templates/yunohost/services.yml | 2 +- debian/control | 2 +- locales/en.json | 14 - src/yunohost/monitor.py | 740 ------------------------- 6 files changed, 2 insertions(+), 902 deletions(-) delete mode 100644 data/templates/glances/glances.default delete mode 100644 src/yunohost/monitor.py diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index e7a1d1ad2..4b76fcb0b 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -968,147 +968,6 @@ backup: pattern: *pattern_backup_archive_name -############################# -# Monitor # -############################# -monitor: - category_help: Monitor the server - actions: - - ### monitor_disk() - disk: - action_help: Monitor disk space and usage - api: GET /monitor/disk - arguments: - -f: - full: --filesystem - help: Show filesystem disk space - action: append_const - const: filesystem - dest: units - -t: - full: --io - help: Show I/O throughput - action: append_const - const: io - dest: units - -m: - full: --mountpoint - help: Monitor only the device mounted on MOUNTPOINT - action: store - -H: - full: --human-readable - help: Print sizes in human readable format - action: store_true - - ### monitor_network() - network: - action_help: Monitor network interfaces - api: GET /monitor/network - arguments: - -u: - full: --usage - help: Show interfaces bit rates - action: append_const - const: usage - dest: units - -i: - full: --infos - help: Show network informations - action: append_const - const: infos - dest: units - -c: - full: --check - help: Check network configuration - action: append_const - const: check - dest: units - -H: - full: --human-readable - help: Print sizes in human readable format - action: store_true - - ### monitor_system() - system: - action_help: Monitor system informations and usage - api: GET /monitor/system - arguments: - -m: - full: --memory - help: Show memory usage - action: append_const - const: memory - dest: units - -c: - full: --cpu - help: Show CPU usage and load - action: append_const - const: cpu - dest: units - -p: - full: --process - help: Show processes summary - action: append_const - const: process - dest: units - -u: - full: --uptime - help: Show the system uptime - action: append_const - const: uptime - dest: units - -i: - full: --infos - help: Show system informations - action: append_const - const: infos - dest: units - -H: - full: --human-readable - help: Print sizes in human readable format - action: store_true - - ### monitor_updatestats() - update-stats: - action_help: Update monitoring statistics - api: POST /monitor/stats - arguments: - period: - help: Time period to update - choices: - - day - - week - - month - - ### monitor_showstats() - show-stats: - action_help: Show monitoring statistics - api: GET /monitor/stats - arguments: - period: - help: Time period to show - choices: - - day - - week - - month - - ### monitor_enable() - enable: - action_help: Enable server monitoring - api: PUT /monitor - arguments: - -s: - full: --with-stats - help: Enable monitoring statistics - action: store_true - - ### monitor_disable() - disable: - api: DELETE /monitor - action_help: Disable server monitoring - - ############################# # Settings # ############################# diff --git a/data/templates/glances/glances.default b/data/templates/glances/glances.default deleted file mode 100644 index 22337a0d9..000000000 --- a/data/templates/glances/glances.default +++ /dev/null @@ -1,5 +0,0 @@ -# Default is to launch glances with '-s' option. -DAEMON_ARGS="-s -B 127.0.0.1" - -# Change to 'true' to have glances running at startup -RUN="true" diff --git a/data/templates/yunohost/services.yml b/data/templates/yunohost/services.yml index 0d79b182f..1c0ee031f 100644 --- a/data/templates/yunohost/services.yml +++ b/data/templates/yunohost/services.yml @@ -17,7 +17,6 @@ redis-server: mysql: log: [/var/log/mysql.log,/var/log/mysql.err] alternates: ['mariadb'] -glances: {} ssh: log: /var/log/auth.log metronome: @@ -32,6 +31,7 @@ yunohost-firewall: need_lock: true nslcd: log: /var/log/syslog +glances: null nsswitch: null ssl: null yunohost: null diff --git a/debian/control b/debian/control index c0604d90e..3b8c257d0 100644 --- a/debian/control +++ b/debian/control @@ -15,7 +15,7 @@ Depends: ${python:Depends}, ${misc:Depends} , python-psutil, python-requests, python-dnspython, python-openssl , python-apt, python-miniupnpc, python-dbus, python-jinja2 , python-toml - , glances, apt-transport-https + , apt-transport-https , dnsutils, bind9utils, unzip, git, curl, cron, wget, jq , ca-certificates, netcat-openbsd, iproute , mariadb-server, php-mysql | php-mysqlnd diff --git a/locales/en.json b/locales/en.json index afcb44edb..e04446ce1 100644 --- a/locales/en.json +++ b/locales/en.json @@ -434,21 +434,9 @@ "migrations_skip_migration": "Skipping migration {id}…", "migrations_success_forward": "Migration {id} completed", "migrations_to_be_ran_manually": "Migration {id} has to be run manually. Please go to Tools → Migrations on the webadmin page, or run `yunohost tools migrations migrate`.", - "monitor_disabled": "Server monitoring now turned off", - "monitor_enabled": "Server monitoring now turned on", - "monitor_glances_con_failed": "Could not connect to Glances server", - "monitor_not_enabled": "Server monitoring is off", - "monitor_period_invalid": "Invalid time period", - "monitor_stats_file_not_found": "Statistics file not found", - "monitor_stats_no_update": "No monitoring statistics to update", - "monitor_stats_period_unavailable": "No available statistics for the period", - "mountpoint_unknown": "Unknown mountpoint", "mysql_db_creation_failed": "MySQL database creation failed", "mysql_db_init_failed": "MySQL database init failed", "mysql_db_initialized": "The MySQL database now initialized", - "network_check_mx_ko": "DNS MX record is not set", - "network_check_smtp_ko": "Outbound e-mail (SMTP port 25) seems to be blocked by your network", - "network_check_smtp_ok": "Outbound e-mail (SMTP port 25) is not blocked", "no_internet_connection": "Server not connected to the Internet", "not_enough_disk_space": "Not enough free space on '{path:s}'", "operation_interrupted": "The operation was manually interrupted?", @@ -536,7 +524,6 @@ "service_description_dnsmasq": "Handles domain name resolution (DNS)", "service_description_dovecot": "Allows e-mail clients to access/fetch email (via IMAP and POP3)", "service_description_fail2ban": "Protects against brute-force and other kinds of attacks from the Internet", - "service_description_glances": "Monitors system info on your server", "service_description_metronome": "Manage XMPP instant messaging accounts", "service_description_mysql": "Stores applications data (SQL database)", "service_description_nginx": "Serves or provides access to all the websites hosted on your server", @@ -588,7 +575,6 @@ "tools_upgrade_special_packages_completed": "YunoHost package upgrade completed.\nPress [Enter] to get the command line back", "unbackup_app": "App '{app:s}' will not be saved", "unexpected_error": "Something unexpected went wrong: {error}", - "unit_unknown": "Unknown unit '{unit:s}'", "unlimit": "No quota", "unrestore_app": "App '{app:s}' will not be restored", "update_apt_cache_failed": "Could not to update the cache of APT (Debian's package manager). Here is a dump of the sources.list lines, which might help identify problematic lines: \n{sourceslist}", diff --git a/src/yunohost/monitor.py b/src/yunohost/monitor.py deleted file mode 100644 index 7af55f287..000000000 --- a/src/yunohost/monitor.py +++ /dev/null @@ -1,740 +0,0 @@ -# -*- coding: utf-8 -*- - -""" License - - Copyright (C) 2013 YunoHost - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program; if not, see http://www.gnu.org/licenses - -""" - -""" yunohost_monitor.py - - Monitoring functions -""" -import re -import json -import time -import psutil -import calendar -import subprocess -import xmlrpclib -import os.path -import os -import dns.resolver -import cPickle as pickle -from datetime import datetime - -from moulinette import m18n -from yunohost.utils.error import YunohostError -from moulinette.utils.log import getActionLogger - -from yunohost.utils.network import get_public_ip -from yunohost.domain import _get_maindomain - -logger = getActionLogger('yunohost.monitor') - -GLANCES_URI = 'http://127.0.0.1:61209' -STATS_PATH = '/var/lib/yunohost/stats' -CRONTAB_PATH = '/etc/cron.d/yunohost-monitor' - - -def monitor_disk(units=None, mountpoint=None, human_readable=False): - """ - Monitor disk space and usage - - Keyword argument: - units -- Unit(s) to monitor - mountpoint -- Device mountpoint - human_readable -- Print sizes in human readable format - - """ - glances = _get_glances_api() - result_dname = None - result = {} - - if units is None: - units = ['io', 'filesystem'] - - _format_dname = lambda d: (os.path.realpath(d)).replace('/dev/', '') - - # Get mounted devices - devices = {} - for p in psutil.disk_partitions(all=True): - if not p.device.startswith('/dev/') or not p.mountpoint: - continue - if mountpoint is None: - devices[_format_dname(p.device)] = p.mountpoint - elif mountpoint == p.mountpoint: - dn = _format_dname(p.device) - devices[dn] = p.mountpoint - result_dname = dn - if len(devices) == 0: - if mountpoint is not None: - raise YunohostError('mountpoint_unknown') - return result - - # Retrieve monitoring for unit(s) - for u in units: - if u == 'io': - # Define setter - if len(units) > 1: - def _set(dn, dvalue): - try: - result[dn][u] = dvalue - except KeyError: - result[dn] = {u: dvalue} - else: - def _set(dn, dvalue): - result[dn] = dvalue - - # Iterate over values - devices_names = devices.keys() - for d in json.loads(glances.getDiskIO()): - dname = d.pop('disk_name') - try: - devices_names.remove(dname) - except: - continue - else: - _set(dname, d) - for dname in devices_names: - _set(dname, 'not-available') - elif u == 'filesystem': - # Define setter - if len(units) > 1: - def _set(dn, dvalue): - try: - result[dn][u] = dvalue - except KeyError: - result[dn] = {u: dvalue} - else: - def _set(dn, dvalue): - result[dn] = dvalue - - # Iterate over values - devices_names = devices.keys() - for d in json.loads(glances.getFs()): - dname = _format_dname(d.pop('device_name')) - try: - devices_names.remove(dname) - except: - continue - else: - d['avail'] = d['size'] - d['used'] - if human_readable: - for i in ['used', 'avail', 'size']: - d[i] = binary_to_human(d[i]) + 'B' - _set(dname, d) - for dname in devices_names: - _set(dname, 'not-available') - else: - raise YunohostError('unit_unknown', unit=u) - - if result_dname is not None: - return result[result_dname] - return result - - -def monitor_network(units=None, human_readable=False): - """ - Monitor network interfaces - - Keyword argument: - units -- Unit(s) to monitor - human_readable -- Print sizes in human readable format - - """ - glances = _get_glances_api() - result = {} - - if units is None: - units = ['check', 'usage', 'infos'] - - # Get network devices and their addresses - # TODO / FIXME : use functions in utils/network.py to manage this - devices = {} - output = subprocess.check_output('ip addr show'.split()) - for d in re.split('^(?:[0-9]+: )', output, flags=re.MULTILINE): - # Extract device name (1) and its addresses (2) - m = re.match('([^\s@]+)(?:@[\S]+)?: (.*)', d, flags=re.DOTALL) - if m: - devices[m.group(1)] = m.group(2) - - # Retrieve monitoring for unit(s) - for u in units: - if u == 'check': - result[u] = {} - domain = _get_maindomain() - cmd_check_smtp = os.system('/bin/nc -z -w1 yunohost.org 25') - if cmd_check_smtp == 0: - smtp_check = m18n.n('network_check_smtp_ok') - else: - smtp_check = m18n.n('network_check_smtp_ko') - - try: - answers = dns.resolver.query(domain, 'MX') - mx_check = {} - i = 0 - for server in answers: - mx_id = 'mx%s' % i - mx_check[mx_id] = server - i = i + 1 - except: - mx_check = m18n.n('network_check_mx_ko') - result[u] = { - 'smtp_check': smtp_check, - 'mx_check': mx_check - } - elif u == 'usage': - result[u] = {} - for i in json.loads(glances.getNetwork()): - iname = i['interface_name'] - if iname in devices.keys(): - del i['interface_name'] - if human_readable: - for k in i.keys(): - if k != 'time_since_update': - i[k] = binary_to_human(i[k]) + 'B' - result[u][iname] = i - else: - logger.debug('interface name %s was not found', iname) - elif u == 'infos': - p_ipv4 = get_public_ip() or 'unknown' - - # TODO / FIXME : use functions in utils/network.py to manage this - l_ip = 'unknown' - for name, addrs in devices.items(): - if name == 'lo': - continue - if not isinstance(l_ip, dict): - l_ip = {} - l_ip[name] = _extract_inet(addrs) - - gateway = 'unknown' - output = subprocess.check_output('ip route show'.split()) - m = re.search('default via (.*) dev ([a-z]+[0-9]?)', output) - if m: - addr = _extract_inet(m.group(1), True) - if len(addr) == 1: - proto, gateway = addr.popitem() - - result[u] = { - 'public_ip': p_ipv4, - 'local_ip': l_ip, - 'gateway': gateway, - } - else: - raise YunohostError('unit_unknown', unit=u) - - if len(units) == 1: - return result[units[0]] - return result - - -def monitor_system(units=None, human_readable=False): - """ - Monitor system informations and usage - - Keyword argument: - units -- Unit(s) to monitor - human_readable -- Print sizes in human readable format - - """ - glances = _get_glances_api() - result = {} - - if units is None: - units = ['memory', 'cpu', 'process', 'uptime', 'infos'] - - # Retrieve monitoring for unit(s) - for u in units: - if u == 'memory': - ram = json.loads(glances.getMem()) - swap = json.loads(glances.getMemSwap()) - if human_readable: - for i in ram.keys(): - if i != 'percent': - ram[i] = binary_to_human(ram[i]) + 'B' - for i in swap.keys(): - if i != 'percent': - swap[i] = binary_to_human(swap[i]) + 'B' - result[u] = { - 'ram': ram, - 'swap': swap - } - elif u == 'cpu': - result[u] = { - 'load': json.loads(glances.getLoad()), - 'usage': json.loads(glances.getCpu()) - } - elif u == 'process': - result[u] = json.loads(glances.getProcessCount()) - elif u == 'uptime': - result[u] = (str(datetime.now() - datetime.fromtimestamp(psutil.boot_time())).split('.')[0]) - elif u == 'infos': - result[u] = json.loads(glances.getSystem()) - else: - raise YunohostError('unit_unknown', unit=u) - - if len(units) == 1 and not isinstance(result[units[0]], str): - return result[units[0]] - return result - - -def monitor_update_stats(period): - """ - Update monitoring statistics - - Keyword argument: - period -- Time period to update (day, week, month) - - """ - if period not in ['day', 'week', 'month']: - raise YunohostError('monitor_period_invalid') - - stats = _retrieve_stats(period) - if not stats: - stats = {'disk': {}, 'network': {}, 'system': {}, 'timestamp': []} - - monitor = None - # Get monitoring stats - if period == 'day': - monitor = _monitor_all('day') - else: - t = stats['timestamp'] - p = 'day' if period == 'week' else 'week' - if len(t) > 0: - monitor = _monitor_all(p, t[len(t) - 1]) - else: - monitor = _monitor_all(p, 0) - if not monitor: - raise YunohostError('monitor_stats_no_update') - - stats['timestamp'].append(time.time()) - - # Append disk stats - for dname, units in monitor['disk'].items(): - disk = {} - # Retrieve current stats for disk name - if dname in stats['disk'].keys(): - disk = stats['disk'][dname] - - for unit, values in units.items(): - # Continue if unit doesn't contain stats - if not isinstance(values, dict): - continue - - # Retrieve current stats for unit and append new ones - curr = disk[unit] if unit in disk.keys() else {} - if unit == 'io': - disk[unit] = _append_to_stats(curr, values, 'time_since_update') - elif unit == 'filesystem': - disk[unit] = _append_to_stats(curr, values, ['fs_type', 'mnt_point']) - stats['disk'][dname] = disk - - # Append network stats - net_usage = {} - for iname, values in monitor['network']['usage'].items(): - # Continue if units doesn't contain stats - if not isinstance(values, dict): - continue - - # Retrieve current stats and append new ones - curr = {} - if 'usage' in stats['network'] and iname in stats['network']['usage']: - curr = stats['network']['usage'][iname] - net_usage[iname] = _append_to_stats(curr, values, 'time_since_update') - stats['network'] = {'usage': net_usage, 'infos': monitor['network']['infos']} - - # Append system stats - for unit, values in monitor['system'].items(): - # Continue if units doesn't contain stats - if not isinstance(values, dict): - continue - - # Set static infos unit - if unit == 'infos': - stats['system'][unit] = values - continue - - # Retrieve current stats and append new ones - curr = stats['system'][unit] if unit in stats['system'].keys() else {} - stats['system'][unit] = _append_to_stats(curr, values) - - _save_stats(stats, period) - - -def monitor_show_stats(period, date=None): - """ - Show monitoring statistics - - Keyword argument: - period -- Time period to show (day, week, month) - - """ - if period not in ['day', 'week', 'month']: - raise YunohostError('monitor_period_invalid') - - result = _retrieve_stats(period, date) - if result is False: - raise YunohostError('monitor_stats_file_not_found') - elif result is None: - raise YunohostError('monitor_stats_period_unavailable') - return result - - -def monitor_enable(with_stats=False): - """ - Enable server monitoring - - Keyword argument: - with_stats -- Enable monitoring statistics - - """ - from yunohost.service import (service_status, service_enable, - service_start) - - glances = service_status('glances') - if glances['status'] != 'running': - service_start('glances') - if glances['loaded'] != 'enabled': - service_enable('glances') - - # Install crontab - if with_stats: - # day: every 5 min # week: every 1 h # month: every 4 h # - rules = ('*/5 * * * * root {cmd} day >> /dev/null\n' - '3 * * * * root {cmd} week >> /dev/null\n' - '6 */4 * * * root {cmd} month >> /dev/null').format( - cmd='/usr/bin/yunohost --quiet monitor update-stats') - with open(CRONTAB_PATH, 'w') as f: - f.write(rules) - - logger.success(m18n.n('monitor_enabled')) - - -def monitor_disable(): - """ - Disable server monitoring - - """ - from yunohost.service import (service_status, service_disable, - service_stop) - - glances = service_status('glances') - if glances['status'] != 'inactive': - service_stop('glances') - if glances['loaded'] != 'disabled': - try: - service_disable('glances') - except YunohostError as e: - logger.warning(e.strerror) - - # Remove crontab - try: - os.remove(CRONTAB_PATH) - except: - pass - - logger.success(m18n.n('monitor_disabled')) - - -def _get_glances_api(): - """ - Retrieve Glances API running on the local server - - """ - try: - p = xmlrpclib.ServerProxy(GLANCES_URI) - p.system.methodHelp('getAll') - except (xmlrpclib.ProtocolError, IOError): - pass - else: - return p - - from yunohost.service import service_status - - if service_status('glances')['status'] != 'running': - raise YunohostError('monitor_not_enabled') - raise YunohostError('monitor_glances_con_failed') - - -def _extract_inet(string, skip_netmask=False, skip_loopback=True): - """ - Extract IP addresses (v4 and/or v6) from a string limited to one - address by protocol - - Keyword argument: - string -- String to search in - skip_netmask -- True to skip subnet mask extraction - skip_loopback -- False to include addresses reserved for the - loopback interface - - Returns: - A dict of {protocol: address} with protocol one of 'ipv4' or 'ipv6' - - """ - ip4_pattern = '((25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}' - ip6_pattern = '(((?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?)::((?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?)' - ip4_pattern += '/[0-9]{1,2})' if not skip_netmask else ')' - ip6_pattern += '/[0-9]{1,3})' if not skip_netmask else ')' - result = {} - - for m in re.finditer(ip4_pattern, string): - addr = m.group(1) - if skip_loopback and addr.startswith('127.'): - continue - - # Limit to only one result - result['ipv4'] = addr - break - - for m in re.finditer(ip6_pattern, string): - addr = m.group(1) - if skip_loopback and addr == '::1': - continue - - # Limit to only one result - result['ipv6'] = addr - break - - return result - - -def binary_to_human(n, customary=False): - """ - Convert bytes or bits into human readable format with binary prefix - - Keyword argument: - n -- Number to convert - customary -- Use customary symbol instead of IEC standard - - """ - symbols = ('Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi') - if customary: - symbols = ('K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y') - prefix = {} - for i, s in enumerate(symbols): - prefix[s] = 1 << (i + 1) * 10 - for s in reversed(symbols): - if n >= prefix[s]: - value = float(n) / prefix[s] - return '%.1f%s' % (value, s) - return "%s" % n - - -def _retrieve_stats(period, date=None): - """ - Retrieve statistics from pickle file - - Keyword argument: - period -- Time period to retrieve (day, week, month) - date -- Date of stats to retrieve - - """ - pkl_file = None - - # Retrieve pickle file - if date is not None: - timestamp = calendar.timegm(date) - pkl_file = '%s/%d_%s.pkl' % (STATS_PATH, timestamp, period) - else: - pkl_file = '%s/%s.pkl' % (STATS_PATH, period) - if not os.path.isfile(pkl_file): - return False - - # Read file and process its content - with open(pkl_file, 'r') as f: - result = pickle.load(f) - if not isinstance(result, dict): - return None - return result - - -def _save_stats(stats, period, date=None): - """ - Save statistics to pickle file - - Keyword argument: - stats -- Stats dict to save - period -- Time period of stats (day, week, month) - date -- Date of stats - - """ - pkl_file = None - - # Set pickle file name - if date is not None: - timestamp = calendar.timegm(date) - pkl_file = '%s/%d_%s.pkl' % (STATS_PATH, timestamp, period) - else: - pkl_file = '%s/%s.pkl' % (STATS_PATH, period) - if not os.path.isdir(STATS_PATH): - os.makedirs(STATS_PATH) - - # Limit stats - if date is None: - t = stats['timestamp'] - limit = {'day': 86400, 'week': 604800, 'month': 2419200} - if (t[len(t) - 1] - t[0]) > limit[period]: - begin = t[len(t) - 1] - limit[period] - stats = _filter_stats(stats, begin) - - # Write file content - with open(pkl_file, 'w') as f: - pickle.dump(stats, f) - return True - - -def _monitor_all(period=None, since=None): - """ - Monitor all units (disk, network and system) for the given period - If since is None, real-time monitoring is returned. Otherwise, the - mean of stats since this timestamp is calculated and returned. - - Keyword argument: - period -- Time period to monitor (day, week, month) - since -- Timestamp of the stats beginning - - """ - result = {'disk': {}, 'network': {}, 'system': {}} - - # Real-time stats - if period == 'day' and since is None: - result['disk'] = monitor_disk() - result['network'] = monitor_network() - result['system'] = monitor_system() - return result - - # Retrieve stats and calculate mean - stats = _retrieve_stats(period) - if not stats: - return None - stats = _filter_stats(stats, since) - if not stats: - return None - result = _calculate_stats_mean(stats) - - return result - - -def _filter_stats(stats, t_begin=None, t_end=None): - """ - Filter statistics by beginning and/or ending timestamp - - Keyword argument: - stats -- Dict stats to filter - t_begin -- Beginning timestamp - t_end -- Ending timestamp - - """ - if t_begin is None and t_end is None: - return stats - - i_begin = i_end = None - # Look for indexes of timestamp interval - for i, t in enumerate(stats['timestamp']): - if t_begin and i_begin is None and t >= t_begin: - i_begin = i - if t_end and i != 0 and i_end is None and t > t_end: - i_end = i - # Check indexes - if i_begin is None: - if t_begin and t_begin > stats['timestamp'][0]: - return None - i_begin = 0 - if i_end is None: - if t_end and t_end < stats['timestamp'][0]: - return None - i_end = len(stats['timestamp']) - if i_begin == 0 and i_end == len(stats['timestamp']): - return stats - - # Filter function - def _filter(s, i, j): - for k, v in s.items(): - if isinstance(v, dict): - s[k] = _filter(v, i, j) - elif isinstance(v, list): - s[k] = v[i:j] - return s - - stats = _filter(stats, i_begin, i_end) - return stats - - -def _calculate_stats_mean(stats): - """ - Calculate the weighted mean for each statistic - - Keyword argument: - stats -- Stats dict to process - - """ - timestamp = stats['timestamp'] - t_sum = sum(timestamp) - del stats['timestamp'] - - # Weighted mean function - def _mean(s, t, ts): - for k, v in s.items(): - if isinstance(v, dict): - s[k] = _mean(v, t, ts) - elif isinstance(v, list): - try: - nums = [float(x * t[i]) for i, x in enumerate(v)] - except: - pass - else: - s[k] = sum(nums) / float(ts) - return s - - stats = _mean(stats, timestamp, t_sum) - return stats - - -def _append_to_stats(stats, monitor, statics=[]): - """ - Append monitoring statistics to current statistics - - Keyword argument: - stats -- Current stats dict - monitor -- Monitoring statistics - statics -- List of stats static keys - - """ - if isinstance(statics, str): - statics = [statics] - - # Appending function - def _append(s, m, st): - for k, v in m.items(): - if k in st: - s[k] = v - elif isinstance(v, dict): - if k not in s: - s[k] = {} - s[k] = _append(s[k], v, st) - else: - if k not in s: - s[k] = [] - if isinstance(v, list): - s[k].extend(v) - else: - s[k].append(v) - return s - - stats = _append(stats, monitor, statics) - return stats From 1372ab916c8db501042397db58a4b91310fff3ab Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 6 Nov 2019 19:05:43 +0100 Subject: [PATCH 61/61] Stale strings + try to keep the namespace-like tidy --- locales/en.json | 11 +++-------- src/yunohost/diagnosis.py | 4 ++-- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/locales/en.json b/locales/en.json index 4a432345f..ebe6b4571 100644 --- a/locales/en.json +++ b/locales/en.json @@ -150,18 +150,11 @@ "confirm_app_install_thirdparty": "DANGER! This app is not part of Yunohost's app catalog. Installing third-party apps may compromise the integrity and security of your system. You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or break your system… If you are willing to take that risk anyway, type '{answers:s}'", "custom_app_url_required": "You must provide a URL to upgrade your custom app {app:s}", "custom_appslist_name_required": "You must provide a name for your custom app list", - "diagnosis_debian_version_error": "Could not retrieve the Debian version: {error}", - "diagnosis_kernel_version_error": "Could not retrieve kernel version: {error}", "diagnosis_basesystem_host": "Server is running Debian {debian_version}.", "diagnosis_basesystem_kernel": "Server is running Linux kernel {kernel_version}", "diagnosis_basesystem_ynh_single_version": "{0} version: {1}", "diagnosis_basesystem_ynh_main_version": "Server is running YunoHost {main_version}", "diagnosis_basesystem_ynh_inconsistent_versions": "You are running inconsistents versions of the YunoHost packages ... most probably because of a failed or partial upgrade.", - "diagnosis_monitor_disk_error": "Could not monitor disks: {error}", - "diagnosis_monitor_system_error": "Could not monitor system: {error}", - "diagnosis_no_apps": "No such installed app", - "dpkg_is_broken": "You cannot do this right now because dpkg/APT (the system package managers) seems to be in a broken state… You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.", - "dpkg_lock_not_available": "This command can't be ran right now because another program seems to be using the lock of dpkg (the system package manager)", "diagnosis_display_tip_web": "You can go to the Diagnosis section (in the home screen) to see the issues found.", "diagnosis_display_tip_cli": "You can run 'yunohost diagnosis show --issues' to display the issues found.", "diagnosis_failed_for_category": "Diagnosis failed for category '{category}' : {error}", @@ -222,6 +215,7 @@ "diagnosis_http_could_not_diagnose": "Could not diagnose if domain is reachable from outside. Error: {error}", "diagnosis_http_ok": "Domain {domain} is reachable from outside.", "diagnosis_http_unreachable": "Domain {domain} is unreachable through HTTP from outside.", + "diagnosis_unknown_categories": "The following categories are unknown : {categories}", "domain_cannot_remove_main": "You cannot remove '{domain:s}' since it's the main domain, you need first to set another domain as the main domain using 'yunohost domain main-domain -n ', here is the list of candidate domains: {other_domains:s}", "domain_cannot_remove_main_add_new_one": "You cannot remove '{domain:s}' since it's the main domain and your only domain, you need to first add another domain using 'yunohost domain add ', then set is as the main domain using 'yunohost domain main-domain -n ' and then you can remove the domain '{domain:s}' using 'yunohost domain remove {domain:s}'.'", "domain_cert_gen_failed": "Could not generate certificate", @@ -239,6 +233,8 @@ "domains_available": "Available domains:", "done": "Done", "downloading": "Downloading…", + "dpkg_is_broken": "You cannot do this right now because dpkg/APT (the system package managers) seems to be in a broken state… You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.", + "dpkg_lock_not_available": "This command can't be ran right now because another program seems to be using the lock of dpkg (the system package manager)", "dyndns_could_not_check_provide": "Could not check if {provider:s} can provide {domain:s}.", "dyndns_could_not_check_available": "Could not check if {domain:s} is available on {provider:s}.", "dyndns_cron_installed": "DynDNS cron job created", @@ -604,7 +600,6 @@ "user_update_failed": "Could not update user {user}: {error}", "user_updated": "User info changed", "users_available": "Available users:", - "unknown_categories": "The following categories are unknown : {categories}", "yunohost_already_installed": "YunoHost is already installed", "yunohost_ca_creation_failed": "Could not create certificate authority", "yunohost_ca_creation_success": "Local certification authority created.", diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 19dd03042..121a0c2ae 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -56,7 +56,7 @@ def diagnosis_show(categories=[], issues=False, full=False, share=False): else: unknown_categories = [c for c in categories if c not in all_categories_names] if unknown_categories: - raise YunohostError('unknown_categories', categories=", ".join(categories)) + raise YunohostError('diagnosis_unknown_categories', categories=", ".join(categories)) # Fetch all reports all_reports = [] @@ -127,7 +127,7 @@ def diagnosis_run(categories=[], force=False): else: unknown_categories = [c for c in categories if c not in all_categories_names] if unknown_categories: - raise YunohostError('unknown_categories', categories=", ".join(unknown_categories)) + raise YunohostError('diagnosis_unknown_categories', categories=", ".join(unknown_categories)) issues = [] # Call the hook ...