From 6a0959dd1d7ba58b1bfa19de9ae9e2d7881ad556 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 20 Jul 2019 06:32:08 +0200 Subject: [PATCH 01/94] [fix] moulinette logs were never displayed #lol --- bin/yunohost | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/yunohost b/bin/yunohost index 10a21a9da..672c1b539 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -145,7 +145,7 @@ def _init_moulinette(debug=False, quiet=False): }, 'moulinette': { 'level': level, - 'handlers': [], + 'handlers': handlers, 'propagate': True, }, 'moulinette.interface': { From 6284ad09c65a3fe377e0a9b542658773aeade1f0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 10 Sep 2019 14:06:18 +0200 Subject: [PATCH 02/94] Simplify madness code about checking requirements --- locales/en.json | 3 -- src/yunohost/app.py | 28 ++----------------- src/yunohost/utils/packages.py | 51 ++++++---------------------------- 3 files changed, 11 insertions(+), 71 deletions(-) diff --git a/locales/en.json b/locales/en.json index 2d708279d..d471f72ca 100644 --- a/locales/en.json +++ b/locales/en.json @@ -21,7 +21,6 @@ "app_change_url_success": "{app:s} URL is now {domain:s}{path:s}", "app_extraction_failed": "Could not extract the installation files", "app_id_invalid": "Invalid app ID", - "app_incompatible": "The app {app} is incompatible with your YunoHost version", "app_install_files_invalid": "These files cannot be installed", "app_location_already_used": "The app '{app}' is already installed in ({path})", "app_make_default_location_already_used": "Can't make the app '{app}' the default on the domain, {domain} is already in use by the other app '{other_app}'", @@ -410,8 +409,6 @@ "no_restore_script": "No restore script found for the app '{app:s}'", "not_enough_disk_space": "Not enough free space on '{path:s}'", "operation_interrupted": "The operation was manually interrupted?", - "package_not_installed": "Package '{pkgname}' is not installed", - "package_unexpected_error": "An unexpected error occurred processing the package '{pkgname}'", "package_unknown": "Unknown package '{pkgname}'", "packages_upgrade_critical_later": "Critical packages ({packages:s}) will be upgraded later", "packages_upgrade_failed": "Could not upgrade all the packages", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index c93c7ba80..dcc46878b 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2461,38 +2461,14 @@ def _check_manifest_requirements(manifest, app_instance_name): """Check if required packages are met from the manifest""" requirements = manifest.get('requirements', dict()) - # FIXME: Deprecate min_version key - if 'min_version' in manifest: - requirements['yunohost'] = '>> {0}'.format(manifest['min_version']) - logger.debug("the manifest key 'min_version' is deprecated, " - "use 'requirements' instead.") - - # Validate multi-instance app - if is_true(manifest.get('multi_instance', False)): - # Handle backward-incompatible change introduced in yunohost >= 2.3.6 - # See https://github.com/YunoHost/issues/issues/156 - yunohost_req = requirements.get('yunohost', None) - if (not yunohost_req or - not packages.SpecifierSet(yunohost_req) & '>= 2.3.6'): - raise YunohostError('{0}{1}'.format( - m18n.g('colon', m18n.n('app_incompatible'), app=app_instance_name), - m18n.n('app_package_need_update', app=app_instance_name))) - elif not requirements: + if not requirements: return logger.debug(m18n.n('app_requirements_checking', app=app_instance_name)) - # Retrieve versions of each required package - try: - versions = packages.get_installed_version( - *requirements.keys(), strict=True, as_dict=True) - except packages.PackageException as e: - raise YunohostError('app_requirements_failed', error=str(e), app=app_instance_name) - # Iterate over requirements for pkgname, spec in requirements.items(): - version = versions[pkgname] - if version not in packages.SpecifierSet(spec): + if not packages.meets_version_specifier(pkgname, spec): raise YunohostError('app_requirements_unmeet', pkgname=pkgname, version=version, spec=spec, app=app_instance_name) diff --git a/src/yunohost/utils/packages.py b/src/yunohost/utils/packages.py index 84901bbff..6df736432 100644 --- a/src/yunohost/utils/packages.py +++ b/src/yunohost/utils/packages.py @@ -33,36 +33,6 @@ logger = logging.getLogger('yunohost.utils.packages') # Exceptions ----------------------------------------------------------------- -class PackageException(Exception): - - """Base exception related to a package - - Represent an exception related to the package named `pkgname`. If no - `message` is provided, it will first try to use the translation key - `message_key` if defined by the derived class. Otherwise, a standard - message will be used. - - """ - message_key = 'package_unexpected_error' - - def __init__(self, pkgname, message=None): - super(PackageException, self).__init__( - message or m18n.n(self.message_key, pkgname=pkgname)) - self.pkgname = pkgname - - -class UnknownPackage(PackageException): - - """The package is not found in the cache.""" - message_key = 'package_unknown' - - -class UninstalledPackage(PackageException): - - """The package is not installed.""" - message_key = 'package_not_installed' - - class InvalidSpecifier(ValueError): """An invalid specifier was found.""" @@ -402,41 +372,38 @@ def get_installed_version(*pkgnames, **kwargs): """Get the installed version of package(s) Retrieve one or more packages named `pkgnames` and return their installed - version as a dict or as a string if only one is requested and `as_dict` is - `False`. If `strict` is `True`, an exception will be raised if a package - is unknown or not installed. + version as a dict or as a string if only one is requested. """ versions = OrderedDict() cache = apt.Cache() # Retrieve options - as_dict = kwargs.get('as_dict', False) - strict = kwargs.get('strict', False) with_repo = kwargs.get('with_repo', False) for pkgname in pkgnames: try: pkg = cache[pkgname] except KeyError: - if strict: - raise UnknownPackage(pkgname) logger.warning(m18n.n('package_unknown', pkgname=pkgname)) + if with_repo: + versions[pkgname] = { + "version": None, + "repo": None, + } + else: + versions[pkgname] = None continue try: version = pkg.installed.version except AttributeError: - if strict: - raise UninstalledPackage(pkgname) version = None try: # stable, testing, unstable repo = pkg.installed.origins[0].component except AttributeError: - if strict: - raise UninstalledPackage(pkgname) repo = "" if with_repo: @@ -449,7 +416,7 @@ def get_installed_version(*pkgnames, **kwargs): else: versions[pkgname] = version - if len(pkgnames) == 1 and not as_dict: + if len(pkgnames) == 1: return versions[pkgnames[0]] return versions From 3bc4945ccf12c1bbc0c47a5c609f8ab1baf161c4 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 11 Sep 2019 04:29:04 +0200 Subject: [PATCH 03/94] [ux] 'new-domain' argument of maindomain command was confusing --- data/actionsmap/yunohost.yml | 2 +- src/yunohost/tools.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 22037f05f..e18a45276 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1547,7 +1547,7 @@ tools: - PUT /domains/main arguments: -n: - full: --new-domain + full: --new-main-domain help: Change the current main domain extra: pattern: *pattern_domain diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 64689fe0c..e4d4c4274 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -165,7 +165,7 @@ def tools_adminpw(new_password, check_strength=True): @is_unit_operation() -def tools_maindomain(operation_logger, new_domain=None): +def tools_maindomain(operation_logger, new_main_domain=None): """ Check the current main domain, or change it From 877cfc1fe550a305a624ba258bf0d30469afab10 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 11 Sep 2019 04:31:22 +0200 Subject: [PATCH 04/94] [ux] move 'maindomain' command from 'tools' to 'domain' section --- data/actionsmap/yunohost.yml | 14 +++++++++ src/yunohost/domain.py | 58 ++++++++++++++++++++++++++++++++++++ src/yunohost/tools.py | 56 ++-------------------------------- 3 files changed, 75 insertions(+), 53 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index e18a45276..b7b4eaf88 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -441,6 +441,19 @@ domain: - !!str ^[0-9]+$ - "pattern_positive_number" + ### domain_maindomain() + maindomain: + action_help: Check the current main domain, or change it + api: + - GET /domains/main + - PUT /domains/main + arguments: + -n: + full: --new-main-domain + help: Change the current main domain + extra: + pattern: *pattern_domain + ### certificate_status() cert-status: action_help: List status of current certificates (all by default). @@ -1542,6 +1555,7 @@ tools: ### tools_maindomain() maindomain: action_help: Check the current main domain, or change it + deprecated: true api: - GET /domains/main - PUT /domains/main diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 3f906748b..8529433ee 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -34,10 +34,12 @@ from moulinette.utils.log import getActionLogger import yunohost.certificate +from yunohost.app import app_ssowatconf from yunohost.regenconf import regen_conf from yunohost.utils.network import get_public_ip from yunohost.log import is_unit_operation from yunohost.hook import hook_callback +from yunohost.tools import _set_hostname logger = getActionLogger('yunohost.domain') @@ -233,6 +235,62 @@ def domain_dns_conf(domain, ttl=None): return result +@is_unit_operation() +def domain_maindomain(operation_logger, new_main_domain=None): + """ + Check the current main domain, or change it + + Keyword argument: + new_main_domain -- The new domain to be set as the main domain + + """ + + # If no new domain specified, we return the current main domain + if not new_main_domain: + return {'current_main_domain': _get_maindomain()} + + # Check domain exists + if new_main_domain not in domain_list()['domains']: + raise YunohostError('domain_unknown') + + operation_logger.related_to.append(('domain', new_main_domain)) + operation_logger.start() + + # Apply changes to ssl certs + ssl_key = "/etc/ssl/private/yunohost_key.pem" + ssl_crt = "/etc/ssl/private/yunohost_crt.pem" + new_ssl_key = "/etc/yunohost/certs/%s/key.pem" % new_main_domain + new_ssl_crt = "/etc/yunohost/certs/%s/crt.pem" % new_main_domain + + try: + if os.path.exists(ssl_key) or os.path.lexists(ssl_key): + os.remove(ssl_key) + if os.path.exists(ssl_crt) or os.path.lexists(ssl_crt): + os.remove(ssl_crt) + + os.symlink(new_ssl_key, ssl_key) + os.symlink(new_ssl_crt, ssl_crt) + + _set_maindomain(new_main_domain) + except Exception as e: + logger.warning("%s" % e, exc_info=1) + raise YunohostError('maindomain_change_failed') + + _set_hostname(new_main_domain) + + # Generate SSOwat configuration file + app_ssowatconf() + + # Regen configurations + try: + with open('/etc/yunohost/installed', 'r'): + regen_conf() + except IOError: + pass + + logger.success(m18n.n('maindomain_changed')) + + def domain_cert_status(domain_list, full=False): return yunohost.certificate.certificate_status(domain_list, full) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index e4d4c4274..5feb04dec 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -39,7 +39,7 @@ from moulinette.utils.log import getActionLogger from moulinette.utils.process import check_output, call_async_output from moulinette.utils.filesystem import read_json, write_to_json, read_yaml, write_to_yaml from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, app_list, _install_appslist_fetch_cron -from yunohost.domain import domain_add, domain_list, _get_maindomain, _set_maindomain +from yunohost.domain import domain_add, domain_list from yunohost.dyndns import _dyndns_available, _dyndns_provides from yunohost.firewall import firewall_upnp from yunohost.service import service_status, service_start, service_enable @@ -166,58 +166,8 @@ def tools_adminpw(new_password, check_strength=True): @is_unit_operation() def tools_maindomain(operation_logger, new_main_domain=None): - """ - Check the current main domain, or change it - - Keyword argument: - new_domain -- The new domain to be set as the main domain - - """ - - # If no new domain specified, we return the current main domain - if not new_domain: - return {'current_main_domain': _get_maindomain()} - - # Check domain exists - if new_domain not in domain_list()['domains']: - raise YunohostError('domain_unknown') - - operation_logger.related_to.append(('domain', new_domain)) - operation_logger.start() - - # Apply changes to ssl certs - ssl_key = "/etc/ssl/private/yunohost_key.pem" - ssl_crt = "/etc/ssl/private/yunohost_crt.pem" - new_ssl_key = "/etc/yunohost/certs/%s/key.pem" % new_domain - new_ssl_crt = "/etc/yunohost/certs/%s/crt.pem" % new_domain - - try: - if os.path.exists(ssl_key) or os.path.lexists(ssl_key): - os.remove(ssl_key) - if os.path.exists(ssl_crt) or os.path.lexists(ssl_crt): - os.remove(ssl_crt) - - os.symlink(new_ssl_key, ssl_key) - os.symlink(new_ssl_crt, ssl_crt) - - _set_maindomain(new_domain) - except Exception as e: - logger.warning("%s" % e, exc_info=1) - raise YunohostError('maindomain_change_failed') - - _set_hostname(new_domain) - - # Generate SSOwat configuration file - app_ssowatconf() - - # Regen configurations - try: - with open('/etc/yunohost/installed', 'r'): - regen_conf() - except IOError: - pass - - logger.success(m18n.n('maindomain_changed')) + from yunohost.domain import domain_maindomain + return domain_main_domain(new_main_domain=new_main_domain) def _set_hostname(hostname, pretty_hostname=None): From f732085d3fb0ada7bc628a034dc5ed5a30774e1d Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 11 Sep 2019 04:48:53 +0200 Subject: [PATCH 05/94] [ux] rename 'yunohost domain maindomain' to 'yunohost domain main-domain' --- data/actionsmap/yunohost.yml | 4 +++- src/yunohost/domain.py | 2 +- src/yunohost/tools.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index b7b4eaf88..76500ac13 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -442,8 +442,10 @@ domain: - "pattern_positive_number" ### domain_maindomain() - maindomain: + main-domain: action_help: Check the current main domain, or change it + deprecated_alias: + - maindomain api: - GET /domains/main - PUT /domains/main diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 8529433ee..bed8e6883 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -236,7 +236,7 @@ def domain_dns_conf(domain, ttl=None): @is_unit_operation() -def domain_maindomain(operation_logger, new_main_domain=None): +def domain_main_domain(operation_logger, new_main_domain=None): """ Check the current main domain, or change it diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 5feb04dec..e33ef8e82 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -166,7 +166,8 @@ def tools_adminpw(new_password, check_strength=True): @is_unit_operation() def tools_maindomain(operation_logger, new_main_domain=None): - from yunohost.domain import domain_maindomain + from yunohost.domain import domain_main_domain + logger.warning(m18n.g("deprecated_command_alias", prog="yunohost", old="tools maindomain", new="domain main-domain")) return domain_main_domain(new_main_domain=new_main_domain) From 1584c907005332405c9f92811297efa658360d61 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 11 Sep 2019 04:50:50 +0200 Subject: [PATCH 06/94] [ux] more useful deprecated warning that mention the new command --- data/actionsmap/yunohost.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 76500ac13..0fbca6efc 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1557,7 +1557,6 @@ tools: ### tools_maindomain() maindomain: action_help: Check the current main domain, or change it - deprecated: true api: - GET /domains/main - PUT /domains/main From 94ba47d171784a4ff755015cfc48c8a5343533c7 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 11 Sep 2019 05:11:33 +0200 Subject: [PATCH 07/94] [ux] better error messages when trying to remove the main domain --- locales/en.json | 7 ++++--- src/yunohost/domain.py | 9 ++++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/locales/en.json b/locales/en.json index 98cbf22e9..3964c17af 100644 --- a/locales/en.json +++ b/locales/en.json @@ -154,12 +154,13 @@ "diagnosis_no_apps": "No installed application", "dpkg_is_broken": "You cannot do this right now because dpkg/APT (the system package managers) seems to be in a broken state… You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.", "dpkg_lock_not_available": "This command can't be ran right now because another program seems to be using the lock of dpkg (the system package manager)", - "domain_cannot_remove_main": "Cannot remove main domain. Set one first", + "domain_cannot_remove_main": "You cannot remove '{domain:s}' since it's the main domain, you need first to set another domain as the main domain using 'yunohost domain main-domain -n ', here is the list of candidate domains: {other_domains:s}", + "domain_cannot_remove_main_add_new_one": "You cannot remove '{domain:s}' since it's the main domain and your only domain, you need to first add another domain using 'yunohost domain add ', then set is as the main domain using 'yunohost domain main-domain -n ' and then you can remove the domain '{domain:s}' using 'yunohost domain remove {domain:s}'.'", "domain_cert_gen_failed": "Could not generate certificate", "domain_created": "Domain created", - "domain_creation_failed": "Could not create domain {domain}: {error}", + "domain_creation_failed": "Unable to create domain {domain}: {error}", "domain_deleted": "Domain deleted", - "domain_deletion_failed": "Could not delete domain {domain}: {error}", + "domain_deletion_failed": "Unable to delete domain {domain}: {error}", "domain_dns_conf_is_just_a_recommendation": "This command shows you the *recommended* configuration. It does not actually set up the DNS configuration for you. It is your responsability to configure your DNS zone in your registrar according to this recommendation.", "domain_dyndns_already_subscribed": "You have already subscribed to a DynDNS domain", "domain_dyndns_root_unknown": "Unknown DynDNS root domain", diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index bed8e6883..f26a19e8d 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -156,7 +156,14 @@ def domain_remove(operation_logger, domain, force=False): # Check domain is not the main domain if domain == _get_maindomain(): - raise YunohostError('domain_cannot_remove_main') + other_domains = domain_list()["domains"] + other_domains.remove(domain) + + if other_domains: + raise YunohostError('domain_cannot_remove_main', + domain=domain, other_domains="\n * " + ("\n * ".join(other_domains))) + else: + raise YunohostError('domain_cannot_remove_main_add_new_one', domain=domain) # Check if apps are installed on the domain for app in os.listdir('/etc/yunohost/apps/'): From 01ad8ec9643220fd835b3c20e39d14176f2c1770 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 14 Sep 2019 17:06:20 +0200 Subject: [PATCH 08/94] [mod] remove now useless decorator --- src/yunohost/tools.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index e33ef8e82..bae17e15a 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -164,8 +164,7 @@ def tools_adminpw(new_password, check_strength=True): logger.success(m18n.n('admin_password_changed')) -@is_unit_operation() -def tools_maindomain(operation_logger, new_main_domain=None): +def tools_maindomain(new_main_domain=None): from yunohost.domain import domain_main_domain logger.warning(m18n.g("deprecated_command_alias", prog="yunohost", old="tools maindomain", new="domain main-domain")) return domain_main_domain(new_main_domain=new_main_domain) From 1cfe32f6f31918f92c095c2d6b400a82b9a16133 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 14 Sep 2019 17:19:09 +0200 Subject: [PATCH 09/94] [fix] circular import --- src/yunohost/domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index f26a19e8d..64c2d9927 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -39,7 +39,6 @@ from yunohost.regenconf import regen_conf from yunohost.utils.network import get_public_ip from yunohost.log import is_unit_operation from yunohost.hook import hook_callback -from yunohost.tools import _set_hostname logger = getActionLogger('yunohost.domain') @@ -251,6 +250,7 @@ def domain_main_domain(operation_logger, new_main_domain=None): new_main_domain -- The new domain to be set as the main domain """ + from yunohost.tools import _set_hostname # If no new domain specified, we return the current main domain if not new_main_domain: From 6eb4b3f89e76223241491019e55fd7a68b53d40a Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 15 Sep 2019 03:11:52 +0200 Subject: [PATCH 10/94] [mod] use renamed domain_main_domain function in postinstall --- src/yunohost/tools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index bae17e15a..fe75bcf61 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -231,6 +231,7 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False, """ from yunohost.utils.password import assert_password_is_strong_enough + from yunohost.domain import domain_main_domain dyndns_provider = "dyndns.yunohost.org" @@ -353,7 +354,7 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False, # New domain config regen_conf(['nsswitch'], force=True) domain_add(domain, dyndns) - tools_maindomain(domain) + domain_main_domain(domain) # Change LDAP admin password tools_adminpw(password, check_strength=not force_password) From f18252d82e6a932dc4d86494d6048668152a5cf5 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 15 Sep 2019 03:39:04 +0200 Subject: [PATCH 11/94] [i18n] change translation key to match new function name --- locales/ar.json | 6 +++--- locales/ca.json | 6 +++--- locales/de.json | 4 ++-- locales/en.json | 6 +++--- locales/es.json | 4 ++-- locales/fr.json | 6 +++--- locales/it.json | 6 +++--- locales/oc.json | 6 +++--- locales/pt.json | 4 ++-- src/yunohost/domain.py | 4 ++-- 10 files changed, 26 insertions(+), 26 deletions(-) diff --git a/locales/ar.json b/locales/ar.json index 46f9315af..fba086bc4 100644 --- a/locales/ar.json +++ b/locales/ar.json @@ -211,8 +211,8 @@ "mail_domain_unknown": "Unknown mail address domain '{domain:s}'", "mail_forward_remove_failed": "Unable to remove mail forward '{mail:s}'", "mailbox_used_space_dovecot_down": "Dovecot mailbox service need to be up, if you want to get mailbox used space", - "maindomain_change_failed": "Unable to change the main domain", - "maindomain_changed": "The main domain has been changed", + "main_domain_change_failed": "Unable to change the main domain", + "main_domain_changed": "The main domain has been changed", "migrate_tsig_end": "Migration to hmac-sha512 finished", "migrate_tsig_failed": "Migrating the dyndns domain {domain} to hmac-sha512 failed, rolling back. Error: {error_code} - {error}", "migrate_tsig_start": "Not secure enough key algorithm detected for TSIG signature of domain '{domain}', initiating migration to the more secure one hmac-sha512", @@ -404,7 +404,7 @@ "log_user_create": "إضافة المستخدم '{}'", "log_user_delete": "حذف المستخدم '{}'", "log_user_update": "تحديث معلومات المستخدم '{}'", - "log_tools_maindomain": "جعل '{}' كنطاق أساسي", + "log_domain_main_domain": "جعل '{}' كنطاق أساسي", "log_tools_upgrade": "تحديث حُزم ديبيان", "log_tools_shutdown": "إطفاء الخادم", "log_tools_reboot": "إعادة تشغيل الخادم", diff --git a/locales/ca.json b/locales/ca.json index f5c040670..6ec3c0890 100644 --- a/locales/ca.json +++ b/locales/ca.json @@ -271,7 +271,7 @@ "log_user_create": "Afegeix l'usuari « {} »", "log_user_delete": "Elimina l'usuari « {} »", "log_user_update": "Actualitza la informació de l'usuari « {} »", - "log_tools_maindomain": "Fes de « {} » el domini principal", + "log_domain_main_domain": "Fes de « {} » el domini principal", "log_tools_migrations_migrate_forward": "Migrar", "log_tools_migrations_migrate_backward": "Migrar endarrera", "log_tools_postinstall": "Fer la post instal·lació del servidor YunoHost", @@ -289,8 +289,8 @@ "mail_forward_remove_failed": "No s'han pogut eliminar el reenviament de correu «{mail:s}»", "mailbox_used_space_dovecot_down": "S'ha d'engegar el servei de correu Dovecot per poder obtenir l'espai utilitzat per la bústia de correu", "mail_unavailable": "Aquesta adreça de correu esta reservada i ha de ser atribuïda automàticament el primer usuari", - "maindomain_change_failed": "No s'ha pogut canviar el domini principal", - "maindomain_changed": "S'ha canviat el domini principal", + "main_domain_change_failed": "No s'ha pogut canviar el domini principal", + "main_domain_changed": "S'ha canviat el domini principal", "migrate_tsig_end": "La migració cap a hmac-sha512 s'ha acabat", "migrate_tsig_failed": "Ha fallat la migració del domini dyndns {domain} cap a hmac-sha512, anul·lant les modificacions. Error: {error_code} - {error}", "migrate_tsig_start": "L'algoritme de generació de claus no es prou segur per a la signatura TSIG del domini «{domain}», començant la migració cap a un de més segur hmac-sha512", diff --git a/locales/de.json b/locales/de.json index d03226187..dc9ea70f1 100644 --- a/locales/de.json +++ b/locales/de.json @@ -104,8 +104,8 @@ "mail_alias_remove_failed": "E-Mail Alias '{mail:s}' konnte nicht entfernt werden", "mail_domain_unknown": "Unbekannte Mail Domain '{domain:s}'", "mail_forward_remove_failed": "Mailweiterleitung '{mail:s}' konnte nicht entfernt werden", - "maindomain_change_failed": "Die Hauptdomain konnte nicht geändert werden", - "maindomain_changed": "Die Hauptdomain wurde geändert", + "main_domain_change_failed": "Die Hauptdomain konnte nicht geändert werden", + "main_domain_changed": "Die Hauptdomain wurde geändert", "monitor_disabled": "Das Servermonitoring wurde erfolgreich deaktiviert", "monitor_enabled": "Das Servermonitoring wurde aktiviert", "monitor_glances_con_failed": "Verbindung mit Glances nicht möglich", diff --git a/locales/en.json b/locales/en.json index 3964c17af..c5c468703 100644 --- a/locales/en.json +++ b/locales/en.json @@ -277,7 +277,7 @@ "log_user_update": "Update user info of '{}'", "log_user_permission_update": "Update accesses for permission '{}'", "log_user_permission_reset": "Reset permission '{}'", - "log_tools_maindomain": "Make '{}' the main domain", + "log_domain_main_domain": "Make '{}' as main domain", "log_tools_migrations_migrate_forward": "Migrate forward", "log_tools_postinstall": "Postinstall your YunoHost server", "log_tools_upgrade": "Upgrade system packages", @@ -292,8 +292,8 @@ "mailbox_disabled": "E-mail turned off for user {user:s}", "mailbox_used_space_dovecot_down": "The Dovecot mailbox service needs to be up, if you want to fetch used mailbox space", "mail_unavailable": "This e-mail address is reserved and shall be automatically allocated to the very first user", - "maindomain_change_failed": "Could not change the main domain", - "maindomain_changed": "The main domain now changed", + "main_domain_change_failed": "Unable to change the main domain", + "main_domain_changed": "The main domain has been changed", "migrate_tsig_end": "Migration to HMAC-SHA-512 finished", "migrate_tsig_failed": "Could not migrate the DynDNS domain '{domain}' to HMAC-SHA-512, rolling back. Error: {error_code}, {error}", "migrate_tsig_start": "Insufficiently secure key algorithm detected for TSIG signature of the domain '{domain}', initiating migration to the more secure HMAC-SHA-512", diff --git a/locales/es.json b/locales/es.json index 02f46652b..a2dadc31c 100644 --- a/locales/es.json +++ b/locales/es.json @@ -121,8 +121,8 @@ "mail_alias_remove_failed": "No se pudo eliminar el alias de correo «{mail:s}»", "mail_domain_unknown": "Dirección de correo desconocida para el dominio «{domain:s}»", "mail_forward_remove_failed": "No se pudo eliminar el reenvío de correo «{mail:s}»", - "maindomain_change_failed": "No se pudo cambiar el dominio principal", - "maindomain_changed": "El dominio principal ha cambiado", + "main_domain_change_failed": "No se pudo cambiar el dominio principal", + "main_domain_changed": "El dominio principal ha cambiado", "monitor_disabled": "Desactivada la monitorización del servidor", "monitor_enabled": "Activada la monitorización del servidor", "monitor_glances_con_failed": "No se pudo conectar al servidor de Glances", diff --git a/locales/fr.json b/locales/fr.json index 8bffec8b2..b09dd54b7 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -122,8 +122,8 @@ "mail_alias_remove_failed": "Impossible de supprimer l’alias de courriel '{mail:s}'", "mail_domain_unknown": "Le domaine '{domain:s}' pour l'adresse de courriel est inconnu", "mail_forward_remove_failed": "Impossible de supprimer le courriel de transfert '{mail:s}'", - "maindomain_change_failed": "Impossible de modifier le domaine principal", - "maindomain_changed": "Le domaine principal modifié", + "main_domain_change_failed": "Impossible de modifier le domaine principal", + "main_domain_changed": "Le domaine principal a été modifié", "monitor_disabled": "La supervision du serveur a été désactivé", "monitor_enabled": "La supervision du serveur a été activé", "monitor_glances_con_failed": "Impossible de se connecter au serveur Glances", @@ -454,7 +454,7 @@ "log_user_create": "Ajouter l’utilisateur '{}'", "log_user_delete": "Supprimer l’utilisateur '{}'", "log_user_update": "Mettre à jour les informations de l’utilisateur '{}'", - "log_tools_maindomain": "Faire de '{}' le domaine principal", + "log_domain_main_domain": "Faire de '{}' le domaine principal", "log_tools_migrations_migrate_forward": "Migrer vers", "log_tools_migrations_migrate_backward": "Revenir en arrière", "log_tools_postinstall": "Faire la post-installation de votre serveur YunoHost", diff --git a/locales/it.json b/locales/it.json index 2c194d5a6..22cf9e2b0 100644 --- a/locales/it.json +++ b/locales/it.json @@ -136,8 +136,8 @@ "mail_domain_unknown": "Dominio d'indirizzo mail '{domain:s}' sconosciuto", "mail_forward_remove_failed": "Impossibile rimuovere la mail inoltrata '{mail:s}'", "mailbox_used_space_dovecot_down": "Il servizio di posta elettronica Dovecot deve essere attivato se vuoi riportare lo spazio usato dalla posta elettronica", - "maindomain_change_failed": "Impossibile cambiare il dominio principale", - "maindomain_changed": "Il dominio principale è stato cambiato", + "main_domain_change_failed": "Impossibile cambiare il dominio principale", + "main_domain_changed": "Il dominio principale è stato cambiato", "monitor_disabled": "Il monitoraggio del sistema è stato disattivato", "monitor_enabled": "Il monitoraggio del sistema è stato attivato", "monitor_glances_con_failed": "Impossibile collegarsi al server Glances", @@ -402,7 +402,7 @@ "log_user_create": "Aggiungi l'utente '{}'", "log_user_delete": "Elimina l'utente '{}'", "log_user_update": "Aggiornate le informazioni dell'utente '{}'", - "log_tools_maindomain": "Rendi '{}' dominio principale", + "log_domain_main_domain": "Rendi '{}' dominio principale", "log_tools_migrations_migrate_forward": "Migra avanti", "log_tools_migrations_migrate_backward": "Migra indietro", "log_tools_postinstall": "Postinstallazione del tuo server YunoHost", diff --git a/locales/oc.json b/locales/oc.json index 320a18341..49063e829 100644 --- a/locales/oc.json +++ b/locales/oc.json @@ -180,8 +180,8 @@ "invalid_url_format": "Format d’URL pas valid", "ldap_initialized": "L’annuari LDAP es inicializat", "license_undefined": "indefinida", - "maindomain_change_failed": "Modificacion impossibla del domeni màger", - "maindomain_changed": "Lo domeni màger es estat modificat", + "main_domain_change_failed": "Modificacion impossibla del domeni màger", + "main_domain_changed": "Lo domeni màger es estat modificat", "migrate_tsig_end": "La migracion cap a hmac-sha512 es acabada", "migrate_tsig_wait_2": "2 minutas…", "migrate_tsig_wait_3": "1 minuta…", @@ -440,7 +440,7 @@ "log_user_create": "Ajustar l’utilizaire « {} »", "log_user_delete": "Levar l’utilizaire « {} »", "log_user_update": "Actualizar las informacions a l’utilizaire « {} »", - "log_tools_maindomain": "Far venir « {} » lo domeni màger", + "log_domain_main_domain": "Far venir « {} » lo domeni màger", "log_tools_migrations_migrate_forward": "Migrar", "log_tools_migrations_migrate_backward": "Tornar en arrièr", "log_tools_postinstall": "Realizar la post installacion del servidor YunoHost", diff --git a/locales/pt.json b/locales/pt.json index 80a0d5ddd..c0ff0284e 100644 --- a/locales/pt.json +++ b/locales/pt.json @@ -74,8 +74,8 @@ "mail_alias_remove_failed": "Não foi possível remover a etiqueta de correio '{mail:s}'", "mail_domain_unknown": "Domínio de endereço de correio desconhecido '{domain:s}'", "mail_forward_remove_failed": "Não foi possível remover o reencaminhamento de correio '{mail:s}'", - "maindomain_change_failed": "Incapaz alterar o domínio raiz", - "maindomain_changed": "Domínio raiz alterado com êxito", + "main_domain_change_failed": "Incapaz alterar o domínio raiz", + "main_domain_changed": "Domínio raiz alterado com êxito", "monitor_disabled": "Monitorização do servidor parada com êxito", "monitor_enabled": "Monitorização do servidor ativada com êxito", "monitor_glances_con_failed": "Não foi possível ligar ao servidor Glances", diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 64c2d9927..8f8a68812 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -281,7 +281,7 @@ def domain_main_domain(operation_logger, new_main_domain=None): _set_maindomain(new_main_domain) except Exception as e: logger.warning("%s" % e, exc_info=1) - raise YunohostError('maindomain_change_failed') + raise YunohostError('main_domain_change_failed') _set_hostname(new_main_domain) @@ -295,7 +295,7 @@ def domain_main_domain(operation_logger, new_main_domain=None): except IOError: pass - logger.success(m18n.n('maindomain_changed')) + logger.success(m18n.n('main_domain_changed')) def domain_cert_status(domain_list, full=False): From f987e7872c9f09bc7320bed2aa16360a72a1ccb1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 28 Aug 2018 23:33:22 +0000 Subject: [PATCH 12/94] Skeleton / draft of API --- data/actionsmap/yunohost.yml | 50 ++++++++++++++++++++++++++++++++++++ src/yunohost/diagnosis.py | 44 +++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 src/yunohost/diagnosis.py diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 22037f05f..4f849160f 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1861,3 +1861,53 @@ log: --share: help: Share the full log using yunopaste action: store_true + + +############################# +# Diagnosis # +############################# +diagnosis: + category_help: Look for possible issues on the server + actions: + + list: + action_help: List diagnosis categories + api: GET /diagnosis/list + + report: + action_help: Show most recents diagnosis results + api: GET /diagnosis/report + arguments: + categories: + help: Diagnosis categories to display (all by default) + nargs: "*" + --full: + help: Display additional information + action: store_true + + run: + action_help: Show most recents diagnosis results + api: POST /diagnosis/run + arguments: + categories: + help: Diagnosis categories to run (all by default) + nargs: "*" + --force: + help: Display additional information + action: store_true + -a: + help: Serialized arguments for diagnosis scripts (e.g. "domain=domain.tld") + full: --args + + ignore: + action_help: Configure some diagnosis results to be ignored + api: PUT /diagnosis/ignore + arguments: + category: + help: Diagnosis category to be affected + -a: + help: Arguments, to be used to ignore only some parts of a report (e.g. "domain=domain.tld") + full: --args + --unignore: + help: Unignore a previously ignored report + action: store_true diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py new file mode 100644 index 000000000..aafbdcec3 --- /dev/null +++ b/src/yunohost/diagnosis.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +""" License + + Copyright (C) 2018 YunoHost + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program; if not, see http://www.gnu.org/licenses + +""" + +""" diagnosis.py + + Look for possible issues on the server +""" + +from moulinette import m18n +from moulinette.core import MoulinetteError +from moulinette.utils import log + +logger = log.getActionLogger('yunohost.diagnosis') + +def diagnosis_list(): + pass + +def diagnosis_report(categories=[], full=False): + pass + +def diagnosis_run(categories=[], force=False, args=""): + pass + +def diagnosis_ignore(category, args="", unignore=False): + pass + From 1d946ad073ed298185da49a6a5f111332c3daf25 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 29 Aug 2018 00:33:02 +0000 Subject: [PATCH 13/94] Implement diagnosis categories listing --- src/yunohost/diagnosis.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index aafbdcec3..0d312a7c1 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -28,10 +28,13 @@ from moulinette import m18n from moulinette.core import MoulinetteError from moulinette.utils import log +from yunohost.hook import hook_list + logger = log.getActionLogger('yunohost.diagnosis') def diagnosis_list(): - pass + all_categories_names = [ h for h, _ in _list_diagnosis_categories() ] + return { "categories": all_categories_names } def diagnosis_report(categories=[], full=False): pass @@ -42,3 +45,13 @@ def diagnosis_run(categories=[], force=False, args=""): def diagnosis_ignore(category, args="", unignore=False): pass +############################################################ + +def _list_diagnosis_categories(): + hooks_raw = hook_list("diagnosis", list_by="priority", show_info=True)["hooks"] + hooks = [] + for _, some_hooks in sorted(hooks_raw.items(), key=lambda h:int(h[0])): + for name, info in some_hooks.items(): + hooks.append((name, info["path"])) + + return hooks From b42bd20311797f59feb9ff7476ed19e49d20f8e5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 29 Aug 2018 01:34:15 +0000 Subject: [PATCH 14/94] First draft for diagnosis_run --- data/actionsmap/yunohost.yml | 2 +- locales/en.json | 1 + src/yunohost/diagnosis.py | 38 +++++++++++++++++++++++++++++++++--- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 4f849160f..aa85fdf70 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1893,7 +1893,7 @@ diagnosis: help: Diagnosis categories to run (all by default) nargs: "*" --force: - help: Display additional information + help: Ignore the cached report even if it is still 'fresh' action: store_true -a: help: Serialized arguments for diagnosis scripts (e.g. "domain=domain.tld") diff --git a/locales/en.json b/locales/en.json index f681fc4ea..a91da4fe9 100644 --- a/locales/en.json +++ b/locales/en.json @@ -547,6 +547,7 @@ "user_update_failed": "Could not update user {user}: {error}", "user_updated": "User info changed", "users_available": "Available users:", + "unknown_categories": "The following categories are unknown : {categories}", "yunohost_already_installed": "YunoHost is already installed", "yunohost_ca_creation_failed": "Could not create certificate authority", "yunohost_ca_creation_success": "Local certification authority created.", diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 0d312a7c1..10f09a576 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -24,14 +24,18 @@ Look for possible issues on the server """ +import errno + from moulinette import m18n from moulinette.core import MoulinetteError from moulinette.utils import log -from yunohost.hook import hook_list +from yunohost.hook import hook_list, hook_exec logger = log.getActionLogger('yunohost.diagnosis') +DIAGNOSIS_CACHE = "/var/cache/yunohost/diagnosis/" + def diagnosis_list(): all_categories_names = [ h for h, _ in _list_diagnosis_categories() ] return { "categories": all_categories_names } @@ -39,8 +43,36 @@ def diagnosis_list(): def diagnosis_report(categories=[], full=False): pass -def diagnosis_run(categories=[], force=False, args=""): - pass +def diagnosis_run(categories=[], force=False, args=None): + + # Get all the categories + all_categories = _list_diagnosis_categories() + all_categories_names = [ category for category, _ in all_categories ] + + # Check the requested category makes sense + if categories == []: + categories = all_categories_names + else: + unknown_categories = [ c for c in categories if c not in all_categories_names ] + if unknown_categories: + raise MoulinetteError(m18n.n('unknown_categories', categories=", ".join(categories))) + + # Transform "arg1=val1&arg2=val2" to { "arg1": "val1", "arg2": "val2" } + if args is not None: + args = { arg.split("=")[0]: arg.split("=")[1] for arg in args.split("&") } + else: + args = {} + args["force"] = force + + + # Call the hook ... + for category in categories: + logger.debug("Running diagnosis for %s ..." % category) + path = [p for n, p in all_categories if n == category ][0] + + # TODO : get the return value and do something with it + hook_exec(path, args=args, env=None) + def diagnosis_ignore(category, args="", unignore=False): pass From 7fb694dbccb62bcac206ffdf0688f7e1a52bd251 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 29 Aug 2018 23:48:16 +0000 Subject: [PATCH 15/94] Add diagnoser example for ip --- data/hooks/diagnosis/10-ip.py | 55 +++++++++++++++++++++++++++++++++++ src/yunohost/diagnosis.py | 15 +++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 data/hooks/diagnosis/10-ip.py diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py new file mode 100644 index 000000000..d8ab53c56 --- /dev/null +++ b/data/hooks/diagnosis/10-ip.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +from moulinette import m18n +from moulinette.utils.network import download_text +from yunohost.diagnosis import Diagnoser + +class IPDiagnoser(Diagnoser): + + def validate_args(self, args): + if "version" not in args.keys(): + return { "versions" : [4, 6] } + else: + if str(args["version"]) not in ["4", "6"]: + raise MoulinetteError(1, "Invalid version, should be 4 or 6.") + return { "versions" : [int(args["version"])] } + + def run(self): + + versions = self.args["versions"] + + if 4 in versions: + ipv4 = self.get_public_ip(4) + yield dict(meta = {"version": "4"}, + result = ipv4, + report = ("SUCCESS", m18n.n("diagnosis_network_connected_ipv4")) if ipv4 \ + else ("ERROR", m18n.n("diagnosis_network_no_ipv4"))) + + if 6 in versions: + ipv6 = self.get_public_ip(6) + yield dict(meta = {"version": "6"}, + result = ipv6, + report = ("SUCCESS", m18n.n("diagnosis_network_connected_ipv6")) if ipv6 \ + else ("WARNING", m18n.n("diagnosis_network_no_ipv6"))) + + + def get_public_ip(self, protocol=4): + + if protocol == 4: + url = 'https://ip.yunohost.org' + elif protocol == 6: + url = 'https://ip6.yunohost.org' + else: + raise ValueError("invalid protocol version") + + try: + return download_text(url, timeout=30).strip() + except Exception as e: + self.logger_debug("Could not get public IPv%s : %s" % (str(protocol), str(e))) + return None + + +def main(args, env, loggers): + + return IPDiagnoser(args, env, loggers).report() + diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 10f09a576..b5e0fa05a 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -71,7 +71,7 @@ def diagnosis_run(categories=[], force=False, args=None): path = [p for n, p in all_categories if n == category ][0] # TODO : get the return value and do something with it - hook_exec(path, args=args, env=None) + return {"report": hook_exec(path, args=args, env=None) } def diagnosis_ignore(category, args="", unignore=False): @@ -79,6 +79,19 @@ def diagnosis_ignore(category, args="", unignore=False): ############################################################ +class Diagnoser(): + + def __init__(self, args, env, loggers): + + self.logger_debug, self.logger_warning, self.logger_info = loggers + self.env = env + self.args = self.validate_args(args) + + def report(self): + + # TODO : implement some caching mecanism in there + return list(self.run()) + def _list_diagnosis_categories(): hooks_raw = hook_list("diagnosis", list_by="priority", show_info=True)["hooks"] hooks = [] From d34ddcaaf2964200b0a1edef167bb792b22cde26 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 30 Aug 2018 13:36:43 +0000 Subject: [PATCH 16/94] Implement caching mechanism --- data/hooks/diagnosis/10-ip.py | 6 ++++++ src/yunohost/diagnosis.py | 38 +++++++++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index d8ab53c56..a9485d7f4 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -1,11 +1,17 @@ #!/usr/bin/env python +import os + from moulinette import m18n from moulinette.utils.network import download_text from yunohost.diagnosis import Diagnoser class IPDiagnoser(Diagnoser): + id_ = os.path.splitext(os.path.basename(__file__))[0] + description = m18n.n("internet_connectivity") + cache_duration = 60 + def validate_args(self, args): if "version" not in args.keys(): return { "versions" : [4, 6] } diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index b5e0fa05a..f0ffcd619 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -25,10 +25,13 @@ """ import errno +import os +import time from moulinette import m18n from moulinette.core import MoulinetteError from moulinette.utils import log +from moulinette.utils.filesystem import read_json, write_to_json from yunohost.hook import hook_list, hook_exec @@ -87,10 +90,41 @@ class Diagnoser(): self.env = env self.args = self.validate_args(args) + @property + def cache_file(self): + return os.path.join(DIAGNOSIS_CACHE, "%s.json" % self.id_) + + def cached_time_ago(self): + + if not os.path.exists(self.cache_file): + return 99999999 + return time.time() - os.path.getmtime(self.cache_file) + + def get_cached_report(self): + return read_json(self.cache_file) + + def write_cache(self, report): + if not os.path.exists(DIAGNOSIS_CACHE): + os.makedirs(DIAGNOSIS_CACHE) + return write_to_json(self.cache_file, report) + def report(self): - # TODO : implement some caching mecanism in there - return list(self.run()) + print(self.cached_time_ago()) + + if self.args.get("force", False) or self.cached_time_ago() < self.cache_duration: + self.logger_debug("Using cached report from %s" % self.cache_file) + return self.get_cached_report() + + new_report = list(self.run()) + + # TODO / FIXME : should handle the case where we only did a partial diagnosis + self.logger_debug("Updating cache %s" % self.cache_file) + self.write_cache(new_report) + + return new_report + + def _list_diagnosis_categories(): hooks_raw = hook_list("diagnosis", list_by="priority", show_info=True)["hooks"] From f11206c0fa16a77aaabb441f33ab77f5c3446f2a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 30 Aug 2018 13:48:14 +0000 Subject: [PATCH 17/94] Turns out it's not really a good idea to do the internationalization right here as the strings will be kept already translated in the cache --- data/hooks/diagnosis/10-ip.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index a9485d7f4..eefa7798d 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -9,7 +9,7 @@ from yunohost.diagnosis import Diagnoser class IPDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0] - description = m18n.n("internet_connectivity") + description = "internet_connectivity" cache_duration = 60 def validate_args(self, args): @@ -26,17 +26,17 @@ class IPDiagnoser(Diagnoser): if 4 in versions: ipv4 = self.get_public_ip(4) - yield dict(meta = {"version": "4"}, + yield dict(meta = {"version": 4}, result = ipv4, - report = ("SUCCESS", m18n.n("diagnosis_network_connected_ipv4")) if ipv4 \ - else ("ERROR", m18n.n("diagnosis_network_no_ipv4"))) + report = ("SUCCESS", "diagnosis_network_connected_ipv4", {}) if ipv4 \ + else ("ERROR", "diagnosis_network_no_ipv4", {})) if 6 in versions: ipv6 = self.get_public_ip(6) - yield dict(meta = {"version": "6"}, + yield dict(meta = {"version": 6}, result = ipv6, - report = ("SUCCESS", m18n.n("diagnosis_network_connected_ipv6")) if ipv6 \ - else ("WARNING", m18n.n("diagnosis_network_no_ipv6"))) + report = ("SUCCESS", "diagnosis_network_connected_ipv6", {}) if ipv6 \ + else ("WARNING", "diagnosis_network_no_ipv6", {})) def get_public_ip(self, protocol=4): From 12df96f33e24404ea578e99e6c6dc370e7cc51b8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 30 Aug 2018 14:05:48 +0000 Subject: [PATCH 18/94] Wrap the report with meta infos --- data/hooks/diagnosis/10-ip.py | 1 - src/yunohost/diagnosis.py | 7 ++++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index eefa7798d..bde96b22e 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -38,7 +38,6 @@ class IPDiagnoser(Diagnoser): report = ("SUCCESS", "diagnosis_network_connected_ipv6", {}) if ipv6 \ else ("WARNING", "diagnosis_network_no_ipv6", {})) - def get_public_ip(self, protocol=4): if protocol == 4: diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index f0ffcd619..8fc46967b 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -110,13 +110,14 @@ class Diagnoser(): def report(self): - print(self.cached_time_ago()) - if self.args.get("force", False) or self.cached_time_ago() < self.cache_duration: self.logger_debug("Using cached report from %s" % self.cache_file) return self.get_cached_report() - new_report = list(self.run()) + new_report = { "id": self.id_, + "cached_for": self.cache_duration, + "reports": list(self.run()) + } # TODO / FIXME : should handle the case where we only did a partial diagnosis self.logger_debug("Updating cache %s" % self.cache_file) From cb6f53fc2bac2f215a6fbd0d43db08ad8d65a76d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 30 Aug 2018 14:18:44 +0000 Subject: [PATCH 19/94] Fix --force mechanism --- src/yunohost/diagnosis.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 8fc46967b..93c2dfc67 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -88,7 +88,8 @@ class Diagnoser(): self.logger_debug, self.logger_warning, self.logger_info = loggers self.env = env - self.args = self.validate_args(args) + self.args = args + self.args.update(self.validate_args(args)) @property def cache_file(self): @@ -110,10 +111,12 @@ class Diagnoser(): def report(self): - if self.args.get("force", False) or self.cached_time_ago() < self.cache_duration: + if not self.args.get("force", False) and self.cached_time_ago() < self.cache_duration: self.logger_debug("Using cached report from %s" % self.cache_file) return self.get_cached_report() + self.logger_debug("Running diagnostic for %s" % self.id_) + new_report = { "id": self.id_, "cached_for": self.cache_duration, "reports": list(self.run()) From faa4682d77cb512f21b12e901a78c2789ab28b3c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 30 Aug 2018 14:53:36 +0000 Subject: [PATCH 20/94] Forgot to keep the description --- src/yunohost/diagnosis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 93c2dfc67..09fcd8dcd 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -118,6 +118,7 @@ class Diagnoser(): self.logger_debug("Running diagnostic for %s" % self.id_) new_report = { "id": self.id_, + "description": self.description, "cached_for": self.cache_duration, "reports": list(self.run()) } From abffba960747378f1f714514ba932a9f0222a5f5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 30 Aug 2018 15:08:46 +0000 Subject: [PATCH 21/94] Remove the priority in the id of the diagnoser --- data/hooks/diagnosis/10-ip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index bde96b22e..b0a3ca1e9 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -8,7 +8,7 @@ from yunohost.diagnosis import Diagnoser class IPDiagnoser(Diagnoser): - id_ = os.path.splitext(os.path.basename(__file__))[0] + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] description = "internet_connectivity" cache_duration = 60 From 8a415579bf54d1515040fbe6f8d84f1d889effc4 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 30 Aug 2018 16:01:01 +0000 Subject: [PATCH 22/94] Implement diagnosis_show --- data/actionsmap/yunohost.yml | 4 +-- src/yunohost/diagnosis.py | 51 +++++++++++++++++++++++++++++------- 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index aa85fdf70..47a858b27 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1874,9 +1874,9 @@ diagnosis: action_help: List diagnosis categories api: GET /diagnosis/list - report: + show: action_help: Show most recents diagnosis results - api: GET /diagnosis/report + api: GET /diagnosis/show arguments: categories: help: Diagnosis categories to display (all by default) diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 09fcd8dcd..5144e9c06 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -43,8 +43,33 @@ def diagnosis_list(): all_categories_names = [ h for h, _ in _list_diagnosis_categories() ] return { "categories": all_categories_names } -def diagnosis_report(categories=[], full=False): - pass +def diagnosis_show(categories=[], full=False): + + # Get all the categories + all_categories = _list_diagnosis_categories() + all_categories_names = [ category for category, _ in all_categories ] + + # Check the requested category makes sense + if categories == []: + categories = all_categories_names + else: + unknown_categories = [ c for c in categories if c not in all_categories_names ] + if unknown_categories: + raise MoulinetteError(m18n.n('unknown_categories', categories=", ".join(categories))) + + # Fetch all reports + all_reports = [ Diagnoser.get_cached_report(c) for c in categories ] + + # "Render" the strings with m18n.n + for report in all_reports: + + report["description"] = m18n.n(report["description"]) + + for r in report["reports"]: + type_, message_key, message_args = r["report"] + r["report"] = (type_, m18n.n(message_key, **message_args)) + + return {"reports": all_reports} def diagnosis_run(categories=[], force=False, args=None): @@ -82,6 +107,7 @@ def diagnosis_ignore(category, args="", unignore=False): ############################################################ + class Diagnoser(): def __init__(self, args, env, loggers): @@ -90,10 +116,7 @@ class Diagnoser(): self.env = env self.args = args self.args.update(self.validate_args(args)) - - @property - def cache_file(self): - return os.path.join(DIAGNOSIS_CACHE, "%s.json" % self.id_) + self.cache_file = Diagnoser.cache_file(self.id_) def cached_time_ago(self): @@ -101,9 +124,6 @@ class Diagnoser(): return 99999999 return time.time() - os.path.getmtime(self.cache_file) - def get_cached_report(self): - return read_json(self.cache_file) - def write_cache(self, report): if not os.path.exists(DIAGNOSIS_CACHE): os.makedirs(DIAGNOSIS_CACHE) @@ -113,7 +133,7 @@ class Diagnoser(): if not self.args.get("force", False) and self.cached_time_ago() < self.cache_duration: self.logger_debug("Using cached report from %s" % self.cache_file) - return self.get_cached_report() + return Diagnoser.get_cached_report(self.id_) self.logger_debug("Running diagnostic for %s" % self.id_) @@ -129,6 +149,17 @@ class Diagnoser(): return new_report + @staticmethod + def cache_file(id_): + return os.path.join(DIAGNOSIS_CACHE, "%s.json" % id_) + + @staticmethod + def get_cached_report(id_): + filename = Diagnoser.cache_file(id_) + report = read_json(filename) + report["timestamp"] = int(os.path.getmtime(filename)) + return report + def _list_diagnosis_categories(): From b03e3a487e8c54d116be839f46217ab8044371fe Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 30 Aug 2018 17:49:42 +0000 Subject: [PATCH 23/94] Handle cases where some category might fail for some reason --- data/hooks/diagnosis/10-ip.py | 4 ++-- src/yunohost/diagnosis.py | 26 +++++++++++++++++--------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index b0a3ca1e9..898a6ac0f 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -55,6 +55,6 @@ class IPDiagnoser(Diagnoser): def main(args, env, loggers): - - return IPDiagnoser(args, env, loggers).report() + IPDiagnoser(args, env, loggers).diagnose() + return 0 diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 5144e9c06..805ac0b97 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -58,7 +58,12 @@ def diagnosis_show(categories=[], full=False): raise MoulinetteError(m18n.n('unknown_categories', categories=", ".join(categories))) # Fetch all reports - all_reports = [ Diagnoser.get_cached_report(c) for c in categories ] + all_reports = [] + for category in categories: + try: + all_reports.append(Diagnoser.get_cached_report(category)) + except Exception as e: + logger.error("Failed to fetch diagnosis result for category '%s' : %s" % (category, str(e))) # FIXME : i18n # "Render" the strings with m18n.n for report in all_reports: @@ -83,7 +88,7 @@ def diagnosis_run(categories=[], force=False, args=None): else: unknown_categories = [ c for c in categories if c not in all_categories_names ] if unknown_categories: - raise MoulinetteError(m18n.n('unknown_categories', categories=", ".join(categories))) + raise MoulinetteError(m18n.n('unknown_categories', categories=", ".join(unknown_categories))) # Transform "arg1=val1&arg2=val2" to { "arg1": "val1", "arg2": "val2" } if args is not None: @@ -92,15 +97,20 @@ def diagnosis_run(categories=[], force=False, args=None): args = {} args["force"] = force - # Call the hook ... + successes = [] for category in categories: logger.debug("Running diagnosis for %s ..." % category) path = [p for n, p in all_categories if n == category ][0] - # TODO : get the return value and do something with it - return {"report": hook_exec(path, args=args, env=None) } + try: + hook_exec(path, args=args, env=None) + successes.append(category) + except Exception as e: + # FIXME / TODO : add stacktrace here ? + logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e))) # FIXME : i18n + return diagnosis_show(successes) def diagnosis_ignore(category, args="", unignore=False): pass @@ -132,8 +142,8 @@ class Diagnoser(): def report(self): if not self.args.get("force", False) and self.cached_time_ago() < self.cache_duration: - self.logger_debug("Using cached report from %s" % self.cache_file) - return Diagnoser.get_cached_report(self.id_) + self.logger_debug("Cache still valid : %s" % self.cache_file) + return self.logger_debug("Running diagnostic for %s" % self.id_) @@ -147,8 +157,6 @@ class Diagnoser(): self.logger_debug("Updating cache %s" % self.cache_file) self.write_cache(new_report) - return new_report - @staticmethod def cache_file(id_): return os.path.join(DIAGNOSIS_CACHE, "%s.json" % id_) From 77b0920dac4d15b6ba70405f63ba023f229be0ae Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 31 Aug 2018 01:35:12 +0000 Subject: [PATCH 24/94] Forgot to change this --- src/yunohost/diagnosis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 805ac0b97..48e9977c1 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -139,7 +139,7 @@ class Diagnoser(): os.makedirs(DIAGNOSIS_CACHE) return write_to_json(self.cache_file, report) - def report(self): + def diagnose(self): if not self.args.get("force", False) and self.cached_time_ago() < self.cache_duration: self.logger_debug("Cache still valid : %s" % self.cache_file) From 85930163a09653fec43336f965d5fa9a2bc20497 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 31 Aug 2018 16:36:10 +0000 Subject: [PATCH 25/94] First draft of DNS diagnoser --- data/hooks/diagnosis/12-dns.py | 93 ++++++++++++++++++++++++++++++++++ locales/en.json | 4 +- 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 data/hooks/diagnosis/12-dns.py diff --git a/data/hooks/diagnosis/12-dns.py b/data/hooks/diagnosis/12-dns.py new file mode 100644 index 000000000..b4cedebad --- /dev/null +++ b/data/hooks/diagnosis/12-dns.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python + +import os + +from moulinette import m18n +from moulinette.utils.network import download_text +from moulinette.core import MoulinetteError, init_authenticator +from moulinette.utils.process import check_output + +from yunohost.diagnosis import Diagnoser +from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain + +# Instantiate LDAP Authenticator +auth_identifier = ('ldap', 'ldap-anonymous') +auth_parameters = {'uri': 'ldap://localhost:389', 'base_dn': 'dc=yunohost,dc=org'} +auth = init_authenticator(auth_identifier, auth_parameters) + +class DNSDiagnoser(Diagnoser): + + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] + description = "dns_configurations" + cache_duration = 3600*24 + + def validate_args(self, args): + all_domains = domain_list(auth)["domains"] + if "domain" not in args.keys(): + return { "domains" : all_domains } + else: + if args["domain"] not in all_domains: + raise MoulinetteError(errno.EINVAL, m18n.n('domain_unknown')) + return { "domains" : [ args["domain"] ] } + + def run(self): + + self.resolver = check_output('grep "$nameserver" /etc/resolv.dnsmasq.conf').split("\n")[0].split(" ")[1] + + main_domain = _get_maindomain() + + for domain in self.args["domains"]: + self.logger_info("Diagnosing DNS conf for %s" % domain) + for report in self.check_domain(domain, domain==main_domain): + yield report + + def check_domain(self, domain, is_main_domain): + + expected_configuration = _build_dns_conf(domain) + + # Here if there are no AAAA record, we should add something to expect "no" AAAA record + # to properly diagnose situations where people have a AAAA record but no IPv6 + + for category, records in expected_configuration.items(): + + discrepancies = [] + + for r in records: + current_value = self.get_current_record(domain, r["name"], r["type"]) or "None" + expected_value = r["value"] if r["value"] != "@" else domain+"." + + if current_value != expected_value: + discrepancies.append((r, expected_value, current_value)) + + if discrepancies: + if category == "basic" or is_main_domain: + level = "ERROR" + else: + level = "WARNING" + report = (level, "diagnosis_dns_bad_conf", {"domain": domain, "category": category}) + else: + level = "SUCCESS" + report = ("SUCCESS", "diagnosis_dns_good_conf", {"domain": domain, "category": category}) + + # FIXME : add management of details of what's wrong if there are discrepancies + yield dict(meta = {"domain": domain, "category": category}, + result = level, report = report ) + + + + def get_current_record(self, domain, name, type_): + if name == "@": + command = "dig +short @%s %s %s" % (self.resolver, type_, domain) + else: + command = "dig +short @%s %s %s.%s" % (self.resolver, type_, name, domain) + output = check_output(command).strip() + output = output.replace("\;",";") + if output.startswith('"') and output.endswith('"'): + output = '"' + ' '.join(output.replace('"',' ').split()) + '"' + return output + + +def main(args, env, loggers): + DNSDiagnoser(args, env, loggers).diagnose() + return 0 + diff --git a/locales/en.json b/locales/en.json index a91da4fe9..77701478c 100644 --- a/locales/en.json +++ b/locales/en.json @@ -155,7 +155,9 @@ "diagnosis_no_apps": "No installed application", "dpkg_is_broken": "You cannot do this right now because dpkg/APT (the system package managers) seems to be in a broken state… You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.", "dpkg_lock_not_available": "This command can't be ran right now because another program seems to be using the lock of dpkg (the system package manager)", - "domain_cannot_remove_main": "Cannot remove main domain. Set one first", + "diagnosis_dns_good_conf": "Good DNS configuration for {domain} : {category}.", + "diagnosis_dns_bad_conf": "Bad DNS configuration for {domain} : {category}.", + "domain_cannot_remove_main": "Cannot remove main domain. Set a new main domain first", "domain_cert_gen_failed": "Could not generate certificate", "domain_created": "Domain created", "domain_creation_failed": "Could not create domain {domain}: {error}", From ded4895b7e590ab60b5349f6a22e983782191adb Mon Sep 17 00:00:00 2001 From: Bram Date: Sat, 1 Sep 2018 02:50:22 +0200 Subject: [PATCH 26/94] [mod] misc, better error message I'm using repr to be able to detect if it's a string or a number since it's an error I'm expecting --- data/hooks/diagnosis/10-ip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index 898a6ac0f..574741da9 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -45,7 +45,7 @@ class IPDiagnoser(Diagnoser): elif protocol == 6: url = 'https://ip6.yunohost.org' else: - raise ValueError("invalid protocol version") + raise ValueError("invalid protocol version, it should be either 4 or 6 and was '%s'" % repr(protocol)) try: return download_text(url, timeout=30).strip() From 2b2ff08f08e8e62761e3a126090ebebd3bce7877 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 2 May 2019 17:59:35 +0200 Subject: [PATCH 27/94] Fix error handling (Yunohost / Moulinette / Asserts) --- data/hooks/diagnosis/10-ip.py | 6 +++--- data/hooks/diagnosis/12-dns.py | 6 ++---- src/yunohost/diagnosis.py | 9 ++++----- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index 574741da9..d229eea8f 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -4,6 +4,7 @@ import os from moulinette import m18n from moulinette.utils.network import download_text + from yunohost.diagnosis import Diagnoser class IPDiagnoser(Diagnoser): @@ -16,8 +17,7 @@ class IPDiagnoser(Diagnoser): if "version" not in args.keys(): return { "versions" : [4, 6] } else: - if str(args["version"]) not in ["4", "6"]: - raise MoulinetteError(1, "Invalid version, should be 4 or 6.") + assert str(args["version"]) in ["4", "6"], "Invalid version, should be 4 or 6." return { "versions" : [int(args["version"])] } def run(self): @@ -30,7 +30,7 @@ class IPDiagnoser(Diagnoser): result = ipv4, report = ("SUCCESS", "diagnosis_network_connected_ipv4", {}) if ipv4 \ else ("ERROR", "diagnosis_network_no_ipv4", {})) - + if 6 in versions: ipv6 = self.get_public_ip(6) yield dict(meta = {"version": 6}, diff --git a/data/hooks/diagnosis/12-dns.py b/data/hooks/diagnosis/12-dns.py index b4cedebad..9bf6a13a3 100644 --- a/data/hooks/diagnosis/12-dns.py +++ b/data/hooks/diagnosis/12-dns.py @@ -2,9 +2,8 @@ import os -from moulinette import m18n from moulinette.utils.network import download_text -from moulinette.core import MoulinetteError, init_authenticator +from moulinette.core import init_authenticator from moulinette.utils.process import check_output from yunohost.diagnosis import Diagnoser @@ -26,8 +25,7 @@ class DNSDiagnoser(Diagnoser): if "domain" not in args.keys(): return { "domains" : all_domains } else: - if args["domain"] not in all_domains: - raise MoulinetteError(errno.EINVAL, m18n.n('domain_unknown')) + assert args["domain"] in all_domains, "Unknown domain" return { "domains" : [ args["domain"] ] } def run(self): diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 48e9977c1..22770ce87 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -24,15 +24,14 @@ Look for possible issues on the server """ -import errno import os import time from moulinette import m18n -from moulinette.core import MoulinetteError from moulinette.utils import log from moulinette.utils.filesystem import read_json, write_to_json +from yunohost.utils.error import YunohostError from yunohost.hook import hook_list, hook_exec logger = log.getActionLogger('yunohost.diagnosis') @@ -55,7 +54,7 @@ def diagnosis_show(categories=[], full=False): else: unknown_categories = [ c for c in categories if c not in all_categories_names ] if unknown_categories: - raise MoulinetteError(m18n.n('unknown_categories', categories=", ".join(categories))) + raise YunohostError('unknown_categories', categories=", ".join(categories)) # Fetch all reports all_reports = [] @@ -88,7 +87,7 @@ def diagnosis_run(categories=[], force=False, args=None): else: unknown_categories = [ c for c in categories if c not in all_categories_names ] if unknown_categories: - raise MoulinetteError(m18n.n('unknown_categories', categories=", ".join(unknown_categories))) + raise YunohostError('unknown_categories', categories=", ".join(unknown_categories)) # Transform "arg1=val1&arg2=val2" to { "arg1": "val1", "arg2": "val2" } if args is not None: @@ -108,7 +107,7 @@ def diagnosis_run(categories=[], force=False, args=None): successes.append(category) except Exception as e: # FIXME / TODO : add stacktrace here ? - logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e))) # FIXME : i18n + logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e))) # FIXME : i18n return diagnosis_show(successes) From 3200fef39c3fb5966031d62153b02baa82f38dfb Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 21 May 2019 20:08:09 +0200 Subject: [PATCH 28/94] Implement detail mechanism for DNS category --- data/hooks/diagnosis/12-dns.py | 17 ++++++++++++----- src/yunohost/diagnosis.py | 3 +++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/data/hooks/diagnosis/12-dns.py b/data/hooks/diagnosis/12-dns.py index 9bf6a13a3..e6370ba05 100644 --- a/data/hooks/diagnosis/12-dns.py +++ b/data/hooks/diagnosis/12-dns.py @@ -54,8 +54,10 @@ class DNSDiagnoser(Diagnoser): current_value = self.get_current_record(domain, r["name"], r["type"]) or "None" expected_value = r["value"] if r["value"] != "@" else domain+"." - if current_value != expected_value: - discrepancies.append((r, expected_value, current_value)) + if current_value == "None": + discrepancies.append(("diagnosis_dns_missing_record", (r["type"], r["name"], expected_value))) + elif current_value != expected_value: + discrepancies.append(("diagnosis_dns_discrepancy", (r["type"], r["name"], expected_value, current_value))) if discrepancies: if category == "basic" or is_main_domain: @@ -66,11 +68,16 @@ class DNSDiagnoser(Diagnoser): else: level = "SUCCESS" report = ("SUCCESS", "diagnosis_dns_good_conf", {"domain": domain, "category": category}) + details = None - # FIXME : add management of details of what's wrong if there are discrepancies - yield dict(meta = {"domain": domain, "category": category}, - result = level, report = report ) + output = dict(meta = {"domain": domain, "category": category}, + result = level, + report = report ) + if discrepancies: + output["details"] = discrepancies + + yield output def get_current_record(self, domain, name, type_): diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 22770ce87..a8fae4124 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -73,6 +73,9 @@ def diagnosis_show(categories=[], full=False): type_, message_key, message_args = r["report"] r["report"] = (type_, m18n.n(message_key, **message_args)) + if "details" in r: + r["details"] = [ m18n.n(key, *values) for key, values in r["details"] ] + return {"reports": all_reports} def diagnosis_run(categories=[], force=False, args=None): From aafef0a8efad86a1c0b223bc7111133d388ce971 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 21 May 2019 20:08:31 +0200 Subject: [PATCH 29/94] Add i18n messages --- locales/en.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/locales/en.json b/locales/en.json index 77701478c..0245bf49d 100644 --- a/locales/en.json +++ b/locales/en.json @@ -155,8 +155,15 @@ "diagnosis_no_apps": "No installed application", "dpkg_is_broken": "You cannot do this right now because dpkg/APT (the system package managers) seems to be in a broken state… You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.", "dpkg_lock_not_available": "This command can't be ran right now because another program seems to be using the lock of dpkg (the system package manager)", + "diagnosis_network_connected_ipv4": "The server is connected to the Internet through IPv4 !", + "diagnosis_network_no_ipv4": "The server does not have a working IPv4.", + "diagnosis_network_connected_ipv6": "The server is connect to the Internet through IPv6 !", + "diagnosis_network_no_ipv6": "The server does not have a working IPv6.", "diagnosis_dns_good_conf": "Good DNS configuration for {domain} : {category}.", "diagnosis_dns_bad_conf": "Bad DNS configuration for {domain} : {category}.", + "diagnosis_dns_missing_record": "According to the recommended DNS configuration, you should add a DNS record with type {0}, name {1} and value {2}", + "diagnosis_dns_discrepancy": "According to the recommended DNS configuration, the value for the DNS record with type {0} and name {1} should be {2}, not {3}.", + "dns_configurations": "Domain name configuration (DNS)", "domain_cannot_remove_main": "Cannot remove main domain. Set a new main domain first", "domain_cert_gen_failed": "Could not generate certificate", "domain_created": "Domain created", @@ -238,6 +245,7 @@ "hook_name_unknown": "Unknown hook name '{name:s}'", "installation_complete": "Installation complete", "installation_failed": "Something went wrong with the installation", + "internet_connectivity": "Internet connectivity", "ip6tables_unavailable": "You cannot play with ip6tables here. You are either in a container or your kernel does not support it", "iptables_unavailable": "You cannot play with iptables here. You are either in a container or your kernel does not support it", "log_corrupted_md_file": "The YAML metadata file associated with logs is damaged: '{md_file}\nError: {error}'", From 06e02de548f4735ab9a9944ac784918700df9bb5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 13 Jul 2019 18:19:16 +0200 Subject: [PATCH 30/94] Add traceback for easier debugging --- src/yunohost/diagnosis.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index a8fae4124..99767e1b8 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -109,8 +109,7 @@ def diagnosis_run(categories=[], force=False, args=None): hook_exec(path, args=args, env=None) successes.append(category) except Exception as e: - # FIXME / TODO : add stacktrace here ? - logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e))) # FIXME : i18n + logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e)), exc_info=True) # FIXME : i18n return diagnosis_show(successes) From 1105b7d943d20dbe2939e5b4310803e9da744ec2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 13 Jul 2019 18:19:50 +0200 Subject: [PATCH 31/94] We don't need this auth madness anymore --- data/hooks/diagnosis/12-dns.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/data/hooks/diagnosis/12-dns.py b/data/hooks/diagnosis/12-dns.py index e6370ba05..3a61b0503 100644 --- a/data/hooks/diagnosis/12-dns.py +++ b/data/hooks/diagnosis/12-dns.py @@ -3,17 +3,11 @@ import os from moulinette.utils.network import download_text -from moulinette.core import init_authenticator from moulinette.utils.process import check_output from yunohost.diagnosis import Diagnoser from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain -# Instantiate LDAP Authenticator -auth_identifier = ('ldap', 'ldap-anonymous') -auth_parameters = {'uri': 'ldap://localhost:389', 'base_dn': 'dc=yunohost,dc=org'} -auth = init_authenticator(auth_identifier, auth_parameters) - class DNSDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] @@ -21,7 +15,7 @@ class DNSDiagnoser(Diagnoser): cache_duration = 3600*24 def validate_args(self, args): - all_domains = domain_list(auth)["domains"] + all_domains = domain_list()["domains"] if "domain" not in args.keys(): return { "domains" : all_domains } else: From bd3a378d285edd907858963d671ebd63cf76c210 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 13 Jul 2019 18:22:29 +0200 Subject: [PATCH 32/94] Use only ipv4 resolver for DNS records diagnosis --- data/hooks/diagnosis/12-dns.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/data/hooks/diagnosis/12-dns.py b/data/hooks/diagnosis/12-dns.py index 3a61b0503..90f52c82d 100644 --- a/data/hooks/diagnosis/12-dns.py +++ b/data/hooks/diagnosis/12-dns.py @@ -4,6 +4,7 @@ import os from moulinette.utils.network import download_text from moulinette.utils.process import check_output +from moulinette.utils.filesystem import read_file from yunohost.diagnosis import Diagnoser from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain @@ -24,8 +25,12 @@ class DNSDiagnoser(Diagnoser): def run(self): - self.resolver = check_output('grep "$nameserver" /etc/resolv.dnsmasq.conf').split("\n")[0].split(" ")[1] + resolvers = read_file("/etc/resolv.dnsmasq.conf").split("\n") + ipv4_resolvers = [r.split(" ")[1] for r in resolvers if r.startswith("nameserver") and ":" not in r] + # FIXME some day ... handle ipv4-only and ipv6-only servers. For now we assume we have at least ipv4 + assert ipv4_resolvers != [], "Uhoh, need at least one IPv4 DNS resolver ..." + self.resolver = ipv4_resolvers[0] main_domain = _get_maindomain() for domain in self.args["domains"]: From 0ce4eb0a27a8c6e128835bc9787d14d7ac294f04 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 13 Jul 2019 19:23:49 +0200 Subject: [PATCH 33/94] Fix the return interface of diagnosis hooks --- data/hooks/diagnosis/10-ip.py | 3 +-- data/hooks/diagnosis/12-dns.py | 3 +-- src/yunohost/diagnosis.py | 13 ++++++++----- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index d229eea8f..19e4806f6 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -55,6 +55,5 @@ class IPDiagnoser(Diagnoser): def main(args, env, loggers): - IPDiagnoser(args, env, loggers).diagnose() - return 0 + return IPDiagnoser(args, env, loggers).diagnose() diff --git a/data/hooks/diagnosis/12-dns.py b/data/hooks/diagnosis/12-dns.py index 90f52c82d..09f8cd4bf 100644 --- a/data/hooks/diagnosis/12-dns.py +++ b/data/hooks/diagnosis/12-dns.py @@ -92,6 +92,5 @@ class DNSDiagnoser(Diagnoser): def main(args, env, loggers): - DNSDiagnoser(args, env, loggers).diagnose() - return 0 + return DNSDiagnoser(args, env, loggers).diagnose() diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 99767e1b8..38c59793f 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -100,18 +100,19 @@ def diagnosis_run(categories=[], force=False, args=None): args["force"] = force # Call the hook ... - successes = [] + diagnosed_categories = [] for category in categories: logger.debug("Running diagnosis for %s ..." % category) path = [p for n, p in all_categories if n == category ][0] try: hook_exec(path, args=args, env=None) - successes.append(category) except Exception as e: logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e)), exc_info=True) # FIXME : i18n + else: + diagnosed_categories.append(category) - return diagnosis_show(successes) + return diagnosis_show(diagnosed_categories) def diagnosis_ignore(category, args="", unignore=False): pass @@ -125,8 +126,8 @@ class Diagnoser(): self.logger_debug, self.logger_warning, self.logger_info = loggers self.env = env - self.args = args - self.args.update(self.validate_args(args)) + self.args = args or {} + self.args.update(self.validate_args(self.args)) self.cache_file = Diagnoser.cache_file(self.id_) def cached_time_ago(self): @@ -158,6 +159,8 @@ class Diagnoser(): self.logger_debug("Updating cache %s" % self.cache_file) self.write_cache(new_report) + return 0, new_report + @staticmethod def cache_file(id_): return os.path.join(DIAGNOSIS_CACHE, "%s.json" % id_) From af23f53d8295affdc54cd3c7fc435e2d1c6bb205 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 13 Jul 2019 19:52:04 +0200 Subject: [PATCH 34/94] Simplify / reorganize i18n management for report and description --- data/hooks/diagnosis/10-ip.py | 1 - .../diagnosis/{12-dns.py => 12-dnsrecords.py} | 7 ++- locales/en.json | 4 +- src/yunohost/diagnosis.py | 44 +++++++++++++------ 4 files changed, 36 insertions(+), 20 deletions(-) rename data/hooks/diagnosis/{12-dns.py => 12-dnsrecords.py} (94%) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index 19e4806f6..665c0ff0d 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -10,7 +10,6 @@ from yunohost.diagnosis import Diagnoser class IPDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] - description = "internet_connectivity" cache_duration = 60 def validate_args(self, args): diff --git a/data/hooks/diagnosis/12-dns.py b/data/hooks/diagnosis/12-dnsrecords.py similarity index 94% rename from data/hooks/diagnosis/12-dns.py rename to data/hooks/diagnosis/12-dnsrecords.py index 09f8cd4bf..5edfc2d41 100644 --- a/data/hooks/diagnosis/12-dns.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -9,10 +9,9 @@ from moulinette.utils.filesystem import read_file from yunohost.diagnosis import Diagnoser from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain -class DNSDiagnoser(Diagnoser): +class DNSRecordsDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] - description = "dns_configurations" cache_duration = 3600*24 def validate_args(self, args): @@ -34,7 +33,7 @@ class DNSDiagnoser(Diagnoser): main_domain = _get_maindomain() for domain in self.args["domains"]: - self.logger_info("Diagnosing DNS conf for %s" % domain) + self.logger_debug("Diagnosing DNS conf for %s" % domain) for report in self.check_domain(domain, domain==main_domain): yield report @@ -92,5 +91,5 @@ class DNSDiagnoser(Diagnoser): def main(args, env, loggers): - return DNSDiagnoser(args, env, loggers).diagnose() + return DNSRecordsDiagnoser(args, env, loggers).diagnose() diff --git a/locales/en.json b/locales/en.json index 0245bf49d..0bb6d7275 100644 --- a/locales/en.json +++ b/locales/en.json @@ -163,7 +163,8 @@ "diagnosis_dns_bad_conf": "Bad DNS configuration for {domain} : {category}.", "diagnosis_dns_missing_record": "According to the recommended DNS configuration, you should add a DNS record with type {0}, name {1} and value {2}", "diagnosis_dns_discrepancy": "According to the recommended DNS configuration, the value for the DNS record with type {0} and name {1} should be {2}, not {3}.", - "dns_configurations": "Domain name configuration (DNS)", + "diagnosis_description_ip": "Internet connectivity", + "diagnosis_description_dnsrecords": "DNS records", "domain_cannot_remove_main": "Cannot remove main domain. Set a new main domain first", "domain_cert_gen_failed": "Could not generate certificate", "domain_created": "Domain created", @@ -245,7 +246,6 @@ "hook_name_unknown": "Unknown hook name '{name:s}'", "installation_complete": "Installation complete", "installation_failed": "Something went wrong with the installation", - "internet_connectivity": "Internet connectivity", "ip6tables_unavailable": "You cannot play with ip6tables here. You are either in a container or your kernel does not support it", "iptables_unavailable": "You cannot play with iptables here. You are either in a container or your kernel does not support it", "log_corrupted_md_file": "The YAML metadata file associated with logs is damaged: '{md_file}\nError: {error}'", diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 38c59793f..fb5220679 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -64,18 +64,6 @@ def diagnosis_show(categories=[], full=False): except Exception as e: logger.error("Failed to fetch diagnosis result for category '%s' : %s" % (category, str(e))) # FIXME : i18n - # "Render" the strings with m18n.n - for report in all_reports: - - report["description"] = m18n.n(report["description"]) - - for r in report["reports"]: - type_, message_key, message_args = r["report"] - r["report"] = (type_, m18n.n(message_key, **message_args)) - - if "details" in r: - r["details"] = [ m18n.n(key, *values) for key, values in r["details"] ] - return {"reports": all_reports} def diagnosis_run(categories=[], force=False, args=None): @@ -130,6 +118,13 @@ class Diagnoser(): self.args.update(self.validate_args(self.args)) self.cache_file = Diagnoser.cache_file(self.id_) + descr_key = "diagnosis_description_" + self.id_ + self.description = m18n.n(descr_key) + # If no description available, fallback to id + if self.description == descr_key: + self.description = report["id"] + + def cached_time_ago(self): if not os.path.exists(self.cache_file): @@ -145,12 +140,12 @@ class Diagnoser(): if not self.args.get("force", False) and self.cached_time_ago() < self.cache_duration: self.logger_debug("Cache still valid : %s" % self.cache_file) + # FIXME uhoh that's not consistent with the other return later return self.logger_debug("Running diagnostic for %s" % self.id_) new_report = { "id": self.id_, - "description": self.description, "cached_for": self.cache_duration, "reports": list(self.run()) } @@ -158,6 +153,7 @@ class Diagnoser(): # TODO / FIXME : should handle the case where we only did a partial diagnosis self.logger_debug("Updating cache %s" % self.cache_file) self.write_cache(new_report) + Diagnoser.i18n(new_report) return 0, new_report @@ -170,8 +166,30 @@ class Diagnoser(): filename = Diagnoser.cache_file(id_) report = read_json(filename) report["timestamp"] = int(os.path.getmtime(filename)) + Diagnoser.i18n(report) return report + @staticmethod + def i18n(report): + + # "Render" the strings with m18n.n + # N.B. : we do those m18n.n right now instead of saving the already-translated report + # because we can't be sure we'll redisplay the infos with the same locale as it + # was generated ... e.g. if the diagnosing happened inside a cron job with locale EN + # instead of FR used by the actual admin... + + descr_key = "diagnosis_description_" + report["id"] + report["description"] = m18n.n(descr_key) + # If no description available, fallback to id + if report["description"] == descr_key: + report["description"] = report["id"] + + for r in report["reports"]: + type_, message_key, message_args = r["report"] + r["report"] = (type_, m18n.n(message_key, **message_args)) + + if "details" in r: + r["details"] = [ m18n.n(key, *values) for key, values in r["details"] ] def _list_diagnosis_categories(): From 9405362caff317f5d90423c21803b0a00e94d66a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 13 Jul 2019 21:00:00 +0200 Subject: [PATCH 35/94] Cooler messages summarizing what's found, instead of displaying a huge unreadable wall of json/yaml --- src/yunohost/diagnosis.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index fb5220679..7297e6d4b 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -27,7 +27,7 @@ import os import time -from moulinette import m18n +from moulinette import m18n, msettings from moulinette.utils import log from moulinette.utils.filesystem import read_json, write_to_json @@ -87,6 +87,7 @@ def diagnosis_run(categories=[], force=False, args=None): args = {} args["force"] = force + found_issues = False # Call the hook ... diagnosed_categories = [] for category in categories: @@ -94,13 +95,23 @@ def diagnosis_run(categories=[], force=False, args=None): path = [p for n, p in all_categories if n == category ][0] try: - hook_exec(path, args=args, env=None) + code, report = hook_exec(path, args=args, env=None) except Exception as e: logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e)), exc_info=True) # FIXME : i18n else: diagnosed_categories.append(category) + if report != {}: + issues = [r for r in report["reports"] if r["report"][0] in ["ERROR", "WARNING"]] + if issues: + found_issues = True - return diagnosis_show(diagnosed_categories) + if found_issues: + if msettings.get("interface") == "api": + logger.info("You can go to the Diagnosis section (in the home screen) to see the issues found.") + else: + logger.info("You can run 'yunohost diagnosis show --issues' to display the issues found.") + + return def diagnosis_ignore(category, args="", unignore=False): pass @@ -140,8 +151,8 @@ class Diagnoser(): if not self.args.get("force", False) and self.cached_time_ago() < self.cache_duration: self.logger_debug("Cache still valid : %s" % self.cache_file) - # FIXME uhoh that's not consistent with the other return later - return + logger.info("(Cache still valid for %s diagnosis. Not re-diagnosing yet!)" % self.description) + return 0, {} self.logger_debug("Running diagnostic for %s" % self.id_) @@ -155,6 +166,17 @@ class Diagnoser(): self.write_cache(new_report) Diagnoser.i18n(new_report) + errors = [r for r in new_report["reports"] if r["report"][0] == "ERROR"] + warnings = [r for r in new_report["reports"] if r["report"][0] == "WARNING"] + + # FIXME : i18n + if errors: + logger.error("Found %s significant issue(s) related to %s!" % (len(errors), new_report["description"])) + elif warnings: + logger.warning("Found %s item(s) that could be improved for %s." % (len(warnings), new_report["description"])) + else: + logger.success("Everything looks good for %s!" % new_report["description"]) + return 0, new_report @staticmethod From 1d8ba7fa95305cf440d3a3888813bde13d5cc564 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 13 Jul 2019 21:38:35 +0200 Subject: [PATCH 36/94] Implement diagnosis show --full and --issues --- data/actionsmap/yunohost.yml | 3 +++ src/yunohost/diagnosis.py | 18 ++++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 47a858b27..6b89a819b 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1884,6 +1884,9 @@ diagnosis: --full: help: Display additional information action: store_true + --issues: + help: Only display issues + action: store_true run: action_help: Show most recents diagnosis results diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 7297e6d4b..de73bd680 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -42,7 +42,7 @@ def diagnosis_list(): all_categories_names = [ h for h, _ in _list_diagnosis_categories() ] return { "categories": all_categories_names } -def diagnosis_show(categories=[], full=False): +def diagnosis_show(categories=[], issues=False, full=False): # Get all the categories all_categories = _list_diagnosis_categories() @@ -60,9 +60,23 @@ def diagnosis_show(categories=[], full=False): all_reports = [] for category in categories: try: - all_reports.append(Diagnoser.get_cached_report(category)) + cat_report = Diagnoser.get_cached_report(category) except Exception as e: logger.error("Failed to fetch diagnosis result for category '%s' : %s" % (category, str(e))) # FIXME : i18n + else: + if not full: + del cat_report["timestamp"] + del cat_report["cached_for"] + for report in cat_report["reports"]: + del report["meta"] + del report["result"] + if issues: + cat_report["reports"] = [ r for r in cat_report["reports"] if r["report"][0] != "SUCCESS" ] + if not cat_report["reports"]: + continue + + all_reports.append(cat_report) + return {"reports": all_reports} From 41c3b054baf7640d5f97164643e9fe779885c843 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 13 Jul 2019 22:07:07 +0200 Subject: [PATCH 37/94] Fix semantic, way too many things called 'report' ... --- data/hooks/diagnosis/10-ip.py | 14 ++++---- data/hooks/diagnosis/12-dnsrecords.py | 16 ++++----- src/yunohost/diagnosis.py | 48 ++++++++++++++------------- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index 665c0ff0d..f38d1fadf 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -26,16 +26,18 @@ class IPDiagnoser(Diagnoser): if 4 in versions: ipv4 = self.get_public_ip(4) yield dict(meta = {"version": 4}, - result = ipv4, - report = ("SUCCESS", "diagnosis_network_connected_ipv4", {}) if ipv4 \ - else ("ERROR", "diagnosis_network_no_ipv4", {})) + data = ipv4, + status = "SUCCESS" if ipv4 else "ERROR", + summary = ("diagnosis_network_connected_ipv4", {}) if ipv4 \ + else ("diagnosis_network_no_ipv4", {})) if 6 in versions: ipv6 = self.get_public_ip(6) yield dict(meta = {"version": 6}, - result = ipv6, - report = ("SUCCESS", "diagnosis_network_connected_ipv6", {}) if ipv6 \ - else ("WARNING", "diagnosis_network_no_ipv6", {})) + data = ipv6, + status = "SUCCESS" if ipv6 else "WARNING", + summary = ("diagnosis_network_connected_ipv6", {}) if ipv6 \ + else ("diagnosis_network_no_ipv6", {})) def get_public_ip(self, protocol=4): diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 5edfc2d41..3ba64445d 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -58,19 +58,15 @@ class DNSRecordsDiagnoser(Diagnoser): discrepancies.append(("diagnosis_dns_discrepancy", (r["type"], r["name"], expected_value, current_value))) if discrepancies: - if category == "basic" or is_main_domain: - level = "ERROR" - else: - level = "WARNING" - report = (level, "diagnosis_dns_bad_conf", {"domain": domain, "category": category}) + status = "ERROR" if (category == "basic" or is_main_domain) else "WARNING" + summary = ("diagnosis_dns_bad_conf", {"domain": domain, "category": category}) else: - level = "SUCCESS" - report = ("SUCCESS", "diagnosis_dns_good_conf", {"domain": domain, "category": category}) - details = None + status = "SUCCESS" + summary = ("diagnosis_dns_good_conf", {"domain": domain, "category": category}) output = dict(meta = {"domain": domain, "category": category}, - result = level, - report = report ) + status = status, + summary = summary) if discrepancies: output["details"] = discrepancies diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index de73bd680..523a5c891 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -60,22 +60,24 @@ def diagnosis_show(categories=[], issues=False, full=False): all_reports = [] for category in categories: try: - cat_report = Diagnoser.get_cached_report(category) + report = Diagnoser.get_cached_report(category) except Exception as e: logger.error("Failed to fetch diagnosis result for category '%s' : %s" % (category, str(e))) # FIXME : i18n else: if not full: - del cat_report["timestamp"] - del cat_report["cached_for"] - for report in cat_report["reports"]: - del report["meta"] - del report["result"] + del report["timestamp"] + del report["cached_for"] + for item in report["items"]: + del item["meta"] + if "data" in item: + del item["data"] if issues: - cat_report["reports"] = [ r for r in cat_report["reports"] if r["report"][0] != "SUCCESS" ] - if not cat_report["reports"]: + report["items"] = [ item for item in report["items"] if item["status"] != "SUCCESS" ] + # Ignore this category if no issue was found + if not report["items"]: continue - all_reports.append(cat_report) + all_reports.append(report) return {"reports": all_reports} @@ -101,7 +103,7 @@ def diagnosis_run(categories=[], force=False, args=None): args = {} args["force"] = force - found_issues = False + issues = [] # Call the hook ... diagnosed_categories = [] for category in categories: @@ -115,11 +117,9 @@ def diagnosis_run(categories=[], force=False, args=None): else: diagnosed_categories.append(category) if report != {}: - issues = [r for r in report["reports"] if r["report"][0] in ["ERROR", "WARNING"]] - if issues: - found_issues = True + issues.extend([item for item in report["items"] if item["status"] != "SUCCESS"]) - if found_issues: + if issues: if msettings.get("interface") == "api": logger.info("You can go to the Diagnosis section (in the home screen) to see the issues found.") else: @@ -147,7 +147,7 @@ class Diagnoser(): self.description = m18n.n(descr_key) # If no description available, fallback to id if self.description == descr_key: - self.description = report["id"] + self.description = self.id_ def cached_time_ago(self): @@ -170,9 +170,11 @@ class Diagnoser(): self.logger_debug("Running diagnostic for %s" % self.id_) + items = list(self.run()) + new_report = { "id": self.id_, "cached_for": self.cache_duration, - "reports": list(self.run()) + "items": items } # TODO / FIXME : should handle the case where we only did a partial diagnosis @@ -180,8 +182,8 @@ class Diagnoser(): self.write_cache(new_report) Diagnoser.i18n(new_report) - errors = [r for r in new_report["reports"] if r["report"][0] == "ERROR"] - warnings = [r for r in new_report["reports"] if r["report"][0] == "WARNING"] + errors = [item for item in new_report["items"] if item["status"] == "ERROR"] + warnings = [item for item in new_report["items"] if item["status"] == "WARNING"] # FIXME : i18n if errors: @@ -220,12 +222,12 @@ class Diagnoser(): if report["description"] == descr_key: report["description"] = report["id"] - for r in report["reports"]: - type_, message_key, message_args = r["report"] - r["report"] = (type_, m18n.n(message_key, **message_args)) + for item in report["items"]: + summary_key, summary_args = item["summary"] + item["summary"] = m18n.n(summary_key, **summary_args) - if "details" in r: - r["details"] = [ m18n.n(key, *values) for key, values in r["details"] ] + if "details" in item: + item["details"] = [ m18n.n(key, *values) for key, values in item["details"] ] def _list_diagnosis_categories(): From 4d5ace06dbc3466013838c61c2f4efc8c15bfa69 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 13 Jul 2019 23:11:28 +0200 Subject: [PATCH 38/94] Add test that we can ping outside before talking to ip.yunohost.org --- data/hooks/diagnosis/10-ip.py | 54 ++++++++++++++++++++++++++- data/hooks/diagnosis/12-dnsrecords.py | 3 ++ locales/en.json | 4 +- 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index f38d1fadf..3259c6a4a 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -1,9 +1,12 @@ #!/usr/bin/env python import os +import random from moulinette import m18n from moulinette.utils.network import download_text +from moulinette.utils.process import check_output +from moulinette.utils.filesystem import read_file from yunohost.diagnosis import Diagnoser @@ -24,7 +27,12 @@ class IPDiagnoser(Diagnoser): versions = self.args["versions"] if 4 in versions: - ipv4 = self.get_public_ip(4) + + if not self.can_ping_outside(4): + ipv4 = None + else: + ipv4 = self.get_public_ip(4) + yield dict(meta = {"version": 4}, data = ipv4, status = "SUCCESS" if ipv4 else "ERROR", @@ -32,15 +40,57 @@ class IPDiagnoser(Diagnoser): else ("diagnosis_network_no_ipv4", {})) if 6 in versions: - ipv6 = self.get_public_ip(6) + + if not self.can_ping_outside(4): + ipv6 = None + else: + ipv6 = self.get_public_ip(6) + yield dict(meta = {"version": 6}, data = ipv6, status = "SUCCESS" if ipv6 else "WARNING", summary = ("diagnosis_network_connected_ipv6", {}) if ipv6 \ else ("diagnosis_network_no_ipv6", {})) + + def can_ping_outside(self, protocol=4): + + assert protocol in [4, 6], "Invalid protocol version, it should be either 4 or 6 and was '%s'" % repr(protocol) + + # We can know that ipv6 is not available directly if this file does not exists + if protocol == 6 and not os.path.exists("/proc/net/if_inet6"): + return False + + # If we are indeed connected in ipv4 or ipv6, we should find a default route + routes = check_output("ip -%s route" % protocol).split("\n") + if not [r for r in routes if r.startswith("default")]: + return False + + # We use the resolver file as a list of well-known, trustable (ie not google ;)) IPs that we can ping + resolver_file = "/usr/share/yunohost/templates/dnsmasq/plain/resolv.dnsmasq.conf" + resolvers = [r.split(" ")[1] for r in read_file(resolver_file).split("\n") if r.startswith("nameserver")] + + if protocol == 4: + resolvers = [r for r in resolvers if ":" not in r] + if protocol == 6: + resolvers = [r for r in resolvers if ":" in r] + + assert resolvers != [], "Uhoh, need at least one IPv%s DNS resolver in %s ..." % (protocol, resolver_file) + + # So let's try to ping the first 4~5 resolvers (shuffled) + # If we succesfully ping any of them, we conclude that we are indeed connected + def ping(protocol, target): + return os.system("ping -c1 -%s -W 3 %s >/dev/null 2>/dev/null" % (protocol, target)) == 0 + + random.shuffle(resolvers) + return any(ping(protocol, resolver) for resolver in resolvers[:5]) + def get_public_ip(self, protocol=4): + # FIXME - TODO : here we assume that DNS resolution for ip.yunohost.org is working + # but if we want to be able to diagnose DNS resolution issues independently from + # internet connectivity, we gotta rely on fixed IPs first.... + if protocol == 4: url = 'https://ip.yunohost.org' elif protocol == 6: diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 3ba64445d..c8b81fd2c 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -79,6 +79,9 @@ class DNSRecordsDiagnoser(Diagnoser): command = "dig +short @%s %s %s" % (self.resolver, type_, domain) else: command = "dig +short @%s %s %s.%s" % (self.resolver, type_, name, domain) + # FIXME : gotta handle case where this command fails ... + # e.g. no internet connectivity (dependency mechanism to good result from 'ip' diagosis ?) + # or the resolver is unavailable for some reason output = check_output(command).strip() output = output.replace("\;",";") if output.startswith('"') and output.endswith('"'): diff --git a/locales/en.json b/locales/en.json index 0bb6d7275..ae5e4dc53 100644 --- a/locales/en.json +++ b/locales/en.json @@ -159,8 +159,8 @@ "diagnosis_network_no_ipv4": "The server does not have a working IPv4.", "diagnosis_network_connected_ipv6": "The server is connect to the Internet through IPv6 !", "diagnosis_network_no_ipv6": "The server does not have a working IPv6.", - "diagnosis_dns_good_conf": "Good DNS configuration for {domain} : {category}.", - "diagnosis_dns_bad_conf": "Bad DNS configuration for {domain} : {category}.", + "diagnosis_dns_good_conf": "Good DNS configuration for domain {domain} (category {category})", + "diagnosis_dns_bad_conf": "Bad DNS configuration for domain {domain} (category {category})", "diagnosis_dns_missing_record": "According to the recommended DNS configuration, you should add a DNS record with type {0}, name {1} and value {2}", "diagnosis_dns_discrepancy": "According to the recommended DNS configuration, the value for the DNS record with type {0} and name {1} should be {2}, not {3}.", "diagnosis_description_ip": "Internet connectivity", From 5f4450ab87f4a0985a877539cba5fc40231c8555 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 14 Jul 2019 00:35:42 +0200 Subject: [PATCH 39/94] Add DNS resolution tests --- data/hooks/diagnosis/10-ip.py | 52 +++++++++++++++++++++++++++++++---- locales/en.json | 11 +++++--- 2 files changed, 54 insertions(+), 9 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index 3259c6a4a..1835927a2 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -28,16 +28,44 @@ class IPDiagnoser(Diagnoser): if 4 in versions: + # If we can't ping, there's not much else we can do if not self.can_ping_outside(4): ipv4 = None + # If we do ping, check that we can resolv domain name else: - ipv4 = self.get_public_ip(4) + can_resolve_dns = self.can_resolve_dns() + # And if we do, then we can fetch the public ip + if can_resolve_dns: + ipv4 = self.get_public_ip(4) + # In every case, we can check that resolvconf seems to be okay + # (symlink managed by resolvconf service + pointing to dnsmasq) + good_resolvconf = self.resolvconf_is_symlink() and self.resolvconf_points_to_localhost() + + # If we can't resolve domain names at all, that's a pretty big issue ... + # If it turns out that at the same time, resolvconf is bad, that's probably + # the cause of this, so we use a different message in that case + if not can_resolve_dns: + yield dict(meta = {"name": "dnsresolution"}, + status = "ERROR", + summary = ("diagnosis_ip_broken_dnsresolution", {}) if good_resolvconf + else ("diagnosis_ip_broken_resolvconf", {})) + # Otherwise, if the resolv conf is bad but we were able to resolve domain name, + # still warn that we're using a weird resolv conf ... + elif not good_resolvconf: + yield dict(meta = {"name": "dnsresolution"}, + status = "WARNING", + summary = ("diagnosis_ip_weird_resolvconf", {})) + else: + # Well, maybe we could report a "success", "dns resolution is working", idk if it's worth it + pass + + # And finally, we actually report the ipv4 connectivity stuff yield dict(meta = {"version": 4}, data = ipv4, status = "SUCCESS" if ipv4 else "ERROR", - summary = ("diagnosis_network_connected_ipv4", {}) if ipv4 \ - else ("diagnosis_network_no_ipv4", {})) + summary = ("diagnosis_ip_connected_ipv4", {}) if ipv4 \ + else ("diagnosis_ip_no_ipv4", {})) if 6 in versions: @@ -49,8 +77,8 @@ class IPDiagnoser(Diagnoser): yield dict(meta = {"version": 6}, data = ipv6, status = "SUCCESS" if ipv6 else "WARNING", - summary = ("diagnosis_network_connected_ipv6", {}) if ipv6 \ - else ("diagnosis_network_no_ipv6", {})) + summary = ("diagnosis_ip_connected_ipv6", {}) if ipv6 \ + else ("diagnosis_ip_no_ipv6", {})) def can_ping_outside(self, protocol=4): @@ -85,6 +113,20 @@ class IPDiagnoser(Diagnoser): random.shuffle(resolvers) return any(ping(protocol, resolver) for resolver in resolvers[:5]) + + def can_resolve_dns(self): + return os.system("dig +short ip.yunohost.org >/dev/null 2>/dev/null") == 0 + + + def resolvconf_is_symlink(self): + return os.path.realpath("/etc/resolv.conf") == "/run/resolvconf/resolv.conf" + + def resolvconf_points_to_localhost(self): + file_ = "/etc/resolv.conf" + resolvers = [r.split(" ")[1] for r in read_file(file_).split("\n") if r.startswith("nameserver")] + return resolvers == ["127.0.0.1"] + + def get_public_ip(self, protocol=4): # FIXME - TODO : here we assume that DNS resolution for ip.yunohost.org is working diff --git a/locales/en.json b/locales/en.json index ae5e4dc53..515993884 100644 --- a/locales/en.json +++ b/locales/en.json @@ -155,10 +155,13 @@ "diagnosis_no_apps": "No installed application", "dpkg_is_broken": "You cannot do this right now because dpkg/APT (the system package managers) seems to be in a broken state… You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.", "dpkg_lock_not_available": "This command can't be ran right now because another program seems to be using the lock of dpkg (the system package manager)", - "diagnosis_network_connected_ipv4": "The server is connected to the Internet through IPv4 !", - "diagnosis_network_no_ipv4": "The server does not have a working IPv4.", - "diagnosis_network_connected_ipv6": "The server is connect to the Internet through IPv6 !", - "diagnosis_network_no_ipv6": "The server does not have a working IPv6.", + "diagnosis_ip_connected_ipv4": "The server is connected to the Internet through IPv4 !", + "diagnosis_ip_no_ipv4": "The server does not have a working IPv4.", + "diagnosis_ip_connected_ipv6": "The server is connected to the Internet through IPv6 !", + "diagnosis_ip_no_ipv6": "The server does not have a working IPv6.", + "diagnosis_ip_broken_dnsresolution": "Domain name resolution seems to be broken for some reason ... Is a firewall blocking DNS requests ?", + "diagnosis_ip_broken_resolvconf": "Domain name resolution seems to be broken on your server, which seems related to /etc/resolv.conf not pointing to 127.0.0.1.", + "diagnosis_ip_weird_resolvconf": "Be careful that you seem to be using a custom /etc/resolv.conf. Instead, this file should be a symlink to /etc/resolvconf/run/resolv.conf itself pointing to 127.0.0.1 (dnsmasq).", "diagnosis_dns_good_conf": "Good DNS configuration for domain {domain} (category {category})", "diagnosis_dns_bad_conf": "Bad DNS configuration for domain {domain} (category {category})", "diagnosis_dns_missing_record": "According to the recommended DNS configuration, you should add a DNS record with type {0}, name {1} and value {2}", From aed53786f2f6e3cac78af79205d956a366b4efc3 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 14 Jul 2019 00:45:09 +0200 Subject: [PATCH 40/94] Make the PEP8 gods less angry --- data/hooks/diagnosis/10-ip.py | 45 ++++++++++++--------------- data/hooks/diagnosis/12-dnsrecords.py | 25 +++++++-------- src/yunohost/diagnosis.py | 39 ++++++++++++----------- 3 files changed, 51 insertions(+), 58 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index 1835927a2..1f6c31f50 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -3,13 +3,13 @@ import os import random -from moulinette import m18n from moulinette.utils.network import download_text from moulinette.utils.process import check_output from moulinette.utils.filesystem import read_file from yunohost.diagnosis import Diagnoser + class IPDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] @@ -17,10 +17,10 @@ class IPDiagnoser(Diagnoser): def validate_args(self, args): if "version" not in args.keys(): - return { "versions" : [4, 6] } + return {"versions": [4, 6]} else: assert str(args["version"]) in ["4", "6"], "Invalid version, should be 4 or 6." - return { "versions" : [int(args["version"])] } + return {"versions": [int(args["version"])]} def run(self): @@ -46,26 +46,26 @@ class IPDiagnoser(Diagnoser): # If it turns out that at the same time, resolvconf is bad, that's probably # the cause of this, so we use a different message in that case if not can_resolve_dns: - yield dict(meta = {"name": "dnsresolution"}, - status = "ERROR", - summary = ("diagnosis_ip_broken_dnsresolution", {}) if good_resolvconf - else ("diagnosis_ip_broken_resolvconf", {})) + yield dict(meta={"name": "dnsresolution"}, + status="ERROR", + summary=("diagnosis_ip_broken_dnsresolution", {}) if good_resolvconf + else ("diagnosis_ip_broken_resolvconf", {})) # Otherwise, if the resolv conf is bad but we were able to resolve domain name, # still warn that we're using a weird resolv conf ... elif not good_resolvconf: - yield dict(meta = {"name": "dnsresolution"}, - status = "WARNING", - summary = ("diagnosis_ip_weird_resolvconf", {})) + yield dict(meta={"name": "dnsresolution"}, + status="WARNING", + summary=("diagnosis_ip_weird_resolvconf", {})) else: # Well, maybe we could report a "success", "dns resolution is working", idk if it's worth it pass # And finally, we actually report the ipv4 connectivity stuff - yield dict(meta = {"version": 4}, - data = ipv4, - status = "SUCCESS" if ipv4 else "ERROR", - summary = ("diagnosis_ip_connected_ipv4", {}) if ipv4 \ - else ("diagnosis_ip_no_ipv4", {})) + yield dict(meta={"version": 4}, + data=ipv4, + status="SUCCESS" if ipv4 else "ERROR", + summary=("diagnosis_ip_connected_ipv4", {}) if ipv4 + else ("diagnosis_ip_no_ipv4", {})) if 6 in versions: @@ -74,12 +74,11 @@ class IPDiagnoser(Diagnoser): else: ipv6 = self.get_public_ip(6) - yield dict(meta = {"version": 6}, - data = ipv6, - status = "SUCCESS" if ipv6 else "WARNING", - summary = ("diagnosis_ip_connected_ipv6", {}) if ipv6 \ - else ("diagnosis_ip_no_ipv6", {})) - + yield dict(meta={"version": 6}, + data=ipv6, + status="SUCCESS" if ipv6 else "WARNING", + summary=("diagnosis_ip_connected_ipv6", {}) if ipv6 + else ("diagnosis_ip_no_ipv6", {})) def can_ping_outside(self, protocol=4): @@ -113,11 +112,9 @@ class IPDiagnoser(Diagnoser): random.shuffle(resolvers) return any(ping(protocol, resolver) for resolver in resolvers[:5]) - def can_resolve_dns(self): return os.system("dig +short ip.yunohost.org >/dev/null 2>/dev/null") == 0 - def resolvconf_is_symlink(self): return os.path.realpath("/etc/resolv.conf") == "/run/resolvconf/resolv.conf" @@ -126,7 +123,6 @@ class IPDiagnoser(Diagnoser): resolvers = [r.split(" ")[1] for r in read_file(file_).split("\n") if r.startswith("nameserver")] return resolvers == ["127.0.0.1"] - def get_public_ip(self, protocol=4): # FIXME - TODO : here we assume that DNS resolution for ip.yunohost.org is working @@ -149,4 +145,3 @@ class IPDiagnoser(Diagnoser): def main(args, env, loggers): return IPDiagnoser(args, env, loggers).diagnose() - diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index c8b81fd2c..493010c59 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -2,25 +2,25 @@ import os -from moulinette.utils.network import download_text from moulinette.utils.process import check_output from moulinette.utils.filesystem import read_file from yunohost.diagnosis import Diagnoser from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain + class DNSRecordsDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] - cache_duration = 3600*24 + cache_duration = 3600 * 24 def validate_args(self, args): all_domains = domain_list()["domains"] if "domain" not in args.keys(): - return { "domains" : all_domains } + return {"domains": all_domains} else: assert args["domain"] in all_domains, "Unknown domain" - return { "domains" : [ args["domain"] ] } + return {"domains": [args["domain"]]} def run(self): @@ -34,7 +34,7 @@ class DNSRecordsDiagnoser(Diagnoser): for domain in self.args["domains"]: self.logger_debug("Diagnosing DNS conf for %s" % domain) - for report in self.check_domain(domain, domain==main_domain): + for report in self.check_domain(domain, domain == main_domain): yield report def check_domain(self, domain, is_main_domain): @@ -44,13 +44,13 @@ class DNSRecordsDiagnoser(Diagnoser): # Here if there are no AAAA record, we should add something to expect "no" AAAA record # to properly diagnose situations where people have a AAAA record but no IPv6 - for category, records in expected_configuration.items(): + for category, records in expected_configuration.items(): discrepancies = [] for r in records: current_value = self.get_current_record(domain, r["name"], r["type"]) or "None" - expected_value = r["value"] if r["value"] != "@" else domain+"." + expected_value = r["value"] if r["value"] != "@" else domain + "." if current_value == "None": discrepancies.append(("diagnosis_dns_missing_record", (r["type"], r["name"], expected_value))) @@ -64,16 +64,15 @@ class DNSRecordsDiagnoser(Diagnoser): status = "SUCCESS" summary = ("diagnosis_dns_good_conf", {"domain": domain, "category": category}) - output = dict(meta = {"domain": domain, "category": category}, - status = status, - summary = summary) + output = dict(meta={"domain": domain, "category": category}, + status=status, + summary=summary) if discrepancies: output["details"] = discrepancies yield output - def get_current_record(self, domain, name, type_): if name == "@": command = "dig +short @%s %s %s" % (self.resolver, type_, domain) @@ -83,12 +82,10 @@ class DNSRecordsDiagnoser(Diagnoser): # e.g. no internet connectivity (dependency mechanism to good result from 'ip' diagosis ?) # or the resolver is unavailable for some reason output = check_output(command).strip() - output = output.replace("\;",";") if output.startswith('"') and output.endswith('"'): - output = '"' + ' '.join(output.replace('"',' ').split()) + '"' + output = '"' + ' '.join(output.replace('"', ' ').split()) + '"' return output def main(args, env, loggers): return DNSRecordsDiagnoser(args, env, loggers).diagnose() - diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 523a5c891..9b17a7457 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -38,21 +38,23 @@ logger = log.getActionLogger('yunohost.diagnosis') DIAGNOSIS_CACHE = "/var/cache/yunohost/diagnosis/" + def diagnosis_list(): - all_categories_names = [ h for h, _ in _list_diagnosis_categories() ] - return { "categories": all_categories_names } + all_categories_names = [h for h, _ in _list_diagnosis_categories()] + return {"categories": all_categories_names} + def diagnosis_show(categories=[], issues=False, full=False): # Get all the categories all_categories = _list_diagnosis_categories() - all_categories_names = [ category for category, _ in all_categories ] + all_categories_names = [category for category, _ in all_categories] # Check the requested category makes sense if categories == []: categories = all_categories_names else: - unknown_categories = [ c for c in categories if c not in all_categories_names ] + unknown_categories = [c for c in categories if c not in all_categories_names] if unknown_categories: raise YunohostError('unknown_categories', categories=", ".join(categories)) @@ -62,7 +64,7 @@ def diagnosis_show(categories=[], issues=False, full=False): try: report = Diagnoser.get_cached_report(category) except Exception as e: - logger.error("Failed to fetch diagnosis result for category '%s' : %s" % (category, str(e))) # FIXME : i18n + logger.error("Failed to fetch diagnosis result for category '%s' : %s" % (category, str(e))) # FIXME : i18n else: if not full: del report["timestamp"] @@ -72,33 +74,33 @@ def diagnosis_show(categories=[], issues=False, full=False): if "data" in item: del item["data"] if issues: - report["items"] = [ item for item in report["items"] if item["status"] != "SUCCESS" ] + report["items"] = [item for item in report["items"] if item["status"] != "SUCCESS"] # Ignore this category if no issue was found if not report["items"]: continue all_reports.append(report) - return {"reports": all_reports} + def diagnosis_run(categories=[], force=False, args=None): # Get all the categories all_categories = _list_diagnosis_categories() - all_categories_names = [ category for category, _ in all_categories ] + all_categories_names = [category for category, _ in all_categories] # Check the requested category makes sense if categories == []: categories = all_categories_names else: - unknown_categories = [ c for c in categories if c not in all_categories_names ] + unknown_categories = [c for c in categories if c not in all_categories_names] if unknown_categories: raise YunohostError('unknown_categories', categories=", ".join(unknown_categories)) # Transform "arg1=val1&arg2=val2" to { "arg1": "val1", "arg2": "val2" } if args is not None: - args = { arg.split("=")[0]: arg.split("=")[1] for arg in args.split("&") } + args = {arg.split("=")[0]: arg.split("=")[1] for arg in args.split("&")} else: args = {} args["force"] = force @@ -108,12 +110,12 @@ def diagnosis_run(categories=[], force=False, args=None): diagnosed_categories = [] for category in categories: logger.debug("Running diagnosis for %s ..." % category) - path = [p for n, p in all_categories if n == category ][0] + path = [p for n, p in all_categories if n == category][0] try: code, report = hook_exec(path, args=args, env=None) except Exception as e: - logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e)), exc_info=True) # FIXME : i18n + logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e)), exc_info=True) # FIXME : i18n else: diagnosed_categories.append(category) if report != {}: @@ -127,6 +129,7 @@ def diagnosis_run(categories=[], force=False, args=None): return + def diagnosis_ignore(category, args="", unignore=False): pass @@ -149,7 +152,6 @@ class Diagnoser(): if self.description == descr_key: self.description = self.id_ - def cached_time_ago(self): if not os.path.exists(self.cache_file): @@ -172,10 +174,9 @@ class Diagnoser(): items = list(self.run()) - new_report = { "id": self.id_, - "cached_for": self.cache_duration, - "items": items - } + new_report = {"id": self.id_, + "cached_for": self.cache_duration, + "items": items} # TODO / FIXME : should handle the case where we only did a partial diagnosis self.logger_debug("Updating cache %s" % self.cache_file) @@ -227,13 +228,13 @@ class Diagnoser(): item["summary"] = m18n.n(summary_key, **summary_args) if "details" in item: - item["details"] = [ m18n.n(key, *values) for key, values in item["details"] ] + item["details"] = [m18n.n(key, *values) for key, values in item["details"]] def _list_diagnosis_categories(): hooks_raw = hook_list("diagnosis", list_by="priority", show_info=True)["hooks"] hooks = [] - for _, some_hooks in sorted(hooks_raw.items(), key=lambda h:int(h[0])): + for _, some_hooks in sorted(hooks_raw.items(), key=lambda h: int(h[0])): for name, info in some_hooks.items(): hooks.append((name, info["path"])) From 1019e95b1d6d4d094c884d2c902a66a3d3deafa4 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 20 Jul 2019 18:44:32 +0200 Subject: [PATCH 41/94] Implement a first version for services status check --- data/hooks/diagnosis/30-services.py | 54 +++++++++++++++++++++++++++++ locales/en.json | 3 ++ 2 files changed, 57 insertions(+) create mode 100644 data/hooks/diagnosis/30-services.py diff --git a/data/hooks/diagnosis/30-services.py b/data/hooks/diagnosis/30-services.py new file mode 100644 index 000000000..4f08247f1 --- /dev/null +++ b/data/hooks/diagnosis/30-services.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python + +import os + +from yunohost.diagnosis import Diagnoser +from yunohost.service import service_status + +# TODO : all these are arbitrary, should be collectively validated +services_ignored = {"glances"} +services_critical = {"dnsmasq", "fail2ban", "yunohost-firewall", "nginx", "slapd", "ssh"} +# TODO / FIXME : we should do something about this postfix thing +# The nominal value is to be "exited" ... some daemon is actually running +# in a different thread that the thing started by systemd, which is fine +# but somehow sometimes it gets killed and there's no easy way to detect it +# Just randomly restarting it will fix ths issue. We should find some trick +# to identify the PID of the process and check it's still up or idk +services_expected_to_be_exited = {"postfix", "yunohost-firewall"} + +class ServicesDiagnoser(Diagnoser): + + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] + cache_duration = 300 + + def validate_args(self, args): + # TODO / FIXME Ugh do we really need this arg system + return {} + + def run(self): + + all_result = service_status() + + for service, result in all_result.items(): + + if service in services_ignored: + continue + + item = dict(meta={"service": service}) + expected_status = "running" if service not in services_expected_to_be_exited else "exited" + + # TODO / FIXME : might also want to check that services are enabled + + if result["active"] != "active" or result["status"] != expected_status: + item["status"] = "WARNING" if service not in services_critical else "ERROR" + item["summary"] = ("diagnosis_services_bad_status", {"service": service, "status": result["active"] + "/" + result["status"]}) + + # TODO : could try to append the tail of the service log to the "details" key ... + else: + item["status"] = "SUCCESS" + item["summary"] = ("diagnosis_services_good_status", {"service": service, "status": result["active"] + "/" + result["status"]}) + + yield item + +def main(args, env, loggers): + return ServicesDiagnoser(args, env, loggers).diagnose() diff --git a/locales/en.json b/locales/en.json index 515993884..8fcb0e773 100644 --- a/locales/en.json +++ b/locales/en.json @@ -166,8 +166,11 @@ "diagnosis_dns_bad_conf": "Bad DNS configuration for domain {domain} (category {category})", "diagnosis_dns_missing_record": "According to the recommended DNS configuration, you should add a DNS record with type {0}, name {1} and value {2}", "diagnosis_dns_discrepancy": "According to the recommended DNS configuration, the value for the DNS record with type {0} and name {1} should be {2}, not {3}.", + "diagnosis_services_good_status": "Service {service} is {status} as expected!", + "diagnosis_services_bad_status": "Service {service} is {status} :/", "diagnosis_description_ip": "Internet connectivity", "diagnosis_description_dnsrecords": "DNS records", + "diagnosis_description_services": "Services status check", "domain_cannot_remove_main": "Cannot remove main domain. Set a new main domain first", "domain_cert_gen_failed": "Could not generate certificate", "domain_created": "Domain created", From 24f9d475b8d79fbf5c57034cba49d4bee013fea5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 20 Jul 2019 18:44:53 +0200 Subject: [PATCH 42/94] Implement a first version for disk usage check --- data/hooks/diagnosis/50-diskusage.py | 42 ++++++++++++++++++++++++++++ locales/en.json | 4 +++ 2 files changed, 46 insertions(+) create mode 100644 data/hooks/diagnosis/50-diskusage.py diff --git a/data/hooks/diagnosis/50-diskusage.py b/data/hooks/diagnosis/50-diskusage.py new file mode 100644 index 000000000..84ce3845c --- /dev/null +++ b/data/hooks/diagnosis/50-diskusage.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +import os +import psutil + +from yunohost.diagnosis import Diagnoser + +class DiskUsageDiagnoser(Diagnoser): + + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] + cache_duration = 3600 * 24 + + def validate_args(self, args): + # TODO / FIXME Ugh do we really need this arg system + return {} + + def run(self): + + disk_partitions = psutil.disk_partitions() + + for disk_partition in disk_partitions: + device = disk_partition.device + mountpoint = disk_partition.mountpoint + + usage = psutil.disk_usage(mountpoint) + free_Go = usage.free / (1024 ** 3) + free_percent = 100 - usage.percent + + item = dict(meta={"mountpoint": mountpoint, "device": device}) + if free_Go < 1 or free_percent < 5: + item["status"] = "ERROR" + item["summary"] = ("diagnosis_diskusage_verylow", {"mountpoint": mountpoint, "device": device, "free_percent": free_percent}) + elif free_Go < 2 or free_percent < 10: + item["status"] = "WARNING" + item["summary"] = ("diagnosis_diskusage_low", {"mountpoint": mountpoint, "device": device, "free_percent": free_percent}) + else: + item["status"] = "SUCCESS" + item["summary"] = ("diagnosis_diskusage_ok", {"mountpoint": mountpoint, "device": device, "free_percent": free_percent}) + + yield item + +def main(args, env, loggers): + return DiskUsageDiagnoser(args, env, loggers).diagnose() diff --git a/locales/en.json b/locales/en.json index 8fcb0e773..2e93e367f 100644 --- a/locales/en.json +++ b/locales/en.json @@ -168,9 +168,13 @@ "diagnosis_dns_discrepancy": "According to the recommended DNS configuration, the value for the DNS record with type {0} and name {1} should be {2}, not {3}.", "diagnosis_services_good_status": "Service {service} is {status} as expected!", "diagnosis_services_bad_status": "Service {service} is {status} :/", + "diagnosis_diskusage_verylow": "Storage {mountpoint} (on device {device}) has only {free_percent}% space remaining. You should really consider cleaning up some space.", + "diagnosis_diskusage_low": "Storage {mountpoint} (on device {device}) has only {free_percent}% space remaining. Be careful", + "diagnosis_diskusage_ok": "Storage {mountpoint} (on device {device}) still has {free_percent}% space left!", "diagnosis_description_ip": "Internet connectivity", "diagnosis_description_dnsrecords": "DNS records", "diagnosis_description_services": "Services status check", + "diagnosis_description_diskusage": "Disk usage", "domain_cannot_remove_main": "Cannot remove main domain. Set a new main domain first", "domain_cert_gen_failed": "Could not generate certificate", "domain_created": "Domain created", From d2bbb5a2b31718054365a0ee5d63c2298776f32e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 20 Jul 2019 19:02:11 +0200 Subject: [PATCH 43/94] This 'args' things sounds like a big YAGNI after all --- data/actionsmap/yunohost.yml | 3 - data/hooks/diagnosis/10-ip.py | 105 ++++++++++++-------------- data/hooks/diagnosis/12-dnsrecords.py | 11 +-- data/hooks/diagnosis/30-services.py | 4 - data/hooks/diagnosis/50-diskusage.py | 4 - src/yunohost/diagnosis.py | 12 +-- 6 files changed, 54 insertions(+), 85 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 6b89a819b..3d72bb57a 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1898,9 +1898,6 @@ diagnosis: --force: help: Ignore the cached report even if it is still 'fresh' action: store_true - -a: - help: Serialized arguments for diagnosis scripts (e.g. "domain=domain.tld") - full: --args ignore: action_help: Configure some diagnosis results to be ignored diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index 1f6c31f50..4a6ee75ce 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -15,70 +15,65 @@ class IPDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] cache_duration = 60 - def validate_args(self, args): - if "version" not in args.keys(): - return {"versions": [4, 6]} - else: - assert str(args["version"]) in ["4", "6"], "Invalid version, should be 4 or 6." - return {"versions": [int(args["version"])]} - def run(self): - versions = self.args["versions"] + # + # IPv4 Diagnosis + # - if 4 in versions: + # If we can't ping, there's not much else we can do + if not self.can_ping_outside(4): + ipv4 = None + # If we do ping, check that we can resolv domain name + else: + can_resolve_dns = self.can_resolve_dns() + # And if we do, then we can fetch the public ip + if can_resolve_dns: + ipv4 = self.get_public_ip(4) - # If we can't ping, there's not much else we can do - if not self.can_ping_outside(4): - ipv4 = None - # If we do ping, check that we can resolv domain name - else: - can_resolve_dns = self.can_resolve_dns() - # And if we do, then we can fetch the public ip - if can_resolve_dns: - ipv4 = self.get_public_ip(4) + # In every case, we can check that resolvconf seems to be okay + # (symlink managed by resolvconf service + pointing to dnsmasq) + good_resolvconf = self.resolvconf_is_symlink() and self.resolvconf_points_to_localhost() - # In every case, we can check that resolvconf seems to be okay - # (symlink managed by resolvconf service + pointing to dnsmasq) - good_resolvconf = self.resolvconf_is_symlink() and self.resolvconf_points_to_localhost() + # If we can't resolve domain names at all, that's a pretty big issue ... + # If it turns out that at the same time, resolvconf is bad, that's probably + # the cause of this, so we use a different message in that case + if not can_resolve_dns: + yield dict(meta={"name": "dnsresolution"}, + status="ERROR", + summary=("diagnosis_ip_broken_dnsresolution", {}) if good_resolvconf + else ("diagnosis_ip_broken_resolvconf", {})) + # Otherwise, if the resolv conf is bad but we were able to resolve domain name, + # still warn that we're using a weird resolv conf ... + elif not good_resolvconf: + yield dict(meta={"name": "dnsresolution"}, + status="WARNING", + summary=("diagnosis_ip_weird_resolvconf", {})) + else: + # Well, maybe we could report a "success", "dns resolution is working", idk if it's worth it + pass - # If we can't resolve domain names at all, that's a pretty big issue ... - # If it turns out that at the same time, resolvconf is bad, that's probably - # the cause of this, so we use a different message in that case - if not can_resolve_dns: - yield dict(meta={"name": "dnsresolution"}, - status="ERROR", - summary=("diagnosis_ip_broken_dnsresolution", {}) if good_resolvconf - else ("diagnosis_ip_broken_resolvconf", {})) - # Otherwise, if the resolv conf is bad but we were able to resolve domain name, - # still warn that we're using a weird resolv conf ... - elif not good_resolvconf: - yield dict(meta={"name": "dnsresolution"}, - status="WARNING", - summary=("diagnosis_ip_weird_resolvconf", {})) - else: - # Well, maybe we could report a "success", "dns resolution is working", idk if it's worth it - pass + # And finally, we actually report the ipv4 connectivity stuff + yield dict(meta={"version": 4}, + data=ipv4, + status="SUCCESS" if ipv4 else "ERROR", + summary=("diagnosis_ip_connected_ipv4", {}) if ipv4 + else ("diagnosis_ip_no_ipv4", {})) - # And finally, we actually report the ipv4 connectivity stuff - yield dict(meta={"version": 4}, - data=ipv4, - status="SUCCESS" if ipv4 else "ERROR", - summary=("diagnosis_ip_connected_ipv4", {}) if ipv4 - else ("diagnosis_ip_no_ipv4", {})) + # + # IPv6 Diagnosis + # - if 6 in versions: + if not self.can_ping_outside(4): + ipv6 = None + else: + ipv6 = self.get_public_ip(6) - if not self.can_ping_outside(4): - ipv6 = None - else: - ipv6 = self.get_public_ip(6) - - yield dict(meta={"version": 6}, - data=ipv6, - status="SUCCESS" if ipv6 else "WARNING", - summary=("diagnosis_ip_connected_ipv6", {}) if ipv6 - else ("diagnosis_ip_no_ipv6", {})) + yield dict(meta={"version": 6}, + data=ipv6, + status="SUCCESS" if ipv6 else "WARNING", + summary=("diagnosis_ip_connected_ipv6", {}) if ipv6 + else ("diagnosis_ip_no_ipv6", {})) def can_ping_outside(self, protocol=4): diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 493010c59..0f47ff136 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -14,14 +14,6 @@ class DNSRecordsDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] cache_duration = 3600 * 24 - def validate_args(self, args): - all_domains = domain_list()["domains"] - if "domain" not in args.keys(): - return {"domains": all_domains} - else: - assert args["domain"] in all_domains, "Unknown domain" - return {"domains": [args["domain"]]} - def run(self): resolvers = read_file("/etc/resolv.dnsmasq.conf").split("\n") @@ -32,7 +24,8 @@ class DNSRecordsDiagnoser(Diagnoser): self.resolver = ipv4_resolvers[0] main_domain = _get_maindomain() - for domain in self.args["domains"]: + all_domains = domain_list()["domains"] + for domain in all_domains: self.logger_debug("Diagnosing DNS conf for %s" % domain) for report in self.check_domain(domain, domain == main_domain): yield report diff --git a/data/hooks/diagnosis/30-services.py b/data/hooks/diagnosis/30-services.py index 4f08247f1..5029e0a5d 100644 --- a/data/hooks/diagnosis/30-services.py +++ b/data/hooks/diagnosis/30-services.py @@ -21,10 +21,6 @@ class ServicesDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] cache_duration = 300 - def validate_args(self, args): - # TODO / FIXME Ugh do we really need this arg system - return {} - def run(self): all_result = service_status() diff --git a/data/hooks/diagnosis/50-diskusage.py b/data/hooks/diagnosis/50-diskusage.py index 84ce3845c..2c6fe387b 100644 --- a/data/hooks/diagnosis/50-diskusage.py +++ b/data/hooks/diagnosis/50-diskusage.py @@ -9,10 +9,6 @@ class DiskUsageDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] cache_duration = 3600 * 24 - def validate_args(self, args): - # TODO / FIXME Ugh do we really need this arg system - return {} - def run(self): disk_partitions = psutil.disk_partitions() diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 9b17a7457..e7aca585f 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -84,7 +84,7 @@ def diagnosis_show(categories=[], issues=False, full=False): return {"reports": all_reports} -def diagnosis_run(categories=[], force=False, args=None): +def diagnosis_run(categories=[], force=False): # Get all the categories all_categories = _list_diagnosis_categories() @@ -98,13 +98,6 @@ def diagnosis_run(categories=[], force=False, args=None): if unknown_categories: raise YunohostError('unknown_categories', categories=", ".join(unknown_categories)) - # Transform "arg1=val1&arg2=val2" to { "arg1": "val1", "arg2": "val2" } - if args is not None: - args = {arg.split("=")[0]: arg.split("=")[1] for arg in args.split("&")} - else: - args = {} - args["force"] = force - issues = [] # Call the hook ... diagnosed_categories = [] @@ -113,7 +106,7 @@ def diagnosis_run(categories=[], force=False, args=None): path = [p for n, p in all_categories if n == category][0] try: - code, report = hook_exec(path, args=args, env=None) + code, report = hook_exec(path, args={"force": force}, env=None) except Exception as e: logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e)), exc_info=True) # FIXME : i18n else: @@ -143,7 +136,6 @@ class Diagnoser(): self.logger_debug, self.logger_warning, self.logger_info = loggers self.env = env self.args = args or {} - self.args.update(self.validate_args(self.args)) self.cache_file = Diagnoser.cache_file(self.id_) descr_key = "diagnosis_description_" + self.id_ From 35f6b778956b3755fec10718be0091d416cfdadc Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 20 Jul 2019 19:30:09 +0200 Subject: [PATCH 44/94] Reclarify ip diagnoser --- data/hooks/diagnosis/10-ip.py | 72 ++++++++++++++++++----------------- locales/en.json | 2 + 2 files changed, 40 insertions(+), 34 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index 4a6ee75ce..a4cfc0a48 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -17,19 +17,26 @@ class IPDiagnoser(Diagnoser): def run(self): - # - # IPv4 Diagnosis - # + # ############################################################ # + # PING : Check that we can ping outside at least in ipv4 or v6 # + # ############################################################ # - # If we can't ping, there's not much else we can do - if not self.can_ping_outside(4): - ipv4 = None - # If we do ping, check that we can resolv domain name - else: - can_resolve_dns = self.can_resolve_dns() - # And if we do, then we can fetch the public ip - if can_resolve_dns: - ipv4 = self.get_public_ip(4) + can_ping_ipv4 = self.can_ping_outside(4) + can_ping_ipv6 = self.can_ping_outside(6) + + if not can_ping_ipv4 and not can_ping_ipv6: + yield dict(meta={"test": "ping"}, + status="ERROR", + summary=("diagnosis_ip_not_connected_at_all", {})) + # Not much else we can do if there's no internet at all + return + + # ###################################################### # + # DNS RESOLUTION : Check that we can resolve domain name # + # (later needed to talk to ip. and ip6.yunohost.org) # + # ###################################################### # + + can_resolve_dns = self.can_resolve_dns() # In every case, we can check that resolvconf seems to be okay # (symlink managed by resolvconf service + pointing to dnsmasq) @@ -39,37 +46,37 @@ class IPDiagnoser(Diagnoser): # If it turns out that at the same time, resolvconf is bad, that's probably # the cause of this, so we use a different message in that case if not can_resolve_dns: - yield dict(meta={"name": "dnsresolution"}, + yield dict(meta={"test": "dnsresolv"}, status="ERROR", summary=("diagnosis_ip_broken_dnsresolution", {}) if good_resolvconf else ("diagnosis_ip_broken_resolvconf", {})) + return # Otherwise, if the resolv conf is bad but we were able to resolve domain name, # still warn that we're using a weird resolv conf ... elif not good_resolvconf: - yield dict(meta={"name": "dnsresolution"}, + yield dict(meta={"test": "dnsresolv"}, status="WARNING", summary=("diagnosis_ip_weird_resolvconf", {})) else: - # Well, maybe we could report a "success", "dns resolution is working", idk if it's worth it - pass + yield dict(meta={"test": "dnsresolv"}, + status="SUCCESS", + summary=("diagnosis_ip_dnsresolution_working", {})) - # And finally, we actually report the ipv4 connectivity stuff - yield dict(meta={"version": 4}, + # ##################################################### # + # IP DIAGNOSIS : Check that we're actually able to talk # + # to a web server to fetch current IPv4 and v6 # + # ##################################################### # + + ipv4 = self.get_public_ip(4) if can_ping_ipv4 else None + ipv6 = self.get_public_ip(6) if can_ping_ipv6 else None + + yield dict(meta={"test": "ip", "version": 4}, data=ipv4, status="SUCCESS" if ipv4 else "ERROR", summary=("diagnosis_ip_connected_ipv4", {}) if ipv4 else ("diagnosis_ip_no_ipv4", {})) - # - # IPv6 Diagnosis - # - - if not self.can_ping_outside(4): - ipv6 = None - else: - ipv6 = self.get_public_ip(6) - - yield dict(meta={"version": 6}, + yield dict(meta={"test": "ip", "version": 6}, data=ipv6, status="SUCCESS" if ipv6 else "WARNING", summary=("diagnosis_ip_connected_ipv6", {}) if ipv6 @@ -124,12 +131,9 @@ class IPDiagnoser(Diagnoser): # but if we want to be able to diagnose DNS resolution issues independently from # internet connectivity, we gotta rely on fixed IPs first.... - if protocol == 4: - url = 'https://ip.yunohost.org' - elif protocol == 6: - url = 'https://ip6.yunohost.org' - else: - raise ValueError("invalid protocol version, it should be either 4 or 6 and was '%s'" % repr(protocol)) + assert protocol in [4, 6], "Invalid protocol version, it should be either 4 or 6 and was '%s'" % repr(protocol) + + url = 'https://ip%s.yunohost.org' % ('6' if protocol == 6 else '') try: return download_text(url, timeout=30).strip() diff --git a/locales/en.json b/locales/en.json index 2e93e367f..8d6828979 100644 --- a/locales/en.json +++ b/locales/en.json @@ -159,6 +159,8 @@ "diagnosis_ip_no_ipv4": "The server does not have a working IPv4.", "diagnosis_ip_connected_ipv6": "The server is connected to the Internet through IPv6 !", "diagnosis_ip_no_ipv6": "The server does not have a working IPv6.", + "diagnosis_ip_not_connected_at_all": "The server does not seem to be connected to the Internet at all!?", + "diagnosis_ip_dnsresolution_working": "Domain name resolution is working!", "diagnosis_ip_broken_dnsresolution": "Domain name resolution seems to be broken for some reason ... Is a firewall blocking DNS requests ?", "diagnosis_ip_broken_resolvconf": "Domain name resolution seems to be broken on your server, which seems related to /etc/resolv.conf not pointing to 127.0.0.1.", "diagnosis_ip_weird_resolvconf": "Be careful that you seem to be using a custom /etc/resolv.conf. Instead, this file should be a symlink to /etc/resolvconf/run/resolv.conf itself pointing to 127.0.0.1 (dnsmasq).", From f690ff6e1e5ed6f80e185f8b0d5af0248ff025d8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 30 Jul 2019 18:53:17 +0200 Subject: [PATCH 45/94] First version of port exposure diagnosis --- data/hooks/diagnosis/14-ports.py | 53 ++++++++++++++++++++++++++++++++ locales/en.json | 4 +++ 2 files changed, 57 insertions(+) create mode 100644 data/hooks/diagnosis/14-ports.py diff --git a/data/hooks/diagnosis/14-ports.py b/data/hooks/diagnosis/14-ports.py new file mode 100644 index 000000000..6b260f3e0 --- /dev/null +++ b/data/hooks/diagnosis/14-ports.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +import os +import requests + +from yunohost.diagnosis import Diagnoser + + +class PortsDiagnoser(Diagnoser): + + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] + cache_duration = 3600 + + def run(self): + + # FIXME / TODO : in the future, maybe we want to report different + # things per port depending on how important they are + # (e.g. XMPP sounds to me much less important than other ports) + # Ideally, a port could be related to a service... + # FIXME / TODO : for now this list of port is hardcoded, might want + # to fetch this from the firewall.yml in /etc/yunohost/ + ports = [ 22, 25, 53, 80, 443, 587, 993, 5222, 5269 ] + + try: + r = requests.post('https://ynhdiagnoser.netlib.re/check-ports', json={'ports': ports}).json() + if not "status" in r.keys(): + raise Exception("Bad syntax for response ? Raw json: %s" % str(r)) + elif r["status"] == "error": + if "content" in r.keys(): + raise Exception(r["content"]) + else: + raise Exception("Bad syntax for response ? Raw json: %s" % str(r)) + elif r["status"] != "ok" or "ports" not in r.keys() or not isinstance(r["ports"], dict): + raise Exception("Bad syntax for response ? Raw json: %s" % str(r)) + except Exception as e: + raise YunohostError("diagnosis_ports_could_not_diagnose", error=e) + + found_issues = False + for port in ports: + if r["ports"].get(str(port), None) != True: + found_issues = True + yield dict(meta={"port": port}, + status="ERROR", + summary=("diagnosis_ports_unreachable", {"port":port})) + + if not found_issues: + yield dict(meta={}, + status="SUCCESS", + summary=("diagnosis_ports_ok",{})) + + +def main(args, env, loggers): + return PortsDiagnoser(args, env, loggers).diagnose() diff --git a/locales/en.json b/locales/en.json index 8d6828979..0a2204725 100644 --- a/locales/en.json +++ b/locales/en.json @@ -177,6 +177,10 @@ "diagnosis_description_dnsrecords": "DNS records", "diagnosis_description_services": "Services status check", "diagnosis_description_diskusage": "Disk usage", + "diagnosis_description_ports": "Ports exposure", + "diagnosis_ports_could_not_diagnose": "Could not diagnose if ports are reachable from outside. Error: {error}", + "diagnosis_ports_unreachable": "Port {port} is not reachable from outside.", + "diagnosis_ports_ok": "Relevant ports are reachable from outside!", "domain_cannot_remove_main": "Cannot remove main domain. Set a new main domain first", "domain_cert_gen_failed": "Could not generate certificate", "domain_created": "Domain created", From 6c48c131a8cb12b56566a09c68d2e59de68182ef Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 31 Jul 2019 01:02:31 +0200 Subject: [PATCH 46/94] Fix small issues in port diagnoser --- data/hooks/diagnosis/14-ports.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/data/hooks/diagnosis/14-ports.py b/data/hooks/diagnosis/14-ports.py index 6b260f3e0..8206474f8 100644 --- a/data/hooks/diagnosis/14-ports.py +++ b/data/hooks/diagnosis/14-ports.py @@ -4,6 +4,7 @@ import os import requests from yunohost.diagnosis import Diagnoser +from yunohost.utils.error import YunohostError class PortsDiagnoser(Diagnoser): @@ -19,11 +20,11 @@ class PortsDiagnoser(Diagnoser): # Ideally, a port could be related to a service... # FIXME / TODO : for now this list of port is hardcoded, might want # to fetch this from the firewall.yml in /etc/yunohost/ - ports = [ 22, 25, 53, 80, 443, 587, 993, 5222, 5269 ] + ports = [22, 25, 53, 80, 443, 587, 993, 5222, 5269] try: - r = requests.post('https://ynhdiagnoser.netlib.re/check-ports', json={'ports': ports}).json() - if not "status" in r.keys(): + r = requests.post('https://ynhdiagnoser.netlib.re/check-ports', json={'ports': ports}, timeout=30).json() + if "status" not in r.keys(): raise Exception("Bad syntax for response ? Raw json: %s" % str(r)) elif r["status"] == "error": if "content" in r.keys(): @@ -37,16 +38,16 @@ class PortsDiagnoser(Diagnoser): found_issues = False for port in ports: - if r["ports"].get(str(port), None) != True: + if r["ports"].get(str(port), None) is not True: found_issues = True yield dict(meta={"port": port}, status="ERROR", - summary=("diagnosis_ports_unreachable", {"port":port})) + summary=("diagnosis_ports_unreachable", {"port": port})) if not found_issues: yield dict(meta={}, status="SUCCESS", - summary=("diagnosis_ports_ok",{})) + summary=("diagnosis_ports_ok", {})) def main(args, env, loggers): From f050b3c5b86bf6c844fc67597d6949324d75be3d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 31 Jul 2019 01:08:21 +0200 Subject: [PATCH 47/94] First version of http exposure diagnosis --- data/hooks/diagnosis/16-http.py | 54 ++++++++++++++++++++++++++++ data/templates/nginx/server.tpl.conf | 4 +++ locales/en.json | 4 +++ src/yunohost/app.py | 3 +- 4 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 data/hooks/diagnosis/16-http.py diff --git a/data/hooks/diagnosis/16-http.py b/data/hooks/diagnosis/16-http.py new file mode 100644 index 000000000..b6b92fc77 --- /dev/null +++ b/data/hooks/diagnosis/16-http.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python + +import os +import random +import requests + +from yunohost.diagnosis import Diagnoser +from yunohost.domain import domain_list +from yunohost.utils.error import YunohostError + + +class HttpDiagnoser(Diagnoser): + + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] + cache_duration = 3600 + + def run(self): + + nonce_digits = "0123456789abcedf" + + all_domains = domain_list()["domains"] + for domain in all_domains: + + nonce = ''.join(random.choice(nonce_digits) for i in range(16)) + os.system("rm -rf /tmp/.well-known/ynh-diagnosis/") + os.system("mkdir -p /tmp/.well-known/ynh-diagnosis/") + os.system("touch /tmp/.well-known/ynh-diagnosis/%s" % nonce) + + try: + r = requests.post('https://ynhdiagnoser.netlib.re/check-http', json={'domain': domain, "nonce": nonce}, timeout=30).json() + print(r) + if "status" not in r.keys(): + raise Exception("Bad syntax for response ? Raw json: %s" % str(r)) + elif r["status"] == "error" and ("code" not in r.keys() or r["code"] not in ["error_http_check_connection_error", "error_http_check_unknown_error"]): + if "content" in r.keys(): + raise Exception(r["content"]) + else: + raise Exception("Bad syntax for response ? Raw json: %s" % str(r)) + except Exception as e: + print(e) + raise YunohostError("diagnosis_http_could_not_diagnose", error=e) + + if r["status"] == "ok": + yield dict(meta={"domain": domain}, + status="SUCCESS", + summary=("diagnosis_http_ok", {"domain": domain})) + else: + yield dict(meta={"domain": domain}, + status="ERROR", + summary=("diagnosis_http_unreachable", {"domain": domain})) + + +def main(args, env, loggers): + return HttpDiagnoser(args, env, loggers).diagnose() diff --git a/data/templates/nginx/server.tpl.conf b/data/templates/nginx/server.tpl.conf index 4a5e91557..9acc6c0fd 100644 --- a/data/templates/nginx/server.tpl.conf +++ b/data/templates/nginx/server.tpl.conf @@ -16,6 +16,10 @@ server { return 301 https://$http_host$request_uri; } + location /.well-known/ynh-diagnosis/ { + alias /tmp/.well-known/ynh-diagnosis/; + } + location /.well-known/autoconfig/mail/ { alias /var/www/.well-known/{{ domain }}/autoconfig/mail/; } diff --git a/locales/en.json b/locales/en.json index 0a2204725..ac44122fe 100644 --- a/locales/en.json +++ b/locales/en.json @@ -178,9 +178,13 @@ "diagnosis_description_services": "Services status check", "diagnosis_description_diskusage": "Disk usage", "diagnosis_description_ports": "Ports exposure", + "diagnosis_description_http": "HTTP exposure", "diagnosis_ports_could_not_diagnose": "Could not diagnose if ports are reachable from outside. Error: {error}", "diagnosis_ports_unreachable": "Port {port} is not reachable from outside.", "diagnosis_ports_ok": "Relevant ports are reachable from outside!", + "diagnosis_http_could_not_diagnose": "Could not diagnose if domain is reachable from outside. Error: {error}", + "diagnosis_http_ok": "Domain {domain} is reachable from outside.", + "diagnosis_http_unreachable": "Domain {domain} is unreachable through HTTP from outside.", "domain_cannot_remove_main": "Cannot remove main domain. Set a new main domain first", "domain_cert_gen_failed": "Could not generate certificate", "domain_created": "Domain created", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index e9e6ce14e..b4962d5f6 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1463,7 +1463,8 @@ def app_ssowatconf(): for domain in domains: skipped_urls.extend([domain + '/yunohost/admin', domain + '/yunohost/api']) - # Authorize ACME challenge url + # Authorize ynh remote diagnosis, ACME challenge and mail autoconfig urls + skipped_regex.append("^[^/]*/%.well%-known/ynh%-diagnosis/.*$") skipped_regex.append("^[^/]*/%.well%-known/acme%-challenge/.*$") skipped_regex.append("^[^/]*/%.well%-known/autoconfig/mail/config%-v1%.1%.xml.*$") From 91ec775ebb695b7a4e3a58951b51a6bd343dfc20 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 31 Jul 2019 16:54:25 +0200 Subject: [PATCH 48/94] Implement basic dependency system between diagnoser --- data/hooks/diagnosis/10-ip.py | 1 + data/hooks/diagnosis/12-dnsrecords.py | 1 + data/hooks/diagnosis/14-ports.py | 1 + data/hooks/diagnosis/16-http.py | 3 +-- data/hooks/diagnosis/30-services.py | 1 + data/hooks/diagnosis/50-diskusage.py | 1 + src/yunohost/diagnosis.py | 29 +++++++++++++++++---------- 7 files changed, 24 insertions(+), 13 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index a4cfc0a48..b29076467 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -14,6 +14,7 @@ class IPDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] cache_duration = 60 + dependencies = [] def run(self): diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 0f47ff136..0e8aaa07e 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -13,6 +13,7 @@ class DNSRecordsDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] cache_duration = 3600 * 24 + dependencies = ["ip"] def run(self): diff --git a/data/hooks/diagnosis/14-ports.py b/data/hooks/diagnosis/14-ports.py index 8206474f8..82a44384a 100644 --- a/data/hooks/diagnosis/14-ports.py +++ b/data/hooks/diagnosis/14-ports.py @@ -11,6 +11,7 @@ class PortsDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] cache_duration = 3600 + dependencies = ["ip"] def run(self): diff --git a/data/hooks/diagnosis/16-http.py b/data/hooks/diagnosis/16-http.py index b6b92fc77..cc335df8b 100644 --- a/data/hooks/diagnosis/16-http.py +++ b/data/hooks/diagnosis/16-http.py @@ -13,6 +13,7 @@ class HttpDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] cache_duration = 3600 + dependencies = ["ip"] def run(self): @@ -28,7 +29,6 @@ class HttpDiagnoser(Diagnoser): try: r = requests.post('https://ynhdiagnoser.netlib.re/check-http', json={'domain': domain, "nonce": nonce}, timeout=30).json() - print(r) if "status" not in r.keys(): raise Exception("Bad syntax for response ? Raw json: %s" % str(r)) elif r["status"] == "error" and ("code" not in r.keys() or r["code"] not in ["error_http_check_connection_error", "error_http_check_unknown_error"]): @@ -37,7 +37,6 @@ class HttpDiagnoser(Diagnoser): else: raise Exception("Bad syntax for response ? Raw json: %s" % str(r)) except Exception as e: - print(e) raise YunohostError("diagnosis_http_could_not_diagnose", error=e) if r["status"] == "ok": diff --git a/data/hooks/diagnosis/30-services.py b/data/hooks/diagnosis/30-services.py index 5029e0a5d..6589d83f2 100644 --- a/data/hooks/diagnosis/30-services.py +++ b/data/hooks/diagnosis/30-services.py @@ -20,6 +20,7 @@ class ServicesDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] cache_duration = 300 + dependencies = [] def run(self): diff --git a/data/hooks/diagnosis/50-diskusage.py b/data/hooks/diagnosis/50-diskusage.py index 2c6fe387b..74b8eb4b9 100644 --- a/data/hooks/diagnosis/50-diskusage.py +++ b/data/hooks/diagnosis/50-diskusage.py @@ -8,6 +8,7 @@ class DiskUsageDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] cache_duration = 3600 * 24 + dependencies = [] def run(self): diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index e7aca585f..14b332fe3 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -137,12 +137,7 @@ class Diagnoser(): self.env = env self.args = args or {} self.cache_file = Diagnoser.cache_file(self.id_) - - descr_key = "diagnosis_description_" + self.id_ - self.description = m18n.n(descr_key) - # If no description available, fallback to id - if self.description == descr_key: - self.description = self.id_ + self.description = Diagnoser.get_description(self.id_) def cached_time_ago(self): @@ -159,9 +154,18 @@ class Diagnoser(): if not self.args.get("force", False) and self.cached_time_ago() < self.cache_duration: self.logger_debug("Cache still valid : %s" % self.cache_file) + # FIXME : i18n logger.info("(Cache still valid for %s diagnosis. Not re-diagnosing yet!)" % self.description) return 0, {} + for dependency in self.dependencies: + dep_report = Diagnoser.get_cached_report(dependency) + dep_errors = [item for item in dep_report["items"] if item["status"] == "ERROR"] + if dep_errors: + # FIXME : i18n + logger.error("Can't run diagnosis for %s while there are important issues related to %s." % (self.description, Diagnoser.get_description(dependency))) + return 1, {} + self.logger_debug("Running diagnostic for %s" % self.id_) items = list(self.run()) @@ -200,6 +204,13 @@ class Diagnoser(): Diagnoser.i18n(report) return report + @staticmethod + def get_description(id_): + key = "diagnosis_description_" + id_ + descr = m18n.n(key) + # If no description available, fallback to id + return descr if descr != key else id_ + @staticmethod def i18n(report): @@ -209,11 +220,7 @@ class Diagnoser(): # was generated ... e.g. if the diagnosing happened inside a cron job with locale EN # instead of FR used by the actual admin... - descr_key = "diagnosis_description_" + report["id"] - report["description"] = m18n.n(descr_key) - # If no description available, fallback to id - if report["description"] == descr_key: - report["description"] = report["id"] + report["description"] = Diagnoser.get_description(report["id"]) for item in report["items"]: summary_key, summary_args = item["summary"] From 612a96e1e2410eabf677aec2c75194b477fe3cc0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 31 Jul 2019 17:36:51 +0200 Subject: [PATCH 49/94] Yield one item per port open to be consistent with other diagnosers --- data/hooks/diagnosis/14-ports.py | 11 ++++------- locales/en.json | 2 +- src/yunohost/diagnosis.py | 2 ++ 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/data/hooks/diagnosis/14-ports.py b/data/hooks/diagnosis/14-ports.py index 82a44384a..b953f35a9 100644 --- a/data/hooks/diagnosis/14-ports.py +++ b/data/hooks/diagnosis/14-ports.py @@ -37,18 +37,15 @@ class PortsDiagnoser(Diagnoser): except Exception as e: raise YunohostError("diagnosis_ports_could_not_diagnose", error=e) - found_issues = False for port in ports: if r["ports"].get(str(port), None) is not True: - found_issues = True yield dict(meta={"port": port}, status="ERROR", summary=("diagnosis_ports_unreachable", {"port": port})) - - if not found_issues: - yield dict(meta={}, - status="SUCCESS", - summary=("diagnosis_ports_ok", {})) + else: + yield dict(meta={}, + status="SUCCESS", + summary=("diagnosis_ports_ok", {"port": port})) def main(args, env, loggers): diff --git a/locales/en.json b/locales/en.json index ac44122fe..6dce769f1 100644 --- a/locales/en.json +++ b/locales/en.json @@ -181,7 +181,7 @@ "diagnosis_description_http": "HTTP exposure", "diagnosis_ports_could_not_diagnose": "Could not diagnose if ports are reachable from outside. Error: {error}", "diagnosis_ports_unreachable": "Port {port} is not reachable from outside.", - "diagnosis_ports_ok": "Relevant ports are reachable from outside!", + "diagnosis_ports_ok": "Port {port} is reachable from outside.", "diagnosis_http_could_not_diagnose": "Could not diagnose if domain is reachable from outside. Error: {error}", "diagnosis_http_ok": "Domain {domain} is reachable from outside.", "diagnosis_http_unreachable": "Domain {domain} is unreachable through HTTP from outside.", diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 14b332fe3..88316a15f 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -116,8 +116,10 @@ def diagnosis_run(categories=[], force=False): if issues: if msettings.get("interface") == "api": + # FIXME: i18n logger.info("You can go to the Diagnosis section (in the home screen) to see the issues found.") else: + # FIXME: i18n logger.info("You can run 'yunohost diagnosis show --issues' to display the issues found.") return From 0dc1909c68b6ef594b992764711c81f0ec169ad1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 31 Jul 2019 20:16:22 +0200 Subject: [PATCH 50/94] Misc small UX stuff --- data/hooks/diagnosis/10-ip.py | 3 ++- data/hooks/diagnosis/12-dnsrecords.py | 2 +- locales/en.json | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index b29076467..8c8dbe95b 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -57,7 +57,8 @@ class IPDiagnoser(Diagnoser): elif not good_resolvconf: yield dict(meta={"test": "dnsresolv"}, status="WARNING", - summary=("diagnosis_ip_weird_resolvconf", {})) + summary=("diagnosis_ip_weird_resolvconf", {}), + details=[("diagnosis_ip_weird_resolvconf_details", ())]) else: yield dict(meta={"test": "dnsresolv"}, status="SUCCESS", diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 0e8aaa07e..b59ffbd54 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -52,7 +52,7 @@ class DNSRecordsDiagnoser(Diagnoser): discrepancies.append(("diagnosis_dns_discrepancy", (r["type"], r["name"], expected_value, current_value))) if discrepancies: - status = "ERROR" if (category == "basic" or is_main_domain) else "WARNING" + status = "ERROR" if (category == "basic" or (is_main_domain and category != "extra")) else "WARNING" summary = ("diagnosis_dns_bad_conf", {"domain": domain, "category": category}) else: status = "SUCCESS" diff --git a/locales/en.json b/locales/en.json index 6dce769f1..65b3ef64d 100644 --- a/locales/en.json +++ b/locales/en.json @@ -163,9 +163,10 @@ "diagnosis_ip_dnsresolution_working": "Domain name resolution is working!", "diagnosis_ip_broken_dnsresolution": "Domain name resolution seems to be broken for some reason ... Is a firewall blocking DNS requests ?", "diagnosis_ip_broken_resolvconf": "Domain name resolution seems to be broken on your server, which seems related to /etc/resolv.conf not pointing to 127.0.0.1.", - "diagnosis_ip_weird_resolvconf": "Be careful that you seem to be using a custom /etc/resolv.conf. Instead, this file should be a symlink to /etc/resolvconf/run/resolv.conf itself pointing to 127.0.0.1 (dnsmasq).", + "diagnosis_ip_weird_resolvconf": "DNS resolution seems to be working, but be careful that you seem to be using a custom /etc/resolv.conf.", + "diagnosis_ip_weird_resolvconf_details": "Instead, this file should be a symlink to /etc/resolvconf/run/resolv.conf itself pointing to 127.0.0.1 (dnsmasq). The actual resolvers should be configured via /etc/resolv.dnsmasq.conf.", "diagnosis_dns_good_conf": "Good DNS configuration for domain {domain} (category {category})", - "diagnosis_dns_bad_conf": "Bad DNS configuration for domain {domain} (category {category})", + "diagnosis_dns_bad_conf": "Bad / missing DNS configuration for domain {domain} (category {category})", "diagnosis_dns_missing_record": "According to the recommended DNS configuration, you should add a DNS record with type {0}, name {1} and value {2}", "diagnosis_dns_discrepancy": "According to the recommended DNS configuration, the value for the DNS record with type {0} and name {1} should be {2}, not {3}.", "diagnosis_services_good_status": "Service {service} is {status} as expected!", From 4cbd1b06c2d572c908a3d7a6d4e82b6738f7a3da Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 31 Jul 2019 21:25:44 +0200 Subject: [PATCH 51/94] Add a regenconf diagnoser to report manually modified files... --- data/hooks/diagnosis/70-regenconf.py | 42 +++++++++++++++++++++++++++ locales/en.json | 5 ++++ src/yunohost/regenconf.py | 43 ++++++++++++++-------------- 3 files changed, 69 insertions(+), 21 deletions(-) create mode 100644 data/hooks/diagnosis/70-regenconf.py diff --git a/data/hooks/diagnosis/70-regenconf.py b/data/hooks/diagnosis/70-regenconf.py new file mode 100644 index 000000000..94c41feb5 --- /dev/null +++ b/data/hooks/diagnosis/70-regenconf.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +import os + +from yunohost.diagnosis import Diagnoser +from yunohost.regenconf import manually_modified_files, manually_modified_files_compared_to_debian_default + + +class RegenconfDiagnoser(Diagnoser): + + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] + cache_duration = 300 + dependencies = [] + + def run(self): + + regenconf_modified_files = manually_modified_files() + debian_modified_files = manually_modified_files_compared_to_debian_default(ignore_handled_by_regenconf=True) + + if regenconf_modified_files == []: + yield dict(meta={"test": "regenconf"}, + status="SUCCESS", + summary=("diagnosis_regenconf_allgood", {}) + ) + else: + for f in regenconf_modified_files: + yield dict(meta={"test": "regenconf", "file": f}, + status="WARNING", + summary=("diagnosis_regenconf_manually_modified", {"file": f}), + details=[("diagnosis_regenconf_manually_modified_details", {})] + ) + + for f in debian_modified_files: + yield dict(meta={"test": "debian", "file": f}, + status="WARNING", + summary=("diagnosis_regenconf_manually_modified_debian", {"file": f}), + details=[("diagnosis_regenconf_manually_modified_debian_details", {})] + ) + + +def main(args, env, loggers): + return RegenconfDiagnoser(args, env, loggers).diagnose() diff --git a/locales/en.json b/locales/en.json index 65b3ef64d..105891571 100644 --- a/locales/en.json +++ b/locales/en.json @@ -174,6 +174,11 @@ "diagnosis_diskusage_verylow": "Storage {mountpoint} (on device {device}) has only {free_percent}% space remaining. You should really consider cleaning up some space.", "diagnosis_diskusage_low": "Storage {mountpoint} (on device {device}) has only {free_percent}% space remaining. Be careful", "diagnosis_diskusage_ok": "Storage {mountpoint} (on device {device}) still has {free_percent}% space left!", + "diagnosis_regenconf_allgood": "All configurations files are in line with the recommended configuration!", + "diagnosis_regenconf_manually_modified": "Configuration file {file} was manually modified.", + "diagnosis_regenconf_manually_modified_details": "This is probably OK as long as you know what you're doing ;) !", + "diagnosis_regenconf_manually_modified_debian": "Configuration file {file} was manually modified compared to Debian's default.", + "diagnosis_regenconf_manually_modified_debian_details": "This may probably be OK, but gotta keep an eye on it...", "diagnosis_description_ip": "Internet connectivity", "diagnosis_description_dnsrecords": "DNS records", "diagnosis_description_services": "Services status check", diff --git a/src/yunohost/regenconf.py b/src/yunohost/regenconf.py index b7a42dd9d..b09824d58 100644 --- a/src/yunohost/regenconf.py +++ b/src/yunohost/regenconf.py @@ -525,31 +525,32 @@ def _process_regen_conf(system_conf, new_conf=None, save=True): def manually_modified_files(): - # We do this to have --quiet, i.e. don't throw a whole bunch of logs - # just to fetch this... - # Might be able to optimize this by looking at what the regen conf does - # and only do the part that checks file hashes... - cmd = "yunohost tools regen-conf --dry-run --output-as json --quiet" - j = json.loads(subprocess.check_output(cmd.split())) - - # j is something like : - # {"postfix": {"applied": {}, "pending": {"/etc/postfix/main.cf": {"status": "modified"}}} - output = [] - for app, actions in j.items(): - for action, files in actions.items(): - for filename, infos in files.items(): - if infos["status"] == "modified": - output.append(filename) + regenconf_categories = _get_regenconf_infos() + for category, infos in regenconf_categories.items(): + conffiles = infos["conffiles"] + for path, hash_ in conffiles.items(): + if hash_ != _calculate_hash(path): + output.append(path) return output -def manually_modified_files_compared_to_debian_default(): +def manually_modified_files_compared_to_debian_default(ignore_handled_by_regenconf=False): # from https://serverfault.com/a/90401 - r = subprocess.check_output("dpkg-query -W -f='${Conffiles}\n' '*' \ - | awk 'OFS=\" \"{print $2,$1}' \ - | md5sum -c 2>/dev/null \ - | awk -F': ' '$2 !~ /OK/{print $1}'", shell=True) - return r.strip().split("\n") + files = subprocess.check_output("dpkg-query -W -f='${Conffiles}\n' '*' \ + | awk 'OFS=\" \"{print $2,$1}' \ + | md5sum -c 2>/dev/null \ + | awk -F': ' '$2 !~ /OK/{print $1}'", shell=True) + files = files.strip().split("\n") + + if ignore_handled_by_regenconf: + regenconf_categories = _get_regenconf_infos() + regenconf_files = [] + for infos in regenconf_categories.values(): + regenconf_files.extend(infos["conffiles"].keys()) + + files = [f for f in files if f not in regenconf_files] + + return files From cee3b4de27dd03fb58f3d4400592c4bf0ec3e017 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 31 Jul 2019 22:04:55 +0200 Subject: [PATCH 52/94] Add nginx -t check to regenconf diagnoser --- data/hooks/diagnosis/70-regenconf.py | 14 ++++++++++++++ locales/en.json | 2 ++ 2 files changed, 16 insertions(+) diff --git a/data/hooks/diagnosis/70-regenconf.py b/data/hooks/diagnosis/70-regenconf.py index 94c41feb5..105d43fa3 100644 --- a/data/hooks/diagnosis/70-regenconf.py +++ b/data/hooks/diagnosis/70-regenconf.py @@ -2,6 +2,7 @@ import os +import subprocess from yunohost.diagnosis import Diagnoser from yunohost.regenconf import manually_modified_files, manually_modified_files_compared_to_debian_default @@ -14,6 +15,19 @@ class RegenconfDiagnoser(Diagnoser): def run(self): + # nginx -t + p = subprocess.Popen("nginx -t".split(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + out, _ = p.communicate() + + if p.returncode != 0: + yield dict(meta={"test": "nginx-t"}, + status="ERROR", + summary=("diagnosis_regenconf_nginx_conf_broken", {}), + details=[(out, ())] + ) + regenconf_modified_files = manually_modified_files() debian_modified_files = manually_modified_files_compared_to_debian_default(ignore_handled_by_regenconf=True) diff --git a/locales/en.json b/locales/en.json index 105891571..f06310679 100644 --- a/locales/en.json +++ b/locales/en.json @@ -179,12 +179,14 @@ "diagnosis_regenconf_manually_modified_details": "This is probably OK as long as you know what you're doing ;) !", "diagnosis_regenconf_manually_modified_debian": "Configuration file {file} was manually modified compared to Debian's default.", "diagnosis_regenconf_manually_modified_debian_details": "This may probably be OK, but gotta keep an eye on it...", + "diagnosis_regenconf_nginx_conf_broken": "The nginx configuration appears to be broken!", "diagnosis_description_ip": "Internet connectivity", "diagnosis_description_dnsrecords": "DNS records", "diagnosis_description_services": "Services status check", "diagnosis_description_diskusage": "Disk usage", "diagnosis_description_ports": "Ports exposure", "diagnosis_description_http": "HTTP exposure", + "diagnosis_description_regenconf": "System configurations", "diagnosis_ports_could_not_diagnose": "Could not diagnose if ports are reachable from outside. Error: {error}", "diagnosis_ports_unreachable": "Port {port} is not reachable from outside.", "diagnosis_ports_ok": "Port {port} is reachable from outside.", From b81cd4fc68ceb0ed63ff9526958bef60c558e1ab Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 31 Jul 2019 22:10:00 +0200 Subject: [PATCH 53/94] Add security diagnoser with meltdown checks --- data/hooks/diagnosis/90-security.py | 98 +++++++++++++++++++++++++++++ locales/en.json | 4 ++ 2 files changed, 102 insertions(+) create mode 100644 data/hooks/diagnosis/90-security.py diff --git a/data/hooks/diagnosis/90-security.py b/data/hooks/diagnosis/90-security.py new file mode 100644 index 000000000..0b1b61226 --- /dev/null +++ b/data/hooks/diagnosis/90-security.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python + +import os +import json +import subprocess + +from yunohost.diagnosis import Diagnoser +from moulinette.utils.filesystem import read_json, write_to_json + + +class SecurityDiagnoser(Diagnoser): + + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] + cache_duration = 3600 + dependencies = [] + + def run(self): + + "CVE-2017-5754" + + if self.is_vulnerable_to_meltdown(): + yield dict(meta={"test": "meltdown"}, + status="ERROR", + summary=("diagnosis_security_vulnerable_to_meltdown", {}), + details=[("diagnosis_security_vulnerable_to_meltdown_details", ())] + ) + else: + yield dict(meta={}, + status="SUCCESS", + summary=("diagnosis_security_all_good", {}) + ) + + + def is_vulnerable_to_meltdown(self): + # meltdown CVE: https://security-tracker.debian.org/tracker/CVE-2017-5754 + + # We use a cache file to avoid re-running the script so many times, + # which can be expensive (up to around 5 seconds on ARM) + # and make the admin appear to be slow (c.f. the calls to diagnosis + # from the webadmin) + # + # The cache is in /tmp and shall disappear upon reboot + # *or* we compare it to dpkg.log modification time + # such that it's re-ran if there was package upgrades + # (e.g. from yunohost) + cache_file = "/tmp/yunohost-meltdown-diagnosis" + dpkg_log = "/var/log/dpkg.log" + if os.path.exists(cache_file): + if not os.path.exists(dpkg_log) or os.path.getmtime(cache_file) > os.path.getmtime(dpkg_log): + self.logger_debug("Using cached results for meltdown checker, from %s" % cache_file) + return read_json(cache_file)[0]["VULNERABLE"] + + # script taken from https://github.com/speed47/spectre-meltdown-checker + # script commit id is store directly in the script + SCRIPT_PATH = "/usr/lib/moulinette/yunohost/vendor/spectre-meltdown-checker/spectre-meltdown-checker.sh" + + # '--variant 3' corresponds to Meltdown + # example output from the script: + # [{"NAME":"MELTDOWN","CVE":"CVE-2017-5754","VULNERABLE":false,"INFOS":"PTI mitigates the vulnerability"}] + try: + self.logger_debug("Running meltdown vulnerability checker") + call = subprocess.Popen("bash %s --batch json --variant 3" % + SCRIPT_PATH, shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + # TODO / FIXME : here we are ignoring error messages ... + # in particular on RPi2 and other hardware, the script complains about + # "missing some kernel info (see -v), accuracy might be reduced" + # Dunno what to do about that but we probably don't want to harass + # users with this warning ... + output, err = call.communicate() + assert call.returncode in (0, 2, 3), "Return code: %s" % call.returncode + + # If there are multiple lines, sounds like there was some messages + # in stdout that are not json >.> ... Try to get the actual json + # stuff which should be the last line + output = output.strip() + if "\n" in output: + self.logger_debug("Original meltdown checker output : %s" % output) + output = output.split("\n")[-1] + + CVEs = json.loads(output) + assert len(CVEs) == 1 + assert CVEs[0]["NAME"] == "MELTDOWN" + except Exception as e: + import traceback + traceback.print_exc() + self.logger_warning("Something wrong happened when trying to diagnose Meltdown vunerability, exception: %s" % e) + raise Exception("Command output for failed meltdown check: '%s'" % output) + + self.logger_debug("Writing results from meltdown checker to cache file, %s" % cache_file) + write_to_json(cache_file, CVEs) + return CVEs[0]["VULNERABLE"] + + +def main(args, env, loggers): + return SecurityDiagnoser(args, env, loggers).diagnose() diff --git a/locales/en.json b/locales/en.json index f06310679..c4330b08a 100644 --- a/locales/en.json +++ b/locales/en.json @@ -180,6 +180,9 @@ "diagnosis_regenconf_manually_modified_debian": "Configuration file {file} was manually modified compared to Debian's default.", "diagnosis_regenconf_manually_modified_debian_details": "This may probably be OK, but gotta keep an eye on it...", "diagnosis_regenconf_nginx_conf_broken": "The nginx configuration appears to be broken!", + "diagnosis_security_all_good": "No critical security vulnerability was found.", + "diagnosis_security_vulnerable_to_meltdown": "You appear vulnerable to the Meltdown criticial security vulnerability", + "diagnosis_security_vulnerable_to_meltdown_details": "To fix this, you should upgrade your system and reboot to load the new linux kernel (or contact your server provider if this doesn't work). See https://meltdownattack.com/ for more infos.", "diagnosis_description_ip": "Internet connectivity", "diagnosis_description_dnsrecords": "DNS records", "diagnosis_description_services": "Services status check", @@ -187,6 +190,7 @@ "diagnosis_description_ports": "Ports exposure", "diagnosis_description_http": "HTTP exposure", "diagnosis_description_regenconf": "System configurations", + "diagnosis_description_security": "Security checks", "diagnosis_ports_could_not_diagnose": "Could not diagnose if ports are reachable from outside. Error: {error}", "diagnosis_ports_unreachable": "Port {port} is not reachable from outside.", "diagnosis_ports_ok": "Port {port} is reachable from outside.", From 0c232b6cb5eb397f09d4d5024218b687a5cfcf46 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 1 Aug 2019 19:22:01 +0200 Subject: [PATCH 54/94] Implement diagnosis show --share --- data/actionsmap/yunohost.yml | 3 +++ src/yunohost/diagnosis.py | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 3d72bb57a..20eb8a0f8 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1887,6 +1887,9 @@ diagnosis: --issues: help: Only display issues action: store_true + --share: + help: Share the logs using yunopaste + action: store_true run: action_help: Show most recents diagnosis results diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 88316a15f..99a798b91 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -44,7 +44,7 @@ def diagnosis_list(): return {"categories": all_categories_names} -def diagnosis_show(categories=[], issues=False, full=False): +def diagnosis_show(categories=[], issues=False, full=False, share=False): # Get all the categories all_categories = _list_diagnosis_categories() @@ -81,7 +81,35 @@ def diagnosis_show(categories=[], issues=False, full=False): all_reports.append(report) - return {"reports": all_reports} + if share: + from yunohost.utils.yunopaste import yunopaste + content = _dump_human_readable_reports(all_reports) + url = yunopaste(content) + + logger.info(m18n.n("log_available_on_yunopaste", url=url)) + if msettings.get('interface') == 'api': + return {"url": url} + else: + return + else: + return {"reports": all_reports} + +def _dump_human_readable_reports(reports): + + output = "" + + for report in reports: + output += "=================================\n" + output += "{description} ({id})\n".format(**report) + output += "=================================\n\n" + for item in report["items"]: + output += "[{status}] {summary}\n".format(**item) + for detail in item.get("details", []): + output += " - " + detail + "\n" + output += "\n" + output += "\n\n" + + return(output) def diagnosis_run(categories=[], force=False): From c4ba8534c5dbc7b214afaedbae5b704a6bcf4339 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 1 Aug 2019 19:54:46 +0200 Subject: [PATCH 55/94] Implement i18n stuff --- locales/en.json | 9 +++++++++ src/yunohost/diagnosis.py | 25 ++++++++++--------------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/locales/en.json b/locales/en.json index c4330b08a..979edbbef 100644 --- a/locales/en.json +++ b/locales/en.json @@ -155,6 +155,15 @@ "diagnosis_no_apps": "No installed application", "dpkg_is_broken": "You cannot do this right now because dpkg/APT (the system package managers) seems to be in a broken state… You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.", "dpkg_lock_not_available": "This command can't be ran right now because another program seems to be using the lock of dpkg (the system package manager)", + "diagnosis_display_tip_web": "You can go to the Diagnosis section (in the home screen) to see the issues found.", + "diagnosis_display_tip_cli": "You can run 'yunohost diagnosis show --issues' to display the issues found.", + "diagnosis_failed_for_category": "Diagnosis failed for category '{category}' : {error}", + "diagnosis_cache_still_valid": "(Cache still valid for {category} diagnosis. Not re-diagnosing yet!)", + "diagnosis_cant_run_because_of_dep": "Can't run diagnosis for {category} while there are important issues related to {dep}.", + "diagnosis_found_issues": "Found {errors} significant issue(s) related to {category}!", + "diagnosis_found_warnings": "Found {warnings} item(s) that could be improved for {category}.", + "diagnosis_everything_ok": "Everything looks good for {category}!", + "diagnosis_failed": "Failed to fetch diagnosis result for category '{category}' : {error}", "diagnosis_ip_connected_ipv4": "The server is connected to the Internet through IPv4 !", "diagnosis_ip_no_ipv4": "The server does not have a working IPv4.", "diagnosis_ip_connected_ipv6": "The server is connected to the Internet through IPv6 !", diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 99a798b91..6cf207282 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -64,7 +64,7 @@ def diagnosis_show(categories=[], issues=False, full=False, share=False): try: report = Diagnoser.get_cached_report(category) except Exception as e: - logger.error("Failed to fetch diagnosis result for category '%s' : %s" % (category, str(e))) # FIXME : i18n + logger.error(m18n.n("diagnosis_failed", category=category, error=str(e))) else: if not full: del report["timestamp"] @@ -136,7 +136,7 @@ def diagnosis_run(categories=[], force=False): try: code, report = hook_exec(path, args={"force": force}, env=None) except Exception as e: - logger.error("Diagnosis failed for category '%s' : %s" % (category, str(e)), exc_info=True) # FIXME : i18n + logger.error(m18n.n("diagnosis_failed_for_category", category=category, error=str(e)), exc_info=True) else: diagnosed_categories.append(category) if report != {}: @@ -144,11 +144,9 @@ def diagnosis_run(categories=[], force=False): if issues: if msettings.get("interface") == "api": - # FIXME: i18n - logger.info("You can go to the Diagnosis section (in the home screen) to see the issues found.") + logger.info(m18n.n("diagnosis_display_tip_web")) else: - # FIXME: i18n - logger.info("You can run 'yunohost diagnosis show --issues' to display the issues found.") + logger.info(m18n.n("diagnosis_display_tip_cli")) return @@ -163,6 +161,7 @@ class Diagnoser(): def __init__(self, args, env, loggers): + # FIXME ? That stuff with custom loggers is weird ... (mainly inherited from the bash hooks, idk) self.logger_debug, self.logger_warning, self.logger_info = loggers self.env = env self.args = args or {} @@ -184,16 +183,14 @@ class Diagnoser(): if not self.args.get("force", False) and self.cached_time_ago() < self.cache_duration: self.logger_debug("Cache still valid : %s" % self.cache_file) - # FIXME : i18n - logger.info("(Cache still valid for %s diagnosis. Not re-diagnosing yet!)" % self.description) + logger.info(m18n.n("diagnosis_cache_still_valid", category=self.description)) return 0, {} for dependency in self.dependencies: dep_report = Diagnoser.get_cached_report(dependency) dep_errors = [item for item in dep_report["items"] if item["status"] == "ERROR"] if dep_errors: - # FIXME : i18n - logger.error("Can't run diagnosis for %s while there are important issues related to %s." % (self.description, Diagnoser.get_description(dependency))) + logger.error(m18n.n("diagnosis_cant_run_because_of_dep", category=self.description, dep=Diagnoser.get_description(dependency))) return 1, {} self.logger_debug("Running diagnostic for %s" % self.id_) @@ -204,7 +201,6 @@ class Diagnoser(): "cached_for": self.cache_duration, "items": items} - # TODO / FIXME : should handle the case where we only did a partial diagnosis self.logger_debug("Updating cache %s" % self.cache_file) self.write_cache(new_report) Diagnoser.i18n(new_report) @@ -212,13 +208,12 @@ class Diagnoser(): errors = [item for item in new_report["items"] if item["status"] == "ERROR"] warnings = [item for item in new_report["items"] if item["status"] == "WARNING"] - # FIXME : i18n if errors: - logger.error("Found %s significant issue(s) related to %s!" % (len(errors), new_report["description"])) + logger.error(m18n.n("diagnosis_found_issues", errors=len(errors), category=new_report["description"])) elif warnings: - logger.warning("Found %s item(s) that could be improved for %s." % (len(warnings), new_report["description"])) + logger.warning(m18n.n("diagnosis_found_warnings", warnings=len(warnings), category=new_report["description"])) else: - logger.success("Everything looks good for %s!" % new_report["description"]) + logger.success(m18n.n("diagnosis_everything_ok", category=new_report["description"])) return 0, new_report From 33180d0947118c53f7e6c96da2882483a9be6df9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 1 Aug 2019 21:11:13 +0200 Subject: [PATCH 56/94] Add base system diagnostic --- data/hooks/diagnosis/00-basesystem.py | 54 +++++++++++++++++++++++++++ locales/en.json | 6 +++ src/yunohost/diagnosis.py | 4 +- 3 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 data/hooks/diagnosis/00-basesystem.py diff --git a/data/hooks/diagnosis/00-basesystem.py b/data/hooks/diagnosis/00-basesystem.py new file mode 100644 index 000000000..8fa90e65e --- /dev/null +++ b/data/hooks/diagnosis/00-basesystem.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python + +import os + +from moulinette.utils.filesystem import read_file +from yunohost.diagnosis import Diagnoser +from yunohost.utils.packages import ynh_packages_version + + +class BaseSystemDiagnoser(Diagnoser): + + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] + cache_duration = 3600 * 24 + dependencies = [] + + def run(self): + + # Kernel version + kernel_version = read_file('/proc/sys/kernel/osrelease').strip() + yield dict(meta={"test": "kernel"}, + status="INFO", + summary=("diagnosis_basesystem_kernel", {"kernel_version": kernel_version})) + + # Debian release + debian_version = read_file("/etc/debian_version").strip() + yield dict(meta={"test": "host"}, + status="INFO", + summary=("diagnosis_basesystem_host", {"debian_version": debian_version})) + + # Yunohost packages versions + ynh_packages = ynh_packages_version() + # We check if versions are consistent (e.g. all 3.6 and not 3 packages with 3.6 and the other with 3.5) + # This is a classical issue for upgrades that failed in the middle + # (or people upgrading half of the package because they did 'apt upgrade' instead of 'dist-upgrade') + # Here, ynh_core_version is for example "3.5.4.12", so [:3] is "3.5" and we check it's the same for all packages + ynh_core_version = ynh_packages["yunohost"]["version"] + consistent_versions = all(infos["version"][:3] == ynh_core_version[:3] for infos in ynh_packages.values()) + ynh_version_details = [("diagnosis_basesystem_ynh_single_version", (package, infos["version"])) + for package, infos in ynh_packages.items()] + + if consistent_versions: + yield dict(meta={"test": "ynh_versions"}, + status="INFO", + summary=("diagnosis_basesystem_ynh_main_version", {"main_version": ynh_core_version[:3]}), + details=ynh_version_details) + else: + yield dict(meta={"test": "ynh_versions"}, + status="ERROR", + summary=("diagnosis_basesystem_ynh_inconsistent_versions", {}), + details=ynh_version_details) + + +def main(args, env, loggers): + return BaseSystemDiagnoser(args, env, loggers).diagnose() diff --git a/locales/en.json b/locales/en.json index 979edbbef..f942d3dc4 100644 --- a/locales/en.json +++ b/locales/en.json @@ -150,6 +150,11 @@ "custom_appslist_name_required": "You must provide a name for your custom app list", "diagnosis_debian_version_error": "Could not retrieve the Debian version: {error}", "diagnosis_kernel_version_error": "Could not retrieve kernel version: {error}", + "diagnosis_basesystem_host": "Server is running Debian {debian_version}.", + "diagnosis_basesystem_kernel": "Server is running Linux kernel {kernel_version}", + "diagnosis_basesystem_ynh_single_version": "{0} version: {1}", + "diagnosis_basesystem_ynh_main_version": "Server is running YunoHost {main_version}", + "diagnosis_basesystem_ynh_inconsistent_versions": "You are running inconsistents versions of the YunoHost packages ... most probably because of a failed or partial upgrade.", "diagnosis_monitor_disk_error": "Could not monitor disks: {error}", "diagnosis_monitor_system_error": "Could not monitor system: {error}", "diagnosis_no_apps": "No installed application", @@ -192,6 +197,7 @@ "diagnosis_security_all_good": "No critical security vulnerability was found.", "diagnosis_security_vulnerable_to_meltdown": "You appear vulnerable to the Meltdown criticial security vulnerability", "diagnosis_security_vulnerable_to_meltdown_details": "To fix this, you should upgrade your system and reboot to load the new linux kernel (or contact your server provider if this doesn't work). See https://meltdownattack.com/ for more infos.", + "diagnosis_description_basesystem": "Base system", "diagnosis_description_ip": "Internet connectivity", "diagnosis_description_dnsrecords": "DNS records", "diagnosis_description_services": "Services status check", diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 6cf207282..b9fe111ed 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -74,7 +74,7 @@ def diagnosis_show(categories=[], issues=False, full=False, share=False): if "data" in item: del item["data"] if issues: - report["items"] = [item for item in report["items"] if item["status"] != "SUCCESS"] + report["items"] = [item for item in report["items"] if item["status"] in ["WARNING", "ERROR"]] # Ignore this category if no issue was found if not report["items"]: continue @@ -140,7 +140,7 @@ def diagnosis_run(categories=[], force=False): else: diagnosed_categories.append(category) if report != {}: - issues.extend([item for item in report["items"] if item["status"] != "SUCCESS"]) + issues.extend([item for item in report["items"] if item["status"] in ["WARNING", "ERROR"]]) if issues: if msettings.get("interface") == "api": From 47c7c72455bbcfaded9a5da0d753ce961511c628 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 1 Aug 2019 21:53:31 +0200 Subject: [PATCH 57/94] Add RAM and swap diagnosis + improve message for disk usage --- data/hooks/diagnosis/50-diskusage.py | 39 ---------- data/hooks/diagnosis/50-systemresources.py | 87 ++++++++++++++++++++++ locales/en.json | 14 +++- 3 files changed, 97 insertions(+), 43 deletions(-) delete mode 100644 data/hooks/diagnosis/50-diskusage.py create mode 100644 data/hooks/diagnosis/50-systemresources.py diff --git a/data/hooks/diagnosis/50-diskusage.py b/data/hooks/diagnosis/50-diskusage.py deleted file mode 100644 index 74b8eb4b9..000000000 --- a/data/hooks/diagnosis/50-diskusage.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python -import os -import psutil - -from yunohost.diagnosis import Diagnoser - -class DiskUsageDiagnoser(Diagnoser): - - id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] - cache_duration = 3600 * 24 - dependencies = [] - - def run(self): - - disk_partitions = psutil.disk_partitions() - - for disk_partition in disk_partitions: - device = disk_partition.device - mountpoint = disk_partition.mountpoint - - usage = psutil.disk_usage(mountpoint) - free_Go = usage.free / (1024 ** 3) - free_percent = 100 - usage.percent - - item = dict(meta={"mountpoint": mountpoint, "device": device}) - if free_Go < 1 or free_percent < 5: - item["status"] = "ERROR" - item["summary"] = ("diagnosis_diskusage_verylow", {"mountpoint": mountpoint, "device": device, "free_percent": free_percent}) - elif free_Go < 2 or free_percent < 10: - item["status"] = "WARNING" - item["summary"] = ("diagnosis_diskusage_low", {"mountpoint": mountpoint, "device": device, "free_percent": free_percent}) - else: - item["status"] = "SUCCESS" - item["summary"] = ("diagnosis_diskusage_ok", {"mountpoint": mountpoint, "device": device, "free_percent": free_percent}) - - yield item - -def main(args, env, loggers): - return DiskUsageDiagnoser(args, env, loggers).diagnose() diff --git a/data/hooks/diagnosis/50-systemresources.py b/data/hooks/diagnosis/50-systemresources.py new file mode 100644 index 000000000..3399c4682 --- /dev/null +++ b/data/hooks/diagnosis/50-systemresources.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +import os +import psutil + +from yunohost.diagnosis import Diagnoser + +class SystemResourcesDiagnoser(Diagnoser): + + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] + cache_duration = 3600 * 24 + dependencies = [] + + def run(self): + + # + # RAM + # + + ram = psutil.virtual_memory() + ram_total_abs_MB = ram.total / (1024**2) + ram_available_abs_MB = ram.available / (1024**2) + ram_available_percent = round(100 * ram.available / ram.total) + item = dict(meta={"test": "ram"}) + infos = {"total_abs_MB": ram_total_abs_MB, "available_abs_MB": ram_available_abs_MB, "available_percent": ram_available_percent} + if ram_available_abs_MB < 100 or ram_available_percent < 5: + item["status"] = "ERROR" + item["summary"] = ("diagnosis_ram_verylow", infos) + elif ram_available_abs_MB < 200 or ram_available_percent < 10: + item["status"] = "WARNING" + item["summary"] = ("diagnosis_ram_low", infos) + else: + item["status"] = "SUCCESS" + item["summary"] = ("diagnosis_ram_ok", infos) + print(item) + yield item + + # + # Swap + # + + swap = psutil.swap_memory() + swap_total_abs_MB = swap.total / (1024*1024) + item = dict(meta={"test": "swap"}) + infos = {"total_MB": swap_total_abs_MB} + if swap_total_abs_MB <= 0: + item["status"] = "ERROR" + item["summary"] = ("diagnosis_swap_none", infos) + elif swap_total_abs_MB <= 256: + item["status"] = "WARNING" + item["summary"] = ("diagnosis_swap_notsomuch", infos) + else: + item["status"] = "SUCCESS" + item["summary"] = ("diagnosis_swap_ok", infos) + print(item) + yield item + + # + # Disks usage + # + + disk_partitions = psutil.disk_partitions() + + for disk_partition in disk_partitions: + device = disk_partition.device + mountpoint = disk_partition.mountpoint + + usage = psutil.disk_usage(mountpoint) + free_abs_GB = usage.free / (1024 ** 3) + free_percent = 100 - usage.percent + + item = dict(meta={"mountpoint": mountpoint, "device": device}) + infos = {"mountpoint": mountpoint, "device": device, "free_abs_GB": free_abs_GB, "free_percent": free_percent} + if free_abs_GB < 1 or free_percent < 5: + item["status"] = "ERROR" + item["summary"] = ("diagnosis_diskusage_verylow", infos) + elif free_abs_GB < 2 or free_percent < 10: + item["status"] = "WARNING" + item["summary"] = ("diagnosis_diskusage_low", infos) + else: + item["status"] = "SUCCESS" + item["summary"] = ("diagnosis_diskusage_ok", infos) + + yield item + + +def main(args, env, loggers): + return SystemResourcesDiagnoser(args, env, loggers).diagnose() diff --git a/locales/en.json b/locales/en.json index f942d3dc4..40edb1425 100644 --- a/locales/en.json +++ b/locales/en.json @@ -185,9 +185,15 @@ "diagnosis_dns_discrepancy": "According to the recommended DNS configuration, the value for the DNS record with type {0} and name {1} should be {2}, not {3}.", "diagnosis_services_good_status": "Service {service} is {status} as expected!", "diagnosis_services_bad_status": "Service {service} is {status} :/", - "diagnosis_diskusage_verylow": "Storage {mountpoint} (on device {device}) has only {free_percent}% space remaining. You should really consider cleaning up some space.", - "diagnosis_diskusage_low": "Storage {mountpoint} (on device {device}) has only {free_percent}% space remaining. Be careful", - "diagnosis_diskusage_ok": "Storage {mountpoint} (on device {device}) still has {free_percent}% space left!", + "diagnosis_diskusage_verylow": "Storage {mountpoint} (on device {device}) has only {free_abs_GB} GB ({free_percent}%) space remaining. You should really consider cleaning up some space.", + "diagnosis_diskusage_low": "Storage {mountpoint} (on device {device}) has only {free_abs_GB} GB ({free_percent}%) space remaining. Be careful.", + "diagnosis_diskusage_ok": "Storage {mountpoint} (on device {device}) still has {free_abs_GB} GB ({free_percent}%) space left!", + "diagnosis_ram_verylow": "The system has only {available_abs_MB} MB ({available_percent}%) RAM left! (out of {total_abs_MB} MB)", + "diagnosis_ram_low": "The system has {available_abs_MB} MB ({available_percent}%) RAM left out of {total_abs_MB} MB. Be careful.", + "diagnosis_ram_ok": "The system still has {available_abs_MB} MB ({available_percent}%) RAM left out of {total_abs_MB} MB.", + "diagnosis_swap_none": "The system has no swap at all. You should consider adding at least 256 MB of swap to avoid situations where the system runs out of memory.", + "diagnosis_swap_notsomuch": "The system has only {total_MB} MB swap. You should consider having at least 256 MB to avoid situations where the system runs out of memory.", + "diagnosis_swap_ok": "The system has {total_MB} MB of swap!", "diagnosis_regenconf_allgood": "All configurations files are in line with the recommended configuration!", "diagnosis_regenconf_manually_modified": "Configuration file {file} was manually modified.", "diagnosis_regenconf_manually_modified_details": "This is probably OK as long as you know what you're doing ;) !", @@ -201,7 +207,7 @@ "diagnosis_description_ip": "Internet connectivity", "diagnosis_description_dnsrecords": "DNS records", "diagnosis_description_services": "Services status check", - "diagnosis_description_diskusage": "Disk usage", + "diagnosis_description_systemresources": "System resources", "diagnosis_description_ports": "Ports exposure", "diagnosis_description_http": "HTTP exposure", "diagnosis_description_regenconf": "System configurations", From 94f3557aeb3c6d2c95728692d59f027a2f7fe793 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 1 Aug 2019 22:02:08 +0200 Subject: [PATCH 58/94] Remove old 'tools diagnosis', superseded by the new diagnosis system --- data/actionsmap/yunohost.yml | 10 -- src/yunohost/tools.py | 188 +---------------------------------- 2 files changed, 3 insertions(+), 195 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 20eb8a0f8..1c96ce3e8 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1606,16 +1606,6 @@ tools: help: Upgrade only the system packages action: store_true - ### tools_diagnosis() - diagnosis: - action_help: YunoHost diagnosis - api: GET /diagnosis - arguments: - -p: - full: --private - help: Show private data (domain, IP) - action: store_true - ### tools_port_available() port-available: action_help: Check availability of a local port diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 64689fe0c..034157e3a 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -30,23 +30,19 @@ import json import subprocess import pwd import socket -from xmlrpclib import Fault from importlib import import_module -from collections import OrderedDict from moulinette import msignals, m18n from moulinette.utils.log import getActionLogger from moulinette.utils.process import check_output, call_async_output from moulinette.utils.filesystem import read_json, write_to_json, read_yaml, write_to_yaml -from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, app_list, _install_appslist_fetch_cron +from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, _install_appslist_fetch_cron from yunohost.domain import domain_add, domain_list, _get_maindomain, _set_maindomain from yunohost.dyndns import _dyndns_available, _dyndns_provides from yunohost.firewall import firewall_upnp -from yunohost.service import service_status, service_start, service_enable +from yunohost.service import service_start, service_enable from yunohost.regenconf import regen_conf -from yunohost.monitor import monitor_disk, monitor_system -from yunohost.utils.packages import ynh_packages_version, _dump_sources_list, _list_upgradable_apt_packages -from yunohost.utils.network import get_public_ip +from yunohost.utils.packages import _dump_sources_list, _list_upgradable_apt_packages from yunohost.utils.error import YunohostError from yunohost.log import is_unit_operation, OperationLogger @@ -726,184 +722,6 @@ def tools_upgrade(operation_logger, apps=None, system=False): operation_logger.success() -def tools_diagnosis(private=False): - """ - Return global info about current yunohost instance to help debugging - - """ - diagnosis = OrderedDict() - - # Debian release - try: - with open('/etc/debian_version', 'r') as f: - debian_version = f.read().rstrip() - except IOError as e: - logger.warning(m18n.n('diagnosis_debian_version_error', error=format(e)), exc_info=1) - else: - diagnosis['host'] = "Debian %s" % debian_version - - # Kernel version - try: - with open('/proc/sys/kernel/osrelease', 'r') as f: - kernel_version = f.read().rstrip() - except IOError as e: - logger.warning(m18n.n('diagnosis_kernel_version_error', error=format(e)), exc_info=1) - else: - diagnosis['kernel'] = kernel_version - - # Packages version - diagnosis['packages'] = ynh_packages_version() - - diagnosis["backports"] = check_output("dpkg -l |awk '/^ii/ && $3 ~ /bpo[6-8]/ {print $2}'").split() - - # Server basic monitoring - diagnosis['system'] = OrderedDict() - try: - disks = monitor_disk(units=['filesystem'], human_readable=True) - except (YunohostError, Fault) as e: - logger.warning(m18n.n('diagnosis_monitor_disk_error', error=format(e)), exc_info=1) - else: - diagnosis['system']['disks'] = {} - for disk in disks: - if isinstance(disks[disk], str): - diagnosis['system']['disks'][disk] = disks[disk] - else: - diagnosis['system']['disks'][disk] = 'Mounted on %s, %s (%s free)' % ( - disks[disk]['mnt_point'], - disks[disk]['size'], - disks[disk]['avail'] - ) - - try: - system = monitor_system(units=['cpu', 'memory'], human_readable=True) - except YunohostError as e: - logger.warning(m18n.n('diagnosis_monitor_system_error', error=format(e)), exc_info=1) - else: - diagnosis['system']['memory'] = { - 'ram': '%s (%s free)' % (system['memory']['ram']['total'], system['memory']['ram']['free']), - 'swap': '%s (%s free)' % (system['memory']['swap']['total'], system['memory']['swap']['free']), - } - - # nginx -t - p = subprocess.Popen("nginx -t".split(), - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - out, _ = p.communicate() - diagnosis["nginx"] = out.strip().split("\n") - if p.returncode != 0: - logger.error(out) - - # Services status - services = service_status() - diagnosis['services'] = {} - - for service in services: - diagnosis['services'][service] = "%s (%s)" % (services[service]['status'], services[service]['loaded']) - - # YNH Applications - try: - applications = app_list()['apps'] - except YunohostError as e: - diagnosis['applications'] = m18n.n('diagnosis_no_apps') - else: - diagnosis['applications'] = {} - for application in applications: - if application['installed']: - diagnosis['applications'][application['id']] = application['label'] if application['label'] else application['name'] - - # Private data - if private: - diagnosis['private'] = OrderedDict() - - # Public IP - diagnosis['private']['public_ip'] = {} - diagnosis['private']['public_ip']['IPv4'] = get_public_ip(4) - diagnosis['private']['public_ip']['IPv6'] = get_public_ip(6) - - # Domains - diagnosis['private']['domains'] = domain_list()['domains'] - - diagnosis['private']['regen_conf'] = regen_conf(with_diff=True, dry_run=True) - - try: - diagnosis['security'] = { - "CVE-2017-5754": { - "name": "meltdown", - "vulnerable": _check_if_vulnerable_to_meltdown(), - } - } - except Exception as e: - import traceback - traceback.print_exc() - logger.warning("Unable to check for meltdown vulnerability: %s" % e) - - return diagnosis - - -def _check_if_vulnerable_to_meltdown(): - # meltdown CVE: https://security-tracker.debian.org/tracker/CVE-2017-5754 - - # We use a cache file to avoid re-running the script so many times, - # which can be expensive (up to around 5 seconds on ARM) - # and make the admin appear to be slow (c.f. the calls to diagnosis - # from the webadmin) - # - # The cache is in /tmp and shall disappear upon reboot - # *or* we compare it to dpkg.log modification time - # such that it's re-ran if there was package upgrades - # (e.g. from yunohost) - cache_file = "/tmp/yunohost-meltdown-diagnosis" - dpkg_log = "/var/log/dpkg.log" - if os.path.exists(cache_file): - if not os.path.exists(dpkg_log) or os.path.getmtime(cache_file) > os.path.getmtime(dpkg_log): - logger.debug("Using cached results for meltdown checker, from %s" % cache_file) - return read_json(cache_file)[0]["VULNERABLE"] - - # script taken from https://github.com/speed47/spectre-meltdown-checker - # script commit id is store directly in the script - file_dir = os.path.split(__file__)[0] - SCRIPT_PATH = os.path.join(file_dir, "./vendor/spectre-meltdown-checker/spectre-meltdown-checker.sh") - - # '--variant 3' corresponds to Meltdown - # example output from the script: - # [{"NAME":"MELTDOWN","CVE":"CVE-2017-5754","VULNERABLE":false,"INFOS":"PTI mitigates the vulnerability"}] - try: - logger.debug("Running meltdown vulnerability checker") - call = subprocess.Popen("bash %s --batch json --variant 3" % - SCRIPT_PATH, shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - # TODO / FIXME : here we are ignoring error messages ... - # in particular on RPi2 and other hardware, the script complains about - # "missing some kernel info (see -v), accuracy might be reduced" - # Dunno what to do about that but we probably don't want to harass - # users with this warning ... - output, err = call.communicate() - assert call.returncode in (0, 2, 3), "Return code: %s" % call.returncode - - # If there are multiple lines, sounds like there was some messages - # in stdout that are not json >.> ... Try to get the actual json - # stuff which should be the last line - output = output.strip() - if "\n" in output: - logger.debug("Original meltdown checker output : %s" % output) - output = output.split("\n")[-1] - - CVEs = json.loads(output) - assert len(CVEs) == 1 - assert CVEs[0]["NAME"] == "MELTDOWN" - except Exception as e: - import traceback - traceback.print_exc() - logger.warning("Something wrong happened when trying to diagnose Meltdown vunerability, exception: %s" % e) - raise Exception("Command output for failed meltdown check: '%s'" % output) - - logger.debug("Writing results from meltdown checker to cache file, %s" % cache_file) - write_to_json(cache_file, CVEs) - return CVEs[0]["VULNERABLE"] - - def tools_port_available(port): """ Check availability of a local port From d113b6a53f126c289d4148f5a210c1bbf21ae118 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 5 Aug 2019 23:53:32 +0200 Subject: [PATCH 59/94] Adding some notes about diagnosis items to be implemented --- data/hooks/diagnosis/12-dnsrecords.py | 2 ++ data/hooks/diagnosis/18-mail.py | 28 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 data/hooks/diagnosis/18-mail.py diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index b59ffbd54..8c6565da9 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -31,6 +31,8 @@ class DNSRecordsDiagnoser(Diagnoser): for report in self.check_domain(domain, domain == main_domain): yield report + # FIXME : somewhere, should implement a check for reverse DNS ... + def check_domain(self, domain, is_main_domain): expected_configuration = _build_dns_conf(domain) diff --git a/data/hooks/diagnosis/18-mail.py b/data/hooks/diagnosis/18-mail.py new file mode 100644 index 000000000..5cf897e72 --- /dev/null +++ b/data/hooks/diagnosis/18-mail.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python + +import os + +from yunohost.diagnosis import Diagnoser + + +class MailDiagnoser(Diagnoser): + + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] + cache_duration = 3600 + dependencies = ["ip"] + + def run(self): + + return # TODO / FIXME TO BE IMPLEMETED in the future ... + + # Mail blacklist using dig requests (c.f. ljf's code) + + # Outgoing port 25 (c.f. code in monitor.py, a simple 'nc -zv yunohost.org 25' IIRC) + + # SMTP reachability (c.f. check-smtp to be implemented on yunohost's remote diagnoser) + + # ideally, SPF / DMARC / DKIM validation ... (c.f. https://github.com/alexAubin/yunoScripts/blob/master/yunoDKIM.py possibly though that looks horrible) + + +def main(args, env, loggers): + return MailDiagnoser(args, env, loggers).diagnose() From 339b6d9cbe2c97ffe249b99fb6b4c7f8c06437d7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 11 Aug 2019 16:23:47 +0200 Subject: [PATCH 60/94] Moar notes about what could be implemented for mail diagnoser --- data/hooks/diagnosis/18-mail.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/data/hooks/diagnosis/18-mail.py b/data/hooks/diagnosis/18-mail.py index 5cf897e72..100ace22f 100644 --- a/data/hooks/diagnosis/18-mail.py +++ b/data/hooks/diagnosis/18-mail.py @@ -23,6 +23,12 @@ class MailDiagnoser(Diagnoser): # ideally, SPF / DMARC / DKIM validation ... (c.f. https://github.com/alexAubin/yunoScripts/blob/master/yunoDKIM.py possibly though that looks horrible) + # check that the mail queue is not filled with hundreds of email pending + + # check that the recent mail logs are not filled with thousand of email sending (unusual number of mail sent) + + # check for unusual failed sending attempt being refused in the logs ? + def main(args, env, loggers): return MailDiagnoser(args, env, loggers).diagnose() From e0fa87cb364cd6d76bf4f80fd5da2c3ae51c9ca5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 18 Aug 2019 04:53:32 +0200 Subject: [PATCH 61/94] Note for the future about trying to diagnose hairpinning --- data/hooks/diagnosis/16-http.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/data/hooks/diagnosis/16-http.py b/data/hooks/diagnosis/16-http.py index cc335df8b..7ca258628 100644 --- a/data/hooks/diagnosis/16-http.py +++ b/data/hooks/diagnosis/16-http.py @@ -48,6 +48,10 @@ class HttpDiagnoser(Diagnoser): status="ERROR", summary=("diagnosis_http_unreachable", {"domain": domain})) + # In there or idk where else ... + # try to diagnose hairpinning situation by crafting a request for the + # global ip (from within local network) and seeing if we're getting the right page ? + def main(args, env, loggers): return HttpDiagnoser(args, env, loggers).diagnose() From 356f2b9ec1362b33b03ca8254d9d25c2bfcc22f1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 22 Aug 2019 12:42:25 +0200 Subject: [PATCH 62/94] Moar ideas --- data/hooks/diagnosis/00-basesystem.py | 2 ++ data/hooks/diagnosis/10-ip.py | 2 ++ data/hooks/diagnosis/12-dnsrecords.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/data/hooks/diagnosis/00-basesystem.py b/data/hooks/diagnosis/00-basesystem.py index 8fa90e65e..8bd522ee7 100644 --- a/data/hooks/diagnosis/00-basesystem.py +++ b/data/hooks/diagnosis/00-basesystem.py @@ -21,6 +21,8 @@ class BaseSystemDiagnoser(Diagnoser): status="INFO", summary=("diagnosis_basesystem_kernel", {"kernel_version": kernel_version})) + # FIXME / TODO : add virt/vm technology using systemd-detect-virt and/or machine arch + # Debian release debian_version = read_file("/etc/debian_version").strip() yield dict(meta={"test": "host"}, diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index 8c8dbe95b..e09dd343b 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -84,6 +84,8 @@ class IPDiagnoser(Diagnoser): summary=("diagnosis_ip_connected_ipv6", {}) if ipv6 else ("diagnosis_ip_no_ipv6", {})) + # TODO / FIXME : add some attempt to detect ISP (using whois ?) ? + def can_ping_outside(self, protocol=4): assert protocol in [4, 6], "Invalid protocol version, it should be either 4 or 6 and was '%s'" % repr(protocol) diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 8c6565da9..e2f7bcc2d 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -33,6 +33,8 @@ class DNSRecordsDiagnoser(Diagnoser): # FIXME : somewhere, should implement a check for reverse DNS ... + # FIXME / TODO : somewhere, could also implement a check for domain expiring soon + def check_domain(self, domain, is_main_domain): expected_configuration = _build_dns_conf(domain) From 3d7f37176cd8839e0455d414e1eb1aeb31274f2e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 10 Oct 2019 16:23:38 +0200 Subject: [PATCH 63/94] Remove debug prints --- data/hooks/diagnosis/50-systemresources.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/data/hooks/diagnosis/50-systemresources.py b/data/hooks/diagnosis/50-systemresources.py index 3399c4682..7e93a9ec0 100644 --- a/data/hooks/diagnosis/50-systemresources.py +++ b/data/hooks/diagnosis/50-systemresources.py @@ -31,7 +31,6 @@ class SystemResourcesDiagnoser(Diagnoser): else: item["status"] = "SUCCESS" item["summary"] = ("diagnosis_ram_ok", infos) - print(item) yield item # @@ -51,7 +50,6 @@ class SystemResourcesDiagnoser(Diagnoser): else: item["status"] = "SUCCESS" item["summary"] = ("diagnosis_swap_ok", infos) - print(item) yield item # From 02d6a0212f508fcd1df78bf2656447f9bd544f6d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 10 Oct 2019 16:40:06 +0200 Subject: [PATCH 64/94] Remove debug prints --- data/hooks/diagnosis/10-ip.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index e09dd343b..9c4257306 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -113,8 +113,10 @@ class IPDiagnoser(Diagnoser): # So let's try to ping the first 4~5 resolvers (shuffled) # If we succesfully ping any of them, we conclude that we are indeed connected def ping(protocol, target): + print("ping -c1 -%s -W 3 %s >/dev/null 2>/dev/null" % (protocol, target)) return os.system("ping -c1 -%s -W 3 %s >/dev/null 2>/dev/null" % (protocol, target)) == 0 + random.shuffle(resolvers) return any(ping(protocol, resolver) for resolver in resolvers[:5]) From e67e9e27ba281913cd0e6b518a252e7c4c536feb Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 10 Oct 2019 16:48:58 +0200 Subject: [PATCH 65/94] Hmm somehow there seem to be different version of ping supporting or not the -4 / -6 ... let's see if this workaroud works in all contexts --- data/hooks/diagnosis/10-ip.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/data/hooks/diagnosis/10-ip.py b/data/hooks/diagnosis/10-ip.py index 9c4257306..552092fe3 100644 --- a/data/hooks/diagnosis/10-ip.py +++ b/data/hooks/diagnosis/10-ip.py @@ -113,9 +113,7 @@ class IPDiagnoser(Diagnoser): # So let's try to ping the first 4~5 resolvers (shuffled) # If we succesfully ping any of them, we conclude that we are indeed connected def ping(protocol, target): - print("ping -c1 -%s -W 3 %s >/dev/null 2>/dev/null" % (protocol, target)) - return os.system("ping -c1 -%s -W 3 %s >/dev/null 2>/dev/null" % (protocol, target)) == 0 - + return os.system("ping%s -c1 -W 3 %s >/dev/null 2>/dev/null" % ("" if protocol == 4 else "6", target)) == 0 random.shuffle(resolvers) return any(ping(protocol, resolver) for resolver in resolvers[:5]) From d6eb55d2a2107e217935256667d4aef52bd64593 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 11 Oct 2019 20:04:53 +0200 Subject: [PATCH 66/94] Add tmp dummy mail report so that the diagnoser kinda works instead of failing miserably --- data/hooks/diagnosis/18-mail.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/data/hooks/diagnosis/18-mail.py b/data/hooks/diagnosis/18-mail.py index 100ace22f..c12c15cff 100644 --- a/data/hooks/diagnosis/18-mail.py +++ b/data/hooks/diagnosis/18-mail.py @@ -13,7 +13,11 @@ class MailDiagnoser(Diagnoser): def run(self): - return # TODO / FIXME TO BE IMPLEMETED in the future ... + # TODO / FIXME TO BE IMPLEMETED in the future ... + + yield dict(meta={}, + status="WARNING", + summary=("nothing_implemented_yet", {})) # Mail blacklist using dig requests (c.f. ljf's code) From f75cd82593cc0feaab0f86f0d57b2f53c895ab5e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 11 Oct 2019 20:05:46 +0200 Subject: [PATCH 67/94] First part of implementing the ignore mechanism --- data/actionsmap/yunohost.yml | 19 ++--- src/yunohost/diagnosis.py | 132 +++++++++++++++++++++++++++++++++-- 2 files changed, 139 insertions(+), 12 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 1c96ce3e8..9b694c853 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1893,14 +1893,17 @@ diagnosis: action: store_true ignore: - action_help: Configure some diagnosis results to be ignored + action_help: Configure some diagnosis results to be ignored and therefore not considered as actual issues api: PUT /diagnosis/ignore arguments: - category: - help: Diagnosis category to be affected - -a: - help: Arguments, to be used to ignore only some parts of a report (e.g. "domain=domain.tld") - full: --args - --unignore: - help: Unignore a previously ignored report + --add-filter: + help: "Add a filter. The first element should be a diagnosis category, and other criterias can be provided using the infos from the 'meta' sections in 'yunohost diagnosis show'. For example: 'dnsrecords domain=yolo.test category=xmpp'" + nargs: "*" + metavar: CRITERIA + --remove-filter: + help: Remove a filter (it should be an existing filter as listed with --list) + nargs: "*" + metavar: CRITERIA + --list: + help: List active ignore filters action: store_true diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index b9fe111ed..da69e5d5e 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -29,7 +29,7 @@ import time from moulinette import m18n, msettings from moulinette.utils import log -from moulinette.utils.filesystem import read_json, write_to_json +from moulinette.utils.filesystem import read_json, write_to_json, read_yaml, write_to_yaml from yunohost.utils.error import YunohostError from yunohost.hook import hook_list, hook_exec @@ -37,7 +37,7 @@ from yunohost.hook import hook_list, hook_exec logger = log.getActionLogger('yunohost.diagnosis') DIAGNOSIS_CACHE = "/var/cache/yunohost/diagnosis/" - +DIAGNOSIS_CONFIG_FILE = '/etc/yunohost/diagnosis.yml' def diagnosis_list(): all_categories_names = [h for h, _ in _list_diagnosis_categories()] @@ -151,8 +151,132 @@ def diagnosis_run(categories=[], force=False): return -def diagnosis_ignore(category, args="", unignore=False): - pass +def diagnosis_ignore(add_filter=None, remove_filter=None, list=False): + """ + This action is meant for the admin to ignore issues reported by the + diagnosis system if they are known and understood by the admin. For + example, the lack of ipv6 on an instance, or badly configured XMPP dns + records if the admin doesn't care so much about XMPP. The point being that + the diagnosis shouldn't keep complaining about those known and "expected" + issues, and instead focus on new unexpected issues that could arise. + + For example, to ignore badly XMPP dnsrecords for domain yolo.test: + + yunohost diagnosis ignore --add-filter dnsrecords domain=yolo.test category=xmpp + ^ ^ ^ + the general additional other + diagnosis criterias criteria + category to to target to target + act on specific specific + reports reports + Or to ignore all dnsrecords issues: + + yunohost diagnosis ignore --add-filter dnsrecords + + The filters are stored in the diagnosis configuration in a data structure like: + + ignore_filters: { + "ip": [ + {"version": 6} # Ignore all issues related to ipv6 + ], + "dnsrecords": [ + {"domain": "yolo.test", "category": "xmpp"}, # Ignore all issues related to DNS xmpp records for yolo.test + {} # Ignore all issues about dnsrecords + ] + } + """ + + # Ignore filters are stored in + configuration = _diagnosis_read_configuration() + + if list: + return {"ignore_filters": configuration.get("ignore_filters", {})} + + def validate_filter_criterias(filter_): + + # Get all the categories + all_categories = _list_diagnosis_categories() + all_categories_names = [category for category, _ in all_categories] + + # Sanity checks for the provided arguments + if len(filter_) == 0: + raise YunohostError("You should provide at least one criteria being the diagnosis category to ignore") + category = filter_[0] + if category not in all_categories_names: + raise YunohostError("%s is not a diagnosis category" % category) + if any("=" not in criteria for criteria in filter_[1:]): + raise YunohostError("Extra criterias should be of the form key=value (e.g. domain=yolo.test)") + + # Convert the provided criteria into a nice dict + criterias = {c.split("=")[0]: c.split("=")[1] for c in filter_[1:]} + + return category, criterias + + if add_filter: + + category, criterias = validate_filter_criterias(add_filter) + + # Fetch current issues for the requested category + current_issues_for_this_category = diagnosis_show(categories=[category], issues=True, full=True) + current_issues_for_this_category = current_issues_for_this_category["reports"][0].get("items", {}) + + # Accept the given filter only if the criteria effectively match an existing issue + if not any(issue_matches_criterias(i, criterias) for i in current_issues_for_this_category): + raise YunohostError("No issues was found matching the given criteria.") + + # Make sure the subdicts/lists exists + if "ignore_filters" not in configuration: + configuration["ignore_filters"] = {} + if category not in configuration["ignore_filters"]: + configuration["ignore_filters"][category] = [] + + if criterias in configuration["ignore_filters"][category]: + logger.warning("This filter already exists.") + return + + configuration["ignore_filters"][category].append(criterias) + _diagnosis_write_configuration(configuration) + logger.success("Filter added") + return + + if remove_filter: + + category, criterias = validate_filter_criterias(remove_filter) + + # Make sure the subdicts/lists exists + if "ignore_filters" not in configuration: + configuration["ignore_filters"] = {} + if category not in configuration["ignore_filters"]: + configuration["ignore_filters"][category] = [] + + if criterias not in configuration["ignore_filters"][category]: + raise YunohostError("This filter does not exists.") + + configuration["ignore_filters"][category].remove(criterias) + _diagnosis_write_configuration(configuration) + logger.success("Filter removed") + return + + +def _diagnosis_read_configuration(): + if not os.path.exists(DIAGNOSIS_CONFIG_FILE): + return {} + + return read_yaml(DIAGNOSIS_CONFIG_FILE) + + +def _diagnosis_write_configuration(conf): + write_to_yaml(DIAGNOSIS_CONFIG_FILE, conf) + + +def issue_matches_criterias(issues, criterias): + for key, value in criterias.items(): + if key not in issues["meta"]: + return False + if str(issues["meta"][key]) != value: + return False + return True + ############################################################ From 97f9d3ea3753db40622b20df967e1644ac678c04 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 11 Oct 2019 22:42:21 +0200 Subject: [PATCH 68/94] Integrate the ignore mechanism with the rest of the code --- locales/en.json | 4 ++- src/yunohost/diagnosis.py | 55 ++++++++++++++++++++++++++++++++------- 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/locales/en.json b/locales/en.json index 40edb1425..afcb44edb 100644 --- a/locales/en.json +++ b/locales/en.json @@ -165,7 +165,9 @@ "diagnosis_failed_for_category": "Diagnosis failed for category '{category}' : {error}", "diagnosis_cache_still_valid": "(Cache still valid for {category} diagnosis. Not re-diagnosing yet!)", "diagnosis_cant_run_because_of_dep": "Can't run diagnosis for {category} while there are important issues related to {dep}.", - "diagnosis_found_issues": "Found {errors} significant issue(s) related to {category}!", + "diagnosis_ignored_issues": "(+ {nb_ignored} ignored issue(s))", + "diagnosis_found_errors": "Found {errors} significant issue(s) related to {category}!", + "diagnosis_found_errors_and_warnings": "Found {errors} significant issue(s) (and {warnings} warning(s)) related to {category}!", "diagnosis_found_warnings": "Found {warnings} item(s) that could be improved for {category}.", "diagnosis_everything_ok": "Everything looks good for {category}!", "diagnosis_failed": "Failed to fetch diagnosis result for category '{category}' : {error}", diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index da69e5d5e..19dd03042 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -66,11 +66,14 @@ def diagnosis_show(categories=[], issues=False, full=False, share=False): except Exception as e: logger.error(m18n.n("diagnosis_failed", category=category, error=str(e))) else: + add_ignore_flag_to_issues(report) if not full: del report["timestamp"] del report["cached_for"] + report["items"] = [item for item in report["items"] if not item["ignored"]] for item in report["items"]: del item["meta"] + del item["ignored"] if "data" in item: del item["data"] if issues: @@ -269,14 +272,42 @@ def _diagnosis_write_configuration(conf): write_to_yaml(DIAGNOSIS_CONFIG_FILE, conf) -def issue_matches_criterias(issues, criterias): +def issue_matches_criterias(issue, criterias): + """ + e.g. an issue with: + meta: + domain: yolo.test + category: xmpp + + matches the criterias {"domain": "yolo.test"} + """ for key, value in criterias.items(): - if key not in issues["meta"]: + if key not in issue["meta"]: return False - if str(issues["meta"][key]) != value: + if str(issue["meta"][key]) != value: return False return True +def add_ignore_flag_to_issues(report): + """ + Iterate over issues in a report, and flag them as ignored if they match an + ignored filter from the configuration + + N.B. : for convenience. we want to make sure the "ignored" key is set for + every item in the report + """ + + ignore_filters = _diagnosis_read_configuration().get("ignore_filters", {}).get(report["id"], []) + + for report_item in report["items"]: + report_item["ignored"] = False + if report_item["status"] not in ["WARNING", "ERROR"]: + continue + for criterias in ignore_filters: + if issue_matches_criterias(report_item, criterias): + report_item["ignored"] = True + break + ############################################################ @@ -328,16 +359,22 @@ class Diagnoser(): self.logger_debug("Updating cache %s" % self.cache_file) self.write_cache(new_report) Diagnoser.i18n(new_report) + add_ignore_flag_to_issues(new_report) - errors = [item for item in new_report["items"] if item["status"] == "ERROR"] - warnings = [item for item in new_report["items"] if item["status"] == "WARNING"] + errors = [item for item in new_report["items"] if item["status"] == "ERROR" and not item["ignored"]] + warnings = [item for item in new_report["items"] if item["status"] == "WARNING" and not item["ignored"]] + errors_ignored = [item for item in new_report["items"] if item["status"] == "ERROR" and item["ignored"]] + warning_ignored = [item for item in new_report["items"] if item["status"] == "WARNING" and item["ignored"]] + ignored_msg = " " + m18n.n("diagnosis_ignored_issues", nb_ignored=len(errors_ignored+warning_ignored)) if errors_ignored or warning_ignored else "" - if errors: - logger.error(m18n.n("diagnosis_found_issues", errors=len(errors), category=new_report["description"])) + if errors and warnings: + logger.error(m18n.n("diagnosis_found_errors_and_warnings", errors=len(errors), warnings=len(warnings), category=new_report["description"]) + ignored_msg) + elif errors: + logger.error(m18n.n("diagnosis_found_errors", errors=len(errors), category=new_report["description"]) + ignored_msg) elif warnings: - logger.warning(m18n.n("diagnosis_found_warnings", warnings=len(warnings), category=new_report["description"])) + logger.warning(m18n.n("diagnosis_found_warnings", warnings=len(warnings), category=new_report["description"]) + ignored_msg) else: - logger.success(m18n.n("diagnosis_everything_ok", category=new_report["description"])) + logger.success(m18n.n("diagnosis_everything_ok", category=new_report["description"]) + ignored_msg) return 0, new_report From 51e7a56522e49edc498677aeb5ec08fa174d713c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 13 Oct 2019 18:42:45 +0200 Subject: [PATCH 69/94] Improve metadata for diskusage tests --- data/hooks/diagnosis/50-systemresources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/hooks/diagnosis/50-systemresources.py b/data/hooks/diagnosis/50-systemresources.py index 7e93a9ec0..95f58ddb7 100644 --- a/data/hooks/diagnosis/50-systemresources.py +++ b/data/hooks/diagnosis/50-systemresources.py @@ -66,7 +66,7 @@ class SystemResourcesDiagnoser(Diagnoser): free_abs_GB = usage.free / (1024 ** 3) free_percent = 100 - usage.percent - item = dict(meta={"mountpoint": mountpoint, "device": device}) + item = dict(meta={"test": "diskusage", "mountpoint": mountpoint}) infos = {"mountpoint": mountpoint, "device": device, "free_abs_GB": free_abs_GB, "free_percent": free_percent} if free_abs_GB < 1 or free_percent < 5: item["status"] = "ERROR" From 0839de2d6a3684056e0d0bf692c4391b1ac153ef Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 13 Oct 2019 23:02:46 +0200 Subject: [PATCH 70/94] Switching to POST method because it's more practical than PUT, idk what im doing --- data/actionsmap/yunohost.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 9b694c853..e7a1d1ad2 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1894,7 +1894,7 @@ diagnosis: ignore: action_help: Configure some diagnosis results to be ignored and therefore not considered as actual issues - api: PUT /diagnosis/ignore + api: POST /diagnosis/ignore arguments: --add-filter: help: "Add a filter. The first element should be a diagnosis category, and other criterias can be provided using the infos from the 'meta' sections in 'yunohost diagnosis show'. For example: 'dnsrecords domain=yolo.test category=xmpp'" From 5818de3a824a1c869c12cc4760b4511d53baa83d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 14 Oct 2019 04:48:56 +0200 Subject: [PATCH 71/94] Remove the whole monitoring / glances stuff --- data/actionsmap/yunohost.yml | 141 ----- data/templates/glances/glances.default | 5 - data/templates/yunohost/services.yml | 2 +- debian/control | 2 +- locales/en.json | 14 - src/yunohost/monitor.py | 740 ------------------------- 6 files changed, 2 insertions(+), 902 deletions(-) delete mode 100644 data/templates/glances/glances.default delete mode 100644 src/yunohost/monitor.py diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index e7a1d1ad2..4b76fcb0b 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -968,147 +968,6 @@ backup: pattern: *pattern_backup_archive_name -############################# -# Monitor # -############################# -monitor: - category_help: Monitor the server - actions: - - ### monitor_disk() - disk: - action_help: Monitor disk space and usage - api: GET /monitor/disk - arguments: - -f: - full: --filesystem - help: Show filesystem disk space - action: append_const - const: filesystem - dest: units - -t: - full: --io - help: Show I/O throughput - action: append_const - const: io - dest: units - -m: - full: --mountpoint - help: Monitor only the device mounted on MOUNTPOINT - action: store - -H: - full: --human-readable - help: Print sizes in human readable format - action: store_true - - ### monitor_network() - network: - action_help: Monitor network interfaces - api: GET /monitor/network - arguments: - -u: - full: --usage - help: Show interfaces bit rates - action: append_const - const: usage - dest: units - -i: - full: --infos - help: Show network informations - action: append_const - const: infos - dest: units - -c: - full: --check - help: Check network configuration - action: append_const - const: check - dest: units - -H: - full: --human-readable - help: Print sizes in human readable format - action: store_true - - ### monitor_system() - system: - action_help: Monitor system informations and usage - api: GET /monitor/system - arguments: - -m: - full: --memory - help: Show memory usage - action: append_const - const: memory - dest: units - -c: - full: --cpu - help: Show CPU usage and load - action: append_const - const: cpu - dest: units - -p: - full: --process - help: Show processes summary - action: append_const - const: process - dest: units - -u: - full: --uptime - help: Show the system uptime - action: append_const - const: uptime - dest: units - -i: - full: --infos - help: Show system informations - action: append_const - const: infos - dest: units - -H: - full: --human-readable - help: Print sizes in human readable format - action: store_true - - ### monitor_updatestats() - update-stats: - action_help: Update monitoring statistics - api: POST /monitor/stats - arguments: - period: - help: Time period to update - choices: - - day - - week - - month - - ### monitor_showstats() - show-stats: - action_help: Show monitoring statistics - api: GET /monitor/stats - arguments: - period: - help: Time period to show - choices: - - day - - week - - month - - ### monitor_enable() - enable: - action_help: Enable server monitoring - api: PUT /monitor - arguments: - -s: - full: --with-stats - help: Enable monitoring statistics - action: store_true - - ### monitor_disable() - disable: - api: DELETE /monitor - action_help: Disable server monitoring - - ############################# # Settings # ############################# diff --git a/data/templates/glances/glances.default b/data/templates/glances/glances.default deleted file mode 100644 index 22337a0d9..000000000 --- a/data/templates/glances/glances.default +++ /dev/null @@ -1,5 +0,0 @@ -# Default is to launch glances with '-s' option. -DAEMON_ARGS="-s -B 127.0.0.1" - -# Change to 'true' to have glances running at startup -RUN="true" diff --git a/data/templates/yunohost/services.yml b/data/templates/yunohost/services.yml index 0d79b182f..1c0ee031f 100644 --- a/data/templates/yunohost/services.yml +++ b/data/templates/yunohost/services.yml @@ -17,7 +17,6 @@ redis-server: mysql: log: [/var/log/mysql.log,/var/log/mysql.err] alternates: ['mariadb'] -glances: {} ssh: log: /var/log/auth.log metronome: @@ -32,6 +31,7 @@ yunohost-firewall: need_lock: true nslcd: log: /var/log/syslog +glances: null nsswitch: null ssl: null yunohost: null diff --git a/debian/control b/debian/control index c0604d90e..3b8c257d0 100644 --- a/debian/control +++ b/debian/control @@ -15,7 +15,7 @@ Depends: ${python:Depends}, ${misc:Depends} , python-psutil, python-requests, python-dnspython, python-openssl , python-apt, python-miniupnpc, python-dbus, python-jinja2 , python-toml - , glances, apt-transport-https + , apt-transport-https , dnsutils, bind9utils, unzip, git, curl, cron, wget, jq , ca-certificates, netcat-openbsd, iproute , mariadb-server, php-mysql | php-mysqlnd diff --git a/locales/en.json b/locales/en.json index afcb44edb..e04446ce1 100644 --- a/locales/en.json +++ b/locales/en.json @@ -434,21 +434,9 @@ "migrations_skip_migration": "Skipping migration {id}…", "migrations_success_forward": "Migration {id} completed", "migrations_to_be_ran_manually": "Migration {id} has to be run manually. Please go to Tools → Migrations on the webadmin page, or run `yunohost tools migrations migrate`.", - "monitor_disabled": "Server monitoring now turned off", - "monitor_enabled": "Server monitoring now turned on", - "monitor_glances_con_failed": "Could not connect to Glances server", - "monitor_not_enabled": "Server monitoring is off", - "monitor_period_invalid": "Invalid time period", - "monitor_stats_file_not_found": "Statistics file not found", - "monitor_stats_no_update": "No monitoring statistics to update", - "monitor_stats_period_unavailable": "No available statistics for the period", - "mountpoint_unknown": "Unknown mountpoint", "mysql_db_creation_failed": "MySQL database creation failed", "mysql_db_init_failed": "MySQL database init failed", "mysql_db_initialized": "The MySQL database now initialized", - "network_check_mx_ko": "DNS MX record is not set", - "network_check_smtp_ko": "Outbound e-mail (SMTP port 25) seems to be blocked by your network", - "network_check_smtp_ok": "Outbound e-mail (SMTP port 25) is not blocked", "no_internet_connection": "Server not connected to the Internet", "not_enough_disk_space": "Not enough free space on '{path:s}'", "operation_interrupted": "The operation was manually interrupted?", @@ -536,7 +524,6 @@ "service_description_dnsmasq": "Handles domain name resolution (DNS)", "service_description_dovecot": "Allows e-mail clients to access/fetch email (via IMAP and POP3)", "service_description_fail2ban": "Protects against brute-force and other kinds of attacks from the Internet", - "service_description_glances": "Monitors system info on your server", "service_description_metronome": "Manage XMPP instant messaging accounts", "service_description_mysql": "Stores applications data (SQL database)", "service_description_nginx": "Serves or provides access to all the websites hosted on your server", @@ -588,7 +575,6 @@ "tools_upgrade_special_packages_completed": "YunoHost package upgrade completed.\nPress [Enter] to get the command line back", "unbackup_app": "App '{app:s}' will not be saved", "unexpected_error": "Something unexpected went wrong: {error}", - "unit_unknown": "Unknown unit '{unit:s}'", "unlimit": "No quota", "unrestore_app": "App '{app:s}' will not be restored", "update_apt_cache_failed": "Could not to update the cache of APT (Debian's package manager). Here is a dump of the sources.list lines, which might help identify problematic lines: \n{sourceslist}", diff --git a/src/yunohost/monitor.py b/src/yunohost/monitor.py deleted file mode 100644 index 7af55f287..000000000 --- a/src/yunohost/monitor.py +++ /dev/null @@ -1,740 +0,0 @@ -# -*- coding: utf-8 -*- - -""" License - - Copyright (C) 2013 YunoHost - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program; if not, see http://www.gnu.org/licenses - -""" - -""" yunohost_monitor.py - - Monitoring functions -""" -import re -import json -import time -import psutil -import calendar -import subprocess -import xmlrpclib -import os.path -import os -import dns.resolver -import cPickle as pickle -from datetime import datetime - -from moulinette import m18n -from yunohost.utils.error import YunohostError -from moulinette.utils.log import getActionLogger - -from yunohost.utils.network import get_public_ip -from yunohost.domain import _get_maindomain - -logger = getActionLogger('yunohost.monitor') - -GLANCES_URI = 'http://127.0.0.1:61209' -STATS_PATH = '/var/lib/yunohost/stats' -CRONTAB_PATH = '/etc/cron.d/yunohost-monitor' - - -def monitor_disk(units=None, mountpoint=None, human_readable=False): - """ - Monitor disk space and usage - - Keyword argument: - units -- Unit(s) to monitor - mountpoint -- Device mountpoint - human_readable -- Print sizes in human readable format - - """ - glances = _get_glances_api() - result_dname = None - result = {} - - if units is None: - units = ['io', 'filesystem'] - - _format_dname = lambda d: (os.path.realpath(d)).replace('/dev/', '') - - # Get mounted devices - devices = {} - for p in psutil.disk_partitions(all=True): - if not p.device.startswith('/dev/') or not p.mountpoint: - continue - if mountpoint is None: - devices[_format_dname(p.device)] = p.mountpoint - elif mountpoint == p.mountpoint: - dn = _format_dname(p.device) - devices[dn] = p.mountpoint - result_dname = dn - if len(devices) == 0: - if mountpoint is not None: - raise YunohostError('mountpoint_unknown') - return result - - # Retrieve monitoring for unit(s) - for u in units: - if u == 'io': - # Define setter - if len(units) > 1: - def _set(dn, dvalue): - try: - result[dn][u] = dvalue - except KeyError: - result[dn] = {u: dvalue} - else: - def _set(dn, dvalue): - result[dn] = dvalue - - # Iterate over values - devices_names = devices.keys() - for d in json.loads(glances.getDiskIO()): - dname = d.pop('disk_name') - try: - devices_names.remove(dname) - except: - continue - else: - _set(dname, d) - for dname in devices_names: - _set(dname, 'not-available') - elif u == 'filesystem': - # Define setter - if len(units) > 1: - def _set(dn, dvalue): - try: - result[dn][u] = dvalue - except KeyError: - result[dn] = {u: dvalue} - else: - def _set(dn, dvalue): - result[dn] = dvalue - - # Iterate over values - devices_names = devices.keys() - for d in json.loads(glances.getFs()): - dname = _format_dname(d.pop('device_name')) - try: - devices_names.remove(dname) - except: - continue - else: - d['avail'] = d['size'] - d['used'] - if human_readable: - for i in ['used', 'avail', 'size']: - d[i] = binary_to_human(d[i]) + 'B' - _set(dname, d) - for dname in devices_names: - _set(dname, 'not-available') - else: - raise YunohostError('unit_unknown', unit=u) - - if result_dname is not None: - return result[result_dname] - return result - - -def monitor_network(units=None, human_readable=False): - """ - Monitor network interfaces - - Keyword argument: - units -- Unit(s) to monitor - human_readable -- Print sizes in human readable format - - """ - glances = _get_glances_api() - result = {} - - if units is None: - units = ['check', 'usage', 'infos'] - - # Get network devices and their addresses - # TODO / FIXME : use functions in utils/network.py to manage this - devices = {} - output = subprocess.check_output('ip addr show'.split()) - for d in re.split('^(?:[0-9]+: )', output, flags=re.MULTILINE): - # Extract device name (1) and its addresses (2) - m = re.match('([^\s@]+)(?:@[\S]+)?: (.*)', d, flags=re.DOTALL) - if m: - devices[m.group(1)] = m.group(2) - - # Retrieve monitoring for unit(s) - for u in units: - if u == 'check': - result[u] = {} - domain = _get_maindomain() - cmd_check_smtp = os.system('/bin/nc -z -w1 yunohost.org 25') - if cmd_check_smtp == 0: - smtp_check = m18n.n('network_check_smtp_ok') - else: - smtp_check = m18n.n('network_check_smtp_ko') - - try: - answers = dns.resolver.query(domain, 'MX') - mx_check = {} - i = 0 - for server in answers: - mx_id = 'mx%s' % i - mx_check[mx_id] = server - i = i + 1 - except: - mx_check = m18n.n('network_check_mx_ko') - result[u] = { - 'smtp_check': smtp_check, - 'mx_check': mx_check - } - elif u == 'usage': - result[u] = {} - for i in json.loads(glances.getNetwork()): - iname = i['interface_name'] - if iname in devices.keys(): - del i['interface_name'] - if human_readable: - for k in i.keys(): - if k != 'time_since_update': - i[k] = binary_to_human(i[k]) + 'B' - result[u][iname] = i - else: - logger.debug('interface name %s was not found', iname) - elif u == 'infos': - p_ipv4 = get_public_ip() or 'unknown' - - # TODO / FIXME : use functions in utils/network.py to manage this - l_ip = 'unknown' - for name, addrs in devices.items(): - if name == 'lo': - continue - if not isinstance(l_ip, dict): - l_ip = {} - l_ip[name] = _extract_inet(addrs) - - gateway = 'unknown' - output = subprocess.check_output('ip route show'.split()) - m = re.search('default via (.*) dev ([a-z]+[0-9]?)', output) - if m: - addr = _extract_inet(m.group(1), True) - if len(addr) == 1: - proto, gateway = addr.popitem() - - result[u] = { - 'public_ip': p_ipv4, - 'local_ip': l_ip, - 'gateway': gateway, - } - else: - raise YunohostError('unit_unknown', unit=u) - - if len(units) == 1: - return result[units[0]] - return result - - -def monitor_system(units=None, human_readable=False): - """ - Monitor system informations and usage - - Keyword argument: - units -- Unit(s) to monitor - human_readable -- Print sizes in human readable format - - """ - glances = _get_glances_api() - result = {} - - if units is None: - units = ['memory', 'cpu', 'process', 'uptime', 'infos'] - - # Retrieve monitoring for unit(s) - for u in units: - if u == 'memory': - ram = json.loads(glances.getMem()) - swap = json.loads(glances.getMemSwap()) - if human_readable: - for i in ram.keys(): - if i != 'percent': - ram[i] = binary_to_human(ram[i]) + 'B' - for i in swap.keys(): - if i != 'percent': - swap[i] = binary_to_human(swap[i]) + 'B' - result[u] = { - 'ram': ram, - 'swap': swap - } - elif u == 'cpu': - result[u] = { - 'load': json.loads(glances.getLoad()), - 'usage': json.loads(glances.getCpu()) - } - elif u == 'process': - result[u] = json.loads(glances.getProcessCount()) - elif u == 'uptime': - result[u] = (str(datetime.now() - datetime.fromtimestamp(psutil.boot_time())).split('.')[0]) - elif u == 'infos': - result[u] = json.loads(glances.getSystem()) - else: - raise YunohostError('unit_unknown', unit=u) - - if len(units) == 1 and not isinstance(result[units[0]], str): - return result[units[0]] - return result - - -def monitor_update_stats(period): - """ - Update monitoring statistics - - Keyword argument: - period -- Time period to update (day, week, month) - - """ - if period not in ['day', 'week', 'month']: - raise YunohostError('monitor_period_invalid') - - stats = _retrieve_stats(period) - if not stats: - stats = {'disk': {}, 'network': {}, 'system': {}, 'timestamp': []} - - monitor = None - # Get monitoring stats - if period == 'day': - monitor = _monitor_all('day') - else: - t = stats['timestamp'] - p = 'day' if period == 'week' else 'week' - if len(t) > 0: - monitor = _monitor_all(p, t[len(t) - 1]) - else: - monitor = _monitor_all(p, 0) - if not monitor: - raise YunohostError('monitor_stats_no_update') - - stats['timestamp'].append(time.time()) - - # Append disk stats - for dname, units in monitor['disk'].items(): - disk = {} - # Retrieve current stats for disk name - if dname in stats['disk'].keys(): - disk = stats['disk'][dname] - - for unit, values in units.items(): - # Continue if unit doesn't contain stats - if not isinstance(values, dict): - continue - - # Retrieve current stats for unit and append new ones - curr = disk[unit] if unit in disk.keys() else {} - if unit == 'io': - disk[unit] = _append_to_stats(curr, values, 'time_since_update') - elif unit == 'filesystem': - disk[unit] = _append_to_stats(curr, values, ['fs_type', 'mnt_point']) - stats['disk'][dname] = disk - - # Append network stats - net_usage = {} - for iname, values in monitor['network']['usage'].items(): - # Continue if units doesn't contain stats - if not isinstance(values, dict): - continue - - # Retrieve current stats and append new ones - curr = {} - if 'usage' in stats['network'] and iname in stats['network']['usage']: - curr = stats['network']['usage'][iname] - net_usage[iname] = _append_to_stats(curr, values, 'time_since_update') - stats['network'] = {'usage': net_usage, 'infos': monitor['network']['infos']} - - # Append system stats - for unit, values in monitor['system'].items(): - # Continue if units doesn't contain stats - if not isinstance(values, dict): - continue - - # Set static infos unit - if unit == 'infos': - stats['system'][unit] = values - continue - - # Retrieve current stats and append new ones - curr = stats['system'][unit] if unit in stats['system'].keys() else {} - stats['system'][unit] = _append_to_stats(curr, values) - - _save_stats(stats, period) - - -def monitor_show_stats(period, date=None): - """ - Show monitoring statistics - - Keyword argument: - period -- Time period to show (day, week, month) - - """ - if period not in ['day', 'week', 'month']: - raise YunohostError('monitor_period_invalid') - - result = _retrieve_stats(period, date) - if result is False: - raise YunohostError('monitor_stats_file_not_found') - elif result is None: - raise YunohostError('monitor_stats_period_unavailable') - return result - - -def monitor_enable(with_stats=False): - """ - Enable server monitoring - - Keyword argument: - with_stats -- Enable monitoring statistics - - """ - from yunohost.service import (service_status, service_enable, - service_start) - - glances = service_status('glances') - if glances['status'] != 'running': - service_start('glances') - if glances['loaded'] != 'enabled': - service_enable('glances') - - # Install crontab - if with_stats: - # day: every 5 min # week: every 1 h # month: every 4 h # - rules = ('*/5 * * * * root {cmd} day >> /dev/null\n' - '3 * * * * root {cmd} week >> /dev/null\n' - '6 */4 * * * root {cmd} month >> /dev/null').format( - cmd='/usr/bin/yunohost --quiet monitor update-stats') - with open(CRONTAB_PATH, 'w') as f: - f.write(rules) - - logger.success(m18n.n('monitor_enabled')) - - -def monitor_disable(): - """ - Disable server monitoring - - """ - from yunohost.service import (service_status, service_disable, - service_stop) - - glances = service_status('glances') - if glances['status'] != 'inactive': - service_stop('glances') - if glances['loaded'] != 'disabled': - try: - service_disable('glances') - except YunohostError as e: - logger.warning(e.strerror) - - # Remove crontab - try: - os.remove(CRONTAB_PATH) - except: - pass - - logger.success(m18n.n('monitor_disabled')) - - -def _get_glances_api(): - """ - Retrieve Glances API running on the local server - - """ - try: - p = xmlrpclib.ServerProxy(GLANCES_URI) - p.system.methodHelp('getAll') - except (xmlrpclib.ProtocolError, IOError): - pass - else: - return p - - from yunohost.service import service_status - - if service_status('glances')['status'] != 'running': - raise YunohostError('monitor_not_enabled') - raise YunohostError('monitor_glances_con_failed') - - -def _extract_inet(string, skip_netmask=False, skip_loopback=True): - """ - Extract IP addresses (v4 and/or v6) from a string limited to one - address by protocol - - Keyword argument: - string -- String to search in - skip_netmask -- True to skip subnet mask extraction - skip_loopback -- False to include addresses reserved for the - loopback interface - - Returns: - A dict of {protocol: address} with protocol one of 'ipv4' or 'ipv6' - - """ - ip4_pattern = '((25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}' - ip6_pattern = '(((?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?)::((?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?)' - ip4_pattern += '/[0-9]{1,2})' if not skip_netmask else ')' - ip6_pattern += '/[0-9]{1,3})' if not skip_netmask else ')' - result = {} - - for m in re.finditer(ip4_pattern, string): - addr = m.group(1) - if skip_loopback and addr.startswith('127.'): - continue - - # Limit to only one result - result['ipv4'] = addr - break - - for m in re.finditer(ip6_pattern, string): - addr = m.group(1) - if skip_loopback and addr == '::1': - continue - - # Limit to only one result - result['ipv6'] = addr - break - - return result - - -def binary_to_human(n, customary=False): - """ - Convert bytes or bits into human readable format with binary prefix - - Keyword argument: - n -- Number to convert - customary -- Use customary symbol instead of IEC standard - - """ - symbols = ('Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi') - if customary: - symbols = ('K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y') - prefix = {} - for i, s in enumerate(symbols): - prefix[s] = 1 << (i + 1) * 10 - for s in reversed(symbols): - if n >= prefix[s]: - value = float(n) / prefix[s] - return '%.1f%s' % (value, s) - return "%s" % n - - -def _retrieve_stats(period, date=None): - """ - Retrieve statistics from pickle file - - Keyword argument: - period -- Time period to retrieve (day, week, month) - date -- Date of stats to retrieve - - """ - pkl_file = None - - # Retrieve pickle file - if date is not None: - timestamp = calendar.timegm(date) - pkl_file = '%s/%d_%s.pkl' % (STATS_PATH, timestamp, period) - else: - pkl_file = '%s/%s.pkl' % (STATS_PATH, period) - if not os.path.isfile(pkl_file): - return False - - # Read file and process its content - with open(pkl_file, 'r') as f: - result = pickle.load(f) - if not isinstance(result, dict): - return None - return result - - -def _save_stats(stats, period, date=None): - """ - Save statistics to pickle file - - Keyword argument: - stats -- Stats dict to save - period -- Time period of stats (day, week, month) - date -- Date of stats - - """ - pkl_file = None - - # Set pickle file name - if date is not None: - timestamp = calendar.timegm(date) - pkl_file = '%s/%d_%s.pkl' % (STATS_PATH, timestamp, period) - else: - pkl_file = '%s/%s.pkl' % (STATS_PATH, period) - if not os.path.isdir(STATS_PATH): - os.makedirs(STATS_PATH) - - # Limit stats - if date is None: - t = stats['timestamp'] - limit = {'day': 86400, 'week': 604800, 'month': 2419200} - if (t[len(t) - 1] - t[0]) > limit[period]: - begin = t[len(t) - 1] - limit[period] - stats = _filter_stats(stats, begin) - - # Write file content - with open(pkl_file, 'w') as f: - pickle.dump(stats, f) - return True - - -def _monitor_all(period=None, since=None): - """ - Monitor all units (disk, network and system) for the given period - If since is None, real-time monitoring is returned. Otherwise, the - mean of stats since this timestamp is calculated and returned. - - Keyword argument: - period -- Time period to monitor (day, week, month) - since -- Timestamp of the stats beginning - - """ - result = {'disk': {}, 'network': {}, 'system': {}} - - # Real-time stats - if period == 'day' and since is None: - result['disk'] = monitor_disk() - result['network'] = monitor_network() - result['system'] = monitor_system() - return result - - # Retrieve stats and calculate mean - stats = _retrieve_stats(period) - if not stats: - return None - stats = _filter_stats(stats, since) - if not stats: - return None - result = _calculate_stats_mean(stats) - - return result - - -def _filter_stats(stats, t_begin=None, t_end=None): - """ - Filter statistics by beginning and/or ending timestamp - - Keyword argument: - stats -- Dict stats to filter - t_begin -- Beginning timestamp - t_end -- Ending timestamp - - """ - if t_begin is None and t_end is None: - return stats - - i_begin = i_end = None - # Look for indexes of timestamp interval - for i, t in enumerate(stats['timestamp']): - if t_begin and i_begin is None and t >= t_begin: - i_begin = i - if t_end and i != 0 and i_end is None and t > t_end: - i_end = i - # Check indexes - if i_begin is None: - if t_begin and t_begin > stats['timestamp'][0]: - return None - i_begin = 0 - if i_end is None: - if t_end and t_end < stats['timestamp'][0]: - return None - i_end = len(stats['timestamp']) - if i_begin == 0 and i_end == len(stats['timestamp']): - return stats - - # Filter function - def _filter(s, i, j): - for k, v in s.items(): - if isinstance(v, dict): - s[k] = _filter(v, i, j) - elif isinstance(v, list): - s[k] = v[i:j] - return s - - stats = _filter(stats, i_begin, i_end) - return stats - - -def _calculate_stats_mean(stats): - """ - Calculate the weighted mean for each statistic - - Keyword argument: - stats -- Stats dict to process - - """ - timestamp = stats['timestamp'] - t_sum = sum(timestamp) - del stats['timestamp'] - - # Weighted mean function - def _mean(s, t, ts): - for k, v in s.items(): - if isinstance(v, dict): - s[k] = _mean(v, t, ts) - elif isinstance(v, list): - try: - nums = [float(x * t[i]) for i, x in enumerate(v)] - except: - pass - else: - s[k] = sum(nums) / float(ts) - return s - - stats = _mean(stats, timestamp, t_sum) - return stats - - -def _append_to_stats(stats, monitor, statics=[]): - """ - Append monitoring statistics to current statistics - - Keyword argument: - stats -- Current stats dict - monitor -- Monitoring statistics - statics -- List of stats static keys - - """ - if isinstance(statics, str): - statics = [statics] - - # Appending function - def _append(s, m, st): - for k, v in m.items(): - if k in st: - s[k] = v - elif isinstance(v, dict): - if k not in s: - s[k] = {} - s[k] = _append(s[k], v, st) - else: - if k not in s: - s[k] = [] - if isinstance(v, list): - s[k].extend(v) - else: - s[k].append(v) - return s - - stats = _append(stats, monitor, statics) - return stats From f5509b7be762f7f0fbdcf19adeef74b4f42a2aa8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 18 Oct 2019 20:20:22 +0200 Subject: [PATCH 72/94] Remove app_debug, unused stuff, not really relevant, now basically superseded by the new log system... --- data/actionsmap/yunohost.yml | 8 -------- src/yunohost/app.py | 22 ---------------------- 2 files changed, 30 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 22037f05f..400599c48 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -754,14 +754,6 @@ app: full: --sql help: Initial SQL file - ### app_debug() - debug: - action_help: Display all debug informations for an application - api: GET /apps//debug - arguments: - app: - help: App name - ### app_makedefault() makedefault: action_help: Redirect domain root to an app diff --git a/src/yunohost/app.py b/src/yunohost/app.py index f4c504505..d8715b278 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1253,28 +1253,6 @@ def app_clearaccess(apps): return {'allowed_users': output} -def app_debug(app): - """ - Display debug informations for an app - - Keyword argument: - app - """ - manifest = _get_manifest_of_app(os.path.join(APPS_SETTING_PATH, app)) - - return { - 'name': manifest['id'], - 'label': manifest['name'], - 'services': [{ - "name": x, - "logs": [{ - "file_name": y, - "file_content": "\n".join(z), - } for (y, z) in sorted(service_log(x).items(), key=lambda x: x[0])], - } for x in sorted(manifest.get("services", []))] - } - - @is_unit_operation() def app_makedefault(operation_logger, app, domain=None): """ From 7c95b8d5082109abfd53146fa21d2789116a9dbc Mon Sep 17 00:00:00 2001 From: Kayou Date: Fri, 25 Oct 2019 19:39:02 +0900 Subject: [PATCH 73/94] fix find udp port --- data/helpers.d/network | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/network b/data/helpers.d/network index 0f75cb165..a4a679d3a 100644 --- a/data/helpers.d/network +++ b/data/helpers.d/network @@ -17,7 +17,7 @@ ynh_find_port () { ynh_handle_getopts_args "$@" test -n "$port" || ynh_die --message="The argument of ynh_find_port must be a valid port." - while netcat -z 127.0.0.1 $port # Check if the port is free + while netstat -ltu | grep -q -w :$port # Check if the port is free do port=$((port+1)) # Else, pass to next port done From 6cd12d450afea7f53d1f40eda83db04b88cecf26 Mon Sep 17 00:00:00 2001 From: Kayou Date: Fri, 25 Oct 2019 19:55:33 +0900 Subject: [PATCH 74/94] Update data/helpers.d/network Co-Authored-By: Alexandre Aubin --- data/helpers.d/network | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/network b/data/helpers.d/network index a4a679d3a..5fc41fa50 100644 --- a/data/helpers.d/network +++ b/data/helpers.d/network @@ -17,7 +17,7 @@ ynh_find_port () { ynh_handle_getopts_args "$@" test -n "$port" || ynh_die --message="The argument of ynh_find_port must be a valid port." - while netstat -ltu | grep -q -w :$port # Check if the port is free + while netstat -nltu | grep -q -w :$port # Check if the port is free do port=$((port+1)) # Else, pass to next port done From 89ab4bd4dc7083d43666ce8ad260fd6dd4c77b48 Mon Sep 17 00:00:00 2001 From: Rafi59 Date: Sat, 26 Oct 2019 13:25:38 +0200 Subject: [PATCH 75/94] Use ss instead of netstat --- data/helpers.d/network | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/network b/data/helpers.d/network index 5fc41fa50..948a327b0 100644 --- a/data/helpers.d/network +++ b/data/helpers.d/network @@ -17,7 +17,7 @@ ynh_find_port () { ynh_handle_getopts_args "$@" test -n "$port" || ynh_die --message="The argument of ynh_find_port must be a valid port." - while netstat -nltu | grep -q -w :$port # Check if the port is free + while ss -nltu | grep -q -w :$port # Check if the port is free do port=$((port+1)) # Else, pass to next port done From b630724ec9f71190419d92b73e9ac8b1ee93e301 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 4 Nov 2019 22:32:30 +0100 Subject: [PATCH 76/94] Can't have duplicated routes --- data/actionsmap/yunohost.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index b7a87ea75..75d51b04e 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1549,9 +1549,6 @@ tools: ### tools_maindomain() maindomain: action_help: Check the current main domain, or change it - api: - - GET /domains/main - - PUT /domains/main arguments: -n: full: --new-main-domain From 9bcbedc356243e78fa3f741f778185f8489c3d75 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 4 Nov 2019 23:02:04 +0100 Subject: [PATCH 77/94] Revert "[fix] moulinette logs were never displayed #lol" This reverts commit 6a0959dd1d7ba58b1bfa19de9ae9e2d7881ad556. --- bin/yunohost | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/yunohost b/bin/yunohost index 672c1b539..10a21a9da 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -145,7 +145,7 @@ def _init_moulinette(debug=False, quiet=False): }, 'moulinette': { 'level': level, - 'handlers': handlers, + 'handlers': [], 'propagate': True, }, 'moulinette.interface': { From 0e98d37c49500182cddff26ba4dad60174bf52f2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 14 Oct 2019 04:48:56 +0200 Subject: [PATCH 78/94] Remove the whole monitoring / glances stuff --- data/actionsmap/yunohost.yml | 141 ----- data/templates/glances/glances.default | 5 - data/templates/yunohost/services.yml | 2 +- debian/control | 2 +- locales/en.json | 14 - src/yunohost/monitor.py | 740 ------------------------- 6 files changed, 2 insertions(+), 902 deletions(-) delete mode 100644 data/templates/glances/glances.default delete mode 100644 src/yunohost/monitor.py diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 75d51b04e..d7de45bfe 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -975,147 +975,6 @@ backup: pattern: *pattern_backup_archive_name -############################# -# Monitor # -############################# -monitor: - category_help: Monitor the server - actions: - - ### monitor_disk() - disk: - action_help: Monitor disk space and usage - api: GET /monitor/disk - arguments: - -f: - full: --filesystem - help: Show filesystem disk space - action: append_const - const: filesystem - dest: units - -t: - full: --io - help: Show I/O throughput - action: append_const - const: io - dest: units - -m: - full: --mountpoint - help: Monitor only the device mounted on MOUNTPOINT - action: store - -H: - full: --human-readable - help: Print sizes in human readable format - action: store_true - - ### monitor_network() - network: - action_help: Monitor network interfaces - api: GET /monitor/network - arguments: - -u: - full: --usage - help: Show interfaces bit rates - action: append_const - const: usage - dest: units - -i: - full: --infos - help: Show network informations - action: append_const - const: infos - dest: units - -c: - full: --check - help: Check network configuration - action: append_const - const: check - dest: units - -H: - full: --human-readable - help: Print sizes in human readable format - action: store_true - - ### monitor_system() - system: - action_help: Monitor system informations and usage - api: GET /monitor/system - arguments: - -m: - full: --memory - help: Show memory usage - action: append_const - const: memory - dest: units - -c: - full: --cpu - help: Show CPU usage and load - action: append_const - const: cpu - dest: units - -p: - full: --process - help: Show processes summary - action: append_const - const: process - dest: units - -u: - full: --uptime - help: Show the system uptime - action: append_const - const: uptime - dest: units - -i: - full: --infos - help: Show system informations - action: append_const - const: infos - dest: units - -H: - full: --human-readable - help: Print sizes in human readable format - action: store_true - - ### monitor_updatestats() - update-stats: - action_help: Update monitoring statistics - api: POST /monitor/stats - arguments: - period: - help: Time period to update - choices: - - day - - week - - month - - ### monitor_showstats() - show-stats: - action_help: Show monitoring statistics - api: GET /monitor/stats - arguments: - period: - help: Time period to show - choices: - - day - - week - - month - - ### monitor_enable() - enable: - action_help: Enable server monitoring - api: PUT /monitor - arguments: - -s: - full: --with-stats - help: Enable monitoring statistics - action: store_true - - ### monitor_disable() - disable: - api: DELETE /monitor - action_help: Disable server monitoring - - ############################# # Settings # ############################# diff --git a/data/templates/glances/glances.default b/data/templates/glances/glances.default deleted file mode 100644 index 22337a0d9..000000000 --- a/data/templates/glances/glances.default +++ /dev/null @@ -1,5 +0,0 @@ -# Default is to launch glances with '-s' option. -DAEMON_ARGS="-s -B 127.0.0.1" - -# Change to 'true' to have glances running at startup -RUN="true" diff --git a/data/templates/yunohost/services.yml b/data/templates/yunohost/services.yml index 0d79b182f..1c0ee031f 100644 --- a/data/templates/yunohost/services.yml +++ b/data/templates/yunohost/services.yml @@ -17,7 +17,6 @@ redis-server: mysql: log: [/var/log/mysql.log,/var/log/mysql.err] alternates: ['mariadb'] -glances: {} ssh: log: /var/log/auth.log metronome: @@ -32,6 +31,7 @@ yunohost-firewall: need_lock: true nslcd: log: /var/log/syslog +glances: null nsswitch: null ssl: null yunohost: null diff --git a/debian/control b/debian/control index b0de9032b..a6d64e0b7 100644 --- a/debian/control +++ b/debian/control @@ -15,7 +15,7 @@ Depends: ${python:Depends}, ${misc:Depends} , python-psutil, python-requests, python-dnspython, python-openssl , python-apt, python-miniupnpc, python-dbus, python-jinja2 , python-toml - , glances, apt-transport-https + , apt-transport-https , dnsutils, bind9utils, unzip, git, curl, cron, wget, jq , ca-certificates, netcat-openbsd, iproute2 , mariadb-server, php-mysql | php-mysqlnd diff --git a/locales/en.json b/locales/en.json index 16c900c85..61c758df1 100644 --- a/locales/en.json +++ b/locales/en.json @@ -375,21 +375,9 @@ "migrations_skip_migration": "Skipping migration {id}…", "migrations_success_forward": "Migration {id} completed", "migrations_to_be_ran_manually": "Migration {id} has to be run manually. Please go to Tools → Migrations on the webadmin page, or run `yunohost tools migrations migrate`.", - "monitor_disabled": "Server monitoring now off", - "monitor_enabled": "Server monitoring now on", - "monitor_glances_con_failed": "Could not connect to Glances server", - "monitor_not_enabled": "Server monitoring is off", - "monitor_period_invalid": "Invalid time period", - "monitor_stats_file_not_found": "Could not find the statistics file", - "monitor_stats_no_update": "No monitoring statistics to update", - "monitor_stats_period_unavailable": "No available statistics for the period", - "mountpoint_unknown": "Unknown mountpoint", "mysql_db_creation_failed": "Could not create MySQL database", "mysql_db_init_failed": "Could not initialize MySQL database", "mysql_db_initialized": "The MySQL database is now initialized", - "network_check_mx_ko": "DNS MX record is not set", - "network_check_smtp_ko": "Outbound e-mail (SMTP port 25) seems to be blocked by your network", - "network_check_smtp_ok": "Outbound e-mail (SMTP port 25) is not blocked", "no_internet_connection": "The server is not connected to the Internet", "not_enough_disk_space": "Not enough free space on '{path:s}'", "operation_interrupted": "The operation was manually interrupted?", @@ -479,7 +467,6 @@ "service_description_dnsmasq": "Handles domain name resolution (DNS)", "service_description_dovecot": "Allows e-mail clients to access/fetch email (via IMAP and POP3)", "service_description_fail2ban": "Protects against brute-force and other kinds of attacks from the Internet", - "service_description_glances": "Monitors system info on your server", "service_description_metronome": "Manage XMPP instant messaging accounts", "service_description_mysql": "Stores app data (SQL database)", "service_description_nginx": "Serves or provides access to all the websites hosted on your server", @@ -529,7 +516,6 @@ "tools_upgrade_special_packages_completed": "YunoHost package upgrade completed.\nPress [Enter] to get the command line back", "unbackup_app": "App '{app:s}' will not be saved", "unexpected_error": "Something unexpected went wrong: {error}", - "unit_unknown": "Unknown unit '{unit:s}'", "unlimit": "No quota", "unrestore_app": "App '{app:s}' will not be restored", "update_apt_cache_failed": "Could not to update the cache of APT (Debian's package manager). Here is a dump of the sources.list lines, which might help identify problematic lines: \n{sourceslist}", diff --git a/src/yunohost/monitor.py b/src/yunohost/monitor.py deleted file mode 100644 index 7af55f287..000000000 --- a/src/yunohost/monitor.py +++ /dev/null @@ -1,740 +0,0 @@ -# -*- coding: utf-8 -*- - -""" License - - Copyright (C) 2013 YunoHost - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program; if not, see http://www.gnu.org/licenses - -""" - -""" yunohost_monitor.py - - Monitoring functions -""" -import re -import json -import time -import psutil -import calendar -import subprocess -import xmlrpclib -import os.path -import os -import dns.resolver -import cPickle as pickle -from datetime import datetime - -from moulinette import m18n -from yunohost.utils.error import YunohostError -from moulinette.utils.log import getActionLogger - -from yunohost.utils.network import get_public_ip -from yunohost.domain import _get_maindomain - -logger = getActionLogger('yunohost.monitor') - -GLANCES_URI = 'http://127.0.0.1:61209' -STATS_PATH = '/var/lib/yunohost/stats' -CRONTAB_PATH = '/etc/cron.d/yunohost-monitor' - - -def monitor_disk(units=None, mountpoint=None, human_readable=False): - """ - Monitor disk space and usage - - Keyword argument: - units -- Unit(s) to monitor - mountpoint -- Device mountpoint - human_readable -- Print sizes in human readable format - - """ - glances = _get_glances_api() - result_dname = None - result = {} - - if units is None: - units = ['io', 'filesystem'] - - _format_dname = lambda d: (os.path.realpath(d)).replace('/dev/', '') - - # Get mounted devices - devices = {} - for p in psutil.disk_partitions(all=True): - if not p.device.startswith('/dev/') or not p.mountpoint: - continue - if mountpoint is None: - devices[_format_dname(p.device)] = p.mountpoint - elif mountpoint == p.mountpoint: - dn = _format_dname(p.device) - devices[dn] = p.mountpoint - result_dname = dn - if len(devices) == 0: - if mountpoint is not None: - raise YunohostError('mountpoint_unknown') - return result - - # Retrieve monitoring for unit(s) - for u in units: - if u == 'io': - # Define setter - if len(units) > 1: - def _set(dn, dvalue): - try: - result[dn][u] = dvalue - except KeyError: - result[dn] = {u: dvalue} - else: - def _set(dn, dvalue): - result[dn] = dvalue - - # Iterate over values - devices_names = devices.keys() - for d in json.loads(glances.getDiskIO()): - dname = d.pop('disk_name') - try: - devices_names.remove(dname) - except: - continue - else: - _set(dname, d) - for dname in devices_names: - _set(dname, 'not-available') - elif u == 'filesystem': - # Define setter - if len(units) > 1: - def _set(dn, dvalue): - try: - result[dn][u] = dvalue - except KeyError: - result[dn] = {u: dvalue} - else: - def _set(dn, dvalue): - result[dn] = dvalue - - # Iterate over values - devices_names = devices.keys() - for d in json.loads(glances.getFs()): - dname = _format_dname(d.pop('device_name')) - try: - devices_names.remove(dname) - except: - continue - else: - d['avail'] = d['size'] - d['used'] - if human_readable: - for i in ['used', 'avail', 'size']: - d[i] = binary_to_human(d[i]) + 'B' - _set(dname, d) - for dname in devices_names: - _set(dname, 'not-available') - else: - raise YunohostError('unit_unknown', unit=u) - - if result_dname is not None: - return result[result_dname] - return result - - -def monitor_network(units=None, human_readable=False): - """ - Monitor network interfaces - - Keyword argument: - units -- Unit(s) to monitor - human_readable -- Print sizes in human readable format - - """ - glances = _get_glances_api() - result = {} - - if units is None: - units = ['check', 'usage', 'infos'] - - # Get network devices and their addresses - # TODO / FIXME : use functions in utils/network.py to manage this - devices = {} - output = subprocess.check_output('ip addr show'.split()) - for d in re.split('^(?:[0-9]+: )', output, flags=re.MULTILINE): - # Extract device name (1) and its addresses (2) - m = re.match('([^\s@]+)(?:@[\S]+)?: (.*)', d, flags=re.DOTALL) - if m: - devices[m.group(1)] = m.group(2) - - # Retrieve monitoring for unit(s) - for u in units: - if u == 'check': - result[u] = {} - domain = _get_maindomain() - cmd_check_smtp = os.system('/bin/nc -z -w1 yunohost.org 25') - if cmd_check_smtp == 0: - smtp_check = m18n.n('network_check_smtp_ok') - else: - smtp_check = m18n.n('network_check_smtp_ko') - - try: - answers = dns.resolver.query(domain, 'MX') - mx_check = {} - i = 0 - for server in answers: - mx_id = 'mx%s' % i - mx_check[mx_id] = server - i = i + 1 - except: - mx_check = m18n.n('network_check_mx_ko') - result[u] = { - 'smtp_check': smtp_check, - 'mx_check': mx_check - } - elif u == 'usage': - result[u] = {} - for i in json.loads(glances.getNetwork()): - iname = i['interface_name'] - if iname in devices.keys(): - del i['interface_name'] - if human_readable: - for k in i.keys(): - if k != 'time_since_update': - i[k] = binary_to_human(i[k]) + 'B' - result[u][iname] = i - else: - logger.debug('interface name %s was not found', iname) - elif u == 'infos': - p_ipv4 = get_public_ip() or 'unknown' - - # TODO / FIXME : use functions in utils/network.py to manage this - l_ip = 'unknown' - for name, addrs in devices.items(): - if name == 'lo': - continue - if not isinstance(l_ip, dict): - l_ip = {} - l_ip[name] = _extract_inet(addrs) - - gateway = 'unknown' - output = subprocess.check_output('ip route show'.split()) - m = re.search('default via (.*) dev ([a-z]+[0-9]?)', output) - if m: - addr = _extract_inet(m.group(1), True) - if len(addr) == 1: - proto, gateway = addr.popitem() - - result[u] = { - 'public_ip': p_ipv4, - 'local_ip': l_ip, - 'gateway': gateway, - } - else: - raise YunohostError('unit_unknown', unit=u) - - if len(units) == 1: - return result[units[0]] - return result - - -def monitor_system(units=None, human_readable=False): - """ - Monitor system informations and usage - - Keyword argument: - units -- Unit(s) to monitor - human_readable -- Print sizes in human readable format - - """ - glances = _get_glances_api() - result = {} - - if units is None: - units = ['memory', 'cpu', 'process', 'uptime', 'infos'] - - # Retrieve monitoring for unit(s) - for u in units: - if u == 'memory': - ram = json.loads(glances.getMem()) - swap = json.loads(glances.getMemSwap()) - if human_readable: - for i in ram.keys(): - if i != 'percent': - ram[i] = binary_to_human(ram[i]) + 'B' - for i in swap.keys(): - if i != 'percent': - swap[i] = binary_to_human(swap[i]) + 'B' - result[u] = { - 'ram': ram, - 'swap': swap - } - elif u == 'cpu': - result[u] = { - 'load': json.loads(glances.getLoad()), - 'usage': json.loads(glances.getCpu()) - } - elif u == 'process': - result[u] = json.loads(glances.getProcessCount()) - elif u == 'uptime': - result[u] = (str(datetime.now() - datetime.fromtimestamp(psutil.boot_time())).split('.')[0]) - elif u == 'infos': - result[u] = json.loads(glances.getSystem()) - else: - raise YunohostError('unit_unknown', unit=u) - - if len(units) == 1 and not isinstance(result[units[0]], str): - return result[units[0]] - return result - - -def monitor_update_stats(period): - """ - Update monitoring statistics - - Keyword argument: - period -- Time period to update (day, week, month) - - """ - if period not in ['day', 'week', 'month']: - raise YunohostError('monitor_period_invalid') - - stats = _retrieve_stats(period) - if not stats: - stats = {'disk': {}, 'network': {}, 'system': {}, 'timestamp': []} - - monitor = None - # Get monitoring stats - if period == 'day': - monitor = _monitor_all('day') - else: - t = stats['timestamp'] - p = 'day' if period == 'week' else 'week' - if len(t) > 0: - monitor = _monitor_all(p, t[len(t) - 1]) - else: - monitor = _monitor_all(p, 0) - if not monitor: - raise YunohostError('monitor_stats_no_update') - - stats['timestamp'].append(time.time()) - - # Append disk stats - for dname, units in monitor['disk'].items(): - disk = {} - # Retrieve current stats for disk name - if dname in stats['disk'].keys(): - disk = stats['disk'][dname] - - for unit, values in units.items(): - # Continue if unit doesn't contain stats - if not isinstance(values, dict): - continue - - # Retrieve current stats for unit and append new ones - curr = disk[unit] if unit in disk.keys() else {} - if unit == 'io': - disk[unit] = _append_to_stats(curr, values, 'time_since_update') - elif unit == 'filesystem': - disk[unit] = _append_to_stats(curr, values, ['fs_type', 'mnt_point']) - stats['disk'][dname] = disk - - # Append network stats - net_usage = {} - for iname, values in monitor['network']['usage'].items(): - # Continue if units doesn't contain stats - if not isinstance(values, dict): - continue - - # Retrieve current stats and append new ones - curr = {} - if 'usage' in stats['network'] and iname in stats['network']['usage']: - curr = stats['network']['usage'][iname] - net_usage[iname] = _append_to_stats(curr, values, 'time_since_update') - stats['network'] = {'usage': net_usage, 'infos': monitor['network']['infos']} - - # Append system stats - for unit, values in monitor['system'].items(): - # Continue if units doesn't contain stats - if not isinstance(values, dict): - continue - - # Set static infos unit - if unit == 'infos': - stats['system'][unit] = values - continue - - # Retrieve current stats and append new ones - curr = stats['system'][unit] if unit in stats['system'].keys() else {} - stats['system'][unit] = _append_to_stats(curr, values) - - _save_stats(stats, period) - - -def monitor_show_stats(period, date=None): - """ - Show monitoring statistics - - Keyword argument: - period -- Time period to show (day, week, month) - - """ - if period not in ['day', 'week', 'month']: - raise YunohostError('monitor_period_invalid') - - result = _retrieve_stats(period, date) - if result is False: - raise YunohostError('monitor_stats_file_not_found') - elif result is None: - raise YunohostError('monitor_stats_period_unavailable') - return result - - -def monitor_enable(with_stats=False): - """ - Enable server monitoring - - Keyword argument: - with_stats -- Enable monitoring statistics - - """ - from yunohost.service import (service_status, service_enable, - service_start) - - glances = service_status('glances') - if glances['status'] != 'running': - service_start('glances') - if glances['loaded'] != 'enabled': - service_enable('glances') - - # Install crontab - if with_stats: - # day: every 5 min # week: every 1 h # month: every 4 h # - rules = ('*/5 * * * * root {cmd} day >> /dev/null\n' - '3 * * * * root {cmd} week >> /dev/null\n' - '6 */4 * * * root {cmd} month >> /dev/null').format( - cmd='/usr/bin/yunohost --quiet monitor update-stats') - with open(CRONTAB_PATH, 'w') as f: - f.write(rules) - - logger.success(m18n.n('monitor_enabled')) - - -def monitor_disable(): - """ - Disable server monitoring - - """ - from yunohost.service import (service_status, service_disable, - service_stop) - - glances = service_status('glances') - if glances['status'] != 'inactive': - service_stop('glances') - if glances['loaded'] != 'disabled': - try: - service_disable('glances') - except YunohostError as e: - logger.warning(e.strerror) - - # Remove crontab - try: - os.remove(CRONTAB_PATH) - except: - pass - - logger.success(m18n.n('monitor_disabled')) - - -def _get_glances_api(): - """ - Retrieve Glances API running on the local server - - """ - try: - p = xmlrpclib.ServerProxy(GLANCES_URI) - p.system.methodHelp('getAll') - except (xmlrpclib.ProtocolError, IOError): - pass - else: - return p - - from yunohost.service import service_status - - if service_status('glances')['status'] != 'running': - raise YunohostError('monitor_not_enabled') - raise YunohostError('monitor_glances_con_failed') - - -def _extract_inet(string, skip_netmask=False, skip_loopback=True): - """ - Extract IP addresses (v4 and/or v6) from a string limited to one - address by protocol - - Keyword argument: - string -- String to search in - skip_netmask -- True to skip subnet mask extraction - skip_loopback -- False to include addresses reserved for the - loopback interface - - Returns: - A dict of {protocol: address} with protocol one of 'ipv4' or 'ipv6' - - """ - ip4_pattern = '((25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}' - ip6_pattern = '(((?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?)::((?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?)' - ip4_pattern += '/[0-9]{1,2})' if not skip_netmask else ')' - ip6_pattern += '/[0-9]{1,3})' if not skip_netmask else ')' - result = {} - - for m in re.finditer(ip4_pattern, string): - addr = m.group(1) - if skip_loopback and addr.startswith('127.'): - continue - - # Limit to only one result - result['ipv4'] = addr - break - - for m in re.finditer(ip6_pattern, string): - addr = m.group(1) - if skip_loopback and addr == '::1': - continue - - # Limit to only one result - result['ipv6'] = addr - break - - return result - - -def binary_to_human(n, customary=False): - """ - Convert bytes or bits into human readable format with binary prefix - - Keyword argument: - n -- Number to convert - customary -- Use customary symbol instead of IEC standard - - """ - symbols = ('Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi') - if customary: - symbols = ('K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y') - prefix = {} - for i, s in enumerate(symbols): - prefix[s] = 1 << (i + 1) * 10 - for s in reversed(symbols): - if n >= prefix[s]: - value = float(n) / prefix[s] - return '%.1f%s' % (value, s) - return "%s" % n - - -def _retrieve_stats(period, date=None): - """ - Retrieve statistics from pickle file - - Keyword argument: - period -- Time period to retrieve (day, week, month) - date -- Date of stats to retrieve - - """ - pkl_file = None - - # Retrieve pickle file - if date is not None: - timestamp = calendar.timegm(date) - pkl_file = '%s/%d_%s.pkl' % (STATS_PATH, timestamp, period) - else: - pkl_file = '%s/%s.pkl' % (STATS_PATH, period) - if not os.path.isfile(pkl_file): - return False - - # Read file and process its content - with open(pkl_file, 'r') as f: - result = pickle.load(f) - if not isinstance(result, dict): - return None - return result - - -def _save_stats(stats, period, date=None): - """ - Save statistics to pickle file - - Keyword argument: - stats -- Stats dict to save - period -- Time period of stats (day, week, month) - date -- Date of stats - - """ - pkl_file = None - - # Set pickle file name - if date is not None: - timestamp = calendar.timegm(date) - pkl_file = '%s/%d_%s.pkl' % (STATS_PATH, timestamp, period) - else: - pkl_file = '%s/%s.pkl' % (STATS_PATH, period) - if not os.path.isdir(STATS_PATH): - os.makedirs(STATS_PATH) - - # Limit stats - if date is None: - t = stats['timestamp'] - limit = {'day': 86400, 'week': 604800, 'month': 2419200} - if (t[len(t) - 1] - t[0]) > limit[period]: - begin = t[len(t) - 1] - limit[period] - stats = _filter_stats(stats, begin) - - # Write file content - with open(pkl_file, 'w') as f: - pickle.dump(stats, f) - return True - - -def _monitor_all(period=None, since=None): - """ - Monitor all units (disk, network and system) for the given period - If since is None, real-time monitoring is returned. Otherwise, the - mean of stats since this timestamp is calculated and returned. - - Keyword argument: - period -- Time period to monitor (day, week, month) - since -- Timestamp of the stats beginning - - """ - result = {'disk': {}, 'network': {}, 'system': {}} - - # Real-time stats - if period == 'day' and since is None: - result['disk'] = monitor_disk() - result['network'] = monitor_network() - result['system'] = monitor_system() - return result - - # Retrieve stats and calculate mean - stats = _retrieve_stats(period) - if not stats: - return None - stats = _filter_stats(stats, since) - if not stats: - return None - result = _calculate_stats_mean(stats) - - return result - - -def _filter_stats(stats, t_begin=None, t_end=None): - """ - Filter statistics by beginning and/or ending timestamp - - Keyword argument: - stats -- Dict stats to filter - t_begin -- Beginning timestamp - t_end -- Ending timestamp - - """ - if t_begin is None and t_end is None: - return stats - - i_begin = i_end = None - # Look for indexes of timestamp interval - for i, t in enumerate(stats['timestamp']): - if t_begin and i_begin is None and t >= t_begin: - i_begin = i - if t_end and i != 0 and i_end is None and t > t_end: - i_end = i - # Check indexes - if i_begin is None: - if t_begin and t_begin > stats['timestamp'][0]: - return None - i_begin = 0 - if i_end is None: - if t_end and t_end < stats['timestamp'][0]: - return None - i_end = len(stats['timestamp']) - if i_begin == 0 and i_end == len(stats['timestamp']): - return stats - - # Filter function - def _filter(s, i, j): - for k, v in s.items(): - if isinstance(v, dict): - s[k] = _filter(v, i, j) - elif isinstance(v, list): - s[k] = v[i:j] - return s - - stats = _filter(stats, i_begin, i_end) - return stats - - -def _calculate_stats_mean(stats): - """ - Calculate the weighted mean for each statistic - - Keyword argument: - stats -- Stats dict to process - - """ - timestamp = stats['timestamp'] - t_sum = sum(timestamp) - del stats['timestamp'] - - # Weighted mean function - def _mean(s, t, ts): - for k, v in s.items(): - if isinstance(v, dict): - s[k] = _mean(v, t, ts) - elif isinstance(v, list): - try: - nums = [float(x * t[i]) for i, x in enumerate(v)] - except: - pass - else: - s[k] = sum(nums) / float(ts) - return s - - stats = _mean(stats, timestamp, t_sum) - return stats - - -def _append_to_stats(stats, monitor, statics=[]): - """ - Append monitoring statistics to current statistics - - Keyword argument: - stats -- Current stats dict - monitor -- Monitoring statistics - statics -- List of stats static keys - - """ - if isinstance(statics, str): - statics = [statics] - - # Appending function - def _append(s, m, st): - for k, v in m.items(): - if k in st: - s[k] = v - elif isinstance(v, dict): - if k not in s: - s[k] = {} - s[k] = _append(s[k], v, st) - else: - if k not in s: - s[k] = [] - if isinstance(v, list): - s[k].extend(v) - else: - s[k].append(v) - return s - - stats = _append(stats, monitor, statics) - return stats From e463e282f28d408d3a1d53fe74000fd238ba0ef8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 6 Nov 2019 17:19:28 +0100 Subject: [PATCH 79/94] Yolo fix to avoid regenerating glances' conf during upgrade to 3.8 --- src/yunohost/regenconf.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/yunohost/regenconf.py b/src/yunohost/regenconf.py index b7a42dd9d..193b23435 100644 --- a/src/yunohost/regenconf.py +++ b/src/yunohost/regenconf.py @@ -131,6 +131,15 @@ def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run show_info=False)['hooks'] names.remove('ssh') + # Dirty hack for legacy code : avoid attempting to regen the conf for + # glances because it got removed ... This is only needed *once* + # during the upgrade from 3.7 to 3.8 because Yunohost will attempt to + # regen glance's conf *before* it gets automatically removed from + # services.yml (which will happens only during the regen-conf of + # 'yunohost', so at the very end of the regen-conf cycle) Anyway, + # this can be safely removed once we're in >= 4.0 + names.remove("glances") + pre_result = hook_callback('conf_regen', names, pre_callback=_pre_call) # Keep only the hook names with at least one success From aee62064bf57b91210be3bc858930b1101deba2a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 6 Nov 2019 18:20:14 +0100 Subject: [PATCH 80/94] Salvage binary_to_human from old monitor.py, needed for backup stuff --- src/yunohost/backup.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 213f2cec1..ddf64774e 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -48,7 +48,6 @@ from yunohost.app import ( from yunohost.hook import ( hook_list, hook_info, hook_callback, hook_exec, CUSTOM_HOOK_FOLDER ) -from yunohost.monitor import binary_to_human from yunohost.tools import tools_postinstall from yunohost.regenconf import regen_conf from yunohost.log import OperationLogger @@ -2492,3 +2491,23 @@ def disk_usage(path): du_output = subprocess.check_output(['du', '-sb', path]) return int(du_output.split()[0].decode('utf-8')) + + +def binary_to_human(n, customary=False): + """ + Convert bytes or bits into human readable format with binary prefix + Keyword argument: + n -- Number to convert + customary -- Use customary symbol instead of IEC standard + """ + symbols = ('Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi') + if customary: + symbols = ('K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y') + prefix = {} + for i, s in enumerate(symbols): + prefix[s] = 1 << (i + 1) * 10 + for s in reversed(symbols): + if n >= prefix[s]: + value = float(n) / prefix[s] + return '%.1f%s' % (value, s) + return "%s" % n From 76cbad0a9a75c06308505eabe58e6f50752da437 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 6 Nov 2019 18:35:21 +0100 Subject: [PATCH 81/94] Make sure the users actually exists when migrating legacy custom permissions --- .../data_migrations/0011_setup_group_permission.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/yunohost/data_migrations/0011_setup_group_permission.py b/src/yunohost/data_migrations/0011_setup_group_permission.py index e9ca32294..c80686344 100644 --- a/src/yunohost/data_migrations/0011_setup_group_permission.py +++ b/src/yunohost/data_migrations/0011_setup_group_permission.py @@ -7,7 +7,7 @@ from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_yaml from yunohost.tools import Migration -from yunohost.user import user_group_create, user_group_update +from yunohost.user import user_list, user_group_create, user_group_update from yunohost.app import app_setting, app_list from yunohost.regenconf import regen_conf, BACKUP_CONF_DIR from yunohost.permission import permission_create, user_permission_update, permission_sync_to_user @@ -109,10 +109,11 @@ class MyMigration(Migration): url = "/" if domain and path else None if permission: - allowed_groups = permission.split(',') + known_users = user_list()["users"].keys() + allowed = [user for user in permission.split(',') if user in known_users] else: - allowed_groups = ["all_users"] - permission_create(app+".main", url=url, allowed=allowed_groups, sync_perm=False) + allowed = ["all_users"] + permission_create(app+".main", url=url, allowed=allowed, sync_perm=False) app_setting(app, 'allowed_users', delete=True) From 1372ab916c8db501042397db58a4b91310fff3ab Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 6 Nov 2019 19:05:43 +0100 Subject: [PATCH 82/94] Stale strings + try to keep the namespace-like tidy --- locales/en.json | 11 +++-------- src/yunohost/diagnosis.py | 4 ++-- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/locales/en.json b/locales/en.json index 4a432345f..ebe6b4571 100644 --- a/locales/en.json +++ b/locales/en.json @@ -150,18 +150,11 @@ "confirm_app_install_thirdparty": "DANGER! This app is not part of Yunohost's app catalog. Installing third-party apps may compromise the integrity and security of your system. You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or break your system… If you are willing to take that risk anyway, type '{answers:s}'", "custom_app_url_required": "You must provide a URL to upgrade your custom app {app:s}", "custom_appslist_name_required": "You must provide a name for your custom app list", - "diagnosis_debian_version_error": "Could not retrieve the Debian version: {error}", - "diagnosis_kernel_version_error": "Could not retrieve kernel version: {error}", "diagnosis_basesystem_host": "Server is running Debian {debian_version}.", "diagnosis_basesystem_kernel": "Server is running Linux kernel {kernel_version}", "diagnosis_basesystem_ynh_single_version": "{0} version: {1}", "diagnosis_basesystem_ynh_main_version": "Server is running YunoHost {main_version}", "diagnosis_basesystem_ynh_inconsistent_versions": "You are running inconsistents versions of the YunoHost packages ... most probably because of a failed or partial upgrade.", - "diagnosis_monitor_disk_error": "Could not monitor disks: {error}", - "diagnosis_monitor_system_error": "Could not monitor system: {error}", - "diagnosis_no_apps": "No such installed app", - "dpkg_is_broken": "You cannot do this right now because dpkg/APT (the system package managers) seems to be in a broken state… You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.", - "dpkg_lock_not_available": "This command can't be ran right now because another program seems to be using the lock of dpkg (the system package manager)", "diagnosis_display_tip_web": "You can go to the Diagnosis section (in the home screen) to see the issues found.", "diagnosis_display_tip_cli": "You can run 'yunohost diagnosis show --issues' to display the issues found.", "diagnosis_failed_for_category": "Diagnosis failed for category '{category}' : {error}", @@ -222,6 +215,7 @@ "diagnosis_http_could_not_diagnose": "Could not diagnose if domain is reachable from outside. Error: {error}", "diagnosis_http_ok": "Domain {domain} is reachable from outside.", "diagnosis_http_unreachable": "Domain {domain} is unreachable through HTTP from outside.", + "diagnosis_unknown_categories": "The following categories are unknown : {categories}", "domain_cannot_remove_main": "You cannot remove '{domain:s}' since it's the main domain, you need first to set another domain as the main domain using 'yunohost domain main-domain -n ', here is the list of candidate domains: {other_domains:s}", "domain_cannot_remove_main_add_new_one": "You cannot remove '{domain:s}' since it's the main domain and your only domain, you need to first add another domain using 'yunohost domain add ', then set is as the main domain using 'yunohost domain main-domain -n ' and then you can remove the domain '{domain:s}' using 'yunohost domain remove {domain:s}'.'", "domain_cert_gen_failed": "Could not generate certificate", @@ -239,6 +233,8 @@ "domains_available": "Available domains:", "done": "Done", "downloading": "Downloading…", + "dpkg_is_broken": "You cannot do this right now because dpkg/APT (the system package managers) seems to be in a broken state… You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.", + "dpkg_lock_not_available": "This command can't be ran right now because another program seems to be using the lock of dpkg (the system package manager)", "dyndns_could_not_check_provide": "Could not check if {provider:s} can provide {domain:s}.", "dyndns_could_not_check_available": "Could not check if {domain:s} is available on {provider:s}.", "dyndns_cron_installed": "DynDNS cron job created", @@ -604,7 +600,6 @@ "user_update_failed": "Could not update user {user}: {error}", "user_updated": "User info changed", "users_available": "Available users:", - "unknown_categories": "The following categories are unknown : {categories}", "yunohost_already_installed": "YunoHost is already installed", "yunohost_ca_creation_failed": "Could not create certificate authority", "yunohost_ca_creation_success": "Local certification authority created.", diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index 19dd03042..121a0c2ae 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -56,7 +56,7 @@ def diagnosis_show(categories=[], issues=False, full=False, share=False): else: unknown_categories = [c for c in categories if c not in all_categories_names] if unknown_categories: - raise YunohostError('unknown_categories', categories=", ".join(categories)) + raise YunohostError('diagnosis_unknown_categories', categories=", ".join(categories)) # Fetch all reports all_reports = [] @@ -127,7 +127,7 @@ def diagnosis_run(categories=[], force=False): else: unknown_categories = [c for c in categories if c not in all_categories_names] if unknown_categories: - raise YunohostError('unknown_categories', categories=", ".join(unknown_categories)) + raise YunohostError('diagnosis_unknown_categories', categories=", ".join(unknown_categories)) issues = [] # Call the hook ... From da7ac5aacb3a70630fd46c76b1303254fe4bbe30 Mon Sep 17 00:00:00 2001 From: Dominik Roesli Date: Wed, 30 Oct 2019 08:27:33 +0000 Subject: [PATCH 83/94] Translated using Weblate (German) Currently translated at 34.3% (193 of 562 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/de/ --- locales/de.json | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/locales/de.json b/locales/de.json index ec7233973..08fdff7e4 100644 --- a/locales/de.json +++ b/locales/de.json @@ -293,7 +293,7 @@ "backup_abstract_method": "Diese Backup-Methode wird noch nicht unterstützt", "backup_applying_method_tar": "Erstellen des Backup-tar Archives…", "backup_applying_method_copy": "Kopiere alle Dateien ins Backup…", - "app_change_url_no_script": "Die Anwendung '{app_name:s}' unterstützt bisher keine URL-Modufikation. Vielleicht gibt es eine Aktualisierung.", + "app_change_url_no_script": "Die Anwendung '{app_name:s}' unterstützt bisher keine URL-Modifikation. Vielleicht sollte sie aktualisiert werden.", "app_location_unavailable": "Diese URL ist nicht verfügbar oder wird von einer installierten Anwendung genutzt:\n{apps:s}", "backup_applying_method_custom": "Rufe die benutzerdefinierte Backup-Methode '{method:s}' auf…", "backup_archive_system_part_not_available": "Der System-Teil '{part:s}' ist in diesem Backup nicht enthalten", @@ -350,7 +350,7 @@ "app_start_remove": "Anwendung {app} wird entfernt…", "app_start_install": "Anwendung {app} wird installiert…", "app_not_upgraded": "Die App '{failed_app}' konnte nicht aktualisiert werden. Infolgedessen wurden die folgenden App-Upgrades abgebrochen: {apps}", - "app_make_default_location_already_used": "Die App \"{app}\" kann nicht als Standard für die Domain \"{domain}\" festgelegt werden. Sie wird bereits von der anderen App \"{other_app}\" verwendet", + "app_make_default_location_already_used": "Die App \"{app}\" kann nicht als Standard für die Domain \"{domain}\" festgelegt werden. Sie wird bereits von der App \"{other_app}\" verwendet", "aborting": "Breche ab.", "app_action_cannot_be_ran_because_required_services_down": "Diese App erfordert einige Dienste, die derzeit nicht verfügbar sind. Bevor Sie fortfahren, sollten Sie versuchen, die folgenden Dienste neu zu starten (und möglicherweise untersuchen, warum sie nicht verfügbar sind): {services}", "already_up_to_date": "Nichts zu tun. Alles ist bereits auf dem neusten Stand.", @@ -414,5 +414,8 @@ "global_settings_key_doesnt_exists": "Der Schlüssel'{settings_key:s}' existiert nicht in den globalen Einstellungen, du kannst alle verfügbaren Schlüssel sehen, indem du 'yunohost settings list' ausführst", "log_app_makedefault": "Mache '{}' zur Standard-Anwendung", "hook_json_return_error": "Konnte die Rückkehr vom Einsprungpunkt {path:s} nicht lesen. Fehler: {msg:s}. Unformatierter Inhalt: {raw_content}", - "app_full_domain_unavailable": "Es tut uns leid, aber diese Anwendung erfordert die Installation einer vollständigen Domäne, aber einige andere Anwendungen sind bereits auf der Domäne'{domain}' installiert. Eine mögliche Lösung ist das Hinzufügen und Verwenden einer Subdomain, die dieser Anwendung zugeordnet ist." + "app_full_domain_unavailable": "Es tut uns leid, aber diese Anwendung erfordert die Installation auf einer eigenen Domain, aber einige andere Anwendungen sind bereits auf der Domäne'{domain}' installiert. Eine mögliche Lösung ist das Hinzufügen und Verwenden einer Subdomain, die dieser Anwendung zugeordnet ist.", + "app_install_failed": "Installation von {app} fehlgeschlagen: {error}", + "app_install_script_failed": "Im Installationsscript ist ein Fehler aufgetreten", + "app_remove_after_failed_install": "Entfernen der App nach fehlgeschlagener Installation…" } From d4dde0e5d76d350d7969dea55e38eeb85327665e Mon Sep 17 00:00:00 2001 From: xaloc33 Date: Sun, 3 Nov 2019 22:26:53 +0000 Subject: [PATCH 84/94] Translated using Weblate (Catalan) Currently translated at 100.0% (562 of 562 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/ca/ --- locales/ca.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/locales/ca.json b/locales/ca.json index e171726b0..c0e500a3c 100644 --- a/locales/ca.json +++ b/locales/ca.json @@ -626,5 +626,6 @@ "permission_already_up_to_date": "No s'ha actualitzat el permís perquè la petició d'afegir/eliminar ja corresponent a l'estat actual.", "permission_currently_allowed_for_visitors": "El permís ja el tenen el grup de visitants a més d'altres grups. Segurament s'hauria de revocar el permís al grup dels visitants o eliminar els altres grups als que s'ha atribuït.", "permission_currently_allowed_for_all_users": "El permís ha el té el grup de tots els usuaris (all_users) a més d'altres grups. Segurament s'hauria de revocar el permís a «all_users» o eliminar els altres grups als que s'ha atribuït.", - "permission_require_account": "El permís {permission} només té sentit per als usuaris que tenen un compte, i per tant no es pot activar per als visitants." + "permission_require_account": "El permís {permission} només té sentit per als usuaris que tenen un compte, i per tant no es pot activar per als visitants.", + "app_remove_after_failed_install": "Eliminant l'aplicació després que hagi fallat la instal·lació…" } From e8c98335f48f83bf5ca8c6a078f190af644fce55 Mon Sep 17 00:00:00 2001 From: Filip Bengtsson Date: Thu, 31 Oct 2019 04:34:15 +0000 Subject: [PATCH 85/94] Translated using Weblate (Swedish) Currently translated at 1.6% (9 of 562 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/sv/ --- locales/sv.json | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/locales/sv.json b/locales/sv.json index 4960d43aa..85572756d 100644 --- a/locales/sv.json +++ b/locales/sv.json @@ -1,3 +1,11 @@ { - "password_too_simple_1": "Lösenordet måste bestå av minst åtta tecken" + "password_too_simple_1": "Lösenordet måste bestå av minst åtta tecken", + "app_action_broke_system": "Åtgärden verkar ha fått följande viktiga tjänster att haverera: {services}", + "already_up_to_date": "Ingenting att göra. Allt är redan uppdaterat.", + "admin_password": "Administratörslösenord", + "admin_password_too_long": "Välj gärna ett lösenord som inte innehåller fler än 127 tecken", + "admin_password_change_failed": "Kan inte byta lösenord", + "action_invalid": "Ej tillåten åtgärd '{action:s}'", + "admin_password_changed": "Administratörskontots lösenord ändrades", + "aborting": "Avbryter." } From ab0deddbb86923d65fa08f8e48e0e9a970cb4b3a Mon Sep 17 00:00:00 2001 From: advocatux Date: Wed, 30 Oct 2019 15:30:49 +0000 Subject: [PATCH 86/94] Translated using Weblate (Spanish) Currently translated at 100.0% (562 of 562 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/es/ --- locales/es.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/locales/es.json b/locales/es.json index fe3e99dbf..3cdeebf56 100644 --- a/locales/es.json +++ b/locales/es.json @@ -637,5 +637,6 @@ "permission_already_up_to_date": "El permiso no se ha actualizado porque las peticiones de incorporación o eliminación ya coinciden con el estado actual.", "permission_currently_allowed_for_visitors": "Este permiso se concede actualmente a los visitantes además de otros grupos. Probablemente quiere o eliminar el permiso de «visitors» o eliminar los otros grupos a los que está otorgado actualmente.", "permission_currently_allowed_for_all_users": "Este permiso se concede actualmente a todos los usuarios además de los otros grupos. Probablemente quiere o eliminar el permiso de «all_users» o eliminar los otros grupos a los que está otorgado actualmente.", - "permission_require_account": "El permiso {permission} solo tiene sentido para usuarios con una cuenta y, por lo tanto, no se puede activar para visitantes." + "permission_require_account": "El permiso {permission} solo tiene sentido para usuarios con una cuenta y, por lo tanto, no se puede activar para visitantes.", + "app_remove_after_failed_install": "Eliminando la aplicación tras el fallo de instalación…" } From da43f025fd56598997afdc4e0ccc15ac6a0592a5 Mon Sep 17 00:00:00 2001 From: amirale qt Date: Tue, 5 Nov 2019 08:45:33 +0000 Subject: [PATCH 87/94] Translated using Weblate (Turkish) Currently translated at 0.2% (1 of 562 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/tr/ --- locales/tr.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/locales/tr.json b/locales/tr.json index 0967ef424..c6eb58ed1 100644 --- a/locales/tr.json +++ b/locales/tr.json @@ -1 +1,3 @@ -{} +{ + "password_too_simple_1": "Şifre en az 8 karakter uzunluğunda olmalı" +} From 42430e427756be42f8e6d7212bb2d6f35001f0ec Mon Sep 17 00:00:00 2001 From: amirale qt Date: Tue, 5 Nov 2019 07:32:56 +0000 Subject: [PATCH 88/94] Translated using Weblate (Basque) Currently translated at 0.2% (1 of 562 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/eu/ --- locales/eu.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/locales/eu.json b/locales/eu.json index 0967ef424..539fb9157 100644 --- a/locales/eu.json +++ b/locales/eu.json @@ -1 +1,3 @@ -{} +{ + "password_too_simple_1": "Pasahitzak gutxienez 8 karaktere izan behar ditu" +} From b56044ea18d84c08e77e9775f85cd7ebcff1521c Mon Sep 17 00:00:00 2001 From: amirale qt Date: Tue, 5 Nov 2019 13:33:02 +0000 Subject: [PATCH 89/94] Translated using Weblate (French) Currently translated at 100.0% (562 of 562 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/fr/ --- locales/fr.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/locales/fr.json b/locales/fr.json index 563c09a1f..866b24281 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -664,5 +664,6 @@ "permission_currently_allowed_for_all_users": "Cette autorisation est actuellement accordée à tous les utilisateurs en plus des autres groupes. Vous voudrez probablement soit supprimer l'autorisation 'all_users', soit supprimer les autres groupes auxquels il est actuellement autorisé.", "app_install_failed": "Impossible d'installer {app}: {error}", "app_install_script_failed": "Une erreur est survenue dans le script d'installation de l'application", - "permission_require_account": "Permission {permission} n'a de sens que pour les utilisateurs ayant un compte et ne peut donc pas être activé pour les visiteurs." + "permission_require_account": "Permission {permission} n'a de sens que pour les utilisateurs ayant un compte et ne peut donc pas être activé pour les visiteurs.", + "app_remove_after_failed_install": "Supprimer l'application après l'échec de l'installation…" } From 01eaea9fbaabed842556d93f001d96e58afaefbd Mon Sep 17 00:00:00 2001 From: amirale qt Date: Tue, 5 Nov 2019 13:33:34 +0000 Subject: [PATCH 90/94] Translated using Weblate (Esperanto) Currently translated at 95.6% (537 of 562 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/eo/ --- locales/eo.json | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/locales/eo.json b/locales/eo.json index 720485ba6..a25fa505e 100644 --- a/locales/eo.json +++ b/locales/eo.json @@ -141,7 +141,7 @@ "field_invalid": "Nevalida kampo '{:s}'", "log_app_makedefault": "Faru '{}' la defaŭlta apliko", "migration_0003_still_on_jessie_after_main_upgrade": "Io okazis malbone dum la ĉefa ĝisdatigo: Ĉu la sistemo ankoraŭ estas en Jessie‽ Por esplori la aferon, bonvolu rigardi {log}:s …", - "migration_0011_can_not_backup_before_migration": "La sekurkopio de la sistemo antaŭ la migrado malsukcesis. Migrado malsukcesis. Eraro: {error:s}", + "migration_0011_can_not_backup_before_migration": "La sekurkopio de la sistemo ne povis finiĝi antaŭ ol la migrado malsukcesis. Eraro: {error:s}", "migration_0011_create_group": "Krei grupon por ĉiu uzanto…", "backup_system_part_failed": "Ne eblis sekurkopi la sistemon de '{part:s}'", "global_settings_setting_security_postfix_compatibility": "Kongruo vs sekureca kompromiso por la Postfix-servilo. Afektas la ĉifradojn (kaj aliajn aspektojn pri sekureco)", @@ -151,8 +151,8 @@ "migration_0011_backup_before_migration": "Krei sekurkopion de LDAP-datumbazo kaj agordojn antaŭ la efektiva migrado.", "migration_0011_LDAP_config_dirty": "Similas ke vi agordis vian LDAP-agordon. Por ĉi tiu migrado la LDAP-agordo bezonas esti ĝisdatigita.\nVi devas konservi vian aktualan agordon, reintaligi la originalan agordon per funkciado de \"yunohost iloj regen-conf -f\" kaj reprovi la migradon", "migration_0011_migrate_permission": "Migrado de permesoj de agordoj al aplikoj al LDAP…", - "migration_0011_migration_failed_trying_to_rollback": "Migrado malsukcesis ... provante reverti la sistemon.", - "migrations_dependencies_not_satisfied": "Ne eblas kuri migradon {id} ĉar unue vi devas ruli ĉi tiujn migradojn: {dependencies_id}", + "migration_0011_migration_failed_trying_to_rollback": "Ne povis migri ... provante redakti la sistemon.", + "migrations_dependencies_not_satisfied": "Rulu ĉi tiujn migradojn: '{dependencies_id}', antaŭ migrado {id}.", "migrations_failed_to_load_migration": "Ne povis ŝarĝi migradon {id}: {error}", "migrations_exclusive_options": "'--auto', '--skip' kaj '--force-rerun' estas reciproke ekskluzivaj ebloj.", "migrations_must_provide_explicit_targets": "Vi devas provizi eksplicitajn celojn kiam vi uzas '--skip' aŭ '--force-rerun'", @@ -162,7 +162,7 @@ "tools_upgrade_cant_hold_critical_packages": "Ne povis teni kritikajn pakojn…", "upnp_dev_not_found": "Neniu UPnP-aparato trovita", "migration_description_0012_postgresql_password_to_md5_authentication": "Devigu PostgreSQL-aŭtentigon uzi MD5 por lokaj ligoj", - "migration_0011_done": "Migrado sukcesis. Vi nun kapablas administri uzantajn grupojn.", + "migration_0011_done": "Migrado finiĝis. Vi nun kapablas administri uzantajn grupojn.", "migration_0011_LDAP_update_failed": "Ne povis ĝisdatigi LDAP. Eraro: {error:s}", "pattern_password": "Devas esti almenaŭ 3 signoj longaj", "root_password_desynchronized": "La pasvorta administranto estis ŝanĝita, sed YunoHost ne povis propagandi ĉi tion al la radika pasvorto!", @@ -194,9 +194,9 @@ "migration_0011_rollback_success": "Sistemo ruliĝis reen.", "migration_0011_update_LDAP_database": "Ĝisdatigante LDAP-datumbazon…", "migration_0011_update_LDAP_schema": "Ĝisdatigante LDAP-skemon…", - "migration_0011_failed_to_remove_stale_object": "Malsukcesis forigi neokazan objekton {dn}: {error}", + "migration_0011_failed_to_remove_stale_object": "Ne povis forigi neuzatan objekton {dn}: {error}", "migrations_already_ran": "Tiuj migradoj estas jam faritaj: {ids}", - "migrations_no_such_migration": "Estas neniu migrado nomata {id}", + "migrations_no_such_migration": "Estas neniu migrado nomata '{id}'", "permission_already_allowed": "Grupo '{group}' jam havas permeson '{permission}' ebligita'", "permission_already_disallowed": "Grupo '{group}' jam havas permeson '{permission}' malebligita'", "permission_cannot_remove_main": "Forigo de ĉefa permeso ne rajtas", @@ -266,7 +266,7 @@ "migration_description_0008_ssh_conf_managed_by_yunohost_step2": "Lasu la SSH-agordon estu administrata de YunoHost (paŝo 2, manlibro)", "restore_confirm_yunohost_installed": "Ĉu vi vere volas restarigi jam instalitan sistemon? [{answers:s}]", "pattern_positive_number": "Devas esti pozitiva nombro", - "monitor_stats_file_not_found": "Statistika dosiero ne trovita", + "monitor_stats_file_not_found": "Ne povis trovi la statistikan dosieron", "certmanager_error_no_A_record": "Neniu DNS 'A' rekordo trovita por '{domain:s}'. Vi bezonas atentigi vian domajnan nomon al via maŝino por povi instali atestilon Lasu-Ĉifri. (Se vi scias, kion vi faras, uzu '--no-checks' por malŝalti tiujn ĉekojn.)", "update_apt_cache_failed": "Ne eblis ĝisdatigi la kaŝmemoron de APT (paka administranto de Debian). Jen rubujo de la sources.list-linioj, kiuj povus helpi identigi problemajn liniojn:\n{sourcelist}", "migrations_no_migrations_to_run": "Neniuj migradoj por funkcii", @@ -339,7 +339,7 @@ "log_app_upgrade": "Ĝisdatigu la aplikon '{}'", "log_help_to_get_failed_log": "La operacio '{desc}' ne povis finiĝi. Bonvolu dividi la plenan ŝtipon de ĉi tiu operacio per la komando 'yunohost log display {name} --share' por akiri helpon", "migration_description_0002_migrate_to_tsig_sha256": "Plibonigu sekurecon de DynDNS TSIG-ĝisdatigoj per SHA-512 anstataŭ MD5", - "monitor_disabled": "Servila monitorado nun malŝaltis", + "monitor_disabled": "Servilo-monitorado nun malŝaltita", "pattern_port": "Devas esti valida havena numero (t.e. 0-65535)", "port_already_closed": "Haveno {port:d} estas jam fermita por {ip_version:s} rilatoj", "hook_name_unknown": "Nekonata hoko-nomo '{name:s}'", @@ -407,7 +407,7 @@ "migration_0003_not_jessie": "La nuna Debian-distribuo ne estas Jessie!", "user_unknown": "Nekonata uzanto: {user:s}", "migrations_to_be_ran_manually": "Migrado {id} devas funkcii permane. Bonvolu iri al Iloj → Migradoj en la retpaĝa paĝo, aŭ kuri `yunohost tools migrations migrate`.", - "migration_0008_warning": "Se vi komprenas tiujn avertojn kaj konsentas lasi YunoHost pretervidi vian nunan agordon, faru la migradon. Alie, vi ankaŭ povas salti la migradon - kvankam ĝi ne rekomendas.", + "migration_0008_warning": "Se vi komprenas tiujn avertojn kaj volas ke YunoHost preterlasu vian nunan agordon, faru la migradon. Alie, vi ankaŭ povas salti la migradon, kvankam ĝi ne rekomendas.", "certmanager_cert_renew_success": "Ni Ĉifru atestilon renovigitan por la domajno '{domain:s}'", "global_settings_reset_success": "Antaŭaj agordoj nun estas rezervitaj al {path:s}", "pattern_domain": "Devas esti valida domajna nomo (t.e. mia-domino.org)", @@ -477,14 +477,14 @@ "log_tools_maindomain": "Faru de '{}' la ĉefa domajno", "maindomain_change_failed": "Ne povis ŝanĝi la ĉefan domajnon", "mail_domain_unknown": "Nevalida retadreso por domajno '{domain:s}'. Bonvolu uzi domajnon administritan de ĉi tiu servilo.", - "migrations_cant_reach_migration_file": "Ne povis aliri migrajn dosierojn ĉe la vojo% s", + "migrations_cant_reach_migration_file": "Ne povis aliri migrajn dosierojn ĉe la vojo '% s'", "pattern_email": "Devas esti valida retpoŝtadreso (t.e.iu@domain.org)", "mail_alias_remove_failed": "Ne povis forigi retpoŝton alias '{mail:s}'", "regenconf_file_manually_removed": "La dosiero de agordo '{conf}' estis forigita permane, kaj ne estos kreita", - "monitor_enabled": "Servila monitorado nun ŝaltis", + "monitor_enabled": "Servilo-monitorado nun", "domain_exists": "La domajno jam ekzistas", "migration_description_0001_change_cert_group_to_sslcert": "Ŝanĝu grupajn permesojn de 'metronomo' al 'ssl-cert'", - "mysql_db_creation_failed": "MySQL-datumbazkreado malsukcesis", + "mysql_db_creation_failed": "Ne povis krei MySQL-datumbazon", "ldap_initialized": "LDAP inicializis", "migrate_tsig_not_needed": "Vi ne ŝajnas uzi DynDNS-domajnon, do neniu migrado necesas.", "certmanager_domain_cert_not_selfsigned": "La atestilo por domajno {domajno:s} ne estas mem-subskribita. Ĉu vi certas, ke vi volas anstataŭigi ĝin? (Uzu '--force' por fari tion.)", @@ -495,7 +495,7 @@ "global_settings_bad_choice_for_enum": "Malbona elekto por agordo {setting:s}, ricevita '{choice:s}', sed disponeblaj elektoj estas: {available_choices:s}", "server_shutdown": "La servilo haltos", "log_tools_migrations_migrate_forward": "Migri antaŭen", - "migration_0008_no_warning": "Neniu grava risko identigita pri superregado de via SSH-agordo, tamen oni ne povas esti absolute certa;)! Ekfunkciu la migradon por superregi ĝin. Alie, vi ankaŭ povas salti la migradon - kvankam ĝi ne rekomendas.", + "migration_0008_no_warning": "Supersalti vian SSH-agordon estu sekura, kvankam ĉi tio ne povas esti promesita! Ekfunkciu la migradon por superregi ĝin. Alie, vi ankaŭ povas salti la migradon, kvankam ĝi ne rekomendas.", "regenconf_now_managed_by_yunohost": "La agorda dosiero '{conf}' nun estas administrata de YunoHost (kategorio {category}).", "server_reboot_confirm": "Ĉu la servilo rekomencos tuj, ĉu vi certas? [{answers:s}]", "log_app_install": "Instalu la aplikon '{}'", @@ -563,5 +563,6 @@ "permission_currently_allowed_for_visitors": "Ĉi tiu permeso estas nuntempe donita al vizitantoj aldone al aliaj grupoj. Vi probable volas aŭ forigi la permeson de \"vizitantoj\" aŭ forigi la aliajn grupojn al kiuj ĝi nun estas koncedita.", "permission_currently_allowed_for_all_users": "Ĉi tiu permeso estas nuntempe donita al ĉiuj uzantoj aldone al aliaj grupoj. Vi probable volas aŭ forigi la permeson \"all_users\" aŭ forigi la aliajn grupojn, kiujn ĝi nuntempe donas.", "app_install_failed": "Ne povis instali {app} : {error}", - "app_install_script_failed": "Eraro okazis en la skripto de instalado de la app" + "app_install_script_failed": "Eraro okazis en la skripto de instalado de la app", + "app_remove_after_failed_install": "Forigado de la app post la instala fiasko …" } From 08c23599e2e1a6989d29fe6ab6c10d1b993b5bd1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 6 Nov 2019 23:48:58 +0100 Subject: [PATCH 91/94] Improve yunohost package version diagnosis --- data/hooks/diagnosis/00-basesystem.py | 8 ++++++-- locales/en.json | 4 ++-- src/yunohost/utils/packages.py | 3 +++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/data/hooks/diagnosis/00-basesystem.py b/data/hooks/diagnosis/00-basesystem.py index 8bd522ee7..4add48fb2 100644 --- a/data/hooks/diagnosis/00-basesystem.py +++ b/data/hooks/diagnosis/00-basesystem.py @@ -37,16 +37,20 @@ class BaseSystemDiagnoser(Diagnoser): # Here, ynh_core_version is for example "3.5.4.12", so [:3] is "3.5" and we check it's the same for all packages ynh_core_version = ynh_packages["yunohost"]["version"] consistent_versions = all(infos["version"][:3] == ynh_core_version[:3] for infos in ynh_packages.values()) - ynh_version_details = [("diagnosis_basesystem_ynh_single_version", (package, infos["version"])) + ynh_version_details = [("diagnosis_basesystem_ynh_single_version", (package, infos["version"], infos["repo"])) for package, infos in ynh_packages.items()] if consistent_versions: yield dict(meta={"test": "ynh_versions"}, + data={"main_version": ynh_core_version, "repo": ynh_packages["yunohost"]["repo"]}, status="INFO", - summary=("diagnosis_basesystem_ynh_main_version", {"main_version": ynh_core_version[:3]}), + summary=("diagnosis_basesystem_ynh_main_version", + {"main_version": ynh_core_version, + "repo": ynh_packages["yunohost"]["repo"]}), details=ynh_version_details) else: yield dict(meta={"test": "ynh_versions"}, + data={"main_version": ynh_core_version, "repo": ynh_packages["yunohost"]["repo"]}, status="ERROR", summary=("diagnosis_basesystem_ynh_inconsistent_versions", {}), details=ynh_version_details) diff --git a/locales/en.json b/locales/en.json index ebe6b4571..751180a37 100644 --- a/locales/en.json +++ b/locales/en.json @@ -152,8 +152,8 @@ "custom_appslist_name_required": "You must provide a name for your custom app list", "diagnosis_basesystem_host": "Server is running Debian {debian_version}.", "diagnosis_basesystem_kernel": "Server is running Linux kernel {kernel_version}", - "diagnosis_basesystem_ynh_single_version": "{0} version: {1}", - "diagnosis_basesystem_ynh_main_version": "Server is running YunoHost {main_version}", + "diagnosis_basesystem_ynh_single_version": "{0} version: {1} ({2})", + "diagnosis_basesystem_ynh_main_version": "Server is running YunoHost {main_version} ({repo})", "diagnosis_basesystem_ynh_inconsistent_versions": "You are running inconsistents versions of the YunoHost packages ... most probably because of a failed or partial upgrade.", "diagnosis_display_tip_web": "You can go to the Diagnosis section (in the home screen) to see the issues found.", "diagnosis_display_tip_cli": "You can run 'yunohost diagnosis show --issues' to display the issues found.", diff --git a/src/yunohost/utils/packages.py b/src/yunohost/utils/packages.py index 6df736432..debba70f4 100644 --- a/src/yunohost/utils/packages.py +++ b/src/yunohost/utils/packages.py @@ -406,6 +406,9 @@ def get_installed_version(*pkgnames, **kwargs): except AttributeError: repo = "" + if repo == "now": + repo = "local" + if with_repo: versions[pkgname] = { "version": version, From c39a1f010ea17b623c6cc0815cf8da7b45b4f14a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 7 Nov 2019 22:19:07 +0100 Subject: [PATCH 92/94] Mistakes were made --- src/yunohost/regenconf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/yunohost/regenconf.py b/src/yunohost/regenconf.py index 5681f12a4..665b906d6 100644 --- a/src/yunohost/regenconf.py +++ b/src/yunohost/regenconf.py @@ -138,7 +138,8 @@ def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run # services.yml (which will happens only during the regen-conf of # 'yunohost', so at the very end of the regen-conf cycle) Anyway, # this can be safely removed once we're in >= 4.0 - names.remove("glances") + if "glances" in names: + names.remove("glances") pre_result = hook_callback('conf_regen', names, pre_callback=_pre_call) From 65d6b02b5604421a9df30206ba29656d2a36e4e9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 8 Nov 2019 19:22:54 +0100 Subject: [PATCH 93/94] Implement basic outgoing port 25 check for email stack --- data/hooks/diagnosis/18-mail.py | 16 ++++++++++------ locales/en.json | 3 +++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/data/hooks/diagnosis/18-mail.py b/data/hooks/diagnosis/18-mail.py index c12c15cff..f0060df52 100644 --- a/data/hooks/diagnosis/18-mail.py +++ b/data/hooks/diagnosis/18-mail.py @@ -13,16 +13,20 @@ class MailDiagnoser(Diagnoser): def run(self): - # TODO / FIXME TO BE IMPLEMETED in the future ... + # Is outgoing port 25 filtered somehow ? + if os.system('/bin/nc -z -w2 yunohost.org 25') == 0: + yield dict(meta={"test": "ougoing_port_25"}, + status="SUCCESS", + summary=("diagnosis_mail_ougoing_port_25_ok",{})) + else: + yield dict(meta={"test": "outgoing_port_25"}, + status="ERROR", + summary=("diagnosis_mail_ougoing_port_25_blocked",{})) + - yield dict(meta={}, - status="WARNING", - summary=("nothing_implemented_yet", {})) # Mail blacklist using dig requests (c.f. ljf's code) - # Outgoing port 25 (c.f. code in monitor.py, a simple 'nc -zv yunohost.org 25' IIRC) - # SMTP reachability (c.f. check-smtp to be implemented on yunohost's remote diagnoser) # ideally, SPF / DMARC / DKIM validation ... (c.f. https://github.com/alexAubin/yunoScripts/blob/master/yunoDKIM.py possibly though that looks horrible) diff --git a/locales/en.json b/locales/en.json index 751180a37..9a9855116 100644 --- a/locales/en.json +++ b/locales/en.json @@ -191,6 +191,8 @@ "diagnosis_swap_none": "The system has no swap at all. You should consider adding at least 256 MB of swap to avoid situations where the system runs out of memory.", "diagnosis_swap_notsomuch": "The system has only {total_MB} MB swap. You should consider having at least 256 MB to avoid situations where the system runs out of memory.", "diagnosis_swap_ok": "The system has {total_MB} MB of swap!", + "diagnosis_mail_ougoing_port_25_ok": "Outgoing port 25 is not blocked and email can be sent to other servers.", + "diagnosis_mail_ougoing_port_25_blocked": "Outgoing port 25 appears to be blocked. You should try to unblock it in your internet service provider (or hoster) configuration panel. Meanwhile, the server won't be able to send emails to other servers.", "diagnosis_regenconf_allgood": "All configurations files are in line with the recommended configuration!", "diagnosis_regenconf_manually_modified": "Configuration file {file} was manually modified.", "diagnosis_regenconf_manually_modified_details": "This is probably OK as long as you know what you're doing ;) !", @@ -207,6 +209,7 @@ "diagnosis_description_systemresources": "System resources", "diagnosis_description_ports": "Ports exposure", "diagnosis_description_http": "HTTP exposure", + "diagnosis_description_mail": "Email", "diagnosis_description_regenconf": "System configurations", "diagnosis_description_security": "Security checks", "diagnosis_ports_could_not_diagnose": "Could not diagnose if ports are reachable from outside. Error: {error}", From 104bba3dd86e995a5d80c17954a5c515b0da60be Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 8 Nov 2019 20:44:27 +0100 Subject: [PATCH 94/94] Sort services during diagnosis to avoid random order --- data/hooks/diagnosis/30-services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/hooks/diagnosis/30-services.py b/data/hooks/diagnosis/30-services.py index 6589d83f2..32f99c84d 100644 --- a/data/hooks/diagnosis/30-services.py +++ b/data/hooks/diagnosis/30-services.py @@ -26,7 +26,7 @@ class ServicesDiagnoser(Diagnoser): all_result = service_status() - for service, result in all_result.items(): + for service, result in sorted(all_result.items()): if service in services_ignored: continue