From de7a66ff0e76f21463f22ee151530dbb4f0d7bcb Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 22 Mar 2019 01:14:45 +0100 Subject: [PATCH 01/23] Rework system-part of tools_update... --- src/yunohost/tools.py | 51 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index a011b1546..482d9305b 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -41,7 +41,7 @@ from moulinette import msettings, msignals, m18n from moulinette.core import init_authenticator from yunohost.utils.error import YunohostError from moulinette.utils.log import getActionLogger -from moulinette.utils.process import check_output +from moulinette.utils.process import check_output, call_async_output from moulinette.utils.filesystem import read_json, write_to_json from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, app_list, _install_appslist_fetch_cron from yunohost.domain import domain_add, domain_list, _get_maindomain, _set_maindomain @@ -469,23 +469,54 @@ def tools_update(ignore_apps=False, ignore_packages=False): # "packages" will list upgradable packages packages = [] if not ignore_packages: - cache = apt.Cache() # Update APT cache + # LC_ALL=C is here to make sure the results are in english + command = "LC_ALL=C apt update" + # TODO : add @is_unit_operation to tools_update so that the + # debug output can be fetched when there's an issue... + callbacks = ( + # stdout goes to debug + lambda l: logger.debug(l.rstrip()), + # stderr goes to warning + # FIXME : filter the damn "CLI interface not stable" from apt >.> + lambda l: logger.warning(l.rstrip()), + ) + logger.info(m18n.n('updating_apt_cache')) - if not cache.update(): + + returncode = call_async_output(command, callbacks, shell=True) + + if returncode != 0: + + # TODO : here, we should run something like a + # `cat /etc/apt/sources.list /etc/apt/sources.list.d/*` + # and append it to the error message to improve debugging + raise YunohostError('update_cache_failed') - cache.open(None) - cache.upgrade(True) + # List upgradable packages + # LC_ALL=C is here to make sure the results are in english + upgradable_raw = check_output("LC_ALL=C apt list --upgradable") - # Add changelogs to the result - for pkg in cache.get_changes(): + # Dirty parsing of the output + upgradable_raw = [l.strip() for l in upgradable_raw.split("\n") if l.strip()] + for line in upgradable_raw: + # Remove stupid warning and verbose messages >.> + if "apt does not have a stable CLI interface" in line or "Listing..." in line: + continue + # line should look like : + # yunohost/stable 3.5.0.2+201903211853 all [upgradable from: 3.4.2.4+201903080053] + line = line.split() + if len(line) != 6: + logger.warning("Failed to parse this line : %s" % ' '.join(line)) + continue packages.append({ - 'name': pkg.name, - 'fullname': pkg.fullname, - 'changelog': pkg.get_changelog() + "name": line[0].split("/")[0], + "new_version": line[1], + "current_version": line[5].strip("]"), }) + logger.debug(m18n.n('done')) # "apps" will list upgradable packages From 51fe6fea277e337732e47239158e68d01a1617b9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 22 Mar 2019 04:01:08 +0100 Subject: [PATCH 02/23] Factorize function to list upgradable packages --- src/yunohost/tools.py | 50 ++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 482d9305b..93e3dd177 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -495,28 +495,7 @@ def tools_update(ignore_apps=False, ignore_packages=False): raise YunohostError('update_cache_failed') - # List upgradable packages - # LC_ALL=C is here to make sure the results are in english - upgradable_raw = check_output("LC_ALL=C apt list --upgradable") - - # Dirty parsing of the output - upgradable_raw = [l.strip() for l in upgradable_raw.split("\n") if l.strip()] - for line in upgradable_raw: - # Remove stupid warning and verbose messages >.> - if "apt does not have a stable CLI interface" in line or "Listing..." in line: - continue - # line should look like : - # yunohost/stable 3.5.0.2+201903211853 all [upgradable from: 3.4.2.4+201903080053] - line = line.split() - if len(line) != 6: - logger.warning("Failed to parse this line : %s" % ' '.join(line)) - continue - packages.append({ - "name": line[0].split("/")[0], - "new_version": line[1], - "current_version": line[5].strip("]"), - }) - + packages = list(_list_upgradable_apt_packages()) logger.debug(m18n.n('done')) # "apps" will list upgradable packages @@ -545,6 +524,33 @@ def tools_update(ignore_apps=False, ignore_packages=False): return {'packages': packages, 'apps': apps} +# TODO : move this to utils/packages.py ? +def _list_upgradable_apt_packages(): + + # List upgradable packages + # LC_ALL=C is here to make sure the results are in english + upgradable_raw = check_output("LC_ALL=C apt list --upgradable") + + # Dirty parsing of the output + upgradable_raw = [l.strip() for l in upgradable_raw.split("\n") if l.strip()] + for line in upgradable_raw: + # Remove stupid warning and verbose messages >.> + if "apt does not have a stable CLI interface" in line or "Listing..." in line: + continue + # line should look like : + # yunohost/stable 3.5.0.2+201903211853 all [upgradable from: 3.4.2.4+201903080053] + line = line.split() + if len(line) != 6: + logger.warning("Failed to parse this line : %s" % ' '.join(line)) + continue + + yield { + "name": line[0].split("/")[0], + "new_version": line[1], + "current_version": line[5].strip("]"), + } + + @is_unit_operation() def tools_upgrade(operation_logger, auth, ignore_apps=False, ignore_packages=False): """ From 096c2a7d7bfa6030206078bc9c19ec2e9305b9c7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 22 Mar 2019 04:02:38 +0100 Subject: [PATCH 03/23] Rework system-part of tools_upgrade... --- src/yunohost/tools.py | 170 +++++++++++++++++++++++++++++------------- 1 file changed, 119 insertions(+), 51 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 93e3dd177..589e49a3f 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -33,6 +33,7 @@ import socket from xmlrpclib import Fault from importlib import import_module from collections import OrderedDict +from datetime import datetime import apt import apt.progress @@ -567,61 +568,127 @@ def tools_upgrade(operation_logger, auth, ignore_apps=False, ignore_packages=Fal failure = False - # Retrieve interface - is_api = True if msettings.get('interface') == 'api' else False - if not ignore_packages: - apt.apt_pkg.init() - apt.apt_pkg.config.set("DPkg::Options::", "--force-confdef") - apt.apt_pkg.config.set("DPkg::Options::", "--force-confold") - - cache = apt.Cache() - cache.open(None) - cache.upgrade(True) - - # If API call - if is_api: - critical_packages = ("moulinette", "yunohost", - "yunohost-admin", "ssowat", "python") - critical_upgrades = set() - - for pkg in cache.get_changes(): - if pkg.name in critical_packages: - critical_upgrades.add(pkg.name) - # Temporarily keep package ... - pkg.mark_keep() - - # ... and set a hourly cron up to upgrade critical packages - if critical_upgrades: - logger.info(m18n.n('packages_upgrade_critical_later', - packages=', '.join(critical_upgrades))) - with open('/etc/cron.d/yunohost-upgrade', 'w+') as f: - f.write('00 * * * * root PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin apt-get install %s -y && rm -f /etc/cron.d/yunohost-upgrade\n' % ' '.join(critical_upgrades)) - - if cache.get_changes(): - logger.info(m18n.n('upgrading_packages')) - - operation_logger.start() - try: - os.environ["DEBIAN_FRONTEND"] = "noninteractive" - # Apply APT changes - # TODO: Logs output for the API - cache.commit(apt.progress.text.AcquireProgress(), - apt.progress.base.InstallProgress()) - except Exception as e: - failure = True - logger.warning('unable to upgrade packages: %s' % str(e)) - logger.error(m18n.n('packages_upgrade_failed')) - operation_logger.error(m18n.n('packages_upgrade_failed')) - else: - logger.info(m18n.n('done')) - operation_logger.success() - finally: - del os.environ["DEBIAN_FRONTEND"] - else: + # Check that there's indeed some packages to upgrade + upgradables = list(_list_upgradable_apt_packages()) + if not upgradables: logger.info(m18n.n('packages_no_upgrade')) + logger.info(m18n.n('upgrading_packages')) + operation_logger.start() + + # Critical packages are packages that we can't just upgrade + # randomly from yunohost itself... upgrading them is likely to + critical_packages = ("moulinette", "yunohost", "yunohost-admin", "ssowat", "python") + + critical_packages_upgradable = [p for p in upgradables if p["name"] in critical_packages] + noncritical_packages_upgradable = [p for p in upgradables if p["name"] not in critical_packages] + + # Prepare dist-upgrade command + dist_upgrade = "DEBIAN_FRONTEND=noninteractive" + dist_upgrade += " APT_LISTCHANGES_FRONTEND=none" + dist_upgrade += " apt-get" + dist_upgrade += " --fix-broken --show-upgraded --assume-yes" + for conf_flag in ["old", "miss", "def"]: + dist_upgrade += ' -o Dpkg::Options::="--force-conf{}"'.format(conf_flag) + dist_upgrade += " dist-upgrade" + + # + # "Regular" packages upgrade + # + if not failure and noncritical_packages_upgradable: + + # TODO : i18n + logger.info("Upgrading 'regular' (non-yunohost-related) packages ...") + + # TODO : factorize this in utils/packages.py ? + # Mark all critical packages as held + for package in critical_packages: + check_output("apt-mark hold %s" % package) + # Doublecheck with apt-mark showhold that packages are indeed held ... + held_packages = check_output("apt-mark showhold").split("\n") + if any(p not in held_packages for p in critical_packages): + failure = True + logger.warning('Unable to hold critical packages ...') + logger.error(m18n.n('packages_upgrade_failed')) + # FIXME : watdo here, should this be an exception or just an + # error + operation_logger.error(m18n.n('packages_upgrade_failed')) + + if not failure: + logger.debug("Running apt command :\n{}".format(dist_upgrade)) + + callbacks = ( + lambda l: logger.info(l.rstrip()), + lambda l: logger.warning(l.rstrip()), + ) + returncode = call_async_output(dist_upgrade, callbacks, shell=True) + if returncode != 0: + failure = True + logger.warning('unable to upgrade packages: %s' % ', '.join(noncritical_packages_upgradable)) + logger.error(m18n.n('packages_upgrade_failed')) + operation_logger.error(m18n.n('packages_upgrade_failed')) + + # + # Critical packages upgrade + # + if not failure and critical_packages_upgradable: + + # TODO : i18n + logger.info("Upgrading 'special' (yunohost-related) packages ...") + + # TODO : factorize this in utils/packages.py ? + # Mark all critical packages as unheld + for package in critical_packages: + check_output("apt-mark unhold %s" % package) + # Doublecheck with apt-mark showhold that packages are indeed unheld ... + unheld_packages = check_output("apt-mark showhold").split("\n") + if any(p in unheld_packages for p in critical_packages): + failure = True + logger.warning('Unable to unhold critical packages ...') + logger.error(m18n.n('packages_upgrade_failed')) + # FIXME : watdo here, should this be an exception or just an + # error + operation_logger.error(m18n.n('packages_upgrade_failed')) + + # + # Here we use a dirty hack to run a command after the current + # "yunohost tools upgrade", because the upgrade of yunohost + # will also trigger other yunohost commands (e.g. "yunohost tools migrations migrate") + # (also the upgrade of the package, if executed from the webadmin, is + # likely to kill/restart the api which is in turn likely to kill this + # command before it ends...) + # + + logfile = "/var/log/yunohost/special_upgrade_%s.log" % datetime.utcnow().strftime("%Y%m%d_%H%M%S") + command = dist_upgrade + " 2>&1 | tee -a {}".format(logfile) + + MOULINETTE_LOCK = "/var/run/moulinette_yunohost.lock" + wait_until_end_of_yunohost_command = "(while [ -f {} ]; do sleep 2; done)".format(MOULINETTE_LOCK) + + # TODO : i18n + upgrade_completed = "YunoHost package upgrade completed ! Press [enter] to get the command line back" + command = "({} && {}; echo '{}') &".format(wait_until_end_of_yunohost_command, + command, + upgrade_completed) + + logger.debug("Running command :\n{}".format(command)) + os.system(command) + + # TODO / FIXME : return from this function immediately, + # otherwise the apps upgrade might happen and it's gonna be a mess + + # FIXME / open question : what about "permanently" mark yunohost + # as "hold" to avoid accidental deletion of it... + # (so, only unhold it during the upgrade) + + if not failure: + + logger.info(m18n.n('done')) + operation_logger.success() + + if not ignore_apps: try: app_upgrade(auth) @@ -634,6 +701,7 @@ def tools_upgrade(operation_logger, auth, ignore_apps=False, ignore_packages=Fal logger.success(m18n.n('system_upgraded')) # Return API logs if it is an API call + is_api = True if msettings.get('interface') == 'api' else False if is_api: return {"log": service_log('yunohost-api', number="100").values()[0]} From ad0f65aad713e3f582f437af4d3bc63dc02044ca Mon Sep 17 00:00:00 2001 From: ljf Date: Fri, 22 Mar 2019 15:17:39 +0100 Subject: [PATCH 04/23] [enh] Log special upgrade into operation_logger --- src/yunohost/log.py | 20 ++++++++++++++++---- src/yunohost/tools.py | 12 +++++++----- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/yunohost/log.py b/src/yunohost/log.py index 857cc3658..7b0401e5b 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -318,14 +318,27 @@ class OperationLogger(object): self.flush() self._register_log() + @property + def md_path(self): + """ + Metadata path file + """ + return os.path.join(self.path, self.name + METADATA_FILE_EXT) + + @property + def log_path(self): + """ + Log path file + """ + return os.path.join(self.path, self.name + LOG_FILE_EXT) + def _register_log(self): """ Register log with a handler connected on log system """ # TODO add a way to not save password on app installation - filename = os.path.join(self.path, self.name + LOG_FILE_EXT) - self.file_handler = FileHandler(filename) + self.file_handler = FileHandler(self.log_path) self.file_handler.formatter = Formatter('%(asctime)s: %(levelname)s - %(message)s') # Listen to the root logger @@ -337,8 +350,7 @@ class OperationLogger(object): Write or rewrite the metadata file with all metadata known """ - filename = os.path.join(self.path, self.name + METADATA_FILE_EXT) - with open(filename, 'w') as outfile: + with open(self.md_path, 'w') as outfile: yaml.safe_dump(self.metadata, outfile, default_flow_style=False) @property diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 589e49a3f..6120ed3a1 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -660,18 +660,20 @@ def tools_upgrade(operation_logger, auth, ignore_apps=False, ignore_packages=Fal # likely to kill/restart the api which is in turn likely to kill this # command before it ends...) # - - logfile = "/var/log/yunohost/special_upgrade_%s.log" % datetime.utcnow().strftime("%Y%m%d_%H%M%S") + logfile = operation_logger.log_path command = dist_upgrade + " 2>&1 | tee -a {}".format(logfile) MOULINETTE_LOCK = "/var/run/moulinette_yunohost.lock" wait_until_end_of_yunohost_command = "(while [ -f {} ]; do sleep 2; done)".format(MOULINETTE_LOCK) + update_log_metadata = "sed -i \"s/ended_at: .*$/ended_at: $(date -u +'%Y-%m-%d %H:%M:%S.%N')/\" {}" + update_log_metadata = update_log_metadata.format(operation_logger.md_path) # TODO : i18n upgrade_completed = "YunoHost package upgrade completed ! Press [enter] to get the command line back" - command = "({} && {}; echo '{}') &".format(wait_until_end_of_yunohost_command, - command, - upgrade_completed) + command = "({} && {} && {}; echo '{}') &".format(wait_until_end_of_yunohost_command, + command, + update_log_metadata, + upgrade_completed) logger.debug("Running command :\n{}".format(command)) os.system(command) From ac05ae655721469255a25885c8a451ede31ce6ca Mon Sep 17 00:00:00 2001 From: ljf Date: Fri, 22 Mar 2019 15:32:51 +0100 Subject: [PATCH 05/23] [enh] Support success info with criticla upgrades --- src/yunohost/tools.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 6120ed3a1..9b27b340a 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -665,15 +665,17 @@ def tools_upgrade(operation_logger, auth, ignore_apps=False, ignore_packages=Fal MOULINETTE_LOCK = "/var/run/moulinette_yunohost.lock" wait_until_end_of_yunohost_command = "(while [ -f {} ]; do sleep 2; done)".format(MOULINETTE_LOCK) + mark_success = "(echo 'success: true' | tee -a {})".format(operation_logger.md_path) update_log_metadata = "sed -i \"s/ended_at: .*$/ended_at: $(date -u +'%Y-%m-%d %H:%M:%S.%N')/\" {}" update_log_metadata = update_log_metadata.format(operation_logger.md_path) # TODO : i18n upgrade_completed = "YunoHost package upgrade completed ! Press [enter] to get the command line back" - command = "({} && {} && {}; echo '{}') &".format(wait_until_end_of_yunohost_command, - command, - update_log_metadata, - upgrade_completed) + command = "({} && {} && {}; {}; echo '{}') &".format(wait_until_end_of_yunohost_command, + command, + mark_success, + update_log_metadata, + upgrade_completed) logger.debug("Running command :\n{}".format(command)) os.system(command) @@ -685,7 +687,7 @@ def tools_upgrade(operation_logger, auth, ignore_apps=False, ignore_packages=Fal # as "hold" to avoid accidental deletion of it... # (so, only unhold it during the upgrade) - if not failure: + elif not failure: logger.info(m18n.n('done')) operation_logger.success() From 15ac51098d9d2ae52eaa50929f843551310a4cd9 Mon Sep 17 00:00:00 2001 From: Romuald du Song Date: Thu, 28 Mar 2019 22:03:25 +0100 Subject: [PATCH 06/23] propose a setting to remove support for TLSv1 and TLSv1.1 --- data/hooks/conf_regen/19-postfix | 11 ++- data/templates/postfix/main.cf | 116 ++++++++++++++++--------------- locales/en.json | 1 + src/yunohost/settings.py | 7 ++ 4 files changed, 76 insertions(+), 59 deletions(-) diff --git a/data/hooks/conf_regen/19-postfix b/data/hooks/conf_regen/19-postfix index a3ad70327..b37425984 100755 --- a/data/hooks/conf_regen/19-postfix +++ b/data/hooks/conf_regen/19-postfix @@ -2,6 +2,8 @@ set -e +. /usr/share/yunohost/helpers + do_pre_regen() { pending_dir=$1 @@ -20,9 +22,12 @@ do_pre_regen() { main_domain=$(cat /etc/yunohost/current_host) domain_list=$(sudo yunohost domain list --output-as plain --quiet | tr '\n' ' ') - cat main.cf \ - | sed "s/{{ main_domain }}/${main_domain}/g" \ - > "${postfix_dir}/main.cf" + # Support different strategy for security configurations + export compatibility="$(yunohost settings get 'security.postfix.compatibility')" + + export main_domain + export domain_list + ynh_render_template "main.cf" "${postfix_dir}/main.cf" cat postsrsd \ | sed "s/{{ main_domain }}/${main_domain}/g" \ diff --git a/data/templates/postfix/main.cf b/data/templates/postfix/main.cf index c38896a3f..e5a3875d4 100644 --- a/data/templates/postfix/main.cf +++ b/data/templates/postfix/main.cf @@ -33,7 +33,11 @@ smtpd_tls_key_file = /etc/yunohost/certs/{{ main_domain }}/key.pem smtpd_tls_exclude_ciphers = aNULL, MD5, DES, ADH, RC4, 3DES smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache smtpd_tls_loglevel=1 +{% if compatibility == "intermediate" %} smtpd_tls_mandatory_protocols=!SSLv2,!SSLv3 +{% else %} +smtpd_tls_mandatory_protocols=!SSLv2,!SSLv3,!TLSv1,!TLSv1.1 +{% endif %} smtpd_tls_mandatory_ciphers=high smtpd_tls_eecdh_grade = ultra @@ -58,7 +62,7 @@ alias_maps = hash:/etc/aliases alias_database = hash:/etc/aliases mydomain = {{ main_domain }} mydestination = localhost -relayhost = +relayhost = mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 mailbox_command = procmail -a "$EXTENSION" mailbox_size_limit = 0 @@ -68,71 +72,71 @@ inet_interfaces = all #### Fit to the maximum message size to 30mb, more than allowed by GMail or Yahoo #### message_size_limit = 31457280 -# Virtual Domains Control -virtual_mailbox_domains = ldap:/etc/postfix/ldap-domains.cf -virtual_mailbox_maps = ldap:/etc/postfix/ldap-accounts.cf -virtual_mailbox_base = -virtual_alias_maps = ldap:/etc/postfix/ldap-aliases.cf -virtual_alias_domains = -virtual_minimum_uid = 100 -virtual_uid_maps = static:vmail +# Virtual Domains Control +virtual_mailbox_domains = ldap:/etc/postfix/ldap-domains.cf +virtual_mailbox_maps = ldap:/etc/postfix/ldap-accounts.cf +virtual_mailbox_base = +virtual_alias_maps = ldap:/etc/postfix/ldap-aliases.cf +virtual_alias_domains = +virtual_minimum_uid = 100 +virtual_uid_maps = static:vmail virtual_gid_maps = static:mail smtpd_sender_login_maps= ldap:/etc/postfix/ldap-accounts.cf -# Dovecot LDA -virtual_transport = dovecot +# Dovecot LDA +virtual_transport = dovecot dovecot_destination_recipient_limit = 1 -# Enable SASL authentication for the smtpd daemon -smtpd_sasl_auth_enable = yes -smtpd_sasl_type = dovecot -smtpd_sasl_path = private/auth -# Fix some outlook's bugs -broken_sasl_auth_clients = yes -# Reject anonymous connections -smtpd_sasl_security_options = noanonymous +# Enable SASL authentication for the smtpd daemon +smtpd_sasl_auth_enable = yes +smtpd_sasl_type = dovecot +smtpd_sasl_path = private/auth +# Fix some outlook's bugs +broken_sasl_auth_clients = yes +# Reject anonymous connections +smtpd_sasl_security_options = noanonymous smtpd_sasl_local_domain = -# Wait until the RCPT TO command before evaluating restrictions -smtpd_delay_reject = yes - -# Basics Restrictions -smtpd_helo_required = yes -strict_rfc821_envelopes = yes - -# Requirements for the connecting server -smtpd_client_restrictions = - permit_mynetworks, - permit_sasl_authenticated, - reject_rbl_client bl.spamcop.net, - reject_rbl_client cbl.abuseat.org, - reject_rbl_client zen.spamhaus.org, - permit - -# Requirements for the HELO statement -smtpd_helo_restrictions = - permit_mynetworks, - permit_sasl_authenticated, - reject_non_fqdn_hostname, - reject_invalid_hostname, - permit - -# Requirements for the sender address +# Wait until the RCPT TO command before evaluating restrictions +smtpd_delay_reject = yes + +# Basics Restrictions +smtpd_helo_required = yes +strict_rfc821_envelopes = yes + +# Requirements for the connecting server +smtpd_client_restrictions = + permit_mynetworks, + permit_sasl_authenticated, + reject_rbl_client bl.spamcop.net, + reject_rbl_client cbl.abuseat.org, + reject_rbl_client zen.spamhaus.org, + permit + +# Requirements for the HELO statement +smtpd_helo_restrictions = + permit_mynetworks, + permit_sasl_authenticated, + reject_non_fqdn_hostname, + reject_invalid_hostname, + permit + +# Requirements for the sender address smtpd_sender_restrictions = - reject_sender_login_mismatch, - permit_mynetworks, - permit_sasl_authenticated, - reject_non_fqdn_sender, + reject_sender_login_mismatch, + permit_mynetworks, + permit_sasl_authenticated, + reject_non_fqdn_sender, reject_unknown_sender_domain, - permit - -# Requirement for the recipient address -smtpd_recipient_restrictions = - permit_mynetworks, - permit_sasl_authenticated, - reject_non_fqdn_recipient, - reject_unknown_recipient_domain, + permit + +# Requirement for the recipient address +smtpd_recipient_restrictions = + permit_mynetworks, + permit_sasl_authenticated, + reject_non_fqdn_recipient, + reject_unknown_recipient_domain, reject_unauth_destination, permit diff --git a/locales/en.json b/locales/en.json index 694df0707..6ab91fd2e 100644 --- a/locales/en.json +++ b/locales/en.json @@ -215,6 +215,7 @@ "global_settings_setting_security_password_admin_strength": "Admin password strength", "global_settings_setting_security_password_user_strength": "User password strength", "global_settings_setting_security_ssh_compatibility": "Compatibility vs. security tradeoff for the SSH server. Affects the ciphers (and other security-related aspects)", + "global_settings_setting_security_postfix_compatibility": "Compatibility vs. security tradeoff for the Postfix server. Affects the ciphers (and other security-related aspects)", "global_settings_unknown_setting_from_settings_file": "Unknown key in settings: '{setting_key:s}', discarding it and save it in /etc/yunohost/settings-unknown.json", "global_settings_setting_service_ssh_allow_deprecated_dsa_hostkey": "Allow the use of (deprecated) DSA hostkey for the SSH daemon configuration", "global_settings_unknown_type": "Unexpected situation, the setting {setting:s} appears to have the type {unknown_type:s} but it's not a type supported by the system.", diff --git a/src/yunohost/settings.py b/src/yunohost/settings.py index 671ad70e9..01f27ba83 100644 --- a/src/yunohost/settings.py +++ b/src/yunohost/settings.py @@ -44,6 +44,8 @@ DEFAULTS = OrderedDict([ "choices": ["intermediate", "modern"]}), ("security.nginx.compatibility", {"type": "enum", "default": "intermediate", "choices": ["intermediate", "modern"]}), + ("security.postfix.compatibility", {"type": "enum", "default": "intermediate", + "choices": ["intermediate", "modern"]}), ]) @@ -292,3 +294,8 @@ def reconfigure_nginx(setting_name, old_value, new_value): def reconfigure_ssh(setting_name, old_value, new_value): if old_value != new_value: service_regen_conf(names=['ssh']) + +@post_change_hook("security.postfix.compatibility") +def reconfigure_ssh(setting_name, old_value, new_value): + if old_value != new_value: + service_regen_conf(names=['postfix']) From a16fb94d82a04bdfbee9cc4647bd51690ea2415c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 21 Apr 2019 02:03:59 +0200 Subject: [PATCH 07/23] Only upgrade system or apps, not both at the same time --- data/actionsmap/yunohost.yml | 10 +++++----- src/yunohost/tools.py | 19 +++++++++++++------ 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index fabdcb923..08188c8c8 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1563,11 +1563,11 @@ tools: authenticate: all authenticator: ldap-anonymous arguments: - --ignore-apps: - help: Ignore apps upgrade - action: store_true - --ignore-packages: - help: Ignore APT packages upgrade + --apps: + help: List of apps to upgrade (all by default) + nargs: "*" + --system: + help: Upgrade only the system packages action: store_true ### tools_diagnosis() diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index ff02e5e39..784f5d512 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -582,22 +582,29 @@ def _dump_sources_list(): @is_unit_operation() -def tools_upgrade(operation_logger, auth, ignore_apps=False, ignore_packages=False): +def tools_upgrade(operation_logger, auth, apps=None, system=False): """ Update apps & package cache, then display changelog Keyword arguments: - ignore_apps -- Ignore apps upgrade - ignore_packages -- Ignore APT packages upgrade - + apps -- List of apps to upgrade (or [] to update all apps) + system -- True to upgrade system """ from yunohost.utils import packages if packages.dpkg_is_broken(): raise YunohostError("dpkg_is_broken") + if system is not False and apps is not None: + # TODO : i18n + raise YunohostError("Cannot upgrade both system and apps at the same time") + + if system is False and apps is None: + # TODO : i18n + raise YunohostError("Please specify --apps OR --system") + failure = False - if not ignore_packages: + if system is True: # Check that there's indeed some packages to upgrade upgradables = list(_list_upgradable_apt_packages()) @@ -722,7 +729,7 @@ def tools_upgrade(operation_logger, auth, ignore_apps=False, ignore_packages=Fal operation_logger.success() - if not ignore_apps: + if apps is not None: try: app_upgrade(auth) except Exception as e: From 9e3d30283268a9c7c07d4b8d6aa33139f5b0e298 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 22 Apr 2019 22:06:35 +0200 Subject: [PATCH 08/23] Feed apps argument to app_upgrade --- src/yunohost/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 784f5d512..5f38971bf 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -731,7 +731,7 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): if apps is not None: try: - app_upgrade(auth) + app_upgrade(auth, app=apps) except Exception as e: failure = True logger.warning('unable to upgrade apps: %s' % str(e)) From 779a16dab1960ce7365c9782ef65cde9e3a90bbc Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 22 Apr 2019 22:15:16 +0200 Subject: [PATCH 09/23] Clarify the whole error / success handling --- src/yunohost/tools.py | 51 +++++++++++++++---------------------------- 1 file changed, 17 insertions(+), 34 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 5f38971bf..20ab7cfae 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -602,8 +602,6 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): # TODO : i18n raise YunohostError("Please specify --apps OR --system") - failure = False - if system is True: # Check that there's indeed some packages to upgrade @@ -633,7 +631,7 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): # # "Regular" packages upgrade # - if not failure and noncritical_packages_upgradable: + if noncritical_packages_upgradable: # TODO : i18n logger.info("Upgrading 'regular' (non-yunohost-related) packages ...") @@ -645,31 +643,26 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): # Doublecheck with apt-mark showhold that packages are indeed held ... held_packages = check_output("apt-mark showhold").split("\n") if any(p not in held_packages for p in critical_packages): - failure = True logger.warning('Unable to hold critical packages ...') - logger.error(m18n.n('packages_upgrade_failed')) - # FIXME : watdo here, should this be an exception or just an - # error operation_logger.error(m18n.n('packages_upgrade_failed')) + raise YunohostError(m18n.n('packages_upgrade_failed')) - if not failure: - logger.debug("Running apt command :\n{}".format(dist_upgrade)) + logger.debug("Running apt command :\n{}".format(dist_upgrade)) - callbacks = ( - lambda l: logger.info(l.rstrip()), - lambda l: logger.warning(l.rstrip()), - ) - returncode = call_async_output(dist_upgrade, callbacks, shell=True) - if returncode != 0: - failure = True - logger.warning('unable to upgrade packages: %s' % ', '.join(noncritical_packages_upgradable)) - logger.error(m18n.n('packages_upgrade_failed')) - operation_logger.error(m18n.n('packages_upgrade_failed')) + callbacks = ( + lambda l: logger.info(l.rstrip()), + lambda l: logger.warning(l.rstrip()), + ) + returncode = call_async_output(dist_upgrade, callbacks, shell=True) + if returncode != 0: + logger.warning('unable to upgrade packages: %s' % ', '.join(noncritical_packages_upgradable)) + operation_logger.error(m18n.n('packages_upgrade_failed')) + raise YunohostError(m18n.n('packages_upgrade_failed')) # # Critical packages upgrade # - if not failure and critical_packages_upgradable: + if critical_packages_upgradable: # TODO : i18n logger.info("Upgrading 'special' (yunohost-related) packages ...") @@ -681,12 +674,9 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): # Doublecheck with apt-mark showhold that packages are indeed unheld ... unheld_packages = check_output("apt-mark showhold").split("\n") if any(p in unheld_packages for p in critical_packages): - failure = True logger.warning('Unable to unhold critical packages ...') - logger.error(m18n.n('packages_upgrade_failed')) - # FIXME : watdo here, should this be an exception or just an - # error operation_logger.error(m18n.n('packages_upgrade_failed')) + raise YunohostError(m18n.n('packages_upgrade_failed')) # # Here we use a dirty hack to run a command after the current @@ -715,17 +705,14 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): logger.debug("Running command :\n{}".format(command)) os.system(command) - - # TODO / FIXME : return from this function immediately, - # otherwise the apps upgrade might happen and it's gonna be a mess + return # FIXME / open question : what about "permanently" mark yunohost # as "hold" to avoid accidental deletion of it... # (so, only unhold it during the upgrade) - elif not failure: - - logger.info(m18n.n('done')) + else: + logger.success(m18n.n('system_upgraded')) operation_logger.success() @@ -733,13 +720,9 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): try: app_upgrade(auth, app=apps) except Exception as e: - failure = True logger.warning('unable to upgrade apps: %s' % str(e)) logger.error(m18n.n('app_upgrade_some_app_failed')) - if not failure: - logger.success(m18n.n('system_upgraded')) - # Return API logs if it is an API call is_api = True if msettings.get('interface') == 'api' else False if is_api: From 845b7448fb23a63ebffdf88511d5d075d2425196 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 22 Apr 2019 22:25:11 +0200 Subject: [PATCH 10/23] Improve tools_upgrade description --- locales/en.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locales/en.json b/locales/en.json index d6a17e7f6..a72f2a654 100644 --- a/locales/en.json +++ b/locales/en.json @@ -270,7 +270,7 @@ "log_tools_migrations_migrate_forward": "Migrate forward", "log_tools_migrations_migrate_backward": "Migrate backward", "log_tools_postinstall": "Postinstall your YunoHost server", - "log_tools_upgrade": "Upgrade debian packages", + "log_tools_upgrade": "Upgrade system packages", "log_tools_shutdown": "Shutdown your server", "log_tools_reboot": "Reboot your server", "ldap_init_failed_to_create_admin": "LDAP initialization failed to create admin user", From ad8f7bbcbc4648b4893a06ebfaaa7f1b00cf7ec0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 22 Apr 2019 22:29:03 +0200 Subject: [PATCH 11/23] Not really conviced about this return thing for the API, we can do better than that ... --- src/yunohost/tools.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 20ab7cfae..edfaa6e32 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -723,11 +723,6 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): logger.warning('unable to upgrade apps: %s' % str(e)) logger.error(m18n.n('app_upgrade_some_app_failed')) - # Return API logs if it is an API call - is_api = True if msettings.get('interface') == 'api' else False - if is_api: - return {"log": service_log('yunohost-api', number="100").values()[0]} - def tools_diagnosis(auth, private=False): """ From 68ac7d3f1cef9e21e94eb7a41029e04281e44385 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 22 Apr 2019 22:31:21 +0200 Subject: [PATCH 12/23] Misc tweak for display --- src/yunohost/tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index edfaa6e32..bfb72d2b6 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -691,12 +691,12 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): MOULINETTE_LOCK = "/var/run/moulinette_yunohost.lock" wait_until_end_of_yunohost_command = "(while [ -f {} ]; do sleep 2; done)".format(MOULINETTE_LOCK) - mark_success = "(echo 'success: true' | tee -a {})".format(operation_logger.md_path) + mark_success = "(echo 'success: true' > {})".format(operation_logger.md_path) update_log_metadata = "sed -i \"s/ended_at: .*$/ended_at: $(date -u +'%Y-%m-%d %H:%M:%S.%N')/\" {}" update_log_metadata = update_log_metadata.format(operation_logger.md_path) # TODO : i18n - upgrade_completed = "YunoHost package upgrade completed ! Press [enter] to get the command line back" + upgrade_completed = "\nYunoHost package upgrade completed !\nPress [Enter] to get the command line back" command = "({} && {} && {}; {}; echo '{}') &".format(wait_until_end_of_yunohost_command, command, mark_success, From 1b84d9a65fe315ff452c73bf0eee86f6f8f34533 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 22 Apr 2019 23:25:57 +0200 Subject: [PATCH 13/23] Reorganize for clarity --- src/yunohost/tools.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index bfb72d2b6..b9056bbb2 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -602,6 +602,23 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): # TODO : i18n raise YunohostError("Please specify --apps OR --system") + # + # Apps + # This is basically just an alias to yunohost app upgrade ... + # + + if apps is not None: + try: + app_upgrade(auth, app=apps) + except Exception as e: + logger.warning('unable to upgrade apps: %s' % str(e)) + logger.error(m18n.n('app_upgrade_some_app_failed')) + + + # + # System + # + if system is True: # Check that there's indeed some packages to upgrade @@ -671,6 +688,7 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): # Mark all critical packages as unheld for package in critical_packages: check_output("apt-mark unhold %s" % package) + # Doublecheck with apt-mark showhold that packages are indeed unheld ... unheld_packages = check_output("apt-mark showhold").split("\n") if any(p in unheld_packages for p in critical_packages): @@ -716,14 +734,6 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): operation_logger.success() - if apps is not None: - try: - app_upgrade(auth, app=apps) - except Exception as e: - logger.warning('unable to upgrade apps: %s' % str(e)) - logger.error(m18n.n('app_upgrade_some_app_failed')) - - def tools_diagnosis(auth, private=False): """ Return global info about current yunohost instance to help debugging From ec9deec9d4acc71e2484e48e50357a77c65b1f3f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 23 Apr 2019 00:08:44 +0200 Subject: [PATCH 14/23] i18n for messages --- locales/en.json | 9 +++++++++ src/yunohost/tools.py | 32 +++++++++++++------------------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/locales/en.json b/locales/en.json index a72f2a654..a7033bfe0 100644 --- a/locales/en.json +++ b/locales/en.json @@ -482,6 +482,15 @@ "system_upgraded": "The system has been upgraded", "system_username_exists": "Username already exists in the system users", "this_action_broke_dpkg": "This action broke dpkg/apt (the system package managers)... You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.", + "tools_upgrade_at_least_one": "Please specify --apps OR --system", + "tools_upgrade_cant_both": "Cannot upgrade both system and apps at the same time", + "tools_upgrade_cant_hold_critical_packages": "Unable to hold critical packages ...", + "tools_upgrade_cant_unhold_critical_packages": "Unable to unhold critical packages ...", + "tools_upgrade_regular_packages": "Now upgrading 'regular' (non-yunohost-related) packages ...", + "tools_upgrade_regular_packages_failed": "Unable to upgrade packages: {packages_list}", + "tools_upgrade_special_packages": "Now upgrading 'special' (yunohost-related) packages ...", + "tools_upgrade_special_packages_explanation": "This action will end but the actual special upgrade will continue in background. Please don't start any other action on your server in the next ~10 minutes (depending on your hardware speed). Once it's done, you may have to re-log on the webadmin.", + "tools_upgrade_special_packages_completed": "YunoHost package upgrade completed !\nPress [Enter] to get the command line back", "unbackup_app": "App '{app:s}' will not be saved", "unexpected_error": "An unexpected error occured: {error}", "unit_unknown": "Unknown unit '{unit:s}'", diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index b9056bbb2..f8f972d2f 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -595,12 +595,10 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): raise YunohostError("dpkg_is_broken") if system is not False and apps is not None: - # TODO : i18n - raise YunohostError("Cannot upgrade both system and apps at the same time") + raise YunohostError("tools_upgrade_cant_both") if system is False and apps is None: - # TODO : i18n - raise YunohostError("Please specify --apps OR --system") + raise YunohostError("tools_upgrade_at_least_one") # # Apps @@ -650,17 +648,17 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): # if noncritical_packages_upgradable: - # TODO : i18n - logger.info("Upgrading 'regular' (non-yunohost-related) packages ...") + logger.info(m18n.n("tools_upgrade_regular_packages")) # TODO : factorize this in utils/packages.py ? # Mark all critical packages as held for package in critical_packages: check_output("apt-mark hold %s" % package) + # Doublecheck with apt-mark showhold that packages are indeed held ... held_packages = check_output("apt-mark showhold").split("\n") if any(p not in held_packages for p in critical_packages): - logger.warning('Unable to hold critical packages ...') + logger.warning(m18n.n("tools_upgrade_cant_hold_critical_packages")) operation_logger.error(m18n.n('packages_upgrade_failed')) raise YunohostError(m18n.n('packages_upgrade_failed')) @@ -672,7 +670,8 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): ) returncode = call_async_output(dist_upgrade, callbacks, shell=True) if returncode != 0: - logger.warning('unable to upgrade packages: %s' % ', '.join(noncritical_packages_upgradable)) + logger.warning('tools_upgrade_regular_packages_failed', + packages_list=', '.join(noncritical_packages_upgradable)) operation_logger.error(m18n.n('packages_upgrade_failed')) raise YunohostError(m18n.n('packages_upgrade_failed')) @@ -681,8 +680,7 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): # if critical_packages_upgradable: - # TODO : i18n - logger.info("Upgrading 'special' (yunohost-related) packages ...") + logger.info(m18n.n("tools_upgrade_special_packages")) # TODO : factorize this in utils/packages.py ? # Mark all critical packages as unheld @@ -690,9 +688,9 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): check_output("apt-mark unhold %s" % package) # Doublecheck with apt-mark showhold that packages are indeed unheld ... - unheld_packages = check_output("apt-mark showhold").split("\n") - if any(p in unheld_packages for p in critical_packages): - logger.warning('Unable to unhold critical packages ...') + held_packages = check_output("apt-mark showhold").split("\n") + if any(p in held_packages for p in critical_packages): + logger.warning(m18n.n("tools_upgrade_cant_unhold_critical_packages")) operation_logger.error(m18n.n('packages_upgrade_failed')) raise YunohostError(m18n.n('packages_upgrade_failed')) @@ -713,22 +711,18 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): update_log_metadata = "sed -i \"s/ended_at: .*$/ended_at: $(date -u +'%Y-%m-%d %H:%M:%S.%N')/\" {}" update_log_metadata = update_log_metadata.format(operation_logger.md_path) - # TODO : i18n - upgrade_completed = "\nYunoHost package upgrade completed !\nPress [Enter] to get the command line back" + upgrade_completed = "\n"+m18n.n("tools_upgrade_special_packages_completed") command = "({} && {} && {}; {}; echo '{}') &".format(wait_until_end_of_yunohost_command, command, mark_success, update_log_metadata, upgrade_completed) + logger.warning(m18n.n("tools_upgrade_special_packages_explanation")) logger.debug("Running command :\n{}".format(command)) os.system(command) return - # FIXME / open question : what about "permanently" mark yunohost - # as "hold" to avoid accidental deletion of it... - # (so, only unhold it during the upgrade) - else: logger.success(m18n.n('system_upgraded')) operation_logger.success() From a192bdd31a7da461f3c17c0d84f26e00b2b763ce Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 23 Apr 2019 16:51:32 +0200 Subject: [PATCH 15/23] Consistent options for tools_update (compared to upgrade) + semantic improvements --- data/actionsmap/yunohost.yml | 8 ++++---- locales/en.json | 5 +++-- src/yunohost/tools.py | 38 ++++++++++++++++++++---------------- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 08188c8c8..59a063a4e 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1548,11 +1548,11 @@ tools: action_help: YunoHost update api: PUT /update arguments: - --ignore-apps: - help: Ignore apps cache update and changelog + --apps: + help: Fetch the application list to check which apps can be upgraded action: store_true - --ignore-packages: - help: Ignore APT cache update and changelog + --system: + help: Fetch available system packages upgrades (equivalent to apt update) action: store_true ### tools_upgrade() diff --git a/locales/en.json b/locales/en.json index a7033bfe0..58b4dbc72 100644 --- a/locales/en.json +++ b/locales/en.json @@ -4,8 +4,9 @@ "admin_password": "Administration password", "admin_password_change_failed": "Unable to change password", "admin_password_changed": "The administration password has been changed", - "app_action_cannot_be_ran_because_required_services_down": "This app requires some services which are currently down. Before continuing, you should try to restart the following services (and possibly investigate why they are down) : {services}", "admin_password_too_long": "Please choose a password shorter than 127 characters", + "already_up_to_date": "Nothing to do! Everything is already up to date!", + "app_action_cannot_be_ran_because_required_services_down": "This app requires some services which are currently down. Before continuing, you should try to restart the following services (and possibly investigate why they are down) : {services}", "app_already_installed": "{app:s} is already installed", "app_already_installed_cant_change_url": "This app is already installed. The url cannot be changed just by this function. Look into `app changeurl` if it's available.", "app_already_up_to_date": "{app:s} is already up to date", @@ -367,7 +368,6 @@ "package_not_installed": "Package '{pkgname}' is not installed", "package_unexpected_error": "An unexpected error occurred processing the package '{pkgname}'", "package_unknown": "Unknown package '{pkgname}'", - "packages_no_upgrade": "There is no package to upgrade", "packages_upgrade_critical_later": "Critical packages ({packages:s}) will be upgraded later", "packages_upgrade_failed": "Unable to upgrade all of the packages", "password_listed": "This password is among the most used password in the world. Please choose something a bit more unique.", @@ -499,6 +499,7 @@ "update_apt_cache_failed": "Unable to update the cache of APT (Debian's package manager). Here is a dump of the sources.list lines which might help to identify problematic lines : \n{sourceslist}", "update_apt_cache_warning": "Some errors happened while updating the cache of APT (Debian's package manager). Here is a dump of the sources.list lines which might help to identify problematic lines : \n{sourceslist}", "updating_apt_cache": "Fetching available upgrades for system packages…", + "updating_app_lists": "Fetching available upgrades for applications…", "upgrade_complete": "Upgrade complete", "upgrading_packages": "Upgrading packages…", "upnp_dev_not_found": "No UPnP device found", diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index f8f972d2f..57870f544 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -470,18 +470,22 @@ def tools_regen_conf(names=[], with_diff=False, force=False, dry_run=False, return regen_conf(names, with_diff, force, dry_run, list_pending) -def tools_update(ignore_apps=False, ignore_packages=False): +def tools_update(apps=False, system=False): """ - Update apps & package cache, then display changelog + Update apps & system package cache Keyword arguments: - ignore_apps -- Ignore app list update and changelog - ignore_packages -- Ignore apt cache update and changelog - + system -- Fetch available system packages upgrades (equivalent to apt update) + apps -- Fetch the application list to check which apps can be upgraded """ - # "packages" will list upgradable packages - packages = [] - if not ignore_packages: + + # If neither --apps nor --system specified, do both + if not apps and not system: + apps = True + system = True + + upgradable_system_packages = [] + if system: # Update APT cache # LC_ALL=C is here to make sure the results are in english @@ -514,12 +518,12 @@ def tools_update(ignore_apps=False, ignore_packages=False): elif warnings: logger.error(m18n.n('update_apt_cache_warning', sourceslist='\n'.join(_dump_sources_list()))) - packages = list(_list_upgradable_apt_packages()) + upgradable_system_packages = list(_list_upgradable_apt_packages()) logger.debug(m18n.n('done')) - # "apps" will list upgradable packages - apps = [] - if not ignore_apps: + upgradable_apps = [] + if apps: + logger.info(m18n.n('updating_app_lists')) try: app_fetchlist() except YunohostError: @@ -532,15 +536,15 @@ def tools_update(ignore_apps=False, ignore_packages=False): app_dict = app_info(app_id, raw=True) if app_dict["upgradable"] == "yes": - apps.append({ + upgradable_apps.append({ 'id': app_id, 'label': app_dict['settings']['label'] }) - if len(apps) == 0 and len(packages) == 0: - logger.info(m18n.n('packages_no_upgrade')) + if len(upgradable_apps) == 0 and len(upgradable_system_packages) == 0: + logger.info(m18n.n('already_up_to_date')) - return {'packages': packages, 'apps': apps} + return {'system': upgradable_system_packages, 'apps': upgradable_apps} # TODO : move this to utils/packages.py ? @@ -622,7 +626,7 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): # Check that there's indeed some packages to upgrade upgradables = list(_list_upgradable_apt_packages()) if not upgradables: - logger.info(m18n.n('packages_no_upgrade')) + logger.info(m18n.n('already_up_to_date')) logger.info(m18n.n('upgrading_packages')) operation_logger.start() From 4f3eaa5e2349fcffc0b26a18ec636bd172ddd934 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 24 Apr 2019 00:49:11 +0200 Subject: [PATCH 16/23] Check there's actually some apps to update when using tools upgrade --apps --- src/yunohost/tools.py | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 57870f544..67ea92bb3 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -530,16 +530,7 @@ def tools_update(apps=False, system=False): # FIXME : silent exception !? pass - app_list_installed = os.listdir(APPS_SETTING_PATH) - for app_id in app_list_installed: - - app_dict = app_info(app_id, raw=True) - - if app_dict["upgradable"] == "yes": - upgradable_apps.append({ - 'id': app_id, - 'label': app_dict['settings']['label'] - }) + upgradable_apps = list(_list_upgradable_apps()) if len(upgradable_apps) == 0 and len(upgradable_system_packages) == 0: logger.info(m18n.n('already_up_to_date')) @@ -547,6 +538,20 @@ def tools_update(apps=False, system=False): return {'system': upgradable_system_packages, 'apps': upgradable_apps} +def _list_upgradable_apps(): + + app_list_installed = os.listdir(APPS_SETTING_PATH) + for app_id in app_list_installed: + + app_dict = app_info(app_id, raw=True) + + if app_dict["upgradable"] == "yes": + yield { + 'id': app_id, + 'label': app_dict['settings']['label'] + } + + # TODO : move this to utils/packages.py ? def _list_upgradable_apt_packages(): @@ -610,12 +615,27 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): # if apps is not None: + + # Make sure there's actually something to upgrade + + upgradable_apps = [app["id"] for app in _list_upgradable_apps()] + + if not upgradable_apps: + logger.info(m18n.n("app_no_upgrade")) + return + elif len(apps) and all(app not in upgradable_apps for app in apps): + logger.info(m18n.n("apps_already_up_to_date")) + return + + # Actually start the upgrades + try: app_upgrade(auth, app=apps) except Exception as e: logger.warning('unable to upgrade apps: %s' % str(e)) logger.error(m18n.n('app_upgrade_some_app_failed')) + return # # System From 12bcd5d0f86dcab3ddff33aba6ddd1710eb17b0d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 24 Apr 2019 00:49:52 +0200 Subject: [PATCH 17/23] Avoid messy output when running apt stuff --- src/yunohost/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 67ea92bb3..d86da719a 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -689,7 +689,7 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): logger.debug("Running apt command :\n{}".format(dist_upgrade)) callbacks = ( - lambda l: logger.info(l.rstrip()), + lambda l: logger.info(l.rstrip() + "\r"), lambda l: logger.warning(l.rstrip()), ) returncode = call_async_output(dist_upgrade, callbacks, shell=True) From 5731e69541a4fa1a8c33546d232691ed2e84f00b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 24 Apr 2019 03:23:47 +0200 Subject: [PATCH 18/23] PEP8 --- src/yunohost/tools.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index d86da719a..fb05df940 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -34,12 +34,8 @@ from glob import glob from xmlrpclib import Fault from importlib import import_module from collections import OrderedDict -from datetime import datetime -import apt -import apt.progress - -from moulinette import msettings, msignals, m18n +from moulinette import msignals, m18n from moulinette.core import init_authenticator from yunohost.utils.error import YunohostError from moulinette.utils.log import getActionLogger @@ -49,7 +45,7 @@ from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, a from yunohost.domain import domain_add, domain_list, _get_maindomain, _set_maindomain from yunohost.dyndns import _dyndns_available, _dyndns_provides from yunohost.firewall import firewall_upnp -from yunohost.service import service_status, service_log, service_start, service_enable +from yunohost.service import service_status, service_start, service_enable from yunohost.regenconf import regen_conf from yunohost.monitor import monitor_disk, monitor_system from yunohost.utils.packages import ynh_packages_version @@ -496,6 +492,7 @@ def tools_update(apps=False, system=False): # Filter boring message about "apt not having a stable CLI interface" # Also keep track of wether or not we encountered a warning... warnings = [] + def is_legit_warning(m): legit_warning = m.rstrip() and "apt does not have a stable CLI interface" not in m.rstrip() if legit_warning: @@ -735,7 +732,7 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): update_log_metadata = "sed -i \"s/ended_at: .*$/ended_at: $(date -u +'%Y-%m-%d %H:%M:%S.%N')/\" {}" update_log_metadata = update_log_metadata.format(operation_logger.md_path) - upgrade_completed = "\n"+m18n.n("tools_upgrade_special_packages_completed") + upgrade_completed = "\n" + m18n.n("tools_upgrade_special_packages_completed") command = "({} && {} && {}; {}; echo '{}') &".format(wait_until_end_of_yunohost_command, command, mark_success, From 1ed65fa1d92731254158ca8456f09c8ffcb864a7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 24 Apr 2019 16:42:37 +0200 Subject: [PATCH 19/23] Add some explanation about where to find the log for special upgrade --- locales/en.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locales/en.json b/locales/en.json index 58b4dbc72..689b2f637 100644 --- a/locales/en.json +++ b/locales/en.json @@ -489,7 +489,7 @@ "tools_upgrade_regular_packages": "Now upgrading 'regular' (non-yunohost-related) packages ...", "tools_upgrade_regular_packages_failed": "Unable to upgrade packages: {packages_list}", "tools_upgrade_special_packages": "Now upgrading 'special' (yunohost-related) packages ...", - "tools_upgrade_special_packages_explanation": "This action will end but the actual special upgrade will continue in background. Please don't start any other action on your server in the next ~10 minutes (depending on your hardware speed). Once it's done, you may have to re-log on the webadmin.", + "tools_upgrade_special_packages_explanation": "This action will end but the actual special upgrade will continue in background. Please don't start any other action on your server in the next ~10 minutes (depending on your hardware speed). Once it's done, you may have to re-log on the webadmin. The upgrade log will be available in Tools > Log (in the webadmin) or through 'yunohost log list' (in command line).", "tools_upgrade_special_packages_completed": "YunoHost package upgrade completed !\nPress [Enter] to get the command line back", "unbackup_app": "App '{app:s}' will not be saved", "unexpected_error": "An unexpected error occured: {error}", From d0faff609e19633a9aeb990be9cac503946d5453 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 24 Apr 2019 17:21:29 +0200 Subject: [PATCH 20/23] Remove weird API behavior for app upgrade, will adapt the webadmin --- src/yunohost/app.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 7813dee58..1426898f2 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -697,10 +697,6 @@ def app_upgrade(auth, app=[], url=None, file=None): logger.success(m18n.n('upgrade_complete')) - # Return API logs if it is an API call - if is_api: - return {"log": service_log('yunohost-api', number="100").values()[0]} - @is_unit_operation() def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on_failure=False, force=False): From f4b87f969522d79a672f4ba2839dfc2f86006111 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 24 Apr 2019 17:46:39 +0200 Subject: [PATCH 21/23] Move low-level apt stuff to utils/packages.py --- src/yunohost/tools.py | 45 +--------------------------------- src/yunohost/utils/packages.py | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index fb05df940..5128d0cbe 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -30,7 +30,6 @@ import json import subprocess import pwd import socket -from glob import glob from xmlrpclib import Fault from importlib import import_module from collections import OrderedDict @@ -48,7 +47,7 @@ from yunohost.firewall import firewall_upnp from yunohost.service import service_status, service_start, service_enable from yunohost.regenconf import regen_conf from yunohost.monitor import monitor_disk, monitor_system -from yunohost.utils.packages import ynh_packages_version +from yunohost.utils.packages import ynh_packages_version, _dump_sources_list, _list_upgradable_apt_packages from yunohost.utils.network import get_public_ip from yunohost.log import is_unit_operation, OperationLogger @@ -486,8 +485,6 @@ def tools_update(apps=False, system=False): # Update APT cache # LC_ALL=C is here to make sure the results are in english command = "LC_ALL=C apt update" - # TODO : add @is_unit_operation to tools_update so that the - # debug output can be fetched when there's an issue... # Filter boring message about "apt not having a stable CLI interface" # Also keep track of wether or not we encountered a warning... @@ -549,44 +546,6 @@ def _list_upgradable_apps(): } -# TODO : move this to utils/packages.py ? -def _list_upgradable_apt_packages(): - - # List upgradable packages - # LC_ALL=C is here to make sure the results are in english - upgradable_raw = check_output("LC_ALL=C apt list --upgradable") - - # Dirty parsing of the output - upgradable_raw = [l.strip() for l in upgradable_raw.split("\n") if l.strip()] - for line in upgradable_raw: - # Remove stupid warning and verbose messages >.> - if "apt does not have a stable CLI interface" in line or "Listing..." in line: - continue - # line should look like : - # yunohost/stable 3.5.0.2+201903211853 all [upgradable from: 3.4.2.4+201903080053] - line = line.split() - if len(line) != 6: - logger.warning("Failed to parse this line : %s" % ' '.join(line)) - continue - - yield { - "name": line[0].split("/")[0], - "new_version": line[1], - "current_version": line[5].strip("]"), - } - - -def _dump_sources_list(): - - filenames = glob("/etc/apt/sources.list") + glob("/etc/apt/sources.list.d/*") - for filename in filenames: - with open(filename, "r") as f: - for line in f.readlines(): - if line.startswith("#") or not line.strip(): - continue - yield filename.replace("/etc/apt/", "") + ":" + line.strip() - - @is_unit_operation() def tools_upgrade(operation_logger, auth, apps=None, system=False): """ @@ -671,7 +630,6 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): logger.info(m18n.n("tools_upgrade_regular_packages")) - # TODO : factorize this in utils/packages.py ? # Mark all critical packages as held for package in critical_packages: check_output("apt-mark hold %s" % package) @@ -703,7 +661,6 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): logger.info(m18n.n("tools_upgrade_special_packages")) - # TODO : factorize this in utils/packages.py ? # Mark all critical packages as unheld for package in critical_packages: check_output("apt-mark unhold %s" % package) diff --git a/src/yunohost/utils/packages.py b/src/yunohost/utils/packages.py index e10de6493..b564d2dea 100644 --- a/src/yunohost/utils/packages.py +++ b/src/yunohost/utils/packages.py @@ -481,3 +481,46 @@ def dpkg_is_broken(): return False return any(re.match("^[0-9]+$", f) for f in os.listdir("/var/lib/dpkg/updates/")) + + +def _list_upgradable_apt_packages(): + + from moulinette.utils.process import check_output + + # List upgradable packages + # LC_ALL=C is here to make sure the results are in english + upgradable_raw = check_output("LC_ALL=C apt list --upgradable") + + # Dirty parsing of the output + upgradable_raw = [l.strip() for l in upgradable_raw.split("\n") if l.strip()] + for line in upgradable_raw: + + # Remove stupid warning and verbose messages >.> + if "apt does not have a stable CLI interface" in line or "Listing..." in line: + continue + + # line should look like : + # yunohost/stable 3.5.0.2+201903211853 all [upgradable from: 3.4.2.4+201903080053] + line = line.split() + if len(line) != 6: + logger.warning("Failed to parse this line : %s" % ' '.join(line)) + continue + + yield { + "name": line[0].split("/")[0], + "new_version": line[1], + "current_version": line[5].strip("]"), + } + + +def _dump_sources_list(): + + from glob import glob + + filenames = glob("/etc/apt/sources.list") + glob("/etc/apt/sources.list.d/*") + for filename in filenames: + with open(filename, "r") as f: + for line in f.readlines(): + if line.startswith("#") or not line.strip(): + continue + yield filename.replace("/etc/apt/", "") + ":" + line.strip() From cf619fe13497f46f70a8c1f4ea6a23a7e99c7efb Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 24 Apr 2019 19:02:35 +0200 Subject: [PATCH 22/23] Fix / improve success mark in logs --- src/yunohost/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 5128d0cbe..8797f7349 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -685,7 +685,7 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): MOULINETTE_LOCK = "/var/run/moulinette_yunohost.lock" wait_until_end_of_yunohost_command = "(while [ -f {} ]; do sleep 2; done)".format(MOULINETTE_LOCK) - mark_success = "(echo 'success: true' > {})".format(operation_logger.md_path) + mark_success = "(echo 'Done!' | tee -a {} && echo 'success: true' >> {})".format(logfile, operation_logger.md_path) update_log_metadata = "sed -i \"s/ended_at: .*$/ended_at: $(date -u +'%Y-%m-%d %H:%M:%S.%N')/\" {}" update_log_metadata = update_log_metadata.format(operation_logger.md_path) From 7ee0e986a6a85420230191dd1c68587722586c06 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 25 Apr 2019 00:26:42 +0200 Subject: [PATCH 23/23] Explicly mark failure if command failed + more readable syntax --- src/yunohost/tools.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 8797f7349..b992fb6c6 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -686,15 +686,18 @@ def tools_upgrade(operation_logger, auth, apps=None, system=False): MOULINETTE_LOCK = "/var/run/moulinette_yunohost.lock" wait_until_end_of_yunohost_command = "(while [ -f {} ]; do sleep 2; done)".format(MOULINETTE_LOCK) mark_success = "(echo 'Done!' | tee -a {} && echo 'success: true' >> {})".format(logfile, operation_logger.md_path) + mark_failure = "(echo 'Failed :(' | tee -a {} && echo 'success: false' >> {})".format(logfile, operation_logger.md_path) update_log_metadata = "sed -i \"s/ended_at: .*$/ended_at: $(date -u +'%Y-%m-%d %H:%M:%S.%N')/\" {}" update_log_metadata = update_log_metadata.format(operation_logger.md_path) upgrade_completed = "\n" + m18n.n("tools_upgrade_special_packages_completed") - command = "({} && {} && {}; {}; echo '{}') &".format(wait_until_end_of_yunohost_command, - command, - mark_success, - update_log_metadata, - upgrade_completed) + command = "(({wait} && {cmd}) && {mark_success} || {mark_failure}; {update_metadata}; echo '{done}') &".format( + wait=wait_until_end_of_yunohost_command, + cmd=command, + mark_success=mark_success, + mark_failure=mark_failure, + update_metadata=update_log_metadata, + done=upgrade_completed) logger.warning(m18n.n("tools_upgrade_special_packages_explanation")) logger.debug("Running command :\n{}".format(command))