diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 7242f9f9d..237a6e404 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1449,11 +1449,11 @@ tools: action_help: YunoHost update api: PUT /update arguments: - --ignore-apps: - help: Ignore apps cache update and changelog + --apps: + help: Fetch the application list to check which apps can be upgraded action: store_true - --ignore-packages: - help: Ignore APT cache update and changelog + --system: + help: Fetch available system packages upgrades (equivalent to apt update) action: store_true ### tools_upgrade() @@ -1461,11 +1461,11 @@ tools: action_help: YunoHost upgrade api: PUT /upgrade arguments: - --ignore-apps: - help: Ignore apps upgrade - action: store_true - --ignore-packages: - help: Ignore APT packages upgrade + --apps: + help: List of apps to upgrade (all by default) + nargs: "*" + --system: + help: Upgrade only the system packages action: store_true ### tools_diagnosis() diff --git a/data/hooks/conf_regen/19-postfix b/data/hooks/conf_regen/19-postfix index a3ad70327..b37425984 100755 --- a/data/hooks/conf_regen/19-postfix +++ b/data/hooks/conf_regen/19-postfix @@ -2,6 +2,8 @@ set -e +. /usr/share/yunohost/helpers + do_pre_regen() { pending_dir=$1 @@ -20,9 +22,12 @@ do_pre_regen() { main_domain=$(cat /etc/yunohost/current_host) domain_list=$(sudo yunohost domain list --output-as plain --quiet | tr '\n' ' ') - cat main.cf \ - | sed "s/{{ main_domain }}/${main_domain}/g" \ - > "${postfix_dir}/main.cf" + # Support different strategy for security configurations + export compatibility="$(yunohost settings get 'security.postfix.compatibility')" + + export main_domain + export domain_list + ynh_render_template "main.cf" "${postfix_dir}/main.cf" cat postsrsd \ | sed "s/{{ main_domain }}/${main_domain}/g" \ diff --git a/data/templates/postfix/main.cf b/data/templates/postfix/main.cf index c38896a3f..e5a3875d4 100644 --- a/data/templates/postfix/main.cf +++ b/data/templates/postfix/main.cf @@ -33,7 +33,11 @@ smtpd_tls_key_file = /etc/yunohost/certs/{{ main_domain }}/key.pem smtpd_tls_exclude_ciphers = aNULL, MD5, DES, ADH, RC4, 3DES smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache smtpd_tls_loglevel=1 +{% if compatibility == "intermediate" %} smtpd_tls_mandatory_protocols=!SSLv2,!SSLv3 +{% else %} +smtpd_tls_mandatory_protocols=!SSLv2,!SSLv3,!TLSv1,!TLSv1.1 +{% endif %} smtpd_tls_mandatory_ciphers=high smtpd_tls_eecdh_grade = ultra @@ -58,7 +62,7 @@ alias_maps = hash:/etc/aliases alias_database = hash:/etc/aliases mydomain = {{ main_domain }} mydestination = localhost -relayhost = +relayhost = mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 mailbox_command = procmail -a "$EXTENSION" mailbox_size_limit = 0 @@ -68,71 +72,71 @@ inet_interfaces = all #### Fit to the maximum message size to 30mb, more than allowed by GMail or Yahoo #### message_size_limit = 31457280 -# Virtual Domains Control -virtual_mailbox_domains = ldap:/etc/postfix/ldap-domains.cf -virtual_mailbox_maps = ldap:/etc/postfix/ldap-accounts.cf -virtual_mailbox_base = -virtual_alias_maps = ldap:/etc/postfix/ldap-aliases.cf -virtual_alias_domains = -virtual_minimum_uid = 100 -virtual_uid_maps = static:vmail +# Virtual Domains Control +virtual_mailbox_domains = ldap:/etc/postfix/ldap-domains.cf +virtual_mailbox_maps = ldap:/etc/postfix/ldap-accounts.cf +virtual_mailbox_base = +virtual_alias_maps = ldap:/etc/postfix/ldap-aliases.cf +virtual_alias_domains = +virtual_minimum_uid = 100 +virtual_uid_maps = static:vmail virtual_gid_maps = static:mail smtpd_sender_login_maps= ldap:/etc/postfix/ldap-accounts.cf -# Dovecot LDA -virtual_transport = dovecot +# Dovecot LDA +virtual_transport = dovecot dovecot_destination_recipient_limit = 1 -# Enable SASL authentication for the smtpd daemon -smtpd_sasl_auth_enable = yes -smtpd_sasl_type = dovecot -smtpd_sasl_path = private/auth -# Fix some outlook's bugs -broken_sasl_auth_clients = yes -# Reject anonymous connections -smtpd_sasl_security_options = noanonymous +# Enable SASL authentication for the smtpd daemon +smtpd_sasl_auth_enable = yes +smtpd_sasl_type = dovecot +smtpd_sasl_path = private/auth +# Fix some outlook's bugs +broken_sasl_auth_clients = yes +# Reject anonymous connections +smtpd_sasl_security_options = noanonymous smtpd_sasl_local_domain = -# Wait until the RCPT TO command before evaluating restrictions -smtpd_delay_reject = yes - -# Basics Restrictions -smtpd_helo_required = yes -strict_rfc821_envelopes = yes - -# Requirements for the connecting server -smtpd_client_restrictions = - permit_mynetworks, - permit_sasl_authenticated, - reject_rbl_client bl.spamcop.net, - reject_rbl_client cbl.abuseat.org, - reject_rbl_client zen.spamhaus.org, - permit - -# Requirements for the HELO statement -smtpd_helo_restrictions = - permit_mynetworks, - permit_sasl_authenticated, - reject_non_fqdn_hostname, - reject_invalid_hostname, - permit - -# Requirements for the sender address +# Wait until the RCPT TO command before evaluating restrictions +smtpd_delay_reject = yes + +# Basics Restrictions +smtpd_helo_required = yes +strict_rfc821_envelopes = yes + +# Requirements for the connecting server +smtpd_client_restrictions = + permit_mynetworks, + permit_sasl_authenticated, + reject_rbl_client bl.spamcop.net, + reject_rbl_client cbl.abuseat.org, + reject_rbl_client zen.spamhaus.org, + permit + +# Requirements for the HELO statement +smtpd_helo_restrictions = + permit_mynetworks, + permit_sasl_authenticated, + reject_non_fqdn_hostname, + reject_invalid_hostname, + permit + +# Requirements for the sender address smtpd_sender_restrictions = - reject_sender_login_mismatch, - permit_mynetworks, - permit_sasl_authenticated, - reject_non_fqdn_sender, + reject_sender_login_mismatch, + permit_mynetworks, + permit_sasl_authenticated, + reject_non_fqdn_sender, reject_unknown_sender_domain, - permit - -# Requirement for the recipient address -smtpd_recipient_restrictions = - permit_mynetworks, - permit_sasl_authenticated, - reject_non_fqdn_recipient, - reject_unknown_recipient_domain, + permit + +# Requirement for the recipient address +smtpd_recipient_restrictions = + permit_mynetworks, + permit_sasl_authenticated, + reject_non_fqdn_recipient, + reject_unknown_recipient_domain, reject_unauth_destination, permit diff --git a/locales/en.json b/locales/en.json index d6a17e7f6..511f69028 100644 --- a/locales/en.json +++ b/locales/en.json @@ -4,8 +4,9 @@ "admin_password": "Administration password", "admin_password_change_failed": "Unable to change password", "admin_password_changed": "The administration password has been changed", - "app_action_cannot_be_ran_because_required_services_down": "This app requires some services which are currently down. Before continuing, you should try to restart the following services (and possibly investigate why they are down) : {services}", "admin_password_too_long": "Please choose a password shorter than 127 characters", + "already_up_to_date": "Nothing to do! Everything is already up to date!", + "app_action_cannot_be_ran_because_required_services_down": "This app requires some services which are currently down. Before continuing, you should try to restart the following services (and possibly investigate why they are down) : {services}", "app_already_installed": "{app:s} is already installed", "app_already_installed_cant_change_url": "This app is already installed. The url cannot be changed just by this function. Look into `app changeurl` if it's available.", "app_already_up_to_date": "{app:s} is already up to date", @@ -216,6 +217,7 @@ "global_settings_setting_security_password_admin_strength": "Admin password strength", "global_settings_setting_security_password_user_strength": "User password strength", "global_settings_setting_security_ssh_compatibility": "Compatibility vs. security tradeoff for the SSH server. Affects the ciphers (and other security-related aspects)", + "global_settings_setting_security_postfix_compatibility": "Compatibility vs. security tradeoff for the Postfix server. Affects the ciphers (and other security-related aspects)", "global_settings_unknown_setting_from_settings_file": "Unknown key in settings: '{setting_key:s}', discarding it and save it in /etc/yunohost/settings-unknown.json", "global_settings_setting_service_ssh_allow_deprecated_dsa_hostkey": "Allow the use of (deprecated) DSA hostkey for the SSH daemon configuration", "global_settings_unknown_type": "Unexpected situation, the setting {setting:s} appears to have the type {unknown_type:s} but it's not a type supported by the system.", @@ -270,7 +272,7 @@ "log_tools_migrations_migrate_forward": "Migrate forward", "log_tools_migrations_migrate_backward": "Migrate backward", "log_tools_postinstall": "Postinstall your YunoHost server", - "log_tools_upgrade": "Upgrade debian packages", + "log_tools_upgrade": "Upgrade system packages", "log_tools_shutdown": "Shutdown your server", "log_tools_reboot": "Reboot your server", "ldap_init_failed_to_create_admin": "LDAP initialization failed to create admin user", @@ -367,7 +369,6 @@ "package_not_installed": "Package '{pkgname}' is not installed", "package_unexpected_error": "An unexpected error occurred processing the package '{pkgname}'", "package_unknown": "Unknown package '{pkgname}'", - "packages_no_upgrade": "There is no package to upgrade", "packages_upgrade_critical_later": "Critical packages ({packages:s}) will be upgraded later", "packages_upgrade_failed": "Unable to upgrade all of the packages", "password_listed": "This password is among the most used password in the world. Please choose something a bit more unique.", @@ -482,6 +483,15 @@ "system_upgraded": "The system has been upgraded", "system_username_exists": "Username already exists in the system users", "this_action_broke_dpkg": "This action broke dpkg/apt (the system package managers)... You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.", + "tools_upgrade_at_least_one": "Please specify --apps OR --system", + "tools_upgrade_cant_both": "Cannot upgrade both system and apps at the same time", + "tools_upgrade_cant_hold_critical_packages": "Unable to hold critical packages ...", + "tools_upgrade_cant_unhold_critical_packages": "Unable to unhold critical packages ...", + "tools_upgrade_regular_packages": "Now upgrading 'regular' (non-yunohost-related) packages ...", + "tools_upgrade_regular_packages_failed": "Unable to upgrade packages: {packages_list}", + "tools_upgrade_special_packages": "Now upgrading 'special' (yunohost-related) packages ...", + "tools_upgrade_special_packages_explanation": "This action will end but the actual special upgrade will continue in background. Please don't start any other action on your server in the next ~10 minutes (depending on your hardware speed). Once it's done, you may have to re-log on the webadmin. The upgrade log will be available in Tools > Log (in the webadmin) or through 'yunohost log list' (in command line).", + "tools_upgrade_special_packages_completed": "YunoHost package upgrade completed !\nPress [Enter] to get the command line back", "unbackup_app": "App '{app:s}' will not be saved", "unexpected_error": "An unexpected error occured: {error}", "unit_unknown": "Unknown unit '{unit:s}'", @@ -490,6 +500,7 @@ "update_apt_cache_failed": "Unable to update the cache of APT (Debian's package manager). Here is a dump of the sources.list lines which might help to identify problematic lines : \n{sourceslist}", "update_apt_cache_warning": "Some errors happened while updating the cache of APT (Debian's package manager). Here is a dump of the sources.list lines which might help to identify problematic lines : \n{sourceslist}", "updating_apt_cache": "Fetching available upgrades for system packages…", + "updating_app_lists": "Fetching available upgrades for applications…", "upgrade_complete": "Upgrade complete", "upgrading_packages": "Upgrading packages…", "upnp_dev_not_found": "No UPnP device found", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index f27a10abd..4ccfb6b47 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -697,10 +697,6 @@ def app_upgrade(app=[], url=None, file=None): logger.success(m18n.n('upgrade_complete')) - # Return API logs if it is an API call - if is_api: - return {"log": service_log('yunohost-api', number="100").values()[0]} - @is_unit_operation() def app_install(operation_logger, app, label=None, args=None, no_remove_on_failure=False, force=False): diff --git a/src/yunohost/log.py b/src/yunohost/log.py index 3620ac990..dd3bbd8b3 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -346,8 +346,7 @@ class OperationLogger(object): """ # TODO add a way to not save password on app installation - filename = os.path.join(self.path, self.name + LOG_FILE_EXT) - self.file_handler = FileHandler(filename) + self.file_handler = FileHandler(self.log_path) self.file_handler.formatter = Formatter('%(asctime)s: %(levelname)s - %(message)s') # Listen to the root logger @@ -359,8 +358,7 @@ class OperationLogger(object): Write or rewrite the metadata file with all metadata known """ - filename = os.path.join(self.path, self.name + METADATA_FILE_EXT) - with open(filename, 'w') as outfile: + with open(self.md_path, 'w') as outfile: yaml.safe_dump(self.metadata, outfile, default_flow_style=False) @property diff --git a/src/yunohost/settings.py b/src/yunohost/settings.py index 671ad70e9..01f27ba83 100644 --- a/src/yunohost/settings.py +++ b/src/yunohost/settings.py @@ -44,6 +44,8 @@ DEFAULTS = OrderedDict([ "choices": ["intermediate", "modern"]}), ("security.nginx.compatibility", {"type": "enum", "default": "intermediate", "choices": ["intermediate", "modern"]}), + ("security.postfix.compatibility", {"type": "enum", "default": "intermediate", + "choices": ["intermediate", "modern"]}), ]) @@ -292,3 +294,8 @@ def reconfigure_nginx(setting_name, old_value, new_value): def reconfigure_ssh(setting_name, old_value, new_value): if old_value != new_value: service_regen_conf(names=['ssh']) + +@post_change_hook("security.postfix.compatibility") +def reconfigure_ssh(setting_name, old_value, new_value): + if old_value != new_value: + service_regen_conf(names=['postfix']) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 572328392..3bb69c961 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -30,15 +30,11 @@ import json import subprocess import pwd import socket -from glob import glob from xmlrpclib import Fault from importlib import import_module from collections import OrderedDict -import apt -import apt.progress - -from moulinette import msettings, msignals, m18n +from moulinette import msignals, m18n from moulinette.utils.log import getActionLogger from moulinette.utils.process import check_output, call_async_output from moulinette.utils.filesystem import read_json, write_to_json @@ -46,10 +42,10 @@ from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, a from yunohost.domain import domain_add, domain_list, _get_maindomain, _set_maindomain from yunohost.dyndns import _dyndns_available, _dyndns_provides from yunohost.firewall import firewall_upnp -from yunohost.service import service_status, service_log, service_start, service_enable +from yunohost.service import service_status, service_start, service_enable from yunohost.regenconf import regen_conf from yunohost.monitor import monitor_disk, monitor_system -from yunohost.utils.packages import ynh_packages_version +from yunohost.utils.packages import ynh_packages_version, _dump_sources_list, _list_upgradable_apt_packages from yunohost.utils.network import get_public_ip from yunohost.utils.error import YunohostError from yunohost.log import is_unit_operation, OperationLogger @@ -463,28 +459,31 @@ def tools_regen_conf(names=[], with_diff=False, force=False, dry_run=False, return regen_conf(names, with_diff, force, dry_run, list_pending) -def tools_update(ignore_apps=False, ignore_packages=False): +def tools_update(apps=False, system=False): """ - Update apps & package cache, then display changelog + Update apps & system package cache Keyword arguments: - ignore_apps -- Ignore app list update and changelog - ignore_packages -- Ignore apt cache update and changelog - + system -- Fetch available system packages upgrades (equivalent to apt update) + apps -- Fetch the application list to check which apps can be upgraded """ - # "packages" will list upgradable packages - packages = [] - if not ignore_packages: + + # If neither --apps nor --system specified, do both + if not apps and not system: + apps = True + system = True + + upgradable_system_packages = [] + if system: # Update APT cache # LC_ALL=C is here to make sure the results are in english command = "LC_ALL=C apt update" - # TODO : add @is_unit_operation to tools_update so that the - # debug output can be fetched when there's an issue... # Filter boring message about "apt not having a stable CLI interface" # Also keep track of wether or not we encountered a warning... warnings = [] + def is_legit_warning(m): legit_warning = m.rstrip() and "apt does not have a stable CLI interface" not in m.rstrip() if legit_warning: @@ -507,158 +506,201 @@ def tools_update(ignore_apps=False, ignore_packages=False): elif warnings: logger.error(m18n.n('update_apt_cache_warning', sourceslist='\n'.join(_dump_sources_list()))) - packages = list(_list_upgradable_apt_packages()) + upgradable_system_packages = list(_list_upgradable_apt_packages()) logger.debug(m18n.n('done')) - # "apps" will list upgradable packages - apps = [] - if not ignore_apps: + upgradable_apps = [] + if apps: + logger.info(m18n.n('updating_app_lists')) try: app_fetchlist() except YunohostError: # FIXME : silent exception !? pass - app_list_installed = os.listdir(APPS_SETTING_PATH) - for app_id in app_list_installed: + upgradable_apps = list(_list_upgradable_apps()) - app_dict = app_info(app_id, raw=True) + if len(upgradable_apps) == 0 and len(upgradable_system_packages) == 0: + logger.info(m18n.n('already_up_to_date')) - if app_dict["upgradable"] == "yes": - apps.append({ - 'id': app_id, - 'label': app_dict['settings']['label'] - }) - - if len(apps) == 0 and len(packages) == 0: - logger.info(m18n.n('packages_no_upgrade')) - - return {'packages': packages, 'apps': apps} + return {'system': upgradable_system_packages, 'apps': upgradable_apps} -# TODO : move this to utils/packages.py ? -def _list_upgradable_apt_packages(): +def _list_upgradable_apps(): - # List upgradable packages - # LC_ALL=C is here to make sure the results are in english - upgradable_raw = check_output("LC_ALL=C apt list --upgradable") + app_list_installed = os.listdir(APPS_SETTING_PATH) + for app_id in app_list_installed: - # Dirty parsing of the output - upgradable_raw = [l.strip() for l in upgradable_raw.split("\n") if l.strip()] - for line in upgradable_raw: - # Remove stupid warning and verbose messages >.> - if "apt does not have a stable CLI interface" in line or "Listing..." in line: - continue - # line should look like : - # yunohost/stable 3.5.0.2+201903211853 all [upgradable from: 3.4.2.4+201903080053] - line = line.split() - if len(line) != 6: - logger.warning("Failed to parse this line : %s" % ' '.join(line)) - continue + app_dict = app_info(app_id, raw=True) - yield { - "name": line[0].split("/")[0], - "new_version": line[1], - "current_version": line[5].strip("]"), - } - - -def _dump_sources_list(): - - filenames = glob("/etc/apt/sources.list") + glob("/etc/apt/sources.list.d/*") - for filename in filenames: - with open(filename, "r") as f: - for line in f.readlines(): - if line.startswith("#") or not line.strip(): - continue - yield filename.replace("/etc/apt/", "") + ":" + line.strip() + if app_dict["upgradable"] == "yes": + yield { + 'id': app_id, + 'label': app_dict['settings']['label'] + } @is_unit_operation() -def tools_upgrade(operation_logger, ignore_apps=False, ignore_packages=False): +def tools_upgrade(operation_logger, apps=None, system=False): """ Update apps & package cache, then display changelog Keyword arguments: - ignore_apps -- Ignore apps upgrade - ignore_packages -- Ignore APT packages upgrade - + apps -- List of apps to upgrade (or [] to update all apps) + system -- True to upgrade system """ from yunohost.utils import packages if packages.dpkg_is_broken(): raise YunohostError("dpkg_is_broken") - failure = False + if system is not False and apps is not None: + raise YunohostError("tools_upgrade_cant_both") - # Retrieve interface - is_api = True if msettings.get('interface') == 'api' else False + if system is False and apps is None: + raise YunohostError("tools_upgrade_at_least_one") - if not ignore_packages: + # + # Apps + # This is basically just an alias to yunohost app upgrade ... + # - apt.apt_pkg.init() - apt.apt_pkg.config.set("DPkg::Options::", "--force-confdef") - apt.apt_pkg.config.set("DPkg::Options::", "--force-confold") + if apps is not None: - cache = apt.Cache() - cache.open(None) - cache.upgrade(True) + # Make sure there's actually something to upgrade - # If API call - if is_api: - critical_packages = ("moulinette", "yunohost", - "yunohost-admin", "ssowat", "python") - critical_upgrades = set() + upgradable_apps = [app["id"] for app in _list_upgradable_apps()] - for pkg in cache.get_changes(): - if pkg.name in critical_packages: - critical_upgrades.add(pkg.name) - # Temporarily keep package ... - pkg.mark_keep() + if not upgradable_apps: + logger.info(m18n.n("app_no_upgrade")) + return + elif len(apps) and all(app not in upgradable_apps for app in apps): + logger.info(m18n.n("apps_already_up_to_date")) + return - # ... and set a hourly cron up to upgrade critical packages - if critical_upgrades: - logger.info(m18n.n('packages_upgrade_critical_later', - packages=', '.join(critical_upgrades))) - with open('/etc/cron.d/yunohost-upgrade', 'w+') as f: - f.write('00 * * * * root PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin apt-get install %s -y && rm -f /etc/cron.d/yunohost-upgrade\n' % ' '.join(critical_upgrades)) + # Actually start the upgrades - if cache.get_changes(): - logger.info(m18n.n('upgrading_packages')) - - operation_logger.start() - try: - os.environ["DEBIAN_FRONTEND"] = "noninteractive" - # Apply APT changes - # TODO: Logs output for the API - cache.commit(apt.progress.text.AcquireProgress(), - apt.progress.base.InstallProgress()) - except Exception as e: - failure = True - logger.warning('unable to upgrade packages: %s' % str(e)) - logger.error(m18n.n('packages_upgrade_failed')) - operation_logger.error(m18n.n('packages_upgrade_failed')) - else: - logger.info(m18n.n('done')) - operation_logger.success() - finally: - del os.environ["DEBIAN_FRONTEND"] - else: - logger.info(m18n.n('packages_no_upgrade')) - - if not ignore_apps: try: - app_upgrade() + app_upgrade(app=apps) except Exception as e: - failure = True logger.warning('unable to upgrade apps: %s' % str(e)) logger.error(m18n.n('app_upgrade_some_app_failed')) - if not failure: - logger.success(m18n.n('system_upgraded')) + return - # Return API logs if it is an API call - if is_api: - return {"log": service_log('yunohost-api', number="100").values()[0]} + # + # System + # + + if system is True: + + # Check that there's indeed some packages to upgrade + upgradables = list(_list_upgradable_apt_packages()) + if not upgradables: + logger.info(m18n.n('already_up_to_date')) + + logger.info(m18n.n('upgrading_packages')) + operation_logger.start() + + # Critical packages are packages that we can't just upgrade + # randomly from yunohost itself... upgrading them is likely to + critical_packages = ("moulinette", "yunohost", "yunohost-admin", "ssowat", "python") + + critical_packages_upgradable = [p for p in upgradables if p["name"] in critical_packages] + noncritical_packages_upgradable = [p for p in upgradables if p["name"] not in critical_packages] + + # Prepare dist-upgrade command + dist_upgrade = "DEBIAN_FRONTEND=noninteractive" + dist_upgrade += " APT_LISTCHANGES_FRONTEND=none" + dist_upgrade += " apt-get" + dist_upgrade += " --fix-broken --show-upgraded --assume-yes" + for conf_flag in ["old", "miss", "def"]: + dist_upgrade += ' -o Dpkg::Options::="--force-conf{}"'.format(conf_flag) + dist_upgrade += " dist-upgrade" + + # + # "Regular" packages upgrade + # + if noncritical_packages_upgradable: + + logger.info(m18n.n("tools_upgrade_regular_packages")) + + # Mark all critical packages as held + for package in critical_packages: + check_output("apt-mark hold %s" % package) + + # Doublecheck with apt-mark showhold that packages are indeed held ... + held_packages = check_output("apt-mark showhold").split("\n") + if any(p not in held_packages for p in critical_packages): + logger.warning(m18n.n("tools_upgrade_cant_hold_critical_packages")) + operation_logger.error(m18n.n('packages_upgrade_failed')) + raise YunohostError(m18n.n('packages_upgrade_failed')) + + logger.debug("Running apt command :\n{}".format(dist_upgrade)) + + callbacks = ( + lambda l: logger.info(l.rstrip() + "\r"), + lambda l: logger.warning(l.rstrip()), + ) + returncode = call_async_output(dist_upgrade, callbacks, shell=True) + if returncode != 0: + logger.warning('tools_upgrade_regular_packages_failed', + packages_list=', '.join(noncritical_packages_upgradable)) + operation_logger.error(m18n.n('packages_upgrade_failed')) + raise YunohostError(m18n.n('packages_upgrade_failed')) + + # + # Critical packages upgrade + # + if critical_packages_upgradable: + + logger.info(m18n.n("tools_upgrade_special_packages")) + + # Mark all critical packages as unheld + for package in critical_packages: + check_output("apt-mark unhold %s" % package) + + # Doublecheck with apt-mark showhold that packages are indeed unheld ... + held_packages = check_output("apt-mark showhold").split("\n") + if any(p in held_packages for p in critical_packages): + logger.warning(m18n.n("tools_upgrade_cant_unhold_critical_packages")) + operation_logger.error(m18n.n('packages_upgrade_failed')) + raise YunohostError(m18n.n('packages_upgrade_failed')) + + # + # Here we use a dirty hack to run a command after the current + # "yunohost tools upgrade", because the upgrade of yunohost + # will also trigger other yunohost commands (e.g. "yunohost tools migrations migrate") + # (also the upgrade of the package, if executed from the webadmin, is + # likely to kill/restart the api which is in turn likely to kill this + # command before it ends...) + # + logfile = operation_logger.log_path + command = dist_upgrade + " 2>&1 | tee -a {}".format(logfile) + + MOULINETTE_LOCK = "/var/run/moulinette_yunohost.lock" + wait_until_end_of_yunohost_command = "(while [ -f {} ]; do sleep 2; done)".format(MOULINETTE_LOCK) + mark_success = "(echo 'Done!' | tee -a {} && echo 'success: true' >> {})".format(logfile, operation_logger.md_path) + mark_failure = "(echo 'Failed :(' | tee -a {} && echo 'success: false' >> {})".format(logfile, operation_logger.md_path) + update_log_metadata = "sed -i \"s/ended_at: .*$/ended_at: $(date -u +'%Y-%m-%d %H:%M:%S.%N')/\" {}" + update_log_metadata = update_log_metadata.format(operation_logger.md_path) + + upgrade_completed = "\n" + m18n.n("tools_upgrade_special_packages_completed") + command = "(({wait} && {cmd}) && {mark_success} || {mark_failure}; {update_metadata}; echo '{done}') &".format( + wait=wait_until_end_of_yunohost_command, + cmd=command, + mark_success=mark_success, + mark_failure=mark_failure, + update_metadata=update_log_metadata, + done=upgrade_completed) + + logger.warning(m18n.n("tools_upgrade_special_packages_explanation")) + logger.debug("Running command :\n{}".format(command)) + os.system(command) + return + + else: + logger.success(m18n.n('system_upgraded')) + operation_logger.success() def tools_diagnosis(private=False): diff --git a/src/yunohost/utils/packages.py b/src/yunohost/utils/packages.py index e10de6493..b564d2dea 100644 --- a/src/yunohost/utils/packages.py +++ b/src/yunohost/utils/packages.py @@ -481,3 +481,46 @@ def dpkg_is_broken(): return False return any(re.match("^[0-9]+$", f) for f in os.listdir("/var/lib/dpkg/updates/")) + + +def _list_upgradable_apt_packages(): + + from moulinette.utils.process import check_output + + # List upgradable packages + # LC_ALL=C is here to make sure the results are in english + upgradable_raw = check_output("LC_ALL=C apt list --upgradable") + + # Dirty parsing of the output + upgradable_raw = [l.strip() for l in upgradable_raw.split("\n") if l.strip()] + for line in upgradable_raw: + + # Remove stupid warning and verbose messages >.> + if "apt does not have a stable CLI interface" in line or "Listing..." in line: + continue + + # line should look like : + # yunohost/stable 3.5.0.2+201903211853 all [upgradable from: 3.4.2.4+201903080053] + line = line.split() + if len(line) != 6: + logger.warning("Failed to parse this line : %s" % ' '.join(line)) + continue + + yield { + "name": line[0].split("/")[0], + "new_version": line[1], + "current_version": line[5].strip("]"), + } + + +def _dump_sources_list(): + + from glob import glob + + filenames = glob("/etc/apt/sources.list") + glob("/etc/apt/sources.list.d/*") + for filename in filenames: + with open(filename, "r") as f: + for line in f.readlines(): + if line.startswith("#") or not line.strip(): + continue + yield filename.replace("/etc/apt/", "") + ":" + line.strip()