diff --git a/locales/en.json b/locales/en.json index 9957c25a0..0f45c7a8b 100644 --- a/locales/en.json +++ b/locales/en.json @@ -224,6 +224,19 @@ "migrate_tsig_not_needed": "You do not appear to use a dyndns domain, so no migration is needed !", "migration_description_0001_change_cert_group_to_sslcert": "Change certificates group permissions from 'metronome' to 'ssl-cert'", "migration_description_0002_migrate_to_tsig_sha256": "Improve security of dyndns TSIG by using SHA512 instead of MD5", + "migration_description_0003_migrate_to_stretch": "Upgrade the system to Debian Stretch and YunoHost 3.0", + "migration_0003_backward_impossible": "The stretch migration cannot be reverted.", + "migration_0003_start": "Starting migration to Stretch. The logs will be available in {logfile}.", + "migration_0003_patching_sources_list": "Patching the sources.lists ...", + "migration_0003_main_upgrade": "Starting main upgrade ...", + "migration_0003_fail2ban_upgrade": "Starting the fail2ban upgrade ...", + "migration_0003_yunohost_upgrade": "Starting the yunohost package upgrade ... The migration will end, but the actual upgrade will happen right after. After the operation is complete, you might have to re-log on the webadmin.", + "migration_0003_not_jessie": "The current debian distribution is not Jessie !", + "migration_0003_system_not_fully_up_to_date": "Your system is not fully up to date. Please perform a regular upgrade before running the migration to stretch.", + "migration_0003_still_on_jessie_after_main_upgrade": "Something wrong happened during the main upgrade : system is still on Jessie !? To investigate the issue, please look at {log} :s ...", + "migration_0003_general_warning": "Please note that this migration is a delicate operation. While the YunoHost team did its best to review and test it, the migration might still break parts of the system or apps.\n\nTherefore, we recommend you to :\n - Perform a backup of any critical data or app ;\n - Be patient after launching the migration : depending on your internet connection and hardware, it might take up to a few hours for everything to upgrade.", + "migration_0003_problematic_apps_warning": "Please note that the following possibly problematic installed apps were detected. It looks like those were not installed from an applist or are not flagged as 'working'. Consequently, we cannot guarantee that they will still work after the upgrade : {problematic_apps}", + "migration_0003_modified_files": "Please note that the following files were found to be manually modified and might be overwritten at the end of the upgrade : {manually_modified_files}", "migrations_backward": "Migrating backward.", "migrations_bad_value_for_target": "Invalid number for target argument, available migrations numbers are 0 or {}", "migrations_cant_reach_migration_file": "Can't access migrations files at path %s", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 6ddf08f52..a4ab8db7b 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2164,3 +2164,20 @@ def normalize_url_path(url_path): return '/' + url_path.strip("/").strip() + '/' return "/" + + +def unstable_apps(): + + raw_app_installed = app_list(installed=True, raw=True) + output = [] + + for app, infos in raw_app_installed.items(): + + repo = infos.get("repository", None) + state = infos.get("state", None) + + if repo is None or state in ["inprogress", "notworking"]: + output.append(app) + + return output + diff --git a/src/yunohost/data_migrations/0003_migrate_to_stretch.py b/src/yunohost/data_migrations/0003_migrate_to_stretch.py new file mode 100644 index 000000000..5f65dbb89 --- /dev/null +++ b/src/yunohost/data_migrations/0003_migrate_to_stretch.py @@ -0,0 +1,291 @@ +import glob +import os +import requests +import base64 +import time +import json +import errno +import platform +from shutil import copy2 + +from moulinette import m18n, msettings +from moulinette.core import MoulinetteError +from moulinette.utils.log import getActionLogger +from moulinette.utils.process import check_output, call_async_output +from moulinette.utils.filesystem import read_file + +from yunohost.tools import Migration +from yunohost.app import unstable_apps +from yunohost.service import _run_service_command, service_regen_conf, manually_modified_files +from yunohost.utils.filesystem import free_space_in_directory +from yunohost.utils.packages import get_installed_version + +logger = getActionLogger('yunohost.migration') + +YUNOHOST_PACKAGES = ["yunohost", "yunohost-admin", "moulinette", "ssowat" ] + +class MyMigration(Migration): + "Upgrade the system to Debian Stretch and Yunohost 3.0" + + mode = "manual" + + def backward(self): + + raise MoulinetteError(m18n.n("migration_0003_backward_impossible")) + + def migrate(self): + + self.logfile = "/tmp/{}.log".format(self.name) + + self.check_assertions() + + logger.warning(m18n.n("migration_0003_start", logfile=self.logfile)) + + # Preparing the upgrade + logger.warning(m18n.n("migration_0003_patching_sources_list")) + self.patch_apt_sources_list() + self.backup_files_to_keep() + self.apt_update() + apps_packages = self.get_apps_equivs_packages() + self.unhold(["metronome"]) + self.hold(YUNOHOST_PACKAGES + apps_packages + ["fail2ban"]) + + # Main dist-upgrade + logger.warning(m18n.n("migration_0003_main_upgrade")) + _run_service_command("stop", "mysql") + self.apt_dist_upgrade(conf_flags=["old", "def"]) + _run_service_command("start", "mysql") + if self.debian_major_version() == 8: + raise MoulinetteError(m18n.n("migration_0003_still_on_jessie_after_main_upgrade", log=self.logfile)) + + # Specific upgrade for fail2ban... + logger.warning(m18n.n("migration_0003_fail2ban_upgrade")) + self.unhold(["fail2ban"]) + # Don't move this if folder already exists. If it does, we probably are + # running this script a 2nd, 3rd, ... time but /etc/fail2ban will + # be re-created only for the first dist-upgrade of fail2ban + if not os.path.exists("/etc/fail2ban.old"): + os.system("mv /etc/fail2ban /etc/fail2ban.old") + self.apt_dist_upgrade(conf_flags=["new", "miss", "def"]) + _run_service_command("restart", "fail2ban") + + # Clean the mess + os.system("apt autoremove --assume-yes") + os.system("apt clean --assume-yes") + + # Upgrade yunohost packages + logger.warning(m18n.n("migration_0003_yunohost_upgrade")) + self.restore_files_to_keep() + self.unhold(YUNOHOST_PACKAGES + apps_packages) + self.upgrade_yunohost_packages() + + def debian_major_version(self): + return int(platform.dist()[1][0]) + + def yunohost_major_version(self): + return int(get_installed_version("yunohost").split('.')[0]) + + def check_assertions(self): + + # Be on jessie (8.x) and yunohost 2.x + # NB : we do both check to cover situations where the upgrade crashed + # in the middle and debian version could be >= 9.x but yunohost package + # would still be in 2.x... + if not self.debian_major_version() == 8 \ + and not self.yunohost_major_version() == 2: + raise MoulinetteError(m18n.n("migration_0003_not_jessie")) + + # Have > 1 Go free space on /var/ ? + if free_space_in_directory("/var/") / (1024**3) < 1.0: + raise MoulinetteError(m18n.n("migration_0003_not_enough_free_space")) + + # Check system is up to date + # (but we don't if 'stretch' is already in the sources.list ... + # which means maybe a previous upgrade crashed and we're re-running it) + if not " stretch " in read_file("/etc/apt/sources.list"): + self.apt_update() + apt_list_upgradable = check_output("apt list --upgradable -a") + if "upgradable" in apt_list_upgradable: + raise MoulinetteError(m18n.n("migration_0003_system_not_fully_up_to_date")) + + @property + def disclaimer(self): + + # Avoid having a super long disclaimer + uncessary check if we ain't + # on jessie / yunohost 2.x anymore + # NB : we do both check to cover situations where the upgrade crashed + # in the middle and debian version could be >= 9.x but yunohost package + # would still be in 2.x... + if not self.debian_major_version() == 8 \ + and not self.yunohost_major_version() == 2: + return None + + # Get list of problematic apps ? I.e. not official or community+working + problematic_apps = unstable_apps() + problematic_apps = "".join(["\n - "+app for app in problematic_apps ]) + + # Manually modified files ? (c.f. yunohost service regen-conf) + modified_files = manually_modified_files() + modified_files = "".join(["\n - "+f for f in modified_files ]) + + message = m18n.n("migration_0003_general_warning") + + if problematic_apps: + message += "\n\n" + m18n.n("migration_0003_problematic_apps_warning", problematic_apps=problematic_apps) + + if modified_files: + message += "\n\n" + m18n.n("migration_0003_modified_files", manually_modified_files=modified_files) + + return message + + def patch_apt_sources_list(self): + + sources_list = glob.glob("/etc/apt/sources.list.d/*.list") + sources_list.append("/etc/apt/sources.list") + + # This : + # - replace single 'jessie' occurence by 'stretch' + # - comments lines containing "backports" + # - replace 'jessie/updates' by 'strech/updates' + # - switch yunohost's repo to forge + for f in sources_list: + command = "sed -i -e 's@ jessie @ stretch @g' " \ + "-e '/backports/ s@^#*@#@' " \ + "-e 's@ jessie/updates @ stretch/updates @g' " \ + "-e 's@repo.yunohost@forge.yunohost@g' " \ + "{}".format(f) + os.system(command) + + def get_apps_equivs_packages(self): + + command = "dpkg --get-selections" \ + " | grep -v deinstall" \ + " | awk '{print $1}'" \ + " | { grep 'ynh-deps$' || true; }" + + output = check_output(command).strip() + + return output.split('\n') if output else [] + + def hold(self, packages): + for package in packages: + os.system("apt-mark hold {}".format(package)) + + def unhold(self, packages): + for package in packages: + os.system("apt-mark unhold {}".format(package)) + + def apt_update(self): + + command = "apt-get update" + logger.debug("Running apt command :\n{}".format(command)) + command += " 2>&1 | tee -a {}".format(self.logfile) + + os.system(command) + + def upgrade_yunohost_packages(self): + + # + # Here we use a dirty hack to run a command after the current + # "yunohost tools migrations migrate", because the upgrade of + # yunohost will also trigger another "yunohost tools migrations migrate" + # (also the upgrade of the package, if executed from the webadmin, is + # likely to kill/restart the api which is in turn likely to kill this + # command before it ends...) + # + + MOULINETTE_LOCK = "/var/run/moulinette_yunohost.lock" + + upgrade_command = "" + upgrade_command += " DEBIAN_FRONTEND=noninteractive" + upgrade_command += " APT_LISTCHANGES_FRONTEND=none" + upgrade_command += " apt-get install" + upgrade_command += " --assume-yes " + upgrade_command += " ".join(YUNOHOST_PACKAGES) + # We also install php-zip to fix an issue with nextcloud and kanboard + # that need it when on stretch. + upgrade_command += " php-zip" + upgrade_command += " 2>&1 | tee -a {}".format(self.logfile) + + wait_until_end_of_yunohost_command = "(while [ -f {} ]; do sleep 2; done)".format(MOULINETTE_LOCK) + + command = "({} && {}; echo 'Done!') &".format(wait_until_end_of_yunohost_command, + upgrade_command) + + logger.debug("Running command :\n{}".format(command)) + + os.system(command) + + + def apt_dist_upgrade(self, conf_flags): + + # Make apt-get happy + os.system("echo 'libc6 libraries/restart-without-asking boolean true' | debconf-set-selections") + + command = "" + command += " DEBIAN_FRONTEND=noninteractive" + command += " APT_LISTCHANGES_FRONTEND=none" + command += " apt-get" + command += " --fix-broken --show-upgraded --assume-yes" + for conf_flag in conf_flags: + command += ' -o Dpkg::Options::="--force-conf{}"'.format(conf_flag) + command += " dist-upgrade" + + logger.debug("Running apt command :\n{}".format(command)) + + command += " 2>&1 | tee -a {}".format(self.logfile) + + is_api = msettings.get('interface') == 'api' + if is_api: + callbacks = ( + lambda l: logger.info(l.rstrip()), + lambda l: logger.warning(l.rstrip()), + ) + call_async_output(command, callbacks, shell=True) + else: + # We do this when running from the cli to have the output of the + # command showing in the terminal, since 'info' channel is only + # enabled if the user explicitly add --verbose ... + os.system(command) + + + # Those are files that should be kept and restored before the final switch + # to yunohost 3.x... They end up being modified by the various dist-upgrades + # (or need to be taken out momentarily), which then blocks the regen-conf + # as they are flagged as "manually modified"... + files_to_keep = [ + "/etc/mysql/my.cnf", + "/etc/nslcd.conf", + "/etc/postfix/master.cf", + "/etc/fail2ban/filter.d/yunohost.conf" + ] + + def backup_files_to_keep(self): + + logger.debug("Backuping specific files to keep ...") + + # Create tmp directory if it does not exists + tmp_dir = os.path.join("/tmp/", self.name) + if not os.path.exists(tmp_dir): + os.mkdir(tmp_dir, 0700) + + for f in self.files_to_keep: + dest_file = f.strip('/').replace("/", "_") + + # If the file is already there, we might be re-running the migration + # because it previously crashed. Hence we keep the existing file. + if os.path.exists(os.path.join(tmp_dir, dest_file)): + continue + + copy2(f, os.path.join(tmp_dir, dest_file)) + + def restore_files_to_keep(self): + + logger.debug("Restoring specific files to keep ...") + + tmp_dir = os.path.join("/tmp/", self.name) + + for f in self.files_to_keep: + dest_file = f.strip('/').replace("/", "_") + copy2(os.path.join(tmp_dir, dest_file), f) + diff --git a/src/yunohost/service.py b/src/yunohost/service.py index f0948c961..1852259ad 100644 --- a/src/yunohost/service.py +++ b/src/yunohost/service.py @@ -26,6 +26,7 @@ import os import time import yaml +import json import glob import subprocess import errno @@ -771,3 +772,24 @@ def _process_regen_conf(system_conf, new_conf=None, save=True): exc_info=1) return False return True + +def manually_modified_files(): + + # We do this to have --quiet, i.e. don't throw a whole bunch of logs + # just to fetch this... + # Might be able to optimize this by looking at what service_regenconf does + # and only do the part that checks file hashes... + cmd = "yunohost service regen-conf --dry-run --output-as json --quiet" + j = json.loads(subprocess.check_output(cmd.split())) + + # j is something like : + # {"postfix": {"applied": {}, "pending": {"/etc/postfix/main.cf": {"status": "modified"}}} + + output = [] + for app, actions in j.items(): + for action, files in actions.items(): + for filename, infos in files.items(): + if infos["status"] == "modified": + output.append(filename) + + return output diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index a60ffa80a..80f7bcaa5 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -395,7 +395,7 @@ def tools_postinstall(domain, password, ignore_dyndns=False): _install_appslist_fetch_cron() # Init migrations (skip them, no need to run them on a fresh system) - tools_migrations_migrate(skip=True, auto=True) + tools_migrations_migrate(target=2, skip=True, auto=True) os.system('touch /etc/yunohost/installed') diff --git a/src/yunohost/utils/filesystem.py b/src/yunohost/utils/filesystem.py new file mode 100644 index 000000000..9b39f5daa --- /dev/null +++ b/src/yunohost/utils/filesystem.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +""" License + + Copyright (C) 2018 YUNOHOST.ORG + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program; if not, see http://www.gnu.org/licenses + +""" +import os + +def free_space_in_directory(dirpath): + stat = os.statvfs(dirpath) + return stat.f_frsize * stat.f_bavail