YunoHost/yunohost
1
0
Fork 0
mirror of https://github.com/YunoHost/yunohost.git synced 2024-09-03 20:06:10 +02:00
Code Issues Projects Releases Packages Wiki Activity

[enh] [wip] Stretch migration (#433)

Browse source 
* Add migration skeleton
* Clumsy attempt to adapt the upgrade script to python
* At the end of the migration, force the regen conf of specific services
* Implement the apt clean/autoremove at the end of migration
* Attempt to fix the upgrade of yunohost packages
* Dumb mistake :|
* Adding strings
* Add test of free space for /var/
* Fix sources.list patching
* Stupid mistake :|
* Check system is up to date
* Working on disclaimer draft
* Add a function to list installed 'unstable' apps
* Get actual list of problemtic apps + improve disclaimer message building
* Use helper to run the apt update
* More simplifications of disclaimer building
* Add helper function to get manually modified files
* Fetch actuall list of manually modified files to build disclaimer
* Internationalize disclaimer
* Don't skip stretch migration when running postinstall on jessie
* Add a done message at the very end of the migration
* Also patch jessie/updates and backports in sources.list
* Backup and restore conf files modified during the upgrade to not mess regen-conf
* Also check for yunohost being in 2.x at the beginning of upgrade
* Fix the check for upgradable packages..
* Try to be more robust if folder already exists (when running multiple times)
* I probably meant fail2ban here o.O
* Try to improve robustness when running multiple time
* Add a check after the main upgrade that we're effectively on stretch
* Hold apps equivs packages during the upgrade
* Show dist-upgrade logs in the yunohost admin, using call_async_output
* Misc fixes because I broke things /o\
* Touch /etc/yunohost/installed at the end, because for some weird reason it get deleted sometimes :|
* Removing this unecessary message, especially because it 'hide' the previous one when running from the webadmin
* Install php-zip for nextcloud and kanboard
* Don't crash if there's no [app]-ynh-deps
* Revert previous commit that added this, should be fixed in the stretch branch now
* [fix] Unhold metronome for migration (#452)
* Let's use forge.yunohost.org as repo now
This commit is contained in:
Alexandre Aubin 2018-05-02 02:10:46 +02:00 • committed by GitHub
parent 01956d22bd
commit 5013965c0e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 369 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
locales/en.json
View file

@ -224,6 +224,19 @@
"migrate_tsig_not_needed": "You do not appear to use a dyndns domain, so no migration is needed !",
"migration_description_0001_change_cert_group_to_sslcert": "Change certificates group permissions from 'metronome' to 'ssl-cert'",
"migration_description_0002_migrate_to_tsig_sha256": "Improve security of dyndns TSIG by using SHA512 instead of MD5",
"migration_description_0003_migrate_to_stretch": "Upgrade the system to Debian Stretch and YunoHost 3.0",
"migration_0003_backward_impossible": "The stretch migration cannot be reverted.",
"migration_0003_start": "Starting migration to Stretch. The logs will be available in {logfile}.",
"migration_0003_patching_sources_list": "Patching the sources.lists ...",
"migration_0003_main_upgrade": "Starting main upgrade ...",
"migration_0003_fail2ban_upgrade": "Starting the fail2ban upgrade ...",
"migration_0003_yunohost_upgrade": "Starting the yunohost package upgrade ... The migration will end, but the actual upgrade will happen right after. After the operation is complete, you might have to re-log on the webadmin.",
"migration_0003_not_jessie": "The current debian distribution is not Jessie !",
"migration_0003_system_not_fully_up_to_date": "Your system is not fully up to date. Please perform a regular upgrade before running the migration to stretch.",
"migration_0003_still_on_jessie_after_main_upgrade": "Something wrong happened during the main upgrade : system is still on Jessie !? To investigate the issue, please look at {log} :s ...",
"migration_0003_general_warning": "Please note that this migration is a delicate operation. While the YunoHost team did its best to review and test it, the migration might still break parts of the system or apps.\n\nTherefore, we recommend you to :\n - Perform a backup of any critical data or app ;\n - Be patient after launching the migration : depending on your internet connection and hardware, it might take up to a few hours for everything to upgrade.",
"migration_0003_problematic_apps_warning": "Please note that the following possibly problematic installed apps were detected. It looks like those were not installed from an applist or are not flagged as 'working'. Consequently, we cannot guarantee that they will still work after the upgrade : {problematic_apps}",
"migration_0003_modified_files": "Please note that the following files were found to be manually modified and might be overwritten at the end of the upgrade : {manually_modified_files}",
"migrations_backward": "Migrating backward.",
"migrations_bad_value_for_target": "Invalid number for target argument, available migrations numbers are 0 or {}",
"migrations_cant_reach_migration_file": "Can't access migrations files at path %s",

17
src/yunohost/app.py
View file

@ -2164,3 +2164,20 @@ def normalize_url_path(url_path):
return '/' + url_path.strip("/").strip() + '/'
return "/"
def unstable_apps():
raw_app_installed = app_list(installed=True, raw=True)
output = []
for app, infos in raw_app_installed.items():
repo = infos.get("repository", None)
state = infos.get("state", None)
if repo is None or state in ["inprogress", "notworking"]:
output.append(app)
return output

291
src/yunohost/data_migrations/0003_migrate_to_stretch.py Normal file
View file

@ -0,0 +1,291 @@
import glob
import os
import requests
import base64
import time
import json
import errno
import platform
from shutil import copy2
from moulinette import m18n, msettings
from moulinette.core import MoulinetteError
from moulinette.utils.log import getActionLogger
from moulinette.utils.process import check_output, call_async_output
from moulinette.utils.filesystem import read_file
from yunohost.tools import Migration
from yunohost.app import unstable_apps
from yunohost.service import _run_service_command, service_regen_conf, manually_modified_files
from yunohost.utils.filesystem import free_space_in_directory
from yunohost.utils.packages import get_installed_version
logger = getActionLogger('yunohost.migration')
YUNOHOST_PACKAGES = ["yunohost", "yunohost-admin", "moulinette", "ssowat" ]
class MyMigration(Migration):
"Upgrade the system to Debian Stretch and Yunohost 3.0"
mode = "manual"
def backward(self):
raise MoulinetteError(m18n.n("migration_0003_backward_impossible"))
def migrate(self):
self.logfile = "/tmp/{}.log".format(self.name)
self.check_assertions()
logger.warning(m18n.n("migration_0003_start", logfile=self.logfile))
# Preparing the upgrade
logger.warning(m18n.n("migration_0003_patching_sources_list"))
self.patch_apt_sources_list()
self.backup_files_to_keep()
self.apt_update()
apps_packages = self.get_apps_equivs_packages()
self.unhold(["metronome"])
self.hold(YUNOHOST_PACKAGES + apps_packages + ["fail2ban"])
# Main dist-upgrade
logger.warning(m18n.n("migration_0003_main_upgrade"))
_run_service_command("stop", "mysql")
self.apt_dist_upgrade(conf_flags=["old", "def"])
_run_service_command("start", "mysql")
if self.debian_major_version() == 8:
raise MoulinetteError(m18n.n("migration_0003_still_on_jessie_after_main_upgrade", log=self.logfile))
# Specific upgrade for fail2ban...
logger.warning(m18n.n("migration_0003_fail2ban_upgrade"))
self.unhold(["fail2ban"])
# Don't move this if folder already exists. If it does, we probably are
# running this script a 2nd, 3rd, ... time but /etc/fail2ban will
# be re-created only for the first dist-upgrade of fail2ban
if not os.path.exists("/etc/fail2ban.old"):
os.system("mv /etc/fail2ban /etc/fail2ban.old")
self.apt_dist_upgrade(conf_flags=["new", "miss", "def"])
_run_service_command("restart", "fail2ban")
# Clean the mess
os.system("apt autoremove --assume-yes")
os.system("apt clean --assume-yes")
# Upgrade yunohost packages
logger.warning(m18n.n("migration_0003_yunohost_upgrade"))
self.restore_files_to_keep()
self.unhold(YUNOHOST_PACKAGES + apps_packages)
self.upgrade_yunohost_packages()
def debian_major_version(self):
return int(platform.dist()[1][0])
def yunohost_major_version(self):
return int(get_installed_version("yunohost").split('.')[0])
def check_assertions(self):
# Be on jessie (8.x) and yunohost 2.x
# NB : we do both check to cover situations where the upgrade crashed
# in the middle and debian version could be >= 9.x but yunohost package
# would still be in 2.x...
if not self.debian_major_version() == 8 \
and not self.yunohost_major_version() == 2:
raise MoulinetteError(m18n.n("migration_0003_not_jessie"))
# Have > 1 Go free space on /var/ ?
if free_space_in_directory("/var/") / (1024**3) < 1.0:
raise MoulinetteError(m18n.n("migration_0003_not_enough_free_space"))
# Check system is up to date
# (but we don't if 'stretch' is already in the sources.list ...
# which means maybe a previous upgrade crashed and we're re-running it)
if not " stretch " in read_file("/etc/apt/sources.list"):
self.apt_update()
apt_list_upgradable = check_output("apt list --upgradable -a")
if "upgradable" in apt_list_upgradable:
raise MoulinetteError(m18n.n("migration_0003_system_not_fully_up_to_date"))
@property
def disclaimer(self):
# Avoid having a super long disclaimer + uncessary check if we ain't
# on jessie / yunohost 2.x anymore
# NB : we do both check to cover situations where the upgrade crashed
# in the middle and debian version could be >= 9.x but yunohost package
# would still be in 2.x...
if not self.debian_major_version() == 8 \
and not self.yunohost_major_version() == 2:
return None
# Get list of problematic apps ? I.e. not official or community+working
problematic_apps = unstable_apps()
problematic_apps = "".join(["\n - "+app for app in problematic_apps ])
# Manually modified files ? (c.f. yunohost service regen-conf)
modified_files = manually_modified_files()
modified_files = "".join(["\n - "+f for f in modified_files ])
message = m18n.n("migration_0003_general_warning")
if problematic_apps:
message += "\n\n" + m18n.n("migration_0003_problematic_apps_warning", problematic_apps=problematic_apps)
if modified_files:
message += "\n\n" + m18n.n("migration_0003_modified_files", manually_modified_files=modified_files)
return message
def patch_apt_sources_list(self):
sources_list = glob.glob("/etc/apt/sources.list.d/*.list")
sources_list.append("/etc/apt/sources.list")
# This :
# - replace single 'jessie' occurence by 'stretch'
# - comments lines containing "backports"
# - replace 'jessie/updates' by 'strech/updates'
# - switch yunohost's repo to forge
for f in sources_list:
command = "sed -i -e 's@ jessie @ stretch @g' " \
"-e '/backports/ s@^#*@#@' " \
"-e 's@ jessie/updates @ stretch/updates @g' " \
"-e 's@repo.yunohost@forge.yunohost@g' " \
"{}".format(f)
os.system(command)
def get_apps_equivs_packages(self):
command = "dpkg --get-selections" \
" | grep -v deinstall" \
" | awk '{print $1}'" \
" | { grep 'ynh-deps$' || true; }"
output = check_output(command).strip()
return output.split('\n') if output else []
def hold(self, packages):
for package in packages:
os.system("apt-mark hold {}".format(package))
def unhold(self, packages):
for package in packages:
os.system("apt-mark unhold {}".format(package))
def apt_update(self):
command = "apt-get update"
logger.debug("Running apt command :\n{}".format(command))
command += " 2>&1 | tee -a {}".format(self.logfile)
os.system(command)
def upgrade_yunohost_packages(self):
#
# Here we use a dirty hack to run a command after the current
# "yunohost tools migrations migrate", because the upgrade of
# yunohost will also trigger another "yunohost tools migrations migrate"
# (also the upgrade of the package, if executed from the webadmin, is
# likely to kill/restart the api which is in turn likely to kill this
# command before it ends...)
#
MOULINETTE_LOCK = "/var/run/moulinette_yunohost.lock"
upgrade_command = ""
upgrade_command += " DEBIAN_FRONTEND=noninteractive"
upgrade_command += " APT_LISTCHANGES_FRONTEND=none"
upgrade_command += " apt-get install"
upgrade_command += " --assume-yes "
upgrade_command += " ".join(YUNOHOST_PACKAGES)
# We also install php-zip to fix an issue with nextcloud and kanboard
# that need it when on stretch.
upgrade_command += " php-zip"
upgrade_command += " 2>&1 | tee -a {}".format(self.logfile)
wait_until_end_of_yunohost_command = "(while [ -f {} ]; do sleep 2; done)".format(MOULINETTE_LOCK)
command = "({} && {}; echo 'Done!') &".format(wait_until_end_of_yunohost_command,
upgrade_command)
logger.debug("Running command :\n{}".format(command))
os.system(command)
def apt_dist_upgrade(self, conf_flags):
# Make apt-get happy
os.system("echo 'libc6 libraries/restart-without-asking boolean true' | debconf-set-selections")
command = ""
command += " DEBIAN_FRONTEND=noninteractive"
command += " APT_LISTCHANGES_FRONTEND=none"
command += " apt-get"
command += " --fix-broken --show-upgraded --assume-yes"
for conf_flag in conf_flags:
command += ' -o Dpkg::Options::="--force-conf{}"'.format(conf_flag)
command += " dist-upgrade"
logger.debug("Running apt command :\n{}".format(command))
command += " 2>&1 | tee -a {}".format(self.logfile)
is_api = msettings.get('interface') == 'api'
if is_api:
callbacks = (
lambda l: logger.info(l.rstrip()),
lambda l: logger.warning(l.rstrip()),
)
call_async_output(command, callbacks, shell=True)
else:
# We do this when running from the cli to have the output of the
# command showing in the terminal, since 'info' channel is only
# enabled if the user explicitly add --verbose ...
os.system(command)
# Those are files that should be kept and restored before the final switch
# to yunohost 3.x... They end up being modified by the various dist-upgrades
# (or need to be taken out momentarily), which then blocks the regen-conf
# as they are flagged as "manually modified"...
files_to_keep = [
"/etc/mysql/my.cnf",
"/etc/nslcd.conf",
"/etc/postfix/master.cf",
"/etc/fail2ban/filter.d/yunohost.conf"
]
def backup_files_to_keep(self):
logger.debug("Backuping specific files to keep ...")
# Create tmp directory if it does not exists
tmp_dir = os.path.join("/tmp/", self.name)
if not os.path.exists(tmp_dir):
os.mkdir(tmp_dir, 0700)
for f in self.files_to_keep:
dest_file = f.strip('/').replace("/", "_")
# If the file is already there, we might be re-running the migration
# because it previously crashed. Hence we keep the existing file.
if os.path.exists(os.path.join(tmp_dir, dest_file)):
continue
copy2(f, os.path.join(tmp_dir, dest_file))
def restore_files_to_keep(self):
logger.debug("Restoring specific files to keep ...")
tmp_dir = os.path.join("/tmp/", self.name)
for f in self.files_to_keep:
dest_file = f.strip('/').replace("/", "_")
copy2(os.path.join(tmp_dir, dest_file), f)

22
src/yunohost/service.py
View file

@ -26,6 +26,7 @@
import os
import time
import yaml
import json
import glob
import subprocess
import errno
@ -771,3 +772,24 @@ def _process_regen_conf(system_conf, new_conf=None, save=True):
exc_info=1)
return False
return True
def manually_modified_files():
# We do this to have --quiet, i.e. don't throw a whole bunch of logs
# just to fetch this...
# Might be able to optimize this by looking at what service_regenconf does
# and only do the part that checks file hashes...
cmd = "yunohost service regen-conf --dry-run --output-as json --quiet"
j = json.loads(subprocess.check_output(cmd.split()))
# j is something like :
# {"postfix": {"applied": {}, "pending": {"/etc/postfix/main.cf": {"status": "modified"}}}
output = []
for app, actions in j.items():
for action, files in actions.items():
for filename, infos in files.items():
if infos["status"] == "modified":
output.append(filename)
return output

2
src/yunohost/tools.py
View file

@ -395,7 +395,7 @@ def tools_postinstall(domain, password, ignore_dyndns=False):
_install_appslist_fetch_cron()
# Init migrations (skip them, no need to run them on a fresh system)
tools_migrations_migrate(skip=True, auto=True)
tools_migrations_migrate(target=2, skip=True, auto=True)
os.system('touch /etc/yunohost/installed')

25
src/yunohost/utils/filesystem.py Normal file
View file

@ -0,0 +1,25 @@
# -*- coding: utf-8 -*-
""" License
Copyright (C) 2018 YUNOHOST.ORG
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program; if not, see http://www.gnu.org/licenses
"""
import os
def free_space_in_directory(dirpath):
stat = os.statvfs(dirpath)
return stat.f_frsize * stat.f_bavail