mirror of
https://github.com/YunoHost/yunohost.git
synced 2024-09-03 20:06:10 +02:00
Merge pull request #768 from YunoHost/rework-migration-system
Rework migration system to have independent migrations
This commit is contained in:
Alexandre Aubin
GitHub
commit
6d604d711f
15 changed files with 297 additions and 305 deletions
|
|
@ -1737,22 +1737,23 @@ tools:
|
|||
|
||||
### tools_migrations_migrate()
|
||||
migrate:
|
||||
action_help: Perform migrations
|
||||
action_help: Run migrations
|
||||
api: POST /migrations/migrate
|
||||
arguments:
|
||||
-t:
|
||||
help: target migration number (or 0), latest one by default
|
||||
type: int
|
||||
full: --target
|
||||
-s:
|
||||
help: skip the migration(s), use it only if you know what you are doing
|
||||
full: --skip
|
||||
targets:
|
||||
help: Migrations to run (all pendings by default)
|
||||
nargs: "*"
|
||||
--skip:
|
||||
help: Skip specified migrations (to be used only if you know what you are doing)
|
||||
action: store_true
|
||||
--force-rerun:
|
||||
help: Re-run already-ran specified migration (to be used only if you know what you are doing)
|
||||
action: store_true
|
||||
--auto:
|
||||
help: automatic mode, won't run manual migrations, use it only if you know what you are doing
|
||||
help: Automatic mode, won't run manual migrations (to be used only if you know what you are doing)
|
||||
action: store_true
|
||||
--accept-disclaimer:
|
||||
help: accept disclaimers of migration (please read them before using this option)
|
||||
help: Accept disclaimers of migrations (please read them before using this option)
|
||||
action: store_true
|
||||
|
||||
### tools_migrations_state()
|
||||
|
|
|
|||
|
|
@ -296,7 +296,6 @@
|
|||
"log_user_permission_remove": "Update '{}' permission",
|
||||
"log_tools_maindomain": "Make '{}' as main domain",
|
||||
"log_tools_migrations_migrate_forward": "Migrate forward",
|
||||
"log_tools_migrations_migrate_backward": "Migrate backward",
|
||||
"log_tools_postinstall": "Postinstall your YunoHost server",
|
||||
"log_tools_upgrade": "Upgrade system packages",
|
||||
"log_tools_shutdown": "Shutdown your server",
|
||||
|
|
@ -332,7 +331,6 @@
|
|||
"migration_description_0010_migrate_to_apps_json": "Remove deprecated appslists and use the new unified 'apps.json' list instead",
|
||||
"migration_description_0011_setup_group_permission": "Setup user group and setup permission for apps and services",
|
||||
"migration_description_0012_postgresql_password_to_md5_authentication": "Force postgresql authentication to use md5 for local connections",
|
||||
"migration_0003_backward_impossible": "The stretch migration cannot be reverted.",
|
||||
"migration_0003_start": "Starting migration to Stretch. The logs will be available in {logfile}.",
|
||||
"migration_0003_patching_sources_list": "Patching the sources.lists…",
|
||||
"migration_0003_main_upgrade": "Starting main upgrade…",
|
||||
|
|
@ -369,22 +367,24 @@
|
|||
"migration_0011_rollback_success": "Rollback succeeded.",
|
||||
"migration_0011_update_LDAP_database": "Updating LDAP database...",
|
||||
"migration_0011_update_LDAP_schema": "Updating LDAP schema...",
|
||||
"migrations_backward": "Migrating backward.",
|
||||
"migrations_bad_value_for_target": "Invalid number for target argument, available migrations numbers are 0 or {}",
|
||||
"migrations_already_ran": "Those migrations have already been ran: {ids}",
|
||||
"migrations_cant_reach_migration_file": "Can't access migrations files at path %s",
|
||||
"migrations_current_target": "Migration target is {}",
|
||||
"migrations_error_failed_to_load_migration": "ERROR: failed to load migration {number} {name}",
|
||||
"migrations_forward": "Migrating forward",
|
||||
"migrations_dependencies_not_satisfied": "Can't run migration {id} because first you need to run these migrations: {dependencies_id}",
|
||||
"migrations_failed_to_load_migration": "Failed to load migration {id} : {error}",
|
||||
"migrations_exclusive_options": "--auto, --skip, and --force-rerun are exclusive options.",
|
||||
"migrations_list_conflict_pending_done": "You cannot use both --previous and --done at the same time.",
|
||||
"migrations_loading_migration": "Loading migration {number} {name}…",
|
||||
"migrations_migration_has_failed": "Migration {number} {name} has failed with exception {exception}, aborting",
|
||||
"migrations_loading_migration": "Loading migration {id}…",
|
||||
"migrations_migration_has_failed": "Migration {id} has failed, aborting. Error: {exception}",
|
||||
"migrations_must_provide_explicit_targets": "You must provide explicit targets when using --skip or --force-rerun",
|
||||
"migrations_need_to_accept_disclaimer": "To run the migration {id}, your must accept the following disclaimer:\n---\n{disclaimer}\n---\nIf you accept to run the migration, please re-run the command with the option --accept-disclaimer.",
|
||||
"migrations_no_migrations_to_run": "No migrations to run",
|
||||
"migrations_show_currently_running_migration": "Running migration {number} {name}…",
|
||||
"migrations_show_last_migration": "Last ran migration is {}",
|
||||
"migrations_skip_migration": "Skipping migration {number} {name}…",
|
||||
"migrations_success": "Successfully ran migration {number} {name}!",
|
||||
"migrations_to_be_ran_manually": "Migration {number} {name} has to be ran manually. Please go to Tools > Migrations on the webadmin, or run `yunohost tools migrations migrate`.",
|
||||
"migrations_need_to_accept_disclaimer": "To run the migration {number} {name}, your must accept the following disclaimer:\n---\n{disclaimer}\n---\nIf you accept to run the migration, please re-run the command with the option --accept-disclaimer.",
|
||||
"migrations_no_such_migration": "No such migration called {id}",
|
||||
"migrations_not_pending_cant_skip": "Those migrations are not pending so cannot be skipped: {ids}",
|
||||
"migrations_pending_cant_rerun": "Those migrations are still pending so cannot be reran: {ids}",
|
||||
"migrations_running_forward": "Running migration {id}…",
|
||||
"migrations_skip_migration": "Skipping migration {id}…",
|
||||
"migrations_success_forward": "Successfully ran migration {id}!",
|
||||
"migrations_to_be_ran_manually": "Migration {id} has to be ran manually. Please go to Tools > Migrations on the webadmin, or run `yunohost tools migrations migrate`.",
|
||||
"monitor_disabled": "The server monitoring has been disabled",
|
||||
"monitor_enabled": "The server monitoring has been enabled",
|
||||
"monitor_glances_con_failed": "Unable to connect to Glances server",
|
||||
|
|
|
|||
|
|
@ -10,10 +10,6 @@ class MyMigration(Migration):
|
|||
|
||||
all_certificate_files = glob.glob("/etc/yunohost/certs/*/*.pem")
|
||||
|
||||
def forward(self):
|
||||
def run(self):
|
||||
for filename in self.all_certificate_files:
|
||||
chown(filename, uid="root", gid="ssl-cert")
|
||||
|
||||
def backward(self):
|
||||
for filename in self.all_certificate_files:
|
||||
chown(filename, uid="root", gid="metronome")
|
||||
|
|
|
|||
|
|
@ -19,11 +19,7 @@ class MyMigration(Migration):
|
|||
|
||||
"Migrate Dyndns stuff from MD5 TSIG to SHA512 TSIG"
|
||||
|
||||
def backward(self):
|
||||
# Not possible because that's a non-reversible operation ?
|
||||
pass
|
||||
|
||||
def migrate(self, dyn_host="dyndns.yunohost.org", domain=None, private_key_path=None):
|
||||
def run(self, dyn_host="dyndns.yunohost.org", domain=None, private_key_path=None):
|
||||
|
||||
if domain is None or private_key_path is None:
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -29,11 +29,7 @@ class MyMigration(Migration):
|
|||
|
||||
mode = "manual"
|
||||
|
||||
def backward(self):
|
||||
|
||||
raise YunohostError("migration_0003_backward_impossible")
|
||||
|
||||
def migrate(self):
|
||||
def run(self):
|
||||
|
||||
self.logfile = "/var/log/yunohost/{}.log".format(self.name)
|
||||
|
||||
|
|
|
|||
|
|
@ -22,8 +22,9 @@ class MyMigration(Migration):
|
|||
|
||||
"Migrate php5-fpm 'pool' conf files to php7 stuff"
|
||||
|
||||
def migrate(self):
|
||||
dependencies = ["migrate_to_stretch"]
|
||||
|
||||
def run(self):
|
||||
# Get list of php5 pool files
|
||||
php5_pool_files = glob.glob("{}/*.conf".format(PHP5_POOLS))
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,9 @@ class MyMigration(Migration):
|
|||
|
||||
"Migrate DBs from Postgresql 9.4 to 9.6 after migrating to Stretch"
|
||||
|
||||
def migrate(self):
|
||||
dependencies = ["migrate_to_stretch"]
|
||||
|
||||
def run(self):
|
||||
|
||||
if not self.package_is_installed("postgresql-9.4"):
|
||||
logger.warning(m18n.n("migration_0005_postgresql_94_not_installed"))
|
||||
|
|
@ -32,10 +34,6 @@ class MyMigration(Migration):
|
|||
subprocess.check_call("pg_dropcluster --stop 9.4 main", shell=True)
|
||||
subprocess.check_call("service postgresql start", shell=True)
|
||||
|
||||
def backward(self):
|
||||
|
||||
pass
|
||||
|
||||
def package_is_installed(self, package_name):
|
||||
|
||||
p = subprocess.Popen("dpkg --list | grep '^ii ' | grep -q -w {}".format(package_name), shell=True)
|
||||
|
|
|
|||
|
|
@ -20,16 +20,13 @@ class MyMigration(Migration):
|
|||
|
||||
"Synchronize admin and root passwords"
|
||||
|
||||
def migrate(self):
|
||||
def run(self):
|
||||
|
||||
new_hash = self._get_admin_hash()
|
||||
self._replace_root_hash(new_hash)
|
||||
|
||||
logger.info(m18n.n("root_password_replaced_by_admin_password"))
|
||||
|
||||
def backward(self):
|
||||
pass
|
||||
|
||||
@property
|
||||
def mode(self):
|
||||
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ class MyMigration(Migration):
|
|||
use the recommended configuration, with an appropriate disclaimer.
|
||||
"""
|
||||
|
||||
def migrate(self):
|
||||
def run(self):
|
||||
|
||||
# Check if deprecated DSA Host Key is in config
|
||||
dsa_rgx = r'^[ \t]*HostKey[ \t]+/etc/ssh/ssh_host_dsa_key[ \t]*(?:#.*)?$'
|
||||
|
|
@ -60,14 +60,11 @@ class MyMigration(Migration):
|
|||
regen_conf(names=['ssh'], force=True)
|
||||
copyfile('/etc/ssh/sshd_config.bkp', SSHD_CONF)
|
||||
|
||||
# Restart ssh and backward if it fail
|
||||
# Restart ssh and rollback if it failed
|
||||
if not _run_service_command('restart', 'ssh'):
|
||||
self.backward()
|
||||
raise YunohostError("migration_0007_cancel")
|
||||
|
||||
def backward(self):
|
||||
|
||||
# We don't backward completely but it should be enough
|
||||
copyfile('/etc/ssh/sshd_config.bkp', SSHD_CONF)
|
||||
if not _run_service_command('restart', 'ssh'):
|
||||
raise YunohostError("migration_0007_cannot_restart")
|
||||
# We don't rollback completely but it should be enough
|
||||
copyfile('/etc/ssh/sshd_config.bkp', SSHD_CONF)
|
||||
if not _run_service_command('restart', 'ssh'):
|
||||
raise YunohostError("migration_0007_cannot_restart")
|
||||
else:
|
||||
raise YunohostError("migration_0007_cancelled")
|
||||
|
|
|
|||
|
|
@ -33,7 +33,9 @@ class MyMigration(Migration):
|
|||
shown - and the user may also choose to skip this migration.
|
||||
"""
|
||||
|
||||
def migrate(self):
|
||||
dependencies = ["ssh_conf_managed_by_yunohost_step1"]
|
||||
|
||||
def run(self):
|
||||
settings_set("service.ssh.allow_deprecated_dsa_hostkey", False)
|
||||
regen_conf(names=['ssh'], force=True)
|
||||
|
||||
|
|
@ -42,10 +44,6 @@ class MyMigration(Migration):
|
|||
if os.path.isdir(ARCHIVES_PATH):
|
||||
chown(ARCHIVES_PATH, uid="admin", gid="root")
|
||||
|
||||
def backward(self):
|
||||
|
||||
raise YunohostError("migration_0008_backward_impossible")
|
||||
|
||||
@property
|
||||
def mode(self):
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ class MyMigration(Migration):
|
|||
Decouple the regen conf mechanism from the concept of services
|
||||
"""
|
||||
|
||||
def migrate(self):
|
||||
def run(self):
|
||||
|
||||
if "conffiles" not in read_file("/etc/yunohost/services.yml") \
|
||||
or os.path.exists(REGEN_CONF_FILE):
|
||||
|
|
@ -37,6 +37,3 @@ class MyMigration(Migration):
|
|||
# (Actually save the modification of services)
|
||||
_save_services(services)
|
||||
|
||||
def backward(self):
|
||||
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ class MyMigration(Migration):
|
|||
|
||||
"Migrate from official.json to apps.json"
|
||||
|
||||
def migrate(self):
|
||||
def run(self):
|
||||
|
||||
# Backup current app list json
|
||||
os.system("cp %s %s" % (APPSLISTS_JSON, APPSLISTS_BACKUP))
|
||||
|
|
@ -28,17 +28,21 @@ class MyMigration(Migration):
|
|||
"labriqueinter.net/apps/labriqueinternet.json",
|
||||
"labriqueinter.net/internetcube.json"
|
||||
]
|
||||
try:
|
||||
appslists = _read_appslist_list()
|
||||
for appslist, infos in appslists.items():
|
||||
if infos["url"].split("//")[-1] in lists_to_remove:
|
||||
app_removelist(name=appslist)
|
||||
|
||||
appslists = _read_appslist_list()
|
||||
for appslist, infos in appslists.items():
|
||||
if infos["url"].split("//")[-1] in lists_to_remove:
|
||||
app_removelist(name=appslist)
|
||||
# Replace by apps.json list
|
||||
app_fetchlist(name="yunohost",
|
||||
url="https://app.yunohost.org/apps.json")
|
||||
except Exception:
|
||||
if os.path.exists(APPSLISTS_BACKUP):
|
||||
os.system("cp %s %s" % (APPSLISTS_BACKUP, APPSLISTS_JSON))
|
||||
raise
|
||||
else:
|
||||
if os.path.exists(APPSLISTS_BACKUP):
|
||||
os.remove(APPSLISTS_BACKUP)
|
||||
|
||||
# Replace by apps.json list
|
||||
app_fetchlist(name="yunohost",
|
||||
url="https://app.yunohost.org/apps.json")
|
||||
|
||||
def backward(self):
|
||||
|
||||
if os.path.exists(APPSLISTS_BACKUP):
|
||||
os.system("cp %s %s" % (APPSLISTS_BACKUP, APPSLISTS_JSON))
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ class MyMigration(Migration):
|
|||
app_setting(app, 'allowed_users', delete=True)
|
||||
|
||||
|
||||
def migrate(self):
|
||||
def run(self):
|
||||
# Check if the migration can be processed
|
||||
ldap_regen_conf_status = regen_conf(names=['slapd'], dry_run=True)
|
||||
# By this we check if the have been customized
|
||||
|
|
|
|||
|
|
@ -10,12 +10,7 @@ class MyMigration(Migration):
|
|||
|
||||
all_hba_files = glob.glob("/etc/postgresql/*/*/pg_hba.conf")
|
||||
|
||||
def forward(self):
|
||||
def run(self):
|
||||
for filename in self.all_hba_files:
|
||||
pg_hba_in = read_file(filename)
|
||||
write_to_file(filename, re.sub(r"local(\s*)all(\s*)all(\s*)password", "local\\1all\\2all\\3md5", pg_hba_in))
|
||||
|
||||
def backward(self):
|
||||
for filename in self.all_hba_files:
|
||||
pg_hba_in = read_file(filename)
|
||||
write_to_file(filename, re.sub(r"local(\s*)all(\s*)all(\s*)md5", "local\\1all\\2all\\3password", pg_hba_in))
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ from collections import OrderedDict
|
|||
from moulinette import msignals, m18n
|
||||
from moulinette.utils.log import getActionLogger
|
||||
from moulinette.utils.process import check_output, call_async_output
|
||||
from moulinette.utils.filesystem import read_json, write_to_json
|
||||
from moulinette.utils.filesystem import read_json, write_to_json, read_yaml, write_to_yaml
|
||||
from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, app_list, _install_appslist_fetch_cron
|
||||
from yunohost.domain import domain_add, domain_list, _get_maindomain, _set_maindomain
|
||||
from yunohost.dyndns import _dyndns_available, _dyndns_provides
|
||||
|
|
@ -52,7 +52,7 @@ from yunohost.log import is_unit_operation, OperationLogger
|
|||
|
||||
# FIXME this is a duplicate from apps.py
|
||||
APPS_SETTING_PATH = '/etc/yunohost/apps/'
|
||||
MIGRATIONS_STATE_PATH = "/etc/yunohost/migrations_state.json"
|
||||
MIGRATIONS_STATE_PATH = "/etc/yunohost/migrations.yaml"
|
||||
|
||||
logger = getActionLogger('yunohost.tools')
|
||||
|
||||
|
|
@ -963,191 +963,6 @@ def tools_reboot(operation_logger, force=False):
|
|||
subprocess.check_call(['systemctl', 'reboot'])
|
||||
|
||||
|
||||
def tools_migrations_list(pending=False, done=False):
|
||||
"""
|
||||
List existing migrations
|
||||
"""
|
||||
|
||||
# Check for option conflict
|
||||
if pending and done:
|
||||
raise YunohostError("migrations_list_conflict_pending_done")
|
||||
|
||||
# Get all migrations
|
||||
migrations = _get_migrations_list()
|
||||
|
||||
# If asked, filter pending or done migrations
|
||||
if pending or done:
|
||||
last_migration = tools_migrations_state()["last_run_migration"]
|
||||
last_migration = last_migration["number"] if last_migration else -1
|
||||
if done:
|
||||
migrations = [m for m in migrations if m.number <= last_migration]
|
||||
if pending:
|
||||
migrations = [m for m in migrations if m.number > last_migration]
|
||||
|
||||
# Reduce to dictionnaries
|
||||
migrations = [{"id": migration.id,
|
||||
"number": migration.number,
|
||||
"name": migration.name,
|
||||
"mode": migration.mode,
|
||||
"description": migration.description,
|
||||
"disclaimer": migration.disclaimer} for migration in migrations]
|
||||
|
||||
return {"migrations": migrations}
|
||||
|
||||
|
||||
def tools_migrations_migrate(target=None, skip=False, auto=False, accept_disclaimer=False):
|
||||
"""
|
||||
Perform migrations
|
||||
"""
|
||||
|
||||
# state is a datastructure that represents the last run migration
|
||||
# it has this form:
|
||||
# {
|
||||
# "last_run_migration": {
|
||||
# "number": "00xx",
|
||||
# "name": "some name",
|
||||
# }
|
||||
# }
|
||||
state = tools_migrations_state()
|
||||
|
||||
last_run_migration_number = state["last_run_migration"]["number"] if state["last_run_migration"] else 0
|
||||
|
||||
# load all migrations
|
||||
migrations = _get_migrations_list()
|
||||
migrations = sorted(migrations, key=lambda x: x.number)
|
||||
|
||||
if not migrations:
|
||||
logger.info(m18n.n('migrations_no_migrations_to_run'))
|
||||
return
|
||||
|
||||
all_migration_numbers = [x.number for x in migrations]
|
||||
|
||||
if target is None:
|
||||
target = migrations[-1].number
|
||||
|
||||
# validate input, target must be "0" or a valid number
|
||||
elif target != 0 and target not in all_migration_numbers:
|
||||
raise YunohostError('migrations_bad_value_for_target', ", ".join(map(str, all_migration_numbers)))
|
||||
|
||||
logger.debug(m18n.n('migrations_current_target', target))
|
||||
|
||||
# no new migrations to run
|
||||
if target == last_run_migration_number:
|
||||
logger.info(m18n.n('migrations_no_migrations_to_run'))
|
||||
return
|
||||
|
||||
logger.debug(m18n.n('migrations_show_last_migration', last_run_migration_number))
|
||||
|
||||
# we need to run missing migrations
|
||||
if last_run_migration_number < target:
|
||||
logger.debug(m18n.n('migrations_forward'))
|
||||
# drop all already run migrations
|
||||
migrations = filter(lambda x: target >= x.number > last_run_migration_number, migrations)
|
||||
mode = "forward"
|
||||
|
||||
# we need to go backward on already run migrations
|
||||
elif last_run_migration_number > target:
|
||||
logger.debug(m18n.n('migrations_backward'))
|
||||
# drop all not already run migrations
|
||||
migrations = filter(lambda x: target < x.number <= last_run_migration_number, migrations)
|
||||
mode = "backward"
|
||||
|
||||
else: # can't happen, this case is handle before
|
||||
raise Exception()
|
||||
|
||||
# effectively run selected migrations
|
||||
for migration in migrations:
|
||||
|
||||
if not skip:
|
||||
# If we are migrating in "automatic mode" (i.e. from debian configure
|
||||
# during an upgrade of the package) but we are asked to run migrations
|
||||
# to be ran manually by the user, stop there and ask the user to
|
||||
# run the migration manually.
|
||||
if auto and migration.mode == "manual":
|
||||
logger.warn(m18n.n('migrations_to_be_ran_manually',
|
||||
number=migration.number,
|
||||
name=migration.name))
|
||||
break
|
||||
|
||||
# If some migrations have disclaimers,
|
||||
if migration.disclaimer:
|
||||
# require the --accept-disclaimer option. Otherwise, stop everything
|
||||
# here and display the disclaimer
|
||||
if not accept_disclaimer:
|
||||
logger.warn(m18n.n('migrations_need_to_accept_disclaimer',
|
||||
number=migration.number,
|
||||
name=migration.name,
|
||||
disclaimer=migration.disclaimer))
|
||||
break
|
||||
# --accept-disclaimer will only work for the first migration
|
||||
else:
|
||||
accept_disclaimer = False
|
||||
|
||||
# Start register change on system
|
||||
operation_logger = OperationLogger('tools_migrations_migrate_' + mode)
|
||||
operation_logger.start()
|
||||
|
||||
if not skip:
|
||||
|
||||
logger.info(m18n.n('migrations_show_currently_running_migration',
|
||||
number=migration.number, name=migration.name))
|
||||
|
||||
try:
|
||||
migration.operation_logger = operation_logger
|
||||
if mode == "forward":
|
||||
migration.migrate()
|
||||
elif mode == "backward":
|
||||
migration.backward()
|
||||
else: # can't happen
|
||||
raise Exception("Illegal state for migration: '%s', should be either 'forward' or 'backward'" % mode)
|
||||
except Exception as e:
|
||||
# migration failed, let's stop here but still update state because
|
||||
# we managed to run the previous ones
|
||||
msg = m18n.n('migrations_migration_has_failed',
|
||||
exception=e,
|
||||
number=migration.number,
|
||||
name=migration.name)
|
||||
logger.error(msg, exc_info=1)
|
||||
operation_logger.error(msg)
|
||||
break
|
||||
else:
|
||||
logger.success(m18n.n('migrations_success',
|
||||
number=migration.number, name=migration.name))
|
||||
|
||||
else: # if skip
|
||||
logger.warn(m18n.n('migrations_skip_migration',
|
||||
number=migration.number,
|
||||
name=migration.name))
|
||||
|
||||
# update the state to include the latest run migration
|
||||
state["last_run_migration"] = {
|
||||
"number": migration.number,
|
||||
"name": migration.name
|
||||
}
|
||||
|
||||
operation_logger.success()
|
||||
|
||||
# Skip migrations one at a time
|
||||
if skip:
|
||||
break
|
||||
|
||||
# special case where we want to go back from the start
|
||||
if target == 0:
|
||||
state["last_run_migration"] = None
|
||||
|
||||
write_to_json(MIGRATIONS_STATE_PATH, state)
|
||||
|
||||
|
||||
def tools_migrations_state():
|
||||
"""
|
||||
Show current migration state
|
||||
"""
|
||||
if not os.path.exists(MIGRATIONS_STATE_PATH):
|
||||
return {"last_run_migration": None}
|
||||
|
||||
return read_json(MIGRATIONS_STATE_PATH)
|
||||
|
||||
|
||||
def tools_shell(command=None):
|
||||
"""
|
||||
Launch an (i)python shell in the YunoHost context.
|
||||
|
|
@ -1179,6 +994,210 @@ def tools_shell(command=None):
|
|||
shell.interact()
|
||||
|
||||
|
||||
# ############################################ #
|
||||
# #
|
||||
# Migrations management #
|
||||
# #
|
||||
# ############################################ #
|
||||
|
||||
def tools_migrations_list(pending=False, done=False):
|
||||
"""
|
||||
List existing migrations
|
||||
"""
|
||||
|
||||
# Check for option conflict
|
||||
if pending and done:
|
||||
raise YunohostError("migrations_list_conflict_pending_done")
|
||||
|
||||
# Get all migrations
|
||||
migrations = _get_migrations_list()
|
||||
|
||||
# Reduce to dictionnaries
|
||||
migrations = [{"id": migration.id,
|
||||
"number": migration.number,
|
||||
"name": migration.name,
|
||||
"mode": migration.mode,
|
||||
"state": migration.state,
|
||||
"description": migration.description,
|
||||
"disclaimer": migration.disclaimer} for migration in migrations]
|
||||
|
||||
# If asked, filter pending or done migrations
|
||||
if pending or done:
|
||||
if done:
|
||||
migrations = [m for m in migrations if m["state"] != "pending"]
|
||||
if pending:
|
||||
migrations = [m for m in migrations if m["state"] == "pending"]
|
||||
|
||||
return {"migrations": migrations}
|
||||
|
||||
|
||||
def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=False, accept_disclaimer=False):
|
||||
"""
|
||||
Perform migrations
|
||||
|
||||
targets A list migrations to run (all pendings by default)
|
||||
--skip Skip specified migrations (to be used only if you know what you are doing) (must explicit which migrations)
|
||||
--auto Automatic mode, won't run manual migrations (to be used only if you know what you are doing)
|
||||
--force-rerun Re-run already-ran migrations (to be used only if you know what you are doing)(must explicit which migrations)
|
||||
--accept-disclaimer Accept disclaimers of migrations (please read them before using this option) (only valid for one migration)
|
||||
"""
|
||||
|
||||
all_migrations = _get_migrations_list()
|
||||
|
||||
# Small utility that allows up to get a migration given a name, id or number later
|
||||
def get_matching_migration(target):
|
||||
for m in all_migrations:
|
||||
if m.id == target or m.name == target or m.id.split("_")[0] == target:
|
||||
return m
|
||||
|
||||
raise YunohostError("migrations_no_such_migration", id=target)
|
||||
|
||||
# auto, skip and force are exclusive options
|
||||
if auto + skip + force_rerun > 1:
|
||||
raise YunohostError("migrations_exclusive_options")
|
||||
|
||||
# If no target specified
|
||||
if not targets:
|
||||
# skip, revert or force require explicit targets
|
||||
if (skip or force_rerun):
|
||||
raise YunohostError("migrations_must_provide_explicit_targets")
|
||||
|
||||
# Otherwise, targets are all pending migrations
|
||||
targets = [m for m in all_migrations if m.state == "pending"]
|
||||
|
||||
# If explicit targets are provided, we shall validate them
|
||||
else:
|
||||
targets = [get_matching_migration(t) for t in targets]
|
||||
done = [t.id for t in targets if t.state != "pending"]
|
||||
pending = [t.id for t in targets if t.state == "pending"]
|
||||
|
||||
if skip and done:
|
||||
raise YunohostError("migrations_not_pending_cant_skip", ids=', '.join(done))
|
||||
if force_rerun and pending:
|
||||
raise YunohostError("migrations_pending_cant_rerun", ids=', '.join(pending))
|
||||
if not (skip or force_rerun) and done:
|
||||
raise YunohostError("migrations_already_ran", ids=', '.join(done))
|
||||
|
||||
# So, is there actually something to do ?
|
||||
if not targets:
|
||||
logger.info(m18n.n('migrations_no_migrations_to_run'))
|
||||
return
|
||||
|
||||
# Actually run selected migrations
|
||||
for migration in targets:
|
||||
|
||||
# If we are migrating in "automatic mode" (i.e. from debian configure
|
||||
# during an upgrade of the package) but we are asked for running
|
||||
# migrations to be ran manually by the user, stop there and ask the
|
||||
# user to run the migration manually.
|
||||
if auto and migration.mode == "manual":
|
||||
logger.warn(m18n.n('migrations_to_be_ran_manually', id=migration.id))
|
||||
|
||||
# We go to the next migration
|
||||
continue
|
||||
|
||||
# Check for migration dependencies
|
||||
if not skip:
|
||||
dependencies = [get_matching_migration(dep) for dep in migration.dependencies]
|
||||
pending_dependencies = [dep.id for dep in dependencies if dep.state == "pending"]
|
||||
if pending_dependencies:
|
||||
logger.error(m18n.n('migrations_dependencies_not_satisfied',
|
||||
id=migration.id,
|
||||
dependencies_id=', '.join(pending_dependencies)))
|
||||
continue
|
||||
|
||||
# If some migrations have disclaimers (and we're not trying to skip them)
|
||||
if migration.disclaimer and not skip:
|
||||
# require the --accept-disclaimer option.
|
||||
# Otherwise, go to the next migration
|
||||
if not accept_disclaimer:
|
||||
logger.warn(m18n.n('migrations_need_to_accept_disclaimer',
|
||||
id=migration.id,
|
||||
disclaimer=migration.disclaimer))
|
||||
continue
|
||||
# --accept-disclaimer will only work for the first migration
|
||||
else:
|
||||
accept_disclaimer = False
|
||||
|
||||
# Start register change on system
|
||||
operation_logger = OperationLogger('tools_migrations_migrate_forward')
|
||||
operation_logger.start()
|
||||
|
||||
if skip:
|
||||
logger.warn(m18n.n('migrations_skip_migration', id=migration.id))
|
||||
migration.state = "skipped"
|
||||
_write_migration_state(migration.id, "skipped")
|
||||
operation_logger.success()
|
||||
else:
|
||||
|
||||
try:
|
||||
migration.operation_logger = operation_logger
|
||||
logger.info(m18n.n('migrations_running_forward', id=migration.id))
|
||||
migration.run()
|
||||
except Exception as e:
|
||||
# migration failed, let's stop here but still update state because
|
||||
# we managed to run the previous ones
|
||||
msg = m18n.n('migrations_migration_has_failed',
|
||||
exception=e, id=migration.id)
|
||||
logger.error(msg, exc_info=1)
|
||||
operation_logger.error(msg)
|
||||
else:
|
||||
logger.success(m18n.n('migrations_success_forward', id=migration.id))
|
||||
migration.state = "done"
|
||||
_write_migration_state(migration.id, "done")
|
||||
|
||||
operation_logger.success()
|
||||
|
||||
|
||||
def tools_migrations_state():
|
||||
"""
|
||||
Show current migration state
|
||||
"""
|
||||
if os.path.exists("/etc/yunohost/migrations_state.json"):
|
||||
_migrate_legacy_migration_json()
|
||||
|
||||
if not os.path.exists(MIGRATIONS_STATE_PATH):
|
||||
return {"migrations": {}}
|
||||
|
||||
return read_yaml(MIGRATIONS_STATE_PATH)
|
||||
|
||||
|
||||
def _migrate_legacy_migration_json():
|
||||
|
||||
from moulinette.utils.filesystem import read_json
|
||||
|
||||
logger.debug("Migrating legacy migration state json to yaml...")
|
||||
|
||||
# We fetch the old state containing the last run migration
|
||||
old_state = read_json("/etc/yunohost/migrations_state.json")["last_run_migration"]
|
||||
last_run_migration_id = str(old_state["number"]) + "_" + old_state["name"]
|
||||
|
||||
# Extract the list of migration ids
|
||||
from . import data_migrations
|
||||
migrations_path = data_migrations.__path__[0]
|
||||
migration_files = filter(lambda x: re.match("^\d+_[a-zA-Z0-9_]+\.py$", x), os.listdir(migrations_path))
|
||||
# (here we remove the .py extension and make sure the ids are sorted)
|
||||
migration_ids = sorted([f.rsplit(".", 1)[0] for f in migration_files])
|
||||
|
||||
# So now build the new dict for every id up to the last run migration
|
||||
migrations = {}
|
||||
for migration_id in migration_ids:
|
||||
migrations[migration_id] = "done"
|
||||
if last_run_migration_id in migration_id:
|
||||
break
|
||||
|
||||
# Write the new file and rename the old one
|
||||
write_to_yaml(MIGRATIONS_STATE_PATH, {"migrations": migrations})
|
||||
os.rename("/etc/yunohost/migrations_state.json", "/etc/yunohost/migrations_state.json.old")
|
||||
|
||||
|
||||
def _write_migration_state(migration_id, state):
|
||||
|
||||
current_states = tools_migrations_state()
|
||||
current_states["migrations"][migration_id] = state
|
||||
write_to_yaml(MIGRATIONS_STATE_PATH, current_states)
|
||||
|
||||
|
||||
def _get_migrations_list():
|
||||
migrations = []
|
||||
|
||||
|
|
@ -1194,8 +1213,21 @@ def _get_migrations_list():
|
|||
logger.warn(m18n.n('migrations_cant_reach_migration_file', migrations_path))
|
||||
return migrations
|
||||
|
||||
# states is a datastructure that represents the last run migration
|
||||
# it has this form:
|
||||
# {
|
||||
# "0001_foo": "skipped",
|
||||
# "0004_baz": "done",
|
||||
# "0002_bar": "skipped",
|
||||
# "0005_zblerg": "done",
|
||||
# }
|
||||
# (in particular, pending migrations / not already ran are not listed
|
||||
states = tools_migrations_state()["migrations"]
|
||||
|
||||
for migration_file in filter(lambda x: re.match("^\d+_[a-zA-Z0-9_]+\.py$", x), os.listdir(migrations_path)):
|
||||
migrations.append(_load_migration(migration_file))
|
||||
m = _load_migration(migration_file)
|
||||
m.state = states.get(m.id, "pending")
|
||||
migrations.append(m)
|
||||
|
||||
return sorted(migrations, key=lambda m: m.id)
|
||||
|
||||
|
|
@ -1222,10 +1254,7 @@ def _load_migration(migration_file):
|
|||
|
||||
migration_id = migration_file[:-len(".py")]
|
||||
|
||||
number, name = migration_id.split("_", 1)
|
||||
|
||||
logger.debug(m18n.n('migrations_loading_migration',
|
||||
number=number, name=name))
|
||||
logger.debug(m18n.n('migrations_loading_migration', id=migration_id))
|
||||
|
||||
try:
|
||||
# this is python builtin method to import a module using a name, we
|
||||
|
|
@ -1233,12 +1262,11 @@ def _load_migration(migration_file):
|
|||
# able to run it in the next loop
|
||||
module = import_module("yunohost.data_migrations.{}".format(migration_id))
|
||||
return module.MyMigration(migration_id)
|
||||
except Exception:
|
||||
except Exception as e:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
raise YunohostError('migrations_error_failed_to_load_migration',
|
||||
number=number, name=name)
|
||||
raise YunohostError('migrations_failed_to_load_migration', id=migration_id, error=e)
|
||||
|
||||
|
||||
def _skip_all_migrations():
|
||||
|
|
@ -1247,18 +1275,11 @@ def _skip_all_migrations():
|
|||
This is meant to be used during postinstall to
|
||||
initialize the migration system.
|
||||
"""
|
||||
state = tools_migrations_state()
|
||||
|
||||
# load all migrations
|
||||
migrations = _get_migrations_list()
|
||||
migrations = sorted(migrations, key=lambda x: x.number)
|
||||
last_migration = migrations[-1]
|
||||
|
||||
state["last_run_migration"] = {
|
||||
"number": last_migration.number,
|
||||
"name": last_migration.name
|
||||
}
|
||||
write_to_json(MIGRATIONS_STATE_PATH, state)
|
||||
all_migrations = _get_migrations_list()
|
||||
new_states = {"migrations": {}}
|
||||
for migration in all_migrations:
|
||||
new_states["migrations"][migration.id] = "skipped"
|
||||
write_to_yaml(MIGRATIONS_STATE_PATH, new_states)
|
||||
|
||||
|
||||
class Migration(object):
|
||||
|
|
@ -1266,21 +1287,16 @@ class Migration(object):
|
|||
# Those are to be implemented by daughter classes
|
||||
|
||||
mode = "auto"
|
||||
|
||||
def forward(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
def backward(self):
|
||||
pass
|
||||
dependencies = [] # List of migration ids required before running this migration
|
||||
|
||||
@property
|
||||
def disclaimer(self):
|
||||
return None
|
||||
|
||||
# The followings shouldn't be overriden
|
||||
def run(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
def migrate(self):
|
||||
self.forward()
|
||||
# The followings shouldn't be overriden
|
||||
|
||||
def __init__(self, id_):
|
||||
self.id = id_
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue