diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 31fb0e6cf..6fac16511 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1555,6 +1555,13 @@ tools: list: action_help: List migrations api: GET /migrations + arguments: + --pending: + help: list only pending migrations + action: store_true + --done: + help: list only migrations already performed + action: store_true ### tools_migrations_migrate() migrate: @@ -1569,7 +1576,12 @@ tools: help: skip the migration(s), use it only if you know what you are doing full: --skip action: store_true - + --auto: + help: automatic mode, won't run manual migrations, use it only if you know what you are doing + action: store_true + --accept-disclaimer: + help: accept disclaimers of migration (please read them before using this option) + action: store_true ### tools_migrations_state() state: diff --git a/debian/postinst b/debian/postinst index 7e91ffbb3..5b6ed8259 100644 --- a/debian/postinst +++ b/debian/postinst @@ -15,7 +15,7 @@ do_configure() { yunohost service regen-conf --output-as none echo "Launching migrations.." - yunohost tools migrations migrate + yunohost tools migrations migrate --auto # restart yunohost-firewall if it's running service yunohost-firewall status >/dev/null \ diff --git a/locales/en.json b/locales/en.json index 66fa93f45..9957c25a0 100644 --- a/locales/en.json +++ b/locales/en.json @@ -222,18 +222,23 @@ "migrate_tsig_wait_3": "1min...", "migrate_tsig_wait_4": "30 secondes...", "migrate_tsig_not_needed": "You do not appear to use a dyndns domain, so no migration is needed !", + "migration_description_0001_change_cert_group_to_sslcert": "Change certificates group permissions from 'metronome' to 'ssl-cert'", + "migration_description_0002_migrate_to_tsig_sha256": "Improve security of dyndns TSIG by using SHA512 instead of MD5", "migrations_backward": "Migrating backward.", - "migrations_bad_value_for_target": "Invalide number for target argument, available migrations numbers are 0 or {}", + "migrations_bad_value_for_target": "Invalid number for target argument, available migrations numbers are 0 or {}", "migrations_cant_reach_migration_file": "Can't access migrations files at path %s", "migrations_current_target": "Migration target is {}", "migrations_error_failed_to_load_migration": "ERROR: failed to load migration {number} {name}", "migrations_forward": "Migrating forward", + "migrations_list_conflict_pending_done": "You cannot use both --previous and --done at the same time.", "migrations_loading_migration": "Loading migration {number} {name}...", "migrations_migration_has_failed": "Migration {number} {name} has failed with exception {exception}, aborting", "migrations_no_migrations_to_run": "No migrations to run", "migrations_show_currently_running_migration": "Running migration {number} {name}...", "migrations_show_last_migration": "Last ran migration is {}", "migrations_skip_migration": "Skipping migration {number} {name}...", + "migrations_to_be_ran_manually": "Migration {number} {name} has to be ran manually. Please go to Tools > Migrations on the webadmin, or run `yunohost tools migrations migrate`.", + "migrations_need_to_accept_disclaimer": "To run the migration {number} {name}, your must accept the following disclaimer:\n---\n{disclaimer}\n---\nIf you accept to run the migration, please re-run the command with the option --accept-disclaimer.", "monitor_disabled": "The server monitoring has been disabled", "monitor_enabled": "The server monitoring has been enabled", "monitor_glances_con_failed": "Unable to connect to Glances server", diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index 0aa6cf36c..f564479fe 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -232,10 +232,13 @@ def dyndns_update(dyn_host="dyndns.yunohost.org", domain=None, key=None, from yunohost.tools import _get_migration_by_name migration = _get_migration_by_name("migrate_to_tsig_sha256") try: - migration["module"].MyMigration().migrate(dyn_host, domain, key) + migration.migrate(dyn_host, domain, key) except Exception as e: - logger.error(m18n.n('migrations_migration_has_failed', exception=e, **migration), exc_info=1) - + logger.error(m18n.n('migrations_migration_has_failed', + exception=e, + number=migration.number, + name=migration.name), + exc_info=1) return # Extract 'host', e.g. 'nohost.me' from 'foo.nohost.me' diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 0b6de942e..a60ffa80a 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -395,7 +395,7 @@ def tools_postinstall(domain, password, ignore_dyndns=False): _install_appslist_fetch_cron() # Init migrations (skip them, no need to run them on a fresh system) - tools_migrations_migrate(skip=True) + tools_migrations_migrate(skip=True, auto=True) os.system('touch /etc/yunohost/installed') @@ -732,24 +732,39 @@ def tools_reboot(force=False): subprocess.check_call(['systemctl', 'reboot']) -def tools_migrations_list(): +def tools_migrations_list(pending=False, done=False): """ List existing migrations """ - migrations = {"migrations": []} + # Check for option conflict + if pending and done: + raise MoulinetteError(errno.EINVAL, m18n.n("migrations_list_conflict_pending_done")) - for migration in _get_migrations_list(): - migrations["migrations"].append({ - "number": int(migration.split("_", 1)[0]), - "name": migration.split("_", 1)[1], - "file_name": migration, - }) + # Get all migrations + migrations = _get_migrations_list() - return migrations + # If asked, filter pending or done migrations + if pending or done: + last_migration = tools_migrations_state()["last_run_migration"] + last_migration = last_migration["number"] if last_migration else -1 + if done: + migrations = [m for m in migrations if m.number <= last_migration] + if pending: + migrations = [m for m in migrations if m.number > last_migration] + + # Reduce to dictionnaries + migrations = [{ "id": migration.id, + "number": migration.number, + "name": migration.name, + "mode": migration.mode, + "description": migration.description, + "disclaimer": migration.disclaimer } for migration in migrations ] + + return {"migrations": migrations} -def tools_migrations_migrate(target=None, skip=False): +def tools_migrations_migrate(target=None, skip=False, auto=False, accept_disclaimer=False): """ Perform migrations """ @@ -766,26 +781,18 @@ def tools_migrations_migrate(target=None, skip=False): last_run_migration_number = state["last_run_migration"]["number"] if state["last_run_migration"] else 0 - migrations = [] - - # loading all migrations - for migration in tools_migrations_list()["migrations"]: - migrations.append({ - "number": migration["number"], - "name": migration["name"], - "module": _get_migration_module(migration), - }) - - migrations = sorted(migrations, key=lambda x: x["number"]) + # load all migrations + migrations = _get_migrations_list() + migrations = sorted(migrations, key=lambda x: x.number) if not migrations: logger.info(m18n.n('migrations_no_migrations_to_run')) return - all_migration_numbers = [x["number"] for x in migrations] + all_migration_numbers = [x.number for x in migrations] if target is None: - target = migrations[-1]["number"] + target = migrations[-1].number # validate input, target must be "0" or a valid number elif target != 0 and target not in all_migration_numbers: @@ -804,44 +811,74 @@ def tools_migrations_migrate(target=None, skip=False): if last_run_migration_number < target: logger.debug(m18n.n('migrations_forward')) # drop all already run migrations - migrations = filter(lambda x: target >= x["number"] > last_run_migration_number, migrations) + migrations = filter(lambda x: target >= x.number > last_run_migration_number, migrations) mode = "forward" # we need to go backward on already run migrations elif last_run_migration_number > target: logger.debug(m18n.n('migrations_backward')) # drop all not already run migrations - migrations = filter(lambda x: target < x["number"] <= last_run_migration_number, migrations) + migrations = filter(lambda x: target < x.number <= last_run_migration_number, migrations) mode = "backward" else: # can't happen, this case is handle before raise Exception() + # If we are migrating in "automatic mode" (i.e. from debian + # configure during an upgrade of the package) but we are asked to run + # migrations is to be ran manually by the user + manual_migrations = [m for m in migrations if m.mode == "manual"] + if not skip and auto and manual_migrations: + for m in manual_migrations: + logger.warn(m18n.n('migrations_to_be_ran_manually', + number=m.number, + name=m.name)) + return + + # If some migrations have disclaimers, require the --accept-disclaimer + # option + migrations_with_disclaimer = [m for m in migrations if m.disclaimer] + if not skip and not accept_disclaimer and migrations_with_disclaimer: + for m in migrations_with_disclaimer: + logger.warn(m18n.n('migrations_need_to_accept_disclaimer', + number=m.number, + name=m.name, + disclaimer=m.disclaimer)) + return + # effectively run selected migrations for migration in migrations: if not skip: - logger.warn(m18n.n('migrations_show_currently_running_migration', **migration)) + + logger.warn(m18n.n('migrations_show_currently_running_migration', + number=migration.number, name=migration.name)) try: if mode == "forward": - migration["module"].MyMigration().migrate() + migration.migrate() elif mode == "backward": - migration["module"].MyMigration().backward() + migration.backward() else: # can't happen raise Exception("Illegal state for migration: '%s', should be either 'forward' or 'backward'" % mode) except Exception as e: # migration failed, let's stop here but still update state because # we managed to run the previous ones - logger.error(m18n.n('migrations_migration_has_failed', exception=e, **migration), exc_info=1) + logger.error(m18n.n('migrations_migration_has_failed', + exception=e, + number=migration.number, + name=migration.name), + exc_info=1) break else: # if skip - logger.warn(m18n.n('migrations_skip_migration', **migration)) + logger.warn(m18n.n('migrations_skip_migration', + number=migration.number, + name=migration.name)) # update the state to include the latest run migration state["last_run_migration"] = { - "number": migration["number"], - "name": migration["name"], + "number": migration.number, + "name": migration.name } # special case where we want to go back from the start @@ -904,60 +941,79 @@ def _get_migrations_list(): logger.warn(m18n.n('migrations_cant_reach_migration_file', migrations_path)) return migrations - for migration in filter(lambda x: re.match("^\d+_[a-zA-Z0-9_]+\.py$", x), os.listdir(migrations_path)): - migrations.append(migration[:-len(".py")]) + for migration_file in filter(lambda x: re.match("^\d+_[a-zA-Z0-9_]+\.py$", x), os.listdir(migrations_path)): + migrations.append(_load_migration(migration_file)) - return sorted(migrations) + return sorted(migrations, key=lambda m: m.id) -def _get_migration_by_name(migration_name, with_module=True): +def _get_migration_by_name(migration_name): """ Low-level / "private" function to find a migration by its name """ - migrations = tools_migrations_list()["migrations"] + try: + import data_migrations + except ImportError: + raise AssertionError("Unable to find migration with name %s" % migration_name) - matches = [ m for m in migrations if m["name"] == migration_name ] + migrations_path = data_migrations.__path__[0] + migrations_found = filter(lambda x: re.match("^\d+_%s\.py$" % migration_name, x), os.listdir(migrations_path)) - assert len(matches) == 1, "Unable to find migration with name %s" % migration_name + assert len(migrations_found) == 1, "Unable to find migration with name %s" % migration_name - migration = matches[0] - - if with_module: - migration["module"] = _get_migration_module(migration) - - return migration + return _load_migration(migrations_found[0]) -def _get_migration_module(migration): +def _load_migration(migration_file): + + migration_id = migration_file[:-len(".py")] + + number, name = migration_id.split("_", 1) logger.debug(m18n.n('migrations_loading_migration', - number=migration["number"], - name=migration["name"], - )) + number=number, name=name)) try: # this is python builtin method to import a module using a name, we # use that to import the migration as a python object so we'll be # able to run it in the next loop - return import_module("yunohost.data_migrations.{file_name}".format(**migration)) + module = import_module("yunohost.data_migrations.{}".format(migration_id)) + return module.MyMigration(migration_id) except Exception: import traceback traceback.print_exc() raise MoulinetteError(errno.EINVAL, m18n.n('migrations_error_failed_to_load_migration', - number=migration["number"], - name=migration["name"], - )) + number=number, name=name)) class Migration(object): - def migrate(self): - self.forward() + # Those are to be implemented by daughter classes + + mode = "auto" def forward(self): raise NotImplementedError() def backward(self): pass + + @property + def disclaimer(self): + return None + + # The followings shouldn't be overriden + + def migrate(self): + self.forward() + + def __init__(self, id_): + self.id = id_ + self.number = int(self.id.split("_", 1)[0]) + self.name = self.id.split("_", 1)[1] + + @property + def description(self): + return m18n.n("migration_description_%s" % self.id)