From bee4fb70fda3857bbfef72b95c7d50198034ad10 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 3 Aug 2019 21:21:07 +0200 Subject: [PATCH 01/19] [mod] Move this block to have a whole part dedicated to migrations --- src/yunohost/tools.py | 68 +++++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 5e14082b5..8b0b981b2 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -970,6 +970,43 @@ def tools_reboot(operation_logger, force=False): subprocess.check_call(['systemctl', 'reboot']) +def tools_shell(command=None): + """ + Launch an (i)python shell in the YunoHost context. + + This is entirely aim for development. + """ + + from yunohost.utils.ldap import _get_ldap_interface + ldap = _get_ldap_interface() + + if command: + exec(command) + return + + logger.warn("The \033[1;34mldap\033[0m interface is available in this context") + try: + from IPython import embed + embed() + except ImportError: + logger.warn("You don't have IPython installed, consider installing it as it is way better than the standard shell.") + logger.warn("Falling back on the standard shell.") + + import readline # will allow Up/Down/History in the console + readline # to please pyflakes + import code + vars = globals().copy() + vars.update(locals()) + shell = code.InteractiveConsole(vars) + shell.interact() + + +# ############################################ # +# # +# Migrations management # +# # +# ############################################ # + def tools_migrations_list(pending=False, done=False): """ List existing migrations @@ -1155,37 +1192,6 @@ def tools_migrations_state(): return read_json(MIGRATIONS_STATE_PATH) -def tools_shell(command=None): - """ - Launch an (i)python shell in the YunoHost context. - - This is entirely aim for development. - """ - - from yunohost.utils.ldap import _get_ldap_interface - ldap = _get_ldap_interface() - - if command: - exec(command) - return - - logger.warn("The \033[1;34mldap\033[0m interface is available in this context") - try: - from IPython import embed - embed() - except ImportError: - logger.warn("You don't have IPython installed, consider installing it as it is way better than the standard shell.") - logger.warn("Falling back on the standard shell.") - - import readline # will allow Up/Down/History in the console - readline # to please pyflakes - import code - vars = globals().copy() - vars.update(locals()) - shell = code.InteractiveConsole(vars) - shell.interact() - - def _get_migrations_list(): migrations = [] From 2723300dcaa1ec9ba803496ad15ebc5c31397566 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 4 Aug 2019 00:28:14 +0200 Subject: [PATCH 02/19] Rework the migration system to have independent, non-ordered migrations, so each migration has a state skipped / done / pending and can be ran independently from others --- data/actionsmap/yunohost.yml | 24 ++-- src/yunohost/tools.py | 228 +++++++++++++++++------------------ 2 files changed, 125 insertions(+), 127 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 6401bdebc..f54cc2254 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1726,22 +1726,26 @@ tools: ### tools_migrations_migrate() migrate: - action_help: Perform migrations + action_help: Run migrations api: POST /migrations/migrate arguments: - -t: - help: target migration number (or 0), latest one by default - type: int - full: --target - -s: - help: skip the migration(s), use it only if you know what you are doing - full: --skip + targets: + help: Migrations to run (all pendings by default) + nargs: "*" + --skip: + help: Skip specified migrations (to be used only if you know what you are doing) + action: store_true + --revert: + help: Attempt to revert already-ran migrations + action: store_true + --force-rerun: + help: Re-run already-ran migrations (to be used only if you know what you are doing) action: store_true --auto: - help: automatic mode, won't run manual migrations, use it only if you know what you are doing + help: Automatic mode, won't run manual migrations (to be used only if you know what you are doing) action: store_true --accept-disclaimer: - help: accept disclaimers of migration (please read them before using this option) + help: Accept disclaimers of migrations (please read them before using this option) action: store_true ### tools_migrations_state() diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 8b0b981b2..f44e98a8d 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -37,7 +37,7 @@ from collections import OrderedDict from moulinette import msignals, m18n from moulinette.utils.log import getActionLogger from moulinette.utils.process import check_output, call_async_output -from moulinette.utils.filesystem import read_json, write_to_json +from moulinette.utils.filesystem import read_json, write_to_json, read_yaml, write_to_yaml from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, app_list, _install_appslist_fetch_cron from yunohost.domain import domain_add, domain_list, _get_maindomain, _set_maindomain from yunohost.dyndns import _dyndns_available, _dyndns_provides @@ -52,7 +52,8 @@ from yunohost.log import is_unit_operation, OperationLogger # FIXME this is a duplicate from apps.py APPS_SETTING_PATH = '/etc/yunohost/apps/' -MIGRATIONS_STATE_PATH = "/etc/yunohost/migrations_state.json" +MIGRATIONS_STATE_PATH = "/etc/yunohost/migrations.yaml" +OLD_MIGRATIONS_STATE_PATH = "/etc/yunohost/migrations_state.json" logger = getActionLogger('yunohost.tools') @@ -1019,131 +1020,126 @@ def tools_migrations_list(pending=False, done=False): # Get all migrations migrations = _get_migrations_list() - # If asked, filter pending or done migrations - if pending or done: - last_migration = tools_migrations_state()["last_run_migration"] - last_migration = last_migration["number"] if last_migration else -1 - if done: - migrations = [m for m in migrations if m.number <= last_migration] - if pending: - migrations = [m for m in migrations if m.number > last_migration] - # Reduce to dictionnaries migrations = [{"id": migration.id, "number": migration.number, "name": migration.name, "mode": migration.mode, + "state": migration.state, "description": migration.description, "disclaimer": migration.disclaimer} for migration in migrations] + # If asked, filter pending or done migrations + if pending or done: + if done: + migrations = [m for m in migrations if m["state"] != "pending"] + if pending: + migrations = [m for m in migrations if m["state"] == "pending"] + return {"migrations": migrations} -def tools_migrations_migrate(target=None, skip=False, auto=False, accept_disclaimer=False): +def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=False, revert=False, accept_disclaimer=False): """ Perform migrations + + targets : a list of migrations to act on (by default : all pending) + auto : automatic mode, run only 'automatic' migrations (compared to manual migrations). Option meant to be used in debian's postinst script during upgades. + skip : skip specified migrations (must explicit which migrations) + revert : to revert already-ran migrations (must explicit which migrations) + force_rerun : to re-run already ran migrations (if you know what you're doing...) (must explicit which migrations) + accept_disclaimer : accept disclaimer for manual migration that requires it (only valid for one migration) """ - # state is a datastructure that represents the last run migration - # it has this form: - # { - # "last_run_migration": { - # "number": "00xx", - # "name": "some name", - # } - # } - state = tools_migrations_state() + all_migrations = _get_migrations_list() - last_run_migration_number = state["last_run_migration"]["number"] if state["last_run_migration"] else 0 + # auto, skip, revert and force are exclusive options + if auto + skip + revert + force_rerun > 1: + raise YunohostError("--auto, --skip, --revert and --force-rerun are exclusive options.") - # load all migrations - migrations = _get_migrations_list() - migrations = sorted(migrations, key=lambda x: x.number) + # If no target specified + if not targets: + # skip, revert or force require explicit targets + if (revert or force_rerun): + raise YunohostError("You must provide explicit targets when using --skip, --revert or --force-rerun") - if not migrations: + # Otherwise, targets are all pending migrations + targets = [m for m in all_migrations if m.state == "pending"] + + # If explicit targets are provided, we shall validate them + else: + def get_matching_migration(target): + for m in all_migrations: + if m.id == target or m.name == target or m.id.split("_")[0] == target: + return m + + raise YunohostError("No such migration called %s" % target) + + targets = [get_matching_migration(t) for t in targets] + done = [t.id for t in targets if t.state != "pending"] + pending = [t.id for t in targets if t.state == "pending"] + + if skip and done: + raise YunohostError("Those migrations are not pending so cannot be skipped: %s" % ', '.join(done)) + if (revert or force_rerun) and pending: + raise YunohostError("Those migrations were not already ran so cannot be reverted or reran: %s" % ', '.join(pending)) + + # So, is there actually something to do ? + if not targets: logger.info(m18n.n('migrations_no_migrations_to_run')) return - all_migration_numbers = [x.number for x in migrations] + # Actually run selected migrations + for migration in targets: - if target is None: - target = migrations[-1].number + # If we are migrating in "automatic mode" (i.e. from debian configure + # during an upgrade of the package) but we are asked to run migrations + # to be ran manually by the user, stop there and ask the user to + # run the migration manually. + if auto and migration.mode == "manual": + logger.warn(m18n.n('migrations_to_be_ran_manually', + number=migration.number, + name=migration.name)) - # validate input, target must be "0" or a valid number - elif target != 0 and target not in all_migration_numbers: - raise YunohostError('migrations_bad_value_for_target', ", ".join(map(str, all_migration_numbers))) + # We go to the next migration + continue - logger.debug(m18n.n('migrations_current_target', target)) - - # no new migrations to run - if target == last_run_migration_number: - logger.info(m18n.n('migrations_no_migrations_to_run')) - return - - logger.debug(m18n.n('migrations_show_last_migration', last_run_migration_number)) - - # we need to run missing migrations - if last_run_migration_number < target: - logger.debug(m18n.n('migrations_forward')) - # drop all already run migrations - migrations = filter(lambda x: target >= x.number > last_run_migration_number, migrations) - mode = "forward" - - # we need to go backward on already run migrations - elif last_run_migration_number > target: - logger.debug(m18n.n('migrations_backward')) - # drop all not already run migrations - migrations = filter(lambda x: target < x.number <= last_run_migration_number, migrations) - mode = "backward" - - else: # can't happen, this case is handle before - raise Exception() - - # effectively run selected migrations - for migration in migrations: - - if not skip: - # If we are migrating in "automatic mode" (i.e. from debian configure - # during an upgrade of the package) but we are asked to run migrations - # to be ran manually by the user, stop there and ask the user to - # run the migration manually. - if auto and migration.mode == "manual": - logger.warn(m18n.n('migrations_to_be_ran_manually', + # If some migrations have disclaimers (and we're not trying to skip them) + if migration.disclaimer and not skip: + # require the --accept-disclaimer option. + # Otherwise, go to the next migration + if not accept_disclaimer: + logger.warn(m18n.n('migrations_need_to_accept_disclaimer', number=migration.number, - name=migration.name)) - break - - # If some migrations have disclaimers, - if migration.disclaimer: - # require the --accept-disclaimer option. Otherwise, stop everything - # here and display the disclaimer - if not accept_disclaimer: - logger.warn(m18n.n('migrations_need_to_accept_disclaimer', - number=migration.number, - name=migration.name, - disclaimer=migration.disclaimer)) - break - # --accept-disclaimer will only work for the first migration - else: - accept_disclaimer = False + name=migration.name, + disclaimer=migration.disclaimer)) + continue + # --accept-disclaimer will only work for the first migration + else: + accept_disclaimer = False # Start register change on system + mode = "backward" if revert else "forward" operation_logger = OperationLogger('tools_migrations_migrate_' + mode) operation_logger.start() - if not skip: + if skip: + logger.warn(m18n.n('migrations_skip_migration', + number=migration.number, + name=migration.name)) + _write_migration_state(migration.id, "skipped") + operation_logger.success() + else: logger.info(m18n.n('migrations_show_currently_running_migration', number=migration.number, name=migration.name)) try: migration.operation_logger = operation_logger - if mode == "forward": - migration.migrate() - elif mode == "backward": + if revert: migration.backward() - else: # can't happen - raise Exception("Illegal state for migration: '%s', should be either 'forward' or 'backward'" % mode) + else: + migration.migrate() except Exception as e: # migration failed, let's stop here but still update state because # we managed to run the previous ones @@ -1153,33 +1149,11 @@ def tools_migrations_migrate(target=None, skip=False, auto=False, accept_disclai name=migration.name) logger.error(msg, exc_info=1) operation_logger.error(msg) - break else: logger.success(m18n.n('migrations_success', number=migration.number, name=migration.name)) - - else: # if skip - logger.warn(m18n.n('migrations_skip_migration', - number=migration.number, - name=migration.name)) - - # update the state to include the latest run migration - state["last_run_migration"] = { - "number": migration.number, - "name": migration.name - } - - operation_logger.success() - - # Skip migrations one at a time - if skip: - break - - # special case where we want to go back from the start - if target == 0: - state["last_run_migration"] = None - - write_to_json(MIGRATIONS_STATE_PATH, state) + _write_migration_state(migration.id, "done") + operation_logger.success() def tools_migrations_state(): @@ -1187,9 +1161,16 @@ def tools_migrations_state(): Show current migration state """ if not os.path.exists(MIGRATIONS_STATE_PATH): - return {"last_run_migration": None} + return {"migrations": {}} - return read_json(MIGRATIONS_STATE_PATH) + return read_yaml(MIGRATIONS_STATE_PATH) + + +def _write_migration_state(migration_id, state): + + current_states = tools_migrations_state() + current_states["migrations"][migration_id] = state + write_to_yaml(MIGRATIONS_STATE_PATH, current_states) def _get_migrations_list(): @@ -1207,8 +1188,21 @@ def _get_migrations_list(): logger.warn(m18n.n('migrations_cant_reach_migration_file', migrations_path)) return migrations + # states is a datastructure that represents the last run migration + # it has this form: + # { + # "0001_foo": "skipped", + # "0004_baz": "done", + # "0002_bar": "skipped", + # "0005_zblerg": "done", + # } + # (in particular, pending migrations / not already ran are not listed + states = tools_migrations_state()["migrations"] + for migration_file in filter(lambda x: re.match("^\d+_[a-zA-Z0-9_]+\.py$", x), os.listdir(migrations_path)): - migrations.append(_load_migration(migration_file)) + m = _load_migration(migration_file) + m.state = states.get(m.id, "pending") + migrations.append(m) return sorted(migrations, key=lambda m: m.id) From da64778b0d73ae9a3b41f92800f62538a7deced7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 4 Aug 2019 00:57:00 +0200 Subject: [PATCH 03/19] Migrate the old migration json to the new yaml format --- src/yunohost/tools.py | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index f44e98a8d..a15b5593e 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -53,7 +53,6 @@ from yunohost.log import is_unit_operation, OperationLogger # FIXME this is a duplicate from apps.py APPS_SETTING_PATH = '/etc/yunohost/apps/' MIGRATIONS_STATE_PATH = "/etc/yunohost/migrations.yaml" -OLD_MIGRATIONS_STATE_PATH = "/etc/yunohost/migrations_state.json" logger = getActionLogger('yunohost.tools') @@ -1160,12 +1159,48 @@ def tools_migrations_state(): """ Show current migration state """ + if os.path.exists("/etc/yunohost/migrations_state.json"): + _migrate_legacy_migration_json() + if not os.path.exists(MIGRATIONS_STATE_PATH): return {"migrations": {}} return read_yaml(MIGRATIONS_STATE_PATH) +def _migrate_legacy_migration_json(): + + from moulinette.utils.filesystem import read_json + + logger.debug("Migrating legacy migration state json to yaml...") + + # We fetch the old state containing the last run migration + old_state = read_json("/etc/yunohost/migrations_state.json")["last_run_migration"] + last_run_migration_id = str(old_state["number"]) + "_" + old_state["name"] + + # Extract the list of migration ids + try: + from . import data_migrations + except ImportError: + # Well ugh what are we supposed to do if we can't do this >.>... + pass + migrations_path = data_migrations.__path__[0] + migration_files = filter(lambda x: re.match("^\d+_[a-zA-Z0-9_]+\.py$", x), os.listdir(migrations_path)) + # (here we remove the .py extension and make sure the ids are sorted) + migration_ids = sorted([f.rsplit(".", 1)[0] for f in migration_files]) + + # So now build the new dict for every id up to the last run migration + migrations = {} + for migration_id in migration_ids: + migrations[migration_id] = "done" + if last_run_migration_id in migration_id: + break + + # Write the new file and rename the old one + write_to_yaml(MIGRATIONS_STATE_PATH, {"migrations": migrations}) + os.rename("/etc/yunohost/migrations_state.json", "/etc/yunohost/migrations_state.json.old") + + def _write_migration_state(migration_id, state): current_states = tools_migrations_state() From add3bd29b014cfe87ce4cc77b5b0eeeebc24ea8a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 5 Aug 2019 21:10:03 +0200 Subject: [PATCH 04/19] Add dependencies between migrations --- locales/en.json | 2 +- .../0003_migrate_to_stretch.py | 2 +- .../0004_php5_to_php7_pools.py | 2 ++ .../0005_postgresql_9p4_to_9p6.py | 2 ++ ...0008_ssh_conf_managed_by_yunohost_step2.py | 4 ++- src/yunohost/tools.py | 25 +++++++++++++------ 6 files changed, 27 insertions(+), 10 deletions(-) diff --git a/locales/en.json b/locales/en.json index ecfba2947..14aee512d 100644 --- a/locales/en.json +++ b/locales/en.json @@ -321,6 +321,7 @@ "migrate_tsig_wait_3": "1min…", "migrate_tsig_wait_4": "30 secondes…", "migrate_tsig_not_needed": "You do not appear to use a dyndns domain, so no migration is needed!", + "migration_backward_impossible": "The migration {name} cannot be reverted.", "migration_description_0001_change_cert_group_to_sslcert": "Change certificates group permissions from 'metronome' to 'ssl-cert'", "migration_description_0002_migrate_to_tsig_sha256": "Improve security of dyndns TSIG by using SHA512 instead of MD5", "migration_description_0003_migrate_to_stretch": "Upgrade the system to Debian Stretch and YunoHost 3.0", @@ -332,7 +333,6 @@ "migration_description_0009_decouple_regenconf_from_services": "Decouple the regen-conf mechanism from services", "migration_description_0010_migrate_to_apps_json": "Remove deprecated appslists and use the new unified 'apps.json' list instead", "migration_description_0011_setup_group_permission": "Setup user group and setup permission for apps and services", - "migration_0003_backward_impossible": "The stretch migration cannot be reverted.", "migration_0003_start": "Starting migration to Stretch. The logs will be available in {logfile}.", "migration_0003_patching_sources_list": "Patching the sources.lists…", "migration_0003_main_upgrade": "Starting main upgrade…", diff --git a/src/yunohost/data_migrations/0003_migrate_to_stretch.py b/src/yunohost/data_migrations/0003_migrate_to_stretch.py index 0db719e15..8b7586701 100644 --- a/src/yunohost/data_migrations/0003_migrate_to_stretch.py +++ b/src/yunohost/data_migrations/0003_migrate_to_stretch.py @@ -31,7 +31,7 @@ class MyMigration(Migration): def backward(self): - raise YunohostError("migration_0003_backward_impossible") + raise YunohostError("migration_backward_impossible", name=self.name) def migrate(self): diff --git a/src/yunohost/data_migrations/0004_php5_to_php7_pools.py b/src/yunohost/data_migrations/0004_php5_to_php7_pools.py index 46a5eb91d..faf9ccbbc 100644 --- a/src/yunohost/data_migrations/0004_php5_to_php7_pools.py +++ b/src/yunohost/data_migrations/0004_php5_to_php7_pools.py @@ -22,6 +22,8 @@ class MyMigration(Migration): "Migrate php5-fpm 'pool' conf files to php7 stuff" + dependencies = ["migrate_to_stretch"] + def migrate(self): # Get list of php5 pool files diff --git a/src/yunohost/data_migrations/0005_postgresql_9p4_to_9p6.py b/src/yunohost/data_migrations/0005_postgresql_9p4_to_9p6.py index 5ae729b60..50c0561cb 100644 --- a/src/yunohost/data_migrations/0005_postgresql_9p4_to_9p6.py +++ b/src/yunohost/data_migrations/0005_postgresql_9p4_to_9p6.py @@ -14,6 +14,8 @@ class MyMigration(Migration): "Migrate DBs from Postgresql 9.4 to 9.6 after migrating to Stretch" + dependencies = ["migrate_to_stretch"] + def migrate(self): if not self.package_is_installed("postgresql-9.4"): diff --git a/src/yunohost/data_migrations/0008_ssh_conf_managed_by_yunohost_step2.py b/src/yunohost/data_migrations/0008_ssh_conf_managed_by_yunohost_step2.py index 8984440bd..0ea0ddc3a 100644 --- a/src/yunohost/data_migrations/0008_ssh_conf_managed_by_yunohost_step2.py +++ b/src/yunohost/data_migrations/0008_ssh_conf_managed_by_yunohost_step2.py @@ -33,6 +33,8 @@ class MyMigration(Migration): shown - and the user may also choose to skip this migration. """ + dependencies = ["ssh_conf_managed_by_yunohost_step1"] + def migrate(self): settings_set("service.ssh.allow_deprecated_dsa_hostkey", False) regen_conf(names=['ssh'], force=True) @@ -44,7 +46,7 @@ class MyMigration(Migration): def backward(self): - raise YunohostError("migration_0008_backward_impossible") + raise YunohostError("migration_backward_impossible", name=self.name) @property def mode(self): diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index a15b5593e..9117f42de 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -1052,6 +1052,15 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal all_migrations = _get_migrations_list() + # Small utility that allows up to get a migration given a name, id or number later + def get_matching_migration(target): + for m in all_migrations: + if m.id == target or m.name == target or m.id.split("_")[0] == target: + return m + + raise YunohostError("No such migration called %s" % target) + + # auto, skip, revert and force are exclusive options if auto + skip + revert + force_rerun > 1: raise YunohostError("--auto, --skip, --revert and --force-rerun are exclusive options.") @@ -1067,13 +1076,6 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal # If explicit targets are provided, we shall validate them else: - def get_matching_migration(target): - for m in all_migrations: - if m.id == target or m.name == target or m.id.split("_")[0] == target: - return m - - raise YunohostError("No such migration called %s" % target) - targets = [get_matching_migration(t) for t in targets] done = [t.id for t in targets if t.state != "pending"] pending = [t.id for t in targets if t.state == "pending"] @@ -1103,6 +1105,14 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal # We go to the next migration continue + # Check for migration dependencies + if not revert and not skip: + dependencies = [get_matching_migration(dep) for dep in migration.dependencies] + pending_dependencies = [dep.id for dep in dependencies if dep.state == "pending"] + if pending_dependencies: + logger.error("Can't run migration %s because first you need to run these migrations: %s" % (migration.id, ', '.join(pending_dependencies))) + continue + # If some migrations have disclaimers (and we're not trying to skip them) if migration.disclaimer and not skip: # require the --accept-disclaimer option. @@ -1308,6 +1318,7 @@ class Migration(object): # Those are to be implemented by daughter classes mode = "auto" + dependencies = [] # List of migration ids required before running this migration def forward(self): raise NotImplementedError() From dedc85e52a9b63df171287395e4da96a3a1227dc Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 5 Aug 2019 21:45:45 +0200 Subject: [PATCH 05/19] Implement / clean strings + fix some edge cases --- locales/en.json | 30 ++++++++++-------- src/yunohost/tools.py | 73 +++++++++++++++++++++---------------------- 2 files changed, 53 insertions(+), 50 deletions(-) diff --git a/locales/en.json b/locales/en.json index 14aee512d..471f0f9dc 100644 --- a/locales/en.json +++ b/locales/en.json @@ -369,22 +369,26 @@ "migration_0011_rollback_success": "Rollback succeeded.", "migration_0011_update_LDAP_database": "Updating LDAP database...", "migration_0011_update_LDAP_schema": "Updating LDAP schema...", - "migrations_backward": "Migrating backward.", - "migrations_bad_value_for_target": "Invalid number for target argument, available migrations numbers are 0 or {}", + "migrations_already_ran": "Those migrations have already been ran: {ids}", "migrations_cant_reach_migration_file": "Can't access migrations files at path %s", - "migrations_current_target": "Migration target is {}", - "migrations_error_failed_to_load_migration": "ERROR: failed to load migration {number} {name}", - "migrations_forward": "Migrating forward", + "migrations_dependencies_not_satisfied": "Can't run migration {id} because first you need to run these migrations: {dependencies_id}", + "migrations_failed_to_load_migration": "Failed to load migration {id} : {error}", + "migrations_exclusive_options": "--auto, --skip, --revert and --force-rerun are exclusive options.", "migrations_list_conflict_pending_done": "You cannot use both --previous and --done at the same time.", - "migrations_loading_migration": "Loading migration {number} {name}…", - "migrations_migration_has_failed": "Migration {number} {name} has failed with exception {exception}, aborting", + "migrations_loading_migration": "Loading migration {id}…", + "migrations_migration_has_failed": "Migration {id} has failed, aborting. Error: {exception}", + "migrations_must_provide_explicit_targets": "You must provide explicit targets when using --skip, --revert or --force-rerun", + "migrations_need_to_accept_disclaimer": "To run the migration {id}, your must accept the following disclaimer:\n---\n{disclaimer}\n---\nIf you accept to run the migration, please re-run the command with the option --accept-disclaimer.", "migrations_no_migrations_to_run": "No migrations to run", - "migrations_show_currently_running_migration": "Running migration {number} {name}…", - "migrations_show_last_migration": "Last ran migration is {}", - "migrations_skip_migration": "Skipping migration {number} {name}…", - "migrations_success": "Successfully ran migration {number} {name}!", - "migrations_to_be_ran_manually": "Migration {number} {name} has to be ran manually. Please go to Tools > Migrations on the webadmin, or run `yunohost tools migrations migrate`.", - "migrations_need_to_accept_disclaimer": "To run the migration {number} {name}, your must accept the following disclaimer:\n---\n{disclaimer}\n---\nIf you accept to run the migration, please re-run the command with the option --accept-disclaimer.", + "migrations_no_such_migration": "No such migration called {id}", + "migrations_not_pending_cant_skip": "Those migrations are not pending so cannot be skipped: {ids}", + "migrations_pending_cant_revert_or_rerun": "Those migrations are still pending so cannot be reverted or reran: {ids}", + "migrations_running_forward": "Running migration {id}…", + "migrations_running_backward": "Attempting to revert migration {id}…", + "migrations_skip_migration": "Skipping migration {id}…", + "migrations_success_forward": "Successfully ran migration {id}!", + "migrations_success_revert": "Successfully reverted migration {id}!", + "migrations_to_be_ran_manually": "Migration {id} has to be ran manually. Please go to Tools > Migrations on the webadmin, or run `yunohost tools migrations migrate`.", "monitor_disabled": "The server monitoring has been disabled", "monitor_enabled": "The server monitoring has been enabled", "monitor_glances_con_failed": "Unable to connect to Glances server", diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 9117f42de..fa46721c9 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -1042,12 +1042,12 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal """ Perform migrations - targets : a list of migrations to act on (by default : all pending) - auto : automatic mode, run only 'automatic' migrations (compared to manual migrations). Option meant to be used in debian's postinst script during upgades. - skip : skip specified migrations (must explicit which migrations) - revert : to revert already-ran migrations (must explicit which migrations) - force_rerun : to re-run already ran migrations (if you know what you're doing...) (must explicit which migrations) - accept_disclaimer : accept disclaimer for manual migration that requires it (only valid for one migration) + targets A list migrations to run (all pendings by default) + --skip Skip specified migrations (to be used only if you know what you are doing) (must explicit which migrations) + --auto Automatic mode, won't run manual migrations (to be used only if you know what you are doing) (must explicit which migrations) + --force-rerun Re-run already-ran migrations (to be used only if you know what you are doing)(must explicit which migrations) + --revert Attempt to revert already-ran migrations (must explicit which migrations) + --accept-disclaimer Accept disclaimers of migrations (please read them before using this option) (only valid for one migration) """ all_migrations = _get_migrations_list() @@ -1058,18 +1058,18 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal if m.id == target or m.name == target or m.id.split("_")[0] == target: return m - raise YunohostError("No such migration called %s" % target) + raise YunohostError("migrations_no_such_migration", id=target) # auto, skip, revert and force are exclusive options if auto + skip + revert + force_rerun > 1: - raise YunohostError("--auto, --skip, --revert and --force-rerun are exclusive options.") + raise YunohostError("migrations_exclusive_options") # If no target specified if not targets: # skip, revert or force require explicit targets if (revert or force_rerun): - raise YunohostError("You must provide explicit targets when using --skip, --revert or --force-rerun") + raise YunohostError("migrations_must_provide_explicit_targets") # Otherwise, targets are all pending migrations targets = [m for m in all_migrations if m.state == "pending"] @@ -1081,9 +1081,11 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal pending = [t.id for t in targets if t.state == "pending"] if skip and done: - raise YunohostError("Those migrations are not pending so cannot be skipped: %s" % ', '.join(done)) + raise YunohostError("migrations_not_pending_cant_skip", ids=', '.join(done)) if (revert or force_rerun) and pending: - raise YunohostError("Those migrations were not already ran so cannot be reverted or reran: %s" % ', '.join(pending)) + raise YunohostError("migrations_pending_cant_revert_or_rerun", ids=', '.join(pending)) + if not (skip or revert or force_rerun) and done: + raise YunohostError("migrations_already_ran", ids=', '.join(done)) # So, is there actually something to do ? if not targets: @@ -1098,9 +1100,7 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal # to be ran manually by the user, stop there and ask the user to # run the migration manually. if auto and migration.mode == "manual": - logger.warn(m18n.n('migrations_to_be_ran_manually', - number=migration.number, - name=migration.name)) + logger.warn(m18n.n('migrations_to_be_ran_manually', id=migration.id)) # We go to the next migration continue @@ -1110,17 +1110,18 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal dependencies = [get_matching_migration(dep) for dep in migration.dependencies] pending_dependencies = [dep.id for dep in dependencies if dep.state == "pending"] if pending_dependencies: - logger.error("Can't run migration %s because first you need to run these migrations: %s" % (migration.id, ', '.join(pending_dependencies))) + logger.error(m18n.n('migrations_dependencies_not_satisfied', + id=migration.id, + dependencies_id=', '.join(pending_dependencies))) continue # If some migrations have disclaimers (and we're not trying to skip them) - if migration.disclaimer and not skip: + if migration.disclaimer and not skip and not revert: # require the --accept-disclaimer option. # Otherwise, go to the next migration if not accept_disclaimer: logger.warn(m18n.n('migrations_need_to_accept_disclaimer', - number=migration.number, - name=migration.name, + id=migration.id, disclaimer=migration.disclaimer)) continue # --accept-disclaimer will only work for the first migration @@ -1133,35 +1134,37 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal operation_logger.start() if skip: - logger.warn(m18n.n('migrations_skip_migration', - number=migration.number, - name=migration.name)) + logger.warn(m18n.n('migrations_skip_migration', id=migration.id)) + migration.state = "skipped" _write_migration_state(migration.id, "skipped") operation_logger.success() else: - logger.info(m18n.n('migrations_show_currently_running_migration', - number=migration.number, name=migration.name)) - try: migration.operation_logger = operation_logger if revert: + logger.info(m18n.n('migrations_running_backward', id=migration.id)) migration.backward() else: + logger.info(m18n.n('migrations_running_forward', id=migration.id)) migration.migrate() except Exception as e: # migration failed, let's stop here but still update state because # we managed to run the previous ones msg = m18n.n('migrations_migration_has_failed', - exception=e, - number=migration.number, - name=migration.name) + exception=e, id=migration.id) logger.error(msg, exc_info=1) operation_logger.error(msg) else: - logger.success(m18n.n('migrations_success', - number=migration.number, name=migration.name)) - _write_migration_state(migration.id, "done") + if revert: + logger.success(m18n.n('migrations_success_revert', id=migration.id)) + migration.state = "pending" + _write_migration_state(migration.id, "pending") + else: + logger.success(m18n.n('migrations_success_forward', id=migration.id)) + migration.state = "done" + _write_migration_state(migration.id, "done") + operation_logger.success() @@ -1274,10 +1277,7 @@ def _load_migration(migration_file): migration_id = migration_file[:-len(".py")] - number, name = migration_id.split("_", 1) - - logger.debug(m18n.n('migrations_loading_migration', - number=number, name=name)) + logger.debug(m18n.n('migrations_loading_migration', id=migration_id)) try: # this is python builtin method to import a module using a name, we @@ -1285,12 +1285,11 @@ def _load_migration(migration_file): # able to run it in the next loop module = import_module("yunohost.data_migrations.{}".format(migration_id)) return module.MyMigration(migration_id) - except Exception: + except Exception as e: import traceback traceback.print_exc() - raise YunohostError('migrations_error_failed_to_load_migration', - number=number, name=name) + raise YunohostError('migrations_failed_to_load_migration', id=migration_id, error=e) def _skip_all_migrations(): From 74b5ea5982099a7661f169c36e89efb0d6b69df2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 5 Aug 2019 22:07:48 +0200 Subject: [PATCH 06/19] Propagate changes to _skip_all_migrations --- src/yunohost/tools.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index fa46721c9..a6379e62b 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -1298,18 +1298,11 @@ def _skip_all_migrations(): This is meant to be used during postinstall to initialize the migration system. """ - state = tools_migrations_state() - - # load all migrations - migrations = _get_migrations_list() - migrations = sorted(migrations, key=lambda x: x.number) - last_migration = migrations[-1] - - state["last_run_migration"] = { - "number": last_migration.number, - "name": last_migration.name - } - write_to_json(MIGRATIONS_STATE_PATH, state) + all_migrations = _get_migrations_list() + new_states = {"migrations": {}} + for migration in all_migrations: + new_states["migrations"][migration.id] = "skipped" + write_to_yaml(MIGRATIONS_STATE_PATH, new_states) class Migration(object): From fc426cff69536417374f334c08dc4e75a59f7941 Mon Sep 17 00:00:00 2001 From: ljf Date: Wed, 14 Aug 2019 12:07:39 +0200 Subject: [PATCH 07/19] [fix] Avoid to revert migrations without backward routine --- src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py | 4 ---- src/yunohost/data_migrations/0003_migrate_to_stretch.py | 6 +----- src/yunohost/data_migrations/0004_php5_to_php7_pools.py | 2 +- src/yunohost/data_migrations/0005_postgresql_9p4_to_9p6.py | 6 +----- .../data_migrations/0006_sync_admin_and_root_passwords.py | 5 +---- .../0007_ssh_conf_managed_by_yunohost_step1.py | 2 +- .../0008_ssh_conf_managed_by_yunohost_step2.py | 6 +----- .../0009_decouple_regenconf_from_services.py | 5 +---- src/yunohost/data_migrations/0010_migrate_to_apps_json.py | 2 +- src/yunohost/data_migrations/0011_setup_group_permission.py | 2 +- src/yunohost/tools.py | 2 +- 11 files changed, 10 insertions(+), 32 deletions(-) diff --git a/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py b/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py index 824245c82..cc5d4c19b 100644 --- a/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py +++ b/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py @@ -19,10 +19,6 @@ class MyMigration(Migration): "Migrate Dyndns stuff from MD5 TSIG to SHA512 TSIG" - def backward(self): - # Not possible because that's a non-reversible operation ? - pass - def migrate(self, dyn_host="dyndns.yunohost.org", domain=None, private_key_path=None): if domain is None or private_key_path is None: diff --git a/src/yunohost/data_migrations/0003_migrate_to_stretch.py b/src/yunohost/data_migrations/0003_migrate_to_stretch.py index 8b7586701..0db6cfd6d 100644 --- a/src/yunohost/data_migrations/0003_migrate_to_stretch.py +++ b/src/yunohost/data_migrations/0003_migrate_to_stretch.py @@ -29,11 +29,7 @@ class MyMigration(Migration): mode = "manual" - def backward(self): - - raise YunohostError("migration_backward_impossible", name=self.name) - - def migrate(self): + def forward(self): self.logfile = "/var/log/yunohost/{}.log".format(self.name) diff --git a/src/yunohost/data_migrations/0004_php5_to_php7_pools.py b/src/yunohost/data_migrations/0004_php5_to_php7_pools.py index faf9ccbbc..e916a9cd7 100644 --- a/src/yunohost/data_migrations/0004_php5_to_php7_pools.py +++ b/src/yunohost/data_migrations/0004_php5_to_php7_pools.py @@ -24,7 +24,7 @@ class MyMigration(Migration): dependencies = ["migrate_to_stretch"] - def migrate(self): + def forward(self): # Get list of php5 pool files php5_pool_files = glob.glob("{}/*.conf".format(PHP5_POOLS)) diff --git a/src/yunohost/data_migrations/0005_postgresql_9p4_to_9p6.py b/src/yunohost/data_migrations/0005_postgresql_9p4_to_9p6.py index 50c0561cb..9cb6e51bd 100644 --- a/src/yunohost/data_migrations/0005_postgresql_9p4_to_9p6.py +++ b/src/yunohost/data_migrations/0005_postgresql_9p4_to_9p6.py @@ -16,7 +16,7 @@ class MyMigration(Migration): dependencies = ["migrate_to_stretch"] - def migrate(self): + def forward(self): if not self.package_is_installed("postgresql-9.4"): logger.warning(m18n.n("migration_0005_postgresql_94_not_installed")) @@ -34,10 +34,6 @@ class MyMigration(Migration): subprocess.check_call("pg_dropcluster --stop 9.4 main", shell=True) subprocess.check_call("service postgresql start", shell=True) - def backward(self): - - pass - def package_is_installed(self, package_name): p = subprocess.Popen("dpkg --list | grep '^ii ' | grep -q -w {}".format(package_name), shell=True) diff --git a/src/yunohost/data_migrations/0006_sync_admin_and_root_passwords.py b/src/yunohost/data_migrations/0006_sync_admin_and_root_passwords.py index cd13d680d..953652111 100644 --- a/src/yunohost/data_migrations/0006_sync_admin_and_root_passwords.py +++ b/src/yunohost/data_migrations/0006_sync_admin_and_root_passwords.py @@ -20,16 +20,13 @@ class MyMigration(Migration): "Synchronize admin and root passwords" - def migrate(self): + def forward(self): new_hash = self._get_admin_hash() self._replace_root_hash(new_hash) logger.info(m18n.n("root_password_replaced_by_admin_password")) - def backward(self): - pass - @property def mode(self): diff --git a/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py b/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py index feffdc27c..7ddcec7fd 100644 --- a/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py +++ b/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py @@ -34,7 +34,7 @@ class MyMigration(Migration): use the recommended configuration, with an appropriate disclaimer. """ - def migrate(self): + def forward(self): # Check if deprecated DSA Host Key is in config dsa_rgx = r'^[ \t]*HostKey[ \t]+/etc/ssh/ssh_host_dsa_key[ \t]*(?:#.*)?$' diff --git a/src/yunohost/data_migrations/0008_ssh_conf_managed_by_yunohost_step2.py b/src/yunohost/data_migrations/0008_ssh_conf_managed_by_yunohost_step2.py index 0ea0ddc3a..c158192f9 100644 --- a/src/yunohost/data_migrations/0008_ssh_conf_managed_by_yunohost_step2.py +++ b/src/yunohost/data_migrations/0008_ssh_conf_managed_by_yunohost_step2.py @@ -35,7 +35,7 @@ class MyMigration(Migration): dependencies = ["ssh_conf_managed_by_yunohost_step1"] - def migrate(self): + def forward(self): settings_set("service.ssh.allow_deprecated_dsa_hostkey", False) regen_conf(names=['ssh'], force=True) @@ -44,10 +44,6 @@ class MyMigration(Migration): if os.path.isdir(ARCHIVES_PATH): chown(ARCHIVES_PATH, uid="admin", gid="root") - def backward(self): - - raise YunohostError("migration_backward_impossible", name=self.name) - @property def mode(self): diff --git a/src/yunohost/data_migrations/0009_decouple_regenconf_from_services.py b/src/yunohost/data_migrations/0009_decouple_regenconf_from_services.py index d552d7c9c..ffe2279a8 100644 --- a/src/yunohost/data_migrations/0009_decouple_regenconf_from_services.py +++ b/src/yunohost/data_migrations/0009_decouple_regenconf_from_services.py @@ -17,7 +17,7 @@ class MyMigration(Migration): Decouple the regen conf mechanism from the concept of services """ - def migrate(self): + def forward(self): if "conffiles" not in read_file("/etc/yunohost/services.yml") \ or os.path.exists(REGEN_CONF_FILE): @@ -37,6 +37,3 @@ class MyMigration(Migration): # (Actually save the modification of services) _save_services(services) - def backward(self): - - pass diff --git a/src/yunohost/data_migrations/0010_migrate_to_apps_json.py b/src/yunohost/data_migrations/0010_migrate_to_apps_json.py index 43ae9a86f..7092d92c7 100644 --- a/src/yunohost/data_migrations/0010_migrate_to_apps_json.py +++ b/src/yunohost/data_migrations/0010_migrate_to_apps_json.py @@ -15,7 +15,7 @@ class MyMigration(Migration): "Migrate from official.json to apps.json" - def migrate(self): + def forward(self): # Backup current app list json os.system("cp %s %s" % (APPSLISTS_JSON, APPSLISTS_BACKUP)) diff --git a/src/yunohost/data_migrations/0011_setup_group_permission.py b/src/yunohost/data_migrations/0011_setup_group_permission.py index 05c426936..0e371944e 100644 --- a/src/yunohost/data_migrations/0011_setup_group_permission.py +++ b/src/yunohost/data_migrations/0011_setup_group_permission.py @@ -92,7 +92,7 @@ class MyMigration(Migration): app_setting(app, 'allowed_users', delete=True) - def migrate(self): + def forward(self): # Check if the migration can be processed ldap_regen_conf_status = regen_conf(names=['slapd'], dry_run=True) # By this we check if the have been customized diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index a6379e62b..59758acdf 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -1316,7 +1316,7 @@ class Migration(object): raise NotImplementedError() def backward(self): - pass + raise YunohostError("migration_backward_impossible", name=self.name) @property def disclaimer(self): From 81ae41a3e4d5b28b6f44c46ad64a76d2e6cc3bfc Mon Sep 17 00:00:00 2001 From: ljf Date: Wed, 14 Aug 2019 12:27:32 +0200 Subject: [PATCH 08/19] [enh] Warn user about --revert option --- data/actionsmap/yunohost.yml | 2 +- src/yunohost/tools.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index f54cc2254..a4dc8d453 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1736,7 +1736,7 @@ tools: help: Skip specified migrations (to be used only if you know what you are doing) action: store_true --revert: - help: Attempt to revert already-ran migrations + help: Attempt to revert already-ran migrations (to be used only if you know what you are doing) action: store_true --force-rerun: help: Re-run already-ran migrations (to be used only if you know what you are doing) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 59758acdf..39333bcaf 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -1046,7 +1046,7 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal --skip Skip specified migrations (to be used only if you know what you are doing) (must explicit which migrations) --auto Automatic mode, won't run manual migrations (to be used only if you know what you are doing) (must explicit which migrations) --force-rerun Re-run already-ran migrations (to be used only if you know what you are doing)(must explicit which migrations) - --revert Attempt to revert already-ran migrations (must explicit which migrations) + --revert Attempt to revert already-ran migrations (to be used only if you know what you are doing)(must explicit which migrations) --accept-disclaimer Accept disclaimers of migrations (please read them before using this option) (only valid for one migration) """ From c81eab396b5d430b268c1581f942eb234148c272 Mon Sep 17 00:00:00 2001 From: ljf Date: Wed, 14 Aug 2019 12:52:47 +0200 Subject: [PATCH 09/19] [enh] Pep8 --- src/yunohost/tools.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 39333bcaf..8c07855a3 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -1060,7 +1060,6 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal raise YunohostError("migrations_no_such_migration", id=target) - # auto, skip, revert and force are exclusive options if auto + skip + revert + force_rerun > 1: raise YunohostError("migrations_exclusive_options") From 7a0dbf79773785634b0e16720ba6067fb2c034d3 Mon Sep 17 00:00:00 2001 From: ljf Date: Wed, 14 Aug 2019 13:05:23 +0200 Subject: [PATCH 10/19] [fix] Ask for specifying migrations to skip --- data/actionsmap/yunohost.yml | 4 ++-- src/yunohost/tools.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index a4dc8d453..054171133 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1736,10 +1736,10 @@ tools: help: Skip specified migrations (to be used only if you know what you are doing) action: store_true --revert: - help: Attempt to revert already-ran migrations (to be used only if you know what you are doing) + help: Attempt to revert already-ran specified migrations (to be used only if you know what you are doing) action: store_true --force-rerun: - help: Re-run already-ran migrations (to be used only if you know what you are doing) + help: Re-run already-ran specified migration (to be used only if you know what you are doing) action: store_true --auto: help: Automatic mode, won't run manual migrations (to be used only if you know what you are doing) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 8c07855a3..9d341ca3d 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -1044,7 +1044,7 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal targets A list migrations to run (all pendings by default) --skip Skip specified migrations (to be used only if you know what you are doing) (must explicit which migrations) - --auto Automatic mode, won't run manual migrations (to be used only if you know what you are doing) (must explicit which migrations) + --auto Automatic mode, won't run manual migrations (to be used only if you know what you are doing) --force-rerun Re-run already-ran migrations (to be used only if you know what you are doing)(must explicit which migrations) --revert Attempt to revert already-ran migrations (to be used only if you know what you are doing)(must explicit which migrations) --accept-disclaimer Accept disclaimers of migrations (please read them before using this option) (only valid for one migration) @@ -1067,7 +1067,7 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal # If no target specified if not targets: # skip, revert or force require explicit targets - if (revert or force_rerun): + if (skip or revert or force_rerun): raise YunohostError("migrations_must_provide_explicit_targets") # Otherwise, targets are all pending migrations From f0c0eff41da7a3a1683d6bf4dc4a028a9865f6e9 Mon Sep 17 00:00:00 2001 From: ljf Date: Wed, 14 Aug 2019 13:13:59 +0200 Subject: [PATCH 11/19] [enh] english sentence in a comment --- src/yunohost/tools.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 9d341ca3d..d84ca3199 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -1095,9 +1095,9 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal for migration in targets: # If we are migrating in "automatic mode" (i.e. from debian configure - # during an upgrade of the package) but we are asked to run migrations - # to be ran manually by the user, stop there and ask the user to - # run the migration manually. + # during an upgrade of the package) but we are asked for running + # migrations to be ran manually by the user, stop there and ask the + # user to run the migration manually. if auto and migration.mode == "manual": logger.warn(m18n.n('migrations_to_be_ran_manually', id=migration.id)) From ed3e014feea6abf17f3baab0f5844510c05dba22 Mon Sep 17 00:00:00 2001 From: ljf Date: Wed, 14 Aug 2019 16:46:14 +0200 Subject: [PATCH 12/19] [enh] Remove revert mechanism --- data/actionsmap/yunohost.yml | 3 - locales/en.json | 2 - .../0001_change_cert_group_to_sslcert.py | 15 +-- .../0002_migrate_to_tsig_sha256.py | 2 +- .../0003_migrate_to_stretch.py | 2 +- .../0004_php5_to_php7_pools.py | 116 +++++++++--------- .../0005_postgresql_9p4_to_9p6.py | 2 +- .../0006_sync_admin_and_root_passwords.py | 2 +- ...0007_ssh_conf_managed_by_yunohost_step1.py | 28 ++--- ...0008_ssh_conf_managed_by_yunohost_step2.py | 2 +- .../0009_decouple_regenconf_from_services.py | 2 +- .../0010_migrate_to_apps_json.py | 28 +++-- .../0011_setup_group_permission.py | 2 +- src/yunohost/tools.py | 50 +++----- 14 files changed, 118 insertions(+), 138 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 054171133..725f14a03 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1735,9 +1735,6 @@ tools: --skip: help: Skip specified migrations (to be used only if you know what you are doing) action: store_true - --revert: - help: Attempt to revert already-ran specified migrations (to be used only if you know what you are doing) - action: store_true --force-rerun: help: Re-run already-ran specified migration (to be used only if you know what you are doing) action: store_true diff --git a/locales/en.json b/locales/en.json index 471f0f9dc..850d89032 100644 --- a/locales/en.json +++ b/locales/en.json @@ -297,7 +297,6 @@ "log_user_permission_remove": "Update '{}' permission", "log_tools_maindomain": "Make '{}' as main domain", "log_tools_migrations_migrate_forward": "Migrate forward", - "log_tools_migrations_migrate_backward": "Migrate backward", "log_tools_postinstall": "Postinstall your YunoHost server", "log_tools_upgrade": "Upgrade system packages", "log_tools_shutdown": "Shutdown your server", @@ -384,7 +383,6 @@ "migrations_not_pending_cant_skip": "Those migrations are not pending so cannot be skipped: {ids}", "migrations_pending_cant_revert_or_rerun": "Those migrations are still pending so cannot be reverted or reran: {ids}", "migrations_running_forward": "Running migration {id}…", - "migrations_running_backward": "Attempting to revert migration {id}…", "migrations_skip_migration": "Skipping migration {id}…", "migrations_success_forward": "Successfully ran migration {id}!", "migrations_success_revert": "Successfully reverted migration {id}!", diff --git a/src/yunohost/data_migrations/0001_change_cert_group_to_sslcert.py b/src/yunohost/data_migrations/0001_change_cert_group_to_sslcert.py index 6485861b7..4338e10f3 100644 --- a/src/yunohost/data_migrations/0001_change_cert_group_to_sslcert.py +++ b/src/yunohost/data_migrations/0001_change_cert_group_to_sslcert.py @@ -10,10 +10,11 @@ class MyMigration(Migration): all_certificate_files = glob.glob("/etc/yunohost/certs/*/*.pem") - def forward(self): - for filename in self.all_certificate_files: - chown(filename, uid="root", gid="ssl-cert") - - def backward(self): - for filename in self.all_certificate_files: - chown(filename, uid="root", gid="metronome") + def run(self): + try: + for filename in self.all_certificate_files: + chown(filename, uid="root", gid="ssl-cert") + except Exception: + for filename in self.all_certificate_files: + chown(filename, uid="root", gid="metronome") + raise diff --git a/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py b/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py index cc5d4c19b..65158ba2c 100644 --- a/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py +++ b/src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py @@ -19,7 +19,7 @@ class MyMigration(Migration): "Migrate Dyndns stuff from MD5 TSIG to SHA512 TSIG" - def migrate(self, dyn_host="dyndns.yunohost.org", domain=None, private_key_path=None): + def run(self, dyn_host="dyndns.yunohost.org", domain=None, private_key_path=None): if domain is None or private_key_path is None: try: diff --git a/src/yunohost/data_migrations/0003_migrate_to_stretch.py b/src/yunohost/data_migrations/0003_migrate_to_stretch.py index 0db6cfd6d..19793bbec 100644 --- a/src/yunohost/data_migrations/0003_migrate_to_stretch.py +++ b/src/yunohost/data_migrations/0003_migrate_to_stretch.py @@ -29,7 +29,7 @@ class MyMigration(Migration): mode = "manual" - def forward(self): + def run(self): self.logfile = "/var/log/yunohost/{}.log".format(self.name) diff --git a/src/yunohost/data_migrations/0004_php5_to_php7_pools.py b/src/yunohost/data_migrations/0004_php5_to_php7_pools.py index e916a9cd7..7bb9123ec 100644 --- a/src/yunohost/data_migrations/0004_php5_to_php7_pools.py +++ b/src/yunohost/data_migrations/0004_php5_to_php7_pools.py @@ -24,77 +24,77 @@ class MyMigration(Migration): dependencies = ["migrate_to_stretch"] - def forward(self): + def run(self): + try: + # Get list of php5 pool files + php5_pool_files = glob.glob("{}/*.conf".format(PHP5_POOLS)) - # Get list of php5 pool files - php5_pool_files = glob.glob("{}/*.conf".format(PHP5_POOLS)) + # Keep only basenames + php5_pool_files = [os.path.basename(f) for f in php5_pool_files] - # Keep only basenames - php5_pool_files = [os.path.basename(f) for f in php5_pool_files] + # Ignore the "www.conf" (default stuff, probably don't want to touch it ?) + php5_pool_files = [f for f in php5_pool_files if f != "www.conf"] - # Ignore the "www.conf" (default stuff, probably don't want to touch it ?) - php5_pool_files = [f for f in php5_pool_files if f != "www.conf"] + for f in php5_pool_files: - for f in php5_pool_files: + # Copy the files to the php7 pool + src = "{}/{}".format(PHP5_POOLS, f) + dest = "{}/{}".format(PHP7_POOLS, f) + copy2(src, dest) - # Copy the files to the php7 pool - src = "{}/{}".format(PHP5_POOLS, f) - dest = "{}/{}".format(PHP7_POOLS, f) - copy2(src, dest) + # Replace the socket prefix if it's found + c = "sed -i -e 's@{}@{}@g' {}".format(PHP5_SOCKETS_PREFIX, PHP7_SOCKETS_PREFIX, dest) + os.system(c) - # Replace the socket prefix if it's found - c = "sed -i -e 's@{}@{}@g' {}".format(PHP5_SOCKETS_PREFIX, PHP7_SOCKETS_PREFIX, dest) - os.system(c) + # Also add a comment that it was automatically moved from php5 + # (for human traceability and backward migration) + c = "sed -i '1i {}' {}".format(MIGRATION_COMMENT, dest) + os.system(c) - # Also add a comment that it was automatically moved from php5 - # (for human traceability and backward migration) - c = "sed -i '1i {}' {}".format(MIGRATION_COMMENT, dest) - os.system(c) + # Some old comments starting with '#' instead of ';' are not + # compatible in php7 + c = "sed -i 's/^#/;#/g' {}".format(dest) + os.system(c) - # Some old comments starting with '#' instead of ';' are not - # compatible in php7 - c = "sed -i 's/^#/;#/g' {}".format(dest) - os.system(c) + # Reload/restart the php pools + _run_service_command("restart", "php7.0-fpm") + _run_service_command("enable", "php7.0-fpm") + os.system("systemctl stop php5-fpm") + os.system("systemctl disable php5-fpm") + os.system("rm /etc/logrotate.d/php5-fpm") # We remove this otherwise the logrotate cron will be unhappy - # Reload/restart the php pools - _run_service_command("restart", "php7.0-fpm") - _run_service_command("enable", "php7.0-fpm") - os.system("systemctl stop php5-fpm") - os.system("systemctl disable php5-fpm") - os.system("rm /etc/logrotate.d/php5-fpm") # We remove this otherwise the logrotate cron will be unhappy + # Get list of nginx conf file + nginx_conf_files = glob.glob("/etc/nginx/conf.d/*.d/*.conf") + for f in nginx_conf_files: + # Replace the socket prefix if it's found + c = "sed -i -e 's@{}@{}@g' {}".format(PHP5_SOCKETS_PREFIX, PHP7_SOCKETS_PREFIX, f) + os.system(c) - # Get list of nginx conf file - nginx_conf_files = glob.glob("/etc/nginx/conf.d/*.d/*.conf") - for f in nginx_conf_files: - # Replace the socket prefix if it's found - c = "sed -i -e 's@{}@{}@g' {}".format(PHP5_SOCKETS_PREFIX, PHP7_SOCKETS_PREFIX, f) - os.system(c) + # Reload nginx + _run_service_command("reload", "nginx") + except Exception: - # Reload nginx - _run_service_command("reload", "nginx") + # Get list of php7 pool files + php7_pool_files = glob.glob("{}/*.conf".format(PHP7_POOLS)) - def backward(self): + # Keep only files which have the migration comment + php7_pool_files = [f for f in php7_pool_files if open(f).readline().strip() == MIGRATION_COMMENT] - # Get list of php7 pool files - php7_pool_files = glob.glob("{}/*.conf".format(PHP7_POOLS)) + # Delete those files + for f in php7_pool_files: + os.remove(f) - # Keep only files which have the migration comment - php7_pool_files = [f for f in php7_pool_files if open(f).readline().strip() == MIGRATION_COMMENT] + # Reload/restart the php pools + _run_service_command("stop", "php7.0-fpm") + os.system("systemctl start php5-fpm") - # Delete those files - for f in php7_pool_files: - os.remove(f) + # Get list of nginx conf file + nginx_conf_files = glob.glob("/etc/nginx/conf.d/*.d/*.conf") + for f in nginx_conf_files: + # Replace the socket prefix if it's found + c = "sed -i -e 's@{}@{}@g' {}".format(PHP7_SOCKETS_PREFIX, PHP5_SOCKETS_PREFIX, f) + os.system(c) - # Reload/restart the php pools - _run_service_command("stop", "php7.0-fpm") - os.system("systemctl start php5-fpm") - - # Get list of nginx conf file - nginx_conf_files = glob.glob("/etc/nginx/conf.d/*.d/*.conf") - for f in nginx_conf_files: - # Replace the socket prefix if it's found - c = "sed -i -e 's@{}@{}@g' {}".format(PHP7_SOCKETS_PREFIX, PHP5_SOCKETS_PREFIX, f) - os.system(c) - - # Reload nginx - _run_service_command("reload", "nginx") + # Reload nginx + _run_service_command("reload", "nginx") + raise diff --git a/src/yunohost/data_migrations/0005_postgresql_9p4_to_9p6.py b/src/yunohost/data_migrations/0005_postgresql_9p4_to_9p6.py index 9cb6e51bd..3127f2c65 100644 --- a/src/yunohost/data_migrations/0005_postgresql_9p4_to_9p6.py +++ b/src/yunohost/data_migrations/0005_postgresql_9p4_to_9p6.py @@ -16,7 +16,7 @@ class MyMigration(Migration): dependencies = ["migrate_to_stretch"] - def forward(self): + def run(self): if not self.package_is_installed("postgresql-9.4"): logger.warning(m18n.n("migration_0005_postgresql_94_not_installed")) diff --git a/src/yunohost/data_migrations/0006_sync_admin_and_root_passwords.py b/src/yunohost/data_migrations/0006_sync_admin_and_root_passwords.py index 953652111..844e3028c 100644 --- a/src/yunohost/data_migrations/0006_sync_admin_and_root_passwords.py +++ b/src/yunohost/data_migrations/0006_sync_admin_and_root_passwords.py @@ -20,7 +20,7 @@ class MyMigration(Migration): "Synchronize admin and root passwords" - def forward(self): + def run(self): new_hash = self._get_admin_hash() self._replace_root_hash(new_hash) diff --git a/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py b/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py index 7ddcec7fd..99eb90b19 100644 --- a/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py +++ b/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py @@ -34,7 +34,7 @@ class MyMigration(Migration): use the recommended configuration, with an appropriate disclaimer. """ - def forward(self): + def run(self): # Check if deprecated DSA Host Key is in config dsa_rgx = r'^[ \t]*HostKey[ \t]+/etc/ssh/ssh_host_dsa_key[ \t]*(?:#.*)?$' @@ -55,19 +55,13 @@ class MyMigration(Migration): # right after the regenconf, such that it will appear as # "manually modified". if os.path.exists('/etc/yunohost/from_script'): - rm('/etc/yunohost/from_script') - copyfile(SSHD_CONF, '/etc/ssh/sshd_config.bkp') - regen_conf(names=['ssh'], force=True) - copyfile('/etc/ssh/sshd_config.bkp', SSHD_CONF) - - # Restart ssh and backward if it fail - if not _run_service_command('restart', 'ssh'): - self.backward() - raise YunohostError("migration_0007_cancel") - - def backward(self): - - # We don't backward completely but it should be enough - copyfile('/etc/ssh/sshd_config.bkp', SSHD_CONF) - if not _run_service_command('restart', 'ssh'): - raise YunohostError("migration_0007_cannot_restart") + try: + rm('/etc/yunohost/from_script') + copyfile(SSHD_CONF, '/etc/ssh/sshd_config.bkp') + regen_conf(names=['ssh'], force=True) + copyfile('/etc/ssh/sshd_config.bkp', SSHD_CONF) + except Exception: + if os.path.exists('/etc/yunohost/sshd_config.bkp'): + copyfile('/etc/ssh/sshd_config.bkp', SSHD_CONF) + _run_service_command('restart', 'ssh') + raise diff --git a/src/yunohost/data_migrations/0008_ssh_conf_managed_by_yunohost_step2.py b/src/yunohost/data_migrations/0008_ssh_conf_managed_by_yunohost_step2.py index c158192f9..ecc8cfdcb 100644 --- a/src/yunohost/data_migrations/0008_ssh_conf_managed_by_yunohost_step2.py +++ b/src/yunohost/data_migrations/0008_ssh_conf_managed_by_yunohost_step2.py @@ -35,7 +35,7 @@ class MyMigration(Migration): dependencies = ["ssh_conf_managed_by_yunohost_step1"] - def forward(self): + def run(self): settings_set("service.ssh.allow_deprecated_dsa_hostkey", False) regen_conf(names=['ssh'], force=True) diff --git a/src/yunohost/data_migrations/0009_decouple_regenconf_from_services.py b/src/yunohost/data_migrations/0009_decouple_regenconf_from_services.py index ffe2279a8..c190e2aaa 100644 --- a/src/yunohost/data_migrations/0009_decouple_regenconf_from_services.py +++ b/src/yunohost/data_migrations/0009_decouple_regenconf_from_services.py @@ -17,7 +17,7 @@ class MyMigration(Migration): Decouple the regen conf mechanism from the concept of services """ - def forward(self): + def run(self): if "conffiles" not in read_file("/etc/yunohost/services.yml") \ or os.path.exists(REGEN_CONF_FILE): diff --git a/src/yunohost/data_migrations/0010_migrate_to_apps_json.py b/src/yunohost/data_migrations/0010_migrate_to_apps_json.py index 7092d92c7..fbcec5117 100644 --- a/src/yunohost/data_migrations/0010_migrate_to_apps_json.py +++ b/src/yunohost/data_migrations/0010_migrate_to_apps_json.py @@ -15,7 +15,7 @@ class MyMigration(Migration): "Migrate from official.json to apps.json" - def forward(self): + def run(self): # Backup current app list json os.system("cp %s %s" % (APPSLISTS_JSON, APPSLISTS_BACKUP)) @@ -28,17 +28,21 @@ class MyMigration(Migration): "labriqueinter.net/apps/labriqueinternet.json", "labriqueinter.net/internetcube.json" ] + try: + appslists = _read_appslist_list() + for appslist, infos in appslists.items(): + if infos["url"].split("//")[-1] in lists_to_remove: + app_removelist(name=appslist) - appslists = _read_appslist_list() - for appslist, infos in appslists.items(): - if infos["url"].split("//")[-1] in lists_to_remove: - app_removelist(name=appslist) + # Replace by apps.json list + app_fetchlist(name="yunohost", + url="https://app.yunohost.org/apps.json") + except Exception: + if os.path.exists(APPSLISTS_BACKUP): + os.system("cp %s %s" % (APPSLISTS_BACKUP, APPSLISTS_JSON)) + raise + else: + if os.path.exists(APPSLISTS_BACKUP): + os.remove(APPSLISTS_BACKUP) - # Replace by apps.json list - app_fetchlist(name="yunohost", - url="https://app.yunohost.org/apps.json") - def backward(self): - - if os.path.exists(APPSLISTS_BACKUP): - os.system("cp %s %s" % (APPSLISTS_BACKUP, APPSLISTS_JSON)) diff --git a/src/yunohost/data_migrations/0011_setup_group_permission.py b/src/yunohost/data_migrations/0011_setup_group_permission.py index 0e371944e..17fd8f4a5 100644 --- a/src/yunohost/data_migrations/0011_setup_group_permission.py +++ b/src/yunohost/data_migrations/0011_setup_group_permission.py @@ -92,7 +92,7 @@ class MyMigration(Migration): app_setting(app, 'allowed_users', delete=True) - def forward(self): + def run(self): # Check if the migration can be processed ldap_regen_conf_status = regen_conf(names=['slapd'], dry_run=True) # By this we check if the have been customized diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index d84ca3199..1c185e90c 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -1038,7 +1038,7 @@ def tools_migrations_list(pending=False, done=False): return {"migrations": migrations} -def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=False, revert=False, accept_disclaimer=False): +def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=False, accept_disclaimer=False): """ Perform migrations @@ -1046,7 +1046,6 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal --skip Skip specified migrations (to be used only if you know what you are doing) (must explicit which migrations) --auto Automatic mode, won't run manual migrations (to be used only if you know what you are doing) --force-rerun Re-run already-ran migrations (to be used only if you know what you are doing)(must explicit which migrations) - --revert Attempt to revert already-ran migrations (to be used only if you know what you are doing)(must explicit which migrations) --accept-disclaimer Accept disclaimers of migrations (please read them before using this option) (only valid for one migration) """ @@ -1060,14 +1059,14 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal raise YunohostError("migrations_no_such_migration", id=target) - # auto, skip, revert and force are exclusive options - if auto + skip + revert + force_rerun > 1: + # auto, skip and force are exclusive options + if auto + skip + force_rerun > 1: raise YunohostError("migrations_exclusive_options") # If no target specified if not targets: # skip, revert or force require explicit targets - if (skip or revert or force_rerun): + if (skip or force_rerun): raise YunohostError("migrations_must_provide_explicit_targets") # Otherwise, targets are all pending migrations @@ -1081,9 +1080,9 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal if skip and done: raise YunohostError("migrations_not_pending_cant_skip", ids=', '.join(done)) - if (revert or force_rerun) and pending: + if force_rerun and pending: raise YunohostError("migrations_pending_cant_revert_or_rerun", ids=', '.join(pending)) - if not (skip or revert or force_rerun) and done: + if not (skip or force_rerun) and done: raise YunohostError("migrations_already_ran", ids=', '.join(done)) # So, is there actually something to do ? @@ -1105,7 +1104,7 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal continue # Check for migration dependencies - if not revert and not skip: + if not skip: dependencies = [get_matching_migration(dep) for dep in migration.dependencies] pending_dependencies = [dep.id for dep in dependencies if dep.state == "pending"] if pending_dependencies: @@ -1115,7 +1114,7 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal continue # If some migrations have disclaimers (and we're not trying to skip them) - if migration.disclaimer and not skip and not revert: + if migration.disclaimer and not skip: # require the --accept-disclaimer option. # Otherwise, go to the next migration if not accept_disclaimer: @@ -1128,8 +1127,7 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal accept_disclaimer = False # Start register change on system - mode = "backward" if revert else "forward" - operation_logger = OperationLogger('tools_migrations_migrate_' + mode) + operation_logger = OperationLogger('tools_migrations_migrate_forward') operation_logger.start() if skip: @@ -1141,12 +1139,8 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal try: migration.operation_logger = operation_logger - if revert: - logger.info(m18n.n('migrations_running_backward', id=migration.id)) - migration.backward() - else: logger.info(m18n.n('migrations_running_forward', id=migration.id)) - migration.migrate() + migration.run() except Exception as e: # migration failed, let's stop here but still update state because # we managed to run the previous ones @@ -1155,14 +1149,9 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal logger.error(msg, exc_info=1) operation_logger.error(msg) else: - if revert: - logger.success(m18n.n('migrations_success_revert', id=migration.id)) - migration.state = "pending" - _write_migration_state(migration.id, "pending") - else: - logger.success(m18n.n('migrations_success_forward', id=migration.id)) - migration.state = "done" - _write_migration_state(migration.id, "done") + logger.success(m18n.n('migrations_success_forward', id=migration.id)) + migration.state = "done" + _write_migration_state(migration.id, "done") operation_logger.success() @@ -1304,6 +1293,9 @@ def _skip_all_migrations(): write_to_yaml(MIGRATIONS_STATE_PATH, new_states) +def _get_revert_dependencies(migration, all_migrations): + + class Migration(object): # Those are to be implemented by daughter classes @@ -1311,20 +1303,14 @@ class Migration(object): mode = "auto" dependencies = [] # List of migration ids required before running this migration - def forward(self): - raise NotImplementedError() - - def backward(self): - raise YunohostError("migration_backward_impossible", name=self.name) - @property def disclaimer(self): return None # The followings shouldn't be overriden - def migrate(self): - self.forward() + def run(self): + raise NotImplementedError() def __init__(self, id_): self.id = id_ From fac44f44d06f120625cbc69fefbb3e28f57093d2 Mon Sep 17 00:00:00 2001 From: ljf Date: Wed, 14 Aug 2019 17:29:24 +0200 Subject: [PATCH 13/19] [enh] Avoid to modify untestable migrations --- .../0001_change_cert_group_to_sslcert.py | 9 ++------- .../0007_ssh_conf_managed_by_yunohost_step1.py | 14 ++++---------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/src/yunohost/data_migrations/0001_change_cert_group_to_sslcert.py b/src/yunohost/data_migrations/0001_change_cert_group_to_sslcert.py index 4338e10f3..5670f3329 100644 --- a/src/yunohost/data_migrations/0001_change_cert_group_to_sslcert.py +++ b/src/yunohost/data_migrations/0001_change_cert_group_to_sslcert.py @@ -11,10 +11,5 @@ class MyMigration(Migration): all_certificate_files = glob.glob("/etc/yunohost/certs/*/*.pem") def run(self): - try: - for filename in self.all_certificate_files: - chown(filename, uid="root", gid="ssl-cert") - except Exception: - for filename in self.all_certificate_files: - chown(filename, uid="root", gid="metronome") - raise + for filename in self.all_certificate_files: + chown(filename, uid="root", gid="ssl-cert") diff --git a/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py b/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py index 99eb90b19..818760e17 100644 --- a/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py +++ b/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py @@ -55,13 +55,7 @@ class MyMigration(Migration): # right after the regenconf, such that it will appear as # "manually modified". if os.path.exists('/etc/yunohost/from_script'): - try: - rm('/etc/yunohost/from_script') - copyfile(SSHD_CONF, '/etc/ssh/sshd_config.bkp') - regen_conf(names=['ssh'], force=True) - copyfile('/etc/ssh/sshd_config.bkp', SSHD_CONF) - except Exception: - if os.path.exists('/etc/yunohost/sshd_config.bkp'): - copyfile('/etc/ssh/sshd_config.bkp', SSHD_CONF) - _run_service_command('restart', 'ssh') - raise + rm('/etc/yunohost/from_script') + copyfile(SSHD_CONF, '/etc/ssh/sshd_config.bkp') + regen_conf(names=['ssh'], force=True) + copyfile('/etc/ssh/sshd_config.bkp', SSHD_CONF) From 0a2ba102c883c0a5b96cf451ab8a7a0c6fedb0dd Mon Sep 17 00:00:00 2001 From: ljf Date: Wed, 14 Aug 2019 17:37:43 +0200 Subject: [PATCH 14/19] [enh] Avoid to modify untestable migrations --- .../0004_php5_to_php7_pools.py | 77 +++++++++---------- 1 file changed, 38 insertions(+), 39 deletions(-) diff --git a/src/yunohost/data_migrations/0004_php5_to_php7_pools.py b/src/yunohost/data_migrations/0004_php5_to_php7_pools.py index 7bb9123ec..3bc6bacc4 100644 --- a/src/yunohost/data_migrations/0004_php5_to_php7_pools.py +++ b/src/yunohost/data_migrations/0004_php5_to_php7_pools.py @@ -25,54 +25,54 @@ class MyMigration(Migration): dependencies = ["migrate_to_stretch"] def run(self): - try: - # Get list of php5 pool files - php5_pool_files = glob.glob("{}/*.conf".format(PHP5_POOLS)) + # Get list of php5 pool files + php5_pool_files = glob.glob("{}/*.conf".format(PHP5_POOLS)) - # Keep only basenames - php5_pool_files = [os.path.basename(f) for f in php5_pool_files] + # Keep only basenames + php5_pool_files = [os.path.basename(f) for f in php5_pool_files] - # Ignore the "www.conf" (default stuff, probably don't want to touch it ?) - php5_pool_files = [f for f in php5_pool_files if f != "www.conf"] + # Ignore the "www.conf" (default stuff, probably don't want to touch it ?) + php5_pool_files = [f for f in php5_pool_files if f != "www.conf"] - for f in php5_pool_files: + for f in php5_pool_files: - # Copy the files to the php7 pool - src = "{}/{}".format(PHP5_POOLS, f) - dest = "{}/{}".format(PHP7_POOLS, f) - copy2(src, dest) + # Copy the files to the php7 pool + src = "{}/{}".format(PHP5_POOLS, f) + dest = "{}/{}".format(PHP7_POOLS, f) + copy2(src, dest) - # Replace the socket prefix if it's found - c = "sed -i -e 's@{}@{}@g' {}".format(PHP5_SOCKETS_PREFIX, PHP7_SOCKETS_PREFIX, dest) - os.system(c) + # Replace the socket prefix if it's found + c = "sed -i -e 's@{}@{}@g' {}".format(PHP5_SOCKETS_PREFIX, PHP7_SOCKETS_PREFIX, dest) + os.system(c) - # Also add a comment that it was automatically moved from php5 - # (for human traceability and backward migration) - c = "sed -i '1i {}' {}".format(MIGRATION_COMMENT, dest) - os.system(c) + # Also add a comment that it was automatically moved from php5 + # (for human traceability and backward migration) + c = "sed -i '1i {}' {}".format(MIGRATION_COMMENT, dest) + os.system(c) - # Some old comments starting with '#' instead of ';' are not - # compatible in php7 - c = "sed -i 's/^#/;#/g' {}".format(dest) - os.system(c) + # Some old comments starting with '#' instead of ';' are not + # compatible in php7 + c = "sed -i 's/^#/;#/g' {}".format(dest) + os.system(c) - # Reload/restart the php pools - _run_service_command("restart", "php7.0-fpm") - _run_service_command("enable", "php7.0-fpm") - os.system("systemctl stop php5-fpm") - os.system("systemctl disable php5-fpm") - os.system("rm /etc/logrotate.d/php5-fpm") # We remove this otherwise the logrotate cron will be unhappy + # Reload/restart the php pools + _run_service_command("restart", "php7.0-fpm") + _run_service_command("enable", "php7.0-fpm") + os.system("systemctl stop php5-fpm") + os.system("systemctl disable php5-fpm") + os.system("rm /etc/logrotate.d/php5-fpm") # We remove this otherwise the logrotate cron will be unhappy - # Get list of nginx conf file - nginx_conf_files = glob.glob("/etc/nginx/conf.d/*.d/*.conf") - for f in nginx_conf_files: - # Replace the socket prefix if it's found - c = "sed -i -e 's@{}@{}@g' {}".format(PHP5_SOCKETS_PREFIX, PHP7_SOCKETS_PREFIX, f) - os.system(c) + # Get list of nginx conf file + nginx_conf_files = glob.glob("/etc/nginx/conf.d/*.d/*.conf") + for f in nginx_conf_files: + # Replace the socket prefix if it's found + c = "sed -i -e 's@{}@{}@g' {}".format(PHP5_SOCKETS_PREFIX, PHP7_SOCKETS_PREFIX, f) + os.system(c) - # Reload nginx - _run_service_command("reload", "nginx") - except Exception: + # Reload nginx + _run_service_command("reload", "nginx") + + def backward(self): # Get list of php7 pool files php7_pool_files = glob.glob("{}/*.conf".format(PHP7_POOLS)) @@ -97,4 +97,3 @@ class MyMigration(Migration): # Reload nginx _run_service_command("reload", "nginx") - raise From f98ad4f191d7ac7aea84a17df09c36c7d2538e70 Mon Sep 17 00:00:00 2001 From: ljf Date: Wed, 14 Aug 2019 17:39:23 +0200 Subject: [PATCH 15/19] [enh] Avoid to modify untestable migrations --- .../0004_php5_to_php7_pools.py | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/yunohost/data_migrations/0004_php5_to_php7_pools.py b/src/yunohost/data_migrations/0004_php5_to_php7_pools.py index 3bc6bacc4..1b90c4ff0 100644 --- a/src/yunohost/data_migrations/0004_php5_to_php7_pools.py +++ b/src/yunohost/data_migrations/0004_php5_to_php7_pools.py @@ -72,28 +72,28 @@ class MyMigration(Migration): # Reload nginx _run_service_command("reload", "nginx") - def backward(self): + def backward(self): - # Get list of php7 pool files - php7_pool_files = glob.glob("{}/*.conf".format(PHP7_POOLS)) + # Get list of php7 pool files + php7_pool_files = glob.glob("{}/*.conf".format(PHP7_POOLS)) - # Keep only files which have the migration comment - php7_pool_files = [f for f in php7_pool_files if open(f).readline().strip() == MIGRATION_COMMENT] + # Keep only files which have the migration comment + php7_pool_files = [f for f in php7_pool_files if open(f).readline().strip() == MIGRATION_COMMENT] - # Delete those files - for f in php7_pool_files: - os.remove(f) + # Delete those files + for f in php7_pool_files: + os.remove(f) - # Reload/restart the php pools - _run_service_command("stop", "php7.0-fpm") - os.system("systemctl start php5-fpm") + # Reload/restart the php pools + _run_service_command("stop", "php7.0-fpm") + os.system("systemctl start php5-fpm") - # Get list of nginx conf file - nginx_conf_files = glob.glob("/etc/nginx/conf.d/*.d/*.conf") - for f in nginx_conf_files: - # Replace the socket prefix if it's found - c = "sed -i -e 's@{}@{}@g' {}".format(PHP7_SOCKETS_PREFIX, PHP5_SOCKETS_PREFIX, f) - os.system(c) + # Get list of nginx conf file + nginx_conf_files = glob.glob("/etc/nginx/conf.d/*.d/*.conf") + for f in nginx_conf_files: + # Replace the socket prefix if it's found + c = "sed -i -e 's@{}@{}@g' {}".format(PHP7_SOCKETS_PREFIX, PHP5_SOCKETS_PREFIX, f) + os.system(c) - # Reload nginx - _run_service_command("reload", "nginx") + # Reload nginx + _run_service_command("reload", "nginx") From b54266cbf3ffb5f3b38ba3280474b01ef1aedcb5 Mon Sep 17 00:00:00 2001 From: ljf Date: Wed, 14 Aug 2019 17:55:17 +0200 Subject: [PATCH 16/19] [fix] Unwanted residual code --- src/yunohost/tools.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 1c185e90c..d2015adff 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -1139,8 +1139,8 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal try: migration.operation_logger = operation_logger - logger.info(m18n.n('migrations_running_forward', id=migration.id)) - migration.run() + logger.info(m18n.n('migrations_running_forward', id=migration.id)) + migration.run() except Exception as e: # migration failed, let's stop here but still update state because # we managed to run the previous ones @@ -1293,9 +1293,6 @@ def _skip_all_migrations(): write_to_yaml(MIGRATIONS_STATE_PATH, new_states) -def _get_revert_dependencies(migration, all_migrations): - - class Migration(object): # Those are to be implemented by daughter classes From 92fa21641bfa8c3c323fe87a4dbb27086a9c8b38 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 9 Sep 2019 12:07:44 +0200 Subject: [PATCH 17/19] Fix some stuff about removing the --revert option --- locales/en.json | 8 +++----- .../0007_ssh_conf_managed_by_yunohost_step1.py | 9 +++++++++ src/yunohost/tools.py | 6 +++--- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/locales/en.json b/locales/en.json index 850d89032..a4596941f 100644 --- a/locales/en.json +++ b/locales/en.json @@ -320,7 +320,6 @@ "migrate_tsig_wait_3": "1min…", "migrate_tsig_wait_4": "30 secondes…", "migrate_tsig_not_needed": "You do not appear to use a dyndns domain, so no migration is needed!", - "migration_backward_impossible": "The migration {name} cannot be reverted.", "migration_description_0001_change_cert_group_to_sslcert": "Change certificates group permissions from 'metronome' to 'ssl-cert'", "migration_description_0002_migrate_to_tsig_sha256": "Improve security of dyndns TSIG by using SHA512 instead of MD5", "migration_description_0003_migrate_to_stretch": "Upgrade the system to Debian Stretch and YunoHost 3.0", @@ -372,20 +371,19 @@ "migrations_cant_reach_migration_file": "Can't access migrations files at path %s", "migrations_dependencies_not_satisfied": "Can't run migration {id} because first you need to run these migrations: {dependencies_id}", "migrations_failed_to_load_migration": "Failed to load migration {id} : {error}", - "migrations_exclusive_options": "--auto, --skip, --revert and --force-rerun are exclusive options.", + "migrations_exclusive_options": "--auto, --skip, and --force-rerun are exclusive options.", "migrations_list_conflict_pending_done": "You cannot use both --previous and --done at the same time.", "migrations_loading_migration": "Loading migration {id}…", "migrations_migration_has_failed": "Migration {id} has failed, aborting. Error: {exception}", - "migrations_must_provide_explicit_targets": "You must provide explicit targets when using --skip, --revert or --force-rerun", + "migrations_must_provide_explicit_targets": "You must provide explicit targets when using --skip or --force-rerun", "migrations_need_to_accept_disclaimer": "To run the migration {id}, your must accept the following disclaimer:\n---\n{disclaimer}\n---\nIf you accept to run the migration, please re-run the command with the option --accept-disclaimer.", "migrations_no_migrations_to_run": "No migrations to run", "migrations_no_such_migration": "No such migration called {id}", "migrations_not_pending_cant_skip": "Those migrations are not pending so cannot be skipped: {ids}", - "migrations_pending_cant_revert_or_rerun": "Those migrations are still pending so cannot be reverted or reran: {ids}", + "migrations_pending_cant_rerun": "Those migrations are still pending so cannot be reran: {ids}", "migrations_running_forward": "Running migration {id}…", "migrations_skip_migration": "Skipping migration {id}…", "migrations_success_forward": "Successfully ran migration {id}!", - "migrations_success_revert": "Successfully reverted migration {id}!", "migrations_to_be_ran_manually": "Migration {id} has to be ran manually. Please go to Tools > Migrations on the webadmin, or run `yunohost tools migrations migrate`.", "monitor_disabled": "The server monitoring has been disabled", "monitor_enabled": "The server monitoring has been enabled", diff --git a/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py b/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py index 818760e17..624288210 100644 --- a/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py +++ b/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py @@ -59,3 +59,12 @@ class MyMigration(Migration): copyfile(SSHD_CONF, '/etc/ssh/sshd_config.bkp') regen_conf(names=['ssh'], force=True) copyfile('/etc/ssh/sshd_config.bkp', SSHD_CONF) + + # Restart ssh and rollback if it failed + if not _run_service_command('restart', 'ssh'): + # We don't rollback completely but it should be enough + copyfile('/etc/ssh/sshd_config.bkp', SSHD_CONF) + if not _run_service_command('restart', 'ssh'): + raise YunohostError("migration_0007_cannot_restart") + else: + raise YunohostError("migration_0007_cancelled") diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index d2015adff..8e2635e2c 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -1081,7 +1081,7 @@ def tools_migrations_migrate(targets=[], skip=False, auto=False, force_rerun=Fal if skip and done: raise YunohostError("migrations_not_pending_cant_skip", ids=', '.join(done)) if force_rerun and pending: - raise YunohostError("migrations_pending_cant_revert_or_rerun", ids=', '.join(pending)) + raise YunohostError("migrations_pending_cant_rerun", ids=', '.join(pending)) if not (skip or force_rerun) and done: raise YunohostError("migrations_already_ran", ids=', '.join(done)) @@ -1304,11 +1304,11 @@ class Migration(object): def disclaimer(self): return None - # The followings shouldn't be overriden - def run(self): raise NotImplementedError() + # The followings shouldn't be overriden + def __init__(self, id_): self.id = id_ self.number = int(self.id.split("_", 1)[0]) From b29731c28b2688547dbbd518cd3897cdede94102 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 10 Sep 2019 14:25:51 +0200 Subject: [PATCH 18/19] We don't need this try/except --- src/yunohost/tools.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 8e2635e2c..7b2ee8526 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -1180,11 +1180,7 @@ def _migrate_legacy_migration_json(): last_run_migration_id = str(old_state["number"]) + "_" + old_state["name"] # Extract the list of migration ids - try: - from . import data_migrations - except ImportError: - # Well ugh what are we supposed to do if we can't do this >.>... - pass + from . import data_migrations migrations_path = data_migrations.__path__[0] migration_files = filter(lambda x: re.match("^\d+_[a-zA-Z0-9_]+\.py$", x), os.listdir(migrations_path)) # (here we remove the .py extension and make sure the ids are sorted) From ec3856a25c532b8b552a428c869bc8a47000b1de Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 10 Sep 2019 17:15:01 +0200 Subject: [PATCH 19/19] Propagate changes about revert mechanism and naming on migration 12 --- .../0012_postgresql_password_to_md5_authentication.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/yunohost/data_migrations/0012_postgresql_password_to_md5_authentication.py b/src/yunohost/data_migrations/0012_postgresql_password_to_md5_authentication.py index 5d36b3e23..636b4f12b 100644 --- a/src/yunohost/data_migrations/0012_postgresql_password_to_md5_authentication.py +++ b/src/yunohost/data_migrations/0012_postgresql_password_to_md5_authentication.py @@ -10,12 +10,7 @@ class MyMigration(Migration): all_hba_files = glob.glob("/etc/postgresql/*/*/pg_hba.conf") - def forward(self): + def run(self): for filename in self.all_hba_files: pg_hba_in = read_file(filename) write_to_file(filename, re.sub(r"local(\s*)all(\s*)all(\s*)password", "local\\1all\\2all\\3md5", pg_hba_in)) - - def backward(self): - for filename in self.all_hba_files: - pg_hba_in = read_file(filename) - write_to_file(filename, re.sub(r"local(\s*)all(\s*)all(\s*)md5", "local\\1all\\2all\\3password", pg_hba_in))