Merge pull request #429 from YunoHost/manual-migrations

[enh] Manual migrations

data/actionsmap/yunohost.yml

@@ -1555,6 +1555,13 @@ tools:
               list:
                   action_help: List migrations
                   api: GET /migrations
+                  arguments:
+                    --pending:
+                        help: list only pending migrations
+                        action: store_true
+                    --done:
+                        help: list only migrations already performed
+                        action: store_true
 
               ### tools_migrations_migrate()
               migrate:
@@ -1569,7 +1576,12 @@ tools:
                         help: skip the migration(s), use it only if you know what you are doing
                         full: --skip
                         action: store_true
-
+                    --auto:
+                        help: automatic mode, won't run manual migrations, use it only if you know what you are doing
+                        action: store_true
+                    --accept-disclaimer:
+                        help: accept disclaimers of migration (please read them before using this option)
+                        action: store_true
 
               ### tools_migrations_state()
               state:

debian/postinst

@@ -15,7 +15,7 @@ do_configure() {
       yunohost service regen-conf --output-as none
 
       echo "Launching migrations.."
-      yunohost tools migrations migrate
+      yunohost tools migrations migrate --auto
 
       # restart yunohost-firewall if it's running
       service yunohost-firewall status >/dev/null \

locales/en.json

@@ -222,18 +222,23 @@
     "migrate_tsig_wait_3": "1min...",
     "migrate_tsig_wait_4": "30 secondes...",
     "migrate_tsig_not_needed": "You do not appear to use a dyndns domain, so no migration is needed !",
+    "migration_description_0001_change_cert_group_to_sslcert": "Change certificates group permissions from 'metronome' to 'ssl-cert'",
+    "migration_description_0002_migrate_to_tsig_sha256": "Improve security of dyndns TSIG by using SHA512 instead of MD5",
     "migrations_backward": "Migrating backward.",
-    "migrations_bad_value_for_target": "Invalide number for target argument, available migrations numbers are 0 or {}",
+    "migrations_bad_value_for_target": "Invalid number for target argument, available migrations numbers are 0 or {}",
     "migrations_cant_reach_migration_file": "Can't access migrations files at path %s",
     "migrations_current_target": "Migration target is {}",
     "migrations_error_failed_to_load_migration": "ERROR: failed to load migration {number} {name}",
     "migrations_forward": "Migrating forward",
+    "migrations_list_conflict_pending_done": "You cannot use both --previous and --done at the same time.",
     "migrations_loading_migration": "Loading migration {number} {name}...",
     "migrations_migration_has_failed": "Migration {number} {name} has failed with exception {exception}, aborting",
     "migrations_no_migrations_to_run": "No migrations to run",
     "migrations_show_currently_running_migration": "Running migration {number} {name}...",
     "migrations_show_last_migration": "Last ran migration is {}",
     "migrations_skip_migration": "Skipping migration {number} {name}...",
+    "migrations_to_be_ran_manually":  "Migration {number} {name} has to be ran manually. Please go to Tools > Migrations on the webadmin, or run `yunohost tools migrations migrate`.",
+    "migrations_need_to_accept_disclaimer": "To run the migration {number} {name}, your must accept the following disclaimer:\n---\n{disclaimer}\n---\nIf you accept to run the migration, please re-run the command with the option --accept-disclaimer.",
     "monitor_disabled": "The server monitoring has been disabled",
     "monitor_enabled": "The server monitoring has been enabled",
     "monitor_glances_con_failed": "Unable to connect to Glances server",

src/yunohost/dyndns.py

@@ -232,10 +232,13 @@ def dyndns_update(dyn_host="dyndns.yunohost.org", domain=None, key=None,
         from yunohost.tools import _get_migration_by_name
         migration = _get_migration_by_name("migrate_to_tsig_sha256")
         try:
-            migration["module"].MyMigration().migrate(dyn_host, domain, key)
+            migration.migrate(dyn_host, domain, key)
         except Exception as e:
-            logger.error(m18n.n('migrations_migration_has_failed', exception=e, **migration), exc_info=1)
+            logger.error(m18n.n('migrations_migration_has_failed',
-
+                                exception=e,
+                                number=migration.number,
+                                name=migration.name),
+                                exc_info=1)
         return
 
     # Extract 'host', e.g. 'nohost.me' from 'foo.nohost.me'

src/yunohost/tools.py

@@ -395,7 +395,7 @@ def tools_postinstall(domain, password, ignore_dyndns=False):
     _install_appslist_fetch_cron()
 
     # Init migrations (skip them, no need to run them on a fresh system)
-    tools_migrations_migrate(skip=True)
+    tools_migrations_migrate(skip=True, auto=True)
 
     os.system('touch /etc/yunohost/installed')
 
@@ -732,24 +732,39 @@ def tools_reboot(force=False):
         subprocess.check_call(['systemctl', 'reboot'])
 
 
-def tools_migrations_list():
+def tools_migrations_list(pending=False, done=False):
     """
     List existing migrations
     """
 
-    migrations = {"migrations": []}
+    # Check for option conflict
+    if pending and done:
+        raise MoulinetteError(errno.EINVAL, m18n.n("migrations_list_conflict_pending_done"))
 
-    for migration in _get_migrations_list():
+    # Get all migrations
-        migrations["migrations"].append({
+    migrations = _get_migrations_list()
-            "number": int(migration.split("_", 1)[0]),
-            "name": migration.split("_", 1)[1],
-            "file_name": migration,
-        })
 
-    return migrations
+    # If asked, filter pending or done migrations
+    if pending or done:
+        last_migration = tools_migrations_state()["last_run_migration"]
+        last_migration = last_migration["number"] if last_migration else -1
+        if done:
+            migrations = [m for m in migrations if m.number <= last_migration]
+        if pending:
+            migrations = [m for m in migrations if m.number > last_migration]
+
+    # Reduce to dictionnaries
+    migrations = [{ "id": migration.id,
+                    "number": migration.number,
+                    "name": migration.name,
+                    "mode": migration.mode,
+                    "description": migration.description,
+                    "disclaimer": migration.disclaimer } for migration in migrations ]
+
+    return {"migrations": migrations}
 
 
-def tools_migrations_migrate(target=None, skip=False):
+def tools_migrations_migrate(target=None, skip=False, auto=False, accept_disclaimer=False):
     """
     Perform migrations
     """
@@ -766,26 +781,18 @@ def tools_migrations_migrate(target=None, skip=False):
 
     last_run_migration_number = state["last_run_migration"]["number"] if state["last_run_migration"] else 0
 
-    migrations = []
+    # load all migrations
-
+    migrations = _get_migrations_list()
-    # loading all migrations
+    migrations = sorted(migrations, key=lambda x: x.number)
-    for migration in tools_migrations_list()["migrations"]:
-        migrations.append({
-            "number": migration["number"],
-            "name": migration["name"],
-            "module": _get_migration_module(migration),
-        })
-
-    migrations = sorted(migrations, key=lambda x: x["number"])
 
     if not migrations:
         logger.info(m18n.n('migrations_no_migrations_to_run'))
         return
 
-    all_migration_numbers = [x["number"] for x in migrations]
+    all_migration_numbers = [x.number for x in migrations]
 
     if target is None:
-        target = migrations[-1]["number"]
+        target = migrations[-1].number
 
     # validate input, target must be "0" or a valid number
     elif target != 0 and target not in all_migration_numbers:
@@ -804,44 +811,74 @@ def tools_migrations_migrate(target=None, skip=False):
     if last_run_migration_number < target:
         logger.debug(m18n.n('migrations_forward'))
         # drop all already run migrations
-        migrations = filter(lambda x: target >= x["number"] > last_run_migration_number, migrations)
+        migrations = filter(lambda x: target >= x.number > last_run_migration_number, migrations)
         mode = "forward"
 
     # we need to go backward on already run migrations
     elif last_run_migration_number > target:
         logger.debug(m18n.n('migrations_backward'))
         # drop all not already run migrations
-        migrations = filter(lambda x: target < x["number"] <= last_run_migration_number, migrations)
+        migrations = filter(lambda x: target < x.number <= last_run_migration_number, migrations)
         mode = "backward"
 
     else:  # can't happen, this case is handle before
         raise Exception()
 
+    # If we are migrating in "automatic mode" (i.e. from debian
+    # configure during an upgrade of the package) but we are asked to run
+    # migrations is to be ran manually by the user
+    manual_migrations = [m for m in migrations if m.mode == "manual"]
+    if not skip and auto and manual_migrations:
+        for m in manual_migrations:
+            logger.warn(m18n.n('migrations_to_be_ran_manually',
+                               number=m.number,
+                               name=m.name))
+        return
+
+    # If some migrations have disclaimers, require the --accept-disclaimer
+    # option
+    migrations_with_disclaimer = [m for m in migrations if m.disclaimer]
+    if not skip and not accept_disclaimer and migrations_with_disclaimer:
+        for m in migrations_with_disclaimer:
+            logger.warn(m18n.n('migrations_need_to_accept_disclaimer',
+                               number=m.number,
+                               name=m.name,
+                               disclaimer=m.disclaimer))
+        return
+
     # effectively run selected migrations
     for migration in migrations:
         if not skip:
-            logger.warn(m18n.n('migrations_show_currently_running_migration', **migration))
+
+            logger.warn(m18n.n('migrations_show_currently_running_migration',
+                               number=migration.number, name=migration.name))
 
             try:
                 if mode == "forward":
-                    migration["module"].MyMigration().migrate()
+                    migration.migrate()
                 elif mode == "backward":
-                    migration["module"].MyMigration().backward()
+                    migration.backward()
                 else:  # can't happen
                     raise Exception("Illegal state for migration: '%s', should be either 'forward' or 'backward'" % mode)
             except Exception as e:
                 # migration failed, let's stop here but still update state because
                 # we managed to run the previous ones
-                logger.error(m18n.n('migrations_migration_has_failed', exception=e, **migration), exc_info=1)
+                logger.error(m18n.n('migrations_migration_has_failed',
+                                    exception=e,
+                                    number=migration.number,
+                                    name=migration.name),
+                                    exc_info=1)
                 break
 
         else:  # if skip
-            logger.warn(m18n.n('migrations_skip_migration', **migration))
+            logger.warn(m18n.n('migrations_skip_migration',
+                               number=migration.number,
+                               name=migration.name))
 
         # update the state to include the latest run migration
         state["last_run_migration"] = {
-            "number": migration["number"],
+            "number": migration.number,
-            "name": migration["name"],
+            "name": migration.name
         }
 
     # special case where we want to go back from the start
@@ -904,60 +941,79 @@ def _get_migrations_list():
         logger.warn(m18n.n('migrations_cant_reach_migration_file', migrations_path))
         return migrations
 
-    for migration in filter(lambda x: re.match("^\d+_[a-zA-Z0-9_]+\.py$", x), os.listdir(migrations_path)):
+    for migration_file in filter(lambda x: re.match("^\d+_[a-zA-Z0-9_]+\.py$", x), os.listdir(migrations_path)):
-        migrations.append(migration[:-len(".py")])
+        migrations.append(_load_migration(migration_file))
 
-    return sorted(migrations)
+    return sorted(migrations, key=lambda m: m.id)
 
 
-def _get_migration_by_name(migration_name, with_module=True):
+def _get_migration_by_name(migration_name):
     """
     Low-level / "private" function to find a migration by its name
     """
 
-    migrations = tools_migrations_list()["migrations"]
+    try:
+        import data_migrations
+    except ImportError:
+        raise AssertionError("Unable to find migration with name %s" % migration_name)
 
-    matches = [ m for m in migrations if m["name"] == migration_name ]
+    migrations_path = data_migrations.__path__[0]
+    migrations_found = filter(lambda x: re.match("^\d+_%s\.py$" % migration_name, x), os.listdir(migrations_path))
 
-    assert len(matches) == 1, "Unable to find migration with name %s" % migration_name
+    assert len(migrations_found) == 1, "Unable to find migration with name %s" % migration_name
 
-    migration = matches[0]
+    return _load_migration(migrations_found[0])
-
-    if with_module:
-        migration["module"] = _get_migration_module(migration)
-
-    return migration
 
 
-def _get_migration_module(migration):
+def _load_migration(migration_file):
+
+    migration_id = migration_file[:-len(".py")]
+
+    number, name = migration_id.split("_", 1)
 
     logger.debug(m18n.n('migrations_loading_migration',
-        number=migration["number"],
+        number=number, name=name))
-        name=migration["name"],
-    ))
 
     try:
         # this is python builtin method to import a module using a name, we
         # use that to import the migration as a python object so we'll be
         # able to run it in the next loop
-        return import_module("yunohost.data_migrations.{file_name}".format(**migration))
+        module = import_module("yunohost.data_migrations.{}".format(migration_id))
+        return module.MyMigration(migration_id)
     except Exception:
         import traceback
         traceback.print_exc()
 
         raise MoulinetteError(errno.EINVAL, m18n.n('migrations_error_failed_to_load_migration',
-            number=migration["number"],
+            number=number, name=name))
-            name=migration["name"],
-        ))
 
 
 class Migration(object):
 
-    def migrate(self):
+    # Those are to be implemented by daughter classes
-        self.forward()
+
+    mode = "auto"
 
     def forward(self):
         raise NotImplementedError()
 
     def backward(self):
         pass
+
+    @property
+    def disclaimer(self):
+        return None
+
+    # The followings shouldn't be overriden
+
+    def migrate(self):
+        self.forward()
+
+    def __init__(self, id_):
+        self.id = id_
+        self.number = int(self.id.split("_", 1)[0])
+        self.name = self.id.split("_", 1)[1]
+
+    @property
+    def description(self):
+        return m18n.n("migration_description_%s" % self.id)