Implement explicit and implicit migrations

locales/en.json

@@ -320,8 +320,9 @@
     "migration_description_0007_ssh_conf_managed_by_yunohost_step1": "Let the SSH configuration be managed by YunoHost (step 1, automatic)",
     "migration_description_0008_ssh_conf_managed_by_yunohost_step2": "Let the SSH configuration be managed by YunoHost (step 2, manual)",
     "migration_description_0009_decouple_regenconf_from_services": "Decouple the regen-conf mechanism from services",
-    "migration_description_0010_migrate_to_apps_json": "Remove deprecated appslists and use the new unified 'apps.json' list instead",
+    "migration_description_0010_migrate_to_apps_json": "Remove deprecated appslists and use the new unified 'apps.json' list instead (outdated, replaced by migration 12)",
     "migration_description_0011_setup_group_permission": "Setup user group and setup permission for apps and services",
+    "migration_description_0012_futureproof_appslist_system": "Migrate to the new future-proof appslist system",
     "migration_0003_backward_impossible": "The stretch migration cannot be reverted.",
     "migration_0003_start": "Starting migration to Stretch. The logs will be available in {logfile}.",
     "migration_0003_patching_sources_list": "Patching the sources.lists…",

src/yunohost/app.py

@@ -2570,6 +2570,12 @@ def _read_appslist_list():
     Read the json corresponding to the list of appslists
     """
 
+    # Legacy code - can be removed after moving to buster (if the migration got merged before buster)
+    if os.path.exists('/etc/yunohost/appslists.json'):
+        from yunohost.tools import _get_migration_by_name
+        migration = _get_migration_by_name("futureproof_appslist_system")
+        migration.migrate()
+
     try:
         list_ = read_yaml(APPSLISTS_CONF)
         # Support the case where file exists but is empty

src/yunohost/data_migrations/0010_migrate_to_apps_json.py

@@ -1,44 +1,18 @@
-import os
-
 from moulinette.utils.log import getActionLogger
-from yunohost.app import app_fetchlist, app_removelist, _read_appslist_list, APPSLISTS_JSON
 from yunohost.tools import Migration
 
 logger = getActionLogger('yunohost.migration')
 
-BASE_CONF_PATH = '/home/yunohost.conf'
-BACKUP_CONF_DIR = os.path.join(BASE_CONF_PATH, 'backup')
-APPSLISTS_BACKUP = os.path.join(BACKUP_CONF_DIR, "appslist_before_migration_to_unified_list.json")
-
 
 class MyMigration(Migration):
 
-    "Migrate from official.json to apps.json"
+    "Migrate from official.json to apps.json (outdated, replaced by migration 12)"
 
     def migrate(self):
 
-        # Backup current app list json
-        os.system("cp %s %s" % (APPSLISTS_JSON, APPSLISTS_BACKUP))
-
-        # Remove all the deprecated lists
-        lists_to_remove = [
-            "app.yunohost.org/list.json",       # Old list on old installs, alias to official.json
-            "app.yunohost.org/official.json",
-            "app.yunohost.org/community.json",
-            "labriqueinter.net/apps/labriqueinternet.json",
-            "labriqueinter.net/internetcube.json"
-        ]
-
-        appslists = _read_appslist_list()
-        for appslist, infos in appslists.items():
-            if infos["url"].split("//")[-1] in lists_to_remove:
-                app_removelist(name=appslist)
-
-        # Replace by apps.json list
-        app_fetchlist(name="yunohost",
-                      url="https://app.yunohost.org/apps.json")
+        logger.info("This is migration is oudated and doesn't do anything anymore. The migration 12 will handle this instead.")
+        pass
 
     def backward(self):
 
-        if os.path.exists(APPSLISTS_BACKUP):
-            os.system("cp %s %s" % (APPSLISTS_BACKUP, APPSLISTS_JSON))
+        pass

src/yunohost/tests/test_appslist.py

@@ -6,7 +6,7 @@ import glob
 import shutil
 
 from moulinette import m18n
-from moulinette.utils.filesystem import read_json, write_to_json, write_to_yaml
+from moulinette.utils.filesystem import read_json, write_to_json, write_to_yaml, mkdir
 
 from yunohost.utils.error import YunohostError
 from yunohost.app import (_initialize_appslists_system,
@@ -49,7 +49,11 @@ def setup_function(function):
 
 
 def teardown_function(function):
-    pass
+
+    # Clear applist cache
+    # Otherwise when using apps stuff after running the test,
+    # we'll still have the dummy unusable list
+    shutil.rmtree(APPSLISTS_CACHE, ignore_errors=True)
 
 
 def cron_job_is_there():
@@ -302,9 +306,53 @@ def test_appslist_load_with_oudated_api_version(mocker):
 
 def test_appslist_migrate_legacy_explicitly():
 
-    raise NotImplementedError
+    open("/etc/yunohost/appslists.json", "w").write('{"yunohost": {"yolo":"swag"}}')
+    mkdir(APPSLISTS_CACHE, 0o750, parents=True)
+    open(APPSLISTS_CACHE+"/yunohost_old.json", "w").write('{"foo":{}, "bar": {}}')
+    open(APPSLISTS_CRON_PATH, "w").write("# Some old cron")
+
+    from yunohost.tools import _get_migration_by_name
+    migration = _get_migration_by_name("futureproof_appslist_system")
+
+    with requests_mock.Mocker() as m:
+
+        # Mock the server response with a dummy applist
+        m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, text=DUMMY_APPLIST)
+        migration.migrate()
+
+    # Old conf shouldnt be there anymore (got renamed to .old)
+    assert not os.path.exists("/etc/yunohost/appslists.json")
+    # Old cache should have been removed
+    assert not os.path.exists(APPSLISTS_CACHE+"/yunohost_old.json")
+    # Cron should have been changed
+    assert "/bin/bash" in open(APPSLISTS_CRON_PATH, "r").read()
+    assert cron_job_is_there()
+
+    # Reading the appslist should work
+    app_dict = _load_appslist()
+    assert "foo" in app_dict.keys()
+    assert "bar" in app_dict.keys()
 
 
 def test_appslist_migrate_legacy_implicitly():
 
-    raise NotImplementedError
+    open("/etc/yunohost/appslists.json", "w").write('{"yunohost": {"yolo":"swag"}}')
+    mkdir(APPSLISTS_CACHE, 0o750, parents=True)
+    open(APPSLISTS_CACHE+"/yunohost_old.json", "w").write('{"old_foo":{}, "old_bar": {}}')
+    open(APPSLISTS_CRON_PATH, "w").write("# Some old cron")
+
+    with requests_mock.Mocker() as m:
+        m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, text=DUMMY_APPLIST)
+        app_dict = _load_appslist()
+
+    assert "foo" in app_dict.keys()
+    assert "bar" in app_dict.keys()
+
+    # Old conf shouldnt be there anymore (got renamed to .old)
+    assert not os.path.exists("/etc/yunohost/appslists.json")
+    # Old cache should have been removed
+    assert not os.path.exists(APPSLISTS_CACHE+"/yunohost_old.json")
+    # Cron should have been changed
+    assert "/bin/bash" in open(APPSLISTS_CRON_PATH, "r").read()
+    assert cron_job_is_there()
+