From 8dd590e6fcd05f4a340511a4c0d7f7ea14088413 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 17 Aug 2019 18:43:34 +0200 Subject: [PATCH] Implement explicit and implicit migrations --- locales/en.json | 3 +- src/yunohost/app.py | 6 ++ .../0010_migrate_to_apps_json.py | 34 ++--------- src/yunohost/tests/test_appslist.py | 56 +++++++++++++++++-- 4 files changed, 64 insertions(+), 35 deletions(-) diff --git a/locales/en.json b/locales/en.json index fdabad9d0..0e3667749 100644 --- a/locales/en.json +++ b/locales/en.json @@ -320,8 +320,9 @@ "migration_description_0007_ssh_conf_managed_by_yunohost_step1": "Let the SSH configuration be managed by YunoHost (step 1, automatic)", "migration_description_0008_ssh_conf_managed_by_yunohost_step2": "Let the SSH configuration be managed by YunoHost (step 2, manual)", "migration_description_0009_decouple_regenconf_from_services": "Decouple the regen-conf mechanism from services", - "migration_description_0010_migrate_to_apps_json": "Remove deprecated appslists and use the new unified 'apps.json' list instead", + "migration_description_0010_migrate_to_apps_json": "Remove deprecated appslists and use the new unified 'apps.json' list instead (outdated, replaced by migration 12)", "migration_description_0011_setup_group_permission": "Setup user group and setup permission for apps and services", + "migration_description_0012_futureproof_appslist_system": "Migrate to the new future-proof appslist system", "migration_0003_backward_impossible": "The stretch migration cannot be reverted.", "migration_0003_start": "Starting migration to Stretch. The logs will be available in {logfile}.", "migration_0003_patching_sources_list": "Patching the sources.lists…", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index aa2163cdf..3a6d2216a 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2570,6 +2570,12 @@ def _read_appslist_list(): Read the json corresponding to the list of appslists """ + # Legacy code - can be removed after moving to buster (if the migration got merged before buster) + if os.path.exists('/etc/yunohost/appslists.json'): + from yunohost.tools import _get_migration_by_name + migration = _get_migration_by_name("futureproof_appslist_system") + migration.migrate() + try: list_ = read_yaml(APPSLISTS_CONF) # Support the case where file exists but is empty diff --git a/src/yunohost/data_migrations/0010_migrate_to_apps_json.py b/src/yunohost/data_migrations/0010_migrate_to_apps_json.py index 43ae9a86f..1333e54e8 100644 --- a/src/yunohost/data_migrations/0010_migrate_to_apps_json.py +++ b/src/yunohost/data_migrations/0010_migrate_to_apps_json.py @@ -1,44 +1,18 @@ -import os - from moulinette.utils.log import getActionLogger -from yunohost.app import app_fetchlist, app_removelist, _read_appslist_list, APPSLISTS_JSON from yunohost.tools import Migration logger = getActionLogger('yunohost.migration') -BASE_CONF_PATH = '/home/yunohost.conf' -BACKUP_CONF_DIR = os.path.join(BASE_CONF_PATH, 'backup') -APPSLISTS_BACKUP = os.path.join(BACKUP_CONF_DIR, "appslist_before_migration_to_unified_list.json") - class MyMigration(Migration): - "Migrate from official.json to apps.json" + "Migrate from official.json to apps.json (outdated, replaced by migration 12)" def migrate(self): - # Backup current app list json - os.system("cp %s %s" % (APPSLISTS_JSON, APPSLISTS_BACKUP)) - - # Remove all the deprecated lists - lists_to_remove = [ - "app.yunohost.org/list.json", # Old list on old installs, alias to official.json - "app.yunohost.org/official.json", - "app.yunohost.org/community.json", - "labriqueinter.net/apps/labriqueinternet.json", - "labriqueinter.net/internetcube.json" - ] - - appslists = _read_appslist_list() - for appslist, infos in appslists.items(): - if infos["url"].split("//")[-1] in lists_to_remove: - app_removelist(name=appslist) - - # Replace by apps.json list - app_fetchlist(name="yunohost", - url="https://app.yunohost.org/apps.json") + logger.info("This is migration is oudated and doesn't do anything anymore. The migration 12 will handle this instead.") + pass def backward(self): - if os.path.exists(APPSLISTS_BACKUP): - os.system("cp %s %s" % (APPSLISTS_BACKUP, APPSLISTS_JSON)) + pass diff --git a/src/yunohost/tests/test_appslist.py b/src/yunohost/tests/test_appslist.py index 057a5f3fe..d7b8e429b 100644 --- a/src/yunohost/tests/test_appslist.py +++ b/src/yunohost/tests/test_appslist.py @@ -6,7 +6,7 @@ import glob import shutil from moulinette import m18n -from moulinette.utils.filesystem import read_json, write_to_json, write_to_yaml +from moulinette.utils.filesystem import read_json, write_to_json, write_to_yaml, mkdir from yunohost.utils.error import YunohostError from yunohost.app import (_initialize_appslists_system, @@ -49,7 +49,11 @@ def setup_function(function): def teardown_function(function): - pass + + # Clear applist cache + # Otherwise when using apps stuff after running the test, + # we'll still have the dummy unusable list + shutil.rmtree(APPSLISTS_CACHE, ignore_errors=True) def cron_job_is_there(): @@ -302,9 +306,53 @@ def test_appslist_load_with_oudated_api_version(mocker): def test_appslist_migrate_legacy_explicitly(): - raise NotImplementedError + open("/etc/yunohost/appslists.json", "w").write('{"yunohost": {"yolo":"swag"}}') + mkdir(APPSLISTS_CACHE, 0o750, parents=True) + open(APPSLISTS_CACHE+"/yunohost_old.json", "w").write('{"foo":{}, "bar": {}}') + open(APPSLISTS_CRON_PATH, "w").write("# Some old cron") + + from yunohost.tools import _get_migration_by_name + migration = _get_migration_by_name("futureproof_appslist_system") + + with requests_mock.Mocker() as m: + + # Mock the server response with a dummy applist + m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, text=DUMMY_APPLIST) + migration.migrate() + + # Old conf shouldnt be there anymore (got renamed to .old) + assert not os.path.exists("/etc/yunohost/appslists.json") + # Old cache should have been removed + assert not os.path.exists(APPSLISTS_CACHE+"/yunohost_old.json") + # Cron should have been changed + assert "/bin/bash" in open(APPSLISTS_CRON_PATH, "r").read() + assert cron_job_is_there() + + # Reading the appslist should work + app_dict = _load_appslist() + assert "foo" in app_dict.keys() + assert "bar" in app_dict.keys() def test_appslist_migrate_legacy_implicitly(): - raise NotImplementedError + open("/etc/yunohost/appslists.json", "w").write('{"yunohost": {"yolo":"swag"}}') + mkdir(APPSLISTS_CACHE, 0o750, parents=True) + open(APPSLISTS_CACHE+"/yunohost_old.json", "w").write('{"old_foo":{}, "old_bar": {}}') + open(APPSLISTS_CRON_PATH, "w").write("# Some old cron") + + with requests_mock.Mocker() as m: + m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, text=DUMMY_APPLIST) + app_dict = _load_appslist() + + assert "foo" in app_dict.keys() + assert "bar" in app_dict.keys() + + # Old conf shouldnt be there anymore (got renamed to .old) + assert not os.path.exists("/etc/yunohost/appslists.json") + # Old cache should have been removed + assert not os.path.exists(APPSLISTS_CACHE+"/yunohost_old.json") + # Cron should have been changed + assert "/bin/bash" in open(APPSLISTS_CRON_PATH, "r").read() + assert cron_job_is_there() +