diff --git a/locales/en.json b/locales/en.json index 45fdbfec2..be31c7599 100644 --- a/locales/en.json +++ b/locales/en.json @@ -420,12 +420,7 @@ "migration_description_0017_postgresql_9p6_to_11": "Migrate databases from PostgreSQL 9.6 to 11", "migration_description_0018_xtable_to_nftable": "Migrate old network traffic rules to the new nftable system", "migration_description_0019_extend_permissions_features": "Extend/rework the app permission management system", - "migration_0011_create_group": "Creating a group for each user...", - "migration_0011_LDAP_update_failed": "Unable to update LDAP. Error: {error:s}", - "migration_0011_migrate_permission": "Migrating permissions from apps settings to LDAP...", - "migration_0011_update_LDAP_database": "Updating LDAP database...", - "migration_0011_update_LDAP_schema": "Updating LDAP schema...", - "migration_0011_failed_to_remove_stale_object": "Unable to remove stale object {dn}: {error}", + "migration_update_LDAP_schema": "Updating LDAP schema...", "migration_0015_start" : "Starting migration to Buster", "migration_0015_patching_sources_list": "Patching the sources.lists...", "migration_0015_main_upgrade": "Starting main upgrade...", @@ -529,6 +524,7 @@ "restore_already_installed_app": "An app with the ID '{app:s}' is already installed", "restore_already_installed_apps": "The following apps can't be restored because they are already installed: {apps}", "restore_app_failed": "Could not restore {app:s}", + "restore_backup_too_old": "This backup archive can not be restored because it comes from a too-old YunoHost version.", "restore_cleaning_failed": "Could not clean up the temporary restoration directory", "restore_complete": "Restoration completed", "restore_confirm_yunohost_installed": "Do you really want to restore an already installed system? [{answers:s}]", diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 93f532525..ed96dac16 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -36,6 +36,7 @@ from datetime import datetime from glob import glob from collections import OrderedDict from functools import reduce +from packaging import version from moulinette import msignals, m18n, msettings from moulinette.utils import filesystem @@ -60,7 +61,7 @@ from yunohost.hook import ( hook_exec, CUSTOM_HOOK_FOLDER, ) -from yunohost.tools import tools_postinstall +from yunohost.tools import tools_postinstall, _tools_migrations_run_after_system_restore, _tools_migrations_run_before_app_restore from yunohost.regenconf import regen_conf from yunohost.log import OperationLogger from yunohost.utils.error import YunohostError, YunohostValidationError @@ -858,6 +859,9 @@ class RestoreManager: # FIXME this way to get the info is not compatible with copy or custom # backup methods self.info = backup_info(name, with_details=True) + if not self.info["from_yunohost_version"] or version.parse(self.info["from_yunohost_version"]) < version.parse("3.8.0"): + raise YunohostValidationError("restore_backup_too_old") + self.archive_path = self.info["path"] self.name = name self.method = BackupMethod.create(method, self) @@ -1215,7 +1219,6 @@ class RestoreManager: if system_targets == []: return - from yunohost.user import user_group_list from yunohost.permission import ( permission_create, permission_delete, @@ -1278,25 +1281,15 @@ class RestoreManager: regen_conf() - # Check that at least a group exists (all_users) to know if we need to - # do the migration 0011 : setup group and permission - # - # Legacy code - if "all_users" not in user_group_list()["groups"].keys(): - from yunohost.utils.legacy import SetupGroupPermissions + _tools_migrations_run_after_system_restore(backup_version=self.info["from_yunohost_version"]) - # Update LDAP schema restart slapd - logger.info(m18n.n("migration_0011_update_LDAP_schema")) - regen_conf(names=["slapd"], force=True) - SetupGroupPermissions.migrate_LDAP_db() - - # Remove all permission for all app which is still in the LDAP + # Remove all permission for all app still in the LDAP for permission_name in user_permission_list(ignore_system_perms=True)[ "permissions" ].keys(): permission_delete(permission_name, force=True, sync_perm=False) - # Restore permission for the app which is installed + # Restore permission for apps installed for permission_name, permission_infos in old_apps_permission.items(): app_name, perm_name = permission_name.split(".") if _is_installed(app_name): @@ -1347,7 +1340,6 @@ class RestoreManager: name should be already install) """ from yunohost.user import user_group_list - from yunohost.app import app_setting from yunohost.permission import ( permission_create, permission_delete, @@ -1421,67 +1413,47 @@ class RestoreManager: restore_script = os.path.join(tmp_folder_for_app_restore, "restore") # Restore permissions - if os.path.isfile("%s/permissions.yml" % app_settings_new_path): + if not os.path.isfile("%s/permissions.yml" % app_settings_new_path): + raise YunohostError("Didnt find a permssions.yml for the app !?", raw_msg=True) - permissions = read_yaml("%s/permissions.yml" % app_settings_new_path) - existing_groups = user_group_list()["groups"] + permissions = read_yaml("%s/permissions.yml" % app_settings_new_path) + existing_groups = user_group_list()["groups"] - for permission_name, permission_infos in permissions.items(): + for permission_name, permission_infos in permissions.items(): - if "allowed" not in permission_infos: - logger.warning( - "'allowed' key corresponding to allowed groups for permission %s not found when restoring app %s … You might have to reconfigure permissions yourself." - % (permission_name, app_instance_name) - ) - should_be_allowed = ["all_users"] - else: - should_be_allowed = [ - g - for g in permission_infos["allowed"] - if g in existing_groups - ] - - perm_name = permission_name.split(".")[1] - permission_create( - permission_name, - allowed=should_be_allowed, - url=permission_infos.get("url"), - additional_urls=permission_infos.get("additional_urls"), - auth_header=permission_infos.get("auth_header"), - label=permission_infos.get("label") - if perm_name == "main" - else permission_infos.get("sublabel"), - show_tile=permission_infos.get("show_tile", True), - protected=permission_infos.get("protected", False), - sync_perm=False, + if "allowed" not in permission_infos: + logger.warning( + "'allowed' key corresponding to allowed groups for permission %s not found when restoring app %s … You might have to reconfigure permissions yourself." + % (permission_name, app_instance_name) ) + should_be_allowed = ["all_users"] + else: + should_be_allowed = [ + g + for g in permission_infos["allowed"] + if g in existing_groups + ] - permission_sync_to_user() + perm_name = permission_name.split(".")[1] + permission_create( + permission_name, + allowed=should_be_allowed, + url=permission_infos.get("url"), + additional_urls=permission_infos.get("additional_urls"), + auth_header=permission_infos.get("auth_header"), + label=permission_infos.get("label") + if perm_name == "main" + else permission_infos.get("sublabel"), + show_tile=permission_infos.get("show_tile", True), + protected=permission_infos.get("protected", False), + sync_perm=False, + ) - os.remove("%s/permissions.yml" % app_settings_new_path) - else: - # Otherwise, we need to migrate the legacy permissions of this - # app (included in its settings.yml) - from yunohost.utils.legacy import SetupGroupPermissions + permission_sync_to_user() - SetupGroupPermissions.migrate_app_permission(app=app_instance_name) + os.remove("%s/permissions.yml" % app_settings_new_path) - # Migrate old settings - legacy_permission_settings = [ - "skipped_uris", - "unprotected_uris", - "protected_uris", - "skipped_regex", - "unprotected_regex", - "protected_regex", - ] - if any( - app_setting(app_instance_name, setting) is not None - for setting in legacy_permission_settings - ): - from yunohost.utils.legacy import migrate_legacy_permission_settings - - migrate_legacy_permission_settings(app=app_instance_name) + _tools_migrations_run_before_app_restore(backup_version=self.info["from_yunohost_version"], app_id=app_instance_name) # Prepare env. var. to pass to script env_dict = _make_environment_for_app_script(app_instance_name) @@ -2446,7 +2418,7 @@ def backup_info(name, with_details=False, human_readable=False): try: files_in_archive = tar.getnames() - except IOError as e: + except (IOError, EOFError) as e: raise YunohostError( "backup_archive_corrupted", archive=archive_file, error=str(e) ) @@ -2530,6 +2502,7 @@ def backup_info(name, with_details=False, human_readable=False): result["apps"] = info["apps"] result["system"] = info[system_key] + result["from_yunohost_version"] = info.get("from_yunohost_version") return result @@ -2559,6 +2532,8 @@ def backup_delete(name): files_to_delete.append(actual_archive) for backup_file in files_to_delete: + if not os.path.exists(backup_file): + continue try: os.remove(backup_file) except Exception: diff --git a/src/yunohost/data_migrations/0019_extend_permissions_features.py b/src/yunohost/data_migrations/0019_extend_permissions_features.py index 07f740a2b..734c11920 100644 --- a/src/yunohost/data_migrations/0019_extend_permissions_features.py +++ b/src/yunohost/data_migrations/0019_extend_permissions_features.py @@ -36,7 +36,7 @@ class MyMigration(Migration): ) # Update LDAP schema restart slapd - logger.info(m18n.n("migration_0011_update_LDAP_schema")) + logger.info(m18n.n("migration_update_LDAP_schema")) regen_conf(names=["slapd"], force=True) logger.info(m18n.n("migration_0019_add_new_attributes_in_ldap")) @@ -78,6 +78,32 @@ class MyMigration(Migration): ldap.update("cn=%s,ou=permission" % permission, update) + introduced_in_version = "4.1" + + def run_after_system_restore(self): + # Update LDAP database + self.add_new_ldap_attributes() + + def run_before_system_restore(self, app_id): + from yunohost.app import app_setting + from yunohost.utils.legacy import migrate_legacy_permission_settings + + # Migrate old settings + legacy_permission_settings = [ + "skipped_uris", + "unprotected_uris", + "protected_uris", + "skipped_regex", + "unprotected_regex", + "protected_regex", + ] + if any( + app_setting(app_id, setting) is not None + for setting in legacy_permission_settings + ): + migrate_legacy_permission_settings(app=app_id) + + def run(self): # FIXME : what do we really want to do here ... diff --git a/src/yunohost/tests/test_backuprestore.py b/src/yunohost/tests/test_backuprestore.py index 021566544..8af9f7149 100644 --- a/src/yunohost/tests/test_backuprestore.py +++ b/src/yunohost/tests/test_backuprestore.py @@ -47,8 +47,8 @@ def setup_function(function): for m in function.__dict__.get("pytestmark", []) } - if "with_wordpress_archive_from_2p4" in markers: - add_archive_wordpress_from_2p4() + if "with_wordpress_archive_from_3p8" in markers: + add_archive_wordpress_from_3p8() assert len(backup_list()["archives"]) == 1 if "with_legacy_app_installed" in markers: @@ -70,8 +70,8 @@ def setup_function(function): ) assert app_is_installed("backup_recommended_app") - if "with_system_archive_from_2p4" in markers: - add_archive_system_from_2p4() + if "with_system_archive_from_3p8" in markers: + add_archive_system_from_3p8() assert len(backup_list()["archives"]) == 1 if "with_permission_app_installed" in markers: @@ -107,7 +107,8 @@ def teardown_function(function): if "with_custom_domain" in markers: domain = markers["with_custom_domain"]["args"][0] - domain_remove(domain) + if domain != maindomain: + domain_remove(domain) @pytest.fixture(autouse=True) @@ -147,7 +148,7 @@ def backup_test_dependencies_are_met(): # Dummy test apps (or backup archives) assert os.path.exists( - os.path.join(get_test_apps_dir(), "backup_wordpress_from_2p4") + os.path.join(get_test_apps_dir(), "backup_wordpress_from_3p8") ) assert os.path.exists(os.path.join(get_test_apps_dir(), "legacy_app_ynh")) assert os.path.exists( @@ -216,39 +217,25 @@ def install_app(app, path, additionnal_args=""): ) -def add_archive_wordpress_from_2p4(): +def add_archive_wordpress_from_3p8(): os.system("mkdir -p /home/yunohost.backup/archives") os.system( "cp " - + os.path.join( - get_test_apps_dir(), "backup_wordpress_from_2p4/backup.info.json" - ) - + " /home/yunohost.backup/archives/backup_wordpress_from_2p4.info.json" - ) - - os.system( - "cp " - + os.path.join(get_test_apps_dir(), "backup_wordpress_from_2p4/backup.tar.gz") - + " /home/yunohost.backup/archives/backup_wordpress_from_2p4.tar.gz" + + os.path.join(get_test_apps_dir(), "backup_wordpress_from_3p8/backup.tar.gz") + + " /home/yunohost.backup/archives/backup_wordpress_from_3p8.tar.gz" ) -def add_archive_system_from_2p4(): +def add_archive_system_from_3p8(): os.system("mkdir -p /home/yunohost.backup/archives") os.system( "cp " - + os.path.join(get_test_apps_dir(), "backup_system_from_2p4/backup.info.json") - + " /home/yunohost.backup/archives/backup_system_from_2p4.info.json" - ) - - os.system( - "cp " - + os.path.join(get_test_apps_dir(), "backup_system_from_2p4/backup.tar.gz") - + " /home/yunohost.backup/archives/backup_system_from_2p4.tar.gz" + + os.path.join(get_test_apps_dir(), "backup_system_from_3p8/backup.tar.gz") + + " /home/yunohost.backup/archives/backup_system_from_3p8.tar.gz" ) @@ -314,12 +301,12 @@ def test_backup_and_restore_all_sys(mocker): # -# System restore from 2.4 # +# System restore from 3.8 # # -@pytest.mark.with_system_archive_from_2p4 -def test_restore_system_from_Ynh2p4(monkeypatch, mocker): +@pytest.mark.with_system_archive_from_3p8 +def test_restore_system_from_Ynh3p8(monkeypatch, mocker): # Backup current system with message(mocker, "backup_created"): @@ -327,7 +314,7 @@ def test_restore_system_from_Ynh2p4(monkeypatch, mocker): archives = backup_list()["archives"] assert len(archives) == 2 - # Restore system archive from 2.4 + # Restore system archive from 3.8 try: with message(mocker, "restore_complete"): backup_restore( @@ -464,9 +451,9 @@ def test_backup_using_copy_method(mocker): # -@pytest.mark.with_wordpress_archive_from_2p4 +@pytest.mark.with_wordpress_archive_from_3p8 @pytest.mark.with_custom_domain("yolo.test") -def test_restore_app_wordpress_from_Ynh2p4(mocker): +def test_restore_app_wordpress_from_Ynh3p8(mocker): with message(mocker, "restore_complete"): backup_restore( @@ -474,7 +461,7 @@ def test_restore_app_wordpress_from_Ynh2p4(mocker): ) -@pytest.mark.with_wordpress_archive_from_2p4 +@pytest.mark.with_wordpress_archive_from_3p8 @pytest.mark.with_custom_domain("yolo.test") def test_restore_app_script_failure_handling(monkeypatch, mocker): def custom_hook_exec(name, *args, **kwargs): @@ -495,7 +482,7 @@ def test_restore_app_script_failure_handling(monkeypatch, mocker): assert not _is_installed("wordpress") -@pytest.mark.with_wordpress_archive_from_2p4 +@pytest.mark.with_wordpress_archive_from_3p8 def test_restore_app_not_enough_free_space(monkeypatch, mocker): def custom_free_space_in_directory(dirpath): return 0 @@ -514,7 +501,7 @@ def test_restore_app_not_enough_free_space(monkeypatch, mocker): assert not _is_installed("wordpress") -@pytest.mark.with_wordpress_archive_from_2p4 +@pytest.mark.with_wordpress_archive_from_3p8 def test_restore_app_not_in_backup(mocker): assert not _is_installed("wordpress") @@ -530,7 +517,7 @@ def test_restore_app_not_in_backup(mocker): assert not _is_installed("yoloswag") -@pytest.mark.with_wordpress_archive_from_2p4 +@pytest.mark.with_wordpress_archive_from_3p8 @pytest.mark.with_custom_domain("yolo.test") def test_restore_app_already_installed(mocker): @@ -648,18 +635,18 @@ def test_restore_archive_with_no_json(mocker): backup_restore(name="badbackup", force=True) -@pytest.mark.with_wordpress_archive_from_2p4 +@pytest.mark.with_wordpress_archive_from_3p8 def test_restore_archive_with_bad_archive(mocker): # Break the archive os.system( - "head -n 1000 /home/yunohost.backup/archives/backup_wordpress_from_2p4.tar.gz > /home/yunohost.backup/archives/backup_wordpress_from_2p4.tar.gz" + "head -n 1000 /home/yunohost.backup/archives/backup_wordpress_from_3p8.tar.gz > /home/yunohost.backup/archives/backup_wordpress_from_3p8_bad.tar.gz" ) - assert "backup_wordpress_from_2p4" in backup_list()["archives"] + assert "backup_wordpress_from_3p8_bad" in backup_list()["archives"] - with raiseYunohostError(mocker, "backup_archive_open_failed"): - backup_restore(name="backup_wordpress_from_2p4", force=True) + with raiseYunohostError(mocker, "backup_archive_corrupted"): + backup_restore(name="backup_wordpress_from_3p8_bad", force=True) clean_tmp_backup_directory() diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index e5699dede..52762932d 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -29,6 +29,7 @@ import yaml import subprocess import pwd from importlib import import_module +from packaging import version from moulinette import msignals, m18n from moulinette.utils.log import getActionLogger @@ -1101,6 +1102,44 @@ def _skip_all_migrations(): write_to_yaml(MIGRATIONS_STATE_PATH, new_states) +def _tools_migrations_run_after_system_restore(backup_version): + + all_migrations = _get_migrations_list() + + for migration in all_migrations: + if hasattr(migration, "introduced_in_version") \ + and version.parse(migration.introduced_in_version) > version.parse(backup_version) \ + and hasattr(migration, "run_after_system_restore"): + try: + logger.info(m18n.n("migrations_running_forward", id=migration.id)) + migration.run_after_system_restore() + except Exception as e: + msg = m18n.n( + "migrations_migration_has_failed", exception=e, id=migration.id + ) + logger.error(msg, exc_info=1) + raise + + +def _tools_migrations_run_before_app_restore(backup_version, app_id): + + all_migrations = _get_migrations_list() + + for migration in all_migrations: + if hasattr(migration, "introduced_in_version") \ + and version.parse(migration.introduced_in_version) > version.parse(backup_version) \ + and hasattr(migration, "run_before_app_restore"): + try: + logger.info(m18n.n("migrations_running_forward", id=migration.id)) + migration.run_before_app_restore(app_id) + except Exception as e: + msg = m18n.n( + "migrations_migration_has_failed", exception=e, id=migration.id + ) + logger.error(msg, exc_info=1) + raise + + class Migration(object): # Those are to be implemented by daughter classes diff --git a/src/yunohost/utils/legacy.py b/src/yunohost/utils/legacy.py index b83a69154..fc00ab586 100644 --- a/src/yunohost/utils/legacy.py +++ b/src/yunohost/utils/legacy.py @@ -1,12 +1,10 @@ import os from moulinette import m18n -from yunohost.utils.error import YunohostError from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import write_to_json, read_yaml -from yunohost.user import user_list, user_group_create, user_group_update +from yunohost.user import user_list from yunohost.app import ( - app_setting, _installed_apps, _get_app_settings, _set_app_settings, @@ -19,149 +17,6 @@ from yunohost.permission import ( logger = getActionLogger("yunohost.legacy") - -class SetupGroupPermissions: - @staticmethod - def remove_if_exists(target): - - from yunohost.utils.ldap import _get_ldap_interface - - ldap = _get_ldap_interface() - - try: - objects = ldap.search(target + ",dc=yunohost,dc=org") - # ldap search will raise an exception if no corresponding object is found >.> ... - except Exception: - logger.debug("%s does not exist, no need to delete it" % target) - return - - objects.reverse() - for o in objects: - for dn in o["dn"]: - dn = dn.replace(",dc=yunohost,dc=org", "") - logger.debug("Deleting old object %s ..." % dn) - try: - ldap.remove(dn) - except Exception as e: - raise YunohostError( - "migration_0011_failed_to_remove_stale_object", dn=dn, error=e - ) - - @staticmethod - def migrate_LDAP_db(): - - logger.info(m18n.n("migration_0011_update_LDAP_database")) - - from yunohost.utils.ldap import _get_ldap_interface - - ldap = _get_ldap_interface() - - ldap_map = read_yaml( - "/usr/share/yunohost/yunohost-config/moulinette/ldap_scheme.yml" - ) - - try: - SetupGroupPermissions.remove_if_exists("ou=permission") - SetupGroupPermissions.remove_if_exists("ou=groups") - - attr_dict = ldap_map["parents"]["ou=permission"] - ldap.add("ou=permission", attr_dict) - - attr_dict = ldap_map["parents"]["ou=groups"] - ldap.add("ou=groups", attr_dict) - - attr_dict = ldap_map["children"]["cn=all_users,ou=groups"] - ldap.add("cn=all_users,ou=groups", attr_dict) - - attr_dict = ldap_map["children"]["cn=visitors,ou=groups"] - ldap.add("cn=visitors,ou=groups", attr_dict) - - for rdn, attr_dict in ldap_map["depends_children"].items(): - ldap.add(rdn, attr_dict) - except Exception as e: - raise YunohostError("migration_0011_LDAP_update_failed", error=e) - - logger.info(m18n.n("migration_0011_create_group")) - - # Create a group for each yunohost user - user_list = ldap.search( - "ou=users,dc=yunohost,dc=org", - "(&(objectclass=person)(!(uid=root))(!(uid=nobody)))", - ["uid", "uidNumber"], - ) - for user_info in user_list: - username = user_info["uid"][0] - ldap.update( - "uid=%s,ou=users" % username, - { - "objectClass": [ - "mailAccount", - "inetOrgPerson", - "posixAccount", - "userPermissionYnh", - ] - }, - ) - user_group_create( - username, - gid=user_info["uidNumber"][0], - primary_group=True, - sync_perm=False, - ) - user_group_update( - groupname="all_users", add=username, force=True, sync_perm=False - ) - - @staticmethod - def migrate_app_permission(app=None): - logger.info(m18n.n("migration_0011_migrate_permission")) - - apps = _installed_apps() - - if app: - if app not in apps: - logger.error( - "Can't migrate permission for app %s because it ain't installed..." - % app - ) - apps = [] - else: - apps = [app] - - for app in apps: - permission = app_setting(app, "allowed_users") - path = app_setting(app, "path") - domain = app_setting(app, "domain") - - url = "/" if domain and path else None - if permission: - known_users = list(user_list()["users"].keys()) - allowed = [ - user for user in permission.split(",") if user in known_users - ] - else: - allowed = ["all_users"] - permission_create( - app + ".main", - url=url, - allowed=allowed, - show_tile=True, - protected=False, - sync_perm=False, - ) - - app_setting(app, "allowed_users", delete=True) - - # Migrate classic public app still using the legacy unprotected_uris - if ( - app_setting(app, "unprotected_uris") == "/" - or app_setting(app, "skipped_uris") == "/" - ): - user_permission_update(app + ".main", add="visitors", sync_perm=False) - - permission_sync_to_user() - - LEGACY_PERMISSION_LABEL = { ("nextcloud", "skipped"): "api", # .well-known ("libreto", "skipped"): "pad access", # /[^/]+