diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index c6f8f2458..af072c1bc 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1478,6 +1478,37 @@ tools: extra: pattern: *pattern_port + subcategories: + + migrations: + subcategory_help: Manage migrations + actions: + + ### tools_migrations_list() + list: + action_help: List migrations + api: GET /migrations + + ### tools_migrations_migrate() + migrate: + action_help: Perform migrations + api: POST /migrations/migrate + arguments: + -t: + help: target migration number (or 0), latest one by default + type: int + full: --target + -s: + help: skip the migration(s), use it only if you know what you are doing + full: --skip + action: store_true + + + ### tools_migrations_state() + state: + action_help: Show current migrations state + api: GET /migrations/state + ############################# # Hook # diff --git a/debian/postinst b/debian/postinst index 124657a10..7e91ffbb3 100644 --- a/debian/postinst +++ b/debian/postinst @@ -14,6 +14,9 @@ do_configure() { echo "Regenerating configuration, this might take a while..." yunohost service regen-conf --output-as none + echo "Launching migrations.." + yunohost tools migrations migrate + # restart yunohost-firewall if it's running service yunohost-firewall status >/dev/null \ && restart_yunohost_firewall \ diff --git a/locales/en.json b/locales/en.json index 50ef2ddf1..a59f87b4d 100644 --- a/locales/en.json +++ b/locales/en.json @@ -5,6 +5,7 @@ "admin_password_changed": "The administration password has been changed", "app_already_installed": "{app:s} is already installed", "app_already_installed_cant_change_url": "This app is already installed. The url cannot be changed just by this function. Look into `app changeurl` if it's available.", + "app_already_up_to_date": "{app:s} is already up to date", "app_argument_choice_invalid": "Invalid choice for argument '{name:s}', it must be one of {choices:s}", "app_argument_invalid": "Invalid value for argument '{name:s}': {error:s}", "app_argument_required": "Argument '{name:s}' is required", @@ -35,17 +36,16 @@ "app_unsupported_remote_type": "Unsupported remote type used for the app", "app_upgrade_failed": "Unable to upgrade {app:s}", "app_upgraded": "{app:s} has been upgraded", - "app_already_up_to_date": "{app:s} is already up to date", - "appslist_fetched": "The application list {appslist:s} has been fetched", - "appslist_removed": "The application list {appslist:s} has been removed", - "appslist_unknown": "Application list {appslist:s} unknown.", - "appslist_retrieve_error": "Unable to retrieve the remote application list {appslist:s}: {error:s}", - "appslist_retrieve_bad_format": "Retrieved file for application list {appslist:s} is not valid", - "appslist_name_already_tracked": "There is already a registered application list with name {name:s}.", - "appslist_url_already_tracked": "There is already a registered application list with url {url:s}.", - "appslist_migrating": "Migrating application list {appslist:s} ...", - "appslist_could_not_migrate": "Could not migrate app list {appslist:s} ! Unable to parse the url... The old cron job has been kept in {bkp_file:s}.", "appslist_corrupted_json": "Could not load the application lists. It looks like {filename:s} is corrupted.", + "appslist_could_not_migrate": "Could not migrate app list {appslist:s} ! Unable to parse the url... The old cron job has been kept in {bkp_file:s}.", + "appslist_fetched": "The application list {appslist:s} has been fetched", + "appslist_migrating": "Migrating application list {appslist:s} ...", + "appslist_name_already_tracked": "There is already a registered application list with name {name:s}.", + "appslist_removed": "The application list {appslist:s} has been removed", + "appslist_retrieve_bad_format": "Retrieved file for application list {appslist:s} is not valid", + "appslist_retrieve_error": "Unable to retrieve the remote application list {appslist:s}: {error:s}", + "appslist_unknown": "Application list {appslist:s} unknown.", + "appslist_url_already_tracked": "There is already a registered application list with url {url:s}.", "ask_current_admin_password": "Current administration password", "ask_email": "Email address", "ask_firstname": "First name", @@ -57,51 +57,75 @@ "backup_abstract_method": "This backup method hasn't yet been implemented", "backup_action_required": "You must specify something to save", "backup_app_failed": "Unable to back up the app '{app:s}'", - "backup_applying_method_tar": "Creating the backup tar archive...", - "backup_applying_method_copy": "Copying all files to backup...", "backup_applying_method_borg": "Sending all files to backup into borg-backup repository...", + "backup_applying_method_copy": "Copying all files to backup...", "backup_applying_method_custom": "Calling the custom backup method '{method:s}'...", + "backup_applying_method_tar": "Creating the backup tar archive...", "backup_archive_app_not_found": "App '{app:s}' not found in the backup archive", "backup_archive_broken_link": "Unable to access backup archive (broken link to {path:s})", - "backup_archive_system_part_not_available": "System part '{part:s}' not available in this backup", + "backup_archive_mount_failed": "Mounting the backup archive failed", "backup_archive_name_exists": "The backup's archive name already exists", "backup_archive_name_unknown": "Unknown local backup archive named '{name:s}'", "backup_archive_open_failed": "Unable to open the backup archive", - "backup_archive_mount_failed": "Mounting the backup archive failed", + "backup_archive_system_part_not_available": "System part '{part:s}' not available in this backup", "backup_archive_writing_error": "Unable to add files to backup into the compressed archive", "backup_ask_for_copying_if_needed": "Your system don't support completely the quick method to organize files in the archive, do you want to organize its by copying {size:s}MB?", "backup_borg_not_implemented": "Borg backup method is not yet implemented", "backup_cant_mount_uncompress_archive": "Unable to mount in readonly mode the uncompress archive directory", "backup_cleaning_failed": "Unable to clean-up the temporary backup directory", - "backup_copying_to_organize_the_archive": "Copying {size:s}MB to organize the archive", + "backup_copying_to_organize_the_archive": "Copying {size:s}MB to organize the archive", "backup_created": "Backup created", "backup_creating_archive": "Creating the backup archive...", "backup_creation_failed": "Backup creation failed", - "backup_csv_creation_failed": "Unable to create the CSV file needed for future restore operations", "backup_csv_addition_failed": "Unable to add files to backup into the CSV file", - "backup_custom_need_mount_error": "Custom backup method failure on 'need_mount' step", + "backup_csv_creation_failed": "Unable to create the CSV file needed for future restore operations", "backup_custom_backup_error": "Custom backup method failure on 'backup' step", "backup_custom_mount_error": "Custom backup method failure on 'mount' step", + "backup_custom_need_mount_error": "Custom backup method failure on 'need_mount' step", "backup_delete_error": "Unable to delete '{path:s}'", "backup_deleted": "The backup has been deleted", "backup_extracting_archive": "Extracting the backup archive...", "backup_hook_unknown": "Backup hook '{hook:s}' unknown", "backup_invalid_archive": "Invalid backup archive", + "backup_method_borg_finished": "Backup into borg finished", + "backup_method_copy_finished": "Backup copy finished", + "backup_method_custom_finished": "Custom backup method '{method:s}' finished", + "backup_method_tar_finished": "Backup tar archive created", "backup_no_uncompress_archive_dir": "Uncompress archive directory doesn't exist", "backup_nothings_done": "There is nothing to save", - "backup_method_tar_finished": "Backup tar archive created", - "backup_method_copy_finished": "Backup copy finished", - "backup_method_borg_finished": "Backup into borg finished", - "backup_method_custom_finished": "Custom backup method '{method:s}' finished", "backup_output_directory_forbidden": "Forbidden output directory. Backups can't be created in /bin, /boot, /dev, /etc, /lib, /root, /run, /sbin, /sys, /usr, /var or /home/yunohost.backup/archives sub-folders", "backup_output_directory_not_empty": "The output directory is not empty", "backup_output_directory_required": "You must provide an output directory for the backup", "backup_running_app_script": "Running backup script of app '{app:s}'...", "backup_running_hooks": "Running backup hooks...", "backup_system_part_failed": "Unable to backup the '{part:s}' system part", - "backup_unable_to_organize_files": "Unable to organize files in the archive with the quick method", + "backup_unable_to_organize_files": "Unable to organize files in the archive with the quick method", "backup_with_no_backup_script_for_app": "App {app:s} has no backup script. Ignoring.", "backup_with_no_restore_script_for_app": "App {app:s} has no restore script, you won't be able to automatically restore the backup of this app.", + "certmanager_acme_not_configured_for_domain": "Certificate for domain {domain:s} does not appear to be correctly installed. Please run cert-install for this domain first.", + "certmanager_attempt_to_renew_nonLE_cert": "The certificate for domain {domain:s} is not issued by Let's Encrypt. Cannot renew it automatically!", + "certmanager_attempt_to_renew_valid_cert": "The certificate for domain {domain:s} is not about to expire! Use --force to bypass", + "certmanager_attempt_to_replace_valid_cert": "You are attempting to overwrite a good and valid certificate for domain {domain:s}! (Use --force to bypass)", + "certmanager_cannot_read_cert": "Something wrong happened when trying to open current certificate for domain {domain:s} (file: {file:s}), reason: {reason:s}", + "certmanager_cert_install_success": "Successfully installed Let's Encrypt certificate for domain {domain:s}!", + "certmanager_cert_install_success_selfsigned": "Successfully installed a self-signed certificate for domain {domain:s}!", + "certmanager_cert_renew_success": "Successfully renewed Let's Encrypt certificate for domain {domain:s}!", + "certmanager_cert_signing_failed": "Signing the new certificate failed", + "certmanager_certificate_fetching_or_enabling_failed": "Sounds like enabling the new certificate for {domain:s} failed somehow...", + "certmanager_conflicting_nginx_file": "Unable to prepare domain for ACME challenge: the nginx configuration file {filepath:s} is conflicting and should be removed first", + "certmanager_couldnt_fetch_intermediate_cert": "Timed out when trying to fetch intermediate certificate from Let's Encrypt. Certificate installation/renewal aborted - please try again later.", + "certmanager_domain_cert_not_selfsigned": "The certificate for domain {domain:s} is not self-signed. Are you sure you want to replace it? (Use --force)", + "certmanager_domain_dns_ip_differs_from_public_ip": "The DNS 'A' record for domain {domain:s} is different from this server IP. If you recently modified your A record, please wait for it to propagate (some DNS propagation checkers are available online). (If you know what you are doing, use --no-checks to disable those checks.)", + "certmanager_domain_http_not_working": "It seems that the domain {domain:s} cannot be accessed through HTTP. Please check your DNS and nginx configuration is okay", + "certmanager_domain_not_resolved_locally": "The domain {domain:s} cannot be resolved from inside your Yunohost server. This might happen if you recently modified your DNS record. If so, please wait a few hours for it to propagate. If the issue persists, consider adding {domain:s} to /etc/hosts. (If you know what you are doing, use --no-checks to disable those checks.)", + "certmanager_domain_unknown": "Unknown domain {domain:s}", + "certmanager_error_no_A_record": "No DNS 'A' record found for {domain:s}. You need to make your domain name point to your machine to be able to install a Let's Encrypt certificate! (If you know what you are doing, use --no-checks to disable those checks.)", + "certmanager_hit_rate_limit": "Too many certificates already issued for exact set of domains {domain:s} recently. Please try again later. See https://letsencrypt.org/docs/rate-limits/ for more details", + "certmanager_http_check_timeout": "Timed out when server tried to contact itself through HTTP using public IP address (domain {domain:s} with ip {ip:s}). You may be experiencing hairpinning issue or the firewall/router ahead of your server is misconfigured.", + "certmanager_no_cert_file": "Unable to read certificate file for domain {domain:s} (file: {file:s})", + "certmanager_old_letsencrypt_app_detected": "\nYunohost detected that the 'letsencrypt' app is installed, which conflits with the new built-in certificate management features in Yunohost. If you wish to use the new built-in features, please run the following commands to migrate your installation:\n\n yunohost app remove letsencrypt\n yunohost domain cert-install\n\nN.B.: this will attempt to re-install certificates for all domains with a Let's Encrypt certificate or self-signed certificate", + "certmanager_self_ca_conf_file_not_found": "Configuration file not found for self-signing authority (file: {file:s})", + "certmanager_unable_to_parse_self_CA_name": "Unable to parse name of self-signing authority (file: {file:s})", "custom_app_url_required": "You must provide a URL to upgrade your custom app {app:s}", "custom_appslist_name_required": "You must provide a name for your custom app list", "diagnosis_debian_version_error": "Can't retrieve the Debian version: {error}", @@ -111,6 +135,8 @@ "diagnosis_monitor_system_error": "Can't monitor system: {error}", "diagnosis_no_apps": "No installed application", "dnsmasq_isnt_installed": "dnsmasq does not seem to be installed, please run 'apt-get remove bind9 && apt-get install dnsmasq'", + "domain_cannot_remove_main": "Cannot remove main domain. Set a new main domain first", + "domain_cert_gen_failed": "Unable to generate certificate", "domain_created": "The domain has been created", "domain_creation_failed": "Unable to create domain", "domain_deleted": "The domain has been deleted", @@ -124,8 +150,8 @@ "domain_unknown": "Unknown domain", "domain_zone_exists": "DNS zone file already exists", "domain_zone_not_found": "DNS zone file not found for domain {:s}", - "done": "Done", "domains_available": "Available domains:", + "done": "Done", "downloading": "Downloading...", "dyndns_cron_installed": "The DynDNS cron job has been installed", "dyndns_cron_remove_failed": "Unable to remove the DynDNS cron job", @@ -154,13 +180,13 @@ "global_settings_key_doesnt_exists": "The key '{settings_key:s}' doesn't exists in the global settings, you can see all the available keys by doing 'yunohost settings list'", "global_settings_reset_success": "Success. Your previous settings have been backuped in {path:s}", "global_settings_setting_example_bool": "Example boolean option", + "global_settings_setting_example_enum": "Example enum option", "global_settings_setting_example_int": "Example int option", "global_settings_setting_example_string": "Example string option", - "global_settings_setting_example_enum": "Example enum option", - "global_settings_unknown_type": "Unexpected situation, the setting {setting:s} appears to have the type {unknown_type:s} but it's not a type supported by the system.", "global_settings_unknown_setting_from_settings_file": "Unknown key in settings: '{setting_key:s}', discarding it and save it in /etc/yunohost/unkown_settings.json", + "global_settings_unknown_type": "Unexpected situation, the setting {setting:s} appears to have the type {unknown_type:s} but it's not a type supported by the system.", "hook_exec_failed": "Script execution failed: {path:s}", - "hook_exec_not_terminated": "Script execution hasn’t terminated: {path:s}", + "hook_exec_not_terminated": "Script execution hasn\u2019t terminated: {path:s}", "hook_list_by_invalid": "Invalid property to list hook by", "hook_name_unknown": "Unknown hook name '{name:s}'", "installation_complete": "Installation complete", @@ -168,8 +194,8 @@ "invalid_url_format": "Invalid URL format", "ip6tables_unavailable": "You cannot play with ip6tables here. You are either in a container or your kernel does not support it", "iptables_unavailable": "You cannot play with iptables here. You are either in a container or your kernel does not support it", - "ldap_initialized": "LDAP has been initialized", "ldap_init_failed_to_create_admin": "LDAP initialization failed to create admin user", + "ldap_initialized": "LDAP has been initialized", "license_undefined": "undefined", "mail_alias_remove_failed": "Unable to remove mail alias '{mail:s}'", "mail_domain_unknown": "Unknown mail address domain '{domain:s}'", @@ -177,6 +203,18 @@ "mailbox_used_space_dovecot_down": "Dovecot mailbox service need to be up, if you want to get mailbox used space", "maindomain_change_failed": "Unable to change the main domain", "maindomain_changed": "The main domain has been changed", + "migrations_backward": "Migrating backward.", + "migrations_bad_value_for_target": "Invalide number for target argument, available migrations numbers are 0 or {}", + "migrations_cant_reach_migration_file": "Can't access migrations files at path %s", + "migrations_current_target": "Migration target is {}", + "migrations_error_failed_to_load_migration": "ERROR: failed to load migration {number} {name}", + "migrations_forward": "Migrating forward", + "migrations_loading_migration": "Loading migration {number} {name}...", + "migrations_migration_has_failed": "Migration {number} {name} has failed with exception {exception}, aborting", + "migrations_no_migrations_to_run": "No migrations to run", + "migrations_show_currently_running_migration": "Running migration {number} {name}...", + "migrations_show_last_migration": "Last ran migration is {}", + "migrations_skip_migration": "Skipping migration {number} {name}...", "monitor_disabled": "The server monitoring has been disabled", "monitor_enabled": "The server monitoring has been enabled", "monitor_glances_con_failed": "Unable to connect to Glances server", @@ -224,17 +262,17 @@ "restore_action_required": "You must specify something to restore", "restore_already_installed_app": "An app is already installed with the id '{app:s}'", "restore_app_failed": "Unable to restore the app '{app:s}'", - "restore_removing_tmp_dir_failed": "Unable to remove an old temporary directory", "restore_cleaning_failed": "Unable to clean-up the temporary restoration directory", "restore_complete": "Restore complete", "restore_confirm_yunohost_installed": "Do you really want to restore an already installed system? [{answers:s}]", "restore_extracting": "Extracting needed files from the archive...", "restore_failed": "Unable to restore the system", "restore_hook_unavailable": "Restoration script for '{part:s}' not available on your system and not in the archive either", - "restore_mounting_archive": "Mounting archive into '{path:s}'", "restore_may_be_not_enough_disk_space": "Your system seems not to have enough disk space (freespace: {free_space:d} B, needed space: {needed_space:d} B, security margin: {margin:d} B)", + "restore_mounting_archive": "Mounting archive into '{path:s}'", "restore_not_enough_disk_space": "Not enough disk space (freespace: {free_space:d} B, needed space: {needed_space:d} B, security margin: {margin:d} B)", "restore_nothings_done": "Nothing has been restored", + "restore_removing_tmp_dir_failed": "Unable to remove an old temporary directory", "restore_running_app_script": "Running restore script of app '{app:s}'...", "restore_running_hooks": "Running restoration hooks...", "restore_system_part_failed": "Unable to restore the '{part:s}' system part", @@ -245,13 +283,13 @@ "service_cmd_exec_failed": "Unable to execute command '{command:s}'", "service_conf_file_backed_up": "The configuration file '{conf}' has been backed up to '{backup}'", "service_conf_file_copy_failed": "Unable to copy the new configuration file '{new}' to '{conf}'", + "service_conf_file_kept_back": "The configuration file '{conf}' is expected to be deleted by service {service} but has been kept back.", "service_conf_file_manually_modified": "The configuration file '{conf}' has been manually modified and will not be updated", "service_conf_file_manually_removed": "The configuration file '{conf}' has been manually removed and will not be created", "service_conf_file_remove_failed": "Unable to remove the configuration file '{conf}'", "service_conf_file_removed": "The configuration file '{conf}' has been removed", "service_conf_file_updated": "The configuration file '{conf}' has been updated", "service_conf_new_managed_file": "The configuration file '{conf}' is now managed by the service {service}.", - "service_conf_file_kept_back": "The configuration file '{conf}' is expected to be deleted by service {service} but has been kept back.", "service_conf_up_to_date": "The configuration is already up-to-date for service '{service}'", "service_conf_updated": "The configuration has been updated for service '{service}'", "service_conf_would_be_updated": "The configuration would have been updated for service '{service}'", @@ -304,31 +342,5 @@ "yunohost_ca_creation_success": "The local certification authority has been created.", "yunohost_configured": "YunoHost has been configured", "yunohost_installing": "Installing YunoHost...", - "yunohost_not_installed": "YunoHost is not or not correctly installed. Please execute 'yunohost tools postinstall'", - "domain_cert_gen_failed": "Unable to generate certificate", - "certmanager_attempt_to_replace_valid_cert": "You are attempting to overwrite a good and valid certificate for domain {domain:s}! (Use --force to bypass)", - "certmanager_domain_unknown": "Unknown domain {domain:s}", - "certmanager_domain_cert_not_selfsigned": "The certificate for domain {domain:s} is not self-signed. Are you sure you want to replace it? (Use --force)", - "certmanager_certificate_fetching_or_enabling_failed": "Sounds like enabling the new certificate for {domain:s} failed somehow...", - "certmanager_attempt_to_renew_nonLE_cert": "The certificate for domain {domain:s} is not issued by Let's Encrypt. Cannot renew it automatically!", - "certmanager_attempt_to_renew_valid_cert": "The certificate for domain {domain:s} is not about to expire! Use --force to bypass", - "certmanager_domain_http_not_working": "It seems that the domain {domain:s} cannot be accessed through HTTP. Please check your DNS and nginx configuration is okay", - "certmanager_error_no_A_record": "No DNS 'A' record found for {domain:s}. You need to make your domain name point to your machine to be able to install a Let's Encrypt certificate! (If you know what you are doing, use --no-checks to disable those checks.)", - "certmanager_domain_dns_ip_differs_from_public_ip": "The DNS 'A' record for domain {domain:s} is different from this server IP. If you recently modified your A record, please wait for it to propagate (some DNS propagation checkers are available online). (If you know what you are doing, use --no-checks to disable those checks.)", - "certmanager_domain_not_resolved_locally": "The domain {domain:s} cannot be resolved from inside your Yunohost server. This might happen if you recently modified your DNS record. If so, please wait a few hours for it to propagate. If the issue persists, consider adding {domain:s} to /etc/hosts. (If you know what you are doing, use --no-checks to disable those checks.)", - "certmanager_cannot_read_cert": "Something wrong happened when trying to open current certificate for domain {domain:s} (file: {file:s}), reason: {reason:s}", - "certmanager_cert_install_success_selfsigned": "Successfully installed a self-signed certificate for domain {domain:s}!", - "certmanager_cert_install_success": "Successfully installed Let's Encrypt certificate for domain {domain:s}!", - "certmanager_cert_renew_success": "Successfully renewed Let's Encrypt certificate for domain {domain:s}!", - "certmanager_old_letsencrypt_app_detected": "\nYunohost detected that the 'letsencrypt' app is installed, which conflits with the new built-in certificate management features in Yunohost. If you wish to use the new built-in features, please run the following commands to migrate your installation:\n\n yunohost app remove letsencrypt\n yunohost domain cert-install\n\nN.B.: this will attempt to re-install certificates for all domains with a Let's Encrypt certificate or self-signed certificate", - "certmanager_hit_rate_limit": "Too many certificates already issued for exact set of domains {domain:s} recently. Please try again later. See https://letsencrypt.org/docs/rate-limits/ for more details", - "certmanager_cert_signing_failed": "Signing the new certificate failed", - "certmanager_no_cert_file": "Unable to read certificate file for domain {domain:s} (file: {file:s})", - "certmanager_conflicting_nginx_file": "Unable to prepare domain for ACME challenge: the nginx configuration file {filepath:s} is conflicting and should be removed first", - "domain_cannot_remove_main": "Cannot remove main domain. Set a new main domain first", - "certmanager_self_ca_conf_file_not_found": "Configuration file not found for self-signing authority (file: {file:s})", - "certmanager_acme_not_configured_for_domain": "Certificate for domain {domain:s} does not appear to be correctly installed. Please run cert-install for this domain first.", - "certmanager_http_check_timeout": "Timed out when server tried to contact itself through HTTP using public IP address (domain {domain:s} with ip {ip:s}). You may be experiencing hairpinning issue or the firewall/router ahead of your server is misconfigured.", - "certmanager_couldnt_fetch_intermediate_cert": "Timed out when trying to fetch intermediate certificate from Let's Encrypt. Certificate installation/renewal aborted - please try again later.", - "certmanager_unable_to_parse_self_CA_name": "Unable to parse name of self-signing authority (file: {file:s})" + "yunohost_not_installed": "YunoHost is not or not correctly installed. Please execute 'yunohost tools postinstall'" } diff --git a/src/yunohost/data_migrations/0001_change_cert_group_to_sslcert.py b/src/yunohost/data_migrations/0001_change_cert_group_to_sslcert.py new file mode 100644 index 000000000..cd39df9fa --- /dev/null +++ b/src/yunohost/data_migrations/0001_change_cert_group_to_sslcert.py @@ -0,0 +1,17 @@ +import subprocess +import glob +from yunohost.tools import Migration +from moulinette.utils.filesystem import chown + +class MyMigration(Migration): + "Change certificates group permissions from 'metronome' to 'ssl-cert'" + + all_certificate_files = glob.glob("/etc/yunohost/certs/*/*.pem") + + def forward(self): + for filename in self.all_certificate_files: + chown(filename, uid="root", gid="ssl-cert") + + def backward(self): + for filename in self.all_certificate_files: + chown(filename, uid="root", gid="metronome") diff --git a/src/yunohost/data_migrations/__init__.py b/src/yunohost/data_migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 0183aebd2..6dc3fcba4 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -23,6 +23,7 @@ Specific tools """ +import re import os import yaml import requests @@ -33,6 +34,7 @@ import subprocess import pwd import socket from collections import OrderedDict +from importlib import import_module import apt import apt.progress @@ -40,6 +42,7 @@ import apt.progress from moulinette import msettings, m18n from moulinette.core import MoulinetteError, init_authenticator from moulinette.utils.log import getActionLogger +from moulinette.utils.filesystem import read_json, write_to_json from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, app_list, _install_appslist_fetch_cron from yunohost.domain import domain_add, domain_list, get_public_ip, _get_maindomain, _set_maindomain from yunohost.dyndns import dyndns_subscribe @@ -50,6 +53,7 @@ from yunohost.utils.packages import ynh_packages_version # FIXME this is a duplicate from apps.py APPS_SETTING_PATH= '/etc/yunohost/apps/' +MIGRATIONS_STATE_PATH = "/etc/yunohost/migrations_state.json" logger = getActionLogger('yunohost.tools') @@ -373,6 +377,9 @@ def tools_postinstall(domain, password, ignore_dyndns=False): _install_appslist_fetch_cron() + # Init migrations (skip them, no need to run them on a fresh system) + tools_migrations_migrate(skip=True) + os.system('touch /etc/yunohost/installed') # Enable and start YunoHost firewall at boot time @@ -623,3 +630,184 @@ def tools_port_available(port): return True else: return False + + +def tools_migrations_list(): + """ + List existing migrations + """ + + migrations = {"migrations": []} + + for migration in _get_migrations_list(): + migrations["migrations"].append({ + "number": int(migration.split("_", 1)[0]), + "name": migration.split("_", 1)[1], + "file_name": migration, + }) + + return migrations + + +def tools_migrations_migrate(target=None, skip=False): + """ + Perform migrations + """ + + # state is a datastructure that represents the last run migration + # it has this form: + # { + # "last_run_migration": { + # "number": "00xx", + # "name": "some name", + # } + # } + state = tools_migrations_state() + + last_run_migration_number = state["last_run_migration"]["number"] if state["last_run_migration"] else 0 + + migrations = [] + + # loading all migrations + for migration in tools_migrations_list()["migrations"]: + logger.debug(m18n.n('migrations_loading_migration', + number=migration["number"], + name=migration["name"], + )) + + try: + # this is python builtin method to import a module using a name, we + # use that to import the migration as a python object so we'll be + # able to run it in the next loop + module = import_module("yunohost.data_migrations.{file_name}".format(**migration)) + except Exception: + import traceback + traceback.print_exc() + + raise MoulinetteError(errno.EINVAL, m18n.n('migrations_error_failed_to_load_migration', + number=migration["number"], + name=migration["name"], + )) + break + + migrations.append({ + "number": migration["number"], + "name": migration["name"], + "module": module, + }) + + migrations = sorted(migrations, key=lambda x: x["number"]) + + if not migrations: + logger.info(m18n.n('migrations_no_migrations_to_run')) + return + + all_migration_numbers = [x["number"] for x in migrations] + + if target is None: + target = migrations[-1]["number"] + + # validate input, target must be "0" or a valid number + elif target != 0 and target not in all_migration_numbers: + raise MoulinetteError(errno.EINVAL, m18n.n('migrations_bad_value_for_target', ", ".join(map(str, all_migration_numbers)))) + + logger.debug(m18n.n('migrations_current_target', target)) + + # no new migrations to run + if target == last_run_migration_number: + logger.warn(m18n.n('migrations_no_migrations_to_run')) + return + + logger.debug(m18n.n('migrations_show_last_migration', last_run_migration_number)) + + # we need to run missing migrations + if last_run_migration_number < target: + logger.debug(m18n.n('migrations_forward')) + # drop all already run migrations + migrations = filter(lambda x: target >= x["number"] > last_run_migration_number, migrations) + mode = "forward" + + # we need to go backward on already run migrations + elif last_run_migration_number > target: + logger.debug(m18n.n('migrations_backward')) + # drop all not already run migrations + migrations = filter(lambda x: target < x["number"] <= last_run_migration_number, migrations) + mode = "backward" + + else: # can't happen, this case is handle before + raise Exception() + + # effectively run selected migrations + for migration in migrations: + if not skip: + logger.warn(m18n.n('migrations_show_currently_running_migration', **migration)) + + try: + if mode == "forward": + migration["module"].MyMigration().migrate() + elif mode == "backward": + migration["module"].MyMigration().backward() + else: # can't happen + raise Exception("Illegal state for migration: '%s', should be either 'forward' or 'backward'" % mode) + except Exception as e: + # migration failed, let's stop here but still update state because + # we managed to run the previous ones + logger.error(m18n.n('migrations_migration_has_failed', exception=e, **migration), exc_info=1) + break + + else: # if skip + logger.warn(m18n.n('migrations_skip_migration', **migration)) + + # update the state to include the latest run migration + state["last_run_migration"] = { + "number": migration["number"], + "name": migration["name"], + } + + # special case where we want to go back from the start + if target == 0: + state["last_run_migration"] = None + + write_to_json(MIGRATIONS_STATE_PATH, state) + + +def tools_migrations_state(): + """ + Show current migration state + """ + if not os.path.exists(MIGRATIONS_STATE_PATH): + return {"last_run_migration": None} + + return read_json(MIGRATIONS_STATE_PATH) + + +def _get_migrations_list(): + migrations = [] + + try: + import data_migrations + except ImportError: + # not data migrations present, return empty list + return migrations + + migrations_path = data_migrations.__path__[0] + + if not os.path.exists(migrations_path): + logger.warn(m18n.n('migrations_cant_reach_migration_file', migrations_path)) + return migrations + + for migration in filter(lambda x: re.match("^\d+_[a-zA-Z0-9_]+\.py$", x), os.listdir(migrations_path)): + migrations.append(migration[:-len(".py")]) + + return sorted(migrations) + + +class Migration(object): + def migrate(self): + self.forward() + + def forward(self): + raise NotImplementedError() + + def backward(self): + pass