diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 09b5687f3..cccc1adbb 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -748,18 +748,25 @@ backup: full: --no-compress help: Do not create an archive file action: store_true - --hooks: - help: List of backup hooks names to execute + --system: + help: List of system parts to backup (all by default) nargs: "*" - --ignore-hooks: - help: Do not execute backup hooks - action: store_true --apps: - help: List of application names to backup + help: List of application names to backup (all by default) nargs: "*" + --hooks: + help: (Deprecated) See --system + nargs: "*" + --ignore-system: + help: Do not backup system + action: store_true --ignore-apps: help: Do not backup apps action: store_true + --ignore-hooks: + help: (Deprecated) See --ignore-system + action: store_true + ### backup_restore() restore: @@ -772,17 +779,23 @@ backup: arguments: name: help: Name of the local backup archive - --hooks: - help: List of restauration hooks names to execute + --system: + help: List of system parts to restore (all by default) nargs: "*" --apps: - help: List of application names to restore + help: List of application names to restore (all by default) nargs: "*" + --hooks: + help: (Deprecated) See --system + nargs: "*" + --ignore-system: + help: Do not restore system parts + action: store_true --ignore-apps: help: Do not restore apps action: store_true --ignore-hooks: - help: Do not restore hooks + help: (Deprecated) See --ignore-system action: store_true --force: help: Force restauration on an already installed system diff --git a/data/helpers.d/filesystem b/data/helpers.d/filesystem index f0f3afb06..6fb073e06 100644 --- a/data/helpers.d/filesystem +++ b/data/helpers.d/filesystem @@ -1,57 +1,205 @@ CAN_BIND=${CAN_BIND:-1} -# Mark a file or a directory for backup -# Note: currently, SRCPATH will be copied or binded to DESTPATH +# Add a file or a directory to the list of paths to backup +# +# Note: this helper could be used in backup hook or in backup script inside an +# app package +# +# Details: ynh_backup writes SRC and the relative DEST into a CSV file. And it +# creates the parent destination directory +# +# If DEST is ended by a slash it complete this path with the basename of SRC. +# +# usage: ynh_backup src [dest [is_big [arg]]] +# | arg: src - file or directory to bind or symlink or copy. it shouldn't be in +# the backup dir. +# | arg: dest - destination file or directory inside the +# backup dir +# | arg: is_big - 1 to indicate data are big (mail, video, image ...) +# | arg: arg - Deprecated arg +# +# example: +# # Wordpress app context +# +# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" +# # => This line will be added into CSV file +# # "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/etc/nginx/conf.d/$domain.d/$app.conf" +# +# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf/nginx.conf" +# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/nginx.conf" +# +# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf/" +# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/$app.conf" +# +# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf" +# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf" +# +# #Deprecated usages (maintained for retro-compatibility) +# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "${backup_dir}/conf/nginx.conf" +# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/nginx.conf" +# +# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "/conf/" +# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/$app.conf" # -# usage: ynh_backup srcdir destdir to_bind no_root -# | arg: srcdir - directory to bind or copy -# | arg: destdir - mountpoint or destination directory -# | arg: to_bind - 1 to bind mounting the directory if possible -# | arg: no_root - 1 to execute commands as current user ynh_backup() { - local SRCPATH=$1 - local DESTPATH=$2 - local TO_BIND=${3:-0} - local SUDO_CMD="sudo" - [[ "${4:-}" = "1" ]] && SUDO_CMD= + # TODO find a way to avoid injection by file strange naming ! + local SRC_PATH="$1" + local DEST_PATH="${2:-}" + local IS_BIG="${3:-0}" - # validate arguments - [[ -e "${SRCPATH}" ]] || { - echo "Source path '${SRCPATH}' does not exist" >&2 + # ============================================================================== + # Format correctly source and destination paths + # ============================================================================== + # Be sure the source path is not empty + [[ -e "${SRC_PATH}" ]] || { + echo "Source path '${SRC_PATH}' does not exist" >&2 return 1 } - # prepend the backup directory - [[ -n "${YNH_APP_BACKUP_DIR:-}" && "${DESTPATH:0:1}" != "/" ]] \ - && DESTPATH="${YNH_APP_BACKUP_DIR}/${DESTPATH}" - [[ ! -e "${DESTPATH}" ]] || { - echo "Destination path '${DESTPATH}' already exist" >&2 - return 1 - } + # Transform the source path as an absolute path + # If it's a dir remove the ending / + SRC_PATH=$(realpath "$SRC_PATH") - # attempt to bind mounting the directory - if [[ "${CAN_BIND}" = "1" && "${TO_BIND}" = "1" ]]; then - eval $SUDO_CMD mkdir -p "${DESTPATH}" + # If there is no destination path, initialize it with the source path + # relative to "/". + # eg: SRC_PATH=/etc/yunohost -> DEST_PATH=etc/yunohost + if [[ -z "$DEST_PATH" ]]; then - if sudo mount --rbind "${SRCPATH}" "${DESTPATH}"; then - # try to remount destination directory as read-only - sudo mount -o remount,ro,bind "${SRCPATH}" "${DESTPATH}" \ - || true - return 0 - else - CAN_BIND=0 - echo "Bind mounting seems to be disabled on your system." - echo "You have maybe to check your apparmor configuration." + DEST_PATH="${SRC_PATH#/}" + + else + if [[ "${DEST_PATH:0:1}" == "/" ]]; then + + # If the destination path is an absolute path, transform it as a path + # relative to the current working directory ($YNH_CWD) + # + # If it's an app backup script that run this helper, YNH_CWD is equal to + # $YNH_BACKUP_DIR/apps/APP_INSTANCE_NAME/backup/ + # + # If it's a system part backup script, YNH_CWD is equal to $YNH_BACKUP_DIR + DEST_PATH="${DEST_PATH#$YNH_CWD/}" + + # Case where $2 is an absolute dir but doesn't begin with $YNH_CWD + [[ "${DEST_PATH:0:1}" == "/" ]] \ + && DEST_PATH="${DEST_PATH#/}" fi - # delete mountpoint directory safely - mountpoint -q "${DESTPATH}" && sudo umount -R "${DESTPATH}" - eval $SUDO_CMD rm -rf "${DESTPATH}" + # Complete DEST_PATH if ended by a / + [[ "${DEST_PATH: -1}" == "/" ]] \ + && DEST_PATH="${DEST_PATH}/$(basename $SRC_PATH)" fi - # ... or just copy the directory - eval $SUDO_CMD mkdir -p $(dirname "${DESTPATH}") - eval $SUDO_CMD cp -a "${SRCPATH}" "${DESTPATH}" + # Check if DEST_PATH already exists in tmp archive + [[ ! -e "${DEST_PATH}" ]] || { + echo "Destination path '${DEST_PATH}' already exist" >&2 + return 1 + } + + # Add the relative current working directory to the destination path + local REL_DIR="${YNH_CWD#$YNH_BACKUP_DIR}" + REL_DIR="${REL_DIR%/}/" + DEST_PATH="${REL_DIR}${DEST_PATH}" + DEST_PATH="${DEST_PATH#/}" + # ============================================================================== + + # ============================================================================== + # Write file to backup into backup_list + # ============================================================================== + local SRC=$(echo "${SRC_PATH}" | sed -r 's/"/\"\"/g') + local DEST=$(echo "${DEST_PATH}" | sed -r 's/"/\"\"/g') + echo "\"${SRC}\",\"${DEST}\"" >> "${YNH_BACKUP_CSV}" + + # ============================================================================== + + # Create the parent dir of the destination path + # It's for retro compatibility, some script consider ynh_backup creates this dir + mkdir -p $(dirname "$YNH_BACKUP_DIR/${DEST_PATH}") +} + +# Restore all files linked to the restore hook or to the restore app script +# +# usage: ynh_restore +# +ynh_restore () { + # Deduce the relative path of $YNH_CWD + local REL_DIR="${YNH_CWD#$YNH_BACKUP_DIR/}" + REL_DIR="${REL_DIR%/}/" + + # For each destination path begining by $REL_DIR + cat ${YNH_BACKUP_CSV} | tr -d $'\r' | grep -ohP "^\".*\",\"$REL_DIR.*\"$" | \ + while read line; do + local ORIGIN_PATH=$(echo "$line" | grep -ohP "^\"\K.*(?=\",\".*\"$)") + local ARCHIVE_PATH=$(echo "$line" | grep -ohP "^\".*\",\"$REL_DIR\K.*(?=\"$)") + ynh_restore_file "$ARCHIVE_PATH" "$ORIGIN_PATH" + done +} + +# Return the path in the archive where has been stocked the origin path +# +# usage: _get_archive_path ORIGIN_PATH +_get_archive_path () { + # For security reasons we use csv python library to read the CSV + sudo python -c " +import sys +import csv +with open(sys.argv[1], 'r') as backup_file: + backup_csv = csv.DictReader(backup_file, fieldnames=['source', 'dest']) + for row in backup_csv: + if row['source']==sys.argv[2].strip('\"'): + print row['dest'] + sys.exit(0) + raise Exception('Original path for %s not found' % sys.argv[2]) + " "${YNH_BACKUP_CSV}" "$1" + return $? +} + +# Restore a file or a directory +# +# Use the registered path in backup_list by ynh_backup to restore the file at +# the good place. +# +# usage: ynh_restore_file ORIGIN_PATH [ DEST_PATH ] +# | arg: ORIGIN_PATH - Path where was located the file or the directory before +# to be backuped or relative path to $YNH_CWD where it is located in the backup archive +# | arg: DEST_PATH - Path where restore the file or the dir, if unspecified, +# the destination will be ORIGIN_PATH or if the ORIGIN_PATH doesn't exist in +# the archive, the destination will be searched into backup.csv +# +# examples: +# ynh_restore_file "/etc/nginx/conf.d/$domain.d/$app.conf" +# # if apps/wordpress/etc/nginx/conf.d/$domain.d/$app.conf exists, restore it into +# # /etc/nginx/conf.d/$domain.d/$app.conf +# # if no, search a correspondance in the csv (eg: conf/nginx.conf) and restore it into +# # /etc/nginx/conf.d/$domain.d/$app.conf +# +# # DON'T GIVE THE ARCHIVE PATH: +# ynh_restore_file "conf/nginx.conf" +# +ynh_restore_file () { + local ORIGIN_PATH="/${1#/}" + local ARCHIVE_PATH="$YNH_CWD${ORIGIN_PATH}" + # Default value for DEST_PATH = /$ORIGIN_PATH + local DEST_PATH="${2:-$ORIGIN_PATH}" + + # If ARCHIVE_PATH doesn't exist, search for a corresponding path in CSV + if [ ! -d "$ARCHIVE_PATH" ] && [ ! -f "$ARCHIVE_PATH" ] && [ ! -L "$ARCHIVE_PATH" ]; then + ARCHIVE_PATH="$YNH_BACKUP_DIR/$(_get_archive_path \"$ORIGIN_PATH\")" + fi + + # Restore ORIGIN_PATH into DEST_PATH + mkdir -p $(dirname "$DEST_PATH") + + # Do a copy if it's just a mounting point + if mountpoint -q $YNH_BACKUP_DIR; then + if [[ -d "${ARCHIVE_PATH}" ]]; then + ARCHIVE_PATH="${ARCHIVE_PATH}/." + mkdir -p "$DEST_PATH" + fi + cp -a "$ARCHIVE_PATH" "${DEST_PATH}" + # Do a move if YNH_BACKUP_DIR is already a copy + else + mv "$ARCHIVE_PATH" "${DEST_PATH}" + fi } # Deprecated helper since it's a dangerous one! @@ -60,7 +208,7 @@ ynh_bind_or_cp() { local NO_ROOT=0 [[ "${AS_ROOT}" = "1" ]] || NO_ROOT=1 echo "This helper is deprecated, you should use ynh_backup instead" >&2 - ynh_backup "$1" "$2" 1 "$NO_ROOT" + ynh_backup "$1" "$2" 1 } # Create a directory under /tmp diff --git a/data/helpers.d/package b/data/helpers.d/package index b292a6c8e..66982cf25 100644 --- a/data/helpers.d/package +++ b/data/helpers.d/package @@ -121,7 +121,7 @@ ynh_install_app_dependencies () { if ynh_package_is_installed "${dep_app}-ynh-deps"; then echo "A package named ${dep_app}-ynh-deps is already installed" >&2 else - cat > ./${dep_app}-ynh-deps.control << EOF # Make a control file for equivs-build + cat > /tmp/${dep_app}-ynh-deps.control << EOF # Make a control file for equivs-build Section: misc Priority: optional Package: ${dep_app}-ynh-deps @@ -131,8 +131,9 @@ Architecture: all Description: Fake package for ${app} (YunoHost app) dependencies This meta-package is only responsible of installing its dependencies. EOF - ynh_package_install_from_equivs ./${dep_app}-ynh-deps.control \ + ynh_package_install_from_equivs /tmp/${dep_app}-ynh-deps.control \ || ynh_die "Unable to install dependencies" # Install the fake package and its dependencies + rm /tmp/${dep_app}-ynh-deps.control ynh_app_setting_set $app apt_dependencies $dependencies fi } diff --git a/data/hooks/post_backup_create/99-umount b/data/hooks/post_backup_create/99-umount deleted file mode 100644 index a9ad5efec..000000000 --- a/data/hooks/post_backup_create/99-umount +++ /dev/null @@ -1,13 +0,0 @@ - -tmp_dir=$1 -retcode=$2 - -FAILURE=0 - -# Iterate over inverted ordered mountpoints to prevent issues -for m in $(mount | grep " ${tmp_dir}" | awk '{ print $3 }' | tac); do - sudo umount $m - [[ $? != 0 ]] && FAILURE=1 -done - -exit $FAILURE diff --git a/data/hooks/restore/11-conf_ynh_mysql b/data/hooks/restore/11-conf_ynh_mysql index b2f8c8e31..8b8438c0e 100644 --- a/data/hooks/restore/11-conf_ynh_mysql +++ b/data/hooks/restore/11-conf_ynh_mysql @@ -6,9 +6,13 @@ service mysql status >/dev/null 2>&1 \ # retrieve current and new password [ -f /etc/yunohost/mysql ] \ - && curr_pwd=$(sudo cat /etc/yunohost/mysql) \ - || curr_pwd="yunohost" + && curr_pwd=$(sudo cat /etc/yunohost/mysql) new_pwd=$(sudo cat "${backup_dir}/root_pwd" || sudo cat "${backup_dir}/mysql") +[ -z "$curr_pwd" ] && curr_pwd="yunohost" +[ -z "$new_pwd" ] && { + . /usr/share/yunohost/helpers.d/string + new_pwd=$(ynh_string_random 10) +} # attempt to change it sudo mysqladmin -s -u root -p"$curr_pwd" password "$new_pwd" || { diff --git a/debian/control b/debian/control index 3be8b917e..e9a0a3d47 100644 --- a/debian/control +++ b/debian/control @@ -26,6 +26,7 @@ Depends: ${python:Depends}, ${misc:Depends} , ssowat, metronome , rspamd (>= 1.2.0), rmilter (>=1.7.0), redis-server, opendkim-tools , haveged + , archivemount Recommends: yunohost-admin , openssh-server, ntp, inetutils-ping | iputils-ping , bash-completion, rsyslog, etckeeper diff --git a/locales/en.json b/locales/en.json index 2ba2a48f6..e4f785002 100644 --- a/locales/en.json +++ b/locales/en.json @@ -54,29 +54,54 @@ "ask_main_domain": "Main domain", "ask_new_admin_password": "New administration password", "ask_password": "Password", + "backup_abstract_method": "This backup method hasn't yet been implemented", "backup_action_required": "You must specify something to save", "backup_app_failed": "Unable to back up the app '{app:s}'", + "backup_applying_method_tar": "Creating the backup tar archive...", + "backup_applying_method_copy": "Copying all files to backup...", + "backup_applying_method_borg": "Sending all files to backup into borg-backup repository...", + "backup_applying_method_custom": "Calling the custom backup method '{method:s}'...", "backup_archive_app_not_found": "App '{app:s}' not found in the backup archive", "backup_archive_broken_link": "Unable to access backup archive (broken link to {path:s})", - "backup_archive_hook_not_exec": "Hook '{hook:s}' not executed in this backup", + "backup_archive_system_part_not_available": "System part '{part:s}' not available in this backup", "backup_archive_name_exists": "The backup's archive name already exists", "backup_archive_name_unknown": "Unknown local backup archive named '{name:s}'", "backup_archive_open_failed": "Unable to open the backup archive", + "backup_archive_mount_failed": "Mounting the backup archive failed", + "backup_archive_writing_error": "Unable to add files to backup into the compressed archive", + "backup_ask_for_copying_if_needed": "Your system don't support completely the quick method to organize files in the archive, do you want to organize its by copying {size:s}MB?", + "backup_borg_not_implemented": "Borg backup method is not yet implemented", + "backup_cant_mount_uncompress_archive": "Unable to mount in readonly mode the uncompress archive directory", "backup_cleaning_failed": "Unable to clean-up the temporary backup directory", + "backup_copying_to_organize_the_archive": "Copying {size:s}MB to organize the archive", "backup_created": "Backup created", "backup_creating_archive": "Creating the backup archive...", "backup_creation_failed": "Backup creation failed", + "backup_csv_creation_failed": "Unable to create the CSV file needed for future restore operations", + "backup_csv_addition_failed": "Unable to add files to backup into the CSV file", + "backup_custom_need_mount_error": "Custom backup method failure on 'need_mount' step", + "backup_custom_backup_error": "Custom backup method failure on 'backup' step", + "backup_custom_mount_error": "Custom backup method failure on 'mount' step", "backup_delete_error": "Unable to delete '{path:s}'", "backup_deleted": "The backup has been deleted", "backup_extracting_archive": "Extracting the backup archive...", "backup_hook_unknown": "Backup hook '{hook:s}' unknown", "backup_invalid_archive": "Invalid backup archive", + "backup_no_uncompress_archive_dir": "Uncompress archive directory doesn't exist", "backup_nothings_done": "There is nothing to save", + "backup_method_tar_finished": "Backup tar archive created", + "backup_method_copy_finished": "Backup copy finished", + "backup_method_borg_finished": "Backup into borg finished", + "backup_method_custom_finished": "Custom backup method '{method:s}' finished", "backup_output_directory_forbidden": "Forbidden output directory. Backups can't be created in /bin, /boot, /dev, /etc, /lib, /root, /run, /sbin, /sys, /usr, /var or /home/yunohost.backup/archives sub-folders", "backup_output_directory_not_empty": "The output directory is not empty", "backup_output_directory_required": "You must provide an output directory for the backup", "backup_running_app_script": "Running backup script of app '{app:s}'...", "backup_running_hooks": "Running backup hooks...", + "backup_system_part_failed": "Unable to backup the '{part:s}' system part", + "backup_unable_to_organize_files": "Unable to organize files in the archive with the quick method", + "backup_with_no_backup_script_for_app": "App {app:s} has no backup script. Ignoring.", + "backup_with_no_restore_script_for_app": "App {app:s} has no restore script, you won't be able to automatically restore the backup of this app.", "custom_app_url_required": "You must provide a URL to upgrade your custom app {app:s}", "custom_appslist_name_required": "You must provide a name for your custom app list", "diagnosis_debian_version_error": "Can't retrieve the Debian version: {error}", @@ -199,14 +224,20 @@ "restore_action_required": "You must specify something to restore", "restore_already_installed_app": "An app is already installed with the id '{app:s}'", "restore_app_failed": "Unable to restore the app '{app:s}'", + "restore_removing_tmp_dir_failed": "Unable to remove an old temporary directory", "restore_cleaning_failed": "Unable to clean-up the temporary restoration directory", "restore_complete": "Restore complete", "restore_confirm_yunohost_installed": "Do you really want to restore an already installed system? [{answers:s}]", + "restore_extracting": "Extracting needed files from the archive...", "restore_failed": "Unable to restore the system", - "restore_hook_unavailable": "Restoration hook '{hook:s}' not available on your system", + "restore_hook_unavailable": "Restoration script for '{part:s}' not available on your system and not in the archive either", + "restore_mounting_archive": "Mounting archive into '{path:s}'", + "restore_may_be_not_enough_disk_space": "Your system seems not to have enough disk space (freespace: {free_space:d} B, needed space: {needed_space:d} B, security margin: {margin:d} B)", + "restore_not_enough_disk_space": "Not enough disk space (freespace: {free_space:d} B, needed space: {needed_space:d} B, security margin: {margin:d} B)", "restore_nothings_done": "Nothing has been restored", "restore_running_app_script": "Running restore script of app '{app:s}'...", "restore_running_hooks": "Running restoration hooks...", + "restore_system_part_failed": "Unable to restore the '{part:s}' system part", "service_add_failed": "Unable to add service '{service:s}'", "service_added": "The service '{service:s}' has been added", "service_already_started": "Service '{service:s}' has already been started", diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 725bf2fda..03598ec48 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -31,6 +31,8 @@ import time import tarfile import shutil import subprocess +import csv +import tempfile from glob import glob from collections import OrderedDict @@ -39,367 +41,2073 @@ from moulinette.utils import filesystem from moulinette.utils.log import getActionLogger from yunohost.app import ( - app_info, app_ssowatconf, _is_installed, _parse_app_instance_name + app_info, _is_installed, _parse_app_instance_name ) from yunohost.hook import ( - hook_info, hook_callback, hook_exec, CUSTOM_HOOK_FOLDER + hook_list, hook_info, hook_callback, hook_exec, CUSTOM_HOOK_FOLDER ) from yunohost.monitor import binary_to_human from yunohost.tools import tools_postinstall +from yunohost.service import service_regen_conf BACKUP_PATH = '/home/yunohost.backup' ARCHIVES_PATH = '%s/archives' % BACKUP_PATH - +APP_MARGIN_SPACE_SIZE = 100 # In MB +CONF_MARGIN_SPACE_SIZE = 10 # IN MB +POSTINSTALL_ESTIMATE_SPACE_SIZE = 5 # In MB +MB_ALLOWED_TO_ORGANIZE = 10 logger = getActionLogger('yunohost.backup') -def backup_create(name=None, description=None, output_directory=None, - no_compress=False, ignore_hooks=False, hooks=[], - ignore_apps=False, apps=[]): +class BackupRestoreTargetsManager(object): + """ + BackupRestoreTargetsManager manage the targets + in BackupManager and RestoreManager + """ + + def __init__(self): + + self.targets = {} + self.results = { + "system": {}, + "apps": {} + } + + + def set_result(self, category, element, value): + """ + Change (or initialize) the current status/result of a given target. + + Args: + category -- The category of the target + + element -- The target for which to change the status/result + + value -- The new status/result, among "Unknown", "Success", + "Warning", "Error" and "Skipped" + """ + + levels = [ "Unknown", "Success", "Warning", "Error", "Skipped" ] + + assert value in levels + + if element not in self.results[category].keys(): + self.results[category][element] = value + else: + currentValue = self.results[category][element] + if (levels.index(currentValue) > levels.index(value)): + return + else: + self.results[category][element] = value + + + def set_wanted(self, category, + wanted_targets, available_targets, + error_if_wanted_target_is_unavailable): + """ + Define and validate targets to be backuped or to be restored (list of + system parts, apps..). The wanted targets are compared and filtered + with respect to the available targets. If a wanted targets is not + available, a call to "error_if_wanted_target_is_unavailable" is made. + + Args: + category -- The category (apps or system) for which to set the + targets ; + + wanted_targets -- List of targets which are wanted by the user. Can be + "None" or [], corresponding to "No targets" or "All + targets" ; + + available_targets -- List of targets which are really available ; + + error_if_wanted_target_is_unavailable + -- Callback for targets which are not available. + """ + + # If no targets wanted, set as empty list + if wanted_targets is None: + self.targets[category] = [] + + # If all targets wanted, use all available targets + elif wanted_targets == []: + self.targets[category] = available_targets + + # If the user manually specified which targets to backup, we need to + # validate that each target is actually available + else: + self.targets[category] = [part for part in wanted_targets + if part in available_targets] + + # Display an error for each target asked by the user but which is + # unknown + unavailable_targets = [part for part in wanted_targets + if part not in available_targets] + + for target in unavailable_targets: + self.set_result(category, target, "Skipped") + error_if_wanted_target_is_unavailable(target) + + # For target with no result yet (like 'Skipped'), set it as unknown + if self.targets[category] is not None: + for target in self.targets[category]: + self.set_result(category, target, "Unknown") + + return self.list(category, exclude=["Skipped"]) + + + def list(self, category, include=None, exclude=None): + """ + List targets in a given category. + + The list is filtered with a whitelist (include) or blacklist (exclude) + with respect to the current 'result' of the target. + """ + + assert (include and isinstance(include, list) and not exclude) \ + or (exclude and isinstance(exclude, list) and not include) + + if include: + return [target for target in self.targets[category] + if self.results[category][target] in include] + + if exclude: + return [target for target in self.targets[category] + if self.results[category][target] not in exclude] + + +class BackupManager(): + """ + This class collect files to backup in a list and apply one or several + backup method on it. + + The list contains dict with source and dest properties. The goal of this csv + is to list all directories and files which need to be backup in this + archive. The `source` property is the path of the source (dir or file). + The `dest` property is the path where it could be placed in the archive. + + The list is filled by app backup scripts and system/user backup hooks. + Files located in the work_dir are automatically added. + + With this list, "backup methods" are able to apply their backup strategy on + data listed in it. It's possible to tar each path (tar methods), to mount + each dir into the work_dir, to copy each files (copy method) or to call a + custom method (via a custom script). + + Note: some future backups methods (like borg) are not able to specify a + different place than the original path. That's why the ynh_restore_file + helpers use primarily the SOURCE_PATH as argument. + + Public properties: + info (getter) + work_dir (getter) # FIXME currently it's not a getter + is_tmp_work_dir (getter) + paths_to_backup (getter) # FIXME not a getter and list is not protected + name (getter) # FIXME currently it's not a getter + size (getter) # FIXME currently it's not a getter + + Public methods: + add(self, method) + set_system_targets(self, system_parts=[]) + set_apps_targets(self, apps=[]) + collect_files(self) + backup(self) + + Usage: + backup_manager = BackupManager(name="mybackup", description="bkp things") + + # Add backup method to apply + backup_manager.add(BackupMethod.create('copy','/mnt/local_fs')) + backup_manager.add(BackupMethod.create('tar','/mnt/remote_fs')) + + # Define targets to be backuped + backup_manager.set_system_targets(["data"]) + backup_manager.set_apps_targets(["wordpress"]) + + # Collect files to backup from targets + backup_manager.collect_files() + + # Apply backup methods + backup_manager.backup() + """ + + def __init__(self, name=None, description='', work_dir=None): + """ + BackupManager constructor + + Args: + name -- (string) The name of this backup (without spaces). If + None, the name will be generated (default: None) + + description -- (string) A description for this future backup archive + (default: '') + + work_dir -- (None|string) A path where prepare the archive. If None, + temporary work_dir will be created (default: None) + """ + self.description = description or '' + self.created_at = int(time.time()) + self.apps_return = {} + self.system_return = {} + self.methods = [] + self.paths_to_backup = [] + self.size_details = { + 'system': {}, + 'apps': {} + } + self.targets = BackupRestoreTargetsManager() + + + # Define backup name if needed + if not name: + name = self._define_backup_name() + self.name = name + + # Define working directory if needed and initialize it + self.work_dir = work_dir + if self.work_dir is None: + self.work_dir = os.path.join(BACKUP_PATH, 'tmp', name) + self._init_work_dir() + + ########################################################################### + # Misc helpers # + ########################################################################### + + @property + def info(self): + """(Getter) Dict containing info about the archive being created""" + return { + 'description': self.description, + 'created_at': self.created_at, + 'size': self.size, + 'size_details': self.size_details, + 'apps': self.apps_return, + 'system': self.system_return + } + + @property + def is_tmp_work_dir(self): + """(Getter) Return true if the working directory is temporary and should + be clean at the end of the backup""" + return self.work_dir == os.path.join(BACKUP_PATH, 'tmp', self.name) + + def __repr__(self): + return json.dumps(self.info) + + def _define_backup_name(self): + """Define backup name + + Return: + (string) A backup name created from current date 'YYMMDD-HHMMSS' + """ + # FIXME: case where this name already exist + return time.strftime('%Y%m%d-%H%M%S') + + def _init_work_dir(self): + """Initialize preparation directory + + Ensure the working directory exists and is empty + + exception: + backup_output_directory_not_empty -- (MoulinetteError) Raised if the + directory was given by the user and isn't empty + + (TODO) backup_cant_clean_tmp_working_directory -- (MoulinetteError) + Raised if the working directory isn't empty, is temporary and can't + be automaticcaly cleaned + + (TODO) backup_cant_create_working_directory -- (MoulinetteError) Raised + if iyunohost can't create the working directory + """ + + # FIXME replace isdir by exists ? manage better the case where the path + # exists + if not os.path.isdir(self.work_dir): + filesystem.mkdir(self.work_dir, 0750, parents=True, uid='admin') + elif self.is_tmp_work_dir: + logger.debug("temporary directory for backup '%s' already exists", + self.work_dir) + # FIXME May be we should clean the workdir here + raise MoulinetteError( + errno.EIO, m18n.n('backup_output_directory_not_empty')) + + ########################################################################### + # Backup target management # + ########################################################################### + + def set_system_targets(self, system_parts=[]): + """ + Define and validate targetted apps to be backuped + + Args: + system_parts -- (list) list of system parts which should be backuped. + If empty list, all system will be backuped. If None, + no system parts will be backuped. + """ + def unknown_error(part): + logger.error(m18n.n('backup_hook_unknown', hook=part)) + + self.targets.set_wanted("system", + system_parts, hook_list('backup')["hooks"], + unknown_error) + + + def set_apps_targets(self, apps=[]): + """ + Define and validate targetted apps to be backuped + + Args: + apps -- (list) list of apps which should be backuped. If given an empty + list, all apps will be backuped. If given None, no apps will be + backuped. + """ + def unknown_error(app): + logger.error(m18n.n('unbackup_app', app=app)) + + target_list = self.targets.set_wanted("apps", apps, + os.listdir('/etc/yunohost/apps'), + unknown_error) + + # Additionnaly, we need to check that each targetted app has a + # backup and restore scripts + + for app in target_list: + app_script_folder = "/etc/yunohost/apps/%s/scripts" % app + backup_script_path = os.path.join(app_script_folder, "backup") + restore_script_path = os.path.join(app_script_folder, "restore") + + if not os.path.isfile(backup_script_path): + logger.warning(m18n.n('backup_with_no_backup_script_for_app', app=app)) + self.targets.set_result("apps", app, "Skipped") + + elif not os.path.isfile(restore_script_path): + logger.warning(m18n.n('backup_with_no_restore_script_for_app', app=app)) + self.targets.set_result("apps", app, "Warning") + + + ########################################################################### + # Management of files to backup / "The CSV" # + ########################################################################### + + def _import_to_list_to_backup(self, tmp_csv): + """ + Commit collected path from system hooks or app scripts + + Args: + tmp_csv -- (string) Path to a temporary csv file with source and + destinations column to add to the list of paths to backup + """ + _call_for_each_path(self, BackupManager._add_to_list_to_backup, tmp_csv) + + + def _add_to_list_to_backup(self, source, dest=None): + """ + Mark file or directory to backup + + This method add source/dest couple to the "paths_to_backup" list. + + Args: + source -- (string) Source path to backup + + dest -- (string) Destination path in the archive. If it ends by a + slash the basename of the source path will be added. If None, + the source path will be used, so source files will be set up + at the same place and with same name than on the system. + (default: None) + + Usage: + self._add_to_list_to_backup('/var/www/wordpress', 'sources') + # => "wordpress" dir will be move and rename as "sources" + + self._add_to_list_to_backup('/var/www/wordpress', 'sources/') + # => "wordpress" dir will be put inside "sources/" and won't be renamed + + """ + if dest is None: + dest = source + source = os.path.join(self.work_dir, source) + if dest.endswith("/"): + dest = os.path.join(dest, os.path.basename(source)) + self.paths_to_backup.append({'source': source, 'dest': dest}) + + + def _write_csv(self): + """ + Write the backup list into a CSV + + The goal of this csv is to list all directories and files which need to + be backup in this archive. For the moment, this CSV contains 2 columns. + The first column `source` is the path of the source (dir or file). The + second `dest` is the path where it could be placed in the archive. + + This CSV is filled by app backup scripts and system/user hooks. + Files in the work_dir are automatically added. + + With this CSV, "backup methods" are able to apply their backup strategy + on data listed in it. It's possible to tar each path (tar methods), to + mount each dir into the work_dir, to copy each files (copy methods) or + a custom method (via a custom script). + + Note: some future backups methods (like borg) are not able to specify a + different place than the original path. That's why the ynh_restore_file + helpers use primarily the SOURCE_PATH as argument. + + Error: + backup_csv_creation_failed -- Raised if the CSV couldn't be created + backup_csv_addition_failed -- Raised if we can't write in the CSV + """ + self.csv_path = os.path.join(self.work_dir, 'backup.csv') + try: + self.csv_file = open(self.csv_path, 'a') + self.fieldnames = ['source', 'dest'] + self.csv = csv.DictWriter(self.csv_file, fieldnames=self.fieldnames, + quoting=csv.QUOTE_ALL) + except (IOError, OSError, csv.Error): + logger.error(m18n.n('backup_csv_creation_failed')) + + for row in self.paths_to_backup: + try: + self.csv.writerow(row) + except csv.Error: + logger.error(m18n.n('backup_csv_addition_failed')) + self.csv_file.close() + + + ########################################################################### + # File collection from system parts and apps # + ########################################################################### + + def collect_files(self): + """ + Collect all files to backup, write its into a CSV and create a + info.json file + + Files to backup are listed by system parts backup hooks and by backup + app scripts that have been defined with the set_targets() method. + + Some files or directories inside the working directory are added by + default: + + info.json -- info about the archive + backup.csv -- a list of paths to backup + apps/ -- some apps generate here temporary files to backup (like + database dump) + conf/ -- system configuration backup scripts could generate here + temporary files to backup + data/ -- system data backup scripts could generate here temporary + files to backup + hooks/ -- restore scripts associated to system backup scripts are + copied here + + Exceptions: + "backup_nothings_done" -- (MoulinetteError) This exception is raised if + nothing has been listed. + """ + + self._collect_system_files() + self._collect_apps_files() + + # Check if something has been saved ('success' or 'warning') + successfull_apps = self.targets.list("apps", include=["Success", "Warning"]) + successfull_system = self.targets.list("system", include=["Success", "Warning"]) + + if not successfull_apps and not successfull_system: + filesystem.rm(self.work_dir, True, True) + raise MoulinetteError(errno.EINVAL, m18n.n('backup_nothings_done')) + + + # Add unlisted files from backup tmp dir + self._add_to_list_to_backup('backup.csv') + self._add_to_list_to_backup('info.json') + if len(self.apps_return) > 0: + self._add_to_list_to_backup('apps') + if os.path.isdir(os.path.join(self.work_dir, 'conf')): + self._add_to_list_to_backup('conf') + if os.path.isdir(os.path.join(self.work_dir, 'data')): + self._add_to_list_to_backup('data') + + # Write CSV file + self._write_csv() + + # Calculate total size + self._compute_backup_size() + + # Create backup info file + with open("%s/info.json" % self.work_dir, 'w') as f: + f.write(json.dumps(self.info)) + + + def _get_env_var(self, app=None): + """ + Define environment variables for apps or system backup scripts. + + Args: + app -- (string|None) The instance name of the app we want the variable + environment. If you want a variable environment for a system backup + script keep None. (default: None) + + Return: + (Dictionnary) The environment variables to apply to the script + """ + env_var = {} + + _, tmp_csv = tempfile.mkstemp(prefix='backupcsv_') + env_var['YNH_BACKUP_DIR'] = self.work_dir + env_var['YNH_BACKUP_CSV'] = tmp_csv + + if app is not None: + app_id, app_instance_nb = _parse_app_instance_name(app) + env_var["YNH_APP_ID"] = app_id + env_var["YNH_APP_INSTANCE_NAME"] = app + env_var["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb) + tmp_app_dir = os.path.join('apps/', app) + tmp_app_bkp_dir = os.path.join(self.work_dir, tmp_app_dir, 'backup') + env_var["YNH_APP_BACKUP_DIR"] = tmp_app_bkp_dir + + return env_var + + + def _collect_system_files(self): + """ + List file to backup for each selected system part + + This corresponds to scripts in data/hooks/backup/ (system hooks) and + to those in /etc/yunohost/hooks.d/backup/ (user hooks) + + Environment variables: + YNH_BACKUP_DIR -- The backup working directory (in + "/home/yunohost.backup/tmp/BACKUPNAME" or could be + defined by the user) + YNH_BACKUP_CSV -- A temporary CSV where the script whould list paths toi + backup + """ + + system_targets = self.targets.list("system", exclude=["Skipped"]) + + # If nothing to backup, return immediately + if system_targets == []: + return + + logger.info(m18n.n('backup_running_hooks')) + + # Prepare environnement + env_dict = self._get_env_var() + + # Actual call to backup scripts/hooks + + ret = hook_callback('backup', + system_targets, + args=[self.work_dir], + env=env_dict, + chdir=self.work_dir) + + if ret["succeed"] != []: + self.system_return = ret["succeed"] + + # Add files from targets (which they put in the CSV) to the list of + # files to backup + self._import_to_list_to_backup(env_dict["YNH_BACKUP_CSV"]) + + # Save restoration hooks for each part that suceeded (and which have + # a restore hook available) + + restore_hooks_dir = os.path.join(self.work_dir, "hooks", "restore") + if not os.path.exists(restore_hooks_dir): + filesystem.mkdir(restore_hooks_dir, mode=0750, + parents=True, uid='admin') + + restore_hooks = hook_list("restore")["hooks"] + + for part in ret['succeed'].keys(): + if part in restore_hooks: + part_restore_hooks = hook_info("restore", part)["hooks"] + for hook in part_restore_hooks: + self._add_to_list_to_backup(hook["path"], "hooks/restore/") + self.targets.set_result("system", part, "Success") + else: + logger.warning(m18n.n('restore_hook_unavailable', hook=part)) + self.targets.set_result("system", part, "Warning") + + for part in ret['failed'].keys(): + logger.error(m18n.n('backup_system_part_failed', part=part)) + self.targets.set_result("system", part, "Error") + + + def _collect_apps_files(self): + """ Prepare backup for each selected apps """ + + apps_targets = self.targets.list("apps", exclude=["Skipped"]) + + for app_instance_name in apps_targets: + self._collect_app_files(app_instance_name) + + + def _collect_app_files(self, app): + """ + List files to backup for the app into the paths_to_backup dict. + + If the app backup script fails, paths from this app already listed for + backup aren't added to the general list and will be ignored + + Environment variables: + YNH_BACKUP_DIR -- The backup working directory (in + "/home/yunohost.backup/tmp/BACKUPNAME" or could be + defined by the user) + YNH_BACKUP_CSV -- A temporary CSV where the script whould list paths toi + backup + YNH_APP_BACKUP_DIR -- The directory where the script should put + temporary files to backup like database dump, + files in this directory don't need to be added to + the temporary CSV. + YNH_APP_ID -- The app id (eg wordpress) + YNH_APP_INSTANCE_NAME -- The app instance name (eg wordpress__3) + YNH_APP_INSTANCE_NUMBER -- The app instance number (eg 3) + + + Args: + app -- (string) an app instance name (already installed) to backup + + Exceptions: + backup_app_failed -- Raised at the end if the app backup script + execution failed + """ + app_setting_path = os.path.join('/etc/yunohost/apps/', app) + + # Prepare environment + env_dict = self._get_env_var(app) + tmp_app_bkp_dir = env_dict["YNH_APP_BACKUP_DIR"] + settings_dir = os.path.join(self.work_dir, 'apps', app, 'settings') + + logger.info(m18n.n('backup_running_app_script', app=app)) + try: + # Prepare backup directory for the app + filesystem.mkdir(tmp_app_bkp_dir, 0750, True, uid='admin') + + # Copy the app settings to be able to call _common.sh + shutil.copytree(app_setting_path, settings_dir) + + # Copy app backup script in a temporary folder and execute it + _, tmp_script = tempfile.mkstemp(prefix='backup_') + app_script = os.path.join(app_setting_path, 'scripts/backup') + subprocess.call(['install', '-Dm555', app_script, tmp_script]) + + hook_exec(tmp_script, args=[tmp_app_bkp_dir, app], + raise_on_error=True, chdir=tmp_app_bkp_dir, env=env_dict, user="root") + + self._import_to_list_to_backup(env_dict["YNH_BACKUP_CSV"]) + except: + abs_tmp_app_dir = os.path.join(self.work_dir, 'apps/', app) + shutil.rmtree(abs_tmp_app_dir, ignore_errors=True) + logger.exception(m18n.n('backup_app_failed', app=app)) + self.targets.set_result("apps", app, "Error") + else: + # Add app info + i = app_info(app) + self.apps_return[app] = { + 'version': i['version'], + 'name': i['name'], + 'description': i['description'], + } + self.targets.set_result("apps", app, "Success") + + # Remove tmp files in all situations + finally: + filesystem.rm(tmp_script, force=True) + filesystem.rm(env_dict["YNH_BACKUP_CSV"], force=True) + + + ########################################################################### + # Actual backup archive creation / method management # + ########################################################################### + + def add(self, method): + """ + Add a backup method that will be applied after the files collection step + + Args: + method -- (BackupMethod) A backup method. Currently, you can use those: + TarBackupMethod + CopyBackupMethod + CustomBackupMethod + """ + self.methods.append(method) + + + def backup(self): + """Apply backup methods""" + + for method in self.methods: + logger.info(m18n.n('backup_applying_method_' + method.method_name)) + method.mount_and_backup(self) + logger.info(m18n.n('backup_method_' + method.method_name + '_finished')) + + + def _compute_backup_size(self): + """ + Compute backup global size and details size for each apps and system + parts + + Update self.size and self.size_details + + Note: currently, these sizes are the size in this archive, not really + the size of needed to restore the archive. To know the size needed to + restore we should consider apt/npm/pip dependencies space and database + dump restore operations. + + Return: + (int) The global size of the archive in bytes + """ + # FIXME Database dump will be loaded, so dump should use almost the + # double of their space + # FIXME Some archive will set up dependencies, those are not in this + # size info + self.size = 0 + for system_key in self.system_return: + self.size_details['system'][system_key] = 0 + for app_key in self.apps_return: + self.size_details['apps'][app_key] = 0 + + for row in self.paths_to_backup: + if row['dest'] != "info.json": + size = disk_usage(row['source']) + + # Add size to apps details + splitted_dest = row['dest'].split('/') + category = splitted_dest[0] + if category == 'apps': + for app_key in self.apps_return: + if row['dest'].startswith('apps/'+app_key): + self.size_details['apps'][app_key] += size + break + # OR Add size to the correct system element + elif category == 'data' or category == 'conf': + for system_key in self.system_return: + if row['dest'].startswith(system_key.replace('_', '/')): + self.size_details['system'][system_key] += size + break + + self.size += size + + return self.size + + +class RestoreManager(): + """ + RestoreManager allow to restore a past backup archive + + Currently it's a tar.gz file, but it could be another kind of archive + + Public properties: + info (getter)i # FIXME + work_dir (getter) # FIXME currently it's not a getter + name (getter) # FIXME currently it's not a getter + success (getter) + result (getter) # FIXME + + Public methods: + set_targets(self, system_parts=[], apps=[]) + restore(self) + + Usage: + restore_manager = RestoreManager(name) + + restore_manager.set_targets(None, ['wordpress__3']) + + restore_manager.restore() + + if restore_manager.success: + logger.success(m18n.n('restore_complete')) + + return restore_manager.result + """ + + def __init__(self, name, repo=None, method='tar'): + """ + RestoreManager constructor + + Args: + name -- (string) Archive name + repo -- (string|None) Repository where is this archive, it could be a + path (default: /home/yunohost.backup/archives) + method -- (string) Method name to use to mount the archive + """ + # Retrieve and open the archive + # FIXME this way to get the info is not compatible with copy or custom + # backup methods + self.info = backup_info(name, with_details=True) + self.archive_path = self.info['path'] + self.name = name + self.method = BackupMethod.create(method) + self.targets = BackupRestoreTargetsManager() + + ########################################################################### + # Misc helpers # + ########################################################################### + + @property + def success(self): + + successful_apps = self.targets.list("apps", include=["Success", "Warning"]) + successful_system = self.targets.list("system", include=["Success", "Warning"]) + + return len(successful_apps) != 0 \ + or len(successful_system) != 0 + + + def _read_info_files(self): + """ + Read the info file from inside an archive + + Exceptions: + backup_invalid_archive -- Raised if we can't read the info + """ + # Retrieve backup info + info_file = os.path.join(self.work_dir, "info.json") + try: + with open(info_file, 'r') as f: + self.info = json.load(f) + + # Historically, "system" was "hooks" + if "system" not in self.info.keys(): + self.info["system"] = self.info["hooks"] + except IOError: + logger.debug("unable to load '%s'", info_file, exc_info=1) + raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive')) + else: + logger.debug("restoring from backup '%s' created on %s", self.name, + time.ctime(self.info['created_at'])) + + + def _postinstall_if_needed(self): + """ + Post install yunohost if needed + + Exceptions: + backup_invalid_archive -- Raised if the current_host isn't in the + archive + """ + # Check if YunoHost is installed + if not os.path.isfile('/etc/yunohost/installed'): + # Retrieve the domain from the backup + try: + with open("%s/conf/ynh/current_host" % self.work_dir, 'r') as f: + domain = f.readline().rstrip() + except IOError: + logger.debug("unable to retrieve current_host from the backup", + exc_info=1) + # FIXME include the current_host by default ? + raise MoulinetteError(errno.EIO, + m18n.n('backup_invalid_archive')) + + logger.debug("executing the post-install...") + tools_postinstall(domain, 'yunohost', True) + + + def clean(self): + """ + End a restore operations by cleaning the working directory and + regenerate ssowat conf (if some apps were restored) + """ + + successfull_apps = self.targets.list("apps", include=["Success", "Warning"]) + + if successfull_apps != []: + # Quickfix: the old app_ssowatconf(auth) instruction failed due to + # ldap restore hooks + os.system('sudo yunohost app ssowatconf') + + if os.path.ismount(self.work_dir): + ret = subprocess.call(["umount", self.work_dir]) + if ret != 0: + logger.warning(m18n.n('restore_cleaning_failed')) + filesystem.rm(self.work_dir, True, True) + + + ########################################################################### + # Restore target manangement # + ########################################################################### + + def set_system_targets(self, system_parts=[]): + """ + Define system parts that will be restored + + Args: + system_parts -- (list) list of system parts which should be restored. + If an empty list if given, restore all system part in + the archive. If None is given, no system will be restored. + """ + + def unknown_error(part): + logger.error(m18n.n("backup_archive_system_part_not_available", + part=part)) + + target_list = self.targets.set_wanted("system", + system_parts, + self.info['system'].keys(), + unknown_error) + + # Now we need to check that the restore hook is actually available for + # all targets we want to restore + + # These are the hooks on the current installation + available_restore_system_hooks = hook_list("restore")["hooks"] + + for system_part in target_list: + # By default, we'll use the restore hooks on the current install + # if available + + # FIXME: so if the restore hook exist we use the new one and not + # the one from backup. So hook should not break compatibility.. + + if system_part in available_restore_system_hooks: + continue + + # Otherwise, attempt to find it (or them?) in the archive + hook_paths = '{:s}/hooks/restore/*-{:s}'.format(self.work_dir, system_part) + hook_paths = glob(hook_paths) + + # If we didn't find it, we ain't gonna be able to restore it + if len(hook_paths) == 0: + logger.exception(m18n.n('restore_hook_unavailable', part=system_part)) + self.targets.set_result("system", system_part, "Skipped") + continue + + # Otherwise, add it from the archive to the system + # FIXME: Refactor hook_add and use it instead + custom_restore_hook_folder = os.path.join(CUSTOM_HOOK_FOLDER, 'restore') + filesystem.mkdir(custom_restore_hook_folder, 755, True) + for hook_path in hook_paths: + logger.debug("Adding restoration script '%s' to the system " + "from the backup archive '%s'", hook_path, + self.archive_path) + shutil.copy(hook_path, custom_restore_hook_folder) + + def set_apps_targets(self, apps=[]): + """ + Define and validate targetted apps to be restored + + Args: + apps -- (list) list of apps which should be restored. If [] is given, + all apps in the archive will be restored. If None is given, + no apps will be restored. + """ + + def unknown_error(app): + logger.error(m18n.n('backup_archive_app_not_found', + app=app)) + + self.targets.set_wanted("apps", + apps, + self.info['apps'].keys(), + unknown_error) + + + ########################################################################### + # Archive mounting # + ########################################################################### + + def mount(self): + """ + Mount the archive. We avoid copy to be able to restore on system without + too many space. + + Use the mount method from the BackupMethod instance and read info about + this archive + + Exceptions: + restore_removing_tmp_dir_failed -- Raised if it's not possible to remove + the working directory + """ + + self.work_dir = os.path.join(BACKUP_PATH, "tmp", self.name) + + if os.path.ismount(self.work_dir): + logger.debug("An already mounting point '%s' already exists", + self.work_dir) + ret = subprocess.call(['umount', self.work_dir]) + if ret == 0: + subprocess.call(['rmdir', self.work_dir]) + logger.debug("Unmount dir: {}".format(self.work_dir)) + else: + raise MoulinetteError(errno.EIO, + m18n.n('restore_removing_tmp_dir_failed')) + elif os.path.isdir(self.work_dir): + logger.debug("temporary restore directory '%s' already exists", + self.work_dir) + ret = subprocess.call(['rm', '-Rf', self.work_dir]) + if ret == 0: + logger.debug("Delete dir: {}".format(self.work_dir)) + else: + raise MoulinetteError(errno.EIO, + m18n.n('restore_removing_tmp_dir_failed')) + + filesystem.mkdir(self.work_dir, parents=True) + + self.method.mount(self) + + self._read_info_files() + + ########################################################################### + # Space computation / checks # + ########################################################################### + + def _compute_needed_space(self): + """ + Compute needed space to be able to restore + + Return: + size -- (int) needed space to backup in bytes + margin -- (int) margin to be sure the backup don't fail by missing space + in bytes + """ + system = self.targets.list("system", exclude=["Skipped"]) + apps = self.targets.list("apps", exclude=["Skipped"]) + restore_all_system = (system == self.info['system'].keys()) + restore_all_apps = (apps == self.info['apps'].keys()) + + # If complete restore operations (or legacy archive) + margin = CONF_MARGIN_SPACE_SIZE * 1024 * 1024 + if (restore_all_system and restore_all_apps) or 'size_details' not in self.info: + size = self.info['size'] + if 'size_details' not in self.info or \ + self.info['size_details']['apps'] != {}: + margin = APP_MARGIN_SPACE_SIZE * 1024 * 1024 + # Partial restore don't need all backup size + else: + size = 0 + if system is not None: + for system_element in system: + size += self.info['size_details']['system'][system_element] + + # TODO how to know the dependencies size ? + if apps is not None: + for app in apps: + size += self.info['size_details']['apps'][app] + margin = APP_MARGIN_SPACE_SIZE * 1024 * 1024 + + if not os.path.isfile('/etc/yunohost/installed'): + size += POSTINSTALL_ESTIMATE_SPACE_SIZE * 1024 * 1024 + return (size, margin) + + def assert_enough_free_space(self): + """ + Check available disk space + + Exceptions: + restore_may_be_not_enough_disk_space -- Raised if there isn't enough + space to cover the security margin space + restore_not_enough_disk_space -- Raised if there isn't enough space + """ + + free_space = free_space_in_directory(BACKUP_PATH) + + (needed_space, margin) = self._compute_needed_space() + if free_space >= needed_space + margin: + return True + elif free_space > needed_space: + # TODO Add --force options to avoid the error raising + raise MoulinetteError(errno.EIO, + m18n.n('restore_may_be_not_enough_disk_space', + free_space=free_space, + needed_space=needed_space, + margin=margin)) + else: + raise MoulinetteError(errno.EIO, + m18n.n('restore_not_enough_disk_space', + free_space=free_space, + needed_space=needed_space, + margin=margin)) + + ########################################################################### + # "Actual restore" (reverse step of the backup collect part) # + ########################################################################### + + def restore(self): + """ + Restore the archive + + Restore system parts and apps after mounting the archive, checking free + space and postinstall if needed + """ + + try: + self._postinstall_if_needed() + self._restore_system() + self._restore_apps() + finally: + self.clean() + + + def _restore_system(self): + """ Restore user and system parts """ + + system_targets = self.targets.list("system", exclude=["Skipped"]) + + # If nothing to restore, return immediately + if system_targets == []: + return + + logger.info(m18n.n('restore_running_hooks')) + + env_dict = self._get_env_var() + ret = hook_callback('restore', + system_targets, + args=[self.work_dir], + env=env_dict, + chdir=self.work_dir) + + for part in ret['succeed'].keys(): + self.targets.set_result("system", part, "Success") + + for part in ret['failed'].keys(): + logger.error(m18n.n('restore_system_part_failed', part=part)) + self.targets.set_result("system", part, "Error") + + service_regen_conf() + + + def _restore_apps(self): + """Restore all apps targeted""" + + apps_targets = self.targets.list("apps", exclude=["Skipped"]) + + for app in apps_targets: + self._restore_app(app) + + + def _restore_app(self, app_instance_name): + """ + Restore an app + + Environment variables: + YNH_BACKUP_DIR -- The backup working directory (in + "/home/yunohost.backup/tmp/BACKUPNAME" or could be + defined by the user) + YNH_BACKUP_CSV -- A temporary CSV where the script whould list paths to + backup + YNH_APP_BACKUP_DIR -- The directory where the script should put + temporary files to backup like database dump, + files in this directory don't need to be added to + the temporary CSV. + YNH_APP_ID -- The app id (eg wordpress) + YNH_APP_INSTANCE_NAME -- The app instance name (eg wordpress__3) + YNH_APP_INSTANCE_NUMBER -- The app instance number (eg 3) + + Args: + app_instance_name -- (string) The app name to restore (no app with this + name should be already install) + + Exceptions: + restore_already_installed_app -- Raised if an app with this app instance + name already exists + restore_app_failed -- Raised if the restore bash script failed + """ + def copytree(src, dst, symlinks=False, ignore=None): + for item in os.listdir(src): + s = os.path.join(src, item) + d = os.path.join(dst, item) + if os.path.isdir(s): + shutil.copytree(s, d, symlinks, ignore) + else: + shutil.copy2(s, d) + + # Check if the app is not already installed + if _is_installed(app_instance_name): + logger.error(m18n.n('restore_already_installed_app', + app=app_instance_name)) + self.targets.set_result("apps", app_instance_name, "Error") + return + + app_dir_in_archive = os.path.join(self.work_dir, 'apps', app_instance_name) + app_backup_in_archive = os.path.join(app_dir_in_archive, 'backup') + app_settings_in_archive = os.path.join(app_dir_in_archive, 'settings') + app_scripts_in_archive = os.path.join(app_settings_in_archive, 'scripts') + + # Check if the app has a restore script + app_restore_script_in_archive = os.path.join(app_scripts_in_archive, + 'restore') + if not os.path.isfile(app_restore_script_in_archive): + logger.warning(m18n.n('unrestore_app', app=app_instance_name)) + self.targets.set_result("apps", app_instance_name, "Warning") + return + + logger.info(m18n.n('restore_running_app_script', app=app_instance_name)) + try: + # Restore app settings + app_settings_new_path = os.path.join('/etc/yunohost/apps/', + app_instance_name) + app_scripts_new_path = os.path.join(app_settings_new_path, 'scripts') + shutil.copytree(app_settings_in_archive, app_settings_new_path) + filesystem.chmod(app_settings_new_path, 0400, 0400, True) + filesystem.chown(app_scripts_new_path, 'admin', None, True) + + # Copy the app scripts to a writable temporary folder + # FIXME : use 'install -Dm555' or something similar to what's done + # in the backup method ? + tmp_folder_for_app_restore = tempfile.mkdtemp(prefix='restore') + copytree(app_scripts_in_archive, tmp_folder_for_app_restore) + filesystem.chmod(tmp_folder_for_app_restore, 0550, 0550, True) + filesystem.chown(tmp_folder_for_app_restore, 'admin', None, True) + restore_script = os.path.join(tmp_folder_for_app_restore, 'restore') + + # Prepare env. var. to pass to script + env_dict = self._get_env_var(app_instance_name) + + # Execute app restore script + hook_exec(restore_script, + args=[app_backup_in_archive, app_instance_name], + chdir=app_backup_in_archive, + raise_on_error=True, + env=env_dict, + user="root") + except: + logger.exception(m18n.n('restore_app_failed', + app=app_instance_name)) + self.targets.set_result("apps", app_instance_name, "Error") + + remove_script = os.path.join(app_scripts_in_archive, 'remove') + + # Setup environment for remove script + app_id, app_instance_nb = _parse_app_instance_name(app_instance_name) + env_dict_remove = {} + env_dict_remove["YNH_APP_ID"] = app_id + env_dict_remove["YNH_APP_INSTANCE_NAME"] = app_instance_name + env_dict_remove["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb) + + # Execute remove script + # TODO: call app_remove instead + if hook_exec(remove_script, args=[app_instance_name], + env=env_dict_remove, user="root") != 0: + logger.warning(m18n.n('app_not_properly_removed', + app=app_instance_name)) + + # Cleaning app directory + shutil.rmtree(app_settings_new_path, ignore_errors=True) + + + # TODO Cleaning app hooks + else: + self.targets.set_result("apps", app_instance_name, "Success") + finally: + # Cleaning temporary scripts directory + shutil.rmtree(tmp_folder_for_app_restore, ignore_errors=True) + + + def _get_env_var(self, app=None): + """ Define environment variable for hooks call """ + env_var = {} + env_var['YNH_BACKUP_DIR'] = self.work_dir + env_var['YNH_BACKUP_CSV'] = os.path.join(self.work_dir, "backup.csv") + + if app is not None: + app_dir_in_archive = os.path.join(self.work_dir, 'apps', app) + app_backup_in_archive = os.path.join(app_dir_in_archive, 'backup') + + # Parse app instance name and id + app_id, app_instance_nb = _parse_app_instance_name(app) + + env_var["YNH_APP_ID"] = app_id + env_var["YNH_APP_INSTANCE_NAME"] = app + env_var["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb) + env_var["YNH_APP_BACKUP_DIR"] = app_backup_in_archive + + return env_var + +############################################################################### +# Backup methods # +############################################################################### + +class BackupMethod(object): + """ + BackupMethod is an abstract class that represents a way to backup and + restore a list of files. + + Daughters of this class can be used by a BackupManager or RestoreManager + instance. Some methods are meant to be used by BackupManager and others + by RestoreManager. + + BackupMethod has a factory method "create" to initialize instances. + + Currently, there are 3 BackupMethods implemented: + + CopyBackupMethod + ---------------- + This method corresponds to a raw (uncompressed) copy of files to a location, + and (could?) reverse the copy when restoring. + + TarBackupMethod + --------------- + This method compresses all files to backup in a .tar.gz archive. When + restoring, it tries to mount the archive using archivemount/fuse instead + of untaring the archive. Some systems don't support fuse (for these, + it automatically falls back to untaring the required parts). + + CustomBackupMethod + ------------------ + This one use a custom bash scrip/hook "backup_method" to do the + backup/restore operations. A user can add his own hook inside + /etc/yunohost/hooks.d/backup_method/ + + Public properties: + method_name + + Public methods: + mount_and_backup(self, backup_manager) + mount(self, restore_manager) + create(cls, method, **kwargs) + + Usage: + method = BackupMethod.create("tar") + method.mount_and_backup(backup_manager) + #or + method = BackupMethod.create("copy") + method.mount(restore_manager) + """ + def __init__(self, repo = None): + """ + BackupMethod constructors + + Note it is an abstract class. You should use the "create" class method + to create instance. + + Args: + repo -- (string|None) A string that represent the repo where put or + get the backup. It could be a path, and in future a + BackupRepository object. If None, the default repo is used : + /home/yunohost.backup/archives/ + """ + self.repo = ARCHIVES_PATH if repo is None else repo + + @property + def method_name(self): + """Return the string name of a BackupMethod (eg "tar" or "copy")""" + raise MoulinetteError(errno.EINVAL, m18n.n('backup_abstract_method')) + + @property + def name(self): + """Return the backup name""" + return self.manager.name + + @property + def work_dir(self): + """ + Return the working directory + + For a BackupManager, it is the directory where we prepare the files to + backup + + For a RestoreManager, it is the directory where we mount the archive + before restoring + """ + return self.manager.work_dir + + def need_mount(self): + """ + Return True if this backup method need to organize path to backup by + binding its in the working directory before to backup its. + + Indeed, some methods like tar or copy method don't need to organize + files before to add it inside the archive, but others like borgbackup + are not able to organize directly the files. In this case we have the + choice to organize in the working directory before to put in the archive + or to organize after mounting the archive before the restoring + operation. + + The default behaviour is to return False. To change it override the + method. + + Note it's not a property because some overrided methods could do long + treatment to get this info + """ + return False + + def mount_and_backup(self, backup_manager): + """ + Run the backup on files listed by the BackupManager instance + + This method shouldn't be overrided, prefer overriding self.backup() and + self.clean() + + Args: + backup_manager -- (BackupManager) A backup manager instance that has + already done the files collection step. + """ + self.manager = backup_manager + if self.need_mount(): + self._organize_files() + + try: + self.backup() + finally: + self.clean() + + def mount(self, restore_manager): + """ + Mount the archive from RestoreManager instance in the working directory + + This method should be extended. + + Args: + restore_manager -- (RestoreManager) A restore manager instance + contains an archive to restore. + """ + self.manager = restore_manager + + def clean(self): + """ + Umount sub directories of working dirextories and delete it if temporary + + Exceptions: + backup_cleaning_failed -- Raise if we were not able to unmount sub + directories of the working directories + """ + if self.need_mount(): + if self._recursive_umount(self.work_dir) > 0: + raise MoulinetteError(errno.EINVAL, + m18n.n('backup_cleaning_failed')) + + if self.manager.is_tmp_work_dir: + filesystem.rm(self.work_dir, True, True) + + def _recursive_umount(directory): + """ + Recursively umount sub directories of a directory + + Args: + directory -- a directory path + """ + mount_lines = subprocess.check_output("mount").split("\n") + + points_to_umount = [ line.split(" ")[2] + for line in mount_lines + if len(line) >= 3 + and line.split(" ")[2].startswith(directory) ] + ret = 0 + for point in reversed(points_to_umount): + ret = subprocess.call(["umount", point]) + if ret != 0: + ret = 1 + logger.warning(m18n.n('backup_cleaning_failed', point)) + continue + + return ret + + def _check_is_enough_free_space(self): + """ + Check free space in repository or output directory before to backup + + Exceptions: + not_enough_disk_space -- Raise if there isn't enough space. + """ + # TODO How to do with distant repo or with deduplicated backup ? + backup_size = self.manager.size + + free_space = free_space_in_directory(self.repo) + + if free_space < backup_size: + logger.debug('Not enough space at %s (free: %s / needed: %d)', + self.repo, free_space, backup_size) + raise MoulinetteError(errno.EIO, m18n.n( + 'not_enough_disk_space', path=self.repo)) + + def _organize_files(self): + """ + Mount all csv src in their related path + + The goal is to organize the files app by app and hook by hook, before + custom backup method or before the restore operation (in the case of an + unorganize archive). + + The usage of binding could be strange for a user because the du -sb + command will return that the working directory is big. + + Exceptions: + backup_unable_to_organize_files + """ + paths_needed_to_be_copied = [] + for path in self.manager.paths_to_backup: + src = path['src'] + + if self.manager is RestoreManager: + # TODO Support to run this before a restore (and not only before + # backup). To do that RestoreManager.unorganized_work_dir should + # be implemented + src = os.path.join(self.unorganized_work_dir, src) + + dest = os.path.join(self.work_dir, path['dest']) + dest_dir = os.path.dirname(dest) + + # Be sure the parent dir of destination exists + filesystem.mkdir(dest_dir, parent=True) + + # Try to bind files + if os.path.isdir(src): + filesystem.mkdir(dest, parent=True) + ret = subprocess.call(["mount", "-r", "--rbind", src, dest]) + if ret == 0: + continue + else: + logger.warning(m18n.n("bind_mouting_disable")) + subprocess.call(["mountpoint", "-q", dest, + "&&", "umount", "-R", dest]) + elif os.path.isfile(src) or os.path.islink(src): + # Create a hardlink if src and dest are on the filesystem + if os.stat(src).st_dev == os.stat(dest_dir).st_dev: + os.link(src, dest) + continue + + # Add to the list to copy + paths_needed_to_be_copied.append(path) + + if len(paths_needed_to_be_copied) == 0: + return + + # Manage the case where we are not able to use mount bind abilities + # It could be just for some small files on different filesystems or due + # to mounting error + + # Compute size to copy + size = sum(disk_usage(path['src']) for path in paths_needed_to_be_copied) + size /= (1024 * 1024) # Convert bytes to megabytes + + # Ask confirmation for copying + if size > MB_ALLOWED_TO_ORGANIZE: + try: + i = msignals.prompt(m18n.n('backup_ask_for_copying_if_needed', + answers='y/N', size=size)) + except NotImplemented: + logger.error(m18n.n('backup_unable_to_organize_files')) + else: + if i != 'y' and i != 'Y': + logger.error(m18n.n('backup_unable_to_organize_files')) + + # Copy unbinded path + logger.info(m18n.n('backup_copying_to_organize_the_archive', size=size)) + for path in paths_needed_to_be_copied: + if os.path.isdir(src): + shutil.copytree(src, dest, symlinks=True) + else: + shutil.copy(src, dest) + + @classmethod + def create(cls, method, *args): + """ + Factory method to create instance of BackupMethod + + Args: + method -- (string) The method name of an existing BackupMethod. If the + name is unknown the CustomBackupMethod will be tried + + ... -- Specific args for the method, could be the repo target by the + method + + Return a BackupMethod instance + """ + if not isinstance(method, basestring): + methods = [] + for m in method: + methods.append(BackupMethod.create(m, *args)) + return methods + + bm_class = { + 'copy': CopyBackupMethod, + 'tar': TarBackupMethod, + 'borg': BorgBackupMethod + } + if method in ["copy", "tar", "borg"]: + return bm_class[method](*args) + else: + return CustomBackupMethod(*args) + + +class CopyBackupMethod(BackupMethod): + """ + This class just do an uncompress copy of each file in a location, and + could be the inverse for restoring + """ + def __init__(self, repo = None): + super(CopyBackupMethod, self).__init__(repo) + + @property + def method_name(self): + return 'copy' + + def backup(self): + """ Copy prepared files into a the repo """ + # Check free space in output + self._check_is_enough_free_space() + + for path in self.manager.paths_to_backup: + source = path['source'] + dest = os.path.join(self.repo, path['dest']) + if source == dest: + logger.debug("Files already copyed") + return + + dest_parent = os.path.dirname(dest) + if not os.path.exists(dest_parent): + filesystem.mkdir(dest_parent, 0750, True, uid='admin') + + if os.path.isdir(source): + shutil.copytree(source, dest) + else: + shutil.copy(source, dest) + + def mount(self): + """ + Mount the uncompress backup in readonly mode to the working directory + + Exceptions: + backup_no_uncompress_archive_dir -- Raised if the repo doesn't exists + backup_cant_mount_uncompress_archive -- Raised if the binding failed + """ + # FIXME: This code is untested because there is no way to run it from + # the ynh cli + super(CopyBackupMethod, self).mount() + + if not os.path.isdir(self.repo): + raise MoulinetteError(errno.EIO, + m18n.n('backup_no_uncompress_archive_dir')) + + filesystem.mkdir(self.work_dir, parent=True) + ret = subprocess.call(["mount", "-r", "--rbind", self.repo, + self.work_dir]) + if ret == 0: + return + else: + logger.warning(m18n.n("bind_mouting_disable")) + subprocess.call(["mountpoint", "-q", dest, + "&&", "umount", "-R", dest]) + raise MoulinetteError(errno.EIO, + m18n.n('backup_cant_mount_uncompress_archive')) + + +class TarBackupMethod(BackupMethod): + """ + This class compress all files to backup in archive. To restore it try to + mount the archive with archivemount (fuse). Some system don't support fuse. + """ + + def __init__(self, repo=None): + super(TarBackupMethod, self).__init__(repo) + + + @property + def method_name(self): + return 'tar' + + + @property + def _archive_file(self): + """Return the compress archive path""" + return os.path.join(self.repo, self.name + '.tar.gz') + + + def backup(self): + """ + Compress prepared files + + It adds the info.json in /home/yunohost.backup/archives and if the + compress archive isn't located here, add a symlink to the archive to. + + Exceptions: + backup_archive_open_failed -- Raised if we can't open the archive + backup_creation_failed -- Raised if we can't write in the + compress archive + """ + + if not os.path.exists(self.repo): + filesystem.mkdir(self.repo, 0750, parents=True, uid='admin') + + # Check free space in output + self._check_is_enough_free_space() + + # Open archive file for writing + try: + tar = tarfile.open(self._archive_file, "w:gz") + except: + logger.debug("unable to open '%s' for writing", + self._archive_file, exc_info=1) + raise MoulinetteError(errno.EIO, + m18n.n('backup_archive_open_failed')) + + # Add files to the archive + try: + for path in self.manager.paths_to_backup: + # Add the "source" into the archive and transform the path into + # "dest" + tar.add(path['source'], arcname=path['dest']) + tar.close() + except IOError: + logger.error(m18n.n('backup_archive_writing_error'), exc_info=1) + raise MoulinetteError(errno.EIO, + m18n.n('backup_creation_failed')) + + # Move info file + shutil.copy(os.path.join(self.work_dir, 'info.json'), + os.path.join(ARCHIVES_PATH, self.name + '.info.json')) + + # If backuped to a non-default location, keep a symlink of the archive + # to that location + link = os.path.join(ARCHIVES_PATH, self.name + '.tar.gz') + if not os.path.isfile(link): + os.symlink(self._archive_file, link) + + + def mount(self, restore_manager): + """ + Mount the archive. We avoid copy to be able to restore on system without + too many space. + + Exceptions: + backup_archive_open_failed -- Raised if the archive can't be open + backup_archive_mount_failed -- Raised if the system don't support + archivemount + """ + super(TarBackupMethod, self).mount(restore_manager) + + # Check the archive can be open + try: + tar = tarfile.open(self._archive_file, "r:gz") + except: + logger.debug("cannot open backup archive '%s'", + self._archive_file, exc_info=1) + raise MoulinetteError(errno.EIO, + m18n.n('backup_archive_open_failed')) + tar.close() + + # Mount the tarball + ret = subprocess.call(['archivemount', '-o', 'readonly', + self._archive_file, self.work_dir]) + if ret != 0: + logger.warning(m18n.n('backup_archive_mount_failed')) + + logger.info(m18n.n("restore_extracting")) + tar = tarfile.open(self._archive_file, "r:gz") + tar.extract('info.json', path=self.work_dir) + + try: + tar.extract('backup.csv', path=self.work_dir) + except KeyError: + # Old backup archive have no backup.csv file + pass + + # Extract system parts backup + conf_extracted = False + + system_targets = self.manager.targets.list("system", exclude=["Skipped"]) + apps_targets = self.manager.targets.list("apps", exclude=["Skipped"]) + + for system_part in system_targets: + # Caution: conf_ynh_currenthost helpers put its files in + # conf/ynh + if system_part.startswith("conf_"): + if conf_extracted: + continue + system_part = "conf/" + conf_extracted = True + else: + system_part = system_part.replace("_", "/") + "/" + subdir_and_files = [ + tarinfo for tarinfo in tar.getmembers() + if tarinfo.name.startswith(system_part) + ] + tar.extractall(members=subdir_and_files, path=self.work_dir) + subdir_and_files = [ + tarinfo for tarinfo in tar.getmembers() + if tarinfo.name.startswith("hooks/restore/") + ] + tar.extractall(members=subdir_and_files, path=self.work_dir) + + + # Extract apps backup + for app in apps_targets: + subdir_and_files = [ + tarinfo for tarinfo in tar.getmembers() + if tarinfo.name.startswith("apps/" + app) + ] + tar.extractall(members=subdir_and_files, path=self.work_dir) + + + +class BorgBackupMethod(BackupMethod): + + @property + def method_name(self): + return 'borg' + + + def backup(self): + """ Backup prepared files with borg """ + super(CopyBackupMethod, self).backup() + + # TODO run borg create command + raise MoulinetteError( + errno.EIO, m18n.n('backup_borg_not_implemented')) + + + def mount(self, mnt_path): + raise MoulinetteError( + errno.EIO, m18n.n('backup_borg_not_implemented')) + + +class CustomBackupMethod(BackupMethod): + """ + This class use a bash script/hook "backup_method" to do the + backup/restore operations. A user can add his own hook inside + /etc/yunohost/hooks.d/backup_method/ + """ + def __init__(self, repo = None, **kwargs): + super(CustomBackupMethod, self).__init__(repo) + self.args = kwargs + self._need_mount = None + + + @property + def method_name(self): + return 'borg' + + + def need_mount(self): + """Call the backup_method hook to know if we need to organize files + + Exceptions: + backup_custom_need_mount_error -- Raised if the hook failed + """ + ret = hook_callback('backup_method', method, + args=self._get_args('need_mount')) + if ret['succeed']: + return True + else: + raise MoulinetteError(errno.EIO, + m18n.n('backup_custom_need_mount_error')) + + + def backup(self): + """ + Launch a custom script to backup + + Exceptions: + backup_custom_backup_error -- Raised if the custom script failed + """ + + ret = hook_callback('backup_method', method, + args=self._get_args('backup')) + if ret['failed']: + raise MoulinetteError(errno.EIO, + m18n.n('backup_custom_backup_error')) + + def mount(self, restore_manager): + """ + Launch a custom script to mount the custom archive + + Exceptions: + backup_custom_mount_error -- Raised if the custom script failed + """ + super(CustomBackupMethod, self).mount(restore_manager) + ret = hook_callback('backup_method', method, + args=self._get_args('mount')) + if ret['failed']: + raise MoulinetteError(errno.EIO, + m18n.n('backup_custom_mount_error')) + + + def _get_args(self, action): + """Return the arguments to give to the custom script""" + return [action, self.work_dir, self.name, self.repo, self.manager.size, + self.manager.description] + + +############################################################################### +# "Front-end" # +############################################################################### + +def backup_create(name=None, description=None, methods=[], + output_directory=None, no_compress=False, + ignore_system=False, system=[], + ignore_apps=False, apps=[], + ignore_hooks=False, hooks=[]): """ Create a backup local archive Keyword arguments: name -- Name of the backup archive description -- Short description of the backup + method -- Method of backup to use output_directory -- Output directory for the backup no_compress -- Do not create an archive file - hooks -- List of backup hooks names to execute - ignore_hooks -- Do not execute backup hooks + system -- List of system elements to backup + ignore_system -- Ignore system elements apps -- List of application names to backup ignore_apps -- Do not backup apps + hooks -- (Deprecated) Renamed to "system" + ignore_hooks -- (Deprecated) Renamed to "ignore_system" """ + # TODO: Add a 'clean' argument to clean output directory - tmp_dir = None - env_var = {} - # Validate what to backup - if ignore_hooks and ignore_apps: - raise MoulinetteError(errno.EINVAL, - m18n.n('backup_action_required')) + ########################################################################### + # Validate / parse arguments # + ########################################################################### - # Validate and define backup name - timestamp = int(time.time()) - if not name: - name = time.strftime('%Y%m%d-%H%M%S') - if name in backup_list()['archives']: - raise MoulinetteError(errno.EINVAL, - m18n.n('backup_archive_name_exists')) + # Historical, deprecated options + if ignore_hooks != False: + logger.warning("--ignore-hooks is deprecated and will be removed in the" + "future. Please use --ignore-system instead.") + ignore_system = ignore_hooks - # Validate additional arguments - if no_compress and not output_directory: + if hooks != [] and hooks is not None: + logger.warning("--hooks is deprecated and will be removed in the" + "future. Please use --system instead.") + system = hooks + + # Validate that there's something to backup + if ignore_system and ignore_apps: raise MoulinetteError(errno.EINVAL, - m18n.n('backup_output_directory_required')) + m18n.n('backup_action_required')) + + # Validate there is no archive with the same name + if name and name in backup_list()['archives']: + raise MoulinetteError(errno.EINVAL, + m18n.n('backup_archive_name_exists')) + + # Validate output_directory option if output_directory: output_directory = os.path.abspath(output_directory) # Check for forbidden folders if output_directory.startswith(ARCHIVES_PATH) or \ - re.match(r'^/(|(bin|boot|dev|etc|lib|root|run|sbin|sys|usr|var)(|/.*))$', - output_directory): + re.match(r'^/(|(bin|boot|dev|etc|lib|root|run|sbin|sys|usr|var)(|/.*))$', + output_directory): raise MoulinetteError(errno.EINVAL, - m18n.n('backup_output_directory_forbidden')) + m18n.n('backup_output_directory_forbidden')) - # Create the output directory - if not os.path.isdir(output_directory): - logger.debug("creating output directory '%s'", output_directory) - os.makedirs(output_directory, 0750) # Check that output directory is empty - elif no_compress and os.listdir(output_directory): + if os.path.isdir(output_directory) and no_compress and \ + os.listdir(output_directory): raise MoulinetteError(errno.EIO, - m18n.n('backup_output_directory_not_empty')) + m18n.n('backup_output_directory_not_empty')) + elif no_compress: + raise MoulinetteError(errno.EINVAL, + m18n.n('backup_output_directory_required')) - # Do not compress, so set temporary directory to output one and - # disable bind mounting to prevent data loss in case of a rm - # See: https://dev.yunohost.org/issues/298 + # Define methods (retro-compat) + if methods == []: if no_compress: - logger.debug('bind mounting will be disabled') - tmp_dir = output_directory - env_var['CAN_BIND'] = 0 + methods = ['copy'] + else: + methods = ['tar'] # In future, borg will be the default actions + + if ignore_system: + system = None + elif system is None: + system = [] + + if ignore_apps: + apps = None + elif apps is None: + apps = [] + + ########################################################################### + # Intialize # + ########################################################################### + + # Create yunohost archives directory if it does not exists + _create_archive_dir() + + # Prepare files to backup + if no_compress: + backup_manager = BackupManager(name, description, + work_dir=output_directory) else: - output_directory = ARCHIVES_PATH + backup_manager = BackupManager(name, description) - # Create archives directory if it does not exists - if not os.path.isdir(ARCHIVES_PATH): - os.mkdir(ARCHIVES_PATH, 0750) + # Add backup methods + if output_directory: + methods = BackupMethod.create(methods, output_directory) + else: + methods = BackupMethod.create(methods) - def _clean_tmp_dir(retcode=0): - ret = hook_callback('post_backup_create', args=[tmp_dir, retcode]) - if not ret['failed']: - filesystem.rm(tmp_dir, True, True) - return True - else: - logger.warning(m18n.n('backup_cleaning_failed')) - return False + for method in methods: + backup_manager.add(method) - # Create temporary directory - if not tmp_dir: - tmp_dir = "%s/tmp/%s" % (BACKUP_PATH, name) - if os.path.isdir(tmp_dir): - logger.debug("temporary directory for backup '%s' already exists", - tmp_dir) - if not _clean_tmp_dir(): - raise MoulinetteError( - errno.EIO, m18n.n('backup_output_directory_not_empty')) - filesystem.mkdir(tmp_dir, 0750, parents=True, uid='admin') + # Add backup targets (system and apps) + backup_manager.set_system_targets(system) + backup_manager.set_apps_targets(apps) - # Initialize backup info - info = { - 'description': description or '', - 'created_at': timestamp, - 'apps': {}, - 'hooks': {}, - } + ########################################################################### + # Collect files and put them in the archive # + ########################################################################### - # Run system hooks - if not ignore_hooks: - # Check hooks availibility - hooks_filtered = set() - if hooks: - for hook in hooks: - try: - hook_info('backup', hook) - except: - logger.error(m18n.n('backup_hook_unknown', hook=hook)) - else: - hooks_filtered.add(hook) + # Collect files to be backup (by calling app backup script / system hooks) + backup_manager.collect_files() - if not hooks or hooks_filtered: - logger.info(m18n.n('backup_running_hooks')) - ret = hook_callback('backup', hooks_filtered, args=[tmp_dir], - env=env_var) - if ret['succeed']: - info['hooks'] = ret['succeed'] - - # Save relevant restoration hooks - tmp_hooks_dir = tmp_dir + '/hooks/restore' - filesystem.mkdir(tmp_hooks_dir, 0750, True, uid='admin') - for h in ret['succeed'].keys(): - try: - i = hook_info('restore', h) - except: - logger.warning(m18n.n('restore_hook_unavailable', - hook=h), exc_info=1) - else: - for f in i['hooks']: - shutil.copy(f['path'], tmp_hooks_dir) - - # Backup apps - if not ignore_apps: - # Filter applications to backup - apps_list = set(os.listdir('/etc/yunohost/apps')) - apps_filtered = set() - if apps: - for a in apps: - if a not in apps_list: - logger.warning(m18n.n('unbackup_app', app=a)) - else: - apps_filtered.add(a) - else: - apps_filtered = apps_list - - # Run apps backup scripts - tmp_script = '/tmp/backup_' + str(timestamp) - for app_instance_name in apps_filtered: - app_setting_path = '/etc/yunohost/apps/' + app_instance_name - - # Check if the app has a backup and restore script - app_script = app_setting_path + '/scripts/backup' - app_restore_script = app_setting_path + '/scripts/restore' - if not os.path.isfile(app_script): - logger.warning(m18n.n('unbackup_app', app=app_instance_name)) - continue - elif not os.path.isfile(app_restore_script): - logger.warning(m18n.n('unrestore_app', app=app_instance_name)) - - tmp_app_dir = '{:s}/apps/{:s}'.format(tmp_dir, app_instance_name) - tmp_app_bkp_dir = tmp_app_dir + '/backup' - logger.info(m18n.n('backup_running_app_script', app=app_instance_name)) - try: - # Prepare backup directory for the app - filesystem.mkdir(tmp_app_bkp_dir, 0750, True, uid='admin') - shutil.copytree(app_setting_path, tmp_app_dir + '/settings') - - # Copy app backup script in a temporary folder and execute it - subprocess.call(['install', '-Dm555', app_script, tmp_script]) - - # Prepare env. var. to pass to script - app_id, app_instance_nb = _parse_app_instance_name( - app_instance_name) - env_dict = env_var.copy() - env_dict["YNH_APP_ID"] = app_id - env_dict["YNH_APP_INSTANCE_NAME"] = app_instance_name - env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb) - env_dict["YNH_APP_BACKUP_DIR"] = tmp_app_bkp_dir - - hook_exec(tmp_script, args=[tmp_app_bkp_dir, app_instance_name], - raise_on_error=True, chdir=tmp_app_bkp_dir, env=env_dict, user="root") - except: - logger.exception(m18n.n('backup_app_failed', app=app_instance_name)) - # Cleaning app backup directory - shutil.rmtree(tmp_app_dir, ignore_errors=True) - else: - # Add app info - i = app_info(app_instance_name) - info['apps'][app_instance_name] = { - 'version': i['version'], - 'name': i['name'], - 'description': i['description'], - } - finally: - filesystem.rm(tmp_script, force=True) - - # Check if something has been saved - if not info['hooks'] and not info['apps']: - _clean_tmp_dir(1) - raise MoulinetteError(errno.EINVAL, m18n.n('backup_nothings_done')) - - # Calculate total size - backup_size = int(subprocess.check_output( - ['du', '-sb', tmp_dir]).split()[0].decode('utf-8')) - info['size'] = backup_size - - # Create backup info file - with open("%s/info.json" % tmp_dir, 'w') as f: - f.write(json.dumps(info)) - - # Create the archive - if not no_compress: - logger.info(m18n.n('backup_creating_archive')) - - # Check free space in output directory at first - avail_output = subprocess.check_output( - ['df', '--block-size=1', '--output=avail', tmp_dir]).split() - if len(avail_output) < 2 or int(avail_output[1]) < backup_size: - logger.debug('not enough space at %s (free: %s / needed: %d)', - output_directory, avail_output[1], backup_size) - _clean_tmp_dir(3) - raise MoulinetteError(errno.EIO, m18n.n( - 'not_enough_disk_space', path=output_directory)) - - # Open archive file for writing - archive_file = "%s/%s.tar.gz" % (output_directory, name) - try: - tar = tarfile.open(archive_file, "w:gz") - except: - logger.debug("unable to open '%s' for writing", - archive_file, exc_info=1) - _clean_tmp_dir(2) - raise MoulinetteError(errno.EIO, - m18n.n('backup_archive_open_failed')) - - # Add files to the archive - try: - tar.add(tmp_dir, arcname='') - tar.close() - except IOError as e: - logger.error(m18n.n('backup_archive_writing_error'), exc_info=1) - _clean_tmp_dir(3) - raise MoulinetteError(errno.EIO, - m18n.n('backup_creation_failed')) - - # FIXME : it looks weird that the "move info file" is not enabled if - # user activated "no_compress" ... or does it really means - # "dont_keep_track_of_this_backup_in_history" ? - - # Move info file - shutil.move(tmp_dir + '/info.json', - '{:s}/{:s}.info.json'.format(ARCHIVES_PATH, name)) - - # If backuped to a non-default location, keep a symlink of the archive - # to that location - if output_directory != ARCHIVES_PATH: - link = "%s/%s.tar.gz" % (ARCHIVES_PATH, name) - os.symlink(archive_file, link) - - # Clean temporary directory - if tmp_dir != output_directory: - _clean_tmp_dir() + # Apply backup methods on prepared files + backup_manager.backup() logger.success(m18n.n('backup_created')) - # Return backup info - info['name'] = name - return {'archive': info} + return { + 'name': backup_manager.name, + 'size': backup_manager.size, + 'results': backup_manager.targets.results + } -def backup_restore(auth, name, hooks=[], ignore_hooks=False, - apps=[], ignore_apps=False, force=False): +def backup_restore(auth, name, + system=[], ignore_system=False, + apps=[], ignore_apps=False, + hooks=[], ignore_hooks=False, + force=False): """ Restore from a local backup archive Keyword argument: name -- Name of the local backup archive - hooks -- List of restoration hooks names to execute - ignore_hooks -- Do not execute backup hooks + force -- Force restauration on an already installed system + system -- List of system parts to restore + ignore_system -- Do not restore any system parts apps -- List of application names to restore ignore_apps -- Do not restore apps - force -- Force restauration on an already installed system + hooks -- (Deprecated) Renamed to "system" + ignore_hooks -- (Deprecated) Renamed to "ignore_system" """ + + ########################################################################### + # Validate / parse arguments # + ########################################################################### + + # Historical, deprecated options + if ignore_hooks != False: + logger.warning("--ignore-hooks is deprecated and will be removed in the" + "future. Please use --ignore-system instead.") + ignore_system = ignore_hooks + if hooks != []: + logger.warning("--hooks is deprecated and will be removed in the" + "future. Please use --system instead.") + system = hooks + # Validate what to restore - if ignore_hooks and ignore_apps: + if ignore_system and ignore_apps: raise MoulinetteError(errno.EINVAL, - m18n.n('restore_action_required')) + m18n.n('restore_action_required')) - # Retrieve and open the archive - info = backup_info(name) - archive_file = info['path'] - try: - tar = tarfile.open(archive_file, "r:gz") - except: - logger.debug("cannot open backup archive '%s'", - archive_file, exc_info=1) - raise MoulinetteError(errno.EIO, m18n.n('backup_archive_open_failed')) + if ignore_system: + system = None + elif system is None: + system = [] - # Check temporary directory - tmp_dir = "%s/tmp/%s" % (BACKUP_PATH, name) - if os.path.isdir(tmp_dir): - logger.debug("temporary directory for restoration '%s' already exists", - tmp_dir) - os.system('rm -rf %s' % tmp_dir) + if ignore_apps: + apps = None + elif apps is None: + apps = [] - # Check available disk space - statvfs = os.statvfs(BACKUP_PATH) - free_space = statvfs.f_frsize * statvfs.f_bavail - if free_space < info['size']: - logger.debug("%dB left but %dB is needed", free_space, info['size']) - raise MoulinetteError( - errno.EIO, m18n.n('not_enough_disk_space', path=BACKUP_PATH)) - - def _clean_tmp_dir(retcode=0): - ret = hook_callback('post_backup_restore', args=[tmp_dir, retcode]) - if not ret['failed']: - filesystem.rm(tmp_dir, True, True) - else: - logger.warning(m18n.n('restore_cleaning_failed')) - - # Extract the tarball - logger.info(m18n.n('backup_extracting_archive')) - tar.extractall(tmp_dir) - tar.close() - - # Retrieve backup info - info_file = "%s/info.json" % tmp_dir - try: - with open(info_file, 'r') as f: - info = json.load(f) - except IOError: - logger.debug("unable to load '%s'", info_file, exc_info=1) - raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive')) - else: - logger.debug("restoring from backup '%s' created on %s", name, - time.ctime(info['created_at'])) - - # Initialize restauration summary result - result = { - 'apps': [], - 'hooks': {}, - } + # TODO don't ask this question when restoring apps only and certain system + # parts # Check if YunoHost is installed - if os.path.isfile('/etc/yunohost/installed'): + if os.path.isfile('/etc/yunohost/installed') and not ignore_system: logger.warning(m18n.n('yunohost_already_installed')) if not force: try: @@ -412,154 +2120,37 @@ def backup_restore(auth, name, hooks=[], ignore_hooks=False, if i == 'y' or i == 'Y': force = True if not force: - _clean_tmp_dir() raise MoulinetteError(errno.EEXIST, m18n.n('restore_failed')) - else: - # Retrieve the domain from the backup - try: - with open("%s/conf/ynh/current_host" % tmp_dir, 'r') as f: - domain = f.readline().rstrip() - except IOError: - logger.debug("unable to retrieve current_host from the backup", - exc_info=1) - raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive')) - logger.debug("executing the post-install...") - tools_postinstall(domain, 'yunohost', True) + # TODO Partial app restore could not work if ldap is not restored before + # TODO repair mysql if broken and it's a complete restore - # Run system hooks - if not ignore_hooks: - # Filter hooks to execute - hooks_list = set(info['hooks'].keys()) - _is_hook_in_backup = lambda h: True - if hooks: - def _is_hook_in_backup(h): - if h in hooks_list: - return True - logger.error(m18n.n('backup_archive_hook_not_exec', hook=h)) - return False - else: - hooks = hooks_list + ########################################################################### + # Initialize # + ########################################################################### - # Check hooks availibility - hooks_filtered = set() - for h in hooks: - if not _is_hook_in_backup(h): - continue - try: - hook_info('restore', h) - except: - tmp_hooks = glob('{:s}/hooks/restore/*-{:s}'.format(tmp_dir, h)) - if not tmp_hooks: - logger.exception(m18n.n('restore_hook_unavailable', hook=h)) - continue - # Add restoration hook from the backup to the system - # FIXME: Refactor hook_add and use it instead - restore_hook_folder = CUSTOM_HOOK_FOLDER + 'restore' - filesystem.mkdir(restore_hook_folder, 755, True) - for f in tmp_hooks: - logger.debug("adding restoration hook '%s' to the system " - "from the backup archive '%s'", f, archive_file) - shutil.copy(f, restore_hook_folder) - hooks_filtered.add(h) + restore_manager = RestoreManager(name) - if hooks_filtered: - logger.info(m18n.n('restore_running_hooks')) - ret = hook_callback('restore', hooks_filtered, args=[tmp_dir]) - result['hooks'] = ret['succeed'] + restore_manager.set_system_targets(system) + restore_manager.set_apps_targets(apps) - # Add apps restore hook - if not ignore_apps: - # Filter applications to restore - apps_list = set(info['apps'].keys()) - apps_filtered = set() - if apps: - for a in apps: - if a not in apps_list: - logger.error(m18n.n('backup_archive_app_not_found', app=a)) - else: - apps_filtered.add(a) - else: - apps_filtered = apps_list + restore_manager.assert_enough_free_space() - for app_instance_name in apps_filtered: - tmp_app_dir = '{:s}/apps/{:s}'.format(tmp_dir, app_instance_name) - tmp_app_bkp_dir = tmp_app_dir + '/backup' + ########################################################################### + # Mount the archive then call the restore for each system part / app # + ########################################################################### - # Parse app instance name and id - # TODO: Use app_id to check if app is installed? - app_id, app_instance_nb = _parse_app_instance_name(app_instance_name) + restore_manager.mount() + restore_manager.restore() - # Check if the app is not already installed - if _is_installed(app_instance_name): - logger.error(m18n.n('restore_already_installed_app', - app=app_instance_name)) - continue - - # Check if the app has a restore script - app_script = tmp_app_dir + '/settings/scripts/restore' - if not os.path.isfile(app_script): - logger.warning(m18n.n('unrestore_app', app=app_instance_name)) - continue - - tmp_settings_dir = tmp_app_dir + '/settings' - app_setting_path = '/etc/yunohost/apps/' + app_instance_name - logger.info(m18n.n('restore_running_app_script', app=app_instance_name)) - try: - # Copy app settings and set permissions - # TODO: Copy app hooks too - shutil.copytree(tmp_app_dir + '/settings', app_setting_path) - filesystem.chmod(app_setting_path, 0555, 0444, True) - filesystem.chmod(app_setting_path + '/settings.yml', 0400) - - # Set correct right to the temporary settings folder - filesystem.chmod(tmp_settings_dir, 0550, 0550, True) - filesystem.chown(tmp_settings_dir, 'admin', None, True) - - # Prepare env. var. to pass to script - env_dict = {} - env_dict["YNH_APP_ID"] = app_id - env_dict["YNH_APP_INSTANCE_NAME"] = app_instance_name - env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb) - env_dict["YNH_APP_BACKUP_DIR"] = tmp_app_bkp_dir - - # Execute app restore script - hook_exec(app_script, args=[tmp_app_bkp_dir, app_instance_name], - raise_on_error=True, chdir=tmp_app_bkp_dir, env=env_dict, user="root") - except: - logger.exception(m18n.n('restore_app_failed', app=app_instance_name)) - - app_script = tmp_app_dir + '/settings/scripts/remove' - - # Setup environment for remove script - env_dict_remove = {} - env_dict_remove["YNH_APP_ID"] = app_id - env_dict_remove["YNH_APP_INSTANCE_NAME"] = app_instance_name - env_dict_remove["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb) - - # Execute remove script - # TODO: call app_remove instead - if hook_exec(app_script, args=[app_instance_name], - env=env_dict_remove, user="root") != 0: - logger.warning(m18n.n('app_not_properly_removed', - app=app_instance_name)) - - # Cleaning app directory - shutil.rmtree(app_setting_path, ignore_errors=True) - else: - result['apps'].append(app_instance_name) # Check if something has been restored - if not result['hooks'] and not result['apps']: - _clean_tmp_dir(1) + if restore_manager.success: + logger.success(m18n.n('restore_complete')) + else: raise MoulinetteError(errno.EINVAL, m18n.n('restore_nothings_done')) - if result['apps']: - app_ssowatconf(auth) - _clean_tmp_dir() - logger.success(m18n.n('restore_complete')) - - return result + return restore_manager.targets.results def backup_list(with_info=False, human_readable=False): @@ -612,7 +2203,7 @@ def backup_info(name, with_details=False, human_readable=False): # Check file exist (even if it's a broken symlink) if not os.path.lexists(archive_file): raise MoulinetteError(errno.EIO, - m18n.n('backup_archive_name_unknown', name=name)) + m18n.n('backup_archive_name_unknown', name=name)) # If symlink, retrieve the real path if os.path.islink(archive_file): @@ -621,16 +2212,24 @@ def backup_info(name, with_details=False, human_readable=False): # Raise exception if link is broken (e.g. on unmounted external storage) if not os.path.exists(archive_file): raise MoulinetteError(errno.EIO, - m18n.n('backup_archive_broken_link', path=archive_file)) + m18n.n('backup_archive_broken_link', + path=archive_file)) info_file = "%s/%s.info.json" % (ARCHIVES_PATH, name) + if not os.path.exists(info_file): + tar = tarfile.open(archive_file, "r:gz") + info_dir = info_file + '.d' + tar.extract('info.json', path=info_dir) + tar.close() + shutil.move(os.path.join(info_dir, 'info.json'), info_file) + os.rmdir(info_dir) + try: with open(info_file) as f: # Retrieve backup info info = json.load(f) except: - # TODO: Attempt to extract backup info file from tarball logger.debug("unable to load '%s'", info_file, exc_info=1) raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive')) @@ -653,8 +2252,13 @@ def backup_info(name, with_details=False, human_readable=False): } if with_details: - for d in ['apps', 'hooks']: - result[d] = info[d] + system_key = "system" + # Historically 'system' was 'hooks' + if "hooks" in info.keys(): + system_key = "hooks" + + result["apps"] = info["apps"] + result["system"] = info[system_key] return result @@ -672,7 +2276,7 @@ def backup_delete(name): info_file = "%s/%s.info.json" % (ARCHIVES_PATH, name) for backup_file in [archive_file, info_file]: - if not os.path.isfile(backup_file): + if not os.path.isfile(backup_file) and not os.path.islink(backup_file): raise MoulinetteError(errno.EIO, m18n.n('backup_archive_name_unknown', name=backup_file)) try: @@ -685,3 +2289,36 @@ def backup_delete(name): hook_callback('post_backup_delete', args=[name]) logger.success(m18n.n('backup_deleted')) + +############################################################################### +# Misc helpers # +############################################################################### + +def _create_archive_dir(): + """ Create the YunoHost archives directory if doesn't exist """ + if not os.path.isdir(ARCHIVES_PATH): + os.mkdir(ARCHIVES_PATH, 0750) + + +def _call_for_each_path(self, callback, csv_path=None): + """ Call a callback for each path in csv """ + if csv_path is None: + csv_path = self.csv_path + with open(csv_path, "r") as backup_file: + backup_csv = csv.DictReader(backup_file, fieldnames=['source', 'dest']) + for row in backup_csv: + callback(self, row['source'], row['dest']) + + +def free_space_in_directory(dirpath): + stat = os.statvfs(dirpath) + return stat.f_frsize * stat.f_bavail + + +def disk_usage(path): + # We don't do this in python with os.stat because we don't want + # to follow symlinks + + du_output = subprocess.check_output(['du', '-sb', path]) + return int(du_output.split()[0].decode('utf-8')) + diff --git a/src/yunohost/tests/test_backuprestore.py b/src/yunohost/tests/test_backuprestore.py new file mode 100644 index 000000000..245dee758 --- /dev/null +++ b/src/yunohost/tests/test_backuprestore.py @@ -0,0 +1,637 @@ +import pytest +import time +import requests +import os +import shutil +import subprocess +from mock import ANY + +from moulinette.core import init_authenticator +from yunohost.app import app_install, app_remove, app_ssowatconf +from yunohost.app import _is_installed +from yunohost.backup import backup_create, backup_restore, backup_list, backup_info, backup_delete +from yunohost.domain import _get_maindomain +from moulinette.core import MoulinetteError + +# Get main domain +maindomain = _get_maindomain() + +# Instantiate LDAP Authenticator +AUTH_IDENTIFIER = ('ldap', 'ldap-anonymous') +AUTH_PARAMETERS = {'uri': 'ldap://localhost:389', 'base_dn': 'dc=yunohost,dc=org'} +auth = None + +def setup_function(function): + + print "" + + global auth + auth = init_authenticator(AUTH_IDENTIFIER, AUTH_PARAMETERS) + + assert backup_test_dependencies_are_met() + + clean_tmp_backup_directory() + reset_ssowat_conf() + delete_all_backups() + uninstall_test_apps_if_needed() + + assert len(backup_list()["archives"]) == 0 + + markers = function.__dict__.keys() + + if "with_wordpress_archive_from_2p4" in markers: + add_archive_wordpress_from_2p4() + assert len(backup_list()["archives"]) == 1 + + if "with_backup_legacy_app_installed" in markers: + assert not app_is_installed("backup_legacy_app") + install_app("backup_legacy_app_ynh", "/yolo") + assert app_is_installed("backup_legacy_app") + + if "with_backup_recommended_app_installed" in markers: + assert not app_is_installed("backup_recommended_app") + install_app("backup_recommended_app_ynh", "/yolo", + "&helper_to_test=ynh_restore_file") + assert app_is_installed("backup_recommended_app") + + if "with_backup_recommended_app_installed_with_ynh_restore" in markers: + assert not app_is_installed("backup_recommended_app") + install_app("backup_recommended_app_ynh", "/yolo", + "&helper_to_test=ynh_restore") + assert app_is_installed("backup_recommended_app") + + if "with_system_archive_from_2p4" in markers: + add_archive_system_from_2p4() + assert len(backup_list()["archives"]) == 1 + + +def teardown_function(function): + + print "" + global auth + auth = init_authenticator(AUTH_IDENTIFIER, AUTH_PARAMETERS) + + assert tmp_backup_directory_is_empty() + + reset_ssowat_conf() + delete_all_backups() + uninstall_test_apps_if_needed() + + markers = function.__dict__.keys() + + if "clean_opt_dir" in markers: + shutil.rmtree("/opt/test_backup_output_directory") + + +############################################################################### +# Helpers # +############################################################################### + +def app_is_installed(app): + + # These are files we know should be installed by the app + app_files = [] + app_files.append("/etc/nginx/conf.d/%s.d/%s.conf" % (maindomain, app)) + app_files.append("/var/www/%s/index.html" % app) + app_files.append("/etc/importantfile") + + return _is_installed(app) and all(os.path.exists(f) for f in app_files) + + +def backup_test_dependencies_are_met(): + + # We need archivemount installed for the backup features to work + assert os.system("which archivemount >/dev/null") == 0 + + # Dummy test apps (or backup archives) + assert os.path.exists("./tests/apps/backup_wordpress_from_2p4") + assert os.path.exists("./tests/apps/backup_legacy_app_ynh") + assert os.path.exists("./tests/apps/backup_recommended_app_ynh") + + return True + +def tmp_backup_directory_is_empty(): + + if not os.path.exists("/home/yunohost.backup/tmp/"): + return True + else: + return len(os.listdir('/home/yunohost.backup/tmp/')) == 0 + +def clean_tmp_backup_directory(): + + if tmp_backup_directory_is_empty(): + return + + mount_lines = subprocess.check_output("mount").split("\n") + + points_to_umount = [ line.split(" ")[2] + for line in mount_lines + if len(line) >= 3 + and line.split(" ")[2].startswith("/home/yunohost.backup/tmp") ] + + for point in reversed(points_to_umount): + os.system("umount %s" % point) + + for f in os.listdir('/home/yunohost.backup/tmp/'): + shutil.rmtree("/home/yunohost.backup/tmp/%s" % f) + + shutil.rmtree("/home/yunohost.backup/tmp/") + +def reset_ssowat_conf(): + + # Make sure we have a ssowat + os.system("mkdir -p /etc/ssowat/") + app_ssowatconf(auth) + + +def delete_all_backups(): + + for archive in backup_list()["archives"]: + backup_delete(archive) + + +def uninstall_test_apps_if_needed(): + + if _is_installed("backup_legacy_app"): + app_remove(auth, "backup_legacy_app") + + if _is_installed("backup_recommended_app"): + app_remove(auth, "backup_recommended_app") + + if _is_installed("wordpress"): + app_remove(auth, "wordpress") + + +def install_app(app, path, additionnal_args=""): + + app_install(auth, "./tests/apps/%s" % app, + args="domain=%s&path=%s%s" % (maindomain, path, + additionnal_args)) + + +def add_archive_wordpress_from_2p4(): + + os.system("mkdir -p /home/yunohost.backup/archives") + + os.system("cp ./tests/apps/backup_wordpress_from_2p4/backup.info.json \ + /home/yunohost.backup/archives/backup_wordpress_from_2p4.info.json") + + os.system("cp ./tests/apps/backup_wordpress_from_2p4/backup.tar.gz \ + /home/yunohost.backup/archives/backup_wordpress_from_2p4.tar.gz") + + +def add_archive_system_from_2p4(): + + os.system("mkdir -p /home/yunohost.backup/archives") + + os.system("cp ./tests/apps/backup_system_from_2p4/backup.info.json \ + /home/yunohost.backup/archives/backup_system_from_2p4.info.json") + + os.system("cp ./tests/apps/backup_system_from_2p4/backup.tar.gz \ + /home/yunohost.backup/archives/backup_system_from_2p4.tar.gz") + +############################################################################### +# System backup # +############################################################################### + +def test_backup_only_ldap(): + + # Create the backup + backup_create(ignore_system=False, ignore_apps=True, system=["conf_ldap"]) + + archives = backup_list()["archives"] + assert len(archives) == 1 + + archives_info = backup_info(archives[0], with_details=True) + assert archives_info["apps"] == {} + assert len(archives_info["system"].keys()) == 1 + assert "conf_ldap" in archives_info["system"].keys() + + +def test_backup_system_part_that_does_not_exists(mocker): + + mocker.spy(m18n, "n") + + # Create the backup + with pytest.raises(MoulinetteError): + backup_create(ignore_system=False, ignore_apps=True, system=["yolol"]) + + m18n.n.assert_any_call('backup_hook_unknown', hook="yolol") + m18n.n.assert_any_call('backup_nothings_done') + +############################################################################### +# System backup and restore # +############################################################################### + +def test_backup_and_restore_all_sys(): + + # Create the backup + backup_create(ignore_system=False, ignore_apps=True) + + archives = backup_list()["archives"] + assert len(archives) == 1 + + archives_info = backup_info(archives[0], with_details=True) + assert archives_info["apps"] == {} + assert (len(archives_info["system"].keys()) == + len(os.listdir("/usr/share/yunohost/hooks/backup/"))) + + # Remove ssowat conf + assert os.path.exists("/etc/ssowat/conf.json") + os.system("rm -rf /etc/ssowat/") + assert not os.path.exists("/etc/ssowat/conf.json") + + # Restore the backup + backup_restore(auth, name=archives[0], force=True, + ignore_system=False, ignore_apps=True) + + # Check ssowat conf is back + assert os.path.exists("/etc/ssowat/conf.json") + + +def test_backup_and_restore_archivemount_failure(monkeypatch, mocker): + + # Create the backup + backup_create(ignore_system=False, ignore_apps=True) + + archives = backup_list()["archives"] + assert len(archives) == 1 + + archives_info = backup_info(archives[0], with_details=True) + assert archives_info["apps"] == {} + assert (len(archives_info["system"].keys()) == + len(os.listdir("/usr/share/yunohost/hooks/backup/"))) + + # Remove ssowat conf + assert os.path.exists("/etc/ssowat/conf.json") + os.system("rm -rf /etc/ssowat/") + assert not os.path.exists("/etc/ssowat/conf.json") + + def custom_subprocess_call(*args, **kwargs): + import subprocess as subprocess2 + if args[0] and args[0][0]=="archivemount": + monkeypatch.undo() + return 1 + return subprocess.call(*args, **kwargs) + + monkeypatch.setattr("subprocess.call", custom_subprocess_call) + mocker.spy(m18n, "n") + + # Restore the backup + backup_restore(auth, name=archives[0], force=True, + ignore_system=False, ignore_apps=True) + + # Check ssowat conf is back + assert os.path.exists("/etc/ssowat/conf.json") + + +############################################################################### +# System restore from 2.4 # +############################################################################### + +@pytest.mark.with_system_archive_from_2p4 +def test_restore_system_from_Ynh2p4(monkeypatch, mocker): + + # Backup current system + backup_create(ignore_system=False, ignore_apps=True) + archives = backup_list()["archives"] + assert len(archives) == 2 + + # Restore system archive from 2.4 + try: + backup_restore(auth, name=backup_list()["archives"][1], + ignore_system=False, + ignore_apps=True, + force=True) + finally: + # Restore system as it was + backup_restore(auth, name=backup_list()["archives"][0], + ignore_system=False, + ignore_apps=True, + force=True) + + +@pytest.mark.with_system_archive_from_2p4 +def test_restore_system_from_Ynh2p4_archivemount_failure(monkeypatch, mocker): + + # Backup current system + backup_create(ignore_system=False, ignore_apps=True) + archives = backup_list()["archives"] + assert len(archives) == 2 + + def custom_subprocess_call(*args, **kwargs): + import subprocess as subprocess2 + if args[0] and args[0][0]=="archivemount": + monkeypatch.undo() + return 1 + return subprocess.call(*args, **kwargs) + + monkeypatch.setattr("subprocess.call", custom_subprocess_call) + + try: + # Restore system from 2.4 + backup_restore(auth, name=backup_list()["archives"][1], + ignore_system=False, + ignore_apps=True, + force=True) + finally: + # Restore system as it was + backup_restore(auth, name=backup_list()["archives"][0], + ignore_system=False, + ignore_apps=True, + force=True) + + +############################################################################### +# App backup # +############################################################################### + +@pytest.mark.with_backup_recommended_app_installed +def test_backup_script_failure_handling(monkeypatch, mocker): + + def custom_hook_exec(name, *args, **kwargs): + + if os.path.basename(name).startswith("backup_"): + raise Exception + else: + return True + + # Create a backup of this app and simulate a crash (patching the backup + # call with monkeypatch). We also patch m18n to check later it's been called + # with the expected error message key + monkeypatch.setattr("yunohost.backup.hook_exec", custom_hook_exec) + mocker.spy(m18n, "n") + + with pytest.raises(MoulinetteError): + backup_create(ignore_system=True, ignore_apps=False, apps=["backup_recommended_app"]) + + m18n.n.assert_any_call('backup_app_failed', app='backup_recommended_app') + +@pytest.mark.with_backup_recommended_app_installed +def test_backup_not_enough_free_space(monkeypatch, mocker): + + def custom_disk_usage(path): + return 99999999999999999 + + def custom_free_space_in_directory(dirpath): + return 0 + + monkeypatch.setattr("yunohost.backup.disk_usage", custom_disk_usage) + monkeypatch.setattr("yunohost.backup.free_space_in_directory", + custom_free_space_in_directory) + + mocker.spy(m18n, "n") + + with pytest.raises(MoulinetteError): + backup_create(ignore_system=True, ignore_apps=False, apps=["backup_recommended_app"]) + + m18n.n.assert_any_call('not_enough_disk_space', path=ANY) + + +def test_backup_app_not_installed(mocker): + + assert not _is_installed("wordpress") + + mocker.spy(m18n, "n") + + with pytest.raises(MoulinetteError): + backup_create(ignore_system=True, ignore_apps=False, apps=["wordpress"]) + + m18n.n.assert_any_call("unbackup_app", app="wordpress") + m18n.n.assert_any_call('backup_nothings_done') + + +@pytest.mark.with_backup_recommended_app_installed +def test_backup_app_with_no_backup_script(mocker): + + backup_script = "/etc/yunohost/apps/backup_recommended_app/scripts/backup" + os.system("rm %s" % backup_script) + assert not os.path.exists(backup_script) + + mocker.spy(m18n, "n") + + with pytest.raises(MoulinetteError): + backup_create(ignore_system=True, ignore_apps=False, apps=["backup_recommended_app"]) + + m18n.n.assert_any_call("backup_with_no_backup_script_for_app", app="backup_recommended_app") + m18n.n.assert_any_call('backup_nothings_done') + + +@pytest.mark.with_backup_recommended_app_installed +def test_backup_app_with_no_restore_script(mocker): + + restore_script = "/etc/yunohost/apps/backup_recommended_app/scripts/restore" + os.system("rm %s" % restore_script) + assert not os.path.exists(restore_script) + + mocker.spy(m18n, "n") + + # Backuping an app with no restore script will only display a warning to the + # user... + + backup_create(ignore_system=True, ignore_apps=False, apps=["backup_recommended_app"]) + + m18n.n.assert_any_call("backup_with_no_restore_script_for_app", app="backup_recommended_app") + + +@pytest.mark.clean_opt_dir +def test_backup_with_different_output_directory(): + + # Create the backup + backup_create(ignore_system=False, ignore_apps=True, system=["conf_ssh"], + output_directory="/opt/test_backup_output_directory", + name="backup") + + assert os.path.exists("/opt/test_backup_output_directory/backup.tar.gz") + + archives = backup_list()["archives"] + assert len(archives) == 1 + + archives_info = backup_info(archives[0], with_details=True) + assert archives_info["apps"] == {} + assert len(archives_info["system"].keys()) == 1 + assert "conf_ssh" in archives_info["system"].keys() + +@pytest.mark.clean_opt_dir +def test_backup_with_no_compress(): + # Create the backup + backup_create(ignore_system=False, ignore_apps=True, system=["conf_nginx"], + output_directory="/opt/test_backup_output_directory", + no_compress=True, + name="backup") + + assert os.path.exists("/opt/test_backup_output_directory/info.json") + + +############################################################################### +# App restore # +############################################################################### + +@pytest.mark.with_wordpress_archive_from_2p4 +def test_restore_app_wordpress_from_Ynh2p4(): + + backup_restore(auth, name=backup_list()["archives"][0], + ignore_system=True, + ignore_apps=False, + apps=["wordpress"]) + + +@pytest.mark.with_wordpress_archive_from_2p4 +def test_restore_app_script_failure_handling(monkeypatch, mocker): + + def custom_hook_exec(name, *args, **kwargs): + if os.path.basename(name).startswith("restore"): + monkeypatch.undo() + raise Exception + + monkeypatch.setattr("yunohost.backup.hook_exec", custom_hook_exec) + mocker.spy(m18n, "n") + + assert not _is_installed("wordpress") + + with pytest.raises(MoulinetteError): + backup_restore(auth, name=backup_list()["archives"][0], + ignore_system=True, + ignore_apps=False, + apps=["wordpress"]) + + m18n.n.assert_any_call('restore_app_failed', app='wordpress') + m18n.n.assert_any_call('restore_nothings_done') + assert not _is_installed("wordpress") + + +@pytest.mark.with_wordpress_archive_from_2p4 +def test_restore_app_not_enough_free_space(monkeypatch, mocker): + + def custom_free_space_in_directory(dirpath): + return 0 + + monkeypatch.setattr("yunohost.backup.free_space_in_directory", + custom_free_space_in_directory) + mocker.spy(m18n, "n") + + assert not _is_installed("wordpress") + + with pytest.raises(MoulinetteError): + backup_restore(auth, name=backup_list()["archives"][0], + ignore_system=True, + ignore_apps=False, + apps=["wordpress"]) + + m18n.n.assert_any_call('restore_not_enough_disk_space', + free_space=0, + margin=ANY, + needed_space=ANY) + assert not _is_installed("wordpress") + + +@pytest.mark.with_wordpress_archive_from_2p4 +def test_restore_app_not_in_backup(mocker): + + assert not _is_installed("wordpress") + assert not _is_installed("yoloswag") + + mocker.spy(m18n, "n") + + with pytest.raises(MoulinetteError): + backup_restore(auth, name=backup_list()["archives"][0], + ignore_system=True, + ignore_apps=False, + apps=["yoloswag"]) + + m18n.n.assert_any_call('backup_archive_app_not_found', app="yoloswag") + assert not _is_installed("wordpress") + assert not _is_installed("yoloswag") + + +@pytest.mark.with_wordpress_archive_from_2p4 +def test_restore_app_archivemount_failure(monkeypatch, mocker): + + def custom_subprocess_call(*args, **kwargs): + import subprocess as subprocess2 + if args[0] and args[0][0]=="archivemount": + monkeypatch.undo() + return 1 + return subprocess.call(*args, **kwargs) + + monkeypatch.setattr("subprocess.call", custom_subprocess_call) + mocker.spy(m18n, "n") + + assert not _is_installed("wordpress") + + backup_restore(auth, name=backup_list()["archives"][0], + ignore_system=True, + ignore_apps=False, + apps=["wordpress"]) + + assert _is_installed("wordpress") + + +@pytest.mark.with_wordpress_archive_from_2p4 +def test_restore_app_already_installed(mocker): + + assert not _is_installed("wordpress") + + backup_restore(auth, name=backup_list()["archives"][0], + ignore_system=True, + ignore_apps=False, + apps=["wordpress"]) + + assert _is_installed("wordpress") + + mocker.spy(m18n, "n") + with pytest.raises(MoulinetteError): + backup_restore(auth, name=backup_list()["archives"][0], + ignore_system=True, + ignore_apps=False, + apps=["wordpress"]) + + m18n.n.assert_any_call('restore_already_installed_app', app="wordpress") + m18n.n.assert_any_call('restore_nothings_done') + + assert _is_installed("wordpress") + + +@pytest.mark.with_backup_legacy_app_installed +def test_backup_and_restore_legacy_app(): + + _test_backup_and_restore_app("backup_legacy_app") + + +@pytest.mark.with_backup_recommended_app_installed +def test_backup_and_restore_recommended_app(): + + _test_backup_and_restore_app("backup_recommended_app") + + +@pytest.mark.with_backup_recommended_app_installed_with_ynh_restore +def test_backup_and_restore_with_ynh_restore(): + + _test_backup_and_restore_app("backup_recommended_app") + + +def _test_backup_and_restore_app(app): + + # Create a backup of this app + backup_create(ignore_system=True, ignore_apps=False, apps=[app]) + + archives = backup_list()["archives"] + assert len(archives) == 1 + + archives_info = backup_info(archives[0], with_details=True) + assert archives_info["system"] == {} + assert len(archives_info["apps"].keys()) == 1 + assert app in archives_info["apps"].keys() + + # Uninstall the app + app_remove(auth, app) + assert not app_is_installed(app) + + # Restore the app + backup_restore(auth, name=archives[0], ignore_system=True, + ignore_apps=False, apps=[app]) + + assert app_is_installed(app) + + +