yunohost/helpers/helpers.v1.d/backup

507 lines
21 KiB
Bash

#!/bin/bash
CAN_BIND=${CAN_BIND:-1}
# Add a file or a directory to the list of paths to backup
#
# usage: ynh_backup --src_path=src_path [--dest_path=dest_path] [--is_big] [--not_mandatory]
# | arg: -s, --src_path= - file or directory to bind or symlink or copy. it shouldn't be in the backup dir.
# | arg: -d, --dest_path= - destination file or directory inside the backup dir
# | arg: -b, --is_big - Indicate data are big (mail, video, image ...)
# | arg: -m, --not_mandatory - Indicate that if the file is missing, the backup can ignore it.
#
# This helper can be used both in a system backup hook, and in an app backup script
#
# `ynh_backup` writes `src_path` and the relative `dest_path` into a CSV file, and it
# creates the parent destination directory
#
# If `dest_path` is ended by a slash it complete this path with the basename of `src_path`.
#
# Example in the context of a wordpress app :
# ```
# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf"
# # => This line will be added into CSV file
# # "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/etc/nginx/conf.d/$domain.d/$app.conf"
#
# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf/nginx.conf"
# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/nginx.conf"
#
# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf/"
# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/$app.conf"
#
# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf"
# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf"
#
# #Deprecated usages (maintained for retro-compatibility)
# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "${backup_dir}/conf/nginx.conf"
# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/nginx.conf"
#
# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "/conf/"
# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/$app.conf"
#
# ```
#
# How to use `--is_big`:
#
# `--is_big` is used to specify that this part of the backup can be quite huge.
# So, you don't want that your package does backup that part during ynh_backup_before_upgrade.
# In the same way, an user may doesn't want to backup this big part of the app for
# each of his backup. And so handle that part differently.
#
# As this part of your backup may not be done, your restore script has to handle it.
# In your restore script, use `--not_mandatory` with `ynh_restore_file`
# As well in your remove script, you should not remove those data ! Or an user may end up with
# a failed upgrade restoring an app without data anymore !
#
# To have the benefit of `--is_big` while doing a backup, you can whether set the environement
# variable `BACKUP_CORE_ONLY` to 1 (`BACKUP_CORE_ONLY=1`) before the backup command. It will affect
# only that backup command.
# Or set the config `do_not_backup_data` to 1 into the `settings.yml` of the app. This will affect
# all backups for this app until the setting is removed.
#
# Requires YunoHost version 2.4.0 or higher.
# Requires YunoHost version 3.5.0 or higher for the argument `--not_mandatory`
ynh_backup() {
# TODO find a way to avoid injection by file strange naming !
# Declare an array to define the options of this helper.
local legacy_args=sdbm
local -A args_array=([s]=src_path= [d]=dest_path= [b]=is_big [m]=not_mandatory)
local src_path
local dest_path
local is_big
local not_mandatory
# Manage arguments with getopts
ynh_handle_getopts_args "$@"
dest_path="${dest_path:-}"
is_big="${is_big:-0}"
not_mandatory="${not_mandatory:-0}"
BACKUP_CORE_ONLY=${BACKUP_CORE_ONLY:-0}
test -n "${app:-}" && do_not_backup_data=$(ynh_app_setting_get --app=$app --key=do_not_backup_data)
# If backing up core only (used by ynh_backup_before_upgrade),
# don't backup big data items
if [ $is_big -eq 1 ] && ([ ${do_not_backup_data:-0} -eq 1 ] || [ $BACKUP_CORE_ONLY -eq 1 ]); then
if [ $BACKUP_CORE_ONLY -eq 1 ]; then
ynh_print_info --message="$src_path will not be saved, because 'BACKUP_CORE_ONLY' is set."
else
ynh_print_info --message="$src_path will not be saved, because 'do_not_backup_data' is set."
fi
return 0
fi
# ==============================================================================
# Format correctly source and destination paths
# ==============================================================================
# Be sure the source path is not empty
if [ ! -e "$src_path" ]; then
ynh_print_warn --message="Source path '${src_path}' does not exist"
if [ "$not_mandatory" == "0" ]; then
# This is a temporary fix for fail2ban config files missing after the migration to stretch.
if echo "${src_path}" | grep --quiet "/etc/fail2ban"; then
touch "${src_path}"
ynh_print_info --message="The missing file will be replaced by a dummy one for the backup !!!"
else
return 1
fi
else
return 0
fi
fi
# Transform the source path as an absolute path
# If it's a dir remove the ending /
src_path=$(realpath "$src_path")
# If there is no destination path, initialize it with the source path
# relative to "/".
# eg: src_path=/etc/yunohost -> dest_path=etc/yunohost
if [[ -z "$dest_path" ]]; then
dest_path="${src_path#/}"
else
if [[ "${dest_path:0:1}" == "/" ]]; then
# If the destination path is an absolute path, transform it as a path
# relative to the current working directory ($YNH_CWD)
#
# If it's an app backup script that run this helper, YNH_CWD is equal to
# $YNH_BACKUP_DIR/apps/APP_INSTANCE_NAME/backup/
#
# If it's a system part backup script, YNH_CWD is equal to $YNH_BACKUP_DIR
dest_path="${dest_path#$YNH_CWD/}"
# Case where $2 is an absolute dir but doesn't begin with $YNH_CWD
if [[ "${dest_path:0:1}" == "/" ]]; then
dest_path="${dest_path#/}"
fi
fi
# Complete dest_path if ended by a /
if [[ "${dest_path: -1}" == "/" ]]; then
dest_path="${dest_path}/$(basename $src_path)"
fi
fi
# Check if dest_path already exists in tmp archive
if [[ -e "${dest_path}" ]]; then
ynh_print_err --message="Destination path '${dest_path}' already exist"
return 1
fi
# Add the relative current working directory to the destination path
local rel_dir="${YNH_CWD#$YNH_BACKUP_DIR}"
rel_dir="${rel_dir%/}/"
dest_path="${rel_dir}${dest_path}"
dest_path="${dest_path#/}"
# ==============================================================================
# ==============================================================================
# Write file to backup into backup_list
# ==============================================================================
local src=$(echo "${src_path}" | sed --regexp-extended 's/"/\"\"/g')
local dest=$(echo "${dest_path}" | sed --regexp-extended 's/"/\"\"/g')
echo "\"${src}\",\"${dest}\"" >>"${YNH_BACKUP_CSV}"
# ==============================================================================
# Create the parent dir of the destination path
# It's for retro compatibility, some script consider ynh_backup creates this dir
mkdir --parents $(dirname "$YNH_BACKUP_DIR/${dest_path}")
}
# Restore all files that were previously backuped in a core backup script or app backup script
#
# usage: ynh_restore
#
# Requires YunoHost version 2.6.4 or higher.
ynh_restore() {
# Deduce the relative path of $YNH_CWD
local REL_DIR="${YNH_CWD#$YNH_BACKUP_DIR/}"
REL_DIR="${REL_DIR%/}/"
# For each destination path begining by $REL_DIR
cat ${YNH_BACKUP_CSV} | tr --delete $'\r' | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR.*\"$" \
| while read line; do
local ORIGIN_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\"\K.*(?=\",\".*\"$)")
local ARCHIVE_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR\K.*(?=\"$)")
ynh_restore_file --origin_path="$ARCHIVE_PATH" --dest_path="$ORIGIN_PATH"
done
}
# Return the path in the archive where has been stocked the origin path
#
# [internal]
#
# usage: _get_archive_path ORIGIN_PATH
_get_archive_path() {
# For security reasons we use csv python library to read the CSV
python3 -c "
import sys
import csv
with open(sys.argv[1], 'r') as backup_file:
backup_csv = csv.DictReader(backup_file, fieldnames=['source', 'dest'])
for row in backup_csv:
if row['source']==sys.argv[2].strip('\"'):
print(row['dest'])
sys.exit(0)
raise Exception('Original path for %s not found' % sys.argv[2])
" "${YNH_BACKUP_CSV}" "$1"
return $?
}
# Restore a file or a directory
#
# usage: ynh_restore_file --origin_path=origin_path [--dest_path=dest_path] [--not_mandatory]
# | arg: -o, --origin_path= - Path where was located the file or the directory before to be backuped or relative path to $YNH_CWD where it is located in the backup archive
# | arg: -d, --dest_path= - Path where restore the file or the dir. If unspecified, the destination will be `ORIGIN_PATH` or if the `ORIGIN_PATH` doesn't exist in the archive, the destination will be searched into `backup.csv`
# | arg: -m, --not_mandatory - Indicate that if the file is missing, the restore process can ignore it.
#
# Use the registered path in backup_list by ynh_backup to restore the file at the right place.
#
# examples:
# ynh_restore_file -o "/etc/nginx/conf.d/$domain.d/$app.conf"
# # You can also use relative paths:
# ynh_restore_file -o "conf/nginx.conf"
#
# If `DEST_PATH` already exists and is lighter than 500 Mo, a backup will be made in
# `/var/cache/yunohost/appconfbackup/`. Otherwise, the existing file is removed.
#
# if `apps/$app/etc/nginx/conf.d/$domain.d/$app.conf` exists, restore it into
# `/etc/nginx/conf.d/$domain.d/$app.conf`
# if no, search for a match in the csv (eg: conf/nginx.conf) and restore it into
# `/etc/nginx/conf.d/$domain.d/$app.conf`
#
# Requires YunoHost version 2.6.4 or higher.
# Requires YunoHost version 3.5.0 or higher for the argument --not_mandatory
ynh_restore_file() {
# Declare an array to define the options of this helper.
local legacy_args=odm
local -A args_array=([o]=origin_path= [d]=dest_path= [m]=not_mandatory)
local origin_path
local dest_path
local not_mandatory
# Manage arguments with getopts
ynh_handle_getopts_args "$@"
origin_path="/${origin_path#/}"
# Default value for dest_path = /$origin_path
dest_path="${dest_path:-$origin_path}"
not_mandatory="${not_mandatory:-0}"
local archive_path="$YNH_CWD${origin_path}"
# If archive_path doesn't exist, search for a corresponding path in CSV
if [ ! -d "$archive_path" ] && [ ! -f "$archive_path" ] && [ ! -L "$archive_path" ]; then
if [ "$not_mandatory" == "0" ]; then
archive_path="$YNH_BACKUP_DIR/$(_get_archive_path \"$origin_path\")"
else
return 0
fi
fi
# Move the old directory if it already exists
if [[ -e "${dest_path}" ]]; then
# Check if the file/dir size is less than 500 Mo
if [[ $(du --summarize --bytes ${dest_path} | cut --delimiter="/" --fields=1) -le "500000000" ]]; then
local backup_file="/var/cache/yunohost/appconfbackup/${dest_path}.backup.$(date '+%Y%m%d.%H%M%S')"
mkdir --parents "$(dirname "$backup_file")"
mv "${dest_path}" "$backup_file" # Move the current file or directory
else
ynh_secure_remove --file=${dest_path}
fi
fi
# Restore origin_path into dest_path
mkdir --parents $(dirname "$dest_path")
# Do a copy if it's just a mounting point
if mountpoint --quiet $YNH_BACKUP_DIR; then
if [[ -d "${archive_path}" ]]; then
archive_path="${archive_path}/."
mkdir --parents "$dest_path"
fi
cp --archive "$archive_path" "${dest_path}"
# Do a move if YNH_BACKUP_DIR is already a copy
else
mv "$archive_path" "${dest_path}"
fi
# Boring hack for nginx conf file mapped to php7.3
# Note that there's no need to patch the fpm config because most php apps
# will call "ynh_add_fpm_config" during restore, effectively recreating the file from scratch
if [[ "${dest_path}" == "/etc/nginx/conf.d/"* ]] && grep 'php7.3.*sock' "${dest_path}"
then
sed -i 's/php7.3/php7.4/g' "${dest_path}"
fi
}
# Calculate and store a file checksum into the app settings
#
# usage: ynh_store_file_checksum --file=file
# | arg: -f, --file= - The file on which the checksum will performed, then stored.
#
# $app should be defined when calling this helper
#
# Requires YunoHost version 2.6.4 or higher.
ynh_store_file_checksum() {
# Declare an array to define the options of this helper.
local legacy_args=f
local -A args_array=([f]=file= [u]=update_only)
local file
local update_only
update_only="${update_only:-0}"
# Manage arguments with getopts
ynh_handle_getopts_args "$@"
local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
# If update only, we don't save the new checksum if no old checksum exist
if [ $update_only -eq 1 ]; then
local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name)
if [ -z "${checksum_value}" ]; then
unset backup_file_checksum
return 0
fi
fi
ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(md5sum "$file" | cut --delimiter=' ' --fields=1)
if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then
# Using a base64 is in fact more reversible than "replace / and space by _" ... So we can in fact obtain the original file path in an easy reliable way ...
local file_path_base64=$(echo "$file" | base64 -w0)
mkdir -p /var/cache/yunohost/appconfbackup/
cat $file > /var/cache/yunohost/appconfbackup/original_${file_path_base64}
fi
# If backup_file_checksum isn't empty, ynh_backup_if_checksum_is_different has made a backup
if [ -n "${backup_file_checksum-}" ]; then
# Print the diff between the previous file and the new one.
# diff return 1 if the files are different, so the || true
diff --report-identical-files --unified --color=always $backup_file_checksum $file >&2 || true
fi
# Unset the variable, so it wouldn't trig a ynh_store_file_checksum without a ynh_backup_if_checksum_is_different before it.
unset backup_file_checksum
}
# Verify the checksum and backup the file if it's different
#
# usage: ynh_backup_if_checksum_is_different --file=file
# | arg: -f, --file= - The file on which the checksum test will be perfomed.
# | ret: the name of a backup file, or nothing
#
# This helper is primarily meant to allow to easily backup personalised/manually
# modified config files.
#
# Requires YunoHost version 2.6.4 or higher.
ynh_backup_if_checksum_is_different() {
# Declare an array to define the options of this helper.
local legacy_args=f
local -A args_array=([f]=file=)
local file
# Manage arguments with getopts
ynh_handle_getopts_args "$@"
local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name)
# backup_file_checksum isn't declare as local, so it can be reuse by ynh_store_file_checksum
backup_file_checksum=""
if [ -n "$checksum_value" ]; then # Proceed only if a value was stored into the app settings
if [ -e $file ] && ! echo "$checksum_value $file" | md5sum --check --status; then # If the checksum is now different
backup_file_checksum="/var/cache/yunohost/appconfbackup/$file.backup.$(date '+%Y%m%d.%H%M%S')"
mkdir --parents "$(dirname "$backup_file_checksum")"
cp --archive "$file" "$backup_file_checksum" # Backup the current file
ynh_print_warn "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum"
echo "$backup_file_checksum" # Return the name of the backup file
if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then
local file_path_base64=$(echo "$file" | base64 -w0)
if test -e /var/cache/yunohost/appconfbackup/original_${file_path_base64}
then
ynh_print_warn "Diff with the original file:"
diff --report-identical-files --unified --color=always /var/cache/yunohost/appconfbackup/original_${file_path_base64} $file >&2 || true
fi
fi
fi
fi
}
# Delete a file checksum from the app settings
#
# usage: ynh_delete_file_checksum --file=file
# | arg: -f, --file= - The file for which the checksum will be deleted
#
# $app should be defined when calling this helper
#
# Requires YunoHost version 3.3.1 or higher.
ynh_delete_file_checksum() {
# Declare an array to define the options of this helper.
local legacy_args=f
local -A args_array=([f]=file=)
local file
# Manage arguments with getopts
ynh_handle_getopts_args "$@"
local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
ynh_app_setting_delete --app=$app --key=$checksum_setting_name
}
# Checks a backup archive exists
#
# [internal]
#
ynh_backup_archive_exists() {
yunohost backup list --output-as json --quiet \
| jq -e --arg archive "$1" '.archives | index($archive)' >/dev/null
}
# Make a backup in case of failed upgrade
#
# [packagingv1]
#
# usage: ynh_backup_before_upgrade
#
# Usage in a package script:
# ```
# ynh_backup_before_upgrade
# ynh_clean_setup () {
# ynh_restore_upgradebackup
# }
# ynh_abort_if_errors
# ```
#
# Requires YunoHost version 2.7.2 or higher.
ynh_backup_before_upgrade() {
if [ ! -e "/etc/yunohost/apps/$app/scripts/backup" ]; then
ynh_print_warn --message="This app doesn't have any backup script."
return
fi
backup_number=1
local old_backup_number=2
local app_bck=${app//_/-} # Replace all '_' by '-'
NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0}
if [ "$NO_BACKUP_UPGRADE" -eq 0 ]; then
# Check if a backup already exists with the prefix 1
if ynh_backup_archive_exists "$app_bck-pre-upgrade1"; then
# Prefix becomes 2 to preserve the previous backup
backup_number=2
old_backup_number=1
fi
# Create backup
BACKUP_CORE_ONLY=1 yunohost backup create --apps $app --name $app_bck-pre-upgrade$backup_number --debug
if [ "$?" -eq 0 ]; then
# If the backup succeeded, remove the previous backup
if ynh_backup_archive_exists "$app_bck-pre-upgrade$old_backup_number"; then
# Remove the previous backup only if it exists
yunohost backup delete $app_bck-pre-upgrade$old_backup_number >/dev/null
fi
else
ynh_die --message="Backup failed, the upgrade process was aborted."
fi
else
ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, backup will be avoided. Be careful, this upgrade is going to be operated without a security backup"
fi
}
# Restore a previous backup if the upgrade process failed
#
# [packagingv1]
#
# usage: ynh_restore_upgradebackup
#
# Usage in a package script:
# ```
# ynh_backup_before_upgrade
# ynh_clean_setup () {
# ynh_restore_upgradebackup
# }
# ynh_abort_if_errors
# ```
#
# Requires YunoHost version 2.7.2 or higher.
ynh_restore_upgradebackup() {
ynh_print_err --message="Upgrade failed."
local app_bck=${app//_/-} # Replace all '_' by '-'
NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0}
if [ "$NO_BACKUP_UPGRADE" -eq 0 ]; then
# Check if an existing backup can be found before removing and restoring the application.
if ynh_backup_archive_exists "$app_bck-pre-upgrade$backup_number"; then
# Remove the application then restore it
yunohost app remove $app
# Restore the backup
yunohost backup restore $app_bck-pre-upgrade$backup_number --apps $app --force --debug
if [[ -d /etc/yunohost/apps/$app ]]
then
ynh_die --message="The app was restored to the way it was before the failed upgrade."
else
ynh_die --message="Uhoh ... Yunohost failed to restore the app to the way it was before the failed upgrade :|"
fi
fi
else
ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, that means there's no backup to restore. You have to fix this upgrade by yourself !"
fi
}