#!/bin/bash CAN_BIND=${CAN_BIND:-1} # Add a file or a directory to the list of paths to backup # # usage: ynh_backup --src_path=src_path [--dest_path=dest_path] [--is_big] [--not_mandatory] # | arg: -s, --src_path= - file or directory to bind or symlink or copy. it shouldn't be in the backup dir. # | arg: -d, --dest_path= - destination file or directory inside the backup dir # | arg: -b, --is_big - Indicate data are big (mail, video, image ...) # | arg: -m, --not_mandatory - Indicate that if the file is missing, the backup can ignore it. # # This helper can be used both in a system backup hook, and in an app backup script # # `ynh_backup` writes `src_path` and the relative `dest_path` into a CSV file, and it # creates the parent destination directory # # If `dest_path` is ended by a slash it complete this path with the basename of `src_path`. # # Example in the context of a wordpress app : # ``` # ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" # # => This line will be added into CSV file # # "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/etc/nginx/conf.d/$domain.d/$app.conf" # # ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf/nginx.conf" # # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/nginx.conf" # # ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf/" # # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/$app.conf" # # ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf" # # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf" # # #Deprecated usages (maintained for retro-compatibility) # ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "${backup_dir}/conf/nginx.conf" # # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/nginx.conf" # # ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "/conf/" # # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/$app.conf" # # ``` # # How to use `--is_big`: # # `--is_big` is used to specify that this part of the backup can be quite huge. # So, you don't want that your package does backup that part during ynh_backup_before_upgrade. # In the same way, an user may doesn't want to backup this big part of the app for # each of his backup. And so handle that part differently. # # As this part of your backup may not be done, your restore script has to handle it. # In your restore script, use `--not_mandatory` with `ynh_restore_file` # As well in your remove script, you should not remove those data ! Or an user may end up with # a failed upgrade restoring an app without data anymore ! # # To have the benefit of `--is_big` while doing a backup, you can whether set the environement # variable `BACKUP_CORE_ONLY` to 1 (`BACKUP_CORE_ONLY=1`) before the backup command. It will affect # only that backup command. # Or set the config `do_not_backup_data` to 1 into the `settings.yml` of the app. This will affect # all backups for this app until the setting is removed. # # Requires YunoHost version 2.4.0 or higher. # Requires YunoHost version 3.5.0 or higher for the argument `--not_mandatory` ynh_backup() { # TODO find a way to avoid injection by file strange naming ! # ============ Argument parsing ============= local -A args_array=([s]=src_path= [d]=dest_path= [b]=is_big [m]=not_mandatory) local src_path local dest_path local is_big local not_mandatory ynh_handle_getopts_args "$@" dest_path="${dest_path:-}" is_big="${is_big:-0}" not_mandatory="${not_mandatory:-0}" # =========================================== BACKUP_CORE_ONLY=${BACKUP_CORE_ONLY:-0} test -n "${app:-}" && do_not_backup_data=$(ynh_app_setting_get --key=do_not_backup_data) # If backing up core only (used by ynh_backup_before_upgrade), # don't backup big data items if [ $is_big -eq 1 ] && ([ ${do_not_backup_data:-0} -eq 1 ] || [ $BACKUP_CORE_ONLY -eq 1 ]); then if [ $BACKUP_CORE_ONLY -eq 1 ]; then ynh_print_info --message="$src_path will not be saved, because 'BACKUP_CORE_ONLY' is set." else ynh_print_info --message="$src_path will not be saved, because 'do_not_backup_data' is set." fi return 0 fi # ============================================================================== # Format correctly source and destination paths # ============================================================================== # Be sure the source path is not empty if [ ! -e "$src_path" ]; then ynh_print_warn --message="Source path '${src_path}' does not exist" if [ "$not_mandatory" == "0" ]; then # This is a temporary fix for fail2ban config files missing after the migration to stretch. if echo "${src_path}" | grep --quiet "/etc/fail2ban"; then touch "${src_path}" ynh_print_info --message="The missing file will be replaced by a dummy one for the backup !!!" else return 1 fi else return 0 fi fi # Transform the source path as an absolute path # If it's a dir remove the ending / src_path=$(realpath "$src_path") # If there is no destination path, initialize it with the source path # relative to "/". # eg: src_path=/etc/yunohost -> dest_path=etc/yunohost if [[ -z "$dest_path" ]]; then dest_path="${src_path#/}" else if [[ "${dest_path:0:1}" == "/" ]]; then # If the destination path is an absolute path, transform it as a path # relative to the current working directory ($YNH_CWD) # # If it's an app backup script that run this helper, YNH_CWD is equal to # $YNH_BACKUP_DIR/apps/APP_INSTANCE_NAME/backup/ # # If it's a system part backup script, YNH_CWD is equal to $YNH_BACKUP_DIR dest_path="${dest_path#$YNH_CWD/}" # Case where $2 is an absolute dir but doesn't begin with $YNH_CWD if [[ "${dest_path:0:1}" == "/" ]]; then dest_path="${dest_path#/}" fi fi # Complete dest_path if ended by a / if [[ "${dest_path: -1}" == "/" ]]; then dest_path="${dest_path}/$(basename $src_path)" fi fi # Check if dest_path already exists in tmp archive if [[ -e "${dest_path}" ]]; then ynh_print_err --message="Destination path '${dest_path}' already exist" return 1 fi # Add the relative current working directory to the destination path local rel_dir="${YNH_CWD#$YNH_BACKUP_DIR}" rel_dir="${rel_dir%/}/" dest_path="${rel_dir}${dest_path}" dest_path="${dest_path#/}" # ============================================================================== # ============================================================================== # Write file to backup into backup_list # ============================================================================== local src=$(echo "${src_path}" | sed --regexp-extended 's/"/\"\"/g') local dest=$(echo "${dest_path}" | sed --regexp-extended 's/"/\"\"/g') echo "\"${src}\",\"${dest}\"" >>"${YNH_BACKUP_CSV}" # ============================================================================== # Create the parent dir of the destination path # It's for retro compatibility, some script consider ynh_backup creates this dir mkdir --parents $(dirname "$YNH_BACKUP_DIR/${dest_path}") } # Restore all files that were previously backuped in a core backup script or app backup script # # usage: ynh_restore # # Requires YunoHost version 2.6.4 or higher. ynh_restore() { # Deduce the relative path of $YNH_CWD local REL_DIR="${YNH_CWD#$YNH_BACKUP_DIR/}" REL_DIR="${REL_DIR%/}/" # For each destination path begining by $REL_DIR cat ${YNH_BACKUP_CSV} | tr --delete $'\r' | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR.*\"$" \ | while read line; do local ORIGIN_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\"\K.*(?=\",\".*\"$)") local ARCHIVE_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR\K.*(?=\"$)") ynh_restore_file --origin_path="$ARCHIVE_PATH" --dest_path="$ORIGIN_PATH" done } # Return the path in the archive where has been stocked the origin path # # [internal] # # usage: _get_archive_path ORIGIN_PATH _get_archive_path() { # For security reasons we use csv python library to read the CSV python3 -c " import sys import csv with open(sys.argv[1], 'r') as backup_file: backup_csv = csv.DictReader(backup_file, fieldnames=['source', 'dest']) for row in backup_csv: if row['source']==sys.argv[2].strip('\"'): print(row['dest']) sys.exit(0) raise Exception('Original path for %s not found' % sys.argv[2]) " "${YNH_BACKUP_CSV}" "$1" return $? } # Restore a file or a directory # # usage: ynh_restore_file --origin_path=origin_path [--dest_path=dest_path] [--not_mandatory] # | arg: -o, --origin_path= - Path where was located the file or the directory before to be backuped or relative path to $YNH_CWD where it is located in the backup archive # | arg: -d, --dest_path= - Path where restore the file or the dir. If unspecified, the destination will be `ORIGIN_PATH` or if the `ORIGIN_PATH` doesn't exist in the archive, the destination will be searched into `backup.csv` # | arg: -m, --not_mandatory - Indicate that if the file is missing, the restore process can ignore it. # # Use the registered path in backup_list by ynh_backup to restore the file at the right place. # # examples: # ynh_restore_file -o "/etc/nginx/conf.d/$domain.d/$app.conf" # # You can also use relative paths: # ynh_restore_file -o "conf/nginx.conf" # # If `DEST_PATH` already exists and is lighter than 500 Mo, a backup will be made in # `/var/cache/yunohost/appconfbackup/`. Otherwise, the existing file is removed. # # if `apps/$app/etc/nginx/conf.d/$domain.d/$app.conf` exists, restore it into # `/etc/nginx/conf.d/$domain.d/$app.conf` # if no, search for a match in the csv (eg: conf/nginx.conf) and restore it into # `/etc/nginx/conf.d/$domain.d/$app.conf` # # Requires YunoHost version 2.6.4 or higher. # Requires YunoHost version 3.5.0 or higher for the argument --not_mandatory ynh_restore_file() { # ============ Argument parsing ============= local -A args_array=([o]=origin_path= [d]=dest_path= [m]=not_mandatory) local origin_path local dest_path local not_mandatory ynh_handle_getopts_args "$@" origin_path="/${origin_path#/}" # Default value for dest_path = /$origin_path dest_path="${dest_path:-$origin_path}" not_mandatory="${not_mandatory:-0}" # =========================================== local archive_path="$YNH_CWD${origin_path}" # If archive_path doesn't exist, search for a corresponding path in CSV if [ ! -d "$archive_path" ] && [ ! -f "$archive_path" ] && [ ! -L "$archive_path" ]; then if [ "$not_mandatory" == "0" ]; then archive_path="$YNH_BACKUP_DIR/$(_get_archive_path \"$origin_path\")" else return 0 fi fi # Move the old directory if it already exists if [[ -e "${dest_path}" ]]; then # Check if the file/dir size is less than 500 Mo if [[ $(du --summarize --bytes ${dest_path} | cut --delimiter="/" --fields=1) -le "500000000" ]]; then local backup_file="/var/cache/yunohost/appconfbackup/${dest_path}.backup.$(date '+%Y%m%d.%H%M%S')" mkdir --parents "$(dirname "$backup_file")" mv "${dest_path}" "$backup_file" # Move the current file or directory else ynh_safe_rm --target=${dest_path} fi fi # Restore origin_path into dest_path mkdir --parents $(dirname "$dest_path") # Do a copy if it's just a mounting point if mountpoint --quiet $YNH_BACKUP_DIR; then if [[ -d "${archive_path}" ]]; then archive_path="${archive_path}/." mkdir --parents "$dest_path" fi cp --archive "$archive_path" "${dest_path}" # Do a move if YNH_BACKUP_DIR is already a copy else mv "$archive_path" "${dest_path}" fi # Boring hack for nginx conf file mapped to php7.3 # Note that there's no need to patch the fpm config because most php apps # will call "ynh_add_fpm_config" during restore, effectively recreating the file from scratch if [[ "${dest_path}" == "/etc/nginx/conf.d/"* ]] && grep 'php7.3.*sock' "${dest_path}" then sed -i 's/php7.3/php7.4/g' "${dest_path}" fi } # Calculate and store a file checksum into the app settings # # usage: ynh_store_file_checksum --file=file # | arg: -f, --file= - The file on which the checksum will performed, then stored. # # $app should be defined when calling this helper # # Requires YunoHost version 2.6.4 or higher. ynh_store_file_checksum() { # ============ Argument parsing ============= local -A args_array=([f]=file= [u]=update_only) local file local update_only update_only="${update_only:-0}" ynh_handle_getopts_args "$@" # =========================================== local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' # If update only, we don't save the new checksum if no old checksum exist if [ $update_only -eq 1 ]; then local checksum_value=$(ynh_app_setting_get --key=$checksum_setting_name) if [ -z "${checksum_value}" ]; then unset backup_file_checksum return 0 fi fi ynh_app_setting_set --key=$checksum_setting_name --value=$(md5sum "$file" | cut --delimiter=' ' --fields=1) if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then # Using a base64 is in fact more reversible than "replace / and space by _" ... So we can in fact obtain the original file path in an easy reliable way ... local file_path_base64=$(echo "$file" | base64 -w0) mkdir -p /var/cache/yunohost/appconfbackup/ cat $file > /var/cache/yunohost/appconfbackup/original_${file_path_base64} fi # If backup_file_checksum isn't empty, ynh_backup_if_checksum_is_different has made a backup if [ -n "${backup_file_checksum-}" ]; then # Print the diff between the previous file and the new one. # diff return 1 if the files are different, so the || true diff --report-identical-files --unified --color=always $backup_file_checksum $file >&2 || true fi # Unset the variable, so it wouldn't trig a ynh_store_file_checksum without a ynh_backup_if_checksum_is_different before it. unset backup_file_checksum } # Verify the checksum and backup the file if it's different # # usage: ynh_backup_if_checksum_is_different --file=file # | arg: -f, --file= - The file on which the checksum test will be perfomed. # | ret: the name of a backup file, or nothing # # This helper is primarily meant to allow to easily backup personalised/manually # modified config files. # # Requires YunoHost version 2.6.4 or higher. ynh_backup_if_checksum_is_different() { # ============ Argument parsing ============= local -A args_array=([f]=file=) local file ynh_handle_getopts_args "$@" # =========================================== local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' local checksum_value=$(ynh_app_setting_get --key=$checksum_setting_name) # backup_file_checksum isn't declare as local, so it can be reuse by ynh_store_file_checksum backup_file_checksum="" if [ -n "$checksum_value" ]; then # Proceed only if a value was stored into the app settings if [ -e $file ] && ! echo "$checksum_value $file" | md5sum --check --status; then # If the checksum is now different backup_file_checksum="/var/cache/yunohost/appconfbackup/$file.backup.$(date '+%Y%m%d.%H%M%S')" mkdir --parents "$(dirname "$backup_file_checksum")" cp --archive "$file" "$backup_file_checksum" # Backup the current file ynh_print_warn --message="File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum" echo "$backup_file_checksum" # Return the name of the backup file if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then local file_path_base64=$(echo "$file" | base64 -w0) if test -e /var/cache/yunohost/appconfbackup/original_${file_path_base64} then ynh_print_warn --message="Diff with the original file:" diff --report-identical-files --unified --color=always /var/cache/yunohost/appconfbackup/original_${file_path_base64} $file >&2 || true fi fi fi fi } # Delete a file checksum from the app settings # # usage: ynh_delete_file_checksum --file=file # | arg: -f, --file= - The file for which the checksum will be deleted # # $app should be defined when calling this helper # # Requires YunoHost version 3.3.1 or higher. ynh_delete_file_checksum() { # ============ Argument parsing ============= local -A args_array=([f]=file=) local file ynh_handle_getopts_args "$@" # =========================================== local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' ynh_app_setting_delete --key=$checksum_setting_name } # Checks a backup archive exists # # [internal] # ynh_backup_archive_exists() { yunohost backup list --output-as json --quiet \ | jq -e --arg archive "$1" '.archives | index($archive)' >/dev/null }