Rename 'filesystem' to 'backup' (backup-related stuff) and migrate some stuff to/from 'utils'

This commit is contained in:
Alexandre Aubin 2019-04-25 20:41:38 +02:00
parent 3c22feb0cb
commit 267c4119a8
2 changed files with 221 additions and 221 deletions

View file

@ -5,17 +5,17 @@ source /usr/share/yunohost/helpers.d/getopts
CAN_BIND=${CAN_BIND:-1} CAN_BIND=${CAN_BIND:-1}
# Add a file or a directory to the list of paths to backup # Add a file or a directory to the list of paths to backup
# #
# Note: this helper could be used in backup hook or in backup script inside an # Note: this helper could be used in backup hook or in backup script inside an
# app package # app package
# #
# Details: ynh_backup writes SRC and the relative DEST into a CSV file. And it # Details: ynh_backup writes SRC and the relative DEST into a CSV file. And it
# creates the parent destination directory # creates the parent destination directory
# #
# If DEST is ended by a slash it complete this path with the basename of SRC. # If DEST is ended by a slash it complete this path with the basename of SRC.
# #
# usage: ynh_backup --src_path=src_path [--dest_path=dest_path] [--is_big] [--not_mandatory] # usage: ynh_backup --src_path=src_path [--dest_path=dest_path] [--is_big] [--not_mandatory]
# | arg: -s, --src_path - file or directory to bind or symlink or copy. it shouldn't be in the backup dir. # | arg: -s, --src_path - file or directory to bind or symlink or copy. it shouldn't be in the backup dir.
# | arg: -d, --dest_path - destination file or directory inside the backup dir # | arg: -d, --dest_path - destination file or directory inside the backup dir
# | arg: -b, --is_big - Indicate data are big (mail, video, image ...) # | arg: -b, --is_big - Indicate data are big (mail, video, image ...)
# | arg: -m, --not_mandatory - Indicate that if the file is missing, the backup can ignore it. # | arg: -m, --not_mandatory - Indicate that if the file is missing, the backup can ignore it.
@ -32,7 +32,7 @@ CAN_BIND=${CAN_BIND:-1}
# #
# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf/" # ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf/"
# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/$app.conf" # # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/$app.conf"
# #
# ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf" # ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf"
# # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf" # # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf"
# #
@ -190,7 +190,7 @@ with open(sys.argv[1], 'r') as backup_file:
return $? return $?
} }
# Restore a file or a directory # Restore a file or a directory
# #
# Use the registered path in backup_list by ynh_backup to restore the file at # Use the registered path in backup_list by ynh_backup to restore the file at
# the right place. # the right place.
@ -205,7 +205,7 @@ with open(sys.argv[1], 'r') as backup_file:
# # You can also use relative paths: # # You can also use relative paths:
# ynh_restore_file "conf/nginx.conf" # ynh_restore_file "conf/nginx.conf"
# #
# If DEST_PATH already exists and is lighter than 500 Mo, a backup will be made in # If DEST_PATH already exists and is lighter than 500 Mo, a backup will be made in
# /home/yunohost.conf/backup/. Otherwise, the existing file is removed. # /home/yunohost.conf/backup/. Otherwise, the existing file is removed.
# #
# if apps/wordpress/etc/nginx/conf.d/$domain.d/$app.conf exists, restore it into # if apps/wordpress/etc/nginx/conf.d/$domain.d/$app.conf exists, restore it into
@ -248,7 +248,7 @@ ynh_restore_file () {
then then
local backup_file="/home/yunohost.conf/backup/${dest_path}.backup.$(date '+%Y%m%d.%H%M%S')" local backup_file="/home/yunohost.conf/backup/${dest_path}.backup.$(date '+%Y%m%d.%H%M%S')"
mkdir -p "$(dirname "$backup_file")" mkdir -p "$(dirname "$backup_file")"
mv "${dest_path}" "$backup_file" # Move the current file or directory mv "${dest_path}" "$backup_file" # Move the current file or directory
else else
ynh_secure_remove --file=${dest_path} ynh_secure_remove --file=${dest_path}
fi fi
@ -282,26 +282,6 @@ ynh_bind_or_cp() {
ynh_backup "$1" "$2" 1 ynh_backup "$1" "$2" 1
} }
# Create a directory under /tmp
#
# [internal]
#
# Deprecated helper
#
# usage: ynh_mkdir_tmp
# | ret: the created directory path
ynh_mkdir_tmp() {
ynh_print_warn --message="The helper ynh_mkdir_tmp is deprecated."
ynh_print_warn --message="You should use 'mktemp -d' instead and manage permissions \
properly with chmod/chown."
local TMP_DIR=$(mktemp -d)
# Give rights to other users could be a security risk.
# But for retrocompatibility we need it. (This helpers is deprecated)
chmod 755 $TMP_DIR
echo $TMP_DIR
}
# Calculate and store a file checksum into the app settings # Calculate and store a file checksum into the app settings
# #
# $app should be defined when calling this helper # $app should be defined when calling this helper
@ -311,29 +291,29 @@ properly with chmod/chown."
# #
# Requires YunoHost version 2.6.4 or higher. # Requires YunoHost version 2.6.4 or higher.
ynh_store_file_checksum () { ynh_store_file_checksum () {
# Declare an array to define the options of this helper. # Declare an array to define the options of this helper.
local legacy_args=f local legacy_args=f
declare -Ar args_array=( [f]=file= ) declare -Ar args_array=( [f]=file= )
local file local file
# Manage arguments with getopts # Manage arguments with getopts
ynh_handle_getopts_args "$@" ynh_handle_getopts_args "$@"
local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(sudo md5sum "$file" | cut -d' ' -f1) ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(sudo md5sum "$file" | cut -d' ' -f1)
# If backup_file_checksum isn't empty, ynh_backup_if_checksum_is_different has made a backup # If backup_file_checksum isn't empty, ynh_backup_if_checksum_is_different has made a backup
if [ -n "${backup_file_checksum-}" ] if [ -n "${backup_file_checksum-}" ]
then then
# Print the diff between the previous file and the new one. # Print the diff between the previous file and the new one.
# diff return 1 if the files are different, so the || true # diff return 1 if the files are different, so the || true
diff --report-identical-files --unified --color=always $backup_file_checksum $file >&2 || true diff --report-identical-files --unified --color=always $backup_file_checksum $file >&2 || true
fi fi
# Unset the variable, so it wouldn't trig a ynh_store_file_checksum without a ynh_backup_if_checksum_is_different before it. # Unset the variable, so it wouldn't trig a ynh_store_file_checksum without a ynh_backup_if_checksum_is_different before it.
unset backup_file_checksum unset backup_file_checksum
} }
# Verify the checksum and backup the file if it's different # Verify the checksum and backup the file if it's different
# This helper is primarily meant to allow to easily backup personalised/manually # This helper is primarily meant to allow to easily backup personalised/manually
# modified config files. # modified config files.
# #
# $app should be defined when calling this helper # $app should be defined when calling this helper
@ -344,28 +324,28 @@ ynh_store_file_checksum () {
# #
# Requires YunoHost version 2.6.4 or higher. # Requires YunoHost version 2.6.4 or higher.
ynh_backup_if_checksum_is_different () { ynh_backup_if_checksum_is_different () {
# Declare an array to define the options of this helper. # Declare an array to define the options of this helper.
local legacy_args=f local legacy_args=f
declare -Ar args_array=( [f]=file= ) declare -Ar args_array=( [f]=file= )
local file local file
# Manage arguments with getopts # Manage arguments with getopts
ynh_handle_getopts_args "$@" ynh_handle_getopts_args "$@"
local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name) local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name)
# backup_file_checksum isn't declare as local, so it can be reuse by ynh_store_file_checksum # backup_file_checksum isn't declare as local, so it can be reuse by ynh_store_file_checksum
backup_file_checksum="" backup_file_checksum=""
if [ -n "$checksum_value" ] if [ -n "$checksum_value" ]
then # Proceed only if a value was stored into the app settings then # Proceed only if a value was stored into the app settings
if ! echo "$checksum_value $file" | sudo md5sum -c --status if ! echo "$checksum_value $file" | sudo md5sum -c --status
then # If the checksum is now different then # If the checksum is now different
backup_file_checksum="/home/yunohost.conf/backup/$file.backup.$(date '+%Y%m%d.%H%M%S')" backup_file_checksum="/home/yunohost.conf/backup/$file.backup.$(date '+%Y%m%d.%H%M%S')"
sudo mkdir -p "$(dirname "$backup_file_checksum")" sudo mkdir -p "$(dirname "$backup_file_checksum")"
sudo cp -a "$file" "$backup_file_checksum" # Backup the current file sudo cp -a "$file" "$backup_file_checksum" # Backup the current file
ynh_print_warn "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum" ynh_print_warn "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum"
echo "$backup_file_checksum" # Return the name of the backup file echo "$backup_file_checksum" # Return the name of the backup file
fi fi
fi fi
} }
# Delete a file checksum from the app settings # Delete a file checksum from the app settings
@ -377,54 +357,94 @@ ynh_backup_if_checksum_is_different () {
# #
# Requires YunoHost version 3.3.1 or higher. # Requires YunoHost version 3.3.1 or higher.
ynh_delete_file_checksum () { ynh_delete_file_checksum () {
# Declare an array to define the options of this helper. # Declare an array to define the options of this helper.
local legacy_args=f local legacy_args=f
declare -Ar args_array=( [f]=file= ) declare -Ar args_array=( [f]=file= )
local file local file
# Manage arguments with getopts # Manage arguments with getopts
ynh_handle_getopts_args "$@" ynh_handle_getopts_args "$@"
local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
ynh_app_setting_delete --app=$app --key=$checksum_setting_name ynh_app_setting_delete --app=$app --key=$checksum_setting_name
} }
# Remove a file or a directory securely # Make a backup in case of failed upgrade
# #
# usage: ynh_secure_remove --file=path_to_remove # usage:
# | arg: -f, --file - File or directory to remove # ynh_backup_before_upgrade
# ynh_clean_setup () {
# ynh_restore_upgradebackup
# }
# ynh_abort_if_errors
# #
# Requires YunoHost version 2.6.4 or higher. # Requires YunoHost version 2.7.2 or higher.
ynh_secure_remove () { ynh_backup_before_upgrade () {
# Declare an array to define the options of this helper. if [ ! -e "/etc/yunohost/apps/$app/scripts/backup" ]
local legacy_args=f then
declare -Ar args_array=( [f]=file= ) ynh_print_warn --message="This app doesn't have any backup script."
local file return
# Manage arguments with getopts fi
ynh_handle_getopts_args "$@" backup_number=1
local old_backup_number=2
local app_bck=${app//_/-} # Replace all '_' by '-'
NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0}
local forbidden_path=" \ if [ "$NO_BACKUP_UPGRADE" -eq 0 ]
/var/www \ then
/home/yunohost.app" # Check if a backup already exists with the prefix 1
if sudo yunohost backup list | grep -q $app_bck-pre-upgrade1
then
# Prefix becomes 2 to preserve the previous backup
backup_number=2
old_backup_number=1
fi
if [ $# -ge 2 ] # Create backup
then sudo BACKUP_CORE_ONLY=1 yunohost backup create --apps $app --name $app_bck-pre-upgrade$backup_number --debug
ynh_print_warn --message="/!\ Packager ! You provided more than one argument to ynh_secure_remove but it will be ignored... Use this helper with one argument at time." if [ "$?" -eq 0 ]
fi then
# If the backup succeeded, remove the previous backup
if [[ "$forbidden_path" =~ "$file" \ if sudo yunohost backup list | grep -q $app_bck-pre-upgrade$old_backup_number
# Match all paths or subpaths in $forbidden_path then
|| "$file" =~ ^/[[:alnum:]]+$ \ # Remove the previous backup only if it exists
# Match all first level paths from / (Like /var, /root, etc...) sudo yunohost backup delete $app_bck-pre-upgrade$old_backup_number > /dev/null
|| "${file:${#file}-1}" = "/" ]] fi
# Match if the path finishes by /. Because it seems there is an empty variable else
then ynh_die --message="Backup failed, the upgrade process was aborted."
ynh_print_warn --message="Avoid deleting $file." fi
else else
if [ -e "$file" ] ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, backup will be avoided. Be careful, this upgrade is going to be operated without a security backup"
then fi
sudo rm -R "$file" }
else
ynh_print_info --message="$file wasn't deleted because it doesn't exist." # Restore a previous backup if the upgrade process failed
fi #
fi # usage:
# ynh_backup_before_upgrade
# ynh_clean_setup () {
# ynh_restore_upgradebackup
# }
# ynh_abort_if_errors
#
# Requires YunoHost version 2.7.2 or higher.
ynh_restore_upgradebackup () {
ynh_print_err --message="Upgrade failed."
local app_bck=${app//_/-} # Replace all '_' by '-'
NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0}
if [ "$NO_BACKUP_UPGRADE" -eq 0 ]
then
# Check if an existing backup can be found before removing and restoring the application.
if sudo yunohost backup list | grep -q $app_bck-pre-upgrade$backup_number
then
# Remove the application then restore it
sudo yunohost app remove $app
# Restore the backup
sudo yunohost backup restore $app_bck-pre-upgrade$backup_number --apps $app --force --debug
ynh_die --message="The app was restored to the way it was before the failed upgrade."
fi
else
ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, that means there's no backup to restore. You have to fix this upgrade by yourself !"
fi
} }

View file

@ -29,87 +29,6 @@ ynh_get_plain_key() {
done done
} }
# Restore a previous backup if the upgrade process failed
#
# usage:
# ynh_backup_before_upgrade
# ynh_clean_setup () {
# ynh_restore_upgradebackup
# }
# ynh_abort_if_errors
#
# Requires YunoHost version 2.7.2 or higher.
ynh_restore_upgradebackup () {
ynh_print_err --message="Upgrade failed."
local app_bck=${app//_/-} # Replace all '_' by '-'
NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0}
if [ "$NO_BACKUP_UPGRADE" -eq 0 ]
then
# Check if an existing backup can be found before removing and restoring the application.
if sudo yunohost backup list | grep -q $app_bck-pre-upgrade$backup_number
then
# Remove the application then restore it
sudo yunohost app remove $app
# Restore the backup
sudo yunohost backup restore $app_bck-pre-upgrade$backup_number --apps $app --force --debug
ynh_die --message="The app was restored to the way it was before the failed upgrade."
fi
else
ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, that means there's no backup to restore. You have to fix this upgrade by yourself !"
fi
}
# Make a backup in case of failed upgrade
#
# usage:
# ynh_backup_before_upgrade
# ynh_clean_setup () {
# ynh_restore_upgradebackup
# }
# ynh_abort_if_errors
#
# Requires YunoHost version 2.7.2 or higher.
ynh_backup_before_upgrade () {
if [ ! -e "/etc/yunohost/apps/$app/scripts/backup" ]
then
ynh_print_warn --message="This app doesn't have any backup script."
return
fi
backup_number=1
local old_backup_number=2
local app_bck=${app//_/-} # Replace all '_' by '-'
NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0}
if [ "$NO_BACKUP_UPGRADE" -eq 0 ]
then
# Check if a backup already exists with the prefix 1
if sudo yunohost backup list | grep -q $app_bck-pre-upgrade1
then
# Prefix becomes 2 to preserve the previous backup
backup_number=2
old_backup_number=1
fi
# Create backup
sudo BACKUP_CORE_ONLY=1 yunohost backup create --apps $app --name $app_bck-pre-upgrade$backup_number --debug
if [ "$?" -eq 0 ]
then
# If the backup succeeded, remove the previous backup
if sudo yunohost backup list | grep -q $app_bck-pre-upgrade$old_backup_number
then
# Remove the previous backup only if it exists
sudo yunohost backup delete $app_bck-pre-upgrade$old_backup_number > /dev/null
fi
else
ynh_die --message="Backup failed, the upgrade process was aborted."
fi
else
ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, backup will be avoided. Be careful, this upgrade is going to be operated without a security backup"
fi
}
# Download, check integrity, uncompress and patch the source from app.src # Download, check integrity, uncompress and patch the source from app.src
# #
# The file conf/app.src need to contains: # The file conf/app.src need to contains:
@ -129,7 +48,7 @@ ynh_backup_before_upgrade () {
# SOURCE_IN_SUBDIR=false # SOURCE_IN_SUBDIR=false
# # (Optionnal) Name of the local archive (offline setup support) # # (Optionnal) Name of the local archive (offline setup support)
# # default: ${src_id}.${src_format} # # default: ${src_id}.${src_format}
# SOURCE_FILENAME=example.tar.gz # SOURCE_FILENAME=example.tar.gz
# # (Optional) If it set as false don't extract the source. # # (Optional) If it set as false don't extract the source.
# # (Useful to get a debian package or a python wheel.) # # (Useful to get a debian package or a python wheel.)
# # default: true # # default: true
@ -212,7 +131,7 @@ ynh_setup_source () {
then then
mv $src_filename $dest_dir mv $src_filename $dest_dir
elif [ "$src_format" = "zip" ] elif [ "$src_format" = "zip" ]
then then
# Zip format # Zip format
# Using of a temp directory, because unzip doesn't manage --strip-components # Using of a temp directory, because unzip doesn't manage --strip-components
if $src_in_subdir ; then if $src_in_subdir ; then
@ -262,7 +181,7 @@ ynh_setup_source () {
# $domain and $path_url should be defined externally (and correspond to the domain.tld and the /path (of the app?)) # $domain and $path_url should be defined externally (and correspond to the domain.tld and the /path (of the app?))
# #
# example: ynh_local_curl "/install.php?installButton" "foo=$var1" "bar=$var2" # example: ynh_local_curl "/install.php?installButton" "foo=$var1" "bar=$var2"
# #
# usage: ynh_local_curl "page_uri" "key1=value1" "key2=value2" ... # usage: ynh_local_curl "page_uri" "key1=value1" "key2=value2" ...
# | arg: page_uri - Path (relative to $path_url) of the page where POST data will be sent # | arg: page_uri - Path (relative to $path_url) of the page where POST data will be sent
# | arg: key1=value1 - (Optionnal) POST key and corresponding value # | arg: key1=value1 - (Optionnal) POST key and corresponding value
@ -271,38 +190,38 @@ ynh_setup_source () {
# #
# Requires YunoHost version 2.6.4 or higher. # Requires YunoHost version 2.6.4 or higher.
ynh_local_curl () { ynh_local_curl () {
# Define url of page to curl # Define url of page to curl
local local_page=$(ynh_normalize_url_path $1) local local_page=$(ynh_normalize_url_path $1)
local full_path=$path_url$local_page local full_path=$path_url$local_page
if [ "${path_url}" == "/" ]; then
full_path=$local_page
fi
local full_page_url=https://localhost$full_path
# Concatenate all other arguments with '&' to prepare POST data if [ "${path_url}" == "/" ]; then
local POST_data="" full_path=$local_page
local arg="" fi
for arg in "${@:2}"
do
POST_data="${POST_data}${arg}&"
done
if [ -n "$POST_data" ]
then
# Add --data arg and remove the last character, which is an unecessary '&'
POST_data="--data ${POST_data::-1}"
fi
# Wait untils nginx has fully reloaded (avoid curl fail with http2)
sleep 2
# Curl the URL local full_page_url=https://localhost$full_path
curl --silent --show-error -kL -H "Host: $domain" --resolve $domain:443:127.0.0.1 $POST_data "$full_page_url"
# Concatenate all other arguments with '&' to prepare POST data
local POST_data=""
local arg=""
for arg in "${@:2}"
do
POST_data="${POST_data}${arg}&"
done
if [ -n "$POST_data" ]
then
# Add --data arg and remove the last character, which is an unecessary '&'
POST_data="--data ${POST_data::-1}"
fi
# Wait untils nginx has fully reloaded (avoid curl fail with http2)
sleep 2
# Curl the URL
curl --silent --show-error -kL -H "Host: $domain" --resolve $domain:443:127.0.0.1 $POST_data "$full_page_url"
} }
# Render templates with Jinja2 # Render templates with Jinja2
# #
# Attention : Variables should be exported before calling this helper to be # Attention : Variables should be exported before calling this helper to be
# accessible inside templates. # accessible inside templates.
# #
@ -318,3 +237,64 @@ ynh_render_template() {
jinja2.Template(sys.stdin.read() jinja2.Template(sys.stdin.read()
).render(os.environ));' < $template_path > $output_path ).render(os.environ));' < $template_path > $output_path
} }
# Create a directory under /tmp
#
# [internal]
#
# Deprecated helper
#
# usage: ynh_mkdir_tmp
# | ret: the created directory path
ynh_mkdir_tmp() {
ynh_print_warn --message="The helper ynh_mkdir_tmp is deprecated."
ynh_print_warn --message="You should use 'mktemp -d' instead and manage permissions \
properly with chmod/chown."
local TMP_DIR=$(mktemp -d)
# Give rights to other users could be a security risk.
# But for retrocompatibility we need it. (This helpers is deprecated)
chmod 755 $TMP_DIR
echo $TMP_DIR
}
# Remove a file or a directory securely
#
# usage: ynh_secure_remove --file=path_to_remove
# | arg: -f, --file - File or directory to remove
#
# Requires YunoHost version 2.6.4 or higher.
ynh_secure_remove () {
# Declare an array to define the options of this helper.
local legacy_args=f
declare -Ar args_array=( [f]=file= )
local file
# Manage arguments with getopts
ynh_handle_getopts_args "$@"
local forbidden_path=" \
/var/www \
/home/yunohost.app"
if [ $# -ge 2 ]
then
ynh_print_warn --message="/!\ Packager ! You provided more than one argument to ynh_secure_remove but it will be ignored... Use this helper with one argument at time."
fi
if [[ "$forbidden_path" =~ "$file" \
# Match all paths or subpaths in $forbidden_path
|| "$file" =~ ^/[[:alnum:]]+$ \
# Match all first level paths from / (Like /var, /root, etc...)
|| "${file:${#file}-1}" = "/" ]]
# Match if the path finishes by /. Because it seems there is an empty variable
then
ynh_print_warn --message="Avoid deleting $file."
else
if [ -e "$file" ]
then
sudo rm -R "$file"
else
ynh_print_info --message="$file wasn't deleted because it doesn't exist."
fi
fi
}