From ade2378dffdf8dd7afb3b862976ad2a510c70ecd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Mon, 12 Aug 2024 13:03:41 +0200 Subject: [PATCH] Refactoring of borgserver, like borg_ynh, stop using /opt/borg-env and /usr/local/bin --- conf/monitor-backup | 2 -- conf/monitor-backup.cron | 2 ++ conf/monitor-backup.sh | 12 ++++++++ manifest.toml | 16 +++++++---- scripts/_common.sh | 60 +++++++++++++++++++++++++++------------- scripts/backup | 28 +++++++++++++++++-- scripts/install | 31 ++++++++------------- scripts/remove | 13 +-------- scripts/restore | 33 +++++++++------------- scripts/upgrade | 58 +++++++++++++++----------------------- tests.toml | 2 ++ 11 files changed, 142 insertions(+), 115 deletions(-) delete mode 100644 conf/monitor-backup create mode 100644 conf/monitor-backup.cron create mode 100644 conf/monitor-backup.sh diff --git a/conf/monitor-backup b/conf/monitor-backup deleted file mode 100644 index 7ecc10b..0000000 --- a/conf/monitor-backup +++ /dev/null @@ -1,2 +0,0 @@ -SHELL=/bin/bash -0 9,20 * * * root : Monitor __SSH_USER__ backup ; ALERT_DELAY="$(grep '^alert_delay: ' /etc/yunohost/apps/__APP__/settings.yml | awk -F\' '{print $2}')"; [[ $(find /home/__SSH_USER__/backup/data -follow -mtime -${ALERT_DELAY} -ls | wc -l) > 0 ]] || ( echo "No file has been backuped in /home/__SSH_USER__ since ${ALERT_DELAY} days" | mail -s "[YNH] Backup missing : __SSH_USER__" $(grep '^alert_mails: ' /etc/yunohost/apps/__APP__/settings.yml | awk '{print $2}')) diff --git a/conf/monitor-backup.cron b/conf/monitor-backup.cron new file mode 100644 index 0000000..e5fd379 --- /dev/null +++ b/conf/monitor-backup.cron @@ -0,0 +1,2 @@ +# Monitor __SSH_USER__ backup every day +0 9,20 * * * root __INSTALL_DIR__/monitor-backup.sh diff --git a/conf/monitor-backup.sh b/conf/monitor-backup.sh new file mode 100644 index 0000000..2618aa0 --- /dev/null +++ b/conf/monitor-backup.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +ALERT_DELAY="$(grep '^alert_delay: ' /etc/yunohost/apps/__APP__/settings.yml | awk -F\' '{print $2}')" + + + +if [[ $(find /home/__SSH_USER__/backup/data -follow -mtime -${ALERT_DELAY} -ls | wc -l) > 0 ]]; then + : +else + echo "No file has been backuped in /home/__SSH_USER__ since ${ALERT_DELAY} days" \ + | mail -s "[YNH] Backup missing : __SSH_USER__" $(grep '^alert_mails: ' /etc/yunohost/apps/__APP__/settings.yml | awk '{print $2}') +fi diff --git a/manifest.toml b/manifest.toml index bef8d3d..2f43d83 100644 --- a/manifest.toml +++ b/manifest.toml @@ -1,3 +1,5 @@ +#:schema https://raw.githubusercontent.com/YunoHost/apps/master/schemas/manifest.v2.schema.json + packaging_format = 2 id = "borgserver" @@ -5,7 +7,7 @@ name = "Borg Server" description.en = "Offer backup storage to a friend" description.fr = "Offrez un espace de stockage à un⋅e ami⋅e" -version = "1.2.8~ynh1" +version = "1.2.8~ynh2" maintainers = ["ljf"] @@ -67,19 +69,23 @@ ram.runtime = "50M" autoupdate.strategy = "latest_github_release" autoupdate.upstream = "https://github.com/borgbackup/borg" + [resources.system_user] + # Not really useful, but required for install_dir + + [resources.install_dir] + [resources.permissions] [resources.apt] packages = [ "python3-pip", "python3-dev", - "libacl1-dev", - "libssl-dev", - "liblz4-dev", "python3-jinja2", "python3-setuptools", "python3-venv", - "virtualenv", + "libacl1-dev", + "libssl-dev", + "liblz4-dev", "libfuse3-dev", "pkg-config", ] diff --git a/scripts/_common.sh b/scripts/_common.sh index 7151311..bf347f7 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -1,23 +1,45 @@ #!/bin/bash -BORG_VERSION=1.2.8 +#================================================= +# COMMON VARIABLES +#================================================= -# Install borg with pip if borg is not here -install_borg_with_pip () { - if [ -d /opt/borg-env ]; then - /opt/borg-env/bin/python /opt/borg-env/bin/pip list | grep "borgbackup *$BORG_VERSION" || ynh_secure_remove /opt/borg-env - fi - if [ ! -d /opt/borg-env ]; then - python3 -m venv /opt/borg-env - /opt/borg-env/bin/python /opt/borg-env/bin/pip install pip -U - /opt/borg-env/bin/python /opt/borg-env/bin/pip install setuptools -U - /opt/borg-env/bin/python /opt/borg-env/bin/pip install wheel -U - ynh_print_info --message="Installing/compiling borg, this may take some time..." - /opt/borg-env/bin/python /opt/borg-env/bin/pip install borgbackup[pyfuse3]==$BORG_VERSION - echo "#!/bin/bash - /opt/borg-env/bin/python /opt/borg-env/bin/borg \"\$@\"" > /usr/local/bin/borg - touch "/opt/borg-env/$(ynh_get_debian_release)" - fi - # We need this to be executable by other borg apps - chmod a+x /usr/local/bin/borg +#================================================= +# PERSONAL HELPERS +#================================================= + +install_borg_with_pip() { + ynh_exec_as "$app" python3 -m venv --upgrade "$install_dir/venv" + venvpy="$install_dir/venv/bin/python3" + + ynh_exec_as "$app" "$venvpy" -m pip install --upgrade setuptools wheel + + BORG_VERSION=$(ynh_app_upstream_version) + ynh_exec_as "$app" "$venvpy" -m pip install borgbackup[pyfuse3]=="$BORG_VERSION" } + +create_ssh_config() { + ssh_dir=$1 + repository=$2 + extra="" + if [[ -n "$quota" ]]; then + extra="--storage-quota $quota" + fi + command="borg serve $extra --restrict-to-repository $repository" + ssh_opts="command=\"$command\",no-pty,no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-user-rc" + + mkdir -p "$ssh_dir" + touch "$ssh_dir/authorized_keys" + echo "$ssh_opts $public_key" >> "$ssh_dir/authorized_keys" + + chown -R "$ssh_user:$ssh_user" "$ssh_dir" + chmod -R u=rwX,go=--- "$ssh_dir" +} + +#================================================= +# EXPERIMENTAL HELPERS +#================================================= + +#================================================= +# FUTURE OFFICIAL HELPERS +#================================================= diff --git a/scripts/backup b/scripts/backup index 5a503fa..50c4e36 100755 --- a/scripts/backup +++ b/scripts/backup @@ -1,17 +1,39 @@ #!/bin/bash +#================================================= +# GENERIC START +#================================================= +# IMPORT GENERIC HELPERS +#================================================= + source /usr/share/yunohost/helpers #================================================= # DECLARE DATA AND CONF FILES TO BACKUP #================================================= ynh_print_info --message="Declaring files to be backed up..." -ynh_backup "/home/$ssh_user/.ssh" -ynh_backup "/home/$ssh_user/.nobackup" -ynh_backup "/etc/cron.d/$app" + +#================================================= +# BACKUP THE APP MAIN DIR +#================================================= + +ynh_backup --src_path="$install_dir" + +#================================================= +# BACKUP THE DATA DIR +#================================================= + +ynh_backup --src_path="/home/$ssh_user/.ssh" +ynh_backup --src_path="/home/$ssh_user/.nobackup" ynh_print_info --message="Borg backup repo in /home/$ssh_user/ won't be backup to avoid backup of backup loop issue." +#================================================= +# BACKUP VARIOUS FILES +#================================================= + +ynh_backup --src_path="/etc/cron.d/$app" + #================================================= # END OF SCRIPT #================================================= diff --git a/scripts/install b/scripts/install index 4faa57f..9343dd5 100755 --- a/scripts/install +++ b/scripts/install @@ -12,48 +12,41 @@ source /usr/share/yunohost/helpers #================================================= # CHECK IF THE APP CAN BE INSTALLED WITH THIS ARGS #================================================= -# Here is a small hack to avoid multi install CI test to fail due +# Here is a small hack to avoid multi install CI test to fail due # to same ssh_user provided if [[ "${PACKAGE_CHECK_EXEC:-}" = "1" ]] && [[ "$YNH_APP_INSTANCE_NUMBER" != "1" ]] ; then ssh_user+="$YNH_APP_INSTANCE_NUMBER" fi -ynh_system_user_exists --username=$ssh_user && ynh_die --message="This user already exists" +ynh_system_user_exists --username="$ssh_user" && ynh_die --message="This user already exists" #================================================= -# INSTALL DEPENDENCIES +# INSTALL BORG #================================================= -ynh_script_progression --message="Installing dependencies..." +ynh_script_progression --message="Installing Borg..." install_borg_with_pip #================================================= # AUTORIZE SSH FOR THIS USER #================================================= -ynh_script_progression --message="Configuring SSH public key for remote connexion..." +ynh_script_progression --message="Configuring user and SSH public key for remote connexion..." -ynh_system_user_create --username=$ssh_user --home_dir=/home/$ssh_user --use_shell --groups ssh.app - -home=/home/$ssh_user -mkdir -p $home/.ssh -chmod o=--- $home -chown -R $ssh_user:$ssh_user $home -touch $home/.ssh/authorized_keys -extra="--storage-quota $quota" -if [ "$quota" = "" ]; then - extra="" -fi -echo "command=\"borg serve $extra --restrict-to-repository $home/backup\",no-pty,no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-user-rc $public_key" >> $home/.ssh/authorized_keys +ynh_system_user_create --username="$ssh_user" --home_dir="/home/$ssh_user" --use_shell --groups ssh.app # Tweak to prevent the backup of the backup itself -touch $home/.nobackup +touch "/home/$ssh_user/.nobackup" + +create_ssh_config "/home/$ssh_user/.ssh" "/home/$ssh_user/backup" #================================================= # SETUP CRON #================================================= ynh_script_progression --message="Configuring cron to monitor backup..." -ynh_add_config --template="monitor-backup" --destination="/etc/cron.d/$app" +ynh_add_config --template="monitor-backup.cron" --destination="/etc/cron.d/$app" +ynh_add_config --template="monitor-backup.sh" --destination="$install_dir/monitor-backup.sh" +chmod +x "$install_dir/monitor-backup.sh" #================================================= # END OF SCRIPT diff --git a/scripts/remove b/scripts/remove index d907141..ed76711 100755 --- a/scripts/remove +++ b/scripts/remove @@ -9,24 +9,13 @@ source _common.sh source /usr/share/yunohost/helpers -#================================================= -# REMOVE DEPENDENCIES -#================================================= -ynh_script_progression --message="Removing dependencies..." - -# Remove borg if we are removing the last borg app on the system -if [ "$(yunohost app list | grep "id: borg" | wc -l)" == "1" ] ; then - ynh_secure_remove "/opt/borg-env" - ynh_secure_remove "/usr/local/bin/borg" -fi - #================================================= # REMOVE USER BUT KEEP FILES #================================================= # We keep files cause we don't know what the user want to do about # backups stored in the home directory -ynh_system_user_delete --username=$ssh_user +ynh_system_user_delete --username="$ssh_user" #================================================= # REMOVE CRON FILES diff --git a/scripts/restore b/scripts/restore index c176e72..a352d10 100755 --- a/scripts/restore +++ b/scripts/restore @@ -10,35 +10,28 @@ source ../settings/scripts/_common.sh source /usr/share/yunohost/helpers #================================================= -# INSTALL DEPENDENCIES +# RESTORE THE APP MAIN DIR #================================================= -ynh_script_progression --message="Reinstalling borg env..." +ynh_script_progression --message="Restoring the app main directory..." --weight=1 -install_borg_with_pip +ynh_restore_file --origin_path="$install_dir" #================================================= -# CREATE SSH USER USED BY BORG +# RESTORE THE DATA DIRECTORY #================================================= -ynh_script_progression --message="Making sure SSH user exists with appropriate permissions..." +ynh_script_progression --message="Restoring the user and SSH configuration..." --weight=1 -ynh_system_user_create --username=$ssh_user --home_dir=/home/$ssh_user --use_shell --groups ssh.app +ynh_system_user_create --username="$ssh_user" --home_dir="/home/$ssh_user" --use_shell --groups ssh.app -home=/home/$ssh_user -mkdir -p $home/.ssh -chmod o=--- $home -extra="--storage-quota $quota" -if [ "$quota" = "" ]; then - extra="" -fi -echo "command=\"borg serve $extra --restrict-to-repository $home/backup\",no-pty,no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-user-rc $public_key" >> $home/.ssh/authorized_keys -chown -R $ssh_user:$ssh_user $home +ynh_restore_file --origin_path="/home/$ssh_user/.ssh" +ynh_restore_file --origin_path="/home/$ssh_user/.nobackup" -# Tweak to prevent the backup of the backup itself -touch $home/.nobackup +#================================================= +# RESTORE SYSTEM CONFIGURATIONS +#================================================= +ynh_script_progression --message="Restoring system configurations related to $app..." --weight=1 - -# Actual restore of ...? -ynh_restore +ynh_restore_file --origin_path="/etc/cron.d/$app" #================================================= # END OF SCRIPT diff --git a/scripts/upgrade b/scripts/upgrade index 4546a08..caed06b 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -9,24 +9,25 @@ source _common.sh source /usr/share/yunohost/helpers -#================================================= -# CHECK IF AN UPGRADE IS NEEDED -#================================================= - -ynh_check_app_version_changed - #================================================= # ENSURE DOWNWARD COMPATIBILITY #================================================= ynh_script_progression --message="Ensuring downward compatibility..." +# Remove legacy stuff if [ -f "/etc/apt/sources.list.d/$app-stretch-backports.list" ]; then - rm -f /etc/apt/sources.list.d/$app-stretch-backports.list - install_borg_with_pip + ynh_secure_remove "/etc/apt/sources.list.d/$app-stretch-backports.list" fi if [ -f "/etc/yunohost/hooks.d/backup/17-data_home" ]; then - ynh_secure_remove /etc/yunohost/hooks.d/backup/17-data_home + ynh_secure_remove "/etc/yunohost/hooks.d/backup/17-data_home" +fi + +# Clear legacy stuff +if [ -d /opt/borg-env ]; then + ynh_secure_remove --file="/opt/borg-env" + ynh_secure_remove --file="/usr/local/bin/borg" + ynh_secure_remove --file="/usr/local/bin/backup-with-borg" fi # Fix broken value ssh_user that mistakenly got replaced by the public key in previous versions... @@ -34,30 +35,23 @@ fi if echo "$ssh_user" | grep -q ' '; then ssh_user=$(grep "$ssh_user" /home/*/.ssh/authorized_keys | grep borg | cut -d/ -f3) [ -n "$ssh_user" ] || ynh_die "Unable to retrieve ssh_user please fix /etc/yunohost/apps/$app/settings.yml manually :( !" - ynh_app_setting_set --app=$app --key=ssh_user --value="$ssh_user" + ynh_app_setting_set --app="$app" --key=ssh_user --value="$ssh_user" fi if echo "$public_key" | grep -q -v ' '; then - ynh_app_setting_set --app=$app --key=public_key --value="$(grep -Po 'no-user-rc \K.*$' /home/$ssh_user/.ssh/authorized_keys)" + ynh_app_setting_set --app="$app" --key=public_key --value="$(grep -Po 'no-user-rc \K.*$' "/home/$ssh_user/.ssh/authorized_keys")" fi # Alert delay and alert mail missing if [ -z "${alert_delay:-}" ]; then - ynh_app_setting_set --app=$app --key=alert_delay --value=1 - ynh_app_setting_set --app=$app --key=alert_mails --value="root" -fi - -# Reinstall borg if debian change of major version -if [ ! -f "/opt/borg-env/$(ynh_get_debian_release)" ] ; then - ynh_secure_remove /opt/borg-env + ynh_app_setting_set --app="$app" --key=alert_delay --value="1" + ynh_app_setting_set --app="$app" --key=alert_mails --value="root" fi #================================================= -# SPECIFIC UPGRADE +# UPGRADE BORG #================================================= -# Upgrade borgbackup -#================================================= -ynh_script_progression --message="Upgrading borgbackup..." --weight=1 +ynh_script_progression --message="Upgrading Borg..." --weight=1 install_borg_with_pip @@ -66,27 +60,21 @@ install_borg_with_pip #================================================= ynh_script_progression --message="Making sure SSH user exists with appropriate permissions..." -ynh_system_user_create --username=$ssh_user --home_dir=/home/$ssh_user --use_shell --groups ssh.app - -home=/home/$ssh_user -mkdir -p $home/.ssh -chmod o=--- $home -extra="--storage-quota $quota" -if [ "$quota" = "" ]; then - extra="" -fi -echo "command=\"borg serve $extra --restrict-to-repository $home/backup\",no-pty,no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-user-rc $public_key" >> $home/.ssh/authorized_keys -chown -R $ssh_user:$ssh_user $home +ynh_system_user_create --username="$ssh_user" --home_dir="/home/$ssh_user" --use_shell --groups ssh.app # Tweak to prevent the backup of the backup itself -touch $home/.nobackup +touch "/home/$ssh_user/.nobackup" + +create_ssh_config "/home/$ssh_user/.ssh" "/home/$ssh_user/backup" #================================================= # SETUP CRON #================================================= ynh_script_progression --message="Configuring cron to monitor backup..." -ynh_add_config --template="monitor-backup" --destination="/etc/cron.d/$app" +ynh_add_config --template="monitor-backup.cron" --destination="/etc/cron.d/$app" +ynh_add_config --template="monitor-backup.sh" --destination="$install_dir/monitor-backup.sh" +chmod +x "$install_dir/monitor-backup.sh" #================================================= # END OF SCRIPT diff --git a/tests.toml b/tests.toml index faaf6c7..08e4328 100644 --- a/tests.toml +++ b/tests.toml @@ -1,3 +1,5 @@ +#:schema https://raw.githubusercontent.com/YunoHost/apps/master/schemas/tests.v1.schema.json + test_format = 1.0 [default]