From b53882a489f9037fba63c8c3eaadcfd3a59441be Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 2 Mar 2021 17:31:12 +0100 Subject: [PATCH 01/21] [fix] Remove borg if no other borg or borgserver are setup --- scripts/remove | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/remove b/scripts/remove index b57c396..27cea14 100755 --- a/scripts/remove +++ b/scripts/remove @@ -43,6 +43,11 @@ ynh_secure_remove "/etc/systemd/system/$app.timer" ynh_secure_remove "/usr/local/bin/backup-with-$app" ynh_secure_remove "/etc/yunohost/hooks.d/backup_method/05-${app}_app" +if [ "$(yunohost app list | grep "id: borg" | wc -l)" == "1" ] ; then + ynh_secure_remove "/opt/borg-env" + ynh_secure_remove "/usr/local/bin/borg" +fi + #================================================= # END OF SCRIPT #================================================= From 4a1133915c8022e5d5e5b173d1289bbad0d8d12e Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 2 Mar 2021 17:31:37 +0100 Subject: [PATCH 02/21] [fix] Permission issue if we install borgserver and next borg --- scripts/_common.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/_common.sh b/scripts/_common.sh index 6bb0320..109f59c 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -15,7 +15,8 @@ install_borg_with_pip () { /opt/borg-env/bin/python /opt/borg-env/bin/pip install borgbackup==1.1.15 echo "#!/bin/bash /opt/borg-env/bin/python /opt/borg-env/bin/borg \"\$@\"" > /usr/local/bin/borg - chmod u+x /usr/local/bin/borg + # We need this to be executable by + chmod a+x /usr/local/bin/borg fi } From 8b769f21d405b70e558bac49eab9a9c68dc32c00 Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 2 Mar 2021 17:32:20 +0100 Subject: [PATCH 03/21] [enh] Add fuse support --- scripts/_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/_common.sh b/scripts/_common.sh index 109f59c..2b122c5 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -12,7 +12,7 @@ pkg_dependencies="python3-pip python3-dev libacl1-dev libssl-dev liblz4-dev pyth install_borg_with_pip () { if [ ! -d /opt/borg-env ]; then virtualenv --python=python3 /opt/borg-env - /opt/borg-env/bin/python /opt/borg-env/bin/pip install borgbackup==1.1.15 + /opt/borg-env/bin/python /opt/borg-env/bin/pip install borgbackup[fuse]==1.1.15 echo "#!/bin/bash /opt/borg-env/bin/python /opt/borg-env/bin/borg \"\$@\"" > /usr/local/bin/borg # We need this to be executable by From 710b0d58b96b813a4038c0e7139d252eb2081c27 Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 2 Mar 2021 17:33:06 +0100 Subject: [PATCH 04/21] [enh] Use python -m venv --- scripts/_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/_common.sh b/scripts/_common.sh index 2b122c5..2b13921 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -11,7 +11,7 @@ pkg_dependencies="python3-pip python3-dev libacl1-dev libssl-dev liblz4-dev pyth # Install borg with pip if borg is not here install_borg_with_pip () { if [ ! -d /opt/borg-env ]; then - virtualenv --python=python3 /opt/borg-env + python3 -m venv /opt/borg-env /opt/borg-env/bin/python /opt/borg-env/bin/pip install borgbackup[fuse]==1.1.15 echo "#!/bin/bash /opt/borg-env/bin/python /opt/borg-env/bin/borg \"\$@\"" > /usr/local/bin/borg From d62d4120fab384958592b393b7df8b533025f82c Mon Sep 17 00:00:00 2001 From: ljf Date: Wed, 3 Mar 2021 19:00:04 +0100 Subject: [PATCH 05/21] [fix] Prune wordpress should not prune wordpress__2 --- conf/backup_method.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conf/backup_method.j2 b/conf/backup_method.j2 index 35ed6a1..37c05d1 100644 --- a/conf/backup_method.j2 +++ b/conf/backup_method.j2 @@ -47,7 +47,9 @@ This is an automated message from your beloved YunoHost server." | /usr/bin/mail borg create $repo::${name}_${current_date} ./ 2>&1 >/dev/null | log_with_timestamp popd - borg prune $repo -P ${name} --keep-hourly 2 --keep-daily=7 --keep-weekly=8 --keep-monthly=12 2>&1 >/dev/null | log_with_timestamp + # About thi _20 it's a crazy fix to avoid pruning wordpress__2 + # if you prune wordpress + borg prune $repo -P ${name}_20 --keep-hourly 2 --keep-daily=7 --keep-weekly=8 --keep-monthly=12 2>&1 >/dev/null | log_with_timestamp } do_mount() { From ff0be0b044d389c282dbb0a2d1876eb236665323 Mon Sep 17 00:00:00 2001 From: ljf Date: Thu, 4 Mar 2021 15:59:07 +0100 Subject: [PATCH 06/21] [enh] Allow local repository --- check_process | 3 +-- conf/backup_method.j2 | 44 +++++++++++++++++++++---------------------- manifest.json | 24 +++++------------------ scripts/install | 12 +++++++++--- scripts/restore | 3 --- scripts/upgrade | 41 +++++++++++++++++++++++++--------------- 6 files changed, 63 insertions(+), 64 deletions(-) diff --git a/check_process b/check_process index c4d634d..363a988 100644 --- a/check_process +++ b/check_process @@ -1,7 +1,6 @@ ;; Test complet ; Manifest - server="domain.tld:22" - ssh_user="sam" + server="ssh://sam@domain.tld:22/~/backup" passphrase="APassphrase" conf=1 data=1 diff --git a/conf/backup_method.j2 b/conf/backup_method.j2 index 37c05d1..790b5fe 100644 --- a/conf/backup_method.j2 +++ b/conf/backup_method.j2 @@ -8,7 +8,7 @@ if ssh-keygen -F "{{ server }}" >/dev/null ; then else BORG_RSH="ssh -i /root/.ssh/id_{{ app }}_ed25519 -oStrictHostKeyChecking=no " fi -repo=ssh://{{ ssh_user }}@{{ server }}/~/backup #$4 +repo="{{ repository }}" #$4 do_need_mount() { true @@ -23,15 +23,15 @@ do_backup() { export BORG_PASSPHRASE export BORG_RSH - work_dir=$1 - name=$2 - repo=$3 - size=$4 - description=$5 + work_dir="$1" + name="$2" + repo="$3" + size="$4" + description="$5" current_date=$(date +"%d_%m_%y_%H:%M") - pushd $work_dir + pushd "$work_dir" set +e - if borg init -e repokey $repo ; then + if borg init -e repokey "$repo" ; then #human_size=`echo $size | awk '{ suffix=" KMGT"; for(i=1; $1>1024 && i < length(suffix); i++) $1/=1024; print int($1) substr(suffix, i, 1), $3; }'` # Speed in Kbps #speed=1000 @@ -44,37 +44,37 @@ This is an automated message from your beloved YunoHost server." | /usr/bin/mail fi set -e - borg create $repo::${name}_${current_date} ./ 2>&1 >/dev/null | log_with_timestamp + borg create "$repo::${name}_${current_date}" ./ 2>&1 >/dev/null | log_with_timestamp popd # About thi _20 it's a crazy fix to avoid pruning wordpress__2 # if you prune wordpress - borg prune $repo -P ${name}_20 --keep-hourly 2 --keep-daily=7 --keep-weekly=8 --keep-monthly=12 2>&1 >/dev/null | log_with_timestamp + borg prune "$repo" -P ${name}_20 --keep-hourly 2 --keep-daily=7 --keep-weekly=8 --keep-monthly=12 2>&1 >/dev/null | log_with_timestamp } do_mount() { export BORG_PASSPHRASE export BORG_RSH - work_dir=$1 - name=$2 - repo=$3 - size=$4 - description=$5 - borg mount $repo::$name $work_dir 2>&1 >/dev/null | log_with_timestamp + work_dir="$1" + name="$2" + repo="$3" + size="$4" + description="$5" + borg mount "$repo::$name" "$work_dir" 2>&1 >/dev/null | log_with_timestamp } -work_dir=$2 -name=$3 +work_dir="$2" +name="$3" -size=$5 -description=$6 +size="$5" +description="$6" case "$1" in need_mount) - do_need_mount $work_dir $name $repo $size $description + do_need_mount "$work_dir" "$name" "$repo" "$size" "$description" ;; backup) - do_backup $work_dir $name $repo $size $description + do_backup "$work_dir" "$name" "$repo" "$size" "$description" ;; mount) do_mount diff --git a/manifest.json b/manifest.json index a51509e..368dfc3 100644 --- a/manifest.json +++ b/manifest.json @@ -22,30 +22,16 @@ "arguments": { "install" : [ { - "name": "server", + "name": "repository", "type": "string", "ask": { - "en": "What is the domain name of the host server where backups will be sent?", - "fr": "Quel est le nom de domaine du serveur distant sur lequel seront envoyés les sauvegardes ?" + "en": "In which borg repository location do you want to backup your files ?", + "fr": "Dans quel repo borg souhaitez-vous sauvegarder vos fichiers ?" }, "help":{ - "en": "If this host server does use a custom SSH port (different from 22), you can specify it with DOMAIN:PORT", - "fr": "Si le serveur distant utilise un port particulier pour SSH (différent de 22), vous pouvez le spécifier avec DOMAIN:PORT" + "en": "You can specify a local by giving direct path or a remote repo in this format: ssh://USER@DOMAIN.TLD:PORT/~/backup . For remote user, it is not meant to be an existing user on the guest server. Instead, it will be created *on the host server* during the installation of the Borg Server App." }, - "example": "host.serverb:22" - }, - { - "name": "ssh_user", - "type": "string", - "ask": { - "en": "Which SSH username should be used to connect to the host server?", - "fr": "Quel utilisateur faut-il utiliser pour se connecter au serveur distant ?" - }, - "help":{ - "en": "It is not meant to be an existing user on this guest server. Instead, it will be created *on the host server* during the installation of the Borg Server App.", - "fr": "Cet utilisateur n'est pas censé exister sur ce serveur. Il sera créé *sur le serveur distant* lors de l'installation de l'App Borg Server sur celui-ci." - }, - "example": "borgservera" + "example": "ssh://john@serverb.tld:22/~/backup" }, { "name": "passphrase", diff --git a/scripts/install b/scripts/install index 9a73927..54b0da4 100755 --- a/scripts/install +++ b/scripts/install @@ -23,13 +23,19 @@ ynh_abort_if_errors export app=$YNH_APP_INSTANCE_NAME # Retrieve arguments -ynh_export server ssh_user passphrase on_calendar conf data apps +ynh_export repository passphrase on_calendar conf data apps #================================================= # STORE SETTINGS FROM MANIFEST #================================================= - -ynh_save_args server ssh_user passphrase on_calendar conf data apps +server="" +if [[ $repository == *"@"* ]]; then + server=$(echo "$repository" | cut -d"@" -f2 | cut -d"/" -f1) + if [[ $server == *":"* ]]; then + server="[$(echo "$server" | cut -d":" -f1)]:$(echo "$server" | cut -d":" -f2)" + fi +fi +ynh_save_args repository server passphrase on_calendar conf data apps #================================================= # INSTALL DEPENDENCIES diff --git a/scripts/restore b/scripts/restore index bc7c282..6310e7f 100755 --- a/scripts/restore +++ b/scripts/restore @@ -23,9 +23,6 @@ ynh_script_progression --message="Loading installation settings..." app=$YNH_APP_INSTANCE_NAME -server=$(ynh_app_setting_get $app server) -ssh_user=$(ynh_app_setting_get $app ssh_user) - #================================================= # INSTALL DEPENDENCIES #================================================= diff --git a/scripts/upgrade b/scripts/upgrade index 79ee142..76aeff5 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -15,21 +15,14 @@ source /usr/share/yunohost/helpers export app=$YNH_APP_INSTANCE_NAME -export server=$(ynh_app_setting_get $app server) -export ssh_user=$(ynh_app_setting_get $app ssh_user) -export passphrase=$(ynh_app_setting_get $app passphrase) -export on_calendar=$(ynh_app_setting_get $app on_calendar) -export conf=$(ynh_app_setting_get $app conf) -export data=$(ynh_app_setting_get $app data) -export apps=$(ynh_app_setting_get $app apps) - -export server=$(ynh_app_setting_get $app server) -export ssh_user=$(ynh_app_setting_get $app ssh_user) -export passphrase=$(ynh_app_setting_get $app passphrase) -export on_calendar=$(ynh_app_setting_get $app on_calendar) -export conf=$(ynh_app_setting_get $app conf) -export data=$(ynh_app_setting_get $app data) -export apps=$(ynh_app_setting_get $app apps) +export repository="$(ynh_app_setting_get $app repository)" +export server="$(ynh_app_setting_get $app server)" +export ssh_user="$(ynh_app_setting_get $app ssh_user)" +export passphrase="$(ynh_app_setting_get $app passphrase)" +export on_calendar="$(ynh_app_setting_get $app on_calendar)" +export conf="$(ynh_app_setting_get $app conf)" +export data="$(ynh_app_setting_get $app data)" +export apps="$(ynh_app_setting_get $app apps)" #================================================= # CHECK IF AN UPGRADE IS NEEDED @@ -51,6 +44,21 @@ ynh_clean_setup () { # Exit if an error occurs during the execution of the script ynh_abort_if_errors +#================================================= +# ENSURE DOWNWARD COMPATIBILITY +#================================================= +ynh_script_progression --message="Ensuring downward compatibility..." --weight=1 + +if [ -z "$repository" ]; then + repository="ssh://$ssh_user@$server/~/backup" + server=$(echo "$repository" | cut -d"@" -f2 | cut -d"/" -f1) + if [[ $server == *":"* ]]; then + server="[$(echo "$server" | cut -d":" -f1)]:$(echo "$server" | cut -d":" -f2)" + fi + ynh_app_setting_set $app repository "$repository" + ynh_app_setting_set $app server "$server" + ynh_app_setting_delete $app ssh_user +fi if grep "borg.timer" /etc/yunohost/services.yml > /dev/null ; then yunohost service remove $app.timer @@ -71,11 +79,13 @@ fi #================================================= # SETUP THE BACKUP METHOD #================================================= +ynh_script_progression --message="Setting up backup method..." --weight=1 ynh_configure backup_method "/etc/yunohost/hooks.d/backup_method/05-${app}_app" #================================================= # CONFIGURE CRON #================================================= +ynh_script_progression --message="Configuring systemd timer..." --weight=1 ynh_configure backup-with-borg "/usr/local/bin/backup-with-$app" chmod u+x "/usr/local/bin/backup-with-$app" ynh_add_systemd_config @@ -86,6 +96,7 @@ systemctl start $app.timer #================================================= # ADVERTISE SERVICE IN ADMIN PANEL #================================================= +ynh_script_progression --message="Exposing borg service in webadmin..." --weight=1 yunohost service add $app --description="Deduplicating backup program" From 5f3085aae3421ebb588afac7dfdf628483a884cb Mon Sep 17 00:00:00 2001 From: ljf Date: Thu, 4 Mar 2021 16:38:12 +0100 Subject: [PATCH 07/21] [fix] Upgrade borgbackup correctly --- manifest.json | 2 +- scripts/_common.sh | 15 ++++++--- scripts/install | 82 ++++++++++++++++++++++++++-------------------- scripts/upgrade | 53 ++++++++++++++++++++++++------ 4 files changed, 101 insertions(+), 51 deletions(-) diff --git a/manifest.json b/manifest.json index 368dfc3..10f8b83 100644 --- a/manifest.json +++ b/manifest.json @@ -6,7 +6,7 @@ "en": "Backup your server on a host server using Borg.", "fr": "Sauvegardez votre serveur sur un serveur distant avec Borg." }, - "version": "1.1.15~ynh1", + "version": "1.1.15~ynh2", "url": "https://borgbackup.readthedocs.io", "license": "BSD-3-Clause", "maintainer": { diff --git a/scripts/_common.sh b/scripts/_common.sh index 2b13921..efcfe2c 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -5,19 +5,26 @@ #================================================= # App package root directory should be the parent folder PKG_DIR=$(cd ../; pwd) +BORG_VERSION=1.1.15 -pkg_dependencies="python3-pip python3-dev libacl1-dev libssl-dev liblz4-dev python3-jinja2 python3-setuptools python-virtualenv virtualenv" +pkg_dependencies="python3-pip python3-dev libacl1-dev libssl-dev liblz4-dev python3-jinja2 python3-setuptools python-virtualenv virtualenv libfuse-dev pkg-config" # Install borg with pip if borg is not here install_borg_with_pip () { + if [ -d /opt/borg-env ]; then + /opt/borg-env/bin/python /opt/borg-env/bin/pip list | grep "Version: $BORG_VERSION" || ynh_secure_remove /opt/borg-env + fi if [ ! -d /opt/borg-env ]; then python3 -m venv /opt/borg-env - /opt/borg-env/bin/python /opt/borg-env/bin/pip install borgbackup[fuse]==1.1.15 + /opt/borg-env/bin/python /opt/borg-env/bin/pip install borgbackup[fuse]==$BORG_VERSION echo "#!/bin/bash /opt/borg-env/bin/python /opt/borg-env/bin/borg \"\$@\"" > /usr/local/bin/borg - # We need this to be executable by - chmod a+x /usr/local/bin/borg + if is_buster; then + touch /opt/borg-env/buster + fi fi + # We need this to be executable by other borg apps + chmod a+x /usr/local/bin/borg } #================================================= diff --git a/scripts/install b/scripts/install index 54b0da4..f215e1c 100755 --- a/scripts/install +++ b/scripts/install @@ -44,7 +44,8 @@ ynh_script_progression --message="Installing dependencies..." ynh_install_app_dependencies $pkg_dependencies install_borg_with_pip - +#================================================= +# SPECIFIC SETUP #================================================= # ACTIVATE BACKUP METHODS #================================================= @@ -56,49 +57,34 @@ mkdir -p /usr/share/yunohost/backup_method # SETUP THE BACKUP METHOD #================================================= ynh_configure backup_method "/etc/yunohost/hooks.d/backup_method/05-${app}_app" +chmod go=--- "/etc/yunohost/hooks.d/backup_method/05-${app}_app" -#================================================= -# CONFIGURE CRON -#================================================= ynh_configure backup-with-borg "/usr/local/bin/backup-with-$app" chmod u+x "/usr/local/bin/backup-with-$app" -ynh_add_systemd_config -ynh_configure systemd.timer "/etc/systemd/system/$app.timer" -systemctl enable $app.timer --quiet -systemctl start $app.timer -#yunohost service add $app.timer -#yunohost service enable $app.timer -#yunohost service start $app.timer -#================================================= -# ADVERTISE SERVICE IN ADMIN PANEL -#================================================= -ynh_script_progression --message="Integrating service in YunoHost..." +if [ ! -z "$server" ]; then + #================================================= + # GENERATE SSH KEY + #================================================= -yunohost service add $app --description="Deduplicating backup program" + private_key="/root/.ssh/id_${app}_ed25519" + test -f $private_key || ssh-keygen -q -t ed25519 -N "" -f $private_key -#================================================= -# GENERATE SSH KEY -#================================================= + #================================================= + # Display key + #================================================= -private_key="/root/.ssh/id_${app}_ed25519" -test -f $private_key || ssh-keygen -q -t ed25519 -N "" -f $private_key + echo "You should now install the \"Borg Server\" app on $server and fill questions like this: + User: ${ssh_user} + Public key: $(cat ${private_key}.pub)" -#================================================= -# Display key -#================================================= + #================================================= + # SEND A README FOR THE ADMIN + #================================================= -echo "You should now install the \"Borg Server\" app on $server and fill questions like this: -User: ${ssh_user} -Public key: $(cat ${private_key}.pub)" - -#================================================= -# SEND A README FOR THE ADMIN -#================================================= - -ynh_print_OFF -message="You should now install the \"Borg Server\" app on $server and fill questions like this: + ynh_print_OFF + message="You should now install the \"Borg Server\" app on $server and fill questions like this: User: ${ssh_user} Public key: $(cat ${private_key}.pub) @@ -108,8 +94,32 @@ yunohost app install https://github.com/YunoHost-Apps/borgserver_ynh -a \"ssh_us If you facing an issue or want to improve this app, please open a new issue in this project: https://github.com/YunoHost-Apps/borg_ynh" -ynh_send_readme_to_admin "$message" "root" -ynh_print_ON + ynh_send_readme_to_admin "$message" "root" + ynh_print_ON +fi + +#================================================= +# SETUP SYSTEMD +#================================================= +ynh_script_progression --message="Configuring a systemd service..." --time --weight=1 + +# Create a dedicated systemd config +ynh_add_systemd_config + +#================================================= +# CONFIGURE SYSTEMD TIMER +#================================================= +ynh_configure systemd.timer "/etc/systemd/system/$app.timer" +systemctl enable $app.timer --quiet +systemctl start $app.timer + +#================================================= +# INTEGRATE SERVICE IN YUNOHOST +#================================================= +ynh_script_progression --message="Integrating service in YunoHost..." --weight=1 + +yunohost service add $app --description="Deduplicating backup program" + #================================================= # END OF SCRIPT diff --git a/scripts/upgrade b/scripts/upgrade index 76aeff5..2720b24 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -67,36 +67,69 @@ if grep "borg.timer" /etc/yunohost/services.yml > /dev/null ; then fi # Replace backports with pip +rm -f /etc/apt/sources.list.d/$app-stretch-backports.list + +# Reinstall borg if stretch -> buster migration if is_buster; then if [ ! -f /opt/borg-env/buster ] ; then - rm -f /etc/apt/sources.list.d/$app-stretch-backports.list - ynh_secure_remove /opt/borg-env - install_borg_with_pip - touch /opt/borg-env/buster + ynh_secure_remove /opt/borg-env fi fi +#================================================= +# UPGRADE DEPENDENCIES +#================================================= +ynh_script_progression --message="Upgrading dependencies..." --weight=1 + +ynh_install_app_dependencies $pkg_dependencies + +#================================================= +# SPECIFIC UPGRADE +#================================================= +# Upgrade borgbackup +#================================================= +ynh_script_progression --message="Upgrading borgbackup..." --weight=1 + +install_borg_with_pip + #================================================= # SETUP THE BACKUP METHOD #================================================= ynh_script_progression --message="Setting up backup method..." --weight=1 ynh_configure backup_method "/etc/yunohost/hooks.d/backup_method/05-${app}_app" +chmod go=--- "/etc/yunohost/hooks.d/backup_method/05-${app}_app" -#================================================= -# CONFIGURE CRON -#================================================= -ynh_script_progression --message="Configuring systemd timer..." --weight=1 ynh_configure backup-with-borg "/usr/local/bin/backup-with-$app" chmod u+x "/usr/local/bin/backup-with-$app" + +#================================================= +# SETUP SYSTEMD +#================================================= +ynh_script_progression --message="Upgrading systemd configuration..." --weight=1 + +# Create a dedicated systemd config ynh_add_systemd_config + +#================================================= +# CONFIGURE SYSTEMD TIMER +#================================================= ynh_configure systemd.timer "/etc/systemd/system/$app.timer" systemctl enable $app.timer --quiet systemctl start $app.timer #================================================= -# ADVERTISE SERVICE IN ADMIN PANEL +# GENERIC FINALIZATION #================================================= -ynh_script_progression --message="Exposing borg service in webadmin..." --weight=1 +# SECURE FILES AND DIRECTORIES +#================================================= + +# Set permissions on app files +chown -R root: $final_path + +#================================================= +# INTEGRATE SERVICE IN YUNOHOST +#================================================= +ynh_script_progression --message="Integrating service in YunoHost..." --weight=1 yunohost service add $app --description="Deduplicating backup program" From d41defe7618832ebce4ed30f6abbc2ffe871e22f Mon Sep 17 00:00:00 2001 From: ljf Date: Thu, 4 Mar 2021 17:07:37 +0100 Subject: [PATCH 08/21] [fix] Support more char in passphrase --- conf/{backup-with-borg.j2 => backup-with-borg} | 18 ++++++++++-------- conf/{backup_method.j2 => backup_method} | 10 +++++----- conf/systemd.service | 2 +- conf/systemd.timer | 8 ++++++++ conf/systemd.timer.j2 | 8 -------- scripts/_common.sh | 10 +--------- scripts/install | 7 ++++--- scripts/restore | 11 +++++++---- scripts/upgrade | 7 ++++--- 9 files changed, 40 insertions(+), 41 deletions(-) rename conf/{backup-with-borg.j2 => backup-with-borg} (58%) rename conf/{backup_method.j2 => backup_method} (87%) create mode 100644 conf/systemd.timer delete mode 100644 conf/systemd.timer.j2 diff --git a/conf/backup-with-borg.j2 b/conf/backup-with-borg similarity index 58% rename from conf/backup-with-borg.j2 rename to conf/backup-with-borg index 68df3b1..2f09a3b 100644 --- a/conf/backup-with-borg.j2 +++ b/conf/backup-with-borg @@ -1,5 +1,7 @@ #!/bin/bash +app=$1 + # Adapt this script to different API 2.x vs 3.x if yunohost -v | grep "version: 2." > /dev/null; then ignore_apps="--ignore-apps" @@ -14,34 +16,34 @@ filter_hooks() { } # Backup system part conf -conf=$(yunohost app setting {{ app }} conf) +conf=$(yunohost app setting $app conf) if [ $conf -eq 1 ] then - yunohost backup create $ignore_apps -n auto_conf --method {{ app }}_app --system $(filter_hooks conf) + yunohost backup create $ignore_apps -n auto_conf --method $app_app --system $(filter_hooks conf) fi # Backup system data -data=$(yunohost app setting {{ app }} data) +data=$(yunohost app setting $app data) if [ $data -eq 1 ] then - yunohost backup create $ignore_apps -n auto_data --method {{ app }}_app --system $(filter_hooks data) + yunohost backup create $ignore_apps -n auto_data --method $app_app --system $(filter_hooks data) fi # Backup all apps independently -apps=$(yunohost app setting {{ app }} apps) -for app in $(ls /etc/yunohost/apps/*/scripts/backup | cut -d / -f 5); do +apps=$(yunohost app setting $app apps) +for application in $(ls /etc/yunohost/apps/*/scripts/backup | cut -d / -f 5); do backup_app=false if [[ "$apps" = "all" ]]; then backup_app=true else for selected_app in $(echo $apps | tr "," " ");do - if [[ "$selected_app" == "$app" ]]; then + if [[ "$selected_app" == "$application" ]]; then backup_app=true break fi done fi if [ "$backup_app" == "true" ];then - yunohost backup create $ignore_system -n auto_$app --method {{ app }}_app --apps $app + yunohost backup create $ignore_system -n auto_$application --method $app_app --apps $app fi done diff --git a/conf/backup_method.j2 b/conf/backup_method similarity index 87% rename from conf/backup_method.j2 rename to conf/backup_method index 790b5fe..4cf3c30 100644 --- a/conf/backup_method.j2 +++ b/conf/backup_method @@ -2,13 +2,13 @@ set -e -BORG_PASSPHRASE="{{ passphrase }}" -if ssh-keygen -F "{{ server }}" >/dev/null ; then - BORG_RSH="ssh -i /root/.ssh/id_{{ app }}_ed25519 -oStrictHostKeyChecking=yes " +BORG_PASSPHRASE='__PASSPHRASE__' +repo="__REPOSITORY__" #$4 +if ssh-keygen -F "__SERVER__" >/dev/null ; then + BORG_RSH="ssh -i /root/.ssh/id___APP___ed25519 -oStrictHostKeyChecking=yes " else - BORG_RSH="ssh -i /root/.ssh/id_{{ app }}_ed25519 -oStrictHostKeyChecking=no " + BORG_RSH="ssh -i /root/.ssh/id___APP___ed25519 -oStrictHostKeyChecking=no " fi -repo="{{ repository }}" #$4 do_need_mount() { true diff --git a/conf/systemd.service b/conf/systemd.service index e0d648d..4c8271b 100644 --- a/conf/systemd.service +++ b/conf/systemd.service @@ -4,7 +4,7 @@ After=network.target [Service] Type=oneshot -ExecStart=/usr/local/bin/backup-with-__APP__ +ExecStart=/usr/local/bin/backup-with-__APP__ __APP__ User=root Group=root diff --git a/conf/systemd.timer b/conf/systemd.timer new file mode 100644 index 0000000..6434e4e --- /dev/null +++ b/conf/systemd.timer @@ -0,0 +1,8 @@ +[Unit] +Description=Run backup __APP__ regularly + +[Timer] +OnCalendar=__ON_CALENDAR__ + +[Install] +WantedBy=timers.target diff --git a/conf/systemd.timer.j2 b/conf/systemd.timer.j2 deleted file mode 100644 index f21a951..0000000 --- a/conf/systemd.timer.j2 +++ /dev/null @@ -1,8 +0,0 @@ -[Unit] -Description=Run backup {{ app }} regularly - -[Timer] -OnCalendar={{ on_calendar }} - -[Install] -WantedBy=timers.target diff --git a/scripts/_common.sh b/scripts/_common.sh index efcfe2c..ed15366 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -51,20 +51,12 @@ ynh_save_args () { if [ "$var" == "path_url" ]; then setting_var="path" fi - ynh_app_setting_set $app $setting_var ${!var} + ynh_app_setting_set $app $setting_var "${!var}" done } -ynh_configure () { - ynh_backup_if_checksum_is_different $2 - ynh_render_template "${PKG_DIR}/conf/$1.j2" "$2" - ynh_store_file_checksum $2 -} - - - # Send an email to inform the administrator # # usage: ynh_send_readme_to_admin app_message [recipients] diff --git a/scripts/install b/scripts/install index f215e1c..c1552fd 100755 --- a/scripts/install +++ b/scripts/install @@ -44,6 +44,7 @@ ynh_script_progression --message="Installing dependencies..." ynh_install_app_dependencies $pkg_dependencies install_borg_with_pip + #================================================= # SPECIFIC SETUP #================================================= @@ -56,11 +57,11 @@ mkdir -p /usr/share/yunohost/backup_method #================================================= # SETUP THE BACKUP METHOD #================================================= -ynh_configure backup_method "/etc/yunohost/hooks.d/backup_method/05-${app}_app" +ynh_add_config --template="backup_method" --destination="/etc/yunohost/hooks.d/backup_method/05-${app}_app" chmod go=--- "/etc/yunohost/hooks.d/backup_method/05-${app}_app" -ynh_configure backup-with-borg "/usr/local/bin/backup-with-$app" +ynh_add_config --template="backup-with-borg" --destination="/usr/local/bin/backup-with-$app" chmod u+x "/usr/local/bin/backup-with-$app" if [ ! -z "$server" ]; then @@ -109,7 +110,7 @@ ynh_add_systemd_config #================================================= # CONFIGURE SYSTEMD TIMER #================================================= -ynh_configure systemd.timer "/etc/systemd/system/$app.timer" +ynh_add_config --template="systemd.timer" --destination="/etc/systemd/system/$app.timer" systemctl enable $app.timer --quiet systemctl start $app.timer diff --git a/scripts/restore b/scripts/restore index 6310e7f..31a35cf 100755 --- a/scripts/restore +++ b/scripts/restore @@ -50,16 +50,19 @@ ynh_restore yunohost service add $app systemctl enable $app.timer --quiet systemctl start $app.timer -#yunohost service add $app.timer -#yunohost service enable $app.timer -#yunohost service start $app.timer #================================================= # RESTORE SYSTEMD #================================================= - systemctl enable $app.service --quiet +#================================================= +# INTEGRATE SERVICE IN YUNOHOST +#================================================= +ynh_script_progression --message="Integrating service in YunoHost..." --weight=1 + +yunohost service add $app --description="Deduplicating backup program" + #================================================= # END OF SCRIPT #================================================= diff --git a/scripts/upgrade b/scripts/upgrade index 2720b24..01a026c 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -96,10 +96,11 @@ install_borg_with_pip # SETUP THE BACKUP METHOD #================================================= ynh_script_progression --message="Setting up backup method..." --weight=1 -ynh_configure backup_method "/etc/yunohost/hooks.d/backup_method/05-${app}_app" +ynh_add_config --template="backup_method" --destination="/etc/yunohost/hooks.d/backup_method/05-${app}_app" chmod go=--- "/etc/yunohost/hooks.d/backup_method/05-${app}_app" -ynh_configure backup-with-borg "/usr/local/bin/backup-with-$app" + +ynh_add_config --template="backup-with-borg" --destination="/usr/local/bin/backup-with-$app" chmod u+x "/usr/local/bin/backup-with-$app" #================================================= @@ -113,7 +114,7 @@ ynh_add_systemd_config #================================================= # CONFIGURE SYSTEMD TIMER #================================================= -ynh_configure systemd.timer "/etc/systemd/system/$app.timer" +ynh_add_config --template="systemd.timer" --destination="/etc/systemd/system/$app.timer" systemctl enable $app.timer --quiet systemctl start $app.timer From f02754e21e872c3922a51ccb13802523c761a20e Mon Sep 17 00:00:00 2001 From: ljf Date: Sat, 6 Mar 2021 22:04:36 +0100 Subject: [PATCH 09/21] I should commit every day --- conf/backup-with-borg | 36 +++++++++++++++++++++++++++++++++--- scripts/install | 1 + 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/conf/backup-with-borg b/conf/backup-with-borg index 2f09a3b..38f1f00 100644 --- a/conf/backup-with-borg +++ b/conf/backup-with-borg @@ -1,6 +1,14 @@ #!/bin/bash +# We don't stop the script on errors cause we want to backup all data we could backuped +#set -eu + app=$1 +errors="" +current_date=$(date +"%y%m%d_%H%M") +log_file="/var/log/$app/$current_date.log" +err_file="/var/log/$app/$current_date.err" +mkdir -p "/var/log/$app" # Adapt this script to different API 2.x vs 3.x if yunohost -v | grep "version: 2." > /dev/null; then @@ -15,18 +23,26 @@ filter_hooks() { ls /usr/share/yunohost/hooks/backup/ /etc/yunohost/hooks.d/backup/ | grep "\-$1_" | cut -d"-" -f2 | uniq } +fail_if_partially_failed() { + grep Skipped|Error +} + # Backup system part conf conf=$(yunohost app setting $app conf) if [ $conf -eq 1 ] then - yunohost backup create $ignore_apps -n auto_conf --method $app_app --system $(filter_hooks conf) + if ! yunohost backup create $ignore_apps -n auto_conf --method $app_app --system $(filter_hooks conf) 2> $err_file > $log_file ; then + errors="$errors\nconf: Error" + fi fi # Backup system data data=$(yunohost app setting $app data) if [ $data -eq 1 ] then - yunohost backup create $ignore_apps -n auto_data --method $app_app --system $(filter_hooks data) + if ! yunohost backup create $ignore_apps -n auto_data --method $app_app --system $(filter_hooks data) 2> $err_file > $log_file ; then + errors="$errors\ndata: Error" + fi fi # Backup all apps independently @@ -44,6 +60,20 @@ for application in $(ls /etc/yunohost/apps/*/scripts/backup | cut -d / -f 5); do done fi if [ "$backup_app" == "true" ];then - yunohost backup create $ignore_system -n auto_$application --method $app_app --apps $app + if ! yunohost backup create $ignore_system -n auto_$application --method $app_app --apps $app 2> $err_file > $log_file ; then + errors="$errors\$application: Error" + fi fi done +partial_errors="$(cat $log_file | grep -E "Error|Skipped")" +if [ ! -z "$partial_errors" ]; then + errors="$errors\n$partial_errors" +fi + +# Send mail on backup (partially) failed +domain=$(hostname) +if [ ! -z "$errors" ]; then + cat $errors | mail -s "[borg] Backup failed from $domain onto $repo" root +else + cat $log_file | mail -s "[borg] Backup succeed from $domain onto $repo" root +fi diff --git a/scripts/install b/scripts/install index c1552fd..85610c6 100755 --- a/scripts/install +++ b/scripts/install @@ -34,6 +34,7 @@ if [[ $repository == *"@"* ]]; then if [[ $server == *":"* ]]; then server="[$(echo "$server" | cut -d":" -f1)]:$(echo "$server" | cut -d":" -f2)" fi + ssh_user=$(echo "$repository" | cut -d"@" -f1 | cut -d"/" -f2) fi ynh_save_args repository server passphrase on_calendar conf data apps From bae5b36e7b648baf3e9453c06b02699d803af3e4 Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 28 Mar 2021 16:16:01 +0200 Subject: [PATCH 10/21] [fix] Some bugs found on Aleks review --- check_process | 2 +- conf/backup-with-borg | 30 +++++++++++++----------------- manifest.json | 2 +- scripts/install | 4 ++-- scripts/remove | 1 + scripts/upgrade | 2 +- 6 files changed, 19 insertions(+), 22 deletions(-) diff --git a/check_process b/check_process index 363a988..c712791 100644 --- a/check_process +++ b/check_process @@ -1,6 +1,6 @@ ;; Test complet ; Manifest - server="ssh://sam@domain.tld:22/~/backup" + repository="ssh://sam@domain.tld:22/~/backup" passphrase="APassphrase" conf=1 data=1 diff --git a/conf/backup-with-borg b/conf/backup-with-borg index 38f1f00..e202f3a 100644 --- a/conf/backup-with-borg +++ b/conf/backup-with-borg @@ -3,20 +3,16 @@ # We don't stop the script on errors cause we want to backup all data we could backuped #set -eu -app=$1 +borg_id=$1 errors="" current_date=$(date +"%y%m%d_%H%M") -log_file="/var/log/$app/$current_date.log" -err_file="/var/log/$app/$current_date.err" -mkdir -p "/var/log/$app" - -# Adapt this script to different API 2.x vs 3.x -if yunohost -v | grep "version: 2." > /dev/null; then - ignore_apps="--ignore-apps" - ignore_system="--ignore-system" -else - ignore_apps="" - ignore_system="" +log_file="/var/log/${borg_id}/${current_date}.log" +err_file="/var/log/${borg_id}/${current_date}.err" +mkdir -p "/var/log/${borg_id}" +if [[ -z "$borg_id" ]] +then + echo "This script expects a borg app id as first argument" >&2 + exit 1 fi filter_hooks() { @@ -28,10 +24,10 @@ fail_if_partially_failed() { } # Backup system part conf -conf=$(yunohost app setting $app conf) +conf=$(yunohost app setting ${borg_id} conf) if [ $conf -eq 1 ] then - if ! yunohost backup create $ignore_apps -n auto_conf --method $app_app --system $(filter_hooks conf) 2> $err_file > $log_file ; then + if ! yunohost backup create -n auto_conf --method ${borg_id}_app --system $(filter_hooks conf) 2> $err_file > $log_file ; then errors="$errors\nconf: Error" fi fi @@ -40,13 +36,13 @@ fi data=$(yunohost app setting $app data) if [ $data -eq 1 ] then - if ! yunohost backup create $ignore_apps -n auto_data --method $app_app --system $(filter_hooks data) 2> $err_file > $log_file ; then + if ! yunohost backup create -n auto_data --method ${borg_id}_app --system $(filter_hooks data) 2> $err_file > $log_file ; then errors="$errors\ndata: Error" fi fi # Backup all apps independently -apps=$(yunohost app setting $app apps) +apps=$(yunohost app setting ${borg_id} apps) for application in $(ls /etc/yunohost/apps/*/scripts/backup | cut -d / -f 5); do backup_app=false if [[ "$apps" = "all" ]]; then @@ -60,7 +56,7 @@ for application in $(ls /etc/yunohost/apps/*/scripts/backup | cut -d / -f 5); do done fi if [ "$backup_app" == "true" ];then - if ! yunohost backup create $ignore_system -n auto_$application --method $app_app --apps $app 2> $err_file > $log_file ; then + if ! yunohost backup create -n auto_$application --method ${borg_id}_app --apps $application 2> $err_file > $log_file ; then errors="$errors\$application: Error" fi fi diff --git a/manifest.json b/manifest.json index 10f8b83..1f075b3 100644 --- a/manifest.json +++ b/manifest.json @@ -29,7 +29,7 @@ "fr": "Dans quel repo borg souhaitez-vous sauvegarder vos fichiers ?" }, "help":{ - "en": "You can specify a local by giving direct path or a remote repo in this format: ssh://USER@DOMAIN.TLD:PORT/~/backup . For remote user, it is not meant to be an existing user on the guest server. Instead, it will be created *on the host server* during the installation of the Borg Server App." + "en": "Specify a remote repository using this format: ssh://USER@DOMAIN.TLD:PORT/~/backup . "USER" is *not* meant to be an existing user on the guest server. Instead, it will be created *on the host server* during the installation of the Borg Server App. It's also possible to use a local repository using a syntax such as /mount/my_external_harddrive/backups" }, "example": "ssh://john@serverb.tld:22/~/backup" }, diff --git a/scripts/install b/scripts/install index 85610c6..1ef275f 100755 --- a/scripts/install +++ b/scripts/install @@ -77,7 +77,7 @@ if [ ! -z "$server" ]; then # Display key #================================================= - echo "You should now install the \"Borg Server\" app on $server and fill questions like this: + echo "You should now install the \"Borg Server\" app on $server and with the following credentials: User: ${ssh_user} Public key: $(cat ${private_key}.pub)" @@ -86,7 +86,7 @@ if [ ! -z "$server" ]; then #================================================= ynh_print_OFF - message="You should now install the \"Borg Server\" app on $server and fill questions like this: + message="You should now install the \"Borg Server\" app on $server and with the following credentials: User: ${ssh_user} Public key: $(cat ${private_key}.pub) diff --git a/scripts/remove b/scripts/remove index 27cea14..e3da1d5 100755 --- a/scripts/remove +++ b/scripts/remove @@ -43,6 +43,7 @@ ynh_secure_remove "/etc/systemd/system/$app.timer" ynh_secure_remove "/usr/local/bin/backup-with-$app" ynh_secure_remove "/etc/yunohost/hooks.d/backup_method/05-${app}_app" +# Remove borg if we are removing the last borg app on the system if [ "$(yunohost app list | grep "id: borg" | wc -l)" == "1" ] ; then ynh_secure_remove "/opt/borg-env" ynh_secure_remove "/usr/local/bin/borg" diff --git a/scripts/upgrade b/scripts/upgrade index 01a026c..9ebde06 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -67,7 +67,7 @@ if grep "borg.timer" /etc/yunohost/services.yml > /dev/null ; then fi # Replace backports with pip -rm -f /etc/apt/sources.list.d/$app-stretch-backports.list +[ ! -e " /etc/apt/sources.list.d/$app-stretch-backports.list" ] || rm -f /etc/apt/sources.list.d/$app-stretch-backports.list # Reinstall borg if stretch -> buster migration if is_buster; then From 070598ea267bd7590e467adfd257c78a0681bca7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 28 Mar 2021 16:40:22 +0200 Subject: [PATCH 11/21] Quotes-inside-quotes, json unhappy --- manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest.json b/manifest.json index 1f075b3..a54aab9 100644 --- a/manifest.json +++ b/manifest.json @@ -29,7 +29,7 @@ "fr": "Dans quel repo borg souhaitez-vous sauvegarder vos fichiers ?" }, "help":{ - "en": "Specify a remote repository using this format: ssh://USER@DOMAIN.TLD:PORT/~/backup . "USER" is *not* meant to be an existing user on the guest server. Instead, it will be created *on the host server* during the installation of the Borg Server App. It's also possible to use a local repository using a syntax such as /mount/my_external_harddrive/backups" + "en": "Specify a remote repository using this format: ssh://USER@DOMAIN.TLD:PORT/~/backup . 'USER' is *not* meant to be an existing user on the guest server. Instead, it will be created *on the host server* during the installation of the Borg Server App. It's also possible to use a local repository using a syntax such as /mount/my_external_harddrive/backups" }, "example": "ssh://john@serverb.tld:22/~/backup" }, From 09d5463b90de5b40bcf06c494e0dc52bfd7afdf8 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 29 Mar 2021 09:56:12 +0200 Subject: [PATCH 12/21] [fix] Install process was broken --- manifest.json | 4 ++-- scripts/_common.sh | 49 ++++++++++++---------------------------------- scripts/install | 2 +- scripts/upgrade | 8 +++----- 4 files changed, 18 insertions(+), 45 deletions(-) diff --git a/manifest.json b/manifest.json index 1f075b3..8268e59 100644 --- a/manifest.json +++ b/manifest.json @@ -6,7 +6,7 @@ "en": "Backup your server on a host server using Borg.", "fr": "Sauvegardez votre serveur sur un serveur distant avec Borg." }, - "version": "1.1.15~ynh2", + "version": "1.1.16~ynh1", "url": "https://borgbackup.readthedocs.io", "license": "BSD-3-Clause", "maintainer": { @@ -29,7 +29,7 @@ "fr": "Dans quel repo borg souhaitez-vous sauvegarder vos fichiers ?" }, "help":{ - "en": "Specify a remote repository using this format: ssh://USER@DOMAIN.TLD:PORT/~/backup . "USER" is *not* meant to be an existing user on the guest server. Instead, it will be created *on the host server* during the installation of the Borg Server App. It's also possible to use a local repository using a syntax such as /mount/my_external_harddrive/backups" + "en": "Specify a remote repository using this format: ssh://USER@DOMAIN.TLD:PORT/~/backup . \"USER\" is *not* meant to be an existing user on the guest server. Instead, it will be created *on the host server* during the installation of the Borg Server App. It's also possible to use a local repository using a syntax such as /mount/my_external_harddrive/backups" }, "example": "ssh://john@serverb.tld:22/~/backup" }, diff --git a/scripts/_common.sh b/scripts/_common.sh index ed15366..aee0e65 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -5,7 +5,7 @@ #================================================= # App package root directory should be the parent folder PKG_DIR=$(cd ../; pwd) -BORG_VERSION=1.1.15 +BORG_VERSION=1.1.16 pkg_dependencies="python3-pip python3-dev libacl1-dev libssl-dev liblz4-dev python3-jinja2 python3-setuptools python-virtualenv virtualenv libfuse-dev pkg-config" @@ -16,12 +16,11 @@ install_borg_with_pip () { fi if [ ! -d /opt/borg-env ]; then python3 -m venv /opt/borg-env + /opt/borg-env/bin/python /opt/borg-env/bin/pip install wheel /opt/borg-env/bin/python /opt/borg-env/bin/pip install borgbackup[fuse]==$BORG_VERSION echo "#!/bin/bash /opt/borg-env/bin/python /opt/borg-env/bin/borg \"\$@\"" > /usr/local/bin/borg - if is_buster; then - touch /opt/borg-env/buster - fi + touch "/opt/borg-env/$(ynh_get_debian_release)" fi # We need this to be executable by other borg apps chmod a+x /usr/local/bin/borg @@ -55,7 +54,16 @@ ynh_save_args () { done } +# Need also the helper https://github.com/YunoHost-Apps/Experimental_helpers/blob/master/ynh_handle_getopts_args/ynh_handle_getopts_args +# Send an email to inform the administrator +# +# usage: ynh_send_readme_to_admin app_message [recipients] +# | arg: -m --app_message= - The message to send to the administrator. +# | arg: -r, --recipients= - The recipients of this email. Use spaces to separate multiples recipients. - default: root +# example: "root admin@domain" +# If you give the name of a YunoHost user, ynh_send_readme_to_admin will find its email adress for you +# example: "root admin@domain user1 user2" # Send an email to inform the administrator # @@ -113,36 +121,3 @@ $(yunohost tools diagnosis | grep -B 100 "services:" | sed '/services:/d')" # Send the email to the recipients echo "$mail_message" | $mail_bin -a "Content-Type: text/plain; charset=UTF-8" -s "$mail_subject" "$recipients" } - - - -ynh_debian_release () { - lsb_release --codename --short -} - -is_buster () { - if [ "$(ynh_debian_release)" == "buster" ] - then - return 0 - else - return 1 - fi -} - -is_stretch () { - if [ "$(ynh_debian_release)" == "stretch" ] - then - return 0 - else - return 1 - fi -} - -is_jessie () { - if [ "$(ynh_debian_release)" == "jessie" ] - then - return 0 - else - return 1 - fi -} diff --git a/scripts/install b/scripts/install index 1ef275f..89090b8 100755 --- a/scripts/install +++ b/scripts/install @@ -96,7 +96,7 @@ yunohost app install https://github.com/YunoHost-Apps/borgserver_ynh -a \"ssh_us If you facing an issue or want to improve this app, please open a new issue in this project: https://github.com/YunoHost-Apps/borg_ynh" - ynh_send_readme_to_admin "$message" "root" + ynh_send_readme_to_admin "$message" ynh_print_ON fi diff --git a/scripts/upgrade b/scripts/upgrade index 9ebde06..5a4ae19 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -69,11 +69,9 @@ fi # Replace backports with pip [ ! -e " /etc/apt/sources.list.d/$app-stretch-backports.list" ] || rm -f /etc/apt/sources.list.d/$app-stretch-backports.list -# Reinstall borg if stretch -> buster migration -if is_buster; then - if [ ! -f /opt/borg-env/buster ] ; then - ynh_secure_remove /opt/borg-env - fi +# Reinstall borg if debian change of major version +if [ ! -f "/opt/borg-env/$(ynh_get_debian_release)" ] ; then + ynh_secure_remove /opt/borg-env fi #================================================= From a95d8002d5826c4d0f5ada12ba82fb74c3e33783 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 29 Mar 2021 10:26:15 +0200 Subject: [PATCH 13/21] [fix] Run borg with a specific user --- conf/sudoer | 1 + conf/systemd.service | 6 +++--- scripts/backup | 1 + scripts/install | 11 ++++++++++- scripts/restore | 1 - scripts/upgrade | 3 ++- 6 files changed, 17 insertions(+), 6 deletions(-) create mode 100644 conf/sudoer diff --git a/conf/sudoer b/conf/sudoer new file mode 100644 index 0000000..7945669 --- /dev/null +++ b/conf/sudoer @@ -0,0 +1 @@ +__APP__ ALL=(root) /usr/local/bin/backup-with-__APP__, /usr/local/bin/borg diff --git a/conf/systemd.service b/conf/systemd.service index 4c8271b..b2d0080 100644 --- a/conf/systemd.service +++ b/conf/systemd.service @@ -4,9 +4,9 @@ After=network.target [Service] Type=oneshot -ExecStart=/usr/local/bin/backup-with-__APP__ __APP__ -User=root -Group=root +ExecStart=/usr/bin/sudo /usr/local/bin/backup-with-__APP__ __APP__ +User=__APP__ +Group=__APP__ #[Install] #WantedBy=multi-user.target diff --git a/scripts/backup b/scripts/backup index 9a0f189..61e6920 100755 --- a/scripts/backup +++ b/scripts/backup @@ -28,6 +28,7 @@ app=$YNH_APP_INSTANCE_NAME ynh_print_info --message="Declaring files to be backed up..." ynh_backup "/usr/local/bin/backup-with-$app" +ynh_backup "/etc/sudoers.d/$app" ynh_backup "/etc/systemd/system/$app.service" ynh_backup "/etc/systemd/system/$app.timer" ynh_backup "/etc/yunohost/hooks.d/backup_method/05-${app}_app" diff --git a/scripts/install b/scripts/install index 89090b8..2be84fd 100755 --- a/scripts/install +++ b/scripts/install @@ -46,6 +46,14 @@ ynh_script_progression --message="Installing dependencies..." ynh_install_app_dependencies $pkg_dependencies install_borg_with_pip +#================================================= +# CREATE DEDICATED USER +#================================================= +ynh_script_progression --message="Configuring system user..." --time --weight=1 + +# Create a system user +ynh_system_user_create --username=$app + #================================================= # SPECIFIC SETUP #================================================= @@ -64,6 +72,7 @@ chmod go=--- "/etc/yunohost/hooks.d/backup_method/05-${app}_app" ynh_add_config --template="backup-with-borg" --destination="/usr/local/bin/backup-with-$app" chmod u+x "/usr/local/bin/backup-with-$app" +ynh_add_config --template="sudoer" --destination="/etc/sudoers.d/$app" if [ ! -z "$server" ]; then #================================================= @@ -103,7 +112,7 @@ fi #================================================= # SETUP SYSTEMD #================================================= -ynh_script_progression --message="Configuring a systemd service..." --time --weight=1 +ynh_script_progression --message="Configuring a systemd service..." --weight=1 # Create a dedicated systemd config ynh_add_systemd_config diff --git a/scripts/restore b/scripts/restore index 31a35cf..f05ed7c 100755 --- a/scripts/restore +++ b/scripts/restore @@ -47,7 +47,6 @@ ynh_restore #================================================= # ADVERTISE SERVICE IN ADMIN PANEL #================================================= -yunohost service add $app systemctl enable $app.timer --quiet systemctl start $app.timer diff --git a/scripts/upgrade b/scripts/upgrade index 5a4ae19..b160e27 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -97,10 +97,11 @@ ynh_script_progression --message="Setting up backup method..." --weight=1 ynh_add_config --template="backup_method" --destination="/etc/yunohost/hooks.d/backup_method/05-${app}_app" chmod go=--- "/etc/yunohost/hooks.d/backup_method/05-${app}_app" - ynh_add_config --template="backup-with-borg" --destination="/usr/local/bin/backup-with-$app" chmod u+x "/usr/local/bin/backup-with-$app" +ynh_add_config --template="sudoer" --destination="/etc/sudoers.d/$app" + #================================================= # SETUP SYSTEMD #================================================= From 034e73da873a6f3eb2615ca1800a3b89569e0724 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 29 Mar 2021 10:33:19 +0200 Subject: [PATCH 14/21] [enh] Support upgrade test --- check_process | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/check_process b/check_process index c712791..16562cf 100644 --- a/check_process +++ b/check_process @@ -20,4 +20,8 @@ change_url=0 ;;; Options Email=ljf+borg_ynh@reflexlibre.net -Notification=down +Notificatio +;;; Upgrade options + ; commit=c1524dd8e37cc671c01f5b5363901dd6a01e6fc1 + name=Merge pull request #63 from YunoHost-Apps/testing + manifest_arg=server=domain.tld:22&ssh_user=sam&passphrase=APassphrase&conf=1&data=1&apps=all&on_calendar=Daily From 15d884afa2756810b69394925e52c91b9d8ae117 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 29 Mar 2021 10:35:54 +0200 Subject: [PATCH 15/21] [fix] Missing deps --- scripts/_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/_common.sh b/scripts/_common.sh index aee0e65..90945f3 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -7,7 +7,7 @@ PKG_DIR=$(cd ../; pwd) BORG_VERSION=1.1.16 -pkg_dependencies="python3-pip python3-dev libacl1-dev libssl-dev liblz4-dev python3-jinja2 python3-setuptools python-virtualenv virtualenv libfuse-dev pkg-config" +pkg_dependencies="python3-pip python3-dev libacl1-dev libssl-dev liblz4-dev python3-jinja2 python3-setuptools python3-venv python-virtualenv virtualenv libfuse-dev pkg-config" # Install borg with pip if borg is not here install_borg_with_pip () { From 1019fa44e34ebf3d35a6c1ce9c1904b539518911 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 29 Mar 2021 12:23:59 +0200 Subject: [PATCH 16/21] [fix] Upgrade process --- scripts/upgrade | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scripts/upgrade b/scripts/upgrade index b160e27..e5a36f4 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -120,11 +120,6 @@ systemctl start $app.timer #================================================= # GENERIC FINALIZATION #================================================= -# SECURE FILES AND DIRECTORIES -#================================================= - -# Set permissions on app files -chown -R root: $final_path #================================================= # INTEGRATE SERVICE IN YUNOHOST From 0788666a335a24950ec889af41debb4fd120f04f Mon Sep 17 00:00:00 2001 From: ljf Date: Sat, 3 Apr 2021 20:17:44 +0200 Subject: [PATCH 17/21] [fix] Backup with borg script was incorrect --- check_process | 1 + conf/backup-with-borg | 62 ++++++++++++++++++++++++------------------- conf/backup_method | 10 ++++--- conf/sudoer | 2 +- manifest.json | 10 ++++--- scripts/install | 6 +++++ scripts/restore | 7 +++++ scripts/upgrade | 6 +++++ 8 files changed, 68 insertions(+), 36 deletions(-) diff --git a/check_process b/check_process index 16562cf..9f5f81b 100644 --- a/check_process +++ b/check_process @@ -14,6 +14,7 @@ setup_private=0 setup_public=0 upgrade=1 + upgrade=1 from_commit=c1524dd8e37cc671c01f5b5363901dd6a01e6fc1 backup_restore=1 multi_instance=1 port_already_use=0 diff --git a/conf/backup-with-borg b/conf/backup-with-borg index e202f3a..f153033 100644 --- a/conf/backup-with-borg +++ b/conf/backup-with-borg @@ -9,14 +9,14 @@ current_date=$(date +"%y%m%d_%H%M") log_file="/var/log/${borg_id}/${current_date}.log" err_file="/var/log/${borg_id}/${current_date}.err" mkdir -p "/var/log/${borg_id}" -if [[ -z "$borg_id" ]] +if [ -z "$borg_id" ] then echo "This script expects a borg app id as first argument" >&2 exit 1 fi filter_hooks() { - ls /usr/share/yunohost/hooks/backup/ /etc/yunohost/hooks.d/backup/ | grep "\-$1_" | cut -d"-" -f2 | uniq + sudo ls /usr/share/yunohost/hooks/backup/ /etc/yunohost/hooks.d/backup/ | grep "\-$1_" | cut -d"-" -f2 | uniq 2>> $err_file } fail_if_partially_failed() { @@ -24,52 +24,58 @@ fail_if_partially_failed() { } # Backup system part conf -conf=$(yunohost app setting ${borg_id} conf) -if [ $conf -eq 1 ] +conf=$(sudo yunohost app setting ${borg_id} conf) +if [[ "$conf" = "1" ]] then - if ! yunohost backup create -n auto_conf --method ${borg_id}_app --system $(filter_hooks conf) 2> $err_file > $log_file ; then - errors="$errors\nconf: Error" + if ! sudo yunohost backup create -n auto_conf --method ${borg_id}_app --system $(filter_hooks conf) 2>> $err_file >> $log_file ; then + errors+="\nThe backup miserably failed to backup system configurations." fi fi # Backup system data -data=$(yunohost app setting $app data) -if [ $data -eq 1 ] +data=$(sudo yunohost app setting ${borg_id} data) +if [[ "$data" = "1" ]] then - if ! yunohost backup create -n auto_data --method ${borg_id}_app --system $(filter_hooks data) 2> $err_file > $log_file ; then - errors="$errors\ndata: Error" + if ! sudo yunohost backup create -n auto_data --method ${borg_id}_app --system $(filter_hooks data) 2>> $err_file >> $log_file ; then + errors+="\nThe backup miserably failed to backup system data." fi fi # Backup all apps independently -apps=$(yunohost app setting ${borg_id} apps) -for application in $(ls /etc/yunohost/apps/*/scripts/backup | cut -d / -f 5); do - backup_app=false - if [[ "$apps" = "all" ]]; then - backup_app=true - else - for selected_app in $(echo $apps | tr "," " ");do - if [[ "$selected_app" == "$application" ]]; then - backup_app=true - break - fi - done +apps=$(sudo yunohost app setting ${borg_id} apps | tr -d ' ') +for application in $(sudo ls /etc/yunohost/apps/); do + + if ( [[ "$apps" =~ ^exclude: ]] && grep -wq "$application" <<< "$apps" ) || + ( [[ "$apps" != "all" ]] && [[ ! "$apps" =~ ^exclude: ]] && ! grep -wq "$application" <<< "$apps" ); + then + continue fi - if [ "$backup_app" == "true" ];then - if ! yunohost backup create -n auto_$application --method ${borg_id}_app --apps $application 2> $err_file > $log_file ; then - errors="$errors\$application: Error" - fi + + if sudo test ! -f /etc/yunohost/apps/$application/scripts/backup ; then + errors+="\nWarning: The application $application has no backup script. This app won't be backuped." + continue + fi + + if ! sudo yunohost backup create -n auto_$application --method ${borg_id}_app --apps $application 2>> $err_file >> $log_file ; then + errors+="\nThe backup miserably failed to backup $application application." fi done + +#========================================================= +# SEND MAIL TO NOTIFY SUCCED OR FAILED OPERATIONS +#========================================================= + partial_errors="$(cat $log_file | grep -E "Error|Skipped")" if [ ! -z "$partial_errors" ]; then - errors="$errors\n$partial_errors" + errors+="\nSome backup partially failed:\n$partial_errors" fi # Send mail on backup (partially) failed domain=$(hostname) if [ ! -z "$errors" ]; then - cat $errors | mail -s "[borg] Backup failed from $domain onto $repo" root + cat <(echo -e "$errors\n\n\n") "$log_file" "$err_file" | mail -s "[borg] Backup failed from $domain onto $repo" root + exit 1 else cat $log_file | mail -s "[borg] Backup succeed from $domain onto $repo" root + exit 0 fi diff --git a/conf/backup_method b/conf/backup_method index 4cf3c30..0b3c1ed 100644 --- a/conf/backup_method +++ b/conf/backup_method @@ -1,13 +1,15 @@ #!/bin/bash set -e +app="${0#"./05-"}" +app="${app%"_app"}" -BORG_PASSPHRASE='__PASSPHRASE__' -repo="__REPOSITORY__" #$4 +BORG_PASSPHRASE="$(yunohost app setting $app passphrase)" +repo="$(yunohost app setting $app repository)" #$4 if ssh-keygen -F "__SERVER__" >/dev/null ; then - BORG_RSH="ssh -i /root/.ssh/id___APP___ed25519 -oStrictHostKeyChecking=yes " + BORG_RSH="ssh -i /root/.ssh/id_${app}_ed25519 -oStrictHostKeyChecking=yes " else - BORG_RSH="ssh -i /root/.ssh/id___APP___ed25519 -oStrictHostKeyChecking=no " + BORG_RSH="ssh -i /root/.ssh/id_${app}_ed25519 -oStrictHostKeyChecking=no " fi do_need_mount() { diff --git a/conf/sudoer b/conf/sudoer index 7945669..6a11357 100644 --- a/conf/sudoer +++ b/conf/sudoer @@ -1 +1 @@ -__APP__ ALL=(root) /usr/local/bin/backup-with-__APP__, /usr/local/bin/borg +__APP__ ALL=(root) NOPASSWD: /usr/local/bin/backup-with-__APP__, /usr/local/bin/borg, /usr/bin/yunohost backup create *, /usr/bin/yunohost app setting *, /bin/ls, /usr/bin/test diff --git a/manifest.json b/manifest.json index 2e04833..5744207 100644 --- a/manifest.json +++ b/manifest.json @@ -6,7 +6,7 @@ "en": "Backup your server on a host server using Borg.", "fr": "Sauvegardez votre serveur sur un serveur distant avec Borg." }, - "version": "1.1.16~ynh1", + "version": "1.1.16~ynh16", "url": "https://borgbackup.readthedocs.io", "license": "BSD-3-Clause", "maintainer": { @@ -67,9 +67,13 @@ "name": "apps", "type": "string", "ask": { - "en": "Which apps should Borg backup (list separated by comma or 'all')?", - "fr": "Quelles applications doivent être sauvegardées par Borg (liste séparée par virgule ou 'all' ?" + "en": "Which apps should Borg backup ?", + "fr": "Quelles applications doivent être sauvegardées par Borg ?" }, + "help":{ + "en": "App list separated by comma. You can write 'all' to select all apps, even those installed after this borg app. You can also select all apps but some apps by writing 'exclude:' following by an app list separated by comma.", + "fr": "Liste d'applications séparées par des virgules. Vous pouvez écrire 'all' pour sélectionner toutes les apps, même celles installées après cette application borg. Vous pouvez aussi sélectionner toutes les apps sauf certaines en écrivant 'exclude:' suivi d'une liste d'applications séparées par des virgules." + }, "default": "all" }, { diff --git a/scripts/install b/scripts/install index 2be84fd..3253316 100755 --- a/scripts/install +++ b/scripts/install @@ -60,8 +60,12 @@ ynh_system_user_create --username=$app # ACTIVATE BACKUP METHODS #================================================= +mkdir -p /etc/yunohost/hooks.d/backup mkdir -p /etc/yunohost/hooks.d/backup_method mkdir -p /usr/share/yunohost/backup_method +mkdir -p /var/log/${app} +chown -R $app:$app /var/log/${app} +chmod u+w /var/log/${app} #================================================= # SETUP THE BACKUP METHOD @@ -72,6 +76,8 @@ chmod go=--- "/etc/yunohost/hooks.d/backup_method/05-${app}_app" ynh_add_config --template="backup-with-borg" --destination="/usr/local/bin/backup-with-$app" chmod u+x "/usr/local/bin/backup-with-$app" +chown $app:$app "/usr/local/bin/backup-with-$app" + ynh_add_config --template="sudoer" --destination="/etc/sudoers.d/$app" if [ ! -z "$server" ]; then diff --git a/scripts/restore b/scripts/restore index f05ed7c..6b396b2 100755 --- a/scripts/restore +++ b/scripts/restore @@ -37,12 +37,19 @@ install_borg_with_pip mkdir -p /etc/yunohost/hooks.d/backup_method mkdir -p /usr/share/yunohost/backup_method +mkdir -p /etc/yunohost/hooks.d/backup +mkdir -p /var/log/${app} +chown -R $app:$app /var/log/${app} +chmod u+w /var/log/${app} #================================================= # RESTORE FILES #================================================= ynh_restore +chmod go=--- "/etc/yunohost/hooks.d/backup_method/05-${app}_app" +chmod u+x "/usr/local/bin/backup-with-$app" +chown $app:$app "/usr/local/bin/backup-with-$app" #================================================= # ADVERTISE SERVICE IN ADMIN PANEL diff --git a/scripts/upgrade b/scripts/upgrade index e5a36f4..37c8520 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -99,6 +99,7 @@ chmod go=--- "/etc/yunohost/hooks.d/backup_method/05-${app}_app" ynh_add_config --template="backup-with-borg" --destination="/usr/local/bin/backup-with-$app" chmod u+x "/usr/local/bin/backup-with-$app" +chown $app:$app "/usr/local/bin/backup-with-$app" ynh_add_config --template="sudoer" --destination="/etc/sudoers.d/$app" @@ -117,6 +118,11 @@ ynh_add_config --template="systemd.timer" --destination="/etc/systemd/system/$ap systemctl enable $app.timer --quiet systemctl start $app.timer +mkdir -p /etc/yunohost/hooks.d/backup +mkdir -p /var/log/${app} +chown -R $app:$app /var/log/${app} +chmod u+w /var/log/${app} + #================================================= # GENERIC FINALIZATION #================================================= From be59eb64820946c7b607b72808657579f0edc0dd Mon Sep 17 00:00:00 2001 From: ljf Date: Sat, 3 Apr 2021 20:46:04 +0200 Subject: [PATCH 18/21] [fix] Restore users --- scripts/restore | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/restore b/scripts/restore index 6b396b2..54eeb83 100755 --- a/scripts/restore +++ b/scripts/restore @@ -31,6 +31,14 @@ ynh_script_progression --message="Reinstalling dependencies..." ynh_install_app_dependencies $pkg_dependencies install_borg_with_pip +#================================================= +# CREATE DEDICATED USER +#================================================= +ynh_script_progression --message="Configuring system user..." --time --weight=1 + +# Create a system user +ynh_system_user_create --username=$app + #================================================= # ACTIVATE BACKUP METHODS #================================================= From 38f7d4d95bf5bbd9170f74e460fa5118839be05b Mon Sep 17 00:00:00 2001 From: ljf Date: Sat, 3 Apr 2021 22:34:55 +0200 Subject: [PATCH 19/21] [fix] Upgrade system user missing --- scripts/upgrade | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/upgrade b/scripts/upgrade index 37c8520..b6dce37 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -81,6 +81,15 @@ ynh_script_progression --message="Upgrading dependencies..." --weight=1 ynh_install_app_dependencies $pkg_dependencies +#================================================= +# CREATE DEDICATED USER +#================================================= +ynh_script_progression --message="Configuring system user..." --time --weight=1 + +# Create a system user +ynh_system_user_create --username=$app + + #================================================= # SPECIFIC UPGRADE #================================================= From 73c2e163120627607d05668c21becc01d1bdb2a0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 4 Apr 2021 01:48:13 +0200 Subject: [PATCH 20/21] No need for --time --- scripts/install | 2 +- scripts/restore | 2 +- scripts/upgrade | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/install b/scripts/install index 3253316..72ad751 100755 --- a/scripts/install +++ b/scripts/install @@ -49,7 +49,7 @@ install_borg_with_pip #================================================= # CREATE DEDICATED USER #================================================= -ynh_script_progression --message="Configuring system user..." --time --weight=1 +ynh_script_progression --message="Configuring system user..." --weight=1 # Create a system user ynh_system_user_create --username=$app diff --git a/scripts/restore b/scripts/restore index 54eeb83..6a70a59 100755 --- a/scripts/restore +++ b/scripts/restore @@ -34,7 +34,7 @@ install_borg_with_pip #================================================= # CREATE DEDICATED USER #================================================= -ynh_script_progression --message="Configuring system user..." --time --weight=1 +ynh_script_progression --message="Configuring system user..." --weight=1 # Create a system user ynh_system_user_create --username=$app diff --git a/scripts/upgrade b/scripts/upgrade index b6dce37..8e0c8a1 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -84,7 +84,7 @@ ynh_install_app_dependencies $pkg_dependencies #================================================= # CREATE DEDICATED USER #================================================= -ynh_script_progression --message="Configuring system user..." --time --weight=1 +ynh_script_progression --message="Configuring system user..." --weight=1 # Create a system user ynh_system_user_create --username=$app From 1dbf95b1af2a8c88bffec87ef1d4fd0c63312ade Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 4 Apr 2021 03:57:14 +0200 Subject: [PATCH 21/21] Fix upgrade test from previous commit --- check_process | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_process b/check_process index 9f5f81b..5fa3e9b 100644 --- a/check_process +++ b/check_process @@ -25,4 +25,4 @@ Notificatio ;;; Upgrade options ; commit=c1524dd8e37cc671c01f5b5363901dd6a01e6fc1 name=Merge pull request #63 from YunoHost-Apps/testing - manifest_arg=server=domain.tld:22&ssh_user=sam&passphrase=APassphrase&conf=1&data=1&apps=all&on_calendar=Daily + manifest_arg=server=domain.tld:22&ssh_user=package_checker&passphrase=APassphrase&conf=1&data=1&apps=all&on_calendar=Daily