Merge branch 'testing'

README.md

@@ -22,7 +22,7 @@ Firstly set up this app on the server A you want to backup:
 $ yunohost app install https://github.com/YunoHost-Apps/restic_ynh
 Indicate the server where you want put your backups: serverb.domain.tld
 sftp port of your server (default: 22): 2222
-The directory where you want to backup repositories to be created in (default: ./): ./servera.domain.tld
+The directory where you want your backup repositories to be created in (default: ./): ./servera.domain.tld
 Indicate the ssh user to use to connect on this server: servera
 You are now about to define a new user password. The password should be at least 8 characters - though it is good practice to use longer password (i.e. a passphrase) and/or to use various kind of characters (uppercase, lowercase, digits and special characters).
 Indicate a strong passphrase, that you will keep preciously if you want to be able to use your backups:
@@ -30,7 +30,9 @@ Would you like to backup your YunoHost configuration ? [yes | no] (default: yes)
 Would you like to backup mails and user home directory ? [yes | no] (default: yes):
 Which apps would you backup (list separated by comma or 'all') ? (default: all): gitlab,blogotext,sogo
 Allow backup method to temporarily use more space? [yes | no] (default: yes):
-Indicate the backup frequency (see systemd OnCalendar format) (default: Daily):
+Indicate the backup frequency (see systemd OnCalendar format) (default: Daily): *-*-* 0:05
+Indicate the backup check frequency (see systemd OnCalendar format) (default: *-*-8,15,22 3:15:00):
+Indicate the complete backup check frequency (see systemd OnCalendar format) (default: *-*-1 1:15:00):
 ```
 
 You can schedule your backup by choosing an other frequency. Some example:
@@ -49,11 +51,12 @@ Sat *-*-1..7 18:00:00 : The first saturday of every month at 18:00
 
 5,17:00 : Every day at 5 AM and at 5 PM
 
+See here for more info : https://wiki.archlinux.org/index.php/Systemd/Timers#Realtime_timer
+
 After each invocation an e-mail will be sent to root@yourdomain.tld with the execution log.
 
-NOTE: After each backup, the repository integrity is checked
-
-See here for more info : https://wiki.archlinux.org/index.php/Systemd/Timers#Realtime_timer
+Restic can check backups consistency and verify the actual backed up data has not been modified.
+If you use the default values for the backup checks frequencies, a full check will be made on the first day of each month and a simple check will be made on each one of the three remaining weeks of the month.
 
 At the end of the installation, the app displays the public_key and the user to give to the person who has access to the server B.
 
@@ -67,7 +70,7 @@ cat << EOPKEY >> ~/.ssh/authorized_keys
 <paste here the privakey displayed at the end of installation>
 EOPKEY
 ```
-If you don't find the mail and you don't see the message in the log bar you can found the public_key with this command:
+If you don't find the mail and you don't see the message in the log bar you can find the public_key with this command:
 ```
 cat /root/.ssh/id_restic_ed25519.pub
 ```
@@ -93,17 +96,25 @@ At this step your backup should schedule.
 
 If you want to be sure, you can test it by running on server A:
 ```
-service restic start
+systemctl start restic.service
 ```
 
-Next you can check by running on server A
+Next you can verify the backup contents by running on server A
 ```
 restic -r sftp:serverb.domain.tld:servera.domain.tld/auto_conf snapshots
 ```
 
 Replace `auto_conf` with `auto_<app>` if you did not choose to backup configuration but only applications.
 
-YOU SHOULD CHECK REGULARLY THAT YOUR BACKUP ARE STILL WORKING.
+If you want to check the backups consistency:
+```
+systemctl start restic_check.service
+```
+
+If you want to make a complete check of the backups - keep in mind that this reads all the backed up data, it can take some time depending on your target server upload speed (more on this topic in [the restic documentation](https://restic.readthedocs.io/en/latest/045_working_with_repos.html#checking-integrity-and-consistency):
+```
+systemctl start restic_check_read_data.service
+```
 
 ## Edit the apps list to backup
 

check_process

@@ -10,6 +10,8 @@ data=1
 app="all"
 allow_extra_space_use=1
 on_calendar="Daily"
+check_on_calendar="*-*-8,15,22 3:15"
+check_read_data_on_calendar="*-*-1 3:15"
 ; Checks
 pkg_linter=1
 setup_sub_dir=0

conf/backup_method.j2

@@ -28,22 +28,19 @@ do_backup() {
     description="$5"
     export RESTIC_PASSWORD
     export RESTIC_REPOSITORY=${RESTIC_REPOSITORY_BASE}/$name
-    LOGFILE=/var/log/backup_restic.log
-    ERRFILE=/var/log/backup_restic.err
+    LOGFILE=/var/log/restic_backup.log
+    ERRFILE=/var/log/restic_backup.err
     current_date=$(date +"%d_%m_%y_%H:%M")
     pushd $work_dir
     $RESTIC_COMMAND backup ./ >> $LOGFILE 2>> $ERRFILE
     backup_return_code="$?"
-    $RESTIC_COMMAND check >> $LOGFILE 2>> $ERRFILE
-    check_return_code="$?"
     popd
 
     # On ne nettoie que si la sauvegarde s'est bien passee
-    if [ "$backup_return_code" -eq "0" ] && [ "$check_return_code" -eq 0 ];then
+    if [ "$backup_return_code" -eq "0" ];then
         $RESTIC_COMMAND forget --keep-daily 7 --keep-weekly 8 --keep-monthly 12 >> $LOGFILE 2>> $ERRFILE
     else
-        [ "$backup_return_code" -ne 0 ] && echo "Something went wrong during backup" >> $ERRFILE
-        [ "$check_return_code" -ne 0 ] && echo "Repository check did not return 0" >> $ERRFILE
+        echo "Something went wrong during backup" >> $ERRFILE
         exit 1
     fi
 }

conf/check-restic.j2

@@ -0,0 +1,46 @@
+#!/bin/bash
+LOCK_FILE=/tmp/{{ app }}_check.lock
+EXIT_PROPERLY() {
+  echo -e "\e[91m \e[1m"    # Shell in light red bold
+  echo -e "!!\n  Caught an interruption signal,  removing lock file...\n!!"
+  echo -e "\e[22m"    # Remove bold
+
+  rm $LOCK_FILE
+  exit 1
+}
+trap EXIT_PROPERLY 1 2 3 6 15
+if [ -f "$LOCK_FILE" ];then
+  echo "Check already launched by process $(grep '.*' $LOCK_FILE), canceling this one" >&2
+  exit 1
+fi
+echo $$ > "$LOCK_FILE"
+
+CHECK_READ_DATA=${1:-0}
+
+# Check system part conf
+conf=$(yunohost app setting {{ app }} conf)
+if [ $conf -eq 1 ];then
+  {{final_path}}/check_method auto_conf ${CHECK_READ_DATA}
+fi
+
+# Check system data
+data=$(yunohost app setting {{ app }} data)
+if [ $data -eq 1 ];then
+  {{final_path}}/check_method auto_data ${CHECK_READ_DATA}
+fi
+
+# Check all apps independently
+apps=$(yunohost app setting {{ app }} apps)
+for app in $(yunohost app list --installed -b | grep id: | cut -d: -f2); do
+  check_app=false
+  for selected_app in $(echo $apps | tr "," " ");do
+    if [[ "$selected_app" == "$app" ]] || [ "$apps" = "all" ]; then
+      check_app=true
+      break
+    fi
+  done
+  if [ "$check_app" == "true" ];then
+    {{final_path}}/check_method auto_${app} ${CHECK_READ_DATA}
+  fi
+done
+rm "$LOCK_FILE"

conf/check_method.j2

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -e
+
+RESTIC_PASSWORD="{{ passphrase }}"
+RESTIC_REPOSITORY_BASE=sftp:{{ server }}:{{ backup_path }}
+RESTIC_COMMAND=/usr/local/bin/restic
+
+do_check() {
+
+    local name="$1"
+    local check_read_data="$2"
+    export RESTIC_PASSWORD
+    export RESTIC_REPOSITORY=${RESTIC_REPOSITORY_BASE}/$name
+    LOGFILE=/var/log/restic_check.log
+    ERRFILE=/var/log/restic_check.err
+    current_date=$(date +"%d_%m_%y_%H:%M")
+    echo -e "\n==============\n${current_date}\n==============\n" | tee -a ${LOGFILE} | tee -a ${ERRFILE}
+    if [ "$check_read_data" -eq "1" ];then
+      $RESTIC_COMMAND check --read-data >> $LOGFILE 2>> $ERRFILE
+    else
+      $RESTIC_COMMAND check >> $LOGFILE 2>> $ERRFILE
+    fi
+    check_return_code="$?"
+    return "${check_return_code}"
+}
+
+name=$1
+check_read_data=${2:-0}
+
+do_check "${name}" "${check_read_data}"
+
+exit 0

conf/systemd_check.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Check backup __APP__
+After=network.target
+
+[Service]
+Type=oneshot
+ExecStart=__FINALPATH__/check-__APP__
+ExecStartPost=/bin/bash -c 'echo -e "Subject: YunoHost Restic check log on $(hostname)\n$(/bin/journalctl _SYSTEMD_INVOCATION_ID=`systemctl show -p InvocationID --value __APP___check.service`)" | /usr/sbin/sendmail root'
+User=root
+Group=root
+
+[Install]
+WantedBy=multi-user.target

conf/systemd_check.timer.j2

@@ -0,0 +1,8 @@
+[Unit]
+Description=Check {{ app }} backup regularly
+
+[Timer]
+OnCalendar={{ check_on_calendar }}
+
+[Install]
+WantedBy=timers.target

conf/systemd_check_read_data.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Complete check backup __APP__
+After=network.target
+
+[Service]
+Type=oneshot
+ExecStart=__FINALPATH__/check-__APP__ "1"
+ExecStartPost=/bin/bash -c 'echo -e "Subject: YunoHost Restic complete check log on $(hostname)\n$(/bin/journalctl _SYSTEMD_INVOCATION_ID=`systemctl show -p InvocationID --value __APP___check_read_data.service`)" | /usr/sbin/sendmail root'
+User=root
+Group=root
+
+[Install]
+WantedBy=multi-user.target

conf/systemd_check_read_data.timer.j2

@@ -0,0 +1,8 @@
+[Unit]
+Description=Complete check {{ app }} backup regularly
+
+[Timer]
+OnCalendar={{ check_read_data_on_calendar }}
+
+[Install]
+WantedBy=timers.target

manifest.json

@@ -6,7 +6,7 @@
     "en": "Backup your server with restic.",
     "fr": "Sauvegardez votre serveur avec restic."
   },
-  "version": "0.9.6~ynh1",
+  "version": "0.9.6~ynh2",
   "url": "https://restic.net/",
   "license": "BSD 2-Clause \"Simplified\" License",
   "maintainer": {
@@ -50,9 +50,9 @@
       },
       {
         "name": "backup_path",
-        "type": "path",
+        "type": "string",
         "ask": {
-          "en": "The directory where you want to backup repositories to be created in",
+          "en": "The directory where you want your backup repositories to be created in",
           "fr": "Le répertoire dans lequel les dépôts restic seront créés"
         },
         "help":{
@@ -128,6 +128,26 @@
         },
         "example": "Daily",
         "default": "Daily"
+      },
+      {
+        "name": "check_on_calendar",
+        "type": "string",
+        "ask": {
+          "en": "Indicate the backup check frequency (see systemd OnCalendar format)",
+          "fr": "Indiquez la fréquence de vérification de la sauvegarde (voir le format OnCalendar de systemd)"
+        },
+        "example": "Tue *-*-* 00:15:00",
+        "default": "*-*-8,15,22 3:15:00"
+      },
+      {
+        "name": "check_read_data_on_calendar",
+        "type": "string",
+        "ask": {
+          "en": "Indicate the complete backup check frequency (see systemd OnCalendar format)",
+          "fr": "Indiquez la fréquence de vérification complète de la sauvegarde (voir le format OnCalendar de systemd)"
+        },
+        "example": "Tue *-*-* 00:15:00",
+        "default": "*-*-1 1:15:00"
       }
     ]
   }

scripts/_common.sh

@@ -67,7 +67,7 @@ ynh_save_args () {
     if [ "$var" == "path_url" ]; then
       setting_var="path"
     fi
-    ynh_app_setting_set $app $setting_var ${!var}
+    ynh_app_setting_set $app $setting_var "${!var}"
   done
 }
 

scripts/install

@@ -20,14 +20,15 @@ ynh_abort_if_errors
 # RETRIEVE ARGUMENTS FROM THE MANIFEST
 #=================================================
 export app=$YNH_APP_INSTANCE_NAME
+export final_path="/opt/yunohost/${app}"
 
 # Retrieve arguments
-ynh_export server port ssh_user backup_path passphrase on_calendar conf data apps allow_extra_space_use
+ynh_export server port ssh_user backup_path passphrase on_calendar check_on_calendar check_read_data_on_calendar conf data apps allow_extra_space_use
 
 #=================================================
 # STORE SETTINGS FROM MANIFEST
 #=================================================
-ynh_save_args server port ssh_user backup_path passphrase on_calendar conf data apps allow_extra_space_use
+ynh_save_args server port ssh_user backup_path passphrase on_calendar check_on_calendar check_read_data_on_calendar conf data apps allow_extra_space_use
 
 #=================================================
 # INSTALL DEPENDENCIES
@@ -49,19 +50,31 @@ mkdir -p /usr/share/yunohost/backup_method
 #=================================================
 ynh_print_info --message="Setting up backup methods"
 ynh_configure backup_method "/etc/yunohost/hooks.d/backup_method/05-${app}_app"
+ynh_configure check_method "${final_path}/check_method"
 
 #=================================================
 # CONFIGURE CRON
 #=================================================
 ynh_print_info --message="Configuring cron"
-ynh_configure backup-with-restic "/usr/local/bin/backup-with-$app"
-ynh_configure backup-with-restic-answerbot "/usr/local/bin/backup-with-$app-answerbot"
-chmod u+x "/usr/local/bin/backup-with-$app"
-chmod u+x "/usr/local/bin/backup-with-$app-answerbot"
-ynh_add_systemd_config
-ynh_configure systemd.timer "/etc/systemd/system/$app.timer"
-systemctl enable $app.timer
-systemctl start $app.timer
+ynh_configure backup-with-restic "/usr/local/bin/backup-with-${app}"
+ynh_configure backup-with-restic-answerbot "/usr/local/bin/backup-with-${app}-answerbot"
+ynh_configure check-restic "${final_path}/check-${app}"
+chmod u+x "/usr/local/bin/backup-with-${app}"
+chmod u+x "/usr/local/bin/backup-with-${app}-answerbot"
+chmod u+x "${final_path}/check-${app}"
+chmod u+x "${final_path}/check_method"
+ynh_add_systemd_config --service=${app} --template=systemd.service
+ynh_add_systemd_config --service=${app}_check --template=systemd_check.service
+ynh_add_systemd_config --service=${app}_check_read_data --template=systemd_check_read_data.service
+ynh_configure systemd.timer "/etc/systemd/system/${app}.timer"
+ynh_configure systemd_check.timer "/etc/systemd/system/${app}_check.timer"
+ynh_configure systemd_check_read_data.timer "/etc/systemd/system/${app}_check_read_data.timer"
+systemctl enable ${app}.timer
+systemctl enable ${app}_check.timer
+systemctl enable ${app}_check_read_data.timer
+systemctl start ${app}.timer
+systemctl start ${app}_check.timer
+systemctl start ${app}_check_read_data.timer
 
 #=================================================
 # GENERATE SSH KEY

scripts/remove

@@ -23,12 +23,20 @@ ynh_remove_app_dependencies
 #=================================================
 # REMOVE FILES
 #=================================================
-systemctl stop $app.timer
-systemctl disable $app.timer
-ynh_remove_systemd_config
-ynh_secure_remove "/etc/systemd/system/$app.timer"
-ynh_secure_remove "/usr/local/bin/backup-with-$app"
+systemctl stop ${app}.timer
+systemctl disable ${app}.timer
+ynh_remove_systemd_config --service=${app}
+ynh_remove_systemd_config --service=${app}_check
+ynh_remove_systemd_config --service=${app}_check_read_data
+ynh_secure_remove "/etc/systemd/system/${app}.timer"
+ynh_secure_remove "/etc/systemd/system/${app}_check.timer"
+ynh_secure_remove "/etc/systemd/system/${app}_check_read_data.timer"
+ynh_secure_remove "/usr/local/bin/backup-with-${app}"
+ynh_secure_remove "/usr/local/bin/check-${app}"
+ynh_secure_remove "/usr/local/bin/check-read-data-${app}"
 ynh_secure_remove "/etc/yunohost/hooks.d/backup_method/05-${app}_app"
+ynh_secure_remove "/etc/yunohost/hooks.d/backup_method/05-${app}_check_app"
+ynh_secure_remove "/etc/yunohost/hooks.d/backup_method/05-${app}_check_read_data_app"
 
 #=================================================
 # REMOVE SSH CONFIG