simple urlwatch integration

that sends an email to the admin of flohmarkt or the admin group of yunohost
when the user database changes

conf/urlwatch.cron

@@ -0,0 +1,9 @@
+#!/bin/sh
+# run urlwatch for flohmarkt
+PATH=/usr/sbin:/sbin:/usr/bin:/bin
+
+sleep $((RANDOM%111)) 
+sudo -u __APP__ urlwatch \
+    --config /var/www/__APP__/urlwatch/config.yaml \
+    --urls=/var/www/__APP__/urlwatch/urls.yaml \
+    --cache /var/www/__APP__/urlwatch/cache.file

conf/urlwatch_config.yaml

@@ -0,0 +1,90 @@
+display:
+  error: true
+  new: true
+  unchanged: false
+job_defaults:
+  all: {}
+  browser: {}
+  shell: {}
+  url: {}
+report:
+  discord:
+    embed: false
+    enabled: false
+    max_message_length: 2000
+    subject: '{count} changes: {jobs}'
+    webhook_url: ''
+  email:
+    enabled: true
+    from: '__MAIL_USER__@__MAIL_DOMAIN__'
+    html: false
+    method: smtp
+    sendmail:
+      path: sendmail
+    smtp:
+      host: localhost
+      auth: true
+      insecure_password: '__MAIL_PWD__'
+      port: 25
+      starttls: true
+      user: '__APP__'
+    subject: 'New or deleted flohmarkt users'
+    to: '__ADMIN_MAIL__'
+  html:
+    diff: unified
+  ifttt:
+    enabled: false
+    event: ''
+    key: ''
+  mailgun:
+    api_key: ''
+    domain: ''
+    enabled: false
+    from_mail: ''
+    from_name: ''
+    region: us
+    subject: '{count} changes: {jobs}'
+    to: ''
+  markdown:
+    details: true
+    footer: true
+    minimal: false
+  matrix:
+    access_token: ''
+    enabled: false
+    homeserver: ''
+    room_id: ''
+  mattermost:
+    enabled: false
+    max_message_length: 40000
+    webhook_url: ''
+  pushbullet:
+    api_key: ''
+    enabled: false
+  pushover:
+    app: ''
+    device: null
+    enabled: false
+    priority: normal
+    sound: spacealarm
+    user: ''
+  slack:
+    enabled: false
+    max_message_length: 40000
+    webhook_url: ''
+  stdout:
+    color: true
+    enabled: true
+  telegram:
+    bot_token: ''
+    chat_id: ''
+    enabled: false
+  text:
+    details: true
+    footer: true
+    line_length: 75
+    minimal: false
+  xmpp:
+    enabled: false
+    recipient: ''
+    sender: ''

conf/urlwatch_urls.yaml

@@ -0,0 +1,11 @@
+name: "watch for changes in user database"
+kind: "url"
+url: "http://__APP__:__PASSWORD_COUCHDB_FLOHMARKT__@127.0.0.1:5984/flohmarkt/_find"
+method: "POST"
+data: '{"selector": { "role": "User", "name": { "$ne": "instance" }}, 
+  "fields": ["name","email","active"], "sort": [ "name" ]}'
+headers: 
+  Content-Type: application/json
+  Accept: application/json
+filter:
+  - shellpipe: 'jq .docs'

manifest.toml

@@ -97,6 +97,12 @@ ram.runtime = "100M"
     ask.en = "name of your flohmarkt instance"
     ask.de = "Name des Flohmarkts"
 
+    [install.admin_mail]
+    ask.en = "email address of this flohmarkts admin"
+    help.en = "you'll get notified about events like changes in the user database of flohmarkt"
+    ask.de = "E-Mail Adresse des Administrators dieses flohmarkts"
+    help.de = "an diese Adresse werden Informationen zu z.B. Änderungen an der Benutzerdatenbank geschickt"
+
 [resources]
     # See the packaging documentation for the full set
     # of explanation regarding the behavior and properties for each of those
@@ -155,7 +161,8 @@ ram.runtime = "100M"
     [resources.apt]
     # python dependencies shall be installed in a venv using pip.
 	# moreutils is needed for `ts` used in systemd.service
-    packages = "python3-pip python3-full curl apt-transport-https gnupg moreutils curl"
+    # urlwatch for email notifications
+    packages = "python3-pip python3-full curl apt-transport-https gnupg moreutils curl urlwatch"
     # repo for couchdb - doesn't work, yet
     # extras.couchdb.repo = "deb https://apache.jfrog.io/artifactory/couchdb-deb/ __YNH_DEBIAN_VERSION__ main"
     # extras.couchdb.key = "https://couchdb.apache.org/repo/keys.asc"

scripts/_common.sh

@@ -373,7 +373,7 @@ flohmarkt_ynh_local_curl() {
 # | arg: -m --method:     request method to use: POST (default), PUT, GET, DELETE
 # | arg: -H --header:     add a header to the request (can be used multiple times)
 # | arg: -d --data:       data to be PUT or POSTed. Can be used multiple times.
-# | arg: -s --seperator:  seperator used to concatenate POST/PUT --date or key=value ('none'=no seperator)
+# | arg: -s --seperator:  seperator used to concatenate POST/PUT --data or key=value ('none'=no seperator)
 # | arg:                  (default for POST: '&', default for PUT: ' ')
 # | arg: -u --user:       login username (requires --password)
 # | arg: -p --password:   login password
@@ -642,9 +642,10 @@ flohmarkt_ynh_dump_couchdb() {
 }
 
 flohmarkt_ynh_import_couchdb() {
-  ls -l ../settings/scripts/couchdb-dump/couchdb-dump.sh ${YNH_CWD}/${app}.json
   ../settings/scripts/couchdb-dump/couchdb-dump.sh -r -c -H 127.0.0.1 -d "${app}" \
     -q -u admin -p "${password_couchdb_admin}" -f "${YNH_CWD}/${app}.json"
+  # TODO: This failed silently during tests when there were leftovers from a former
+  # install. Check couchdb-dump.sh restore for error codes and handling.
 }
 
 flohmarkt_ynh_delete_couchdb_user() {
@@ -725,6 +726,15 @@ flohmarkt_ynh_rename_couchdb() {
   flohmarkt_ynh_delete_couchdb_db "$old_name"
 }
 
+flohmarkt_ynh_backup_old_couchdb() {
+  flohmarkt_couchdb_rename_to="${app}_$(date '+%Y-%m-%d_%H-%M-%S_%N')"
+  if flohmarkt_ynh_rename_couchdb "${app}" "${flohmarkt_couchdb_rename_to}"; then
+    ynh_print_warn --message="renamed existing database ${app} to ${flohmarkt_couchdb_rename_to}"
+  else
+    ynh_die --message="could not rename existing couchdb database and cannot overwrite it"
+  fi
+}
+
 flohmarkt_ynh_restore_couchdb() {
   flohmarkt_ynh_import_couchdb
   flohmarkt_ynh_create_couchdb_user
@@ -748,6 +758,24 @@ flohmarkt_ynh_venv_requirements() {
   )
 }
 
+flohmarkt_ynh_urlwatch_cron() {
+    mkdir -m 770 -p "${flohmarkt_install}/urlwatch"
+    chown root:${app} "${flohmarkt_install}/urlwatch"
+    ynh_add_config --template="../conf/urlwatch_config.yaml" \
+        --destination="${flohmarkt_install}/urlwatch/config.yaml"
+    ynh_add_config --template="../conf/urlwatch_urls.yaml" \
+        --destination="${flohmarkt_install}/urlwatch/urls.yaml"
+    ynh_add_config --template="../conf/urlwatch.cron" \
+        --destination="/etc/cron.hourly/${flohmarkt_filename}"
+    chown root:root "/etc/cron.hourly/${flohmarkt_filename}"
+    chmod 755 "/etc/cron.hourly/${flohmarkt_filename}"
+    #  run it once to initialize
+    sudo -u ${app} urlwatch \
+    --config /var/www/${app}/urlwatch/config.yaml \
+    --urls=/var/www/${app}/urlwatch/urls.yaml \
+    --cache /var/www/${app}/urlwatch/cache.file
+}
+
 # move files and directories to their new places
 flohmarkt_ynh_upgrade_path_ynh5() {
   # flohmarkt and couchdb are already stopped in upgrade script

scripts/backup

@@ -11,6 +11,7 @@ ynh_backup --src_path="/etc/fail2ban/filter.d/$app.conf"
 ynh_backup --src_path="/etc/logrotate.d/$app"
 ynh_backup --src_path="/opt/couchdb/etc/local.d/05-flohmarkt.ini"
 ynh_backup --src_path="/etc/systemd/system/$flohmarkt_filename.service"
+ynh_backup --src_path="/etc/cron.hourly/$flohmarkt_filename"
 ynh_backup --src_path="${flohmarkt_log_dir}"
 
 # for the following backups we'll want to stop flohmarkt and couchdb

scripts/install

@@ -121,4 +121,8 @@ ls -l $flohmarkt_logfile /bin/bash /usr/bin/bash || true
 ynh_script_progression --message="Starting flohmarkt..." --weight=10
 flohmarkt_ynh_start_service
 
+# integrate urlwatch
+ynh_script_progression --message="Configure urlwatch and its cron job..." --weight=1
+flohmarkt_ynh_urlwatch_cron
+
 ynh_script_progression --message="Installation of $id completed" --last

scripts/restore

@@ -23,14 +23,10 @@ else
   ynh_script_progression --message="CouchDB configuration file already exists." --weight=1
 fi
 
-# check if couchdb already exists and if so rename it to make room for the restore
+# if exists rename existing couchdb
+ynh_script_progression --message="Check if old database exists..." --weight=1
 if flohmarkt_ynh_exists_couchdb_db; then 
-  flohmarkt_couchdb_rename_to="${app}_$(date '+%Y-%m-%d_%H-%M-%S_%N')"
+  flohmarkt_ynh_backup_old_couchdb
-  if flohmarkt_ynh_rename_couchdb "${app}" "${flohmarkt_couchdb_rename_to}"; then
-    ynh_print_warn --message="renamed existing database ${app} to ${flohmarkt_couchdb_rename_to}"
-  else
-    ynh_die --message="could not rename existing couchdb database and cannot overwrite it"
-  fi
 fi
 
 # restore couchdb from json
@@ -88,4 +84,8 @@ ynh_systemd_action --service_name=nginx --action=reload
 ynh_script_progression --message="Creating symlinks..." --weight=1
 flohmarkt_ynh_create_symlinks
 
+# restore cronjob file for urlwatch
+ynh_script_progression --message="Restoring cronjob for urlwatch..." --weight=1
+ynh_restore_file --origin_path="/etc/cron.hourly/$flohmarkt_filename"
+
 ynh_script_progression --message="Restoration completed for $app" --last

scripts/upgrade

@@ -88,4 +88,26 @@ ynh_add_fail2ban_config --logpath="$flohmarkt_logfile" --failregex='INFO: +<HOST
 ynh_script_progression --message="Starting flohmarkt..." --weight=3
 flohmarkt_ynh_start_service
 
+# check if admin_mail is already set and if not use a default
+if ! [[ -v admin_mail ]] || [[ -z "${admin_mail}" ]]; then
+  ynh_print_warn --message="new setting admin_mail - trying to set a default"
+  # try to read admin email address from flohmarkt installation
+  admin_mail=$( flohmarkt_ynh_local_curl -n -m POST -u admin -p "$password_couchdb_admin" \
+    -H "Accept: application/json" -H "Content-Type: application/json" -s none \
+    -d '{"selector": { "role": "User", "admin": true }, "fields": ["email"]}' \
+    --location 'http://127.0.0.1:5984/flohmarkt/_find' \
+    | jq -r '.docs[0] .email'
+  )
+  if [[ -z "${admin_mail}" ]] || [[ "${admin_mail}" == 'null' ]]; then
+    # use yunohost admin group
+    admin_mail="admin@$(yunohost domain main-domain | cut -d' ' -f2)"
+  fi
+  ynh_print_warn --message="using ${admin_mail} for new setting admin_mail"
+  ynh_app_setting_set --app=$app --key=admin_mail --value="${admin_mail}"
+fi
+
+# upgrade urlwatch configuration
+ynh_script_progression --message="Upgrade urlwatch configuration..." --weight=1
+flohmarkt_ynh_urlwatch_cron
+
 ynh_script_progression --message="Upgrade of $app completed" --last