Merge pull request #57 from YunoHost-Apps/testing

Multiple fixes

README.md

@@ -1,6 +1,7 @@
 # Funkwhale
 A modern, convivial and free music server on YunoHost
 
+[![Integration level](https://dash.yunohost.org/integration/funkwhale.svg)](https://dash.yunohost.org/appci/app/funkwhale)  
 [![Install Funkwhale with YunoHost](https://install-app.yunohost.org/install-with-yunohost.png)](https://install-app.yunohost.org/?app=funkwhale)
 
 Installation requires a dedicated domain for now. I hope subpath installation will be possible in the future.

check_process

@@ -16,8 +16,8 @@
 		setup_private=1
 		setup_public=1
 		upgrade=1
-		upgrade=1		from_commit=11c81c1f503691272df5002dad8278bd82f34535
 		upgrade=1		from_commit=80a5044d13a6782063fee2d7fb7a01832a6aa767
+		upgrade=1               from_commit=806bcb0f320d8e3e28711a9ae1d2b175b95c65ce
 		backup_restore=1
 		multi_instance=1
 		incorrect_path=0
@@ -40,9 +40,9 @@
 Email=jean-baptiste@holcroft.fr
 Notification=all
 ;;; Upgrade options
-	; commit=11c81c1f503691272df5002dad8278bd82f34535
-		name=Upgrade from 0.15~ynh2
-manifest_arg=domain=DOMAIN&path=/&admin=USER&is_public=1
 	; commit=80a5044d13a6782063fee2d7fb7a01832a6aa767
 		name=Upgrade from 0.16.3
+manifest_arg=domain=DOMAIN&path=/&admin=USER&is_public=1
+	; commit=806bcb0f320d8e3e28711a9ae1d2b175b95c65ce
+		name=Upgrade from 0.17.0~ynh2
 manifest_arg=domain=DOMAIN&path=/&admin=USER&is_public=1

conf/funkwhale-beat.service

@@ -8,7 +8,9 @@ User=__APP__
 Group=__APP__
 WorkingDirectory=__FINALPATH__/code/api
 EnvironmentFile=__FINALPATH__/code/config/.env
-ExecStart=__FINALPATH__/code/virtualenv/bin/celery -A funkwhale_api.taskapp beat -l INFO
+
+ExecStart=__FINALPATH__/code/virtualenv/bin/celery -A funkwhale_api.taskapp beat \
+	--loglevel INFO --logfile=/var/log/__APP__/beat.log
 
 NoNewPrivileges=true
 PrivateDevices=true
@@ -18,7 +20,7 @@ ProtectSystem=strict
 ProtectControlGroups=yes
 ProtectKernelModules=yes
 ReadOnlyPaths=__FINALPATH__/code/config/.env __FINALPATH__/code/
-ReadWritePaths=__FINALPATH__/media __FINALPATH__/import __FINALPATH__/code/api
+ReadWritePaths=__FINALPATH__/media __FINALPATH__/import __FINALPATH__/code/api /var/log/__APP__
 
 StandardOutput=syslog
 StandardError=syslog

conf/funkwhale-server.service

@@ -8,7 +8,9 @@ User=__APP__
 Group=__APP__
 WorkingDirectory=__FINALPATH__/code/api
 EnvironmentFile=__FINALPATH__/code/config/.env
-ExecStart=__FINALPATH__/code/virtualenv/bin/daphne -b ${FUNKWHALE_API_IP} -p ${FUNKWHALE_API_PORT} config.asgi:application --proxy-headers
+
+ExecStart=__FINALPATH__/code/virtualenv/bin/daphne -b ${FUNKWHALE_API_IP} -p ${FUNKWHALE_API_PORT} config.asgi:application --proxy-headers \
+	--verbosity 1 --access-log=/var/log/__APP__/server.log
 
 NoNewPrivileges=true
 PrivateDevices=true
@@ -18,7 +20,7 @@ ProtectSystem=strict
 ProtectControlGroups=yes
 ProtectKernelModules=yes
 ReadOnlyPaths=__FINALPATH__/code/config/.env __FINALPATH__/code/
-ReadWritePaths=__FINALPATH__/media __FINALPATH__/import
+ReadWritePaths=__FINALPATH__/media __FINALPATH__/import /var/log/__APP__
 
 StandardOutput=syslog
 StandardError=syslog

conf/funkwhale-worker.service

@@ -8,7 +8,9 @@ User=__APP__
 Group=__APP__
 WorkingDirectory=__FINALPATH__/code/api
 EnvironmentFile=__FINALPATH__/code/config/.env
-ExecStart=__FINALPATH__/code/virtualenv/bin/celery -A funkwhale_api.taskapp worker -l INFO
+
+ExecStart=__FINALPATH__/code/virtualenv/bin/celery -A funkwhale_api.taskapp worker -l INFO \
+	--loglevel INFO --logfile=/var/log/__APP__/worker.log
 
 NoNewPrivileges=true
 PrivateDevices=true
@@ -18,7 +20,7 @@ ProtectSystem=strict
 ProtectControlGroups=yes
 ProtectKernelModules=yes
 ReadOnlyPaths=__FINALPATH__/code/config/.env __FINALPATH__/code/
-ReadWritePaths=__FINALPATH__/media __FINALPATH__/import
+ReadWritePaths=__FINALPATH__/media __FINALPATH__/import /var/log/__APP__
 
 StandardOutput=syslog
 StandardError=syslog

conf/nginx.conf

@@ -15,7 +15,7 @@ location @rewrites {
 location /api/ {
 	include __FINALPATH__/code/deploy/funkwhale_proxy.conf;
 	# this is needed if you have file import via upload enabled
-	client_max_body_size 30M;
+	client_max_body_size 100M;
 	proxy_pass   http://127.0.0.1:__PORT__/api/;
 }
 

manifest.json

@@ -8,7 +8,7 @@
 	"description": {
 		"en": "A modern, convivial and free music server"
 	},
-	"version": "0.17.0~ynh2",
+	"version": "0.17.0~ynh3",
 	"url": "https://funkwhale.audio",
 	"license": "BSD-3-Clause",
 	"maintainer": {
@@ -47,4 +47,4 @@
 			}
 		]
 	}
-}
+}

scripts/_common.sh

@@ -106,3 +106,71 @@ $(yunohost tools diagnosis | grep -B 100 "services:" | sed '/services:/d')"
 	echo "$mail_message" | $mail_bin -a "Content-Type: text/plain; charset=UTF-8" -s "$mail_subject" "$recipients"
 }
 
+#=================================================
+# fail2ban helpers
+# taken from https://github.com/YunoHost-Apps/shaarli_ynh
+#=================================================
+
+# Create a dedicated fail2ban config (jail and filter conf files)
+#
+# usage: ynh_add_fail2ban_config log_file filter [max_retry [ports]]
+# | arg: log_file - Log file to be checked by fail2ban
+# | arg: failregex - Failregex to be looked for by fail2ban
+# | arg: max_retry - Maximum number of retries allowed before banning IP address - default: 3
+# | arg: ports - Ports blocked for a banned IP address - default: http,https
+ynh_add_fail2ban_config () {
+	local logpath
+	local failregex
+	local max_retry
+	local ports
+
+	logpath=$1
+	failregex=$2
+	max_retry=${3:-3}
+	ports=${4:-http,https}
+
+	test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing."
+	test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing."
+
+	finalfail2banjailconf="/etc/fail2ban/jail.d/$app.conf"
+	finalfail2banfilterconf="/etc/fail2ban/filter.d/$app.conf"
+	ynh_backup_if_checksum_is_different "$finalfail2banjailconf" 1
+	ynh_backup_if_checksum_is_different "$finalfail2banfilterconf" 1
+
+	sudo tee "$finalfail2banjailconf" <<EOF
+[$app]
+enabled = true
+port = $ports
+filter = $app
+logpath = $logpath
+maxretry = $max_retry
+EOF
+
+	sudo tee "$finalfail2banfilterconf" <<EOF
+[INCLUDES]
+before = common.conf
+[Definition]
+failregex = $failregex
+ignoreregex =
+EOF
+
+	ynh_store_file_checksum "$finalfail2banjailconf"
+	ynh_store_file_checksum "$finalfail2banfilterconf"
+
+	systemctl restart fail2ban
+	local fail2ban_error="$(journalctl -u fail2ban | tail -n50 | grep "WARNING.*$app.*")"
+	if [ -n "$fail2ban_error" ]
+	then
+		echo "[ERR] Fail2ban failed to load the jail for $app" >&2
+		echo "WARNING${fail2ban_error#*WARNING}" >&2
+	fi
+}
+
+# Remove the dedicated fail2ban config (jail and filter conf files)
+#
+# usage: ynh_remove_fail2ban_config
+ynh_remove_fail2ban_config () {
+	ynh_secure_remove "/etc/fail2ban/jail.d/$app.conf"
+	ynh_secure_remove "/etc/fail2ban/filter.d/$app.conf"
+	systemctl reload fail2ban
+}

scripts/install

@@ -63,7 +63,6 @@ ynh_app_setting_set "$app" admin "$admin"
 # Find a free port
 port=$(ynh_find_port 5000)
 # Open this port
-yunohost firewall allow --no-upnp TCP "$port" 2>&1
 ynh_app_setting_set "$app" port "$port"
 
 #=================================================
@@ -71,7 +70,7 @@ ynh_app_setting_set "$app" port "$port"
 #=================================================
 
 ynh_install_app_dependencies build-essential curl ffmpeg \
-	libjpeg-dev libmagic-dev libpq-dev postgresql python3-dev python3-venv \
+	libjpeg-dev libmagic-dev libpq-dev postgresql python3-dev virtualenv \
 	redis-server libldap2-dev libsasl2-dev \
 	`# add arm support` \
 	zlib1g-dev libffi-dev libssl-dev
@@ -139,7 +138,7 @@ ynh_system_user_create "$app" "$final_path"
 # PYTHON DEPENDENCIES
 #=================================================
 
-python3 -m venv "$final_path/code/virtualenv"
+virtualenv -p python3 "$final_path/code/virtualenv"
 (
 	set +o nounset
 	source "${final_path}/code/virtualenv/bin/activate"
@@ -173,12 +172,16 @@ ynh_replace_string "__DBNAME__"    "$app"         "$configfile"
 ynh_replace_string "__FINALPATH__" "$final_path"  "$configfile"
 ynh_replace_string "__KEY__"       "$key"         "$configfile"
 
-cat > "$final_path/code/load_env" <<'EOL'
+loadfile="$final_path/code/load_env"
+
+cat > "$loadfile" <<'EOL'
 #!/bin/bash
-export $(cat "$final_path/code/config/.env" | grep -v ^# | xargs)
+export $(cat "__FINALPATH__/code/config/.env" | grep -v ^# | xargs)
 EOL
 
-chmod +x "$final_path/code/load_env"
+chmod +x "$loadfile"
+
+ynh_replace_string "__FINALPATH__" "$final_path"  "$loadfile"
 
 #=================================================
 # MODIFY THE CONFIG FILE
@@ -188,7 +191,7 @@ admin_mail=$(ynh_user_get_info "$admin" "mail")
 (
 	set +o nounset
 	source "${final_path}/code/virtualenv/bin/activate"
-	source "${final_path}/code/load_env"
+	source "$loadfile"
 	set -o nounset
 	cd "$final_path/code/"
 
@@ -223,6 +226,24 @@ systemctl restart "$app".target
 chown -R "$app": "$final_path"
 chmod -R 755 "$final_path/code/front/dist/"
 
+mkdir -p "/var/log/$app"
+chown -R "$app": "/var/log/$app"
+
+#=================================================
+# SETUP FAIL2BAN
+#=================================================
+
+ynh_add_fail2ban_config "/var/log/nginx/$domain-access.log" "<HOST>.* \"POST /api/v1/token/ HTTP/1.1\" 400 68.*$" 5
+
+
+#=================================================
+# ADVERTISE SERVICE IN ADMIN PANEL
+#=================================================
+
+yunohost service add "$app-server" --log "/var/log/$app/server.log"
+yunohost service add "$app-worker" --log "/var/log/$app/worker.log"
+yunohost service add "$app-beat"   --log "/var/log/$app/beat.log"
+
 #=================================================
 # SETUP SSOWAT
 #=================================================

scripts/remove

@@ -24,6 +24,29 @@ redis_db=$(ynh_app_setting_get $app redis_db)
 
 #=================================================
 # STANDARD REMOVE
+#=================================================
+# REMOVE SERVICE FROM ADMIN PANEL
+#=================================================
+
+# Remove a service from the admin panel, added by `yunohost service add`
+if yunohost service status | grep -q "$app-server"
+then
+	echo "Remove $app-server service"
+	yunohost service remove "$app-server"
+fi
+
+if yunohost service status | grep -q "$app-worker"
+then
+	echo "Remove $app-worker service"
+	yunohost service remove "$app-worker"
+fi
+
+if yunohost service status | grep -q "$app-beat"
+then
+	echo "Remove $app-beat service"
+	yunohost service remove "$app-beat"
+fi
+
 #=================================================
 # STOP AND REMOVE SERVICE
 #=================================================
@@ -65,6 +88,8 @@ ynh_remove_app_dependencies
 # Remove the app directory securely
 ynh_secure_remove "$final_path"
 
+ynh_secure_remove "/var/log/$app"
+
 #=================================================
 # REMOVE NGINX CONFIGURATION
 #=================================================
@@ -91,9 +116,15 @@ fi
 
 #=================================================
 # GENERIC FINALIZATION
+#=================================================
+# REMOVE FAIL2BAN CONFIGURATION
+#=================================================
+
+ynh_remove_fail2ban_config
+
 #=================================================
 # REMOVE DEDICATED USER
 #=================================================
 
 # Delete a system user
-ynh_system_user_delete "$app"
+ynh_system_user_delete "$app"

scripts/restore

@@ -79,6 +79,7 @@ then
 	mv "$final_path/media" "$final_path/code/data/media"
 	mv "$final_path/import" "$final_path/code/data/music"
 	ynh_secure_remove "$final_path/code"
+	ynh_app_setting_delete "$app" code_migration
 fi
 
 #=================================================
@@ -95,7 +96,7 @@ ynh_system_user_create "$app"
 #=================================================
 
 ynh_install_app_dependencies build-essential curl ffmpeg \
-	libjpeg-dev libmagic-dev libpq-dev postgresql python3-dev python3-venv \
+	libjpeg-dev libmagic-dev libpq-dev postgresql python3-dev virtualenv \
 	redis-server libldap2-dev libsasl2-dev \
 	`# add arm support` \
 	zlib1g-dev libffi-dev libssl-dev
@@ -115,6 +116,17 @@ ynh_psql_execute_file_as_root ./db.sql "$db_name"
 # Restore permissions on app files
 chown -R "$app": "$final_path"
 
+mkdir -p "/var/log/$app"
+chown -R "$app": "/var/log/$app"
+
+#=================================================
+# ADVERTISE SERVICE IN ADMIN PANEL
+#=================================================
+
+yunohost service add "$app-server" --log "/var/log/$app/server.log"
+yunohost service add "$app-worker" --log "/var/log/$app/worker.log"
+yunohost service add "$app-beat"   --log "/var/log/$app/beat.log"
+
 #=================================================
 # RESTORE SYSTEMD
 #=================================================

scripts/upgrade

@@ -24,6 +24,7 @@ db_user=$db_name
 port=$(ynh_app_setting_get "$app" port)
 db_pwd=$(ynh_app_setting_get "$app" psqlpwd)
 redis_db=$(ynh_app_setting_get "$app" redis_db)
+code_migration=$(ynh_app_setting_get "$app" code_migration)
 
 #=================================================
 # ENSURE DOWNWARD COMPATIBILITY
@@ -37,10 +38,15 @@ fi
 
 # make sure we have the last code organization
 if [ ! -d "$final_path/code/" ]; then
+	mkdir "$final_path-tmp"
+	mv "$final_path"/* "$final_path-tmp/"
+
 	mkdir "$final_path/code"
-	mv "$final_path/!(code)" "$final_path/code/"
+	mv "$final_path-tmp/data/media" "$final_path/media"
-	mv "$final_path/code/data/media" "$final_path/media"
+	mv "$final_path-tmp/data/music" "$final_path/import"
-	mv "$final_path/code/data/music" "$final_path/import"
+	mv "$final_path-tmp"/* "$final_path/code"
+
+	ynh_secure_remove "$final_path-tmp/"
 	ynh_app_setting_set "$app" code_migration 1
 fi
 
@@ -60,6 +66,14 @@ ynh_clean_setup () {
 # Exit if an error occurs during the execution of the script
 ynh_abort_if_errors
 
+#=================================================
+# STOP SERVICES
+#=================================================
+
+systemctl stop "$app-beat.service"
+systemctl stop "$app-server.service"
+systemctl stop "$app-worker.service"
+
 #=================================================
 # CHECK THE PATH
 #=================================================
@@ -69,6 +83,16 @@ path_url=$(ynh_normalize_url_path "$path_url")
 
 #=================================================
 # STANDARD UPGRADE STEPS
+#=================================================
+# CLOSE A PORT
+#=================================================
+
+if yunohost firewall list | grep -q "\- $port$"
+then
+	echo "Close port $port"
+	yunohost firewall disallow TCP $port 2>&1
+fi
+
 #=================================================
 # DOWNLOAD, CHECK AND UNPACK SOURCE
 #=================================================
@@ -119,7 +143,7 @@ ynh_system_user_create "$app"
 #=================================================
 
 ynh_install_app_dependencies build-essential curl ffmpeg \
-	libjpeg-dev libmagic-dev libpq-dev postgresql python3-dev python3-venv \
+	libjpeg-dev libmagic-dev libpq-dev postgresql python3-dev virtualenv \
 	redis-server libldap2-dev libsasl2-dev \
 	`# add arm support` \
 	zlib1g-dev libffi-dev libssl-dev
@@ -130,7 +154,8 @@ ynh_install_app_dependencies build-essential curl ffmpeg \
 # PYTHON DEPENDENCIES
 #=================================================
 
-python3 -m venv "$final_path/code/virtualenv"
+ynh_secure_remove "$final_path/code/virtualenv"
+virtualenv -p python3 "$final_path/code/virtualenv"
 (
 	set +o nounset
 	source "${final_path}/code/virtualenv/bin/activate"
@@ -165,12 +190,16 @@ ynh_replace_string "__DBNAME__"    "$app"         "$configfile"
 ynh_replace_string "__FINALPATH__" "$final_path"  "$configfile"
 ynh_replace_string "__KEY__"       "$key"         "$configfile"
 
-cat > "$final_path/code/load_env" <<'EOL'
+loadfile="$final_path/code/load_env"
+
+cat > "$loadfile" <<'EOL'
 #!/bin/bash
-export $(cat "$final_path/code/config/.env" | grep -v ^# | xargs)
+export $(cat "__FINALPATH__/code/config/.env" | grep -v ^# | xargs)
 EOL
 
-chmod +x "$final_path/code/load_env"
+chmod +x "$loadfile"
+
+ynh_replace_string "__FINALPATH__" "$final_path"  "$loadfile"
 
 #=================================================
 # MIGRATE
@@ -179,7 +208,7 @@ chmod +x "$final_path/code/load_env"
 (
 	set +o nounset
 	source "${final_path}/code/virtualenv/bin/activate"
-	source "${final_path}/code/load_env"
+	source "$loadfile"
 	set -o nounset
 	cd "$final_path/code"
 
@@ -222,6 +251,15 @@ systemctl restart "$app".target
 chown -R "$app": "$final_path"
 chmod -R 755 "$final_path/code/front/dist/"
 
+mkdir -p "/var/log/$app"
+chown -R "$app": "/var/log/$app"
+
+#=================================================
+# SETUP FAIL2BAN
+#=================================================
+
+ynh_add_fail2ban_config "/var/log/nginx/$domain-access.log" "<HOST>.* \"POST /api/v1/token/ HTTP/1.1\" 400 68.*$" 5
+
 #=================================================
 # SETUP SSOWAT
 #=================================================
@@ -243,4 +281,10 @@ systemctl reload nginx
 # REMOVE CODE MIGRATION FLAG
 #=================================================
 
-ynh_app_setting_delete "$app" code_migration
+ynh_app_setting_set "$app" code_migration 2
+
+#=================================================
+# RESTART Funkwhale
+#=================================================
+
+systemctl restart "$app.target"