Upgrade yo 0.20.0

README.md

@@ -11,7 +11,7 @@ A modern, convivial and free music server on YunoHost
 
 Installation requires a dedicated domain for now. I hope subpath installation will be possible in the future.
 
-**Shipped version:** 0.19.1 (this is an Alpha version!)
+**Shipped version:** 0.20.0 (this is an Alpha version!)
 
 ## Screenshots
 

conf/app-frontend.src

@@ -1,5 +1,5 @@
-SOURCE_URL=https://dev.funkwhale.audio/funkwhale/funkwhale/-/jobs/artifacts/0.19.1/download?job=build_front
+SOURCE_URL=https://dev.funkwhale.audio/funkwhale/funkwhale/-/jobs/artifacts/0.20.0/download?job=build_front
-SOURCE_SUM=5ee39dd1cca26b244c53b24539e01906d3d0b04c9df5290e0070290a38fd1c83
+SOURCE_SUM=b42e10e3c57103eb8e39a1987aebae819843bb90fd9d3116ac233df5d7c9091e
 SOURCE_SUM_PRG=sha256sum
 SOURCE_FORMAT=zip
 SOURCE_IN_SUBDIR=false

conf/app.src

@@ -1,5 +1,5 @@
-SOURCE_URL=https://dev.funkwhale.audio/funkwhale/funkwhale/-/archive/0.19.1/funkwhale-0.19.1.tar.gz
+SOURCE_URL=https://dev.funkwhale.audio/funkwhale/funkwhale/-/archive/0.20.0/funkwhale-0.20.0.tar.gz
-SOURCE_SUM=a5f2b8b51f4259b0964f7adeceeaadab6b39de3417b019b4ec505d5154adff82
+SOURCE_SUM=d010f9db31473b86cdbad7f1bab1b2df3326604e8b0db18acb7f56b77e75c048
 SOURCE_SUM_PRG=sha256sum
 SOURCE_FORMAT=tar.bz2
 SOURCE_IN_SUBDIR=true

conf/funkwhale-beat.service

@@ -8,9 +8,7 @@ User=__APP__
 Group=__APP__
 WorkingDirectory=__FINALPATH__/code/api
 EnvironmentFile=__FINALPATH__/code/config/.env
-
+ExecStart=__FINALPATH__/code/ve3/bin/celery -A funkwhale_api.taskapp beat --loglevel INFO --logfile=/var/log/__APP__/beat.log
-ExecStart=__FINALPATH__/code/virtualenv/bin/celery -A funkwhale_api.taskapp beat \
-	--loglevel INFO --logfile=/var/log/__APP__/beat.log
 
 #NoNewPrivileges=true
 #PrivateDevices=true

conf/funkwhale-server.service

@@ -8,9 +8,7 @@ User=__APP__
 Group=__APP__
 WorkingDirectory=__FINALPATH__/code/api
 EnvironmentFile=__FINALPATH__/code/config/.env
-
+ExecStart=__FINALPATH__/code/ve3/bin/daphne -b ${FUNKWHALE_API_IP} -p ${FUNKWHALE_API_PORT} config.asgi:application --proxy-headers --verbosity 1 --access-log=/var/log/__APP__/server.log
-ExecStart=__FINALPATH__/code/virtualenv/bin/daphne -b ${FUNKWHALE_API_IP} -p ${FUNKWHALE_API_PORT} config.asgi:application --proxy-headers \
-	--verbosity 1 --access-log=/var/log/__APP__/server.log
 
 #NoNewPrivileges=true
 #PrivateDevices=true

conf/funkwhale-worker.service

@@ -8,9 +8,7 @@ User=__APP__
 Group=__APP__
 WorkingDirectory=__FINALPATH__/code/api
 EnvironmentFile=__FINALPATH__/code/config/.env
-
+ExecStart=__FINALPATH__/code/ve3/bin/celery -A funkwhale_api.taskapp worker -l INFO --loglevel INFO --logfile=/var/log/__APP__/worker.log
-ExecStart=__FINALPATH__/code/virtualenv/bin/celery -A funkwhale_api.taskapp worker -l INFO \
-	--loglevel INFO --logfile=/var/log/__APP__/worker.log
 
 #NoNewPrivileges=true
 #PrivateDevices=true

conf/pythonz.src

@@ -0,0 +1,6 @@
+SOURCE_URL=https://github.com/saghul/pythonz/archive/pythonz-2.0.1.tar.gz
+SOURCE_SUM=36980ef3b0ad632fb9596c60eb32a3275b2ef3641b07b376924d06ff79b4d539
+SOURCE_SUM_PRG=sha256sum
+SOURCE_FORMAT=tar.gz
+SOURCE_IN_SUBDIR=true
+SOURCE_FILENAME=

manifest.json

@@ -5,7 +5,7 @@
 	"description": {
 		"en": "A modern, convivial and free music server"
 	},
-	"version": "0.19.1~ynh2",
+	"version": "0.20.0~ynh1",
 	"url": "https://funkwhale.audio",
 	"license": "AGPL-3.0-or-later",
 	"maintainer": {

scripts/backup

@@ -46,6 +46,7 @@ then
 else
 	echo "Data dir will not be saved, because backup_core_only is set." >&2
 	ynh_backup --src_path="$final_path/code"
+	ynh_backup --src_path="$final_path/.pythonz"
 fi
 
 #=================================================

scripts/install

@@ -96,10 +96,9 @@ ynh_app_setting_set --app="$app" --key=final_path --value="$final_path"
 ynh_setup_source --dest_dir="$final_path/code"
 ynh_setup_source --dest_dir="$final_path/code" --source_id="app-frontend"
 
-(
+pushd "$final_path"
-	cd "$final_path"
 	mkdir -p code/config code/api code/data/static media import code/front
-)
+popd
 
 #=================================================
 # NGINX CONFIGURATION
@@ -122,20 +121,33 @@ ynh_system_user_create --username="$app" --home_dir="$final_path"
 
 #=================================================
 # SPECIFIC SETUP
+#=================================================
+# INSTALL PYTHONZ TO GET PYTHON 3.7.5
+#=================================================
+
+# Get pythonz
+# curl -kL https://raw.githubusercontent.com/saghul/pythonz/master/pythonz-install
+# ./pythonz-install
+ynh_setup_source --dest_dir="$final_path/.pythonz" --source_id="pythonz"
+export PYTHONZ_ROOT="$final_path/.pythonz"
+# Then install it
+python $final_path/.pythonz/pythonz_install.py
+
+# Install Python 3.7.5
+$final_path/.pythonz/bin/pythonz install 3.7.5
+
 #=================================================
 # PYTHON DEPENDENCIES
 #=================================================
 
-virtualenv -p python3 "$final_path/code/virtualenv"
+pushd "$final_path/code"
-(
+	virtualenv -p $($final_path/.pythonz/bin/pythonz locate 3.7.5) ve3
-	set +o nounset
+	ve3/bin/pip3 install --upgrade pip
-	source "${final_path}/code/virtualenv/bin/activate"
+	ve3/bin/pip3 install --upgrade setuptools
-	set -o nounset
+	ve3/bin/pip3 install wheel
-	pip install --upgrade pip
+	ve3/bin/pip3 install service_identity
-	pip install --upgrade setuptools
+	ve3/bin/pip3 install -r "${final_path}/code/api/requirements.txt"
-	pip install wheel
+popd
-	pip install -r "${final_path}/code/api/requirements.txt"
-)
 
 #=================================================
 # MODIFY THE CONFIG FILE
@@ -176,20 +188,15 @@ ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path"
 #=================================================
 
 admin_mail=$(ynh_user_get_info --username="$admin" --key="mail")
-(
-	set +o nounset
-	source "${final_path}/code/virtualenv/bin/activate"
-	source "$loadfile"
-	set -o nounset
-	cd "$final_path/code/"
 
+pushd "$final_path/code"
 	# needed for enabling the 'unaccent' extension
 	ynh_psql_execute_as_root --sql="ALTER USER $db_user WITH SUPERUSER;" --database="$db_name"
-	python api/manage.py migrate
+	ve3/bin/python3 api/manage.py migrate
 	ynh_psql_execute_as_root --sql="ALTER USER $db_user WITH NOSUPERUSER;" --database="$db_name"
-	echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser('$admin', '$admin_mail', 'funkwhale') " | python api/manage.py shell
+	echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser('$admin', '$admin_mail', 'funkwhale') " | ve3/bin/python3 api/manage.py shell
-	python api/manage.py collectstatic
+	ve3/bin/python3 api/manage.py collectstatic
-)
+popd
 
 #=================================================
 # SETUP SYSTEMD
@@ -236,9 +243,9 @@ yunohost service add "$app-beat"   --log "/var/log/$app/beat.log"
 #=================================================
 ynh_print_info --message="Starting a systemd service..."
 
-ynh_systemd_action --action="start" --service_name="${app}-beat" --log_path="systemd" --line_match="celery beat process"
+ynh_systemd_action --action="start" --service_name="${app}-beat" --log_path="systemd" --line_match="Running with"
-ynh_systemd_action --action="start" --service_name="${app}-server" --log_path="systemd" --line_match="Listening on TCP address"
+ynh_systemd_action --action="start" --service_name="${app}-server" --log_path="systemd" --line_match="Running with"
-ynh_systemd_action --action="start" --service_name="${app}-worker" --log_path="systemd" --line_match="celery worker"
+ynh_systemd_action --action="start" --service_name="${app}-worker" --log_path="systemd" --line_match="Running with"
 
 #=================================================
 # SETUP FAIL2BAN

scripts/remove

@@ -53,10 +53,6 @@ fi
 #=================================================
 ynh_print_info --message="Stopping and removing the systemd service"
 
-ynh_systemd_action --action="stop" --service_name="${app}-beat" --log_path="systemd" --line_match="Stopped Funkwhale"
-ynh_systemd_action --action="stop" --service_name="${app}-server" --log_path="systemd" --line_match="Stopped Funkwhale"
-ynh_systemd_action --action="stop" --service_name="${app}-worker" --log_path="systemd" --line_match="Stopped Funkwhale"
-
 # Remove the dedicated systemd config
 ynh_remove_systemd_config --service="$app-server"
 ynh_remove_systemd_config --service="$app-worker"

scripts/restore

@@ -15,6 +15,7 @@ source /usr/share/yunohost/helpers
 #=================================================
 
 ynh_clean_setup () {
+	read -p "key"
 	ynh_clean_check_starting
 }
 # Exit if an error occurs during the execution of the script
@@ -64,6 +65,7 @@ backup_core_only=$(ynh_app_setting_get --app="$app" --key=backup_core_only)
 if [ -z "$backup_core_only" ]
 then
 	ynh_restore_file --origin_path="$final_path/code"
+	ynh_restore_file --origin_path="$final_path/.pythonz"
 else
 	ynh_restore_file --origin_path="$final_path"
 fi
@@ -148,9 +150,9 @@ yunohost service add "$app-beat"   --log "/var/log/$app/beat.log"
 #=================================================
 ynh_print_info --message="Starting a systemd service..."
 
-ynh_systemd_action --action="start" --service_name="${app}-beat" --log_path="systemd" --line_match="celery beat process"
+ynh_systemd_action --action="start" --service_name="${app}-beat" --log_path="systemd" --line_match="Running with"
-ynh_systemd_action --action="start" --service_name="${app}-server" --log_path="systemd" --line_match="Listening on TCP address"
+ynh_systemd_action --action="start" --service_name="${app}-server" --log_path="systemd" --line_match="Running with"
-ynh_systemd_action --action="start" --service_name="${app}-worker" --log_path="systemd" --line_match="celery worker"
+ynh_systemd_action --action="start" --service_name="${app}-worker" --log_path="systemd" --line_match="Running with"
 
 #=================================================
 # GENERIC FINALIZATION

scripts/upgrade

@@ -144,24 +144,37 @@ ynh_system_user_create --username="$app" --home_dir="$final_path"
 
 #=================================================
 # SPECIFIC UPGRADE
+#=================================================
+# INSTALL PYTHONZ TO GET PYTHON 3.7.5
+#=================================================
+
+# Get pythonz
+# curl -kL https://raw.githubusercontent.com/saghul/pythonz/master/pythonz-install
+# ./pythonz-install
+ynh_setup_source --dest_dir="$final_path/.pythonz" --source_id="pythonz"
+export PYTHONZ_ROOT="$final_path/.pythonz"
+# Then install it
+python $final_path/.pythonz/pythonz_install.py
+
+# Install Python 3.7.5
+$final_path/.pythonz/bin/pythonz install 3.7.5
+
 #=================================================
 # PYTHON DEPENDENCIES
 #=================================================
 
 ynh_secure_remove --file="$final_path/code/virtualenv"
-virtualenv -p python3 "$final_path/code/virtualenv"
+pushd "$final_path/code"
-(
+	virtualenv -p $($final_path/.pythonz/bin/pythonz locate 3.7.5) ve3
-	set +o nounset
+	ve3/bin/pip3 install --upgrade pip
-	source "${final_path}/code/virtualenv/bin/activate"
+	ve3/bin/pip3 install --upgrade setuptools
-	set -o nounset
+	ve3/bin/pip3 install wheel
-	pip install --upgrade pip
+	ve3/bin/pip3 install service_identity
-	pip install --upgrade setuptools
+	ve3/bin/pip3 install -r "${final_path}/code/api/requirements.txt"
-	pip install wheel
-	pip install -r "${final_path}/code/api/requirements.txt"
 
 	# https://code.eliotberriot.com/funkwhale/funkwhale/tags/0.16
-	pip uninstall django-cacheops --yes
+	ve3/bin/pip3 uninstall django-cacheops --yes
-)
+popd
 
 #=================================================
 # MODIFY THE CONFIG FILE
@@ -199,32 +212,26 @@ ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path"
 # MIGRATE
 #=================================================
 
-(
+pushd "$final_path/code"
-	set +o nounset
-	source "${final_path}/code/virtualenv/bin/activate"
-	source "$loadfile"
-	set -o nounset
-	cd "$final_path/code"
-
 	# needed for enabling the 'unaccent' extension
 	ynh_psql_execute_as_root --sql="ALTER USER $db_user WITH SUPERUSER;" --database="$db_name"
-	python api/manage.py migrate
+	ve3/bin/python3 api/manage.py migrate
 	ynh_psql_execute_as_root --sql="ALTER USER $db_user WITH NOSUPERUSER;" --database="$db_name"
 
-	python api/manage.py collectstatic --clear --noinput
+	ve3/bin/python3 api/manage.py collectstatic --clear --noinput
 
 	# https://code.eliotberriot.com/funkwhale/funkwhale/tags/0.16 # users-now-have-an-activitypub-actor-manual-action-required
-	python api/manage.py script create_actors --no-input
+	ve3/bin/python3 api/manage.py script create_actors --no-input
 	# https://code.eliotberriot.com/funkwhale/funkwhale/tags/0.16 #image-thumbnails-manual-action-required
-	python api/manage.py script create_image_variations --no-input
+	ve3/bin/python3 api/manage.py script create_image_variations --no-input
 
 	# https://docs.funkwhale.audio/upgrading/0.17.html#upgrade-instructions
-	python api/manage.py script migrate_to_user_libraries --no-input
+	ve3/bin/python3 api/manage.py script migrate_to_user_libraries --no-input
 
 	# Delete pre 0.17 federated tracks [manual action suggested]
 	# https://dev.funkwhale.audio/funkwhale/funkwhale/tags/0.18
-	python api/manage.py script delete_pre_017_federated_uploads --no-input
+	ve3/bin/python3 api/manage.py script delete_pre_017_federated_uploads --no-input
-)
+popd
 
 #=================================================
 # STORE THE CONFIG FILE CHECKSUM
@@ -282,9 +289,9 @@ fi
 #=================================================
 ynh_print_info --message="Starting a systemd service..."
 
-ynh_systemd_action --action="start" --service_name="${app}-beat" --log_path="systemd" --line_match="celery beat process"
+ynh_systemd_action --action="start" --service_name="${app}-beat" --log_path="systemd" --line_match="Running with"
-ynh_systemd_action --action="start" --service_name="${app}-server" --log_path="systemd" --line_match="Listening on TCP address"
+ynh_systemd_action --action="start" --service_name="${app}-server" --log_path="systemd" --line_match="Running with"
-ynh_systemd_action --action="start" --service_name="${app}-worker" --log_path="systemd" --line_match="celery worker"
+ynh_systemd_action --action="start" --service_name="${app}-worker" --log_path="systemd" --line_match="Running with"
 
 #=================================================
 # RELOAD NGINX

sources/extra_files/app/api/requirements/base.txt

@@ -1,11 +1,11 @@
 # Bleeding edge Django
-django>=2.2.1,<2.3
+django>=2.2.4,<2.3
 
 # Configuration
 django-environ>=0.4,<0.5
 
 # Images
-Pillow>=5.4,<5.5
+Pillow>=6.2.0
 
 # For user registration, either via email or social
 # Well-built with regular release cycles!
@@ -29,7 +29,7 @@ celery>=4.3,<4.4
 # Your custom requirements go here
 django-cors-headers>=2.5.3,<2.6
 musicbrainzngs==0.6
-djangorestframework>=3.9,<3.10
+djangorestframework>=3.10,<3.11
 djangorestframework-jwt>=1.11,<1.12
 pendulum>=2,<3
 persisting-theory>=0.2,<0.3
@@ -39,11 +39,9 @@ django-rest-auth>=0.9,<0.10
 ipython>=7,<8
 mutagen>=1.42,<1.43
 
-
-django-taggit>=0.24,<0.25
 pymemoize==1.0.3
 
-django-dynamic-preferences>=1.7,<1.8
+django-dynamic-preferences>=1.7.1,<1.8
 raven>=6.10,<7
 python-magic==0.4.15
 # XXX: until https://github.com/django/channels/issues/1240 is fixed
@@ -53,12 +51,16 @@ channels==2.1.6
 channels_redis==2.3.2
 
 daphne>=2.2,<2.3
+uvicorn
+gunicorn
 
 cryptography>=2,<3
 # requests-http-signature==0.0.3
 # clone until the branch is merged and released upstream
 git+https://github.com/EliotBerriot/requests-http-signature.git@signature-header-support
 django-cleanup==3.2.0
+requests>=2.22<2.23
+pyOpenSSL>=19<20
 
 # for LDAP authentication
 python-ldap==3.2.0