migrate to python3, add celery and redis

README.md

@@ -63,13 +63,11 @@ It doesn't work yet, but while [it looks doable](https://docs.weblate.org/en/lat
 [ ] ARM support
 
 * to be added:
-[ ] strenghen uwsgi systemd security (https://github.com/YunoHost-Apps/kresus_ynh/pull/21/commits/0ce4979d5037b111aa9698ed5552135bf0ed7165)
 [ ] use jq instead of grep/sed
 [ ] change URL script
 [ ] use debian package for lxml (may unlock ARM support)
 [ ] Add configuration options using the YunoHost interface (https://forum.yunohost.org/t/yunohost-3-1-minor-stable-release-version-stable-mineure/5445)
 [ ] Add fail2ban script
-[ ] Use redis+hiredis for cache?
 [ ] Enable CHECK_LIST?
 [ ] Enable AUTOFIX_LIST?
 [ ] Enable Translation Memory?

conf/celery-weblate

@@ -0,0 +1,27 @@
+# Name of nodes to start
+# here we have a single node
+CELERYD_NODES="w1"
+# or we could have three nodes:
+#CELERYD_NODES="w1 w2 w3"
+
+# Absolute or relative path to the 'celery' command:
+CELERY_BIN="/usr/bin/celery"
+
+# App instance to use
+# comment out this line if you don't use an app
+CELERY_APP="weblate"
+
+# How to call manage.py
+CELERYD_MULTI="multi"
+
+# Extra command-line arguments to the worker
+CELERYD_OPTS="--beat"
+
+# - %n will be replaced with the first part of the nodename.
+# - %I will be replaced with the current child process index
+#   and is important when using the prefork pool to avoid race conditions.
+CELERYD_PID_FILE="/var/run/celery-__APP__/weblate-%n.pid"
+CELERYD_LOG_FILE="/var/log/uwsgi/app/__APP__-celery-%n%I.log"
+CELERYD_LOG_LEVEL="INFO"
+
+CELERY_WORKER_RUNNING="1"

conf/celery-weblate.service

@@ -0,0 +1,23 @@
+[Unit]
+Description=Celery Service for Weblate (__APP__)
+After=network.target
+
+[Service]
+Type=forking
+User=__APP__
+Group=__APP__
+PermissionsStartOnly=true
+EnvironmentFile=__FINALPATH__/celery-weblate
+Environment=PATH=__FINALPATH__/venv/bin:$PATH
+WorkingDirectory=__FINALPATH__/
+ExecStart=/bin/sh -c '${CELERY_BIN} multi start ${CELERYD_NODES} \
+  -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} \
+  --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}'
+ExecStop=/bin/sh -c '${CELERY_BIN} multi stopwait ${CELERYD_NODES} \
+  --pidfile=${CELERYD_PID_FILE}'
+ExecReload=/bin/sh -c '${CELERY_BIN} multi restart ${CELERYD_NODES} \
+  -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} \
+  --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}'
+
+[Install]
+WantedBy=multi-user.target

conf/settings_history/settings.3.2.py

@@ -56,7 +56,7 @@ BASE_DIR = '__FINALPATH__'
 
 # Data directory
 DATA_DIR = os.path.join(BASE_DIR, 'data')
-TTF_PATH = '__FINALPATH__/venv/lib/python2.7/site-packages/weblate/ttf/'
+TTF_PATH = '__FINALPATH__/venv/lib/python3.5/site-packages/weblate/ttf/'
 
 # Local time zone for this installation. Choices can be found here:
 # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
@@ -715,8 +715,12 @@ ALLOWED_HOSTS = ['__DOMAIN__']
 
 CACHES = {
     'default': {
-        'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
+        'BACKEND': 'django_redis.cache.RedisCache',
-        'LOCATION': '127.0.0.1:__MEMCPORT__',
+        'LOCATION': 'redis://127.0.0.1:6379/_REDIS_DB__',
+        'OPTIONS': {
+            'CLIENT_CLASS': 'django_redis.client.DefaultClient',
+            'PARSER_CLASS': 'redis.connection.HiredisParser',
+        }
     },
     'avatar': {
         'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
@@ -806,11 +810,11 @@ REST_FRAMEWORK = {
 # )
 
 # Celery worker configuration for testing
-CELERY_TASK_ALWAYS_EAGER = True
+# CELERY_TASK_ALWAYS_EAGER = True
-CELERY_BROKER_URL = 'memory://'
+# CELERY_BROKER_URL = 'memory://'
 # Celery worker configuration for production
-# CELERY_TASK_ALWAYS_EAGER = False
+CELERY_TASK_ALWAYS_EAGER = False
-# CELERY_BROKER_URL = 'redis://localhost:6379'
+CELERY_BROKER_URL = 'redis://localhost:6379/__REDIS_DB__'
 
 # Celery settings, it is not recommended to change these
 CELERY_WORKER_PREFETCH_MULTIPLIER = 0

conf/uwsgi.ini

@@ -1,12 +1,12 @@
 [uwsgi]
-plugins       = python
+plugins       = python3
 master        = true
 protocol      = uwsgi
 socket        = /var/run/uwsgi/__APP__.socket
 virtualenv    = __FINALPATH__/venv
 
 # http://uwsgi-docs.readthedocs.io/en/latest/Nginx.html#hosting-multiple-apps-in-the-same-process-aka-managing-script-name-and-path-info
-mount = __PATH__=__FINALPATH__/venv/lib/python2.7/site-packages/weblate/wsgi.py
+mount = __PATH__=__FINALPATH__/venv/lib/python3.5/site-packages/weblate/wsgi.py
 manage-script-name = true
 
 # Needed for OAuth/OpenID

scripts/_common.sh

@@ -91,8 +91,8 @@ weblate_fill_settings() {
 	ynh_replace_string "__DOMAIN__"     "$domain"         "$settings"
 	ynh_replace_string "__KEY__"        "$key"            "$settings"
 	ynh_replace_string "__FINALPATH__"  "$final_path"     "$settings"
-	ynh_replace_string "__MEMCPORT__"   "$memc_port"      "$settings"
 	ynh_replace_string "__GITHUBUSER__" "$github_account" "$settings"
+	ynh_replace_string "__REDIS_DB__"   "$redis_db"       "$settings"
 
 	# root install as an empty PATHURL to prevent '//static'
 	if [ "$path_url" == "/" ]
@@ -177,3 +177,48 @@ $(yunohost tools diagnosis | grep -B 100 "services:" | sed '/services:/d')"
 	# Send the email to the recipients
 	echo "$mail_message" | $mail_bin -a "Content-Type: text/plain; charset=UTF-8" -s "$mail_subject" "$recipients"
 }
+
+#=================================================
+#
+# Redis HELPERS
+#
+# Point of contact : Jean-Baptiste Holcroft <jean-baptiste@holcroft.fr>
+#=================================================
+
+# get the first available redis database
+#
+# usage: ynh_redis_get_free_db
+# | returns: the database number to use
+ynh_redis_get_free_db() {
+	local result max db
+	result=$(redis-cli INFO keyspace)
+
+	# get the num
+	max=$(cat /etc/redis/redis.conf | grep ^databases | grep -Eow "[0-9]+")
+
+	db=0
+	# default Debian setting is 15 databases
+	for i in $(seq 0 "$max")
+	do
+	 	if ! echo "$result" | grep -q "db$i"
+	 	then
+			db=$i
+	 		break 1
+ 		fi
+ 		db=-1
+	done
+
+	test "$db" -eq -1 && ynh_die "No available Redis databases..."
+
+	echo "$db"
+}
+
+# Create a master password and set up global settings
+# Please always call this script in install and restore scripts
+#
+# usage: ynh_redis_remove_db database
+# | arg: database - the database to erase
+ynh_redis_remove_db() {
+	local db=$1
+	redis-cli -n "$db" flushall
+}

scripts/backup

@@ -56,6 +56,12 @@ ynh_backup "db.sql"
 
 ynh_backup "/etc/cron.d/$app"
 
+#=================================================
+# BACKUP THE CRON FILE
+#=================================================
+
+ynh_backup "/usr/lib/tmpfiles.d/$app.conf"
+
 #=================================================
 # BACKUP THE uwsgi files
 #=================================================

scripts/install

@@ -77,9 +77,9 @@ ynh_app_setting_set "$app" github_token "$github_token"
 #=================================================
 
 ynh_install_app_dependencies libxml2-dev libxslt-dev libfreetype6-dev \
-	libjpeg-dev libz-dev libyaml-dev python-dev python-pip python-virtualenv \
+	libjpeg-dev libz-dev libyaml-dev python3-dev python3-pip python3-virtualenv \
-	postgresql libpq-dev uwsgi uwsgi-plugin-python memcached \
+	postgresql libpq-dev uwsgi uwsgi-plugin-python3 \
-	mailutils
+	mailutils python-celery-common virtualenv redis-server
 
 #=================================================
 # CREATE A PostgreSQL DATABASE
@@ -185,7 +185,7 @@ fi
 #=================================================
 # PIP INSTALLATION
 #=================================================
-virtualenv "${final_path}/venv"
+virtualenv --python=python3 "${final_path}/venv"
 #run source in a 'sub shell'
 (
 	set +o nounset
@@ -195,11 +195,9 @@ virtualenv "${final_path}/venv"
 	# prevent error: "command: 'install_requires' must be a string or list of strings containing valid project/version requirement specifiers"
 	pip install --upgrade setuptools
 	pip install Weblate=="$current_version"
-	pip install pytz python-bidi PyYaML Babel pyuca pylibravatar pydns psycopg2-binary python-memcached phply
+	pip install pytz python-bidi PyYaML Babel pyuca pylibravatar py3dns psycopg2-binary phply django-redis hiredis
 	# specific to YunoHost package:
 	pip install django_sendmail_backend
-	# Fix ImportError: No module named backports
-	pip install backports.csv
 )
 
 #=================================================
@@ -210,12 +208,13 @@ virtualenv "${final_path}/venv"
 db_pwd=$(ynh_app_setting_get "$app" psqlpwd)
 admin_mail=$(ynh_user_get_info "$admin" mail)
 key=$(ynh_string_random 24)$(ynh_string_random 24)$(ynh_string_random 2)
-memc_port=$(ynh_find_port 8080)
+redis_db=$(ynh_redis_get_free_db)
-settings="$final_path/venv/lib/python2.7/site-packages/weblate/settings.py"
+
+settings="$final_path/venv/lib/python3.5/site-packages/weblate/settings.py"
 cp "../conf/settings_history/settings.$current_version.py" "$settings"
 
 weblate_fill_settings "$settings"
-ynh_app_setting_set "$app" memc_port "$memc_port"
+ynh_app_setting_set "$app" redis_db "$redis_db"
 
 #=================================================
 # SPECIFIC SETUP Filling up the database
@@ -249,7 +248,21 @@ ynh_replace_string "__FINALPATH__" "$final_path/" "/etc/cron.d/$app"
 #=================================================
 
 # Calculate and store the config file checksum into the app settings
-ynh_store_file_checksum "$final_path/venv/lib/python2.7/site-packages/weblate/settings.py"
+ynh_store_file_checksum "$final_path/venv/lib/python3.5/site-packages/weblate/settings.py"
+
+#=================================================
+# ACTIVATE CELERY
+#=================================================
+
+celeryconf="$final_path/celery-weblate"
+cp ../conf/celery-weblate "$celeryconf"
+
+ynh_replace_string "__APP__"  "$app"  "$celeryconf"
+
+echo "d     /var/run/celery-$app   0775 $app $app" > "/usr/lib/tmpfiles.d/$app.conf"
+systemd-tmpfiles --create
+
+ynh_add_systemd_config "$app-celery" "celery-weblate.service"
 
 #=================================================
 # GENERIC FINALIZATION
@@ -287,6 +300,7 @@ fi
 # Start weblate
 #=================================================
 
+systemctl start "$app-celery"
 systemctl start "uwsgi-app@$app.service"
 
 #=================================================

scripts/remove

@@ -18,10 +18,11 @@ domain=$(ynh_app_setting_get "$app" domain)
 db_name=$(ynh_app_setting_get "$app" db_name)
 
 #=================================================
-# STOP WEBLATE'S SERVICE
+# STOP WEBLATE'S SERVICES
 #=================================================
 
 systemctl stop "uwsgi-app@$app.service"
+systemctl stop "$app-celery.service"
 
 #=================================================
 # REMOVE THE PostgreSQL DATABASE
@@ -60,11 +61,19 @@ ynh_remove_nginx_config
 # Remove a cron file
 ynh_secure_remove "/etc/cron.d/$app"
 
+#=================================================
+# REMOVE TMPFILES.D
+#=================================================
+
+ynh_secure_remove "/var/run/celery-$app"
+ynh_secure_remove "/usr/lib/tmpfiles.d/$app.conf"
+
 #=================================================
 # REMOVE uwsgi and systemd files
 #=================================================
 
 ynh_remove_uwsgi_service
+ynh_remove_systemd_config "$app-celery"
 
 #=================================================
 # GENERIC FINALIZATION

scripts/restore

@@ -75,9 +75,9 @@ chown -R "$app": "$final_path"
 #=================================================
 
 ynh_install_app_dependencies libxml2-dev libxslt-dev libfreetype6-dev \
-	libjpeg-dev libz-dev libyaml-dev python-dev python-pip python-virtualenv \
+	libjpeg-dev libz-dev libyaml-dev python3-dev python3-pip python3-virtualenv \
-	postgresql libpq-dev uwsgi uwsgi-plugin-python memcached \
+	postgresql libpq-dev uwsgi uwsgi-plugin-python3 memcached \
-	mailutils
+	mailutils python-celery-common virtualenv redis-server
 
 #=================================================
 # RESTORE THE PostgreSQL DATABASE
@@ -123,6 +123,13 @@ yunohost service add "uwsgi-app@$app.service" --log "/var/log/uwsgi/app/$app"
 
 ynh_restore_file "/etc/cron.d/$app"
 
+#=================================================
+# RESTORE THE TMPFILES.D
+#=================================================
+
+ynh_restore_file "/usr/lib/tmpfiles.d/$app.conf"
+systemd-tmpfiles --create
+
 #=================================================
 # RESTORE THE HUB BINARY FILE
 #=================================================
@@ -135,6 +142,7 @@ ynh_restore_file "/usr/bin/hub"
 # Start weblate
 #=================================================
 
+systemctl start "$app-celery"
 systemctl start "uwsgi-app@$app.service"
 
 #=================================================

scripts/upgrade

@@ -186,9 +186,9 @@ chsh --shell /bin/bash "$app"
 #=================================================
 
 ynh_install_app_dependencies libxml2-dev libxslt-dev libfreetype6-dev \
-	libjpeg-dev libz-dev libyaml-dev python-dev python-pip python-virtualenv \
+	libjpeg-dev libz-dev libyaml-dev python3-dev python3-pip python3-virtualenv \
-	postgresql libpq-dev uwsgi uwsgi-plugin-python memcached \
+	postgresql libpq-dev uwsgi uwsgi-plugin-python3 memcached \
-	mailutils
+	mailutils python-celery-common virtualenv redis-server
 
 #=================================================
 # SPECIFIC SETUP uwsgi
@@ -215,12 +215,10 @@ fi
 	# prevent error: "command: 'install_requires' must be a string or list of strings containing valid project/version requirement specifiers"
 	pip install --upgrade setuptools
 	pip install Weblate=="$current_version"
-	pip install pytz python-bidi PyYaML Babel pyuca pylibravatar pydns psycopg2-binary python-memcached phply
+	pip install pytz python-bidi PyYaML Babel pyuca pylibravatar py3dns psycopg2-binary phply django-redis hiredis
 	# specific to YunoHost package:
 	pip install django_sendmail_backend
-	# Fix ImportError: No module named backports
+))
-	pip install backports.csv
-)
 
 #=================================================
 # CONFIG FILE UPGRADE