mirror of
https://github.com/YunoHost-Apps/weblate_ynh.git
synced 2024-10-01 13:35:04 +02:00
migrate to python3, add celery and redis
This commit is contained in:
Jean-Baptiste Holcroft
parent
c38e8df3f4
commit
9cf6fd0e89
11 changed files with 168 additions and 36 deletions
|
|
@ -63,13 +63,11 @@ It doesn't work yet, but while [it looks doable](https://docs.weblate.org/en/lat
|
|||
[ ] ARM support
|
||||
|
||||
* to be added:
|
||||
[ ] strenghen uwsgi systemd security (https://github.com/YunoHost-Apps/kresus_ynh/pull/21/commits/0ce4979d5037b111aa9698ed5552135bf0ed7165)
|
||||
[ ] use jq instead of grep/sed
|
||||
[ ] change URL script
|
||||
[ ] use debian package for lxml (may unlock ARM support)
|
||||
[ ] Add configuration options using the YunoHost interface (https://forum.yunohost.org/t/yunohost-3-1-minor-stable-release-version-stable-mineure/5445)
|
||||
[ ] Add fail2ban script
|
||||
[ ] Use redis+hiredis for cache?
|
||||
[ ] Enable CHECK_LIST?
|
||||
[ ] Enable AUTOFIX_LIST?
|
||||
[ ] Enable Translation Memory?
|
||||
|
|
|
|||
27
conf/celery-weblate
Normal file
27
conf/celery-weblate
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
# Name of nodes to start
|
||||
# here we have a single node
|
||||
CELERYD_NODES="w1"
|
||||
# or we could have three nodes:
|
||||
#CELERYD_NODES="w1 w2 w3"
|
||||
|
||||
# Absolute or relative path to the 'celery' command:
|
||||
CELERY_BIN="/usr/bin/celery"
|
||||
|
||||
# App instance to use
|
||||
# comment out this line if you don't use an app
|
||||
CELERY_APP="weblate"
|
||||
|
||||
# How to call manage.py
|
||||
CELERYD_MULTI="multi"
|
||||
|
||||
# Extra command-line arguments to the worker
|
||||
CELERYD_OPTS="--beat"
|
||||
|
||||
# - %n will be replaced with the first part of the nodename.
|
||||
# - %I will be replaced with the current child process index
|
||||
# and is important when using the prefork pool to avoid race conditions.
|
||||
CELERYD_PID_FILE="/var/run/celery-__APP__/weblate-%n.pid"
|
||||
CELERYD_LOG_FILE="/var/log/uwsgi/app/__APP__-celery-%n%I.log"
|
||||
CELERYD_LOG_LEVEL="INFO"
|
||||
|
||||
CELERY_WORKER_RUNNING="1"
|
||||
23
conf/celery-weblate.service
Normal file
23
conf/celery-weblate.service
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
[Unit]
|
||||
Description=Celery Service for Weblate (__APP__)
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=forking
|
||||
User=__APP__
|
||||
Group=__APP__
|
||||
PermissionsStartOnly=true
|
||||
EnvironmentFile=__FINALPATH__/celery-weblate
|
||||
Environment=PATH=__FINALPATH__/venv/bin:$PATH
|
||||
WorkingDirectory=__FINALPATH__/
|
||||
ExecStart=/bin/sh -c '${CELERY_BIN} multi start ${CELERYD_NODES} \
|
||||
-A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} \
|
||||
--logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}'
|
||||
ExecStop=/bin/sh -c '${CELERY_BIN} multi stopwait ${CELERYD_NODES} \
|
||||
--pidfile=${CELERYD_PID_FILE}'
|
||||
ExecReload=/bin/sh -c '${CELERY_BIN} multi restart ${CELERYD_NODES} \
|
||||
-A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} \
|
||||
--logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}'
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -56,7 +56,7 @@ BASE_DIR = '__FINALPATH__'
|
|||
|
||||
# Data directory
|
||||
DATA_DIR = os.path.join(BASE_DIR, 'data')
|
||||
TTF_PATH = '__FINALPATH__/venv/lib/python2.7/site-packages/weblate/ttf/'
|
||||
TTF_PATH = '__FINALPATH__/venv/lib/python3.5/site-packages/weblate/ttf/'
|
||||
|
||||
# Local time zone for this installation. Choices can be found here:
|
||||
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
|
||||
|
|
@ -715,8 +715,12 @@ ALLOWED_HOSTS = ['__DOMAIN__']
|
|||
|
||||
CACHES = {
|
||||
'default': {
|
||||
'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
|
||||
'LOCATION': '127.0.0.1:__MEMCPORT__',
|
||||
'BACKEND': 'django_redis.cache.RedisCache',
|
||||
'LOCATION': 'redis://127.0.0.1:6379/_REDIS_DB__',
|
||||
'OPTIONS': {
|
||||
'CLIENT_CLASS': 'django_redis.client.DefaultClient',
|
||||
'PARSER_CLASS': 'redis.connection.HiredisParser',
|
||||
}
|
||||
},
|
||||
'avatar': {
|
||||
'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
|
||||
|
|
@ -806,11 +810,11 @@ REST_FRAMEWORK = {
|
|||
# )
|
||||
|
||||
# Celery worker configuration for testing
|
||||
CELERY_TASK_ALWAYS_EAGER = True
|
||||
CELERY_BROKER_URL = 'memory://'
|
||||
# CELERY_TASK_ALWAYS_EAGER = True
|
||||
# CELERY_BROKER_URL = 'memory://'
|
||||
# Celery worker configuration for production
|
||||
# CELERY_TASK_ALWAYS_EAGER = False
|
||||
# CELERY_BROKER_URL = 'redis://localhost:6379'
|
||||
CELERY_TASK_ALWAYS_EAGER = False
|
||||
CELERY_BROKER_URL = 'redis://localhost:6379/__REDIS_DB__'
|
||||
|
||||
# Celery settings, it is not recommended to change these
|
||||
CELERY_WORKER_PREFETCH_MULTIPLIER = 0
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
[uwsgi]
|
||||
plugins = python
|
||||
plugins = python3
|
||||
master = true
|
||||
protocol = uwsgi
|
||||
socket = /var/run/uwsgi/__APP__.socket
|
||||
virtualenv = __FINALPATH__/venv
|
||||
|
||||
# http://uwsgi-docs.readthedocs.io/en/latest/Nginx.html#hosting-multiple-apps-in-the-same-process-aka-managing-script-name-and-path-info
|
||||
mount = __PATH__=__FINALPATH__/venv/lib/python2.7/site-packages/weblate/wsgi.py
|
||||
mount = __PATH__=__FINALPATH__/venv/lib/python3.5/site-packages/weblate/wsgi.py
|
||||
manage-script-name = true
|
||||
|
||||
# Needed for OAuth/OpenID
|
||||
|
|
|
|||
|
|
@ -91,8 +91,8 @@ weblate_fill_settings() {
|
|||
ynh_replace_string "__DOMAIN__" "$domain" "$settings"
|
||||
ynh_replace_string "__KEY__" "$key" "$settings"
|
||||
ynh_replace_string "__FINALPATH__" "$final_path" "$settings"
|
||||
ynh_replace_string "__MEMCPORT__" "$memc_port" "$settings"
|
||||
ynh_replace_string "__GITHUBUSER__" "$github_account" "$settings"
|
||||
ynh_replace_string "__REDIS_DB__" "$redis_db" "$settings"
|
||||
|
||||
# root install as an empty PATHURL to prevent '//static'
|
||||
if [ "$path_url" == "/" ]
|
||||
|
|
@ -177,3 +177,48 @@ $(yunohost tools diagnosis | grep -B 100 "services:" | sed '/services:/d')"
|
|||
# Send the email to the recipients
|
||||
echo "$mail_message" | $mail_bin -a "Content-Type: text/plain; charset=UTF-8" -s "$mail_subject" "$recipients"
|
||||
}
|
||||
|
||||
#=================================================
|
||||
#
|
||||
# Redis HELPERS
|
||||
#
|
||||
# Point of contact : Jean-Baptiste Holcroft <jean-baptiste@holcroft.fr>
|
||||
#=================================================
|
||||
|
||||
# get the first available redis database
|
||||
#
|
||||
# usage: ynh_redis_get_free_db
|
||||
# | returns: the database number to use
|
||||
ynh_redis_get_free_db() {
|
||||
local result max db
|
||||
result=$(redis-cli INFO keyspace)
|
||||
|
||||
# get the num
|
||||
max=$(cat /etc/redis/redis.conf | grep ^databases | grep -Eow "[0-9]+")
|
||||
|
||||
db=0
|
||||
# default Debian setting is 15 databases
|
||||
for i in $(seq 0 "$max")
|
||||
do
|
||||
if ! echo "$result" | grep -q "db$i"
|
||||
then
|
||||
db=$i
|
||||
break 1
|
||||
fi
|
||||
db=-1
|
||||
done
|
||||
|
||||
test "$db" -eq -1 && ynh_die "No available Redis databases..."
|
||||
|
||||
echo "$db"
|
||||
}
|
||||
|
||||
# Create a master password and set up global settings
|
||||
# Please always call this script in install and restore scripts
|
||||
#
|
||||
# usage: ynh_redis_remove_db database
|
||||
# | arg: database - the database to erase
|
||||
ynh_redis_remove_db() {
|
||||
local db=$1
|
||||
redis-cli -n "$db" flushall
|
||||
}
|
||||
|
|
@ -56,6 +56,12 @@ ynh_backup "db.sql"
|
|||
|
||||
ynh_backup "/etc/cron.d/$app"
|
||||
|
||||
#=================================================
|
||||
# BACKUP THE CRON FILE
|
||||
#=================================================
|
||||
|
||||
ynh_backup "/usr/lib/tmpfiles.d/$app.conf"
|
||||
|
||||
#=================================================
|
||||
# BACKUP THE uwsgi files
|
||||
#=================================================
|
||||
|
|
|
|||
|
|
@ -77,9 +77,9 @@ ynh_app_setting_set "$app" github_token "$github_token"
|
|||
#=================================================
|
||||
|
||||
ynh_install_app_dependencies libxml2-dev libxslt-dev libfreetype6-dev \
|
||||
libjpeg-dev libz-dev libyaml-dev python-dev python-pip python-virtualenv \
|
||||
postgresql libpq-dev uwsgi uwsgi-plugin-python memcached \
|
||||
mailutils
|
||||
libjpeg-dev libz-dev libyaml-dev python3-dev python3-pip python3-virtualenv \
|
||||
postgresql libpq-dev uwsgi uwsgi-plugin-python3 \
|
||||
mailutils python-celery-common virtualenv redis-server
|
||||
|
||||
#=================================================
|
||||
# CREATE A PostgreSQL DATABASE
|
||||
|
|
@ -185,7 +185,7 @@ fi
|
|||
#=================================================
|
||||
# PIP INSTALLATION
|
||||
#=================================================
|
||||
virtualenv "${final_path}/venv"
|
||||
virtualenv --python=python3 "${final_path}/venv"
|
||||
#run source in a 'sub shell'
|
||||
(
|
||||
set +o nounset
|
||||
|
|
@ -195,11 +195,9 @@ virtualenv "${final_path}/venv"
|
|||
# prevent error: "command: 'install_requires' must be a string or list of strings containing valid project/version requirement specifiers"
|
||||
pip install --upgrade setuptools
|
||||
pip install Weblate=="$current_version"
|
||||
pip install pytz python-bidi PyYaML Babel pyuca pylibravatar pydns psycopg2-binary python-memcached phply
|
||||
pip install pytz python-bidi PyYaML Babel pyuca pylibravatar py3dns psycopg2-binary phply django-redis hiredis
|
||||
# specific to YunoHost package:
|
||||
pip install django_sendmail_backend
|
||||
# Fix ImportError: No module named backports
|
||||
pip install backports.csv
|
||||
)
|
||||
|
||||
#=================================================
|
||||
|
|
@ -210,12 +208,13 @@ virtualenv "${final_path}/venv"
|
|||
db_pwd=$(ynh_app_setting_get "$app" psqlpwd)
|
||||
admin_mail=$(ynh_user_get_info "$admin" mail)
|
||||
key=$(ynh_string_random 24)$(ynh_string_random 24)$(ynh_string_random 2)
|
||||
memc_port=$(ynh_find_port 8080)
|
||||
settings="$final_path/venv/lib/python2.7/site-packages/weblate/settings.py"
|
||||
redis_db=$(ynh_redis_get_free_db)
|
||||
|
||||
settings="$final_path/venv/lib/python3.5/site-packages/weblate/settings.py"
|
||||
cp "../conf/settings_history/settings.$current_version.py" "$settings"
|
||||
|
||||
weblate_fill_settings "$settings"
|
||||
ynh_app_setting_set "$app" memc_port "$memc_port"
|
||||
ynh_app_setting_set "$app" redis_db "$redis_db"
|
||||
|
||||
#=================================================
|
||||
# SPECIFIC SETUP Filling up the database
|
||||
|
|
@ -249,7 +248,21 @@ ynh_replace_string "__FINALPATH__" "$final_path/" "/etc/cron.d/$app"
|
|||
#=================================================
|
||||
|
||||
# Calculate and store the config file checksum into the app settings
|
||||
ynh_store_file_checksum "$final_path/venv/lib/python2.7/site-packages/weblate/settings.py"
|
||||
ynh_store_file_checksum "$final_path/venv/lib/python3.5/site-packages/weblate/settings.py"
|
||||
|
||||
#=================================================
|
||||
# ACTIVATE CELERY
|
||||
#=================================================
|
||||
|
||||
celeryconf="$final_path/celery-weblate"
|
||||
cp ../conf/celery-weblate "$celeryconf"
|
||||
|
||||
ynh_replace_string "__APP__" "$app" "$celeryconf"
|
||||
|
||||
echo "d /var/run/celery-$app 0775 $app $app" > "/usr/lib/tmpfiles.d/$app.conf"
|
||||
systemd-tmpfiles --create
|
||||
|
||||
ynh_add_systemd_config "$app-celery" "celery-weblate.service"
|
||||
|
||||
#=================================================
|
||||
# GENERIC FINALIZATION
|
||||
|
|
@ -287,6 +300,7 @@ fi
|
|||
# Start weblate
|
||||
#=================================================
|
||||
|
||||
systemctl start "$app-celery"
|
||||
systemctl start "uwsgi-app@$app.service"
|
||||
|
||||
#=================================================
|
||||
|
|
|
|||
|
|
@ -18,10 +18,11 @@ domain=$(ynh_app_setting_get "$app" domain)
|
|||
db_name=$(ynh_app_setting_get "$app" db_name)
|
||||
|
||||
#=================================================
|
||||
# STOP WEBLATE'S SERVICE
|
||||
# STOP WEBLATE'S SERVICES
|
||||
#=================================================
|
||||
|
||||
systemctl stop "uwsgi-app@$app.service"
|
||||
systemctl stop "$app-celery.service"
|
||||
|
||||
#=================================================
|
||||
# REMOVE THE PostgreSQL DATABASE
|
||||
|
|
@ -60,11 +61,19 @@ ynh_remove_nginx_config
|
|||
# Remove a cron file
|
||||
ynh_secure_remove "/etc/cron.d/$app"
|
||||
|
||||
#=================================================
|
||||
# REMOVE TMPFILES.D
|
||||
#=================================================
|
||||
|
||||
ynh_secure_remove "/var/run/celery-$app"
|
||||
ynh_secure_remove "/usr/lib/tmpfiles.d/$app.conf"
|
||||
|
||||
#=================================================
|
||||
# REMOVE uwsgi and systemd files
|
||||
#=================================================
|
||||
|
||||
ynh_remove_uwsgi_service
|
||||
ynh_remove_systemd_config "$app-celery"
|
||||
|
||||
#=================================================
|
||||
# GENERIC FINALIZATION
|
||||
|
|
|
|||
|
|
@ -75,9 +75,9 @@ chown -R "$app": "$final_path"
|
|||
#=================================================
|
||||
|
||||
ynh_install_app_dependencies libxml2-dev libxslt-dev libfreetype6-dev \
|
||||
libjpeg-dev libz-dev libyaml-dev python-dev python-pip python-virtualenv \
|
||||
postgresql libpq-dev uwsgi uwsgi-plugin-python memcached \
|
||||
mailutils
|
||||
libjpeg-dev libz-dev libyaml-dev python3-dev python3-pip python3-virtualenv \
|
||||
postgresql libpq-dev uwsgi uwsgi-plugin-python3 memcached \
|
||||
mailutils python-celery-common virtualenv redis-server
|
||||
|
||||
#=================================================
|
||||
# RESTORE THE PostgreSQL DATABASE
|
||||
|
|
@ -123,6 +123,13 @@ yunohost service add "uwsgi-app@$app.service" --log "/var/log/uwsgi/app/$app"
|
|||
|
||||
ynh_restore_file "/etc/cron.d/$app"
|
||||
|
||||
#=================================================
|
||||
# RESTORE THE TMPFILES.D
|
||||
#=================================================
|
||||
|
||||
ynh_restore_file "/usr/lib/tmpfiles.d/$app.conf"
|
||||
systemd-tmpfiles --create
|
||||
|
||||
#=================================================
|
||||
# RESTORE THE HUB BINARY FILE
|
||||
#=================================================
|
||||
|
|
@ -135,10 +142,11 @@ ynh_restore_file "/usr/bin/hub"
|
|||
# Start weblate
|
||||
#=================================================
|
||||
|
||||
systemctl start "$app-celery"
|
||||
systemctl start "uwsgi-app@$app.service"
|
||||
|
||||
#=================================================
|
||||
# RELOAD NGINX & uwsgi
|
||||
#=================================================
|
||||
|
||||
systemctl reload nginx
|
||||
systemctl reload nginx
|
||||
|
|
@ -186,9 +186,9 @@ chsh --shell /bin/bash "$app"
|
|||
#=================================================
|
||||
|
||||
ynh_install_app_dependencies libxml2-dev libxslt-dev libfreetype6-dev \
|
||||
libjpeg-dev libz-dev libyaml-dev python-dev python-pip python-virtualenv \
|
||||
postgresql libpq-dev uwsgi uwsgi-plugin-python memcached \
|
||||
mailutils
|
||||
libjpeg-dev libz-dev libyaml-dev python3-dev python3-pip python3-virtualenv \
|
||||
postgresql libpq-dev uwsgi uwsgi-plugin-python3 memcached \
|
||||
mailutils python-celery-common virtualenv redis-server
|
||||
|
||||
#=================================================
|
||||
# SPECIFIC SETUP uwsgi
|
||||
|
|
@ -215,12 +215,10 @@ fi
|
|||
# prevent error: "command: 'install_requires' must be a string or list of strings containing valid project/version requirement specifiers"
|
||||
pip install --upgrade setuptools
|
||||
pip install Weblate=="$current_version"
|
||||
pip install pytz python-bidi PyYaML Babel pyuca pylibravatar pydns psycopg2-binary python-memcached phply
|
||||
pip install pytz python-bidi PyYaML Babel pyuca pylibravatar py3dns psycopg2-binary phply django-redis hiredis
|
||||
# specific to YunoHost package:
|
||||
pip install django_sendmail_backend
|
||||
# Fix ImportError: No module named backports
|
||||
pip install backports.csv
|
||||
)
|
||||
))
|
||||
|
||||
#=================================================
|
||||
# CONFIG FILE UPGRADE
|
||||
|
|
@ -374,4 +372,4 @@ systemctl start "uwsgi-app@$app.service"
|
|||
# RELOAD NGINX
|
||||
#=================================================
|
||||
|
||||
systemctl reload nginx
|
||||
systemctl reload nginx
|
||||
Loading…
Reference in a new issue