From 9cf6fd0e89d0829825a1a06ebd576db744123ca4 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Fri, 19 Oct 2018 23:28:40 +0200 Subject: [PATCH] migrate to python3, add celery and redis --- README.md | 2 -- conf/celery-weblate | 27 +++++++++++++++ conf/celery-weblate.service | 23 +++++++++++++ conf/settings_history/settings.3.2.py | 18 ++++++---- conf/uwsgi.ini | 4 +-- scripts/_common.sh | 47 ++++++++++++++++++++++++++- scripts/backup | 6 ++++ scripts/install | 36 +++++++++++++------- scripts/remove | 11 ++++++- scripts/restore | 16 ++++++--- scripts/upgrade | 14 ++++---- 11 files changed, 168 insertions(+), 36 deletions(-) create mode 100644 conf/celery-weblate create mode 100644 conf/celery-weblate.service diff --git a/README.md b/README.md index b4ba459..5e95f42 100644 --- a/README.md +++ b/README.md @@ -63,13 +63,11 @@ It doesn't work yet, but while [it looks doable](https://docs.weblate.org/en/lat [ ] ARM support * to be added: -[ ] strenghen uwsgi systemd security (https://github.com/YunoHost-Apps/kresus_ynh/pull/21/commits/0ce4979d5037b111aa9698ed5552135bf0ed7165) [ ] use jq instead of grep/sed [ ] change URL script [ ] use debian package for lxml (may unlock ARM support) [ ] Add configuration options using the YunoHost interface (https://forum.yunohost.org/t/yunohost-3-1-minor-stable-release-version-stable-mineure/5445) [ ] Add fail2ban script -[ ] Use redis+hiredis for cache? [ ] Enable CHECK_LIST? [ ] Enable AUTOFIX_LIST? [ ] Enable Translation Memory? diff --git a/conf/celery-weblate b/conf/celery-weblate new file mode 100644 index 0000000..b1b3091 --- /dev/null +++ b/conf/celery-weblate @@ -0,0 +1,27 @@ +# Name of nodes to start +# here we have a single node +CELERYD_NODES="w1" +# or we could have three nodes: +#CELERYD_NODES="w1 w2 w3" + +# Absolute or relative path to the 'celery' command: +CELERY_BIN="/usr/bin/celery" + +# App instance to use +# comment out this line if you don't use an app +CELERY_APP="weblate" + +# How to call manage.py +CELERYD_MULTI="multi" + +# Extra command-line arguments to the worker +CELERYD_OPTS="--beat" + +# - %n will be replaced with the first part of the nodename. +# - %I will be replaced with the current child process index +# and is important when using the prefork pool to avoid race conditions. +CELERYD_PID_FILE="/var/run/celery-__APP__/weblate-%n.pid" +CELERYD_LOG_FILE="/var/log/uwsgi/app/__APP__-celery-%n%I.log" +CELERYD_LOG_LEVEL="INFO" + +CELERY_WORKER_RUNNING="1" \ No newline at end of file diff --git a/conf/celery-weblate.service b/conf/celery-weblate.service new file mode 100644 index 0000000..59e566e --- /dev/null +++ b/conf/celery-weblate.service @@ -0,0 +1,23 @@ +[Unit] +Description=Celery Service for Weblate (__APP__) +After=network.target + +[Service] +Type=forking +User=__APP__ +Group=__APP__ +PermissionsStartOnly=true +EnvironmentFile=__FINALPATH__/celery-weblate +Environment=PATH=__FINALPATH__/venv/bin:$PATH +WorkingDirectory=__FINALPATH__/ +ExecStart=/bin/sh -c '${CELERY_BIN} multi start ${CELERYD_NODES} \ + -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} \ + --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}' +ExecStop=/bin/sh -c '${CELERY_BIN} multi stopwait ${CELERYD_NODES} \ + --pidfile=${CELERYD_PID_FILE}' +ExecReload=/bin/sh -c '${CELERY_BIN} multi restart ${CELERYD_NODES} \ + -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} \ + --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}' + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/conf/settings_history/settings.3.2.py b/conf/settings_history/settings.3.2.py index 435b688..d614042 100644 --- a/conf/settings_history/settings.3.2.py +++ b/conf/settings_history/settings.3.2.py @@ -56,7 +56,7 @@ BASE_DIR = '__FINALPATH__' # Data directory DATA_DIR = os.path.join(BASE_DIR, 'data') -TTF_PATH = '__FINALPATH__/venv/lib/python2.7/site-packages/weblate/ttf/' +TTF_PATH = '__FINALPATH__/venv/lib/python3.5/site-packages/weblate/ttf/' # Local time zone for this installation. Choices can be found here: # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name @@ -715,8 +715,12 @@ ALLOWED_HOSTS = ['__DOMAIN__'] CACHES = { 'default': { - 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', - 'LOCATION': '127.0.0.1:__MEMCPORT__', + 'BACKEND': 'django_redis.cache.RedisCache', + 'LOCATION': 'redis://127.0.0.1:6379/_REDIS_DB__', + 'OPTIONS': { + 'CLIENT_CLASS': 'django_redis.client.DefaultClient', + 'PARSER_CLASS': 'redis.connection.HiredisParser', + } }, 'avatar': { 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', @@ -806,11 +810,11 @@ REST_FRAMEWORK = { # ) # Celery worker configuration for testing -CELERY_TASK_ALWAYS_EAGER = True -CELERY_BROKER_URL = 'memory://' +# CELERY_TASK_ALWAYS_EAGER = True +# CELERY_BROKER_URL = 'memory://' # Celery worker configuration for production -# CELERY_TASK_ALWAYS_EAGER = False -# CELERY_BROKER_URL = 'redis://localhost:6379' +CELERY_TASK_ALWAYS_EAGER = False +CELERY_BROKER_URL = 'redis://localhost:6379/__REDIS_DB__' # Celery settings, it is not recommended to change these CELERY_WORKER_PREFETCH_MULTIPLIER = 0 diff --git a/conf/uwsgi.ini b/conf/uwsgi.ini index 2c3929e..aff091c 100644 --- a/conf/uwsgi.ini +++ b/conf/uwsgi.ini @@ -1,12 +1,12 @@ [uwsgi] -plugins = python +plugins = python3 master = true protocol = uwsgi socket = /var/run/uwsgi/__APP__.socket virtualenv = __FINALPATH__/venv # http://uwsgi-docs.readthedocs.io/en/latest/Nginx.html#hosting-multiple-apps-in-the-same-process-aka-managing-script-name-and-path-info -mount = __PATH__=__FINALPATH__/venv/lib/python2.7/site-packages/weblate/wsgi.py +mount = __PATH__=__FINALPATH__/venv/lib/python3.5/site-packages/weblate/wsgi.py manage-script-name = true # Needed for OAuth/OpenID diff --git a/scripts/_common.sh b/scripts/_common.sh index 6b10cee..2c5e87c 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -91,8 +91,8 @@ weblate_fill_settings() { ynh_replace_string "__DOMAIN__" "$domain" "$settings" ynh_replace_string "__KEY__" "$key" "$settings" ynh_replace_string "__FINALPATH__" "$final_path" "$settings" - ynh_replace_string "__MEMCPORT__" "$memc_port" "$settings" ynh_replace_string "__GITHUBUSER__" "$github_account" "$settings" + ynh_replace_string "__REDIS_DB__" "$redis_db" "$settings" # root install as an empty PATHURL to prevent '//static' if [ "$path_url" == "/" ] @@ -177,3 +177,48 @@ $(yunohost tools diagnosis | grep -B 100 "services:" | sed '/services:/d')" # Send the email to the recipients echo "$mail_message" | $mail_bin -a "Content-Type: text/plain; charset=UTF-8" -s "$mail_subject" "$recipients" } + +#================================================= +# +# Redis HELPERS +# +# Point of contact : Jean-Baptiste Holcroft +#================================================= + +# get the first available redis database +# +# usage: ynh_redis_get_free_db +# | returns: the database number to use +ynh_redis_get_free_db() { + local result max db + result=$(redis-cli INFO keyspace) + + # get the num + max=$(cat /etc/redis/redis.conf | grep ^databases | grep -Eow "[0-9]+") + + db=0 + # default Debian setting is 15 databases + for i in $(seq 0 "$max") + do + if ! echo "$result" | grep -q "db$i" + then + db=$i + break 1 + fi + db=-1 + done + + test "$db" -eq -1 && ynh_die "No available Redis databases..." + + echo "$db" +} + +# Create a master password and set up global settings +# Please always call this script in install and restore scripts +# +# usage: ynh_redis_remove_db database +# | arg: database - the database to erase +ynh_redis_remove_db() { + local db=$1 + redis-cli -n "$db" flushall +} \ No newline at end of file diff --git a/scripts/backup b/scripts/backup index e159b99..3578fbe 100755 --- a/scripts/backup +++ b/scripts/backup @@ -56,6 +56,12 @@ ynh_backup "db.sql" ynh_backup "/etc/cron.d/$app" +#================================================= +# BACKUP THE CRON FILE +#================================================= + +ynh_backup "/usr/lib/tmpfiles.d/$app.conf" + #================================================= # BACKUP THE uwsgi files #================================================= diff --git a/scripts/install b/scripts/install index e3a6fce..d382546 100755 --- a/scripts/install +++ b/scripts/install @@ -77,9 +77,9 @@ ynh_app_setting_set "$app" github_token "$github_token" #================================================= ynh_install_app_dependencies libxml2-dev libxslt-dev libfreetype6-dev \ - libjpeg-dev libz-dev libyaml-dev python-dev python-pip python-virtualenv \ - postgresql libpq-dev uwsgi uwsgi-plugin-python memcached \ - mailutils + libjpeg-dev libz-dev libyaml-dev python3-dev python3-pip python3-virtualenv \ + postgresql libpq-dev uwsgi uwsgi-plugin-python3 \ + mailutils python-celery-common virtualenv redis-server #================================================= # CREATE A PostgreSQL DATABASE @@ -185,7 +185,7 @@ fi #================================================= # PIP INSTALLATION #================================================= -virtualenv "${final_path}/venv" +virtualenv --python=python3 "${final_path}/venv" #run source in a 'sub shell' ( set +o nounset @@ -195,11 +195,9 @@ virtualenv "${final_path}/venv" # prevent error: "command: 'install_requires' must be a string or list of strings containing valid project/version requirement specifiers" pip install --upgrade setuptools pip install Weblate=="$current_version" - pip install pytz python-bidi PyYaML Babel pyuca pylibravatar pydns psycopg2-binary python-memcached phply + pip install pytz python-bidi PyYaML Babel pyuca pylibravatar py3dns psycopg2-binary phply django-redis hiredis # specific to YunoHost package: pip install django_sendmail_backend - # Fix ImportError: No module named backports - pip install backports.csv ) #================================================= @@ -210,12 +208,13 @@ virtualenv "${final_path}/venv" db_pwd=$(ynh_app_setting_get "$app" psqlpwd) admin_mail=$(ynh_user_get_info "$admin" mail) key=$(ynh_string_random 24)$(ynh_string_random 24)$(ynh_string_random 2) -memc_port=$(ynh_find_port 8080) -settings="$final_path/venv/lib/python2.7/site-packages/weblate/settings.py" +redis_db=$(ynh_redis_get_free_db) + +settings="$final_path/venv/lib/python3.5/site-packages/weblate/settings.py" cp "../conf/settings_history/settings.$current_version.py" "$settings" weblate_fill_settings "$settings" -ynh_app_setting_set "$app" memc_port "$memc_port" +ynh_app_setting_set "$app" redis_db "$redis_db" #================================================= # SPECIFIC SETUP Filling up the database @@ -249,7 +248,21 @@ ynh_replace_string "__FINALPATH__" "$final_path/" "/etc/cron.d/$app" #================================================= # Calculate and store the config file checksum into the app settings -ynh_store_file_checksum "$final_path/venv/lib/python2.7/site-packages/weblate/settings.py" +ynh_store_file_checksum "$final_path/venv/lib/python3.5/site-packages/weblate/settings.py" + +#================================================= +# ACTIVATE CELERY +#================================================= + +celeryconf="$final_path/celery-weblate" +cp ../conf/celery-weblate "$celeryconf" + +ynh_replace_string "__APP__" "$app" "$celeryconf" + +echo "d /var/run/celery-$app 0775 $app $app" > "/usr/lib/tmpfiles.d/$app.conf" +systemd-tmpfiles --create + +ynh_add_systemd_config "$app-celery" "celery-weblate.service" #================================================= # GENERIC FINALIZATION @@ -287,6 +300,7 @@ fi # Start weblate #================================================= +systemctl start "$app-celery" systemctl start "uwsgi-app@$app.service" #================================================= diff --git a/scripts/remove b/scripts/remove index 841b4ff..6adfad6 100755 --- a/scripts/remove +++ b/scripts/remove @@ -18,10 +18,11 @@ domain=$(ynh_app_setting_get "$app" domain) db_name=$(ynh_app_setting_get "$app" db_name) #================================================= -# STOP WEBLATE'S SERVICE +# STOP WEBLATE'S SERVICES #================================================= systemctl stop "uwsgi-app@$app.service" +systemctl stop "$app-celery.service" #================================================= # REMOVE THE PostgreSQL DATABASE @@ -60,11 +61,19 @@ ynh_remove_nginx_config # Remove a cron file ynh_secure_remove "/etc/cron.d/$app" +#================================================= +# REMOVE TMPFILES.D +#================================================= + +ynh_secure_remove "/var/run/celery-$app" +ynh_secure_remove "/usr/lib/tmpfiles.d/$app.conf" + #================================================= # REMOVE uwsgi and systemd files #================================================= ynh_remove_uwsgi_service +ynh_remove_systemd_config "$app-celery" #================================================= # GENERIC FINALIZATION diff --git a/scripts/restore b/scripts/restore index a5518bd..ae0ea38 100755 --- a/scripts/restore +++ b/scripts/restore @@ -75,9 +75,9 @@ chown -R "$app": "$final_path" #================================================= ynh_install_app_dependencies libxml2-dev libxslt-dev libfreetype6-dev \ - libjpeg-dev libz-dev libyaml-dev python-dev python-pip python-virtualenv \ - postgresql libpq-dev uwsgi uwsgi-plugin-python memcached \ - mailutils + libjpeg-dev libz-dev libyaml-dev python3-dev python3-pip python3-virtualenv \ + postgresql libpq-dev uwsgi uwsgi-plugin-python3 memcached \ + mailutils python-celery-common virtualenv redis-server #================================================= # RESTORE THE PostgreSQL DATABASE @@ -123,6 +123,13 @@ yunohost service add "uwsgi-app@$app.service" --log "/var/log/uwsgi/app/$app" ynh_restore_file "/etc/cron.d/$app" +#================================================= +# RESTORE THE TMPFILES.D +#================================================= + +ynh_restore_file "/usr/lib/tmpfiles.d/$app.conf" +systemd-tmpfiles --create + #================================================= # RESTORE THE HUB BINARY FILE #================================================= @@ -135,10 +142,11 @@ ynh_restore_file "/usr/bin/hub" # Start weblate #================================================= +systemctl start "$app-celery" systemctl start "uwsgi-app@$app.service" #================================================= # RELOAD NGINX & uwsgi #================================================= -systemctl reload nginx +systemctl reload nginx \ No newline at end of file diff --git a/scripts/upgrade b/scripts/upgrade index a91a3c0..90a2760 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -186,9 +186,9 @@ chsh --shell /bin/bash "$app" #================================================= ynh_install_app_dependencies libxml2-dev libxslt-dev libfreetype6-dev \ - libjpeg-dev libz-dev libyaml-dev python-dev python-pip python-virtualenv \ - postgresql libpq-dev uwsgi uwsgi-plugin-python memcached \ - mailutils + libjpeg-dev libz-dev libyaml-dev python3-dev python3-pip python3-virtualenv \ + postgresql libpq-dev uwsgi uwsgi-plugin-python3 memcached \ + mailutils python-celery-common virtualenv redis-server #================================================= # SPECIFIC SETUP uwsgi @@ -215,12 +215,10 @@ fi # prevent error: "command: 'install_requires' must be a string or list of strings containing valid project/version requirement specifiers" pip install --upgrade setuptools pip install Weblate=="$current_version" - pip install pytz python-bidi PyYaML Babel pyuca pylibravatar pydns psycopg2-binary python-memcached phply + pip install pytz python-bidi PyYaML Babel pyuca pylibravatar py3dns psycopg2-binary phply django-redis hiredis # specific to YunoHost package: pip install django_sendmail_backend - # Fix ImportError: No module named backports - pip install backports.csv -) +)) #================================================= # CONFIG FILE UPGRADE @@ -374,4 +372,4 @@ systemctl start "uwsgi-app@$app.service" # RELOAD NGINX #================================================= -systemctl reload nginx +systemctl reload nginx \ No newline at end of file