From 3adff298487e4fa53f33519a197689a564a9806f Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Mon, 25 Feb 2019 23:09:47 +0100 Subject: [PATCH] upgrade to 3.4 (and remove systemd template) --- conf/celery-weblate.service | 6 +- conf/nginx.conf | 28 +++++----- conf/settings_history/settings.3.3.py | 1 - conf/settings_history/settings.3.4.py | 30 ++++++++-- conf/uwsgi-app@.service | 21 ------- conf/uwsgi.ini | 11 +++- conf/weblate.service | 33 +++++++++++ manifest.json | 4 +- scripts/_common.sh | 80 +-------------------------- scripts/install | 43 +++++++++----- scripts/remove | 33 +++++++---- scripts/upgrade | 14 +++++ 12 files changed, 155 insertions(+), 149 deletions(-) delete mode 100644 conf/uwsgi-app@.service create mode 100644 conf/weblate.service diff --git a/conf/celery-weblate.service b/conf/celery-weblate.service index 87be424..a0f4a98 100644 --- a/conf/celery-weblate.service +++ b/conf/celery-weblate.service @@ -9,10 +9,12 @@ Group=__APP__ PermissionsStartOnly=true EnvironmentFile=__FINALPATH__/celery-weblate WorkingDirectory=__FINALPATH__/ -ExecStartPre=/bin/mkdir -p /var/run/__APP__-celery -ExecStartPre=/bin/chown -R __APP__ /var/run/__APP__-celery +# https://www.freedesktop.org/software/systemd/man/systemd.exec.html#RuntimeDirectory= +RuntimeDirectory=__APP__-celery + ExecStartPre=/bin/mkdir -p /var/log/__APP__-celery ExecStartPre=/bin/chown -R __APP__ /var/log/__APP__-celery + ExecStart=__FINALPATH__/venv/bin/celery multi start ${CELERYD_NODES} \ -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} \ --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS} diff --git a/conf/nginx.conf b/conf/nginx.conf index 25c4237..f13dd41 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -1,17 +1,4 @@ -location __PATH__ { - #Source: https://docs.weblate.org/en/latest/admin/install.html#sample-configuration-for-nginx-and-uwsgi - # Path to source - alias __FINALPATH__/ ; - - include uwsgi_params; - # Needed for long running operations in admin interface - uwsgi_read_timeout 3600; - uwsgi_pass unix:///var/run/uwsgi/__NAME__.socket; - - # Include SSOWAT user panel. - include conf.d/yunohost_panel.conf.inc; -} - +# https://github.com/WeblateOrg/weblate/blob/master/examples/weblate.nginx.conf location ^__PATH__/favicon.ico$ { # DATA_DIR/static/favicon.ico alias __FINALPATH__/data/static/favicon.ico; @@ -35,3 +22,16 @@ location __PATH__/media/ { alias __FINALPATH__/data/media/; expires 30d; } + +location __PATH__ { + # Path to source + alias __FINALPATH__/ ; + + include uwsgi_params; + # Needed for long running operations in admin interface + uwsgi_read_timeout 3600; + uwsgi_pass unix:///var/run/__NAME__/socket; + + # Include SSOWAT user panel. + include conf.d/yunohost_panel.conf.inc; +} \ No newline at end of file diff --git a/conf/settings_history/settings.3.3.py b/conf/settings_history/settings.3.3.py index f2e4562..3f69df4 100644 --- a/conf/settings_history/settings.3.3.py +++ b/conf/settings_history/settings.3.3.py @@ -832,4 +832,3 @@ CELERY_WORKER_PREFETCH_MULTIPLIER = 0 CELERY_BEAT_SCHEDULE_FILENAME = os.path.join( DATA_DIR, 'celery', 'beat-schedule' ) - diff --git a/conf/settings_history/settings.3.4.py b/conf/settings_history/settings.3.4.py index f2e4562..6edec99 100644 --- a/conf/settings_history/settings.3.4.py +++ b/conf/settings_history/settings.3.4.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright © 2012 - 2018 Michal Čihař +# Copyright © 2012 - 2019 Michal Čihař # # This file is part of Weblate # @@ -488,11 +488,16 @@ LOGGING = { 'handlers': [DEFAULT_LOG], 'level': 'DEBUG', }, + # Logging search operations + 'weblate.search': { + 'handlers': [DEFAULT_LOG], + 'level': 'INFO', + }, # Logging VCS operations - # 'weblate-vcs': { - # 'handlers': [DEFAULT_LOG], - # 'level': 'DEBUG', - # }, + 'weblate.vcs': { + 'handlers': [DEFAULT_LOG], + 'level': 'WARNING', + }, # Python Social Auth logging # 'social': { # 'handlers': [DEFAULT_LOG], @@ -710,6 +715,7 @@ CRISPY_TEMPLATE_PACK = 'bootstrap3' # 'weblate.addons.generate.GenerateFileAddon', # 'weblate.addons.json.JSONCustomizeAddon', # 'weblate.addons.properties.PropertiesSortAddon', +# 'weblate.addons.git.GitSquashAddon', # ) # E-mail address that error messages come from. @@ -818,6 +824,13 @@ REST_FRAMEWORK = { # r'/legal/(.*)$', # Optional for legal app # ) +# Silence some of the Django system checks +SILENCED_SYSTEM_CHECKS = [ + # We have modified django.contrib.auth.middleware.AuthenticationMiddleware + # as weblate.accounts.middleware.AuthenticationMiddleware + 'admin.E408', +] + # Celery worker configuration for testing # CELERY_TASK_ALWAYS_EAGER = True # CELERY_BROKER_URL = 'memory://' @@ -833,3 +846,10 @@ CELERY_BEAT_SCHEDULE_FILENAME = os.path.join( DATA_DIR, 'celery', 'beat-schedule' ) +CELERY_TASK_ROUTES = { + 'weblate.trans.search.*': {'queue': 'search'}, + 'weblate.trans.tasks.optimize_fulltext': {'queue': 'search'}, + 'weblate.trans.tasks.cleanup_fulltext': {'queue': 'search'}, + 'weblate.memory.tasks.*': {'queue': 'memory'}, +} + diff --git a/conf/uwsgi-app@.service b/conf/uwsgi-app@.service deleted file mode 100644 index 8b316e2..0000000 --- a/conf/uwsgi-app@.service +++ /dev/null @@ -1,21 +0,0 @@ -[Unit] -Description=%i uWSGI app -After=syslog.target - -[Service] -ExecStart=/usr/bin/uwsgi \ - --ini /etc/uwsgi/apps-available/%i.ini \ - --socket /var/run/uwsgi/%i.socket \ - --chmod-socket=775 \ - --logto /var/log/uwsgi/app/%i \ - --processes 4 -User=%i -Group=www-data -Restart=on-failure -KillSignal=SIGQUIT -Type=notify -StandardError=syslog -NotifyAccess=all - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/conf/uwsgi.ini b/conf/uwsgi.ini index aff091c..1218909 100644 --- a/conf/uwsgi.ini +++ b/conf/uwsgi.ini @@ -2,7 +2,7 @@ plugins = python3 master = true protocol = uwsgi -socket = /var/run/uwsgi/__APP__.socket +socket = /var/run/__APP__/socket virtualenv = __FINALPATH__/venv # http://uwsgi-docs.readthedocs.io/en/latest/Nginx.html#hosting-multiple-apps-in-the-same-process-aka-managing-script-name-and-path-info @@ -16,3 +16,12 @@ buffer-size = 8192 enable-threads = true # Child processes do not need file descriptors close-on-exec = true + +# Run as weblate user +#uid = __APP__ +#gid = www-data + +# Do not log some errors caused by client disconnects +ignore-sigpipe = true +ignore-write-errors = true +disable-write-exception = true \ No newline at end of file diff --git a/conf/weblate.service b/conf/weblate.service new file mode 100644 index 0000000..e9a947d --- /dev/null +++ b/conf/weblate.service @@ -0,0 +1,33 @@ +[Unit] +Description=__APP__ uWSGI app +After=syslog.target + +[Service] +Type=forking +User=__APP__ +Group=www-data +PermissionsStartOnly=true + + +# https://www.freedesktop.org/software/systemd/man/systemd.exec.html#RuntimeDirectory= +RuntimeDirectory=__APP__ + +ExecStartPre=/bin/mkdir -p /var/log/uwsgi/app/__APP__ +ExecStartPre=/bin/chown -R __APP__ /var/log/uwsgi/app/__APP__ + +ExecStart=/usr/bin/uwsgi \ + --ini /etc/uwsgi/apps-available/__APP__.ini \ + --socket /var/run/__APP__/socket \ + --chmod-socket=775 \ + --logto /var/log/uwsgi/app/__APP__/weblate \ + --processes 4 + + +Restart=on-failure +KillSignal=SIGQUIT +Type=notify +StandardError=syslog +NotifyAccess=all + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/manifest.json b/manifest.json index 68baf84..4c2ed63 100644 --- a/manifest.json +++ b/manifest.json @@ -3,12 +3,12 @@ "id": "weblate", "packaging_format": 1, "requirements": { - "yunohost": ">= 3.3.0" + "yunohost": ">= 3.4.0" }, "description": { "en": "A translation platform using Git and Python" }, - "version": "3.3~ynh1", + "version": "3.4~ynh1", "url": "https://weblate.org", "license": "AGPL-3.0", "maintainer": { diff --git a/scripts/_common.sh b/scripts/_common.sh index 9684200..0106d5c 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -1,84 +1,6 @@ #!/bin/bash -current_version="3.3" - -ynh_check_global_uwsgi_config () { - uwsgi --version || ynh_die "You need to add uwsgi (and appropriate plugin) as a dependency" - - cp ../conf/uwsgi-app@.service /etc/systemd/system/uwsgi-app@.service - - # make sure the folder for sockets exists and set authorizations - # make sure it exists on every startup - echo "d /var/run/uwsgi 0775 root www-data" > /usr/lib/tmpfiles.d/uwsgi.conf - systemd-tmpfiles --create - - # make sure the folder for logs exists and set authorizations - mkdir -p /var/log/uwsgi/app/ - chown root:www-data /var/log/uwsgi/app/ - chmod -R 775 /var/log/uwsgi/app/ -} - -# Create a dedicated uwsgi ini file to use with generic uwsgi service -# It will install generic uwsgi.socket and -# -# This will use a template in ../conf/uwsgi.ini -# and will replace the following keywords with -# global variables that should be defined before calling -# this helper : -# -# __APP__ by $app -# __PATH__ by $path_url -# __FINALPATH__ by $final_path -# -# usage: ynh_add_systemd_config -# -# to interact with your service: `systemctl uwsgi-app@app` -ynh_add_uwsgi_service () { - ynh_check_global_uwsgi_config - - # www-data group is needed since it is this nginx who will start the service - usermod --append --groups www-data "$app" || ynh_die "It wasn't possible to add user $app to group www-data" - - finaluwsgiini="/etc/uwsgi/apps-available/$app.ini" - ynh_backup_if_checksum_is_different "$finaluwsgiini" - cp ../conf/uwsgi.ini "$finaluwsgiini" - - # To avoid a break by set -u, use a void substitution ${var:-}. If the variable is not set, it's simply set with an empty variable. - # Substitute in a nginx config file only if the variable is not empty - if test -n "${final_path:-}"; then - ynh_replace_string "__FINALPATH__" "$final_path" "$finaluwsgiini" - fi - if test -n "${path_url:-}"; then - ynh_replace_string "__PATH__" "$path_url" "$finaluwsgiini" - fi - if test -n "${app:-}"; then - ynh_replace_string "__APP__" "$app" "$finaluwsgiini" - fi - ynh_store_file_checksum "$finaluwsgiini" - - chown root: "$finaluwsgiini" - - systemctl daemon-reload - systemctl enable "uwsgi-app@$app.service" - - # Add as a service - yunohost service add "uwsgi-app@$app.service" --log "/var/log/uwsgi/app/$app" -} - -# Remove the dedicated uwsgi ini file -# -# usage: ynh_remove_systemd_config -ynh_remove_uwsgi_service () { - finaluwsgiini="/etc/uwsgi/apps-available/$app.ini" - if [ -e "$finaluwsgiini" ]; then - systemctl stop "uwsgi-app@$app.service" - systemctl disable "uwsgi-app@$app.service" - yunohost service remove "uwsgi-app@$app.service" - - ynh_secure_remove "$finaluwsgiini" - ynh_secure_remove "/var/log/uwsgi/app/$app" - fi -} +current_version="3.4" weblate_fill_settings() { diff --git a/scripts/install b/scripts/install index c6236b8..2bd0b7f 100755 --- a/scripts/install +++ b/scripts/install @@ -169,19 +169,6 @@ EOF #================================================= # SPECIFIC SETUP -#================================================= -# SPECIFIC SETUP uwsgi -#================================================= - -ynh_add_uwsgi_service - -# root install doesn't require uwsgi to handle script names -if [ "$path_url" == "/" ] -then - ynh_replace_string "manage-script-name = true" "manage-script-name = false" "$finaluwsgiini" - ynh_store_file_checksum "$finaluwsgiini" -fi - #================================================= # PIP INSTALLATION #================================================= @@ -250,6 +237,27 @@ ynh_replace_string "__FINALPATH__" "$final_path/" "/etc/cron.d/$app" # Calculate and store the config file checksum into the app settings ynh_store_file_checksum "$final_path/venv/lib/python3.5/site-packages/weblate/settings.py" +# SPECIFIC SETUP uwsgi +#================================================= + +finaluwsgiini="/etc/uwsgi/apps-available/$app.ini" +cp ../conf/uwsgi.ini "$finaluwsgiini" + +ynh_replace_string "__FINALPATH__" "$final_path" "$finaluwsgiini" +ynh_replace_string "__PATH__" "$path_url" "$finaluwsgiini" +ynh_replace_string "__APP__" "$app" "$finaluwsgiini" + + +# root install doesn't require uwsgi to handle script names +if [ "$path_url" == "/" ] +then + ynh_replace_string "manage-script-name = true" "manage-script-name = false" "$finaluwsgiini" +fi + +ynh_add_systemd_config "$app" "weblate.service" + +ynh_store_file_checksum "$finaluwsgiini" + #================================================= # ACTIVATE CELERY #================================================= @@ -294,12 +302,19 @@ then ynh_store_file_checksum "$finalnginxconf" fi +#================================================= +# ADVERTISE SERVICES IN ADMIN PANEL +#================================================= + +yunohost service add "$app.service" --log "/var/log/uwsgi/app/$app" +yunohost service add "$app-celery.service" --log "/var/log/$app-celery" + #================================================= # Start weblate #================================================= +systemctl start "$app.service" systemctl start "$app-celery" -systemctl start "uwsgi-app@$app.service" #================================================= # RELOAD NGINX diff --git a/scripts/remove b/scripts/remove index b341da1..74e3273 100755 --- a/scripts/remove +++ b/scripts/remove @@ -17,12 +17,32 @@ app=$YNH_APP_INSTANCE_NAME domain=$(ynh_app_setting_get "$app" domain) db_name=$(ynh_app_setting_get "$app" db_name) +#================================================= +# STANDARD REMOVE +#================================================= +# REMOVE SERVICE FROM ADMIN PANEL +#================================================= + +# Remove a service from the admin panel, added by `yunohost service add` +if yunohost service status | grep -q "$app" +then + echo "Remove $app service" + yunohost service remove "$app" +fi + +# Remove a service from the admin panel, added by `yunohost service add` +if yunohost service status | grep -q "$app-celery" +then + echo "Remove $app service" + yunohost service remove "$app-celery" +fi + #================================================= # STOP WEBLATE'S SERVICES #================================================= -systemctl stop "uwsgi-app@$app.service" -systemctl stop "$app-celery.service" +ynh_remove_systemd_config "$app" +ynh_remove_systemd_config "$app-celery" #================================================= # REMOVE THE PostgreSQL DATABASE @@ -67,13 +87,6 @@ ynh_secure_remove "/etc/cron.d/$app" ynh_secure_remove "/var/run/$app-celery" -#================================================= -# REMOVE uwsgi and systemd files -#================================================= - -ynh_remove_uwsgi_service -ynh_remove_systemd_config "$app-celery" - #================================================= # GENERIC FINALIZATION #================================================= @@ -81,4 +94,4 @@ ynh_remove_systemd_config "$app-celery" #================================================= # Delete a system user -ynh_system_user_delete "$app" +ynh_system_user_delete "$app" \ No newline at end of file diff --git a/scripts/upgrade b/scripts/upgrade index 767f4df..abd3ec1 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -177,6 +177,20 @@ path_url=$(ynh_normalize_url_path "$path_url") # STANDARD UPGRADE STEPS #================================================= +finaluwsgiini="/etc/uwsgi/apps-available/$app.ini" +if [ -e "$finaluwsgiini" ]; then + systemctl stop "uwsgi-app@$app.service" + systemctl disable "uwsgi-app@$app.service" + yunohost service remove "uwsgi-app@$app.service" + + ynh_secure_remove "$finaluwsgiini" + ynh_secure_remove "/var/log/uwsgi/app/$app" +fi + +#================================================= +# STANDARD UPGRADE STEPS +#================================================= + # Create a dedicated nginx config ynh_add_nginx_config