From f67986641f48d9e7b5df99f13fefe460094a5d81 Mon Sep 17 00:00:00 2001 From: Jimmy Monin Date: Fri, 4 May 2018 12:51:16 +0200 Subject: [PATCH] Use unicorn instead of puma, hence not needing a separate sidekiq service and drastically reducing memory footprint Limit number of workers to 2 on ARM devices See here: https://meta.discourse.org/t/does-discourse-require-2gb-to-work-lag-free/25278/8 --- conf/discourse-sidekiq.service | 15 ------ conf/nginx.conf | 12 ++--- ...discourse-puma.service => systemd.service} | 9 ++-- scripts/backup | 9 +--- scripts/install | 52 +++++++------------ scripts/remove | 6 +-- scripts/restore | 17 +++--- scripts/upgrade | 37 ++++++------- 8 files changed, 55 insertions(+), 102 deletions(-) delete mode 100644 conf/discourse-sidekiq.service rename conf/{discourse-puma.service => systemd.service} (54%) diff --git a/conf/discourse-sidekiq.service b/conf/discourse-sidekiq.service deleted file mode 100644 index a226b1b..0000000 --- a/conf/discourse-sidekiq.service +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=__APP__ sidekiq service -After=network.target - -[Service] -User=__APP__ -Group=__APP__ -WorkingDirectory=__FINALPATH__ -Environment=RAILS_ENV=production -ExecStart=__RBENVROOT__/shims/bundle exec sidekiq -C config/sidekiq.yml -Restart=always -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/conf/nginx.conf b/conf/nginx.conf index 516893d..c8b637d 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -56,7 +56,7 @@ proxy_set_header X-Request-Start "t=${msec}"; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto https; - proxy_pass http://unix:__FINALPATH__/tmp/sockets/puma.sock; + proxy_pass http://unix:__FINALPATH__/tmp/sockets/unicorn.sock; break; } @@ -123,7 +123,7 @@ try_files $uri =404; } - proxy_pass http://unix:__FINALPATH__/tmp/sockets/puma.sock; + proxy_pass http://unix:__FINALPATH__/tmp/sockets/unicorn.sock; break; } @@ -135,7 +135,7 @@ proxy_set_header X-Forwarded-Proto https; proxy_set_header X-Sendfile-Type X-Accel-Redirect; proxy_set_header X-Accel-Mapping __FINALPATH__/public/=/downloads/; - proxy_pass http://unix:__FINALPATH__/tmp/sockets/puma.sock; + proxy_pass http://unix:__FINALPATH__/tmp/sockets/unicorn.sock; break; } @@ -158,7 +158,7 @@ # proxy_cache one; proxy_cache_valid 200 301 302 7d; proxy_cache_valid any 1m; - proxy_pass http://unix:__FINALPATH__/tmp/sockets/puma.sock; + proxy_pass http://unix:__FINALPATH__/tmp/sockets/unicorn.sock; break; } @@ -192,7 +192,7 @@ proxy_set_header X-Forwarded-Proto https; proxy_http_version 1.1; proxy_buffering off; - proxy_pass http://unix:__FINALPATH__/tmp/sockets/puma.sock; + proxy_pass http://unix:__FINALPATH__/tmp/sockets/unicorn.sock; break; } @@ -212,5 +212,5 @@ proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto https; - proxy_pass http://unix:__FINALPATH__/tmp/sockets/puma.sock; + proxy_pass http://unix:__FINALPATH__/tmp/sockets/unicorn.sock; } diff --git a/conf/discourse-puma.service b/conf/systemd.service similarity index 54% rename from conf/discourse-puma.service rename to conf/systemd.service index 1a06240..359d689 100644 --- a/conf/discourse-puma.service +++ b/conf/systemd.service @@ -1,11 +1,9 @@ [Unit] -Description=__APP__ puma service +Description=__APP__ service Wants=postgresql.service Wants=redis-server.service After=redis-server.service After=postgresql.service -After=discourse-sidekiq.service -Requires=discourse-sidekiq.service [Service] User=__APP__ @@ -13,9 +11,8 @@ Group=__APP__ WorkingDirectory=__FINALPATH__ Environment=__ADDITIONAL_ENV__ Environment=RAILS_ENV=production -ExecStart=__RBENVROOT__/shims/bundle exec puma --config config/puma.rb -e production -ExecStop=__RBENVROOT__/shims/bundle exec pumactl stop -RemainAfterExit=true +Environment=LD_PRELOAD=__LIBJEMALLOC__ +ExecStart=__RBENVROOT__/shims/bundle exec unicorn --config config/unicorn.conf.rb -E production Restart=always RestartSec=10 diff --git a/scripts/backup b/scripts/backup index 6a84785..1250b27 100644 --- a/scripts/backup +++ b/scripts/backup @@ -48,17 +48,10 @@ ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" # BACKUP SYSTEMD #================================================= -ynh_backup "/etc/systemd/system/$app-puma.service" -ynh_backup "/etc/systemd/system/$app-sidekiq.service" +ynh_backup "/etc/systemd/system/$app.service" #================================================= # BACKUP THE POSTGRESQL DATABASE #================================================= ynh_psql_dump_db "$db_name" > ${YNH_CWD}/db.sql - -#================================================= -# BACKUP LOG FILE -#================================================= - -ynh_backup "/var/log/$app" diff --git a/scripts/install b/scripts/install index 6061282..7048e49 100644 --- a/scripts/install +++ b/scripts/install @@ -161,19 +161,14 @@ ynh_store_file_checksum "$ldap_config_file" echo "svgo: false" > $final_path/.image_optim.yml #================================================= -# SETUP PUMA, A RUBY SERVER +# SETUP UNICORN, A RUBY SERVER #================================================= -puma_config_file="$final_path/config/puma.rb" -# Set log files location -ynh_replace_string "#{APP_ROOT}/log/puma" "/var/log/$app/puma" "$puma_config_file" -# Set application absolute path -ynh_replace_string "/home/discourse/discourse" "/var/www/$app" "$puma_config_file" -# Don't daemonize so we get logs in journalctl -ynh_replace_string "daemonize true" "daemonize false" "$puma_config_file" -# Don't preload threads to avoid warnings -ynh_replace_string "preload_app" "#preload_app" "$puma_config_file" +unicorn_config_file="$final_path/config/unicorn.conf.rb" +# Use socket connection +ynh_replace_string 'listen (ENV\["UNICORN_PORT"\] || 3000).to_i' '# listen (ENV["UNICORN_PORT"] || 3000).to_i' "$unicorn_config_file" +ynh_replace_string '# listen "#{discourse_path}/tmp/sockets/unicorn.sock"' 'listen "#{discourse_path}/tmp/sockets/unicorn.sock"' "$unicorn_config_file" # Calculate and store the config file checksum -ynh_store_file_checksum "$puma_config_file" +ynh_store_file_checksum "$unicorn_config_file" # Set a secret value cp ../conf/secrets.yml "$final_path/config/secrets.yml" @@ -206,11 +201,6 @@ if [ -n "$(uname -m | grep arm)" ] ; then ln -s $(ldconfig -p | grep libpsl | awk 'END {print $NF}') libpsl.so) fi -#================================================= -# SETUP SIDEKIQ -#================================================= -cp ../conf/sidekiq.yml "$final_path/config/sidekiq.yml" - #================================================= # PREPARE THE DATABASE #================================================= @@ -271,21 +261,20 @@ patch -p1 < $YNH_CWD/../conf/ldap-auth-fix-subfolder.patch) # SETUP SYSTEMD #================================================= -ynh_replace_string "__RBENVROOT__" "$RBENV_ROOT" "../conf/discourse-puma.service" -ynh_replace_string "__RBENVROOT__" "$RBENV_ROOT" "../conf/discourse-sidekiq.service" +ynh_replace_string "__RBENVROOT__" "$RBENV_ROOT" "../conf/systemd.service" # We assume for the moment that ARM devices are only dual core, so -# we restrict the number of workers to 2 (the default is 4) +# we restrict the number of workers to 2 (the default is 3) if [ -n "$(uname -m | grep arm)" ] ; then - additional_env="NUM_WEBS=2" + additional_env="UNICORN_WORKERS=2" + unicorn_workers=2 else additional_env="" + unicorn_workers=3 fi -ynh_replace_string "__ADDITIONAL_ENV__" "$additional_env" "../conf/discourse-puma.service" - -ynh_add_systemd_config $app-puma discourse-puma.service -ynh_add_systemd_config $app-sidekiq discourse-sidekiq.service +ynh_replace_string "__ADDITIONAL_ENV__" "$additional_env" "../conf/systemd.service" +ynh_add_systemd_config #================================================= # GENERIC FINALIZATION @@ -304,18 +293,14 @@ chown -R $app: $final_path # SETUP LOGROTATE #================================================= -mkdir -p /var/log/$app -chown -R $app: /var/log/$app - # Use logrotate to manage application logfile(s) -ynh_use_logrotate +ynh_use_logrotate "$final_path/log/unicorn.stderr.log" #================================================= # ADVERTISE SERVICE IN ADMIN PANEL #================================================= -yunohost service add $app-puma --log "/var/log/$app/puma.stderr.log" -yunohost service add $app-sidekiq --log "/var/www/$app/log/production.log" +yunohost service add $app --log "$final_path/log/unicorn.stderr.log" #================================================= # SETUP SSOWAT @@ -334,9 +319,8 @@ fi systemctl reload nginx #================================================= -# START PUMA AND SIDEKIQ +# START UNICORN #================================================= -# Wait for discourse-puma to be fully started -# As discourse-sidekiq is a dependency, it is automatically started before -ynh_check_starting "Use Ctrl-C to stop" systemd "120" "$app-puma" +# Wait for discourse to be fully started +ynh_check_starting "INFO -- : worker=$((unicorn_workers-1)) ready" "$final_path/log/unicorn.stderr.log" "120" "$app" diff --git a/scripts/remove b/scripts/remove index 8e933f4..f9584da 100644 --- a/scripts/remove +++ b/scripts/remove @@ -25,8 +25,7 @@ final_path=$(ynh_app_setting_get $app final_path) # STOP AND REMOVE SERVICE #================================================= -ynh_remove_systemd_config discourse-puma -ynh_remove_systemd_config discourse-sidekiq +ynh_remove_systemd_config #================================================= # REMOVE SERVICE FROM ADMIN PANEL @@ -35,8 +34,7 @@ ynh_remove_systemd_config discourse-sidekiq if yunohost service status | grep -q $app then echo "Remove $app services" - yunohost service remove $app-puma - yunohost service remove $app-sidekiq + yunohost service remove $app fi #================================================= diff --git a/scripts/restore b/scripts/restore index 5f7bb3f..56f4fc6 100644 --- a/scripts/restore +++ b/scripts/restore @@ -90,8 +90,7 @@ ynh_psql_execute_file_as_root ./db.sql "$db_name" #================================================= systemctl daemon-reload -systemctl enable $app-puma.service -systemctl enable $app-sidekiq.service +systemctl enable $app.service #================================================= # SETUP LOGROTATE @@ -117,8 +116,7 @@ chown -R $app: /var/log/$app # ADVERTISE SERVICE IN ADMIN PANEL #================================================= -yunohost service add $app-puma --log "/var/log/$app/puma.stderr.log" -yunohost service add $app-sidekiq --log "/var/www/$app/log/production.log" +yunohost service add $app --log "$final_path/log/unicorn.stderr.log" #================================================= # GENERIC FINALIZATION @@ -126,8 +124,13 @@ yunohost service add $app-sidekiq --log "/var/www/$app/log/production.log" # RELOAD NGINX AND DISCOURSE #================================================= -# Wait for discourse-puma to be fully started -# As discourse-sidekiq is a dependency, it is automatically started before -ynh_check_starting "Use Ctrl-C to stop" systemd "120" "$app-puma" +if [ -n "$(uname -m | grep arm)" ] ; then + unicorn_workers=2 +else + unicorn_workers=3 +fi +# Wait for discourse to be fully started +ynh_check_starting "INFO -- : worker=$((unicorn_workers-1)) ready" "$final_path/log/unicorn.stderr.log" "120" "$app" + systemctl reload nginx diff --git a/scripts/upgrade b/scripts/upgrade index 813f86d..eb254bc 100644 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -43,8 +43,7 @@ check_memory_requirements_upgrade #================================================= # Stop services -systemctl stop $app-sidekiq -systemctl stop $app-puma +systemctl stop $app # Backup the current version of the app if [[ $(ynh_app_setting_get $app disable_backup_before_upgrade) != '1' ]] @@ -137,19 +136,14 @@ if ! ynh_is_upstream_up_to_date ; then ynh_store_file_checksum "$ldap_config_file" #================================================= - # SETUP PUMA, A RUBY SERVER + # SETUP UNICORN, A RUBY SERVER #================================================= - puma_config_file="$final_path/config/puma.rb" - # Set log files location - ynh_replace_string "#{APP_ROOT}/log/puma" "/var/log/$app/puma" "$puma_config_file" - # Set application absolute path - ynh_replace_string "/home/discourse/discourse" "/var/www/$app" "$puma_config_file" - # Don't daemonize so we get logs in journalctl - ynh_replace_string "daemonize true" "daemonize false" "$puma_config_file" - # Don't preload threads to avoid warnings - ynh_replace_string "preload_app" "#preload_app" "$puma_config_file" + unicorn_config_file="$final_path/config/unicorn.conf.rb" + # Use socket connection + ynh_replace_string 'listen (ENV\["UNICORN_PORT"\] || 3000).to_i' '# listen (ENV["UNICORN_PORT"] || 3000).to_i' "$unicorn_config_file" + ynh_replace_string '# listen "#{discourse_path}/tmp/sockets/unicorn.sock"' 'listen "#{discourse_path}/tmp/sockets/unicorn.sock"' "$unicorn_config_file" # Calculate and store the config file checksum - ynh_store_file_checksum "$puma_config_file" + ynh_store_file_checksum "$unicorn_config_file" # Set a secret value cp ../conf/secrets.yml "$final_path/config/secrets.yml" @@ -177,11 +171,6 @@ if ! ynh_is_upstream_up_to_date ; then ln -s $(ldconfig -p | grep libpsl | awk 'END {print $NF}') libpsl.so) fi - #================================================= - # SETUP SIDEKIQ - #================================================= - cp ../conf/sidekiq.yml "$final_path/config/sidekiq.yml" - #================================================= # PREPARE THE DATABASE #================================================= @@ -217,9 +206,13 @@ fi systemctl reload nginx #================================================= -# START PUMA AND SIDEKIQ +# START UNICORN #================================================= -# Wait for discourse-puma to be fully started -# As discourse-sidekiq is a dependency, it is automatically started before -ynh_check_starting "Use Ctrl-C to stop" systemd "120" "$app-puma" +if [ -n "$(uname -m | grep arm)" ] ; then + unicorn_workers=2 +else + unicorn_workers=3 +fi +# Wait for discourse to be fully started +ynh_check_starting "INFO -- : worker=$((unicorn_workers-1)) ready" "$final_path/log/unicorn.stderr.log" "120" "$app"