Merge pull request #6 from ewagoner/testing

Installation fixes for testing

conf/.env.production

@@ -2,8 +2,8 @@
 SECRET_KEY="7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr"
 
 # SECURITY WARNING: don 't run with debug turned on in production!
-DEBUG=false       
-USE_HTTPS=true                 
+DEBUG=false
+USE_HTTPS=true
 
 DOMAIN=__DOMAIN__
 
@@ -26,15 +26,16 @@ POSTGRES_HOST=localhost
 
 # Redis activity stream manager
 MAX_STREAM_LENGTH=200
-REDIS_ACTIVITY_HOST=redis_activity
+REDIS_ACTIVITY_HOST=localhost
 REDIS_ACTIVITY_PORT=6379
-REDIS_ACTIVITY_PASSWORD=redispassword345
+REDIS_ACTIVITY_PASSWORD=
 # Optional, use a different redis database (defaults to 0)
 # REDIS_ACTIVITY_DB_INDEX=0
 
 # Redis as celery broker
+REDIS_BROKER_HOST=localhost
 REDIS_BROKER_PORT=6379
-REDIS_BROKER_PASSWORD=redispassword123
+REDIS_BROKER_PASSWORD=
 # Optional, use a different redis database (defaults to 0)
 # REDIS_BROKER_DB_INDEX=0
 
@@ -46,8 +47,8 @@ FLOWER_PASSWORD=changeme
 # Email config
 EMAIL_HOST=localhost
 EMAIL_PORT=587
-#EMAIL_HOST_USER=mail@your.domain.here
-#EMAIL_HOST_PASSWORD=emailpassword123
+EMAIL_HOST_USER=__ADMIN_MAIL__
+EMAIL_HOST_PASSWORD=
 EMAIL_USE_TLS=true
 EMAIL_USE_SSL=false
 EMAIL_SENDER_NAME=__ADMIN_MAIL__
@@ -113,4 +114,4 @@ OTEL_SERVICE_NAME=
 # via https. Please refer to the Django-Documentation and assess the risks
 # for your instance:
 # https://docs.djangoproject.com/en/3.2/ref/settings/#secure-proxy-ssl-header
-HTTP_X_FORWARDED_PROTO=false 
+HTTP_X_FORWARDED_PROTO=false

conf/bookwyrm-beat.service

@@ -0,0 +1,15 @@
+
+[Unit]
+Description=__APP__ celery beat process
+After=redis.service postgresql.service
+PartOf=__APP__.target
+
+[Service]
+User=__APP__
+Group=__APP__
+WorkingDirectory=__FINALPATH__/
+EnvironmentFile=__FINALPATH__/.env
+ExecStart=__FINALPATH__/venv/bin/celery -A celerywyrm beat -l INFO --scheduler django_celery_beat.schedulers:DatabaseScheduler
+
+[Install]
+WantedBy=multi-user.target

conf/bookwyrm-server.service

@@ -0,0 +1,15 @@
+
+[Unit]
+Description=__APP__ application server
+After=redis.service postgresql.service
+PartOf=__APP__.target
+
+[Service]
+User=__APP__
+Group=__APP__
+WorkingDirectory=__FINALPATH__/
+EnvironmentFile=__FINALPATH__/.env
+ExecStart=__FINALPATH__/venv/bin/gunicorn bookwyrm.wsgi:application --bind 0.0.0.0:__PORT__
+
+[Install]
+WantedBy=multi-user.target

conf/bookwyrm-worker.service

@@ -0,0 +1,15 @@
+
+[Unit]
+Description=__APP__ celery worker
+After=redis.service postgresql.service
+PartOf=__APP__.target
+
+[Service]
+User=__APP__
+Group=__APP__
+WorkingDirectory=__FINALPATH__/
+EnvironmentFile=__FINALPATH__/.env
+ExecStart=__FINALPATH__/venv/bin/celery -A celerywyrm worker -l info -Q high_priority,medium_priority,low_priority
+
+[Install]
+WantedBy=multi-user.target

conf/bookwyrm.target

@@ -0,0 +1,3 @@
+[Unit]
+Description=Bookwyrm Services (__APP__)
+Wants=__APP__-server.service __APP__-worker.service __APP__-beat.service

conf/nginx.conf

@@ -1,48 +1,41 @@
-#sub_path_only rewrite ^__PATH__$ __PATH__/ permanent;
-location __PATH__/ {
+client_max_body_size 10m;
 
-  proxy_pass http://127.0.0.1:__PORT__;
+location / {
+  try_files $uri @proxy;
+
+  # Include SSOWAT user panel.
+  include conf.d/yunohost_panel.conf.inc;
+}
+
+location @proxy {
   proxy_set_header Host $host;
   proxy_set_header X-Real-IP $remote_addr;
   proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
   proxy_set_header X-Forwarded-Proto https;
-  proxy_http_version 1.1;
+  proxy_set_header Proxy "";
+  proxy_pass_header Server;
+
+  proxy_pass http://127.0.0.1:__PORT__;
+  proxy_buffering on;
   proxy_redirect off;
-
-  # For WebSocket
+  proxy_http_version 1.1;
   proxy_set_header Upgrade $http_upgrade;
-  proxy_set_header Connection $connection_upgrade;
+  proxy_set_header Connection "upgrade";
 
-  # Cache settings
-  #proxy_cache cache1;
-  proxy_cache_lock on;
-  proxy_cache_use_stale updating;
-  more_set_headers "X-Cache: $upstream_cache_status";
-  # Change to upload limit
-  client_max_body_size 80m;
-  # Include SSOWAT user panel.
-  include conf.d/yunohost_panel.conf.inc;
-
-
-    # rate limit the login or password reset pages
-    location ~ ^/(login[^-/]|password-reset|resend-link|2fa-check) {
-        limit_req zone=loginlimit;
-        proxy_pass http://127.0.0.1:__PORT__;
-    }
-
-    location /api/updates/ {
-        access_log off;
-        proxy_pass http://127.0.0.1:__PORT__;
-    }
-
-    # directly serve images and static files from the
-    # bookwyrm filesystem using sendfile.
-    # make the logs quieter by not reporting these requests
-    location ~ ^/(images|static)/ {
-        root /app;
-        try_files $uri =404;
-        more_set_headers "X-Cache-Status: STATIC";
-        access_log off;
-    }
+  #proxy_cache CACHE;
+  proxy_cache_valid 200 7d;
+  proxy_cache_valid 410 24h;
+  proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;
+  more_set_headers "X-Cached: $upstream_cache_status";
+  more_set_headers "Strict-Transport-Security: max-age=31536000";
 
+  tcp_nodelay on;
+}
+
+location /images/ {
+  alias __FINALPATH__/images/;
+}
+
+location /static/ {
+  alias __FINALPATH__/static/;
 }

conf/systemd.service

@@ -1,45 +0,0 @@
-[Unit]
-Description=Small description of the service
-After=network.target
-
-[Service]
-Type=simple
-User=__APP__
-Group=__APP__
-WorkingDirectory=__FINALPATH__/
-ExecStart=__FINALPATH__/script
-StandardOutput=append:/var/log/__APP__/__APP__.log
-StandardError=inherit
-
-# Sandboxing options to harden security
-# Depending on specificities of your service/app, you may need to tweak these 
-# .. but this should be a good baseline
-# Details for these options: https://www.freedesktop.org/software/systemd/man/systemd.exec.html
-NoNewPrivileges=yes
-PrivateTmp=yes
-PrivateDevices=yes
-RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
-RestrictNamespaces=yes
-RestrictRealtime=yes
-DevicePolicy=closed
-ProtectSystem=full
-ProtectControlGroups=yes
-ProtectKernelModules=yes
-ProtectKernelTunables=yes
-LockPersonality=yes
-SystemCallFilter=~@clock @debug @module @mount @obsolete @reboot @setuid @swap
-
-# Denying access to capabilities that should not be relevant for webapps
-# Doc: https://man7.org/linux/man-pages/man7/capabilities.7.html
-CapabilityBoundingSet=~CAP_RAWIO CAP_MKNOD
-CapabilityBoundingSet=~CAP_AUDIT_CONTROL CAP_AUDIT_READ CAP_AUDIT_WRITE
-CapabilityBoundingSet=~CAP_SYS_BOOT CAP_SYS_TIME CAP_SYS_MODULE CAP_SYS_PACCT
-CapabilityBoundingSet=~CAP_LEASE CAP_LINUX_IMMUTABLE CAP_IPC_LOCK
-CapabilityBoundingSet=~CAP_BLOCK_SUSPEND CAP_WAKE_ALARM
-CapabilityBoundingSet=~CAP_SYS_TTY_CONFIG
-CapabilityBoundingSet=~CAP_MAC_ADMIN CAP_MAC_OVERRIDE
-CapabilityBoundingSet=~CAP_NET_ADMIN CAP_NET_BROADCAST CAP_NET_RAW
-CapabilityBoundingSet=~CAP_SYS_ADMIN CAP_SYS_PTRACE CAP_SYSLOG 
-
-[Install]
-WantedBy=multi-user.target

manifest.json

@@ -3,10 +3,10 @@
     "id": "bookwyrm",
     "packaging_format": 1,
     "description": {
-        "en": "$app is a platform for social reading.",
-        "fr": "$app est une plateforme de lecture sociale."
+        "en": "BookWyrm is a platform for social reading.",
+        "fr": "BookWyrm est une plateforme de lecture sociale."
     },
-    "version": "0.5.3~ynh1",
+    "version": "0.5.3~ynh1.0.5",
     "url": "https://github.com/bookwyrm-social/bookwyrm",
     "upstream": {
         "license": "AGPL-3.0-or-later",
@@ -33,7 +33,7 @@
                 "name": "domain",
                 "type": "domain"
             },
-   
+
             {
                 "name": "is_public",
                 "type": "boolean",

scripts/backup

@@ -72,7 +72,10 @@ ynh_backup --src_path="/etc/nginx/conf.d/$domain.d/$app.conf"
 # BACKUP SYSTEMD
 #=================================================
 
-ynh_backup --src_path="/etc/systemd/system/$app.service"
+ynh_backup --src_path="/etc/systemd/system/${app}-beat.service"
+ynh_backup --src_path="/etc/systemd/system/${app}-server.service"
+ynh_backup --src_path="/etc/systemd/system/${app}-worker.service"
+ynh_backup --src_path="/etc/systemd/system/$app.target"
 
 
 #=================================================

scripts/install

@@ -109,6 +109,29 @@ chmod 750 "$final_path"
 chmod -R o-rwx "$final_path"
 chown -R $app:www-data "$final_path"
 
+#=================================================
+# CONFIGURE THEN INSTALL SCRIPT AND DEPENDENCIES
+#=================================================
+ynh_script_progression --message="Installing service script..." --weight=1
+
+ynh_add_config --template="../conf/.env.production" --destination="$final_path/.env"
+chmod 600 $final_path/.env
+chown $app:www-data "$final_path/.env"
+
+set -a; source "$final_path/.env"; set +a
+
+mkdir "$final_path/venv"
+python3 -m venv "$final_path/venv"
+$final_path/venv/bin/pip3 install -r "$final_path/requirements.txt"
+
+#=================================================
+# INITIALIZE DATABASE
+#=================================================
+ynh_script_progression --message="Initializing database..." --weight=1
+
+$final_path/venv/bin/python3 "$final_path/manage.py" migrate
+$final_path/venv/bin/python3 "$final_path/manage.py" initdb
+$final_path/venv/bin/python3 "$final_path/manage.py" collectstatic --no-input
 
 #=================================================
 # NGINX CONFIGURATION
@@ -140,9 +163,12 @@ chown -R $app:www-data "$datadir"
 #=================================================
 ynh_script_progression --message="Configuring a systemd service..." --weight=1
 
+ynh_add_config --template="../conf/${app}.target" --destination="/etc/systemd/system/$app.target"
 
 # Create a dedicated systemd config
-ynh_add_systemd_config
+ynh_add_systemd_config --service="${app}-server" --template="${app}-server.service"
+ynh_add_systemd_config --service="${app}-worker" --template="${app}-worker.service"
+ynh_add_systemd_config --service="${app}-beat"   --template="${app}-beat.service"
 
 
 #=================================================
@@ -154,8 +180,9 @@ ynh_add_systemd_config
 #=================================================
 ynh_script_progression --message="Integrating service in YunoHost..." --weight=1
 
-
-yunohost service add $app --description="A short description of the app" --log="/var/log/$app/$app.log"
+yunohost service add "${app}-beat"
+yunohost service add "${app}-server"
+yunohost service add "${app}-worker"
 
 
 #=================================================
@@ -163,9 +190,10 @@ yunohost service add $app --description="A short description of the app" --log="
 #=================================================
 ynh_script_progression --message="Starting a systemd service..." --weight=1
 
-
 # Start a systemd service
-ynh_systemd_action --service_name=$app --action="start" --log_path="/var/log/$app/$app.log"
+ynh_systemd_action --service_name="${app}-beat" --action="start" --log_path="systemd" --line_match="Started"
+ynh_systemd_action --service_name="${app}-server" --action="start" --log_path="systemd" --line_match="Booting worker with pid"
+ynh_systemd_action --service_name="${app}-worker" --action="start" --log_path="systemd" --line_match="ready"
 
 
 #=================================================

scripts/remove

@@ -41,8 +41,16 @@ fi
 #=================================================
 ynh_script_progression --message="Stopping and removing the systemd service..."  --weight=1
 
+ynh_systemd_action --service_name="${app}-beat" --action="stop" --log_path="systemd" --line_match="Stopped $app"
+ynh_systemd_action --service_name="${app}-server" --action="stop" --log_path="systemd" --line_match="Stopped $app"
+ynh_systemd_action --service_name="${app}-worker" --action="stop" --log_path="systemd" --line_match="Stopped $app"
+
 # Remove the dedicated systemd config
-ynh_remove_systemd_config
+ynh_remove_systemd_config --service="${app}-beat"
+ynh_remove_systemd_config --service="${app}-server"
+ynh_remove_systemd_config --service="${app}-worker"
+
+ynh_secure_remove --file="/etc/systemd/system/$app.target"
 
 #=================================================
 # REMOVE LOGROTATE CONFIGURATION

scripts/restore

@@ -124,22 +124,32 @@ ynh_psql_execute_file_as_root --file="./db.sql" --database="$db_name"
 #=================================================
 ynh_script_progression --message="Restoring the systemd configuration..." --weight=1
 
-ynh_restore_file --origin_path="/etc/systemd/system/$app.service"
-systemctl enable $app.service --quiet
+ynh_restore_file --origin_path="/etc/systemd/system/${app}-beat.service"
+ynh_restore_file --origin_path="/etc/systemd/system/${app}-server.service"
+ynh_restore_file --origin_path="/etc/systemd/system/${app}-worker.service"
+ynh_restore_file --origin_path="/etc/systemd/system/$app.target"
+
+systemctl enable "${app}-beat.service" --quiet
+systemctl enable "${app}-server.service" --quiet
+systemctl enable "${app}-worker.service" --quiet
 
 #=================================================
 # INTEGRATE SERVICE IN YUNOHOST
 #=================================================
 ynh_script_progression --message="Integrating service in YunoHost..." --weight=1
 
-yunohost service add $app --description="A short description of the app" --log="/var/log/$app/$app.log"
+yunohost service add "${app}-beat"
+yunohost service add "${app}-server"
+yunohost service add "${app}-worker"
 
 #=================================================
 # START SYSTEMD SERVICE
 #=================================================
 ynh_script_progression --message="Starting a systemd service..." --weight=1
 
-ynh_systemd_action --service_name=$app --action="start" --log_path="/var/log/$app/$app.log"
+ynh_systemd_action --service_name="${app}-beat" --action="start" --log_path="systemd" --line_match="Started"
+ynh_systemd_action --service_name="${app}-server" --action="start" --log_path="systemd" --line_match="Booting worker with pid"
+ynh_systemd_action --service_name="${app}-worker" --action="start" --log_path="systemd" --line_match="ready"
 
 #=================================================
 # GENERIC FINALIZATION

scripts/upgrade

@@ -22,6 +22,7 @@ language=$(ynh_app_setting_get --app=$app --key=language)
 admin=$(ynh_app_setting_get --app=$app --key=admin)
 final_path=$(ynh_app_setting_get --app=$app --key=final_path)
 db_name=$(ynh_app_setting_get --app=$app --key=db_name)
+port=$(ynh_app_setting_get --app=$app --key=port)
 
 #=================================================
 # CHECK VERSION
@@ -56,7 +57,9 @@ ynh_abort_if_errors
 #=================================================
 ynh_script_progression --message="Stopping a systemd service..." --weight=1
 
-ynh_systemd_action --service_name=$app --action="stop" --log_path="/var/log/$app/$app.log"
+ynh_systemd_action --action="stop" --service_name="${app}-beat" --log_path="systemd" --line_match="Stopped"
+ynh_systemd_action --action="stop" --service_name="${app}-server" --log_path="systemd" --line_match="Stopped"
+ynh_systemd_action --action="stop" --service_name="${app}-worker" --log_path="systemd" --line_match="Stopped"
 
 #=================================================
 # ENSURE DOWNWARD COMPATIBILITY
@@ -152,8 +155,12 @@ ynh_install_app_dependencies $pkg_dependencies
 #=================================================
 ynh_script_progression --message="Upgrading systemd configuration..." --weight=1
 
+ynh_add_config --template="../conf/${app}.target" --destination="/etc/systemd/system/$app.target"
+
 # Create a dedicated systemd config
-ynh_add_systemd_config
+ynh_add_systemd_config --service="${app}-server" --template="${app}-server.service"
+ynh_add_systemd_config --service="${app}-worker" --template="${app}-worker.service"
+ynh_add_systemd_config --service="${app}-beat"   --template="${app}-beat.service"
 
 #=================================================
 # GENERIC FINALIZATION
@@ -163,14 +170,18 @@ ynh_add_systemd_config
 #=================================================
 ynh_script_progression --message="Integrating service in YunoHost..." --weight=1
 
-yunohost service add $app --description="A short description of the app" --log="/var/log/$app/$app.log"
+yunohost service add "${app}-beat"
+yunohost service add "${app}-server"
+yunohost service add "${app}-worker"
 
 #=================================================
 # START SYSTEMD SERVICE
 #=================================================
 ynh_script_progression --message="Starting a systemd service..." --weight=1
 
-ynh_systemd_action --service_name=$app --action="start" --log_path="/var/log/$app/$app.log"
+ynh_systemd_action --service_name="${app}-beat" --action="start" --log_path="systemd" --line_match="Started"
+ynh_systemd_action --service_name="${app}-server" --action="start" --log_path="systemd" --line_match="Booting worker with pid"
+ynh_systemd_action --service_name="${app}-worker" --action="start" --log_path="systemd" --line_match="ready"
 
 
 #=================================================