From ddd514a1faa45f76256681c54658b0f8bfc6cdba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Sun, 19 Jun 2022 18:25:06 +0200 Subject: [PATCH] Use again gunicorn as uwsgi is badly maintained and bugy for pypy --- check_process | 7 +- conf/nginx.conf | 10 ++- conf/{uwsgi.ini => syncserver.ini} | 16 ++-- conf/systemd.service | 51 +++++++++++ conf/uwsgi-app@override.service | 9 -- manifest.json | 2 +- scripts/_common.sh | 29 +++++-- scripts/backup | 11 +-- scripts/experimental_helper.sh | 130 ----------------------------- scripts/install | 18 ++-- scripts/remove | 7 +- scripts/restore | 11 ++- scripts/upgrade | 41 +++++++-- 13 files changed, 146 insertions(+), 196 deletions(-) rename conf/{uwsgi.ini => syncserver.ini} (84%) create mode 100644 conf/systemd.service delete mode 100644 conf/uwsgi-app@override.service delete mode 100644 scripts/experimental_helper.sh diff --git a/check_process b/check_process index da4e358..cf3e7f8 100644 --- a/check_process +++ b/check_process @@ -10,10 +10,11 @@ setup_private=0 setup_public=1 upgrade=1 + upgrade=1 from_commit=028501b35335139cff4fc41477a9dbc969657576 backup_restore=1 multi_instance=1 port_already_use=0 change_url=0 -;;; Options -Email=jean-baptiste@holcroft.fr -Notification=fail +;;; Upgrade options + ; commit=028501b35335139cff4fc41477a9dbc969657576 + name=Before migration to pypy diff --git a/conf/nginx.conf b/conf/nginx.conf index 460eb29..a4db401 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -11,8 +11,10 @@ location __PATH__/ { include uwsgi_params; # Needed for long running operations in admin interface - uwsgi_read_timeout 3600; - __IS_SUBPATH__uwsgi_param SCRIPT_NAME __PATH__; - __IS_SUBPATH__uwsgi_modifier1 30; - uwsgi_pass unix:///run/__NAME__/app.socket; + proxy_pass http://localhost:__PORT__/; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_buffering off; + client_max_body_size 200M; + proxy_set_header X-Real-IP $remote_addr; } diff --git a/conf/uwsgi.ini b/conf/syncserver.ini similarity index 84% rename from conf/uwsgi.ini rename to conf/syncserver.ini index 8cc5c48..ca247f5 100644 --- a/conf/uwsgi.ini +++ b/conf/syncserver.ini @@ -1,13 +1,9 @@ -[uwsgi] -master = true -protocol = uwsgi -socket = /run/__APP__/app.socket -chmod-socket = 660 -virtualenv = __FINALPATH__/local -wsgi-file = __FINALPATH__/syncserver.wsgi -python-path = __FINALPATH__/local -enable-threads = true -close-on-exec = true +[server:main] +use = egg:gunicorn +host = 127.0.0.1 +port = __PORT__ +workers = 1 +timeout = 30 [app:main] use = egg:syncserver diff --git a/conf/systemd.service b/conf/systemd.service new file mode 100644 index 0000000..12cc844 --- /dev/null +++ b/conf/systemd.service @@ -0,0 +1,51 @@ +[Unit] +Description=Firefox sync server +After=network.target +After=mysql.service + +[Service] +# Modify these two values and uncomment them if you have +# repos with lots of files and get an HTTP error 500 because +# of that +### +#LimitMEMLOCK=infinity +#LimitNOFILE=65535 +Type=simple +User=__APP__ +Group=www-data +WorkingDirectory=/opt/yunohost/__APP__ +ExecStart=/opt/yunohost/__APP__/local/bin/gunicorn --paste /opt/yunohost/__APP__/syncserver.ini +Restart=always + +# Sandboxing options to harden security +# Depending on specificities of your service/app, you may need to tweak these +# .. but this should be a good baseline +# Details for these options: https://www.freedesktop.org/software/systemd/man/systemd.exec.html +NoNewPrivileges=yes +PrivateTmp=yes +PrivateDevices=yes +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +RestrictNamespaces=yes +RestrictRealtime=yes +DevicePolicy=closed +ProtectSystem=full +ProtectControlGroups=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +LockPersonality=yes +SystemCallFilter=~@clock @debug @module @mount @obsolete @reboot @swap + +# Denying access to capabilities that should not be relevant for webapps +# Doc: https://man7.org/linux/man-pages/man7/capabilities.7.html +CapabilityBoundingSet=~CAP_RAWIO CAP_MKNOD +CapabilityBoundingSet=~CAP_AUDIT_CONTROL CAP_AUDIT_READ CAP_AUDIT_WRITE +CapabilityBoundingSet=~CAP_SYS_BOOT CAP_SYS_TIME CAP_SYS_MODULE CAP_SYS_PACCT +CapabilityBoundingSet=~CAP_LEASE CAP_LINUX_IMMUTABLE CAP_IPC_LOCK +CapabilityBoundingSet=~CAP_BLOCK_SUSPEND CAP_WAKE_ALARM +CapabilityBoundingSet=~CAP_SYS_TTY_CONFIG +CapabilityBoundingSet=~CAP_MAC_ADMIN CAP_MAC_OVERRIDE +CapabilityBoundingSet=~CAP_NET_ADMIN CAP_NET_BROADCAST CAP_NET_RAW +CapabilityBoundingSet=~CAP_SYS_ADMIN CAP_SYS_PTRACE CAP_SYSLOG + +[Install] +WantedBy=multi-user.target diff --git a/conf/uwsgi-app@override.service b/conf/uwsgi-app@override.service deleted file mode 100644 index 8bba75b..0000000 --- a/conf/uwsgi-app@override.service +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -After=mysql.service - -[Service] -ExecStart= -ExecStart=/opt/yunohost/__app__/local/bin/uwsgi \ - --ini /etc/uwsgi/apps-available/%i.ini \ - --socket /run/%i/app.socket \ - --logto /var/log/uwsgi/%i/%i.log diff --git a/manifest.json b/manifest.json index 0f3bf0d..6cec3f4 100644 --- a/manifest.json +++ b/manifest.json @@ -6,7 +6,7 @@ "en": "Mozilla’s Sync-Server to host your Firefox account data", "fr": "Le serveur de synchronisation de Mozilla, pour héberger vos données Firefox" }, - "version": "1.9.1~ynh2", + "version": "1.9.1~ynh3", "url": "https://github.com/mozilla-services/syncserver", "license": "MPL-2.0", "maintainer": { diff --git a/scripts/_common.sh b/scripts/_common.sh index 204f2bb..ca90663 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -7,12 +7,27 @@ # Note that we also need some specific pkg_dependencies for build with arm architectures # dependencies used by the app -pkg_dependencies="pypy pypy-dev python3-virtualenv uwsgi build-essential libssl-dev libffi-dev libmariadb-dev-compat" +pkg_dependencies="pypy pypy-dev python3-virtualenv build-essential libssl-dev libffi-dev libmariadb-dev-compat" #================================================= # PERSONAL HELPERS #================================================= +call_pip() { + # Sometime we get a segfault error while we invoke pip + # As we don't have a really clean way to fix this really bad error we just try many time utils it works + i=0 + result_ok=false + while [ $i -lt 5 ] && ! $result_ok; do + ynh_exec_warn_less pip $@ && result_ok=true + i=$((i+1)) + done + if ! $result_ok; then + echo "Error on build package" + false + fi +} + install_sources() { ynh_setup_source --dest_dir "$final_path" @@ -26,13 +41,15 @@ install_sources() { source "$final_path/local/bin/activate" set -o nounset pushd "$final_path" - pip install --upgrade 'pip<20.2' - pip install setuptools==44.1.1 - pip install --upgrade pyramid_chameleon 'soupsieve<2.0' uwsgi + call_pip install --upgrade 'pip<20.2' + call_pip install setuptools==44.1.1 + call_pip install --upgrade pyramid_chameleon 'soupsieve<2.0' CFLAGS="-Wno-error -Wno-error=format-security" \ ARCHFLAGS="-Wno-error=unused-command-line-argument-hard-error-in-future" \ - pip install --upgrade --requirement "$final_path/requirements.txt" + call_pip install --upgrade --requirement "$final_path/requirements.txt" pypy "$final_path/setup.py" develop + test -e $final_path/local/lib_pypy/_sysconfigdata.py || ln -s /usr/lib/pypy/lib_pypy/_sysconfigdata.py $final_path/local/lib_pypy/_sysconfigdata.py + test -e $final_path/local/lib_pypy/cffi || ln -s /usr/lib/pypy/lib_pypy/cffi $final_path/local/lib_pypy/cffi popd # Add nice homepage @@ -43,8 +60,6 @@ install_sources() { set_permissions() { chown $app -R $final_path chmod u=rwX,g=rX,o= -R $final_path - chown $app:root /var/log/uwsgi/$app - chmod -R u=rwX,g=rX,o= /var/log/uwsgi/$app } #================================================= diff --git a/scripts/backup b/scripts/backup index 5b51f01..5a22674 100644 --- a/scripts/backup +++ b/scripts/backup @@ -7,7 +7,6 @@ #================================================= #Keep this path for calling _common.sh inside the execution's context of backup and restore scripts -source ../settings/scripts/experimental_helper.sh source ../settings/scripts/_common.sh source /usr/share/yunohost/helpers @@ -51,14 +50,10 @@ ynh_mysql_dump_db --database="$db_name" > db.sql #================================================= # SPECIFIC BACKUP #================================================= -# Backup Log -ynh_print_info --message="Backing up logs" -ynh_backup --src_path="/var/log/uwsgi/$app" -# BACKUP THE UWSGI FILES -ynh_print_info --message="Backing up UWSGI..." -ynh_backup --src_path="/etc/uwsgi/apps-available/$app.ini" -ynh_backup --src_path="/etc/systemd/system/uwsgi-app@.service" +# BACKUP THE systemd FILES +ynh_print_info --message="Backing up systemd..." +ynh_backup --src_path="/etc/systemd/system/$app.service" #================================================= # END OF SCRIPT diff --git a/scripts/experimental_helper.sh b/scripts/experimental_helper.sh deleted file mode 100644 index f9351fe..0000000 --- a/scripts/experimental_helper.sh +++ /dev/null @@ -1,130 +0,0 @@ -# Check if system wide templates are available and correcly configured -# -# usage: ynh_check_global_uwsgi_config -ynh_check_global_uwsgi_config () { - uwsgi --version || ynh_die --message="You need to add uwsgi (and appropriate plugin) as a dependency" - - cat > /etc/systemd/system/uwsgi-app@.service < uwsgi-app@$app` -ynh_add_uwsgi_service () { - ynh_check_global_uwsgi_config - - local others_var=${1:-} - local finaluwsgiini="/etc/uwsgi/apps-available/$app.ini" - - # www-data group is needed since it is this nginx who will start the service - usermod --append --groups www-data "$app" || ynh_die --message="It wasn't possible to add user $app to group www-data" - - ynh_backup_if_checksum_is_different --file="$finaluwsgiini" - cp ../conf/uwsgi.ini "$finaluwsgiini" - - # To avoid a break by set -u, use a void substitution ${var:-}. If the variable is not set, it's simply set with an empty variable. - # Substitute in a nginx config file only if the variable is not empty - if test -n "${final_path:-}"; then - ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finaluwsgiini" - fi - if test -n "${path_url:-}"; then - ynh_replace_string --match_string="__PATH__" --replace_string="$path_url" --target_file="$finaluwsgiini" - fi - if test -n "${app:-}"; then - ynh_replace_string --match_string="__APP__" --replace_string="$app" --target_file="$finaluwsgiini" - fi - - # Replace all other variable given as arguments - for var_to_replace in $others_var - do - # ${var_to_replace^^} make the content of the variable on upper-cases - # ${!var_to_replace} get the content of the variable named $var_to_replace - ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finaluwsgiini" - done - - ynh_store_file_checksum --file="$finaluwsgiini" - - chown $app:root "$finaluwsgiini" - - # make sure the folder for logs exists and set authorizations - mkdir -p /var/log/uwsgi/$app - chown $app:root /var/log/uwsgi/$app - chmod -R u=rwX,g=rX,o= /var/log/uwsgi/$app - - # Setup specific Systemd rules if necessary - test -e ../conf/uwsgi-app@override.service && \ - mkdir /etc/systemd/system/uwsgi-app@$app.service.d && \ - ynh_add_config --template="uwsgi-app@override.service" --destination="/etc/systemd/system/uwsgi-app@$app.service.d/override.conf" - - systemctl daemon-reload - systemctl enable "uwsgi-app@$app.service" - - # Add as a service - yunohost service add "uwsgi-app@$app" --log "/var/log/uwsgi/$app/$app.log" -} - -# Remove the dedicated uwsgi ini file -# -# usage: ynh_remove_uwsgi_service -ynh_remove_uwsgi_service () { - local finaluwsgiini="/etc/uwsgi/apps-available/$app.ini" - if [ -e "$finaluwsgiini" ]; then - yunohost service remove "uwsgi-app@$app" - systemctl stop "uwsgi-app@$app.service" - systemctl disable "uwsgi-app@$app.service" - - ynh_secure_remove --file="$finaluwsgiini" - ynh_secure_remove --file="/var/log/uwsgi/$app" - ynh_secure_remove --file="/etc/systemd/system/uwsgi-app@$app.service.d" - fi -} - -ynh_restore_uwsgi_service () { - ynh_check_global_uwsgi_config - systemctl enable "uwsgi-app@$app" --quiet - yunohost service add "uwsgi-app@$app" --log "/var/log/uwsgi/$app/$app.log" -} diff --git a/scripts/install b/scripts/install index 090abd6..107aa82 100644 --- a/scripts/install +++ b/scripts/install @@ -6,7 +6,6 @@ # IMPORT GENERIC HELPERS #================================================= -source ./experimental_helper.sh source ./_common.sh source /usr/share/yunohost/helpers @@ -36,6 +35,9 @@ test ! -e "$final_path" || ynh_die --message="This path already contains a folde # Register (book) web path ynh_webpath_register --app=$app --domain=$domain --path_url=$path_url +# Find available ports +port=$(ynh_find_port --port 6000) + #================================================= # STORE SETTINGS FROM MANIFEST #================================================= @@ -43,6 +45,7 @@ ynh_webpath_register --app=$app --domain=$domain --path_url=$path_url ynh_app_setting_set --app=$app --key=domain --value=$domain ynh_app_setting_set --app=$app --key=path --value=$path_url ynh_app_setting_set --app=$app --key=secret --value="$secret" +ynh_app_setting_set --app $app --key web_port --value $port #================================================= # STANDARD MODIFICATIONS @@ -104,11 +107,12 @@ ynh_system_user_create --username=$app --home_dir=$final_path # create config file syncserver.ini ynh_script_progression --message="Configuring application..." -rm "$final_path/syncserver.ini" -ln -s "/etc/uwsgi/apps-available/$app.ini" "$final_path/syncserver.ini" +ynh_add_config --template="syncserver.ini" --destination="$final_path/syncserver.ini" -# configure uwsgi -ynh_add_uwsgi_service 'domain secret db_user db_pwd db_name' +# Configure init script +ynh_script_progression --message="Configuring a systemd service..." --weight=2 +ynh_add_systemd_config +yunohost service add "$app" #================================================= # MODIFY A CONFIG FILE @@ -142,8 +146,8 @@ ynh_script_progression --message="Restart services..." ynh_systemd_action --service_name=nginx --action=reload ynh_script_progression --message="Starting $app services..." --weight=3 -ynh_systemd_action --service_name "uwsgi-app@$app.service" \ - --line_match "WSGI app 0 \(mountpoint='[/[:alnum:]_-]*'\) ready in [[:digit:]]* seconds on interpreter" --log_path "/var/log/uwsgi/$app/$app.log" +ynh_systemd_action --service_name "$app.service" \ + --line_match "Booting worker with pid" --log_path "systemd" -t 20 #================================================= # END OF SCRIPT diff --git a/scripts/remove b/scripts/remove index 39f10dc..5ca6d88 100644 --- a/scripts/remove +++ b/scripts/remove @@ -6,7 +6,6 @@ # IMPORT GENERIC HELPERS #================================================= -source ./experimental_helper.sh source ./_common.sh source /usr/share/yunohost/helpers @@ -40,8 +39,10 @@ fi #================================================= ynh_script_progression --message="Removing configuration..." -# Remove the dedicated systemd config -ynh_remove_uwsgi_service +# Remove init script +ynh_script_progression --message="Removing systemd units..." +ynh_remove_systemd_config +yunohost service remove "$app" #================================================= # REMOVE THE MYSQL DATABASE diff --git a/scripts/restore b/scripts/restore index 56004b5..c3e3831 100644 --- a/scripts/restore +++ b/scripts/restore @@ -5,7 +5,6 @@ #================================================= # IMPORT GENERIC HELPERS #================================================= -source ../settings/scripts/experimental_helper.sh source ../settings/scripts/_common.sh source /usr/share/yunohost/helpers @@ -82,8 +81,8 @@ ynh_mysql_connect_as --user=$db_user --password=$db_pwd --database=$db_name < ./ #================================================= ynh_script_progression --message="Reloading services..." --weight=3 - -ynh_restore_uwsgi_service +systemctl daemon-reload +systemctl enable $app.service #================================================= # GENERIC FINALIZATION @@ -92,8 +91,8 @@ ynh_restore_uwsgi_service #================================================= ynh_script_progression --message="Starting pgadmin services..." --weight=3 -ynh_systemd_action --service_name "uwsgi-app@$app.service" \ - --line_match "WSGI app 0 \(mountpoint='[/[:alnum:]_-]*'\) ready in [[:digit:]]* seconds on interpreter" --log_path "/var/log/uwsgi/$app/$app.log" -ynh_systemd_action --service_name=nginx --action=reload +ynh_systemd_action --service_name "$app.service" \ + --line_match "Booting worker with pid" --log_path "systemd" +ynh_systemd_action --service_name=nginx --action=reload -t 20 ynh_script_progression --message="Restoration completed for $app" --last diff --git a/scripts/upgrade b/scripts/upgrade index 6736e53..101af48 100644 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -6,7 +6,6 @@ # IMPORT GENERIC HELPERS #================================================= -source ./experimental_helper.sh source ./_common.sh source /usr/share/yunohost/helpers @@ -23,6 +22,7 @@ final_path=$(ynh_app_setting_get --app $app --key=final_path) db_name=$(ynh_app_setting_get --app=$app --key=db_name) secret=$(ynh_app_setting_get --app $app --key=secret) db_pwd=$(ynh_app_setting_get --app=$app --key mysqlpwd) +port=$(ynh_app_setting_get --app=$app --key web_port) db_user=$app #================================================= @@ -88,6 +88,23 @@ ynh_clean_setup () { # Exit if an error occurs during the execution of the script ynh_abort_if_errors +#================================================= +# ENSURE DOWNWARD COMPATIBILITY +#================================================= +ynh_script_progression --message="Checking backware compatibility..." --weight=10 + +# Detect old installation with uwsgi +if [ -e /etc/uwsgi/apps-available/$app.ini ]; then + systemctl stop uwsgi-app@$app.service + systemctl disable uwsgi-app@$app.service + yunohost service remove "uwsgi-app@$app" + ynh_secure_remove --file=/etc/uwsgi/apps-available/$app.ini + ynh_secure_remove --file=/etc/systemd/system/uwsgi-app@$app.service.d + ynh_secure_remove --file=$final_path +else + systemctl stop $app.service +fi + #================================================= # STANDARD UPGRADE STEPS #================================================= @@ -105,7 +122,13 @@ ynh_install_app_dependencies $pkg_dependencies # Download, check integrity, uncompress and patch the source from app.src ynh_script_progression --message="Upgrading source files..." --weight=6 -install_sources +if [ -e $final_path/syncserver.ini ]; then + config_backup="$(cat $final_path/syncserver.ini)" + install_sources + echo "$config_backup" > $final_path/syncserver.ini +else + install_sources +fi #================================================= # NGINX CONFIGURATION @@ -134,11 +157,12 @@ ynh_system_user_create --username="$app" ynh_script_progression --message="Configuring application..." # create config file syncserver.ini -rm "$final_path/syncserver.ini" -ln -s "/etc/uwsgi/apps-available/$app.ini" "$final_path/syncserver.ini" +ynh_script_progression --message="Configuring application..." +ynh_add_config --template="syncserver.ini" --destination="$final_path/syncserver.ini" -# configure uwsgi -ynh_add_uwsgi_service 'domain secret db_user db_pwd db_name' +# Configure init script +ynh_script_progression --message="Configuring a systemd service..." --weight=2 +ynh_add_systemd_config # Upgrade database table ynh_mysql_execute_as_root --sql='ALTER TABLE `users` ADD COLUMN IF NOT EXISTS `keys_changed_at` BIGINT NULL AFTER `replaced_at`;' --database=$db_name @@ -166,10 +190,11 @@ then fi ynh_permission_update --permission=main --add=visitors --protected=true --show_tile=true +yunohost service add "$app" ynh_script_progression --message="Restarting $app services..." --weight=3 -ynh_systemd_action --service_name "uwsgi-app@$app.service" \ - --line_match "WSGI app 0 \(mountpoint='[/[:alnum:]_-]*'\) ready in [[:digit:]]* seconds on interpreter" --log_path "/var/log/uwsgi/$app/$app.log" +ynh_systemd_action --service_name "$app.service" \ + --line_match "Booting worker with pid" --log_path "systemd" -t 20 #================================================= # END OF SCRIPT