Use again gunicorn as uwsgi is badly maintained and bugy for pypy

check_process

@@ -10,10 +10,11 @@
 		setup_private=0
 		setup_public=1
 		upgrade=1
+		upgrade=1    from_commit=028501b35335139cff4fc41477a9dbc969657576
 		backup_restore=1
 		multi_instance=1
 		port_already_use=0
 		change_url=0
-;;; Options
+;;; Upgrade options
-Email=jean-baptiste@holcroft.fr
+    ; commit=028501b35335139cff4fc41477a9dbc969657576
-Notification=fail
+        name=Before migration to pypy

conf/nginx.conf

@@ -11,8 +11,10 @@ location __PATH__/ {
 
 	include uwsgi_params;
 	# Needed for long running operations in admin interface
-	uwsgi_read_timeout 3600;
+    proxy_pass                  http://localhost:__PORT__/;
-	__IS_SUBPATH__uwsgi_param SCRIPT_NAME __PATH__;
+    proxy_set_header            Host $host;
-	__IS_SUBPATH__uwsgi_modifier1 30;
+	proxy_set_header   X-Forwarded-Proto $scheme;
-	uwsgi_pass unix:///run/__NAME__/app.socket;
+    proxy_buffering off;
+    client_max_body_size        200M;
+    proxy_set_header X-Real-IP $remote_addr;
 }

conf/syncserver.ini

@@ -1,13 +1,9 @@
-[uwsgi]
+[server:main]
-master        = true
+use = egg:gunicorn
-protocol      = uwsgi
+host = 127.0.0.1
-socket        = /run/__APP__/app.socket
+port = __PORT__
-chmod-socket  = 660
+workers = 1
-virtualenv    = __FINALPATH__/local
+timeout = 30
-wsgi-file     = __FINALPATH__/syncserver.wsgi
-python-path   = __FINALPATH__/local
-enable-threads = true
-close-on-exec  = true
 
 [app:main]
 use = egg:syncserver

conf/systemd.service

@@ -0,0 +1,51 @@
+[Unit]
+Description=Firefox sync server
+After=network.target
+After=mysql.service
+
+[Service]
+# Modify these two values and uncomment them if you have
+# repos with lots of files and get an HTTP error 500 because
+# of that
+###
+#LimitMEMLOCK=infinity
+#LimitNOFILE=65535
+Type=simple
+User=__APP__
+Group=www-data
+WorkingDirectory=/opt/yunohost/__APP__
+ExecStart=/opt/yunohost/__APP__/local/bin/gunicorn --paste /opt/yunohost/__APP__/syncserver.ini
+Restart=always
+
+# Sandboxing options to harden security
+# Depending on specificities of your service/app, you may need to tweak these 
+# .. but this should be a good baseline
+# Details for these options: https://www.freedesktop.org/software/systemd/man/systemd.exec.html
+NoNewPrivileges=yes
+PrivateTmp=yes
+PrivateDevices=yes
+RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
+RestrictNamespaces=yes
+RestrictRealtime=yes
+DevicePolicy=closed
+ProtectSystem=full
+ProtectControlGroups=yes
+ProtectKernelModules=yes
+ProtectKernelTunables=yes
+LockPersonality=yes
+SystemCallFilter=~@clock @debug @module @mount @obsolete @reboot @swap
+
+# Denying access to capabilities that should not be relevant for webapps
+# Doc: https://man7.org/linux/man-pages/man7/capabilities.7.html
+CapabilityBoundingSet=~CAP_RAWIO CAP_MKNOD
+CapabilityBoundingSet=~CAP_AUDIT_CONTROL CAP_AUDIT_READ CAP_AUDIT_WRITE
+CapabilityBoundingSet=~CAP_SYS_BOOT CAP_SYS_TIME CAP_SYS_MODULE CAP_SYS_PACCT
+CapabilityBoundingSet=~CAP_LEASE CAP_LINUX_IMMUTABLE CAP_IPC_LOCK
+CapabilityBoundingSet=~CAP_BLOCK_SUSPEND CAP_WAKE_ALARM
+CapabilityBoundingSet=~CAP_SYS_TTY_CONFIG
+CapabilityBoundingSet=~CAP_MAC_ADMIN CAP_MAC_OVERRIDE
+CapabilityBoundingSet=~CAP_NET_ADMIN CAP_NET_BROADCAST CAP_NET_RAW
+CapabilityBoundingSet=~CAP_SYS_ADMIN CAP_SYS_PTRACE CAP_SYSLOG 
+
+[Install]
+WantedBy=multi-user.target

conf/uwsgi-app@override.service

@@ -1,9 +0,0 @@
-[Unit]
-After=mysql.service
-
-[Service]
-ExecStart=
-ExecStart=/opt/yunohost/__app__/local/bin/uwsgi \
-        --ini /etc/uwsgi/apps-available/%i.ini \
-        --socket /run/%i/app.socket \
-        --logto /var/log/uwsgi/%i/%i.log

manifest.json

@@ -6,7 +6,7 @@
         "en": "Mozilla’s Sync-Server to host your Firefox account data",
         "fr": "Le serveur de synchronisation de Mozilla, pour héberger vos données Firefox"
     },
-    "version": "1.9.1~ynh2",
+    "version": "1.9.1~ynh3",
     "url": "https://github.com/mozilla-services/syncserver",
     "license": "MPL-2.0",
     "maintainer": {

scripts/_common.sh

@@ -7,12 +7,27 @@
 
 # Note that we also need some specific pkg_dependencies for build with arm architectures
 # dependencies used by the app
-pkg_dependencies="pypy pypy-dev python3-virtualenv uwsgi build-essential libssl-dev libffi-dev libmariadb-dev-compat"
+pkg_dependencies="pypy pypy-dev python3-virtualenv build-essential libssl-dev libffi-dev libmariadb-dev-compat"
 
 #=================================================
 # PERSONAL HELPERS
 #=================================================
 
+call_pip() {
+    # Sometime we get a segfault error while we invoke pip
+    # As we don't have a really clean way to fix this really bad error we just try many time utils it works
+    i=0
+    result_ok=false
+    while [ $i -lt 5 ] && ! $result_ok; do
+        ynh_exec_warn_less pip $@ && result_ok=true
+        i=$((i+1))
+    done
+    if ! $result_ok; then
+        echo "Error on build package"
+        false
+    fi
+}
+
 install_sources() {
     ynh_setup_source --dest_dir "$final_path"
 
@@ -26,13 +41,15 @@ install_sources() {
     source "$final_path/local/bin/activate"
     set -o nounset
     pushd "$final_path"
-    pip install --upgrade 'pip<20.2'
+    call_pip install --upgrade 'pip<20.2'
-    pip install setuptools==44.1.1
+    call_pip install setuptools==44.1.1
-    pip install --upgrade pyramid_chameleon 'soupsieve<2.0' uwsgi
+    call_pip install --upgrade pyramid_chameleon 'soupsieve<2.0'
     CFLAGS="-Wno-error -Wno-error=format-security" \
         ARCHFLAGS="-Wno-error=unused-command-line-argument-hard-error-in-future" \
-        pip install --upgrade --requirement "$final_path/requirements.txt"
+        call_pip install --upgrade --requirement "$final_path/requirements.txt"
     pypy "$final_path/setup.py" develop
+    test -e $final_path/local/lib_pypy/_sysconfigdata.py || ln -s /usr/lib/pypy/lib_pypy/_sysconfigdata.py $final_path/local/lib_pypy/_sysconfigdata.py
+    test -e $final_path/local/lib_pypy/cffi || ln -s /usr/lib/pypy/lib_pypy/cffi $final_path/local/lib_pypy/cffi
     popd
 
     # Add nice homepage
@@ -43,8 +60,6 @@ install_sources() {
 set_permissions() {
     chown $app -R $final_path
     chmod u=rwX,g=rX,o= -R $final_path
-    chown $app:root /var/log/uwsgi/$app
-    chmod -R u=rwX,g=rX,o= /var/log/uwsgi/$app
 }
 
 #=================================================

scripts/backup

@@ -7,7 +7,6 @@
 #=================================================
 
 #Keep this path for calling _common.sh inside the execution's context of backup and restore scripts
-source ../settings/scripts/experimental_helper.sh
 source ../settings/scripts/_common.sh
 source /usr/share/yunohost/helpers
 
@@ -51,14 +50,10 @@ ynh_mysql_dump_db --database="$db_name" > db.sql
 #=================================================
 # SPECIFIC BACKUP
 #=================================================
-# Backup Log
-ynh_print_info --message="Backing up logs"
-ynh_backup --src_path="/var/log/uwsgi/$app"
 
-# BACKUP THE UWSGI FILES
+# BACKUP THE systemd FILES
-ynh_print_info --message="Backing up UWSGI..."
+ynh_print_info --message="Backing up systemd..."
-ynh_backup --src_path="/etc/uwsgi/apps-available/$app.ini"
+ynh_backup --src_path="/etc/systemd/system/$app.service"
-ynh_backup --src_path="/etc/systemd/system/uwsgi-app@.service"
 
 #=================================================
 # END OF SCRIPT

scripts/experimental_helper.sh

@@ -1,130 +0,0 @@
-# Check if system wide templates are available and correcly configured
-#
-# usage: ynh_check_global_uwsgi_config
-ynh_check_global_uwsgi_config () {
-    uwsgi --version || ynh_die --message="You need to add uwsgi (and appropriate plugin) as a dependency"
-
-    cat > /etc/systemd/system/uwsgi-app@.service <<EOF
-[Unit]
-Description=%i uWSGI app
-
-[Service]
-RuntimeDirectory=%i
-ExecStart=/usr/bin/uwsgi \
-        --ini /etc/uwsgi/apps-available/%i.ini \
-        --socket /run/%i/app.socket \
-        --logto /var/log/uwsgi/%i/%i.log
-User=%i
-Group=www-data
-Restart=always
-RestartSec=10
-KillSignal=SIGQUIT
-Type=notify
-NotifyAccess=all
-
-[Install]
-WantedBy=multi-user.target
-EOF
-
-    systemctl daemon-reload
-}
-
-# Create a dedicated uwsgi ini file to use with generic uwsgi service
-#
-# This will use a template in ../conf/uwsgi.ini
-# and will replace the following keywords with
-# global variables that should be defined before calling
-# this helper :
-#
-#   __APP__       by  $app
-#   __PATH__      by  $path_url
-#   __FINALPATH__ by  $final_path
-#
-#  And dynamic variables (from the last example) :
-#   __PATH_2__    by $path_2
-#   __PORT_2__    by $port_2
-#
-# To be able to customise the settings of the systemd unit you can override the rules with the file "conf/uwsgi-app@override.service".
-# This file will be automatically placed on the good place
-# 
-# Note that the service need to be started manually at the end of the installation.
-# Generally you can start the service with this command:
-# # ynh_systemd_action --service_name "uwsgi-app@$app.service" --line_match "WSGI app 0 \(mountpoint='[/[:alnum:]_-]*'\) ready in [[:digit:]]* seconds on interpreter" --log_path "/var/log/uwsgi/$app/$app.log"
-#
-# usage: ynh_add_uwsgi_service
-#
-# to interact with your service: `systemctl <action> uwsgi-app@$app`
-ynh_add_uwsgi_service () {
-    ynh_check_global_uwsgi_config
-
-    local others_var=${1:-}
-    local finaluwsgiini="/etc/uwsgi/apps-available/$app.ini"
-
-    # www-data group is needed since it is this nginx who will start the service
-    usermod --append --groups www-data "$app" || ynh_die --message="It wasn't possible to add user $app to group www-data"
-
-    ynh_backup_if_checksum_is_different --file="$finaluwsgiini"
-    cp ../conf/uwsgi.ini "$finaluwsgiini"
-
-    # To avoid a break by set -u, use a void substitution ${var:-}. If the variable is not set, it's simply set with an empty variable.
-    # Substitute in a nginx config file only if the variable is not empty
-    if test -n "${final_path:-}"; then
-        ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finaluwsgiini"
-    fi
-    if test -n "${path_url:-}"; then
-        ynh_replace_string --match_string="__PATH__" --replace_string="$path_url" --target_file="$finaluwsgiini"
-    fi
-    if test -n "${app:-}"; then
-        ynh_replace_string --match_string="__APP__" --replace_string="$app" --target_file="$finaluwsgiini"
-    fi
-
-    # Replace all other variable given as arguments
-    for var_to_replace in $others_var
-    do
-        # ${var_to_replace^^} make the content of the variable on upper-cases
-        # ${!var_to_replace} get the content of the variable named $var_to_replace 
-        ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finaluwsgiini"
-    done
-
-    ynh_store_file_checksum --file="$finaluwsgiini"
-
-    chown $app:root "$finaluwsgiini"
-
-    # make sure the folder for logs exists and set authorizations
-    mkdir -p /var/log/uwsgi/$app
-    chown $app:root /var/log/uwsgi/$app
-    chmod -R u=rwX,g=rX,o= /var/log/uwsgi/$app
-
-    # Setup specific Systemd rules if necessary
-    test -e ../conf/uwsgi-app@override.service && \
-        mkdir /etc/systemd/system/uwsgi-app@$app.service.d && \
-        ynh_add_config --template="uwsgi-app@override.service" --destination="/etc/systemd/system/uwsgi-app@$app.service.d/override.conf"
-
-    systemctl daemon-reload
-    systemctl enable "uwsgi-app@$app.service"
-
-    # Add as a service
-    yunohost service add "uwsgi-app@$app" --log "/var/log/uwsgi/$app/$app.log"
-}
-
-# Remove the dedicated uwsgi ini file
-#
-# usage: ynh_remove_uwsgi_service
-ynh_remove_uwsgi_service () {
-    local finaluwsgiini="/etc/uwsgi/apps-available/$app.ini"
-    if [ -e "$finaluwsgiini" ]; then
-        yunohost service remove "uwsgi-app@$app"
-        systemctl stop "uwsgi-app@$app.service"
-        systemctl disable "uwsgi-app@$app.service"
-
-        ynh_secure_remove --file="$finaluwsgiini"
-        ynh_secure_remove --file="/var/log/uwsgi/$app"
-        ynh_secure_remove --file="/etc/systemd/system/uwsgi-app@$app.service.d"
-    fi
-}
-
-ynh_restore_uwsgi_service () {
-    ynh_check_global_uwsgi_config
-    systemctl enable "uwsgi-app@$app" --quiet
-    yunohost service add "uwsgi-app@$app" --log "/var/log/uwsgi/$app/$app.log"
-}

scripts/install

@@ -6,7 +6,6 @@
 # IMPORT GENERIC HELPERS
 #=================================================
 
-source ./experimental_helper.sh
 source ./_common.sh
 source /usr/share/yunohost/helpers
 
@@ -36,6 +35,9 @@ test ! -e "$final_path" || ynh_die --message="This path already contains a folde
 # Register (book) web path
 ynh_webpath_register --app=$app --domain=$domain --path_url=$path_url
 
+# Find available ports
+port=$(ynh_find_port --port 6000)
+
 #=================================================
 # STORE SETTINGS FROM MANIFEST
 #=================================================
@@ -43,6 +45,7 @@ ynh_webpath_register --app=$app --domain=$domain --path_url=$path_url
 ynh_app_setting_set --app=$app --key=domain --value=$domain
 ynh_app_setting_set --app=$app --key=path --value=$path_url
 ynh_app_setting_set --app=$app --key=secret --value="$secret"
+ynh_app_setting_set --app $app --key web_port --value $port
 
 #=================================================
 # STANDARD MODIFICATIONS
@@ -104,11 +107,12 @@ ynh_system_user_create --username=$app --home_dir=$final_path
 
 # create config file syncserver.ini
 ynh_script_progression --message="Configuring application..."
-rm "$final_path/syncserver.ini"
+ynh_add_config --template="syncserver.ini" --destination="$final_path/syncserver.ini"
-ln -s "/etc/uwsgi/apps-available/$app.ini" "$final_path/syncserver.ini"
 
-# configure uwsgi
+# Configure init script
-ynh_add_uwsgi_service 'domain secret db_user db_pwd db_name'
+ynh_script_progression --message="Configuring a systemd service..." --weight=2
+ynh_add_systemd_config
+yunohost service add "$app"
 
 #=================================================
 # MODIFY A CONFIG FILE
@@ -142,8 +146,8 @@ ynh_script_progression --message="Restart services..."
 ynh_systemd_action --service_name=nginx --action=reload
 
 ynh_script_progression --message="Starting $app services..." --weight=3
-ynh_systemd_action --service_name "uwsgi-app@$app.service" \
+ynh_systemd_action --service_name "$app.service" \
-    --line_match "WSGI app 0 \(mountpoint='[/[:alnum:]_-]*'\) ready in [[:digit:]]* seconds on interpreter" --log_path "/var/log/uwsgi/$app/$app.log"
+    --line_match "Booting worker with pid" --log_path "systemd" -t 20
 
 #=================================================
 # END OF SCRIPT

scripts/remove

@@ -6,7 +6,6 @@
 # IMPORT GENERIC HELPERS
 #=================================================
 
-source ./experimental_helper.sh
 source ./_common.sh
 source /usr/share/yunohost/helpers
 
@@ -40,8 +39,10 @@ fi
 #=================================================
 ynh_script_progression --message="Removing configuration..."
 
-# Remove the dedicated systemd config
+# Remove init script
-ynh_remove_uwsgi_service
+ynh_script_progression --message="Removing systemd units..."
+ynh_remove_systemd_config
+yunohost service remove "$app"
 
 #=================================================
 # REMOVE THE MYSQL DATABASE

scripts/restore

@@ -5,7 +5,6 @@
 #=================================================
 # IMPORT GENERIC HELPERS
 #=================================================
-source ../settings/scripts/experimental_helper.sh
 source ../settings/scripts/_common.sh
 source /usr/share/yunohost/helpers
 
@@ -82,8 +81,8 @@ ynh_mysql_connect_as --user=$db_user --password=$db_pwd --database=$db_name < ./
 #=================================================
 
 ynh_script_progression --message="Reloading services..." --weight=3
-
+systemctl daemon-reload
-ynh_restore_uwsgi_service
+systemctl enable $app.service
 
 #=================================================
 # GENERIC FINALIZATION
@@ -92,8 +91,8 @@ ynh_restore_uwsgi_service
 #=================================================
 
 ynh_script_progression --message="Starting pgadmin services..." --weight=3
-ynh_systemd_action --service_name "uwsgi-app@$app.service" \
+ynh_systemd_action --service_name "$app.service" \
-    --line_match "WSGI app 0 \(mountpoint='[/[:alnum:]_-]*'\) ready in [[:digit:]]* seconds on interpreter" --log_path "/var/log/uwsgi/$app/$app.log"
+    --line_match "Booting worker with pid" --log_path "systemd"
-ynh_systemd_action --service_name=nginx --action=reload
+ynh_systemd_action --service_name=nginx --action=reload -t 20
 
 ynh_script_progression --message="Restoration completed for $app" --last

scripts/upgrade

@@ -6,7 +6,6 @@
 # IMPORT GENERIC HELPERS
 #=================================================
 
-source ./experimental_helper.sh
 source ./_common.sh
 source /usr/share/yunohost/helpers
 
@@ -23,6 +22,7 @@ final_path=$(ynh_app_setting_get --app $app --key=final_path)
 db_name=$(ynh_app_setting_get --app=$app --key=db_name)
 secret=$(ynh_app_setting_get --app $app --key=secret)
 db_pwd=$(ynh_app_setting_get --app=$app --key mysqlpwd)
+port=$(ynh_app_setting_get --app=$app --key web_port)
 db_user=$app
 
 #=================================================
@@ -88,6 +88,23 @@ ynh_clean_setup () {
 # Exit if an error occurs during the execution of the script
 ynh_abort_if_errors
 
+#=================================================
+# ENSURE DOWNWARD COMPATIBILITY
+#=================================================
+ynh_script_progression --message="Checking backware compatibility..." --weight=10
+
+# Detect old installation with uwsgi
+if [ -e /etc/uwsgi/apps-available/$app.ini ]; then
+	systemctl stop uwsgi-app@$app.service
+	systemctl disable uwsgi-app@$app.service
+	yunohost service remove "uwsgi-app@$app"
+	ynh_secure_remove --file=/etc/uwsgi/apps-available/$app.ini
+	ynh_secure_remove --file=/etc/systemd/system/uwsgi-app@$app.service.d
+	ynh_secure_remove --file=$final_path
+else
+	systemctl stop $app.service
+fi
+
 #=================================================
 # STANDARD UPGRADE STEPS
 #=================================================
@@ -105,7 +122,13 @@ ynh_install_app_dependencies $pkg_dependencies
 
 # Download, check integrity, uncompress and patch the source from app.src
 ynh_script_progression --message="Upgrading source files..." --weight=6
+if [ -e $final_path/syncserver.ini ]; then
+	config_backup="$(cat $final_path/syncserver.ini)"
 	install_sources
+	echo "$config_backup" > $final_path/syncserver.ini
+else
+	install_sources
+fi
 
 #=================================================
 # NGINX CONFIGURATION
@@ -134,11 +157,12 @@ ynh_system_user_create --username="$app"
 ynh_script_progression --message="Configuring application..."
 
 # create config file syncserver.ini
-rm "$final_path/syncserver.ini"
+ynh_script_progression --message="Configuring application..."
-ln -s "/etc/uwsgi/apps-available/$app.ini" "$final_path/syncserver.ini"
+ynh_add_config --template="syncserver.ini" --destination="$final_path/syncserver.ini"
 
-# configure uwsgi
+# Configure init script
-ynh_add_uwsgi_service 'domain secret db_user db_pwd db_name'
+ynh_script_progression --message="Configuring a systemd service..." --weight=2
+ynh_add_systemd_config
 
 # Upgrade database table
 ynh_mysql_execute_as_root --sql='ALTER TABLE `users` ADD COLUMN IF NOT EXISTS `keys_changed_at` BIGINT NULL AFTER `replaced_at`;' --database=$db_name
@@ -166,10 +190,11 @@ then
 fi
 ynh_permission_update --permission=main --add=visitors --protected=true --show_tile=true
 
+yunohost service add "$app"
 
 ynh_script_progression --message="Restarting $app services..." --weight=3
-ynh_systemd_action --service_name "uwsgi-app@$app.service" \
+ynh_systemd_action --service_name "$app.service" \
-    --line_match "WSGI app 0 \(mountpoint='[/[:alnum:]_-]*'\) ready in [[:digit:]]* seconds on interpreter" --log_path "/var/log/uwsgi/$app/$app.log"
+    --line_match "Booting worker with pid" --log_path "systemd" -t 20
 
 #=================================================
 # END OF SCRIPT