First implementation of matrix sliding proxy

conf/nginx.conf

@@ -12,6 +12,19 @@ location /_matrix/ {
 
         client_max_body_size __MAX_UPLOAD_SIZE__;
 }
+location /_matrix/client/unstable/org.matrix.msc3575/ {
+    proxy_pass http://localhost:__PORT_SLIDING_PROXY__;
+    proxy_set_header X-Forwarded-For $remote_addr;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header Host $host;
+}
+
+location /client/ {
+    proxy_pass http://localhost:__PORT_SLIDING_PROXY__;
+    proxy_set_header X-Forwarded-For $remote_addr;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header Host $host;
+}
 
 # Use the specific path for the php file. It's more secure than global php path
 location /_matrix/cas_server.php/ {

conf/server_name.conf

@@ -7,6 +7,7 @@ location = /.well-known/matrix/server {
 location = /.well-known/matrix/client {
     return 200 '{
         "m.homeserver": { "base_url": "https://__DOMAIN__" },
+        "org.matrix.msc3575.proxy": { "url": "https://__DOMAIN__"},
         "im.vector.riot.jitsi": {"preferredDomain": "__JITSI_SERVER__"},
         "im.vector.riot.e2ee": {"default": __E2E_ENABLED_BY_DEFAULT_CLIENT_CONFIG__ }
     }';

conf/sliding_proxy.conf

@@ -0,0 +1,34 @@
+# Required. The destination homeserver to talk to (CS API HTTPS URL) e.g 'https://matrix-client.matrix.org'
+SYNCV3_SERVER=https://__DOMAIN__
+# Required. The postgres connection string
+SYNCV3_DB=postgresql://__DB_USER__:__DB_PWD__@localhost/__DB_NAME_SLIDINGPROXY__
+# Required. A secret to use to encrypt access tokens. Must remain the same for the lifetime of the database.
+SYNCV3_SECRET=__SYNC_PROXY_SECRET__
+# Default: 0.0.0.0:8008.  The interface and port to listen on. (Supports unix socket: /path/to/socket)
+SYNCV3_BINDADDR=localhost:__PORT_SLIDING_PROXY__
+# Default: unset. Path to a certificate file to serve to HTTPS clients. Specifying this enables TLS on the bound address.
+# SYNCV3_TLS_CERT=
+# Default: unset. Path to a key file for the certificate. Must be provided along with the certificate file.
+# SYNCV3_TLS_KEY=
+# Default: unset. The bind addr for pprof debugging e.g ':6060'. If not set, does not listen.
+# SYNCV3_PPROF=
+# Default: unset. The bind addr for Prometheus metrics, which will be accessible at /metrics at this address.
+# SYNCV3_PROM=
+# Default: unset. The OTLP HTTP URL to send spans to e.g https://localhost:4318 - if unset does not send OTLP traces.
+# SYNCV3_OTLP_URL=
+# Default: unset. The OTLP username for Basic auth. If unset, does not send an Authorization header.
+# SYNCV3_OTLP_USERNAME=
+# Default: unset. The OTLP password for Basic auth. If unset, does not send an Authorization header.
+# SYNCV3_OTLP_PASSWORD=
+# Default: unset. The Sentry DSN to report events to e.g https://sliding-sync@sentry.example.com/123 - if unset does not send sentry events.
+# SYNCV3_SENTRY_DSN=
+# Default: info. The level of verbosity for messages logged. Available values are trace, debug, info, warn, error and fatal
+# SYNCV3_LOG_LEVEL=
+# Default: unset. Max database connections to use when communicating with postgres. Unset or 0 means no limit.
+# SYNCV3_MAX_DB_CONN=
+# Default: 3600. The maximum amount of time a database connection may be idle, in seconds. 0 means no limit.
+# SYNCV3_DB_IDLE_TIMEOUT_SECS=
+# Default: 300. The timeout in seconds for normal HTTP requests.
+# SYNCV3_HTTP_TIMEOUT_SECS=
+# Default: 1800. The timeout in seconds for initial sync requests.
+# SYNCV3_HTTP_INITIAL_TIMEOUT_SECS=

conf/synapse-sliding-proxy.service

@@ -0,0 +1,59 @@
+[Unit]
+Description=Matrix sliding proxy
+After=network.target
+
+#
+# Big fat warning
+#
+
+# This unit file MUST be reworked completly when we will be on debian bookworm
+# To make it working on debian bullseye many workaround was many, mainly chroot
+# So mainly this unit should be completly reworkd and cleaned since we dropt debian bullseye support
+
+
+[Service]
+Type=simple
+User=__APP__
+# WorkingDirectory=/opt/yunohost/matrix-__APP__
+EnvironmentFile=/etc/matrix-__APP__/sliding_proxy.conf
+ExecStart=/bin/sliding-proxy
+Restart=always
+RestartSec=3
+RootDirectory=/opt/yunohost/matrix-__APP__/sliding-chroot
+BindReadOnlyPaths=/etc
+BindReadOnlyPaths=/usr/share/ca-certificates
+
+# Sandboxing options to harden security
+# Details for these options: https://www.freedesktop.org/software/systemd/man/systemd.exec.html
+NoNewPrivileges=yes
+PrivateTmp=yes
+PrivateDevices=yes
+RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
+RestrictNamespaces=yes
+RestrictRealtime=yes
+DevicePolicy=closed
+ProtectClock=yes
+ProtectHostname=yes
+ProtectProc=invisible
+ProtectSystem=full
+ProtectControlGroups=yes
+ProtectKernelModules=yes
+ProtectKernelTunables=yes
+LockPersonality=yes
+SystemCallArchitectures=native
+# SystemCallFilter=~@clock @debug @module @mount @obsolete @reboot @setuid @swap @cpu-emulation @privileged
+
+# Denying access to capabilities that should not be relevant for webapps
+# Doc: https://man7.org/linux/man-pages/man7/capabilities.7.html
+CapabilityBoundingSet=~CAP_RAWIO CAP_MKNOD
+CapabilityBoundingSet=~CAP_AUDIT_CONTROL CAP_AUDIT_READ CAP_AUDIT_WRITE
+CapabilityBoundingSet=~CAP_SYS_BOOT CAP_SYS_TIME CAP_SYS_MODULE CAP_SYS_PACCT
+CapabilityBoundingSet=~CAP_LEASE CAP_LINUX_IMMUTABLE CAP_IPC_LOCK
+CapabilityBoundingSet=~CAP_BLOCK_SUSPEND CAP_WAKE_ALARM
+CapabilityBoundingSet=~CAP_SYS_TTY_CONFIG
+CapabilityBoundingSet=~CAP_MAC_ADMIN CAP_MAC_OVERRIDE
+CapabilityBoundingSet=~CAP_NET_ADMIN CAP_NET_BROADCAST CAP_NET_RAW
+CapabilityBoundingSet=~CAP_SYS_ADMIN CAP_SYS_PTRACE CAP_SYSLOG
+
+[Install]
+WantedBy=multi-user.target

manifest.toml

@@ -19,7 +19,7 @@ fund = "https://matrix.org/support/#"
 
 [integration]
 yunohost = ">= 11.2.10"
-architectures = "all"
+architectures = ["amd64", "arm64"]
 multi_instance = true
 ldap = true
 sso = "not_relevant"
@@ -71,6 +71,26 @@ ram.runtime = "200M"
     armhf.url = "https://github.com/YunoHost-Apps/synapse_python_build/releases/download/v1.102.0/matrix-synapse_1.102.0-bullseye-bin1_armv7l.tar.gz"
     armhf.sha256 = "4027b8e41950aeec151dfa19eda2402e7974de56d33abb9eee3879e211227eef"
 
+    [resources.sources.sliding_proxy]
+    prefetch = true
+    extract = false
+    rename = "sliding-proxy"
+
+    amd64.url = "https://github.com/matrix-org/sliding-sync/releases/download/v0.99.15/syncv3_linux_amd64"
+    amd64.sha256 = "f5a57a26a5a478470774d8f6387accf5ed7644c8258c8ff7570998b99f7afe01"
+    arm64.url = "https://github.com/matrix-org/sliding-sync/releases/download/v0.99.15/syncv3_linux_arm64"
+    arm64.sha256 = "24bd8c5ebf89806104465b8b218aba7e05b15c095d010aba2fb666172a106f75"
+
+    [resources.sources.sliding_proxy_rootfs]
+    prefetch = true
+    extract = true
+    in_subdir = false
+
+    amd64.url = "https://github.com/YunoHost-Apps/synapse_python_build/releases/download/sp-mfs-v1.0/min_rootfs_x86-64.tar.gz"
+    amd64.sha256 = "4626879513fedd8fca33eb98668a7d3967c6b04d2ea1eed86c5059ef394bc925"
+    arm64.url = "https://github.com/YunoHost-Apps/synapse_python_build/releases/download/sp-mfs-v1.0/min_rootfs_arm64.tar.gz"
+    arm64.sha256 = "3f71a0c1746d18cd5081fc68ba81d18d4dca717509b8de75ef1b23b99af325e0"
+
     [resources.system_user]
     allow_email = true
     home = "/opt/yunohost/matrix-__APP__"
@@ -92,6 +112,7 @@ ram.runtime = "200M"
     main.protected = true
 
     server_api.url = "/_matrix"
+    server_api.additional_urls = ["/client"]
     server_api.label = "Server access for client apps"
     server_api.allowed = "visitors"
     server_api.auth_header = false
@@ -109,6 +130,7 @@ ram.runtime = "200M"
     synapse_tls.default = 8448
     synapse_tls.exposed = "TCP"
     synapse.default = 8008
+    sliding_proxy.default = 8009
     turnserver_tls.default = 5349
     turnserver_tls.exposed = "Both"
     turnserver_alt_tls.default = 5350

scripts/_common.sh

@@ -1,6 +1,6 @@
 python_version="$(python3 -V | cut -d' ' -f2 | cut -d. -f1-2)"
 code_dir="/opt/yunohost/matrix-$app"
-
+db_name_slidingproxy=${db_name}_slidingproxy
 install_sources() {
     # Install/upgrade synapse in virtualenv
 
@@ -51,6 +51,13 @@ install_sources() {
         patch < $YNH_APP_BASEDIR/sources/ldap_auth_filter_anonymous_user.patch
         popd
     fi
+
+    # Setup chroot for sliding proxy
+    # Note that on debian bullseye we can't support run directly sliding proxy as it require new version of libc not available on debian bullseye
+    mkdir -p  $code_dir/sliding-chroot
+    ynh_setup_source -r --dest_dir=$code_dir/sliding-chroot/ --source_id=sliding_proxy_rootfs
+    mkdir -p  $code_dir/sliding-chroot/bin
+    ynh_setup_source --dest_dir=$code_dir/sliding-chroot/bin/ --source_id=sliding_proxy
 }
 
 configure_synapse() {
@@ -160,6 +167,7 @@ set_permissions() {
     chmod 770 $code_dir/Coturn_config_rotate.sh
     chmod 700 $code_dir/update_synapse_for_appservice.sh
     chmod 700 $code_dir/set_admin_user.sh
+    chmod 755 $code_dir/sliding-chroot/bin/sliding-proxy
 
     if [ "${1:-}" == data ]; then
         find $data_dir \(   \! -perm -o= \

scripts/backup

@@ -73,6 +73,7 @@ ynh_backup --src_path="/etc/matrix-$app"
 ynh_backup --src_path="/etc/systemd/system/$app.service"
 ynh_backup --src_path="/etc/default/coturn-$app"
 ynh_backup --src_path="/etc/systemd/system/$app-coturn.service"
+ynh_backup --src_path=/etc/systemd/system/$app-sliding-proxy.service
 
 #=================================================
 # BACKUP SYNAPSE DATA

scripts/install

@@ -132,12 +132,13 @@ adduser $app ssl-cert
 adduser turnserver ssl-cert
 
 #=================================================
-# FIX DB CONFIG
+# FIX DB CONFIG AND CREATE SYLING PROXY DB
 #=================================================
 ynh_script_progression --message="Fixing database type..." --weight=1
 
 ynh_psql_execute_as_root \
     --sql="update pg_database set datcollate='C', datctype='C' where datname='$db_name';"
+ynh_''psql_setup_db --db_user=$db_user --db_pwd=$db_pwd --db_name=$db_name_slidingproxy
 
 #=================================================
 # DOWNLOAD, CHECK AND UNPACK SOURCE
@@ -179,11 +180,13 @@ $code_dir/bin/python -m synapse.app.homeserver --keys-directory /etc/matrix-$app
 registration_shared_secret=$(egrep "^registration_shared_secret:" homeserver.yml | cut -d'"' -f2)
 form_secret=$(egrep "^form_secret:" homeserver.yml | cut -d'"' -f2)
 macaroon_secret_key=$(egrep "^macaroon_secret_key:" homeserver.yml | cut -d'"' -f2)
+sync_proxy_secret=$(ynh_string_random -l 40)
 
 # store in yunohost settings
 ynh_app_setting_set --app=$app --key=registration_shared_secret --value="$registration_shared_secret"
 ynh_app_setting_set --app=$app --key=form_secret --value="$form_secret"
 ynh_app_setting_set --app=$app --key=macaroon_secret_key --value="$macaroon_secret_key"
+ynh_app_setting_set --app=$app --key=sync_proxy_secret --value="$sync_proxy_secret"
 
 #=================================================
 # SETUP SYSTEMD
@@ -196,6 +199,8 @@ ynh_add_systemd_config --service=$app --template=synapse.service
 cp ../conf/default_coturn /etc/default/coturn-$app
 ynh_add_systemd_config --service=$app-coturn --template=synapse-coturn.service
 
+ynh_add_systemd_config --service=$app-sliding-proxy --template=synapse-sliding-proxy.service
+
 #=================================================
 # NGINX CONFIGURATION
 #=================================================
@@ -218,6 +223,7 @@ turnserver_pwd=$(ynh_string_random --length=30)
 ynh_app_setting_set --app=$app --key=turnserver_pwd --value=$turnserver_pwd
 
 configure_synapse
+ynh_add_config --template=sliding_proxy.conf --destination=/etc/matrix-$app/sliding_proxy.conf
 
 #=================================================
 # SET COTURN CONFIG
@@ -282,6 +288,7 @@ set_permissions data
 
 yunohost service add $app --log "/var/log/matrix-$app/homeserver.log" --needs_exposed_ports $port_synapse_tls --description 'Main matrix server service.'
 yunohost service add $app-coturn --needs_exposed_ports $port_turnserver_tls --description 'Turn server for matrix server. Used for audio and video call.'
+yunohost service add $app-sliding-proxy --description 'Matrix v2 service for clients.'
 
 #=================================================
 # RELOAD SERVICES
@@ -290,6 +297,7 @@ ynh_script_progression --message="Restarting Synapse services..." --weight=11
 
 ynh_systemd_action --service_name=$app-coturn.service --action=restart
 ynh_systemd_action --service_name=$app.service --action=restart --line_match="Synapse now listening on TCP port $port_synapse_tls" --log_path="/var/log/matrix-$app/homeserver.log" --timeout=300
+ynh_systemd_action --service_name=$app-sliding-proxy.service --action=restart
 
 #=================================================
 # SETUP FAIL2BAN

scripts/remove

@@ -19,6 +19,7 @@ source /usr/share/yunohost/helpers
 # Remove a service from the admin panel, added by `yunohost service add`
 yunohost service remove $app
 yunohost service remove $app-coturn
+yunohost service remove $app-sliding-proxy
 
 #=================================================
 # STOP AND REMOVE SERVICE
@@ -27,6 +28,7 @@ ynh_script_progression --message="Stopping and removing the systemd service" --w
 
 ynh_remove_systemd_config --service=$app
 ynh_remove_systemd_config --service=$app-coturn
+ynh_remove_systemd_config --service=$app-sliding-proxy
 
 #=================================================
 # REMOVE APP MAIN DIR
@@ -39,6 +41,13 @@ ynh_secure_remove --file=/etc/matrix-$app
 ynh_secure_remove --file=/etc/default/coturn-$app
 ynh_secure_remove --file=/etc/nginx/conf.d/${server_name}.d/${app}_server_name.conf
 
+#=================================================
+# REMOVE DATABASE
+#=================================================
+ynh_script_progression --message="Removing database" --weight=2
+
+ynh_''psql_remove_db --db_user=$db_user --db_name=$db_name_slidingproxy
+
 #=================================================
 # REMOVE NGINX CONFIGURATION
 #=================================================

scripts/restore

@@ -68,6 +68,7 @@ ynh_script_progression --message="Enable systemd services" --weight=2
 # systemctl daemon-reload
 systemctl enable $app.service --quiet
 systemctl enable $app-coturn.service --quiet
+systemctl enable $app-sliding-proxy.service --quiet
 
 #=================================================
 # ADVERTISE SERVICE IN ADMIN PANEL
@@ -75,6 +76,7 @@ systemctl enable $app-coturn.service --quiet
 
 yunohost service add $app --log "/var/log/matrix-$app/homeserver.log" --needs_exposed_ports $port_synapse_tls --description 'Main matrix server service.'
 yunohost service add $app-coturn --needs_exposed_ports $port_turnserver_tls --description 'Turn server for matrix server. Used for audio and video call.'
+yunohost service add $app-sliding-proxy --description 'Matrix v2 service for clients.'
 
 #=================================================
 # CREATE A DH FILE
@@ -122,6 +124,7 @@ ynh_script_progression --message="Restarting Synapse services..." --weight=7
 
 ynh_systemd_action --service_name=$app-coturn.service --action=restart
 ynh_systemd_action --service_name=$app.service --action=restart --line_match="Synapse now listening on TCP port $port_synapse_tls" --log_path="/var/log/matrix-$app/homeserver.log" --timeout=300
+ynh_systemd_action --service_name=$app-sliding-proxy.service --action=restart
 
 #=================================================
 # GENERIC FINALIZATION

scripts/upgrade

@@ -204,6 +204,11 @@ then
     enable_dtls_for_audio_video_turn_call="true"
     ynh_app_setting_set --app=$app --key=enable_dtls_for_audio_video_turn_call --value=$enable_dtls_for_audio_video_turn_call
 fi
+if [ -z "${sync_proxy_secret:-}" ]
+then
+    sync_proxy_secret=$(ynh_string_random -l 40)
+    ynh_app_setting_set --app=$app --key=sync_proxy_secret --value=$sync_proxy_secret
+fi
 
 #=================================================
 # MIGRATION 7 : STANDARDIZE SYSTEMD UNIT
@@ -381,6 +386,15 @@ then
     ynh_app_setting_set --app=$app --key=form_secret --value="$form_secret"
 fi
 
+#=================================================
+# MIGRATION 13 : ADD SLIDING PROXY DB
+#=================================================
+ynh_script_progression --message="Updating db if needed..."
+
+if ! ynh_''psql_database_exists -d $db_name_slidingproxy; then
+    ynh_''psql_setup_db --db_user=$db_user --db_pwd=$db_pwd --db_name=$db_name_slidingproxy
+fi
+
 #=================================================
 # STANDARD UPGRADE STEPS
 #=================================================
@@ -391,6 +405,7 @@ fi
 ynh_script_progression --message="Updating synapse config..." --weight=2
 
 configure_synapse
+ynh_add_config --template=sliding_proxy.conf --destination=/etc/matrix-$app/sliding_proxy.conf
 
 #=================================================
 # CREATE SMALL CAS SERVER
@@ -453,6 +468,7 @@ fi
 
 yunohost service add $app --log "/var/log/matrix-$app/homeserver.log" --needs_exposed_ports $port_synapse_tls --description 'Main matrix server service.'
 yunohost service add $app-coturn --needs_exposed_ports $port_turnserver_tls --description 'Turn server for matrix server. Used for audio and video call.'
+yunohost service add $app-sliding-proxy --description 'Matrix v2 service for clients.'
 
 #=================================================
 # UPDATE SYSTEMD
@@ -465,6 +481,8 @@ ynh_add_systemd_config --service=$app --template=synapse.service
 cp ../conf/default_coturn /etc/default/coturn-$app
 ynh_add_systemd_config --service=$app-coturn --template=synapse-coturn.service
 
+ynh_add_systemd_config --service=$app-sliding-proxy --template=synapse-sliding-proxy.service
+
 #=================================================
 # UPGRADE FAIL2BAN
 #=================================================
@@ -516,6 +534,7 @@ ynh_script_progression --message="Restarting Synapse services..." --weight=5
 
 ynh_systemd_action --service_name=$app-coturn.service --action=restart
 ynh_systemd_action --service_name=$app.service --action=restart --line_match="Synapse now listening on TCP port $port_synapse_tls" --log_path="/var/log/matrix-$app/homeserver.log" --timeout=300
+ynh_systemd_action --service_name=$app-sliding-proxy.service --action=restart
 
 #=================================================
 # END OF SCRIPT

tests.toml

@@ -7,3 +7,5 @@ test_format = 1.0
     test_upgrade_from.672791a51c1d239918562d7a0d4420ec137e6694.name = "Post app user creation (branch old_version_for_CI_7)"
 
     test_upgrade_from.971f2eb590325fb1d6e1ca5723f59aacd639c9ce.name = "Before packaging v2 (branch old_version_for_CI_6)"
+
+    test_upgrade_from.214c337b35d34fbdb1ad28324e7c45437e7cae48.name = "Before Matrix v2 implementation"