Merge 155514feb8 into 09fcfc6f56

conf/nginx.conf

@@ -12,6 +12,19 @@ location /_matrix/ {
 
         client_max_body_size __MAX_UPLOAD_SIZE__;
 }
+location /_matrix/client/unstable/org.matrix.msc3575/ {
+    proxy_pass http://localhost:__PORT_SLIDING_PROXY__;
+    proxy_set_header X-Forwarded-For $remote_addr;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header Host $host;
+}
+
+location /client/ {
+    proxy_pass http://localhost:__PORT_SLIDING_PROXY__;
+    proxy_set_header X-Forwarded-For $remote_addr;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header Host $host;
+}
 
 # Use the specific path for the php file. It's more secure than global php path
 location /_matrix/cas_server.php/ {

conf/server_name.conf

@@ -7,6 +7,7 @@ location = /.well-known/matrix/server {
 location = /.well-known/matrix/client {
     return 200 '{
         "m.homeserver": { "base_url": "https://__DOMAIN__" },
+        "org.matrix.msc3575.proxy": { "url": "https://__DOMAIN__"},
         "im.vector.riot.jitsi": {"preferredDomain": "__JITSI_SERVER__"},
         "im.vector.riot.e2ee": {"default": __E2E_ENABLED_BY_DEFAULT_CLIENT_CONFIG__ }
     }';

conf/sliding_proxy.conf

@@ -0,0 +1,34 @@
+# Required. The destination homeserver to talk to (CS API HTTPS URL) e.g 'https://matrix-client.matrix.org'
+SYNCV3_SERVER=https://__DOMAIN__
+# Required. The postgres connection string
+SYNCV3_DB=postgresql://__DB_USER__:__DB_PWD__@localhost/__DB_NAME_SLIDINGPROXY__
+# Required. A secret to use to encrypt access tokens. Must remain the same for the lifetime of the database.
+SYNCV3_SECRET=__SYNC_PROXY_SECRET__
+# Default: 0.0.0.0:8008.  The interface and port to listen on. (Supports unix socket: /path/to/socket)
+SYNCV3_BINDADDR=localhost:__PORT_SLIDING_PROXY__
+# Default: unset. Path to a certificate file to serve to HTTPS clients. Specifying this enables TLS on the bound address.
+# SYNCV3_TLS_CERT=
+# Default: unset. Path to a key file for the certificate. Must be provided along with the certificate file.
+# SYNCV3_TLS_KEY=
+# Default: unset. The bind addr for pprof debugging e.g ':6060'. If not set, does not listen.
+# SYNCV3_PPROF=
+# Default: unset. The bind addr for Prometheus metrics, which will be accessible at /metrics at this address.
+# SYNCV3_PROM=
+# Default: unset. The OTLP HTTP URL to send spans to e.g https://localhost:4318 - if unset does not send OTLP traces.
+# SYNCV3_OTLP_URL=
+# Default: unset. The OTLP username for Basic auth. If unset, does not send an Authorization header.
+# SYNCV3_OTLP_USERNAME=
+# Default: unset. The OTLP password for Basic auth. If unset, does not send an Authorization header.
+# SYNCV3_OTLP_PASSWORD=
+# Default: unset. The Sentry DSN to report events to e.g https://sliding-sync@sentry.example.com/123 - if unset does not send sentry events.
+# SYNCV3_SENTRY_DSN=
+# Default: info. The level of verbosity for messages logged. Available values are trace, debug, info, warn, error and fatal
+# SYNCV3_LOG_LEVEL=
+# Default: unset. Max database connections to use when communicating with postgres. Unset or 0 means no limit.
+# SYNCV3_MAX_DB_CONN=
+# Default: 3600. The maximum amount of time a database connection may be idle, in seconds. 0 means no limit.
+# SYNCV3_DB_IDLE_TIMEOUT_SECS=
+# Default: 300. The timeout in seconds for normal HTTP requests.
+# SYNCV3_HTTP_TIMEOUT_SECS=
+# Default: 1800. The timeout in seconds for initial sync requests.
+# SYNCV3_HTTP_INITIAL_TIMEOUT_SECS=

conf/synapse-sliding-proxy.service

@@ -0,0 +1,59 @@
+[Unit]
+Description=Matrix sliding proxy
+After=network.target
+
+#
+# Big fat warning
+#
+
+# This unit file MUST be reworked completly when we will be on debian bookworm
+# To make it working on debian bullseye many workaround was many, mainly chroot
+# So mainly this unit should be completly reworkd and cleaned since we dropt debian bullseye support
+
+
+[Service]
+Type=simple
+User=__APP__
+# WorkingDirectory=/opt/yunohost/matrix-__APP__
+EnvironmentFile=/etc/matrix-__APP__/sliding_proxy.conf
+ExecStart=/bin/sliding-proxy
+Restart=always
+RestartSec=3
+RootDirectory=/opt/yunohost/matrix-__APP__/sliding-chroot
+BindReadOnlyPaths=/etc
+BindReadOnlyPaths=/usr/share/ca-certificates
+
+# Sandboxing options to harden security
+# Details for these options: https://www.freedesktop.org/software/systemd/man/systemd.exec.html
+NoNewPrivileges=yes
+PrivateTmp=yes
+PrivateDevices=yes
+RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
+RestrictNamespaces=yes
+RestrictRealtime=yes
+DevicePolicy=closed
+ProtectClock=yes
+ProtectHostname=yes
+ProtectProc=invisible
+ProtectSystem=full
+ProtectControlGroups=yes
+ProtectKernelModules=yes
+ProtectKernelTunables=yes
+LockPersonality=yes
+SystemCallArchitectures=native
+# SystemCallFilter=~@clock @debug @module @mount @obsolete @reboot @setuid @swap @cpu-emulation @privileged
+
+# Denying access to capabilities that should not be relevant for webapps
+# Doc: https://man7.org/linux/man-pages/man7/capabilities.7.html
+CapabilityBoundingSet=~CAP_RAWIO CAP_MKNOD
+CapabilityBoundingSet=~CAP_AUDIT_CONTROL CAP_AUDIT_READ CAP_AUDIT_WRITE
+CapabilityBoundingSet=~CAP_SYS_BOOT CAP_SYS_TIME CAP_SYS_MODULE CAP_SYS_PACCT
+CapabilityBoundingSet=~CAP_LEASE CAP_LINUX_IMMUTABLE CAP_IPC_LOCK
+CapabilityBoundingSet=~CAP_BLOCK_SUSPEND CAP_WAKE_ALARM
+CapabilityBoundingSet=~CAP_SYS_TTY_CONFIG
+CapabilityBoundingSet=~CAP_MAC_ADMIN CAP_MAC_OVERRIDE
+CapabilityBoundingSet=~CAP_NET_ADMIN CAP_NET_BROADCAST CAP_NET_RAW
+CapabilityBoundingSet=~CAP_SYS_ADMIN CAP_SYS_PTRACE CAP_SYSLOG
+
+[Install]
+WantedBy=multi-user.target

config_panel.toml

@@ -14,12 +14,13 @@ services = ["__APP__"]
         no = "false"
         help = "Defaults to 'false'. If 'true', it is highly recommended to use either captcha, email, or token-based verification to avoid SPAM."
 
-        [main.welcome.password_enabled]
+        # Disabled for matrix V2 because some client like Element X don't support cas and so require to have password authentication enabled
-        ask = "Enable Password Login"
+        # [main.welcome.password_enabled]
-        type = "boolean"
+        # ask = "Enable Password Login"
-        yes = "true"
+        # type = "boolean"
-        no = "false"
+        # yes = "true"
-        help = "If disabled, Login with Non-YunoHost Users impossible. But it simplies Login process if your Matrix server only has YunoHost SSO Users."
+        # no = "false"
+        # help = "If disabled, Login with Non-YunoHost Users impossible. But it simplies Login process if your Matrix server only has YunoHost SSO Users."
         # Temporary disable the visible flag due of https://github.com/YunoHost/issues/issues/2331
         # visible = "! enable_registration"
 

manifest.toml

@@ -19,7 +19,7 @@ fund = "https://matrix.org/support/#"
 
 [integration]
 yunohost = ">= 11.2.13"
-architectures = "all"
+architectures = ["amd64", "arm64"]
 multi_instance = true
 ldap = true
 sso = "not_relevant"
@@ -71,6 +71,26 @@ ram.runtime = "1G"
     armhf.url = "https://github.com/YunoHost-Apps/synapse_python_build/releases/download/v1.113.0/matrix-synapse_1.113.0-bullseye-bin1_armv7l.tar.gz"
     armhf.sha256 = "9df230b5d0d34ec978f2f1fe63ac4bb9383f03b13e5268624cfaa7779f747300"
 
+    [resources.sources.sliding_proxy]
+    prefetch = true
+    extract = false
+    rename = "sliding-proxy"
+
+    amd64.url = "https://github.com/matrix-org/sliding-sync/releases/download/v0.99.18/syncv3_linux_amd64"
+    amd64.sha256 = "852ffefe59332d0ad55db4518b2be44e0b120e563afe3fb52afe078a0feacb7c"
+    arm64.url = "https://github.com/matrix-org/sliding-sync/releases/download/v0.99.18/syncv3_linux_arm64"
+    arm64.sha256 = "bfff6be2f57e1a55698ea1945e29bdb5ffd7ed6043069edaf2f0e21c384e2872"
+
+    [resources.sources.sliding_proxy_rootfs]
+    prefetch = true
+    extract = true
+    in_subdir = false
+
+    amd64.url = "https://github.com/YunoHost-Apps/synapse_python_build/releases/download/sp-mfs-v1.0/min_rootfs_x86-64.tar.gz"
+    amd64.sha256 = "4626879513fedd8fca33eb98668a7d3967c6b04d2ea1eed86c5059ef394bc925"
+    arm64.url = "https://github.com/YunoHost-Apps/synapse_python_build/releases/download/sp-mfs-v1.0/min_rootfs_arm64.tar.gz"
+    arm64.sha256 = "3f71a0c1746d18cd5081fc68ba81d18d4dca717509b8de75ef1b23b99af325e0"
+
     [resources.system_user]
     allow_email = true
     home = "/opt/yunohost/matrix-__APP__"
@@ -92,6 +112,7 @@ ram.runtime = "1G"
     main.protected = true
 
     server_api.url = "/_matrix"
+    server_api.additional_urls = ["/client"]
     server_api.label = "Server access for client apps"
     server_api.allowed = "visitors"
     server_api.auth_header = false
@@ -109,6 +130,7 @@ ram.runtime = "1G"
     synapse_tls.default = 8448
     synapse_tls.exposed = "TCP"
     synapse.default = 8008
+    sliding_proxy.default = 8009
     turnserver_tls.default = 5349
     turnserver_tls.exposed = "Both"
     turnserver_alt_tls.default = 5350

scripts/_common.sh

@@ -1,6 +1,7 @@
 readonly python_version="$(python3 -V | cut -d' ' -f2 | cut -d. -f1-2)"
 readonly code_dir="/opt/yunohost/matrix-$app"
 readonly domain_whitelist_client="$(yunohost --output-as json domain list  | jq -r '.domains | .[]')"
+readonly db_name_slidingproxy="${db_name}"_slidingproxy
 
 install_sources() {
     # Install/upgrade synapse in virtualenv
@@ -53,6 +54,13 @@ install_sources() {
         patch < "$YNH_APP_BASEDIR"/scripts/patch/ldap_auth_filter_anonymous_user.patch
         popd
     fi
+
+    # Setup chroot for sliding proxy
+    # Note that on debian bullseye we can't support run directly sliding proxy as it require new version of libc not available on debian bullseye
+    mkdir -p  $code_dir/sliding-chroot
+    ynh_setup_source -r --dest_dir=$code_dir/sliding-chroot/ --source_id=sliding_proxy_rootfs
+    mkdir -p  $code_dir/sliding-chroot/bin
+    ynh_setup_source --dest_dir=$code_dir/sliding-chroot/bin/ --source_id=sliding_proxy
 }
 
 configure_coturn() {
@@ -244,6 +252,12 @@ ensure_vars_set() {
         enable_dtls_for_audio_video_turn_call=true
         ynh_app_setting_set --app="$app" --key=enable_dtls_for_audio_video_turn_call --value="$enable_dtls_for_audio_video_turn_call"
     fi
+
+    if [ -z "${sync_proxy_secret:-}" ]
+    then
+        sync_proxy_secret=$(ynh_string_random -l 40)
+        ynh_app_setting_set --app=$app --key=sync_proxy_secret --value=$sync_proxy_secret
+    fi
 }
 
 set_permissions() {
@@ -253,6 +267,7 @@ set_permissions() {
     chmod 770 "$code_dir"/Coturn_config_rotate.sh
     chmod 700 "$code_dir"/update_synapse_for_appservice.sh
     chmod 700 "$code_dir"/set_admin_user.sh
+    chmod 755 "$code_dir"/sliding-chroot/bin/sliding-proxy
 
     if [ "${1:-}" == data ]; then
         find "$data_dir" \(   \! -perm -o= \

scripts/backup

@@ -72,6 +72,7 @@ ynh_backup --src_path="/etc/matrix-$app"
 
 ynh_backup --src_path="/etc/systemd/system/$app.service"
 ynh_backup --src_path="/etc/systemd/system/$app-coturn.service"
+ynh_backup --src_path=/etc/systemd/system/"$app"-sliding-proxy.service
 
 #=================================================
 # BACKUP SYNAPSE DATA

scripts/change_url

@@ -28,6 +28,10 @@ configure_nginx
 #=================================================
 
 ynh_script_progression --message="Updating Synapse config..." --weight=2
+
+# Force enable it because some client like Element X don't support CAS and so require to have password authentication enabled
+password_enabled=true
+
 ynh_add_config --jinja --template="homeserver.yaml" --destination="/etc/matrix-$app/homeserver.yaml"
 ynh_add_config --template="log.yaml" --destination="/etc/matrix-$app/log.yaml"
 

scripts/config

@@ -27,6 +27,10 @@ ynh_app_config_validate() {
 ynh_app_config_apply() {
     _ynh_app_config_apply
     configure_nginx
+
+    # Force enable it because some client like Element X don't support CAS and so require to have password authentication enabled
+    password_enabled=true
+
     ynh_add_config --jinja --template="homeserver.yaml" --destination="/etc/matrix-$app/homeserver.yaml"
     ynh_add_config --template="log.yaml" --destination="/etc/matrix-$app/log.yaml"
     set_permissions

scripts/install

@@ -75,12 +75,13 @@ adduser "$app" ssl-cert
 adduser turnserver ssl-cert
 
 #=================================================
-# FIX DB CONFIG
+# FIX DB CONFIG AND CREATE SYLING PROXY DB
 #=================================================
 ynh_script_progression --message="Fixing database type..." --weight=1
 
 ynh_psql_execute_as_root \
     --sql="update pg_database set datcollate='C', datctype='C' where datname='$db_name';"
+ynh_''psql_setup_db --db_user=$db_user --db_pwd=$db_pwd --db_name=$db_name_slidingproxy
 
 #=================================================
 # DOWNLOAD, CHECK AND UNPACK SOURCE
@@ -139,6 +140,8 @@ ynh_add_systemd_config --service="$app" --template=synapse.service
 cp ../conf/default_coturn /etc/matrix-"$app"/coturn_env
 ynh_add_systemd_config --service="$app"-coturn --template=synapse-coturn.service
 
+ynh_add_systemd_config --service=$app-sliding-proxy --template=synapse-sliding-proxy.service
+
 #=================================================
 # NGINX CONFIGURATION
 #=================================================
@@ -156,8 +159,12 @@ configure_nginx
 #=================================================
 ynh_script_progression --message="Configuring Synapse..." --weight=2
 
+# Force enable it because some client like Element X don't support CAS and so require to have password authentication enabled
+password_enabled=true
+
 ynh_add_config --jinja --template="homeserver.yaml" --destination="/etc/matrix-$app/homeserver.yaml"
 ynh_add_config --template="log.yaml" --destination="/etc/matrix-$app/log.yaml"
+ynh_add_config --template=sliding_proxy.conf --destination=/etc/matrix-$app/sliding_proxy.conf
 
 #=================================================
 # SET COTURN CONFIG
@@ -222,6 +229,7 @@ set_permissions data
 
 yunohost service add "$app" --log "/var/log/matrix-$app/homeserver.log" --needs_exposed_ports "$port_synapse_tls" --description 'Main matrix server service.'
 yunohost service add "$app"-coturn --needs_exposed_ports "$port_turnserver_tls" --description 'Turn server for matrix server. Used for audio and video call.'
+yunohost service add "$app"-sliding-proxy --description 'Matrix v2 service for clients.'
 
 #=================================================
 # RELOAD SERVICES
@@ -230,6 +238,7 @@ ynh_script_progression --message="Restarting Synapse services..." --weight=11
 
 ynh_systemd_action --service_name="$app"-coturn.service --action=restart
 ynh_systemd_action --service_name="$app".service --action=restart --line_match="Synapse now listening on TCP port $port_synapse_tls" --log_path="/var/log/matrix-$app/homeserver.log" --timeout=300
+ynh_systemd_action --service_name="$app"-sliding-proxy.service --action=restart
 
 #=================================================
 # SETUP FAIL2BAN

scripts/remove

@@ -19,6 +19,7 @@ source /usr/share/yunohost/helpers
 # Remove a service from the admin panel, added by `yunohost service add`
 yunohost service remove "$app"
 yunohost service remove "$app"-coturn
+yunohost service remove "$app"-sliding-proxy
 
 #=================================================
 # STOP AND REMOVE SERVICE
@@ -27,6 +28,7 @@ ynh_script_progression --message="Stopping and removing the systemd service" --w
 
 ynh_remove_systemd_config --service="$app"
 ynh_remove_systemd_config --service="$app"-coturn
+ynh_remove_systemd_config --service="$app"-sliding-proxy
 
 #=================================================
 # REMOVE APP MAIN DIR
@@ -37,6 +39,13 @@ ynh_secure_remove --file="$code_dir"
 ynh_secure_remove --file=/etc/matrix-"$app"
 ynh_secure_remove --file="/etc/nginx/conf.d/${server_name}.d/${app}_server_name.conf"
 
+#=================================================
+# REMOVE DATABASE
+#=================================================
+ynh_script_progression --message="Removing database" --weight=2
+
+ynh_''psql_remove_db --db_user=user_wich_must_dont_exist_and_keep_current_user --db_name=$db_name_slidingproxy
+
 #=================================================
 # REMOVE NGINX CONFIGURATION
 #=================================================

scripts/restore

@@ -32,6 +32,7 @@ ynh_script_progression --message="Fixing database type..." --weight=1
 
 ynh_psql_execute_as_root \
     --sql="update pg_database set datcollate='C', datctype='C' where datname='$db_name';"
+ynh_''psql_setup_db --db_user="$db_user" --db_pwd="$db_pwd" --db_name="$db_name_slidingproxy"
 
 #=================================================
 # RESTORE ALL CONFIG AND DATA
@@ -68,6 +69,7 @@ ynh_script_progression --message="Enable systemd services" --weight=2
 # systemctl daemon-reload
 systemctl enable $app.service --quiet
 systemctl enable $app-coturn.service --quiet
+systemctl enable $app-sliding-proxy.service --quiet
 
 #=================================================
 # ADVERTISE SERVICE IN ADMIN PANEL
@@ -75,6 +77,7 @@ systemctl enable $app-coturn.service --quiet
 
 yunohost service add $app --log "/var/log/matrix-$app/homeserver.log" --needs_exposed_ports $port_synapse_tls --description 'Main matrix server service.'
 yunohost service add $app-coturn --needs_exposed_ports $port_turnserver_tls --description 'Turn server for matrix server. Used for audio and video call.'
+yunohost service add $app-sliding-proxy --description 'Matrix v2 service for clients.'
 
 #=================================================
 # CREATE A DH FILE
@@ -122,6 +125,7 @@ ynh_script_progression --message="Restarting Synapse services..." --weight=7
 
 ynh_systemd_action --service_name=$app-coturn.service --action=restart
 ynh_systemd_action --service_name=$app.service --action=restart --line_match="Synapse now listening on TCP port $port_synapse_tls" --log_path="/var/log/matrix-$app/homeserver.log" --timeout=300
+ynh_systemd_action --service_name=$app-sliding-proxy.service --action=restart
 
 #=================================================
 # GENERIC FINALIZATION

scripts/upgrade

@@ -271,6 +271,15 @@ then
     ynh_app_setting_set --app=$app --key=form_secret --value="$form_secret"
 fi
 
+#=================================================
+# MIGRATION 13 : ADD SLIDING PROXY DB
+#=================================================
+ynh_script_progression --message="Updating db if needed..."
+
+if ! ynh_''psql_database_exists -d $db_name_slidingproxy; then
+    ynh_''psql_setup_db --db_user=$db_user --db_pwd=$db_pwd --db_name=$db_name_slidingproxy
+fi
+
 #=================================================
 # STANDARD UPGRADE STEPS
 #=================================================
@@ -279,8 +288,13 @@ fi
 # UPDATE SYNAPSE CONFIG
 #=================================================
 ynh_script_progression --message="Updating synapse config..." --weight=2
+
+# Force enable it because some client like Element X don't support CAS and so require to have password authentication enabled
+password_enabled=true
+
 ynh_add_config --jinja --template="homeserver.yaml" --destination="/etc/matrix-$app/homeserver.yaml"
 ynh_add_config --template="log.yaml" --destination="/etc/matrix-$app/log.yaml"
+ynh_add_config --template=sliding_proxy.conf --destination=/etc/matrix-$app/sliding_proxy.conf
 
 #=================================================
 # CREATE SMALL CAS SERVER
@@ -343,6 +357,7 @@ fi
 
 yunohost service add $app --log "/var/log/matrix-$app/homeserver.log" --needs_exposed_ports $port_synapse_tls --description 'Main matrix server service.'
 yunohost service add $app-coturn --needs_exposed_ports $port_turnserver_tls --description 'Turn server for matrix server. Used for audio and video call.'
+yunohost service add $app-sliding-proxy --description 'Matrix v2 service for clients.'
 
 #=================================================
 # UPDATE SYSTEMD
@@ -355,6 +370,8 @@ ynh_add_systemd_config --service=$app --template=synapse.service
 cp ../conf/default_coturn /etc/matrix-$app/coturn_env
 ynh_add_systemd_config --service=$app-coturn --template=synapse-coturn.service
 
+ynh_add_systemd_config --service=$app-sliding-proxy --template=synapse-sliding-proxy.service
+
 #=================================================
 # UPGRADE FAIL2BAN
 #=================================================
@@ -406,6 +423,7 @@ ynh_script_progression --message="Restarting Synapse services..." --weight=5
 
 ynh_systemd_action --service_name=$app-coturn.service --action=restart
 ynh_systemd_action --service_name=$app.service --action=restart --line_match="Synapse now listening on TCP port $port_synapse_tls" --log_path="/var/log/matrix-$app/homeserver.log" --timeout=300
+ynh_systemd_action --service_name=$app-sliding-proxy.service --action=restart
 
 #=================================================
 # END OF SCRIPT

tests.toml

@@ -7,3 +7,5 @@ test_format = 1.0
     test_upgrade_from.672791a51c1d239918562d7a0d4420ec137e6694.name = "Post app user creation (branch old_version_for_CI_7)"
 
     test_upgrade_from.971f2eb590325fb1d6e1ca5723f59aacd639c9ce.name = "Before packaging v2 (branch old_version_for_CI_6)"
+
+    test_upgrade_from.0d433dc2d756cf39b13b1be8a7b9c00655edf4fb.name = "Before Matrix v2 implementation"