Merge pull request #1369 from YunoHost/modules-sortof

Making mariadb, php and metronome optional + better integration for postgresql + add sury by default

data/helpers.d/apt

@@ -253,9 +253,6 @@ ynh_install_app_dependencies() {
     # The (?<=php) syntax corresponds to lookbehind ;)
     local specific_php_version=$(echo $dependencies | grep -oP '(?<=php)[0-9.]+(?=-|\>)' | sort -u)
 
-    # Ignore case where the php version found is the one available in debian vanilla
-    [[ "$specific_php_version" != "$YNH_DEFAULT_PHP_VERSION" ]] || specific_php_version=""
-
     if [[ -n "$specific_php_version" ]]
     then
         # Cover a small edge case where a packager could have specified "php7.4-pwet php5-gni" which is confusing
@@ -263,10 +260,9 @@ ynh_install_app_dependencies() {
             || ynh_die --message="Inconsistent php versions in dependencies ... found : $specific_php_version"
 
         dependencies+=", php${specific_php_version}, php${specific_php_version}-fpm, php${specific_php_version}-common"
-
-        ynh_add_sury
     fi
 
+    local psql_installed="$(ynh_package_is_installed "postgresql-$PSQL_VERSION" && echo yes || echo no)"
 
     # The first time we run ynh_install_app_dependencies, we will replace the
     # entire control file (This is in particular meant to cover the case of
@@ -286,23 +282,6 @@ ynh_install_app_dependencies() {
         dependencies="$current_dependencies, $dependencies"
     fi
 
-    #
-    # Epic ugly hack to fix the goddamn dependency nightmare of sury
-    # Sponsored by the "Djeezusse Fokin Kraiste Why Do Adminsys Has To Be So Fucking Complicated I Should Go Grow Potatoes Instead Of This Shit" collective
-    # https://github.com/YunoHost/issues/issues/1407
-    #
-    # If we require to install php dependency
-    if grep --quiet 'php' <<< "$dependencies"; then
-        # And we have packages from sury installed (7.0.33-10+weirdshiftafter instead of 7.0.33-0 on debian)
-        if dpkg --list | grep "php7.0" | grep --quiet --invert-match "7.0.33-0+deb9"; then
-            # And sury ain't already in sources.lists
-            if ! grep --recursive --quiet "^ *deb.*sury" /etc/apt/sources.list*; then
-                # Re-add sury
-                ynh_add_sury
-            fi
-        fi
-    fi
-
     cat >/tmp/${dep_app}-ynh-deps.control <<EOF # Make a control file for equivs-build
 Section: misc
 Priority: optional
@@ -319,34 +298,21 @@ EOF
 
     if [[ -n "$specific_php_version" ]]
     then
-        # Set the default php version back as the default version for php-cli.
-        update-alternatives --set php /usr/bin/php$YNH_DEFAULT_PHP_VERSION
-
-        # Store phpversion into the config of this app
         ynh_app_setting_set --app=$app --key=phpversion --value=$specific_php_version
 
-        # Integrate new php-fpm service in yunohost
+        # Set the default php version back as the default version for php-cli.
-        yunohost service add php${specific_php_version}-fpm --log "/var/log/php${phpversion}-fpm.log"
+        update-alternatives --set php /usr/bin/php$YNH_DEFAULT_PHP_VERSION
-    elif grep --quiet 'php' <<< "$dependencies"; then
-        # Store phpversion into the config of this app
-        ynh_app_setting_set --app=$app --key=phpversion --value=$YNH_DEFAULT_PHP_VERSION
     fi
-}
 
-# Add sury repository with adequate pin strategy
+    # Trigger postgresql regenconf if we may have just installed postgresql
-#
+    local psql_installed2="$(ynh_package_is_installed "postgresql-$PSQL_VERSION" && echo yes || echo no)"
-# [internal]
+    if [[ "$psql_installed" != "$psql_installed2" ]]
-#
+    then
-# usage: ynh_add_sury
+        yunohost tools regen-conf postgresql
-#
+    fi
-ynh_add_sury() {
-
-    # Add an extra repository for those packages
-    ynh_install_extra_repo --repo="https://packages.sury.org/php/ $(ynh_get_debian_release) main" --key="https://packages.sury.org/php/apt.gpg" --name=extra_php_version --priority=600
 
 }
 
-
 # Add dependencies to install with ynh_install_app_dependencies
 #
 # usage: ynh_add_app_dependencies --package=phpversion [--replace]

data/helpers.d/postgresql

@@ -281,6 +281,8 @@ ynh_psql_remove_db() {
 
 # Create a master password and set up global settings
 #
+# [internal]
+#
 # usage: ynh_psql_test_if_first_run
 #
 # It also make sure that postgresql is installed and running
@@ -292,34 +294,5 @@ ynh_psql_test_if_first_run() {
     # Make sure postgresql is indeed installed
     dpkg --list | grep -q "ii  postgresql-$PSQL_VERSION" || ynh_die --message="postgresql-$PSQL_VERSION is not installed !?"
 
-    # Check for some weird issue where postgresql could be installed but etc folder would not exist ...
+    yunohost tools regen-conf postgresql
-    [ -e "/etc/postgresql/$PSQL_VERSION" ] || ynh_die --message="It looks like postgresql was not properly configured ? /etc/postgresql/$PSQL_VERSION is missing ... Could be due to a locale issue, c.f.https://serverfault.com/questions/426989/postgresql-etc-postgresql-doesnt-exist"
-
-    # Make sure postgresql is started and enabled
-    # (N.B. : to check the active state, we check the cluster state because
-    # postgresql could be flagged as active even though the cluster is in
-    # failed state because of how the service is configured..)
-    systemctl is-active postgresql@$PSQL_VERSION-main -q || ynh_systemd_action --service_name=postgresql --action=restart
-    systemctl is-enabled postgresql -q || systemctl enable postgresql --quiet
-
-    # If this is the very first time, we define the root password
-    # and configure a few things
-    if [ ! -f "$PSQL_ROOT_PWD_FILE" ]; then
-        local pg_hba=/etc/postgresql/$PSQL_VERSION/main/pg_hba.conf
-
-        local psql_root_password="$(ynh_string_random)"
-        echo "$psql_root_password" >$PSQL_ROOT_PWD_FILE
-        sudo --login --user=postgres psql -c"ALTER user postgres WITH PASSWORD '$psql_root_password'" postgres
-
-        # force all user to connect to local databases using hashed passwords
-        # https://www.postgresql.org/docs/current/static/auth-pg-hba-conf.html#EXAMPLE-PG-HBA.CONF
-        # Note: we can't use peer since YunoHost create users with nologin
-        #  See: https://github.com/YunoHost/yunohost/blob/unstable/data/helpers.d/user
-        ynh_replace_string --match_string="local\(\s*\)all\(\s*\)all\(\s*\)peer" --replace_string="local\1all\2all\3md5" --target_file="$pg_hba"
-
-        # Integrate postgresql service in yunohost
-        yunohost service add postgresql --log "/var/log/postgresql/"
-
-        ynh_systemd_action --service_name=postgresql --action=reload
-    fi
 }

data/hooks/conf_regen/10-apt

@@ -7,6 +7,11 @@ do_pre_regen() {
 
     mkdir --parents "${pending_dir}/etc/apt/preferences.d"
 
+    # Add sury
+    mkdir -p ${pending_dir}/etc/apt/sources.list.d/
+    echo "deb https://packages.sury.org/php/ $(lsb_release --codename --short) main" > "${pending_dir}/etc/apt/sources.list.d/extra_php_version.list"
+
+    # Ban some packages from sury
     packages_to_refuse_from_sury="php php-fpm php-mysql php-xml php-zip php-mbstring php-ldap php-gd php-curl php-bz2 php-json php-sqlite3 php-intl openssl libssl1.1 libssl-dev"
     for package in $packages_to_refuse_from_sury; do
         echo "
@@ -15,6 +20,7 @@ Pin: origin \"packages.sury.org\"
 Pin-Priority: -1" >>"${pending_dir}/etc/apt/preferences.d/extra_php_version"
     done
 
+    # Ban some packages that users may inadvertendly try to install such as apache2 ...
     echo "
 
 # PLEASE READ THIS WARNING AND DON'T EDIT THIS FILE
@@ -44,11 +50,19 @@ Pin: release *
 Pin-Priority: -1
 " >>"${pending_dir}/etc/apt/preferences.d/ban_packages"
 
+
 }
 
 do_post_regen() {
     regen_conf_files=$1
 
+    # Add sury key
+    # We do this only at the post regen and if the key doesn't already exists, because we don't want the regenconf to fuck everything up if the regenconf runs while the network is down
+    if [[ ! -s /etc/apt/trusted.gpg.d/extra_php_version.gpg ]]
+    then
+        wget --timeout 900 --quiet "https://packages.sury.org/php/apt.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/extra_php_version.gpg"
+    fi
+
     # Make sure php7.4 is the default version when using php in cli
     update-alternatives --set php /usr/bin/php7.4
 }

data/hooks/conf_regen/12-metronome

@@ -2,6 +2,12 @@
 
 set -e
 
+if ! dpkg --list | grep -q 'ii *metronome '
+then
+    echo 'metronome is not installed, skipping'
+    exit 0
+fi
+
 do_pre_regen() {
     pending_dir=$1
 

data/hooks/conf_regen/34-mysql

@@ -3,6 +3,12 @@
 set -e
 . /usr/share/yunohost/helpers
 
+if ! dpkg --list | grep -q 'ii *mariadb-server '
+then
+    echo 'mysql/mariadb is not installed, skipping'
+    exit 0
+fi
+
 do_pre_regen() {
     pending_dir=$1
 

data/hooks/conf_regen/35-postgresql

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+set -e
+. /usr/share/yunohost/helpers
+
+if ! dpkg --list | grep -q "ii *postgresql-$PSQL_VERSION "
+then
+    echo 'postgresql is not installed, skipping'
+    exit 0
+fi
+
+if [ ! -e "/etc/postgresql/$PSQL_VERSION" ]
+then
+    ynh_die --message="It looks like postgresql was not properly configured ? /etc/postgresql/$PSQL_VERSION is missing ... Could be due to a locale issue, c.f.https://serverfault.com/questions/426989/postgresql-etc-postgresql-doesnt-exist"
+fi
+
+
+do_pre_regen() {
+    return 0
+}
+
+do_post_regen() {
+    regen_conf_files=$1
+  
+    # Make sure postgresql is started and enabled
+    # (N.B. : to check the active state, we check the cluster state because
+    # postgresql could be flagged as active even though the cluster is in
+    # failed state because of how the service is configured..)
+    systemctl is-active postgresql@$PSQL_VERSION-main -q || ynh_systemd_action --service_name=postgresql --action=restart
+    systemctl is-enabled postgresql -q || systemctl enable postgresql --quiet
+  
+    # If this is the very first time, we define the root password
+    # and configure a few things
+    if [ ! -f "$PSQL_ROOT_PWD_FILE" ] || [ -z "$(cat $PSQL_ROOT_PWD_FILE)" ]; then
+        ynh_string_random >$PSQL_ROOT_PWD_FILE
+    fi
+    
+    sudo --login --user=postgres psql -c"ALTER user postgres WITH PASSWORD '$(cat $PSQL_ROOT_PWD_FILE)'" postgres
+  
+    # force all user to connect to local databases using hashed passwords
+    # https://www.postgresql.org/docs/current/static/auth-pg-hba-conf.html#EXAMPLE-PG-HBA.CONF
+    # Note: we can't use peer since YunoHost create users with nologin
+    #  See: https://github.com/YunoHost/yunohost/blob/unstable/data/helpers.d/user
+    local pg_hba=/etc/postgresql/$PSQL_VERSION/main/pg_hba.conf
+    ynh_replace_string --match_string="local\(\s*\)all\(\s*\)all\(\s*\)peer" --replace_string="local\1all\2all\3md5" --target_file="$pg_hba"
+
+    ynh_systemd_action --service_name=postgresql --action=reload
+}
+
+FORCE=${2:-0}
+DRY_RUN=${3:-0}
+
+case "$1" in
+  pre)
+    do_pre_regen $4
+    ;;
+  post)
+    do_post_regen $4
+    ;;
+  *)
+    echo "hook called with unknown argument \`$1'" >&2
+    exit 1
+    ;;
+esac
+
+exit 0

data/templates/yunohost/services.yml

@@ -12,24 +12,31 @@ metronome:
   log: [/var/log/metronome/metronome.log,/var/log/metronome/metronome.err]
   needs_exposed_ports: [5222, 5269]
   category: xmpp
+  ignore_if_package_is_not_installed: metronome
 mysql:
   log: [/var/log/mysql.log,/var/log/mysql.err,/var/log/mysql/error.log]
   actual_systemd_service: mariadb
   category: database
+  ignore_if_package_is_not_installed: mariadb-server
 nginx:
   log: /var/log/nginx
   test_conf: nginx -t
   needs_exposed_ports: [80, 443]
   category: web
-php7.4-fpm:
+# Yunohost will dynamically add installed php-fpm services (7.3, 7.4, 8.0, ...) in services.py
-  log: /var/log/php7.4-fpm.log
+#php7.4-fpm:
-  test_conf: php-fpm7.4 --test
+#  log: /var/log/php7.4-fpm.log
-  category: web
+#  test_conf: php-fpm7.4 --test
+#  category: web
 postfix:
   log: [/var/log/mail.log,/var/log/mail.err]
   actual_systemd_service: postfix@-
   needs_exposed_ports: [25, 587]
   category: email
+postgresql:
+  actual_systemd_service: 'postgresql@13-main'
+  category: database
+  ignore_if_package_is_not_installed: postgresql-13
 redis-server:
   log: /var/log/redis/redis-server.log
   category: database

debian/control

@@ -18,8 +18,6 @@ Depends: ${python3:Depends}, ${misc:Depends}
  , python-is-python3
  , nginx, nginx-extras (>=1.18)
  , apt, apt-transport-https, apt-utils, dirmngr
- , php7.4-common, php7.4-fpm, php7.4-ldap, php7.4-intl
- , mariadb-server, php7.4-mysql
  , openssh-server, iptables, fail2ban, dnsutils, bind9utils
  , openssl, ca-certificates, netcat-openbsd, iproute2
  , slapd, ldap-utils, sudo-ldap, libnss-ldapd, unscd, libpam-ldapd
@@ -34,6 +32,8 @@ Depends: ${python3:Depends}, ${misc:Depends}
 Recommends: yunohost-admin
  , ntp, inetutils-ping | iputils-ping
  , bash-completion, rsyslog
+ , php7.4-common, php7.4-fpm, php7.4-ldap, php7.4-intl
+ , mariadb-server, php7.4-mysql
  , php7.4-gd, php7.4-curl, php-php-gettext
  , python3-pip
  , unattended-upgrades

locales/en.json

@@ -614,8 +614,8 @@
     "service_description_fail2ban": "Protects against brute-force and other kinds of attacks from the Internet",
     "service_description_metronome": "Manage XMPP instant messaging accounts",
     "service_description_mysql": "Stores app data (SQL database)",
+    "service_description_postgresql": "Stores app data (SQL database)",
     "service_description_nginx": "Serves or provides access to all the websites hosted on your server",
-    "service_description_php7.4-fpm": "Runs apps written in PHP with NGINX",
     "service_description_postfix": "Used to send and receive e-mails",
     "service_description_redis-server": "A specialized database used for rapid data access, task queue, and communication between programs",
     "service_description_rspamd": "Filters spam, and other e-mail related features",

src/yunohost/service.py

@@ -110,7 +110,7 @@ def service_add(
         # Try to get the description from systemd service
         _, systemd_info = _get_service_information_from_systemd(name)
         type_ = systemd_info.get("Type") if systemd_info is not None else ""
-        if type_ == "oneshot" and name != "postgresql":
+        if type_ == "oneshot":
             logger.warning(
                 "/!\\ Packagers! Please provide a --test_status when adding oneshot-type services in Yunohost, such that it has a reliable way to check if the service is running or not."
             )
@@ -699,13 +699,20 @@ def _get_services():
         if "log" not in services["ynh-vpnclient"]:
             services["ynh-vpnclient"]["log"] = ["/var/log/ynh-vpnclient.log"]
 
-    # Stupid hack for postgresql which ain't an official service ... Can't
+    services_with_package_condition = [name for name, infos in services.items() if infos.get("ignore_if_package_is_not_installed")]
-    # really inject that info otherwise. Real service we want to check for
+    for name in services_with_package_condition:
-    # status and log is in fact postgresql@x.y-main (x.y being the version)
+        package = services[name]["ignore_if_package_is_not_installed"]
-    if "postgresql" in services:
+        if os.system(f"dpkg --list | grep -q 'ii *{package}'") != 0:
-        if "description" in services["postgresql"]:
+            del services[name]
-            del services["postgresql"]["description"]
+
-        services["postgresql"]["actual_systemd_service"] = "postgresql@13-main"
+    php_fpm_versions = check_output(r"dpkg --list | grep -P 'ii  php\d.\d-fpm' | awk '{print $2}' | grep -o -P '\d.\d'")
+    php_fpm_versions = [v for v in php_fpm_versions.split('\n') if v.strip()]
+    for version in php_fpm_versions:
+        services[f"php{version}-fpm"] = {
+            "log": f"/var/log/php{version}-fpm.log",
+            "test_conf": f"php-fpm{version} --test",  # ofc the service is phpx.y-fpm but the program is php-fpmx.y because why not ...
+            "category": "web"
+        }
 
     # Remove legacy /var/log/daemon.log and /var/log/syslog from log entries
     # because they are too general. Instead, now the journalctl log is
@@ -825,11 +832,7 @@ def _get_journalctl_logs(service, number="all"):
     services = _get_services()
     systemd_service = services.get(service, {}).get("actual_systemd_service", service)
     try:
-        return check_output(
+        return check_output(f"journalctl --no-hostname --no-pager -u {systemd_service} -n{number}")
-            "journalctl --no-hostname --no-pager -u {0} -n{1}".format(
-                systemd_service, number
-            )
-        )
     except Exception:
         import traceback