Fix version number >.>

We need this because the regen-conf need to get the state of the slapd config database if it is customized or not and if some update are need.

data/helpers.d/getopts

@@ -80,10 +80,10 @@ ynh_handle_getopts_args () {
                 arguments[arg]="${arguments[arg]//--${args_array[$option_flag]}-/--${args_array[$option_flag]}\\TOBEREMOVED\\-}"
                 # And replace long option (value of the option_flag) by the short option, the option_flag itself
                 # (e.g. for [u]=user, --user will be -u)
-                # Replace long option with =
-                arguments[arg]="${arguments[arg]//--${args_array[$option_flag]}/-${option_flag} }"
-                # And long option without =
-                arguments[arg]="${arguments[arg]//--${args_array[$option_flag]%=}/-${option_flag}}"
+                # Replace long option with = (match the beginning of the argument)
+                arguments[arg]="$(echo "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]}/-${option_flag} /")"
+                # And long option without = (match the whole line)
+                arguments[arg]="$(echo "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]%=}$/-${option_flag} /")"
             done
         done
 

data/helpers.d/mysql

@@ -44,8 +44,13 @@ ynh_mysql_execute_as_root() {
     ynh_handle_getopts_args "$@"
     database="${database:-}"
 
+    if [ -n "$database" ]
+    then
+        database="--database=$database"
+    fi
+
     ynh_mysql_connect_as --user="root" --password="$(cat $MYSQL_ROOT_PWD_FILE)" \
-        --database="$database" <<< "$sql"
+        $database <<< "$sql"
 }
 
 # Execute a command from a file as root user
@@ -65,8 +70,14 @@ ynh_mysql_execute_file_as_root() {
     ynh_handle_getopts_args "$@"
     database="${database:-}"
 
+    if [ -n "$database" ]
+    then
+        database="--database=$database"
+    fi
+
+
     ynh_mysql_connect_as --user="root" --password="$(cat $MYSQL_ROOT_PWD_FILE)" \
-        --database="$database" < "$file"
+        $database < "$file"
 }
 
 # Create a database and grant optionnaly privilegies to a user

data/helpers.d/php

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-readonly YNH_DEFAULT_PHP_VERSION=7.0
+readonly YNH_DEFAULT_PHP_VERSION=7.3
 # Declare the actual php version to use.
 # A packager willing to use another version of php can override the variable into its _common.sh.
 YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION}

data/helpers.d/postgresql

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 PSQL_ROOT_PWD_FILE=/etc/yunohost/psql
-PSQL_VERSION=9.6
+PSQL_VERSION=11
 
 # Open a connection as a user
 #
@@ -46,8 +46,13 @@ ynh_psql_execute_as_root() {
     ynh_handle_getopts_args "$@"
     database="${database:-}"
 
+    if [ -n "$database" ]
+    then
+        database="--database=$database"
+    fi
+
     ynh_psql_connect_as --user="postgres" --password="$(cat $PSQL_ROOT_PWD_FILE)" \
-        --database="$database" <<<"$sql"
+        $database <<<"$sql"
 }
 
 # Execute a command from a file as root user
@@ -67,8 +72,13 @@ ynh_psql_execute_file_as_root() {
     ynh_handle_getopts_args "$@"
     database="${database:-}"
 
+    if [ -n "$database" ]
+    then
+        database="--database=$database"
+    fi
+
     ynh_psql_connect_as --user="postgres" --password="$(cat $PSQL_ROOT_PWD_FILE)" \
-        --database="$database" <"$file"
+        $database <"$file"
 }
 
 # Create a database and grant optionnaly privilegies to a user

data/hooks/backup/05-conf_ldap

@@ -10,7 +10,8 @@ source /usr/share/yunohost/helpers
 backup_dir="${1}/conf/ldap"
 
 # Backup the configuration
-ynh_backup "/etc/ldap/slapd.conf" "${backup_dir}/slapd.conf"
+ynh_backup "/etc/ldap/ldap.conf" "${backup_dir}/ldap.conf"
+ynh_backup "/etc/ldap/slapd.ldif" "${backup_dir}/slapd.ldif"
 slapcat -b cn=config -l "${backup_dir}/cn=config.master.ldif"
 
 # Backup the database

data/hooks/backup/11-conf_ynh_mysql

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-# Exit hook on subcommand error or unset variable
-set -eu
-
-# Source YNH helpers
-source /usr/share/yunohost/helpers
-
-# Backup destination
-backup_dir="${1}/conf/ynh/mysql"
-
-# Save MySQL root password
-ynh_backup "/etc/yunohost/mysql" "${backup_dir}/root_pwd"

data/hooks/conf_regen/01-yunohost

@@ -75,7 +75,16 @@ EOF
 ConditionCapability=CAP_SYS_TIME
 ConditionVirtualization=!container
 " > ${pending_dir}/etc/systemd/system/ntp.service.d/ynh-override.conf
-
+  
+  # Make nftable conflict with yunohost-firewall
+  mkdir -p ${pending_dir}/etc/systemd/system/nftables.service.d/
+  cat > ${pending_dir}/etc/systemd/system/nftables.service.d/ynh-override.conf << EOF
+[Unit]
+# yunohost-firewall and nftables conflict with each other
+Conflicts=yunohost-firewall.service
+ConditionFileIsExecutable=!/etc/init.d/yunohost-firewall
+ConditionPathExists=!/etc/systemd/system/multi-user.target.wants/yunohost-firewall.service
+EOF
 }
 
 do_post_regen() {
@@ -100,7 +109,9 @@ do_post_regen() {
   [[ ! -e /etc/yunohost/hooks.d ]] || (chown root /etc/yunohost/hooks.d && chmod 700 /etc/yunohost/hooks.d)
   [[ ! -e /etc/yunohost/apps ]] || (chown root /etc/yunohost/apps && chmod 700 /etc/yunohost/apps)
 
+  # Propagates changes in systemd service config overrides
   [[ ! "$regen_conf_files" =~ "ntp.service.d/ynh-override.conf" ]] || { systemctl daemon-reload; systemctl restart ntp; }
+  [[ ! "$regen_conf_files" =~ "nftables.service.d/ynh-override.conf" ]] || systemctl daemon-reload
 }
 
 _update_services() {

data/hooks/conf_regen/06-slapd

@@ -13,28 +13,51 @@ do_init_regen() {
   do_pre_regen ""
 
   systemctl daemon-reload
+  
+  _regenerate_slapd_conf
 
-  # fix some permissions
-  chown root:openldap /etc/ldap/slapd.conf
+   # Enforce permissions
+  chown root:openldap /etc/ldap/slapd.ldif
   chown -R openldap:openldap /etc/ldap/schema/
   usermod -aG ssl-cert openldap
 
-  # check the slapd config file at first
-  slaptest -Q -u -f /etc/ldap/slapd.conf
-
-  # regenerate LDAP config directory from slapd.conf
-  rm -Rf /etc/ldap/slapd.d
-  mkdir /etc/ldap/slapd.d
-  slaptest -f /etc/ldap/slapd.conf -F /etc/ldap/slapd.d/ 2>&1
-  chown -R openldap:openldap /etc/ldap/slapd.d/
-
   service slapd restart
 }
 
+_regenerate_slapd_conf() {
+
+  # Validate the new slapd config
+  # To do so, we have to use the .ldif to generate the config directory
+  # so we use a temporary directory slapd_new.d
+  rm -Rf /etc/ldap/slapd_new.d
+  mkdir /etc/ldap/slapd_new.d
+  slapadd -n0 -l /etc/ldap/slapd.ldif -F /etc/ldap/slapd_new.d/ 2>&1
+  # Actual validation (-Q is for quiet, -u is for dry-run)
+  slaptest -Q -u -F /etc/ldap/slapd_new.d
+
+  # "Commit" / apply the new config (meaning we delete the old one and replace
+  # it with the new one)
+  rm -Rf /etc/ldap/slapd.d
+  mv /etc/ldap/slapd_new.d /etc/ldap/slapd.d
+
+  chown -R openldap:openldap /etc/ldap/slapd.d/
+}
+
 do_pre_regen() {
   pending_dir=$1
 
-  cd /usr/share/yunohost/templates/slapd
+  # remove temporary backup file
+  rm -f "$tmp_backup_dir_file"
+
+  # Define if we need to migrate from hdb to mdb
+  curr_backend=$(grep '^database' /etc/ldap/slapd.conf 2>/dev/null | awk '{print $2}')
+  if [ -e /etc/ldap/slapd.conf ] && [ -n "$curr_backend" ] && \
+     [ $curr_backend != 'mdb' ]; then
+    backup_dir="/var/backups/dc=yunohost,dc=org-${curr_backend}-$(date +%s)"
+    mkdir -p "$backup_dir"
+    slapcat -b dc=yunohost,dc=org -l "${backup_dir}/dc=yunohost-dc=org.ldif"
+    echo "$backup_dir" > "$tmp_backup_dir_file"
+  fi
 
   # create needed directories
   ldap_dir="${pending_dir}/etc/ldap"
@@ -42,28 +65,15 @@ do_pre_regen() {
   mkdir -p "$ldap_dir" "$schema_dir"
 
   # remove legacy configuration file
-  [ ! -f /etc/ldap/slapd-yuno.conf ] \
-    || touch "${pending_dir}/etc/ldap/slapd-yuno.conf"
+  [ ! -f /etc/ldap/slapd-yuno.conf ] || touch "${ldap_dir}/slapd-yuno.conf"
+  [ ! -f /etc/ldap/slapd.conf ] || touch "${ldap_dir}/slapd.conf"
+  [ ! -f /etc/ldap/schema/yunohost.schema ] || touch "${schema_dir}/yunohost.schema"
 
-  # remove temporary backup file
-  rm -f "$tmp_backup_dir_file"
-
-  # retrieve current and new backends
-  curr_backend=$(grep '^database' /etc/ldap/slapd.conf 2>/dev/null | awk '{print $2}')
-  new_backend=$(grep '^database' slapd.conf | awk '{print $2}')
-
-  # save current database before any conf changes
-  if [[ -n "$curr_backend" && "$curr_backend" != "$new_backend" ]]; then
-    backup_dir="/var/backups/dc=yunohost,dc=org-${curr_backend}-$(date +%s)"
-    mkdir -p "$backup_dir"
-    slapcat -b dc=yunohost,dc=org \
-      -l "${backup_dir}/dc=yunohost-dc=org.ldif"
-    echo "$backup_dir" > "$tmp_backup_dir_file"
-  fi
+  cd /usr/share/yunohost/templates/slapd
 
   # copy configuration files
-  cp -a ldap.conf slapd.conf "$ldap_dir"
-  cp -a sudo.schema mailserver.schema yunohost.schema "$schema_dir"
+  cp -a ldap.conf slapd.ldif "$ldap_dir"
+  cp -a sudo.ldif mailserver.ldif permission.ldif "$schema_dir"
 
   mkdir -p ${pending_dir}/etc/systemd/system/slapd.service.d/
   cp systemd-override.conf ${pending_dir}/etc/systemd/system/slapd.service.d/ynh-override.conf
@@ -74,14 +84,11 @@ do_pre_regen() {
 do_post_regen() {
   regen_conf_files=$1
 
-  # ensure that slapd.d exists
-  mkdir -p /etc/ldap/slapd.d
-
   # fix some permissions
-  echo "Making sure we have the right permissions needed ..."
+  echo "Enforce permissions on ldap/slapd directories and certs ..."
   # penldap user should be in the ssl-cert group to let it access the certificate for TLS
   usermod -aG ssl-cert openldap
-  chown root:openldap /etc/ldap/slapd.conf
+  chown root:openldap /etc/ldap/slapd.ldif
   chown -R openldap:openldap /etc/ldap/schema/
   chown -R openldap:openldap /etc/ldap/slapd.d/
 
@@ -94,29 +101,17 @@ do_post_regen() {
 
   [ -z "$regen_conf_files" ] && exit 0
 
-  # check the slapd config file at first
-  slaptest -Q -u -f /etc/ldap/slapd.conf
+  # regenerate LDAP config directory from slapd.conf
+  echo "Regenerate LDAP config directory from slapd.ldif"
+  _regenerate_slapd_conf
 
-  # check if a backup should be restored
+  # If there's a backup, re-import its data
   backup_dir=$(cat "$tmp_backup_dir_file" 2>/dev/null || true)
   if [[ -n "$backup_dir" && -f "${backup_dir}/dc=yunohost-dc=org.ldif" ]]; then
       # regenerate LDAP config directory and import database as root
-      # since the admin user may be unavailable
-      echo "Regenerate LDAP config directory and import the database using slapadd"
-      sh -c "rm -Rf /etc/ldap/slapd.d;
-  mkdir /etc/ldap/slapd.d;
-  slaptest -f /etc/ldap/slapd.conf -F /etc/ldap/slapd.d;
-  chown -R openldap:openldap /etc/ldap/slapd.d;
-  slapadd -F /etc/ldap/slapd.d -b dc=yunohost,dc=org \
-    -l '${backup_dir}/dc=yunohost-dc=org.ldif';
-  chown -R openldap:openldap /var/lib/ldap" 2>&1
-  else
-      # regenerate LDAP config directory from slapd.conf
-      echo "Regenerate LDAP config directory from slapd.conf"
-      rm -Rf /etc/ldap/slapd.d
-      mkdir /etc/ldap/slapd.d
-      slaptest -f /etc/ldap/slapd.conf -F /etc/ldap/slapd.d/ 2>&1
-      chown -R openldap:openldap /etc/ldap/slapd.d/
+    echo "Import the database using slapadd"
+    slapadd -F /etc/ldap/slapd.d -b dc=yunohost,dc=org -l "${backup_dir}/dc=yunohost-dc=org.ldif"
+    chown -R openldap:openldap /var/lib/ldap 2>&1
   fi
 
   echo "Running slapdindex"
@@ -156,6 +151,9 @@ case "$1" in
   init)
     do_init_regen
     ;;
+  apply_config)
+    do_post_regen /etc/ldap/slapd.ldif
+    ;;
   *)
     echo "hook called with unknown argument \`$1'" >&2
     exit 1

data/hooks/conf_regen/43-dnsmasq

@@ -69,8 +69,16 @@ do_post_regen() {
   short_hostname=$(hostname -s)
   grep -q "127.0.0.1.*$short_hostname" /etc/hosts || echo -e "\n127.0.0.1\t$short_hostname" >>/etc/hosts
 
-  [[ -z "$regen_conf_files" ]] \
-    || service dnsmasq restart
+  [[ -n "$regen_conf_files" ]] || return
+
+  # Remove / disable services likely to conflict with dnsmasq
+  for SERVICE in systemd-resolved bind9
+  do
+      systemctl is-enabled $SERVICE &>/dev/null && systemctl disable $SERVICE 2>/dev/null
+      systemctl is-active $SERVICE &>/dev/null && systemctl stop $SERVICE
+  done
+
+  systemctl restart dnsmasq
 }
 
 FORCE=${2:-0}

data/hooks/restore/05-conf_ldap

@@ -39,6 +39,9 @@ else
     # Restore the configuration
     mv /etc/ldap/slapd.d "$TMPDIR"
     mkdir -p /etc/ldap/slapd.d
+    cp -a "${backup_dir}/ldap.conf" /etc/ldap/ldap.conf
+    cp -a "${backup_dir}/slapd.ldif" /etc/ldap/slapd.ldif
+    # Legacy thing but we need it to force the regen-conf in case of it exist
     cp -a "${backup_dir}/slapd.conf" /etc/ldap/slapd.conf
     slapadd -F /etc/ldap/slapd.d -b cn=config \
         -l "${backup_dir}/cn=config.master.ldif" \

data/hooks/restore/11-conf_ynh_mysql

@@ -1,42 +1,5 @@
-backup_dir="$1/conf/ynh/mysql"
-MYSQL_PKG="$(dpkg --list | sed -ne 's/^ii  \(mariadb-server-[[:digit:].]\+\) .*$/\1/p')"
+# We don't backup/restore mysql password anymore
+# c.f. https://github.com/YunoHost/yunohost/pull/912
 
-. /usr/share/yunohost/helpers
-
-# ensure that mysql is running
-service mysql status >/dev/null 2>&1 \
-  || service mysql start
-
-# retrieve current and new password
-[ -f /etc/yunohost/mysql ] \
-  && curr_pwd=$(cat /etc/yunohost/mysql)
-new_pwd=$(cat "${backup_dir}/root_pwd" || cat "${backup_dir}/mysql")
-[ -z "$curr_pwd" ] && curr_pwd="yunohost"
-[ -z "$new_pwd" ] && {
-    new_pwd=$(ynh_string_random 10)
-}
-
-# attempt to change it
-mysqladmin -s -u root -p"$curr_pwd" password "$new_pwd" || {
-
-  echo "It seems that you have already configured MySQL." \
-    "YunoHost needs to have a root access to MySQL to runs its" \
-    "applications, and is going to reset the MySQL root password." \
-    "You can find this new password in /etc/yunohost/mysql." >&2
-
-  # set new password with debconf
-  debconf-set-selections << EOF
-$MYSQL_PKG mysql-server/root_password password $new_pwd
-$MYSQL_PKG mysql-server/root_password_again password $new_pwd
-EOF
-
-  # reconfigure Debian package
-  dpkg-reconfigure -freadline -u "$MYSQL_PKG" 2>&1
-}
-
-# store new root password
-echo "$new_pwd" | tee /etc/yunohost/mysql
-chmod 400 /etc/yunohost/mysql
-
-# reload the grant tables
-mysqladmin -s -u root -p"$new_pwd" reload
+# This is a dummy empty file as a workaround for
+# https://github.com/YunoHost/issues/issues/1553 until it is fixed

data/templates/dovecot/dovecot.conf

@@ -14,18 +14,16 @@ mail_plugins = $mail_plugins quota
 
 ###############################################################################
 
-# generated 2020-04-03, Mozilla Guideline v5.4, Dovecot 2.2.27, OpenSSL 1.1.0l, intermediate configuration
-# https://ssl-config.mozilla.org/#server=dovecot&version=2.2.27&config=intermediate&openssl=1.1.0l&guideline=5.4
+# generated 2020-05-02, Mozilla Guideline v5.4, Dovecot 2.3.4.1, OpenSSL 1.1.1d, intermediate configuration
+# https://ssl-config.mozilla.org/#server=dovecot&version=2.3.4.1&config=intermediate&openssl=1.1.1d&guideline=5.4
 
 ssl = required
 
 ssl_cert = </etc/yunohost/certs/{{ main_domain }}/crt.pem
 ssl_key = </etc/yunohost/certs/{{ main_domain }}/key.pem
 
-ssl_dh_parameters_length = 2048
-
 # intermediate configuration
-ssl_protocols = TLSv1.2
+ssl_min_protocol = TLSv1.2
 ssl_cipher_list = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
 ssl_prefer_server_ciphers = no
 

data/templates/nslcd/nslcd.conf

@@ -15,6 +15,18 @@ base dc=yunohost,dc=org
 # The LDAP protocol version to use.
 #ldap_version 3
 
+# The DN to bind with for normal lookups.
+#binddn cn=annonymous,dc=example,dc=net
+#bindpw secret
+
+# The DN used for password modifications by root.
+#rootpwmoddn cn=admin,dc=example,dc=com
+
+# SSL options
+#ssl off
+#tls_reqcert never
+tls_cacertfile /etc/ssl/certs/ca-certificates.crt
+
 # The search scope.
 #scope sub
 

data/templates/nsswitch/nsswitch.conf

@@ -1,12 +1,8 @@
 # /etc/nsswitch.conf
-#
-# Example configuration of GNU Name Service Switch functionality.
-# If you have the `glibc-doc-reference' and `info' packages installed, try:
-# `info libc "Name Service Switch"' for information about this file.
 
-passwd:         compat ldap
-group:          compat ldap
-shadow:         compat ldap
+passwd:         files systemd ldap
+group:          files systemd ldap
+shadow:         files ldap
 gshadow:        files
 
 hosts:          files myhostname mdns4_minimal [NOTFOUND=return] dns

data/templates/slapd/mailserver.ldif

@@ -2,58 +2,62 @@
 ## Version 0.1
 ## Adrien Beudin
 
+dn: cn=mailserver,cn=schema,cn=config
+objectClass: olcSchemaConfig
+cn: mailserver
+#
 # Attributes
-attributetype ( 1.3.6.1.4.1.40328.1.20.2.1
+olcAttributeTypes: ( 1.3.6.1.4.1.40328.1.20.2.1
         NAME 'maildrop'
         DESC 'Mail addresses where mails are forwarded -- ie forwards'
         EQUALITY caseIgnoreMatch
         SUBSTR caseIgnoreSubstringsMatch
         SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{512})
-
-attributetype ( 1.3.6.1.4.1.40328.1.20.2.2
+#
+olcAttributeTypes: ( 1.3.6.1.4.1.40328.1.20.2.2
         NAME 'mailalias'
         DESC 'Mail addresses accepted by this account -- ie aliases'
         EQUALITY caseIgnoreMatch
         SUBSTR caseIgnoreSubstringsMatch
         SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{512})
-
-attributetype ( 1.3.6.1.4.1.40328.1.20.2.3
+#
+olcAttributeTypes: ( 1.3.6.1.4.1.40328.1.20.2.3
         NAME 'mailenable'
         DESC 'Mail Account validity'
         EQUALITY caseIgnoreMatch
         SUBSTR caseIgnoreSubstringsMatch
         SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{8})
-
-attributetype ( 1.3.6.1.4.1.40328.1.20.2.4
+#
+olcAttributeTypes: ( 1.3.6.1.4.1.40328.1.20.2.4
         NAME 'mailbox'
         DESC 'Mailbox path where mails are delivered'
         EQUALITY caseIgnoreMatch
         SUBSTR caseIgnoreSubstringsMatch
         SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{512})
-
-attributetype ( 1.3.6.1.4.1.40328.1.20.2.5
+#
+olcAttributeTypes: ( 1.3.6.1.4.1.40328.1.20.2.5
         NAME 'virtualdomain'
         DESC 'A mail domain name'
         EQUALITY caseIgnoreMatch
         SUBSTR caseIgnoreSubstringsMatch
         SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{512})
-
-attributetype ( 1.3.6.1.4.1.40328.1.20.2.6
+#
+olcAttributeTypes: ( 1.3.6.1.4.1.40328.1.20.2.6
         NAME 'virtualdomaindescription'
         DESC 'Virtual domain description'
         EQUALITY caseIgnoreMatch
         SUBSTR caseIgnoreSubstringsMatch
         SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{512})
-
-attributetype ( 1.3.6.1.4.1.40328.1.20.2.7
+#
+olcAttributeTypes: ( 1.3.6.1.4.1.40328.1.20.2.7
         NAME 'mailuserquota'
         DESC 'Mailbox quota for a user'
         EQUALITY caseIgnoreMatch
         SUBSTR caseIgnoreSubstringsMatch
         SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{16} SINGLE-VALUE )
-
+#
 # Mail Account Objectclass
-objectclass ( 1.3.6.1.4.1.40328.1.1.2.1
+olcObjectClasses: ( 1.3.6.1.4.1.40328.1.1.2.1
         NAME 'mailAccount'
         DESC 'Mail Account'
         SUP top
@@ -65,9 +69,9 @@ objectclass ( 1.3.6.1.4.1.40328.1.1.2.1
             mailalias $ maildrop $ mailenable $ mailbox $ mailuserquota 
             )
         )
-
+#
 # Mail Domain Objectclass
-objectclass ( 1.3.6.1.4.1.40328.1.1.2.2
+olcObjectClasses: ( 1.3.6.1.4.1.40328.1.1.2.2
         NAME 'mailDomain'
         DESC 'Domain mail entry'
         SUP top
@@ -79,9 +83,9 @@ objectclass ( 1.3.6.1.4.1.40328.1.1.2.2
             virtualdomaindescription $ mailuserquota
             )
         )
-
+#
 # Mail Group Objectclass
-objectclass ( 1.3.6.1.4.1.40328.1.1.2.3
+olcObjectClasses: ( 1.3.6.1.4.1.40328.1.1.2.3
         NAME 'mailGroup' SUP top AUXILIARY
         DESC 'Mail Group'
         MUST ( mail )

data/templates/slapd/permission.ldif

@@ -1,33 +1,35 @@
-#dn: cn=yunohost,cn=schema,cn=config
-#objectClass: olcSchemaConfig
-#cn: yunohost
+# Yunohost schema for group and permission support
+
+dn: cn=yunohost,cn=schema,cn=config
+objectClass: olcSchemaConfig
+cn: yunohost
 # ATTRIBUTES
 # For Permission
-attributetype ( 1.3.6.1.4.1.17953.9.1.1 NAME 'permission'
+olcAttributeTypes: ( 1.3.6.1.4.1.17953.9.1.1 NAME 'permission'
   DESC 'Yunohost permission on user and group side'
   SUP distinguishedName )
-attributetype ( 1.3.6.1.4.1.17953.9.1.2 NAME 'groupPermission'
+olcAttributeTypes: ( 1.3.6.1.4.1.17953.9.1.2 NAME 'groupPermission'
   DESC 'Yunohost permission for a group on permission side'
   SUP distinguishedName )
-attributetype ( 1.3.6.1.4.1.17953.9.1.3 NAME 'inheritPermission'
+olcAttributeTypes: ( 1.3.6.1.4.1.17953.9.1.3 NAME 'inheritPermission'
   DESC 'Yunohost permission for user on permission side'
   SUP distinguishedName )
-attributetype ( 1.3.6.1.4.1.17953.9.1.4 NAME 'URL'
+olcAttributeTypes: ( 1.3.6.1.4.1.17953.9.1.4 NAME 'URL'
   DESC 'Yunohost application URL'
   SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{128} )
 # OBJECTCLASS
 # For Applications
-objectclass ( 1.3.6.1.4.1.17953.9.2.1 NAME 'groupOfNamesYnh'
+olcObjectClasses: ( 1.3.6.1.4.1.17953.9.2.1 NAME 'groupOfNamesYnh'
   DESC 'Yunohost user group'
   SUP top AUXILIARY
   MAY ( member $ businessCategory $ seeAlso $ owner $ ou $ o $ permission ) )
-objectclass ( 1.3.6.1.4.1.17953.9.2.2 NAME 'permissionYnh'
+olcObjectClasses: ( 1.3.6.1.4.1.17953.9.2.2 NAME 'permissionYnh'
   DESC 'a Yunohost application'
   SUP top AUXILIARY
   MUST cn
   MAY ( groupPermission $ inheritPermission $ URL ) )
 # For User
-objectclass ( 1.3.6.1.4.1.17953.9.2.3 NAME 'userPermissionYnh'
+olcObjectClasses: ( 1.3.6.1.4.1.17953.9.2.3 NAME 'userPermissionYnh'
   DESC 'a Yunohost application'
   SUP top AUXILIARY
   MAY ( permission ) )

data/templates/slapd/slapd.conf

@@ -1,154 +0,0 @@
-# This is the main slapd configuration file. See slapd.conf(5) for more
-# info on the configuration options.
-
-#######################################################################
-# Global Directives:
-
-# Features to permit
-#allow bind_v2
-
-# Schema and objectClass definitions
-include         /etc/ldap/schema/core.schema
-include         /etc/ldap/schema/cosine.schema
-include         /etc/ldap/schema/nis.schema
-include         /etc/ldap/schema/inetorgperson.schema
-include         /etc/ldap/schema/mailserver.schema
-include         /etc/ldap/schema/sudo.schema
-include         /etc/ldap/schema/yunohost.schema
-
-# Where the pid file is put. The init.d script
-# will not stop the server if you change this.
-pidfile         /var/run/slapd/slapd.pid
-
-# List of arguments that were passed to the server
-argsfile        /var/run/slapd/slapd.args
-
-# Read slapd.conf(5) for possible values
-loglevel        none
-
-# Hashes to be used in generation of user passwords
-password-hash   {SSHA}
-
-# Where the dynamically loaded modules are stored
-modulepath	/usr/lib/ldap
-moduleload	back_mdb
-moduleload	memberof
-
-# The maximum number of entries that is returned for a search operation
-sizelimit 500
-
-# The tool-threads parameter sets the actual amount of cpu's that is used
-# for indexing.
-tool-threads 1
-
-# TLS Support
-TLSCertificateFile /etc/yunohost/certs/yunohost.org/crt.pem
-TLSCertificateKeyFile /etc/yunohost/certs/yunohost.org/key.pem
-
-#######################################################################
-# Specific Backend Directives for mdb:
-# Backend specific directives apply to this backend until another
-# 'backend' directive occurs
-backend		mdb
-
-#######################################################################
-# Specific Directives for database #1, of type mdb:
-# Database specific directives apply to this databasse until another
-# 'database' directive occurs
-database        mdb
-
-# The base of your directory in database #1
-suffix          "dc=yunohost,dc=org"
-
-# rootdn directive for specifying a superuser on the database. This is needed
-# for syncrepl.
-# rootdn          "cn=admin,dc=yunohost,dc=org"
-
-# Where the database file are physically stored for database #1
-directory       "/var/lib/ldap"
-
-# Indexing options for database #1
-index  objectClass                   eq
-index  uid,sudoUser                  eq,sub
-index  entryCSN,entryUUID            eq
-index  cn,mail                       eq
-index  gidNumber,uidNumber           eq
-index  member,memberUid,uniqueMember eq
-index  virtualdomain                 eq
-index  permission                    eq
-
-# Save the time that the entry gets modified, for database #1
-lastmod         on
-
-# Checkpoint the BerkeleyDB database periodically in case of system
-# failure and to speed slapd shutdown.
-checkpoint      512 30
-
-# The userPassword by default can be changed
-# by the entry owning it if they are authenticated.
-# Others should not be able to see it, except the
-# admin entry below
-# These access lines apply to database #1 only
-access to attrs=userPassword,shadowLastChange
-        by dn="cn=admin,dc=yunohost,dc=org" write
-        by dn.exact="gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth" write
-        by anonymous auth
-        by self write
-        by * none
-
-# Personnal information can be changed by the entry
-# owning it if they are authenticated.
-# Others should be able to see it.
-access to attrs=cn,gecos,givenName,mail,maildrop,displayName,sn
-        by dn="cn=admin,dc=yunohost,dc=org" write
-        by dn.exact="gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth" write
-        by self write
-        by * read
-
-# Ensure read access to the base for things like
-# supportedSASLMechanisms.  Without this you may
-# have problems with SASL not knowing what
-# mechanisms are available and the like.
-# Note that this is covered by the 'access to *'
-# ACL below too but if you change that as people
-# are wont to do you'll still need this if you
-# want SASL (and possible other things) to work 
-# happily.
-access to dn.base="" by * read
-
-# The admin dn has full write access, everyone else
-# can read everything.
-access to *
-        by dn="cn=admin,dc=yunohost,dc=org" write
-        by dn.exact="gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth" write
-        by group/groupOfNames/Member="cn=admin,ou=groups,dc=yunohost,dc=org" write
-        by * read
-
-# Configure Memberof Overlay (used for Yunohost permission)
-
-# Link user <-> group
-#dn: olcOverlay={0}memberof,olcDatabase={1}mdb,cn=config
-overlay                 memberof
-memberof-group-oc       groupOfNamesYnh
-memberof-member-ad      member
-memberof-memberof-ad    memberOf
-memberof-dangling       error
-memberof-refint         TRUE
-
-# Link permission <-> groupes
-#dn: olcOverlay={1}memberof,olcDatabase={1}mdb,cn=config
-overlay                 memberof
-memberof-group-oc       permissionYnh
-memberof-member-ad      groupPermission
-memberof-memberof-ad    permission
-memberof-dangling       error
-memberof-refint         TRUE
-
-# Link permission <-> user
-#dn: olcOverlay={2}memberof,olcDatabase={1}mdb,cn=config
-overlay                 memberof
-memberof-group-oc       permissionYnh
-memberof-member-ad      inheritPermission
-memberof-memberof-ad    permission
-memberof-dangling       error
-memberof-refint         TRUE

data/templates/slapd/slapd.ldif

@@ -0,0 +1,235 @@
+# OpenLDAP server configuration for Yunohost
+# ------------------------------------------
+#
+# Because of the Yunohost's regen-conf mechanism, it is NOT POSSIBLE to
+# edit the config database using an LDAP request.
+#
+# If you wish to edit the config database, you should edit THIS file
+# and update the config database based on this file.
+#
+# Config database customization:
+# 1. Edit this file as you want.
+# 2. Apply your modifications. For this just run this following command in a shell:
+#    $ /usr/share/yunohost/hooks/conf_regen/06-slapd apply_config
+#
+# Note that if you customize this file, YunoHost's regen-conf will NOT
+# overwrite this file. But that also means that you should be careful about
+# upgrades, because they may ship important/necessary changes to this
+# configuration that you will have to propagate yourself.
+
+#
+# Main configuration
+#
+dn: cn=config
+objectClass: olcGlobal
+cn: config
+olcConfigFile: /etc/ldap/slapd.conf
+olcConfigDir: /etc/ldap/slapd.d/
+# List of arguments that were passed to the server
+olcArgsFile: /var/run/slapd/slapd.args
+#
+olcAttributeOptions: lang-
+olcAuthzPolicy: none
+olcConcurrency: 0
+olcConnMaxPending: 100
+olcConnMaxPendingAuth: 1000
+olcIdleTimeout: 0
+olcIndexSubstrIfMaxLen: 4
+olcIndexSubstrIfMinLen: 2
+olcIndexSubstrAnyLen: 4
+olcIndexSubstrAnyStep: 2
+olcIndexIntLen: 4
+olcListenerThreads: 1
+olcLocalSSF: 71
+# Read slapd.conf(5) for possible values
+olcLogLevel: None
+# Where the pid file is put. The init.d script
+# will not stop the server if you change this.
+olcPidFile: /var/run/slapd/slapd.pid
+olcReverseLookup: FALSE
+olcThreads: 16
+# TLS Support
+olcTLSCertificateFile: /etc/yunohost/certs/yunohost.org/crt.pem
+olcTLSCertificateKeyFile: /etc/yunohost/certs/yunohost.org/key.pem
+olcTLSVerifyClient: never
+olcTLSProtocolMin: 0.0
+# The tool-threads parameter sets the actual amount of cpu's that is used
+# for indexing.
+olcToolThreads: 1
+structuralObjectClass: olcGlobal
+
+#
+# Schema and objectClass definitions
+#
+dn: cn=schema,cn=config
+objectClass: olcSchemaConfig
+cn: schema
+
+include: file:///etc/ldap/schema/core.ldif
+include: file:///etc/ldap/schema/cosine.ldif
+include: file:///etc/ldap/schema/nis.ldif
+include: file:///etc/ldap/schema/inetorgperson.ldif
+include: file:///etc/ldap/schema/mailserver.ldif
+include: file:///etc/ldap/schema/sudo.ldif
+include: file:///etc/ldap/schema/permission.ldif
+
+#
+# Module management
+#
+dn: cn=module{0},cn=config
+objectClass: olcModuleList
+cn: module{0}
+# Where the dynamically loaded modules are stored
+olcModulePath: /usr/lib/ldap
+olcModuleLoad: {0}back_mdb
+olcModuleLoad: {1}memberof
+structuralObjectClass: olcModuleList
+
+#
+# Frontend database
+#
+dn: olcDatabase={-1}frontend,cn=config
+objectClass: olcDatabaseConfig
+objectClass: olcFrontendConfig
+olcDatabase: {-1}frontend
+olcAddContentAcl: FALSE
+olcLastMod: TRUE
+olcSchemaDN: cn=Subschema
+# Hashes to be used in generation of user passwords
+olcPasswordHash: {SSHA}
+structuralObjectClass: olcDatabaseConfig
+
+#
+# Config database Configuration (#0)
+#
+dn: olcDatabase={0}config,cn=config
+objectClass: olcDatabaseConfig
+olcDatabase: {0}config
+# Give access to root user.
+# This give the possiblity to the admin to customize the LDAP configuration
+olcAccess: {0}to *  by * none
+olcAddContentAcl: TRUE
+olcLastMod: TRUE
+olcRootDN: cn=config
+structuralObjectClass: olcDatabaseConfig
+
+#
+# Main database Configuration (#1)
+#
+dn: olcDatabase={1}mdb,cn=config
+objectClass: olcDatabaseConfig
+objectClass: olcMdbConfig
+olcDatabase: {1}mdb
+# The base of your directory in database #1
+olcSuffix: dc=yunohost,dc=org
+#
+# The userPassword by default can be changed
+# by the entry owning it if they are authenticated.
+# Others should not be able to see it, except the
+# admin entry below
+# These access lines apply to database #1 only
+olcAccess: {0}to attrs=userPassword,shadowLastChange
+    by dn.base="cn=admin,dc=yunohost,dc=org" write
+    by dn.base="gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth" write
+    by anonymous auth
+    by self write
+    by * none
+#
+# Personnal information can be changed by the entry
+# owning it if they are authenticated.
+# Others should be able to see it.
+olcAccess: {1}to attrs=cn,gecos,givenName,mail,maildrop,displayName,sn
+    by dn.base="cn=admin,dc=yunohost,dc=org" write
+    by dn.base="gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth" write
+    by self write
+    by * read
+#
+# Ensure read access to the base for things like
+# supportedSASLMechanisms.  Without this you may
+# have problems with SASL not knowing what
+# mechanisms are available and the like.
+# Note that this is covered by the 'access to *'
+# ACL below too but if you change that as people
+# are wont to do you'll still need this if you
+# want SASL (and possible other things) to work
+# happily.
+olcAccess: {2}to dn.base=""
+    by * read
+#
+# The admin dn has full write access, everyone else
+# can read everything.
+olcAccess: {3}to *
+    by dn.base="cn=admin,dc=yunohost,dc=org" write
+    by dn.base="gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth" write
+    by group/groupOfNames/member.exact="cn=admin,ou=groups,dc=yunohost,dc=org" write
+    by * read
+#
+olcAddContentAcl: FALSE
+# Save the time that the entry gets modified, for database #1
+olcLastMod: TRUE
+# Where the database file are physically stored for database #1
+olcDbDirectory: /var/lib/ldap
+# Checkpoint the BerkeleyDB database periodically in case of system
+# failure and to speed slapd shutdown.
+olcDbCheckpoint: 512 30
+olcDbNoSync: FALSE
+# Indexing options for database #1
+olcDbIndex: objectClass eq
+olcDbIndex: entryUUID eq
+olcDbIndex: entryCSN eq
+olcDbIndex: cn eq
+olcDbIndex: uid eq,sub
+olcDbIndex: uidNumber eq
+olcDbIndex: gidNumber eq
+olcDbIndex: sudoUser eq,sub
+olcDbIndex: member eq
+olcDbIndex: mail eq
+olcDbIndex: memberUid eq
+olcDbIndex: uniqueMember eq
+olcDbIndex: virtualdomain eq
+olcDbIndex: permission eq
+olcDbMaxSize: 10485760
+structuralObjectClass: olcMdbConfig
+
+#
+# Configure Memberof Overlay (used for Yunohost permission)
+#
+
+# Link user <-> group
+dn: olcOverlay={0}memberof,olcDatabase={1}mdb,cn=config
+objectClass: olcOverlayConfig
+objectClass: olcMemberOf
+olcOverlay: {0}memberof
+olcMemberOfDangling: error
+olcMemberOfDanglingError: constraintViolation
+olcMemberOfRefInt: TRUE
+olcMemberOfGroupOC: groupOfNamesYnh
+olcMemberOfMemberAD: member
+olcMemberOfMemberOfAD: memberOf
+structuralObjectClass: olcMemberOf
+
+# Link permission <-> groupes
+dn: olcOverlay={1}memberof,olcDatabase={1}mdb,cn=config
+objectClass: olcOverlayConfig
+objectClass: olcMemberOf
+olcOverlay: {1}memberof
+olcMemberOfDangling: error
+olcMemberOfDanglingError: constraintViolation
+olcMemberOfRefInt: TRUE
+olcMemberOfGroupOC: permissionYnh
+olcMemberOfMemberAD: groupPermission
+olcMemberOfMemberOfAD: permission
+structuralObjectClass: olcMemberOf
+
+# Link permission <-> user
+dn: olcOverlay={2}memberof,olcDatabase={1}mdb,cn=config
+objectClass: olcOverlayConfig
+objectClass: olcMemberOf
+olcOverlay: {2}memberof
+olcMemberOfDangling: error
+olcMemberOfDanglingError: constraintViolation
+olcMemberOfRefInt: TRUE
+olcMemberOfGroupOC: permissionYnh
+olcMemberOfMemberAD: inheritPermission
+olcMemberOfMemberOfAD: permission
+structuralObjectClass: olcMemberOf

data/templates/slapd/sudo.ldif

@@ -1,76 +1,78 @@
 #
 # OpenLDAP schema file for Sudo
-# Save as /etc/openldap/schema/sudo.schema
+# Save as /etc/openldap/schema/sudo.ldif
 #
 
-attributetype ( 1.3.6.1.4.1.15953.9.1.1
+dn: cn=sudo,cn=schema,cn=config
+objectClass: olcSchemaConfig
+cn: sudo
+olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.1
     NAME 'sudoUser'
     DESC 'User(s) who may  run sudo'
     EQUALITY caseExactIA5Match
     SUBSTR caseExactIA5SubstringsMatch
     SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 )
-
-attributetype ( 1.3.6.1.4.1.15953.9.1.2
+#
+olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.2
     NAME 'sudoHost'
     DESC 'Host(s) who may run sudo'
     EQUALITY caseExactIA5Match
     SUBSTR caseExactIA5SubstringsMatch
     SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 )
-
-attributetype ( 1.3.6.1.4.1.15953.9.1.3
+#
+olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.3
     NAME 'sudoCommand'
     DESC 'Command(s) to be executed by sudo'
     EQUALITY caseExactIA5Match
     SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 )
-
-attributetype ( 1.3.6.1.4.1.15953.9.1.4
+#
+olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.4
     NAME 'sudoRunAs'
     DESC 'User(s) impersonated by sudo (deprecated)'
     EQUALITY caseExactIA5Match
     SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 )
-
-attributetype ( 1.3.6.1.4.1.15953.9.1.5
+#
+olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.5
     NAME 'sudoOption'
     DESC 'Options(s) followed by sudo'
     EQUALITY caseExactIA5Match
     SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 )
-
-attributetype ( 1.3.6.1.4.1.15953.9.1.6
+#
+olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.6
     NAME 'sudoRunAsUser'
     DESC 'User(s) impersonated by sudo'
     EQUALITY caseExactIA5Match
     SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 )
-
-attributetype ( 1.3.6.1.4.1.15953.9.1.7
+#
+olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.7
     NAME 'sudoRunAsGroup'
     DESC 'Group(s) impersonated by sudo'
     EQUALITY caseExactIA5Match
     SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 )
-
-attributetype ( 1.3.6.1.4.1.15953.9.1.8
+#
+olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.8
     NAME 'sudoNotBefore'
     DESC 'Start of time interval for which the entry is valid'
     EQUALITY generalizedTimeMatch
     ORDERING generalizedTimeOrderingMatch
     SYNTAX 1.3.6.1.4.1.1466.115.121.1.24 )
-
-attributetype ( 1.3.6.1.4.1.15953.9.1.9
+#
+olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.9
     NAME 'sudoNotAfter'
     DESC 'End of time interval for which the entry is valid'
     EQUALITY generalizedTimeMatch
     ORDERING generalizedTimeOrderingMatch
     SYNTAX 1.3.6.1.4.1.1466.115.121.1.24 )
-
-attributeTypes ( 1.3.6.1.4.1.15953.9.1.10
+#
+olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.10
     NAME 'sudoOrder'
     DESC 'an integer to order the sudoRole entries'
     EQUALITY integerMatch
     ORDERING integerOrderingMatch
     SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 )
-
-objectclass ( 1.3.6.1.4.1.15953.9.2.1 NAME 'sudoRole' SUP top STRUCTURAL
+#
+olcObjectClasses: ( 1.3.6.1.4.1.15953.9.2.1 NAME 'sudoRole' SUP top STRUCTURAL
     DESC 'Sudoer Entries'
     MUST ( cn )
-    MAY ( sudoUser $ sudoHost $ sudoCommand $ sudoRunAs $ sudoRunAsUser $ sudoRunAsGroup $ sudoOption $ sudoOrder $ sudoNotBefore $ sudoNotAfter $
-	    description )
+    MAY ( sudoUser $ sudoHost $ sudoCommand $ sudoRunAs $ sudoRunAsUser $ sudoRunAsGroup $ sudoOption $ sudoOrder $ sudoNotBefore $ sudoNotAfter $ description )
     )

data/templates/ssh/sshd_config

@@ -27,9 +27,6 @@ HostKey {{ key }}{% endfor %}
   MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-512,hmac-sha2-256,umac-128@openssh.com
 {% endif %}
 
-# Use kernel sandbox mechanisms where possible in unprivileged processes
-UsePrivilegeSeparation sandbox
-
 # LogLevel VERBOSE logs user's key fingerprint on login.
 # Needed to have a clear audit track of which key was using to log in.
 SyslogFacility AUTH

data/templates/unattended/02periodic

@@ -1,5 +0,0 @@
-# https://wiki.debian.org/UnattendedUpgrades#automatic_call_via_.2Fetc.2Fapt.2Fapt.conf.d.2F02periodic
-APT::Periodic::Enable "1";
-APT::Periodic::Update-Package-Lists "1";
-APT::Periodic::Unattended-Upgrade "1";
-APT::Periodic::Verbose "1";

data/templates/unattended/50unattended-upgrades

@@ -1,36 +0,0 @@
-// Automatically upgrade packages from these (origin, archive) pairs
-Unattended-Upgrade::Allowed-Origins {
-        "${distro_id} stable";
-        "${distro_id} testing";
-        "Depot-Debian testing";
-        "${distro_id} ${distro_codename}-security";
-        "${distro_id} ${distro_codename}-updates";
-//      "${distro_id} ${distro_codename}-proposed-updates";
-};
-
-// List of packages to not update
-Unattended-Upgrade::Package-Blacklist {
-//      "vim";
-//      "libc6";
-//      "libc6-dev";
-//      "libc6-i686";
-};
-
-// Send email to this address for problems or packages upgrades
-// If empty or unset then no email is sent, make sure that you
-// have a working mail setup on your system. The package 'mailx'
-// must be installed or anything that provides /usr/bin/mail.
-//Unattended-Upgrade::Mail "root@localhost";
-
-// Do automatic removal of new unused dependencies after the upgrade
-// (equivalent to apt-get autoremove)
-Unattended-Upgrade::Remove-Unused-Dependencies "true";
-
-// Automatically reboot *WITHOUT CONFIRMATION* if a
-// the file /var/run/reboot-required is found after the upgrade
-Unattended-Upgrade::Automatic-Reboot "false";
-
-
-// Use apt bandwidth limit feature, this example limits the download
-// speed to 70kb/sec
-//Acquire::http::Dl-Limit "70";

data/templates/yunohost/services.yml

@@ -20,10 +20,9 @@ nginx:
   test_conf: nginx -t
   needs_exposed_ports: [80, 443]
   category: web
-nslcd: {}
-php7.0-fpm:
-  log: /var/log/php7.0-fpm.log
-  test_conf: php-fpm7.0 --test
+php7.3-fpm:
+  log: /var/log/php7.3-fpm.log
+  test_conf: php-fpm7.3 --test
   category: web
 postfix:
   log: [/var/log/mail.log,/var/log/mail.err]
@@ -64,3 +63,5 @@ postgrey: null
 spamassassin: null
 rmilter: null
 php5-fpm: null
+php7.0-fpm: null
+nslcd: null

debian/changelog

@@ -1,3 +1,25 @@
+yunohost (4.0.1~alpha) testing; urgency=low
+
+  - [fix] It just make no sense to backup/restore the mysql password... (#911)
+  - [fix] Fix getopts and helpers (#885, #886)
+  - [fix] Explicitly create home using mkhomedir_helper instead of obscure pam rule that doesn't work anymore (b67ff314)
+  - [fix] Ldap interface seems to expect lists everywhere now? (fb8c2b7b)
+  - [deb] Clean control file, remove some legacy Conflicts and Replaces (ca0d4933)
+  - [deb] Add conflicts with versions from backports for critical dependencies (#967)
+  - [cleanup] Stale / legacy code (217aaa36, d77da6a0, af047468, 82d468a3)
+  - [conf] Automatically disable/stop systemd-resolved that conflicts with dnsmasq on fresh setups ... (e7214b37)
+  - [conf] Remove deprecated option in sshd conf, c.f. https://patchwork.openembedded.org/patch/139981/ (2723d245)
+  - [conf] Small tweak in dovecot conf (deprecated settings) (dc0481e2)
+  - [conf] Update nslcd and nsswitch stuff using new Buster's default configs + get rid of nslcd service, only keep the regen-conf part (6ef3520f)
+  - [php] Migrate from php7.0 to php7.3 (3374e653, 9be10506, dd9564d3, 9679c291, 212a15e4, 25fcaa19, c4ad66f5)
+  - [psql] Migrate from psql 11 to 9.6 (e88aed72, 4920d4f9, c70b0ae4)
+  - [firewall] Migrate from xtable to nftable (05fb58f2, 2c4a8b73, 625d5372)
+  - [slapd] Rework slapd regenconf to use new backend (#984)
+
+  Thanks to all contributors <3 ! (Étienne M., Josué, Kay0u)
+
+ -- Alexandre Aubin <alex.aubin@mailoo.org>  Fri, 05 Jun 2020 03:10:09 +0200
+
 yunohost (3.8.4.7) stable; urgency=low
 
   - [fix] Remove some remains of glances (17eec25e)

debian/conf/pam/mkhomedir

@@ -1,6 +0,0 @@
-Name: Create home directory during login
-Default: yes
-Priority: 900
-Session-Type: Additional
-Session:
-        required        pam_mkhomedir.so umask=0022 skel=/etc/skel

debian/control

@@ -11,12 +11,12 @@ Package: yunohost
 Essential: yes
 Architecture: all
 Depends: ${python:Depends}, ${misc:Depends}
- , moulinette (>= 3.7), ssowat (>= 3.7)
+ , moulinette (>= 4.0.0~alpha), ssowat (>= 4.0.0~alpha)
  , python-psutil, python-requests, python-dnspython, python-openssl
  , python-miniupnpc, python-dbus, python-jinja2
- , python-toml, python-packaging
+ , python-toml, python-packaging, python-publicsuffix
  , apt, apt-transport-https
- , nginx, nginx-extras (>=1.6.2)
+ , nginx, nginx-extras
  , php-fpm, php-ldap, php-intl
  , mariadb-server, php-mysql | php-mysqlnd
  , openssh-server, iptables, fail2ban, dnsutils, bind9utils
@@ -25,31 +25,29 @@ Depends: ${python:Depends}, ${misc:Depends}
  , dnsmasq, avahi-daemon, libnss-mdns, resolvconf, libnss-myhostname
  , postfix, postfix-ldap, postfix-policyd-spf-perl, postfix-pcre
  , dovecot-core, dovecot-ldap, dovecot-lmtpd, dovecot-managesieved, dovecot-antispam
- , rspamd (>= 1.6.0), opendkim-tools, postsrsd, procmail, mailutils
+ , rspamd, opendkim-tools, postsrsd, procmail, mailutils
  , redis-server
  , metronome (>=3.14.0)
- , git, curl, wget, cron, unzip, jq
+ , git, curl, wget, cron, unzip, jq, bc
  , lsb-release, haveged, fake-hwclock, equivs, lsof, whois, python-publicsuffix
 Recommends: yunohost-admin
  , ntp, inetutils-ping | iputils-ping
  , bash-completion, rsyslog
- , php-gd, php-curl, php-gettext, php-mcrypt
+ , php-gd, php-curl, php-gettext
  , python-pip
  , unattended-upgrades
  , libdbd-ldap-perl, libnet-dns-perl
 Suggests: htop, vim, rsync, acpi-support-base, udisks2
 Conflicts: iptables-persistent
- , moulinette-yunohost, yunohost-config
- , yunohost-config-others, yunohost-config-postfix
- , yunohost-config-dovecot, yunohost-config-slapd
- , yunohost-config-nginx, yunohost-config-amavis
- , yunohost-config-mysql, yunohost-predepends
- , apache2, bind9
-Replaces: moulinette-yunohost, yunohost-config
- , yunohost-config-others, yunohost-config-postfix
- , yunohost-config-dovecot, yunohost-config-slapd
- , yunohost-config-nginx, yunohost-config-amavis
- , yunohost-config-mysql, yunohost-predepends
+ , apache2
+ , bind9
+ , nginx-extras (>= 1.16)
+ , openssl (>= 1.1.1g)
+ , slapd (>= 2.4.49)
+ , dovecot-core (>= 1:2.3.7)
+ , redis-server (>= 5:5.0.7)
+ , fail2ban (>= 0.11)
+ , iptables (>= 1.8.3)
 Description: manageable and configured self-hosting server
  YunoHost aims to make self-hosting accessible to everyone. It configures
  an email, Web and IM server alongside a LDAP base. It also provides

debian/install

@@ -13,7 +13,6 @@ data/other/* /usr/share/yunohost/yunohost-config/moulinette/
 data/templates/* /usr/share/yunohost/templates/
 data/helpers /usr/share/yunohost/
 data/helpers.d/* /usr/share/yunohost/helpers.d/
-debian/conf/pam/* /usr/share/pam-configs/
 lib/metronome/modules/* /usr/lib/metronome/modules/
 locales/* /usr/lib/moulinette/yunohost/locales/
 src/yunohost /usr/lib/moulinette

debian/postinst

@@ -29,11 +29,6 @@ do_configure() {
 
   # Yunoprompt
   systemctl enable yunoprompt.service
-
-  # remove old PAM config and update it
-  [[ ! -f /usr/share/pam-configs/my_mkhomedir ]] \
-    || rm /usr/share/pam-configs/my_mkhomedir
-  pam-auth-update --package
 }
 
 # summary of how this script can be called:

locales/en.json

@@ -106,7 +106,6 @@
     "backup_output_directory_required": "You must provide an output directory for the backup",
     "backup_output_symlink_dir_broken": "Your archive directory '{path:s}' is a broken symlink. Maybe you forgot to re/mount or plug in the storage medium it points to.",
     "backup_permission": "Backup permission for app {app:s}",
-    "backup_php5_to_php7_migration_may_fail": "Could not convert your archive to support PHP 7, you may be unable to restore your PHP apps (reason: {error:s})",
     "backup_running_hooks": "Running backup hooks…",
     "backup_system_part_failed": "Could not backup the '{part:s}' system part",
     "backup_unable_to_organize_files": "Could not use the quick method to organize files in the archive",
@@ -406,63 +405,13 @@
     "mail_unavailable": "This e-mail address is reserved and shall be automatically allocated to the very first user",
     "main_domain_change_failed": "Unable to change the main domain",
     "main_domain_changed": "The main domain has been changed",
-    "migrate_tsig_end": "Migration to HMAC-SHA-512 finished",
-    "migrate_tsig_failed": "Could not migrate the DynDNS domain '{domain}' to HMAC-SHA-512, rolling back. Error: {error_code}, {error}",
-    "migrate_tsig_start": "Insufficiently secure key algorithm detected for TSIG signature of the domain '{domain}', initiating migration to the more secure HMAC-SHA-512",
-    "migrate_tsig_wait": "Waiting three minutes for the DynDNS server to take the new key into account…",
-    "migrate_tsig_wait_2": "2min…",
-    "migrate_tsig_wait_3": "1min…",
-    "migrate_tsig_wait_4": "30 seconds…",
-    "migrate_tsig_not_needed": "You do not appear to use a DynDNS domain, so no migration is needed.",
-    "migration_description_0001_change_cert_group_to_sslcert": "Change certificates group permissions from 'metronome' to 'ssl-cert'",
-    "migration_description_0002_migrate_to_tsig_sha256": "Improve security of DynDNS TSIG updates by using SHA-512 instead of MD5",
-    "migration_description_0003_migrate_to_stretch": "Upgrade the system to Debian Stretch and YunoHost 3.0",
-    "migration_description_0004_php5_to_php7_pools": "Reconfigure the PHP pools to use PHP 7 instead of 5",
-    "migration_description_0005_postgresql_9p4_to_9p6": "Migrate databases from PostgreSQL 9.4 to 9.6",
-    "migration_description_0006_sync_admin_and_root_passwords": "Synchronize admin and root passwords",
-    "migration_description_0007_ssh_conf_managed_by_yunohost_step1": "Let the SSH configuration be managed by YunoHost (step 1, automatic)",
-    "migration_description_0008_ssh_conf_managed_by_yunohost_step2": "Let the SSH configuration be managed by YunoHost (step 2, manual)",
-    "migration_description_0009_decouple_regenconf_from_services": "Decouple the regen-conf mechanism from services",
-    "migration_description_0010_migrate_to_apps_json": "Remove deprecated apps catalogs and use the new unified 'apps.json' list instead (outdated, replaced by migration 13)",
-    "migration_description_0011_setup_group_permission": "Set up user groups and permissions for apps and services",
-    "migration_description_0012_postgresql_password_to_md5_authentication": "Force PostgreSQL authentication to use MD5 for local connections",
-    "migration_description_0013_futureproof_apps_catalog_system": "Migrate to the new future-proof apps catalog system",
-    "migration_description_0014_remove_app_status_json": "Remove legacy status.json app files",
     "migration_description_0015_migrate_to_buster": "Upgrade the system to Debian Buster and YunoHost 4.x",
-    "migration_0003_start": "Starting migration to Stretch. The logs will be available in {logfile}.",
-    "migration_0003_patching_sources_list": "Patching the sources.lists…",
-    "migration_0003_main_upgrade": "Starting main upgrade…",
-    "migration_0003_fail2ban_upgrade": "Starting the Fail2Ban upgrade…",
-    "migration_0003_restoring_origin_nginx_conf": "Your file /etc/nginx/nginx.conf was edited somehow. The migration is going to reset it to its original state first… The previous file will be available as {backup_dest}.",
-    "migration_0003_yunohost_upgrade": "Starting the YunoHost package upgrade… The migration will end, but the actual upgrade will happen immediately afterwards. After the operation is complete, you might have to log in to the webadmin page again.",
-    "migration_0003_not_jessie": "The current Debian distribution is not Jessie!",
-    "migration_0003_system_not_fully_up_to_date": "Your system is not fully up-to-date. Please perform a regular upgrade before running the migration to Stretch.",
-    "migration_0003_still_on_jessie_after_main_upgrade": "Something went wrong during the main upgrade: Is the system still on Jessie‽ To investigate the issue, please look at {log}:s…",
-    "migration_0003_general_warning": "Please note that this migration is a delicate operation. The YunoHost team did its best to review and test it, but the migration might still break parts of the system or its apps.\n\nTherefore, it is recommended to:\n    - Perform a backup of any critical data or app. More info on https://yunohost.org/backup;\n    - Be patient after launching the migration: Depending on your Internet connection and hardware, it might take up to a few hours for everything to upgrade.\n\nAdditionally, the port for SMTP, used by external e-mail clients (like Thunderbird or K9-Mail) was changed from 465 (SSL/TLS) to 587 (STARTTLS). The old port (465) will automatically be closed, and the new port (587) will be opened in the firewall. You and your users *will* have to adapt the configuration of your e-mail clients accordingly.",
-    "migration_0003_problematic_apps_warning": "Please note that the following possibly problematic installed apps were detected. It looks like those were not installed from an app catalog, or are not flagged as 'working'. Consequently, it cannot be guaranteed that they will still work after the upgrade: {problematic_apps}",
-    "migration_0003_modified_files": "Please note that the following files were found to be manually modified and might be overwritten following the upgrade: {manually_modified_files}",
-    "migration_0005_postgresql_94_not_installed": "PostgreSQL was not installed on your system. Nothing to do.",
-    "migration_0005_postgresql_96_not_installed": "PostgreSQL 9.4 is installed, but not postgresql 9.6‽ Something weird might have happened on your system :(…",
-    "migration_0005_not_enough_space": "Make sufficient space available in {path} to run the migration.",
-    "migration_0006_disclaimer": "YunoHost now expects the admin and root passwords to be synchronized. This migration replaces your root password with the admin password.",
-    "migration_0007_cancelled": "Could not improve the way your SSH configuration is managed.",
-    "migration_0007_cannot_restart": "SSH can't be restarted after trying to cancel migration number 6.",
-    "migration_0008_general_disclaimer": "To improve the security of your server, it is recommended to let YunoHost manage the SSH configuration. Your current SSH setup differs from the recommendation. If you let YunoHost reconfigure it, the way you connect to your server through SSH will change thusly:",
-    "migration_0008_port": "• You will have to connect using port 22 instead of your current custom SSH port. Feel free to reconfigure it;",
-    "migration_0008_root": "• You will not be able to connect as root through SSH. Instead you should use the admin user;",
-    "migration_0008_dsa": "• The DSA key will be turned off. Hence, you might need to invalidate a spooky warning from your SSH client, and recheck the fingerprint of your server;",
-    "migration_0008_warning": "If you understand those warnings and want YunoHost to override your current configuration, run the migration. Otherwise, you can also skip the migration, though it is not recommended.",
-    "migration_0008_no_warning": "Overriding your SSH configuration should be safe, though this cannot be promised! Run the migration to override it. Otherwise, you can also skip the migration, though it is not recommended.",
-    "migration_0009_not_needed": "This migration already happened somehow… (?) Skipping.",
-    "migration_0011_backup_before_migration": "Creating a backup of LDAP database and apps settings prior to the actual migration.",
-    "migration_0011_can_not_backup_before_migration": "The backup of the system could not be completed before the migration failed. Error: {error:s}",
+    "migration_description_0016_php70_to_php73_pools": "Migrate php7.0-fpm 'pool' conf files to php7.3",
+    "migration_description_0017_postgresql_9p6_to_11": "Migrate databases from PostgreSQL 9.6 to 11",
+    "migration_description_0018_xtable_to_nftable": "Migrate old network traffic rules to the new nftable system",
     "migration_0011_create_group": "Creating a group for each user…",
-    "migration_0011_done": "Migration completed. You are now able to manage usergroups.",
-    "migration_0011_slapd_config_will_be_overwritten": "It looks like you manually edited the slapd configuration. For this critical migration, YunoHost needs to force the update of the slapd configuration. The original files will be backuped in {conf_backup_folder}.",
     "migration_0011_LDAP_update_failed": "Could not update LDAP. Error: {error:s}",
     "migration_0011_migrate_permission": "Migrating permissions from apps settings to LDAP…",
-    "migration_0011_migration_failed_trying_to_rollback": "Could not migrate… trying to roll back the system.",
-    "migration_0011_rollback_success": "System rolled back.",
     "migration_0011_update_LDAP_database": "Updating LDAP database…",
     "migration_0011_update_LDAP_schema": "Updating LDAP schema…",
     "migration_0011_failed_to_remove_stale_object": "Could not remove stale object {dn}: {error}",
@@ -479,6 +428,11 @@
     "migration_0015_modified_files": "Please note that the following files were found to be manually modified and might be overwritten following the upgrade: {manually_modified_files}",
     "migration_0015_specific_upgrade": "Starting upgrade of system packages that needs to be upgrade independently…",
     "migration_0015_cleaning_up": "Cleaning up cache and packages not useful anymore…",
+    "migration_0017_postgresql_96_not_installed": "PostgreSQL was not installed on your system. Nothing to do.",
+    "migration_0017_postgresql_11_not_installed": "PostgreSQL 9.6 is installed, but not postgresql 11‽ Something weird might have happened on your system :(…",
+    "migration_0017_not_enough_space": "Make sufficient space available in {path} to run the migration.",
+    "migration_0018_failed_to_migrate_iptables_rules": "Failed to migrate legacy iptables rules to nftables: {error}",
+    "migration_0018_failed_to_reset_legacy_rules": "Failed to reset legacy iptables rules: {error}",
     "migrations_already_ran": "Those migrations are already done: {ids}",
     "migrations_cant_reach_migration_file": "Could not access migrations files at the path '%s'",
     "migrations_dependencies_not_satisfied": "Run these migrations: '{dependencies_id}', before migration {id}.",
@@ -497,7 +451,6 @@
     "migrations_skip_migration": "Skipping migration {id}…",
     "migrations_success_forward": "Migration {id} completed",
     "migrations_to_be_ran_manually":  "Migration {id} has to be run manually. Please go to Tools → Migrations on the webadmin page, or run `yunohost tools migrations migrate`.",
-    "no_internet_connection": "The server is not connected to the Internet",
     "not_enough_disk_space": "Not enough free space on '{path:s}'",
     "operation_interrupted": "The operation was manually interrupted?",
     "packages_upgrade_failed": "Could not upgrade all the packages",
@@ -583,8 +536,7 @@
     "service_description_metronome": "Manage XMPP instant messaging accounts",
     "service_description_mysql": "Stores app data (SQL database)",
     "service_description_nginx": "Serves or provides access to all the websites hosted on your server",
-    "service_description_nslcd": "Handles YunoHost user shell connection",
-    "service_description_php7.0-fpm": "Runs apps written in PHP with NGINX",
+    "service_description_php7.3-fpm": "Runs apps written in PHP with NGINX",
     "service_description_postfix": "Used to send and receive e-mails",
     "service_description_redis-server": "A specialized database used for rapid data access, task queue, and communication between programs",
     "service_description_rspamd": "Filters spam, and other e-mail related features",

src/yunohost/app.py

@@ -521,7 +521,7 @@ def app_upgrade(app=[], url=None, file=None):
         _patch_legacy_helpers(extracted_app_folder)
 
         # Apply dirty patch to make php5 apps compatible with php7
-        _patch_php5(extracted_app_folder)
+        _patch_legacy_php_versions(extracted_app_folder)
 
         # Start register change on system
         related_to = [('app', app_instance_name)]
@@ -736,7 +736,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu
     _patch_legacy_helpers(extracted_app_folder)
 
     # Apply dirty patch to make php5 apps compatible with php7
-    _patch_php5(extracted_app_folder)
+    _patch_legacy_php_versions(extracted_app_folder)
 
     # Prepare env. var. to pass to script
     env_dict = _make_environment_dict(args_odict)
@@ -865,6 +865,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu
                     os.path.join(extracted_app_folder, 'scripts/remove'),
                     args=[app_instance_name], env=env_dict_remove
                 )[0]
+
             # Here again, calling hook_exec could fail miserably, or get
             # manually interrupted (by mistake or because script was stuck)
             # In that case we still want to proceed with the rest of the
@@ -1032,7 +1033,7 @@ def app_remove(operation_logger, app):
 
     # Apply dirty patch to make php5 apps compatible with php7 (e.g. the remove
     # script might date back from jessie install)
-    _patch_php5(app_setting_path)
+    _patch_legacy_php_versions(app_setting_path)
 
     manifest = _get_manifest_of_app(app_setting_path)
 
@@ -2682,12 +2683,6 @@ def _read_apps_catalog_list():
     Read the json corresponding to the list of apps catalogs
     """
 
-    # Legacy code - can be removed after moving to buster (if the migration got merged before buster)
-    if os.path.exists('/etc/yunohost/appslists.json'):
-        from yunohost.tools import _get_migration_by_name
-        migration = _get_migration_by_name("futureproof_apps_catalog_system")
-        migration.run()
-
     try:
         list_ = read_yaml(APPS_CATALOG_CONF)
         # Support the case where file exists but is empty
@@ -2844,8 +2839,8 @@ def _assert_system_is_sane_for_app(manifest, when):
 
     # Some apps use php-fpm or php5-fpm which is now php7.0-fpm
     def replace_alias(service):
-        if service in ["php-fpm", "php5-fpm"]:
-            return "php7.0-fpm"
+        if service in ["php-fpm", "php5-fpm", "php7.0-fpm"]:
+            return "php7.3-fpm"
         else:
             return service
     services = [replace_alias(s) for s in services]
@@ -2853,7 +2848,7 @@ def _assert_system_is_sane_for_app(manifest, when):
     # We only check those, mostly to ignore "custom" services
     # (added by apps) and because those are the most popular
     # services
-    service_filter = ["nginx", "php7.0-fpm", "mysql", "postfix"]
+    service_filter = ["nginx", "php7.3-fpm", "mysql", "postfix"]
     services = [str(s) for s in services if s in service_filter]
 
     if "nginx" not in services:
@@ -2878,11 +2873,24 @@ def _assert_system_is_sane_for_app(manifest, when):
             raise YunohostError("this_action_broke_dpkg")
 
 
-def _patch_php5(app_folder):
+LEGACY_PHP_VERSION_REPLACEMENTS = [
+    ("/etc/php5", "/etc/php/7.3"),
+    ("/etc/php/7.0", "/etc/php/7.3"),
+    ("/var/run/php5-fpm", "/var/run/php/php7.3-fpm"),
+    ("/var/run/php/php7.0-fpm", "/var/run/php/php7.3-fpm"),
+    ("php5", "php7.3"),
+    ("php7.0", "php7.3"),
+    ('phpversion="${phpversion:-7.0}"', 'phpversion="${phpversion:-7.3}"'),  # Many helpers like the composer ones use 7.0 by default ...
+    ('"$phpversion" == "7.0"', '$(bc <<< "$phpversion >= 7.3") -eq 1')  # patch ynh_install_php to refuse installing/removing php <= 7.3
+]
+
+
+def _patch_legacy_php_versions(app_folder):
 
     files_to_patch = []
     files_to_patch.extend(glob.glob("%s/conf/*" % app_folder))
     files_to_patch.extend(glob.glob("%s/scripts/*" % app_folder))
+    files_to_patch.extend(glob.glob("%s/scripts/*/*" % app_folder))
     files_to_patch.extend(glob.glob("%s/scripts/.*" % app_folder))
     files_to_patch.append("%s/manifest.json" % app_folder)
     files_to_patch.append("%s/manifest.toml" % app_folder)
@@ -2893,12 +2901,32 @@ def _patch_php5(app_folder):
         if not os.path.isfile(filename):
             continue
 
-        c = "sed -i -e 's@/etc/php5@/etc/php/7.0@g' " \
-            "-e 's@/var/run/php5-fpm@/var/run/php/php7.0-fpm@g' " \
-            "-e 's@php5@php7.0@g' " \
-            "%s" % filename
+        c = "sed -i " \
+            + "".join("-e 's@{pattern}@{replace}@g' ".format(pattern=p, replace=r) for p, r in LEGACY_PHP_VERSION_REPLACEMENTS) \
+            + "%s" % filename
         os.system(c)
 
+
+def _patch_legacy_php_versions_in_settings(app_folder):
+
+    settings = read_yaml(os.path.join(app_folder, 'settings.yml'))
+
+    if settings.get("fpm_config_dir") == "/etc/php/7.0/fpm":
+        settings["fpm_config_dir"] = "/etc/php/7.3/fpm"
+    if settings.get("fpm_service") == "php7.0-fpm":
+        settings["fpm_service"] = "php7.3-fpm"
+    if settings.get("phpversion") == "7.0":
+        settings["phpversion"] = "7.3"
+
+    # We delete these checksums otherwise the file will appear as manually modified
+    list_to_remove = ["checksum__etc_php_7.0_fpm_pool",
+                      "checksum__etc_nginx_conf.d"]
+    settings = {k: v for k, v in settings.items()
+                if not any(k.startswith(to_remove) for to_remove in list_to_remove)}
+
+    write_to_yaml(app_folder + '/settings.yml', settings)
+
+
 def _patch_legacy_helpers(app_folder):
 
     files_to_patch = []

src/yunohost/backup.py

@@ -43,7 +43,13 @@ from moulinette.utils.log import getActionLogger
 from moulinette.utils.filesystem import read_file, mkdir, write_to_yaml, read_yaml
 
 from yunohost.app import (
-    app_info, _is_installed, _parse_app_instance_name, _patch_php5, dump_app_log_extract_for_debugging, _patch_legacy_helpers
+    app_info, _is_installed,
+    _parse_app_instance_name,
+    dump_app_log_extract_for_debugging,
+    _patch_legacy_helpers,
+    _patch_legacy_php_versions,
+    _patch_legacy_php_versions_in_settings,
+    LEGACY_PHP_VERSION_REPLACEMENTS
 )
 from yunohost.hook import (
     hook_list, hook_info, hook_callback, hook_exec, CUSTOM_HOOK_FOLDER
@@ -1141,7 +1147,7 @@ class RestoreManager():
             self._postinstall_if_needed()
 
             # Apply dirty patch to redirect php5 file on php7
-            self._patch_backup_csv_file()
+            self._patch_legacy_php_versions_in_csv_file()
 
             self._restore_system()
             self._restore_apps()
@@ -1150,9 +1156,9 @@ class RestoreManager():
         finally:
             self.clean()
 
-    def _patch_backup_csv_file(self):
+    def _patch_legacy_php_versions_in_csv_file(self):
         """
-        Apply dirty patch to redirect php5 file on php7
+        Apply dirty patch to redirect php5 and php7.0 files to php7.3
         """
 
         backup_csv = os.path.join(self.work_dir, 'backup.csv')
@@ -1160,32 +1166,27 @@ class RestoreManager():
         if not os.path.isfile(backup_csv):
             return
 
-        contains_php5 = False
+        replaced_something = False
         with open(backup_csv) as csvfile:
             reader = csv.DictReader(csvfile, fieldnames=['source', 'dest'])
             newlines = []
             for row in reader:
-                if 'php5' in row['source']:
-                    contains_php5 = True
-                    row['source'] = row['source'].replace('/etc/php5', '/etc/php/7.0') \
-                        .replace('/var/run/php5-fpm', '/var/run/php/php7.0-fpm') \
-                        .replace('php5', 'php7')
+                for pattern, replace in LEGACY_PHP_VERSION_REPLACEMENTS:
+                    if pattern in row['source']:
+                        replaced_something = True
+                        row['source'] = row['source'].replace(pattern, replace)
 
                 newlines.append(row)
 
-        if not contains_php5:
+        if not replaced_something:
             return
 
-        try:
-            with open(backup_csv, 'w') as csvfile:
-                writer = csv.DictWriter(csvfile,
-                                        fieldnames=['source', 'dest'],
-                                        quoting=csv.QUOTE_ALL)
-                for row in newlines:
-                    writer.writerow(row)
-        except (IOError, OSError, csv.Error) as e:
-            logger.warning(m18n.n('backup_php5_to_php7_migration_may_fail',
-                                  error=str(e)))
+        with open(backup_csv, 'w') as csvfile:
+            writer = csv.DictWriter(csvfile,
+                                    fieldnames=['source', 'dest'],
+                                    quoting=csv.QUOTE_ALL)
+            for row in newlines:
+                writer.writerow(row)
 
     def _restore_system(self):
         """ Restore user and system parts """
@@ -1244,12 +1245,11 @@ class RestoreManager():
         #
         # Legacy code
         if not "all_users" in user_group_list()["groups"].keys():
-            from yunohost.tools import _get_migration_by_name
-            setup_group_permission = _get_migration_by_name("setup_group_permission")
+            from yunohost.utils.legacy import SetupGroupPermissions
             # Update LDAP schema restart slapd
             logger.info(m18n.n("migration_0011_update_LDAP_schema"))
             regen_conf(names=['slapd'], force=True)
-            setup_group_permission.migrate_LDAP_db()
+            SetupGroupPermissions.migrate_LDAP_db()
 
         # Remove all permission for all app which is still in the LDAP
         for permission_name in user_permission_list(ignore_system_perms=True)["permissions"].keys():
@@ -1334,7 +1334,8 @@ class RestoreManager():
         _patch_legacy_helpers(app_settings_in_archive)
 
         # Apply dirty patch to make php5 apps compatible with php7
-        _patch_php5(app_settings_in_archive)
+        _patch_legacy_php_versions(app_settings_in_archive)
+        _patch_legacy_php_versions_in_settings(app_settings_in_archive)
 
         # Delete _common.sh file in backup
         common_file = os.path.join(app_backup_in_archive, '_common.sh')
@@ -1389,9 +1390,8 @@ class RestoreManager():
             else:
                 # Otherwise, we need to migrate the legacy permissions of this
                 # app (included in its settings.yml)
-                from yunohost.tools import _get_migration_by_name
-                setup_group_permission = _get_migration_by_name("setup_group_permission")
-                setup_group_permission.migrate_app_permission(app=app_instance_name)
+                from yunohost.utils.legacy import SetupGroupPermissions
+                SetupGroupPermissions.migrate_app_permission(app=app_instance_name)
 
             # Prepare env. var. to pass to script
             env_dict = self._get_env_var(app_instance_name)

src/yunohost/data_migrations/0001_change_cert_group_to_sslcert.py

@@ -1,15 +0,0 @@
-import subprocess
-import glob
-from yunohost.tools import Migration
-from moulinette.utils.filesystem import chown
-
-
-class MyMigration(Migration):
-
-    "Change certificates group permissions from 'metronome' to 'ssl-cert'"
-
-    all_certificate_files = glob.glob("/etc/yunohost/certs/*/*.pem")
-
-    def run(self):
-        for filename in self.all_certificate_files:
-            chown(filename, uid="root", gid="ssl-cert")

src/yunohost/data_migrations/0002_migrate_to_tsig_sha256.py

@@ -1,86 +0,0 @@
-import glob
-import os
-import requests
-import base64
-import time
-import json
-
-from moulinette import m18n
-from yunohost.utils.error import YunohostError
-from moulinette.utils.log import getActionLogger
-
-from yunohost.tools import Migration
-from yunohost.dyndns import _guess_current_dyndns_domain
-
-logger = getActionLogger('yunohost.migration')
-
-
-class MyMigration(Migration):
-
-    "Migrate Dyndns stuff from MD5 TSIG to SHA512 TSIG"
-
-    def run(self, dyn_host="dyndns.yunohost.org", domain=None, private_key_path=None):
-
-        if domain is None or private_key_path is None:
-            try:
-                (domain, private_key_path) = _guess_current_dyndns_domain(dyn_host)
-                assert "+157" in private_key_path
-            except (YunohostError, AssertionError):
-                logger.info(m18n.n("migrate_tsig_not_needed"))
-                return
-
-        logger.info(m18n.n('migrate_tsig_start', domain=domain))
-        public_key_path = private_key_path.rsplit(".private", 1)[0] + ".key"
-        public_key_md5 = open(public_key_path).read().strip().split(' ')[-1]
-
-        os.system('cd /etc/yunohost/dyndns && '
-                  'dnssec-keygen -a hmac-sha512 -b 512 -r /dev/urandom -n USER %s' % domain)
-        os.system('chmod 600 /etc/yunohost/dyndns/*.key /etc/yunohost/dyndns/*.private')
-
-        # +165 means that this file store a hmac-sha512 key
-        new_key_path = glob.glob('/etc/yunohost/dyndns/*+165*.key')[0]
-        public_key_sha512 = open(new_key_path).read().strip().split(' ', 6)[-1]
-
-        try:
-            r = requests.put('https://%s/migrate_key_to_sha512/' % (dyn_host),
-                             data={
-                               'public_key_md5': base64.b64encode(public_key_md5),
-                               'public_key_sha512': base64.b64encode(public_key_sha512),
-                             }, timeout=30)
-        except requests.ConnectionError:
-            raise YunohostError('no_internet_connection')
-
-        if r.status_code != 201:
-            try:
-                error = json.loads(r.text)['error']
-            except Exception:
-                # failed to decode json
-                error = r.text
-
-                import traceback
-                from StringIO import StringIO
-                stack = StringIO()
-                traceback.print_stack(file=stack)
-                logger.error(stack.getvalue())
-
-            # Migration didn't succeed, so we rollback and raise an exception
-            os.system("mv /etc/yunohost/dyndns/*+165* /tmp")
-
-            raise YunohostError('migrate_tsig_failed', domain=domain,
-                                error_code=str(r.status_code), error=error)
-
-        # remove old certificates
-        os.system("mv /etc/yunohost/dyndns/*+157* /tmp")
-
-        # sleep to wait for dyndns cache invalidation
-        logger.info(m18n.n('migrate_tsig_wait'))
-        time.sleep(60)
-        logger.info(m18n.n('migrate_tsig_wait_2'))
-        time.sleep(60)
-        logger.info(m18n.n('migrate_tsig_wait_3'))
-        time.sleep(30)
-        logger.info(m18n.n('migrate_tsig_wait_4'))
-        time.sleep(30)
-
-        logger.info(m18n.n('migrate_tsig_end'))
-        return

src/yunohost/data_migrations/0003_migrate_to_stretch.py

@@ -1,379 +0,0 @@
-import glob
-import os
-from shutil import copy2
-
-from moulinette import m18n, msettings
-from yunohost.utils.error import YunohostError
-from moulinette.utils.log import getActionLogger
-from moulinette.utils.process import check_output, call_async_output
-from moulinette.utils.filesystem import read_file
-
-from yunohost.tools import Migration
-from yunohost.app import unstable_apps
-from yunohost.service import _run_service_command
-from yunohost.regenconf import (manually_modified_files,
-                                manually_modified_files_compared_to_debian_default)
-from yunohost.utils.filesystem import free_space_in_directory
-from yunohost.utils.packages import get_ynh_package_version
-from yunohost.utils.network import get_network_interfaces
-from yunohost.firewall import firewall_allow, firewall_disallow
-
-logger = getActionLogger('yunohost.migration')
-
-YUNOHOST_PACKAGES = ["yunohost", "yunohost-admin", "moulinette", "ssowat"]
-
-
-class MyMigration(Migration):
-
-    "Upgrade the system to Debian Stretch and Yunohost 3.0"
-
-    mode = "manual"
-
-    def run(self):
-
-        self.logfile = "/var/log/yunohost/{}.log".format(self.name)
-
-        self.check_assertions()
-
-        logger.info(m18n.n("migration_0003_start", logfile=self.logfile))
-
-        # Preparing the upgrade
-        self.restore_original_nginx_conf_if_needed()
-
-        logger.info(m18n.n("migration_0003_patching_sources_list"))
-        self.patch_apt_sources_list()
-        self.backup_files_to_keep()
-        self.apt_update()
-        apps_packages = self.get_apps_equivs_packages()
-        self.unhold(["metronome"])
-        self.hold(YUNOHOST_PACKAGES + apps_packages + ["fail2ban"])
-
-        # Main dist-upgrade
-        logger.info(m18n.n("migration_0003_main_upgrade"))
-        _run_service_command("stop", "mysql")
-        self.apt_dist_upgrade(conf_flags=["old", "miss", "def"])
-        _run_service_command("start", "mysql")
-        if self.debian_major_version() == 8:
-            raise YunohostError("migration_0003_still_on_jessie_after_main_upgrade", log=self.logfile)
-
-        # Specific upgrade for fail2ban...
-        logger.info(m18n.n("migration_0003_fail2ban_upgrade"))
-        self.unhold(["fail2ban"])
-        # Don't move this if folder already exists. If it does, we probably are
-        # running this script a 2nd, 3rd, ... time but /etc/fail2ban will
-        # be re-created only for the first dist-upgrade of fail2ban
-        if not os.path.exists("/etc/fail2ban.old"):
-            os.system("mv /etc/fail2ban /etc/fail2ban.old")
-        self.apt_dist_upgrade(conf_flags=["new", "miss", "def"])
-        _run_service_command("restart", "fail2ban")
-
-        self.disable_predicable_interface_names()
-
-        # Clean the mess
-        os.system("apt autoremove --assume-yes")
-        os.system("apt clean --assume-yes")
-
-        # We moved to port 587 for SMTP
-        # https://busylog.net/smtp-tls-ssl-25-465-587/
-        firewall_allow("Both", 587)
-        firewall_disallow("Both", 465)
-
-        # Upgrade yunohost packages
-        logger.info(m18n.n("migration_0003_yunohost_upgrade"))
-        self.restore_files_to_keep()
-        self.unhold(YUNOHOST_PACKAGES + apps_packages)
-        self.upgrade_yunohost_packages()
-
-    def debian_major_version(self):
-        # The python module "platform" and lsb_release are not reliable because
-        # on some setup, they still return Release=8 even after upgrading to
-        # stretch ... (Apparently this is related to OVH overriding some stuff
-        # with /etc/lsb-release for instance -_-)
-        # Instead, we rely on /etc/os-release which should be the raw info from
-        # the distribution...
-        return int(check_output("grep VERSION_ID /etc/os-release | head -n 1 | tr '\"' ' ' | cut -d ' ' -f2"))
-
-    def yunohost_major_version(self):
-        return int(get_ynh_package_version("yunohost")["version"].split('.')[0])
-
-    def check_assertions(self):
-
-        # Be on jessie (8.x) and yunohost 2.x
-        # NB : we do both check to cover situations where the upgrade crashed
-        # in the middle and debian version could be >= 9.x but yunohost package
-        # would still be in 2.x...
-        if not self.debian_major_version() == 8 \
-           and not self.yunohost_major_version() == 2:
-            raise YunohostError("migration_0003_not_jessie")
-
-        # Have > 1 Go free space on /var/ ?
-        if free_space_in_directory("/var/") / (1024**3) < 1.0:
-            raise YunohostError("There is not enough free space in /var/ to run the migration. You need at least 1GB free space")
-
-        # Check system is up to date
-        # (but we don't if 'stretch' is already in the sources.list ...
-        # which means maybe a previous upgrade crashed and we're re-running it)
-        if " stretch " not in read_file("/etc/apt/sources.list"):
-            self.apt_update()
-            apt_list_upgradable = check_output("apt list --upgradable -a")
-            if "upgradable" in apt_list_upgradable:
-                raise YunohostError("migration_0003_system_not_fully_up_to_date")
-
-    @property
-    def disclaimer(self):
-
-        # Avoid having a super long disclaimer + uncessary check if we ain't
-        # on jessie / yunohost 2.x anymore
-        # NB : we do both check to cover situations where the upgrade crashed
-        # in the middle and debian version could be >= 9.x but yunohost package
-        # would still be in 2.x...
-        if not self.debian_major_version() == 8 \
-           and not self.yunohost_major_version() == 2:
-            return None
-
-        # Get list of problematic apps ? I.e. not official or community+working
-        problematic_apps = unstable_apps()
-        problematic_apps = "".join(["\n    - " + app for app in problematic_apps])
-
-        # Manually modified files ? (c.f. yunohost service regen-conf)
-        modified_files = manually_modified_files()
-        # We also have a specific check for nginx.conf which some people
-        # modified and needs to be upgraded...
-        if "/etc/nginx/nginx.conf" in manually_modified_files_compared_to_debian_default():
-            modified_files.append("/etc/nginx/nginx.conf")
-        modified_files = "".join(["\n    - " + f for f in modified_files])
-
-        message = m18n.n("migration_0003_general_warning")
-
-        if problematic_apps:
-            message += "\n\n" + m18n.n("migration_0003_problematic_apps_warning", problematic_apps=problematic_apps)
-
-        if modified_files:
-            message += "\n\n" + m18n.n("migration_0003_modified_files", manually_modified_files=modified_files)
-
-        return message
-
-    def patch_apt_sources_list(self):
-
-        sources_list = glob.glob("/etc/apt/sources.list.d/*.list")
-        sources_list.append("/etc/apt/sources.list")
-
-        # This :
-        # - replace single 'jessie' occurence by 'stretch'
-        # - comments lines containing "backports"
-        # - replace 'jessie/updates' by 'strech/updates' (or same with a -)
-        # - switch yunohost's repo to forge
-        for f in sources_list:
-            command = "sed -i -e 's@ jessie @ stretch @g' " \
-                      "-e '/backports/ s@^#*@#@' " \
-                      "-e 's@ jessie/updates @ stretch/updates @g' " \
-                      "-e 's@ jessie-updates @ stretch-updates @g' " \
-                      "-e 's@repo.yunohost@forge.yunohost@g' " \
-                      "{}".format(f)
-            os.system(command)
-
-    def get_apps_equivs_packages(self):
-
-        command = "dpkg --get-selections" \
-                  " | grep -v deinstall" \
-                  " | awk '{print $1}'" \
-                  " | { grep 'ynh-deps$' || true; }"
-
-        output = check_output(command).strip()
-
-        return output.split('\n') if output else []
-
-    def hold(self, packages):
-        for package in packages:
-            os.system("apt-mark hold {}".format(package))
-
-    def unhold(self, packages):
-        for package in packages:
-            os.system("apt-mark unhold {}".format(package))
-
-    def apt_update(self):
-
-        command = "apt-get update"
-        logger.debug("Running apt command :\n{}".format(command))
-        command += " 2>&1 | tee -a {}".format(self.logfile)
-
-        os.system(command)
-
-    def upgrade_yunohost_packages(self):
-
-        #
-        # Here we use a dirty hack to run a command after the current
-        # "yunohost tools migrations migrate", because the upgrade of
-        # yunohost will also trigger another "yunohost tools migrations migrate"
-        # (also the upgrade of the package, if executed from the webadmin, is
-        # likely to kill/restart the api which is in turn likely to kill this
-        # command before it ends...)
-        #
-
-        MOULINETTE_LOCK = "/var/run/moulinette_yunohost.lock"
-
-        upgrade_command = ""
-        upgrade_command += " DEBIAN_FRONTEND=noninteractive"
-        upgrade_command += " APT_LISTCHANGES_FRONTEND=none"
-        upgrade_command += " apt-get install"
-        upgrade_command += " --assume-yes "
-        upgrade_command += " ".join(YUNOHOST_PACKAGES)
-        # We also install php-zip and php7.0-acpu to fix an issue with
-        # nextcloud and kanboard that need it when on stretch.
-        upgrade_command += " php-zip php7.0-apcu"
-        upgrade_command += " 2>&1 | tee -a {}".format(self.logfile)
-
-        wait_until_end_of_yunohost_command = "(while [ -f {} ]; do sleep 2; done)".format(MOULINETTE_LOCK)
-
-        command = "({} && {}; echo 'Migration complete!') &".format(wait_until_end_of_yunohost_command,
-                                                                    upgrade_command)
-
-        logger.debug("Running command :\n{}".format(command))
-
-        os.system(command)
-
-    def apt_dist_upgrade(self, conf_flags):
-
-        # Make apt-get happy
-        os.system("echo 'libc6 libraries/restart-without-asking boolean true' | debconf-set-selections")
-        # Don't send an email to root about the postgresql migration. It should be handled automatically after.
-        os.system("echo 'postgresql-common postgresql-common/obsolete-major seen true' | debconf-set-selections")
-
-        command = ""
-        command += " DEBIAN_FRONTEND=noninteractive"
-        command += " APT_LISTCHANGES_FRONTEND=none"
-        command += " apt-get"
-        command += " --fix-broken --show-upgraded --assume-yes"
-        for conf_flag in conf_flags:
-            command += ' -o Dpkg::Options::="--force-conf{}"'.format(conf_flag)
-        command += " dist-upgrade"
-
-        logger.debug("Running apt command :\n{}".format(command))
-
-        command += " 2>&1 | tee -a {}".format(self.logfile)
-
-        is_api = msettings.get('interface') == 'api'
-        if is_api:
-            callbacks = (
-                lambda l: logger.info(l.rstrip()),
-                lambda l: logger.warning(l.rstrip()),
-            )
-            call_async_output(command, callbacks, shell=True)
-        else:
-            # We do this when running from the cli to have the output of the
-            # command showing in the terminal, since 'info' channel is only
-            # enabled if the user explicitly add --verbose ...
-            os.system(command)
-
-    # Those are files that should be kept and restored before the final switch
-    # to yunohost 3.x... They end up being modified by the various dist-upgrades
-    # (or need to be taken out momentarily), which then blocks the regen-conf
-    # as they are flagged as "manually modified"...
-    files_to_keep = [
-        "/etc/mysql/my.cnf",
-        "/etc/nslcd.conf",
-        "/etc/postfix/master.cf",
-        "/etc/fail2ban/filter.d/yunohost.conf"
-    ]
-
-    def backup_files_to_keep(self):
-
-        logger.debug("Backuping specific files to keep ...")
-
-        # Create tmp directory if it does not exists
-        tmp_dir = os.path.join("/tmp/", self.name)
-        if not os.path.exists(tmp_dir):
-            os.mkdir(tmp_dir, 0o700)
-
-        for f in self.files_to_keep:
-            dest_file = f.strip('/').replace("/", "_")
-
-            # If the file is already there, we might be re-running the migration
-            # because it previously crashed. Hence we keep the existing file.
-            if os.path.exists(os.path.join(tmp_dir, dest_file)):
-                continue
-
-            copy2(f, os.path.join(tmp_dir, dest_file))
-
-    def restore_files_to_keep(self):
-
-        logger.debug("Restoring specific files to keep ...")
-
-        tmp_dir = os.path.join("/tmp/", self.name)
-
-        for f in self.files_to_keep:
-            dest_file = f.strip('/').replace("/", "_")
-            copy2(os.path.join(tmp_dir, dest_file), f)
-
-    # On some setups, /etc/nginx/nginx.conf got edited. But this file needs
-    # to be upgraded because of the way the new module system works for nginx.
-    # (in particular, having the line that include the modules at the top)
-    #
-    # So here, if it got edited, we force the restore of the original conf
-    # *before* starting the actual upgrade...
-    #
-    # An alternative strategy that was attempted was to hold the nginx-common
-    # package and have a specific upgrade for it like for fail2ban, but that
-    # leads to apt complaining about not being able to upgrade for shitty
-    # reasons >.>
-    def restore_original_nginx_conf_if_needed(self):
-        if "/etc/nginx/nginx.conf" not in manually_modified_files_compared_to_debian_default():
-            return
-
-        if not os.path.exists("/etc/nginx/nginx.conf"):
-            return
-
-        # If stretch is in the sources.list, we already started migrating on
-        # stretch so we don't re-do this
-        if " stretch " in read_file("/etc/apt/sources.list"):
-            return
-
-        backup_dest = "/home/yunohost.conf/backup/nginx.conf.bkp_before_stretch"
-
-        logger.warning(m18n.n("migration_0003_restoring_origin_nginx_conf",
-                              backup_dest=backup_dest))
-
-        os.system("mv /etc/nginx/nginx.conf %s" % backup_dest)
-
-        command = ""
-        command += " DEBIAN_FRONTEND=noninteractive"
-        command += " APT_LISTCHANGES_FRONTEND=none"
-        command += " apt-get"
-        command += " --fix-broken --show-upgraded --assume-yes"
-        command += ' -o Dpkg::Options::="--force-confmiss"'
-        command += " install --reinstall"
-        command += " nginx-common"
-
-        logger.debug("Running apt command :\n{}".format(command))
-
-        command += " 2>&1 | tee -a {}".format(self.logfile)
-
-        is_api = msettings.get('interface') == 'api'
-        if is_api:
-            callbacks = (
-                lambda l: logger.info(l.rstrip()),
-                lambda l: logger.warning(l.rstrip()),
-            )
-            call_async_output(command, callbacks, shell=True)
-        else:
-            # We do this when running from the cli to have the output of the
-            # command showing in the terminal, since 'info' channel is only
-            # enabled if the user explicitly add --verbose ...
-            os.system(command)
-
-    def disable_predicable_interface_names(self):
-
-        # Try to see if currently used interface names are predictable ones or not...
-        # If we ain't using "eth0" or "wlan0", assume we are using predictable interface
-        # names and therefore they shouldnt be disabled
-        network_interfaces = get_network_interfaces().keys()
-        if "eth0" not in network_interfaces and "wlan0" not in network_interfaces:
-            return
-
-        interfaces_config = read_file("/etc/network/interfaces")
-        if "eth0" not in interfaces_config and "wlan0" not in interfaces_config:
-            return
-
-        # Disable predictive interface names
-        # c.f. https://unix.stackexchange.com/a/338730
-        os.system("ln -s /dev/null /etc/systemd/network/99-default.link")

src/yunohost/data_migrations/0004_php5_to_php7_pools.py

@@ -1,99 +0,0 @@
-import os
-import glob
-from shutil import copy2
-
-from moulinette.utils.log import getActionLogger
-
-from yunohost.tools import Migration
-from yunohost.service import _run_service_command
-
-logger = getActionLogger('yunohost.migration')
-
-PHP5_POOLS = "/etc/php5/fpm/pool.d"
-PHP7_POOLS = "/etc/php/7.0/fpm/pool.d"
-
-PHP5_SOCKETS_PREFIX = "/var/run/php5-fpm"
-PHP7_SOCKETS_PREFIX = "/run/php/php7.0-fpm"
-
-MIGRATION_COMMENT = "; YunoHost note : this file was automatically moved from {}".format(PHP5_POOLS)
-
-
-class MyMigration(Migration):
-
-    "Migrate php5-fpm 'pool' conf files to php7 stuff"
-
-    dependencies = ["migrate_to_stretch"]
-
-    def run(self):
-        # Get list of php5 pool files
-        php5_pool_files = glob.glob("{}/*.conf".format(PHP5_POOLS))
-
-        # Keep only basenames
-        php5_pool_files = [os.path.basename(f) for f in php5_pool_files]
-
-        # Ignore the "www.conf" (default stuff, probably don't want to touch it ?)
-        php5_pool_files = [f for f in php5_pool_files if f != "www.conf"]
-
-        for f in php5_pool_files:
-
-            # Copy the files to the php7 pool
-            src = "{}/{}".format(PHP5_POOLS, f)
-            dest = "{}/{}".format(PHP7_POOLS, f)
-            copy2(src, dest)
-
-            # Replace the socket prefix if it's found
-            c = "sed -i -e 's@{}@{}@g' {}".format(PHP5_SOCKETS_PREFIX, PHP7_SOCKETS_PREFIX, dest)
-            os.system(c)
-
-            # Also add a comment that it was automatically moved from php5
-            # (for human traceability and backward migration)
-            c = "sed -i '1i {}' {}".format(MIGRATION_COMMENT, dest)
-            os.system(c)
-
-            # Some old comments starting with '#' instead of ';' are not
-            # compatible in php7
-            c = "sed -i 's/^#/;#/g' {}".format(dest)
-            os.system(c)
-
-        # Reload/restart the php pools
-        _run_service_command("restart", "php7.0-fpm")
-        _run_service_command("enable", "php7.0-fpm")
-        os.system("systemctl stop php5-fpm")
-        os.system("systemctl disable php5-fpm")
-        os.system("rm /etc/logrotate.d/php5-fpm")  # We remove this otherwise the logrotate cron will be unhappy
-
-        # Get list of nginx conf file
-        nginx_conf_files = glob.glob("/etc/nginx/conf.d/*.d/*.conf")
-        for f in nginx_conf_files:
-            # Replace the socket prefix if it's found
-            c = "sed -i -e 's@{}@{}@g' {}".format(PHP5_SOCKETS_PREFIX, PHP7_SOCKETS_PREFIX, f)
-            os.system(c)
-
-        # Reload nginx
-        _run_service_command("reload", "nginx")
-
-    def backward(self):
-
-        # Get list of php7 pool files
-        php7_pool_files = glob.glob("{}/*.conf".format(PHP7_POOLS))
-
-        # Keep only files which have the migration comment
-        php7_pool_files = [f for f in php7_pool_files if open(f).readline().strip() == MIGRATION_COMMENT]
-
-        # Delete those files
-        for f in php7_pool_files:
-            os.remove(f)
-
-        # Reload/restart the php pools
-        _run_service_command("stop", "php7.0-fpm")
-        os.system("systemctl start php5-fpm")
-
-        # Get list of nginx conf file
-        nginx_conf_files = glob.glob("/etc/nginx/conf.d/*.d/*.conf")
-        for f in nginx_conf_files:
-            # Replace the socket prefix if it's found
-            c = "sed -i -e 's@{}@{}@g' {}".format(PHP7_SOCKETS_PREFIX, PHP5_SOCKETS_PREFIX, f)
-            os.system(c)
-
-        # Reload nginx
-        _run_service_command("reload", "nginx")

src/yunohost/data_migrations/0006_sync_admin_and_root_passwords.py

@@ -1,78 +0,0 @@
-import spwd
-import crypt
-import random
-import string
-import subprocess
-
-from moulinette import m18n
-from yunohost.utils.error import YunohostError
-from moulinette.utils.log import getActionLogger
-from moulinette.utils.process import run_commands, check_output
-from moulinette.utils.filesystem import append_to_file
-from moulinette.authenticators.ldap import Authenticator
-from yunohost.tools import Migration
-
-logger = getActionLogger('yunohost.migration')
-SMALL_PWD_LIST = ["yunohost", "olinuxino", "olinux", "raspberry", "admin", "root", "test", "rpi"]
-
-
-class MyMigration(Migration):
-
-    "Synchronize admin and root passwords"
-
-    def run(self):
-
-        new_hash = self._get_admin_hash()
-        self._replace_root_hash(new_hash)
-
-        logger.info(m18n.n("root_password_replaced_by_admin_password"))
-
-    @property
-    def mode(self):
-
-        # If the root password is still a "default" value,
-        # then this is an emergency and migration shall
-        # be applied automatically
-        #
-        # Otherwise, as playing with root password is touchy,
-        # we set this as a manual migration.
-        return "auto" if self._is_root_pwd_listed(SMALL_PWD_LIST) else "manual"
-
-    @property
-    def disclaimer(self):
-        if self._is_root_pwd_listed(SMALL_PWD_LIST):
-            return None
-
-        return m18n.n("migration_0006_disclaimer")
-
-    def _get_admin_hash(self):
-        """
-        Fetch the admin hash from the LDAP db using slapcat
-        """
-        admin_hash = check_output("slapcat \
-            | grep 'dn: cn=admin,dc=yunohost,dc=org' -A20 \
-            | grep userPassword -A2 \
-            | tr -d '\n ' \
-            | tr ':' ' ' \
-            | awk '{print $2}' \
-            | base64 -d \
-            | sed 's/{CRYPT}//g'")
-        return admin_hash
-
-    def _replace_root_hash(self, new_hash):
-        hash_root = spwd.getspnam("root").sp_pwd
-
-        with open('/etc/shadow', 'r') as before_file:
-            before = before_file.read()
-
-        with open('/etc/shadow', 'w') as after_file:
-            after_file.write(before.replace("root:" + hash_root,
-                                            "root:" + new_hash))
-
-    def _is_root_pwd_listed(self, pwd_list):
-        hash_root = spwd.getspnam("root").sp_pwd
-
-        for password in pwd_list:
-            if hash_root == crypt.crypt(password, hash_root):
-                return True
-        return False

src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py

@@ -1,70 +0,0 @@
-import os
-import re
-
-from shutil import copyfile
-
-from moulinette.utils.log import getActionLogger
-from moulinette.utils.filesystem import mkdir, rm
-
-from yunohost.tools import Migration
-from yunohost.service import _run_service_command
-from yunohost.regenconf import regen_conf
-from yunohost.settings import settings_set
-from yunohost.utils.error import YunohostError
-
-logger = getActionLogger('yunohost.migration')
-
-SSHD_CONF = '/etc/ssh/sshd_config'
-
-
-class MyMigration(Migration):
-
-    """
-    This is the first step of a couple of migrations that ensure SSH conf is
-    managed by YunoHost (even if the "from_script" flag is present, which was
-    previously preventing it from being managed by YunoHost)
-
-    The goal of this first (automatic) migration is to make sure that the
-    sshd_config is managed by the regen-conf mechanism.
-
-    If the from_script flag exists, then we keep the current SSH conf such that it
-    will appear as "manually modified" to the regenconf.
-
-    In step 2 (manual), the admin will be able to choose wether or not to actually
-    use the recommended configuration, with an appropriate disclaimer.
-    """
-
-    def run(self):
-
-        # Check if deprecated DSA Host Key is in config
-        dsa_rgx = r'^[ \t]*HostKey[ \t]+/etc/ssh/ssh_host_dsa_key[ \t]*(?:#.*)?$'
-        dsa = False
-        for line in open(SSHD_CONF):
-            if re.match(dsa_rgx, line) is not None:
-                dsa = True
-                break
-        if dsa:
-            settings_set("service.ssh.allow_deprecated_dsa_hostkey", True)
-
-        # Here, we make it so that /etc/ssh/sshd_config is managed
-        # by the regen conf (in particular in the case where the
-        # from_script flag is present - in which case it was *not*
-        # managed by the regenconf)
-        # But because we can't be sure the user wants to use the
-        # recommended conf, we backup then restore the /etc/ssh/sshd_config
-        # right after the regenconf, such that it will appear as
-        # "manually modified".
-        if os.path.exists('/etc/yunohost/from_script'):
-            rm('/etc/yunohost/from_script')
-            copyfile(SSHD_CONF, '/etc/ssh/sshd_config.bkp')
-            regen_conf(names=['ssh'], force=True)
-            copyfile('/etc/ssh/sshd_config.bkp', SSHD_CONF)
-
-        # Restart ssh and rollback if it failed
-        if not _run_service_command('restart', 'ssh'):
-            # We don't rollback completely but it should be enough
-            copyfile('/etc/ssh/sshd_config.bkp', SSHD_CONF)
-            if not _run_service_command('restart', 'ssh'):
-                raise YunohostError("migration_0007_cannot_restart")
-            else:
-                raise YunohostError("migration_0007_cancelled")

src/yunohost/data_migrations/0008_ssh_conf_managed_by_yunohost_step2.py

@@ -1,105 +0,0 @@
-import os
-import re
-
-from moulinette import m18n
-from moulinette.utils.log import getActionLogger
-from moulinette.utils.filesystem import chown
-
-from yunohost.tools import Migration
-from yunohost.regenconf import _get_conf_hashes, _calculate_hash
-from yunohost.regenconf import regen_conf
-from yunohost.settings import settings_set, settings_get
-from yunohost.utils.error import YunohostError
-from yunohost.backup import ARCHIVES_PATH
-
-
-logger = getActionLogger('yunohost.migration')
-
-SSHD_CONF = '/etc/ssh/sshd_config'
-
-
-class MyMigration(Migration):
-
-    """
-    In this second step, the admin is asked if it's okay to use
-    the recommended SSH configuration - which also implies
-    disabling deprecated DSA key.
-
-    This has important implications in the way the user may connect
-    to its server (key change, and a spooky warning might be given
-    by SSH later)
-
-    A disclaimer explaining the various things to be aware of is
-    shown - and the user may also choose to skip this migration.
-    """
-
-    dependencies = ["ssh_conf_managed_by_yunohost_step1"]
-
-    def run(self):
-        settings_set("service.ssh.allow_deprecated_dsa_hostkey", False)
-        regen_conf(names=['ssh'], force=True)
-
-        # Update local archives folder permissions, so that
-        # admin can scp archives out of the server
-        if os.path.isdir(ARCHIVES_PATH):
-            chown(ARCHIVES_PATH, uid="admin", gid="root")
-
-    @property
-    def mode(self):
-
-        # If the conf is already up to date
-        # and no DSA key is used, then we're good to go
-        # and the migration can be done automatically
-        # (basically nothing shall change)
-        ynh_hash = _get_conf_hashes('ssh').get(SSHD_CONF, None)
-        current_hash = _calculate_hash(SSHD_CONF)
-        dsa = settings_get("service.ssh.allow_deprecated_dsa_hostkey")
-        if ynh_hash == current_hash and not dsa:
-            return "auto"
-
-        return "manual"
-
-    @property
-    def disclaimer(self):
-
-        if self.mode == "auto":
-            return None
-
-        # Detect key things to be aware of before enabling the
-        # recommended configuration
-        dsa_key_enabled = False
-        ports = []
-        root_login = []
-        port_rgx = r'^[ \t]*Port[ \t]+(\d+)[ \t]*(?:#.*)?$'
-        root_rgx = r'^[ \t]*PermitRootLogin[ \t]([^# \t]*)[ \t]*(?:#.*)?$'
-        dsa_rgx = r'^[ \t]*HostKey[ \t]+/etc/ssh/ssh_host_dsa_key[ \t]*(?:#.*)?$'
-        for line in open(SSHD_CONF):
-
-            ports = ports + re.findall(port_rgx, line)
-
-            root_login = root_login + re.findall(root_rgx, line)
-
-            if not dsa_key_enabled and re.match(dsa_rgx, line) is not None:
-                dsa_key_enabled = True
-
-        custom_port = ports != ['22'] and ports != []
-        root_login_enabled = root_login and root_login[-1] != 'no'
-
-        # Build message
-        message = m18n.n("migration_0008_general_disclaimer")
-
-        if custom_port:
-            message += "\n\n" + m18n.n("migration_0008_port")
-
-        if root_login_enabled:
-            message += "\n\n" + m18n.n("migration_0008_root")
-
-        if dsa_key_enabled:
-            message += "\n\n" + m18n.n("migration_0008_dsa")
-
-        if custom_port or root_login_enabled or dsa_key_enabled:
-            message += "\n\n" + m18n.n("migration_0008_warning")
-        else:
-            message += "\n\n" + m18n.n("migration_0008_no_warning")
-
-        return message

src/yunohost/data_migrations/0009_decouple_regenconf_from_services.py

@@ -1,39 +0,0 @@
-import os
-
-from moulinette import m18n
-from moulinette.utils.log import getActionLogger
-
-from moulinette.utils.filesystem import read_file
-from yunohost.service import _get_services, _save_services
-from yunohost.regenconf import _update_conf_hashes, REGEN_CONF_FILE
-
-from yunohost.tools import Migration
-
-logger = getActionLogger('yunohost.migration')
-
-
-class MyMigration(Migration):
-    """
-    Decouple the regen conf mechanism from the concept of services
-    """
-
-    def run(self):
-
-        if "conffiles" not in read_file("/etc/yunohost/services.yml") \
-           or os.path.exists(REGEN_CONF_FILE):
-            logger.warning(m18n.n("migration_0009_not_needed"))
-            return
-
-        # For all services
-        services = _get_services()
-        for service, infos in services.items():
-            # If there are some conffiles (file hashes)
-            if "conffiles" in infos.keys():
-                # Save them using the new regen conf thingy
-                _update_conf_hashes(service, infos["conffiles"])
-                # And delete the old conffile key from the service infos
-                del services[service]["conffiles"]
-
-        # (Actually save the modification of services)
-        _save_services(services)
-

src/yunohost/data_migrations/0010_migrate_to_apps_json.py

@@ -1,13 +0,0 @@
-from moulinette.utils.log import getActionLogger
-from yunohost.tools import Migration
-
-logger = getActionLogger('yunohost.migration')
-
-
-class MyMigration(Migration):
-
-    "Migrate from official.json to apps.json (outdated, replaced by migration 13)"
-
-    def run(self):
-        logger.info("This migration is oudated and doesn't do anything anymore. The migration 13 will handle this instead.")
-        pass

src/yunohost/data_migrations/0012_postgresql_password_to_md5_authentication.py

@@ -1,16 +0,0 @@
-import glob
-import re
-from yunohost.tools import Migration
-from moulinette.utils.filesystem import read_file, write_to_file
-
-
-class MyMigration(Migration):
-
-    "Force authentication in md5 for local connexions"
-
-    all_hba_files = glob.glob("/etc/postgresql/*/*/pg_hba.conf")
-
-    def run(self):
-        for filename in self.all_hba_files:
-            pg_hba_in = read_file(filename)
-            write_to_file(filename, re.sub(r"local(\s*)all(\s*)all(\s*)password", "local\\1all\\2all\\3md5", pg_hba_in))

src/yunohost/data_migrations/0013_futureproof_apps_catalog_system.py

@@ -1,51 +0,0 @@
-
-import os
-import shutil
-
-from moulinette.utils.log import getActionLogger
-from moulinette.utils.filesystem import read_json
-
-from yunohost.tools import Migration
-from yunohost.app import (_initialize_apps_catalog_system,
-                          _update_apps_catalog,
-                          APPS_CATALOG_CACHE,
-                          APPS_CATALOG_CONF)
-
-logger = getActionLogger('yunohost.migration')
-
-LEGACY_APPS_CATALOG_CONF = '/etc/yunohost/appslists.json'
-LEGACY_APPS_CATALOG_CONF_BACKUP = LEGACY_APPS_CATALOG_CONF + ".old"
-
-
-class MyMigration(Migration):
-
-    "Migrate to the new future-proof apps catalog system"
-
-    def run(self):
-
-        if not os.path.exists(LEGACY_APPS_CATALOG_CONF):
-            logger.info("No need to do anything")
-
-        # Destroy old lecacy cache
-        if os.path.exists(APPS_CATALOG_CACHE):
-            shutil.rmtree(APPS_CATALOG_CACHE)
-
-        # and legacy cron
-        if os.path.exists("/etc/cron.daily/yunohost-fetch-appslists"):
-            os.remove("/etc/cron.daily/yunohost-fetch-appslists")
-
-        # Backup the legacy file
-        try:
-            legacy_catalogs = read_json(LEGACY_APPS_CATALOG_CONF)
-            # If there's only one catalog, we assume it's just the old official catalog
-            # Otherwise, warn the (power-?)users that they should migrate their old catalogs manually
-            if len(legacy_catalogs) > 1:
-                logger.warning("It looks like you had additional apps_catalog in the configuration file %s! YunoHost now uses %s instead, but it won't migrate your custom apps_catalog. You should do this manually. The old file has been backuped in %s." % (LEGACY_APPS_CATALOG_CONF, APPS_CATALOG_CONF, LEGACY_APPS_CATALOG_CONF_BACKUP))
-        except Exception as e:
-            logger.warning("Unable to parse the legacy conf %s (error : %s) ... migrating anyway" % (LEGACY_APPS_CATALOG_CONF, str(e)))
-
-        if os.path.exists(LEGACY_APPS_CATALOG_CONF):
-            os.rename(LEGACY_APPS_CATALOG_CONF, LEGACY_APPS_CATALOG_CONF_BACKUP)
-
-        _initialize_apps_catalog_system()
-        _update_apps_catalog()

src/yunohost/data_migrations/0014_remove_app_status_json.py

@@ -1,31 +0,0 @@
-import os
-
-from moulinette.utils.log import getActionLogger
-from moulinette.utils.filesystem import read_json
-
-from yunohost.tools import Migration
-from yunohost.app import app_setting, APPS_SETTING_PATH
-
-logger = getActionLogger('yunohost.migration')
-
-class MyMigration(Migration):
-
-    """Remove legacy app status.json files"""
-
-    def run(self):
-
-        apps = os.listdir(APPS_SETTING_PATH)
-
-        for app in apps:
-            status_file = os.path.join(APPS_SETTING_PATH, app, "status.json")
-            if not os.path.exists(status_file):
-                continue
-
-            try:
-                status = read_json(status_file)
-                current_revision = status.get("remote", {}).get("revision", "?")
-                app_setting(app, 'current_revision', current_revision)
-            except Exception as e:
-                logger.warning("Could not migrate status.json from app %s: %s", (app, str(e)))
-            else:
-                os.system("rm %s" % status_file)

src/yunohost/data_migrations/0016_php70_to_php73_pools.py

@@ -0,0 +1,73 @@
+import os
+import glob
+from shutil import copy2
+
+from moulinette.utils.log import getActionLogger
+
+from yunohost.app import _is_installed, _get_app_settings, _set_app_settings, _patch_legacy_php_versions_in_settings
+from yunohost.tools import Migration
+from yunohost.service import _run_service_command
+
+logger = getActionLogger('yunohost.migration')
+
+PHP70_POOLS = "/etc/php/7.0/fpm/pool.d"
+PHP73_POOLS = "/etc/php/7.3/fpm/pool.d"
+
+PHP70_SOCKETS_PREFIX = "/run/php/php7.0-fpm"
+PHP73_SOCKETS_PREFIX = "/run/php/php7.3-fpm"
+
+MIGRATION_COMMENT = "; YunoHost note : this file was automatically moved from {}".format(PHP70_POOLS)
+
+
+class MyMigration(Migration):
+
+    "Migrate php7.0-fpm 'pool' conf files to php7.3"
+
+    dependencies = ["migrate_to_buster"]
+
+    def run(self):
+        # Get list of php7.0 pool files
+        php70_pool_files = glob.glob("{}/*.conf".format(PHP70_POOLS))
+
+        # Keep only basenames
+        php70_pool_files = [os.path.basename(f) for f in php70_pool_files]
+
+        # Ignore the "www.conf" (default stuff, probably don't want to touch it ?)
+        php70_pool_files = [f for f in php70_pool_files if f != "www.conf"]
+
+        for f in php70_pool_files:
+
+            # Copy the files to the php7.3 pool
+            src = "{}/{}".format(PHP70_POOLS, f)
+            dest = "{}/{}".format(PHP73_POOLS, f)
+            copy2(src, dest)
+
+            # Replace the socket prefix if it's found
+            c = "sed -i -e 's@{}@{}@g' {}".format(PHP70_SOCKETS_PREFIX, PHP73_SOCKETS_PREFIX, dest)
+            os.system(c)
+
+            # Also add a comment that it was automatically moved from php7.0
+            # (for human traceability and backward migration)
+            c = "sed -i '1i {}' {}".format(MIGRATION_COMMENT, dest)
+            os.system(c)
+
+            app_id = os.path.basename(f)[:-len(".conf")]
+            if _is_installed(app_id):
+                _patch_legacy_php_versions_in_settings("/etc/yunohost/apps/%s/" % app_id)
+
+            nginx_conf_files = glob.glob("/etc/nginx/conf.d/*.d/%s.conf" % app_id)
+            for f in nginx_conf_files:
+                # Replace the socket prefix if it's found
+                c = "sed -i -e 's@{}@{}@g' {}".format(PHP70_SOCKETS_PREFIX, PHP73_SOCKETS_PREFIX, f)
+                os.system(c)
+
+        os.system("rm /etc/logrotate.d/php7.0-fpm")  # We remove this otherwise the logrotate cron will be unhappy
+
+        # Reload/restart the php pools
+        _run_service_command("restart", "php7.3-fpm")
+        _run_service_command("enable", "php7.3-fpm")
+        os.system("systemctl stop php7.0-fpm")
+        os.system("systemctl disable php7.0-fpm")
+
+        # Reload nginx
+        _run_service_command("reload", "nginx")

src/yunohost/data_migrations/0017_postgresql_9p6_to_11.py

@@ -12,27 +12,27 @@ logger = getActionLogger('yunohost.migration')
 
 class MyMigration(Migration):
 
-    "Migrate DBs from Postgresql 9.4 to 9.6 after migrating to Stretch"
+    "Migrate DBs from Postgresql 9.6 to 11 after migrating to Buster"
 
-    dependencies = ["migrate_to_stretch"]
+    dependencies = ["migrate_to_buster"]
 
     def run(self):
 
-        if not self.package_is_installed("postgresql-9.4"):
-            logger.warning(m18n.n("migration_0005_postgresql_94_not_installed"))
+        if not self.package_is_installed("postgresql-9.6"):
+            logger.warning(m18n.n("migration_0017_postgresql_96_not_installed"))
             return
 
-        if not self.package_is_installed("postgresql-9.6"):
-            raise YunohostError("migration_0005_postgresql_96_not_installed")
+        if not self.package_is_installed("postgresql-11"):
+            raise YunohostError("migration_0017_postgresql_11_not_installed")
 
-        if not space_used_by_directory("/var/lib/postgresql/9.4") > free_space_in_directory("/var/lib/postgresql"):
-            raise YunohostError("migration_0005_not_enough_space", path="/var/lib/postgresql/")
+        if not space_used_by_directory("/var/lib/postgresql/9.6") > free_space_in_directory("/var/lib/postgresql"):
+            raise YunohostError("migration_0017_not_enough_space", path="/var/lib/postgresql/")
 
-        subprocess.check_call("service postgresql stop", shell=True)
+        subprocess.check_call("systemctl stop postgresql", shell=True)
+        subprocess.check_call("pg_dropcluster --stop 11 main", shell=True)
+        subprocess.check_call("pg_upgradecluster -m upgrade 9.6 main", shell=True)
         subprocess.check_call("pg_dropcluster --stop 9.6 main", shell=True)
-        subprocess.check_call("pg_upgradecluster -m upgrade 9.4 main", shell=True)
-        subprocess.check_call("pg_dropcluster --stop 9.4 main", shell=True)
-        subprocess.check_call("service postgresql start", shell=True)
+        subprocess.check_call("systemctl start postgresql", shell=True)
 
     def package_is_installed(self, package_name):
 

src/yunohost/data_migrations/0018_xtable_to_nftable.py

@@ -0,0 +1,92 @@
+import os
+import subprocess
+
+from moulinette import m18n
+from yunohost.utils.error import YunohostError
+from moulinette.utils.log import getActionLogger
+
+from yunohost.firewall import firewall_reload
+from yunohost.service import service_restart
+from yunohost.tools import Migration
+
+logger = getActionLogger('yunohost.migration')
+
+
+class MyMigration(Migration):
+
+    "Migrate legacy iptables rules from stretch that relied on xtable and should now rely on nftable"
+
+    dependencies = ["migrate_to_buster"]
+
+    def run(self):
+
+        self.do_ipv4 = os.system("iptables -w -L >/dev/null") == 0
+        self.do_ipv6 = os.system("ip6tables -w -L >/dev/null") == 0
+
+        if not self.do_ipv4:
+            logger.warning(m18n.n('iptables_unavailable'))
+        if not self.do_ipv6:
+            logger.warning(m18n.n('ip6tables_unavailable'))
+
+        backup_folder = "/home/yunohost.backup/premigration/xtable_to_nftable/"
+        if not os.path.exists(backup_folder):
+            os.makedirs(backup_folder, 0o750)
+        self.backup_rules_ipv4 = os.path.join(backup_folder, "legacy_rules_ipv4")
+        self.backup_rules_ipv6 = os.path.join(backup_folder, "legacy_rules_ipv6")
+
+        # Backup existing legacy rules to be able to rollback
+        if self.do_ipv4 and not os.path.exists(self.backup_rules_ipv4):
+            os.system("iptables-legacy -L >/dev/null")  # For some reason if we don't do this, iptables-legacy-save is empty ?
+            subprocess.check_call("iptables-legacy-save > %s" % self.backup_rules_ipv4, shell=True)
+            assert subprocess.check_output("cat %s" % self.backup_rules_ipv4, shell=True).strip(), "Uhoh backup of legacy ipv4 rules is empty !?"
+        if self.do_ipv6 and not os.path.exists(self.backup_rules_ipv6):
+            os.system("ip6tables-legacy -L >/dev/null")  # For some reason if we don't do this, iptables-legacy-save is empty ?
+            subprocess.check_call("ip6tables-legacy-save > %s" % self.backup_rules_ipv6, shell=True)
+            assert subprocess.check_output("cat %s" % self.backup_rules_ipv6, shell=True).strip(), "Uhoh backup of legacy ipv6 rules is empty !?"
+
+        # We inject the legacy rules (iptables-legacy) into the new iptable (just "iptables")
+        try:
+            if self.do_ipv4:
+                subprocess.check_call("iptables-legacy-save | iptables-restore", shell=True)
+            if self.do_ipv6:
+                subprocess.check_call("ip6tables-legacy-save | ip6tables-restore", shell=True)
+        except Exception as e:
+            self.rollback()
+            raise YunohostError("migration_0018_failed_to_migrate_iptables_rules", error=e)
+
+        # Reset everything in iptables-legacy
+        # Stolen from https://serverfault.com/a/200642
+        try:
+            if self.do_ipv4:
+                subprocess.check_call(
+                    "iptables-legacy-save | awk '/^[*]/ { print $1 }"                         # Keep lines like *raw, *filter and *nat
+                    "                            /^:[A-Z]+ [^-]/ { print $1 \" ACCEPT\" ; }"  # Turn all policies to accept
+                    "                            /COMMIT/ { print $0; }'"                     # Keep the line COMMIT
+                    " | iptables-legacy-restore",
+                    shell=True)
+            if self.do_ipv6:
+                subprocess.check_call(
+                    "ip6tables-legacy-save | awk '/^[*]/ { print $1 }"                        # Keep lines like *raw, *filter and *nat
+                    "                            /^:[A-Z]+ [^-]/ { print $1 \" ACCEPT\" ; }"  # Turn all policies to accept
+                    "                            /COMMIT/ { print $0; }'"                     # Keep the line COMMIT
+                    " | ip6tables-legacy-restore",
+                    shell=True)
+        except Exception as e:
+            self.rollback()
+            raise YunohostError("migration_0018_failed_to_reset_legacy_rules", error=e)
+
+        # You might be wondering "uh but is it really useful to
+        # iptables-legacy-save | iptables-restore considering firewall_reload()
+        # flush/resets everything anyway ?"
+        # But the answer is : firewall_reload() only resets the *filter table.
+        # On more complex setups (e.g. internet cube or docker) you will also
+        # have rules in the *nat (or maybe *raw?) sections of iptables.
+        firewall_reload()
+        service_restart("fail2ban")
+
+    def rollback(self):
+
+        if self.do_ipv4:
+            subprocess.check_call("iptables-legacy-restore < %s" % self.backup_rules_ipv4, shell=True)
+        if self.do_ipv6:
+            subprocess.check_call("iptables-legacy-restore < %s" % self.backup_rules_ipv6, shell=True)

src/yunohost/dyndns.py

@@ -205,22 +205,6 @@ def dyndns_update(operation_logger, dyn_host="dyndns.yunohost.org", domain=None,
 
             key = keys[0]
 
-    # This mean that hmac-md5 is used
-    # (Re?)Trigger the migration to sha256 and return immediately.
-    # The actual update will be done in next run.
-    if "+157" in key:
-        from yunohost.tools import _get_migration_by_name
-        migration = _get_migration_by_name("migrate_to_tsig_sha256")
-        try:
-            migration.run(dyn_host, domain, key)
-        except Exception as e:
-            logger.error(m18n.n('migrations_migration_has_failed',
-                                exception=e,
-                                number=migration.number,
-                                name=migration.name),
-                         exc_info=1)
-        return
-
     # Extract 'host', e.g. 'nohost.me' from 'foo.nohost.me'
     host = domain.split('.')[1:]
     host = '.'.join(host)

src/yunohost/firewall.py

@@ -26,11 +26,7 @@
 import os
 import sys
 import yaml
-try:
-    import miniupnpc
-except ImportError:
-    sys.stderr.write('Error: Yunohost CLI Require miniupnpc lib\n')
-    sys.exit(1)
+import miniupnpc
 
 from moulinette import m18n
 from yunohost.utils.error import YunohostError

src/yunohost/regenconf.py

@@ -62,16 +62,6 @@ def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run
 
     """
 
-    # Legacy code to automatically run the migration
-    # This is required because regen_conf is called before the migration call
-    # in debian's postinst script
-    if os.path.exists("/etc/yunohost/installed") \
-       and ("conffiles" in read_file("/etc/yunohost/services.yml") \
-            or not os.path.exists(REGEN_CONF_FILE)):
-        from yunohost.tools import _get_migration_by_name
-        migration = _get_migration_by_name("decouple_regenconf_from_services")
-        migration.run()
-
     result = {}
 
     # Return the list of pending conf

src/yunohost/service.py

@@ -618,7 +618,7 @@ def _get_services():
     if "postgresql" in services:
         if "description" in services["postgresql"]:
             del services["postgresql"]["description"]
-        services["postgresql"]["actual_systemd_service"] = "postgresql@9.6-main"
+        services["postgresql"]["actual_systemd_service"] = "postgresql@11-main"
 
     return services
 

src/yunohost/ssh.py

@@ -25,7 +25,7 @@ def user_ssh_allow(username):
 
     from yunohost.utils.ldap import _get_ldap_interface
     ldap = _get_ldap_interface()
-    ldap.update('uid=%s,ou=users' % username, {'loginShell': '/bin/bash'})
+    ldap.update('uid=%s,ou=users' % username, {'loginShell': ['/bin/bash']})
 
     # Somehow this is needed otherwise the PAM thing doesn't forget about the
     # old loginShell value ?
@@ -46,7 +46,7 @@ def user_ssh_disallow(username):
 
     from yunohost.utils.ldap import _get_ldap_interface
     ldap = _get_ldap_interface()
-    ldap.update('uid=%s,ou=users' % username, {'loginShell': '/bin/false'})
+    ldap.update('uid=%s,ou=users' % username, {'loginShell': ['/bin/false']})
 
     # Somehow this is needed otherwise the PAM thing doesn't forget about the
     # old loginShell value ?

src/yunohost/tests/test_appscatalog.py

@@ -315,58 +315,3 @@ def test_apps_catalog_load_with_oudated_api_version(mocker):
     for cache_file in glob.glob(APPS_CATALOG_CACHE + "/*"):
         cache_json = read_json(cache_file)
         assert cache_json["from_api_version"] == APPS_CATALOG_API_VERSION
-
-
-
-def test_apps_catalog_migrate_legacy_explicitly():
-
-    open("/etc/yunohost/appslists.json", "w").write('{"yunohost": {"yolo":"swag"}}')
-    mkdir(APPS_CATALOG_CACHE, 0o750, parents=True)
-    open(APPS_CATALOG_CACHE+"/yunohost_old.json", "w").write('{"foo":{}, "bar": {}}')
-    open(APPS_CATALOG_CRON_PATH, "w").write("# Some old cron")
-
-    from yunohost.tools import _get_migration_by_name
-    migration = _get_migration_by_name("futureproof_apps_catalog_system")
-
-    with requests_mock.Mocker() as m:
-
-        # Mock the server response with a dummy apps catalog
-        m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG)
-        migration.run()
-
-    # Old conf shouldnt be there anymore (got renamed to .old)
-    assert not os.path.exists("/etc/yunohost/appslists.json")
-    # Old cache should have been removed
-    assert not os.path.exists(APPS_CATALOG_CACHE+"/yunohost_old.json")
-    # Cron should have been changed
-    assert "/bin/bash" in open(APPS_CATALOG_CRON_PATH, "r").read()
-    assert cron_job_is_there()
-
-    # Reading the apps_catalog should work
-    app_dict = _load_apps_catalog()["apps"]
-    assert "foo" in app_dict.keys()
-    assert "bar" in app_dict.keys()
-
-
-def test_apps_catalog_migrate_legacy_implicitly():
-
-    open("/etc/yunohost/appslists.json", "w").write('{"yunohost": {"yolo":"swag"}}')
-    mkdir(APPS_CATALOG_CACHE, 0o750, parents=True)
-    open(APPS_CATALOG_CACHE+"/yunohost_old.json", "w").write('{"old_foo":{}, "old_bar": {}}')
-    open(APPS_CATALOG_CRON_PATH, "w").write("# Some old cron")
-
-    with requests_mock.Mocker() as m:
-        m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG)
-        app_dict = _load_apps_catalog()["apps"]
-
-    assert "foo" in app_dict.keys()
-    assert "bar" in app_dict.keys()
-
-    # Old conf shouldnt be there anymore (got renamed to .old)
-    assert not os.path.exists("/etc/yunohost/appslists.json")
-    # Old cache should have been removed
-    assert not os.path.exists(APPS_CATALOG_CACHE+"/yunohost_old.json")
-    # Cron should have been changed
-    assert "/bin/bash" in open(APPS_CATALOG_CRON_PATH, "r").read()
-    assert cron_job_is_there()
-

src/yunohost/tools.py

@@ -88,15 +88,15 @@ def tools_ldapinit():
             logger.warn("Error when trying to inject '%s' -> '%s' into ldap: %s" % (rdn, attr_dict, e))
 
     admin_dict = {
-        'cn': 'admin',
-        'uid': 'admin',
-        'description': 'LDAP Administrator',
-        'gidNumber': '1007',
-        'uidNumber': '1007',
-        'homeDirectory': '/home/admin',
-        'loginShell': '/bin/bash',
+        'cn': ['admin'],
+        'uid': ['admin'],
+        'description': ['LDAP Administrator'],
+        'gidNumber': ['1007'],
+        'uidNumber': ['1007'],
+        'homeDirectory': ['/home/admin'],
+        'loginShell': ['/bin/bash'],
         'objectClass': ['organizationalRole', 'posixAccount', 'simpleSecurityObject'],
-        'userPassword': 'yunohost'
+        'userPassword': ['yunohost']
     }
 
     ldap.update('cn=admin', admin_dict)
@@ -140,7 +140,7 @@ def tools_adminpw(new_password, check_strength=True):
     ldap = _get_ldap_interface()
 
     try:
-        ldap.update("cn=admin", {"userPassword": new_hash, })
+        ldap.update("cn=admin", {"userPassword": [ new_hash ], })
     except:
         logger.exception('unable to change admin password')
         raise YunohostError('admin_password_change_failed')

src/yunohost/user.py

@@ -178,19 +178,19 @@ def user_create(operation_logger, username, firstname, lastname, mail, password,
     fullname = '%s %s' % (firstname, lastname)
     attr_dict = {
         'objectClass': ['mailAccount', 'inetOrgPerson', 'posixAccount', 'userPermissionYnh'],
-        'givenName': firstname,
-        'sn': lastname,
-        'displayName': fullname,
-        'cn': fullname,
-        'uid': username,
-        'mail': mail,
-        'maildrop': username,
-        'mailuserquota': mailbox_quota,
-        'userPassword': _hash_user_password(password),
-        'gidNumber': uid,
-        'uidNumber': uid,
-        'homeDirectory': '/home/' + username,
-        'loginShell': '/bin/false'
+        'givenName': [firstname],
+        'sn': [lastname],
+        'displayName': [fullname],
+        'cn': [fullname],
+        'uid': [username],
+        'mail': mail,  # NOTE: this one seems to be already a list
+        'maildrop': [username],
+        'mailuserquota': [mailbox_quota],
+        'userPassword': [_hash_user_password(password)],
+        'gidNumber': [uid],
+        'uidNumber': [uid],
+        'homeDirectory': ['/home/' + username],
+        'loginShell': ['/bin/false']
     }
 
     # If it is the first user, add some aliases
@@ -208,8 +208,7 @@ def user_create(operation_logger, username, firstname, lastname, mail, password,
 
     try:
         # Attempt to create user home folder
-        subprocess.check_call(
-            ['su', '-', username, '-c', "''"])
+        subprocess.check_call(["mkhomedir_helper", username])
     except subprocess.CalledProcessError:
         if not os.path.isdir('/home/{0}'.format(username)):
             logger.warning(m18n.n('user_home_creation_failed'),
@@ -317,21 +316,21 @@ def user_update(operation_logger, username, firstname=None, lastname=None, mail=
     # Get modifications from arguments
     new_attr_dict = {}
     if firstname:
-        new_attr_dict['givenName'] = firstname  # TODO: Validate
-        new_attr_dict['cn'] = new_attr_dict['displayName'] = firstname + ' ' + user['sn'][0]
+        new_attr_dict['givenName'] = [firstname]  # TODO: Validate
+        new_attr_dict['cn'] = new_attr_dict['displayName'] = [firstname + ' ' + user['sn'][0]]
 
     if lastname:
-        new_attr_dict['sn'] = lastname  # TODO: Validate
-        new_attr_dict['cn'] = new_attr_dict['displayName'] = user['givenName'][0] + ' ' + lastname
+        new_attr_dict['sn'] = [lastname]  # TODO: Validate
+        new_attr_dict['cn'] = new_attr_dict['displayName'] = [user['givenName'][0] + ' ' + lastname]
 
     if lastname and firstname:
-        new_attr_dict['cn'] = new_attr_dict['displayName'] = firstname + ' ' + lastname
+        new_attr_dict['cn'] = new_attr_dict['displayName'] = [firstname + ' ' + lastname]
 
     if change_password:
         # Ensure sufficiently complex password
         assert_password_is_strong_enough("user", change_password)
 
-        new_attr_dict['userPassword'] = _hash_user_password(change_password)
+        new_attr_dict['userPassword'] = [_hash_user_password(change_password)]
 
     if mail:
         main_domain = _get_maindomain()
@@ -396,7 +395,7 @@ def user_update(operation_logger, username, firstname=None, lastname=None, mail=
         new_attr_dict['maildrop'] = user['maildrop']
 
     if mailbox_quota is not None:
-        new_attr_dict['mailuserquota'] = mailbox_quota
+        new_attr_dict['mailuserquota'] = [mailbox_quota]
 
     operation_logger.start()
 

src/yunohost/utils/legacy.py

@@ -1,34 +1,19 @@
-import time
-import os
-
 from moulinette import m18n
 from yunohost.utils.error import YunohostError
 from moulinette.utils.log import getActionLogger
 from moulinette.utils.filesystem import read_yaml
 
-from yunohost.tools import Migration
 from yunohost.user import user_list, user_group_create, user_group_update
 from yunohost.app import app_setting, _installed_apps
-from yunohost.regenconf import regen_conf, BACKUP_CONF_DIR
 from yunohost.permission import permission_create, user_permission_update, permission_sync_to_user
 
-logger = getActionLogger('yunohost.migration')
-
-###################################################
-# Tools used also for restoration
-###################################################
+logger = getActionLogger('yunohost.legacy')
 
 
-class MyMigration(Migration):
-    """
-        Update the LDAP DB to be able to store the permission
-        Create a group for each yunohost user
-        Migrate app permission from apps setting to LDAP
-    """
+class SetupGroupPermissions():
 
-    required = True
-
-    def remove_if_exists(self, target):
+    @staticmethod
+    def remove_if_exists(target):
 
         from yunohost.utils.ldap import _get_ldap_interface
         ldap = _get_ldap_interface()
@@ -50,7 +35,8 @@ class MyMigration(Migration):
                 except Exception as e:
                     raise YunohostError("migration_0011_failed_to_remove_stale_object", dn=dn, error=e)
 
-    def migrate_LDAP_db(self):
+    @staticmethod
+    def migrate_LDAP_db():
 
         logger.info(m18n.n("migration_0011_update_LDAP_database"))
 
@@ -60,8 +46,8 @@ class MyMigration(Migration):
         ldap_map = read_yaml('/usr/share/yunohost/yunohost-config/moulinette/ldap_scheme.yml')
 
         try:
-            self.remove_if_exists("ou=permission")
-            self.remove_if_exists('ou=groups')
+            SetupGroupPermissions.remove_if_exists("ou=permission")
+            SetupGroupPermissions.remove_if_exists('ou=groups')
 
             attr_dict = ldap_map['parents']['ou=permission']
             ldap.add('ou=permission', attr_dict)
@@ -93,7 +79,8 @@ class MyMigration(Migration):
             user_group_create(username, gid=user_info['uidNumber'][0], primary_group=True, sync_perm=False)
             user_group_update(groupname='all_users', add=username, force=True, sync_perm=False)
 
-    def migrate_app_permission(self, app=None):
+    @staticmethod
+    def migrate_app_permission(app=None):
         logger.info(m18n.n("migration_0011_migrate_permission"))
 
         apps = _installed_apps()
@@ -116,66 +103,12 @@ class MyMigration(Migration):
                 allowed = [user for user in permission.split(',') if user in known_users]
             else:
                 allowed = ["all_users"]
-            permission_create(app+".main", url=url, allowed=allowed, sync_perm=False)
+            permission_create(app + ".main", url=url, allowed=allowed, sync_perm=False)
 
             app_setting(app, 'allowed_users', delete=True)
 
             # Migrate classic public app still using the legacy unprotected_uris
             if app_setting(app, "unprotected_uris") == "/" or app_setting(app, "skipped_uris") == "/":
-                user_permission_update(app+".main", add="visitors", sync_perm=False)
+                user_permission_update(app + ".main", add="visitors", sync_perm=False)
 
         permission_sync_to_user()
-
-    def run(self):
-
-        # FIXME : what do we really want to do here ...
-        # Imho we should just force-regen the conf in all case, and maybe
-        # just display a warning if we detect that the conf was manually modified
-
-        # Check if the migration can be processed
-        ldap_regen_conf_status = regen_conf(names=['slapd'], dry_run=True)
-        # By this we check if the have been customized
-        if ldap_regen_conf_status and ldap_regen_conf_status['slapd']['pending']:
-            logger.warning(m18n.n("migration_0011_slapd_config_will_be_overwritten", conf_backup_folder=BACKUP_CONF_DIR))
-
-        # Backup LDAP and the apps settings before to do the migration
-        logger.info(m18n.n("migration_0011_backup_before_migration"))
-        try:
-            backup_folder = "/home/yunohost.backup/premigration/" + time.strftime('%Y%m%d-%H%M%S', time.gmtime())
-            os.makedirs(backup_folder, 0o750)
-            os.system("systemctl stop slapd")
-            os.system("cp -r --preserve /etc/ldap %s/ldap_config" % backup_folder)
-            os.system("cp -r --preserve /var/lib/ldap %s/ldap_db" % backup_folder)
-            os.system("cp -r --preserve /etc/yunohost/apps %s/apps_settings" % backup_folder)
-        except Exception as e:
-            raise YunohostError("migration_0011_can_not_backup_before_migration", error=e)
-        finally:
-            os.system("systemctl start slapd")
-
-        try:
-            # Update LDAP schema restart slapd
-            logger.info(m18n.n("migration_0011_update_LDAP_schema"))
-            regen_conf(names=['slapd'], force=True)
-
-            # Update LDAP database
-            self.migrate_LDAP_db()
-
-            # Migrate permission
-            self.migrate_app_permission()
-
-            permission_sync_to_user()
-        except Exception as e:
-            logger.warn(m18n.n("migration_0011_migration_failed_trying_to_rollback"))
-            os.system("systemctl stop slapd")
-            os.system("rm -r /etc/ldap/slapd.d")  # To be sure that we don't keep some part of the old config
-            os.system("cp -r --preserve %s/ldap_config/. /etc/ldap/" % backup_folder)
-            os.system("cp -r --preserve %s/ldap_db/. /var/lib/ldap/" % backup_folder)
-            os.system("cp -r --preserve %s/apps_settings/. /etc/yunohost/apps/" % backup_folder)
-            os.system("systemctl start slapd")
-            os.system("rm -r " + backup_folder)
-            logger.info(m18n.n("migration_0011_rollback_success"))
-            raise
-        else:
-            os.system("rm -r " + backup_folder)
-
-            logger.info(m18n.n("migration_0011_done"))