diff --git a/data/helpers.d/backend b/data/helpers.d/backend index ac6f3e5de..6a574ab9a 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -164,10 +164,10 @@ ynh_remove_systemd_config () { local finalsystemdconf="/etc/systemd/system/$service.service" if [ -e "$finalsystemdconf" ]; then - sudo systemctl stop $service - sudo systemctl disable $service + ynh_systemd_action --service_name=$service --action=stop + systemctl disable $service ynh_secure_remove --file="$finalsystemdconf" - sudo systemctl daemon-reload + systemctl daemon-reload fi } @@ -234,7 +234,7 @@ ynh_add_nginx_config () { ynh_store_file_checksum --file="$finalnginxconf" - sudo systemctl reload nginx + ynh_systemd_action --service_name=nginx --action=reload } # Remove the dedicated nginx config @@ -244,7 +244,7 @@ ynh_add_nginx_config () { # Requires YunoHost version 2.7.2 or higher. ynh_remove_nginx_config () { ynh_secure_remove --file="/etc/nginx/conf.d/$domain.d/$app.conf" - sudo systemctl reload nginx + ynh_systemd_action --service_name=nginx --action=reload } # Create a dedicated php-fpm config @@ -281,7 +281,7 @@ ynh_add_fpm_config () { sudo chown root: "$finalphpini" ynh_store_file_checksum "$finalphpini" fi - sudo systemctl reload $fpm_service + ynh_systemd_action --service_name=$fpm_service --action=reload } # Remove the dedicated php-fpm config @@ -299,7 +299,7 @@ ynh_remove_fpm_config () { fi ynh_secure_remove --file="$fpm_config_dir/pool.d/$app.conf" ynh_secure_remove --file="$fpm_config_dir/conf.d/20-$app.ini" 2>&1 - sudo systemctl reload $fpm_service + ynh_systemd_action --service_name=$fpm_service --action=reload } # Create a dedicated fail2ban config (jail and filter conf files) @@ -366,6 +366,7 @@ ynh_remove_fpm_config () { # Requires YunoHost version 3.?.? or higher. ynh_add_fail2ban_config () { # Declare an array to define the options of this helper. + local legacy_args=lrmptv declare -Ar args_array=( [l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= [t]=use_template [v]=others_var=) local logpath local failregex diff --git a/data/helpers.d/nodejs b/data/helpers.d/nodejs index 34583328d..b51bcd7c3 100644 --- a/data/helpers.d/nodejs +++ b/data/helpers.d/nodejs @@ -105,7 +105,7 @@ ynh_install_nodejs () { # Install the requested version of nodejs uname=$(uname -m) - if [[ $uname =~ aarch64 || $uname =~ arm64]] + if [[ $uname =~ aarch64 || $uname =~ arm64 ]] then n $nodejs_version --arch=arm64 else diff --git a/data/helpers.d/package b/data/helpers.d/package index 75323521d..9c2b58458 100644 --- a/data/helpers.d/package +++ b/data/helpers.d/package @@ -27,7 +27,7 @@ ynh_wait_dpkg_free() { while read dpkg_file <&9 do # Check if the name of this file contains only numbers. - if echo "$dpkg_file" | grep -Pq "^[[:digit:]]*$" + if echo "$dpkg_file" | grep -Pq "^[[:digit:]]+$" then # If so, that a remaining of dpkg. ynh_print_err "E: dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem." diff --git a/data/helpers.d/print b/data/helpers.d/print index 6e7b2b1d7..95d2af139 100644 --- a/data/helpers.d/print +++ b/data/helpers.d/print @@ -100,6 +100,7 @@ ynh_print_err () { # usage: ynh_exec_err command to execute # usage: ynh_exec_err "command to execute | following command" # In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe. +# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed. # # | arg: command - command to execute # @@ -113,6 +114,7 @@ ynh_exec_err () { # usage: ynh_exec_warn command to execute # usage: ynh_exec_warn "command to execute | following command" # In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe. +# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed. # # | arg: command - command to execute # @@ -126,6 +128,7 @@ ynh_exec_warn () { # usage: ynh_exec_warn_less command to execute # usage: ynh_exec_warn_less "command to execute | following command" # In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe. +# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed. # # | arg: command - command to execute # @@ -139,6 +142,7 @@ ynh_exec_warn_less () { # usage: ynh_exec_quiet command to execute # usage: ynh_exec_quiet "command to execute | following command" # In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe. +# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed. # # | arg: command - command to execute # @@ -152,6 +156,7 @@ ynh_exec_quiet () { # usage: ynh_exec_fully_quiet command to execute # usage: ynh_exec_fully_quiet "command to execute | following command" # In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe. +# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed. # # | arg: command - command to execute # diff --git a/data/helpers.d/psql b/data/helpers.d/psql index 70ea58af4..a9ea5dadc 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -1,23 +1,276 @@ +#!/bin/bash + +PSQL_ROOT_PWD_FILE=/etc/yunohost/psql + +# Open a connection as a user +# +# example: ynh_psql_connect_as 'user' 'pass' <<< "UPDATE ...;" +# example: ynh_psql_connect_as 'user' 'pass' < /path/to/file.sql +# +# usage: ynh_psql_connect_as --user=user --password=password [--database=database] +# | arg: -u, --user - the user name to connect as +# | arg: -p, --password - the user password +# | arg: -d, --database - the database to connect to +# +# Requires YunoHost version 3.?.? or higher. +ynh_psql_connect_as() { + # Declare an array to define the options of this helper. + local legacy_args=upd + declare -Ar args_array=([u]=user= [p]=password= [d]=database=) + local user + local password + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + + sudo --login --user=postgres PGUSER="$user" PGPASSWORD="$password" psql "$database" +} + +# Execute a command as root user +# +# usage: ynh_psql_execute_as_root --sql=sql [--database=database] +# | arg: -s, --sql - the SQL command to execute +# | arg: -d, --database - the database to connect to +# +# Requires YunoHost version 3.?.? or higher. +ynh_psql_execute_as_root() { + # Declare an array to define the options of this helper. + local legacy_args=sd + declare -Ar args_array=([s]=sql= [d]=database=) + local sql + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + + ynh_psql_connect_as --user="postgres" --password="$(sudo cat $PSQL_ROOT_PWD_FILE)" \ + --database="$database" <<<"$sql" +} + +# Execute a command from a file as root user +# +# usage: ynh_psql_execute_file_as_root --file=file [--database=database] +# | arg: -f, --file - the file containing SQL commands +# | arg: -d, --database - the database to connect to +# +# Requires YunoHost version 3.?.? or higher. +ynh_psql_execute_file_as_root() { + # Declare an array to define the options of this helper. + local legacy_args=fd + declare -Ar args_array=([f]=file= [d]=database=) + local file + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + + ynh_psql_connect_as --user="postgres" --password="$(sudo cat $PSQL_ROOT_PWD_FILE)" \ + --database="$database" <"$file" +} + +# Create a database and grant optionnaly privilegies to a user +# +# [internal] +# +# usage: ynh_psql_create_db db [user] +# | arg: db - the database name to create +# | arg: user - the user to grant privilegies +# +# Requires YunoHost version 3.?.? or higher. +ynh_psql_create_db() { + local db=$1 + local user=${2:-} + + local sql="CREATE DATABASE ${db};" + + # grant all privilegies to user + if [ -n "$user" ]; then + sql+="GRANT ALL PRIVILEGES ON DATABASE ${db} TO ${user} WITH GRANT OPTION;" + fi + + ynh_psql_execute_as_root --sql="$sql" +} + +# Drop a database +# +# [internal] +# +# If you intend to drop the database *and* the associated user, +# consider using ynh_psql_remove_db instead. +# +# usage: ynh_psql_drop_db db +# | arg: db - the database name to drop +# +# Requires YunoHost version 3.?.? or higher. +ynh_psql_drop_db() { + local db=$1 + sudo --login --user=postgres dropdb $db +} + +# Dump a database +# +# example: ynh_psql_dump_db 'roundcube' > ./dump.sql +# +# usage: ynh_psql_dump_db --database=database +# | arg: -d, --database - the database name to dump +# | ret: the psqldump output +# +# Requires YunoHost version 3.?.? or higher. +ynh_psql_dump_db() { + # Declare an array to define the options of this helper. + local legacy_args=d + declare -Ar args_array=([d]=database=) + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + sudo --login --user=postgres pg_dump "$database" +} + +# Create a user +# +# [internal] +# +# usage: ynh_psql_create_user user pwd +# | arg: user - the user name to create +# +# Requires YunoHost version 3.?.? or higher. +ynh_psql_create_user() { + local user=$1 + local pwd=$2 + ynh_psql_execute_as_root --sql="CREATE USER $user WITH ENCRYPTED PASSWORD '$pwd'" +} + +# Check if a psql user exists +# +# usage: ynh_psql_user_exists --user=user +# | arg: -u, --user - the user for which to check existence +ynh_psql_user_exists() { + # Declare an array to define the options of this helper. + local legacy_args=u + declare -Ar args_array=([u]=user=) + local user + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(sudo cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT rolname FROM pg_roles WHERE rolname='$user';" | grep --quiet "$user" ; then + return 1 + else + return 0 + fi +} + +# Check if a psql database exists +# +# usage: ynh_psql_database_exists --database=database +# | arg: -d, --database - the database for which to check existence +ynh_psql_database_exists() { + # Declare an array to define the options of this helper. + local legacy_args=d + declare -Ar args_array=([d]=database=) + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(sudo cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT datname FROM pg_database WHERE datname='$database';" | grep --quiet "$user"; then + return 1 + else + return 0 + fi +} + +# Drop a user +# +# [internal] +# +# usage: ynh_psql_drop_user user +# | arg: user - the user name to drop +# +# Requires YunoHost version 3.?.? or higher. +ynh_psql_drop_user() { + ynh_psql_execute_as_root --sql="DROP USER ${1};" +} + +# Create a database, an user and its password. Then store the password in the app's config +# +# After executing this helper, the password of the created database will be available in $db_pwd +# It will also be stored as "psqlpwd" into the app settings. +# +# usage: ynh_psql_setup_db --db_user=user --db_name=name [--db_pwd=pwd] +# | arg: -u, --db_user - Owner of the database +# | arg: -n, --db_name - Name of the database +# | arg: -p, --db_pwd - Password of the database. If not given, a password will be generated +ynh_psql_setup_db() { + # Declare an array to define the options of this helper. + local legacy_args=unp + declare -Ar args_array=([u]=db_user= [n]=db_name= [p]=db_pwd=) + local db_user + local db_name + db_pwd="" + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local new_db_pwd=$(ynh_string_random) # Generate a random password + # If $db_pwd is not given, use new_db_pwd instead for db_pwd + db_pwd="${db_pwd:-$new_db_pwd}" + + if ! ynh_psql_user_exists --user=$db_user; then + ynh_psql_create_user "$db_user" "$db_pwd" + fi + + ynh_psql_create_db "$db_name" "$db_user" # Create the database + ynh_app_setting_set --app=$app --key=psqlpwd --value=$db_pwd # Store the password in the app's config +} + +# Remove a database if it exists, and the associated user +# +# usage: ynh_psql_remove_db --db_user=user --db_name=name +# | arg: -u, --db_user - Owner of the database +# | arg: -n, --db_name - Name of the database +ynh_psql_remove_db() { + # Declare an array to define the options of this helper. + local legacy_args=un + declare -Ar args_array=([u]=db_user= [n]=db_name=) + local db_user + local db_name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local psql_root_password=$(sudo cat $PSQL_ROOT_PWD_FILE) + if ynh_psql_database_exists --database=$db_name; then # Check if the database exists + echo "Removing database $db_name" >&2 + ynh_psql_drop_db $db_name # Remove the database + else + echo "Database $db_name not found" >&2 + fi + + # Remove psql user if it exists + if ynh_psql_user_exists --user=$db_user; then + echo "Removing user $db_user" >&2 + ynh_psql_drop_user $db_user + else + echo "User $db_user not found" >&2 + fi +} + # Create a master password and set up global settings # Please always call this script in install and restore scripts # # usage: ynh_psql_test_if_first_run -# -# Requires YunoHost version 3.?.? or higher. ynh_psql_test_if_first_run() { - if [ -f /etc/yunohost/psql ]; - then + if [ -f "$PSQL_ROOT_PWD_FILE" ]; then echo "PostgreSQL is already installed, no need to create master password" else local pgsql="$(ynh_string_random)" - echo "$pgsql" > /etc/yunohost/psql + echo "$pgsql" >/etc/yunohost/psql - if [ -e /etc/postgresql/9.4/ ] - then + if [ -e /etc/postgresql/9.4/ ]; then local pg_hba=/etc/postgresql/9.4/main/pg_hba.conf - elif [ -e /etc/postgresql/9.6/ ] - then + local logfile=/var/log/postgresql/postgresql-9.4-main.log + elif [ -e /etc/postgresql/9.6/ ]; then local pg_hba=/etc/postgresql/9.6/main/pg_hba.conf + local logfile=/var/log/postgresql/postgresql-9.6-main.log else ynh_die "postgresql shoud be 9.4 or 9.6" fi @@ -29,140 +282,12 @@ ynh_psql_test_if_first_run() { # https://www.postgresql.org/docs/current/static/auth-pg-hba-conf.html#EXAMPLE-PG-HBA.CONF # Note: we can't use peer since YunoHost create users with nologin # See: https://github.com/YunoHost/yunohost/blob/unstable/data/helpers.d/user - sed -i '/local\s*all\s*all\s*peer/i \ - local all all password' "$pg_hba" + ynh_replace_string --match_string="local\(\s*\)all\(\s*\)all\(\s*\)peer" --replace_string="local\1all\2all\3password" --target_file="$pg_hba" + + # Advertise service in admin panel + yunohost service add postgresql --log "$logfile" + systemctl enable postgresql systemctl reload postgresql fi } - -# Open a connection as a user -# -# example: ynh_psql_connect_as 'user' 'pass' <<< "UPDATE ...;" -# example: ynh_psql_connect_as 'user' 'pass' < /path/to/file.sql -# -# usage: ynh_psql_connect_as user pwd [db] -# | arg: user - the user name to connect as -# | arg: pwd - the user password -# | arg: db - the database to connect to -# -# Requires YunoHost version 3.?.? or higher. -ynh_psql_connect_as() { - local user="$1" - local pwd="$2" - local db="$3" - sudo --login --user=postgres PGUSER="$user" PGPASSWORD="$pwd" psql "$db" -} - -# # Execute a command as root user -# -# usage: ynh_psql_execute_as_root sql [db] -# | arg: sql - the SQL command to execute -# -# Requires YunoHost version 3.?.? or higher. -ynh_psql_execute_as_root () { - local sql="$1" - sudo --login --user=postgres psql <<< "$sql" -} - -# Execute a command from a file as root user -# -# usage: ynh_psql_execute_file_as_root file [db] -# | arg: file - the file containing SQL commands -# | arg: db - the database to connect to -# -# Requires YunoHost version 3.?.? or higher. -ynh_psql_execute_file_as_root() { - local file="$1" - local db="$2" - sudo --login --user=postgres psql "$db" < "$file" -} - -# Create a database, an user and its password. Then store the password in the app's config -# -# After executing this helper, the password of the created database will be available in $db_pwd -# It will also be stored as "psqlpwd" into the app settings. -# -# usage: ynh_psql_setup_db user name [pwd] -# | arg: user - Owner of the database -# | arg: name - Name of the database -# | arg: pwd - Password of the database. If not given, a password will be generated -# -# Requires YunoHost version 3.?.? or higher. -ynh_psql_setup_db () { - local db_user="$1" - local db_name="$2" - local new_db_pwd=$(ynh_string_random) # Generate a random password - # If $3 is not given, use new_db_pwd instead for db_pwd. - local db_pwd="${3:-$new_db_pwd}" - ynh_psql_create_db "$db_name" "$db_user" "$db_pwd" # Create the database - ynh_app_setting_set "$app" psqlpwd "$db_pwd" # Store the password in the app's config -} - -# Create a database and grant privilegies to a user -# -# usage: ynh_psql_create_db db [user [pwd]] -# | arg: db - the database name to create -# | arg: user - the user to grant privilegies -# | arg: pwd - the user password -# -# Requires YunoHost version 3.?.? or higher. -ynh_psql_create_db() { - local db="$1" - local user="$2" - local pwd="$3" - ynh_psql_create_user "$user" "$pwd" - sudo --login --user=postgres createdb --owner="$user" "$db" -} - -# Drop a database -# -# usage: ynh_psql_drop_db db -# | arg: db - the database name to drop -# | arg: user - the user to drop -# -# Requires YunoHost version 3.?.? or higher. -ynh_psql_remove_db() { - local db="$1" - local user="$2" - sudo --login --user=postgres dropdb "$db" - ynh_psql_drop_user "$user" -} - -# Dump a database -# -# example: ynh_psql_dump_db 'roundcube' > ./dump.sql -# -# usage: ynh_psql_dump_db db -# | arg: db - the database name to dump -# | ret: the psqldump output -# -# Requires YunoHost version 3.?.? or higher. -ynh_psql_dump_db() { - local db="$1" - sudo --login --user=postgres pg_dump "$db" -} - - -# Create a user -# -# usage: ynh_psql_create_user user pwd [host] -# | arg: user - the user name to create -# -# Requires YunoHost version 3.?.? or higher. -ynh_psql_create_user() { - local user="$1" - local pwd="$2" - sudo --login --user=postgres psql -c"CREATE USER $user WITH PASSWORD '$pwd'" postgres -} - -# Drop a user -# -# usage: ynh_psql_drop_user user -# | arg: user - the user name to drop -# -# Requires YunoHost version 3.?.? or higher. -ynh_psql_drop_user() { - local user="$1" - sudo --login --user=postgres dropuser "$user" -} diff --git a/data/helpers.d/setting b/data/helpers.d/setting index 0c3698061..63d9104f3 100644 --- a/data/helpers.d/setting +++ b/data/helpers.d/setting @@ -16,7 +16,7 @@ ynh_app_setting_get() { # Manage arguments with getopts ynh_handle_getopts_args "$@" - sudo yunohost app setting "$app" "$key" --output-as plain --quiet + ynh_app_setting "get" "$app" "$key" } # Set an application setting @@ -37,7 +37,7 @@ ynh_app_setting_set() { # Manage arguments with getopts ynh_handle_getopts_args "$@" - sudo yunohost app setting "$app" "$key" --value="$value" --quiet + ynh_app_setting "set" "$app" "$key" "$value" } # Delete an application setting @@ -56,5 +56,38 @@ ynh_app_setting_delete() { # Manage arguments with getopts ynh_handle_getopts_args "$@" - sudo yunohost app setting -d "$app" "$key" --quiet + ynh_app_setting "delete" "$app" "$key" +} + +# Small "hard-coded" interface to avoid calling "yunohost app" directly each +# time dealing with a setting is needed (which may be so slow on ARM boards) +# +# [internal] +# +ynh_app_setting() +{ + ACTION="$1" APP="$2" KEY="$3" VALUE="${4:-}" python - < "$templog" & + # Get the PID of the journalctl command + local pid_tail=$! + else + # Read the specified log file + tail -F -n0 "$log_path" > "$templog" 2>&1 & + # Get the PID of the tail command + local pid_tail=$! + fi + fi + + ynh_print_info --message="${action^} the service $service_name" + + # Use reload-or-restart instead of reload. So it wouldn't fail if the service isn't running. + if [ "$action" == "reload" ]; then + action="reload-or-restart" + fi + + systemctl $action $service_name \ + || ( journalctl --no-pager --lines=$length -u $service_name >&2 \ + ; test -e "$log_path" && echo "--" >&2 && tail --lines=$length "$log_path" >&2 \ + ; false ) + + # Start the timeout and try to find line_match + if [[ -n "${line_match:-}" ]] + then + local i=0 + for i in $(seq 1 $timeout) + do + # Read the log until the sentence is found, that means the app finished to start. Or run until the timeout + if grep --quiet "$line_match" "$templog" + then + ynh_print_info --message="The service $service_name has correctly started." + break + fi + if [ $i -eq 3 ]; then + echo -n "Please wait, the service $service_name is ${action}ing" >&2 + fi + if [ $i -ge 3 ]; then + echo -n "." >&2 + fi + sleep 1 + done + if [ $i -ge 3 ]; then + echo "" >&2 + fi + if [ $i -eq $timeout ] + then + ynh_print_warn --message="The service $service_name didn't fully started before the timeout." + ynh_print_warn --message="Please find here an extract of the end of the log of the service $service_name:" + journalctl --no-pager --lines=$length -u $service_name >&2 + test -e "$log_path" && echo "--" >&2 && tail --lines=$length "$log_path" >&2 + fi + ynh_clean_check_starting + fi +} + +# Clean temporary process and file used by ynh_check_starting +# (usually used in ynh_clean_setup scripts) +# +# usage: ynh_clean_check_starting +ynh_clean_check_starting () { + # Stop the execution of tail. + kill -s 15 $pid_tail 2>&1 + ynh_secure_remove "$templog" 2>&1 +} + # Read the value of a key in a ynh manifest file # # usage: ynh_read_manifest manifest key diff --git a/data/helpers.d/user b/data/helpers.d/user index f19739993..83fa47aa8 100644 --- a/data/helpers.d/user +++ b/data/helpers.d/user @@ -71,6 +71,21 @@ ynh_system_user_exists() { getent passwd "$username" &>/dev/null } +# Check if a group exists on the system +# +# usage: ynh_system_group_exists --group=group +# | arg: -g, --group - the group to check +ynh_system_group_exists() { + # Declare an array to define the options of this helper. + local legacy_args=g + declare -Ar args_array=( [g]=group= ) + local group + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + getent group "$group" &>/dev/null +} + # Create a system user # # examples: @@ -128,11 +143,19 @@ ynh_system_user_delete () { # Manage arguments with getopts ynh_handle_getopts_args "$@" - if ynh_system_user_exists "$username" # Check if the user exists on the system + # Check if the user exists on the system + if ynh_system_user_exists "$username" then echo "Remove the user $username" >&2 - sudo userdel $username + deluser $username else echo "The user $username was not found" >&2 fi + + # Check if the group exists on the system + if ynh_system_group_exists "$username" + then + echo "Remove the group $username" >&2 + delgroup $username + fi } diff --git a/data/templates/dnsmasq/plain/resolv.dnsmasq.conf b/data/templates/dnsmasq/plain/resolv.dnsmasq.conf index 7eed1142f..197ee2d64 100644 --- a/data/templates/dnsmasq/plain/resolv.dnsmasq.conf +++ b/data/templates/dnsmasq/plain/resolv.dnsmasq.conf @@ -9,25 +9,37 @@ # (FR) FDN nameserver 80.67.169.12 +nameserver 2001:910:800::12 nameserver 80.67.169.40 +nameserver 2001:910:800::40 # (FR) LDN nameserver 80.67.188.188 +nameserver 2001:913::8 # (FR) ARN nameserver 89.234.141.66 +nameserver 2a00:5881:8100:1000::3 # (FR) Aquilenet nameserver 185.233.100.100 +nameserver 2a0c:e300::100 nameserver 185.233.100.101 +nameserver 2a0c:e300::101 # (FR) gozmail / grifon nameserver 80.67.190.200 +nameserver 2a00:5884:8218::1 # (DE) FoeBud / Digital Courage nameserver 85.214.20.141 # (DE) CCC Berlin nameserver 195.160.173.53 # (DE) AS250 nameserver 194.150.168.168 +nameserver 2001:4ce8::53 # (DE) Ideal-Hosting nameserver 84.200.69.80 +nameserver 2001:1608:10:25::1c04:b12f nameserver 84.200.70.40 +nameserver 2001:1608:10:25::9249:d69b # (DK) censurfridns nameserver 91.239.100.100 +nameserver 2001:67c:28a4:: nameserver 89.233.43.71 +nameserver 2002:d596:2a92:1:71:53:: diff --git a/data/templates/nginx/plain/global.conf b/data/templates/nginx/plain/global.conf index ca8721afb..b3a5f356a 100644 --- a/data/templates/nginx/plain/global.conf +++ b/data/templates/nginx/plain/global.conf @@ -1,2 +1 @@ server_tokens off; -gzip_types text/css text/javascript application/javascript; diff --git a/data/templates/nginx/plain/yunohost_admin.conf b/data/templates/nginx/plain/yunohost_admin.conf index 2493e4033..ff61b8638 100644 --- a/data/templates/nginx/plain/yunohost_admin.conf +++ b/data/templates/nginx/plain/yunohost_admin.conf @@ -51,6 +51,10 @@ server { more_set_headers "X-Permitted-Cross-Domain-Policies : none"; more_set_headers "X-Frame-Options : SAMEORIGIN"; + # Disable gzip to protect against BREACH + # Read https://trac.nginx.org/nginx/ticket/1720 (text/html cannot be disabled!) + gzip off; + location / { return 302 https://$http_host/yunohost/admin; } diff --git a/data/templates/nginx/server.tpl.conf b/data/templates/nginx/server.tpl.conf index 43d38ca98..d8793ef05 100644 --- a/data/templates/nginx/server.tpl.conf +++ b/data/templates/nginx/server.tpl.conf @@ -71,6 +71,10 @@ server { resolver_timeout 5s; {% endif %} + # Disable gzip to protect against BREACH + # Read https://trac.nginx.org/nginx/ticket/1720 (text/html cannot be disabled!) + gzip off; + access_by_lua_file /usr/share/ssowat/access.lua; include /etc/nginx/conf.d/{{ domain }}.d/*.conf; diff --git a/debian/changelog b/debian/changelog index 7be4212fe..444d797e1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,69 @@ +yunohost (3.5.0.2) testing; urgency=low + + - [fix] Make sure that `ynh_system_user_delete` also deletes the group (#680) + - [enh] `ynh_systemd_action` : reload-or-restart instead of just reload (#681) + + Last minute fixes by Maniack ;) + + -- Alexandre Aubin Thu, 14 Mar 2019 03:45:00 +0000 + +yunohost (3.5.0.1) testing; urgency=low + + - [fix] #675 introduced a bug in nginx conf ... + + -- Alexandre Aubin Wed, 13 Mar 2019 19:23:00 +0000 + +yunohost (3.5.0) testing; urgency=low + + Core + ---- + + - [fix] Disable gzip entirely to avoid BREACH attacks (#675) + - [fix] Backup tests were broken (#673) + - [fix] Backup fails because output directory not empty (#672) + - [fix] Reject app password if they contains { or } (#671) + - [enh] Allow `display_text` 'fake' argument in manifest.json (#669) + - [fix] Optimize dyndns requests (#662) + - [enh] Don't add Strict-Transport-Security header in nginx conf if using a selfsigned cert (#661) + - [enh] Add apt-transport-https to dependencies (#658) + - [enh] Cache results from meltdown vulnerability checker (#656) + - [enh] Ensure the tar file is closed during the backup (#655) + - [enh] Be able to define hook to trigger when changing a setting (#654) + - [enh] Assert dpkg is not broken before app install (#652) + - [fix] Loading only one helper file leads to errors because missing getopts (#651) + - [enh] Improve / add some messages to improve UX (#650) + - [enh] Reload fail2ban instead of restart (#649) + - [enh] Add IPv6 resolvers from diyisp.org to resolv.dnsmasq.conf (#639) + - [fix] Remove old SMTP port (465) from fail2ban jail.conf (#637) + - [enh] Improve protection against indexation from the robots. (#622) + - [enh] Allow hooks to return data (#526) + - [fix] Do not make version number available from web API to unauthenticated users (#291) + - [i18n] Improve Russian and Chinese (Mandarin) translations + + App helpers + ----------- + + - [enh] Optimize app setting helpers (#663, #676) + - [enh] Handle `ynh_install_nodejs` for arm64 / aarch64 (#660) + - [enh] Update postgresql helpers (#657) + - [enh] Print diff of files when backup by `ynh_backup_if_checksum_is_different` (#648) + - [enh] Add app debugger helper (#647) + - [fix] Escape double quote before eval in getopts (#646) + - [fix] `ynh_local_curl` not using the right url in some cases (#644) + - [fix] Get rid of annoying 'unable to initialize frontend' messages (#643) + - [enh] Check if dpkg is not broken when calling `ynh_wait_dpkg_free` (#638) + - [enh] Warn the packager that `ynh_secure_remove` should be used with only one arg… (#635, #642) + - [enh] Add `ynh_script_progression` helper (#634) + - [enh] Add `ynh_systemd_action` helper (#633) + - [enh] Allow to dig deeper into an archive with `ynh_setup_source` (#630) + - [enh] Use getops (#561) + - [enh] Add `ynh_check_app_version_changed` helper (#521) + - [enh] Add fail2ban helpers (#364) + + Contributors: Alexandre Aubin, Jimmy Monin, Josué Tille, Kayou, Laurent Peuch, Lukas Fülling, Maniack Crudelis, Taekiro, frju365, ljf, opi, yalh76, Алексей + + -- Alexandre Aubin Wed, 13 Mar 2019 16:10:00 +0000 + yunohost (3.4.2.4) stable; urgency=low - [fix] Meltdown vulnerability checker something outputing trash instead of pure json diff --git a/locales/ca.json b/locales/ca.json index 6c06d55b3..bfad4d2bd 100644 --- a/locales/ca.json +++ b/locales/ca.json @@ -21,5 +21,64 @@ "app_location_already_used": "L'aplicació '{app}' ja està instal·lada en aquest camí ({path})", "app_make_default_location_already_used": "No es pot fer l'aplicació '{app}' per defecte en el domini {domain} ja que ja és utilitzat per una altra aplicació '{other_app}'", "app_location_install_failed": "No s'ha pogut instal·lar l'aplicació en aquest camí ja que entra en conflicte amb l'aplicació '{other_app}' ja instal·lada a '{other_path}'", - "app_location_unavailable": "Aquesta url no està disponible o entra en conflicte amb aplicacions ja instal·lades:\n{apps:s}" + "app_location_unavailable": "Aquesta url no està disponible o entra en conflicte amb aplicacions ja instal·lades:\n{apps:s}", + "app_manifest_invalid": "Manifest d'aplicació incorrecte: {error}", + "app_no_upgrade": "No hi ha cap aplicació per actualitzar", + "app_not_correctly_installed": "{app:s} sembla estar mal instal·lada", + "app_not_installed": "{app:s} no està instal·lada", + "app_not_properly_removed": "{app:s} no s'ha pogut suprimir correctament", + "app_package_need_update": "El paquet de l'aplicació {app} ha de ser actualitzat per poder seguir els canvis de YunoHost", + "app_removed": "{app:s} ha estat suprimida", + "app_requirements_checking": "Verificació dels paquets requerits per {app}", + "app_requirements_failed": "No es poden satisfer els requeriments per {app}: {error}", + "app_requirements_unmeet": "No es compleixen els requeriments per {app}, el paquet {pkgname} ({version}) ha de ser {spec}", + "app_sources_fetch_failed": "No s'han pogut carregar els fitxers font", + "app_unknown": "Aplicació desconeguda", + "app_unsupported_remote_type": "El tipus remot utilitzat per l'aplicació no està suportat", + "app_upgrade_app_name": "Actualitzant l'aplicació {app}...", + "app_upgrade_failed": "No s'ha pogut actualitzar {app:s}", + "app_upgrade_some_app_failed": "No s'han pogut actualitzar algunes aplicacions", + "app_upgraded": "{app:s} ha estat actualitzada", + "appslist_corrupted_json": "No s'han pogut carregar les llistes d'aplicacions. Sembla que {filename:s} està danyat.", + "appslist_could_not_migrate": "No s'ha pogut migrar la llista d'aplicacions {appslist:s}! No s'ha pogut analitzar la URL... L'antic cronjob s'ha guardat a {bkp_file:s}.", + "appslist_fetched": "S'ha descarregat la llista d'aplicacions {appslist:s} correctament", + "appslist_migrating": "Migrant la llista d'aplicacions {appslist:s} ...", + "appslist_name_already_tracked": "Ja hi ha una llista d'aplicacions registrada amb el nom {name:s}.", + "appslist_removed": "S'ha eliminat la llista d'aplicacions {appslist:s}", + "appslist_retrieve_bad_format": "L'arxiu obtingut per la llista d'aplicacions {appslist:s} no és vàlid", + "appslist_retrieve_error": "No s'ha pogut obtenir la llista d'aplicacions remota {appslist:s}: {error:s}", + "appslist_unknown": "La llista d'aplicacions {appslist:s} es desconeguda.", + "appslist_url_already_tracked": "Ja hi ha una llista d'aplicacions registrada amb al URL {url:s}.", + "ask_current_admin_password": "Contrasenya d'administrador actual", + "ask_email": "Correu electrònic", + "ask_firstname": "Nom", + "ask_lastname": "Cognom", + "ask_list_to_remove": "Llista per a suprimir", + "ask_main_domain": "Domini principal", + "ask_new_admin_password": "Nova contrasenya d'administrador", + "ask_password": "Contrasenya", + "ask_path": "Camí", + "backup_abstract_method": "Encara no s'ha implementat aquest mètode de copia de seguretat", + "backup_action_required": "S'ha d'especificar què s'ha de guardar", + "backup_app_failed": "No s'ha pogut fer la còpia de seguretat de l'aplicació \"{app:s}\"", + "backup_applying_method_borg": "Enviant tots els fitxers de la còpia de seguretat al repositori borg-backup...", + "backup_applying_method_copy": "Còpia de tots els fitxers a la còpia de seguretat...", + "backup_applying_method_custom": "Crida del mètode de còpia de seguretat personalitzat \"{method:s}\"...", + "backup_applying_method_tar": "Creació de l'arxiu tar de la còpia de seguretat...", + "backup_archive_app_not_found": "L'aplicació \"{app:s}\" no es troba dins l'arxiu de la còpia de seguretat", + "backup_archive_broken_link": "No s'ha pogut accedir a l'arxiu de la còpia de seguretat (enllaç invàlid cap a {path:s})", + "backup_archive_mount_failed": "No s'ha pogut carregar l'arxiu de la còpia de seguretat", + "backup_archive_name_exists": "Ja hi ha una còpia de seguretat amb aquest nom", + "backup_archive_name_unknown": "Còpia de seguretat local \"{name:s}\" desconeguda", + "backup_archive_open_failed": "No s'ha pogut obrir l'arxiu de la còpia de seguretat", + "backup_archive_system_part_not_available": "La part \"{part:s}\" del sistema no està disponible en aquesta copia de seguretat", + "backup_archive_writing_error": "No es poden afegir arxius a l'arxiu comprimit de la còpia de seguretat", + "backup_ask_for_copying_if_needed": "Alguns fitxers no s'han pogut preparar per la còpia de seguretat utilitzant el mètode que evita malgastar espai del sistema temporalment. Per fer la còpia de seguretat, s'han d'utilitzar {size:s}MB temporalment. Hi esteu d'acord?", + "backup_borg_not_implemented": "El mètode de còpia de seguretat Borg encara no està implementat", + "backup_cant_mount_uncompress_archive": "No es pot carregar en mode de lectura només el directori de l'arxiu descomprimit", + "backup_cleaning_failed": "No s'ha pogut netejar el directori temporal de la còpia de seguretat", + "backup_copying_to_organize_the_archive": "Copiant {size:s}MB per organitzar l'arxiu", + "backup_couldnt_bind": "No es pot lligar {src:s} amb {dest:s}.", + "backup_created": "S'ha creat la còpia de seguretat", + "backup_creating_archive": "Creant l'arxiu de la còpia de seguretat" } diff --git a/locales/el.json b/locales/el.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/locales/el.json @@ -0,0 +1 @@ +{} diff --git a/locales/en.json b/locales/en.json index 1157e0a54..9a509048e 100644 --- a/locales/en.json +++ b/locales/en.json @@ -218,7 +218,8 @@ "good_practices_about_admin_password": "You are now about to define a new administration password. The password should be at least 8 characters - though it is good practice to use longer password (i.e. a passphrase) and/or to use various kind of characters (uppercase, lowercase, digits and special characters).", "good_practices_about_user_password": "You are now about to define a new user password. The password should be at least 8 characters - though it is good practice to use longer password (i.e. a passphrase) and/or to use various kind of characters (uppercase, lowercase, digits and special characters).", "hook_exec_failed": "Script execution failed: {path:s}", - "hook_exec_not_terminated": "Script execution hasn\u2019t terminated: {path:s}", + "hook_exec_not_terminated": "Script execution did not finish properly: {path:s}", + "hook_json_return_error": "Failed to read return from hook {path:s}. Error: {msg:s}. Raw content: {raw_content}", "hook_list_by_invalid": "Invalid property to list hook by", "hook_name_unknown": "Unknown hook name '{name:s}'", "installation_complete": "Installation complete", @@ -380,6 +381,7 @@ "pattern_port_or_range": "Must be a valid port number (i.e. 0-65535) or range of ports (e.g. 100:200)", "pattern_positive_number": "Must be a positive number", "pattern_username": "Must be lower-case alphanumeric and underscore characters only", + "pattern_password_app": "Sorry, passwords should not contain the following characters: {forbidden_chars}", "port_already_closed": "Port {port:d} is already closed for {ip_version:s} connections", "port_already_opened": "Port {port:d} is already opened for {ip_version:s} connections", "port_available": "Port {port:d} is available", diff --git a/locales/pl.json b/locales/pl.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/locales/pl.json @@ -0,0 +1 @@ +{} diff --git a/locales/ru.json b/locales/ru.json index 2658446bc..306a8763a 100644 --- a/locales/ru.json +++ b/locales/ru.json @@ -6,5 +6,41 @@ "app_already_installed": "{app:s} уже установлено", "app_already_installed_cant_change_url": "Это приложение уже установлено. URL не может быть изменен только с помощью этой функции. Изучите `app changeurl`, если это доступно.", "app_argument_choice_invalid": "Неверный выбор для аргумента '{name:s}', Это должно быть '{choices:s}'", - "app_argument_invalid": "Недопустимое значение аргумента '{name:s}': {error:s}'" + "app_argument_invalid": "Недопустимое значение аргумента '{name:s}': {error:s}'", + "app_already_up_to_date": "{app:s} уже обновлено", + "app_argument_required": "Аргумент '{name:s}' необходим", + "app_change_no_change_url_script": "Приложение {app_name:s} не поддерживает изменение URL, вы должны обновить его.", + "app_change_url_identical_domains": "Старый и новый domain/url_path идентичны ('{domain:s}{path:s}'), ничего делать не надо.", + "app_change_url_no_script": "Приложение '{app_name:s}' не поддерживает изменение url. Наверное, вам нужно обновить приложение.", + "app_change_url_success": "Успешно изменён {app:s} url на {domain:s}{path:s}", + "app_extraction_failed": "Невозможно извлечь файлы для инсталляции", + "app_id_invalid": "Неправильный id приложения", + "app_incompatible": "Приложение {app} несовместимо с вашей версией YonoHost", + "app_install_files_invalid": "Неправильные файлы инсталляции", + "app_location_already_used": "Приложение '{app}' уже установлено по этому адресу ({path})", + "app_location_install_failed": "Невозможно установить приложение в это место, потому что оно конфликтует с приложением, '{other_app}' установленном на '{other_path}'", + "app_location_unavailable": "Этот url отсутствует или конфликтует с уже установленным приложением или приложениями: {apps:s}", + "app_manifest_invalid": "Недопустимый манифест приложения: {error}", + "app_no_upgrade": "Нет приложений, требующих обновления", + "app_not_correctly_installed": "{app:s} , кажется, установлены неправильно", + "app_not_installed": "{app:s} не установлены", + "app_not_properly_removed": "{app:s} удалены неправильно", + "app_package_need_update": "Пакет приложения {app} должен быть обновлён в соответствии с изменениями YonoHost", + "app_removed": "{app:s} удалено", + "app_requirements_checking": "Проверяю необходимые пакеты для {app}...", + "app_sources_fetch_failed": "Невозможно получить исходные файлы", + "app_unknown": "Неизвестное приложение", + "app_upgrade_app_name": "Обновление приложения {app}...", + "app_upgrade_failed": "Невозможно обновить {app:s}", + "app_upgrade_some_app_failed": "Невозможно обновить некоторые приложения", + "app_upgraded": "{app:s} обновлено", + "appslist_corrupted_json": "Не могу загрузить список приложений. Кажется, {filename:s} поврежден.", + "appslist_fetched": "Был выбран список приложений {appslist:s}", + "appslist_name_already_tracked": "Уже есть зарегистрированный список приложений по имени {name:s}.", + "appslist_removed": "Список приложений {appslist:s} удалён", + "appslist_retrieve_bad_format": "Неверный файл списка приложений{appslist:s}", + "appslist_retrieve_error": "Невозможно получить список удаленных приложений {appslist:s}: {error:s}", + "appslist_unknown": "Список приложений {appslist:s} неизвестен.", + "appslist_url_already_tracked": "Это уже зарегистрированный список приложений с url{url:s}.", + "installation_complete": "Установка завершена" } diff --git a/locales/sv.json b/locales/sv.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/locales/sv.json @@ -0,0 +1 @@ +{} diff --git a/locales/zh_Hans.json b/locales/zh_Hans.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/locales/zh_Hans.json @@ -0,0 +1 @@ +{} diff --git a/src/yunohost/app.py b/src/yunohost/app.py index e84187d6b..f21352fc2 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -97,7 +97,7 @@ def app_fetchlist(url=None, name=None): name -- Name of the list url -- URL of remote JSON list """ - if not url.endswith(".json"): + if url and not url.endswith(".json"): raise YunohostError("This is not a valid application list url. It should end with .json.") # If needed, create folder where actual appslists are stored @@ -523,7 +523,7 @@ def app_change_url(operation_logger, auth, app, domain, path): os.system('chmod +x %s' % os.path.join(os.path.join(APP_TMP_FOLDER, "scripts", "change_url"))) if hook_exec(os.path.join(APP_TMP_FOLDER, 'scripts/change_url'), - args=args_list, env=env_dict) != 0: + args=args_list, env=env_dict)[0] != 0: msg = "Failed to change '%s' url." % app logger.error(msg) operation_logger.error(msg) @@ -583,28 +583,28 @@ def app_upgrade(auth, app=[], url=None, file=None): not_upgraded_apps = [] apps = app - user_specified_list = True # If no app is specified, upgrade all apps if not apps: + # FIXME : not sure what's supposed to happen if there is a url and a file but no apps... if not url and not file: apps = [app["id"] for app in app_list(installed=True)["apps"]] - user_specified_list = False elif not isinstance(app, list): apps = [app] # Remove possible duplicates - apps = [app for i,app in enumerate(apps) if apps not in L[:i]] + apps = [app for i,app in enumerate(apps) if apps not in apps[:i]] + + # Abort if any of those app is in fact not installed.. + for app in [app for app in apps if not _is_installed(app)]: + raise YunohostError('app_not_installed', app=app) if len(apps) == 0: raise YunohostError('app_no_upgrade') if len(apps) > 1: - logger.info(m18n.n("app_upgrade_several_apps", apps=", ".join(app))) + logger.info(m18n.n("app_upgrade_several_apps", apps=", ".join(apps))) for app_instance_name in apps: logger.info(m18n.n('app_upgrade_app_name', app=app_instance_name)) - installed = _is_installed(app_instance_name) - if not installed: - raise YunohostError('app_not_installed', app=app_instance_name) app_dict = app_info(app_instance_name, raw=True) @@ -618,8 +618,7 @@ def app_upgrade(auth, app=[], url=None, file=None): elif app_dict["upgradable"] == "yes": manifest, extracted_app_folder = _fetch_app_from_git(app_instance_name) else: - if user_specified_list: - logger.success(m18n.n('app_already_up_to_date', app=app_instance_name)) + logger.success(m18n.n('app_already_up_to_date', app=app_instance_name)) continue # Check requirements @@ -655,7 +654,7 @@ def app_upgrade(auth, app=[], url=None, file=None): # Execute App upgrade script os.system('chown -hR admin: %s' % INSTALL_TMP) if hook_exec(extracted_app_folder + '/scripts/upgrade', - args=args_list, env=env_dict) != 0: + args=args_list, env=env_dict)[0] != 0: msg = m18n.n('app_upgrade_failed', app=app_instance_name) not_upgraded_apps.append(app_instance_name) logger.error(msg) @@ -848,7 +847,7 @@ def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on install_retcode = hook_exec( os.path.join(extracted_app_folder, 'scripts/install'), args=args_list, env=env_dict - ) + )[0] except (KeyboardInterrupt, EOFError): install_retcode = -1 except Exception: @@ -873,7 +872,7 @@ def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on remove_retcode = hook_exec( os.path.join(extracted_app_folder, 'scripts/remove'), args=[app_instance_name], env=env_dict_remove - ) + )[0] if remove_retcode != 0: msg = m18n.n('app_not_properly_removed', app=app_instance_name) @@ -964,7 +963,7 @@ def app_remove(operation_logger, auth, app): operation_logger.flush() if hook_exec('/tmp/yunohost_remove/scripts/remove', args=args_list, - env=env_dict) == 0: + env=env_dict)[0] == 0: logger.success(m18n.n('app_removed', app=app)) hook_callback('post_app_remove', args=args_list, env=env_dict) @@ -1563,7 +1562,7 @@ def app_action_run(app, action, args=None): env=env_dict, chdir=cwd, user=action_declaration.get("user", "root"), - ) + )[0] if retcode not in action_declaration.get("accepted_return_codes", [0]): raise YunohostError("Error while executing action '%s' of app '%s': return code %s" % (action, app, retcode), raw_msg=True) @@ -2203,6 +2202,11 @@ def _parse_action_args_in_yunohost_format(args, action_args, auth=None): if arg_type == 'boolean': arg_default = 1 if arg_default else 0 + # do not print for webadmin + if arg_type == 'display_text' and msettings.get('interface') != 'api': + print(arg["text"]) + continue + # Attempt to retrieve argument value if arg_name in args: arg_value = args[arg_name] @@ -2288,6 +2292,9 @@ def _parse_action_args_in_yunohost_format(args, action_args, auth=None): else: raise YunohostError('app_argument_choice_invalid', name=arg_name, choices='yes, no, y, n, 1, 0') elif arg_type == 'password': + forbidden_chars = "{}" + if any(char in arg_value for char in forbidden_chars): + raise YunohostError('pattern_password_app', forbidden_chars=forbidden_chars) from yunohost.utils.password import assert_password_is_strong_enough assert_password_is_strong_enough('user', arg_value) args_dict[arg_name] = arg_value diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index f9505fb66..6f969327b 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -326,10 +326,19 @@ class BackupManager(): if not os.path.isdir(self.work_dir): filesystem.mkdir(self.work_dir, 0o750, parents=True, uid='admin') elif self.is_tmp_work_dir: - logger.debug("temporary directory for backup '%s' already exists", + + logger.debug("temporary directory for backup '%s' already exists... attempting to clean it", self.work_dir) - # FIXME May be we should clean the workdir here - raise YunohostError('backup_output_directory_not_empty') + + # Try to recursively unmount stuff (from a previously failed backup ?) + if not _recursive_umount(self.work_dir): + raise YunohostError('backup_output_directory_not_empty') + else: + # If umount succeeded, remove the directory (we checked that + # we're in /home/yunohost.backup/tmp so that should be okay... + # c.f. method clean() which also does this) + filesystem.rm(self.work_dir, recursive=True, force=True) + filesystem.mkdir(self.work_dir, 0o750, parents=True, uid='admin') # # Backup target management # @@ -593,8 +602,15 @@ class BackupManager(): env=env_dict, chdir=self.work_dir) - if ret["succeed"] != []: - self.system_return = ret["succeed"] + ret_succeed = {hook: {path:result["state"] for path, result in infos.items()} + for hook, infos in ret.items() + if any(result["state"] == "succeed" for result in infos.values())} + ret_failed = {hook: {path:result["state"] for path, result in infos.items.items()} + for hook, infos in ret.items() + if any(result["state"] == "failed" for result in infos.values())} + + if ret_succeed.keys() != []: + self.system_return = ret_succeed # Add files from targets (which they put in the CSV) to the list of # files to backup @@ -610,7 +626,7 @@ class BackupManager(): restore_hooks = hook_list("restore")["hooks"] - for part in ret['succeed'].keys(): + for part in ret_succeed.keys(): if part in restore_hooks: part_restore_hooks = hook_info("restore", part)["hooks"] for hook in part_restore_hooks: @@ -620,7 +636,7 @@ class BackupManager(): logger.warning(m18n.n('restore_hook_unavailable', hook=part)) self.targets.set_result("system", part, "Warning") - for part in ret['failed'].keys(): + for part in ret_failed.keys(): logger.error(m18n.n('backup_system_part_failed', part=part)) self.targets.set_result("system", part, "Error") @@ -682,7 +698,7 @@ class BackupManager(): subprocess.call(['install', '-Dm555', app_script, tmp_script]) hook_exec(tmp_script, args=[tmp_app_bkp_dir, app], - raise_on_error=True, chdir=tmp_app_bkp_dir, env=env_dict) + raise_on_error=True, chdir=tmp_app_bkp_dir, env=env_dict)[0] self._import_to_list_to_backup(env_dict["YNH_BACKUP_CSV"]) except: @@ -904,7 +920,7 @@ class RestoreManager(): ret = subprocess.call(["umount", self.work_dir]) if ret != 0: logger.warning(m18n.n('restore_cleaning_failed')) - filesystem.rm(self.work_dir, True, True) + filesystem.rm(self.work_dir, recursive=True, force=True) # # Restore target manangement # @@ -1177,16 +1193,21 @@ class RestoreManager(): env=env_dict, chdir=self.work_dir) - for part in ret['succeed'].keys(): + ret_succeed = [hook for hook, infos in ret.items() + if any(result["state"] == "succeed" for result in infos.values())] + ret_failed = [hook for hook, infos in ret.items() + if any(result["state"] == "failed" for result in infos.values())] + + for part in ret_succeed: self.targets.set_result("system", part, "Success") error_part = [] - for part in ret['failed'].keys(): + for part in ret_failed: logger.error(m18n.n('restore_system_part_failed', part=part)) self.targets.set_result("system", part, "Error") error_part.append(part) - if ret['failed']: + if ret_failed: operation_logger.error(m18n.n('restore_system_part_failed', part=', '.join(error_part))) else: operation_logger.success() @@ -1301,7 +1322,7 @@ class RestoreManager(): args=[app_backup_in_archive, app_instance_name], chdir=app_backup_in_archive, raise_on_error=True, - env=env_dict) + env=env_dict)[0] except: msg = m18n.n('restore_app_failed', app=app_instance_name) logger.exception(msg) @@ -1326,7 +1347,7 @@ class RestoreManager(): # Execute remove script # TODO: call app_remove instead if hook_exec(remove_script, args=[app_instance_name], - env=env_dict_remove) != 0: + env=env_dict_remove)[0] != 0: msg = m18n.n('app_not_properly_removed', app=app_instance_name) logger.warning(msg) operation_logger.error(msg) @@ -1514,34 +1535,12 @@ class BackupMethod(object): directories of the working directories """ if self.need_mount(): - if self._recursive_umount(self.work_dir) > 0: + if not _recursive_umount(self.work_dir): raise YunohostError('backup_cleaning_failed') if self.manager.is_tmp_work_dir: filesystem.rm(self.work_dir, True, True) - def _recursive_umount(self, directory): - """ - Recursively umount sub directories of a directory - - Args: - directory -- a directory path - """ - mount_lines = subprocess.check_output("mount").split("\n") - - points_to_umount = [line.split(" ")[2] - for line in mount_lines - if len(line) >= 3 and line.split(" ")[2].startswith(directory)] - ret = 0 - for point in reversed(points_to_umount): - ret = subprocess.call(["umount", point]) - if ret != 0: - ret = 1 - logger.warning(m18n.n('backup_cleaning_failed', point)) - continue - - return ret - def _check_is_enough_free_space(self): """ Check free space in repository or output directory before to backup @@ -1621,9 +1620,18 @@ class BackupMethod(object): # 'NUMBER OF HARD LINKS > 1' see #1043 cron_path = os.path.abspath('/etc/cron') + '.' if not os.path.abspath(src).startswith(cron_path): - os.link(src, dest) - # Success, go to next file to organize - continue + try: + os.link(src, dest) + except Exception as e: + # This kind of situation may happen when src and dest are on different + # logical volume ... even though the st_dev check previously match... + # E.g. this happens when running an encrypted hard drive + # where everything is mapped to /dev/mapper/some-stuff + # yet there are different devices behind it or idk ... + logger.warning("Could not link %s to %s (%s) ... falling back to regular copy." % (src, dest, str(e))) + else: + # Success, go to next file to organize + continue # If mountbind or hardlink couldnt be created, # prepare a list of files that need to be copied @@ -1932,8 +1940,9 @@ class CustomBackupMethod(BackupMethod): ret = hook_callback('backup_method', [self.method], args=self._get_args('need_mount')) - - self._need_mount = True if ret['succeed'] else False + ret_succeed = [hook for hook, infos in ret.items() + if any(result["state"] == "succeed" for result in infos.values())] + self._need_mount = True if ret_succeed else False return self._need_mount def backup(self): @@ -1946,7 +1955,10 @@ class CustomBackupMethod(BackupMethod): ret = hook_callback('backup_method', [self.method], args=self._get_args('backup')) - if ret['failed']: + + ret_failed = [hook for hook, infos in ret.items() + if any(result["state"] == "failed" for result in infos.values())] + if ret_failed: raise YunohostError('backup_custom_backup_error') def mount(self, restore_manager): @@ -1959,7 +1971,10 @@ class CustomBackupMethod(BackupMethod): super(CustomBackupMethod, self).mount(restore_manager) ret = hook_callback('backup_method', [self.method], args=self._get_args('mount')) - if ret['failed']: + + ret_failed = [hook for hook, infos in ret.items() + if any(result["state"] == "failed" for result in infos.values())] + if ret_failed: raise YunohostError('backup_custom_mount_error') def _get_args(self, action): @@ -2011,6 +2026,7 @@ def backup_create(name=None, description=None, methods=[], # Check that output directory is empty if os.path.isdir(output_directory) and no_compress and \ os.listdir(output_directory): + raise YunohostError('backup_output_directory_not_empty') elif no_compress: raise YunohostError('backup_output_directory_required') @@ -2315,6 +2331,30 @@ def _call_for_each_path(self, callback, csv_path=None): callback(self, row['source'], row['dest']) +def _recursive_umount(directory): + """ + Recursively umount sub directories of a directory + + Args: + directory -- a directory path + """ + mount_lines = subprocess.check_output("mount").split("\n") + + points_to_umount = [line.split(" ")[2] + for line in mount_lines + if len(line) >= 3 and line.split(" ")[2].startswith(directory)] + + everything_went_fine = True + for point in reversed(points_to_umount): + ret = subprocess.call(["umount", point]) + if ret != 0: + everything_went_fine = False + logger.warning(m18n.n('backup_cleaning_failed', point)) + continue + + return everything_went_fine + + def free_space_in_directory(dirpath): stat = os.statvfs(dirpath) return stat.f_frsize * stat.f_bavail diff --git a/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py b/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py index 080cc0163..d188ff024 100644 --- a/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py +++ b/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py @@ -49,10 +49,6 @@ class MyMigration(Migration): if dsa: settings_set("service.ssh.allow_deprecated_dsa_hostkey", True) - # Create sshd_config.d dir - if not os.path.exists(SSHD_CONF + '.d'): - mkdir(SSHD_CONF + '.d', 0o755, uid='root', gid='root') - # Here, we make it so that /etc/ssh/sshd_config is managed # by the regen conf (in particular in the case where the # from_script flag is present - in which case it was *not* diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index 2f8d63135..2dadcef52 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -119,6 +119,9 @@ def dyndns_subscribe(operation_logger, subscribe_host="dyndns.yunohost.org", dom subscribe_host -- Dynette HTTP API to subscribe to """ + if len(glob.glob('/etc/yunohost/dyndns/*.key')) != 0 or os.path.exists('/etc/cron.d/yunohost-dyndns'): + raise YunohostError('domain_dyndns_already_subscribed') + if domain is None: domain = _get_maindomain() operation_logger.related_to.append(('domain', domain)) @@ -144,7 +147,8 @@ def dyndns_subscribe(operation_logger, subscribe_host="dyndns.yunohost.org", dom 'dnssec-keygen -a hmac-sha512 -b 512 -r /dev/urandom -n USER %s' % domain) os.system('chmod 600 /etc/yunohost/dyndns/*.key /etc/yunohost/dyndns/*.private') - key_file = glob.glob('/etc/yunohost/dyndns/*.key')[0] + private_file = glob.glob('/etc/yunohost/dyndns/*%s*.private' % domain)[0] + key_file = glob.glob('/etc/yunohost/dyndns/*%s*.key' % domain)[0] with open(key_file) as f: key = f.readline().strip().split(' ', 6)[-1] @@ -152,9 +156,13 @@ def dyndns_subscribe(operation_logger, subscribe_host="dyndns.yunohost.org", dom # Send subscription try: r = requests.post('https://%s/key/%s?key_algo=hmac-sha512' % (subscribe_host, base64.b64encode(key)), data={'subdomain': domain}, timeout=30) - except requests.ConnectionError: - raise YunohostError('no_internet_connection') + except Exception as e: + os.system("rm -f %s" % private_file) + os.system("rm -f %s" % key_file) + raise YunohostError('dyndns_registration_failed', error=str(e)) if r.status_code != 201: + os.system("rm -f %s" % private_file) + os.system("rm -f %s" % key_file) try: error = json.loads(r.text)['error'] except: @@ -333,7 +341,8 @@ def _guess_current_dyndns_domain(dyn_host): """ # Retrieve the first registered domain - for path in glob.iglob('/etc/yunohost/dyndns/K*.private'): + paths = list(glob.iglob('/etc/yunohost/dyndns/K*.private')) + for path in paths: match = RE_DYNDNS_PRIVATE_KEY_MD5.match(path) if not match: match = RE_DYNDNS_PRIVATE_KEY_SHA512.match(path) @@ -343,7 +352,9 @@ def _guess_current_dyndns_domain(dyn_host): # Verify if domain is registered (i.e., if it's available, skip # current domain beause that's not the one we want to update..) - if _dyndns_available(dyn_host, _domain): + # If there's only 1 such key found, then avoid doing the request + # for nothing (that's very probably the one we want to find ...) + if len(paths) > 1 and _dyndns_available(dyn_host, _domain): continue else: return (_domain, path) diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index ca93c7f03..c4605b6e8 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -31,6 +31,7 @@ from glob import iglob from moulinette import m18n from yunohost.utils.error import YunohostError from moulinette.utils import log +from moulinette.utils.filesystem import read_json HOOK_FOLDER = '/usr/share/yunohost/hooks/' CUSTOM_HOOK_FOLDER = '/etc/yunohost/hooks.d/' @@ -228,7 +229,7 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None, (name, priority, path, succeed) as arguments """ - result = {'succeed': {}, 'failed': {}} + result = {} hooks_dict = {} # Retrieve hooks @@ -278,20 +279,20 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None, try: hook_args = pre_callback(name=name, priority=priority, path=path, args=args) - hook_exec(path, args=hook_args, chdir=chdir, env=env, - no_trace=no_trace, raise_on_error=True) + hook_return = hook_exec(path, args=hook_args, chdir=chdir, env=env, + no_trace=no_trace, raise_on_error=True)[1] except YunohostError as e: state = 'failed' + hook_return = {} logger.error(e.strerror, exc_info=1) post_callback(name=name, priority=priority, path=path, succeed=False) else: post_callback(name=name, priority=priority, path=path, succeed=True) - try: - result[state][name].append(path) - except KeyError: - result[state][name] = [path] + if not name in result: + result[name] = {} + result[name][path] = {'state' : state, 'stdreturn' : hook_return } return result @@ -339,6 +340,11 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False, stdinfo = os.path.join(tempfile.mkdtemp(), "stdinfo") env['YNH_STDINFO'] = stdinfo + stdreturn = os.path.join(tempfile.mkdtemp(), "stdreturn") + with open(stdreturn, 'w') as f: + f.write('') + env['YNH_STDRETURN'] = stdreturn + # Construct command to execute if user == "root": command = ['sh', '-c'] @@ -385,10 +391,27 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False, raise YunohostError('hook_exec_not_terminated', path=path) else: logger.error(m18n.n('hook_exec_not_terminated', path=path)) - return 1 + return 1, {} elif raise_on_error and returncode != 0: raise YunohostError('hook_exec_failed', path=path) - return returncode + + raw_content = None + try: + with open(stdreturn, 'r') as f: + raw_content = f.read() + if raw_content != '': + returnjson = read_json(stdreturn) + else: + returnjson = {} + except Exception as e: + raise YunohostError('hook_json_return_error', path=path, msg=str(e), + raw_content=raw_content) + finally: + stdreturndir = os.path.split(stdreturn)[0] + os.remove(stdreturn) + os.rmdir(stdreturndir) + + return returncode, returnjson def _extract_filename_parts(filename): diff --git a/src/yunohost/service.py b/src/yunohost/service.py index 60729053b..61274aaac 100644 --- a/src/yunohost/service.py +++ b/src/yunohost/service.py @@ -493,12 +493,16 @@ def service_regen_conf(operation_logger, names=[], with_diff=False, force=False, pre_result = hook_callback('conf_regen', names, pre_callback=_pre_call) - # Update the services name - names = pre_result['succeed'].keys() + # Keep only the hook names with at least one success + names = [hook for hook, infos in pre_result.items() + if any(result["state"] == "succeed" for result in infos.values())] + # FIXME : what do in case of partial success/failure ... if not names: + ret_failed = [hook for hook, infos in pre_result.items() + if any(result["state"] == "failed" for result in infos.values())] raise YunohostError('service_regenconf_failed', - services=', '.join(pre_result['failed'])) + services=', '.join(ret_failed)) # Set the processing method _regen = _process_regen_conf if not dry_run else lambda *a, **k: True diff --git a/src/yunohost/tests/test_backuprestore.py b/src/yunohost/tests/test_backuprestore.py index 14c479d9a..353b88f27 100644 --- a/src/yunohost/tests/test_backuprestore.py +++ b/src/yunohost/tests/test_backuprestore.py @@ -10,7 +10,7 @@ from moulinette import m18n from moulinette.core import init_authenticator from yunohost.app import app_install, app_remove, app_ssowatconf from yunohost.app import _is_installed -from yunohost.backup import backup_create, backup_restore, backup_list, backup_info, backup_delete +from yunohost.backup import backup_create, backup_restore, backup_list, backup_info, backup_delete, _recursive_umount from yunohost.domain import _get_maindomain from yunohost.utils.error import YunohostError @@ -42,7 +42,7 @@ def setup_function(function): assert len(backup_list()["archives"]) == 0 - markers = function.__dict__.keys() + markers = [m.name for m in function.__dict__.get("pytestmark",[])] if "with_wordpress_archive_from_2p4" in markers: add_archive_wordpress_from_2p4() @@ -82,7 +82,7 @@ def teardown_function(function): delete_all_backups() uninstall_test_apps_if_needed() - markers = function.__dict__.keys() + markers = [m.name for m in function.__dict__.get("pytestmark",[])] if "clean_opt_dir" in markers: shutil.rmtree("/opt/test_backup_output_directory") @@ -571,7 +571,7 @@ def test_backup_binds_are_readonly(monkeypatch): assert "Read-only file system" in output - if self._recursive_umount(self.work_dir) > 0: + if not _recursive_umount(self.work_dir): raise Exception("Backup cleaning failed !") self.clean()