diff --git a/bin/yunohost-api b/bin/yunohost-api index 6e47df7ef..e518c34b0 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -201,12 +201,10 @@ if __name__ == '__main__': _init_moulinette(opts.use_websocket, opts.debug, opts.verbose) # Run the server - from yunohost.utils.packages import ynh_packages_version ret = moulinette.api( _retrieve_namespaces(), host=opts.host, port=opts.port, routes={ ('GET', '/installed'): is_installed, - ('GET', '/version'): ynh_packages_version, }, use_cache=opts.use_cache, use_websocket=opts.use_websocket ) sys.exit(ret) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index c96315238..1532601a8 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -288,3 +288,150 @@ ynh_remove_fpm_config () { ynh_secure_remove --file="$fpm_config_dir/conf.d/20-$app.ini" 2>&1 sudo systemctl reload $fpm_service } + +# Create a dedicated fail2ban config (jail and filter conf files) +# +# usage 1: ynh_add_fail2ban_config --logpath=log_file --failregex=filter [--max_retry=max_retry] [--ports=ports] +# | arg: -l, --logpath= - Log file to be checked by fail2ban +# | arg: -r, --failregex= - Failregex to be looked for by fail2ban +# | arg: -m, --max_retry= - Maximum number of retries allowed before banning IP address - default: 3 +# | arg: -p, --ports= - Ports blocked for a banned IP address - default: http,https +# +# ----------------------------------------------------------------------------- +# +# usage 2: ynh_add_fail2ban_config --use_template [--others_var="list of others variables to replace"] +# | arg: -t, --use_template - Use this helper in template mode +# | arg: -v, --others_var= - List of others variables to replace separeted by a space +# | for example : 'var_1 var_2 ...' +# +# This will use a template in ../conf/f2b_jail.conf and ../conf/f2b_filter.conf +# __APP__ by $app +# +# You can dynamically replace others variables by example : +# __VAR_1__ by $var_1 +# __VAR_2__ by $var_2 +# +# Generally your template will look like that by example (for synapse): +# +# f2b_jail.conf: +# [__APP__] +# enabled = true +# port = http,https +# filter = __APP__ +# logpath = /var/log/__APP__/logfile.log +# maxretry = 3 +# +# f2b_filter.conf: +# [INCLUDES] +# before = common.conf +# [Definition] +# +# # Part of regex definition (just used to make more easy to make the global regex) +# __synapse_start_line = .? \- synapse\..+ \- +# +# # Regex definition. +# failregex = ^%(__synapse_start_line)s INFO \- POST\-(\d+)\- \- \d+ \- Received request\: POST /_matrix/client/r0/login\??%(__synapse_start_line)s INFO \- POST\-\1\- Got login request with identifier: \{u'type': u'm.id.user', u'user'\: u'(.+?)'\}, medium\: None, address: None, user\: u'\5'%(__synapse_start_line)s WARNING \- \- (Attempted to login as @\5\:.+ but they do not exist|Failed password login for user @\5\:.+)$ +# +# ignoreregex = +# +# ----------------------------------------------------------------------------- +# +# Note about the "failregex" option: +# regex to match the password failure messages in the logfile. The +# host must be matched by a group named "host". The tag "" can +# be used for standard IP/hostname matching and is only an alias for +# (?:::f{4,6}:)?(?P[\w\-.^_]+) +# +# You can find some more explainations about how to make a regex here : +# https://www.fail2ban.org/wiki/index.php/MANUAL_0_8#Filters +# +# Note that the logfile need to exist before to call this helper !! +# +# To validate your regex you can test with this command: +# fail2ban-regex /var/log/YOUR_LOG_FILE_PATH /etc/fail2ban/filter.d/YOUR_APP.conf +# +ynh_add_fail2ban_config () { + # Declare an array to define the options of this helper. + local legacy_args=lrmptv + declare -Ar args_array=( [l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= [t]=use_template [v]=others_var=) + local logpath + local failregex + local max_retry + local ports + local others_var + local use_template + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + use_template="${use_template:-0}" + max_retry=${max_retry:-3} + ports=${ports:-http,https} + + finalfail2banjailconf="/etc/fail2ban/jail.d/$app.conf" + finalfail2banfilterconf="/etc/fail2ban/filter.d/$app.conf" + ynh_backup_if_checksum_is_different "$finalfail2banjailconf" + ynh_backup_if_checksum_is_different "$finalfail2banfilterconf" + + if [ $use_template -eq 1 ] + then + # Usage 2, templates + cp ../conf/f2b_jail.conf $finalfail2banjailconf + cp ../conf/f2b_filter.conf $finalfail2banfilterconf + + if [ -n "${app:-}" ] + then + ynh_replace_string "__APP__" "$app" "$finalfail2banjailconf" + ynh_replace_string "__APP__" "$app" "$finalfail2banfilterconf" + fi + + # Replace all other variable given as arguments + for var_to_replace in ${others_var:-}; do + # ${var_to_replace^^} make the content of the variable on upper-cases + # ${!var_to_replace} get the content of the variable named $var_to_replace + ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banjailconf" + ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banfilterconf" + done + + else + # Usage 1, no template. Build a config file from scratch. + test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." + test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." + + tee $finalfail2banjailconf <&2 + fi + + if [ "$trace" == "1" ] + then + ynh_debug --message="Enable debugging" + set +x + # Get the current file descriptor of xtrace + old_bash_xtracefd=$BASH_XTRACEFD + # Add the current file name and the line number of any command currently running while tracing. + PS4='$(basename ${BASH_SOURCE[0]})-L${LINENO}: ' + # Force xtrace to stderr + BASH_XTRACEFD=2 + fi + if [ "$trace" == "0" ] + then + ynh_debug --message="Disable debugging" + set +x + # Put xtrace back to its original fild descriptor + BASH_XTRACEFD=$old_bash_xtracefd + fi + # Renable set xtrace + set -x +} + +# Execute a command and print the result as debug +# +# usage: ynh_debug_exec command to execute +# usage: ynh_debug_exec "command to execute | following command" +# In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe. +# +# | arg: command - command to execute +ynh_debug_exec () { + ynh_debug --message="$(eval $@)" +} diff --git a/data/helpers.d/filesystem b/data/helpers.d/filesystem index 88d51750b..10123dea4 100644 --- a/data/helpers.d/filesystem +++ b/data/helpers.d/filesystem @@ -323,6 +323,16 @@ ynh_store_file_checksum () { local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(sudo md5sum "$file" | cut -d' ' -f1) + + # If backup_file_checksum isn't empty, ynh_backup_if_checksum_is_different has made a backup + if [ -n "${backup_file_checksum-}" ] + then + # Print the diff between the previous file and the new one. + # diff return 1 if the files are different, so the || true + diff --report-identical-files --unified --color=always $backup_file_checksum $file >&2 || true + fi + # Unset the variable, so it wouldn't trig a ynh_store_file_checksum without a ynh_backup_if_checksum_is_different before it. + unset backup_file_checksum } # Verify the checksum and backup the file if it's different @@ -345,15 +355,17 @@ ynh_backup_if_checksum_is_different () { local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name) + # backup_file_checksum isn't declare as local, so it can be reuse by ynh_store_file_checksum + backup_file_checksum="" if [ -n "$checksum_value" ] then # Proceed only if a value was stored into the app settings if ! echo "$checksum_value $file" | sudo md5sum -c --status then # If the checksum is now different - local backup_file="/home/yunohost.conf/backup/$file.backup.$(date '+%Y%m%d.%H%M%S')" - sudo mkdir -p "$(dirname "$backup_file")" - sudo cp -a "$file" "$backup_file" # Backup the current file - echo "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file" >&2 - echo "$backup_file" # Return the name of the backup file + backup_file_checksum="/home/yunohost.conf/backup/$file.backup.$(date '+%Y%m%d.%H%M%S')" + sudo mkdir -p "$(dirname "$backup_file_checksum")" + sudo cp -a "$file" "$backup_file_checksum" # Backup the current file + ynh_print_warn "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum" + echo "$backup_file_checksum" # Return the name of the backup file fi fi } diff --git a/data/helpers.d/nodejs b/data/helpers.d/nodejs index c4332b60c..098ed4410 100644 --- a/data/helpers.d/nodejs +++ b/data/helpers.d/nodejs @@ -98,7 +98,13 @@ ynh_install_nodejs () { test -x /usr/bin/npm_n && mv /usr/bin/npm_n /usr/bin/npm # Install the requested version of nodejs - n $nodejs_version + uname=$(uname -m) + if [[ $uname =~ aarch64 || $uname =~ arm64 ]] + then + n $nodejs_version --arch=arm64 + else + n $nodejs_version + fi # Find the last "real" version for this major version of node. real_nodejs_version=$(find $node_version_path/$nodejs_version* -maxdepth 0 | sort --version-sort | tail --lines=1) diff --git a/data/helpers.d/package b/data/helpers.d/package index c0617abb2..000b0ee74 100644 --- a/data/helpers.d/package +++ b/data/helpers.d/package @@ -17,6 +17,21 @@ ynh_wait_dpkg_free() { # Sleep an exponential time at each round sleep $(( try * try )) else + # Check if dpkg hasn't been interrupted and is fully available. + # See this for more information: https://sources.debian.org/src/apt/1.4.9/apt-pkg/deb/debsystem.cc/#L141-L174 + local dpkg_dir="/var/lib/dpkg/updates/" + + # For each file in $dpkg_dir + while read dpkg_file <&9 + do + # Check if the name of this file contains only numbers. + if echo "$dpkg_file" | grep -Pq "^[[:digit:]]+$" + then + # If so, that a remaining of dpkg. + ynh_print_err "E: dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem." + return 1 + fi + done 9<<< "$(ls -1 $dpkg_dir)" return 0 fi done @@ -71,7 +86,7 @@ ynh_package_version() { # usage: ynh_apt update ynh_apt() { ynh_wait_dpkg_free - DEBIAN_FRONTEND=noninteractive sudo apt-get -y $@ + DEBIAN_FRONTEND=noninteractive apt-get -y $@ } # Update package index files diff --git a/data/helpers.d/print b/data/helpers.d/print index 7f37021ae..b46192d87 100644 --- a/data/helpers.d/print +++ b/data/helpers.d/print @@ -89,6 +89,7 @@ ynh_print_err () { # usage: ynh_exec_err command to execute # usage: ynh_exec_err "command to execute | following command" # In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe. +# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed. # # | arg: command - command to execute ynh_exec_err () { @@ -100,6 +101,7 @@ ynh_exec_err () { # usage: ynh_exec_warn command to execute # usage: ynh_exec_warn "command to execute | following command" # In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe. +# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed. # # | arg: command - command to execute ynh_exec_warn () { @@ -111,6 +113,7 @@ ynh_exec_warn () { # usage: ynh_exec_warn_less command to execute # usage: ynh_exec_warn_less "command to execute | following command" # In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe. +# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed. # # | arg: command - command to execute ynh_exec_warn_less () { @@ -122,6 +125,7 @@ ynh_exec_warn_less () { # usage: ynh_exec_quiet command to execute # usage: ynh_exec_quiet "command to execute | following command" # In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe. +# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed. # # | arg: command - command to execute ynh_exec_quiet () { @@ -133,6 +137,7 @@ ynh_exec_quiet () { # usage: ynh_exec_fully_quiet command to execute # usage: ynh_exec_fully_quiet "command to execute | following command" # In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe. +# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed. # # | arg: command - command to execute ynh_exec_fully_quiet () { diff --git a/data/helpers.d/system b/data/helpers.d/system index 968135f16..9a4219e11 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -55,3 +55,107 @@ ynh_abort_if_errors () { ynh_get_debian_release () { echo $(lsb_release --codename --short) } + +# Read the value of a key in a ynh manifest file +# +# usage: ynh_read_manifest manifest key +# | arg: -m, --manifest= - Path of the manifest to read +# | arg: -k, --key= - Name of the key to find +ynh_read_manifest () { + # Declare an array to define the options of this helper. + declare -Ar args_array=( [m]=manifest= [k]=manifest_key= ) + local manifest + local manifest_key + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if [ ! -e "$manifest" ]; then + # If the manifest isn't found, try the common place for backup and restore script. + manifest="../settings/manifest.json" + fi + + jq ".$manifest_key" "$manifest" --raw-output +} + +# Read the upstream version from the manifest +# The version number in the manifest is defined by ~ynh +# For example : 4.3-2~ynh3 +# This include the number before ~ynh +# In the last example it return 4.3-2 +# +# usage: ynh_app_upstream_version [-m manifest] +# | arg: -m, --manifest= - Path of the manifest to read +ynh_app_upstream_version () { + declare -Ar args_array=( [m]=manifest= ) + local manifest + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + manifest="${manifest:-../manifest.json}" + version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") + echo "${version_key/~ynh*/}" +} + +# Read package version from the manifest +# The version number in the manifest is defined by ~ynh +# For example : 4.3-2~ynh3 +# This include the number after ~ynh +# In the last example it return 3 +# +# usage: ynh_app_package_version [-m manifest] +# | arg: -m, --manifest= - Path of the manifest to read +ynh_app_package_version () { + declare -Ar args_array=( [m]=manifest= ) + local manifest + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + manifest="${manifest:-../manifest.json}" + version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") + echo "${version_key/*~ynh/}" +} + +# Checks the app version to upgrade with the existing app version and returns: +# - UPGRADE_APP if the upstream app version has changed +# - UPGRADE_PACKAGE if only the YunoHost package has changed +# +## It stops the current script without error if the package is up-to-date +# +# This helper should be used to avoid an upgrade of an app, or the upstream part +# of it, when it's not needed +# +# To force an upgrade, even if the package is up to date, +# you have to set the variable YNH_FORCE_UPGRADE before. +# example: sudo YNH_FORCE_UPGRADE=1 yunohost app upgrade MyApp +# +# usage: ynh_check_app_version_changed +ynh_check_app_version_changed () { + local force_upgrade=${YNH_FORCE_UPGRADE:-0} + local package_check=${PACKAGE_CHECK_EXEC:-0} + + # By default, upstream app version has changed + local return_value="UPGRADE_APP" + + local current_version=$(ynh_read_manifest --manifest="/etc/yunohost/apps/$YNH_APP_INSTANCE_NAME/manifest.json" --manifest_key="version" || echo 1.0) + local current_upstream_version="$(ynh_app_upstream_version --manifest="/etc/yunohost/apps/$YNH_APP_INSTANCE_NAME/manifest.json")" + local update_version=$(ynh_read_manifest --manifest="../manifest.json" --manifest_key="version" || echo 1.0) + local update_upstream_version="$(ynh_app_upstream_version)" + + if [ "$current_version" == "$update_version" ] ; then + # Complete versions are the same + if [ "$force_upgrade" != "0" ] + then + echo "Upgrade forced by YNH_FORCE_UPGRADE." >&2 + unset YNH_FORCE_UPGRADE + elif [ "$package_check" != "0" ] + then + echo "Upgrade forced for package check." >&2 + else + ynh_die "Up-to-date, nothing to do" 0 + fi + elif [ "$current_upstream_version" == "$update_upstream_version" ] ; then + # Upstream versions are the same, only YunoHost package versions differ + return_value="UPGRADE_PACKAGE" + fi + echo $return_value +} diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 360174847..5ba2946a2 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -257,7 +257,14 @@ ynh_setup_source () { # | arg: ... - (Optionnal) More POST keys and values ynh_local_curl () { # Define url of page to curl - local full_page_url=https://localhost$path_url$1 + local local_page=$(ynh_normalize_url_path $1) + local full_path=$path_url$local_page + + if [ "${path_url}" == "/" ]; then + full_path=$local_page + fi + + local full_page_url=https://localhost$full_path # Concatenate all other arguments with '&' to prepare POST data local POST_data="" diff --git a/data/hooks/backup/05-conf_ldap b/data/hooks/backup/05-conf_ldap index b21103ede..9ae22095e 100755 --- a/data/hooks/backup/05-conf_ldap +++ b/data/hooks/backup/05-conf_ldap @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/ldap" diff --git a/data/hooks/backup/08-conf_ssh b/data/hooks/backup/08-conf_ssh index ae422617e..ee976080c 100755 --- a/data/hooks/backup/08-conf_ssh +++ b/data/hooks/backup/08-conf_ssh @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/ssh" diff --git a/data/hooks/backup/11-conf_ynh_mysql b/data/hooks/backup/11-conf_ynh_mysql index 60bd8c017..031707337 100755 --- a/data/hooks/backup/11-conf_ynh_mysql +++ b/data/hooks/backup/11-conf_ynh_mysql @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/ynh/mysql" diff --git a/data/hooks/backup/14-conf_ssowat b/data/hooks/backup/14-conf_ssowat index ca42d3369..d4db72493 100755 --- a/data/hooks/backup/14-conf_ssowat +++ b/data/hooks/backup/14-conf_ssowat @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/ssowat" diff --git a/data/hooks/backup/17-data_home b/data/hooks/backup/17-data_home index f7a797b6b..af00d67e8 100755 --- a/data/hooks/backup/17-data_home +++ b/data/hooks/backup/17-data_home @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/data/home" diff --git a/data/hooks/backup/20-conf_ynh_firewall b/data/hooks/backup/20-conf_ynh_firewall index 4e08114e7..98be3eb09 100755 --- a/data/hooks/backup/20-conf_ynh_firewall +++ b/data/hooks/backup/20-conf_ynh_firewall @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/ynh/firewall" diff --git a/data/hooks/backup/21-conf_ynh_certs b/data/hooks/backup/21-conf_ynh_certs index f9687164d..a3912a995 100755 --- a/data/hooks/backup/21-conf_ynh_certs +++ b/data/hooks/backup/21-conf_ynh_certs @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/ynh/certs" diff --git a/data/hooks/backup/23-data_mail b/data/hooks/backup/23-data_mail index 618a0aafe..7fdc883fd 100755 --- a/data/hooks/backup/23-data_mail +++ b/data/hooks/backup/23-data_mail @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/data/mail" diff --git a/data/hooks/backup/26-conf_xmpp b/data/hooks/backup/26-conf_xmpp index 12300a00a..b55ad2bfc 100755 --- a/data/hooks/backup/26-conf_xmpp +++ b/data/hooks/backup/26-conf_xmpp @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/xmpp" diff --git a/data/hooks/backup/29-conf_nginx b/data/hooks/backup/29-conf_nginx index d900c7535..81e145e24 100755 --- a/data/hooks/backup/29-conf_nginx +++ b/data/hooks/backup/29-conf_nginx @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/nginx" diff --git a/data/hooks/backup/32-conf_cron b/data/hooks/backup/32-conf_cron index 2fea9f53f..063ec1a3f 100755 --- a/data/hooks/backup/32-conf_cron +++ b/data/hooks/backup/32-conf_cron @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/cron" diff --git a/data/hooks/backup/40-conf_ynh_currenthost b/data/hooks/backup/40-conf_ynh_currenthost index e4a684576..6a98fd0d2 100755 --- a/data/hooks/backup/40-conf_ynh_currenthost +++ b/data/hooks/backup/40-conf_ynh_currenthost @@ -4,7 +4,7 @@ set -eu # Source YNH helpers -source /usr/share/yunohost/helpers.d/filesystem +source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/conf/ynh" diff --git a/data/hooks/conf_regen/03-ssh b/data/hooks/conf_regen/03-ssh index 9de527518..5bb9cf916 100755 --- a/data/hooks/conf_regen/03-ssh +++ b/data/hooks/conf_regen/03-ssh @@ -2,7 +2,7 @@ set -e -. /usr/share/yunohost/helpers.d/utils +. /usr/share/yunohost/helpers do_pre_regen() { pending_dir=$1 diff --git a/data/hooks/conf_regen/15-nginx b/data/hooks/conf_regen/15-nginx index 461c10c0c..7ca63c003 100755 --- a/data/hooks/conf_regen/15-nginx +++ b/data/hooks/conf_regen/15-nginx @@ -2,7 +2,7 @@ set -e -. /usr/share/yunohost/helpers.d/utils +. /usr/share/yunohost/helpers do_init_regen() { if [[ $EUID -ne 0 ]]; then diff --git a/data/hooks/conf_regen/34-mysql b/data/hooks/conf_regen/34-mysql index 5ee91827b..9f35fec18 100755 --- a/data/hooks/conf_regen/34-mysql +++ b/data/hooks/conf_regen/34-mysql @@ -2,6 +2,7 @@ set -e MYSQL_PKG="mariadb-server-10.1" +. /usr/share/yunohost/helpers do_pre_regen() { pending_dir=$1 @@ -15,7 +16,6 @@ do_post_regen() { regen_conf_files=$1 if [ ! -f /etc/yunohost/mysql ]; then - . /usr/share/yunohost/helpers.d/string # ensure that mysql is running sudo systemctl -q is-active mysql.service \ @@ -25,8 +25,6 @@ do_post_regen() { mysql_password=$(ynh_string_random 10) sudo mysqladmin -s -u root -pyunohost password "$mysql_password" || { if [ $FORCE -eq 1 ]; then - . /usr/share/yunohost/helpers.d/package - echo "It seems that you have already configured MySQL." \ "YunoHost needs to have a root access to MySQL to runs its" \ "applications, and is going to reset the MySQL root password." \ diff --git a/data/hooks/conf_regen/43-dnsmasq b/data/hooks/conf_regen/43-dnsmasq index 2c8ce797b..ed795c058 100755 --- a/data/hooks/conf_regen/43-dnsmasq +++ b/data/hooks/conf_regen/43-dnsmasq @@ -1,13 +1,11 @@ #!/bin/bash set -e +. /usr/share/yunohost/helpers do_pre_regen() { pending_dir=$1 - # source ip helpers - . /usr/share/yunohost/helpers.d/ip - cd /usr/share/yunohost/templates/dnsmasq # create directory for pending conf diff --git a/data/hooks/restore/11-conf_ynh_mysql b/data/hooks/restore/11-conf_ynh_mysql index 0aaaccd54..1336a2cc2 100644 --- a/data/hooks/restore/11-conf_ynh_mysql +++ b/data/hooks/restore/11-conf_ynh_mysql @@ -1,6 +1,8 @@ backup_dir="$1/conf/ynh/mysql" MYSQL_PKG="mariadb-server-10.1" +. /usr/share/yunohost/helpers + # ensure that mysql is running service mysql status >/dev/null 2>&1 \ || service mysql start @@ -11,13 +13,11 @@ service mysql status >/dev/null 2>&1 \ new_pwd=$(sudo cat "${backup_dir}/root_pwd" || sudo cat "${backup_dir}/mysql") [ -z "$curr_pwd" ] && curr_pwd="yunohost" [ -z "$new_pwd" ] && { - . /usr/share/yunohost/helpers.d/string new_pwd=$(ynh_string_random 10) } # attempt to change it sudo mysqladmin -s -u root -p"$curr_pwd" password "$new_pwd" || { - . /usr/share/yunohost/helpers.d/package echo "It seems that you have already configured MySQL." \ "YunoHost needs to have a root access to MySQL to runs its" \ diff --git a/data/templates/dnsmasq/plain/resolv.dnsmasq.conf b/data/templates/dnsmasq/plain/resolv.dnsmasq.conf index 7eed1142f..197ee2d64 100644 --- a/data/templates/dnsmasq/plain/resolv.dnsmasq.conf +++ b/data/templates/dnsmasq/plain/resolv.dnsmasq.conf @@ -9,25 +9,37 @@ # (FR) FDN nameserver 80.67.169.12 +nameserver 2001:910:800::12 nameserver 80.67.169.40 +nameserver 2001:910:800::40 # (FR) LDN nameserver 80.67.188.188 +nameserver 2001:913::8 # (FR) ARN nameserver 89.234.141.66 +nameserver 2a00:5881:8100:1000::3 # (FR) Aquilenet nameserver 185.233.100.100 +nameserver 2a0c:e300::100 nameserver 185.233.100.101 +nameserver 2a0c:e300::101 # (FR) gozmail / grifon nameserver 80.67.190.200 +nameserver 2a00:5884:8218::1 # (DE) FoeBud / Digital Courage nameserver 85.214.20.141 # (DE) CCC Berlin nameserver 195.160.173.53 # (DE) AS250 nameserver 194.150.168.168 +nameserver 2001:4ce8::53 # (DE) Ideal-Hosting nameserver 84.200.69.80 +nameserver 2001:1608:10:25::1c04:b12f nameserver 84.200.70.40 +nameserver 2001:1608:10:25::9249:d69b # (DK) censurfridns nameserver 91.239.100.100 +nameserver 2001:67c:28a4:: nameserver 89.233.43.71 +nameserver 2002:d596:2a92:1:71:53:: diff --git a/data/templates/nginx/plain/yunohost_admin.conf b/data/templates/nginx/plain/yunohost_admin.conf index b6fabf8e3..2493e4033 100644 --- a/data/templates/nginx/plain/yunohost_admin.conf +++ b/data/templates/nginx/plain/yunohost_admin.conf @@ -60,7 +60,8 @@ server { if ($http_user_agent ~ (crawl|Googlebot|Slurp|spider|bingbot|tracker|click|parser|spider|facebookexternalhit) ) { return 403; } - + # X-Robots-Tag to precise the rules applied. + add_header X-Robots-Tag "nofollow, noindex, noarchive, nosnippet"; # Redirect most of 404 to maindomain.tld/yunohost/sso access_by_lua_file /usr/share/ssowat/access.lua; } diff --git a/data/templates/nginx/server.tpl.conf b/data/templates/nginx/server.tpl.conf index 0c221f188..43d38ca98 100644 --- a/data/templates/nginx/server.tpl.conf +++ b/data/templates/nginx/server.tpl.conf @@ -51,7 +51,9 @@ server { # Follows the Web Security Directives from the Mozilla Dev Lab and the Mozilla Obervatory + Partners # https://wiki.mozilla.org/Security/Guidelines/Web_Security # https://observatory.mozilla.org/ - more_set_headers "Strict-Transport-Security : max-age=63072000; includeSubDomains; preload"; + {% if domain_cert_ca != "Self-signed" %} + more_set_headers "Strict-Transport-Security : max-age=63072000; includeSubDomains; preload"; + {% endif %} more_set_headers "Content-Security-Policy : upgrade-insecure-requests"; more_set_headers "Content-Security-Policy-Report-Only : default-src https: data: 'unsafe-inline' 'unsafe-eval'"; more_set_headers "X-Content-Type-Options : nosniff"; diff --git a/debian/changelog b/debian/changelog index 3f3a6a5ef..7be4212fe 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +yunohost (3.4.2.4) stable; urgency=low + + - [fix] Meltdown vulnerability checker something outputing trash instead of pure json + + -- Alexandre Aubin Tue, 19 Feb 2019 19:11:38 +0000 + yunohost (3.4.2.3) stable; urgency=low - [fix] Admin password appearing in logs after logging in on webadmin diff --git a/debian/control b/debian/control index b359d5ec4..685c194ba 100644 --- a/debian/control +++ b/debian/control @@ -13,7 +13,7 @@ Depends: ${python:Depends}, ${misc:Depends} , moulinette (>= 2.7.1), ssowat (>= 2.7.1) , python-psutil, python-requests, python-dnspython, python-openssl , python-apt, python-miniupnpc, python-dbus, python-jinja2 - , glances + , glances, apt-transport-https , dnsutils, bind9utils, unzip, git, curl, cron, wget, jq , ca-certificates, netcat-openbsd, iproute , mariadb-server, php-mysql | php-mysqlnd diff --git a/locales/en.json b/locales/en.json index ca1d9c8a1..47e452450 100644 --- a/locales/en.json +++ b/locales/en.json @@ -24,7 +24,8 @@ "app_location_install_failed": "Unable to install the app in this location because it conflit with the app '{other_app}' already installed on '{other_path}'", "app_location_unavailable": "This url is not available or conflicts with the already installed app(s):\n{apps:s}", "app_manifest_invalid": "Invalid app manifest: {error}", - "app_no_upgrade": "No app to upgrade", + "app_no_upgrade": "No apps to upgrade", + "app_not_upgraded": "The following apps were not upgraded: {apps}", "app_not_correctly_installed": "{app:s} seems to be incorrectly installed", "app_not_installed": "{app:s} is not installed", "app_not_properly_removed": "{app:s} has not been properly removed", @@ -34,9 +35,14 @@ "app_requirements_failed": "Unable to meet requirements for {app}: {error}", "app_requirements_unmeet": "Requirements are not met for {app}, the package {pkgname} ({version}) must be {spec}", "app_sources_fetch_failed": "Unable to fetch sources files", + "app_start_install": "Installing application {app}…", + "app_start_remove": "Removing application {app}…", + "app_start_backup": "Collecting files to be backuped for {app}…", + "app_start_restore": "Restoring application {app}…", "app_unknown": "Unknown app", "app_unsupported_remote_type": "Unsupported remote type used for the app", - "app_upgrade_app_name": "Upgrading app {app}…", + "app_upgrade_several_apps": "The following apps will be upgraded : {apps}", + "app_upgrade_app_name": "Now upgrading app {app}…", "app_upgrade_failed": "Unable to upgrade {app:s}", "app_upgrade_some_app_failed": "Unable to upgrade some applications", "app_upgraded": "{app:s} has been upgraded", @@ -63,6 +69,7 @@ "ask_path": "Path", "backup_abstract_method": "This backup method hasn't yet been implemented", "backup_action_required": "You must specify something to save", + "backup_actually_backuping": "Now creating a backup archive from the files collected…", "backup_app_failed": "Unable to back up the app '{app:s}'", "backup_applying_method_borg": "Sending all files to backup into borg-backup repository…", "backup_applying_method_copy": "Copying all files to backup…", @@ -75,7 +82,7 @@ "backup_archive_name_unknown": "Unknown local backup archive named '{name:s}'", "backup_archive_open_failed": "Unable to open the backup archive", "backup_archive_system_part_not_available": "System part '{part:s}' not available in this backup", - "backup_archive_writing_error": "Unable to add files to backup into the compressed archive", + "backup_archive_writing_error": "Unable to add files '{source:s}' (named in the archive: '{dest:s}') to backup into the compressed archive '{archive:s}'", "backup_ask_for_copying_if_needed": "Some files couldn't be prepared to be backuped using the method that avoid to temporarily waste space on the system. To perform the backup, {size:s}MB should be used temporarily. Do you agree?", "backup_borg_not_implemented": "Borg backup method is not yet implemented", "backup_cant_mount_uncompress_archive": "Unable to mount in readonly mode the uncompress archive directory", @@ -99,6 +106,7 @@ "backup_method_copy_finished": "Backup copy finished", "backup_method_custom_finished": "Custom backup method '{method:s}' finished", "backup_method_tar_finished": "Backup tar archive created", + "backup_mount_archive_for_restore": "Preparing archive for restoration…", "backup_no_uncompress_archive_dir": "Uncompress archive directory doesn't exist", "backup_nothings_done": "There is nothing to save", "backup_output_directory_forbidden": "Forbidden output directory. Backups can't be created in /bin, /boot, /dev, /etc, /lib, /root, /run, /sbin, /sys, /usr, /var or /home/yunohost.backup/archives sub-folders", @@ -106,7 +114,6 @@ "backup_output_directory_required": "You must provide an output directory for the backup", "backup_output_symlink_dir_broken": "You have a broken symlink instead of your archives directory '{path:s}'. You may have a specific setup to backup your data on an other filesystem, in this case you probably forgot to remount or plug your hard dirve or usb key.", "backup_php5_to_php7_migration_may_fail": "Could not convert your archive to support php7, your php apps may fail to restore (reason: {error:s})", - "backup_running_app_script": "Running backup script of app '{app:s}'…", "backup_running_hooks": "Running backup hooks…", "backup_system_part_failed": "Unable to backup the '{part:s}' system part", "backup_unable_to_organize_files": "Unable to organize files in the archive with the quick method", @@ -146,6 +153,7 @@ "diagnosis_monitor_network_error": "Can't monitor network: {error}", "diagnosis_monitor_system_error": "Can't monitor system: {error}", "diagnosis_no_apps": "No installed application", + "dpkg_is_broken": "You cannot do this right now because dpkg/apt (the system package managers) seems to be in a broken state... You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.", "dnsmasq_isnt_installed": "dnsmasq does not seem to be installed, please run 'apt-get remove bind9 && apt-get install dnsmasq'", "domain_cannot_remove_main": "Cannot remove main domain. Set a new main domain first", "domain_cert_gen_failed": "Unable to generate certificate", @@ -210,7 +218,7 @@ "good_practices_about_admin_password": "You are now about to define a new administration password. The password should be at least 8 characters - though it is good practice to use longer password (i.e. a passphrase) and/or to use various kind of characters (uppercase, lowercase, digits and special characters).", "good_practices_about_user_password": "You are now about to define a new user password. The password should be at least 8 characters - though it is good practice to use longer password (i.e. a passphrase) and/or to use various kind of characters (uppercase, lowercase, digits and special characters).", "hook_exec_failed": "Script execution failed: {path:s}", - "hook_exec_not_terminated": "Script execution hasn\u2019t terminated: {path:s}", + "hook_exec_not_terminated": "Script execution did not finish properly: {path:s}", "hook_json_return_error": "Failed to read return from hook {path:s}. Error: {msg:s}. Raw content: {raw_content}", "hook_list_by_invalid": "Invalid property to list hook by", "hook_name_unknown": "Unknown hook name '{name:s}'", @@ -464,6 +472,7 @@ "ssowat_persistent_conf_write_error": "Error while saving SSOwat persistent configuration: {error:s}. Edit /etc/ssowat/conf.json.persistent file to fix the JSON syntax", "system_upgraded": "The system has been upgraded", "system_username_exists": "Username already exists in the system users", + "this_action_broke_dpkg": "This action broke dpkg/apt (the system package managers)... You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.", "unbackup_app": "App '{app:s}' will not be saved", "unexpected_error": "An unexpected error occured: {error}", "unit_unknown": "Unknown unit '{unit:s}'", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 19a308ffc..de759f04f 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -97,6 +97,9 @@ def app_fetchlist(url=None, name=None): name -- Name of the list url -- URL of remote JSON list """ + if not url.endswith(".json"): + raise YunohostError("This is not a valid application list url. It should end with .json.") + # If needed, create folder where actual appslists are stored if not os.path.exists(REPO_PATH): os.makedirs(REPO_PATH) @@ -564,6 +567,9 @@ def app_upgrade(auth, app=[], url=None, file=None): url -- Git url to fetch for upgrade """ + if packages.dpkg_is_broken(): + raise YunohostError("dpkg_is_broken") + from yunohost.hook import hook_add, hook_remove, hook_exec, hook_callback # Retrieve interface @@ -574,28 +580,31 @@ def app_upgrade(auth, app=[], url=None, file=None): except YunohostError: raise YunohostError('app_no_upgrade') - upgraded_apps = [] + not_upgraded_apps = [] apps = app - user_specified_list = True # If no app is specified, upgrade all apps if not apps: + # FIXME : not sure what's supposed to happen if there is a url and a file but no apps... if not url and not file: apps = [app["id"] for app in app_list(installed=True)["apps"]] - user_specified_list = False elif not isinstance(app, list): apps = [app] - logger.info("Upgrading apps %s", ", ".join(app)) + # Remove possible duplicates + apps = [app for i,app in enumerate(apps) if apps not in apps[:i]] + + # Abort if any of those app is in fact not installed.. + for app in [app for app in apps if not _is_installed(app)]: + raise YunohostError('app_not_installed', app=app) + + if len(apps) == 0: + raise YunohostError('app_no_upgrade') + if len(apps) > 1: + logger.info(m18n.n("app_upgrade_several_apps", apps=", ".join(apps))) for app_instance_name in apps: logger.info(m18n.n('app_upgrade_app_name', app=app_instance_name)) - installed = _is_installed(app_instance_name) - if not installed: - raise YunohostError('app_not_installed', app=app_instance_name) - - if app_instance_name in upgraded_apps: - continue app_dict = app_info(app_instance_name, raw=True) @@ -609,8 +618,7 @@ def app_upgrade(auth, app=[], url=None, file=None): elif app_dict["upgradable"] == "yes": manifest, extracted_app_folder = _fetch_app_from_git(app_instance_name) else: - if user_specified_list: - logger.success(m18n.n('app_already_up_to_date', app=app_instance_name)) + logger.success(m18n.n('app_already_up_to_date', app=app_instance_name)) continue # Check requirements @@ -648,6 +656,7 @@ def app_upgrade(auth, app=[], url=None, file=None): if hook_exec(extracted_app_folder + '/scripts/upgrade', args=args_list, env=env_dict)[0] != 0: msg = m18n.n('app_upgrade_failed', app=app_instance_name) + not_upgraded_apps.append(app_instance_name) logger.error(msg) operation_logger.error(msg) else: @@ -675,14 +684,13 @@ def app_upgrade(auth, app=[], url=None, file=None): os.system('cp -R %s/%s %s' % (extracted_app_folder, file_to_copy, app_setting_path)) # So much win - upgraded_apps.append(app_instance_name) logger.success(m18n.n('app_upgraded', app=app_instance_name)) hook_callback('post_app_upgrade', args=args_list, env=env_dict) operation_logger.success() - if not upgraded_apps: - raise YunohostError('app_no_upgrade') + if not_upgraded_apps: + raise YunohostError('app_not_upgraded', apps=', '.join(not_upgraded_apps)) app_ssowatconf(auth) @@ -705,6 +713,9 @@ def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on no_remove_on_failure -- Debug option to avoid removing the app on a failed installation force -- Do not ask for confirmation when installing experimental / low-quality apps """ + if packages.dpkg_is_broken(): + raise YunohostError("dpkg_is_broken") + from yunohost.hook import hook_add, hook_remove, hook_exec, hook_callback from yunohost.log import OperationLogger @@ -798,6 +809,8 @@ def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on operation_logger.related_to.append(("app", app_id)) operation_logger.start() + logger.info(m18n.n("app_start_install", app=app_id)) + # Create app directory app_setting_path = os.path.join(APPS_SETTING_PATH, app_instance_name) if os.path.exists(app_setting_path): @@ -874,6 +887,9 @@ def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on app_ssowatconf(auth) + if packages.dpkg_is_broken(): + logger.error(m18n.n("this_action_broke_dpkg")) + if install_retcode == -1: msg = m18n.n('operation_interrupted') + " " + error_msg raise YunohostError(msg, raw_msg=True) @@ -918,6 +934,8 @@ def app_remove(operation_logger, auth, app): operation_logger.start() + logger.info(m18n.n("app_start_remove", app=app)) + app_setting_path = APPS_SETTING_PATH + app # TODO: display fail messages from script @@ -956,6 +974,9 @@ def app_remove(operation_logger, auth, app): hook_remove(app) app_ssowatconf(auth) + if packages.dpkg_is_broken(): + raise YunohostError("this_action_broke_dpkg") + def app_addaccess(auth, apps, users=[]): """ @@ -2181,6 +2202,11 @@ def _parse_action_args_in_yunohost_format(args, action_args, auth=None): if arg_type == 'boolean': arg_default = 1 if arg_default else 0 + # do not print for webadmin + if arg_type == 'display_text' and msettings.get('interface') != 'api': + print(arg["text"]) + continue + # Attempt to retrieve argument value if arg_name in args: arg_value = args[arg_name] @@ -2345,6 +2371,7 @@ def _parse_app_instance_name(app_instance_name): True """ match = re_app_instance_name.match(app_instance_name) + assert match, "Could not parse app instance name : %s" % app_instance_name appid = match.groupdict().get('appid') app_instance_nb = int(match.groupdict().get('appinstancenb')) if match.groupdict().get('appinstancenb') is not None else 1 return (appid, app_instance_nb) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 867de3ea9..53919f2cc 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -675,7 +675,7 @@ class BackupManager(): tmp_app_bkp_dir = env_dict["YNH_APP_BACKUP_DIR"] settings_dir = os.path.join(self.work_dir, 'apps', app, 'settings') - logger.debug(m18n.n('backup_running_app_script', app=app)) + logger.info(m18n.n("app_start_backup", app=app)) try: # Prepare backup directory for the app filesystem.mkdir(tmp_app_bkp_dir, 0o750, True, uid='admin') @@ -1254,6 +1254,8 @@ class RestoreManager(): operation_logger = OperationLogger('backup_restore_app', related_to) operation_logger.start() + logger.info(m18n.n("app_start_restore", app=app_instance_name)) + # Check if the app is not already installed if _is_installed(app_instance_name): logger.error(m18n.n('restore_already_installed_app', @@ -1814,10 +1816,11 @@ class TarBackupMethod(BackupMethod): # Add the "source" into the archive and transform the path into # "dest" tar.add(path['source'], arcname=path['dest']) - tar.close() except IOError: - logger.error(m18n.n('backup_archive_writing_error'), exc_info=1) + logger.error(m18n.n('backup_archive_writing_error', source=path['source'], archive=self._archive_file, dest=path['dest']), exc_info=1) raise YunohostError('backup_creation_failed') + finally: + tar.close() # Move info file shutil.copy(os.path.join(self.work_dir, 'info.json'), @@ -2078,6 +2081,7 @@ def backup_create(name=None, description=None, methods=[], backup_manager.collect_files() # Apply backup methods on prepared files + logger.info(m18n.n("backup_actually_backuping")) backup_manager.backup() logger.success(m18n.n('backup_created')) @@ -2146,6 +2150,7 @@ def backup_restore(auth, name, system=[], apps=[], force=False): # Mount the archive then call the restore for each system part / app # # + logger.info(m18n.n("backup_mount_archive_for_restore")) restore_manager.mount() restore_manager.restore() diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index 2f8d63135..2dadcef52 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -119,6 +119,9 @@ def dyndns_subscribe(operation_logger, subscribe_host="dyndns.yunohost.org", dom subscribe_host -- Dynette HTTP API to subscribe to """ + if len(glob.glob('/etc/yunohost/dyndns/*.key')) != 0 or os.path.exists('/etc/cron.d/yunohost-dyndns'): + raise YunohostError('domain_dyndns_already_subscribed') + if domain is None: domain = _get_maindomain() operation_logger.related_to.append(('domain', domain)) @@ -144,7 +147,8 @@ def dyndns_subscribe(operation_logger, subscribe_host="dyndns.yunohost.org", dom 'dnssec-keygen -a hmac-sha512 -b 512 -r /dev/urandom -n USER %s' % domain) os.system('chmod 600 /etc/yunohost/dyndns/*.key /etc/yunohost/dyndns/*.private') - key_file = glob.glob('/etc/yunohost/dyndns/*.key')[0] + private_file = glob.glob('/etc/yunohost/dyndns/*%s*.private' % domain)[0] + key_file = glob.glob('/etc/yunohost/dyndns/*%s*.key' % domain)[0] with open(key_file) as f: key = f.readline().strip().split(' ', 6)[-1] @@ -152,9 +156,13 @@ def dyndns_subscribe(operation_logger, subscribe_host="dyndns.yunohost.org", dom # Send subscription try: r = requests.post('https://%s/key/%s?key_algo=hmac-sha512' % (subscribe_host, base64.b64encode(key)), data={'subdomain': domain}, timeout=30) - except requests.ConnectionError: - raise YunohostError('no_internet_connection') + except Exception as e: + os.system("rm -f %s" % private_file) + os.system("rm -f %s" % key_file) + raise YunohostError('dyndns_registration_failed', error=str(e)) if r.status_code != 201: + os.system("rm -f %s" % private_file) + os.system("rm -f %s" % key_file) try: error = json.loads(r.text)['error'] except: @@ -333,7 +341,8 @@ def _guess_current_dyndns_domain(dyn_host): """ # Retrieve the first registered domain - for path in glob.iglob('/etc/yunohost/dyndns/K*.private'): + paths = list(glob.iglob('/etc/yunohost/dyndns/K*.private')) + for path in paths: match = RE_DYNDNS_PRIVATE_KEY_MD5.match(path) if not match: match = RE_DYNDNS_PRIVATE_KEY_SHA512.match(path) @@ -343,7 +352,9 @@ def _guess_current_dyndns_domain(dyn_host): # Verify if domain is registered (i.e., if it's available, skip # current domain beause that's not the one we want to update..) - if _dyndns_available(dyn_host, _domain): + # If there's only 1 such key found, then avoid doing the request + # for nothing (that's very probably the one we want to find ...) + if len(paths) > 1 and _dyndns_available(dyn_host, _domain): continue else: return (_domain, path) diff --git a/src/yunohost/firewall.py b/src/yunohost/firewall.py index 1c44efe99..9d209dbb8 100644 --- a/src/yunohost/firewall.py +++ b/src/yunohost/firewall.py @@ -195,6 +195,7 @@ def firewall_reload(skip_upnp=False): """ from yunohost.hook import hook_callback + from yunohost.service import _run_service_command reloaded = False errors = False @@ -276,8 +277,7 @@ def firewall_reload(skip_upnp=False): # Refresh port forwarding with UPnP firewall_upnp(no_refresh=False) - # TODO: Use service_restart - os.system("service fail2ban restart") + _run_service_command("reload", "fail2ban") if errors: logger.warning(m18n.n('firewall_rules_cmd_failed')) diff --git a/src/yunohost/settings.py b/src/yunohost/settings.py index bbfb3ca56..81ea46114 100644 --- a/src/yunohost/settings.py +++ b/src/yunohost/settings.py @@ -115,10 +115,18 @@ def settings_set(key, value): raise YunohostError('global_settings_unknown_type', setting=key, unknown_type=key_type) + old_value = settings[key].get("value") settings[key]["value"] = value - _save_settings(settings) + # TODO : whatdo if the old value is the same as + # the new value... + try: + trigger_post_change_hook(key, old_value, value) + except Exception as e: + logger.error("Post-change hook for setting %s failed : %s" % (key, e)) + raise + def settings_reset(key): """ @@ -235,3 +243,45 @@ def _save_settings(settings, location=SETTINGS_PATH): settings_fd.write(result) except Exception as e: raise YunohostError('global_settings_cant_write_settings', reason=e) + + +# Meant to be a dict of setting_name -> function to call +post_change_hooks = {} + + +def post_change_hook(setting_name): + def decorator(func): + assert setting_name in DEFAULTS.keys(), "The setting %s does not exists" % setting_name + assert setting_name not in post_change_hooks, "You can only register one post change hook per setting (in particular for %s)" % setting_name + post_change_hooks[setting_name] = func + return func + return decorator + + +def trigger_post_change_hook(setting_name, old_value, new_value): + if setting_name not in post_change_hooks: + logger.debug("Nothing to do after changing setting %s" % setting_name) + return + + f = post_change_hooks[setting_name] + f(setting_name, old_value, new_value) + + +# =========================================== +# +# Actions to trigger when changing a setting +# You can define such an action with : +# +# @post_change_hook("your.setting.name") +# def some_function_name(setting_name, old_value, new_value): +# # Do some stuff +# +# =========================================== + + +#@post_change_hook("example.int") +#def myfunc(setting_name, old_value, new_value): +# print("In hook") +# print(setting_name) +# print(old_value) +# print(new_value) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index a220e21ca..a011b1546 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -524,6 +524,10 @@ def tools_upgrade(operation_logger, auth, ignore_apps=False, ignore_packages=Fal ignore_packages -- Ignore APT packages upgrade """ + from yunohost.utils import packages + if packages.dpkg_is_broken(): + raise YunohostError("dpkg_is_broken") + failure = False # Retrieve interface @@ -713,6 +717,22 @@ def tools_diagnosis(auth, private=False): def _check_if_vulnerable_to_meltdown(): # meltdown CVE: https://security-tracker.debian.org/tracker/CVE-2017-5754 + # We use a cache file to avoid re-running the script so many times, + # which can be expensive (up to around 5 seconds on ARM) + # and make the admin appear to be slow (c.f. the calls to diagnosis + # from the webadmin) + # + # The cache is in /tmp and shall disappear upon reboot + # *or* we compare it to dpkg.log modification time + # such that it's re-ran if there was package upgrades + # (e.g. from yunohost) + cache_file = "/tmp/yunohost-meltdown-diagnosis" + dpkg_log = "/var/log/dpkg.log" + if os.path.exists(cache_file): + if not os.path.exists(dpkg_log) or os.path.getmtime(cache_file) > os.path.getmtime(dpkg_log): + logger.debug("Using cached results for meltdown checker, from %s" % cache_file) + return read_json(cache_file)[0]["VULNERABLE"] + # script taken from https://github.com/speed47/spectre-meltdown-checker # script commit id is store directly in the script file_dir = os.path.split(__file__)[0] @@ -722,6 +742,7 @@ def _check_if_vulnerable_to_meltdown(): # example output from the script: # [{"NAME":"MELTDOWN","CVE":"CVE-2017-5754","VULNERABLE":false,"INFOS":"PTI mitigates the vulnerability"}] try: + logger.debug("Running meltdown vulnerability checker") call = subprocess.Popen("bash %s --batch json --variant 3" % SCRIPT_PATH, shell=True, stdout=subprocess.PIPE, @@ -735,6 +756,14 @@ def _check_if_vulnerable_to_meltdown(): output, err = call.communicate() assert call.returncode in (0, 2, 3), "Return code: %s" % call.returncode + # If there are multiple lines, sounds like there was some messages + # in stdout that are not json >.> ... Try to get the actual json + # stuff which should be the last line + output = output.strip() + if "\n" in output: + logger.debug("Original meltdown checker output : %s" % output) + output = output.split("\n")[-1] + CVEs = json.loads(output) assert len(CVEs) == 1 assert CVEs[0]["NAME"] == "MELTDOWN" @@ -744,6 +773,8 @@ def _check_if_vulnerable_to_meltdown(): logger.warning("Something wrong happened when trying to diagnose Meltdown vunerability, exception: %s" % e) raise Exception("Command output for failed meltdown check: '%s'" % output) + logger.debug("Writing results from meltdown checker to cache file, %s" % cache_file) + write_to_json(cache_file, CVEs) return CVEs[0]["VULNERABLE"] diff --git a/src/yunohost/utils/packages.py b/src/yunohost/utils/packages.py index 5ef97618b..e10de6493 100644 --- a/src/yunohost/utils/packages.py +++ b/src/yunohost/utils/packages.py @@ -19,6 +19,7 @@ """ import re +import os import logging from collections import OrderedDict @@ -470,3 +471,13 @@ def ynh_packages_version(*args, **kwargs): 'yunohost', 'yunohost-admin', 'moulinette', 'ssowat', with_repo=True ) + + +def dpkg_is_broken(): + # If dpkg is broken, /var/lib/dpkg/updates + # will contains files like 0001, 0002, ... + # ref: https://sources.debian.org/src/apt/1.4.9/apt-pkg/deb/debsystem.cc/#L141-L174 + if not os.path.isdir("/var/lib/dpkg/updates/"): + return False + return any(re.match("^[0-9]+$", f) + for f in os.listdir("/var/lib/dpkg/updates/"))