Merge branch 'stretch-unstable' into use_ynh_systemd_action

This commit is contained in:
Alexandre Aubin 2019-04-25 13:48:43 +02:00 committed by GitHub
commit 2426a786c3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 813 additions and 582 deletions

View file

@ -1477,13 +1477,6 @@ tools:
category_help: Specific tools
actions:
### tools_ldapinit()
ldapinit:
action_help: YunoHost LDAP initialization
api: POST /ldap
configuration:
authenticate: all
### tools_adminpw()
adminpw:
action_help: Change password of admin and root users
@ -1623,6 +1616,32 @@ tools:
full: --force
action: store_true
### tools_regen_conf()
regen-conf:
action_help: Regenerate the configuration file(s)
api: PUT /tools/regenconf
arguments:
names:
help: Categories to regenerate configuration of (all by default)
nargs: "*"
metavar: NAME
-d:
full: --with-diff
help: Show differences in case of configuration changes
action: store_true
-f:
full: --force
help: Override all manual modifications in configuration files
action: store_true
-n:
full: --dry-run
help: Show what would have been regenerated
action: store_true
-p:
full: --list-pending
help: List pending configuration files and exit
action: store_true
subcategories:
migrations:

View file

@ -55,7 +55,7 @@ ynh_use_logrotate () {
if [ -n "$logfile" ]
then
if [ "$(echo ${logfile##*.})" != "log" ]; then # Keep only the extension to check if it's a logfile
local logfile="$1/*.log" # Else, uses the directory and all logfile into it.
local logfile="$logfile/*.log" # Else, uses the directory and all logfile into it.
fi
else
logfile="/var/log/${app}/*.log" # Without argument, use a defaut directory in /var/log

View file

@ -65,25 +65,23 @@ ynh_backup() {
# If backing up core only (used by ynh_backup_before_upgrade),
# don't backup big data items
if [ "$is_big" == "1" ] && [ "$BACKUP_CORE_ONLY" == "1" ] ; then
echo "$src_path will not be saved, because backup_core_only is set." >&2
ynh_print_info --message="$src_path will not be saved, because backup_core_only is set."
return 0
fi
# ==============================================================================
# Format correctly source and destination paths
# ==============================================================================
# Be sure the source path is not empty
[[ -e "${src_path}" ]] || {
echo "Source path '${src_path}' does not exist" >&2
ynh_print_warn --message="Source path '${src_path}' does not exist"
if [ "$not_mandatory" == "0" ]
then
echo "Source path '${src_path}' does not exist" >&2
# This is a temporary fix for fail2ban config files missing after the migration to stretch.
if echo "${src_path}" | grep --quiet "/etc/fail2ban"
then
touch "${src_path}"
echo "The missing file will be replaced by a dummy one for the backup !!!" >&2
ynh_print_info --message="The missing file will be replaced by a dummy one for the backup !!!"
else
return 1
fi
@ -127,7 +125,7 @@ ynh_backup() {
# Check if dest_path already exists in tmp archive
[[ ! -e "${dest_path}" ]] || {
echo "Destination path '${dest_path}' already exist" >&2
ynh_print_err --message="Destination path '${dest_path}' already exist"
return 1
}
@ -280,7 +278,7 @@ ynh_bind_or_cp() {
local AS_ROOT=${3:-0}
local NO_ROOT=0
[[ "${AS_ROOT}" = "1" ]] || NO_ROOT=1
echo "This helper is deprecated, you should use ynh_backup instead" >&2
ynh_print_warn --message="This helper is deprecated, you should use ynh_backup instead"
ynh_backup "$1" "$2" 1
}
@ -293,9 +291,9 @@ ynh_bind_or_cp() {
# usage: ynh_mkdir_tmp
# | ret: the created directory path
ynh_mkdir_tmp() {
echo "The helper ynh_mkdir_tmp is deprecated." >&2
echo "You should use 'mktemp -d' instead and manage permissions \
properly with chmod/chown." >&2
ynh_print_warn --message="The helper ynh_mkdir_tmp is deprecated."
ynh_print_warn --message="You should use 'mktemp -d' instead and manage permissions \
properly with chmod/chown."
local TMP_DIR=$(mktemp -d)
# Give rights to other users could be a security risk.
@ -410,7 +408,7 @@ ynh_secure_remove () {
if [ $# -ge 2 ]
then
echo "/!\ Packager ! You provided more than one argument to ynh_secure_remove but it will be ignored... Use this helper with one argument at time." >&2
ynh_print_warn --message="/!\ Packager ! You provided more than one argument to ynh_secure_remove but it will be ignored... Use this helper with one argument at time."
fi
if [[ "$forbidden_path" =~ "$file" \
@ -420,13 +418,13 @@ ynh_secure_remove () {
|| "${file:${#file}-1}" = "/" ]]
# Match if the path finishes by /. Because it seems there is an empty variable
then
echo "Avoid deleting $file." >&2
ynh_print_warn --message="Avoid deleting $file."
else
if [ -e "$file" ]
then
sudo rm -R "$file"
else
echo "$file wasn't deleted because it doesn't exist." >&2
ynh_print_info --message="$file wasn't deleted because it doesn't exist."
fi
fi
}

View file

@ -225,10 +225,10 @@ ynh_mysql_remove_db () {
local mysql_root_password=$(sudo cat $MYSQL_ROOT_PWD_FILE)
if mysqlshow -u root -p$mysql_root_password | grep -q "^| $db_name"; then # Check if the database exists
echo "Removing database $db_name" >&2
ynh_print_info --message="Removing database $db_name"
ynh_mysql_drop_db $db_name # Remove the database
else
echo "Database $db_name not found" >&2
ynh_print_warn --message="Database $db_name not found"
fi
# Remove mysql user if it exists

View file

@ -13,7 +13,7 @@ export N_PREFIX="$n_install_dir"
#
# Requires YunoHost version 2.7.12 or higher.
ynh_install_n () {
echo "Installation of N - Node.js version management" >&2
ynh_print_info --message="Installation of N - Node.js version management"
# Build an app.src for n
mkdir -p "../conf"
echo "SOURCE_URL=https://github.com/tj/n/archive/v2.1.7.tar.gz

View file

@ -166,7 +166,7 @@ ynh_package_install_from_equivs () {
local pkgname=$(grep '^Package: ' $controlfile | cut -d' ' -f 2) # Retrieve the name of the debian package
local pkgversion=$(grep '^Version: ' $controlfile | cut -d' ' -f 2) # And its version number
[[ -z "$pkgname" || -z "$pkgversion" ]] \
&& echo "Invalid control file" && exit 1 # Check if this 2 variables aren't empty.
&& ynh_die --message="Invalid control file" # Check if this 2 variables aren't empty.
# Update packages cache
ynh_package_update

View file

@ -192,6 +192,7 @@ ynh_print_ON () {
# | arg: -m, --message= - The text to print
# | arg: -w, --weight= - The weight for this progression. This value is 1 by default. Use a bigger value for a longer part of the script.
# | arg: -t, --time= - Print the execution time since the last call to this helper. Especially usefull to define weights.
# The execution time is given for the duration since the previous call. So the weight should be applied to this previous call.
# | arg: -l, --last= - Use for the last call of the helper, to fill te progression bar.
#
# Requires YunoHost version 3.?.? or higher.
@ -223,9 +224,9 @@ ynh_script_progression () {
local weight_calls=$(grep --perl-regexp --count "^[^#]*ynh_script_progression.*(--weight|-w )" $0)
# Get the weight of each occurrences of 'ynh_script_progression' in the script using --weight
local weight_valuesA="$(grep --perl-regexp "^[^#]*ynh_script_progression.*--weight" $0 | sed 's/.*--weight[= ]\([[:digit:]].*\)/\1/g')"
# Get the weight of each occurrences of 'ynh_script_progression' in the script using -w
local weight_valuesB="$(grep --perl-regexp "^[^#]*ynh_script_progression.*-w " $0 | sed 's/.*-w[= ]\([[:digit:]].*\)/\1/g')"
local weight_valuesA="$(grep --perl-regexp "^[^#]*ynh_script_progression.*--weight" $0 | sed 's/.*--weight[= ]\([[:digit:]]*\).*/\1/g')"
# Get the weight of each occurrences of 'ynh_script_progression' in the script using -w
local weight_valuesB="$(grep --perl-regexp "^[^#]*ynh_script_progression.*-w " $0 | sed 's/.*-w[= ]\([[:digit:]]*\).*/\1/g')"
# Each value will be on a different line.
# Remove each 'end of line' and replace it by a '+' to sum the values.
local weight_values=$(( $(echo "$weight_valuesA" | tr '\n' '+') + $(echo "$weight_valuesB" | tr '\n' '+') 0 ))
@ -242,8 +243,9 @@ ynh_script_progression () {
# Set the scale of the progression bar
local scale=20
# progress_string(1,2) should have the size of the scale.
local progress_string1="####################"
# progress_string(0,1,2) should have the size of the scale.
local progress_string2="####################"
local progress_string1="++++++++++++++++++++"
local progress_string0="...................."
# Reduce $increment_progression to the size of the scale
@ -255,8 +257,17 @@ ynh_script_progression () {
local effective_progression=$scale
fi
# Build $progression_bar from progress_string(1,2) according to $effective_progression
local progression_bar="${progress_string1:0:$effective_progression}${progress_string0:0:$(( $scale - $effective_progression ))}"
# Build $progression_bar from progress_string(0,1,2) according to $effective_progression and the weight of the current task
# expected_progression is the progression expected after the current task
local expected_progression="$(( ( $increment_progression + $weight ) * $scale / $max_progression - $effective_progression ))"
if [ $last -eq 1 ]
then
expected_progression=0
fi
# left_progression is the progression not yet done
local left_progression="$(( $scale - $effective_progression - $expected_progression ))"
# Build the progression bar with $effective_progression, work done, $expected_progression, current work and $left_progression, work to be done.
local progression_bar="${progress_string2:0:$effective_progression}${progress_string1:0:$expected_progression}${progress_string0:0:$left_progression}"
local print_exec_time=""
if [ $time -eq 1 ]

View file

@ -105,6 +105,10 @@ ynh_psql_create_db() {
# Requires YunoHost version 3.?.? or higher.
ynh_psql_drop_db() {
local db=$1
# First, force disconnection of all clients connected to the database
# https://stackoverflow.com/questions/5408156/how-to-drop-a-postgresql-database-if-there-are-active-connections-to-it
# https://dba.stackexchange.com/questions/16426/how-to-drop-all-connections-to-a-specific-database-without-stopping-the-server
ynh_psql_execute_as_root --sql="SELECT pg_terminate_backend (pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$db';" --database="$db"
sudo --login --user=postgres dropdb $db
}
@ -174,7 +178,7 @@ ynh_psql_database_exists() {
# Manage arguments with getopts
ynh_handle_getopts_args "$@"
if ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(sudo cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT datname FROM pg_database WHERE datname='$database';" | grep --quiet "$user"; then
if ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(sudo cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT datname FROM pg_database WHERE datname='$database';" | grep --quiet "$database"; then
return 1
else
return 0
@ -240,18 +244,18 @@ ynh_psql_remove_db() {
local psql_root_password=$(sudo cat $PSQL_ROOT_PWD_FILE)
if ynh_psql_database_exists --database=$db_name; then # Check if the database exists
echo "Removing database $db_name" >&2
ynh_print_info --message="Removing database $db_name"
ynh_psql_drop_db $db_name # Remove the database
else
echo "Database $db_name not found" >&2
ynh_print_warn --message="Database $db_name not found"
fi
# Remove psql user if it exists
if ynh_psql_user_exists --user=$db_user; then
echo "Removing user $db_user" >&2
ynh_print_info --message="Removing user $db_user"
ynh_psql_drop_user $db_user
else
echo "User $db_user not found" >&2
ynh_print_warn --message="User $db_user not found"
fi
}

View file

@ -26,7 +26,7 @@ ynh_exit_properly () {
trap '' EXIT # Ignore new exit signals
set +eu # Do not exit anymore if a command fail or if a variable is empty
echo -e "!!\n $app's script has encountered an error. Its execution was cancelled.\n!!" >&2
ynh_print_err --message="!!\n $app's script has encountered an error. Its execution was cancelled.\n!!"
if type -t ynh_clean_setup > /dev/null; then # Check if the function exist in the app script.
ynh_clean_setup # Call the function to do specific cleaning for the app.
@ -269,11 +269,11 @@ ynh_check_app_version_changed () {
# Complete versions are the same
if [ "$force_upgrade" != "0" ]
then
echo "Upgrade forced by YNH_FORCE_UPGRADE." >&2
ynh_print_info --message="Upgrade forced by YNH_FORCE_UPGRADE."
unset YNH_FORCE_UPGRADE
elif [ "$package_check" != "0" ]
then
echo "Upgrade forced for package check." >&2
ynh_print_info --message="Upgrade forced for package check."
else
ynh_die "Up-to-date, nothing to do" 0
fi

View file

@ -145,16 +145,16 @@ ynh_system_user_delete () {
# Check if the user exists on the system
if ynh_system_user_exists "$username"
then
echo "Remove the user $username" >&2
ynh_print_info --message="Remove the user $username"
deluser $username
else
echo "The user $username was not found" >&2
ynh_print_warn --message="The user $username was not found"
fi
# Check if the group exists on the system
if ynh_system_group_exists "$username"
then
echo "Remove the group $username" >&2
ynh_print_info --message="Remove the group $username"
delgroup $username
fi
}

View file

@ -40,7 +40,7 @@ ynh_get_plain_key() {
#
# Requires YunoHost version 2.7.2 or higher.
ynh_restore_upgradebackup () {
echo "Upgrade failed." >&2
ynh_print_err --message="Upgrade failed."
local app_bck=${app//_/-} # Replace all '_' by '-'
NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0}
@ -57,7 +57,7 @@ ynh_restore_upgradebackup () {
ynh_die --message="The app was restored to the way it was before the failed upgrade."
fi
else
echo "\$NO_BACKUP_UPGRADE is set, that means there's no backup to restore. You have to fix this upgrade by yourself !" >&2
ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, that means there's no backup to restore. You have to fix this upgrade by yourself !"
fi
}
@ -74,7 +74,7 @@ ynh_restore_upgradebackup () {
ynh_backup_before_upgrade () {
if [ ! -e "/etc/yunohost/apps/$app/scripts/backup" ]
then
echo "This app doesn't have any backup script." >&2
ynh_print_warn --message="This app doesn't have any backup script."
return
fi
backup_number=1
@ -106,7 +106,7 @@ ynh_backup_before_upgrade () {
ynh_die --message="Backup failed, the upgrade process was aborted."
fi
else
echo "\$NO_BACKUP_UPGRADE is set, backup will be avoided. Be careful, this upgrade is going to be operated without a security backup"
ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, backup will be avoided. Be careful, this upgrade is going to be operated without a security backup"
fi
}
@ -166,15 +166,22 @@ ynh_setup_source () {
ynh_handle_getopts_args "$@"
source_id="${source_id:-app}" # If the argument is not given, source_id equals "app"
local src_file_path="$YNH_CWD/../conf/${source_id}.src"
# In case of restore script the src file is in an other path.
# So try to use the restore path if the general path point to no file.
if [ ! -e "$src_file_path" ]; then
src_file_path="$YNH_CWD/../settings/conf/${source_id}.src"
fi
# Load value from configuration file (see above for a small doc about this file
# format)
local src_url=$(grep 'SOURCE_URL=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-)
local src_sum=$(grep 'SOURCE_SUM=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-)
local src_sumprg=$(grep 'SOURCE_SUM_PRG=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-)
local src_format=$(grep 'SOURCE_FORMAT=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-)
local src_extract=$(grep 'SOURCE_EXTRACT=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-)
local src_in_subdir=$(grep 'SOURCE_IN_SUBDIR=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-)
local src_filename=$(grep 'SOURCE_FILENAME=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-)
local src_url=$(grep 'SOURCE_URL=' "$src_file_path" | cut -d= -f2-)
local src_sum=$(grep 'SOURCE_SUM=' "$src_file_path" | cut -d= -f2-)
local src_sumprg=$(grep 'SOURCE_SUM_PRG=' "$src_file_path" | cut -d= -f2-)
local src_format=$(grep 'SOURCE_FORMAT=' "$src_file_path" | cut -d= -f2-)
local src_extract=$(grep 'SOURCE_EXTRACT=' "$src_file_path" | cut -d= -f2-)
local src_in_subdir=$(grep 'SOURCE_IN_SUBDIR=' "$src_file_path" | cut -d= -f2-)
local src_filename=$(grep 'SOURCE_FILENAME=' "$src_file_path" | cut -d= -f2-)
# Default value
src_sumprg=${src_sumprg:-sha256sum}

View file

@ -20,8 +20,6 @@ mysql:
glances: {}
ssh:
log: /var/log/auth.log
ssl:
status: null
metronome:
log: [/var/log/metronome/metronome.log,/var/log/metronome/metronome.err]
slapd:
@ -34,10 +32,9 @@ yunohost-firewall:
need_lock: true
nslcd:
log: /var/log/syslog
nsswitch:
status: null
yunohost:
status: null
nsswitch: null
ssl: null
yunohost: null
bind9: null
tahoe-lafs: null
memcached: null

6
debian/changelog vendored
View file

@ -1,3 +1,9 @@
yunohost (3.5.2.2) stable; urgency=low
- Hotfix for ynh_psql_remove_db (from ljf)
-- Alexandre Aubin <alex.aubin@mailoo.org> Thu, 18 Apr 2019 17:32:00 +0000
yunohost (3.5.2.1) stable; urgency=low
- [fix] Fresh install was broken because of yunohost_admin.conf initialization

1
debian/control vendored
View file

@ -8,6 +8,7 @@ X-Python-Version: >= 2.7
Homepage: https://yunohost.org/
Package: yunohost
Essential: yes
Architecture: all
Depends: ${python:Depends}, ${misc:Depends}
, moulinette (>= 2.7.1), ssowat (>= 2.7.1)

2
debian/postinst vendored
View file

@ -12,7 +12,7 @@ do_configure() {
bash /usr/share/yunohost/hooks/conf_regen/15-nginx init
else
echo "Regenerating configuration, this might take a while..."
yunohost service regen-conf --output-as none
yunohost tools regen-conf --output-as none
echo "Launching migrations.."
yunohost tools migrations migrate --auto

View file

@ -262,7 +262,7 @@
"log_selfsigned_cert_install": "Install self signed certificate on '{}' domain",
"log_letsencrypt_cert_renew": "Renew '{}' Let's encrypt certificate",
"log_service_enable": "Enable '{}' service",
"log_service_regen_conf": "Regenerate system configurations '{}'",
"log_regen_conf": "Regenerate system configurations '{}'",
"log_user_create": "Add '{}' user",
"log_user_delete": "Delete '{}' user",
"log_user_update": "Update information of '{}' user",
@ -299,6 +299,8 @@
"migration_description_0006_sync_admin_and_root_passwords": "Synchronize admin and root passwords",
"migration_description_0007_ssh_conf_managed_by_yunohost_step1": "Let the SSH configuration be managed by YunoHost (step 1, automatic)",
"migration_description_0008_ssh_conf_managed_by_yunohost_step2": "Let the SSH configuration be managed by YunoHost (step 2, manual)",
"migration_description_0009_decouple_regenconf_from_services": "Decouple the regen-conf mechanism from services",
"migration_description_0010_migrate_to_apps_json": "Remove deprecated appslists and use the new unified 'apps.json' list instead",
"migration_0003_backward_impossible": "The stretch migration cannot be reverted.",
"migration_0003_start": "Starting migration to Stretch. The logs will be available in {logfile}.",
"migration_0003_patching_sources_list": "Patching the sources.lists…",
@ -324,6 +326,7 @@
"migration_0008_dsa": " - the DSA key will be disabled. Hence, you might need to invalidate a spooky warning from your SSH client, and recheck the fingerprint of your server;",
"migration_0008_warning": "If you understand those warnings and agree to let YunoHost override your current configuration, run the migration. Otherwise, you can also skip the migration - though it is not recommended.",
"migration_0008_no_warning": "No major risk has been indentified about overriding your SSH configuration - but we can't be absolutely sure ;)! If you agree to let YunoHost override your current configuration, run the migration. Otherwise, you can also skip the migration - though it is not recommended.",
"migration_0009_not_needed": "This migration already happened somehow ? Skipping.",
"migrations_backward": "Migrating backward.",
"migrations_bad_value_for_target": "Invalid number for target argument, available migrations numbers are 0 or {}",
"migrations_cant_reach_migration_file": "Can't access migrations files at path %s",
@ -391,6 +394,21 @@
"port_available": "Port {port:d} is available",
"port_unavailable": "Port {port:d} is not available",
"recommend_to_add_first_user": "The post-install is finished but YunoHost needs at least one user to work correctly, you should add one using 'yunohost user create' or the admin interface.",
"regenconf_file_backed_up": "The configuration file '{conf}' has been backed up to '{backup}'",
"regenconf_file_copy_failed": "Unable to copy the new configuration file '{new}' to '{conf}'",
"regenconf_file_kept_back": "The configuration file '{conf}' is expected to be deleted by regen-conf (category {category}) but has been kept back.",
"regenconf_file_manually_modified": "The configuration file '{conf}' has been manually modified and will not be updated",
"regenconf_file_manually_removed": "The configuration file '{conf}' has been manually removed and will not be created",
"regenconf_file_remove_failed": "Unable to remove the configuration file '{conf}'",
"regenconf_file_removed": "The configuration file '{conf}' has been removed",
"regenconf_file_updated": "The configuration file '{conf}' has been updated",
"regenconf_now_managed_by_yunohost": "The configuration file '{conf}' is now managed by YunoHost (category {category}).",
"regenconf_up_to_date": "The configuration is already up-to-date for category '{category}'",
"regenconf_updated": "The configuration has been updated for category '{category}'",
"regenconf_would_be_updated": "The configuration would have been updated for category '{category}'",
"regenconf_dry_pending_applying": "Checking pending configuration which would have been applied for category '{category}'…",
"regenconf_failed": "Unable to regenerate the configuration for category(s): {categories}",
"regenconf_pending_applying": "Applying pending configuration for category '{category}'…",
"restore_action_required": "You must specify something to restore",
"restore_already_installed_app": "An app is already installed with the id '{app:s}'",
"restore_app_failed": "Unable to restore the app '{app:s}'",
@ -419,18 +437,6 @@
"service_already_started": "Service '{service:s}' has already been started",
"service_already_stopped": "Service '{service:s}' has already been stopped",
"service_cmd_exec_failed": "Unable to execute command '{command:s}'",
"service_conf_file_backed_up": "The configuration file '{conf}' has been backed up to '{backup}'",
"service_conf_file_copy_failed": "Unable to copy the new configuration file '{new}' to '{conf}'",
"service_conf_file_kept_back": "The configuration file '{conf}' is expected to be deleted by service {service} but has been kept back.",
"service_conf_file_manually_modified": "The configuration file '{conf}' has been manually modified and will not be updated",
"service_conf_file_manually_removed": "The configuration file '{conf}' has been manually removed and will not be created",
"service_conf_file_remove_failed": "Unable to remove the configuration file '{conf}'",
"service_conf_file_removed": "The configuration file '{conf}' has been removed",
"service_conf_file_updated": "The configuration file '{conf}' has been updated",
"service_conf_now_managed_by_yunohost": "The configuration file '{conf}' is now managed by YunoHost.",
"service_conf_up_to_date": "The configuration is already up-to-date for service '{service}'",
"service_conf_updated": "The configuration has been updated for service '{service}'",
"service_conf_would_be_updated": "The configuration would have been updated for service '{service}'",
"service_description_avahi-daemon": "allows to reach your server using yunohost.local on your local network",
"service_description_dnsmasq": "handles domain name resolution (DNS)",
"service_description_dovecot": "allows e-mail client to access/fetch email (via IMAP and POP3)",
@ -454,9 +460,7 @@
"service_enable_failed": "Unable to enable service '{service:s}'\n\nRecent service logs:{logs:s}",
"service_enabled": "The service '{service:s}' has been enabled",
"service_no_log": "No log to display for service '{service:s}'",
"service_regenconf_dry_pending_applying": "Checking pending configuration which would have been applied for service '{service}'…",
"service_regenconf_failed": "Unable to regenerate the configuration for service(s): {services}",
"service_regenconf_pending_applying": "Applying pending configuration for service '{service}'…",
"service_regen_conf_is_deprecated": "'yunohost service regen-conf' is deprecated! Please use 'yunohost tools regen-conf' instead.",
"service_remove_failed": "Unable to remove service '{service:s}'",
"service_removed": "The service '{service:s}' has been removed",
"service_reload_failed": "Unable to reload service '{service:s}'\n\nRecent service logs:{logs:s}",

View file

@ -1629,7 +1629,7 @@ def app_config_show_panel(app):
args=["show"],
env=env,
stdout_callback=parse_stdout,
)
)[0]
if return_code != 0:
raise Exception("script/config show return value code: %s (considered as an error)", return_code)
@ -1715,7 +1715,7 @@ def app_config_apply(app, args):
return_code = hook_exec(config_script,
args=["apply"],
env=env,
)
)[0]
if return_code != 0:
raise Exception("'script/config apply' return value code: %s (considered as an error)", return_code)

View file

@ -50,7 +50,7 @@ from yunohost.hook import (
)
from yunohost.monitor import binary_to_human
from yunohost.tools import tools_postinstall
from yunohost.service import service_regen_conf
from yunohost.regenconf import regen_conf
from yunohost.log import OperationLogger
from functools import reduce
@ -1212,7 +1212,7 @@ class RestoreManager():
else:
operation_logger.success()
service_regen_conf()
regen_conf()
def _restore_apps(self):
"""Restore all apps targeted"""

View file

@ -43,7 +43,8 @@ from yunohost.utils.network import get_public_ip
from moulinette import m18n
from yunohost.app import app_ssowatconf
from yunohost.service import _run_service_command, service_regen_conf
from yunohost.service import _run_service_command
from yunohost.regenconf import regen_conf
from yunohost.log import OperationLogger
logger = getActionLogger('yunohost.certmanager')
@ -806,7 +807,7 @@ def _enable_certificate(domain, new_cert_folder):
if os.path.isfile('/etc/yunohost/installed'):
# regen nginx conf to be sure it integrates OCSP Stapling
# (We don't do this yet if postinstall is not finished yet)
service_regen_conf(names=['nginx'])
regen_conf(names=['nginx'])
_run_service_command("reload", "nginx")
@ -924,7 +925,7 @@ def _regen_dnsmasq_if_needed():
break
if do_regen:
service_regen_conf(["dnsmasq"])
regen_conf(["dnsmasq"])
def _name_self_CA():

View file

@ -10,9 +10,9 @@ from moulinette.utils.filesystem import read_file
from yunohost.tools import Migration
from yunohost.app import unstable_apps
from yunohost.service import (_run_service_command,
manually_modified_files,
manually_modified_files_compared_to_debian_default)
from yunohost.service import _run_service_command
from yunohost.regenconf import (manually_modified_files,
manually_modified_files_compared_to_debian_default)
from yunohost.utils.filesystem import free_space_in_directory
from yunohost.utils.packages import get_installed_version
from yunohost.utils.network import get_network_interfaces

View file

@ -3,15 +3,12 @@ import re
from shutil import copyfile
from moulinette import m18n
from moulinette.utils.log import getActionLogger
from moulinette.utils.filesystem import mkdir, rm
from yunohost.tools import Migration
from yunohost.service import service_regen_conf, \
_get_conf_hashes, \
_calculate_hash, \
_run_service_command
from yunohost.service import _run_service_command
from yunohost.regenconf import regen_conf
from yunohost.settings import settings_set
from yunohost.utils.error import YunohostError
@ -60,7 +57,7 @@ class MyMigration(Migration):
if os.path.exists('/etc/yunohost/from_script'):
rm('/etc/yunohost/from_script')
copyfile(SSHD_CONF, '/etc/ssh/sshd_config.bkp')
service_regen_conf(names=['ssh'], force=True)
regen_conf(names=['ssh'], force=True)
copyfile('/etc/ssh/sshd_config.bkp', SSHD_CONF)
# Restart ssh and backward if it fail

View file

@ -6,9 +6,8 @@ from moulinette.utils.log import getActionLogger
from moulinette.utils.filesystem import chown
from yunohost.tools import Migration
from yunohost.service import service_regen_conf, \
_get_conf_hashes, \
_calculate_hash
from yunohost.regenconf import _get_conf_hashes, _calculate_hash
from yunohost.regenconf import regen_conf
from yunohost.settings import settings_set, settings_get
from yunohost.utils.error import YunohostError
from yunohost.backup import ARCHIVES_PATH
@ -36,7 +35,7 @@ class MyMigration(Migration):
def migrate(self):
settings_set("service.ssh.allow_deprecated_dsa_hostkey", False)
service_regen_conf(names=['ssh'], force=True)
regen_conf(names=['ssh'], force=True)
# Update local archives folder permissions, so that
# admin can scp archives out of the server

View file

@ -0,0 +1,42 @@
import os
from moulinette import m18n
from moulinette.utils.log import getActionLogger
from moulinette.utils.filesystem import read_file
from yunohost.service import _get_services, _save_services
from yunohost.regenconf import _update_conf_hashes, REGEN_CONF_FILE
from yunohost.tools import Migration
logger = getActionLogger('yunohost.migration')
class MyMigration(Migration):
"""
Decouple the regen conf mechanism from the concept of services
"""
def migrate(self):
if "conffiles" not in read_file("/etc/yunohost/services.yml") \
or os.path.exists(REGEN_CONF_FILE):
logger.warning(m18n.n("migration_0009_not_needed"))
return
# For all services
services = _get_services()
for service, infos in services.items():
# If there are some conffiles (file hashes)
if "conffiles" in infos.keys():
# Save them using the new regen conf thingy
_update_conf_hashes(service, infos["conffiles"])
# And delete the old conffile key from the service infos
del services[service]["conffiles"]
# (Actually save the modification of services)
_save_services(services)
def backward(self):
pass

View file

@ -0,0 +1,42 @@
import os
from moulinette.utils.log import getActionLogger
from yunohost.app import app_fetchlist, app_removelist, _read_appslist_list, APPSLISTS_JSON
from yunohost.tools import Migration
logger = getActionLogger('yunohost.migration')
BASE_CONF_PATH = '/home/yunohost.conf'
BACKUP_CONF_DIR = os.path.join(BASE_CONF_PATH, 'backup')
APPSLISTS_BACKUP = os.path.join(BACKUP_CONF_DIR, "appslist_before_migration_to_unified_list.json")
class MyMigration(Migration):
"Migrate from official.json to apps.json"
def migrate(self):
# Backup current app list json
os.system("cp %s %s" % (APPSLISTS_JSON, APPSLISTS_BACKUP))
# Remove all the deprecated lists
lists_to_remove = [
"https://app.yunohost.org/official.json",
"https://app.yunohost.org/community.json",
"https://labriqueinter.net/apps/labriqueinternet.json"
]
appslists = _read_appslist_list()
for appslist, infos in appslists.items():
if infos["url"] in lists_to_remove:
app_removelist(name=appslist)
# Replace by apps.json list
app_fetchlist(name="yunohost",
url="https://app.yunohost.org/apps.json")
def backward(self):
if os.path.exists(APPSLISTS_BACKUP):
os.system("cp %s %s" % (APPSLISTS_BACKUP, APPSLISTS_JSON))

View file

@ -34,7 +34,7 @@ from moulinette.utils.log import getActionLogger
import yunohost.certificate
from yunohost.service import service_regen_conf
from yunohost.regenconf import regen_conf
from yunohost.utils.network import get_public_ip
from yunohost.log import is_unit_operation
from yunohost.hook import hook_callback
@ -112,7 +112,7 @@ def domain_add(operation_logger, auth, domain, dyndns=False):
# Don't regen these conf if we're still in postinstall
if os.path.exists('/etc/yunohost/installed'):
service_regen_conf(names=['nginx', 'metronome', 'dnsmasq', 'postfix', 'rspamd'])
regen_conf(names=['nginx', 'metronome', 'dnsmasq', 'postfix', 'rspamd'])
app_ssowatconf(auth)
except Exception:
@ -165,7 +165,7 @@ def domain_remove(operation_logger, auth, domain, force=False):
else:
raise YunohostError('domain_deletion_failed')
service_regen_conf(names=['nginx', 'metronome', 'dnsmasq', 'postfix'])
regen_conf(names=['nginx', 'metronome', 'dnsmasq', 'postfix'])
app_ssowatconf(auth)
hook_callback('post_domain_remove', args=[domain])

View file

@ -370,7 +370,7 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False,
# Define output callbacks and call command
callbacks = (
stdout_callback if stdout_callback else lambda l: logger.debug(l.rstrip()),
stdout_callback if stdout_callback else lambda l: logger.debug(l.rstrip()+"\r"),
stderr_callback if stderr_callback else lambda l: logger.warning(l.rstrip()),
)

555
src/yunohost/regenconf.py Normal file
View file

@ -0,0 +1,555 @@
# -*- coding: utf-8 -*-
""" License
Copyright (C) 2019 YunoHost
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program; if not, see http://www.gnu.org/licenses
"""
import os
import yaml
import json
import subprocess
import shutil
import hashlib
from difflib import unified_diff
from datetime import datetime
from moulinette import m18n
from moulinette.utils import log, filesystem
from moulinette.utils.filesystem import read_file
from yunohost.utils.error import YunohostError
from yunohost.log import is_unit_operation
from yunohost.hook import hook_callback, hook_list
BASE_CONF_PATH = '/home/yunohost.conf'
BACKUP_CONF_DIR = os.path.join(BASE_CONF_PATH, 'backup')
PENDING_CONF_DIR = os.path.join(BASE_CONF_PATH, 'pending')
REGEN_CONF_FILE = '/etc/yunohost/regenconf.yml'
logger = log.getActionLogger('yunohost.regenconf')
# FIXME : those ain't just services anymore ... what are we supposed to do with this ...
# FIXME : check for all reference of 'service' close to operation_logger stuff
@is_unit_operation([('names', 'configuration')])
def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run=False,
list_pending=False):
"""
Regenerate the configuration file(s)
Keyword argument:
names -- Categories to regenerate configuration of
with_diff -- Show differences in case of configuration changes
force -- Override all manual modifications in configuration files
dry_run -- Show what would have been regenerated
list_pending -- List pending configuration files and exit
"""
# Legacy code to automatically run the migration
# This is required because regen_conf is called before the migration call
# in debian's postinst script
if os.path.exists("/etc/yunohost/installed") \
and ("conffiles" in read_file("/etc/yunohost/services.yml") \
or not os.path.exists(REGEN_CONF_FILE)):
from yunohost.tools import _get_migration_by_name
migration = _get_migration_by_name("decouple_regenconf_from_services")
migration.migrate()
result = {}
# Return the list of pending conf
if list_pending:
pending_conf = _get_pending_conf(names)
if not with_diff:
return pending_conf
for category, conf_files in pending_conf.items():
for system_path, pending_path in conf_files.items():
pending_conf[category][system_path] = {
'pending_conf': pending_path,
'diff': _get_files_diff(
system_path, pending_path, True),
}
return pending_conf
if not dry_run:
operation_logger.related_to = [('configuration', x) for x in names]
if not names:
operation_logger.name_parameter_override = 'all'
elif len(names) != 1:
operation_logger.name_parameter_override = str(len(operation_logger.related_to)) + '_categories'
operation_logger.start()
# Clean pending conf directory
if os.path.isdir(PENDING_CONF_DIR):
if not names:
shutil.rmtree(PENDING_CONF_DIR, ignore_errors=True)
else:
for name in names:
shutil.rmtree(os.path.join(PENDING_CONF_DIR, name),
ignore_errors=True)
else:
filesystem.mkdir(PENDING_CONF_DIR, 0o755, True)
# Format common hooks arguments
common_args = [1 if force else 0, 1 if dry_run else 0]
# Execute hooks for pre-regen
pre_args = ['pre', ] + common_args
def _pre_call(name, priority, path, args):
# create the pending conf directory for the category
category_pending_path = os.path.join(PENDING_CONF_DIR, name)
filesystem.mkdir(category_pending_path, 0o755, True, uid='root')
# return the arguments to pass to the script
return pre_args + [category_pending_path, ]
# Don't regen SSH if not specifically specified
if not names:
names = hook_list('conf_regen', list_by='name',
show_info=False)['hooks']
names.remove('ssh')
pre_result = hook_callback('conf_regen', names, pre_callback=_pre_call)
# Keep only the hook names with at least one success
names = [hook for hook, infos in pre_result.items()
if any(result["state"] == "succeed" for result in infos.values())]
# FIXME : what do in case of partial success/failure ...
if not names:
ret_failed = [hook for hook, infos in pre_result.items()
if any(result["state"] == "failed" for result in infos.values())]
raise YunohostError('regenconf_failed',
categories=', '.join(ret_failed))
# Set the processing method
_regen = _process_regen_conf if not dry_run else lambda *a, **k: True
operation_logger.related_to = []
# Iterate over categories and process pending conf
for category, conf_files in _get_pending_conf(names).items():
if not dry_run:
operation_logger.related_to.append(('configuration', category))
logger.debug(m18n.n(
'regenconf_pending_applying' if not dry_run else
'regenconf_dry_pending_applying',
category=category))
conf_hashes = _get_conf_hashes(category)
succeed_regen = {}
failed_regen = {}
for system_path, pending_path in conf_files.items():
logger.debug("processing pending conf '%s' to system conf '%s'",
pending_path, system_path)
conf_status = None
regenerated = False
# Get the diff between files
conf_diff = _get_files_diff(
system_path, pending_path, True) if with_diff else None
# Check if the conf must be removed
to_remove = True if os.path.getsize(pending_path) == 0 else False
# Retrieve and calculate hashes
system_hash = _calculate_hash(system_path)
saved_hash = conf_hashes.get(system_path, None)
new_hash = None if to_remove else _calculate_hash(pending_path)
# -> system conf does not exists
if not system_hash:
if to_remove:
logger.debug("> system conf is already removed")
os.remove(pending_path)
continue
if not saved_hash or force:
if force:
logger.debug("> system conf has been manually removed")
conf_status = 'force-created'
else:
logger.debug("> system conf does not exist yet")
conf_status = 'created'
regenerated = _regen(
system_path, pending_path, save=False)
else:
logger.info(m18n.n(
'regenconf_file_manually_removed',
conf=system_path))
conf_status = 'removed'
# -> system conf is not managed yet
elif not saved_hash:
logger.debug("> system conf is not managed yet")
if system_hash == new_hash:
logger.debug("> no changes to system conf has been made")
conf_status = 'managed'
regenerated = True
elif not to_remove:
# If the conf exist but is not managed yet, and is not to be removed,
# we assume that it is safe to regen it, since the file is backuped
# anyway (by default in _regen), as long as we warn the user
# appropriately.
logger.info(m18n.n('regenconf_now_managed_by_yunohost',
conf=system_path, category=category))
regenerated = _regen(system_path, pending_path)
conf_status = 'new'
elif force:
regenerated = _regen(system_path)
conf_status = 'force-removed'
else:
logger.info(m18n.n('regenconf_file_kept_back',
conf=system_path, category=category))
conf_status = 'unmanaged'
# -> system conf has not been manually modified
elif system_hash == saved_hash:
if to_remove:
regenerated = _regen(system_path)
conf_status = 'removed'
elif system_hash != new_hash:
regenerated = _regen(system_path, pending_path)
conf_status = 'updated'
else:
logger.debug("> system conf is already up-to-date")
os.remove(pending_path)
continue
else:
logger.debug("> system conf has been manually modified")
if system_hash == new_hash:
logger.debug("> new conf is as current system conf")
conf_status = 'managed'
regenerated = True
elif force:
regenerated = _regen(system_path, pending_path)
conf_status = 'force-updated'
else:
logger.warning(m18n.n(
'regenconf_file_manually_modified',
conf=system_path))
conf_status = 'modified'
# Store the result
conf_result = {'status': conf_status}
if conf_diff is not None:
conf_result['diff'] = conf_diff
if regenerated:
succeed_regen[system_path] = conf_result
conf_hashes[system_path] = new_hash
if os.path.isfile(pending_path):
os.remove(pending_path)
else:
failed_regen[system_path] = conf_result
# Check for category conf changes
if not succeed_regen and not failed_regen:
logger.debug(m18n.n('regenconf_up_to_date', category=category))
continue
elif not failed_regen:
logger.success(m18n.n(
'regenconf_updated' if not dry_run else
'regenconf_would_be_updated',
category=category))
if succeed_regen and not dry_run:
_update_conf_hashes(category, conf_hashes)
# Append the category results
result[category] = {
'applied': succeed_regen,
'pending': failed_regen
}
# Return in case of dry run
if dry_run:
return result
# Execute hooks for post-regen
post_args = ['post', ] + common_args
def _pre_call(name, priority, path, args):
# append coma-separated applied changes for the category
if name in result and result[name]['applied']:
regen_conf_files = ','.join(result[name]['applied'].keys())
else:
regen_conf_files = ''
return post_args + [regen_conf_files, ]
hook_callback('conf_regen', names, pre_callback=_pre_call)
operation_logger.success()
return result
def _get_regenconf_infos():
"""
Get a dict of regen conf informations
"""
try:
with open(REGEN_CONF_FILE, 'r') as f:
return yaml.load(f)
except:
return {}
def _save_regenconf_infos(infos):
"""
Save the regen conf informations
Keyword argument:
categories -- A dict containing the regenconf infos
"""
try:
with open(REGEN_CONF_FILE, 'w') as f:
yaml.safe_dump(infos, f, default_flow_style=False)
except Exception as e:
logger.warning('Error while saving regenconf infos, exception: %s', e, exc_info=1)
raise
def _get_files_diff(orig_file, new_file, as_string=False, skip_header=True):
"""Compare two files and return the differences
Read and compare two files. The differences are returned either as a delta
in unified diff format or a formatted string if as_string is True. The
header can also be removed if skip_header is True.
"""
if os.path.exists(orig_file):
with open(orig_file, 'r') as orig_file:
orig_file = orig_file.readlines()
else:
orig_file = []
if os.path.exists(new_file):
with open(new_file, 'r') as new_file:
new_file = new_file.readlines()
else:
new_file = []
# Compare files and format output
diff = unified_diff(orig_file, new_file)
if skip_header:
try:
next(diff)
next(diff)
except:
pass
if as_string:
return ''.join(diff).rstrip()
return diff
def _calculate_hash(path):
"""Calculate the MD5 hash of a file"""
if not os.path.exists(path):
return None
hasher = hashlib.md5()
try:
with open(path, 'rb') as f:
hasher.update(f.read())
return hasher.hexdigest()
except IOError as e:
logger.warning("Error while calculating file '%s' hash: %s", path, e, exc_info=1)
return None
def _get_pending_conf(categories=[]):
"""Get pending configuration for categories
Iterate over the pending configuration directory for given categories - or
all if empty - and look for files inside. Each file is considered as a
pending configuration file and therefore must be in the same directory
tree than the system file that it replaces.
The result is returned as a dict of categories with pending configuration as
key and a dict of `system_conf_path` => `pending_conf_path` as value.
"""
result = {}
if not os.path.isdir(PENDING_CONF_DIR):
return result
if not categories:
categories = os.listdir(PENDING_CONF_DIR)
for name in categories:
category_pending_path = os.path.join(PENDING_CONF_DIR, name)
if not os.path.isdir(category_pending_path):
continue
path_index = len(category_pending_path)
category_conf = {}
for root, dirs, files in os.walk(category_pending_path):
for filename in files:
pending_path = os.path.join(root, filename)
category_conf[pending_path[path_index:]] = pending_path
if category_conf:
result[name] = category_conf
else:
# remove empty directory
shutil.rmtree(category_pending_path, ignore_errors=True)
return result
def _get_conf_hashes(category):
"""Get the registered conf hashes for a category"""
categories = _get_regenconf_infos()
if category not in categories:
logger.debug("category %s is not in categories.yml yet.", category)
return {}
elif categories[category] is None or 'conffiles' not in categories[category]:
logger.debug("No configuration files for category %s.", category)
return {}
else:
return categories[category]['conffiles']
def _update_conf_hashes(category, hashes):
"""Update the registered conf hashes for a category"""
logger.debug("updating conf hashes for '%s' with: %s",
category, hashes)
categories = _get_regenconf_infos()
category_conf = categories.get(category, {})
# Handle the case where categories[category] is set to null in the yaml
if category_conf is None:
category_conf = {}
category_conf['conffiles'] = hashes
categories[category] = category_conf
_save_regenconf_infos(categories)
def _process_regen_conf(system_conf, new_conf=None, save=True):
"""Regenerate a given system configuration file
Replace a given system configuration file by a new one or delete it if
new_conf is None. A backup of the file - keeping its directory tree - will
be done in the backup conf directory before any operation if save is True.
"""
if save:
backup_path = os.path.join(BACKUP_CONF_DIR, '{0}-{1}'.format(
system_conf.lstrip('/'), datetime.utcnow().strftime("%Y%m%d.%H%M%S")))
backup_dir = os.path.dirname(backup_path)
if not os.path.isdir(backup_dir):
filesystem.mkdir(backup_dir, 0o755, True)
shutil.copy2(system_conf, backup_path)
logger.debug(m18n.n('regenconf_file_backed_up',
conf=system_conf, backup=backup_path))
try:
if not new_conf:
os.remove(system_conf)
logger.debug(m18n.n('regenconf_file_removed',
conf=system_conf))
else:
system_dir = os.path.dirname(system_conf)
if not os.path.isdir(system_dir):
filesystem.mkdir(system_dir, 0o755, True)
shutil.copyfile(new_conf, system_conf)
logger.debug(m18n.n('regenconf_file_updated',
conf=system_conf))
except Exception as e:
logger.warning("Exception while trying to regenerate conf '%s': %s", system_conf, e, exc_info=1)
if not new_conf and os.path.exists(system_conf):
logger.warning(m18n.n('regenconf_file_remove_failed',
conf=system_conf),
exc_info=1)
return False
elif new_conf:
try:
# From documentation:
# Raise an exception if an os.stat() call on either pathname fails.
# (os.stats returns a series of information from a file like type, size...)
copy_succeed = os.path.samefile(system_conf, new_conf)
except:
copy_succeed = False
finally:
if not copy_succeed:
logger.warning(m18n.n('regenconf_file_copy_failed',
conf=system_conf, new=new_conf),
exc_info=1)
return False
return True
def manually_modified_files():
# We do this to have --quiet, i.e. don't throw a whole bunch of logs
# just to fetch this...
# Might be able to optimize this by looking at what the regen conf does
# and only do the part that checks file hashes...
cmd = "yunohost tools regen-conf --dry-run --output-as json --quiet"
j = json.loads(subprocess.check_output(cmd.split()))
# j is something like :
# {"postfix": {"applied": {}, "pending": {"/etc/postfix/main.cf": {"status": "modified"}}}
output = []
for app, actions in j.items():
for action, files in actions.items():
for filename, infos in files.items():
if infos["status"] == "modified":
output.append(filename)
return output
def manually_modified_files_compared_to_debian_default():
# from https://serverfault.com/a/90401
r = subprocess.check_output("dpkg-query -W -f='${Conffiles}\n' '*' \
| awk 'OFS=\" \"{print $2,$1}' \
| md5sum -c 2>/dev/null \
| awk -F': ' '$2 !~ /OK/{print $1}'", shell=True)
return r.strip().split("\n")

View file

@ -26,13 +26,9 @@
import os
import time
import yaml
import json
import subprocess
import shutil
import hashlib
from glob import glob
from difflib import unified_diff
from datetime import datetime
from moulinette import m18n
@ -40,11 +36,7 @@ from yunohost.utils.error import YunohostError
from moulinette.utils import log, filesystem
from yunohost.log import is_unit_operation
from yunohost.hook import hook_callback, hook_list
BASE_CONF_PATH = '/home/yunohost.conf'
BACKUP_CONF_DIR = os.path.join(BASE_CONF_PATH, 'backup')
PENDING_CONF_DIR = os.path.join(BASE_CONF_PATH, 'pending')
MOULINETTE_LOCK = "/var/run/moulinette_yunohost.lock"
logger = log.getActionLogger('yunohost.service')
@ -424,253 +416,25 @@ def service_log(name, number=50):
return result
@is_unit_operation([('names', 'service')])
def service_regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run=False,
def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
list_pending=False):
"""
Regenerate the configuration file(s) for a service
Keyword argument:
names -- Services name to regenerate configuration of
with_diff -- Show differences in case of configuration changes
force -- Override all manual modifications in configuration files
dry_run -- Show what would have been regenerated
list_pending -- List pending configuration files and exit
services = _get_services()
"""
result = {}
if isinstance(names, str):
names = [names]
# Return the list of pending conf
if list_pending:
pending_conf = _get_pending_conf(names)
for name in names:
if name not in services.keys():
raise YunohostError('service_unknown', service=name)
if not with_diff:
return pending_conf
if names is []:
names = services.keys()
for service, conf_files in pending_conf.items():
for system_path, pending_path in conf_files.items():
logger.warning(m18n.n("service_regen_conf_is_deprecated"))
pending_conf[service][system_path] = {
'pending_conf': pending_path,
'diff': _get_files_diff(
system_path, pending_path, True),
}
return pending_conf
if not dry_run:
operation_logger.related_to = [('service', x) for x in names]
if not names:
operation_logger.name_parameter_override = 'all'
elif len(names) != 1:
operation_logger.name_parameter_override = str(len(operation_logger.related_to)) + '_services'
operation_logger.start()
# Clean pending conf directory
if os.path.isdir(PENDING_CONF_DIR):
if not names:
shutil.rmtree(PENDING_CONF_DIR, ignore_errors=True)
else:
for name in names:
shutil.rmtree(os.path.join(PENDING_CONF_DIR, name),
ignore_errors=True)
else:
filesystem.mkdir(PENDING_CONF_DIR, 0o755, True)
# Format common hooks arguments
common_args = [1 if force else 0, 1 if dry_run else 0]
# Execute hooks for pre-regen
pre_args = ['pre', ] + common_args
def _pre_call(name, priority, path, args):
# create the pending conf directory for the service
service_pending_path = os.path.join(PENDING_CONF_DIR, name)
filesystem.mkdir(service_pending_path, 0o755, True, uid='root')
# return the arguments to pass to the script
return pre_args + [service_pending_path, ]
# Don't regen SSH if not specifically specified
if not names:
names = hook_list('conf_regen', list_by='name',
show_info=False)['hooks']
names.remove('ssh')
pre_result = hook_callback('conf_regen', names, pre_callback=_pre_call)
# Keep only the hook names with at least one success
names = [hook for hook, infos in pre_result.items()
if any(result["state"] == "succeed" for result in infos.values())]
# FIXME : what do in case of partial success/failure ...
if not names:
ret_failed = [hook for hook, infos in pre_result.items()
if any(result["state"] == "failed" for result in infos.values())]
raise YunohostError('service_regenconf_failed',
services=', '.join(ret_failed))
# Set the processing method
_regen = _process_regen_conf if not dry_run else lambda *a, **k: True
operation_logger.related_to = []
# Iterate over services and process pending conf
for service, conf_files in _get_pending_conf(names).items():
if not dry_run:
operation_logger.related_to.append(('service', service))
logger.debug(m18n.n(
'service_regenconf_pending_applying' if not dry_run else
'service_regenconf_dry_pending_applying',
service=service))
conf_hashes = _get_conf_hashes(service)
succeed_regen = {}
failed_regen = {}
for system_path, pending_path in conf_files.items():
logger.debug("processing pending conf '%s' to system conf '%s'",
pending_path, system_path)
conf_status = None
regenerated = False
# Get the diff between files
conf_diff = _get_files_diff(
system_path, pending_path, True) if with_diff else None
# Check if the conf must be removed
to_remove = True if os.path.getsize(pending_path) == 0 else False
# Retrieve and calculate hashes
system_hash = _calculate_hash(system_path)
saved_hash = conf_hashes.get(system_path, None)
new_hash = None if to_remove else _calculate_hash(pending_path)
# -> system conf does not exists
if not system_hash:
if to_remove:
logger.debug("> system conf is already removed")
os.remove(pending_path)
continue
if not saved_hash or force:
if force:
logger.debug("> system conf has been manually removed")
conf_status = 'force-created'
else:
logger.debug("> system conf does not exist yet")
conf_status = 'created'
regenerated = _regen(
system_path, pending_path, save=False)
else:
logger.info(m18n.n(
'service_conf_file_manually_removed',
conf=system_path))
conf_status = 'removed'
# -> system conf is not managed yet
elif not saved_hash:
logger.debug("> system conf is not managed yet")
if system_hash == new_hash:
logger.debug("> no changes to system conf has been made")
conf_status = 'managed'
regenerated = True
elif not to_remove:
# If the conf exist but is not managed yet, and is not to be removed,
# we assume that it is safe to regen it, since the file is backuped
# anyway (by default in _regen), as long as we warn the user
# appropriately.
logger.info(m18n.n('service_conf_now_managed_by_yunohost',
conf=system_path))
regenerated = _regen(system_path, pending_path)
conf_status = 'new'
elif force:
regenerated = _regen(system_path)
conf_status = 'force-removed'
else:
logger.info(m18n.n('service_conf_file_kept_back',
conf=system_path, service=service))
conf_status = 'unmanaged'
# -> system conf has not been manually modified
elif system_hash == saved_hash:
if to_remove:
regenerated = _regen(system_path)
conf_status = 'removed'
elif system_hash != new_hash:
regenerated = _regen(system_path, pending_path)
conf_status = 'updated'
else:
logger.debug("> system conf is already up-to-date")
os.remove(pending_path)
continue
else:
logger.debug("> system conf has been manually modified")
if system_hash == new_hash:
logger.debug("> new conf is as current system conf")
conf_status = 'managed'
regenerated = True
elif force:
regenerated = _regen(system_path, pending_path)
conf_status = 'force-updated'
else:
logger.warning(m18n.n(
'service_conf_file_manually_modified',
conf=system_path))
conf_status = 'modified'
# Store the result
conf_result = {'status': conf_status}
if conf_diff is not None:
conf_result['diff'] = conf_diff
if regenerated:
succeed_regen[system_path] = conf_result
conf_hashes[system_path] = new_hash
if os.path.isfile(pending_path):
os.remove(pending_path)
else:
failed_regen[system_path] = conf_result
# Check for service conf changes
if not succeed_regen and not failed_regen:
logger.debug(m18n.n('service_conf_up_to_date', service=service))
continue
elif not failed_regen:
logger.success(m18n.n(
'service_conf_updated' if not dry_run else
'service_conf_would_be_updated',
service=service))
if succeed_regen and not dry_run:
_update_conf_hashes(service, conf_hashes)
# Append the service results
result[service] = {
'applied': succeed_regen,
'pending': failed_regen
}
# Return in case of dry run
if dry_run:
return result
# Execute hooks for post-regen
post_args = ['post', ] + common_args
def _pre_call(name, priority, path, args):
# append coma-separated applied changes for the service
if name in result and result[name]['applied']:
regen_conf_files = ','.join(result[name]['applied'].keys())
else:
regen_conf_files = ''
return post_args + [regen_conf_files, ]
hook_callback('conf_regen', names, pre_callback=_pre_call)
operation_logger.success()
return result
from yunohost.regenconf import regen_conf
return regen_conf(names, with_diff, force, dry_run, list_pending)
def _run_service_command(action, service):
@ -870,231 +634,9 @@ def _find_previous_log_file(file):
return None
def _get_files_diff(orig_file, new_file, as_string=False, skip_header=True):
"""Compare two files and return the differences
Read and compare two files. The differences are returned either as a delta
in unified diff format or a formatted string if as_string is True. The
header can also be removed if skip_header is True.
"""
if os.path.exists(orig_file):
with open(orig_file, 'r') as orig_file:
orig_file = orig_file.readlines()
else:
orig_file = []
if os.path.exists(new_file):
with open(new_file, 'r') as new_file:
new_file = new_file.readlines()
else:
new_file = []
# Compare files and format output
diff = unified_diff(orig_file, new_file)
if skip_header:
try:
next(diff)
next(diff)
except:
pass
if as_string:
return ''.join(diff).rstrip()
return diff
def _calculate_hash(path):
"""Calculate the MD5 hash of a file"""
if not os.path.exists(path):
return None
hasher = hashlib.md5()
try:
with open(path, 'rb') as f:
hasher.update(f.read())
return hasher.hexdigest()
except IOError as e:
logger.warning("Error while calculating file '%s' hash: %s", path, e, exc_info=1)
return None
def _get_pending_conf(services=[]):
"""Get pending configuration for service(s)
Iterate over the pending configuration directory for given service(s) - or
all if empty - and look for files inside. Each file is considered as a
pending configuration file and therefore must be in the same directory
tree than the system file that it replaces.
The result is returned as a dict of services with pending configuration as
key and a dict of `system_conf_path` => `pending_conf_path` as value.
"""
result = {}
if not os.path.isdir(PENDING_CONF_DIR):
return result
if not services:
services = os.listdir(PENDING_CONF_DIR)
for name in services:
service_pending_path = os.path.join(PENDING_CONF_DIR, name)
if not os.path.isdir(service_pending_path):
continue
path_index = len(service_pending_path)
service_conf = {}
for root, dirs, files in os.walk(service_pending_path):
for filename in files:
pending_path = os.path.join(root, filename)
service_conf[pending_path[path_index:]] = pending_path
if service_conf:
result[name] = service_conf
else:
# remove empty directory
shutil.rmtree(service_pending_path, ignore_errors=True)
return result
def _get_conf_hashes(service):
"""Get the registered conf hashes for a service"""
services = _get_services()
if service not in services:
logger.debug("Service %s is not in services.yml yet.", service)
return {}
elif services[service] is None or 'conffiles' not in services[service]:
logger.debug("No configuration files for service %s.", service)
return {}
else:
return services[service]['conffiles']
def _update_conf_hashes(service, hashes):
"""Update the registered conf hashes for a service"""
logger.debug("updating conf hashes for '%s' with: %s",
service, hashes)
services = _get_services()
service_conf = services.get(service, {})
# Handle the case where services[service] is set to null in the yaml
if service_conf is None:
service_conf = {}
service_conf['conffiles'] = hashes
services[service] = service_conf
_save_services(services)
def _process_regen_conf(system_conf, new_conf=None, save=True):
"""Regenerate a given system configuration file
Replace a given system configuration file by a new one or delete it if
new_conf is None. A backup of the file - keeping its directory tree - will
be done in the backup conf directory before any operation if save is True.
"""
if save:
backup_path = os.path.join(BACKUP_CONF_DIR, '{0}-{1}'.format(
system_conf.lstrip('/'), datetime.utcnow().strftime("%Y%m%d.%H%M%S")))
backup_dir = os.path.dirname(backup_path)
if not os.path.isdir(backup_dir):
filesystem.mkdir(backup_dir, 0o755, True)
shutil.copy2(system_conf, backup_path)
logger.debug(m18n.n('service_conf_file_backed_up',
conf=system_conf, backup=backup_path))
try:
if not new_conf:
os.remove(system_conf)
logger.debug(m18n.n('service_conf_file_removed',
conf=system_conf))
else:
system_dir = os.path.dirname(system_conf)
if not os.path.isdir(system_dir):
filesystem.mkdir(system_dir, 0o755, True)
shutil.copyfile(new_conf, system_conf)
logger.debug(m18n.n('service_conf_file_updated',
conf=system_conf))
except Exception as e:
logger.warning("Exception while trying to regenerate conf '%s': %s", system_conf, e, exc_info=1)
if not new_conf and os.path.exists(system_conf):
logger.warning(m18n.n('service_conf_file_remove_failed',
conf=system_conf),
exc_info=1)
return False
elif new_conf:
try:
# From documentation:
# Raise an exception if an os.stat() call on either pathname fails.
# (os.stats returns a series of information from a file like type, size...)
copy_succeed = os.path.samefile(system_conf, new_conf)
except:
copy_succeed = False
finally:
if not copy_succeed:
logger.warning(m18n.n('service_conf_file_copy_failed',
conf=system_conf, new=new_conf),
exc_info=1)
return False
return True
def manually_modified_files():
# We do this to have --quiet, i.e. don't throw a whole bunch of logs
# just to fetch this...
# Might be able to optimize this by looking at what service_regenconf does
# and only do the part that checks file hashes...
cmd = "yunohost service regen-conf --dry-run --output-as json --quiet"
j = json.loads(subprocess.check_output(cmd.split()))
# j is something like :
# {"postfix": {"applied": {}, "pending": {"/etc/postfix/main.cf": {"status": "modified"}}}
output = []
for app, actions in j.items():
for action, files in actions.items():
for filename, infos in files.items():
if infos["status"] == "modified":
output.append(filename)
return output
def _get_journalctl_logs(service, number="all"):
try:
return subprocess.check_output("journalctl -xn -u {0} -n{1}".format(service, number), shell=True)
except:
import traceback
return "error while get services logs from journalctl:\n%s" % traceback.format_exc()
def manually_modified_files_compared_to_debian_default():
# from https://serverfault.com/a/90401
r = subprocess.check_output("dpkg-query -W -f='${Conffiles}\n' '*' \
| awk 'OFS=\" \"{print $2,$1}' \
| md5sum -c 2>/dev/null \
| awk -F': ' '$2 !~ /OK/{print $1}'", shell=True)
return r.strip().split("\n")

View file

@ -48,7 +48,8 @@ from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, a
from yunohost.domain import domain_add, domain_list, _get_maindomain, _set_maindomain
from yunohost.dyndns import _dyndns_available, _dyndns_provides
from yunohost.firewall import firewall_upnp
from yunohost.service import service_status, service_regen_conf, service_log, service_start, service_enable
from yunohost.service import service_status, service_log, service_start, service_enable
from yunohost.regenconf import regen_conf
from yunohost.monitor import monitor_disk, monitor_system
from yunohost.utils.packages import ynh_packages_version
from yunohost.utils.network import get_public_ip
@ -213,7 +214,7 @@ def tools_maindomain(operation_logger, auth, new_domain=None):
# Regen configurations
try:
with open('/etc/yunohost/installed', 'r'):
service_regen_conf()
regen_conf()
except IOError:
pass
@ -331,7 +332,7 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False,
operation_logger.start()
logger.info(m18n.n('yunohost_installing'))
service_regen_conf(['nslcd', 'nsswitch'], force=True)
regen_conf(['nslcd', 'nsswitch'], force=True)
# Initialize LDAP for YunoHost
# TODO: Improve this part by integrate ldapinit into conf_regen hook
@ -382,7 +383,7 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False,
os.system('chmod 644 /etc/ssowat/conf.json.persistent')
# Create SSL CA
service_regen_conf(['ssl'], force=True)
regen_conf(['ssl'], force=True)
ssl_dir = '/usr/share/yunohost/yunohost-config/ssl/yunoCA'
# (Update the serial so that it's specific to this very instance)
os.system("openssl rand -hex 19 > %s/serial" % ssl_dir)
@ -411,7 +412,7 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False,
logger.success(m18n.n('yunohost_ca_creation_success'))
# New domain config
service_regen_conf(['nsswitch'], force=True)
regen_conf(['nsswitch'], force=True)
domain_add(auth, domain, dyndns)
tools_maindomain(auth, domain)
@ -421,10 +422,10 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False,
# Enable UPnP silently and reload firewall
firewall_upnp('enable', no_refresh=True)
# Setup the default official app list with cron job
# Setup the default apps list with cron job
try:
app_fetchlist(name="yunohost",
url="https://app.yunohost.org/official.json")
url="https://app.yunohost.org/apps.json")
except Exception as e:
logger.warning(str(e))
@ -439,7 +440,7 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False,
service_enable("yunohost-firewall")
service_start("yunohost-firewall")
service_regen_conf(force=True)
regen_conf(force=True)
# Restore original ssh conf, as chosen by the
# admin during the initial install
@ -456,13 +457,18 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False,
else:
# We need to explicitly ask the regen conf to regen ssh
# (by default, i.e. first argument = None, it won't because it's too touchy)
service_regen_conf(names=["ssh"], force=True)
regen_conf(names=["ssh"], force=True)
logger.success(m18n.n('yunohost_configured'))
logger.warning(m18n.n('recommend_to_add_first_user'))
def tools_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
list_pending=False):
return regen_conf(names, with_diff, force, dry_run, list_pending)
def tools_update(ignore_apps=False, ignore_packages=False):
"""
Update apps & package cache, then display changelog
@ -758,7 +764,7 @@ def tools_diagnosis(auth, private=False):
# Domains
diagnosis['private']['domains'] = domain_list(auth)['domains']
diagnosis['private']['regen_conf'] = service_regen_conf(with_diff=True, dry_run=True)
diagnosis['private']['regen_conf'] = regen_conf(with_diff=True, dry_run=True)
try:
diagnosis['security'] = {