From 70d0f8a68f276274bd09cf31788185ed02b480b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Sun, 26 Aug 2018 12:46:42 +0200 Subject: [PATCH 01/91] Add return value in hook_exec --- src/yunohost/hook.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index 87844ce17..143742df1 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -280,8 +280,8 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None, try: hook_args = pre_callback(name=name, priority=priority, path=path, args=args) - hook_exec(path, args=hook_args, chdir=chdir, env=env, - no_trace=no_trace, raise_on_error=True, user="root") + hook_return = hook_exec(path, args=hook_args, chdir=chdir, env=env, + no_trace=no_trace, raise_on_error=True, user="root")[1] except MoulinetteError as e: state = 'failed' logger.error(e.strerror, exc_info=1) @@ -294,6 +294,10 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None, result[state][name].append(path) except KeyError: result[state][name] = [path] + try: + result['stdreturn'].append(hook_return) + except KeyError: + result['stdreturn'] = [hook_return] return result @@ -341,6 +345,11 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False, stdinfo = os.path.join(tempfile.mkdtemp(), "stdinfo") env['YNH_STDINFO'] = stdinfo + stdreturn = os.path.join(tempfile.mkdtemp(), "stdreturn") + with open(stdreturn, 'w') as f: + f.write('') + env['YNH_STDRETURN'] = stdreturn + # Construct command to execute if user == "root": command = ['sh', '-c'] @@ -388,11 +397,18 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False, errno.EIO, m18n.n('hook_exec_not_terminated', path=path)) else: logger.error(m18n.n('hook_exec_not_terminated', path=path)) - return 1 + return 1, '' elif raise_on_error and returncode != 0: raise MoulinetteError( errno.EIO, m18n.n('hook_exec_failed', path=path)) - return returncode + + with open(stdreturn, 'r') as f: + returnstring = f.read() + stdreturndir = os.path.split(stdreturn)[0] + os.remove(stdreturn) + os.rmdir(stdreturndir) + + return returncode, returnstring def _extract_filename_parts(filename): From b64196c47ddc80926c4d46717da915bb78833a3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Sun, 26 Aug 2018 12:31:20 +0200 Subject: [PATCH 02/91] Change return of hook_exec everywhere --- src/yunohost/app.py | 12 ++++++------ src/yunohost/backup.py | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 1fed09425..bad1797b3 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -513,7 +513,7 @@ def app_change_url(operation_logger, auth, app, domain, path): os.system('chmod +x %s' % os.path.join(os.path.join(APP_TMP_FOLDER, "scripts", "change_url"))) if hook_exec(os.path.join(APP_TMP_FOLDER, 'scripts/change_url'), - args=args_list, env=env_dict, user="root") != 0: + args=args_list, env=env_dict, user="root")[0] != 0: msg = "Failed to change '%s' url." % app logger.error(msg) operation_logger.error(msg) @@ -640,7 +640,7 @@ def app_upgrade(auth, app=[], url=None, file=None): # Execute App upgrade script os.system('chown -hR admin: %s' % INSTALL_TMP) if hook_exec(extracted_app_folder + '/scripts/upgrade', - args=args_list, env=env_dict, user="root") != 0: + args=args_list, env=env_dict, user="root")[0] != 0: msg = m18n.n('app_upgrade_failed', app=app_instance_name) logger.error(msg) operation_logger.error(msg) @@ -801,7 +801,7 @@ def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on install_retcode = hook_exec( os.path.join(extracted_app_folder, 'scripts/install'), args=args_list, env=env_dict, user="root" - ) + )[0] except (KeyboardInterrupt, EOFError): install_retcode = -1 except: @@ -825,7 +825,7 @@ def app_install(operation_logger, auth, app, label=None, args=None, no_remove_on remove_retcode = hook_exec( os.path.join(extracted_app_folder, 'scripts/remove'), args=[app_instance_name], env=env_dict_remove, user="root" - ) + )[0] if remove_retcode != 0: msg = m18n.n('app_not_properly_removed', app=app_instance_name) @@ -912,7 +912,7 @@ def app_remove(operation_logger, auth, app): operation_logger.flush() if hook_exec('/tmp/yunohost_remove/scripts/remove', args=args_list, - env=env_dict, user="root") == 0: + env=env_dict, user="root")[0] == 0: logger.success(m18n.n('app_removed', app=app)) hook_callback('post_app_remove', args=args_list, env=env_dict) @@ -1518,7 +1518,7 @@ def app_action_run(app_id, action, args=None): env=env_dict, chdir=cwd, user=action_declaration.get("user", "root"), - ) + )[0] if retcode not in action_declaration.get("accepted_return_codes", [0]): raise MoulinetteError(retcode, "Error while executing action '%s' of app '%s': return code %s" % (action, app_id, retcode)) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 88959cc2f..bad2f53c2 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -680,7 +680,7 @@ class BackupManager(): subprocess.call(['install', '-Dm555', app_script, tmp_script]) hook_exec(tmp_script, args=[tmp_app_bkp_dir, app], - raise_on_error=True, chdir=tmp_app_bkp_dir, env=env_dict, user="root") + raise_on_error=True, chdir=tmp_app_bkp_dir, env=env_dict, user="root")[0] self._import_to_list_to_backup(env_dict["YNH_BACKUP_CSV"]) except: @@ -1311,7 +1311,7 @@ class RestoreManager(): chdir=app_backup_in_archive, raise_on_error=True, env=env_dict, - user="root") + user="root")[0] except: msg = m18n.n('restore_app_failed',app=app_instance_name) logger.exception(msg) @@ -1336,7 +1336,7 @@ class RestoreManager(): # Execute remove script # TODO: call app_remove instead if hook_exec(remove_script, args=[app_instance_name], - env=env_dict_remove, user="root") != 0: + env=env_dict_remove, user="root")[0] != 0: msg = m18n.n('app_not_properly_removed', app=app_instance_name) logger.warning(msg) operation_logger.error(msg) From b7554dec2176f38df20a0d49a42d522a7c702ff9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Sun, 26 Aug 2018 14:59:26 +0200 Subject: [PATCH 03/91] Use json for return --- locales/en.json | 1 + src/yunohost/hook.py | 24 ++++++++++++++++-------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/locales/en.json b/locales/en.json index 074512311..1e7e78490 100644 --- a/locales/en.json +++ b/locales/en.json @@ -199,6 +199,7 @@ "global_settings_unknown_type": "Unexpected situation, the setting {setting:s} appears to have the type {unknown_type:s} but it's not a type supported by the system.", "hook_exec_failed": "Script execution failed: {path:s}", "hook_exec_not_terminated": "Script execution hasn\u2019t terminated: {path:s}", + "hook_json_return_error": "Faild to read return from hook {path:s}. Error: {msg:s}", "hook_list_by_invalid": "Invalid property to list hook by", "hook_name_unknown": "Unknown hook name '{name:s}'", "installation_complete": "Installation complete", diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index 143742df1..3ef05980a 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -32,6 +32,7 @@ from glob import iglob from moulinette import m18n from moulinette.core import MoulinetteError from moulinette.utils import log +from moulinette.utils.filesystem import read_json HOOK_FOLDER = '/usr/share/yunohost/hooks/' CUSTOM_HOOK_FOLDER = '/etc/yunohost/hooks.d/' @@ -229,7 +230,7 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None, (name, priority, path, succeed) as arguments """ - result = {'succeed': {}, 'failed': {}} + result = {'succeed': {}, 'failed': {}, 'stdreturn' : []} hooks_dict = {} # Retrieve hooks @@ -294,10 +295,13 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None, result[state][name].append(path) except KeyError: result[state][name] = [path] - try: - result['stdreturn'].append(hook_return) - except KeyError: - result['stdreturn'] = [hook_return] + + #print(hook_return) + #for r in hook_return.: + result['stdreturn'].extend(hook_return) #for r in hook_return + #print(r) + + #print(result['stdreturn']) return result @@ -402,13 +406,17 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False, raise MoulinetteError( errno.EIO, m18n.n('hook_exec_failed', path=path)) - with open(stdreturn, 'r') as f: - returnstring = f.read() + try: + returnjson = read_json(stdreturn) + except Exception as e: + returnjson = {} + errno.EIO, m18n.n('hook_json_return_error', path=path, msg=str(e)) + stdreturndir = os.path.split(stdreturn)[0] os.remove(stdreturn) os.rmdir(stdreturndir) - return returncode, returnstring + return returncode, returnjson def _extract_filename_parts(filename): From 29bf70c57cd7ec24ac33d5688ccd75b2ac5bd101 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Wed, 29 Aug 2018 18:54:39 +0200 Subject: [PATCH 04/91] Add hook name in return structure in hook_callback --- src/yunohost/hook.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index 3ef05980a..efd14ca75 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -230,7 +230,6 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None, (name, priority, path, succeed) as arguments """ - result = {'succeed': {}, 'failed': {}, 'stdreturn' : []} hooks_dict = {} # Retrieve hooks @@ -292,16 +291,11 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None, post_callback(name=name, priority=priority, path=path, succeed=True) try: - result[state][name].append(path) + result[name][state].append(path) except KeyError: - result[state][name] = [path] + result[name][state] = [path] - #print(hook_return) - #for r in hook_return.: - result['stdreturn'].extend(hook_return) #for r in hook_return - #print(r) - - #print(result['stdreturn']) + result[name]['stdreturn'] = hook_return return result From e4e981c0fe4e635001619e3bd339aa5c32834954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Wed, 29 Aug 2018 20:56:00 +0200 Subject: [PATCH 05/91] Change struct returned by hook_callback --- src/yunohost/hook.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index efd14ca75..ce83ce011 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -230,6 +230,7 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None, (name, priority, path, succeed) as arguments """ + result = {} hooks_dict = {} # Retrieve hooks @@ -290,12 +291,8 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None, else: post_callback(name=name, priority=priority, path=path, succeed=True) - try: - result[name][state].append(path) - except KeyError: - result[name][state] = [path] - result[name]['stdreturn'] = hook_return + result[name] = {'path' : path, 'state' : state, 'stdreturn' : hook_return } return result From 8ec6f2b81a2ee91e450a3a49d7fa627713d61020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Thu, 30 Aug 2018 17:37:28 +0200 Subject: [PATCH 06/91] Fix error if hook return nothing --- src/yunohost/hook.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index ce83ce011..7ad95c8ab 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -398,10 +398,15 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False, errno.EIO, m18n.n('hook_exec_failed', path=path)) try: - returnjson = read_json(stdreturn) + with open(stdreturn, 'r') as f: + if f.read() != '': + returnjson = read_json(stdreturn) + else: + returnjson = {} except Exception as e: returnjson = {} - errno.EIO, m18n.n('hook_json_return_error', path=path, msg=str(e)) + raise MoulinetteError( + errno.EIO, m18n.n('hook_json_return_error', path=path, msg=str(e))) stdreturndir = os.path.split(stdreturn)[0] os.remove(stdreturn) From 8012f12797e7025f6832db6fa60e60db3c8c247a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Wed, 5 Dec 2018 00:32:42 +0100 Subject: [PATCH 07/91] Fix raise error --- src/yunohost/hook.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index 121630e9f..4e219d226 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -404,7 +404,8 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False, else: returnjson = {} except Exception as e: - returnjson = {} + os.remove(stdreturn) + os.rmdir(stdreturndir) raise MoulinetteError( errno.EIO, m18n.n('hook_json_return_error', path=path, msg=str(e))) From 2e1ccf2b99f9faec47f0455d73cbcc5ea62d15ad Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Mon, 28 Jan 2019 16:20:47 +0100 Subject: [PATCH 08/91] Add ynh_systemd_action helper --- data/helpers.d/system | 89 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/data/helpers.d/system b/data/helpers.d/system index 70cc57493..5fe62dcef 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -53,3 +53,92 @@ ynh_abort_if_errors () { ynh_get_debian_release () { echo $(lsb_release --codename --short) } + +# Start (or other actions) a service, print a log in case of failure and optionnaly wait until the service is completely started +# +# usage: ynh_systemd_action [-n service_name] [-a action] [ [-l "line to match"] [-p log_path] [-t timeout] [-e length] ] +# | arg: -n, --service_name= - Name of the service to reload. Default : $app +# | arg: -a, --action= - Action to perform with systemctl. Default: start +# | arg: -l, --line_match= - Line to match - The line to find in the log to attest the service have finished to boot. +# If not defined it don't wait until the service is completely started. +# | arg: -p, --log_path= - Log file - Path to the log file. Default : /var/log/$app/$app.log +# | arg: -t, --timeout= - Timeout - The maximum time to wait before ending the watching. Default : 300 seconds. +# | arg: -e, --length= - Length of the error log : Default : 20 +ynh_systemd_action() { + # Declare an array to define the options of this helper. + declare -Ar args_array=( [n]=service_name= [a]=action= [l]=line_match= [p]=log_path= [t]=timeout= [e]=length= ) + local service_name + local action + local line_match + local length + local log_path + local timeout + + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local service_name="${service_name:-$app}" + local action=${action:-start} + local log_path="${log_path:-/var/log/$service_name/$service_name.log}" + local length=${length:-20} + local timeout=${timeout:-300} + + # Start to read the log + if [[ -n "${line_match:-}" ]] + then + local templog="$(mktemp)" + # Following the starting of the app in its log + if [ "$log_path" == "systemd" ] ; then + # Read the systemd journal + journalctl -u $service_name -f --since=-45 > "$templog" & + else + # Read the specified log file + tail -F -n0 "$log_path" > "$templog" & + fi + # Get the PID of the tail command + local pid_tail=$! + fi + + echo "${action^} the service $service_name" >&2 + systemctl $action $service_name \ + || ( journalctl --lines=$length -u $service_name >&2 \ + ; test -n "$log_path" && echo "--" && tail --lines=$length "$log_path" >&2 \ + ; false ) + + # Start the timeout and try to find line_match + if [[ -n "${line_match:-}" ]] + then + local i=0 + for i in $(seq 1 $timeout) + do + # Read the log until the sentence is found, that means the app finished to start. Or run until the timeout + if grep --quiet "$line_match" "$templog" + then + echo "The service $service_name has correctly started." >&2 + break + fi + echo -n "." >&2 + sleep 1 + done + if [ $i -eq $timeout ] + then + echo "The service $service_name didn't fully started before the timeout." >&2 + echo "Please find here an extract of the end of the log of the service $service_name:" + journalctl --lines=$length -u $service_name >&2 + test -n "$log_path" && echo "--" && tail --lines=$length "$log_path" >&2 + fi + + echo "" + ynh_clean_check_starting + fi +} + +# Clean temporary process and file used by ynh_check_starting +# (usually used in ynh_clean_setup scripts) +# +# usage: ynh_clean_check_starting +ynh_clean_check_starting () { + # Stop the execution of tail. + kill -s 15 $pid_tail 2>&1 + ynh_secure_remove "$templog" 2>&1 +} From 35ffadbe1c7f5967a485f1282257d7244e9641f1 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Mon, 28 Jan 2019 16:26:43 +0100 Subject: [PATCH 09/91] Use ynh_systemd_action for backend helpers --- data/helpers.d/backend | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index e73644d2d..8d353365d 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -117,10 +117,10 @@ ynh_remove_systemd_config () { local finalsystemdconf="/etc/systemd/system/$service_name.service" if [ -e "$finalsystemdconf" ]; then - sudo systemctl stop $service_name - sudo systemctl disable $service_name + ynh_systemd_action --service_name=$service_name --action=stop + systemctl disable $service_name ynh_secure_remove "$finalsystemdconf" - sudo systemctl daemon-reload + systemctl daemon-reload fi } @@ -186,7 +186,7 @@ ynh_add_nginx_config () { ynh_store_file_checksum "$finalnginxconf" - sudo systemctl reload nginx + ynh_systemd_action --service_name=nginx --action=reload } # Remove the dedicated nginx config @@ -194,7 +194,7 @@ ynh_add_nginx_config () { # usage: ynh_remove_nginx_config ynh_remove_nginx_config () { ynh_secure_remove "/etc/nginx/conf.d/$domain.d/$app.conf" - sudo systemctl reload nginx + ynh_systemd_action --service_name=nginx --action=reload } # Create a dedicated php-fpm config @@ -229,7 +229,7 @@ ynh_add_fpm_config () { sudo chown root: "$finalphpini" ynh_store_file_checksum "$finalphpini" fi - sudo systemctl reload $fpm_service + ynh_systemd_action --service_name=$fpm_service --action=reload } # Remove the dedicated php-fpm config @@ -245,5 +245,5 @@ ynh_remove_fpm_config () { fi ynh_secure_remove "$fpm_config_dir/pool.d/$app.conf" ynh_secure_remove "$fpm_config_dir/conf.d/20-$app.ini" 2>&1 - sudo systemctl reload $fpm_service + ynh_systemd_action --service_name=$fpm_service --action=reload } From 66ef3e208bbac0472ae583c6ef4e18013e892b64 Mon Sep 17 00:00:00 2001 From: Taekiro Date: Sun, 3 Feb 2019 09:54:48 +0100 Subject: [PATCH 10/91] Add IPv6 nameserver to resolv.dnsmasq.conf --- data/templates/dnsmasq/plain/resolv.dnsmasq.conf | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/data/templates/dnsmasq/plain/resolv.dnsmasq.conf b/data/templates/dnsmasq/plain/resolv.dnsmasq.conf index 7eed1142f..197ee2d64 100644 --- a/data/templates/dnsmasq/plain/resolv.dnsmasq.conf +++ b/data/templates/dnsmasq/plain/resolv.dnsmasq.conf @@ -9,25 +9,37 @@ # (FR) FDN nameserver 80.67.169.12 +nameserver 2001:910:800::12 nameserver 80.67.169.40 +nameserver 2001:910:800::40 # (FR) LDN nameserver 80.67.188.188 +nameserver 2001:913::8 # (FR) ARN nameserver 89.234.141.66 +nameserver 2a00:5881:8100:1000::3 # (FR) Aquilenet nameserver 185.233.100.100 +nameserver 2a0c:e300::100 nameserver 185.233.100.101 +nameserver 2a0c:e300::101 # (FR) gozmail / grifon nameserver 80.67.190.200 +nameserver 2a00:5884:8218::1 # (DE) FoeBud / Digital Courage nameserver 85.214.20.141 # (DE) CCC Berlin nameserver 195.160.173.53 # (DE) AS250 nameserver 194.150.168.168 +nameserver 2001:4ce8::53 # (DE) Ideal-Hosting nameserver 84.200.69.80 +nameserver 2001:1608:10:25::1c04:b12f nameserver 84.200.70.40 +nameserver 2001:1608:10:25::9249:d69b # (DK) censurfridns nameserver 91.239.100.100 +nameserver 2001:67c:28a4:: nameserver 89.233.43.71 +nameserver 2002:d596:2a92:1:71:53:: From 1895e1ac6339c1e1a4547c509108c606a88a4ce1 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Wed, 6 Feb 2019 11:18:16 +0100 Subject: [PATCH 11/91] since 0 instead of since 45 --- data/helpers.d/system | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/system b/data/helpers.d/system index 5fe62dcef..25d135401 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -90,7 +90,7 @@ ynh_systemd_action() { # Following the starting of the app in its log if [ "$log_path" == "systemd" ] ; then # Read the systemd journal - journalctl -u $service_name -f --since=-45 > "$templog" & + journalctl --unit=$service_name --follow --since=-0 --quiet > "$templog" & else # Read the specified log file tail -F -n0 "$log_path" > "$templog" & From 8e8ed223951d56c9f115efbe008b6b110a0ec4a7 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sat, 9 Feb 2019 11:38:15 +0100 Subject: [PATCH 12/91] Fix usage of $service_name --- data/helpers.d/backend | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index 66f10baba..95ef95820 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -158,8 +158,8 @@ ynh_remove_systemd_config () { local finalsystemdconf="/etc/systemd/system/$service.service" if [ -e "$finalsystemdconf" ]; then - ynh_systemd_action --service_name=$service_name --action=stop - systemctl disable $service_name + ynh_systemd_action --service_name=$service --action=stop + systemctl disable $service ynh_secure_remove --file="$finalsystemdconf" systemctl daemon-reload fi From fc0e3d4830ddbcf1565380833e30292dffa1f322 Mon Sep 17 00:00:00 2001 From: Taekiro Date: Sat, 9 Feb 2019 16:37:23 +0100 Subject: [PATCH 13/91] Allow query to local IPv6 --- data/templates/dnsmasq/plain/dnsmasq.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/data/templates/dnsmasq/plain/dnsmasq.conf b/data/templates/dnsmasq/plain/dnsmasq.conf index 12a14048a..2dc7775e1 100644 --- a/data/templates/dnsmasq/plain/dnsmasq.conf +++ b/data/templates/dnsmasq/plain/dnsmasq.conf @@ -2,5 +2,6 @@ domain-needed expand-hosts listen-address=127.0.0.1 +listen-address=::1 resolv-file=/etc/resolv.dnsmasq.conf cache-size=256 From d0fbcb43454397dc98d0aef64327e5bd2d60276d Mon Sep 17 00:00:00 2001 From: Taekiro Date: Sat, 9 Feb 2019 16:42:59 +0100 Subject: [PATCH 14/91] Update dnsmasq.conf --- data/templates/dnsmasq/plain/dnsmasq.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/data/templates/dnsmasq/plain/dnsmasq.conf b/data/templates/dnsmasq/plain/dnsmasq.conf index 2dc7775e1..12a14048a 100644 --- a/data/templates/dnsmasq/plain/dnsmasq.conf +++ b/data/templates/dnsmasq/plain/dnsmasq.conf @@ -2,6 +2,5 @@ domain-needed expand-hosts listen-address=127.0.0.1 -listen-address=::1 resolv-file=/etc/resolv.dnsmasq.conf cache-size=256 From 6ab5d716037b76a913c30a43b6ad1e2d57870428 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Mon, 11 Feb 2019 20:05:21 +0100 Subject: [PATCH 15/91] Add the possibility to have multiple path per hook name --- src/yunohost/hook.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index 02e3cb2dd..d9cad9c7a 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -283,14 +283,16 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None, no_trace=no_trace, raise_on_error=True)[1] except YunohostError as e: state = 'failed' + hook_return = {} logger.error(e.strerror, exc_info=1) post_callback(name=name, priority=priority, path=path, succeed=False) else: post_callback(name=name, priority=priority, path=path, succeed=True) - - result[name] = {'path' : path, 'state' : state, 'stdreturn' : hook_return } + if not name in result: + result[name] = {} + result[name][path] = {'state' : state, 'stdreturn' : hook_return } return result @@ -389,7 +391,7 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False, raise YunohostError('hook_exec_not_terminated', path=path) else: logger.error(m18n.n('hook_exec_not_terminated', path=path)) - return 1, '' + return 1, {} elif raise_on_error and returncode != 0: raise YunohostError('hook_exec_failed', path=path) From 37a2cc2e1c155f420f283d05bb0eecdf7ad6d1a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Mon, 11 Feb 2019 20:05:50 +0100 Subject: [PATCH 16/91] Adapt the service and backup to support new result of hook_callback result --- src/yunohost/backup.py | 30 +++++++++++++++++++----------- src/yunohost/service.py | 5 +++-- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 3874a4461..d33f75e3a 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -593,8 +593,11 @@ class BackupManager(): env=env_dict, chdir=self.work_dir) - if ret["succeed"] != []: - self.system_return = ret["succeed"] + ret_succeed = {k: val for k, val in {n: [p for p, c in v.items() if c['state'] == "succeed"] for n, v in ret.items()}.items() if val} + ret_failed = {k: val for k, val in {n: [p for p, c in v.items() if c['state'] == "failed"] for n, v in ret.items()}.items() if val} + + if ret_succeed != []: + self.system_return = ret_succeed # Add files from targets (which they put in the CSV) to the list of # files to backup @@ -610,7 +613,7 @@ class BackupManager(): restore_hooks = hook_list("restore")["hooks"] - for part in ret['succeed'].keys(): + for part in ret_succeed.keys(): if part in restore_hooks: part_restore_hooks = hook_info("restore", part)["hooks"] for hook in part_restore_hooks: @@ -620,7 +623,7 @@ class BackupManager(): logger.warning(m18n.n('restore_hook_unavailable', hook=part)) self.targets.set_result("system", part, "Warning") - for part in ret['failed'].keys(): + for part in ret_failed.keys(): logger.error(m18n.n('backup_system_part_failed', part=part)) self.targets.set_result("system", part, "Error") @@ -1177,16 +1180,19 @@ class RestoreManager(): env=env_dict, chdir=self.work_dir) - for part in ret['succeed'].keys(): + ret_succeed = {k: val for k, val in {n: [p for p, c in v.items() if c['state'] == "succeed"] for n, v in ret.items()}.items() if val} + ret_failed = {k: val for k, val in {n: [p for p, c in v.items() if c['state'] == "failed"] for n, v in ret.items()}.items() if val} + + for part in ret_succeed.keys(): self.targets.set_result("system", part, "Success") error_part = [] - for part in ret['failed'].keys(): + for part in ret_failed.keys(): logger.error(m18n.n('restore_system_part_failed', part=part)) self.targets.set_result("system", part, "Error") error_part.append(part) - if ret['failed']: + if ret_failed: operation_logger.error(m18n.n('restore_system_part_failed', part=', '.join(error_part))) else: operation_logger.success() @@ -1929,8 +1935,8 @@ class CustomBackupMethod(BackupMethod): ret = hook_callback('backup_method', [self.method], args=self._get_args('need_mount')) - - self._need_mount = True if ret['succeed'] else False + ret_succeed = {k: val for k, val in {n: [p for p, c in v.items() if c['state'] == "succeed"] for n, v in ret.items()}.items() if val} + self._need_mount = True if ret_succeed else False return self._need_mount def backup(self): @@ -1943,7 +1949,8 @@ class CustomBackupMethod(BackupMethod): ret = hook_callback('backup_method', [self.method], args=self._get_args('backup')) - if ret['failed']: + ret_failed = {k: val for k, val in {n: [p for p, c in v.items() if c['state'] == "failed"] for n, v in ret.items()}.items() if val} + if ret_failed: raise YunohostError('backup_custom_backup_error') def mount(self, restore_manager): @@ -1956,7 +1963,8 @@ class CustomBackupMethod(BackupMethod): super(CustomBackupMethod, self).mount(restore_manager) ret = hook_callback('backup_method', [self.method], args=self._get_args('mount')) - if ret['failed']: + ret_failed = {k: val for k, val in {n: [p for p, c in v.items() if c['state'] == "failed"] for n, v in ret.items()}.items() if val} + if ret_failed: raise YunohostError('backup_custom_mount_error') def _get_args(self, action): diff --git a/src/yunohost/service.py b/src/yunohost/service.py index 60729053b..151a877b9 100644 --- a/src/yunohost/service.py +++ b/src/yunohost/service.py @@ -494,11 +494,12 @@ def service_regen_conf(operation_logger, names=[], with_diff=False, force=False, pre_result = hook_callback('conf_regen', names, pre_callback=_pre_call) # Update the services name - names = pre_result['succeed'].keys() + names = {n: [p for p, c in v.items() if c['state'] == "failed"] for n, v in pre_result.items()}.keys() if not names: + ret_failed = {n: [p for p, c in v.items() if c['state'] == "failed"] for n, v in pre_result.items()} raise YunohostError('service_regenconf_failed', - services=', '.join(pre_result['failed'])) + services=', '.join(ret_failed)) # Set the processing method _regen = _process_regen_conf if not dry_run else lambda *a, **k: True From af82be4ffdc68e5514fda773a750566f3972558f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Sat, 16 Feb 2019 15:43:09 +0100 Subject: [PATCH 17/91] Check if file exist --- data/helpers.d/system | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/data/helpers.d/system b/data/helpers.d/system index 24e89dccb..71db85390 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -89,14 +89,14 @@ ynh_systemd_action() { if [[ -n "${line_match:-}" ]] then local templog="$(mktemp)" - # Following the starting of the app in its log - if [ "$log_path" == "systemd" ] ; then + # Following the starting of the app in its log + if [ "$log_path" == "systemd" ] ; then # Read the systemd journal journalctl --unit=$service_name --follow --since=-0 --quiet > "$templog" & - else + else # Read the specified log file tail -F -n0 "$log_path" > "$templog" & - fi + fi # Get the PID of the tail command local pid_tail=$! fi @@ -104,7 +104,7 @@ ynh_systemd_action() { echo "${action^} the service $service_name" >&2 systemctl $action $service_name \ || ( journalctl --lines=$length -u $service_name >&2 \ - ; test -n "$log_path" && echo "--" && tail --lines=$length "$log_path" >&2 \ + ; test -e "$log_path" && echo "--" && tail --lines=$length "$log_path" >&2 \ ; false ) # Start the timeout and try to find line_match @@ -127,7 +127,7 @@ ynh_systemd_action() { echo "The service $service_name didn't fully started before the timeout." >&2 echo "Please find here an extract of the end of the log of the service $service_name:" journalctl --lines=$length -u $service_name >&2 - test -n "$log_path" && echo "--" && tail --lines=$length "$log_path" >&2 + test -e "$log_path" && echo "--" && tail --lines=$length "$log_path" >&2 fi echo "" From 9f3bf11fd06e3d16136e3de9a376c38d3179b956 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Mon, 18 Feb 2019 00:04:55 +0100 Subject: [PATCH 18/91] Add warning about --line_match usage --- data/helpers.d/system | 3 +++ 1 file changed, 3 insertions(+) diff --git a/data/helpers.d/system b/data/helpers.d/system index 71db85390..d63a4b482 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -63,6 +63,9 @@ ynh_get_debian_release () { # | arg: -a, --action= - Action to perform with systemctl. Default: start # | arg: -l, --line_match= - Line to match - The line to find in the log to attest the service have finished to boot. # If not defined it don't wait until the service is completely started. +# WARNING: When using --line_match, you should always add `ynh_clean_check_starting` into your +# `ynh_clean_setup` at the beginning of the script. Otherwise, tail will not stop in case of failure +# of the script. The script will then hang forever. # | arg: -p, --log_path= - Log file - Path to the log file. Default : /var/log/$app/$app.log # | arg: -t, --timeout= - Timeout - The maximum time to wait before ending the watching. Default : 300 seconds. # | arg: -e, --length= - Length of the error log : Default : 20 From 9052bb0b78acc46327ba524ac8010b8ad3fc9133 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Mon, 18 Feb 2019 00:11:04 +0100 Subject: [PATCH 19/91] Do not pipe into less --- data/helpers.d/system | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/system b/data/helpers.d/system index d63a4b482..58bc77582 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -106,7 +106,7 @@ ynh_systemd_action() { echo "${action^} the service $service_name" >&2 systemctl $action $service_name \ - || ( journalctl --lines=$length -u $service_name >&2 \ + || ( journalctl --no-pager --lines=$length -u $service_name >&2 \ ; test -e "$log_path" && echo "--" && tail --lines=$length "$log_path" >&2 \ ; false ) @@ -129,7 +129,7 @@ ynh_systemd_action() { then echo "The service $service_name didn't fully started before the timeout." >&2 echo "Please find here an extract of the end of the log of the service $service_name:" - journalctl --lines=$length -u $service_name >&2 + journalctl --no-pager --lines=$length -u $service_name >&2 test -e "$log_path" && echo "--" && tail --lines=$length "$log_path" >&2 fi From 631ca4763664dde1c517fd21dbcc94abafc95599 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Mon, 18 Feb 2019 00:12:01 +0100 Subject: [PATCH 20/91] Be sure to get the correct PID --- data/helpers.d/system | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/system b/data/helpers.d/system index 58bc77582..a846ee590 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -96,12 +96,14 @@ ynh_systemd_action() { if [ "$log_path" == "systemd" ] ; then # Read the systemd journal journalctl --unit=$service_name --follow --since=-0 --quiet > "$templog" & + # Get the PID of the journalctl command + local pid_tail=$! else # Read the specified log file tail -F -n0 "$log_path" > "$templog" & + # Get the PID of the tail command + local pid_tail=$! fi - # Get the PID of the tail command - local pid_tail=$! fi echo "${action^} the service $service_name" >&2 From 1b7d25de96d215101547d08babfa2d884343518d Mon Sep 17 00:00:00 2001 From: Kayou Date: Tue, 19 Feb 2019 00:03:46 +0100 Subject: [PATCH 21/91] Update psql --- data/helpers.d/psql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index 2ef13482a..2feb6b0ac 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -1,3 +1,5 @@ +#!/bin/bash + # Create a master password and set up global settings # Please always call this script in install and restore scripts # From b5ae91b34e8edd4cefe99ef56be891b72b663e48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre=20Bourr=C3=A9?= Date: Tue, 19 Feb 2019 01:39:00 +0100 Subject: [PATCH 22/91] [WIP] Update --- data/helpers.d/psql | 216 +++++++++++++++++++++++++++++++++----------- 1 file changed, 162 insertions(+), 54 deletions(-) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index 2feb6b0ac..20edb2fc5 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -1,75 +1,147 @@ #!/bin/bash -# Create a master password and set up global settings -# Please always call this script in install and restore scripts -# -# usage: ynh_psql_test_if_first_run -ynh_psql_test_if_first_run() { - if [ -f /etc/yunohost/psql ]; - then - echo "PostgreSQL is already installed, no need to create master password" - else - local pgsql="$(ynh_string_random)" - echo "$pgsql" > /etc/yunohost/psql - - if [ -e /etc/postgresql/9.4/ ] - then - local pg_hba=/etc/postgresql/9.4/main/pg_hba.conf - elif [ -e /etc/postgresql/9.6/ ] - then - local pg_hba=/etc/postgresql/9.6/main/pg_hba.conf - else - ynh_die "postgresql shoud be 9.4 or 9.6" - fi - - systemctl start postgresql - sudo --login --user=postgres psql -c"ALTER user postgres WITH PASSWORD '$pgsql'" postgres - - # force all user to connect to local database using passwords - # https://www.postgresql.org/docs/current/static/auth-pg-hba-conf.html#EXAMPLE-PG-HBA.CONF - # Note: we can't use peer since YunoHost create users with nologin - # See: https://github.com/YunoHost/yunohost/blob/unstable/data/helpers.d/user - sed -i '/local\s*all\s*all\s*peer/i \ - local all all password' "$pg_hba" - systemctl enable postgresql - systemctl reload postgresql - fi -} +PSQL_ROOT_PWD_FILE=/etc/yunohost/psql # Open a connection as a user # # example: ynh_psql_connect_as 'user' 'pass' <<< "UPDATE ...;" # example: ynh_psql_connect_as 'user' 'pass' < /path/to/file.sql # -# usage: ynh_psql_connect_as user pwd [db] -# | arg: user - the user name to connect as -# | arg: pwd - the user password -# | arg: db - the database to connect to +# usage: ynh_psql_connect_as --user=user --password=password [--database=database] +# | arg: -u, --user - the user name to connect as +# | arg: -p, --password - the user password +# | arg: -d, --database - the database to connect to ynh_psql_connect_as() { - local user="$1" - local pwd="$2" - local db="$3" - sudo --login --user=postgres PGUSER="$user" PGPASSWORD="$pwd" psql "$db" + # Declare an array to define the options of this helper. + local legacy_args=upd + declare -Ar args_array=( [u]=user= [p]=password= [d]=database= ) + local user + local password + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + + sudo --login --user=postgres PGUSER="$user" PGPASSWORD="$password" psql "$database" } -# # Execute a command as root user +# Execute a command as root user # -# usage: ynh_psql_execute_as_root sql [db] -# | arg: sql - the SQL command to execute +# usage: ynh_psql_execute_as_root --sql=sql [--database=database] +# | arg: -s, --sql - the SQL command to execute +# | arg: -d, --database - the database to connect to ynh_psql_execute_as_root () { - local sql="$1" - sudo --login --user=postgres psql <<< "$sql" + # Declare an array to define the options of this helper. + local legacy_args=sd + declare -Ar args_array=( [s]=sql= [d]=database= ) + local sql + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + + ynh_psql_connect_as --user="postgres" --password="$(sudo cat $PSQL_ROOT_PWD_FILE)" \ + --database="$database" <<< "$sql" } # Execute a command from a file as root user # -# usage: ynh_psql_execute_file_as_root file [db] -# | arg: file - the file containing SQL commands -# | arg: db - the database to connect to +# usage: ynh_psql_execute_file_as_root --file=file [--database=database] +# | arg: -f, --file - the file containing SQL commands +# | arg: -d, --database - the database to connect to ynh_psql_execute_file_as_root() { - local file="$1" - local db="$2" - sudo --login --user=postgres psql "$db" < "$file" + # Declare an array to define the options of this helper. + local legacy_args=fd + declare -Ar args_array=( [f]=file= [d]=database= ) + local file + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" + + ynh_psql_connect_as --user="postgres" --password="$(sudo cat $PSQL_ROOT_PWD_FILE)" \ + --database="$database" < "$file" +} + +# Create a database and grant optionnaly privilegies to a user +# +# [internal] +# +# usage: ynh_psql_create_db db [user [pwd]] +# | arg: db - the database name to create +# | arg: user - the user to grant privilegies +# | arg: pwd - the password to identify user by +ynh_psql_create_db() { + local db=$1 + + ynh_psql_create_user "$user" "$pwd" + sudo --login --user=postgres createdb --owner="$user" "$db" +} + +# Drop a database +# +# [internal] +# +# If you intend to drop the database *and* the associated user, +# consider using ynh_psql_remove_db instead. +# +# usage: ynh_psql_drop_db db +# | arg: db - the database name to drop +ynh_psql_drop_db() { + local db=$1 + sudo --login --user=postgres dropdb $db +} + +# Dump a database +# +# example: ynh_psql_dump_db 'roundcube' > ./dump.sql +# +# usage: ynh_psql_dump_db --database=database +# | arg: -d, --database - the database name to dump +# | ret: the psqldump output +ynh_psql_dump_db() { + # Declare an array to define the options of this helper. + local legacy_args=d + declare -Ar args_array=( [d]=database= ) + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + sudo --login --user=postgres pg_dump "$database" +} + +# Create a user +# +# [internal] +# +# usage: ynh_psql_create_user user pwd [host] +# | arg: user - the user name to create +# | arg: pwd - the password to identify user by +ynh_psql_create_user() { + local user=$1 + local psql=$2 + ynh_psql_execute_as_root "CREATE USER $user WITH PASSWORD '$pwd'" +} + +# Check if a psql user exists +# +# usage: ynh_psql_user_exists --user=user +# | arg: -u, --user - the user for which to check existence +ynh_psql_user_exists() +{ + # Declare an array to define the options of this helper. + local legacy_args=u + declare -Ar args_array=( [u]=user= ) + local user + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if [[ -z $(ynh_psql_execute_as_root --sql="SELECT 1 FROM pg_roles WHERE rolename='$user';") ]] + then + return 1 + else + return 0 + fi } # Create a database, an user and its password. Then store the password in the app's config @@ -148,3 +220,39 @@ ynh_psql_drop_user() { local user="$1" sudo --login --user=postgres dropuser "$user" } + +# Create a master password and set up global settings +# Please always call this script in install and restore scripts +# +# usage: ynh_psql_test_if_first_run +ynh_psql_test_if_first_run() { + if [ -f /etc/yunohost/psql ]; + then + echo "PostgreSQL is already installed, no need to create master password" + else + local pgsql="$(ynh_string_random)" + echo "$pgsql" > /etc/yunohost/psql + + if [ -e /etc/postgresql/9.4/ ] + then + local pg_hba=/etc/postgresql/9.4/main/pg_hba.conf + elif [ -e /etc/postgresql/9.6/ ] + then + local pg_hba=/etc/postgresql/9.6/main/pg_hba.conf + else + ynh_die "postgresql shoud be 9.4 or 9.6" + fi + + systemctl start postgresql + sudo --login --user=postgres psql -c"ALTER user postgres WITH PASSWORD '$pgsql'" postgres + + # force all user to connect to local database using passwords + # https://www.postgresql.org/docs/current/static/auth-pg-hba-conf.html#EXAMPLE-PG-HBA.CONF + # Note: we can't use peer since YunoHost create users with nologin + # See: https://github.com/YunoHost/yunohost/blob/unstable/data/helpers.d/user + sed -i '/local\s*all\s*all\s*peer/i \ + local all all password' "$pg_hba" + systemctl enable postgresql + systemctl reload postgresql + fi +} \ No newline at end of file From d5ca4dd88b3fe49d3c8b9a0780b97e61c1c0e89e Mon Sep 17 00:00:00 2001 From: Kayou Date: Wed, 20 Feb 2019 00:21:44 +0100 Subject: [PATCH 23/91] Fix ynh_psql_user_exists --- data/helpers.d/psql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index 20edb2fc5..b6ba5afaf 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -120,7 +120,7 @@ ynh_psql_dump_db() { ynh_psql_create_user() { local user=$1 local psql=$2 - ynh_psql_execute_as_root "CREATE USER $user WITH PASSWORD '$pwd'" + ynh_psql_execute_as_root --sql="CREATE USER $user WITH PASSWORD '$pwd'" } # Check if a psql user exists @@ -136,7 +136,7 @@ ynh_psql_user_exists() # Manage arguments with getopts ynh_handle_getopts_args "$@" - if [[ -z $(ynh_psql_execute_as_root --sql="SELECT 1 FROM pg_roles WHERE rolename='$user';") ]] + if [[ -n $(sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(sudo cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT rolname FROM pg_roles WHERE rolname='$user';") ]] then return 1 else From d030628a9b36d869e39921f502b9e7f5b58bb1ad Mon Sep 17 00:00:00 2001 From: Kayou Date: Wed, 20 Feb 2019 01:27:27 +0100 Subject: [PATCH 24/91] Update psql helper --- data/helpers.d/psql | 264 +++++++++++++++++++++++--------------------- 1 file changed, 137 insertions(+), 127 deletions(-) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index b6ba5afaf..fb9ffe013 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -12,15 +12,15 @@ PSQL_ROOT_PWD_FILE=/etc/yunohost/psql # | arg: -p, --password - the user password # | arg: -d, --database - the database to connect to ynh_psql_connect_as() { - # Declare an array to define the options of this helper. - local legacy_args=upd - declare -Ar args_array=( [u]=user= [p]=password= [d]=database= ) - local user - local password - local database - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - database="${database:-}" + # Declare an array to define the options of this helper. + local legacy_args=upd + declare -Ar args_array=([u]=user= [p]=password= [d]=database=) + local user + local password + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" sudo --login --user=postgres PGUSER="$user" PGPASSWORD="$password" psql "$database" } @@ -30,18 +30,18 @@ ynh_psql_connect_as() { # usage: ynh_psql_execute_as_root --sql=sql [--database=database] # | arg: -s, --sql - the SQL command to execute # | arg: -d, --database - the database to connect to -ynh_psql_execute_as_root () { - # Declare an array to define the options of this helper. - local legacy_args=sd - declare -Ar args_array=( [s]=sql= [d]=database= ) - local sql - local database - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - database="${database:-}" +ynh_psql_execute_as_root() { + # Declare an array to define the options of this helper. + local legacy_args=sd + declare -Ar args_array=([s]=sql= [d]=database=) + local sql + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" - ynh_psql_connect_as --user="postgres" --password="$(sudo cat $PSQL_ROOT_PWD_FILE)" \ - --database="$database" <<< "$sql" + ynh_psql_connect_as --user="postgres" --password="$(sudo cat $PSQL_ROOT_PWD_FILE)" \ + --database="$database" <<<"$sql" } # Execute a command from a file as root user @@ -50,17 +50,17 @@ ynh_psql_execute_as_root () { # | arg: -f, --file - the file containing SQL commands # | arg: -d, --database - the database to connect to ynh_psql_execute_file_as_root() { - # Declare an array to define the options of this helper. - local legacy_args=fd - declare -Ar args_array=( [f]=file= [d]=database= ) - local file - local database - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - database="${database:-}" + # Declare an array to define the options of this helper. + local legacy_args=fd + declare -Ar args_array=([f]=file= [d]=database=) + local file + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + database="${database:-}" - ynh_psql_connect_as --user="postgres" --password="$(sudo cat $PSQL_ROOT_PWD_FILE)" \ - --database="$database" < "$file" + ynh_psql_connect_as --user="postgres" --password="$(sudo cat $PSQL_ROOT_PWD_FILE)" \ + --database="$database" <"$file" } # Create a database and grant optionnaly privilegies to a user @@ -72,10 +72,18 @@ ynh_psql_execute_file_as_root() { # | arg: user - the user to grant privilegies # | arg: pwd - the password to identify user by ynh_psql_create_db() { - local db=$1 + local db=$1 - ynh_psql_create_user "$user" "$pwd" - sudo --login --user=postgres createdb --owner="$user" "$db" + local sql="CREATE DATABASE ${db};" + + # grant all privilegies to user + if [[ $# -gt 1 ]]; then + #ynh_psql_create_user "$user" "$pwd" + sql+=" GRANT ALL PRIVILEGES ON ${db} TO ${2} WITH GRANT OPTION;" + fi + + #sudo --login --user=postgres createdb --owner="$user" "$db" + ynh_psql_execute_as_root --sql="$sql" } # Drop a database @@ -89,7 +97,7 @@ ynh_psql_create_db() { # | arg: db - the database name to drop ynh_psql_drop_db() { local db=$1 - sudo --login --user=postgres dropdb $db + sudo --login --user=postgres dropdb $db } # Dump a database @@ -100,14 +108,14 @@ ynh_psql_drop_db() { # | arg: -d, --database - the database name to dump # | ret: the psqldump output ynh_psql_dump_db() { - # Declare an array to define the options of this helper. - local legacy_args=d - declare -Ar args_array=( [d]=database= ) - local database - # Manage arguments with getopts - ynh_handle_getopts_args "$@" + # Declare an array to define the options of this helper. + local legacy_args=d + declare -Ar args_array=([d]=database=) + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" - sudo --login --user=postgres pg_dump "$database" + sudo --login --user=postgres pg_dump "$database" } # Create a user @@ -127,21 +135,48 @@ ynh_psql_create_user() { # # usage: ynh_psql_user_exists --user=user # | arg: -u, --user - the user for which to check existence -ynh_psql_user_exists() -{ - # Declare an array to define the options of this helper. - local legacy_args=u - declare -Ar args_array=( [u]=user= ) - local user - # Manage arguments with getopts - ynh_handle_getopts_args "$@" +ynh_psql_user_exists() { + # Declare an array to define the options of this helper. + local legacy_args=u + declare -Ar args_array=([u]=user=) + local user + # Manage arguments with getopts + ynh_handle_getopts_args "$@" - if [[ -n $(sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(sudo cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT rolname FROM pg_roles WHERE rolname='$user';") ]] - then - return 1 - else - return 0 - fi + if [[ -n $(sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(sudo cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT rolname FROM pg_roles WHERE rolname='$user';") ]]; then + return 1 + else + return 0 + fi +} + +# Check if a psql database exists +# +# usage: ynh_psql_database_exists --database=database +# | arg: -d, --database - the database for which to check existence +ynh_psql_database_exists() { + # Declare an array to define the options of this helper. + local legacy_args=u + declare -Ar args_array=([u]=database=) + local database + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if [[ -n $(sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(sudo cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT datname FROM pg_database WHERE datname='$database';") ]]; then + return 1 + else + return 0 + fi +} + +# Drop a user +# +# [internal] +# +# usage: ynh_psql_drop_user user +# | arg: user - the user name to drop +ynh_psql_drop_user() { + ynh_psql_execute_as_root --sql="DROP USER '${1}';" } # Create a database, an user and its password. Then store the password in the app's config @@ -149,76 +184,54 @@ ynh_psql_user_exists() # After executing this helper, the password of the created database will be available in $db_pwd # It will also be stored as "psqlpwd" into the app settings. # -# usage: ynh_psql_setup_db user name [pwd] -# | arg: user - Owner of the database -# | arg: name - Name of the database -# | arg: pwd - Password of the database. If not given, a password will be generated -ynh_psql_setup_db () { - local db_user="$1" - local db_name="$2" - local new_db_pwd=$(ynh_string_random) # Generate a random password - # If $3 is not given, use new_db_pwd instead for db_pwd. - local db_pwd="${3:-$new_db_pwd}" - ynh_psql_create_db "$db_name" "$db_user" "$db_pwd" # Create the database - ynh_app_setting_set "$app" psqlpwd "$db_pwd" # Store the password in the app's config +# usage: ynh_psql_setup_db --db_user=user --db_name=name [--db_pwd=pwd] +# | arg: -u, --db_user - Owner of the database +# | arg: -n, --db_name - Name of the database +# | arg: -p, --db_pwd - Password of the database. If not given, a password will be generated +ynh_psql_setup_db() { + # Declare an array to define the options of this helper. + local legacy_args=unp + declare -Ar args_array=([u]=db_user= [n]=db_name= [p]=db_pwd=) + local db_user + local db_name + db_pwd="" + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local new_db_pwd=$(ynh_string_random) # Generate a random password + # If $db_pwd is not given, use new_db_pwd instead for db_pwd + db_pwd="${db_pwd:-$new_db_pwd}" + + ynh_psql_create_db "$db_name" "$db_user" "$db_pwd" # Create the database + ynh_app_setting_set --app=$app --key=psqlpwd --value=$db_pwd # Store the password in the app's config } -# Create a database and grant privilegies to a user +# Remove a database if it exists, and the associated user # -# usage: ynh_psql_create_db db [user [pwd]] -# | arg: db - the database name to create -# | arg: user - the user to grant privilegies -# | arg: pwd - the user password -ynh_psql_create_db() { - local db="$1" - local user="$2" - local pwd="$3" - ynh_psql_create_user "$user" "$pwd" - sudo --login --user=postgres createdb --owner="$user" "$db" -} - -# Drop a database -# -# usage: ynh_psql_drop_db db -# | arg: db - the database name to drop -# | arg: user - the user to drop +# usage: ynh_psql_remove_db --db_user=user --db_name=name +# | arg: -u, --db_user - Owner of the database +# | arg: -n, --db_name - Name of the database ynh_psql_remove_db() { - local db="$1" - local user="$2" - sudo --login --user=postgres dropdb "$db" - ynh_psql_drop_user "$user" -} + # Declare an array to define the options of this helper. + local legacy_args=un + declare -Ar args_array=([u]=db_user= [n]=db_name=) + local db_user + local db_name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" -# Dump a database -# -# example: ynh_psql_dump_db 'roundcube' > ./dump.sql -# -# usage: ynh_psql_dump_db db -# | arg: db - the database name to dump -# | ret: the psqldump output -ynh_psql_dump_db() { - local db="$1" - sudo --login --user=postgres pg_dump "$db" -} + local psql_root_password=$(sudo cat $PSQL_ROOT_PWD_FILE) + if ynh_psql_database_exists "$db_name"; then # Check if the database exists + echo "Removing database $db_name" >&2 + ynh_psql_drop_db $db_name # Remove the database + else + echo "Database $db_name not found" >&2 + fi - -# Create a user -# -# usage: ynh_psql_create_user user pwd [host] -# | arg: user - the user name to create -ynh_psql_create_user() { - local user="$1" - local pwd="$2" - sudo --login --user=postgres psql -c"CREATE USER $user WITH PASSWORD '$pwd'" postgres -} - -# Drop a user -# -# usage: ynh_psql_drop_user user -# | arg: user - the user name to drop -ynh_psql_drop_user() { - local user="$1" - sudo --login --user=postgres dropuser "$user" + # Remove psql user if it exists + if $(ynh_psql_user_exists --user=$db_user); then + ynh_psql_drop_user $db_user + fi } # Create a master password and set up global settings @@ -226,18 +239,15 @@ ynh_psql_drop_user() { # # usage: ynh_psql_test_if_first_run ynh_psql_test_if_first_run() { - if [ -f /etc/yunohost/psql ]; - then + if [ -f /etc/yunohost/psql ]; then echo "PostgreSQL is already installed, no need to create master password" else local pgsql="$(ynh_string_random)" - echo "$pgsql" > /etc/yunohost/psql + echo "$pgsql" >/etc/yunohost/psql - if [ -e /etc/postgresql/9.4/ ] - then + if [ -e /etc/postgresql/9.4/ ]; then local pg_hba=/etc/postgresql/9.4/main/pg_hba.conf - elif [ -e /etc/postgresql/9.6/ ] - then + elif [ -e /etc/postgresql/9.6/ ]; then local pg_hba=/etc/postgresql/9.6/main/pg_hba.conf else ynh_die "postgresql shoud be 9.4 or 9.6" @@ -255,4 +265,4 @@ ynh_psql_test_if_first_run() { systemctl enable postgresql systemctl reload postgresql fi -} \ No newline at end of file +} From f2a4be29920261f0f95921be1cbf84ac2c516374 Mon Sep 17 00:00:00 2001 From: Kayou Date: Wed, 20 Feb 2019 01:39:22 +0100 Subject: [PATCH 25/91] fix ynh_psql_create_db --- data/helpers.d/psql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index fb9ffe013..7843212eb 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -79,7 +79,7 @@ ynh_psql_create_db() { # grant all privilegies to user if [[ $# -gt 1 ]]; then #ynh_psql_create_user "$user" "$pwd" - sql+=" GRANT ALL PRIVILEGES ON ${db} TO ${2} WITH GRANT OPTION;" + sql+="GRANT ALL PRIVILEGES ON DATABASE ${db} TO ${2} WITH GRANT OPTION;" fi #sudo --login --user=postgres createdb --owner="$user" "$db" From a7af86832eaa134f6613ebff47244841dec8361a Mon Sep 17 00:00:00 2001 From: Kayou Date: Wed, 20 Feb 2019 01:56:14 +0100 Subject: [PATCH 26/91] Small fix --- data/helpers.d/psql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index 7843212eb..d4888982c 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -221,7 +221,7 @@ ynh_psql_remove_db() { ynh_handle_getopts_args "$@" local psql_root_password=$(sudo cat $PSQL_ROOT_PWD_FILE) - if ynh_psql_database_exists "$db_name"; then # Check if the database exists + if $(ynh_psql_database_exists "$db_name"); then # Check if the database exists echo "Removing database $db_name" >&2 ynh_psql_drop_db $db_name # Remove the database else From 203b8c06a999766ecf2229588787c203ca37c582 Mon Sep 17 00:00:00 2001 From: Kayou Date: Wed, 20 Feb 2019 02:18:24 +0100 Subject: [PATCH 27/91] Fix ynh_psql_user_exists and ynh_psql_database_exists --- data/helpers.d/psql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index d4888982c..a3069ce11 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -143,7 +143,7 @@ ynh_psql_user_exists() { # Manage arguments with getopts ynh_handle_getopts_args "$@" - if [[ -n $(sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(sudo cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT rolname FROM pg_roles WHERE rolname='$user';") ]]; then + if [[ -z $(sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(sudo cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT rolname FROM pg_roles WHERE rolname='$user';") ]]; then return 1 else return 0 @@ -162,7 +162,7 @@ ynh_psql_database_exists() { # Manage arguments with getopts ynh_handle_getopts_args "$@" - if [[ -n $(sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(sudo cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT datname FROM pg_database WHERE datname='$database';") ]]; then + if [[ -z $(sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(sudo cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT datname FROM pg_database WHERE datname='$database';") ]]; then return 1 else return 0 From a20b0e96c809a4b7d23597eea82bbc7f6ce0baa5 Mon Sep 17 00:00:00 2001 From: Kayou Date: Wed, 20 Feb 2019 02:18:38 +0100 Subject: [PATCH 28/91] Fix ynh_psql_setup_db --- data/helpers.d/psql | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index a3069ce11..a48aef0fb 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -78,11 +78,9 @@ ynh_psql_create_db() { # grant all privilegies to user if [[ $# -gt 1 ]]; then - #ynh_psql_create_user "$user" "$pwd" sql+="GRANT ALL PRIVILEGES ON DATABASE ${db} TO ${2} WITH GRANT OPTION;" fi - #sudo --login --user=postgres createdb --owner="$user" "$db" ynh_psql_execute_as_root --sql="$sql" } @@ -128,7 +126,7 @@ ynh_psql_dump_db() { ynh_psql_create_user() { local user=$1 local psql=$2 - ynh_psql_execute_as_root --sql="CREATE USER $user WITH PASSWORD '$pwd'" + ynh_psql_execute_as_root --sql="CREATE USER $user WITH PASSWORD $pwd" } # Check if a psql user exists @@ -176,7 +174,7 @@ ynh_psql_database_exists() { # usage: ynh_psql_drop_user user # | arg: user - the user name to drop ynh_psql_drop_user() { - ynh_psql_execute_as_root --sql="DROP USER '${1}';" + ynh_psql_execute_as_root --sql="DROP USER ${1};" } # Create a database, an user and its password. Then store the password in the app's config @@ -202,6 +200,10 @@ ynh_psql_setup_db() { # If $db_pwd is not given, use new_db_pwd instead for db_pwd db_pwd="${db_pwd:-$new_db_pwd}" + if [ $(ynh_psql_user_exists --user=$db_user) ]; then + ynh_psql_create_user "$db_name" "$db_user" "$db_pwd" + fi + ynh_psql_create_db "$db_name" "$db_user" "$db_pwd" # Create the database ynh_app_setting_set --app=$app --key=psqlpwd --value=$db_pwd # Store the password in the app's config } @@ -230,7 +232,10 @@ ynh_psql_remove_db() { # Remove psql user if it exists if $(ynh_psql_user_exists --user=$db_user); then + echo "Removing user $db_user" >&2 ynh_psql_drop_user $db_user + else + echo "User $db_user not found" >&2 fi } From f0d8f88121361d2677bb5c2a403c8a2f094d0ed4 Mon Sep 17 00:00:00 2001 From: Kayou Date: Wed, 20 Feb 2019 23:57:12 +0100 Subject: [PATCH 29/91] Change $psql to $pwd --- data/helpers.d/psql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index a48aef0fb..0c7d70caf 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -125,8 +125,8 @@ ynh_psql_dump_db() { # | arg: pwd - the password to identify user by ynh_psql_create_user() { local user=$1 - local psql=$2 - ynh_psql_execute_as_root --sql="CREATE USER $user WITH PASSWORD $pwd" + local pwd=$2 + ynh_psql_execute_as_root --sql="CREATE USER $user WITH ENCRYPTED PASSWORD '$pwd'" } # Check if a psql user exists From 3ae5955590e0e9ade4cf3c3625e1cb54e574b66c Mon Sep 17 00:00:00 2001 From: Kayou Date: Wed, 20 Feb 2019 23:58:26 +0100 Subject: [PATCH 30/91] Remove some $() and [] --- data/helpers.d/psql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index 0c7d70caf..f960e6297 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -200,8 +200,8 @@ ynh_psql_setup_db() { # If $db_pwd is not given, use new_db_pwd instead for db_pwd db_pwd="${db_pwd:-$new_db_pwd}" - if [ $(ynh_psql_user_exists --user=$db_user) ]; then - ynh_psql_create_user "$db_name" "$db_user" "$db_pwd" + if ! ynh_psql_user_exists --user=$db_user; then + ynh_psql_create_user "$db_user" "$db_pwd" fi ynh_psql_create_db "$db_name" "$db_user" "$db_pwd" # Create the database @@ -223,7 +223,7 @@ ynh_psql_remove_db() { ynh_handle_getopts_args "$@" local psql_root_password=$(sudo cat $PSQL_ROOT_PWD_FILE) - if $(ynh_psql_database_exists "$db_name"); then # Check if the database exists + if ynh_psql_database_exists --database=$db_name; then # Check if the database exists echo "Removing database $db_name" >&2 ynh_psql_drop_db $db_name # Remove the database else @@ -231,7 +231,7 @@ ynh_psql_remove_db() { fi # Remove psql user if it exists - if $(ynh_psql_user_exists --user=$db_user); then + if ynh_psql_user_exists --user=$db_user; then echo "Removing user $db_user" >&2 ynh_psql_drop_user $db_user else From 9b8bd79a37655585e806d04ef96534c96caaf19b Mon Sep 17 00:00:00 2001 From: Kayou Date: Thu, 21 Feb 2019 00:16:57 +0100 Subject: [PATCH 31/91] Don't use [[]] anymore --- data/helpers.d/psql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index f960e6297..bdc9a07c2 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -141,7 +141,7 @@ ynh_psql_user_exists() { # Manage arguments with getopts ynh_handle_getopts_args "$@" - if [[ -z $(sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(sudo cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT rolname FROM pg_roles WHERE rolname='$user';") ]]; then + if ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(sudo cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT rolname FROM pg_roles WHERE rolname='$user';" | grep --quiet "$user" ; then return 1 else return 0 @@ -160,7 +160,7 @@ ynh_psql_database_exists() { # Manage arguments with getopts ynh_handle_getopts_args "$@" - if [[ -z $(sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(sudo cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT datname FROM pg_database WHERE datname='$database';") ]]; then + if ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(sudo cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT datname FROM pg_database WHERE datname='$database';" | grep --quiet "$user"; then return 1 else return 0 From 81bc9987bdb8b75414fabf8dfd54ce6f59e68ab1 Mon Sep 17 00:00:00 2001 From: Kayou Date: Thu, 21 Feb 2019 01:03:10 +0100 Subject: [PATCH 32/91] rework ynh_psql_create_db --- data/helpers.d/psql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index bdc9a07c2..e427582d6 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -67,18 +67,18 @@ ynh_psql_execute_file_as_root() { # # [internal] # -# usage: ynh_psql_create_db db [user [pwd]] +# usage: ynh_psql_create_db db [user] # | arg: db - the database name to create # | arg: user - the user to grant privilegies -# | arg: pwd - the password to identify user by ynh_psql_create_db() { local db=$1 + local user=$2 local sql="CREATE DATABASE ${db};" # grant all privilegies to user - if [[ $# -gt 1 ]]; then - sql+="GRANT ALL PRIVILEGES ON DATABASE ${db} TO ${2} WITH GRANT OPTION;" + if [ $# -gt 1 ]; then + sql+="GRANT ALL PRIVILEGES ON DATABASE ${db} TO ${user} WITH GRANT OPTION;" fi ynh_psql_execute_as_root --sql="$sql" From b1b14a399d0587258ecef91c8ac3dab2203cf6e8 Mon Sep 17 00:00:00 2001 From: Kayou Date: Thu, 21 Feb 2019 01:03:21 +0100 Subject: [PATCH 33/91] fix ynh_psql_database_exists --- data/helpers.d/psql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index e427582d6..8051736c1 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -154,8 +154,8 @@ ynh_psql_user_exists() { # | arg: -d, --database - the database for which to check existence ynh_psql_database_exists() { # Declare an array to define the options of this helper. - local legacy_args=u - declare -Ar args_array=([u]=database=) + local legacy_args=d + declare -Ar args_array=([d]=database=) local database # Manage arguments with getopts ynh_handle_getopts_args "$@" From 95dd4303344aead0603dbfad765850820a37c4ce Mon Sep 17 00:00:00 2001 From: Kayou Date: Thu, 21 Feb 2019 01:06:38 +0100 Subject: [PATCH 34/91] ynh_psql_create_db take only 2 arguments --- data/helpers.d/psql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index 8051736c1..f5e076d4f 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -204,7 +204,7 @@ ynh_psql_setup_db() { ynh_psql_create_user "$db_user" "$db_pwd" fi - ynh_psql_create_db "$db_name" "$db_user" "$db_pwd" # Create the database + ynh_psql_create_db "$db_name" "$db_user" # Create the database ynh_app_setting_set --app=$app --key=psqlpwd --value=$db_pwd # Store the password in the app's config } From 49ec93a9c57f5a3fd1c734c8da09967e2d1809ab Mon Sep 17 00:00:00 2001 From: Kayou Date: Thu, 21 Feb 2019 01:31:31 +0100 Subject: [PATCH 35/91] default argument for a optional argument --- data/helpers.d/psql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index f5e076d4f..47804f585 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -72,7 +72,7 @@ ynh_psql_execute_file_as_root() { # | arg: user - the user to grant privilegies ynh_psql_create_db() { local db=$1 - local user=$2 + local user=${2:-} local sql="CREATE DATABASE ${db};" From c24d45beff554a7de94b0e9dcc560d0c66896a4d Mon Sep 17 00:00:00 2001 From: Kayou Date: Thu, 21 Feb 2019 01:39:15 +0100 Subject: [PATCH 36/91] remove false promises --- data/helpers.d/psql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index 47804f585..e3c3bb96e 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -120,7 +120,7 @@ ynh_psql_dump_db() { # # [internal] # -# usage: ynh_psql_create_user user pwd [host] +# usage: ynh_psql_create_user user pwd # | arg: user - the user name to create # | arg: pwd - the password to identify user by ynh_psql_create_user() { From d7f381518348cde6feb6be8a64c510745d35180f Mon Sep 17 00:00:00 2001 From: Kayou Date: Fri, 22 Feb 2019 00:58:12 +0100 Subject: [PATCH 37/91] For your eyes --- data/helpers.d/psql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index e3c3bb96e..1d992d268 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -77,7 +77,7 @@ ynh_psql_create_db() { local sql="CREATE DATABASE ${db};" # grant all privilegies to user - if [ $# -gt 1 ]; then + if [ -n "$user" ]; then sql+="GRANT ALL PRIVILEGES ON DATABASE ${db} TO ${user} WITH GRANT OPTION;" fi From abc091911c717539cb259a9a353159ba590cb017 Mon Sep 17 00:00:00 2001 From: Kayou Date: Fri, 22 Feb 2019 01:13:23 +0100 Subject: [PATCH 38/91] User ynh_replace_string, add postgresql in the admin panel --- data/helpers.d/psql | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index 1d992d268..324cfee83 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -252,8 +252,10 @@ ynh_psql_test_if_first_run() { if [ -e /etc/postgresql/9.4/ ]; then local pg_hba=/etc/postgresql/9.4/main/pg_hba.conf + local logfile=/var/log/postgresql/postgresql-9.4-main.log elif [ -e /etc/postgresql/9.6/ ]; then local pg_hba=/etc/postgresql/9.6/main/pg_hba.conf + local logfile=/var/log/postgresql/postgresql-9.6-main.log else ynh_die "postgresql shoud be 9.4 or 9.6" fi @@ -265,8 +267,11 @@ ynh_psql_test_if_first_run() { # https://www.postgresql.org/docs/current/static/auth-pg-hba-conf.html#EXAMPLE-PG-HBA.CONF # Note: we can't use peer since YunoHost create users with nologin # See: https://github.com/YunoHost/yunohost/blob/unstable/data/helpers.d/user - sed -i '/local\s*all\s*all\s*peer/i \ - local all all password' "$pg_hba" + ynh_replace_string --match_string="local\(\s*\)all\(\s*\)all\(\s*\)peer" --replace_string="local\1all\2all\3password" --target_file="$pg_hba" + + # Advertise service in admin panel + yunohost service add postgresql --log "$logfile" + systemctl enable postgresql systemctl reload postgresql fi From 283e01db8fa3c2322670f465427710cbbb300640 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 24 Feb 2019 01:58:33 +0100 Subject: [PATCH 39/91] Don't make a whole HTTP request every dyndns update... --- src/yunohost/dyndns.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index 2f8d63135..53486241a 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -333,7 +333,8 @@ def _guess_current_dyndns_domain(dyn_host): """ # Retrieve the first registered domain - for path in glob.iglob('/etc/yunohost/dyndns/K*.private'): + paths = list(glob.iglob('/etc/yunohost/dyndns/K*.private')) + for path in paths: match = RE_DYNDNS_PRIVATE_KEY_MD5.match(path) if not match: match = RE_DYNDNS_PRIVATE_KEY_SHA512.match(path) @@ -343,7 +344,9 @@ def _guess_current_dyndns_domain(dyn_host): # Verify if domain is registered (i.e., if it's available, skip # current domain beause that's not the one we want to update..) - if _dyndns_available(dyn_host, _domain): + # If there's only 1 such key found, then avoid doing the request + # for nothing (that's very probably the one we want to find ...) + if len(paths) > 1 and _dyndns_available(dyn_host, _domain): continue else: return (_domain, path) From 3f8f328fae8967a6294495484c7c197e8497947d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 24 Feb 2019 02:04:41 +0100 Subject: [PATCH 40/91] Delete the key if subscribing failed, to avoid confusion later trying to detect registered domains --- src/yunohost/dyndns.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index 53486241a..090ee3901 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -144,7 +144,8 @@ def dyndns_subscribe(operation_logger, subscribe_host="dyndns.yunohost.org", dom 'dnssec-keygen -a hmac-sha512 -b 512 -r /dev/urandom -n USER %s' % domain) os.system('chmod 600 /etc/yunohost/dyndns/*.key /etc/yunohost/dyndns/*.private') - key_file = glob.glob('/etc/yunohost/dyndns/*.key')[0] + private_file = glob.glob('/etc/yunohost/dyndns/*%s*.private' % domain)[0] + key_file = glob.glob('/etc/yunohost/dyndns/*%s*.key' % domain)[0] with open(key_file) as f: key = f.readline().strip().split(' ', 6)[-1] @@ -153,8 +154,12 @@ def dyndns_subscribe(operation_logger, subscribe_host="dyndns.yunohost.org", dom try: r = requests.post('https://%s/key/%s?key_algo=hmac-sha512' % (subscribe_host, base64.b64encode(key)), data={'subdomain': domain}, timeout=30) except requests.ConnectionError: + os.system("rm %s" % private_file) + os.system("rm %s" % key_file) raise YunohostError('no_internet_connection') if r.status_code != 201: + os.system("rm %s" % private_file) + os.system("rm %s" % key_file) try: error = json.loads(r.text)['error'] except: From 9657387746137d0a4e1adfefd2757759325951a7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 24 Feb 2019 02:10:24 +0100 Subject: [PATCH 41/91] Prevent calling dyndns_subscribe if already subscribed --- src/yunohost/dyndns.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index 090ee3901..36509b0e4 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -119,6 +119,9 @@ def dyndns_subscribe(operation_logger, subscribe_host="dyndns.yunohost.org", dom subscribe_host -- Dynette HTTP API to subscribe to """ + if len(glob.glob('/etc/yunohost/dyndns/*.key')) != 0 or os.path.exists('/etc/cron.d/yunohost-dyndns'): + raise YunohostError('domain_dyndns_already_subscribed') + if domain is None: domain = _get_maindomain() operation_logger.related_to.append(('domain', domain)) From f17f51f3f6de24c69eaf552209947a7414cb5b12 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 24 Feb 2019 03:32:14 +0100 Subject: [PATCH 42/91] Add ynh_app_setting helper to optimize app setting interface --- data/helpers.d/setting | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/data/helpers.d/setting b/data/helpers.d/setting index 6f75f6c80..18a57780a 100644 --- a/data/helpers.d/setting +++ b/data/helpers.d/setting @@ -14,7 +14,7 @@ ynh_app_setting_get() { # Manage arguments with getopts ynh_handle_getopts_args "$@" - sudo yunohost app setting "$app" "$key" --output-as plain --quiet + ynh_app_setting "get" "$app" "$key" } # Set an application setting @@ -33,7 +33,7 @@ ynh_app_setting_set() { # Manage arguments with getopts ynh_handle_getopts_args "$@" - sudo yunohost app setting "$app" "$key" --value="$value" --quiet + ynh_app_setting "set" "$app" "$key" "$value" } # Delete an application setting @@ -50,5 +50,36 @@ ynh_app_setting_delete() { # Manage arguments with getopts ynh_handle_getopts_args "$@" - sudo yunohost app setting -d "$app" "$key" --quiet + ynh_app_setting "delete" "$app" "$key" +} + +# Small "hard-coded" interface to avoid calling "yunohost app" directly each +# time dealing with a setting is needed (which may be so slow on ARM boards) +# +# [internal] +# +ynh_app_setting() +{ + ACTION="$1" APP="$2" KEY="$3" VALUE="$4" python - < Date: Mon, 25 Feb 2019 01:23:05 +0100 Subject: [PATCH 43/91] Typo --- locales/en.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locales/en.json b/locales/en.json index ac9091c19..a6d95ada1 100644 --- a/locales/en.json +++ b/locales/en.json @@ -211,7 +211,7 @@ "good_practices_about_user_password": "You are now about to define a new user password. The password should be at least 8 characters - though it is good practice to use longer password (i.e. a passphrase) and/or to use various kind of characters (uppercase, lowercase, digits and special characters).", "hook_exec_failed": "Script execution failed: {path:s}", "hook_exec_not_terminated": "Script execution hasn\u2019t terminated: {path:s}", - "hook_json_return_error": "Faild to read return from hook {path:s}. Error: {msg:s}", + "hook_json_return_error": "Failed to read return from hook {path:s}. Error: {msg:s}", "hook_list_by_invalid": "Invalid property to list hook by", "hook_name_unknown": "Unknown hook name '{name:s}'", "installation_complete": "Installation complete", From 990eeb76e1cf4092695ce92490a46925a797884a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 26 Feb 2019 00:40:16 +0100 Subject: [PATCH 44/91] Missing space breaks bash parsing of the whole file --- data/helpers.d/nodejs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/nodejs b/data/helpers.d/nodejs index 61a1414ef..098ed4410 100644 --- a/data/helpers.d/nodejs +++ b/data/helpers.d/nodejs @@ -99,7 +99,7 @@ ynh_install_nodejs () { # Install the requested version of nodejs uname=$(uname -m) - if [[ $uname =~ aarch64 || $uname =~ arm64]] + if [[ $uname =~ aarch64 || $uname =~ arm64 ]] then n $nodejs_version --arch=arm64 else From ac6f24671cce306251d12c9913e49a622be83981 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 26 Feb 2019 00:59:31 +0100 Subject: [PATCH 45/91] In case no file in dir, still an empty string will be read, which matches [[:digit:]]* (but won't with +) --- data/helpers.d/package | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/package b/data/helpers.d/package index 3924fc14e..000b0ee74 100644 --- a/data/helpers.d/package +++ b/data/helpers.d/package @@ -25,7 +25,7 @@ ynh_wait_dpkg_free() { while read dpkg_file <&9 do # Check if the name of this file contains only numbers. - if echo "$dpkg_file" | grep -Pq "^[[:digit:]]*$" + if echo "$dpkg_file" | grep -Pq "^[[:digit:]]+$" then # If so, that a remaining of dpkg. ynh_print_err "E: dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem." From b3edc2a63ccf23c95d7e58f9b3b17635372686b9 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Tue, 26 Feb 2019 01:59:03 +0100 Subject: [PATCH 46/91] Fix ynh_add_fail2ban_config legacy mode Merged as a micro decision Fix https://github.com/YunoHost/issues/issues/1313 --- data/helpers.d/backend | 1 + 1 file changed, 1 insertion(+) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index e710da9c7..1532601a8 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -352,6 +352,7 @@ ynh_remove_fpm_config () { # ynh_add_fail2ban_config () { # Declare an array to define the options of this helper. + local legacy_args=lrmptv declare -Ar args_array=( [l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= [t]=use_template [v]=others_var=) local logpath local failregex From b49eeddcf94ee4d733d10a9db657281d36648e8a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 26 Feb 2019 02:20:41 +0100 Subject: [PATCH 47/91] Drunk commits were drunk --- src/yunohost/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index e84187d6b..a622c0f24 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -593,12 +593,12 @@ def app_upgrade(auth, app=[], url=None, file=None): apps = [app] # Remove possible duplicates - apps = [app for i,app in enumerate(apps) if apps not in L[:i]] + apps = [app for i,app in enumerate(apps) if apps not in apps[:i]] if len(apps) == 0: raise YunohostError('app_no_upgrade') if len(apps) > 1: - logger.info(m18n.n("app_upgrade_several_apps", apps=", ".join(app))) + logger.info(m18n.n("app_upgrade_several_apps", apps=", ".join(apps))) for app_instance_name in apps: logger.info(m18n.n('app_upgrade_app_name', app=app_instance_name)) From 366d8231eb7d29708e4dea7a243c210acaa11c3f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 26 Feb 2019 02:29:18 +0100 Subject: [PATCH 48/91] Makes no sense to not show those messages only if that's a user-specified list ... --- src/yunohost/app.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index a622c0f24..a91ba61eb 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -583,12 +583,11 @@ def app_upgrade(auth, app=[], url=None, file=None): not_upgraded_apps = [] apps = app - user_specified_list = True # If no app is specified, upgrade all apps if not apps: + # FIXME : not sure what's supposed to happen if there is a url and a file but no apps... if not url and not file: apps = [app["id"] for app in app_list(installed=True)["apps"]] - user_specified_list = False elif not isinstance(app, list): apps = [app] @@ -618,8 +617,7 @@ def app_upgrade(auth, app=[], url=None, file=None): elif app_dict["upgradable"] == "yes": manifest, extracted_app_folder = _fetch_app_from_git(app_instance_name) else: - if user_specified_list: - logger.success(m18n.n('app_already_up_to_date', app=app_instance_name)) + logger.success(m18n.n('app_already_up_to_date', app=app_instance_name)) continue # Check requirements From 15b3733e2241cdc8322e19399827c2f65772399f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 26 Feb 2019 02:34:02 +0100 Subject: [PATCH 49/91] Don't raise an exception in the middle of the loop, that check can be done before --- src/yunohost/app.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index a91ba61eb..be0bb5a55 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -594,6 +594,10 @@ def app_upgrade(auth, app=[], url=None, file=None): # Remove possible duplicates apps = [app for i,app in enumerate(apps) if apps not in apps[:i]] + # Abort if any of those app is in fact not installed.. + for app in [app for app in apps if not _is_installed(app)]: + raise YunohostError('app_not_installed', app=app) + if len(apps) == 0: raise YunohostError('app_no_upgrade') if len(apps) > 1: @@ -601,9 +605,6 @@ def app_upgrade(auth, app=[], url=None, file=None): for app_instance_name in apps: logger.info(m18n.n('app_upgrade_app_name', app=app_instance_name)) - installed = _is_installed(app_instance_name) - if not installed: - raise YunohostError('app_not_installed', app=app_instance_name) app_dict = app_info(app_instance_name, raw=True) From f51101152a04fd6672198d1712c0886d27a80cc7 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Tue, 26 Feb 2019 15:37:50 +0100 Subject: [PATCH 50/91] Missing } --- data/helpers.d/system | 1 + 1 file changed, 1 insertion(+) diff --git a/data/helpers.d/system b/data/helpers.d/system index 604cf7bc6..cc1dbbcc5 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -148,6 +148,7 @@ ynh_clean_check_starting () { # Stop the execution of tail. kill -s 15 $pid_tail 2>&1 ynh_secure_remove "$templog" 2>&1 +} # Read the value of a key in a ynh manifest file # From e6f1845448c86c32a379353c8c0a70797135a52b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 1 Mar 2019 01:40:52 +0100 Subject: [PATCH 51/91] Add -f to rm --- src/yunohost/dyndns.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index 36509b0e4..ddc285e8d 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -157,12 +157,12 @@ def dyndns_subscribe(operation_logger, subscribe_host="dyndns.yunohost.org", dom try: r = requests.post('https://%s/key/%s?key_algo=hmac-sha512' % (subscribe_host, base64.b64encode(key)), data={'subdomain': domain}, timeout=30) except requests.ConnectionError: - os.system("rm %s" % private_file) - os.system("rm %s" % key_file) + os.system("rm -f %s" % private_file) + os.system("rm -f %s" % key_file) raise YunohostError('no_internet_connection') if r.status_code != 201: - os.system("rm %s" % private_file) - os.system("rm %s" % key_file) + os.system("rm -f %s" % private_file) + os.system("rm -f %s" % key_file) try: error = json.loads(r.text)['error'] except: From 4a8563ed9e86d4aaedc8b189373056b505a250b6 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 1 Mar 2019 01:43:16 +0100 Subject: [PATCH 52/91] Clarify the if case for 'get' --- data/helpers.d/setting | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data/helpers.d/setting b/data/helpers.d/setting index 18a57780a..ccd2ac0a6 100644 --- a/data/helpers.d/setting +++ b/data/helpers.d/setting @@ -75,9 +75,11 @@ else: if action == "delete": if key in settings: del settings[key] - else: + elif action == "get": if key in ['redirected_urls', 'redirected_regex']: value = yaml.load(value) + else: + raise ValueError("action should either be get, set or delete") settings[key] = value with open(setting_file, "w") as f: yaml.safe_dump(settings, f, default_flow_style=False) From 078b9b90bc22c56fc78255ae5fc72bea9e0e5af5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 3 Mar 2019 00:50:36 +0100 Subject: [PATCH 53/91] Drunk commits were made >.> --- data/helpers.d/setting | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/setting b/data/helpers.d/setting index ccd2ac0a6..fb833281a 100644 --- a/data/helpers.d/setting +++ b/data/helpers.d/setting @@ -75,12 +75,12 @@ else: if action == "delete": if key in settings: del settings[key] - elif action == "get": + elif action == "set": if key in ['redirected_urls', 'redirected_regex']: value = yaml.load(value) + settings[key] = value else: raise ValueError("action should either be get, set or delete") - settings[key] = value with open(setting_file, "w") as f: yaml.safe_dump(settings, f, default_flow_style=False) EOF From 12c61eeb6507faddf902a0bdc9a7c13fbba2fdbe Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sun, 3 Mar 2019 16:38:54 +0100 Subject: [PATCH 54/91] Use printers --- data/helpers.d/system | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/data/helpers.d/system b/data/helpers.d/system index cc1dbbcc5..f319f3f7b 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -106,10 +106,10 @@ ynh_systemd_action() { fi fi - echo "${action^} the service $service_name" >&2 + ynh_print_info --message="${action^} the service $service_name" systemctl $action $service_name \ || ( journalctl --no-pager --lines=$length -u $service_name >&2 \ - ; test -e "$log_path" && echo "--" && tail --lines=$length "$log_path" >&2 \ + ; test -e "$log_path" && echo "--" >&2 && tail --lines=$length "$log_path" >&2 \ ; false ) # Start the timeout and try to find line_match @@ -121,21 +121,20 @@ ynh_systemd_action() { # Read the log until the sentence is found, that means the app finished to start. Or run until the timeout if grep --quiet "$line_match" "$templog" then - echo "The service $service_name has correctly started." >&2 + ynh_print_info --message="The service $service_name has correctly started." break fi echo -n "." >&2 sleep 1 done + echo "" >&2 if [ $i -eq $timeout ] then - echo "The service $service_name didn't fully started before the timeout." >&2 - echo "Please find here an extract of the end of the log of the service $service_name:" + ynh_print_warn --message="The service $service_name didn't fully started before the timeout." + ynh_print_warn --message="Please find here an extract of the end of the log of the service $service_name:" journalctl --no-pager --lines=$length -u $service_name >&2 - test -e "$log_path" && echo "--" && tail --lines=$length "$log_path" >&2 + test -e "$log_path" && echo "--" >&2 && tail --lines=$length "$log_path" >&2 fi - - echo "" ynh_clean_check_starting fi } From 58af343a84d2319b2a70b7e06133953c99d1af60 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sun, 3 Mar 2019 17:25:07 +0100 Subject: [PATCH 55/91] Add warning about exec_* printers --- data/helpers.d/print | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/data/helpers.d/print b/data/helpers.d/print index 7f37021ae..b46192d87 100644 --- a/data/helpers.d/print +++ b/data/helpers.d/print @@ -89,6 +89,7 @@ ynh_print_err () { # usage: ynh_exec_err command to execute # usage: ynh_exec_err "command to execute | following command" # In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe. +# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed. # # | arg: command - command to execute ynh_exec_err () { @@ -100,6 +101,7 @@ ynh_exec_err () { # usage: ynh_exec_warn command to execute # usage: ynh_exec_warn "command to execute | following command" # In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe. +# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed. # # | arg: command - command to execute ynh_exec_warn () { @@ -111,6 +113,7 @@ ynh_exec_warn () { # usage: ynh_exec_warn_less command to execute # usage: ynh_exec_warn_less "command to execute | following command" # In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe. +# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed. # # | arg: command - command to execute ynh_exec_warn_less () { @@ -122,6 +125,7 @@ ynh_exec_warn_less () { # usage: ynh_exec_quiet command to execute # usage: ynh_exec_quiet "command to execute | following command" # In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe. +# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed. # # | arg: command - command to execute ynh_exec_quiet () { @@ -133,6 +137,7 @@ ynh_exec_quiet () { # usage: ynh_exec_fully_quiet command to execute # usage: ynh_exec_fully_quiet "command to execute | following command" # In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe. +# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed. # # | arg: command - command to execute ynh_exec_fully_quiet () { From 337a065b4a074a0a72b38d195875e1956136a804 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sun, 3 Mar 2019 18:57:41 +0100 Subject: [PATCH 56/91] Print progression only after 3s --- data/helpers.d/system | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/system b/data/helpers.d/system index f319f3f7b..7061e261f 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -124,10 +124,17 @@ ynh_systemd_action() { ynh_print_info --message="The service $service_name has correctly started." break fi - echo -n "." >&2 + if [ $i -eq 3 ]; then + echo -n "Please wait, the service $service_name is ${action}ing" >&2 + fi + if [ $i -ge 3 ]; then + echo -n "." >&2 + fi sleep 1 done - echo "" >&2 + if [ $i -ge 3 ]; then + echo "" >&2 + fi if [ $i -eq $timeout ] then ynh_print_warn --message="The service $service_name didn't fully started before the timeout." From 08e51f2680a82bf2fbf5a5de826e49cda86422ff Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sun, 3 Mar 2019 18:58:38 +0100 Subject: [PATCH 57/91] Do not show errors about log not existing yet. --- data/helpers.d/system | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/system b/data/helpers.d/system index 7061e261f..bfc20cb33 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -100,7 +100,7 @@ ynh_systemd_action() { local pid_tail=$! else # Read the specified log file - tail -F -n0 "$log_path" > "$templog" & + tail -F -n0 "$log_path" > "$templog" 2>&1 & # Get the PID of the tail command local pid_tail=$! fi From b8c5e9f1b6b0333c652ae89541baa80204ac054f Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 4 Mar 2019 02:59:14 +0100 Subject: [PATCH 58/91] [enh] allow 'display_text' ~fake~ argument in manifest.json --- src/yunohost/app.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index be0bb5a55..8561e2667 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2202,6 +2202,11 @@ def _parse_action_args_in_yunohost_format(args, action_args, auth=None): if arg_type == 'boolean': arg_default = 1 if arg_default else 0 + # do not print for webadmin + if arg_type == 'display_text' and msettings.get('interface') != 'api': + print(arg["text"]) + continue + # Attempt to retrieve argument value if arg_name in args: arg_value = args[arg_name] From f37a6b43e1ab4e163877b6afdd3226009cfd70a2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 4 Mar 2019 17:09:31 +0100 Subject: [PATCH 59/91] More general exception to handle other cases (e.g. SSL errors, ...) --- src/yunohost/dyndns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index ddc285e8d..2dadcef52 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -156,10 +156,10 @@ def dyndns_subscribe(operation_logger, subscribe_host="dyndns.yunohost.org", dom # Send subscription try: r = requests.post('https://%s/key/%s?key_algo=hmac-sha512' % (subscribe_host, base64.b64encode(key)), data={'subdomain': domain}, timeout=30) - except requests.ConnectionError: + except Exception as e: os.system("rm -f %s" % private_file) os.system("rm -f %s" % key_file) - raise YunohostError('no_internet_connection') + raise YunohostError('dyndns_registration_failed', error=str(e)) if r.status_code != 201: os.system("rm -f %s" % private_file) os.system("rm -f %s" % key_file) From 877fe506850b388807cfebf0e9d7561776ac2637 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 4 Mar 2019 17:21:57 +0100 Subject: [PATCH 60/91] Use a finally clause to avoid duplicated code --- src/yunohost/hook.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index d9cad9c7a..961caf606 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -403,13 +403,11 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False, else: returnjson = {} except Exception as e: + raise YunohostError('hook_json_return_error', path=path, msg=str(e)) + finally: + stdreturndir = os.path.split(stdreturn)[0] os.remove(stdreturn) os.rmdir(stdreturndir) - raise YunohostError('hook_json_return_error', path=path, msg=str(e)) - - stdreturndir = os.path.split(stdreturn)[0] - os.remove(stdreturn) - os.rmdir(stdreturndir) return returncode, returnjson From c74bff31d12e08acc80f54f628e09ce8c16f7adc Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 4 Mar 2019 17:44:20 +0100 Subject: [PATCH 61/91] Fix listing of succeed/failed hook names --- src/yunohost/service.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/yunohost/service.py b/src/yunohost/service.py index 151a877b9..61274aaac 100644 --- a/src/yunohost/service.py +++ b/src/yunohost/service.py @@ -493,11 +493,14 @@ def service_regen_conf(operation_logger, names=[], with_diff=False, force=False, pre_result = hook_callback('conf_regen', names, pre_callback=_pre_call) - # Update the services name - names = {n: [p for p, c in v.items() if c['state'] == "failed"] for n, v in pre_result.items()}.keys() + # Keep only the hook names with at least one success + names = [hook for hook, infos in pre_result.items() + if any(result["state"] == "succeed" for result in infos.values())] + # FIXME : what do in case of partial success/failure ... if not names: - ret_failed = {n: [p for p, c in v.items() if c['state'] == "failed"] for n, v in pre_result.items()} + ret_failed = [hook for hook, infos in pre_result.items() + if any(result["state"] == "failed" for result in infos.values())] raise YunohostError('service_regenconf_failed', services=', '.join(ret_failed)) From 16245f53d9591fcfeca70e5c54604ba3d26c2185 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 4 Mar 2019 18:33:09 +0100 Subject: [PATCH 62/91] Fix listing of succeed/failed for backup and restore --- src/yunohost/backup.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index d33f75e3a..872896ecb 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -593,10 +593,14 @@ class BackupManager(): env=env_dict, chdir=self.work_dir) - ret_succeed = {k: val for k, val in {n: [p for p, c in v.items() if c['state'] == "succeed"] for n, v in ret.items()}.items() if val} - ret_failed = {k: val for k, val in {n: [p for p, c in v.items() if c['state'] == "failed"] for n, v in ret.items()}.items() if val} + ret_succeed = {hook: {path:result["state"] for path, result in infos.items()} + for hook, infos in ret.items() + if any(result["state"] == "succeed" for result in infos.values())} + ret_failed = {hook: {path:result["state"] for path, result in infos.items.items()} + for hook, infos in ret.items() + if any(result["state"] == "failed" for result in infos.values())} - if ret_succeed != []: + if ret_succeed.keys() != []: self.system_return = ret_succeed # Add files from targets (which they put in the CSV) to the list of @@ -1180,14 +1184,16 @@ class RestoreManager(): env=env_dict, chdir=self.work_dir) - ret_succeed = {k: val for k, val in {n: [p for p, c in v.items() if c['state'] == "succeed"] for n, v in ret.items()}.items() if val} - ret_failed = {k: val for k, val in {n: [p for p, c in v.items() if c['state'] == "failed"] for n, v in ret.items()}.items() if val} + ret_succeed = [hook for hook, infos in ret.items() + if any(result["state"] == "succeed" for result in infos.values())] + ret_failed = [hook for hook, infos in ret.items() + if any(result["state"] == "failed" for result in infos.values())] - for part in ret_succeed.keys(): + for part in ret_succeed: self.targets.set_result("system", part, "Success") error_part = [] - for part in ret_failed.keys(): + for part in ret_failed: logger.error(m18n.n('restore_system_part_failed', part=part)) self.targets.set_result("system", part, "Error") error_part.append(part) From 88b7d77610d6493d62a490fa3ff758a992e7e270 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 4 Mar 2019 18:58:04 +0100 Subject: [PATCH 63/91] Display raw json content in case of exception --- locales/en.json | 2 +- src/yunohost/hook.py | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/locales/en.json b/locales/en.json index a6d95ada1..ca1d9c8a1 100644 --- a/locales/en.json +++ b/locales/en.json @@ -211,7 +211,7 @@ "good_practices_about_user_password": "You are now about to define a new user password. The password should be at least 8 characters - though it is good practice to use longer password (i.e. a passphrase) and/or to use various kind of characters (uppercase, lowercase, digits and special characters).", "hook_exec_failed": "Script execution failed: {path:s}", "hook_exec_not_terminated": "Script execution hasn\u2019t terminated: {path:s}", - "hook_json_return_error": "Failed to read return from hook {path:s}. Error: {msg:s}", + "hook_json_return_error": "Failed to read return from hook {path:s}. Error: {msg:s}. Raw content: {raw_content}", "hook_list_by_invalid": "Invalid property to list hook by", "hook_name_unknown": "Unknown hook name '{name:s}'", "installation_complete": "Installation complete", diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index 961caf606..c4605b6e8 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -395,15 +395,17 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False, elif raise_on_error and returncode != 0: raise YunohostError('hook_exec_failed', path=path) + raw_content = None try: - - with open(stdreturn, 'r') as f: - if f.read() != '': - returnjson = read_json(stdreturn) - else: - returnjson = {} + with open(stdreturn, 'r') as f: + raw_content = f.read() + if raw_content != '': + returnjson = read_json(stdreturn) + else: + returnjson = {} except Exception as e: - raise YunohostError('hook_json_return_error', path=path, msg=str(e)) + raise YunohostError('hook_json_return_error', path=path, msg=str(e), + raw_content=raw_content) finally: stdreturndir = os.path.split(stdreturn)[0] os.remove(stdreturn) From 5516e96d076eba0fc98f714b73e365693ddfc7a2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 4 Mar 2019 19:05:43 +0100 Subject: [PATCH 64/91] Simpler fetching of failed/succeeded hooks in CustomBackupMethod --- src/yunohost/backup.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 872896ecb..867de3ea9 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -1941,7 +1941,8 @@ class CustomBackupMethod(BackupMethod): ret = hook_callback('backup_method', [self.method], args=self._get_args('need_mount')) - ret_succeed = {k: val for k, val in {n: [p for p, c in v.items() if c['state'] == "succeed"] for n, v in ret.items()}.items() if val} + ret_succeed = [hook for hook, infos in ret.items() + if any(result["state"] == "succeed" for result in infos.values())] self._need_mount = True if ret_succeed else False return self._need_mount @@ -1955,7 +1956,9 @@ class CustomBackupMethod(BackupMethod): ret = hook_callback('backup_method', [self.method], args=self._get_args('backup')) - ret_failed = {k: val for k, val in {n: [p for p, c in v.items() if c['state'] == "failed"] for n, v in ret.items()}.items() if val} + + ret_failed = [hook for hook, infos in ret.items() + if any(result["state"] == "failed" for result in infos.values())] if ret_failed: raise YunohostError('backup_custom_backup_error') @@ -1969,7 +1972,9 @@ class CustomBackupMethod(BackupMethod): super(CustomBackupMethod, self).mount(restore_manager) ret = hook_callback('backup_method', [self.method], args=self._get_args('mount')) - ret_failed = {k: val for k, val in {n: [p for p, c in v.items() if c['state'] == "failed"] for n, v in ret.items()}.items() if val} + + ret_failed = [hook for hook, infos in ret.items() + if any(result["state"] == "failed" for result in infos.values())] if ret_failed: raise YunohostError('backup_custom_mount_error') From e3dfd63481fd72109ff2bf8bac562b123b95ad90 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 4 Mar 2019 22:54:02 +0100 Subject: [PATCH 65/91] Replace hard-corded value with constant defined above --- data/helpers.d/psql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index 324cfee83..705aeeb9a 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -244,7 +244,7 @@ ynh_psql_remove_db() { # # usage: ynh_psql_test_if_first_run ynh_psql_test_if_first_run() { - if [ -f /etc/yunohost/psql ]; then + if [ -f "$PSQL_ROOT_PWD_FILE" ]; then echo "PostgreSQL is already installed, no need to create master password" else local pgsql="$(ynh_string_random)" From def005d2da29763fc7d45d3398e0f82a896641d6 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 5 Mar 2019 00:42:20 +0100 Subject: [PATCH 66/91] Fix weird message --- locales/en.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locales/en.json b/locales/en.json index 1157e0a54..f7a65a883 100644 --- a/locales/en.json +++ b/locales/en.json @@ -218,7 +218,7 @@ "good_practices_about_admin_password": "You are now about to define a new administration password. The password should be at least 8 characters - though it is good practice to use longer password (i.e. a passphrase) and/or to use various kind of characters (uppercase, lowercase, digits and special characters).", "good_practices_about_user_password": "You are now about to define a new user password. The password should be at least 8 characters - though it is good practice to use longer password (i.e. a passphrase) and/or to use various kind of characters (uppercase, lowercase, digits and special characters).", "hook_exec_failed": "Script execution failed: {path:s}", - "hook_exec_not_terminated": "Script execution hasn\u2019t terminated: {path:s}", + "hook_exec_not_terminated": "Script execution did not finish properly: {path:s}", "hook_list_by_invalid": "Invalid property to list hook by", "hook_name_unknown": "Unknown hook name '{name:s}'", "installation_complete": "Installation complete", From f1e097a5bdb4bbc462fa823a42cb21e5e9fd7d1a Mon Sep 17 00:00:00 2001 From: Kayou Date: Thu, 7 Mar 2019 11:25:32 +0100 Subject: [PATCH 67/91] Fix tab --- data/helpers.d/psql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index 705aeeb9a..f0be628ea 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -74,9 +74,9 @@ ynh_psql_create_db() { local db=$1 local user=${2:-} - local sql="CREATE DATABASE ${db};" + local sql="CREATE DATABASE ${db};" - # grant all privilegies to user + # grant all privilegies to user if [ -n "$user" ]; then sql+="GRANT ALL PRIVILEGES ON DATABASE ${db} TO ${user} WITH GRANT OPTION;" fi From 5e0f63eab488ef6430bfa6bd92f050baf93cc175 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 7 Mar 2019 15:45:48 +0100 Subject: [PATCH 68/91] Reject app password if they contains { or } --- locales/en.json | 1 + src/yunohost/app.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/locales/en.json b/locales/en.json index f7a65a883..a70da7b82 100644 --- a/locales/en.json +++ b/locales/en.json @@ -380,6 +380,7 @@ "pattern_port_or_range": "Must be a valid port number (i.e. 0-65535) or range of ports (e.g. 100:200)", "pattern_positive_number": "Must be a positive number", "pattern_username": "Must be lower-case alphanumeric and underscore characters only", + "pattern_password_app": "Sorry, passwords should not contain the following characters: {forbidden_chars}", "port_already_closed": "Port {port:d} is already closed for {ip_version:s} connections", "port_already_opened": "Port {port:d} is already opened for {ip_version:s} connections", "port_available": "Port {port:d} is available", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index be0bb5a55..fa05ebe47 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2287,6 +2287,9 @@ def _parse_action_args_in_yunohost_format(args, action_args, auth=None): else: raise YunohostError('app_argument_choice_invalid', name=arg_name, choices='yes, no, y, n, 1, 0') elif arg_type == 'password': + forbidden_chars = "{}" + if any(char in arg_value for char in forbidden_chars): + raise YunohostError('pattern_password_app', forbidden_chars=forbidden_chars) from yunohost.utils.password import assert_password_is_strong_enough assert_password_is_strong_enough('user', arg_value) args_dict[arg_name] = arg_value From 55e9df75d0dd97bbf1eece1e0a39bbc645d49df2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 7 Mar 2019 17:03:59 +0100 Subject: [PATCH 69/91] Clean tmp directory if it not empty --- src/yunohost/backup.py | 64 +++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index f9505fb66..98758a57a 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -326,10 +326,19 @@ class BackupManager(): if not os.path.isdir(self.work_dir): filesystem.mkdir(self.work_dir, 0o750, parents=True, uid='admin') elif self.is_tmp_work_dir: - logger.debug("temporary directory for backup '%s' already exists", + + logger.debug("temporary directory for backup '%s' already exists... attempting to clean it", self.work_dir) - # FIXME May be we should clean the workdir here - raise YunohostError('backup_output_directory_not_empty') + + # Try to recursively unmount stuff (from a previously failed backup ?) + if _recursive_umount(self.work_dir) > 0: + raise YunohostError('backup_output_directory_not_empty') + else: + # If umount succeeded, remove the directory (we checked that + # we're in /home/yunohost.backup/tmp so that should be okay... + # c.f. method clean() which also does this) + filesystem.rm(self.work_dir, True, True) + filesystem.mkdir(self.work_dir, 0o750, parents=True, uid='admin') # # Backup target management # @@ -1514,34 +1523,12 @@ class BackupMethod(object): directories of the working directories """ if self.need_mount(): - if self._recursive_umount(self.work_dir) > 0: + if _recursive_umount(self.work_dir) > 0: raise YunohostError('backup_cleaning_failed') if self.manager.is_tmp_work_dir: filesystem.rm(self.work_dir, True, True) - def _recursive_umount(self, directory): - """ - Recursively umount sub directories of a directory - - Args: - directory -- a directory path - """ - mount_lines = subprocess.check_output("mount").split("\n") - - points_to_umount = [line.split(" ")[2] - for line in mount_lines - if len(line) >= 3 and line.split(" ")[2].startswith(directory)] - ret = 0 - for point in reversed(points_to_umount): - ret = subprocess.call(["umount", point]) - if ret != 0: - ret = 1 - logger.warning(m18n.n('backup_cleaning_failed', point)) - continue - - return ret - def _check_is_enough_free_space(self): """ Check free space in repository or output directory before to backup @@ -2011,6 +1998,7 @@ def backup_create(name=None, description=None, methods=[], # Check that output directory is empty if os.path.isdir(output_directory) and no_compress and \ os.listdir(output_directory): + raise YunohostError('backup_output_directory_not_empty') elif no_compress: raise YunohostError('backup_output_directory_required') @@ -2315,6 +2303,30 @@ def _call_for_each_path(self, callback, csv_path=None): callback(self, row['source'], row['dest']) +def _recursive_umount(directory): + """ + Recursively umount sub directories of a directory + + Args: + directory -- a directory path + """ + mount_lines = subprocess.check_output("mount").split("\n") + + points_to_umount = [line.split(" ")[2] + for line in mount_lines + if len(line) >= 3 and line.split(" ")[2].startswith(directory)] + + ret = 0 + for point in reversed(points_to_umount): + ret = subprocess.call(["umount", point]) + if ret != 0: + ret = 1 + logger.warning(m18n.n('backup_cleaning_failed', point)) + continue + + return ret + + def free_space_in_directory(dirpath): stat = os.statvfs(dirpath) return stat.f_frsize * stat.f_bavail From 135ac60d94d81e35ce790d0c0e7d7d037a490e03 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 7 Mar 2019 17:04:40 +0100 Subject: [PATCH 70/91] Clarify arguments usage --- src/yunohost/backup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 98758a57a..5608c7478 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -337,7 +337,7 @@ class BackupManager(): # If umount succeeded, remove the directory (we checked that # we're in /home/yunohost.backup/tmp so that should be okay... # c.f. method clean() which also does this) - filesystem.rm(self.work_dir, True, True) + filesystem.rm(self.work_dir, recursive=True, force=True) filesystem.mkdir(self.work_dir, 0o750, parents=True, uid='admin') # @@ -913,7 +913,7 @@ class RestoreManager(): ret = subprocess.call(["umount", self.work_dir]) if ret != 0: logger.warning(m18n.n('restore_cleaning_failed')) - filesystem.rm(self.work_dir, True, True) + filesystem.rm(self.work_dir, recursive=True, force=True) # # Restore target manangement # From 7f779ddd1cf60ba808f5ca0569c121be5fb75e62 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 7 Mar 2019 17:10:01 +0100 Subject: [PATCH 71/91] This ambiguity about 'ret' definitely looks like a bug ... --- src/yunohost/backup.py | 10 +++++----- src/yunohost/tests/test_backuprestore.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 5608c7478..039944264 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -331,7 +331,7 @@ class BackupManager(): self.work_dir) # Try to recursively unmount stuff (from a previously failed backup ?) - if _recursive_umount(self.work_dir) > 0: + if not _recursive_umount(self.work_dir): raise YunohostError('backup_output_directory_not_empty') else: # If umount succeeded, remove the directory (we checked that @@ -1523,7 +1523,7 @@ class BackupMethod(object): directories of the working directories """ if self.need_mount(): - if _recursive_umount(self.work_dir) > 0: + if not _recursive_umount(self.work_dir): raise YunohostError('backup_cleaning_failed') if self.manager.is_tmp_work_dir: @@ -2316,15 +2316,15 @@ def _recursive_umount(directory): for line in mount_lines if len(line) >= 3 and line.split(" ")[2].startswith(directory)] - ret = 0 + everything_went_fine = True for point in reversed(points_to_umount): ret = subprocess.call(["umount", point]) if ret != 0: - ret = 1 + everything_went_fine = False logger.warning(m18n.n('backup_cleaning_failed', point)) continue - return ret + return everything_went_fine def free_space_in_directory(dirpath): diff --git a/src/yunohost/tests/test_backuprestore.py b/src/yunohost/tests/test_backuprestore.py index 14c479d9a..73728240f 100644 --- a/src/yunohost/tests/test_backuprestore.py +++ b/src/yunohost/tests/test_backuprestore.py @@ -571,7 +571,7 @@ def test_backup_binds_are_readonly(monkeypatch): assert "Read-only file system" in output - if self._recursive_umount(self.work_dir) > 0: + if not _recursive_umount(self.work_dir): raise Exception("Backup cleaning failed !") self.clean() From a974d08897741e577353644acd28cad6c4727685 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 8 Mar 2019 01:22:32 +0100 Subject: [PATCH 72/91] Fix use case of app_fetchlist with url=None --- src/yunohost/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index de759f04f..99d3784bc 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -97,7 +97,7 @@ def app_fetchlist(url=None, name=None): name -- Name of the list url -- URL of remote JSON list """ - if not url.endswith(".json"): + if url and not url.endswith(".json"): raise YunohostError("This is not a valid application list url. It should end with .json.") # If needed, create folder where actual appslists are stored From ea98cf557ef3c93fdb3f47304a5745d38418a894 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 8 Mar 2019 19:53:56 +0100 Subject: [PATCH 73/91] Avoid miserably failing when attempting to create link in some edge case --- src/yunohost/backup.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 53919f2cc..80e13c273 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -1633,9 +1633,18 @@ class BackupMethod(object): # 'NUMBER OF HARD LINKS > 1' see #1043 cron_path = os.path.abspath('/etc/cron') + '.' if not os.path.abspath(src).startswith(cron_path): - os.link(src, dest) - # Success, go to next file to organize - continue + try: + os.link(src, dest) + except Exception as e: + # This kind of situation may happen when src and dest are on different + # logical volume ... even though the st_dev check previously match... + # E.g. this happens when running an encrypted hard drive + # where everything is mapped to /dev/mapper/some-stuff + # yet there are different devices behind it or idk ... + logger.warning("Could not link %s to %s (%s) ... falling back to regular copy." % (src, dest, str(e))) + else: + # Success, go to next file to organize + continue # If mountbind or hardlink couldnt be created, # prepare a list of files that need to be copied From bf659db300bbe658bbe180c3a3deb306f8c0f6e9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 8 Mar 2019 19:54:26 +0100 Subject: [PATCH 74/91] Not sure why this stop working, maybe new pytest version or something... --- src/yunohost/tests/test_backuprestore.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/tests/test_backuprestore.py b/src/yunohost/tests/test_backuprestore.py index 14c479d9a..54f31bcb2 100644 --- a/src/yunohost/tests/test_backuprestore.py +++ b/src/yunohost/tests/test_backuprestore.py @@ -42,7 +42,7 @@ def setup_function(function): assert len(backup_list()["archives"]) == 0 - markers = function.__dict__.keys() + markers = [m.name for m in function.__dict__.get("pytestmark",[])] if "with_wordpress_archive_from_2p4" in markers: add_archive_wordpress_from_2p4() @@ -82,7 +82,7 @@ def teardown_function(function): delete_all_backups() uninstall_test_apps_if_needed() - markers = function.__dict__.keys() + markers = [m.name for m in function.__dict__.get("pytestmark",[])] if "clean_opt_dir" in markers: shutil.rmtree("/opt/test_backup_output_directory") From 2818abe5000e2a8d08624bf459c908f2c3b76696 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 9 Mar 2019 16:45:53 +0100 Subject: [PATCH 75/91] Update global.conf --- data/templates/nginx/plain/global.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/templates/nginx/plain/global.conf b/data/templates/nginx/plain/global.conf index ca8721afb..07f7c49ea 100644 --- a/data/templates/nginx/plain/global.conf +++ b/data/templates/nginx/plain/global.conf @@ -1,2 +1,2 @@ server_tokens off; -gzip_types text/css text/javascript application/javascript; +gzip off; From 945912cc983c9b2fe67cf9a6905f0a07d37d2c4c Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sun, 10 Mar 2019 13:22:07 +0100 Subject: [PATCH 76/91] [fix] ynh_app_setting_get and ynh_app_setting_delete don't have a fourth argument. --- data/helpers.d/setting | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/setting b/data/helpers.d/setting index fb833281a..7aae6be7b 100644 --- a/data/helpers.d/setting +++ b/data/helpers.d/setting @@ -60,7 +60,7 @@ ynh_app_setting_delete() { # ynh_app_setting() { - ACTION="$1" APP="$2" KEY="$3" VALUE="$4" python - < Date: Sun, 10 Mar 2019 23:57:37 +0100 Subject: [PATCH 77/91] [microdecision] c.f. 25efab7 .. there's no way to include files in sshd_config --- .../0007_ssh_conf_managed_by_yunohost_step1.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py b/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py index 080cc0163..d188ff024 100644 --- a/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py +++ b/src/yunohost/data_migrations/0007_ssh_conf_managed_by_yunohost_step1.py @@ -49,10 +49,6 @@ class MyMigration(Migration): if dsa: settings_set("service.ssh.allow_deprecated_dsa_hostkey", True) - # Create sshd_config.d dir - if not os.path.exists(SSHD_CONF + '.d'): - mkdir(SSHD_CONF + '.d', 0o755, uid='root', gid='root') - # Here, we make it so that /etc/ssh/sshd_config is managed # by the regen conf (in particular in the case where the # from_script flag is present - in which case it was *not* From 1d37dd4fa28a7529dd438f1aaa544d2551894b0e Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Wed, 13 Mar 2019 13:32:50 +0100 Subject: [PATCH 78/91] [fix] Missing import --- src/yunohost/tests/test_backuprestore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/tests/test_backuprestore.py b/src/yunohost/tests/test_backuprestore.py index 73728240f..893925e83 100644 --- a/src/yunohost/tests/test_backuprestore.py +++ b/src/yunohost/tests/test_backuprestore.py @@ -10,7 +10,7 @@ from moulinette import m18n from moulinette.core import init_authenticator from yunohost.app import app_install, app_remove, app_ssowatconf from yunohost.app import _is_installed -from yunohost.backup import backup_create, backup_restore, backup_list, backup_info, backup_delete +from yunohost.backup import backup_create, backup_restore, backup_list, backup_info, backup_delete, _recursive_umount from yunohost.domain import _get_maindomain from yunohost.utils.error import YunohostError From 73aefc6bc8aaf3f01e983300afad1124d7b9eecd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9?= Date: Tue, 5 Feb 2019 20:09:15 +0000 Subject: [PATCH 79/91] Translated using Weblate (Russian) Currently translated at 9.4% (44 of 465 strings) --- locales/ru.json | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/locales/ru.json b/locales/ru.json index 2658446bc..306a8763a 100644 --- a/locales/ru.json +++ b/locales/ru.json @@ -6,5 +6,41 @@ "app_already_installed": "{app:s} уже установлено", "app_already_installed_cant_change_url": "Это приложение уже установлено. URL не может быть изменен только с помощью этой функции. Изучите `app changeurl`, если это доступно.", "app_argument_choice_invalid": "Неверный выбор для аргумента '{name:s}', Это должно быть '{choices:s}'", - "app_argument_invalid": "Недопустимое значение аргумента '{name:s}': {error:s}'" + "app_argument_invalid": "Недопустимое значение аргумента '{name:s}': {error:s}'", + "app_already_up_to_date": "{app:s} уже обновлено", + "app_argument_required": "Аргумент '{name:s}' необходим", + "app_change_no_change_url_script": "Приложение {app_name:s} не поддерживает изменение URL, вы должны обновить его.", + "app_change_url_identical_domains": "Старый и новый domain/url_path идентичны ('{domain:s}{path:s}'), ничего делать не надо.", + "app_change_url_no_script": "Приложение '{app_name:s}' не поддерживает изменение url. Наверное, вам нужно обновить приложение.", + "app_change_url_success": "Успешно изменён {app:s} url на {domain:s}{path:s}", + "app_extraction_failed": "Невозможно извлечь файлы для инсталляции", + "app_id_invalid": "Неправильный id приложения", + "app_incompatible": "Приложение {app} несовместимо с вашей версией YonoHost", + "app_install_files_invalid": "Неправильные файлы инсталляции", + "app_location_already_used": "Приложение '{app}' уже установлено по этому адресу ({path})", + "app_location_install_failed": "Невозможно установить приложение в это место, потому что оно конфликтует с приложением, '{other_app}' установленном на '{other_path}'", + "app_location_unavailable": "Этот url отсутствует или конфликтует с уже установленным приложением или приложениями: {apps:s}", + "app_manifest_invalid": "Недопустимый манифест приложения: {error}", + "app_no_upgrade": "Нет приложений, требующих обновления", + "app_not_correctly_installed": "{app:s} , кажется, установлены неправильно", + "app_not_installed": "{app:s} не установлены", + "app_not_properly_removed": "{app:s} удалены неправильно", + "app_package_need_update": "Пакет приложения {app} должен быть обновлён в соответствии с изменениями YonoHost", + "app_removed": "{app:s} удалено", + "app_requirements_checking": "Проверяю необходимые пакеты для {app}...", + "app_sources_fetch_failed": "Невозможно получить исходные файлы", + "app_unknown": "Неизвестное приложение", + "app_upgrade_app_name": "Обновление приложения {app}...", + "app_upgrade_failed": "Невозможно обновить {app:s}", + "app_upgrade_some_app_failed": "Невозможно обновить некоторые приложения", + "app_upgraded": "{app:s} обновлено", + "appslist_corrupted_json": "Не могу загрузить список приложений. Кажется, {filename:s} поврежден.", + "appslist_fetched": "Был выбран список приложений {appslist:s}", + "appslist_name_already_tracked": "Уже есть зарегистрированный список приложений по имени {name:s}.", + "appslist_removed": "Список приложений {appslist:s} удалён", + "appslist_retrieve_bad_format": "Неверный файл списка приложений{appslist:s}", + "appslist_retrieve_error": "Невозможно получить список удаленных приложений {appslist:s}: {error:s}", + "appslist_unknown": "Список приложений {appslist:s} неизвестен.", + "appslist_url_already_tracked": "Это уже зарегистрированный список приложений с url{url:s}.", + "installation_complete": "Установка завершена" } From 4ed63926804086fff2a037fb522796b0dcb145e4 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 13 Mar 2019 16:26:18 +0100 Subject: [PATCH 80/91] Update changelog for 3.5.0 testing --- debian/changelog | 51 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/debian/changelog b/debian/changelog index 7be4212fe..f8485e020 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,54 @@ +yunohost (3.5.0) testing; urgency=low + + Core + ---- + + - [fix] Disable gzip entirely to avoid BREACH attacks (#675) + - [fix] Backup tests were broken (#673) + - [fix] Backup fails because output directory not empty (#672) + - [fix] Reject app password if they contains { or } (#671) + - [enh] Allow `display_text` 'fake' argument in manifest.json (#669) + - [fix] Optimize dyndns requests (#662) + - [enh] Don't add Strict-Transport-Security header in nginx conf if using a selfsigned cert (#661) + - [enh] Add apt-transport-https to dependencies (#658) + - [enh] Cache results from meltdown vulnerability checker (#656) + - [enh] Ensure the tar file is closed during the backup (#655) + - [enh] Be able to define hook to trigger when changing a setting (#654) + - [enh] Assert dpkg is not broken before app install (#652) + - [fix] Loading only one helper file leads to errors because missing getopts (#651) + - [enh] Improve / add some messages to improve UX (#650) + - [enh] Reload fail2ban instead of restart (#649) + - [enh] Add IPv6 resolvers from diyisp.org to resolv.dnsmasq.conf (#639) + - [fix] Remove old SMTP port (465) from fail2ban jail.conf (#637) + - [enh] Improve protection against indexation from the robots. (#622) + - [enh] Allow hooks to return data (#526) + - [fix] Do not make version number available from web API to unauthenticated users (#291) + - [i18n] Improve Russian and Chinese (Mandarin) translations + + App helpers + ----------- + + - [enh] Optimize app setting helpers (#663, #676) + - [enh] Handle `ynh_install_nodejs` for arm64 / aarch64 (#660) + - [enh] Update postgresql helpers (#657) + - [enh] Print diff of files when backup by `ynh_backup_if_checksum_is_different` (#648) + - [enh] Add app debugger helper (#647) + - [fix] Escape double quote before eval in getopts (#646) + - [fix] `ynh_local_curl` not using the right url in some cases (#644) + - [fix] Get rid of annoying 'unable to initialize frontend' messages (#643) + - [enh] Check if dpkg is not broken when calling `ynh_wait_dpkg_free` (#638) + - [enh] Warn the packager that `ynh_secure_remove` should be used with only one arg… (#635, #642) + - [enh] Add `ynh_script_progression` helper (#634) + - [enh] Add `ynh_systemd_action` helper (#633) + - [enh] Allow to dig deeper into an archive with `ynh_setup_source` (#630) + - [enh] Use getops (#561) + - [enh] Add `ynh_check_app_version_changed` helper (#521) + - [enh] Add fail2ban helpers (#364) + + Contributors: Alexandre Aubin, Jimmy Monin, Josué Tille, Kayou, Laurent Peuch, Lukas Fülling, Maniack Crudelis, Taekiro, frju365, ljf, opi, yalh76, Алексей + + -- Alexandre Aubin Wed, 13 Mar 2019 16:10:00 +0000 + yunohost (3.4.2.4) stable; urgency=low - [fix] Meltdown vulnerability checker something outputing trash instead of pure json From 485cb4b5c75a7a74ab95104ffdc6eb4c7b00bfeb Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Wed, 13 Mar 2019 18:42:07 +0100 Subject: [PATCH 81/91] [fix] Delete properly group as well as user --- data/helpers.d/user | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/user b/data/helpers.d/user index d716bf03b..b28ee490a 100644 --- a/data/helpers.d/user +++ b/data/helpers.d/user @@ -63,6 +63,21 @@ ynh_system_user_exists() { getent passwd "$username" &>/dev/null } +# Check if a group exists on the system +# +# usage: ynh_system_group_exists --group=group +# | arg: -g, --group - the group to check +ynh_system_group_exists() { + # Declare an array to define the options of this helper. + local legacy_args=g + declare -Ar args_array=( [g]=group= ) + local group + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + getent group "$group" &>/dev/null +} + # Create a system user # # examples: @@ -116,11 +131,21 @@ ynh_system_user_delete () { # Manage arguments with getopts ynh_handle_getopts_args "$@" - if ynh_system_user_exists "$username" # Check if the user exists on the system + # Check if the user exists on the system + if ynh_system_user_exists "$username" then echo "Remove the user $username" >&2 - sudo userdel $username + deluser $username else echo "The user $username was not found" >&2 fi + + # Check if the group exists on the system + if ynh_system_group_exists "$username" + then + echo "Remove the group $username" >&2 + delgroup $username + else + echo "The group $username was not found" >&2 + fi } From ab574bb218a414f975651ac5b386bed9d9317363 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Wed, 13 Mar 2019 18:57:27 +0100 Subject: [PATCH 82/91] [enh] reload-or-restart instead of reload --- data/helpers.d/system | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/data/helpers.d/system b/data/helpers.d/system index bfc20cb33..fd552db7f 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -59,7 +59,7 @@ ynh_get_debian_release () { # Start (or other actions) a service, print a log in case of failure and optionnaly wait until the service is completely started # # usage: ynh_systemd_action [-n service_name] [-a action] [ [-l "line to match"] [-p log_path] [-t timeout] [-e length] ] -# | arg: -n, --service_name= - Name of the service to reload. Default : $app +# | arg: -n, --service_name= - Name of the service to start. Default : $app # | arg: -a, --action= - Action to perform with systemctl. Default: start # | arg: -l, --line_match= - Line to match - The line to find in the log to attest the service have finished to boot. # If not defined it don't wait until the service is completely started. @@ -107,6 +107,12 @@ ynh_systemd_action() { fi ynh_print_info --message="${action^} the service $service_name" + + # Use reload-or-restart instead of reload. So it wouldn't fail if the service isn't running. + if [ "$action" == "reload" ]; then + action="reload-or-restart" + fi + systemctl $action $service_name \ || ( journalctl --no-pager --lines=$length -u $service_name >&2 \ ; test -e "$log_path" && echo "--" >&2 && tail --lines=$length "$log_path" >&2 \ From a8f88e7232ca39005e9d78e45481ee8bb1e6e771 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 13 Mar 2019 19:11:49 +0100 Subject: [PATCH 83/91] [yolo] gzip off in global.conf breaks everything because conflict with gzip on; in nginx.conf ... Moving it to server blocs --- data/templates/nginx/plain/global.conf | 1 - data/templates/nginx/plain/yunohost_admin.conf | 4 ++++ data/templates/nginx/server.tpl.conf | 4 ++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/data/templates/nginx/plain/global.conf b/data/templates/nginx/plain/global.conf index 07f7c49ea..b3a5f356a 100644 --- a/data/templates/nginx/plain/global.conf +++ b/data/templates/nginx/plain/global.conf @@ -1,2 +1 @@ server_tokens off; -gzip off; diff --git a/data/templates/nginx/plain/yunohost_admin.conf b/data/templates/nginx/plain/yunohost_admin.conf index 2493e4033..ff61b8638 100644 --- a/data/templates/nginx/plain/yunohost_admin.conf +++ b/data/templates/nginx/plain/yunohost_admin.conf @@ -51,6 +51,10 @@ server { more_set_headers "X-Permitted-Cross-Domain-Policies : none"; more_set_headers "X-Frame-Options : SAMEORIGIN"; + # Disable gzip to protect against BREACH + # Read https://trac.nginx.org/nginx/ticket/1720 (text/html cannot be disabled!) + gzip off; + location / { return 302 https://$http_host/yunohost/admin; } diff --git a/data/templates/nginx/server.tpl.conf b/data/templates/nginx/server.tpl.conf index 43d38ca98..d8793ef05 100644 --- a/data/templates/nginx/server.tpl.conf +++ b/data/templates/nginx/server.tpl.conf @@ -71,6 +71,10 @@ server { resolver_timeout 5s; {% endif %} + # Disable gzip to protect against BREACH + # Read https://trac.nginx.org/nginx/ticket/1720 (text/html cannot be disabled!) + gzip off; + access_by_lua_file /usr/share/ssowat/access.lua; include /etc/nginx/conf.d/{{ domain }}.d/*.conf; From e3d8929dec650d6980e64a3b17f000150ac27c6e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 13 Mar 2019 19:24:00 +0100 Subject: [PATCH 84/91] Update changelog for 3.5.0.1 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index f8485e020..7290e935d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +yunohost (3.5.0.1) testing; urgency=low + + - [fix] #675 introduced a bug in nginx conf ... + + -- Alexandre Aubin Wed, 13 Mar 2019 19:23:00 +0000 + yunohost (3.5.0) testing; urgency=low Core From 1507e11e6eb84551e4e44ce4daf5241d536e22dd Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Wed, 13 Mar 2019 19:32:10 +0100 Subject: [PATCH 85/91] Do not warn if group isn't exist --- data/helpers.d/user | 2 -- 1 file changed, 2 deletions(-) diff --git a/data/helpers.d/user b/data/helpers.d/user index b28ee490a..5a65f7642 100644 --- a/data/helpers.d/user +++ b/data/helpers.d/user @@ -145,7 +145,5 @@ ynh_system_user_delete () { then echo "Remove the group $username" >&2 delgroup $username - else - echo "The group $username was not found" >&2 fi } From e24890c2f2f01bd6f3a934bf30ee713ad6503564 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 14 Mar 2019 03:48:35 +0100 Subject: [PATCH 86/91] Update changelog for 3.5.0.2 --- debian/changelog | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/debian/changelog b/debian/changelog index 7290e935d..444d797e1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +yunohost (3.5.0.2) testing; urgency=low + + - [fix] Make sure that `ynh_system_user_delete` also deletes the group (#680) + - [enh] `ynh_systemd_action` : reload-or-restart instead of just reload (#681) + + Last minute fixes by Maniack ;) + + -- Alexandre Aubin Thu, 14 Mar 2019 03:45:00 +0000 + yunohost (3.5.0.1) testing; urgency=low - [fix] #675 introduced a bug in nginx conf ... From 88deea7cec5032536ca23ddaf546bd982e1ce688 Mon Sep 17 00:00:00 2001 From: Weblate Date: Sat, 16 Mar 2019 21:34:46 +0000 Subject: [PATCH 87/91] Dunno why those changes were not actually commited... --- locales/ca.json | 61 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/locales/ca.json b/locales/ca.json index 6c06d55b3..bfad4d2bd 100644 --- a/locales/ca.json +++ b/locales/ca.json @@ -21,5 +21,64 @@ "app_location_already_used": "L'aplicació '{app}' ja està instal·lada en aquest camí ({path})", "app_make_default_location_already_used": "No es pot fer l'aplicació '{app}' per defecte en el domini {domain} ja que ja és utilitzat per una altra aplicació '{other_app}'", "app_location_install_failed": "No s'ha pogut instal·lar l'aplicació en aquest camí ja que entra en conflicte amb l'aplicació '{other_app}' ja instal·lada a '{other_path}'", - "app_location_unavailable": "Aquesta url no està disponible o entra en conflicte amb aplicacions ja instal·lades:\n{apps:s}" + "app_location_unavailable": "Aquesta url no està disponible o entra en conflicte amb aplicacions ja instal·lades:\n{apps:s}", + "app_manifest_invalid": "Manifest d'aplicació incorrecte: {error}", + "app_no_upgrade": "No hi ha cap aplicació per actualitzar", + "app_not_correctly_installed": "{app:s} sembla estar mal instal·lada", + "app_not_installed": "{app:s} no està instal·lada", + "app_not_properly_removed": "{app:s} no s'ha pogut suprimir correctament", + "app_package_need_update": "El paquet de l'aplicació {app} ha de ser actualitzat per poder seguir els canvis de YunoHost", + "app_removed": "{app:s} ha estat suprimida", + "app_requirements_checking": "Verificació dels paquets requerits per {app}", + "app_requirements_failed": "No es poden satisfer els requeriments per {app}: {error}", + "app_requirements_unmeet": "No es compleixen els requeriments per {app}, el paquet {pkgname} ({version}) ha de ser {spec}", + "app_sources_fetch_failed": "No s'han pogut carregar els fitxers font", + "app_unknown": "Aplicació desconeguda", + "app_unsupported_remote_type": "El tipus remot utilitzat per l'aplicació no està suportat", + "app_upgrade_app_name": "Actualitzant l'aplicació {app}...", + "app_upgrade_failed": "No s'ha pogut actualitzar {app:s}", + "app_upgrade_some_app_failed": "No s'han pogut actualitzar algunes aplicacions", + "app_upgraded": "{app:s} ha estat actualitzada", + "appslist_corrupted_json": "No s'han pogut carregar les llistes d'aplicacions. Sembla que {filename:s} està danyat.", + "appslist_could_not_migrate": "No s'ha pogut migrar la llista d'aplicacions {appslist:s}! No s'ha pogut analitzar la URL... L'antic cronjob s'ha guardat a {bkp_file:s}.", + "appslist_fetched": "S'ha descarregat la llista d'aplicacions {appslist:s} correctament", + "appslist_migrating": "Migrant la llista d'aplicacions {appslist:s} ...", + "appslist_name_already_tracked": "Ja hi ha una llista d'aplicacions registrada amb el nom {name:s}.", + "appslist_removed": "S'ha eliminat la llista d'aplicacions {appslist:s}", + "appslist_retrieve_bad_format": "L'arxiu obtingut per la llista d'aplicacions {appslist:s} no és vàlid", + "appslist_retrieve_error": "No s'ha pogut obtenir la llista d'aplicacions remota {appslist:s}: {error:s}", + "appslist_unknown": "La llista d'aplicacions {appslist:s} es desconeguda.", + "appslist_url_already_tracked": "Ja hi ha una llista d'aplicacions registrada amb al URL {url:s}.", + "ask_current_admin_password": "Contrasenya d'administrador actual", + "ask_email": "Correu electrònic", + "ask_firstname": "Nom", + "ask_lastname": "Cognom", + "ask_list_to_remove": "Llista per a suprimir", + "ask_main_domain": "Domini principal", + "ask_new_admin_password": "Nova contrasenya d'administrador", + "ask_password": "Contrasenya", + "ask_path": "Camí", + "backup_abstract_method": "Encara no s'ha implementat aquest mètode de copia de seguretat", + "backup_action_required": "S'ha d'especificar què s'ha de guardar", + "backup_app_failed": "No s'ha pogut fer la còpia de seguretat de l'aplicació \"{app:s}\"", + "backup_applying_method_borg": "Enviant tots els fitxers de la còpia de seguretat al repositori borg-backup...", + "backup_applying_method_copy": "Còpia de tots els fitxers a la còpia de seguretat...", + "backup_applying_method_custom": "Crida del mètode de còpia de seguretat personalitzat \"{method:s}\"...", + "backup_applying_method_tar": "Creació de l'arxiu tar de la còpia de seguretat...", + "backup_archive_app_not_found": "L'aplicació \"{app:s}\" no es troba dins l'arxiu de la còpia de seguretat", + "backup_archive_broken_link": "No s'ha pogut accedir a l'arxiu de la còpia de seguretat (enllaç invàlid cap a {path:s})", + "backup_archive_mount_failed": "No s'ha pogut carregar l'arxiu de la còpia de seguretat", + "backup_archive_name_exists": "Ja hi ha una còpia de seguretat amb aquest nom", + "backup_archive_name_unknown": "Còpia de seguretat local \"{name:s}\" desconeguda", + "backup_archive_open_failed": "No s'ha pogut obrir l'arxiu de la còpia de seguretat", + "backup_archive_system_part_not_available": "La part \"{part:s}\" del sistema no està disponible en aquesta copia de seguretat", + "backup_archive_writing_error": "No es poden afegir arxius a l'arxiu comprimit de la còpia de seguretat", + "backup_ask_for_copying_if_needed": "Alguns fitxers no s'han pogut preparar per la còpia de seguretat utilitzant el mètode que evita malgastar espai del sistema temporalment. Per fer la còpia de seguretat, s'han d'utilitzar {size:s}MB temporalment. Hi esteu d'acord?", + "backup_borg_not_implemented": "El mètode de còpia de seguretat Borg encara no està implementat", + "backup_cant_mount_uncompress_archive": "No es pot carregar en mode de lectura només el directori de l'arxiu descomprimit", + "backup_cleaning_failed": "No s'ha pogut netejar el directori temporal de la còpia de seguretat", + "backup_copying_to_organize_the_archive": "Copiant {size:s}MB per organitzar l'arxiu", + "backup_couldnt_bind": "No es pot lligar {src:s} amb {dest:s}.", + "backup_created": "S'ha creat la còpia de seguretat", + "backup_creating_archive": "Creant l'arxiu de la còpia de seguretat" } From 5ec499e75827fb6158e4c403cf31756c3ae9dfef Mon Sep 17 00:00:00 2001 From: Weblate Date: Sun, 17 Mar 2019 12:46:20 +0000 Subject: [PATCH 88/91] Added translation using Weblate (Greek) --- locales/el.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 locales/el.json diff --git a/locales/el.json b/locales/el.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/locales/el.json @@ -0,0 +1 @@ +{} From aa0c1985d08a5ce4c236b6d2d6ce180f3ddd4459 Mon Sep 17 00:00:00 2001 From: Weblate Date: Sun, 17 Mar 2019 12:47:04 +0000 Subject: [PATCH 89/91] Added translation using Weblate (Polish) --- locales/pl.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 locales/pl.json diff --git a/locales/pl.json b/locales/pl.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/locales/pl.json @@ -0,0 +1 @@ +{} From b5909d7d3445fc845daf5f11444b62ca20f6ebe2 Mon Sep 17 00:00:00 2001 From: Weblate Date: Sun, 17 Mar 2019 12:47:38 +0000 Subject: [PATCH 90/91] Added translation using Weblate (Swedish) --- locales/sv.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 locales/sv.json diff --git a/locales/sv.json b/locales/sv.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/locales/sv.json @@ -0,0 +1 @@ +{} From b9fe9cb3a569e6a451f733142726db979194ba12 Mon Sep 17 00:00:00 2001 From: Weblate Date: Sun, 17 Mar 2019 12:49:23 +0000 Subject: [PATCH 91/91] Added translation using Weblate (Chinese (Simplified)) --- locales/zh_Hans.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 locales/zh_Hans.json diff --git a/locales/zh_Hans.json b/locales/zh_Hans.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/locales/zh_Hans.json @@ -0,0 +1 @@ +{}