From 0c172cd3f95799f20d3ab55faa9d3d02437a35d4 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 2 Mar 2021 01:49:23 +0100 Subject: [PATCH 01/11] Catch more secrets not redacted --- src/yunohost/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/log.py b/src/yunohost/log.py index 7e9ae18e6..c51628c50 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -387,7 +387,7 @@ class RedactingFormatter(Formatter): # This matches stuff like db_pwd=the_secret or admin_password=other_secret # (the secret part being at least 3 chars to avoid catching some lines like just "db_pwd=") # Some names like "key" or "manifest_key" are ignored, used in helpers like ynh_app_setting_set or ynh_read_manifest - match = re.search(r'(pwd|pass|password|secret|\w+key|token)=(\S{3,})$', record.strip()) + match = re.search(r'(pwd|pass|password|secret\w*|\w+key|token)=(\S{3,})$', record.strip()) if match and match.group(2) not in self.data_to_redact and match.group(1) not in ["key", "manifest_key"]: self.data_to_redact.append(match.group(2)) except Exception as e: From a43cd72c72524b2f9eb22fa86a2edac0465ab432 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 25 Feb 2021 16:36:28 +0100 Subject: [PATCH 02/11] Fix ynh_replace_vars again, mystical bash is mystic... --- data/helpers.d/utils | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 5e02ca762..c67e1fae1 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -393,8 +393,11 @@ ynh_replace_vars () { for one_var in "${uniques_vars[@]}" do # Validate that one_var is indeed defined - # Explanation for the weird '+x' syntax: https://stackoverflow.com/a/13864829 - test -n "${one_var+x}" || ynh_die --message="Variable \$$one_var wasn't initialized when trying to replace __${one_var^^}__ in $file" + # -v checks if the variable is defined, for example: + # -v FOO tests if $FOO is defined + # -v $FOO tests if ${!FOO} is defined + # More info: https://stackoverflow.com/questions/3601515/how-to-check-if-a-variable-is-set-in-bash/17538964#comment96392525_17538964 + [[ -v "${one_var:-}" ]] || ynh_die --message="Variable \$$one_var wasn't initialized when trying to replace __${one_var^^}__ in $file" # Escape delimiter in match/replace string match_string="__${one_var^^}__" From 2728801d1798be9eff25294dc85dbce2c4bb02ad Mon Sep 17 00:00:00 2001 From: yalh76 Date: Sat, 27 Feb 2021 20:39:55 +0100 Subject: [PATCH 03/11] Force destination to be replaced --- data/helpers.d/utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index c67e1fae1..fee7e009b 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -321,7 +321,7 @@ ynh_add_config () { ynh_backup_if_checksum_is_different --file="$destination" - cp "$template_path" "$destination" + cp -f "$template_path" "$destination" ynh_replace_vars --file="$destination" From 9bbc3b72ae30092b717af7095359b16975700ca6 Mon Sep 17 00:00:00 2001 From: yalh76 Date: Sun, 28 Feb 2021 13:38:10 +0100 Subject: [PATCH 04/11] Make grep lazy --- data/helpers.d/utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index fee7e009b..b1b265b8b 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -386,7 +386,7 @@ ynh_replace_vars () { # Replace others variables # List other unique (__ __) variables in $file - local uniques_vars=( $(grep -o '__[A-Z0-9_]*__' $file | sort --unique | sed "s@__\([^.]*\)__@\L\1@g" )) + local uniques_vars=( $(grep -oP '__[A-Z0-9_]+?__' $file | sort --unique | sed "s@__\([^.]*\)__@\L\1@g" )) # Do the replacement local delimit=@ From 2402a1db39bf3a294459e1c0123b254e36c9f19c Mon Sep 17 00:00:00 2001 From: yalh76 Date: Sun, 28 Feb 2021 18:56:30 +0100 Subject: [PATCH 05/11] Fixing ___APP__ --- data/helpers.d/utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index b1b265b8b..53a7ea3b7 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -386,7 +386,7 @@ ynh_replace_vars () { # Replace others variables # List other unique (__ __) variables in $file - local uniques_vars=( $(grep -oP '__[A-Z0-9_]+?__' $file | sort --unique | sed "s@__\([^.]*\)__@\L\1@g" )) + local uniques_vars=( $(grep -oP '__[A-Z0-9]+?[A-Z0-9_]*?[A-Z0-9]*?__' $file | sort --unique | sed "s@__\([^.]*\)__@\L\1@g" )) # Do the replacement local delimit=@ From 88b414c8f3e0d3192a057f18260b65b9e3b1b23e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 25 Feb 2021 16:44:55 +0100 Subject: [PATCH 06/11] Don't redact empty string... --- src/yunohost/log.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/yunohost/log.py b/src/yunohost/log.py index c51628c50..a5501557d 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -376,7 +376,11 @@ class RedactingFormatter(Formatter): msg = super(RedactingFormatter, self).format(record) self.identify_data_to_redact(msg) for data in self.data_to_redact: - msg = msg.replace(data, "**********") + # we check that data is not empty string, + # otherwise this may lead to super epic stuff + # (try to run "foo".replace("", "bar")) + if data: + msg = msg.replace(data, "**********") return msg def identify_data_to_redact(self, record): From d12f403fe3235364f80f33c0522fb56e1998cb26 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 25 Feb 2021 16:57:57 +0100 Subject: [PATCH 07/11] Fix permission helper doc format --- data/helpers.d/permission | 47 +++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/data/helpers.d/permission b/data/helpers.d/permission index 1791425b5..4b51902eb 100644 --- a/data/helpers.d/permission +++ b/data/helpers.d/permission @@ -25,19 +25,14 @@ # usage: ynh_permission_create --permission="permission" [--url="url"] [--additional_urls="second-url" [ "third-url" ]] [--auth_header=true|false] # [--allowed=group1 [ group2 ]] [--label="label"] [--show_tile=true|false] # [--protected=true|false] -# | arg: -p, permission= - the name for the permission (by default a permission named "main" already exist) -# | arg: -u, url= - (optional) URL for which access will be allowed/forbidden. -# | Not that if 'show_tile' is enabled, this URL will be the URL of the tile. -# | arg: -A, additional_urls= - (optional) List of additional URL for which access will be allowed/forbidden -# | arg: -h, auth_header= - (optional) Define for the URL of this permission, if SSOwat pass the authentication header to the application. Default is true -# | arg: -a, allowed= - (optional) A list of group/user to allow for the permission -# | arg: -l, label= - (optional) Define a name for the permission. This label will be shown on the SSO and in the admin. -# | Default is "APP_LABEL (permission name)". -# | arg: -t, show_tile= - (optional) Define if a tile will be shown in the SSO. If yes the name of the tile will be the 'label' parameter. -# | Default is false (for the permission different than 'main'). -# | arg: -P, protected= - (optional) Define if this permission is protected. If it is protected the administrator -# | won't be able to add or remove the visitors group of this permission. -# | By default it's 'false' +# | arg: -p, --permission= - the name for the permission (by default a permission named "main" already exist) +# | arg: -u, --url= - (optional) URL for which access will be allowed/forbidden. Note that if 'show_tile' is enabled, this URL will be the URL of the tile. +# | arg: -A, --additional_urls= - (optional) List of additional URL for which access will be allowed/forbidden +# | arg: -h, --auth_header= - (optional) Define for the URL of this permission, if SSOwat pass the authentication header to the application. Default is true +# | arg: -a, --allowed= - (optional) A list of group/user to allow for the permission +# | arg: -l, --label= - (optional) Define a name for the permission. This label will be shown on the SSO and in the admin. Default is "APP_LABEL (permission name)". +# | arg: -t, --show_tile= - (optional) Define if a tile will be shown in the SSO. If yes the name of the tile will be the 'label' parameter. Defaults to false for the permission different than 'main'. +# | arg: -P, --protected= - (optional) Define if this permission is protected. If it is protected the administrator won't be able to add or remove the visitors group of this permission. Defaults to 'false'. # # If provided, 'url' or 'additional_urls' is assumed to be relative to the app domain/path if they # start with '/'. For example: @@ -192,13 +187,12 @@ ynh_permission_exists() { # # usage: ynh_permission_url --permission "permission" [--url="url"] [--add_url="new-url" [ "other-new-url" ]] [--remove_url="old-url" [ "other-old-url" ]] # [--auth_header=true|false] [--clear_urls] -# | arg: -p, permission= - the name for the permission (by default a permission named "main" is removed automatically when the app is removed) -# | arg: -u, url= - (optional) URL for which access will be allowed/forbidden. -# | Note that if you want to remove url you can pass an empty sting as arguments (""). -# | arg: -a, add_url= - (optional) List of additional url to add for which access will be allowed/forbidden. -# | arg: -r, remove_url= - (optional) List of additional url to remove for which access will be allowed/forbidden -# | arg: -h, auth_header= - (optional) Define for the URL of this permission, if SSOwat pass the authentication header to the application -# | arg: -c, clear_urls - (optional) Clean all urls (url and additional_urls) +# | arg: -p, --permission= - the name for the permission (by default a permission named "main" is removed automatically when the app is removed) +# | arg: -u, --url= - (optional) URL for which access will be allowed/forbidden. Note that if you want to remove url you can pass an empty sting as arguments (""). +# | arg: -a, --add_url= - (optional) List of additional url to add for which access will be allowed/forbidden. +# | arg: -r, --remove_url= - (optional) List of additional url to remove for which access will be allowed/forbidden +# | arg: -h, --auth_header= - (optional) Define for the URL of this permission, if SSOwat pass the authentication header to the application +# | arg: -c, --clear_urls - (optional) Clean all urls (url and additional_urls) # # Requires YunoHost version 3.7.0 or higher. ynh_permission_url() { @@ -268,13 +262,12 @@ ynh_permission_url() { # # usage: ynh_permission_update --permission "permission" [--add="group" ["group" ...]] [--remove="group" ["group" ...]] # [--label="label"] [--show_tile=true|false] [--protected=true|false] -# | arg: -p, permission= - the name for the permission (by default a permission named "main" already exist) -# | arg: -a, add= - the list of group or users to enable add to the permission -# | arg: -r, remove= - the list of group or users to remove from the permission -# | arg: -l, label= - (optional) Define a name for the permission. This label will be shown on the SSO and in the admin. -# | arg: -t, show_tile= - (optional) Define if a tile will be shown in the SSO -# | arg: -P, protected= - (optional) Define if this permission is protected. If it is protected the administrator -# | won't be able to add or remove the visitors group of this permission. +# | arg: -p, --permission= - the name for the permission (by default a permission named "main" already exist) +# | arg: -a, --add= - the list of group or users to enable add to the permission +# | arg: -r, --remove= - the list of group or users to remove from the permission +# | arg: -l, --label= - (optional) Define a name for the permission. This label will be shown on the SSO and in the admin. +# | arg: -t, --show_tile= - (optional) Define if a tile will be shown in the SSO +# | arg: -P, --protected= - (optional) Define if this permission is protected. If it is protected the administrator won't be able to add or remove the visitors group of this permission. # # Requires YunoHost version 3.7.0 or higher. ynh_permission_update() { From 05969184feda1cb6c554ff0be2857229d140fde9 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Thu, 25 Feb 2021 11:16:34 +0100 Subject: [PATCH 08/11] fix ynh_systemd_action in case of service fails --- data/helpers.d/systemd | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data/helpers.d/systemd b/data/helpers.d/systemd index b0e175d4d..7f316cf2b 100644 --- a/data/helpers.d/systemd +++ b/data/helpers.d/systemd @@ -92,7 +92,7 @@ ynh_remove_systemd_config () { # usage: ynh_systemd_action [--service_name=service_name] [--action=action] [ [--line_match="line to match"] [--log_path=log_path] [--timeout=300] [--length=20] ] # | arg: -n, --service_name= - Name of the service to start. Default : $app # | arg: -a, --action= - Action to perform with systemctl. Default: start -# | arg: -l, --line_match= - Line to match - The line to find in the log to attest the service have finished to boot. If not defined it don't wait until the service is completely started. WARNING: When using --line_match, you should always add `ynh_clean_check_starting` into your `ynh_clean_setup` at the beginning of the script. Otherwise, tail will not stop in case of failure of the script. The script will then hang forever. +# | arg: -l, --line_match= - Line to match - The line to find in the log to attest the service have finished to boot. If not defined it don't wait until the service is completely started. # | arg: -p, --log_path= - Log file - Path to the log file. Default : /var/log/$app/$app.log # | arg: -t, --timeout= - Timeout - The maximum time to wait before ending the watching. Default : 300 seconds. # | arg: -e, --length= - Length of the error log : Default : 20 @@ -151,6 +151,7 @@ ynh_systemd_action() { then ynh_exec_err tail --lines=$length "$log_path" fi + ynh_clean_check_starting return 1 fi @@ -195,9 +196,8 @@ ynh_systemd_action() { } # Clean temporary process and file used by ynh_check_starting -# (usually used in ynh_clean_setup scripts) # -# usage: ynh_clean_check_starting +# [internal] # # Requires YunoHost version 3.5.0 or higher. ynh_clean_check_starting () { @@ -208,7 +208,7 @@ ynh_clean_check_starting () { fi if [ -n "$templog" ] then - ynh_secure_remove "$templog" 2>&1 + ynh_secure_remove --file="$templog" 2>&1 fi } From 6ce02270ae9b6f5ad0c14b790769795b80aa0afa Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 22 Feb 2021 23:46:12 +0100 Subject: [PATCH 09/11] Gotta escape \ during ynh_replace_vars --- data/helpers.d/utils | 1 + 1 file changed, 1 insertion(+) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 53a7ea3b7..e26cac5ba 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -403,6 +403,7 @@ ynh_replace_vars () { match_string="__${one_var^^}__" match_string=${match_string//${delimit}/"\\${delimit}"} replace_string="${!one_var}" + replace_string=${replace_string//\\/\\\\} replace_string=${replace_string//${delimit}/"\\${delimit}"} # Actually replace (sed is used instead of ynh_replace_string to avoid triggering an epic amount of debug logs) From bd8644a651a77b16713aec2d83ca04468c31b5b5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 20 Feb 2021 17:07:38 +0100 Subject: [PATCH 10/11] Translation typo.. --- locales/it.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locales/it.json b/locales/it.json index 9b5d18f1a..0a5095b9e 100644 --- a/locales/it.json +++ b/locales/it.json @@ -591,7 +591,7 @@ "log_user_permission_update": "Aggiorna gli accessi del permesso '{}'", "log_user_group_update": "Aggiorna il gruppo '{}'", "log_user_group_delete": "Cancella il gruppo '{}'", - "log_user_group_create": "Crea il gruppo '[}'", + "log_user_group_create": "Crea il gruppo '{}'", "log_permission_url": "Aggiorna l'URL collegato al permesso '{}'", "log_permission_delete": "Cancella permesso '{}'", "log_permission_create": "Crea permesso '{}'", From 234ae15d771b620b1239236c789cd5be66f7062f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 2 Mar 2021 02:07:58 +0100 Subject: [PATCH 11/11] Update changelog for 4.1.7.3 --- debian/changelog | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 95cca2eb8..5fb0e563d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,17 @@ -yunohost (4.1.7.2) testing; urgency=low +yunohost (4.1.7.3) stable; urgency=low + + - [fix] log: Some secrets were not redacted (0c172cd3) + - [fix] log: For some reason sometimes we were redacting 'empty string' which made everything explode (88b414c8) + - [fix] helpers: Various fixes for ynh_add_config / ynh_replace_vars (a43cd72c, 2728801d, 9bbc3b72, 2402a1db, 6ce02270) + - [fix] helpers: Fix permission helpers doc format (d12f403f) + - [fix] helpers: ynh_systemd_action did not properly clean the 'tail' process when service action failed (05969184) + - [fix] i18n: Translation typo in italian translation ... (bd8644a6) + + Thanks to all contributors <3 ! (Kay0u, yalh76) + + -- Alexandre Aubin Tue, 02 Mar 2021 02:03:35 +0100 + +yunohost (4.1.7.2) stable; urgency=low - [fix] When migration legacy protected permissions, all users were allowed on the new perm (29bd3c4a) - [fix] Mysql is a fucking joke (... trying to fix the mysql issue on RPi ...) (cd4fdb2b)