From 3550be63f2e5cb556a0c080ca58b62836f052d84 Mon Sep 17 00:00:00 2001 From: Christophe Vuillot Date: Mon, 11 Mar 2019 14:53:40 +0100 Subject: [PATCH 01/42] Added a python script (yunohost_completion.py) which generates a bash completion file for the yunohost command based on yunohost.yml, in data/actionsmap Added the output of the script in data/bash-completion.d/yunohost_completion This is probably not the correct place for the script and the generation should be done at some other time and place also. --- data/actionsmap/yunohost_completion.py | 84 ++++++++++++++++++++++ data/bash-completion.d/yunohost_completion | 77 ++++++++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 data/actionsmap/yunohost_completion.py create mode 100644 data/bash-completion.d/yunohost_completion diff --git a/data/actionsmap/yunohost_completion.py b/data/actionsmap/yunohost_completion.py new file mode 100644 index 000000000..9b5472837 --- /dev/null +++ b/data/actionsmap/yunohost_completion.py @@ -0,0 +1,84 @@ +""" +Simple automated generation of a bash_completion file +for yunohost command from the actionsmap. + +Generates a bash completion file assuming the structure +`yunohost domain action` +adds `--help` at the end if one presses [tab] again. + +author: Christophe Vuillot +""" +import yaml + +ACTIONSMAP_FILE = 'yunohost.yml' +BASH_COMPLETION_FILE = '../bash-completion.d/yunohost_completion' + +with open(ACTIONSMAP_FILE, 'r') as stream: + + # Getting the dictionary containning what actions are possible per domain + OPTION_TREE = yaml.load(stream) + DOMAINS = [str for str in OPTION_TREE.keys() if not str.startswith('_')] + DOMAINS_STR = '"{}"'.format(' '.join(DOMAINS)) + ACTIONS_DICT = {} + for domain in DOMAINS: + ACTIONS = [str for str in OPTION_TREE[domain]['actions'].keys() + if not str.startswith('_')] + ACTIONS_STR = '"{}"'.format(' '.join(ACTIONS)) + ACTIONS_DICT[domain] = ACTIONS_STR + + with open(BASH_COMPLETION_FILE, 'w') as generated_file: + + # header of the file + generated_file.write('#\n') + generated_file.write('# completion for yunohost\n') + generated_file.write('# automatically generated from the actionsmap\n') + generated_file.write('#\n\n') + + # Start of the completion function + generated_file.write('_yunohost_completion()\n') + generated_file.write('{\n') + + # Defining local variable for previously and currently typed words + generated_file.write('\tlocal cur prev opts narg\n') + generated_file.write('\tCOMPREPLY=()\n\n') + generated_file.write('\t# the number of words already typed\n') + generated_file.write('\tnarg=${#COMP_WORDS[@]}\n\n') + generated_file.write('\t# the current word being typed\n') + generated_file.write('\tcur="${COMP_WORDS[COMP_CWORD]}"\n\n') + generated_file.write('\t# the last typed word\n') + generated_file.write('\tprev="${COMP_WORDS[COMP_CWORD-1]}"\n\n') + + # If one is currently typing a domain then match with the domain list + generated_file.write('\t# If one is currently typing a domain,\n') + generated_file.write('\t# match with domains\n') + generated_file.write('\tif [[ $narg == 2 ]]; then\n') + generated_file.write('\t\topts={}\n'.format(DOMAINS_STR)) + generated_file.write('\tfi\n\n') + + # If one is currently typing an action then match with the action list + # of the previously typed domain + generated_file.write('\t# If one already typed a domain,\n') + generated_file.write('\t# match the actions of that domain\n') + generated_file.write('\tif [[ $narg == 3 ]]; then\n') + for domain in DOMAINS: + generated_file.write('\t\tif [[ $prev == "{}" ]]; then\n'.format(domain)) + generated_file.write('\t\t\topts={}\n'.format(ACTIONS_DICT[domain])) + generated_file.write('\t\tfi\n') + generated_file.write('\tfi\n\n') + + # If both domain and action have been typed or the domain + # was not recognized propose --help (only once) + generated_file.write('\t# If no options were found propose --help\n') + generated_file.write('\tif [ -z "$opts" ]; then\n') + generated_file.write('\t\tif [[ $prev != "--help" ]]; then\n') + generated_file.write('\t\t\topts=( --help )\n') + generated_file.write('\t\tfi\n') + generated_file.write('\tfi\n') + + # generate the completion list from the possible options + generated_file.write('\tCOMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )\n') + generated_file.write('\treturn 0\n') + generated_file.write('}\n\n') + + # Add the function to bash completion + generated_file.write('complete -F _yunohost_completion yunohost') diff --git a/data/bash-completion.d/yunohost_completion b/data/bash-completion.d/yunohost_completion new file mode 100644 index 000000000..715073475 --- /dev/null +++ b/data/bash-completion.d/yunohost_completion @@ -0,0 +1,77 @@ +# +# completion for yunohost +# automatically generated from the actionsmap +# + +_yunohost_completion() +{ + local cur prev opts narg + COMPREPLY=() + + # the number of words already typed + narg=${#COMP_WORDS[@]} + + # the current word being typed + cur="${COMP_WORDS[COMP_CWORD]}" + + # the last typed word + prev="${COMP_WORDS[COMP_CWORD-1]}" + + # If one is currently typing a domain, + # match with domains + if [[ $narg == 2 ]]; then + opts="user domain log service settings firewall backup app hook dyndns tools monitor" + fi + + # If one already typed a domain, + # match the actions of that domain + if [[ $narg == 3 ]]; then + if [[ $prev == "user" ]]; then + opts="info create list update delete" + fi + if [[ $prev == "domain" ]]; then + opts="cert-install cert-status list remove url-available add dns-conf cert-renew" + fi + if [[ $prev == "log" ]]; then + opts="list display" + fi + if [[ $prev == "service" ]]; then + opts="status enable reload_or_restart log start stop remove reload add disable regen-conf restart" + fi + if [[ $prev == "settings" ]]; then + opts="reset set list reset-all get" + fi + if [[ $prev == "firewall" ]]; then + opts="reload allow stop list upnp disallow" + fi + if [[ $prev == "backup" ]]; then + opts="info restore create list delete" + fi + if [[ $prev == "app" ]]; then + opts="map checkurl install makedefault checkport listlists change-url removelist info change-label upgrade fetchlist clearaccess ssowatconf list remove register-url removeaccess setting initdb debug addaccess" + fi + if [[ $prev == "hook" ]]; then + opts="info callback add exec list remove" + fi + if [[ $prev == "dyndns" ]]; then + opts="subscribe update installcron removecron" + fi + if [[ $prev == "tools" ]]; then + opts="upgrade ldapinit postinstall maindomain update reboot shell adminpw shutdown diagnosis port-available" + fi + if [[ $prev == "monitor" ]]; then + opts="enable network show-stats update-stats disk system disable" + fi + fi + + # If no options were found propose --help + if [ -z "$opts" ]; then + if [[ $prev != "--help" ]]; then + opts=( --help ) + fi + fi + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 +} + +complete -F _yunohost_completion yunohost \ No newline at end of file From f35eb1785437d9f7b1fd3c66edeb83868dc26418 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 26 Mar 2019 18:49:32 +0100 Subject: [PATCH 02/42] Mark YunoHost as essential to avoid removing it inadvertenly --- debian/control | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/control b/debian/control index 685c194ba..a6fd95154 100644 --- a/debian/control +++ b/debian/control @@ -1,6 +1,7 @@ Source: yunohost Section: utils Priority: extra +Essential: yes Maintainer: YunoHost Contributors Build-Depends: debhelper (>=9), dh-systemd, dh-python, python-all (>= 2.7) Standards-Version: 3.9.6 From ccdd7e645d7a745d5293a188820e6e75492330c5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 28 Mar 2019 16:47:54 +0100 Subject: [PATCH 03/42] Be able to run the script from a distant folder --- data/actionsmap/yunohost_completion.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/data/actionsmap/yunohost_completion.py b/data/actionsmap/yunohost_completion.py index 9b5472837..915c6ac89 100644 --- a/data/actionsmap/yunohost_completion.py +++ b/data/actionsmap/yunohost_completion.py @@ -8,10 +8,12 @@ adds `--help` at the end if one presses [tab] again. author: Christophe Vuillot """ +import os import yaml -ACTIONSMAP_FILE = 'yunohost.yml' -BASH_COMPLETION_FILE = '../bash-completion.d/yunohost_completion' +THIS_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +ACTIONSMAP_FILE = THIS_SCRIPT_DIR + '/yunohost.yml' +BASH_COMPLETION_FILE = THIS_SCRIPT_DIR + '/../bash-completion.d/yunohost_completion' with open(ACTIONSMAP_FILE, 'r') as stream: From b3d8167548803364b62ecb8cafafe0f78802497c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 2 Apr 2019 01:53:10 +0200 Subject: [PATCH 04/42] Generate yunohost_completion.py during debian builds --- data/bash-completion.d/yunohost | 15 +---- data/bash-completion.d/yunohost_completion | 77 ---------------------- debian/rules | 4 ++ 3 files changed, 7 insertions(+), 89 deletions(-) delete mode 100644 data/bash-completion.d/yunohost_completion diff --git a/data/bash-completion.d/yunohost b/data/bash-completion.d/yunohost index 106f8fbdf..2572a391d 100644 --- a/data/bash-completion.d/yunohost +++ b/data/bash-completion.d/yunohost @@ -1,12 +1,3 @@ -# -# Bash completion for yunohost -# - -_python_argcomplete() { - local IFS=' ' - COMPREPLY=( $(IFS="$IFS" COMP_LINE="$COMP_LINE" COMP_POINT="$COMP_POINT" _ARGCOMPLETE_COMP_WORDBREAKS="$COMP_WORDBREAKS" _ARGCOMPLETE=1 "$1" 8>&1 9>&2 1>/dev/null 2>/dev/null) ) - if [[ $? != 0 ]]; then - unset COMPREPLY - fi -} -complete -o nospace -o default -F _python_argcomplete "yunohost" +# This file is automatically generated +# during Debian's package build by the script +# data/actionsmap/yunohost_completion.py diff --git a/data/bash-completion.d/yunohost_completion b/data/bash-completion.d/yunohost_completion deleted file mode 100644 index 715073475..000000000 --- a/data/bash-completion.d/yunohost_completion +++ /dev/null @@ -1,77 +0,0 @@ -# -# completion for yunohost -# automatically generated from the actionsmap -# - -_yunohost_completion() -{ - local cur prev opts narg - COMPREPLY=() - - # the number of words already typed - narg=${#COMP_WORDS[@]} - - # the current word being typed - cur="${COMP_WORDS[COMP_CWORD]}" - - # the last typed word - prev="${COMP_WORDS[COMP_CWORD-1]}" - - # If one is currently typing a domain, - # match with domains - if [[ $narg == 2 ]]; then - opts="user domain log service settings firewall backup app hook dyndns tools monitor" - fi - - # If one already typed a domain, - # match the actions of that domain - if [[ $narg == 3 ]]; then - if [[ $prev == "user" ]]; then - opts="info create list update delete" - fi - if [[ $prev == "domain" ]]; then - opts="cert-install cert-status list remove url-available add dns-conf cert-renew" - fi - if [[ $prev == "log" ]]; then - opts="list display" - fi - if [[ $prev == "service" ]]; then - opts="status enable reload_or_restart log start stop remove reload add disable regen-conf restart" - fi - if [[ $prev == "settings" ]]; then - opts="reset set list reset-all get" - fi - if [[ $prev == "firewall" ]]; then - opts="reload allow stop list upnp disallow" - fi - if [[ $prev == "backup" ]]; then - opts="info restore create list delete" - fi - if [[ $prev == "app" ]]; then - opts="map checkurl install makedefault checkport listlists change-url removelist info change-label upgrade fetchlist clearaccess ssowatconf list remove register-url removeaccess setting initdb debug addaccess" - fi - if [[ $prev == "hook" ]]; then - opts="info callback add exec list remove" - fi - if [[ $prev == "dyndns" ]]; then - opts="subscribe update installcron removecron" - fi - if [[ $prev == "tools" ]]; then - opts="upgrade ldapinit postinstall maindomain update reboot shell adminpw shutdown diagnosis port-available" - fi - if [[ $prev == "monitor" ]]; then - opts="enable network show-stats update-stats disk system disable" - fi - fi - - # If no options were found propose --help - if [ -z "$opts" ]; then - if [[ $prev != "--help" ]]; then - opts=( --help ) - fi - fi - COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) - return 0 -} - -complete -F _yunohost_completion yunohost \ No newline at end of file diff --git a/debian/rules b/debian/rules index ce03d0e31..d012c73f3 100755 --- a/debian/rules +++ b/debian/rules @@ -7,6 +7,10 @@ %: dh ${@} --with=python2,systemd +override_dh_auto_build: + # Generate bash completion file + python data/actionsmap/yunohost_completion.py + override_dh_installinit: dh_installinit -pyunohost --name=yunohost-api --restart-after-upgrade dh_installinit -pyunohost --name=yunohost-firewall --noscripts From 15cfe22b3c835e313a44ea44d07105ba9e3a0b2d Mon Sep 17 00:00:00 2001 From: Kayou Date: Wed, 3 Apr 2019 00:25:50 +0200 Subject: [PATCH 05/42] Add size of apps in backup_info result --- src/yunohost/backup.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 6f969327b..f10b112ac 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -2272,6 +2272,14 @@ def backup_info(name, with_details=False, human_readable=False): if "hooks" in info.keys(): system_key = "hooks" + + if "size_details" in info.keys(): + for name, key_info in info["apps"].items(): + if name in info["size_details"]["apps"].keys(): + key_info["size"] = info["size_details"]["apps"][name] + else: + key_info["size"] = 0 + result["apps"] = info["apps"] result["system"] = info[system_key] return result From 1a1fc0bdff52fbc9d77ed01c8148e818d358749b Mon Sep 17 00:00:00 2001 From: Kayou Date: Thu, 4 Apr 2019 12:28:00 +0200 Subject: [PATCH 06/42] Add connection_upgrade --- data/templates/nginx/server.tpl.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/data/templates/nginx/server.tpl.conf b/data/templates/nginx/server.tpl.conf index fa8b6586b..14cb52abc 100644 --- a/data/templates/nginx/server.tpl.conf +++ b/data/templates/nginx/server.tpl.conf @@ -1,3 +1,8 @@ +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + server { listen 80; listen [::]:80; From e9d399f7cd92ba50ea19b7da1fcc25997ecb650b Mon Sep 17 00:00:00 2001 From: Kayou Date: Thu, 4 Apr 2019 12:29:23 +0200 Subject: [PATCH 07/42] Format --- data/templates/nginx/server.tpl.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/templates/nginx/server.tpl.conf b/data/templates/nginx/server.tpl.conf index 14cb52abc..4a5e91557 100644 --- a/data/templates/nginx/server.tpl.conf +++ b/data/templates/nginx/server.tpl.conf @@ -1,6 +1,6 @@ map $http_upgrade $connection_upgrade { - default upgrade; - '' close; + default upgrade; + '' close; } server { From 214d1ce8fc14b2c1afb7a7852574d6853e9c5be0 Mon Sep 17 00:00:00 2001 From: Josue-T Date: Sun, 7 Apr 2019 11:27:38 +0200 Subject: [PATCH 08/42] Add support in restore script of ynh_setup_source Actually the path to the app.src file in the helper ynh_setup_source work only in install and upgrade script. Add the support for restore script. --- data/helpers.d/utils | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 5f5e61015..40e6fe045 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -166,15 +166,22 @@ ynh_setup_source () { ynh_handle_getopts_args "$@" source_id="${source_id:-app}" # If the argument is not given, source_id equals "app" + local src_file_path="$YNH_CWD/../conf/${src_id}.src" + # In case of restore script the src file is in an other path. + # So try to use the restore path if the general path point to no file. + if [[ ! -e "$src_file_path" ]]; then + src_file_path="$YNH_CWD/../settings/conf/${src_id}.src" + fi + # Load value from configuration file (see above for a small doc about this file # format) - local src_url=$(grep 'SOURCE_URL=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-) - local src_sum=$(grep 'SOURCE_SUM=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-) - local src_sumprg=$(grep 'SOURCE_SUM_PRG=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-) - local src_format=$(grep 'SOURCE_FORMAT=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-) - local src_extract=$(grep 'SOURCE_EXTRACT=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-) - local src_in_subdir=$(grep 'SOURCE_IN_SUBDIR=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-) - local src_filename=$(grep 'SOURCE_FILENAME=' "$YNH_CWD/../conf/${source_id}.src" | cut -d= -f2-) + local src_url=$(grep 'SOURCE_URL=' "$src_file_path" | cut -d= -f2-) + local src_sum=$(grep 'SOURCE_SUM=' "$src_file_path" | cut -d= -f2-) + local src_sumprg=$(grep 'SOURCE_SUM_PRG=' "$src_file_path" | cut -d= -f2-) + local src_format=$(grep 'SOURCE_FORMAT=' "$src_file_path" | cut -d= -f2-) + local src_extract=$(grep 'SOURCE_EXTRACT=' "$src_file_path" | cut -d= -f2-) + local src_in_subdir=$(grep 'SOURCE_IN_SUBDIR=' "$src_file_path" | cut -d= -f2-) + local src_filename=$(grep 'SOURCE_FILENAME=' "$src_file_path" | cut -d= -f2-) # Default value src_sumprg=${src_sumprg:-sha256sum} From e9aed65b0c1e8029d4a7980292bb3b4dd009e43e Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Tue, 9 Apr 2019 19:12:05 +0200 Subject: [PATCH 09/42] [Fix] src_id doesn't exist --- data/helpers.d/utils | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 40e6fe045..be2972e49 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -166,11 +166,11 @@ ynh_setup_source () { ynh_handle_getopts_args "$@" source_id="${source_id:-app}" # If the argument is not given, source_id equals "app" - local src_file_path="$YNH_CWD/../conf/${src_id}.src" + local src_file_path="$YNH_CWD/../conf/${source_id}.src" # In case of restore script the src file is in an other path. # So try to use the restore path if the general path point to no file. if [[ ! -e "$src_file_path" ]]; then - src_file_path="$YNH_CWD/../settings/conf/${src_id}.src" + src_file_path="$YNH_CWD/../settings/conf/${source_id}.src" fi # Load value from configuration file (see above for a small doc about this file From 55f8ffb2c9f3f5b96d2e6e5e020c9f3350ba31bb Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Tue, 9 Apr 2019 19:13:14 +0200 Subject: [PATCH 10/42] [enh] Double brackets not needed --- data/helpers.d/utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/utils b/data/helpers.d/utils index be2972e49..7d943c9df 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -169,7 +169,7 @@ ynh_setup_source () { local src_file_path="$YNH_CWD/../conf/${source_id}.src" # In case of restore script the src file is in an other path. # So try to use the restore path if the general path point to no file. - if [[ ! -e "$src_file_path" ]]; then + if [ ! -e "$src_file_path" ]; then src_file_path="$YNH_CWD/../settings/conf/${source_id}.src" fi From b7bf95153805373f964abbc2d4895576a247c17a Mon Sep 17 00:00:00 2001 From: Kayou Date: Wed, 10 Apr 2019 12:15:21 +0200 Subject: [PATCH 11/42] Tired of seeing this warning message --- data/hooks/backup/32-conf_cron | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/hooks/backup/32-conf_cron b/data/hooks/backup/32-conf_cron index 063ec1a3f..acbd009ab 100755 --- a/data/hooks/backup/32-conf_cron +++ b/data/hooks/backup/32-conf_cron @@ -10,6 +10,6 @@ source /usr/share/yunohost/helpers backup_dir="${1}/conf/cron" # Backup the configuration -for f in $(ls -1B /etc/cron.d/yunohost*); do +for f in $(ls -1B /etc/cron.d/yunohost* 2> /dev/null); do ynh_backup "$f" "${backup_dir}/${f##*/}" done From 0ef7ebbba6962fa96dbd5bdc2a1be72067929760 Mon Sep 17 00:00:00 2001 From: Romuald du Song Date: Thu, 11 Apr 2019 23:03:57 +0200 Subject: [PATCH 12/42] only open socket to localhost --- data/templates/slapd/slapd.default | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/templates/slapd/slapd.default b/data/templates/slapd/slapd.default index 372b8f4ab..0041b30c5 100644 --- a/data/templates/slapd/slapd.default +++ b/data/templates/slapd/slapd.default @@ -21,7 +21,7 @@ SLAPD_PIDFILE= # sockets. # Example usage: # SLAPD_SERVICES="ldap://127.0.0.1:389/ ldaps:/// ldapi:///" -SLAPD_SERVICES="ldap:/// ldapi:///" +SLAPD_SERVICES="ldap://127.0.0.1:389/ ldap://[::1]:389/ ldapi:///" # If SLAPD_NO_START is set, the init script will not start or restart # slapd (but stop will still work). Uncomment this if you are From 7ff9d849c414b8037720d4e92876126480fb3027 Mon Sep 17 00:00:00 2001 From: Christophe Vuillot Date: Fri, 12 Apr 2019 14:41:59 +0200 Subject: [PATCH 13/42] fixed naming convention --- data/actionsmap/yunohost_completion.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data/actionsmap/yunohost_completion.py b/data/actionsmap/yunohost_completion.py index 915c6ac89..a4c17c4d6 100644 --- a/data/actionsmap/yunohost_completion.py +++ b/data/actionsmap/yunohost_completion.py @@ -13,7 +13,7 @@ import yaml THIS_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) ACTIONSMAP_FILE = THIS_SCRIPT_DIR + '/yunohost.yml' -BASH_COMPLETION_FILE = THIS_SCRIPT_DIR + '/../bash-completion.d/yunohost_completion' +BASH_COMPLETION_FILE = THIS_SCRIPT_DIR + '/../bash-completion.d/yunohost' with open(ACTIONSMAP_FILE, 'r') as stream: @@ -37,7 +37,7 @@ with open(ACTIONSMAP_FILE, 'r') as stream: generated_file.write('#\n\n') # Start of the completion function - generated_file.write('_yunohost_completion()\n') + generated_file.write('_yunohost()\n') generated_file.write('{\n') # Defining local variable for previously and currently typed words @@ -83,4 +83,4 @@ with open(ACTIONSMAP_FILE, 'r') as stream: generated_file.write('}\n\n') # Add the function to bash completion - generated_file.write('complete -F _yunohost_completion yunohost') + generated_file.write('complete -F _yunohost yunohost') From 7f040f25f63c2ccbcfbf9639d3cd6582621ca684 Mon Sep 17 00:00:00 2001 From: Romuald du Song Date: Sat, 13 Apr 2019 16:35:36 +0200 Subject: [PATCH 14/42] specify -a parameter on dovecot lda for sieve --- data/templates/postfix/plain/master.cf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data/templates/postfix/plain/master.cf b/data/templates/postfix/plain/master.cf index 2d8712604..377a90971 100644 --- a/data/templates/postfix/plain/master.cf +++ b/data/templates/postfix/plain/master.cf @@ -122,6 +122,6 @@ mailman unix - n n - - pipe flags=FR user=list argv=/usr/lib/mailman/bin/postfix-to-mailman.py ${nexthop} ${user} -# Dovecot LDA -dovecot unix - n n - - pipe - flags=DRhu user=vmail:mail argv=/usr/lib/dovecot/deliver -f ${sender} -d ${user}@${nexthop} -m ${extension} +# Dovecot LDA +dovecot unix - n n - - pipe + flags=DRhu user=vmail:mail argv=/usr/lib/dovecot/deliver -f ${sender} -d ${user}@${nexthop} -m ${extension} -a ${recipient} From ebb32f0d9c34b1a6af28bb4d7277863451d5cf83 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sat, 13 Apr 2019 17:42:47 +0200 Subject: [PATCH 15/42] Use ynh_systemd_action --- data/helpers.d/backend | 4 ++-- data/helpers.d/psql | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index 710e6299b..715ad2353 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -442,7 +442,7 @@ EOF ynh_store_file_checksum "$finalfail2banjailconf" ynh_store_file_checksum "$finalfail2banfilterconf" - systemctl try-reload-or-restart fail2ban + ynh_systemd_action --service_name=fail2ban --action=reload local fail2ban_error="$(journalctl -u fail2ban | tail -n50 | grep "WARNING.*$app.*")" if [[ -n "$fail2ban_error" ]]; then @@ -459,5 +459,5 @@ EOF ynh_remove_fail2ban_config () { ynh_secure_remove "/etc/fail2ban/jail.d/$app.conf" ynh_secure_remove "/etc/fail2ban/filter.d/$app.conf" - systemctl try-reload-or-restart fail2ban + ynh_systemd_action --service_name=fail2ban --action=reload } diff --git a/data/helpers.d/psql b/data/helpers.d/psql index 2212d692a..f36ef8128 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -276,7 +276,8 @@ ynh_psql_test_if_first_run() { ynh_die "postgresql shoud be 9.4 or 9.6" fi - systemctl start postgresql + ynh_systemd_action --service_name=postgresql --action=start + sudo --login --user=postgres psql -c"ALTER user postgres WITH PASSWORD '$pgsql'" postgres # force all user to connect to local database using passwords @@ -289,6 +290,6 @@ ynh_psql_test_if_first_run() { yunohost service add postgresql --log "$logfile" systemctl enable postgresql - systemctl reload postgresql + ynh_systemd_action --service_name=postgresql --action=reload fi } From 30c776702fba0dd87df68292b77752a1d8da68ab Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sat, 13 Apr 2019 10:44:48 +0200 Subject: [PATCH 16/42] Use printers in helpers --- data/helpers.d/filesystem | 25 ++++++++++++------------- data/helpers.d/mysql | 4 ++-- data/helpers.d/nodejs | 2 +- data/helpers.d/package | 2 +- data/helpers.d/psql | 9 +++++---- data/helpers.d/system | 6 +++--- data/helpers.d/user | 6 +++--- data/helpers.d/utils | 8 ++++---- 8 files changed, 31 insertions(+), 31 deletions(-) diff --git a/data/helpers.d/filesystem b/data/helpers.d/filesystem index 07614d179..cb838e35a 100644 --- a/data/helpers.d/filesystem +++ b/data/helpers.d/filesystem @@ -65,25 +65,24 @@ ynh_backup() { # If backing up core only (used by ynh_backup_before_upgrade), # don't backup big data items if [ "$is_big" == "1" ] && [ "$BACKUP_CORE_ONLY" == "1" ] ; then - echo "$src_path will not be saved, because backup_core_only is set." >&2 + ynh_print_info --message="$src_path will not be saved, because backup_core_only is set." return 0 fi - + # ============================================================================== # Format correctly source and destination paths # ============================================================================== # Be sure the source path is not empty [[ -e "${src_path}" ]] || { - echo "Source path '${src_path}' does not exist" >&2 + ynh_print_warn --message="Source path '${src_path}' does not exist" if [ "$not_mandatory" == "0" ] then - echo "Source path '${SRC_PATH}' does not exist" >&2 # This is a temporary fix for fail2ban config files missing after the migration to stretch. if echo "${src_path}" | grep --quiet "/etc/fail2ban" then touch "${src_path}" - echo "The missing file will be replaced by a dummy one for the backup !!!" >&2 + ynh_print_info --message="The missing file will be replaced by a dummy one for the backup !!!" else return 1 fi @@ -127,7 +126,7 @@ ynh_backup() { # Check if dest_path already exists in tmp archive [[ ! -e "${dest_path}" ]] || { - echo "Destination path '${dest_path}' already exist" >&2 + ynh_print_err --message="Destination path '${dest_path}' already exist" return 1 } @@ -280,7 +279,7 @@ ynh_bind_or_cp() { local AS_ROOT=${3:-0} local NO_ROOT=0 [[ "${AS_ROOT}" = "1" ]] || NO_ROOT=1 - echo "This helper is deprecated, you should use ynh_backup instead" >&2 + ynh_print_warn --message="This helper is deprecated, you should use ynh_backup instead" ynh_backup "$1" "$2" 1 } @@ -293,9 +292,9 @@ ynh_bind_or_cp() { # usage: ynh_mkdir_tmp # | ret: the created directory path ynh_mkdir_tmp() { - echo "The helper ynh_mkdir_tmp is deprecated." >&2 - echo "You should use 'mktemp -d' instead and manage permissions \ -properly with chmod/chown." >&2 + ynh_print_warn --message="The helper ynh_mkdir_tmp is deprecated." + ynh_print_warn --message="You should use 'mktemp -d' instead and manage permissions \ +properly with chmod/chown." local TMP_DIR=$(mktemp -d) # Give rights to other users could be a security risk. @@ -410,7 +409,7 @@ ynh_secure_remove () { if [ $# -ge 2 ] then - echo "/!\ Packager ! You provided more than one argument to ynh_secure_remove but it will be ignored... Use this helper with one argument at time." >&2 + ynh_print_warn --message="/!\ Packager ! You provided more than one argument to ynh_secure_remove but it will be ignored... Use this helper with one argument at time." fi if [[ "$forbidden_path" =~ "$file" \ @@ -420,13 +419,13 @@ ynh_secure_remove () { || "${file:${#file}-1}" = "/" ]] # Match if the path finishes by /. Because it seems there is an empty variable then - echo "Avoid deleting $file." >&2 + ynh_print_warn --message="Avoid deleting $file." else if [ -e "$file" ] then sudo rm -R "$file" else - echo "$file wasn't deleted because it doesn't exist." >&2 + ynh_print_info --message="$file wasn't deleted because it doesn't exist." fi fi } diff --git a/data/helpers.d/mysql b/data/helpers.d/mysql index d7400db2d..1dad94e13 100644 --- a/data/helpers.d/mysql +++ b/data/helpers.d/mysql @@ -225,10 +225,10 @@ ynh_mysql_remove_db () { local mysql_root_password=$(sudo cat $MYSQL_ROOT_PWD_FILE) if mysqlshow -u root -p$mysql_root_password | grep -q "^| $db_name"; then # Check if the database exists - echo "Removing database $db_name" >&2 + ynh_print_info --message="Removing database $db_name" ynh_mysql_drop_db $db_name # Remove the database else - echo "Database $db_name not found" >&2 + ynh_print_warn --message="Database $db_name not found" fi # Remove mysql user if it exists diff --git a/data/helpers.d/nodejs b/data/helpers.d/nodejs index 6833b7593..9295c4348 100644 --- a/data/helpers.d/nodejs +++ b/data/helpers.d/nodejs @@ -13,7 +13,7 @@ export N_PREFIX="$n_install_dir" # # Requires YunoHost version 2.7.12 or higher. ynh_install_n () { - echo "Installation of N - Node.js version management" >&2 + ynh_print_info --message="Installation of N - Node.js version management" # Build an app.src for n mkdir -p "../conf" echo "SOURCE_URL=https://github.com/tj/n/archive/v2.1.7.tar.gz diff --git a/data/helpers.d/package b/data/helpers.d/package index 9c2b58458..9d5ad3ac2 100644 --- a/data/helpers.d/package +++ b/data/helpers.d/package @@ -166,7 +166,7 @@ ynh_package_install_from_equivs () { local pkgname=$(grep '^Package: ' $controlfile | cut -d' ' -f 2) # Retrieve the name of the debian package local pkgversion=$(grep '^Version: ' $controlfile | cut -d' ' -f 2) # And its version number [[ -z "$pkgname" || -z "$pkgversion" ]] \ - && echo "Invalid control file" && exit 1 # Check if this 2 variables aren't empty. + && ynh_die --message="Invalid control file" # Check if this 2 variables aren't empty. # Update packages cache ynh_package_update diff --git a/data/helpers.d/psql b/data/helpers.d/psql index 2212d692a..12c9ed096 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -240,18 +240,18 @@ ynh_psql_remove_db() { local psql_root_password=$(sudo cat $PSQL_ROOT_PWD_FILE) if ynh_psql_database_exists --database=$db_name; then # Check if the database exists - echo "Removing database $db_name" >&2 + ynh_print_info --message="Removing database $db_name" ynh_psql_drop_db $db_name # Remove the database else - echo "Database $db_name not found" >&2 + ynh_print_warn --message="Database $db_name not found" fi # Remove psql user if it exists if ynh_psql_user_exists --user=$db_user; then - echo "Removing user $db_user" >&2 + ynh_print_info --message="Removing user $db_user" ynh_psql_drop_user $db_user else - echo "User $db_user not found" >&2 + ynh_print_warn --message="User $db_user not found" fi } @@ -277,6 +277,7 @@ ynh_psql_test_if_first_run() { fi systemctl start postgresql + sudo --login --user=postgres psql -c"ALTER user postgres WITH PASSWORD '$pgsql'" postgres # force all user to connect to local database using passwords diff --git a/data/helpers.d/system b/data/helpers.d/system index 757fdf93c..b8971c80f 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -26,7 +26,7 @@ ynh_exit_properly () { trap '' EXIT # Ignore new exit signals set +eu # Do not exit anymore if a command fail or if a variable is empty - echo -e "!!\n $app's script has encountered an error. Its execution was cancelled.\n!!" >&2 + ynh_print_err --message="!!\n $app's script has encountered an error. Its execution was cancelled.\n!!" if type -t ynh_clean_setup > /dev/null; then # Check if the function exist in the app script. ynh_clean_setup # Call the function to do specific cleaning for the app. @@ -269,11 +269,11 @@ ynh_check_app_version_changed () { # Complete versions are the same if [ "$force_upgrade" != "0" ] then - echo "Upgrade forced by YNH_FORCE_UPGRADE." >&2 + ynh_print_info --message="Upgrade forced by YNH_FORCE_UPGRADE." unset YNH_FORCE_UPGRADE elif [ "$package_check" != "0" ] then - echo "Upgrade forced for package check." >&2 + ynh_print_info --message="Upgrade forced for package check." else ynh_die "Up-to-date, nothing to do" 0 fi diff --git a/data/helpers.d/user b/data/helpers.d/user index 9ee44515d..0c4591dcd 100644 --- a/data/helpers.d/user +++ b/data/helpers.d/user @@ -145,16 +145,16 @@ ynh_system_user_delete () { # Check if the user exists on the system if ynh_system_user_exists "$username" then - echo "Remove the user $username" >&2 + ynh_print_info --message="Remove the user $username" deluser $username else - echo "The user $username was not found" >&2 + ynh_print_warn --message="The user $username was not found" fi # Check if the group exists on the system if ynh_system_group_exists "$username" then - echo "Remove the group $username" >&2 + ynh_print_info --message="Remove the group $username" delgroup $username fi } diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 5f5e61015..f020ba212 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -40,7 +40,7 @@ ynh_get_plain_key() { # # Requires YunoHost version 2.7.2 or higher. ynh_restore_upgradebackup () { - echo "Upgrade failed." >&2 + ynh_print_err --message="Upgrade failed." local app_bck=${app//_/-} # Replace all '_' by '-' NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0} @@ -57,7 +57,7 @@ ynh_restore_upgradebackup () { ynh_die --message="The app was restored to the way it was before the failed upgrade." fi else - echo "\$NO_BACKUP_UPGRADE is set, that means there's no backup to restore. You have to fix this upgrade by yourself !" >&2 + ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, that means there's no backup to restore. You have to fix this upgrade by yourself !" fi } @@ -74,7 +74,7 @@ ynh_restore_upgradebackup () { ynh_backup_before_upgrade () { if [ ! -e "/etc/yunohost/apps/$app/scripts/backup" ] then - echo "This app doesn't have any backup script." >&2 + ynh_print_warn --message="This app doesn't have any backup script." return fi backup_number=1 @@ -106,7 +106,7 @@ ynh_backup_before_upgrade () { ynh_die --message="Backup failed, the upgrade process was aborted." fi else - echo "\$NO_BACKUP_UPGRADE is set, backup will be avoided. Be careful, this upgrade is going to be operated without a security backup" + ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, backup will be avoided. Be careful, this upgrade is going to be operated without a security backup" fi } From 06a066fe17d4f2561020e94d3a5bc7edf4ad6692 Mon Sep 17 00:00:00 2001 From: yalh76 Date: Mon, 15 Apr 2019 14:18:04 +0200 Subject: [PATCH 17/42] Force disconnection of all clients connected to the database --- data/helpers.d/psql | 1 + 1 file changed, 1 insertion(+) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index 2212d692a..dc83cc0e8 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -105,6 +105,7 @@ ynh_psql_create_db() { # Requires YunoHost version 3.?.? or higher. ynh_psql_drop_db() { local db=$1 + ynh_psql_execute_as_root --sql="SELECT pg_terminate_backend (pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$db';" --database="$db" sudo --login --user=postgres dropdb $db } From fc9ad45beef3899f880bc6f96ad11e76a09249a9 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Wed, 17 Apr 2019 20:24:38 +0200 Subject: [PATCH 18/42] [enh] Add the current work to the progression bar --- data/helpers.d/print | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/data/helpers.d/print b/data/helpers.d/print index 468c41e96..8a2e1ee79 100644 --- a/data/helpers.d/print +++ b/data/helpers.d/print @@ -243,8 +243,9 @@ ynh_script_progression () { # Set the scale of the progression bar local scale=20 - # progress_string(1,2) should have the size of the scale. - local progress_string1="####################" + # progress_string(0,1,2) should have the size of the scale. + local progress_string2="####################" + local progress_string1="++++++++++++++++++++" local progress_string0="...................." # Reduce $increment_progression to the size of the scale @@ -256,8 +257,13 @@ ynh_script_progression () { local effective_progression=$scale fi - # Build $progression_bar from progress_string(1,2) according to $effective_progression - local progression_bar="${progress_string1:0:$effective_progression}${progress_string0:0:$(( $scale - $effective_progression ))}" + # Build $progression_bar from progress_string(0,1,2) according to $effective_progression and the weight of the current task + # expected_progression is the progression expected after the current task + local expected_progression="$(( ( $increment_progression + $weight ) * $scale / $max_progression - $effective_progression ))" + # left_progression is the progression not yet done + local left_progression="$(( $scale - $effective_progression - $expected_progression ))" + # Build the progression bar with $effective_progression, work done, $expected_progression, current work and $left_progression, work to be done. + local progression_bar="${progress_string2:0:$effective_progression}${progress_string1:0:$expected_progression}${progress_string0:0:$left_progression}" local print_exec_time="" if [ $time -eq 1 ] From 4ed252e894003195e74f7b491693174cb06294ff Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Wed, 17 Apr 2019 20:37:19 +0200 Subject: [PATCH 19/42] Fix behavior with --last --- data/helpers.d/print | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/data/helpers.d/print b/data/helpers.d/print index 8a2e1ee79..1558e4d21 100644 --- a/data/helpers.d/print +++ b/data/helpers.d/print @@ -260,6 +260,10 @@ ynh_script_progression () { # Build $progression_bar from progress_string(0,1,2) according to $effective_progression and the weight of the current task # expected_progression is the progression expected after the current task local expected_progression="$(( ( $increment_progression + $weight ) * $scale / $max_progression - $effective_progression ))" + if [ $last -eq 1 ] + then + expected_progression=0 + fi # left_progression is the progression not yet done local left_progression="$(( $scale - $effective_progression - $expected_progression ))" # Build the progression bar with $effective_progression, work done, $expected_progression, current work and $left_progression, work to be done. From efc848fe7d5f6e7ccba601cb8e3dfd6bd7bde70a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 19 Apr 2019 02:06:56 +0200 Subject: [PATCH 20/42] Essential thing should be in the second block --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index a6fd95154..bb697d074 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,6 @@ Source: yunohost Section: utils Priority: extra -Essential: yes Maintainer: YunoHost Contributors Build-Depends: debhelper (>=9), dh-systemd, dh-python, python-all (>= 2.7) Standards-Version: 3.9.6 @@ -9,6 +8,7 @@ X-Python-Version: >= 2.7 Homepage: https://yunohost.org/ Package: yunohost +Essential: yes Architecture: all Depends: ${python:Depends}, ${misc:Depends} , moulinette (>= 2.7.1), ssowat (>= 2.7.1) From d11d31dbf24ab264dc8eb6d0d22b60a0f03fadf6 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Mon, 22 Apr 2019 00:27:01 +0200 Subject: [PATCH 21/42] Fix ynh_use_logrotate with getopts $1 contains --logfile= Merged as a micro decision. --- data/helpers.d/backend | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/backend b/data/helpers.d/backend index 710e6299b..db7f94d4f 100644 --- a/data/helpers.d/backend +++ b/data/helpers.d/backend @@ -55,7 +55,7 @@ ynh_use_logrotate () { if [ -n "$logfile" ] then if [ "$(echo ${logfile##*.})" != "log" ]; then # Keep only the extension to check if it's a logfile - local logfile="$1/*.log" # Else, uses the directory and all logfile into it. + local logfile="$logfile/*.log" # Else, uses the directory and all logfile into it. fi else logfile="/var/log/${app}/*.log" # Without argument, use a defaut directory in /var/log From dc19e579ff8731848bb46e401162ce7fdcf86ab3 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 22 Apr 2019 14:31:38 +0200 Subject: [PATCH 22/42] Add comment explaining the mystic incantation --- data/helpers.d/psql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/data/helpers.d/psql b/data/helpers.d/psql index dc83cc0e8..a5bccedd6 100644 --- a/data/helpers.d/psql +++ b/data/helpers.d/psql @@ -105,6 +105,9 @@ ynh_psql_create_db() { # Requires YunoHost version 3.?.? or higher. ynh_psql_drop_db() { local db=$1 + # First, force disconnection of all clients connected to the database + # https://stackoverflow.com/questions/5408156/how-to-drop-a-postgresql-database-if-there-are-active-connections-to-it + # https://dba.stackexchange.com/questions/16426/how-to-drop-all-connections-to-a-specific-database-without-stopping-the-server ynh_psql_execute_as_root --sql="SELECT pg_terminate_backend (pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$db';" --database="$db" sudo --login --user=postgres dropdb $db } From 7de5251af697a726e8973f30cbd60b64715376c4 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 22 Apr 2019 15:40:47 +0200 Subject: [PATCH 23/42] Handle human size option for apps size as well --- src/yunohost/backup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index f10b112ac..8c7d53b1c 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -2272,13 +2272,16 @@ def backup_info(name, with_details=False, human_readable=False): if "hooks" in info.keys(): system_key = "hooks" - if "size_details" in info.keys(): for name, key_info in info["apps"].items(): if name in info["size_details"]["apps"].keys(): key_info["size"] = info["size_details"]["apps"][name] + if human_readable: + key_info["size"] = binary_to_human(key_info["size"]) + 'B' else: key_info["size"] = 0 + if human_readable: + key_info["size"] = "?" result["apps"] = info["apps"] result["system"] = info[system_key] From 0b70ee84dd684f45442eac50bf96c78464043eb9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 22 Apr 2019 15:41:27 +0200 Subject: [PATCH 24/42] Set size to -1 to me more explicit about the size not being available ? --- src/yunohost/backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 8c7d53b1c..5db17a047 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -2279,7 +2279,7 @@ def backup_info(name, with_details=False, human_readable=False): if human_readable: key_info["size"] = binary_to_human(key_info["size"]) + 'B' else: - key_info["size"] = 0 + key_info["size"] = -1 if human_readable: key_info["size"] = "?" From 3668bf7bc36c9169f79fb6dc368cbb395d0370c9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 23 Apr 2019 18:59:32 +0200 Subject: [PATCH 25/42] [fix] Microdecision : fix wobbly / messed up debug logs when running apt stuff inside app scripts --- src/yunohost/hook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index c4605b6e8..2841dd425 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -370,7 +370,7 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False, # Define output callbacks and call command callbacks = ( - stdout_callback if stdout_callback else lambda l: logger.debug(l.rstrip()), + stdout_callback if stdout_callback else lambda l: logger.debug(l.rstrip()+"\r"), stderr_callback if stderr_callback else lambda l: logger.warning(l.rstrip()), ) From c7eb5bbed47bec49130d66bb6d5c385a14c017a8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 24 Apr 2019 17:51:17 +0200 Subject: [PATCH 26/42] [fix] Microdecision : exposing ldapinit does not make sense --- data/actionsmap/yunohost.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index fabdcb923..87fd0c272 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1477,13 +1477,6 @@ tools: category_help: Specific tools actions: - ### tools_ldapinit() - ldapinit: - action_help: YunoHost LDAP initialization - api: POST /ldap - configuration: - authenticate: all - ### tools_adminpw() adminpw: action_help: Change password of admin and root users From 94f33b617b2ff55983924f10c3f02a1274dcf991 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 24 Apr 2019 20:19:55 +0200 Subject: [PATCH 27/42] Add a --with-details option for log list --- data/actionsmap/yunohost.yml | 4 ++++ src/yunohost/log.py | 29 ++++++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 87fd0c272..32c1b0e42 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -1802,6 +1802,10 @@ log: full: --limit help: Maximum number of logs type: int + -d: + full: --with-details + help: Show additional infos (e.g. operation success) but may significantly increase command time. Consider using --limit in combination with this. + action: store_true ### log_display() display: diff --git a/src/yunohost/log.py b/src/yunohost/log.py index 857cc3658..e576ae536 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -47,12 +47,13 @@ RELATED_CATEGORIES = ['app', 'domain', 'service', 'user'] logger = getActionLogger('yunohost.log') -def log_list(category=[], limit=None): +def log_list(category=[], limit=None, with_details=False): """ List available logs Keyword argument: limit -- Maximum number of logs + with_details -- Include details (e.g. if the operation was a success). Likely to increase the command time as it needs to open and parse the metadata file for each log... So try to use this in combination with --limit. """ categories = category @@ -69,12 +70,11 @@ def log_list(category=[], limit=None): category_path = os.path.join(CATEGORIES_PATH, category) if not os.path.exists(category_path): logger.debug(m18n.n('log_category_404', category=category)) - continue logs = filter(lambda x: x.endswith(METADATA_FILE_EXT), os.listdir(category_path)) - logs = reversed(sorted(logs)) + logs = list(reversed(sorted(logs))) if limit is not None: logs = logs[:limit] @@ -100,6 +100,15 @@ def log_list(category=[], limit=None): else: entry["started_at"] = log_datetime + if with_details: + with open(md_path, "r") as md_file: + try: + metadata = yaml.safe_load(md_file) + except yaml.YAMLError: + logger.warning(m18n.n('log_corrupted_md_file', file=md_path)) + + entry["success"] = metadata.get("success", "?") + result[category].append(entry) # Reverse the order of log when in cli, more comfortable to read (avoid @@ -318,6 +327,20 @@ class OperationLogger(object): self.flush() self._register_log() + @property + def md_path(self): + """ + Metadata path file + """ + return os.path.join(self.path, self.name + METADATA_FILE_EXT) + + @property + def log_path(self): + """ + Log path file + """ + return os.path.join(self.path, self.name + LOG_FILE_EXT) + def _register_log(self): """ Register log with a handler connected on log system From 161c18831e8c0d009833dce39229c89abe091490 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 25 Apr 2019 01:04:59 +0200 Subject: [PATCH 28/42] Also get size info about system ? --- src/yunohost/backup.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 5db17a047..e5a21669c 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -2273,15 +2273,16 @@ def backup_info(name, with_details=False, human_readable=False): system_key = "hooks" if "size_details" in info.keys(): - for name, key_info in info["apps"].items(): - if name in info["size_details"]["apps"].keys(): - key_info["size"] = info["size_details"]["apps"][name] - if human_readable: - key_info["size"] = binary_to_human(key_info["size"]) + 'B' - else: - key_info["size"] = -1 - if human_readable: - key_info["size"] = "?" + for category in ["apps", "system"]: + for name, key_info in info[category].items(): + if name in info["size_details"][category].keys(): + key_info["size"] = info["size_details"][category][name] + if human_readable: + key_info["size"] = binary_to_human(key_info["size"]) + 'B' + else: + key_info["size"] = -1 + if human_readable: + key_info["size"] = "?" result["apps"] = info["apps"] result["system"] = info[system_key] From eb89fc04b1f1cdc274a49e43bedc86b3900651eb Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 25 Apr 2019 20:09:57 +0200 Subject: [PATCH 29/42] Split 'backend' into fail2ban, logrotate, nginx, php, systemd --- data/helpers.d/backend | 463 --------------------------------------- data/helpers.d/fail2ban | 151 +++++++++++++ data/helpers.d/logrotate | 103 +++++++++ data/helpers.d/nginx | 76 +++++++ data/helpers.d/php | 67 ++++++ data/helpers.d/systemd | 70 ++++++ 6 files changed, 467 insertions(+), 463 deletions(-) delete mode 100644 data/helpers.d/backend create mode 100644 data/helpers.d/fail2ban create mode 100644 data/helpers.d/logrotate create mode 100644 data/helpers.d/nginx create mode 100644 data/helpers.d/php create mode 100644 data/helpers.d/systemd diff --git a/data/helpers.d/backend b/data/helpers.d/backend deleted file mode 100644 index 305c016ad..000000000 --- a/data/helpers.d/backend +++ /dev/null @@ -1,463 +0,0 @@ -#!/bin/bash - -# Use logrotate to manage the logfile -# -# usage: ynh_use_logrotate [--logfile=/log/file] [--nonappend] [--specific_user=user/group] -# | arg: -l, --logfile - absolute path of logfile -# | arg: -n, --nonappend - (optional) Replace the config file instead of appending this new config. -# | arg: -u, --specific_user : run logrotate as the specified user and group. If not specified logrotate is runned as root. -# -# If no --logfile is provided, /var/log/${app} will be used as default. -# logfile can be just a directory, or a full path to a logfile : -# /parentdir/logdir -# /parentdir/logdir/logfile.log -# -# It's possible to use this helper multiple times, each config will be added to -# the same logrotate config file. Unless you use the option --non-append -# -# Requires YunoHost version 2.6.4 or higher. -ynh_use_logrotate () { - # Declare an array to define the options of this helper. - local legacy_args=lnuya - declare -Ar args_array=( [l]=logfile= [n]=nonappend [u]=specific_user= [y]=non [a]=append ) - # [y]=non [a]=append are only for legacy purpose, to not fail on the old option '--non-append' - local logfile - local nonappend - local specific_user - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - local logfile="${logfile:-}" - local nonappend="${nonappend:-0}" - local specific_user="${specific_user:-}" - - # LEGACY CODE - PRE GETOPTS - if [ $# -gt 0 ] && [ "$1" == "--non-append" ]; then - nonappend=1 - # Destroy this argument for the next command. - shift - elif [ $# -gt 1 ] && [ "$2" == "--non-append" ]; then - nonappend=1 - fi - - if [ $# -gt 0 ] && [ "$(echo ${1:0:1})" != "-" ]; then - if [ "$(echo ${1##*.})" == "log" ]; then # Keep only the extension to check if it's a logfile - local logfile=$1 # In this case, focus logrotate on the logfile - else - local logfile=$1/*.log # Else, uses the directory and all logfile into it. - fi - fi - # LEGACY CODE - - local customtee="tee -a" - if [ "$nonappend" -eq 1 ]; then - customtee="tee" - fi - if [ -n "$logfile" ] - then - if [ "$(echo ${logfile##*.})" != "log" ]; then # Keep only the extension to check if it's a logfile - local logfile="$logfile/*.log" # Else, uses the directory and all logfile into it. - fi - else - logfile="/var/log/${app}/*.log" # Without argument, use a defaut directory in /var/log - fi - local su_directive="" - if [[ -n $specific_user ]]; then - su_directive=" # Run logorotate as specific user - group - su ${specific_user%/*} ${specific_user#*/}" - fi - - cat > ./${app}-logrotate << EOF # Build a config file for logrotate -$logfile { - # Rotate if the logfile exceeds 100Mo - size 100M - # Keep 12 old log maximum - rotate 12 - # Compress the logs with gzip - compress - # Compress the log at the next cycle. So keep always 2 non compressed logs - delaycompress - # Copy and truncate the log to allow to continue write on it. Instead of move the log. - copytruncate - # Do not do an error if the log is missing - missingok - # Not rotate if the log is empty - notifempty - # Keep old logs in the same dir - noolddir - $su_directive -} -EOF - sudo mkdir -p $(dirname "$logfile") # Create the log directory, if not exist - cat ${app}-logrotate | sudo $customtee /etc/logrotate.d/$app > /dev/null # Append this config to the existing config file, or replace the whole config file (depending on $customtee) -} - -# Remove the app's logrotate config. -# -# usage: ynh_remove_logrotate -# -# Requires YunoHost version 2.6.4 or higher. -ynh_remove_logrotate () { - if [ -e "/etc/logrotate.d/$app" ]; then - sudo rm "/etc/logrotate.d/$app" - fi -} - -# Create a dedicated systemd config -# -# usage: ynh_add_systemd_config [--service=service] [--template=template] -# | arg: -s, --service - Service name (optionnal, $app by default) -# | arg: -t, --template - Name of template file (optionnal, this is 'systemd' by default, meaning ./conf/systemd.service will be used as template) -# -# This will use the template ../conf/.service -# to generate a systemd config, by replacing the following keywords -# with global variables that should be defined before calling -# this helper : -# -# __APP__ by $app -# __FINALPATH__ by $final_path -# -# Requires YunoHost version 2.7.2 or higher. -ynh_add_systemd_config () { - # Declare an array to define the options of this helper. - local legacy_args=st - declare -Ar args_array=( [s]=service= [t]=template= ) - local service - local template - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - local service="${service:-$app}" - local template="${template:-systemd.service}" - - finalsystemdconf="/etc/systemd/system/$service.service" - ynh_backup_if_checksum_is_different --file="$finalsystemdconf" - sudo cp ../conf/$template "$finalsystemdconf" - - # To avoid a break by set -u, use a void substitution ${var:-}. If the variable is not set, it's simply set with an empty variable. - # Substitute in a nginx config file only if the variable is not empty - if test -n "${final_path:-}"; then - ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalsystemdconf" - fi - if test -n "${app:-}"; then - ynh_replace_string --match_string="__APP__" --replace_string="$app" --target_file="$finalsystemdconf" - fi - ynh_store_file_checksum --file="$finalsystemdconf" - - sudo chown root: "$finalsystemdconf" - sudo systemctl enable $service - sudo systemctl daemon-reload -} - -# Remove the dedicated systemd config -# -# usage: ynh_remove_systemd_config [--service=service] -# | arg: -s, --service - Service name (optionnal, $app by default) -# -# Requires YunoHost version 2.7.2 or higher. -ynh_remove_systemd_config () { - # Declare an array to define the options of this helper. - local legacy_args=s - declare -Ar args_array=( [s]=service= ) - local service - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - local service="${service:-$app}" - - local finalsystemdconf="/etc/systemd/system/$service.service" - if [ -e "$finalsystemdconf" ]; then - ynh_systemd_action --service_name=$service --action=stop - systemctl disable $service - ynh_secure_remove --file="$finalsystemdconf" - systemctl daemon-reload - fi -} - -# Create a dedicated nginx config -# -# usage: ynh_add_nginx_config "list of others variables to replace" -# -# | arg: list - (Optional) list of others variables to replace separated by spaces. For example : 'path_2 port_2 ...' -# -# This will use a template in ../conf/nginx.conf -# __PATH__ by $path_url -# __DOMAIN__ by $domain -# __PORT__ by $port -# __NAME__ by $app -# __FINALPATH__ by $final_path -# -# And dynamic variables (from the last example) : -# __PATH_2__ by $path_2 -# __PORT_2__ by $port_2 -# -# Requires YunoHost version 2.7.2 or higher. -ynh_add_nginx_config () { - finalnginxconf="/etc/nginx/conf.d/$domain.d/$app.conf" - local others_var=${1:-} - ynh_backup_if_checksum_is_different --file="$finalnginxconf" - sudo cp ../conf/nginx.conf "$finalnginxconf" - - # To avoid a break by set -u, use a void substitution ${var:-}. If the variable is not set, it's simply set with an empty variable. - # Substitute in a nginx config file only if the variable is not empty - if test -n "${path_url:-}"; then - # path_url_slash_less is path_url, or a blank value if path_url is only '/' - local path_url_slash_less=${path_url%/} - ynh_replace_string --match_string="__PATH__/" --replace_string="$path_url_slash_less/" --target_file="$finalnginxconf" - ynh_replace_string --match_string="__PATH__" --replace_string="$path_url" --target_file="$finalnginxconf" - fi - if test -n "${domain:-}"; then - ynh_replace_string --match_string="__DOMAIN__" --replace_string="$domain" --target_file="$finalnginxconf" - fi - if test -n "${port:-}"; then - ynh_replace_string --match_string="__PORT__" --replace_string="$port" --target_file="$finalnginxconf" - fi - if test -n "${app:-}"; then - ynh_replace_string --match_string="__NAME__" --replace_string="$app" --target_file="$finalnginxconf" - fi - if test -n "${final_path:-}"; then - ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalnginxconf" - fi - - # Replace all other variable given as arguments - for var_to_replace in $others_var - do - # ${var_to_replace^^} make the content of the variable on upper-cases - # ${!var_to_replace} get the content of the variable named $var_to_replace - ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalnginxconf" - done - - if [ "${path_url:-}" != "/" ] - then - ynh_replace_string --match_string="^#sub_path_only" --replace_string="" --target_file="$finalnginxconf" - else - ynh_replace_string --match_string="^#root_path_only" --replace_string="" --target_file="$finalnginxconf" - fi - - ynh_store_file_checksum --file="$finalnginxconf" - - ynh_systemd_action --service_name=nginx --action=reload -} - -# Remove the dedicated nginx config -# -# usage: ynh_remove_nginx_config -# -# Requires YunoHost version 2.7.2 or higher. -ynh_remove_nginx_config () { - ynh_secure_remove --file="/etc/nginx/conf.d/$domain.d/$app.conf" - ynh_systemd_action --service_name=nginx --action=reload -} - -# Create a dedicated php-fpm config -# -# usage: ynh_add_fpm_config [--phpversion=7.X] -# | arg: -v, --phpversion - Version of php to use. -# -# Requires YunoHost version 2.7.2 or higher. -ynh_add_fpm_config () { - # Declare an array to define the options of this helper. - local legacy_args=v - declare -Ar args_array=( [v]=phpversion= ) - local phpversion - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - - # Configure PHP-FPM 7.0 by default - phpversion="${phpversion:-7.0}" - - local fpm_config_dir="/etc/php/$phpversion/fpm" - local fpm_service="php${phpversion}-fpm" - # Configure PHP-FPM 5 on Debian Jessie - if [ "$(ynh_get_debian_release)" == "jessie" ]; then - fpm_config_dir="/etc/php5/fpm" - fpm_service="php5-fpm" - fi - ynh_app_setting_set --app=$app --key=fpm_config_dir --value="$fpm_config_dir" - ynh_app_setting_set --app=$app --key=fpm_service --value="$fpm_service" - finalphpconf="$fpm_config_dir/pool.d/$app.conf" - ynh_backup_if_checksum_is_different --file="$finalphpconf" - sudo cp ../conf/php-fpm.conf "$finalphpconf" - ynh_replace_string --match_string="__NAMETOCHANGE__" --replace_string="$app" --target_file="$finalphpconf" - ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalphpconf" - ynh_replace_string --match_string="__USER__" --replace_string="$app" --target_file="$finalphpconf" - ynh_replace_string --match_string="__PHPVERSION__" --replace_string="$phpversion" --target_file="$finalphpconf" - sudo chown root: "$finalphpconf" - ynh_store_file_checksum --file="$finalphpconf" - - if [ -e "../conf/php-fpm.ini" ] - then - echo "Packagers ! Please do not use a separate php ini file, merge your directives in the pool file instead." >&2 - finalphpini="$fpm_config_dir/conf.d/20-$app.ini" - ynh_backup_if_checksum_is_different "$finalphpini" - sudo cp ../conf/php-fpm.ini "$finalphpini" - sudo chown root: "$finalphpini" - ynh_store_file_checksum "$finalphpini" - fi - ynh_systemd_action --service_name=$fpm_service --action=reload -} - -# Remove the dedicated php-fpm config -# -# usage: ynh_remove_fpm_config -# -# Requires YunoHost version 2.7.2 or higher. -ynh_remove_fpm_config () { - local fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir) - local fpm_service=$(ynh_app_setting_get --app=$app --key=fpm_service) - # Assume php version 7 if not set - if [ -z "$fpm_config_dir" ]; then - fpm_config_dir="/etc/php/7.0/fpm" - fpm_service="php7.0-fpm" - fi - ynh_secure_remove --file="$fpm_config_dir/pool.d/$app.conf" - ynh_secure_remove --file="$fpm_config_dir/conf.d/20-$app.ini" 2>&1 - ynh_systemd_action --service_name=$fpm_service --action=reload -} - -# Create a dedicated fail2ban config (jail and filter conf files) -# -# usage 1: ynh_add_fail2ban_config --logpath=log_file --failregex=filter [--max_retry=max_retry] [--ports=ports] -# | arg: -l, --logpath= - Log file to be checked by fail2ban -# | arg: -r, --failregex= - Failregex to be looked for by fail2ban -# | arg: -m, --max_retry= - Maximum number of retries allowed before banning IP address - default: 3 -# | arg: -p, --ports= - Ports blocked for a banned IP address - default: http,https -# -# ----------------------------------------------------------------------------- -# -# usage 2: ynh_add_fail2ban_config --use_template [--others_var="list of others variables to replace"] -# | arg: -t, --use_template - Use this helper in template mode -# | arg: -v, --others_var= - List of others variables to replace separeted by a space -# | for example : 'var_1 var_2 ...' -# -# This will use a template in ../conf/f2b_jail.conf and ../conf/f2b_filter.conf -# __APP__ by $app -# -# You can dynamically replace others variables by example : -# __VAR_1__ by $var_1 -# __VAR_2__ by $var_2 -# -# Generally your template will look like that by example (for synapse): -# -# f2b_jail.conf: -# [__APP__] -# enabled = true -# port = http,https -# filter = __APP__ -# logpath = /var/log/__APP__/logfile.log -# maxretry = 3 -# -# f2b_filter.conf: -# [INCLUDES] -# before = common.conf -# [Definition] -# -# # Part of regex definition (just used to make more easy to make the global regex) -# __synapse_start_line = .? \- synapse\..+ \- -# -# # Regex definition. -# failregex = ^%(__synapse_start_line)s INFO \- POST\-(\d+)\- \- \d+ \- Received request\: POST /_matrix/client/r0/login\??%(__synapse_start_line)s INFO \- POST\-\1\- Got login request with identifier: \{u'type': u'm.id.user', u'user'\: u'(.+?)'\}, medium\: None, address: None, user\: u'\5'%(__synapse_start_line)s WARNING \- \- (Attempted to login as @\5\:.+ but they do not exist|Failed password login for user @\5\:.+)$ -# -# ignoreregex = -# -# ----------------------------------------------------------------------------- -# -# Note about the "failregex" option: -# regex to match the password failure messages in the logfile. The -# host must be matched by a group named "host". The tag "" can -# be used for standard IP/hostname matching and is only an alias for -# (?:::f{4,6}:)?(?P[\w\-.^_]+) -# -# You can find some more explainations about how to make a regex here : -# https://www.fail2ban.org/wiki/index.php/MANUAL_0_8#Filters -# -# Note that the logfile need to exist before to call this helper !! -# -# To validate your regex you can test with this command: -# fail2ban-regex /var/log/YOUR_LOG_FILE_PATH /etc/fail2ban/filter.d/YOUR_APP.conf -# -# Requires YunoHost version 3.?.? or higher. -ynh_add_fail2ban_config () { - # Declare an array to define the options of this helper. - local legacy_args=lrmptv - declare -Ar args_array=( [l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= [t]=use_template [v]=others_var=) - local logpath - local failregex - local max_retry - local ports - local others_var - local use_template - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - use_template="${use_template:-0}" - max_retry=${max_retry:-3} - ports=${ports:-http,https} - - finalfail2banjailconf="/etc/fail2ban/jail.d/$app.conf" - finalfail2banfilterconf="/etc/fail2ban/filter.d/$app.conf" - ynh_backup_if_checksum_is_different "$finalfail2banjailconf" - ynh_backup_if_checksum_is_different "$finalfail2banfilterconf" - - if [ $use_template -eq 1 ] - then - # Usage 2, templates - cp ../conf/f2b_jail.conf $finalfail2banjailconf - cp ../conf/f2b_filter.conf $finalfail2banfilterconf - - if [ -n "${app:-}" ] - then - ynh_replace_string "__APP__" "$app" "$finalfail2banjailconf" - ynh_replace_string "__APP__" "$app" "$finalfail2banfilterconf" - fi - - # Replace all other variable given as arguments - for var_to_replace in ${others_var:-}; do - # ${var_to_replace^^} make the content of the variable on upper-cases - # ${!var_to_replace} get the content of the variable named $var_to_replace - ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banjailconf" - ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banfilterconf" - done - - else - # Usage 1, no template. Build a config file from scratch. - test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." - test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." - - tee $finalfail2banjailconf < \- \d+ \- Received request\: POST /_matrix/client/r0/login\??%(__synapse_start_line)s INFO \- POST\-\1\- Got login request with identifier: \{u'type': u'm.id.user', u'user'\: u'(.+?)'\}, medium\: None, address: None, user\: u'\5'%(__synapse_start_line)s WARNING \- \- (Attempted to login as @\5\:.+ but they do not exist|Failed password login for user @\5\:.+)$ +# +# ignoreregex = +# +# ----------------------------------------------------------------------------- +# +# Note about the "failregex" option: +# regex to match the password failure messages in the logfile. The +# host must be matched by a group named "host". The tag "" can +# be used for standard IP/hostname matching and is only an alias for +# (?:::f{4,6}:)?(?P[\w\-.^_]+) +# +# You can find some more explainations about how to make a regex here : +# https://www.fail2ban.org/wiki/index.php/MANUAL_0_8#Filters +# +# Note that the logfile need to exist before to call this helper !! +# +# To validate your regex you can test with this command: +# fail2ban-regex /var/log/YOUR_LOG_FILE_PATH /etc/fail2ban/filter.d/YOUR_APP.conf +# +# Requires YunoHost version 3.?.? or higher. +ynh_add_fail2ban_config () { + # Declare an array to define the options of this helper. + local legacy_args=lrmptv + declare -Ar args_array=( [l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= [t]=use_template [v]=others_var=) + local logpath + local failregex + local max_retry + local ports + local others_var + local use_template + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + use_template="${use_template:-0}" + max_retry=${max_retry:-3} + ports=${ports:-http,https} + + finalfail2banjailconf="/etc/fail2ban/jail.d/$app.conf" + finalfail2banfilterconf="/etc/fail2ban/filter.d/$app.conf" + ynh_backup_if_checksum_is_different "$finalfail2banjailconf" + ynh_backup_if_checksum_is_different "$finalfail2banfilterconf" + + if [ $use_template -eq 1 ] + then + # Usage 2, templates + cp ../conf/f2b_jail.conf $finalfail2banjailconf + cp ../conf/f2b_filter.conf $finalfail2banfilterconf + + if [ -n "${app:-}" ] + then + ynh_replace_string "__APP__" "$app" "$finalfail2banjailconf" + ynh_replace_string "__APP__" "$app" "$finalfail2banfilterconf" + fi + + # Replace all other variable given as arguments + for var_to_replace in ${others_var:-}; do + # ${var_to_replace^^} make the content of the variable on upper-cases + # ${!var_to_replace} get the content of the variable named $var_to_replace + ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banjailconf" + ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banfilterconf" + done + + else + # Usage 1, no template. Build a config file from scratch. + test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing." + test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing." + + tee $finalfail2banjailconf < ./${app}-logrotate << EOF # Build a config file for logrotate +$logfile { + # Rotate if the logfile exceeds 100Mo + size 100M + # Keep 12 old log maximum + rotate 12 + # Compress the logs with gzip + compress + # Compress the log at the next cycle. So keep always 2 non compressed logs + delaycompress + # Copy and truncate the log to allow to continue write on it. Instead of move the log. + copytruncate + # Do not do an error if the log is missing + missingok + # Not rotate if the log is empty + notifempty + # Keep old logs in the same dir + noolddir + $su_directive +} +EOF + sudo mkdir -p $(dirname "$logfile") # Create the log directory, if not exist + cat ${app}-logrotate | sudo $customtee /etc/logrotate.d/$app > /dev/null # Append this config to the existing config file, or replace the whole config file (depending on $customtee) +} + +# Remove the app's logrotate config. +# +# usage: ynh_remove_logrotate +# +# Requires YunoHost version 2.6.4 or higher. +ynh_remove_logrotate () { + if [ -e "/etc/logrotate.d/$app" ]; then + sudo rm "/etc/logrotate.d/$app" + fi +} diff --git a/data/helpers.d/nginx b/data/helpers.d/nginx new file mode 100644 index 000000000..ce6b61d3c --- /dev/null +++ b/data/helpers.d/nginx @@ -0,0 +1,76 @@ +#!/bin/bash + +# Create a dedicated nginx config +# +# usage: ynh_add_nginx_config "list of others variables to replace" +# +# | arg: list - (Optional) list of others variables to replace separated by spaces. For example : 'path_2 port_2 ...' +# +# This will use a template in ../conf/nginx.conf +# __PATH__ by $path_url +# __DOMAIN__ by $domain +# __PORT__ by $port +# __NAME__ by $app +# __FINALPATH__ by $final_path +# +# And dynamic variables (from the last example) : +# __PATH_2__ by $path_2 +# __PORT_2__ by $port_2 +# +# Requires YunoHost version 2.7.2 or higher. +ynh_add_nginx_config () { + finalnginxconf="/etc/nginx/conf.d/$domain.d/$app.conf" + local others_var=${1:-} + ynh_backup_if_checksum_is_different --file="$finalnginxconf" + sudo cp ../conf/nginx.conf "$finalnginxconf" + + # To avoid a break by set -u, use a void substitution ${var:-}. If the variable is not set, it's simply set with an empty variable. + # Substitute in a nginx config file only if the variable is not empty + if test -n "${path_url:-}"; then + # path_url_slash_less is path_url, or a blank value if path_url is only '/' + local path_url_slash_less=${path_url%/} + ynh_replace_string --match_string="__PATH__/" --replace_string="$path_url_slash_less/" --target_file="$finalnginxconf" + ynh_replace_string --match_string="__PATH__" --replace_string="$path_url" --target_file="$finalnginxconf" + fi + if test -n "${domain:-}"; then + ynh_replace_string --match_string="__DOMAIN__" --replace_string="$domain" --target_file="$finalnginxconf" + fi + if test -n "${port:-}"; then + ynh_replace_string --match_string="__PORT__" --replace_string="$port" --target_file="$finalnginxconf" + fi + if test -n "${app:-}"; then + ynh_replace_string --match_string="__NAME__" --replace_string="$app" --target_file="$finalnginxconf" + fi + if test -n "${final_path:-}"; then + ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalnginxconf" + fi + + # Replace all other variable given as arguments + for var_to_replace in $others_var + do + # ${var_to_replace^^} make the content of the variable on upper-cases + # ${!var_to_replace} get the content of the variable named $var_to_replace + ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalnginxconf" + done + + if [ "${path_url:-}" != "/" ] + then + ynh_replace_string --match_string="^#sub_path_only" --replace_string="" --target_file="$finalnginxconf" + else + ynh_replace_string --match_string="^#root_path_only" --replace_string="" --target_file="$finalnginxconf" + fi + + ynh_store_file_checksum --file="$finalnginxconf" + + ynh_systemd_action --service_name=nginx --action=reload +} + +# Remove the dedicated nginx config +# +# usage: ynh_remove_nginx_config +# +# Requires YunoHost version 2.7.2 or higher. +ynh_remove_nginx_config () { + ynh_secure_remove --file="/etc/nginx/conf.d/$domain.d/$app.conf" + ynh_systemd_action --service_name=nginx --action=reload +} diff --git a/data/helpers.d/php b/data/helpers.d/php new file mode 100644 index 000000000..c9e3ba9ed --- /dev/null +++ b/data/helpers.d/php @@ -0,0 +1,67 @@ +#!/bin/bash + +# Create a dedicated php-fpm config +# +# usage: ynh_add_fpm_config [--phpversion=7.X] +# | arg: -v, --phpversion - Version of php to use. +# +# Requires YunoHost version 2.7.2 or higher. +ynh_add_fpm_config () { + # Declare an array to define the options of this helper. + local legacy_args=v + declare -Ar args_array=( [v]=phpversion= ) + local phpversion + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Configure PHP-FPM 7.0 by default + phpversion="${phpversion:-7.0}" + + local fpm_config_dir="/etc/php/$phpversion/fpm" + local fpm_service="php${phpversion}-fpm" + # Configure PHP-FPM 5 on Debian Jessie + if [ "$(ynh_get_debian_release)" == "jessie" ]; then + fpm_config_dir="/etc/php5/fpm" + fpm_service="php5-fpm" + fi + ynh_app_setting_set --app=$app --key=fpm_config_dir --value="$fpm_config_dir" + ynh_app_setting_set --app=$app --key=fpm_service --value="$fpm_service" + finalphpconf="$fpm_config_dir/pool.d/$app.conf" + ynh_backup_if_checksum_is_different --file="$finalphpconf" + sudo cp ../conf/php-fpm.conf "$finalphpconf" + ynh_replace_string --match_string="__NAMETOCHANGE__" --replace_string="$app" --target_file="$finalphpconf" + ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalphpconf" + ynh_replace_string --match_string="__USER__" --replace_string="$app" --target_file="$finalphpconf" + ynh_replace_string --match_string="__PHPVERSION__" --replace_string="$phpversion" --target_file="$finalphpconf" + sudo chown root: "$finalphpconf" + ynh_store_file_checksum --file="$finalphpconf" + + if [ -e "../conf/php-fpm.ini" ] + then + echo "Packagers ! Please do not use a separate php ini file, merge your directives in the pool file instead." >&2 + finalphpini="$fpm_config_dir/conf.d/20-$app.ini" + ynh_backup_if_checksum_is_different "$finalphpini" + sudo cp ../conf/php-fpm.ini "$finalphpini" + sudo chown root: "$finalphpini" + ynh_store_file_checksum "$finalphpini" + fi + ynh_systemd_action --service_name=$fpm_service --action=reload +} + +# Remove the dedicated php-fpm config +# +# usage: ynh_remove_fpm_config +# +# Requires YunoHost version 2.7.2 or higher. +ynh_remove_fpm_config () { + local fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir) + local fpm_service=$(ynh_app_setting_get --app=$app --key=fpm_service) + # Assume php version 7 if not set + if [ -z "$fpm_config_dir" ]; then + fpm_config_dir="/etc/php/7.0/fpm" + fpm_service="php7.0-fpm" + fi + ynh_secure_remove --file="$fpm_config_dir/pool.d/$app.conf" + ynh_secure_remove --file="$fpm_config_dir/conf.d/20-$app.ini" 2>&1 + ynh_systemd_action --service_name=$fpm_service --action=reload +} diff --git a/data/helpers.d/systemd b/data/helpers.d/systemd new file mode 100644 index 000000000..54a376cd6 --- /dev/null +++ b/data/helpers.d/systemd @@ -0,0 +1,70 @@ +#!/bin/bash + +# Create a dedicated systemd config +# +# usage: ynh_add_systemd_config [--service=service] [--template=template] +# | arg: -s, --service - Service name (optionnal, $app by default) +# | arg: -t, --template - Name of template file (optionnal, this is 'systemd' by default, meaning ./conf/systemd.service will be used as template) +# +# This will use the template ../conf/.service +# to generate a systemd config, by replacing the following keywords +# with global variables that should be defined before calling +# this helper : +# +# __APP__ by $app +# __FINALPATH__ by $final_path +# +# Requires YunoHost version 2.7.2 or higher. +ynh_add_systemd_config () { + # Declare an array to define the options of this helper. + local legacy_args=st + declare -Ar args_array=( [s]=service= [t]=template= ) + local service + local template + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + local service="${service:-$app}" + local template="${template:-systemd.service}" + + finalsystemdconf="/etc/systemd/system/$service.service" + ynh_backup_if_checksum_is_different --file="$finalsystemdconf" + sudo cp ../conf/$template "$finalsystemdconf" + + # To avoid a break by set -u, use a void substitution ${var:-}. If the variable is not set, it's simply set with an empty variable. + # Substitute in a nginx config file only if the variable is not empty + if test -n "${final_path:-}"; then + ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalsystemdconf" + fi + if test -n "${app:-}"; then + ynh_replace_string --match_string="__APP__" --replace_string="$app" --target_file="$finalsystemdconf" + fi + ynh_store_file_checksum --file="$finalsystemdconf" + + sudo chown root: "$finalsystemdconf" + sudo systemctl enable $service + sudo systemctl daemon-reload +} + +# Remove the dedicated systemd config +# +# usage: ynh_remove_systemd_config [--service=service] +# | arg: -s, --service - Service name (optionnal, $app by default) +# +# Requires YunoHost version 2.7.2 or higher. +ynh_remove_systemd_config () { + # Declare an array to define the options of this helper. + local legacy_args=s + declare -Ar args_array=( [s]=service= ) + local service + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + local service="${service:-$app}" + + local finalsystemdconf="/etc/systemd/system/$service.service" + if [ -e "$finalsystemdconf" ]; then + ynh_systemd_action --service_name=$service --action=stop + systemctl disable $service + ynh_secure_remove --file="$finalsystemdconf" + systemctl daemon-reload + fi +} From 23e31daf64cf49a7780cec242cb521cef7afd720 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 25 Apr 2019 20:13:35 +0200 Subject: [PATCH 30/42] Merge 'ip' in 'network' ... --- data/helpers.d/ip | 76 ------------------------------------------ data/helpers.d/network | 75 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 76 deletions(-) delete mode 100644 data/helpers.d/ip diff --git a/data/helpers.d/ip b/data/helpers.d/ip deleted file mode 100644 index 2ca4053d9..000000000 --- a/data/helpers.d/ip +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash - -# Validate an IP address -# -# usage: ynh_validate_ip --family=family --ip_address=ip_address -# | ret: 0 for valid ip addresses, 1 otherwise -# -# example: ynh_validate_ip 4 111.222.333.444 -# -# Requires YunoHost version 2.2.4 or higher. -ynh_validate_ip() -{ - # http://stackoverflow.com/questions/319279/how-to-validate-ip-address-in-python#319298 - - # Declare an array to define the options of this helper. - local legacy_args=fi - declare -Ar args_array=( [f]=family= [i]=ip_address= ) - local family - local ip_address - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - - [ "$family" == "4" ] || [ "$family" == "6" ] || return 1 - - python /dev/stdin << EOF -import socket -import sys -family = { "4" : socket.AF_INET, "6" : socket.AF_INET6 } -try: - socket.inet_pton(family["$family"], "$ip_address") -except socket.error: - sys.exit(1) -sys.exit(0) -EOF -} - -# Validate an IPv4 address -# -# example: ynh_validate_ip4 111.222.333.444 -# -# usage: ynh_validate_ip4 --ip_address=ip_address -# | ret: 0 for valid ipv4 addresses, 1 otherwise -# -# Requires YunoHost version 2.2.4 or higher. -ynh_validate_ip4() -{ - # Declare an array to define the options of this helper. - local legacy_args=i - declare -Ar args_array=( [i]=ip_address= ) - local ip_address - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - - ynh_validate_ip 4 $ip_address -} - - -# Validate an IPv6 address -# -# example: ynh_validate_ip6 2000:dead:beef::1 -# -# usage: ynh_validate_ip6 --ip_address=ip_address -# | ret: 0 for valid ipv6 addresses, 1 otherwise -# -# Requires YunoHost version 2.2.4 or higher. -ynh_validate_ip6() -{ - # Declare an array to define the options of this helper. - local legacy_args=i - declare -Ar args_array=( [i]=ip_address= ) - local ip_address - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - - ynh_validate_ip 6 $ip_address -} diff --git a/data/helpers.d/network b/data/helpers.d/network index 4dc080203..d750d549f 100644 --- a/data/helpers.d/network +++ b/data/helpers.d/network @@ -101,3 +101,78 @@ ynh_webpath_register () { sudo yunohost app register-url $app $domain $path_url } + +# Validate an IP address +# +# usage: ynh_validate_ip --family=family --ip_address=ip_address +# | ret: 0 for valid ip addresses, 1 otherwise +# +# example: ynh_validate_ip 4 111.222.333.444 +# +# Requires YunoHost version 2.2.4 or higher. +ynh_validate_ip() +{ + # http://stackoverflow.com/questions/319279/how-to-validate-ip-address-in-python#319298 + + # Declare an array to define the options of this helper. + local legacy_args=fi + declare -Ar args_array=( [f]=family= [i]=ip_address= ) + local family + local ip_address + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + [ "$family" == "4" ] || [ "$family" == "6" ] || return 1 + + python /dev/stdin << EOF +import socket +import sys +family = { "4" : socket.AF_INET, "6" : socket.AF_INET6 } +try: + socket.inet_pton(family["$family"], "$ip_address") +except socket.error: + sys.exit(1) +sys.exit(0) +EOF +} + +# Validate an IPv4 address +# +# example: ynh_validate_ip4 111.222.333.444 +# +# usage: ynh_validate_ip4 --ip_address=ip_address +# | ret: 0 for valid ipv4 addresses, 1 otherwise +# +# Requires YunoHost version 2.2.4 or higher. +ynh_validate_ip4() +{ + # Declare an array to define the options of this helper. + local legacy_args=i + declare -Ar args_array=( [i]=ip_address= ) + local ip_address + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + ynh_validate_ip 4 $ip_address +} + + +# Validate an IPv6 address +# +# example: ynh_validate_ip6 2000:dead:beef::1 +# +# usage: ynh_validate_ip6 --ip_address=ip_address +# | ret: 0 for valid ipv6 addresses, 1 otherwise +# +# Requires YunoHost version 2.2.4 or higher. +ynh_validate_ip6() +{ + # Declare an array to define the options of this helper. + local legacy_args=i + declare -Ar args_array=( [i]=ip_address= ) + local ip_address + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + ynh_validate_ip 6 $ip_address +} From 73b1ff169ff91d79ac73d824d140d4e207230d4c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 25 Apr 2019 20:14:52 +0200 Subject: [PATCH 31/42] Move ynh_webpath_available/register to setting --- data/helpers.d/network | 44 ------------------------------------------ data/helpers.d/setting | 44 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/data/helpers.d/network b/data/helpers.d/network index d750d549f..d2a34f8d7 100644 --- a/data/helpers.d/network +++ b/data/helpers.d/network @@ -58,50 +58,6 @@ ynh_find_port () { echo $port } -# Check availability of a web path -# -# example: ynh_webpath_available --domain=some.domain.tld --path_url=/coffee -# -# usage: ynh_webpath_available --domain=domain --path_url=path -# | arg: -d, --domain - the domain/host of the url -# | arg: -p, --path_url - the web path to check the availability of -# -# Requires YunoHost version 2.6.4 or higher. -ynh_webpath_available () { - # Declare an array to define the options of this helper. - local legacy_args=dp - declare -Ar args_array=( [d]=domain= [p]=path_url= ) - local domain - local path_url - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - - sudo yunohost domain url-available $domain $path_url -} - -# Register/book a web path for an app -# -# example: ynh_webpath_register --app=wordpress --domain=some.domain.tld --path_url=/coffee -# -# usage: ynh_webpath_register --app=app --domain=domain --path_url=path -# | arg: -a, --app - the app for which the domain should be registered -# | arg: -d, --domain - the domain/host of the web path -# | arg: -p, --path_url - the web path to be registered -# -# Requires YunoHost version 2.6.4 or higher. -ynh_webpath_register () { - # Declare an array to define the options of this helper. - local legacy_args=adp - declare -Ar args_array=( [a]=app= [d]=domain= [p]=path_url= ) - local app - local domain - local path_url - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - - sudo yunohost app register-url $app $domain $path_url -} - # Validate an IP address # # usage: ynh_validate_ip --family=family --ip_address=ip_address diff --git a/data/helpers.d/setting b/data/helpers.d/setting index 63d9104f3..799e26554 100644 --- a/data/helpers.d/setting +++ b/data/helpers.d/setting @@ -91,3 +91,47 @@ else: yaml.safe_dump(settings, f, default_flow_style=False) EOF } + +# Check availability of a web path +# +# example: ynh_webpath_available --domain=some.domain.tld --path_url=/coffee +# +# usage: ynh_webpath_available --domain=domain --path_url=path +# | arg: -d, --domain - the domain/host of the url +# | arg: -p, --path_url - the web path to check the availability of +# +# Requires YunoHost version 2.6.4 or higher. +ynh_webpath_available () { + # Declare an array to define the options of this helper. + local legacy_args=dp + declare -Ar args_array=( [d]=domain= [p]=path_url= ) + local domain + local path_url + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + sudo yunohost domain url-available $domain $path_url +} + +# Register/book a web path for an app +# +# example: ynh_webpath_register --app=wordpress --domain=some.domain.tld --path_url=/coffee +# +# usage: ynh_webpath_register --app=app --domain=domain --path_url=path +# | arg: -a, --app - the app for which the domain should be registered +# | arg: -d, --domain - the domain/host of the web path +# | arg: -p, --path_url - the web path to be registered +# +# Requires YunoHost version 2.6.4 or higher. +ynh_webpath_register () { + # Declare an array to define the options of this helper. + local legacy_args=adp + declare -Ar args_array=( [a]=app= [d]=domain= [p]=path_url= ) + local app + local domain + local path_url + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + sudo yunohost app register-url $app $domain $path_url +} From 125a52e54b14a669388acfc8f010e5b8344c6b28 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 25 Apr 2019 20:16:16 +0200 Subject: [PATCH 32/42] Rename package to apt to be more accurate about what it does --- data/helpers.d/{package => apt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename data/helpers.d/{package => apt} (100%) diff --git a/data/helpers.d/package b/data/helpers.d/apt similarity index 100% rename from data/helpers.d/package rename to data/helpers.d/apt From 3c22feb0cb08760b3a0a1c09c008ad6f619fc548 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 25 Apr 2019 20:19:04 +0200 Subject: [PATCH 33/42] Move string normalization helpers to 'string' --- data/helpers.d/mysql | 21 ---------------- data/helpers.d/network | 34 ------------------------- data/helpers.d/string | 56 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 55 deletions(-) diff --git a/data/helpers.d/mysql b/data/helpers.d/mysql index 1dad94e13..39f93891c 100644 --- a/data/helpers.d/mysql +++ b/data/helpers.d/mysql @@ -237,24 +237,3 @@ ynh_mysql_remove_db () { fi } -# Sanitize a string intended to be the name of a database -# (More specifically : replace - and . by _) -# -# example: dbname=$(ynh_sanitize_dbid $app) -# -# usage: ynh_sanitize_dbid --db_name=name -# | arg: -n, --db_name - name to correct/sanitize -# | ret: the corrected name -# -# Requires YunoHost version 2.2.4 or higher. -ynh_sanitize_dbid () { - # Declare an array to define the options of this helper. - local legacy_args=n - declare -Ar args_array=( [n]=db_name= ) - local db_name - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - - # We should avoid having - and . in the name of databases. They are replaced by _ - echo ${db_name//[-.]/_} -} diff --git a/data/helpers.d/network b/data/helpers.d/network index d2a34f8d7..0f75cb165 100644 --- a/data/helpers.d/network +++ b/data/helpers.d/network @@ -1,39 +1,5 @@ #!/bin/bash -# Normalize the url path syntax -# -# Handle the slash at the beginning of path and its absence at ending -# Return a normalized url path -# -# examples: -# url_path=$(ynh_normalize_url_path $url_path) -# ynh_normalize_url_path example # -> /example -# ynh_normalize_url_path /example # -> /example -# ynh_normalize_url_path /example/ # -> /example -# ynh_normalize_url_path / # -> / -# -# usage: ynh_normalize_url_path --path_url=path_to_normalize -# | arg: -p, --path_url - URL path to normalize before using it -# -# Requires YunoHost version 2.6.4 or higher. -ynh_normalize_url_path () { - # Declare an array to define the options of this helper. - local legacy_args=p - declare -Ar args_array=( [p]=path_url= ) - local path_url - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - - test -n "$path_url" || ynh_die --message="ynh_normalize_url_path expect a URL path as first argument and received nothing." - if [ "${path_url:0:1}" != "/" ]; then # If the first character is not a / - path_url="/$path_url" # Add / at begin of path variable - fi - if [ "${path_url:${#path_url}-1}" == "/" ] && [ ${#path_url} -gt 1 ]; then # If the last character is a / and that not the only character. - path_url="${path_url:0:${#path_url}-1}" # Delete the last character - fi - echo $path_url -} - # Find a free port and return it # # example: port=$(ynh_find_port --port=8080) diff --git a/data/helpers.d/string b/data/helpers.d/string index 52eede872..fcbc5190d 100644 --- a/data/helpers.d/string +++ b/data/helpers.d/string @@ -83,3 +83,59 @@ ynh_replace_special_string () { ynh_replace_string --match_string="$match_string" --replace_string="$replace_string" --target_file="$target_file" } + +# Sanitize a string intended to be the name of a database +# (More specifically : replace - and . by _) +# +# example: dbname=$(ynh_sanitize_dbid $app) +# +# usage: ynh_sanitize_dbid --db_name=name +# | arg: -n, --db_name - name to correct/sanitize +# | ret: the corrected name +# +# Requires YunoHost version 2.2.4 or higher. +ynh_sanitize_dbid () { + # Declare an array to define the options of this helper. + local legacy_args=n + declare -Ar args_array=( [n]=db_name= ) + local db_name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # We should avoid having - and . in the name of databases. They are replaced by _ + echo ${db_name//[-.]/_} +} + +# Normalize the url path syntax +# +# Handle the slash at the beginning of path and its absence at ending +# Return a normalized url path +# +# examples: +# url_path=$(ynh_normalize_url_path $url_path) +# ynh_normalize_url_path example # -> /example +# ynh_normalize_url_path /example # -> /example +# ynh_normalize_url_path /example/ # -> /example +# ynh_normalize_url_path / # -> / +# +# usage: ynh_normalize_url_path --path_url=path_to_normalize +# | arg: -p, --path_url - URL path to normalize before using it +# +# Requires YunoHost version 2.6.4 or higher. +ynh_normalize_url_path () { + # Declare an array to define the options of this helper. + local legacy_args=p + declare -Ar args_array=( [p]=path_url= ) + local path_url + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + test -n "$path_url" || ynh_die --message="ynh_normalize_url_path expect a URL path as first argument and received nothing." + if [ "${path_url:0:1}" != "/" ]; then # If the first character is not a / + path_url="/$path_url" # Add / at begin of path variable + fi + if [ "${path_url:${#path_url}-1}" == "/" ] && [ ${#path_url} -gt 1 ]; then # If the last character is a / and that not the only character. + path_url="${path_url:0:${#path_url}-1}" # Delete the last character + fi + echo $path_url +} From 267c4119a882158d53074a03140cf51597a7900a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 25 Apr 2019 20:41:38 +0200 Subject: [PATCH 34/42] Rename 'filesystem' to 'backup' (backup-related stuff) and migrate some stuff to/from 'utils' --- data/helpers.d/{filesystem => backup} | 240 ++++++++++++++------------ data/helpers.d/utils | 202 ++++++++++------------ 2 files changed, 221 insertions(+), 221 deletions(-) rename data/helpers.d/{filesystem => backup} (67%) diff --git a/data/helpers.d/filesystem b/data/helpers.d/backup similarity index 67% rename from data/helpers.d/filesystem rename to data/helpers.d/backup index 3a7d6da25..2e08a5417 100644 --- a/data/helpers.d/filesystem +++ b/data/helpers.d/backup @@ -5,17 +5,17 @@ source /usr/share/yunohost/helpers.d/getopts CAN_BIND=${CAN_BIND:-1} # Add a file or a directory to the list of paths to backup -# +# # Note: this helper could be used in backup hook or in backup script inside an # app package # # Details: ynh_backup writes SRC and the relative DEST into a CSV file. And it -# creates the parent destination directory -# +# creates the parent destination directory +# # If DEST is ended by a slash it complete this path with the basename of SRC. # # usage: ynh_backup --src_path=src_path [--dest_path=dest_path] [--is_big] [--not_mandatory] -# | arg: -s, --src_path - file or directory to bind or symlink or copy. it shouldn't be in the backup dir. +# | arg: -s, --src_path - file or directory to bind or symlink or copy. it shouldn't be in the backup dir. # | arg: -d, --dest_path - destination file or directory inside the backup dir # | arg: -b, --is_big - Indicate data are big (mail, video, image ...) # | arg: -m, --not_mandatory - Indicate that if the file is missing, the backup can ignore it. @@ -32,7 +32,7 @@ CAN_BIND=${CAN_BIND:-1} # # ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf/" # # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/$app.conf" -# +# # ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "conf" # # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf" # @@ -190,7 +190,7 @@ with open(sys.argv[1], 'r') as backup_file: return $? } -# Restore a file or a directory +# Restore a file or a directory # # Use the registered path in backup_list by ynh_backup to restore the file at # the right place. @@ -205,7 +205,7 @@ with open(sys.argv[1], 'r') as backup_file: # # You can also use relative paths: # ynh_restore_file "conf/nginx.conf" # -# If DEST_PATH already exists and is lighter than 500 Mo, a backup will be made in +# If DEST_PATH already exists and is lighter than 500 Mo, a backup will be made in # /home/yunohost.conf/backup/. Otherwise, the existing file is removed. # # if apps/wordpress/etc/nginx/conf.d/$domain.d/$app.conf exists, restore it into @@ -248,7 +248,7 @@ ynh_restore_file () { then local backup_file="/home/yunohost.conf/backup/${dest_path}.backup.$(date '+%Y%m%d.%H%M%S')" mkdir -p "$(dirname "$backup_file")" - mv "${dest_path}" "$backup_file" # Move the current file or directory + mv "${dest_path}" "$backup_file" # Move the current file or directory else ynh_secure_remove --file=${dest_path} fi @@ -282,26 +282,6 @@ ynh_bind_or_cp() { ynh_backup "$1" "$2" 1 } -# Create a directory under /tmp -# -# [internal] -# -# Deprecated helper -# -# usage: ynh_mkdir_tmp -# | ret: the created directory path -ynh_mkdir_tmp() { - ynh_print_warn --message="The helper ynh_mkdir_tmp is deprecated." - ynh_print_warn --message="You should use 'mktemp -d' instead and manage permissions \ -properly with chmod/chown." - local TMP_DIR=$(mktemp -d) - - # Give rights to other users could be a security risk. - # But for retrocompatibility we need it. (This helpers is deprecated) - chmod 755 $TMP_DIR - echo $TMP_DIR -} - # Calculate and store a file checksum into the app settings # # $app should be defined when calling this helper @@ -311,29 +291,29 @@ properly with chmod/chown." # # Requires YunoHost version 2.6.4 or higher. ynh_store_file_checksum () { - # Declare an array to define the options of this helper. - local legacy_args=f - declare -Ar args_array=( [f]=file= ) - local file - # Manage arguments with getopts - ynh_handle_getopts_args "$@" + # Declare an array to define the options of this helper. + local legacy_args=f + declare -Ar args_array=( [f]=file= ) + local file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" - local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' - ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(sudo md5sum "$file" | cut -d' ' -f1) + local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' + ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(sudo md5sum "$file" | cut -d' ' -f1) - # If backup_file_checksum isn't empty, ynh_backup_if_checksum_is_different has made a backup - if [ -n "${backup_file_checksum-}" ] - then - # Print the diff between the previous file and the new one. - # diff return 1 if the files are different, so the || true - diff --report-identical-files --unified --color=always $backup_file_checksum $file >&2 || true - fi - # Unset the variable, so it wouldn't trig a ynh_store_file_checksum without a ynh_backup_if_checksum_is_different before it. - unset backup_file_checksum + # If backup_file_checksum isn't empty, ynh_backup_if_checksum_is_different has made a backup + if [ -n "${backup_file_checksum-}" ] + then + # Print the diff between the previous file and the new one. + # diff return 1 if the files are different, so the || true + diff --report-identical-files --unified --color=always $backup_file_checksum $file >&2 || true + fi + # Unset the variable, so it wouldn't trig a ynh_store_file_checksum without a ynh_backup_if_checksum_is_different before it. + unset backup_file_checksum } # Verify the checksum and backup the file if it's different -# This helper is primarily meant to allow to easily backup personalised/manually +# This helper is primarily meant to allow to easily backup personalised/manually # modified config files. # # $app should be defined when calling this helper @@ -344,28 +324,28 @@ ynh_store_file_checksum () { # # Requires YunoHost version 2.6.4 or higher. ynh_backup_if_checksum_is_different () { - # Declare an array to define the options of this helper. - local legacy_args=f - declare -Ar args_array=( [f]=file= ) - local file - # Manage arguments with getopts - ynh_handle_getopts_args "$@" + # Declare an array to define the options of this helper. + local legacy_args=f + declare -Ar args_array=( [f]=file= ) + local file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" - local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' - local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name) - # backup_file_checksum isn't declare as local, so it can be reuse by ynh_store_file_checksum - backup_file_checksum="" - if [ -n "$checksum_value" ] - then # Proceed only if a value was stored into the app settings - if ! echo "$checksum_value $file" | sudo md5sum -c --status - then # If the checksum is now different - backup_file_checksum="/home/yunohost.conf/backup/$file.backup.$(date '+%Y%m%d.%H%M%S')" - sudo mkdir -p "$(dirname "$backup_file_checksum")" - sudo cp -a "$file" "$backup_file_checksum" # Backup the current file - ynh_print_warn "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum" - echo "$backup_file_checksum" # Return the name of the backup file - fi - fi + local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' + local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name) + # backup_file_checksum isn't declare as local, so it can be reuse by ynh_store_file_checksum + backup_file_checksum="" + if [ -n "$checksum_value" ] + then # Proceed only if a value was stored into the app settings + if ! echo "$checksum_value $file" | sudo md5sum -c --status + then # If the checksum is now different + backup_file_checksum="/home/yunohost.conf/backup/$file.backup.$(date '+%Y%m%d.%H%M%S')" + sudo mkdir -p "$(dirname "$backup_file_checksum")" + sudo cp -a "$file" "$backup_file_checksum" # Backup the current file + ynh_print_warn "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum" + echo "$backup_file_checksum" # Return the name of the backup file + fi + fi } # Delete a file checksum from the app settings @@ -377,54 +357,94 @@ ynh_backup_if_checksum_is_different () { # # Requires YunoHost version 3.3.1 or higher. ynh_delete_file_checksum () { - # Declare an array to define the options of this helper. - local legacy_args=f - declare -Ar args_array=( [f]=file= ) - local file - # Manage arguments with getopts - ynh_handle_getopts_args "$@" + # Declare an array to define the options of this helper. + local legacy_args=f + declare -Ar args_array=( [f]=file= ) + local file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" - local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' - ynh_app_setting_delete --app=$app --key=$checksum_setting_name + local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_' + ynh_app_setting_delete --app=$app --key=$checksum_setting_name } -# Remove a file or a directory securely +# Make a backup in case of failed upgrade # -# usage: ynh_secure_remove --file=path_to_remove -# | arg: -f, --file - File or directory to remove +# usage: +# ynh_backup_before_upgrade +# ynh_clean_setup () { +# ynh_restore_upgradebackup +# } +# ynh_abort_if_errors # -# Requires YunoHost version 2.6.4 or higher. -ynh_secure_remove () { - # Declare an array to define the options of this helper. - local legacy_args=f - declare -Ar args_array=( [f]=file= ) - local file - # Manage arguments with getopts - ynh_handle_getopts_args "$@" +# Requires YunoHost version 2.7.2 or higher. +ynh_backup_before_upgrade () { + if [ ! -e "/etc/yunohost/apps/$app/scripts/backup" ] + then + ynh_print_warn --message="This app doesn't have any backup script." + return + fi + backup_number=1 + local old_backup_number=2 + local app_bck=${app//_/-} # Replace all '_' by '-' + NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0} - local forbidden_path=" \ - /var/www \ - /home/yunohost.app" + if [ "$NO_BACKUP_UPGRADE" -eq 0 ] + then + # Check if a backup already exists with the prefix 1 + if sudo yunohost backup list | grep -q $app_bck-pre-upgrade1 + then + # Prefix becomes 2 to preserve the previous backup + backup_number=2 + old_backup_number=1 + fi - if [ $# -ge 2 ] - then - ynh_print_warn --message="/!\ Packager ! You provided more than one argument to ynh_secure_remove but it will be ignored... Use this helper with one argument at time." - fi - - if [[ "$forbidden_path" =~ "$file" \ - # Match all paths or subpaths in $forbidden_path - || "$file" =~ ^/[[:alnum:]]+$ \ - # Match all first level paths from / (Like /var, /root, etc...) - || "${file:${#file}-1}" = "/" ]] - # Match if the path finishes by /. Because it seems there is an empty variable - then - ynh_print_warn --message="Avoid deleting $file." - else - if [ -e "$file" ] - then - sudo rm -R "$file" - else - ynh_print_info --message="$file wasn't deleted because it doesn't exist." - fi - fi + # Create backup + sudo BACKUP_CORE_ONLY=1 yunohost backup create --apps $app --name $app_bck-pre-upgrade$backup_number --debug + if [ "$?" -eq 0 ] + then + # If the backup succeeded, remove the previous backup + if sudo yunohost backup list | grep -q $app_bck-pre-upgrade$old_backup_number + then + # Remove the previous backup only if it exists + sudo yunohost backup delete $app_bck-pre-upgrade$old_backup_number > /dev/null + fi + else + ynh_die --message="Backup failed, the upgrade process was aborted." + fi + else + ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, backup will be avoided. Be careful, this upgrade is going to be operated without a security backup" + fi +} + +# Restore a previous backup if the upgrade process failed +# +# usage: +# ynh_backup_before_upgrade +# ynh_clean_setup () { +# ynh_restore_upgradebackup +# } +# ynh_abort_if_errors +# +# Requires YunoHost version 2.7.2 or higher. +ynh_restore_upgradebackup () { + ynh_print_err --message="Upgrade failed." + local app_bck=${app//_/-} # Replace all '_' by '-' + + NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0} + + if [ "$NO_BACKUP_UPGRADE" -eq 0 ] + then + # Check if an existing backup can be found before removing and restoring the application. + if sudo yunohost backup list | grep -q $app_bck-pre-upgrade$backup_number + then + # Remove the application then restore it + sudo yunohost app remove $app + # Restore the backup + sudo yunohost backup restore $app_bck-pre-upgrade$backup_number --apps $app --force --debug + ynh_die --message="The app was restored to the way it was before the failed upgrade." + fi + else + ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, that means there's no backup to restore. You have to fix this upgrade by yourself !" + fi } diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 1519e6150..5335cfbe6 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -29,87 +29,6 @@ ynh_get_plain_key() { done } -# Restore a previous backup if the upgrade process failed -# -# usage: -# ynh_backup_before_upgrade -# ynh_clean_setup () { -# ynh_restore_upgradebackup -# } -# ynh_abort_if_errors -# -# Requires YunoHost version 2.7.2 or higher. -ynh_restore_upgradebackup () { - ynh_print_err --message="Upgrade failed." - local app_bck=${app//_/-} # Replace all '_' by '-' - - NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0} - - if [ "$NO_BACKUP_UPGRADE" -eq 0 ] - then - # Check if an existing backup can be found before removing and restoring the application. - if sudo yunohost backup list | grep -q $app_bck-pre-upgrade$backup_number - then - # Remove the application then restore it - sudo yunohost app remove $app - # Restore the backup - sudo yunohost backup restore $app_bck-pre-upgrade$backup_number --apps $app --force --debug - ynh_die --message="The app was restored to the way it was before the failed upgrade." - fi - else - ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, that means there's no backup to restore. You have to fix this upgrade by yourself !" - fi -} - -# Make a backup in case of failed upgrade -# -# usage: -# ynh_backup_before_upgrade -# ynh_clean_setup () { -# ynh_restore_upgradebackup -# } -# ynh_abort_if_errors -# -# Requires YunoHost version 2.7.2 or higher. -ynh_backup_before_upgrade () { - if [ ! -e "/etc/yunohost/apps/$app/scripts/backup" ] - then - ynh_print_warn --message="This app doesn't have any backup script." - return - fi - backup_number=1 - local old_backup_number=2 - local app_bck=${app//_/-} # Replace all '_' by '-' - NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0} - - if [ "$NO_BACKUP_UPGRADE" -eq 0 ] - then - # Check if a backup already exists with the prefix 1 - if sudo yunohost backup list | grep -q $app_bck-pre-upgrade1 - then - # Prefix becomes 2 to preserve the previous backup - backup_number=2 - old_backup_number=1 - fi - - # Create backup - sudo BACKUP_CORE_ONLY=1 yunohost backup create --apps $app --name $app_bck-pre-upgrade$backup_number --debug - if [ "$?" -eq 0 ] - then - # If the backup succeeded, remove the previous backup - if sudo yunohost backup list | grep -q $app_bck-pre-upgrade$old_backup_number - then - # Remove the previous backup only if it exists - sudo yunohost backup delete $app_bck-pre-upgrade$old_backup_number > /dev/null - fi - else - ynh_die --message="Backup failed, the upgrade process was aborted." - fi - else - ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, backup will be avoided. Be careful, this upgrade is going to be operated without a security backup" - fi -} - # Download, check integrity, uncompress and patch the source from app.src # # The file conf/app.src need to contains: @@ -129,7 +48,7 @@ ynh_backup_before_upgrade () { # SOURCE_IN_SUBDIR=false # # (Optionnal) Name of the local archive (offline setup support) # # default: ${src_id}.${src_format} -# SOURCE_FILENAME=example.tar.gz +# SOURCE_FILENAME=example.tar.gz # # (Optional) If it set as false don't extract the source. # # (Useful to get a debian package or a python wheel.) # # default: true @@ -212,7 +131,7 @@ ynh_setup_source () { then mv $src_filename $dest_dir elif [ "$src_format" = "zip" ] - then + then # Zip format # Using of a temp directory, because unzip doesn't manage --strip-components if $src_in_subdir ; then @@ -262,7 +181,7 @@ ynh_setup_source () { # $domain and $path_url should be defined externally (and correspond to the domain.tld and the /path (of the app?)) # # example: ynh_local_curl "/install.php?installButton" "foo=$var1" "bar=$var2" -# +# # usage: ynh_local_curl "page_uri" "key1=value1" "key2=value2" ... # | arg: page_uri - Path (relative to $path_url) of the page where POST data will be sent # | arg: key1=value1 - (Optionnal) POST key and corresponding value @@ -271,38 +190,38 @@ ynh_setup_source () { # # Requires YunoHost version 2.6.4 or higher. ynh_local_curl () { - # Define url of page to curl - local local_page=$(ynh_normalize_url_path $1) - local full_path=$path_url$local_page - - if [ "${path_url}" == "/" ]; then - full_path=$local_page - fi - - local full_page_url=https://localhost$full_path + # Define url of page to curl + local local_page=$(ynh_normalize_url_path $1) + local full_path=$path_url$local_page - # Concatenate all other arguments with '&' to prepare POST data - local POST_data="" - local arg="" - for arg in "${@:2}" - do - POST_data="${POST_data}${arg}&" - done - if [ -n "$POST_data" ] - then - # Add --data arg and remove the last character, which is an unecessary '&' - POST_data="--data ${POST_data::-1}" - fi - - # Wait untils nginx has fully reloaded (avoid curl fail with http2) - sleep 2 + if [ "${path_url}" == "/" ]; then + full_path=$local_page + fi - # Curl the URL - curl --silent --show-error -kL -H "Host: $domain" --resolve $domain:443:127.0.0.1 $POST_data "$full_page_url" + local full_page_url=https://localhost$full_path + + # Concatenate all other arguments with '&' to prepare POST data + local POST_data="" + local arg="" + for arg in "${@:2}" + do + POST_data="${POST_data}${arg}&" + done + if [ -n "$POST_data" ] + then + # Add --data arg and remove the last character, which is an unecessary '&' + POST_data="--data ${POST_data::-1}" + fi + + # Wait untils nginx has fully reloaded (avoid curl fail with http2) + sleep 2 + + # Curl the URL + curl --silent --show-error -kL -H "Host: $domain" --resolve $domain:443:127.0.0.1 $POST_data "$full_page_url" } # Render templates with Jinja2 -# +# # Attention : Variables should be exported before calling this helper to be # accessible inside templates. # @@ -318,3 +237,64 @@ ynh_render_template() { jinja2.Template(sys.stdin.read() ).render(os.environ));' < $template_path > $output_path } + +# Create a directory under /tmp +# +# [internal] +# +# Deprecated helper +# +# usage: ynh_mkdir_tmp +# | ret: the created directory path +ynh_mkdir_tmp() { + ynh_print_warn --message="The helper ynh_mkdir_tmp is deprecated." + ynh_print_warn --message="You should use 'mktemp -d' instead and manage permissions \ +properly with chmod/chown." + local TMP_DIR=$(mktemp -d) + + # Give rights to other users could be a security risk. + # But for retrocompatibility we need it. (This helpers is deprecated) + chmod 755 $TMP_DIR + echo $TMP_DIR +} + +# Remove a file or a directory securely +# +# usage: ynh_secure_remove --file=path_to_remove +# | arg: -f, --file - File or directory to remove +# +# Requires YunoHost version 2.6.4 or higher. +ynh_secure_remove () { + # Declare an array to define the options of this helper. + local legacy_args=f + declare -Ar args_array=( [f]=file= ) + local file + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local forbidden_path=" \ + /var/www \ + /home/yunohost.app" + + if [ $# -ge 2 ] + then + ynh_print_warn --message="/!\ Packager ! You provided more than one argument to ynh_secure_remove but it will be ignored... Use this helper with one argument at time." + fi + + if [[ "$forbidden_path" =~ "$file" \ + # Match all paths or subpaths in $forbidden_path + || "$file" =~ ^/[[:alnum:]]+$ \ + # Match all first level paths from / (Like /var, /root, etc...) + || "${file:${#file}-1}" = "/" ]] + # Match if the path finishes by /. Because it seems there is an empty variable + then + ynh_print_warn --message="Avoid deleting $file." + else + if [ -e "$file" ] + then + sudo rm -R "$file" + else + ynh_print_info --message="$file wasn't deleted because it doesn't exist." + fi + fi +} From c35aaea945e419ae11812cc11b92b9e9d676c58d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 25 Apr 2019 20:46:10 +0200 Subject: [PATCH 35/42] Move systemd_action in 'systemd' --- data/helpers.d/system | 107 ---------------------------------------- data/helpers.d/systemd | 109 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 107 deletions(-) diff --git a/data/helpers.d/system b/data/helpers.d/system index b8971c80f..fd42ce726 100644 --- a/data/helpers.d/system +++ b/data/helpers.d/system @@ -60,113 +60,6 @@ ynh_get_debian_release () { echo $(lsb_release --codename --short) } -# Start (or other actions) a service, print a log in case of failure and optionnaly wait until the service is completely started -# -# usage: ynh_systemd_action [-n service_name] [-a action] [ [-l "line to match"] [-p log_path] [-t timeout] [-e length] ] -# | arg: -n, --service_name= - Name of the service to start. Default : $app -# | arg: -a, --action= - Action to perform with systemctl. Default: start -# | arg: -l, --line_match= - Line to match - The line to find in the log to attest the service have finished to boot. -# If not defined it don't wait until the service is completely started. -# WARNING: When using --line_match, you should always add `ynh_clean_check_starting` into your -# `ynh_clean_setup` at the beginning of the script. Otherwise, tail will not stop in case of failure -# of the script. The script will then hang forever. -# | arg: -p, --log_path= - Log file - Path to the log file. Default : /var/log/$app/$app.log -# | arg: -t, --timeout= - Timeout - The maximum time to wait before ending the watching. Default : 300 seconds. -# | arg: -e, --length= - Length of the error log : Default : 20 -ynh_systemd_action() { - # Declare an array to define the options of this helper. - local legacy_args=nalpte - declare -Ar args_array=( [n]=service_name= [a]=action= [l]=line_match= [p]=log_path= [t]=timeout= [e]=length= ) - local service_name - local action - local line_match - local length - local log_path - local timeout - - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - - local service_name="${service_name:-$app}" - local action=${action:-start} - local log_path="${log_path:-/var/log/$service_name/$service_name.log}" - local length=${length:-20} - local timeout=${timeout:-300} - - # Start to read the log - if [[ -n "${line_match:-}" ]] - then - local templog="$(mktemp)" - # Following the starting of the app in its log - if [ "$log_path" == "systemd" ] ; then - # Read the systemd journal - journalctl --unit=$service_name --follow --since=-0 --quiet > "$templog" & - # Get the PID of the journalctl command - local pid_tail=$! - else - # Read the specified log file - tail -F -n0 "$log_path" > "$templog" 2>&1 & - # Get the PID of the tail command - local pid_tail=$! - fi - fi - - ynh_print_info --message="${action^} the service $service_name" - - # Use reload-or-restart instead of reload. So it wouldn't fail if the service isn't running. - if [ "$action" == "reload" ]; then - action="reload-or-restart" - fi - - systemctl $action $service_name \ - || ( journalctl --no-pager --lines=$length -u $service_name >&2 \ - ; test -e "$log_path" && echo "--" >&2 && tail --lines=$length "$log_path" >&2 \ - ; false ) - - # Start the timeout and try to find line_match - if [[ -n "${line_match:-}" ]] - then - local i=0 - for i in $(seq 1 $timeout) - do - # Read the log until the sentence is found, that means the app finished to start. Or run until the timeout - if grep --quiet "$line_match" "$templog" - then - ynh_print_info --message="The service $service_name has correctly started." - break - fi - if [ $i -eq 3 ]; then - echo -n "Please wait, the service $service_name is ${action}ing" >&2 - fi - if [ $i -ge 3 ]; then - echo -n "." >&2 - fi - sleep 1 - done - if [ $i -ge 3 ]; then - echo "" >&2 - fi - if [ $i -eq $timeout ] - then - ynh_print_warn --message="The service $service_name didn't fully started before the timeout." - ynh_print_warn --message="Please find here an extract of the end of the log of the service $service_name:" - journalctl --no-pager --lines=$length -u $service_name >&2 - test -e "$log_path" && echo "--" >&2 && tail --lines=$length "$log_path" >&2 - fi - ynh_clean_check_starting - fi -} - -# Clean temporary process and file used by ynh_check_starting -# (usually used in ynh_clean_setup scripts) -# -# usage: ynh_clean_check_starting -ynh_clean_check_starting () { - # Stop the execution of tail. - kill -s 15 $pid_tail 2>&1 - ynh_secure_remove "$templog" 2>&1 -} - # Read the value of a key in a ynh manifest file # # usage: ynh_read_manifest manifest key diff --git a/data/helpers.d/systemd b/data/helpers.d/systemd index 54a376cd6..c4100bf8a 100644 --- a/data/helpers.d/systemd +++ b/data/helpers.d/systemd @@ -68,3 +68,112 @@ ynh_remove_systemd_config () { systemctl daemon-reload fi } + +# Start (or other actions) a service, print a log in case of failure and optionnaly wait until the service is completely started +# +# usage: ynh_systemd_action [-n service_name] [-a action] [ [-l "line to match"] [-p log_path] [-t timeout] [-e length] ] +# | arg: -n, --service_name= - Name of the service to start. Default : $app +# | arg: -a, --action= - Action to perform with systemctl. Default: start +# | arg: -l, --line_match= - Line to match - The line to find in the log to attest the service have finished to boot. +# If not defined it don't wait until the service is completely started. +# WARNING: When using --line_match, you should always add `ynh_clean_check_starting` into your +# `ynh_clean_setup` at the beginning of the script. Otherwise, tail will not stop in case of failure +# of the script. The script will then hang forever. +# | arg: -p, --log_path= - Log file - Path to the log file. Default : /var/log/$app/$app.log +# | arg: -t, --timeout= - Timeout - The maximum time to wait before ending the watching. Default : 300 seconds. +# | arg: -e, --length= - Length of the error log : Default : 20 +ynh_systemd_action() { + # Declare an array to define the options of this helper. + local legacy_args=nalpte + declare -Ar args_array=( [n]=service_name= [a]=action= [l]=line_match= [p]=log_path= [t]=timeout= [e]=length= ) + local service_name + local action + local line_match + local length + local log_path + local timeout + + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local service_name="${service_name:-$app}" + local action=${action:-start} + local log_path="${log_path:-/var/log/$service_name/$service_name.log}" + local length=${length:-20} + local timeout=${timeout:-300} + + # Start to read the log + if [[ -n "${line_match:-}" ]] + then + local templog="$(mktemp)" + # Following the starting of the app in its log + if [ "$log_path" == "systemd" ] ; then + # Read the systemd journal + journalctl --unit=$service_name --follow --since=-0 --quiet > "$templog" & + # Get the PID of the journalctl command + local pid_tail=$! + else + # Read the specified log file + tail -F -n0 "$log_path" > "$templog" 2>&1 & + # Get the PID of the tail command + local pid_tail=$! + fi + fi + + ynh_print_info --message="${action^} the service $service_name" + + # Use reload-or-restart instead of reload. So it wouldn't fail if the service isn't running. + if [ "$action" == "reload" ]; then + action="reload-or-restart" + fi + + systemctl $action $service_name \ + || ( journalctl --no-pager --lines=$length -u $service_name >&2 \ + ; test -e "$log_path" && echo "--" >&2 && tail --lines=$length "$log_path" >&2 \ + ; false ) + + # Start the timeout and try to find line_match + if [[ -n "${line_match:-}" ]] + then + local i=0 + for i in $(seq 1 $timeout) + do + # Read the log until the sentence is found, that means the app finished to start. Or run until the timeout + if grep --quiet "$line_match" "$templog" + then + ynh_print_info --message="The service $service_name has correctly started." + break + fi + if [ $i -eq 3 ]; then + echo -n "Please wait, the service $service_name is ${action}ing" >&2 + fi + if [ $i -ge 3 ]; then + echo -n "." >&2 + fi + sleep 1 + done + if [ $i -ge 3 ]; then + echo "" >&2 + fi + if [ $i -eq $timeout ] + then + ynh_print_warn --message="The service $service_name didn't fully started before the timeout." + ynh_print_warn --message="Please find here an extract of the end of the log of the service $service_name:" + journalctl --no-pager --lines=$length -u $service_name >&2 + test -e "$log_path" && echo "--" >&2 && tail --lines=$length "$log_path" >&2 + fi + ynh_clean_check_starting + fi +} + +# Clean temporary process and file used by ynh_check_starting +# (usually used in ynh_clean_setup scripts) +# +# usage: ynh_clean_check_starting +ynh_clean_check_starting () { + # Stop the execution of tail. + kill -s 15 $pid_tail 2>&1 + ynh_secure_remove "$templog" 2>&1 +} + + From 6019a38919536a2b8f9827db3b703ad2f63b93ea Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 25 Apr 2019 20:49:58 +0200 Subject: [PATCH 36/42] Merge 'system' in 'utils' --- data/helpers.d/system | 178 --------------------------------- data/helpers.d/utils | 225 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 201 insertions(+), 202 deletions(-) delete mode 100644 data/helpers.d/system diff --git a/data/helpers.d/system b/data/helpers.d/system deleted file mode 100644 index fd42ce726..000000000 --- a/data/helpers.d/system +++ /dev/null @@ -1,178 +0,0 @@ -#!/bin/bash - -# Manage a fail of the script -# -# [internal] -# -# usage: -# ynh_exit_properly is used only by the helper ynh_abort_if_errors. -# You should not use it directly. -# Instead, add to your script: -# ynh_clean_setup () { -# instructions... -# } -# -# This function provide a way to clean some residual of installation that not managed by remove script. -# -# It prints a warning to inform that the script was failed, and execute the ynh_clean_setup function if used in the app script -# -# Requires YunoHost version 2.6.4 or higher. -ynh_exit_properly () { - local exit_code=$? - if [ "$exit_code" -eq 0 ]; then - exit 0 # Exit without error if the script ended correctly - fi - - trap '' EXIT # Ignore new exit signals - set +eu # Do not exit anymore if a command fail or if a variable is empty - - ynh_print_err --message="!!\n $app's script has encountered an error. Its execution was cancelled.\n!!" - - if type -t ynh_clean_setup > /dev/null; then # Check if the function exist in the app script. - ynh_clean_setup # Call the function to do specific cleaning for the app. - fi - - ynh_die # Exit with error status -} - -# Exits if an error occurs during the execution of the script. -# -# usage: ynh_abort_if_errors -# -# This configure the rest of the script execution such that, if an error occurs -# or if an empty variable is used, the execution of the script stops -# immediately and a call to `ynh_clean_setup` is triggered if it has been -# defined by your script. -# -# Requires YunoHost version 2.6.4 or higher. -ynh_abort_if_errors () { - set -eu # Exit if a command fail, and if a variable is used unset. - trap ynh_exit_properly EXIT # Capturing exit signals on shell script -} - -# Fetch the Debian release codename -# -# usage: ynh_get_debian_release -# | ret: The Debian release codename (i.e. jessie, stretch, ...) -# -# Requires YunoHost version 2.7.12 or higher. -ynh_get_debian_release () { - echo $(lsb_release --codename --short) -} - -# Read the value of a key in a ynh manifest file -# -# usage: ynh_read_manifest manifest key -# | arg: -m, --manifest= - Path of the manifest to read -# | arg: -k, --key= - Name of the key to find -# -# Requires YunoHost version 3.?.? or higher. -ynh_read_manifest () { - # Declare an array to define the options of this helper. - local legacy_args=mk - declare -Ar args_array=( [m]=manifest= [k]=manifest_key= ) - local manifest - local manifest_key - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - - if [ ! -e "$manifest" ]; then - # If the manifest isn't found, try the common place for backup and restore script. - manifest="../settings/manifest.json" - fi - - jq ".$manifest_key" "$manifest" --raw-output -} - -# Read the upstream version from the manifest -# The version number in the manifest is defined by ~ynh -# For example : 4.3-2~ynh3 -# This include the number before ~ynh -# In the last example it return 4.3-2 -# -# usage: ynh_app_upstream_version [-m manifest] -# | arg: -m, --manifest= - Path of the manifest to read -# -# Requires YunoHost version 3.?.? or higher. -ynh_app_upstream_version () { - # Declare an array to define the options of this helper. - local legacy_args=m - declare -Ar args_array=( [m]=manifest= ) - local manifest - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - - manifest="${manifest:-../manifest.json}" - version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") - echo "${version_key/~ynh*/}" -} - -# Read package version from the manifest -# The version number in the manifest is defined by ~ynh -# For example : 4.3-2~ynh3 -# This include the number after ~ynh -# In the last example it return 3 -# -# usage: ynh_app_package_version [-m manifest] -# | arg: -m, --manifest= - Path of the manifest to read -# -# Requires YunoHost version 3.?.? or higher. -ynh_app_package_version () { - # Declare an array to define the options of this helper. - local legacy_args=m - declare -Ar args_array=( [m]=manifest= ) - local manifest - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - - manifest="${manifest:-../manifest.json}" - version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") - echo "${version_key/*~ynh/}" -} - -# Checks the app version to upgrade with the existing app version and returns: -# - UPGRADE_APP if the upstream app version has changed -# - UPGRADE_PACKAGE if only the YunoHost package has changed -# -# It stops the current script without error if the package is up-to-date -# -# This helper should be used to avoid an upgrade of an app, or the upstream part -# of it, when it's not needed -# -# To force an upgrade, even if the package is up to date, -# you have to set the variable YNH_FORCE_UPGRADE before. -# example: sudo YNH_FORCE_UPGRADE=1 yunohost app upgrade MyApp -# -# usage: ynh_check_app_version_changed -# -# Requires YunoHost version 3.?.? or higher. -ynh_check_app_version_changed () { - local force_upgrade=${YNH_FORCE_UPGRADE:-0} - local package_check=${PACKAGE_CHECK_EXEC:-0} - - # By default, upstream app version has changed - local return_value="UPGRADE_APP" - - local current_version=$(ynh_read_manifest --manifest="/etc/yunohost/apps/$YNH_APP_INSTANCE_NAME/manifest.json" --manifest_key="version" || echo 1.0) - local current_upstream_version="$(ynh_app_upstream_version --manifest="/etc/yunohost/apps/$YNH_APP_INSTANCE_NAME/manifest.json")" - local update_version=$(ynh_read_manifest --manifest="../manifest.json" --manifest_key="version" || echo 1.0) - local update_upstream_version="$(ynh_app_upstream_version)" - - if [ "$current_version" == "$update_version" ] ; then - # Complete versions are the same - if [ "$force_upgrade" != "0" ] - then - ynh_print_info --message="Upgrade forced by YNH_FORCE_UPGRADE." - unset YNH_FORCE_UPGRADE - elif [ "$package_check" != "0" ] - then - ynh_print_info --message="Upgrade forced for package check." - else - ynh_die "Up-to-date, nothing to do" 0 - fi - elif [ "$current_upstream_version" == "$update_upstream_version" ] ; then - # Upstream versions are the same, only YunoHost package versions differ - return_value="UPGRADE_PACKAGE" - fi - echo $return_value -} diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 5335cfbe6..1dd83c0e2 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -1,32 +1,53 @@ #!/bin/bash -# Extract a key from a plain command output +# Manage a fail of the script # -# example: yunohost user info tata --output-as plain | ynh_get_plain_key mail +# [internal] # -# usage: ynh_get_plain_key key [subkey [subsubkey ...]] -# | ret: string - the key's value +# usage: +# ynh_exit_properly is used only by the helper ynh_abort_if_errors. +# You should not use it directly. +# Instead, add to your script: +# ynh_clean_setup () { +# instructions... +# } # -# Requires YunoHost version 2.2.4 or higher. -ynh_get_plain_key() { - local prefix="#" - local founded=0 - local key=$1 - shift - while read line; do - if [[ "$founded" == "1" ]] ; then - [[ "$line" =~ ^${prefix}[^#] ]] && return - echo $line - elif [[ "$line" =~ ^${prefix}${key}$ ]]; then - if [[ -n "${1:-}" ]]; then - prefix+="#" - key=$1 - shift - else - founded=1 - fi - fi - done +# This function provide a way to clean some residual of installation that not managed by remove script. +# +# It prints a warning to inform that the script was failed, and execute the ynh_clean_setup function if used in the app script +# +# Requires YunoHost version 2.6.4 or higher. +ynh_exit_properly () { + local exit_code=$? + if [ "$exit_code" -eq 0 ]; then + exit 0 # Exit without error if the script ended correctly + fi + + trap '' EXIT # Ignore new exit signals + set +eu # Do not exit anymore if a command fail or if a variable is empty + + ynh_print_err --message="!!\n $app's script has encountered an error. Its execution was cancelled.\n!!" + + if type -t ynh_clean_setup > /dev/null; then # Check if the function exist in the app script. + ynh_clean_setup # Call the function to do specific cleaning for the app. + fi + + ynh_die # Exit with error status +} + +# Exits if an error occurs during the execution of the script. +# +# usage: ynh_abort_if_errors +# +# This configure the rest of the script execution such that, if an error occurs +# or if an empty variable is used, the execution of the script stops +# immediately and a call to `ynh_clean_setup` is triggered if it has been +# defined by your script. +# +# Requires YunoHost version 2.6.4 or higher. +ynh_abort_if_errors () { + set -eu # Exit if a command fail, and if a variable is used unset. + trap ynh_exit_properly EXIT # Capturing exit signals on shell script } # Download, check integrity, uncompress and patch the source from app.src @@ -238,6 +259,16 @@ ynh_render_template() { ).render(os.environ));' < $template_path > $output_path } +# Fetch the Debian release codename +# +# usage: ynh_get_debian_release +# | ret: The Debian release codename (i.e. jessie, stretch, ...) +# +# Requires YunoHost version 2.7.12 or higher. +ynh_get_debian_release () { + echo $(lsb_release --codename --short) +} + # Create a directory under /tmp # # [internal] @@ -298,3 +329,149 @@ ynh_secure_remove () { fi fi } + +# Extract a key from a plain command output +# +# example: yunohost user info tata --output-as plain | ynh_get_plain_key mail +# +# usage: ynh_get_plain_key key [subkey [subsubkey ...]] +# | ret: string - the key's value +# +# Requires YunoHost version 2.2.4 or higher. +ynh_get_plain_key() { + local prefix="#" + local founded=0 + local key=$1 + shift + while read line; do + if [[ "$founded" == "1" ]] ; then + [[ "$line" =~ ^${prefix}[^#] ]] && return + echo $line + elif [[ "$line" =~ ^${prefix}${key}$ ]]; then + if [[ -n "${1:-}" ]]; then + prefix+="#" + key=$1 + shift + else + founded=1 + fi + fi + done +} + +# Read the value of a key in a ynh manifest file +# +# usage: ynh_read_manifest manifest key +# | arg: -m, --manifest= - Path of the manifest to read +# | arg: -k, --key= - Name of the key to find +# +# Requires YunoHost version 3.?.? or higher. +ynh_read_manifest () { + # Declare an array to define the options of this helper. + local legacy_args=mk + declare -Ar args_array=( [m]=manifest= [k]=manifest_key= ) + local manifest + local manifest_key + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if [ ! -e "$manifest" ]; then + # If the manifest isn't found, try the common place for backup and restore script. + manifest="../settings/manifest.json" + fi + + jq ".$manifest_key" "$manifest" --raw-output +} + +# Read the upstream version from the manifest +# The version number in the manifest is defined by ~ynh +# For example : 4.3-2~ynh3 +# This include the number before ~ynh +# In the last example it return 4.3-2 +# +# usage: ynh_app_upstream_version [-m manifest] +# | arg: -m, --manifest= - Path of the manifest to read +# +# Requires YunoHost version 3.?.? or higher. +ynh_app_upstream_version () { + # Declare an array to define the options of this helper. + local legacy_args=m + declare -Ar args_array=( [m]=manifest= ) + local manifest + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + manifest="${manifest:-../manifest.json}" + version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") + echo "${version_key/~ynh*/}" +} + +# Read package version from the manifest +# The version number in the manifest is defined by ~ynh +# For example : 4.3-2~ynh3 +# This include the number after ~ynh +# In the last example it return 3 +# +# usage: ynh_app_package_version [-m manifest] +# | arg: -m, --manifest= - Path of the manifest to read +# +# Requires YunoHost version 3.?.? or higher. +ynh_app_package_version () { + # Declare an array to define the options of this helper. + local legacy_args=m + declare -Ar args_array=( [m]=manifest= ) + local manifest + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + manifest="${manifest:-../manifest.json}" + version_key=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version") + echo "${version_key/*~ynh/}" +} + +# Checks the app version to upgrade with the existing app version and returns: +# - UPGRADE_APP if the upstream app version has changed +# - UPGRADE_PACKAGE if only the YunoHost package has changed +# +# It stops the current script without error if the package is up-to-date +# +# This helper should be used to avoid an upgrade of an app, or the upstream part +# of it, when it's not needed +# +# To force an upgrade, even if the package is up to date, +# you have to set the variable YNH_FORCE_UPGRADE before. +# example: sudo YNH_FORCE_UPGRADE=1 yunohost app upgrade MyApp +# +# usage: ynh_check_app_version_changed +# +# Requires YunoHost version 3.?.? or higher. +ynh_check_app_version_changed () { + local force_upgrade=${YNH_FORCE_UPGRADE:-0} + local package_check=${PACKAGE_CHECK_EXEC:-0} + + # By default, upstream app version has changed + local return_value="UPGRADE_APP" + + local current_version=$(ynh_read_manifest --manifest="/etc/yunohost/apps/$YNH_APP_INSTANCE_NAME/manifest.json" --manifest_key="version" || echo 1.0) + local current_upstream_version="$(ynh_app_upstream_version --manifest="/etc/yunohost/apps/$YNH_APP_INSTANCE_NAME/manifest.json")" + local update_version=$(ynh_read_manifest --manifest="../manifest.json" --manifest_key="version" || echo 1.0) + local update_upstream_version="$(ynh_app_upstream_version)" + + if [ "$current_version" == "$update_version" ] ; then + # Complete versions are the same + if [ "$force_upgrade" != "0" ] + then + ynh_print_info --message="Upgrade forced by YNH_FORCE_UPGRADE." + unset YNH_FORCE_UPGRADE + elif [ "$package_check" != "0" ] + then + ynh_print_info --message="Upgrade forced for package check." + else + ynh_die "Up-to-date, nothing to do" 0 + fi + elif [ "$current_upstream_version" == "$update_upstream_version" ] ; then + # Upstream versions are the same, only YunoHost package versions differ + return_value="UPGRADE_PACKAGE" + fi + echo $return_value +} From 81be638e805d852b89fcc5f40b9cb8a5aee75117 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 25 Apr 2019 20:51:14 +0200 Subject: [PATCH 37/42] Unecessary source of getopts ? --- data/helpers.d/backup | 2 -- 1 file changed, 2 deletions(-) diff --git a/data/helpers.d/backup b/data/helpers.d/backup index 2e08a5417..ee524ef7f 100644 --- a/data/helpers.d/backup +++ b/data/helpers.d/backup @@ -1,7 +1,5 @@ #!/bin/bash -source /usr/share/yunohost/helpers.d/getopts - CAN_BIND=${CAN_BIND:-1} # Add a file or a directory to the list of paths to backup From 0c08caca89008afd6b2394bacfb708785f3ff1d0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 25 Apr 2019 20:51:59 +0200 Subject: [PATCH 38/42] Rename 'print' to 'logging' --- data/helpers.d/{print => logging} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename data/helpers.d/{print => logging} (100%) diff --git a/data/helpers.d/print b/data/helpers.d/logging similarity index 100% rename from data/helpers.d/print rename to data/helpers.d/logging From 1572717034bf00afbd584a6046b6eca9ca5d9c17 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 25 Apr 2019 20:54:33 +0200 Subject: [PATCH 39/42] Merge 'debug' in 'logging' --- data/helpers.d/debug | 67 ------------------------------------------ data/helpers.d/logging | 66 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 67 deletions(-) delete mode 100644 data/helpers.d/debug diff --git a/data/helpers.d/debug b/data/helpers.d/debug deleted file mode 100644 index 7ad097dbd..000000000 --- a/data/helpers.d/debug +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash - -# Debugger for app packagers -# -# usage: ynh_debug [--message=message] [--trace=1/0] -# | arg: -m, --message= - The text to print -# | arg: -t, --trace= - Turn on or off the trace of the script. Usefull to trace nonly a small part of a script. -# -# Requires YunoHost version 3.?.? or higher. -ynh_debug () { - # Disable set xtrace for the helper itself, to not pollute the debug log - set +x - # Declare an array to define the options of this helper. - local legacy_args=mt - declare -Ar args_array=( [m]=message= [t]=trace= ) - local message - local trace - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - # Redisable xtrace, ynh_handle_getopts_args set it back - set +x - message=${message:-} - trace=${trace:-} - - if [ -n "$message" ] - then - ynh_print_log "\e[34m\e[1m[DEBUG]\e[0m ${message}" >&2 - fi - - if [ "$trace" == "1" ] - then - ynh_debug --message="Enable debugging" - set +x - # Get the current file descriptor of xtrace - old_bash_xtracefd=$BASH_XTRACEFD - # Add the current file name and the line number of any command currently running while tracing. - PS4='$(basename ${BASH_SOURCE[0]})-L${LINENO}: ' - # Force xtrace to stderr - BASH_XTRACEFD=2 - # Force stdout to stderr - exec 1>&2 - fi - if [ "$trace" == "0" ] - then - ynh_debug --message="Disable debugging" - set +x - # Put xtrace back to its original fild descriptor - BASH_XTRACEFD=$old_bash_xtracefd - # Restore stdout - exec 1>&1 - fi - # Renable set xtrace - set -x -} - -# Execute a command and print the result as debug -# -# usage: ynh_debug_exec command to execute -# usage: ynh_debug_exec "command to execute | following command" -# In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe. -# -# | arg: command - command to execute -# -# Requires YunoHost version 3.?.? or higher. -ynh_debug_exec () { - ynh_debug --message="$(eval $@)" -} diff --git a/data/helpers.d/logging b/data/helpers.d/logging index 1558e4d21..087cfc80c 100644 --- a/data/helpers.d/logging +++ b/data/helpers.d/logging @@ -277,3 +277,69 @@ ynh_script_progression () { ynh_print_info "[$progression_bar] > ${message}${print_exec_time}" } + +# Debugger for app packagers +# +# usage: ynh_debug [--message=message] [--trace=1/0] +# | arg: -m, --message= - The text to print +# | arg: -t, --trace= - Turn on or off the trace of the script. Usefull to trace nonly a small part of a script. +# +# Requires YunoHost version 3.?.? or higher. +ynh_debug () { + # Disable set xtrace for the helper itself, to not pollute the debug log + set +x + # Declare an array to define the options of this helper. + local legacy_args=mt + declare -Ar args_array=( [m]=message= [t]=trace= ) + local message + local trace + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + # Redisable xtrace, ynh_handle_getopts_args set it back + set +x + message=${message:-} + trace=${trace:-} + + if [ -n "$message" ] + then + ynh_print_log "\e[34m\e[1m[DEBUG]\e[0m ${message}" >&2 + fi + + if [ "$trace" == "1" ] + then + ynh_debug --message="Enable debugging" + set +x + # Get the current file descriptor of xtrace + old_bash_xtracefd=$BASH_XTRACEFD + # Add the current file name and the line number of any command currently running while tracing. + PS4='$(basename ${BASH_SOURCE[0]})-L${LINENO}: ' + # Force xtrace to stderr + BASH_XTRACEFD=2 + # Force stdout to stderr + exec 1>&2 + fi + if [ "$trace" == "0" ] + then + ynh_debug --message="Disable debugging" + set +x + # Put xtrace back to its original fild descriptor + BASH_XTRACEFD=$old_bash_xtracefd + # Restore stdout + exec 1>&1 + fi + # Renable set xtrace + set -x +} + +# Execute a command and print the result as debug +# +# usage: ynh_debug_exec command to execute +# usage: ynh_debug_exec "command to execute | following command" +# In case of use of pipes, you have to use double quotes. Otherwise, this helper will be executed with the first command, then be sent to the next pipe. +# +# | arg: command - command to execute +# +# Requires YunoHost version 3.?.? or higher. +ynh_debug_exec () { + ynh_debug --message="$(eval $@)" +} From 719c5c49c9e4596db72bc3737aab23cc3169b4f9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 25 Apr 2019 20:57:02 +0200 Subject: [PATCH 40/42] 'psql' -> 'postgresql' --- data/helpers.d/{psql => postgresql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename data/helpers.d/{psql => postgresql} (100%) diff --git a/data/helpers.d/psql b/data/helpers.d/postgresql similarity index 100% rename from data/helpers.d/psql rename to data/helpers.d/postgresql From 9b08afc11862f6e7ec2c13f7dcc1a24f93a39b3f Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 28 Apr 2019 03:43:32 +0200 Subject: [PATCH 41/42] [mod] small refactoring, remove useless exception --- src/yunohost/tools.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 36ed62642..b5533a6ae 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -347,11 +347,8 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False, '/home/yunohost.app' ] - for folder in folders_to_create: - try: - os.listdir(folder) - except OSError: - os.makedirs(folder) + for folder in filter(not os.path.exists, folders_to_create): + os.makedirs(folder) # Change folders permissions os.system('chmod 755 /home/yunohost.app') From cecaee4f13d70befd6881e37bb8aee26d4a5e17f Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 29 Apr 2019 00:22:24 +0200 Subject: [PATCH 42/42] [fix] meh, it's not haskell --- src/yunohost/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index b5533a6ae..4ca594cf1 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -347,7 +347,7 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False, '/home/yunohost.app' ] - for folder in filter(not os.path.exists, folders_to_create): + for folder in filter(lambda x: not os.path.exists(x), folders_to_create): os.makedirs(folder) # Change folders permissions