From 7523ef3aed280553d6bd15b1bdf0e4f9d08ded07 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 15 May 2019 20:00:37 +0200 Subject: [PATCH 01/88] Remove unecessary log messages --- data/helpers.d/mysql | 1 - data/helpers.d/postgresql | 2 -- data/helpers.d/systemd | 2 -- data/helpers.d/user | 2 -- 4 files changed, 7 deletions(-) diff --git a/data/helpers.d/mysql b/data/helpers.d/mysql index 39f93891c..372819025 100644 --- a/data/helpers.d/mysql +++ b/data/helpers.d/mysql @@ -225,7 +225,6 @@ ynh_mysql_remove_db () { local mysql_root_password=$(sudo cat $MYSQL_ROOT_PWD_FILE) if mysqlshow -u root -p$mysql_root_password | grep -q "^| $db_name"; then # Check if the database exists - ynh_print_info --message="Removing database $db_name" ynh_mysql_drop_db $db_name # Remove the database else ynh_print_warn --message="Database $db_name not found" diff --git a/data/helpers.d/postgresql b/data/helpers.d/postgresql index 8e3297458..fcf2b806e 100644 --- a/data/helpers.d/postgresql +++ b/data/helpers.d/postgresql @@ -244,7 +244,6 @@ ynh_psql_remove_db() { local psql_root_password=$(sudo cat $PSQL_ROOT_PWD_FILE) if ynh_psql_database_exists --database=$db_name; then # Check if the database exists - ynh_print_info --message="Removing database $db_name" ynh_psql_drop_db $db_name # Remove the database else ynh_print_warn --message="Database $db_name not found" @@ -252,7 +251,6 @@ ynh_psql_remove_db() { # Remove psql user if it exists if ynh_psql_user_exists --user=$db_user; then - ynh_print_info --message="Removing user $db_user" ynh_psql_drop_user $db_user else ynh_print_warn --message="User $db_user not found" diff --git a/data/helpers.d/systemd b/data/helpers.d/systemd index c4100bf8a..c3a8da822 100644 --- a/data/helpers.d/systemd +++ b/data/helpers.d/systemd @@ -120,8 +120,6 @@ ynh_systemd_action() { fi fi - ynh_print_info --message="${action^} the service $service_name" - # Use reload-or-restart instead of reload. So it wouldn't fail if the service isn't running. if [ "$action" == "reload" ]; then action="reload-or-restart" diff --git a/data/helpers.d/user b/data/helpers.d/user index 0c4591dcd..e7890ccb2 100644 --- a/data/helpers.d/user +++ b/data/helpers.d/user @@ -145,7 +145,6 @@ ynh_system_user_delete () { # Check if the user exists on the system if ynh_system_user_exists "$username" then - ynh_print_info --message="Remove the user $username" deluser $username else ynh_print_warn --message="The user $username was not found" @@ -154,7 +153,6 @@ ynh_system_user_delete () { # Check if the group exists on the system if ynh_system_group_exists "$username" then - ynh_print_info --message="Remove the group $username" delgroup $username fi } From 5d2d194dc62f065b290009469dc44dcf91cecd7d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 18 May 2019 16:37:27 +0200 Subject: [PATCH 02/88] [fix] Stupid issue with files inside tar : foo is not the same as ./foo ... --- src/yunohost/backup.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 82a778491..63a5ec6fc 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -1850,11 +1850,19 @@ class TarBackupMethod(BackupMethod): # Mount the tarball logger.debug(m18n.n("restore_extracting")) tar = tarfile.open(self._archive_file, "r:gz") - tar.extract('info.json', path=self.work_dir) - try: + if "info.json" in tar.getnames(): + leading_dot = "" + tar.extract('info.json', path=self.work_dir) + elif "./info.json" in tar.getnames(): + leading_dot = "./" + tar.extract('./info.json', path=self.work_dir) + + if "backup.csv" in tar.getnames(): tar.extract('backup.csv', path=self.work_dir) - except KeyError: + elif "./backup.csv" in tar.getnames(): + tar.extract('./backup.csv', path=self.work_dir) + else: # Old backup archive have no backup.csv file pass @@ -1876,12 +1884,12 @@ class TarBackupMethod(BackupMethod): system_part = system_part.replace("_", "/") + "/" subdir_and_files = [ tarinfo for tarinfo in tar.getmembers() - if tarinfo.name.startswith(system_part) + if tarinfo.name.startswith(leading_dot+system_part) ] tar.extractall(members=subdir_and_files, path=self.work_dir) subdir_and_files = [ tarinfo for tarinfo in tar.getmembers() - if tarinfo.name.startswith("hooks/restore/") + if tarinfo.name.startswith(leading_dot+"hooks/restore/") ] tar.extractall(members=subdir_and_files, path=self.work_dir) @@ -1889,7 +1897,7 @@ class TarBackupMethod(BackupMethod): for app in apps_targets: subdir_and_files = [ tarinfo for tarinfo in tar.getmembers() - if tarinfo.name.startswith("apps/" + app) + if tarinfo.name.startswith(leading_dot+"apps/" + app) ] tar.extractall(members=subdir_and_files, path=self.work_dir) @@ -2230,7 +2238,12 @@ def backup_info(name, with_details=False, human_readable=False): tar = tarfile.open(archive_file, "r:gz") info_dir = info_file + '.d' try: - tar.extract('info.json', path=info_dir) + if "info.json" in tar.getnames(): + tar.extract('info.json', path=info_dir) + elif "./info.json" in tar.getnames(): + tar.extract('./info.json', path=info_dir) + else: + raise KeyError except KeyError: logger.debug("unable to retrieve '%s' inside the archive", info_file, exc_info=1) From bd43e5cde388df27cf0cbaea7a7abb96b0350551 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Sat, 25 May 2019 15:13:24 +0200 Subject: [PATCH 03/88] add current version in app_info --- src/yunohost/app.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 4ccfb6b47..a88a8dbb7 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -362,6 +362,12 @@ def app_info(app, show_status=False, raw=False): ret['upgradable'] = upgradable ret['change_url'] = os.path.exists(os.path.join(app_setting_path, "scripts", "change_url")) + + with open(os.path.join(APPS_SETTING_PATH, app, 'manifest.json')) as json_manifest: + manifest = json.load(json_manifest) + + if 'version' in manifest: + ret['version'] = manifest.get('version', '-') return ret From 97695f6afb91e4e32a163b4a7aa1176f17f90c00 Mon Sep 17 00:00:00 2001 From: Kayou Date: Tue, 28 May 2019 17:51:05 +0200 Subject: [PATCH 04/88] delete an unnecessary "if" --- src/yunohost/app.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index a88a8dbb7..1f172a267 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -366,8 +366,7 @@ def app_info(app, show_status=False, raw=False): with open(os.path.join(APPS_SETTING_PATH, app, 'manifest.json')) as json_manifest: manifest = json.load(json_manifest) - if 'version' in manifest: - ret['version'] = manifest.get('version', '-') + ret['version'] = manifest.get('version', '-') return ret From 68c30a943a9ef06c58553e3982226be19bef6c8d Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Wed, 29 May 2019 12:57:31 +0200 Subject: [PATCH 05/88] Add exception for data backup --- data/helpers.d/backup | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/data/helpers.d/backup b/data/helpers.d/backup index ee524ef7f..b347f1700 100644 --- a/data/helpers.d/backup +++ b/data/helpers.d/backup @@ -59,12 +59,18 @@ ynh_backup() { local not_mandatory="${not_mandatory:-0}" BACKUP_CORE_ONLY=${BACKUP_CORE_ONLY:-0} + test -n "${app:-}" && do_not_backup_data=$(ynh_app_setting_get $app do_not_backup_data) # If backing up core only (used by ynh_backup_before_upgrade), # don't backup big data items - if [ "$is_big" == "1" ] && [ "$BACKUP_CORE_ONLY" == "1" ] ; then - ynh_print_info --message="$src_path will not be saved, because backup_core_only is set." - return 0 + if [ $is_big -eq 1 ] && ( [ ${do_not_backup_data:-0} -eq 1 ] || [ $BACKUP_CORE_ONLY -eq 1 ] ) + then + if [ $BACKUP_CORE_ONLY -eq 1 ]; then + ynh_print_warn --message="$src_path will not be saved, because 'BACKUP_CORE_ONLY' is set." + else + ynh_print_warn --message="$src_path will not be saved, because 'do_not_backup_data' is set." + fi + return 0 fi # ============================================================================== From 4084bddb54acb8f230b73182d8efac68ad1f169c Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 1 Jun 2019 01:44:33 +0200 Subject: [PATCH 06/88] [enh] support config_panel in TOML format --- debian/control | 1 + src/yunohost/app.py | 140 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 131 insertions(+), 10 deletions(-) diff --git a/debian/control b/debian/control index f19ddd465..b17be5148 100644 --- a/debian/control +++ b/debian/control @@ -14,6 +14,7 @@ Depends: ${python:Depends}, ${misc:Depends} , moulinette (>= 2.7.1), ssowat (>= 2.7.1) , python-psutil, python-requests, python-dnspython, python-openssl , python-apt, python-miniupnpc, python-dbus, python-jinja2 + , python-toml , glances, apt-transport-https , dnsutils, bind9utils, unzip, git, curl, cron, wget, jq , ca-certificates, netcat-openbsd, iproute diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 4ccfb6b47..4323a5410 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -24,6 +24,7 @@ Manage apps """ import os +import toml import json import shutil import yaml @@ -680,7 +681,7 @@ def app_upgrade(app=[], url=None, file=None): os.system('rm -rf "%s/scripts" "%s/manifest.json %s/conf"' % (app_setting_path, app_setting_path, app_setting_path)) os.system('mv "%s/manifest.json" "%s/scripts" %s' % (extracted_app_folder, extracted_app_folder, app_setting_path)) - for file_to_copy in ["actions.json", "config_panel.json", "conf"]: + for file_to_copy in ["actions.json", "config_panel.json", "config_panel.toml", "conf"]: if os.path.exists(os.path.join(extracted_app_folder, file_to_copy)): os.system('cp -R %s/%s %s' % (extracted_app_folder, file_to_copy, app_setting_path)) @@ -835,7 +836,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu os.system('cp %s/manifest.json %s' % (extracted_app_folder, app_setting_path)) os.system('cp -R %s/scripts %s' % (extracted_app_folder, app_setting_path)) - for file_to_copy in ["actions.json", "config_panel.json", "conf"]: + for file_to_copy in ["actions.json", "config_panel.json", "config_panel.toml", "conf"]: if os.path.exists(os.path.join(extracted_app_folder, file_to_copy)): os.system('cp -R %s/%s %s' % (extracted_app_folder, file_to_copy, app_setting_path)) @@ -1581,12 +1582,12 @@ def app_config_show_panel(app): # this will take care of checking if the app is installed app_info_dict = app_info(app) - config_panel = os.path.join(APPS_SETTING_PATH, app, 'config_panel.json') + config_panel = _get_app_config_panel(app) config_script = os.path.join(APPS_SETTING_PATH, app, 'scripts', 'config') app_id, app_instance_nb = _parse_app_instance_name(app) - if not os.path.exists(config_panel) or not os.path.exists(config_script): + if not config_panel or not os.path.exists(config_script): return { "app_id": app_id, "app": app, @@ -1594,8 +1595,6 @@ def app_config_show_panel(app): "config_panel": [], } - config_panel = read_json(config_panel) - env = { "YNH_APP_ID": app_id, "YNH_APP_INSTANCE_NAME": app, @@ -1672,15 +1671,13 @@ def app_config_apply(app, args): if not installed: raise YunohostError('app_not_installed', app=app) - config_panel = os.path.join(APPS_SETTING_PATH, app, 'config_panel.json') + config_panel = _get_app_config_panel(app) config_script = os.path.join(APPS_SETTING_PATH, app, 'scripts', 'config') - if not os.path.exists(config_panel) or not os.path.exists(config_script): + if not config_panel or not os.path.exists(config_script): # XXX real exception raise Exception("Not config-panel.json nor scripts/config") - config_panel = read_json(config_panel) - app_id, app_instance_nb = _parse_app_instance_name(app) env = { "YNH_APP_ID": app_id, @@ -1719,6 +1716,129 @@ def app_config_apply(app, args): logger.success("Config updated as expected") +def _get_app_config_panel(app_id): + "Get app config panel stored in json or in toml" + config_panel_toml_path = os.path.join(APPS_SETTING_PATH, app_id, 'config_panel.toml') + config_panel_json_path = os.path.join(APPS_SETTING_PATH, app_id, 'config_panel.json') + + # sample data to get an idea of what is going on + # this toml extract: + # + # version = "0.1" + # name = "Unattended-upgrades configuration panel" + # + # [main] + # name = "Unattended-upgrades configuration" + # + # [main.unattended_configuration] + # name = "50unattended-upgrades configuration file" + # + # [main.unattended_configuration.upgrade_level] + # name = "Choose the sources of packages to automatically upgrade." + # default = "Security only" + # type = "text" + # help = "We can't use a choices field for now. In the meantime please choose between one of this values:
Security only, Security and updates." + # # choices = ["Security only", "Security and updates"] + + # [main.unattended_configuration.ynh_update] + # name = "Would you like to update YunoHost packages automatically ?" + # type = "bool" + # default = true + # + # will be parsed into this: + # + # OrderedDict([(u'version', u'0.1'), + # (u'name', u'Unattended-upgrades configuration panel'), + # (u'main', + # OrderedDict([(u'name', u'Unattended-upgrades configuration'), + # (u'unattended_configuration', + # OrderedDict([(u'name', + # u'50unattended-upgrades configuration file'), + # (u'upgrade_level', + # OrderedDict([(u'name', + # u'Choose the sources of packages to automatically upgrade.'), + # (u'default', + # u'Security only'), + # (u'type', u'text'), + # (u'help', + # u"We can't use a choices field for now. In the meantime please choose between one of this values:
Security only, Security and updates.")])), + # (u'ynh_update', + # OrderedDict([(u'name', + # u'Would you like to update YunoHost packages automatically ?'), + # (u'type', u'bool'), + # (u'default', True)])), + # + # and needs to be converted into this: + # + # {u'name': u'Unattended-upgrades configuration panel', + # u'panel': [{u'id': u'main', + # u'name': u'Unattended-upgrades configuration', + # u'sections': [{u'id': u'unattended_configuration', + # u'name': u'50unattended-upgrades configuration file', + # u'options': [{u'//': u'"choices" : ["Security only", "Security and updates"]', + # u'default': u'Security only', + # u'help': u"We can't use a choices field for now. In the meantime please choose between one of this values:
Security only, Security and updates.", + # u'id': u'upgrade_level', + # u'name': u'Choose the sources of packages to automatically upgrade.', + # u'type': u'text'}, + # {u'default': True, + # u'id': u'ynh_update', + # u'name': u'Would you like to update YunoHost packages automatically ?', + # u'type': u'bool'}, + + if os.path.exists(config_panel_toml_path): + toml_config_panel = toml.load(open(config_panel_toml_path, "r"), _dict=OrderedDict) + + # transform toml format into json format + config_panel = { + "name": toml_config_panel["name"], + "version": toml_config_panel["version"], + "panel": [], + } + + panels = filter(lambda (key, value): key not in ("name", "version") + and isinstance(value, OrderedDict), + toml_config_panel.items()) + + for key, value in panels: + panel = { + "id": key, + "name": value["name"], + "sections": [], + } + + sections = filter(lambda (k, v): k not in ("name",) + and isinstance(v, OrderedDict), + value.items()) + + for section_key, section_value in sections: + section = { + "id": section_key, + "name": section_value["name"], + "options": [], + } + + options = filter(lambda (k, v): k not in ("name",) + and isinstance(v, OrderedDict), + section_value.items()) + + for option_key, option_value in options: + option = dict(option_value) + option["id"] = option_key + section["options"].append(option) + + panel["sections"].append(section) + + config_panel["panel"].append(panel) + + return config_panel + + elif os.path.exists(config_panel_json_path): + return json.load(open(config_panel_json_path)) + + return None + + def _get_app_settings(app_id): """ Get settings of an installed app From dcc493934c0548f76edf4a44db47ee1b04a92121 Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Sat, 1 Jun 2019 22:47:22 +0200 Subject: [PATCH 07/88] Fix ynh_print_OFF when set -x is used in other helpers --- data/helpers.d/logging | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/logging b/data/helpers.d/logging index c46000509..9e8cc06cb 100644 --- a/data/helpers.d/logging +++ b/data/helpers.d/logging @@ -172,7 +172,7 @@ ynh_exec_fully_quiet () { # # Requires YunoHost version 3.2.0 or higher. ynh_print_OFF () { - set +x + exec {BASH_XTRACEFD}>/dev/null } # Restore the logging after ynh_print_OFF @@ -181,7 +181,7 @@ ynh_print_OFF () { # # Requires YunoHost version 3.2.0 or higher. ynh_print_ON () { - set -x + exec {BASH_XTRACEFD}>&1 # Print an echo only for the log, to be able to know that ynh_print_ON has been called. echo ynh_print_ON > /dev/null } From 58a03fc53ce0f1caaadbdd2812ce3144315fa1c2 Mon Sep 17 00:00:00 2001 From: toitoinebzh Date: Thu, 14 Mar 2019 00:22:18 +0100 Subject: [PATCH 08/88] Script to generate manpages - Fix #284 --- manpages_auto.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 manpages_auto.py diff --git a/manpages_auto.py b/manpages_auto.py new file mode 100644 index 000000000..077aba963 --- /dev/null +++ b/manpages_auto.py @@ -0,0 +1,43 @@ + GNU nano 2.7.4 Fichier : manpages_auto.py + +""" +Inspired by yunohost_completion.py (author: Christophe Vuillot) +======= + +This script generates man pages for yunohost. +Pages are stored in OUTPUT_DIR +""" + +import yaml +import os + +ACTIONSMAP_FILE = '/usr/share/moulinette/actionsmap/yunohost.yml' +OUTPUT_DIR ="output/" + +# creates output directory +os.system("mkdir "+ OUTPUT_DIR ) + + +# man page of yunohost +cmd = "sudo help2man \" yunohost \" -o " + OUTPUT_DIR + "yunohost" +print(cmd) +os.system(cmd) + +# man pages of "yunohost *" +with open(ACTIONSMAP_FILE, 'r') as stream: + + # Getting the dictionary containning what actions are possible per domain + OPTION_TREE = yaml.load(stream) + DOMAINS = [str for str in OPTION_TREE.keys() if not str.startswith('_')] + DOMAINS_STR = '"{}"'.format(' '.join(DOMAINS)) + ACTIONS_DICT = {} + for domain in DOMAINS: + ACTIONS = [str for str in OPTION_TREE[domain]['actions'].keys() + if not str.startswith('_')] + ACTIONS_STR = '"{}"'.format(' '.join(ACTIONS)) + ACTIONS_DICT[domain] = ACTIONS_STR + for action in ACTIONS: + #print("yunohost", domain, action) + cmd = "sudo help2man \" yunohost " + domain + " " + action + " --help \" -o " +$ + print(cmd) + os.system(cmd) From 934d15cb5106a6b9a391a964812dc52a46f3f6a5 Mon Sep 17 00:00:00 2001 From: toitoinebzh <44553182+toitoinebzh@users.noreply.github.com> Date: Thu, 14 Mar 2019 00:32:57 +0100 Subject: [PATCH 09/88] Correct some typo --- manpages_auto.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/manpages_auto.py b/manpages_auto.py index 077aba963..d6aedb7e2 100644 --- a/manpages_auto.py +++ b/manpages_auto.py @@ -1,5 +1,3 @@ - GNU nano 2.7.4 Fichier : manpages_auto.py - """ Inspired by yunohost_completion.py (author: Christophe Vuillot) ======= @@ -38,6 +36,6 @@ with open(ACTIONSMAP_FILE, 'r') as stream: ACTIONS_DICT[domain] = ACTIONS_STR for action in ACTIONS: #print("yunohost", domain, action) - cmd = "sudo help2man \" yunohost " + domain + " " + action + " --help \" -o " +$ + cmd = "sudo help2man \" yunohost " + domain + " " + action + " --help \" -o " + OUTPUT_DIR + "yunohost_" + domain+ "_" + action print(cmd) os.system(cmd) From 1e95a5d7381fd63f9957028fe04d03e4f5d3b262 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 22 Apr 2019 19:07:10 +0200 Subject: [PATCH 10/88] Move / rename / make script more robust w.r.t to path from which its called --- manpages_auto.py => doc/generate_manpages.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) rename manpages_auto.py => doc/generate_manpages.py (86%) diff --git a/manpages_auto.py b/doc/generate_manpages.py similarity index 86% rename from manpages_auto.py rename to doc/generate_manpages.py index d6aedb7e2..56cc974d4 100644 --- a/manpages_auto.py +++ b/doc/generate_manpages.py @@ -9,7 +9,9 @@ Pages are stored in OUTPUT_DIR import yaml import os -ACTIONSMAP_FILE = '/usr/share/moulinette/actionsmap/yunohost.yml' + +THIS_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +ACTIONSMAP_FILE = THIS_SCRIPT_DIR + '/../data/actionsmap/yunohost.yml' OUTPUT_DIR ="output/" # creates output directory @@ -17,7 +19,7 @@ os.system("mkdir "+ OUTPUT_DIR ) # man page of yunohost -cmd = "sudo help2man \" yunohost \" -o " + OUTPUT_DIR + "yunohost" +cmd = "sudo help2man \" yunohost \" -o " + OUTPUT_DIR + "yunohost" print(cmd) os.system(cmd) @@ -36,6 +38,6 @@ with open(ACTIONSMAP_FILE, 'r') as stream: ACTIONS_DICT[domain] = ACTIONS_STR for action in ACTIONS: #print("yunohost", domain, action) - cmd = "sudo help2man \" yunohost " + domain + " " + action + " --help \" -o " + OUTPUT_DIR + "yunohost_" + domain+ "_" + action + cmd = "sudo help2man \" yunohost " + domain + " " + action + " --help \" -o " + OUTPUT_DIR + "yunohost_" + domain+ "_" + action print(cmd) - os.system(cmd) + os.system(cmd) From 218029b6283974d98f0a79330d227fd688166861 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 28 Apr 2019 03:48:13 +0200 Subject: [PATCH 11/88] [mod] autopep8 --- doc/generate_manpages.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/generate_manpages.py b/doc/generate_manpages.py index 56cc974d4..eb9b0df53 100644 --- a/doc/generate_manpages.py +++ b/doc/generate_manpages.py @@ -12,10 +12,10 @@ import os THIS_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) ACTIONSMAP_FILE = THIS_SCRIPT_DIR + '/../data/actionsmap/yunohost.yml' -OUTPUT_DIR ="output/" +OUTPUT_DIR = "output/" # creates output directory -os.system("mkdir "+ OUTPUT_DIR ) +os.system("mkdir " + OUTPUT_DIR) # man page of yunohost @@ -37,7 +37,7 @@ with open(ACTIONSMAP_FILE, 'r') as stream: ACTIONS_STR = '"{}"'.format(' '.join(ACTIONS)) ACTIONS_DICT[domain] = ACTIONS_STR for action in ACTIONS: - #print("yunohost", domain, action) - cmd = "sudo help2man \" yunohost " + domain + " " + action + " --help \" -o " + OUTPUT_DIR + "yunohost_" + domain+ "_" + action + # print("yunohost", domain, action) + cmd = "sudo help2man \" yunohost " + domain + " " + action + " --help \" -o " + OUTPUT_DIR + "yunohost_" + domain + "_" + action print(cmd) os.system(cmd) From e23604e763cf6a35cbc46c4d94fe4a8f4d483384 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 28 Apr 2019 03:49:59 +0200 Subject: [PATCH 12/88] [mod] use os.path --- doc/generate_manpages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/generate_manpages.py b/doc/generate_manpages.py index eb9b0df53..c25b01ef6 100644 --- a/doc/generate_manpages.py +++ b/doc/generate_manpages.py @@ -6,12 +6,12 @@ This script generates man pages for yunohost. Pages are stored in OUTPUT_DIR """ -import yaml import os +import yaml THIS_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -ACTIONSMAP_FILE = THIS_SCRIPT_DIR + '/../data/actionsmap/yunohost.yml' +ACTIONSMAP_FILE = os.path.join(THIS_SCRIPT_DIR, '../data/actionsmap/yunohost.yml') OUTPUT_DIR = "output/" # creates output directory From 51f9dbe5aeae2eeda414534a3fc97dd0252338ce Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 28 Apr 2019 03:50:14 +0200 Subject: [PATCH 13/88] [mod] use builtin python function for creating dirs --- doc/generate_manpages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/generate_manpages.py b/doc/generate_manpages.py index c25b01ef6..85c6812cc 100644 --- a/doc/generate_manpages.py +++ b/doc/generate_manpages.py @@ -15,7 +15,8 @@ ACTIONSMAP_FILE = os.path.join(THIS_SCRIPT_DIR, '../data/actionsmap/yunohost.yml OUTPUT_DIR = "output/" # creates output directory -os.system("mkdir " + OUTPUT_DIR) +if not os.path.exists(OUTPUT_DIR): + os.makedirs(OUTPUT_DIR) # man page of yunohost From 31482e7d3129beebc4958829bbb2b478672b468b Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 28 Apr 2019 03:50:49 +0200 Subject: [PATCH 14/88] [mod] use subprocess instead of os.system to ensure function has run correctly --- doc/generate_manpages.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/generate_manpages.py b/doc/generate_manpages.py index 85c6812cc..408ff883d 100644 --- a/doc/generate_manpages.py +++ b/doc/generate_manpages.py @@ -8,6 +8,7 @@ Pages are stored in OUTPUT_DIR import os import yaml +import subprocess THIS_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -22,7 +23,7 @@ if not os.path.exists(OUTPUT_DIR): # man page of yunohost cmd = "sudo help2man \" yunohost \" -o " + OUTPUT_DIR + "yunohost" print(cmd) -os.system(cmd) +subprocess.check_call(cmd, shell=True) # man pages of "yunohost *" with open(ACTIONSMAP_FILE, 'r') as stream: @@ -41,4 +42,4 @@ with open(ACTIONSMAP_FILE, 'r') as stream: # print("yunohost", domain, action) cmd = "sudo help2man \" yunohost " + domain + " " + action + " --help \" -o " + OUTPUT_DIR + "yunohost_" + domain + "_" + action print(cmd) - os.system(cmd) + subprocess.check_call(cmd, shell=True) From 6cceddf70c33eca3db3a9b22084651498c2779a6 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 28 Apr 2019 03:51:25 +0200 Subject: [PATCH 15/88] [mod] use more secure yaml.safe_load --- doc/generate_manpages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/generate_manpages.py b/doc/generate_manpages.py index 408ff883d..d92073ca1 100644 --- a/doc/generate_manpages.py +++ b/doc/generate_manpages.py @@ -29,7 +29,7 @@ subprocess.check_call(cmd, shell=True) with open(ACTIONSMAP_FILE, 'r') as stream: # Getting the dictionary containning what actions are possible per domain - OPTION_TREE = yaml.load(stream) + OPTION_TREE = yaml.safe_load(stream) DOMAINS = [str for str in OPTION_TREE.keys() if not str.startswith('_')] DOMAINS_STR = '"{}"'.format(' '.join(DOMAINS)) ACTIONS_DICT = {} From 340f3617a25419044906f4d60706a90ab2b97a64 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 28 Apr 2019 03:51:44 +0200 Subject: [PATCH 16/88] [mod] 'str' is a python builtin --- doc/generate_manpages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/generate_manpages.py b/doc/generate_manpages.py index d92073ca1..c9399dce9 100644 --- a/doc/generate_manpages.py +++ b/doc/generate_manpages.py @@ -30,7 +30,7 @@ with open(ACTIONSMAP_FILE, 'r') as stream: # Getting the dictionary containning what actions are possible per domain OPTION_TREE = yaml.safe_load(stream) - DOMAINS = [str for str in OPTION_TREE.keys() if not str.startswith('_')] + DOMAINS = [x for x in OPTION_TREE.keys() if not x.startswith('_')] DOMAINS_STR = '"{}"'.format(' '.join(DOMAINS)) ACTIONS_DICT = {} for domain in DOMAINS: From 1fcffed58db2a8bf5127194c2fa258cce56cf065 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 28 Apr 2019 03:52:38 +0200 Subject: [PATCH 17/88] [mod] blank lines for lisibility --- doc/generate_manpages.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/generate_manpages.py b/doc/generate_manpages.py index c9399dce9..8159fe946 100644 --- a/doc/generate_manpages.py +++ b/doc/generate_manpages.py @@ -30,14 +30,19 @@ with open(ACTIONSMAP_FILE, 'r') as stream: # Getting the dictionary containning what actions are possible per domain OPTION_TREE = yaml.safe_load(stream) + DOMAINS = [x for x in OPTION_TREE.keys() if not x.startswith('_')] DOMAINS_STR = '"{}"'.format(' '.join(DOMAINS)) + ACTIONS_DICT = {} + for domain in DOMAINS: ACTIONS = [str for str in OPTION_TREE[domain]['actions'].keys() if not str.startswith('_')] + ACTIONS_STR = '"{}"'.format(' '.join(ACTIONS)) ACTIONS_DICT[domain] = ACTIONS_STR + for action in ACTIONS: # print("yunohost", domain, action) cmd = "sudo help2man \" yunohost " + domain + " " + action + " --help \" -o " + OUTPUT_DIR + "yunohost_" + domain + "_" + action From 491e3efb5a3ac6b2e13d666006269cad5842f78d Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Mon, 29 Apr 2019 23:37:44 +0200 Subject: [PATCH 18/88] [enh] generate manpages from actionsmap --- doc/generate_manpages.py | 131 ++++++++++++++++++++++++++++++++------- 1 file changed, 110 insertions(+), 21 deletions(-) diff --git a/doc/generate_manpages.py b/doc/generate_manpages.py index 8159fe946..917b16447 100644 --- a/doc/generate_manpages.py +++ b/doc/generate_manpages.py @@ -10,41 +10,130 @@ import os import yaml import subprocess +from datetime import date +from collections import OrderedDict + +from jinja2 import Template + +template = Template('''\ +.TH YunoHost "1" "{{ month }} {{ year }}" "YunoHost Collectif" +.SH NAME +YunoHost \\- yunohost server administration command +.SH SYNOPSIS +yunohost \\fI\\,CATEGORY\\/\\fR \\fI\\,COMMAND\\/\\fR [\\fI\\,SUBCOMMAND\\/\\fR] [\\fI\\,ARGUMENTS\\/\\fR]... [\\fI\\,OPTIONS\\/\\fR]... +.SH DESCRIPTION +usage: yunohost +{{ '{' }}{{ ",".join(categories) }}{{ '}' }} +\&... +[\\-h|\\-\\-help] [\\-\\-no\\-cache] [\\-\\-output\\-as {json,plain,none}] [\\-\\-debug] +[\\-\\-quiet] [\\-\\-timeout ==SUPPRESS==] [\\-\\-admin\\-password PASSWORD] +[\\-v|\\-\\-version] +.SS "optional arguments:" +.TP +\\fB\\-h\\fR, \\fB\\-\\-help\\fR +show this help message and exit +.SS "categories:" +.IP +{{ '{' }}{{ ",".join(categories) }}{{ '}' }} +{% for name, value in categories.items() %} +.TP +{{ name }} +{{ value["category_help"] }} +{% endfor %} +Manage debug logs +.SS "global arguments:" +.TP +\\fB\\-\\-no\\-cache\\fR +Don't use actions map cache +.TP +\\fB\\-\\-output\\-as\\fR {json,plain,none} +Output result in another format +.TP +\\fB\\-\\-debug\\fR +Log and print debug messages +.TP +\\fB\\-\\-quiet\\fR +Don't produce any output +.TP +\\fB\\-\\-timeout\\fR SECONDS +Number of seconds before this command will timeout +because it can't acquire the lock (meaning that +another command is currently running), by default +there is no timeout and the command will wait until it +can get the lock +.TP +\\fB\\-\\-admin\\-password\\fR PASSWORD +The admin password to use to authenticate +.TP +\\fB\\-v\\fR, \\fB\\-\\-version\\fR +Display YunoHost packages versions +{% for name, value in categories.items() %} +.SH YUNOHOST {{ name.upper() }} +usage: yunohost {{ name }} {{ '{' }}{{ ",".join(value["actions"].keys()) }}{{ '}' }} +\\&... +.SS "description:" +.IP +{{ value["category_help"] }} +{% for action, action_value in value["actions"].items() %} +.SS "yunohost {{ name }} {{ action }} \ +{% for argument_name, argument_value in action_value.get("arguments", {}).items() %}\ +{% set required=(not str(argument_name).startswith("-")) or argument_value.get("extra", {}).get("required", False) %}\ +{% if not required %}[{% endif %}\ +\\fI\\,{{ argument_name }}\\/\\fR{% if argument_value.get("full") %}|\\fI\\,{{ argument_value["full"] }}\\fR{% endif %}\ +{% if str(argument_name).startswith("-") and not argument_value.get("action") == "store_true" %} {{ (argument_value.get("full", argument_name)).lstrip("-") }}{% endif %}\ +{% if not required %}]{% endif %} \ +{% endfor %}" +{{ action_value["action_help"] }} +{% if "arguments" in action_value %} +{% for argument_name, argument_value in action_value["arguments"].items() %} +.TP +\\fB{{ argument_name }}\\fR{% if argument_value.get("full") %}, \\fB{{ argument_value["full"] }}\\fR{% endif %}\ +{% if str(argument_name).startswith("-") and not argument_value.get("action") == "store_true" %} \\fI\\,{{ (argument_value.get("full", argument_name)).lstrip("-") }}\\fR{% endif %} +{{ argument_value.get("help", "")}} +{% endfor %} +{% endif %} +{% endfor %} +{% endfor %} +''') + THIS_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) ACTIONSMAP_FILE = os.path.join(THIS_SCRIPT_DIR, '../data/actionsmap/yunohost.yml') OUTPUT_DIR = "output/" + +def ordered_yaml_load(stream): + class OrderedLoader(yaml.Loader): + pass + OrderedLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + lambda loader, node: OrderedDict(loader.construct_pairs(node))) + return yaml.load(stream, OrderedLoader) + + # creates output directory if not os.path.exists(OUTPUT_DIR): os.makedirs(OUTPUT_DIR) -# man page of yunohost -cmd = "sudo help2man \" yunohost \" -o " + OUTPUT_DIR + "yunohost" -print(cmd) -subprocess.check_call(cmd, shell=True) - # man pages of "yunohost *" -with open(ACTIONSMAP_FILE, 'r') as stream: +with open(ACTIONSMAP_FILE, 'r') as actionsmap: # Getting the dictionary containning what actions are possible per domain - OPTION_TREE = yaml.safe_load(stream) + actionsmap = ordered_yaml_load(actionsmap) - DOMAINS = [x for x in OPTION_TREE.keys() if not x.startswith('_')] - DOMAINS_STR = '"{}"'.format(' '.join(DOMAINS)) + for i in actionsmap.keys(): + if i.startswith("_"): + del actionsmap[i] - ACTIONS_DICT = {} + today = date.today() - for domain in DOMAINS: - ACTIONS = [str for str in OPTION_TREE[domain]['actions'].keys() - if not str.startswith('_')] + result = template.render( + month=today.strftime("%B"), + year=today.year, + categories=actionsmap, + str=str, + ) - ACTIONS_STR = '"{}"'.format(' '.join(ACTIONS)) - ACTIONS_DICT[domain] = ACTIONS_STR - - for action in ACTIONS: - # print("yunohost", domain, action) - cmd = "sudo help2man \" yunohost " + domain + " " + action + " --help \" -o " + OUTPUT_DIR + "yunohost_" + domain + "_" + action - print(cmd) - subprocess.check_call(cmd, shell=True) +with open(os.path.join(OUTPUT_DIR, "yunohost"), "w") as output: + output.write(result) From 2725fba1f497fb7a8f9ab402aa3760a92f91c82d Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 19 May 2019 03:17:12 +0200 Subject: [PATCH 19/88] [mod] remove unused import --- doc/generate_manpages.py | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/generate_manpages.py b/doc/generate_manpages.py index 917b16447..a8c40f754 100644 --- a/doc/generate_manpages.py +++ b/doc/generate_manpages.py @@ -8,7 +8,6 @@ Pages are stored in OUTPUT_DIR import os import yaml -import subprocess from datetime import date from collections import OrderedDict From ec67f5af61569611f153e3877e844af419357490 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 19 May 2019 03:17:18 +0200 Subject: [PATCH 20/88] [mod] add a main function --- doc/generate_manpages.py | 52 +++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/doc/generate_manpages.py b/doc/generate_manpages.py index a8c40f754..e2fc25787 100644 --- a/doc/generate_manpages.py +++ b/doc/generate_manpages.py @@ -110,29 +110,33 @@ def ordered_yaml_load(stream): return yaml.load(stream, OrderedLoader) -# creates output directory -if not os.path.exists(OUTPUT_DIR): - os.makedirs(OUTPUT_DIR) +def main(): + # creates output directory + if not os.path.exists(OUTPUT_DIR): + os.makedirs(OUTPUT_DIR) + + # man pages of "yunohost *" + with open(ACTIONSMAP_FILE, 'r') as actionsmap: + + # Getting the dictionary containning what actions are possible per domain + actionsmap = ordered_yaml_load(actionsmap) + + for i in actionsmap.keys(): + if i.startswith("_"): + del actionsmap[i] + + today = date.today() + + result = template.render( + month=today.strftime("%B"), + year=today.year, + categories=actionsmap, + str=str, + ) + + with open(os.path.join(OUTPUT_DIR, "yunohost"), "w") as output: + output.write(result) -# man pages of "yunohost *" -with open(ACTIONSMAP_FILE, 'r') as actionsmap: - - # Getting the dictionary containning what actions are possible per domain - actionsmap = ordered_yaml_load(actionsmap) - - for i in actionsmap.keys(): - if i.startswith("_"): - del actionsmap[i] - - today = date.today() - - result = template.render( - month=today.strftime("%B"), - year=today.year, - categories=actionsmap, - str=str, - ) - -with open(os.path.join(OUTPUT_DIR, "yunohost"), "w") as output: - output.write(result) +if __name__ == '__main__': + main() From d334baf78fd54600ce7b07480db9250039bce8a6 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 19 May 2019 03:37:30 +0200 Subject: [PATCH 21/88] [enh] allow to specify a path for the output --- doc/generate_manpages.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/doc/generate_manpages.py b/doc/generate_manpages.py index e2fc25787..0c7fab3c2 100644 --- a/doc/generate_manpages.py +++ b/doc/generate_manpages.py @@ -8,6 +8,7 @@ Pages are stored in OUTPUT_DIR import os import yaml +import argparse from datetime import date from collections import OrderedDict @@ -98,7 +99,6 @@ usage: yunohost {{ name }} {{ '{' }}{{ ",".join(value["actions"].keys()) }}{{ '} THIS_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) ACTIONSMAP_FILE = os.path.join(THIS_SCRIPT_DIR, '../data/actionsmap/yunohost.yml') -OUTPUT_DIR = "output/" def ordered_yaml_load(stream): @@ -111,9 +111,23 @@ def ordered_yaml_load(stream): def main(): - # creates output directory - if not os.path.exists(OUTPUT_DIR): - os.makedirs(OUTPUT_DIR) + parser = argparse.ArgumentParser(description="generate yunohost manpage based on actionsmap.yml") + parser.add_argument("-o", "--output", default="output/yunohost") + + args = parser.parse_args() + + if os.path.isdir(args.output): + if not os.path.exists(args.output): + os.makedirs(args.output) + + output_path = os.path.join(args.output, "yunohost") + else: + output_dir = os.path.split(args.output)[0] + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + output_path = args.output # man pages of "yunohost *" with open(ACTIONSMAP_FILE, 'r') as actionsmap: @@ -134,7 +148,7 @@ def main(): str=str, ) - with open(os.path.join(OUTPUT_DIR, "yunohost"), "w") as output: + with open(output_path, "w") as output: output.write(result) From 3e0dde747e5c0588b83361bac68ce41043842c3b Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 25 May 2019 17:44:10 +0200 Subject: [PATCH 22/88] [fix] invalide \ --- doc/generate_manpages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/generate_manpages.py b/doc/generate_manpages.py index 0c7fab3c2..28ac3ecbe 100644 --- a/doc/generate_manpages.py +++ b/doc/generate_manpages.py @@ -24,7 +24,7 @@ yunohost \\fI\\,CATEGORY\\/\\fR \\fI\\,COMMAND\\/\\fR [\\fI\\,SUBCOMMAND\\/\\fR] .SH DESCRIPTION usage: yunohost {{ '{' }}{{ ",".join(categories) }}{{ '}' }} -\&... +\\&... [\\-h|\\-\\-help] [\\-\\-no\\-cache] [\\-\\-output\\-as {json,plain,none}] [\\-\\-debug] [\\-\\-quiet] [\\-\\-timeout ==SUPPRESS==] [\\-\\-admin\\-password PASSWORD] [\\-v|\\-\\-version] From 3c074e637cee63d79ad141328efb15299976c261 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 26 May 2019 02:42:15 +0200 Subject: [PATCH 23/88] [enh] add support to write gzip compressed man page --- doc/generate_manpages.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/generate_manpages.py b/doc/generate_manpages.py index 28ac3ecbe..1c0586b2e 100644 --- a/doc/generate_manpages.py +++ b/doc/generate_manpages.py @@ -8,6 +8,7 @@ Pages are stored in OUTPUT_DIR import os import yaml +import gzip import argparse from datetime import date @@ -113,6 +114,7 @@ def ordered_yaml_load(stream): def main(): parser = argparse.ArgumentParser(description="generate yunohost manpage based on actionsmap.yml") parser.add_argument("-o", "--output", default="output/yunohost") + parser.add_argument("-z", "--gzip", action="store_true", default=False) args = parser.parse_args() @@ -148,8 +150,12 @@ def main(): str=str, ) - with open(output_path, "w") as output: - output.write(result) + if not args.gzip: + with open(output_path, "w") as output: + output.write(result) + else: + with gzip.open(output_path, mode="w", compresslevel=9) as output: + output.write(result) if __name__ == '__main__': From 8428385cd44083aff79402328df58a60548a6b7d Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 26 May 2019 02:42:42 +0200 Subject: [PATCH 24/88] [fix] if output is directly a filename without a path no need to create a dir --- doc/generate_manpages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/generate_manpages.py b/doc/generate_manpages.py index 1c0586b2e..9d4201f5d 100644 --- a/doc/generate_manpages.py +++ b/doc/generate_manpages.py @@ -126,7 +126,7 @@ def main(): else: output_dir = os.path.split(args.output)[0] - if not os.path.exists(output_dir): + if output_dir and not os.path.exists(output_dir): os.makedirs(output_dir) output_path = args.output From c75b9f7fa5a45585c9f4cae5f3327f2f58b17868 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 26 May 2019 03:06:53 +0200 Subject: [PATCH 25/88] [fix] duplicated line in manpage --- doc/generate_manpages.py | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/generate_manpages.py b/doc/generate_manpages.py index 9d4201f5d..bd4d9c3f2 100644 --- a/doc/generate_manpages.py +++ b/doc/generate_manpages.py @@ -41,7 +41,6 @@ show this help message and exit {{ name }} {{ value["category_help"] }} {% endfor %} -Manage debug logs .SS "global arguments:" .TP \\fB\\-\\-no\\-cache\\fR From ebba9967d0a9329d8e02160e8515297716efb568 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 26 May 2019 03:16:07 +0200 Subject: [PATCH 26/88] [mod] try to make manpages template less disastrously unreadable --- doc/generate_manpages.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/doc/generate_manpages.py b/doc/generate_manpages.py index bd4d9c3f2..1b0d85a69 100644 --- a/doc/generate_manpages.py +++ b/doc/generate_manpages.py @@ -20,8 +20,11 @@ template = Template('''\ .TH YunoHost "1" "{{ month }} {{ year }}" "YunoHost Collectif" .SH NAME YunoHost \\- yunohost server administration command + .SH SYNOPSIS yunohost \\fI\\,CATEGORY\\/\\fR \\fI\\,COMMAND\\/\\fR [\\fI\\,SUBCOMMAND\\/\\fR] [\\fI\\,ARGUMENTS\\/\\fR]... [\\fI\\,OPTIONS\\/\\fR]... + +{# generale command format #} .SH DESCRIPTION usage: yunohost {{ '{' }}{{ ",".join(categories) }}{{ '}' }} @@ -29,10 +32,12 @@ usage: yunohost [\\-h|\\-\\-help] [\\-\\-no\\-cache] [\\-\\-output\\-as {json,plain,none}] [\\-\\-debug] [\\-\\-quiet] [\\-\\-timeout ==SUPPRESS==] [\\-\\-admin\\-password PASSWORD] [\\-v|\\-\\-version] + .SS "optional arguments:" .TP \\fB\\-h\\fR, \\fB\\-\\-help\\fR show this help message and exit + .SS "categories:" .IP {{ '{' }}{{ ",".join(categories) }}{{ '}' }} @@ -41,6 +46,7 @@ show this help message and exit {{ name }} {{ value["category_help"] }} {% endfor %} + .SS "global arguments:" .TP \\fB\\-\\-no\\-cache\\fR @@ -67,6 +73,8 @@ The admin password to use to authenticate .TP \\fB\\-v\\fR, \\fB\\-\\-version\\fR Display YunoHost packages versions + +{# each categories #} {% for name, value in categories.items() %} .SH YUNOHOST {{ name.upper() }} usage: yunohost {{ name }} {{ '{' }}{{ ",".join(value["actions"].keys()) }}{{ '}' }} @@ -74,6 +82,8 @@ usage: yunohost {{ name }} {{ '{' }}{{ ",".join(value["actions"].keys()) }}{{ '} .SS "description:" .IP {{ value["category_help"] }} + +{# each command of each category #} {% for action, action_value in value["actions"].items() %} .SS "yunohost {{ name }} {{ action }} \ {% for argument_name, argument_value in action_value.get("arguments", {}).items() %}\ @@ -83,7 +93,11 @@ usage: yunohost {{ name }} {{ '{' }}{{ ",".join(value["actions"].keys()) }}{{ '} {% if str(argument_name).startswith("-") and not argument_value.get("action") == "store_true" %} {{ (argument_value.get("full", argument_name)).lstrip("-") }}{% endif %}\ {% if not required %}]{% endif %} \ {% endfor %}" + +{# help of the command #} {{ action_value["action_help"] }} + +{# arguments of the command #} {% if "arguments" in action_value %} {% for argument_name, argument_value in action_value["arguments"].items() %} .TP @@ -91,6 +105,7 @@ usage: yunohost {{ name }} {{ '{' }}{{ ",".join(value["actions"].keys()) }}{{ '} {% if str(argument_name).startswith("-") and not argument_value.get("action") == "store_true" %} \\fI\\,{{ (argument_value.get("full", argument_name)).lstrip("-") }}\\fR{% endif %} {{ argument_value.get("help", "")}} {% endfor %} + {% endif %} {% endfor %} {% endfor %} From 388d332ba1cbc9826dc0c86491e96accb6bb1174 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 26 May 2019 03:20:23 +0200 Subject: [PATCH 27/88] [enh] display default value in manpages --- doc/generate_manpages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/generate_manpages.py b/doc/generate_manpages.py index 1b0d85a69..694b82cf3 100644 --- a/doc/generate_manpages.py +++ b/doc/generate_manpages.py @@ -102,7 +102,7 @@ usage: yunohost {{ name }} {{ '{' }}{{ ",".join(value["actions"].keys()) }}{{ '} {% for argument_name, argument_value in action_value["arguments"].items() %} .TP \\fB{{ argument_name }}\\fR{% if argument_value.get("full") %}, \\fB{{ argument_value["full"] }}\\fR{% endif %}\ -{% if str(argument_name).startswith("-") and not argument_value.get("action") == "store_true" %} \\fI\\,{{ (argument_value.get("full", argument_name)).lstrip("-") }}\\fR{% endif %} +{% if str(argument_name).startswith("-") and not argument_value.get("action") == "store_true" %} \\fI\\,{{ (argument_value.get("full", argument_name)).lstrip("-") }}\\fR {% if "default" in argument_value %}(default: {{ argument_value["default"] }}){% endif %}{% endif %} {{ argument_value.get("help", "")}} {% endfor %} From 47d2383c11f9b79936327e4fcededb608f721ed5 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 2 Jun 2019 04:33:10 +0200 Subject: [PATCH 28/88] [enh] generate and copy manpages on debian installation --- debian/install | 1 + debian/rules | 1 + 2 files changed, 2 insertions(+) diff --git a/debian/install b/debian/install index b540ca749..3cb7e356a 100644 --- a/debian/install +++ b/debian/install @@ -1,6 +1,7 @@ bin/* /usr/bin/ sbin/* /usr/sbin/ data/bash-completion.d/yunohost /etc/bash_completion.d/ +doc/yunohost.8.gz /usr/share/man/man8/yunohost.8.gz data/actionsmap/* /usr/share/moulinette/actionsmap/ data/hooks/* /usr/share/yunohost/hooks/ data/other/yunoprompt.service /etc/systemd/system/ diff --git a/debian/rules b/debian/rules index d012c73f3..8afe372b5 100755 --- a/debian/rules +++ b/debian/rules @@ -10,6 +10,7 @@ override_dh_auto_build: # Generate bash completion file python data/actionsmap/yunohost_completion.py + python doc/generate_manpages.py --gzip --output doc/yunohost.8.gz override_dh_installinit: dh_installinit -pyunohost --name=yunohost-api --restart-after-upgrade From e2f43b43a50924bdbf329b4462b499c497694a5c Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 2 Jun 2019 04:45:49 +0200 Subject: [PATCH 29/88] [fix] make function name clearer, it's not only for actions --- src/yunohost/app.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 4323a5410..38f62d51d 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2278,7 +2278,7 @@ def _parse_args_from_manifest(manifest, action, args={}): return OrderedDict() action_args = manifest['arguments'][action] - return _parse_action_args_in_yunohost_format(args, action_args) + return _parse_args_in_yunohost_format(args, action_args) def _parse_args_for_action(action, args={}): @@ -2302,10 +2302,10 @@ def _parse_args_for_action(action, args={}): action_args = action['arguments'] - return _parse_action_args_in_yunohost_format(args, action_args) + return _parse_args_in_yunohost_format(args, action_args) -def _parse_action_args_in_yunohost_format(args, action_args): +def _parse_args_in_yunohost_format(args, action_args): """Parse arguments store in either manifest.json or actions.json """ from yunohost.domain import (domain_list, _get_maindomain, From d75e9d7a52bae20184ef57aedae7ee13b8fff3df Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 4 Jun 2019 13:21:37 +0200 Subject: [PATCH 30/88] Update changelog for 3.6.1 --- debian/changelog | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/debian/changelog b/debian/changelog index 0a92b9e51..49be5e222 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,16 @@ +yunohost (3.6.1) testing; urgency=low + + - [fix] current version in app_info (#730) + - [fix] Add indexes for fields listed by slapd in the logs (#729) + - [fix] Allow to display logs when postinstall fails (#728) + - [fix] Stupid issue with files inside tar : foo is not the same as ./foo (#726) + - [enh] Remove unecessary log messages (#724) + - [enh] Check for obvious conflict with already running apt/dpkg commands when running yunohost upgrade (d0c982a) + + Thanks to all contributors <3 ! (Aleks, Kay0u, Bram, L. Murphy, MCMic) + + -- Alexandre Aubin Tue, 04 Jun 2019 13:20:00 +0000 + yunohost (3.6.0) testing; urgency=low ## Major changes From 769ba5771d571b3c293bca2fac9f6293625d1cd5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 4 Jun 2019 15:09:09 +0200 Subject: [PATCH 31/88] Indexing gidNumber / uidNumber trigger weird issues when logging in through SSH (or the 'I have no name' shell when loggin in as admin) --- data/templates/slapd/slapd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/templates/slapd/slapd.conf b/data/templates/slapd/slapd.conf index cc7ac9230..de13173c4 100644 --- a/data/templates/slapd/slapd.conf +++ b/data/templates/slapd/slapd.conf @@ -67,7 +67,7 @@ index objectClass eq index uid,sudoUser eq,sub index entryCSN,entryUUID eq index cn,mail eq -index gidNumber,uidNumber eq +#index gidNumber,uidNumber eq index member,memberUid,uniqueMember eq index virtualdomain eq From 4fb59482ce08fa182c640577c77a755ebf51a63d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 4 Jun 2019 15:10:22 +0200 Subject: [PATCH 32/88] Update changelog for 3.6.1.1 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 49be5e222..8ef0b3e71 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +yunohost (3.6.1.1) testing; urgency=low + + - [fix] Weird issue in slapd triggered by indexing uidNumber / gidNumber + + -- Alexandre Aubin Tue, 04 Jun 2019 15:10:00 +0000 + yunohost (3.6.1) testing; urgency=low - [fix] current version in app_info (#730) From 8027e9268373410cb11989cf21c6e37624742471 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Tue, 4 Jun 2019 19:47:16 +0200 Subject: [PATCH 33/88] [mod] using yunohost format for config-panel arguments --- src/yunohost/app.py | 56 ++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 38f62d51d..a200ecae1 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1635,24 +1635,28 @@ def app_config_show_panel(app): for section in tab.get("sections", []): section_id = section["id"] for option in section.get("options", []): - option_id = option["id"] - generated_id = ("YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_id)).upper() - option["id"] = generated_id - logger.debug(" * '%s'.'%s'.'%s' -> %s", tab.get("name"), section.get("name"), option.get("name"), generated_id) + option_name = option["name"] + generated_name = ("YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_name)).upper() + option["name"] = generated_name + logger.debug(" * '%s'.'%s'.'%s' -> %s", tab.get("name"), section.get("name"), option.get("name"), generated_name) - if generated_id in parsed_values: - # XXX we should probably uses the one of install here but it's at a POC state right now - option_type = option["type"] - if option_type == "bool": - assert parsed_values[generated_id].lower() in ("true", "false") - option["value"] = True if parsed_values[generated_id].lower() == "true" else False - elif option_type == "integer": - option["value"] = int(parsed_values[generated_id]) - elif option_type == "text": - option["value"] = parsed_values[generated_id] + if generated_name in parsed_values: + # code is not adapted for that so we have to mock expected format :/ + if option.get("type") == "boolean": + if parsed_values[generated_name].lower() in ("true", "1", "y"): + option["default"] = parsed_values[generated_name] + else: + del option["default"] + + args_dict = _parse_args_in_yunohost_format( + [{option["name"]: parsed_values[generated_name]}], + [option] + ) + print("%s ----> %s ----> %s" % (option["name"], repr(option.get("default", "[removed -> false]")), args_dict[option["name"]])) + option["default"] = args_dict[option["name"]] else: - logger.debug("Variable '%s' is not declared by config script, using default", generated_id) - option["value"] = option["default"] + logger.debug("Variable '%s' is not declared by config script, using default", generated_name) + option["default"] = option["default"] return { "app_id": app_id, @@ -1685,26 +1689,29 @@ def app_config_apply(app, args): "YNH_APP_INSTANCE_NUMBER": str(app_instance_nb), } args = dict(urlparse.parse_qsl(args, keep_blank_values=True)) if args else {} + print(json.dumps(args, sort_keys=True, indent=4)) + print("------------------------------------------") for tab in config_panel.get("panel", []): tab_id = tab["id"] # this makes things easier to debug on crash for section in tab.get("sections", []): section_id = section["id"] for option in section.get("options", []): - option_id = option["id"] - generated_id = ("YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_id)).upper() + option_name = option["name"] + generated_name = ("YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_name)).upper() - if generated_id in args: - logger.debug("include into env %s=%s", generated_id, args[generated_id]) - env[generated_id] = args[generated_id] + if generated_name in args: + logger.debug("include into env %s=%s", generated_name, args[generated_name]) + env[generated_name] = args[generated_name] else: - logger.debug("no value for key id %s", generated_id) + logger.debug("no value for key id %s", generated_name) # for debug purpose for key in args: if key not in env: logger.warning("Ignore key '%s' from arguments because it is not in the config", key) + print(json.dumps(env, sort_keys=True, indent=4)) return_code = hook_exec(config_script, args=["apply"], env=env, @@ -1824,7 +1831,10 @@ def _get_app_config_panel(app_id): for option_key, option_value in options: option = dict(option_value) - option["id"] = option_key + option["name"] = option_key + option["ask"] = {"en": option["ask"]} + if "help" in option: + option["help"] = {"en": option["help"]} section["options"].append(option) panel["sections"].append(section) From f7b5886d10027ae7bcf4fc2ffc930f70992be82a Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Wed, 5 Jun 2019 12:23:31 +0200 Subject: [PATCH 34/88] [mod] ux, the user create command without arg fails --- locales/en.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locales/en.json b/locales/en.json index 901d36574..b5c804391 100644 --- a/locales/en.json +++ b/locales/en.json @@ -395,7 +395,7 @@ "port_already_opened": "Port {port:d} is already opened for {ip_version:s} connections", "port_available": "Port {port:d} is available", "port_unavailable": "Port {port:d} is not available", - "recommend_to_add_first_user": "The post-install is finished but YunoHost needs at least one user to work correctly, you should add one using 'yunohost user create' or the admin interface.", + "recommend_to_add_first_user": "The post-install is finished but YunoHost needs at least one user to work correctly, you should add one using 'yunohost user create $username' or the admin interface.", "regenconf_file_backed_up": "The configuration file '{conf}' has been backed up to '{backup}'", "regenconf_file_copy_failed": "Unable to copy the new configuration file '{new}' to '{conf}'", "regenconf_file_kept_back": "The configuration file '{conf}' is expected to be deleted by regen-conf (category {category}) but has been kept back.", From 5e93b5cfd94f417240ae469f461f6bff6cae1680 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 5 Jun 2019 16:22:51 +0200 Subject: [PATCH 35/88] [fix] When self-upgrading, use a hack to not fill the ended_at and success key right away --- src/yunohost/tools.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index b0ed1c449..23fc50483 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -688,6 +688,13 @@ def tools_upgrade(operation_logger, apps=None, system=False): update_log_metadata = "sed -i \"s/ended_at: .*$/ended_at: $(date -u +'%Y-%m-%d %H:%M:%S.%N')/\" {}" update_log_metadata = update_log_metadata.format(operation_logger.md_path) + # Dirty hack such that the operation_logger does not add ended_at + # and success keys in the log metadata. (c.f. the code of the + # is_unit_operation + operation_logger.close()) We take care of + # this ourselves (c.f. the mark_success and updated_log_metadata in + # the huge command launched by os.system) + operation_logger.ended_at = "notyet" + upgrade_completed = "\n" + m18n.n("tools_upgrade_special_packages_completed") command = "(({wait} && {cmd}) && {mark_success} || {mark_failure}; {update_metadata}; echo '{done}') &".format( wait=wait_until_end_of_yunohost_command, From 7ecd606d04bd1d0269e4829d0c4afa3f5622be6a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 5 Jun 2019 16:25:25 +0200 Subject: [PATCH 36/88] [fix] Do not index cn,mail and virtualdomain as it's causing some weird issues in the mail stack ... --- data/templates/slapd/slapd.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/templates/slapd/slapd.conf b/data/templates/slapd/slapd.conf index de13173c4..ebde7fdb3 100644 --- a/data/templates/slapd/slapd.conf +++ b/data/templates/slapd/slapd.conf @@ -66,10 +66,10 @@ directory "/var/lib/ldap" index objectClass eq index uid,sudoUser eq,sub index entryCSN,entryUUID eq -index cn,mail eq +#index cn,mail eq #index gidNumber,uidNumber eq index member,memberUid,uniqueMember eq -index virtualdomain eq +#index virtualdomain eq # Save the time that the entry gets modified, for database #1 lastmod on From 90172260196396916adcd30f0398f6b5d5c8392c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 5 Jun 2019 16:27:11 +0200 Subject: [PATCH 37/88] Update changelog for 3.6.1.2 --- debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index 8ef0b3e71..7168ba793 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +yunohost (3.6.1.2) testing; urgency=low + + - [fix] More weird issues with slapd indexation ... + - [fix] Small issue with operation logging during failed upgrade (success status set to true) + + -- Alexandre Aubin Wed, 05 Jun 2019 16:25:00 +0000 + yunohost (3.6.1.1) testing; urgency=low - [fix] Weird issue in slapd triggered by indexing uidNumber / gidNumber From e63d3986b22aa02484d22cdc40943b7b4b2a3dd8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 5 Jun 2019 17:33:53 +0200 Subject: [PATCH 38/88] [fix] This need quotes because this can be sourced by another program --- data/templates/postfix/postsrsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/templates/postfix/postsrsd b/data/templates/postfix/postsrsd index 56bfd091e..a0451faf6 100644 --- a/data/templates/postfix/postsrsd +++ b/data/templates/postfix/postsrsd @@ -12,7 +12,7 @@ SRS_DOMAIN={{ main_domain }} # the domain itself. Separate multiple domains by space or comma. # We have to put some "dummy" stuff at start and end... see this comment : # https://github.com/roehling/postsrsd/issues/64#issuecomment-284003762 -SRS_EXCLUDE_DOMAINS=dummy {{ domain_list }} dummy +SRS_EXCLUDE_DOMAINS="dummy {{ domain_list }} dummy" # First separator character after SRS0 or SRS1. # Can be one of: -+= From 3b6ddd02625d4464d4283be079b9883942d1ca39 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 6 Jun 2019 18:40:42 +0200 Subject: [PATCH 39/88] [i18n] Fix / improve message --- locales/en.json | 2 +- locales/fr.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/locales/en.json b/locales/en.json index b5c804391..22247c97e 100644 --- a/locales/en.json +++ b/locales/en.json @@ -124,7 +124,7 @@ "backup_with_no_restore_script_for_app": "App {app:s} has no restore script, you won't be able to automatically restore the backup of this app.", "certmanager_acme_not_configured_for_domain": "Certificate for domain {domain:s} does not appear to be correctly installed. Please run cert-install for this domain first.", "certmanager_attempt_to_renew_nonLE_cert": "The certificate for domain {domain:s} is not issued by Let's Encrypt. Cannot renew it automatically!", - "certmanager_attempt_to_renew_valid_cert": "The certificate for domain {domain:s} is not about to expire! Use --force to bypass", + "certmanager_attempt_to_renew_valid_cert": "The certificate for domain {domain:s} is not about to expire! (You may use --force if you know what you're doing)", "certmanager_attempt_to_replace_valid_cert": "You are attempting to overwrite a good and valid certificate for domain {domain:s}! (Use --force to bypass)", "certmanager_cannot_read_cert": "Something wrong happened when trying to open current certificate for domain {domain:s} (file: {file:s}), reason: {reason:s}", "certmanager_cert_install_success": "Successfully installed Let's Encrypt certificate for domain {domain:s}!", diff --git a/locales/fr.json b/locales/fr.json index ef9954a6e..59d02b119 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -254,7 +254,7 @@ "certmanager_domain_cert_not_selfsigned": "Le certificat du domaine {domain:s} n’est pas auto-signé. Voulez-vous vraiment le remplacer ? (Utilisez --force pour cela)", "certmanager_certificate_fetching_or_enabling_failed": "Il semble que l’activation du nouveau certificat pour {domain:s} a échoué …", "certmanager_attempt_to_renew_nonLE_cert": "Le certificat pour le domaine {domain:s} n’est pas émis par Let’s Encrypt. Impossible de le renouveler automatiquement !", - "certmanager_attempt_to_renew_valid_cert": "Le certificat pour le domaine {domain:s} est sur le point d’expirer ! Utilisez --force pour contourner cela", + "certmanager_attempt_to_renew_valid_cert": "Le certificat pour le domaine {domain:s} n'est pas sur le point d’expirer ! (Vous pouvez utiliser --force si vous savez ce que vous faites)", "certmanager_domain_http_not_working": "Il semble que le domaine {domain:s} ne soit pas accessible via HTTP. Veuillez vérifier que vos configuration DNS et Nginx sont correctes", "certmanager_error_no_A_record": "Aucun enregistrement DNS 'A' n’a été trouvé pour {domain:s}. Vous devez faire pointer votre nom de domaine vers votre machine pour être en mesure d’installer un certificat Let’s Encrypt ! (Si vous savez ce que vous faites, utilisez --no-checks pour désactiver ces contrôles)", "certmanager_domain_dns_ip_differs_from_public_ip": "L’enregistrement DNS 'A' du domaine {domain:s} est différent de l’adresse IP de ce serveur. Si vous avez récemment modifié votre enregistrement 'A', veuillez attendre sa propagation (quelques vérificateur de propagation DNS sont disponibles en ligne). (Si vous savez ce que vous faites, utilisez --no-checks pour désactiver ces contrôles)", From d8b086a06ad4d33b08890fac0b80e1d0f608a94e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 6 Jun 2019 19:26:49 +0200 Subject: [PATCH 40/88] [fix] Running slapindex seems to fix the previous issues about LDAP indexing stuff --- data/hooks/conf_regen/06-slapd | 2 ++ data/templates/slapd/slapd.conf | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/data/hooks/conf_regen/06-slapd b/data/hooks/conf_regen/06-slapd index d0a1fad63..bd6b427dd 100755 --- a/data/hooks/conf_regen/06-slapd +++ b/data/hooks/conf_regen/06-slapd @@ -101,6 +101,8 @@ do_post_regen() { sudo chown -R openldap:openldap /etc/ldap/slapd.d/ fi + sudo -u openldap slapindex + sudo service slapd force-reload # on slow hardware/vm this regen conf would exit before the admin user that diff --git a/data/templates/slapd/slapd.conf b/data/templates/slapd/slapd.conf index ebde7fdb3..cc7ac9230 100644 --- a/data/templates/slapd/slapd.conf +++ b/data/templates/slapd/slapd.conf @@ -66,10 +66,10 @@ directory "/var/lib/ldap" index objectClass eq index uid,sudoUser eq,sub index entryCSN,entryUUID eq -#index cn,mail eq -#index gidNumber,uidNumber eq +index cn,mail eq +index gidNumber,uidNumber eq index member,memberUid,uniqueMember eq -#index virtualdomain eq +index virtualdomain eq # Save the time that the entry gets modified, for database #1 lastmod on From ad9430b3eab006979e81b9feccedee6847b95393 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 7 Jun 2019 18:38:32 +0200 Subject: [PATCH 41/88] Update changelog for 3.6.1.3 --- debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index 7168ba793..6bed53f47 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +yunohost (3.6.1.3) testing; urgency=low + + - [fix] Missing quotes led to an issue during when upgrading postsrsd + - [fix] Running slapindex seems to fix the previous issues about LDAP indexing stuff + + -- Alexandre Aubin Fri, 07 Jun 2019 06:38:00 +0000 + yunohost (3.6.1.2) testing; urgency=low - [fix] More weird issues with slapd indexation ... From 7ebb8bade3a31f3da4276709bce7c27deeecfbf5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 7 Jun 2019 19:47:09 +0200 Subject: [PATCH 42/88] Add current and new version for apps in tools_update output --- src/yunohost/tools.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 23fc50483..acde7a8d5 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -534,9 +534,21 @@ def _list_upgradable_apps(): app_dict = app_info(app_id, raw=True) if app_dict["upgradable"] == "yes": + + current_version = app_dict.get("version", "?") + current_commit = app_dict.get("status", {}).get("remote", {}).get("revision", "?")[:7] + new_version = app_dict.get("manifest",{}).get("version","?") + new_commit = app_dict.get("git", {}).get("revision", "?")[:7] + + if current_version == new_version: + current_version += " (" + current_commit + ")" + new_version += " (" + new_commit + ")" + yield { 'id': app_id, - 'label': app_dict['settings']['label'] + 'label': app_dict['settings']['label'], + 'current_version': current_version, + 'new_version': new_version } From 01e3be24da700ffe833e764f2cfd8730384f860a Mon Sep 17 00:00:00 2001 From: Maniack Crudelis Date: Fri, 14 Jun 2019 22:31:47 +0200 Subject: [PATCH 43/88] Update data/helpers.d/backup Co-Authored-By: Kayou --- data/helpers.d/backup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/backup b/data/helpers.d/backup index b347f1700..cfffe13d0 100644 --- a/data/helpers.d/backup +++ b/data/helpers.d/backup @@ -59,7 +59,7 @@ ynh_backup() { local not_mandatory="${not_mandatory:-0}" BACKUP_CORE_ONLY=${BACKUP_CORE_ONLY:-0} - test -n "${app:-}" && do_not_backup_data=$(ynh_app_setting_get $app do_not_backup_data) + test -n "${app:-}" && do_not_backup_data=$(ynh_app_setting_get --app=$app --key=do_not_backup_data) # If backing up core only (used by ynh_backup_before_upgrade), # don't backup big data items From 3a4bf0ea10c9631fd64c606c60c85de220361729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quent=C3=AD?= Date: Sun, 9 Jun 2019 17:01:04 +0000 Subject: [PATCH 44/88] Translated using Weblate (Occitan) Currently translated at 91.4% (480 of 525 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/oc/ --- locales/oc.json | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/locales/oc.json b/locales/oc.json index 90641fabd..f601b2a04 100644 --- a/locales/oc.json +++ b/locales/oc.json @@ -53,7 +53,7 @@ "app_manifest_invalid": "Manifest d’aplicacion incorrècte : {error}", "app_package_need_update": "Lo paquet de l’aplicacion {app} deu èsser mes a jorn per seguir los cambiaments de YunoHost", "app_requirements_checking": "Verificacion dels paquets requesits per {app}…", - "app_sources_fetch_failed": "Recuperacion dels fichièrs fonts impossibla", + "app_sources_fetch_failed": "Recuperacion dels fichièrs fonts impossibla, l’URL es corrècta ?", "app_unsupported_remote_type": "Lo tipe alonhat utilizat per l’aplicacion es pas suportat", "appslist_retrieve_error": "Impossible de recuperar la lista d’aplicacions alonhadas {appslist:s} : {error:s}", "backup_archive_app_not_found": "L’aplicacion « {app:s} » es pas estada trobada dins l’archiu de la salvagarda", @@ -121,7 +121,7 @@ "backup_with_no_restore_script_for_app": "L’aplicacion {app:s} a pas cap de script de restauracion, poiretz pas restaurar automaticament la salvagarda d’aquesta aplicacion.", "certmanager_acme_not_configured_for_domain": "Lo certificat del domeni {domain:s} sembla pas corrèctament installat. Mercés de lançar d’en primièr cert-install per aqueste domeni.", "certmanager_attempt_to_renew_nonLE_cert": "Lo certificat pel domeni {domain:s} es pas provesit per Let’s Encrypt. Impossible de lo renovar automaticament !", - "certmanager_attempt_to_renew_valid_cert": "Lo certificat pel domeni {domain:s} es a man d’expirar ! Utilizatz --force per cortcircuitar", + "certmanager_attempt_to_renew_valid_cert": "Lo certificat pel domeni {domain:s} es a man d’expirar ! (Podètz utilizar --force se sabètz çò que fasètz)", "certmanager_cannot_read_cert": "Quicòm a trucat en ensajar de dobrir lo certificat actual pel domeni {domain:s} (fichièr : {file:s}), rason : {reason:s}", "certmanager_cert_install_success": "Installacion capitada del certificat Let’s Encrypt pel domeni {domain:s} !", "certmanager_cert_install_success_selfsigned": "Installacion capitada del certificat auto-signat pel domeni {domain:s} !", @@ -175,7 +175,7 @@ "global_settings_key_doesnt_exists": "La clau « {settings_key:s} » existís pas dins las configuracions globalas, podètz veire totas las claus disponiblas en picant « yunohost settings list »", "global_settings_reset_success": "Capitada ! Vòstra configuracion precedenta es estada salvagarda dins {path:s}", "global_settings_setting_example_bool": "Exemple d’opcion booleana", - "global_settings_unknown_setting_from_settings_file": "Clau desconeguda dins los paramètres : {setting_key:s}, apartada e salvagardada dins /etc/yunohost/unkown_settings.json", + "global_settings_unknown_setting_from_settings_file": "Clau desconeguda dins los paramètres : {setting_key:s}, apartada e salvagardada dins /etc/yunohost/settings-unknown.json", "installation_failed": "Fracàs de l’installacion", "invalid_url_format": "Format d’URL pas valid", "ldap_initialized": "L’annuari LDAP es inicializat", @@ -444,7 +444,7 @@ "log_tools_migrations_migrate_forward": "Migrar", "log_tools_migrations_migrate_backward": "Tornar en arrièr", "log_tools_postinstall": "Realizar la post installacion del servidor YunoHost", - "log_tools_upgrade": "Mesa a jorn dels paquets Debian", + "log_tools_upgrade": "Mesa a jorn dels paquets sistèma", "log_tools_shutdown": "Atudar lo servidor", "log_tools_reboot": "Reaviar lo servidor", "mail_unavailable": "Aquesta adreça electronica es reservada e deu èsser automaticament atribuida al tot bèl just primièr utilizaire", @@ -453,7 +453,7 @@ "migration_0005_postgresql_94_not_installed": "Postgresql es pas installat sul sistèma. Pas res de far !", "migration_0005_postgresql_96_not_installed": "Avèm trobat que Postgresql 9.4 es installat, mas cap de version de Postgresql 9.6 pas trobada !? Quicòm d’estranh a degut arribar a vòstre sistèma :( …", "migration_0005_not_enough_space": "I a pas pro d’espaci disponible sus {path} per lançar la migracion d’aquela passa :(.", - "recommend_to_add_first_user": "La post installacion es acabada, mas YunoHost fa besonh d’almens un utilizaire per foncionar coma cal. Vos cal n’ajustar un en utilizant la comanda « yunohost user create » o ben l’interfàcia d’administracion.", + "recommend_to_add_first_user": "La post installacion es acabada, mas YunoHost fa besonh d’almens un utilizaire per foncionar coma cal. Vos cal n’ajustar un en utilizant la comanda « yunohost user create $username » o ben l’interfàcia d’administracion.", "service_description_php7.0-fpm": "executa d’aplicacions escrichas en PHP amb nginx", "users_available": "Lista dels utilizaires disponibles :", "good_practices_about_admin_password": "Sètz per definir un nòu senhal per l’administracion. Lo senhal deu almens conténer 8 caractèrs - encara que siá de bon far d’utilizar un senhal mai long qu’aquò (ex. una passafrasa) e/o d’utilizar mantun tipes de caractèrs (majuscula, minuscula, nombre e caractèrs especials).", @@ -491,5 +491,16 @@ "migration_0007_cannot_restart": "SSH pòt pas èsser reavit aprèp aver ensajat d’anullar la migracion numèro 6.", "migrations_success": "Migracion {number} {name} reüssida !", "service_conf_now_managed_by_yunohost": "Lo fichièr de configuracion « {conf} » es ara gerit per YunoHost.", - "service_reloaded": "Lo servici « {servici:s} » es estat tornat cargar" + "service_reloaded": "Lo servici « {servici:s} » es estat tornat cargar", + "already_up_to_date": "I a pas res a far ! Tot es ja a jorn !", + "app_action_cannot_be_ran_because_required_services_down": "Aquesta aplicacion necessita unes servicis que son actualament encalats. Abans de contunhar deuriatz ensajar de reaviar los servicis seguents (e tanben cercar perque son tombats en pana) : {services}", + "confirm_app_install_warning": "Atencion : aquesta aplicacion fonciona mas non es pas ben integrada amb YunoHost. Unas foncionalitats coma l’autentificacion unica e la còpia de seguretat/restauracion pòdon èsser indisponiblas. volètz l’installar de totas manièras ? [{answers:s}] ", + "confirm_app_install_danger": "ATENCION ! Aquesta aplicacion es encara experimentala (autrament dich, fonciona pas) e es possible que còpe lo sistèma ! Deuriatz PAS l’installar se non sabètz çò que fasètz. Volètz vertadièrament córrer aqueste risc ? [{answers:s}] ", + "confirm_app_install_thirdparty": "ATENCION ! L’installacion d’aplicacions tèrças pòt comprometre l’integralitat e la seguretat del sistèma. Deuriatz PAS l’installar se non sabètz pas çò que fasètz. Volètz vertadièrament córrer aqueste risc ? [{answers:s}] ", + "dpkg_lock_not_available": "Aquesta comanda pòt pas s’executar pel moment perque un autre programa sembla utilizar lo varrolh de dpkg (lo gestionari de paquets del sistèma)", + "log_regen_conf": "Regenerar las configuracions del sistèma « {} »", + "service_reloaded_or_restarted": "Lo servici « {service:s} » es estat recargat o reaviat", + "tools_upgrade_regular_packages_failed": "Actualizacion impossibla dels paquets seguents : {packages_list}", + "tools_upgrade_special_packages_completed": "L’actualizacion dels paquets de YunoHost es acabada !\nQuichatz [Entrada] per tornar a la linha de comanda", + "updating_app_lists": "Recuperacion de las mesas a jorn disponiblas per las aplicacions…" } From 32ac0682b1ad77c210efc9c36d06427952a59099 Mon Sep 17 00:00:00 2001 From: Locness Date: Sun, 9 Jun 2019 12:17:53 +0000 Subject: [PATCH 45/88] Translated using Weblate (French) Currently translated at 95.8% (503 of 525 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/fr/ --- locales/fr.json | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/locales/fr.json b/locales/fr.json index 59d02b119..944a0b3a8 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -25,7 +25,7 @@ "app_requirements_checking": "Vérification des paquets requis pour {app} …", "app_requirements_failed": "Impossible de satisfaire les pré-requis pour {app} : {error}", "app_requirements_unmeet": "Les pré-requis de {app} ne sont pas satisfaits, le paquet {pkgname} ({version}) doit être {spec}", - "app_sources_fetch_failed": "Impossible de récupérer les fichiers sources", + "app_sources_fetch_failed": "Impossible de récupérer les fichiers sources, l'URL est-elle correcte ?", "app_unknown": "Application inconnue", "app_unsupported_remote_type": "Ce type de commande à distance utilisé pour cette application n'est pas supporté", "app_upgrade_failed": "Impossible de mettre à jour {app:s}", @@ -111,7 +111,7 @@ "hook_choice_invalid": "Choix incorrect : '{:s}'", "hook_exec_failed": "Échec de l’exécution du script : {path:s}", "hook_exec_not_terminated": "L’exécution du script {path:s} ne s’est pas terminée correctement", - "hook_list_by_invalid": "Propriété invalide pour lister les actions par", + "hook_list_by_invalid": "Propriété invalide pour lister les actions par celle-ci", "hook_name_unknown": "Nom de l'action '{name:s}' inconnu", "installation_complete": "Installation terminée", "installation_failed": "Échec de l’installation", @@ -458,7 +458,7 @@ "log_tools_migrations_migrate_forward": "Migrer vers", "log_tools_migrations_migrate_backward": "Revenir en arrière", "log_tools_postinstall": "Faire la post-installation de votre serveur YunoHost", - "log_tools_upgrade": "Mettre à jour les paquets système", + "log_tools_upgrade": "Mettre à jour les paquets du système", "log_tools_shutdown": "Éteindre votre serveur", "log_tools_reboot": "Redémarrer votre serveur", "mail_unavailable": "Cette adresse de courriel est réservée et doit être automatiquement attribuée au tout premier utilisateur", @@ -467,7 +467,7 @@ "migration_0005_postgresql_94_not_installed": "PostgreSQL n’a pas été installé sur votre système. Rien à faire !", "migration_0005_postgresql_96_not_installed": "PostgreSQL 9.4 a été trouvé et installé, mais pas PostgreSQL 9.6 !? Quelque chose d’étrange a dû arriver à votre système :( …", "migration_0005_not_enough_space": "Il n’y a pas assez d’espace libre de disponible sur {path} pour lancer maintenant la migration :(.", - "recommend_to_add_first_user": "La post-installation est terminée. YunoHost a besoin d’au moins un utilisateur pour fonctionner correctement. Vous devez en ajouter un en utilisant 'yunohost user create' ou bien via l’interface d’administration web.", + "recommend_to_add_first_user": "La post-installation est terminée mais YunoHost a besoin d’au moins un utilisateur pour fonctionner correctement. Vous devez en ajouter un en utilisant 'yunohost user create $nomdutilisateur' ou bien via l’interface d’administration web.", "service_description_php7.0-fpm": "exécute des applications écrites en PHP avec Nginx", "users_available": "Liste des utilisateurs disponibles :", "good_practices_about_admin_password": "Vous êtes maintenant sur le point de définir un nouveau mot de passe d’administration. Le mot de passe doit comporter au moins 8 caractères – bien qu’il soit recommandé d’utiliser un mot de passe plus long (c’est-à-dire une phrase secrète) et/ou d’utiliser différents types de caractères (majuscules, minuscules, chiffres et caractères spéciaux).", @@ -524,7 +524,7 @@ "service_reloaded_or_restarted": "Le service '{service:s}' a été rechargé ou redémarré", "this_action_broke_dpkg": "Cette action a laissé des paquets non configurés par dpkg/apt (les gestionnaires de paquets système). Vous pouvez essayer de résoudre ce problème en vous connectant via SSH et en exécutant `sudo dpkg --configure -a`.", "app_action_cannot_be_ran_because_required_services_down": "Cette application requiert certains services qui sont actuellement en pannes. Avant de continuer, vous devriez essayer de redémarrer les services suivant (et éventuellement rechercher pourquoi ils sont en panne) : {services}", - "admin_password_too_long": "Choisissez un mot de passe de 127 caractères maximum.", + "admin_password_too_long": "Choisissez un mot de passe plus court que 127 caractères", "log_regen_conf": "Régénérer les configurations du système '{}'", "migration_0009_not_needed": "Cette migration semble avoir déjà été jouée ? On l'ignore.", "regenconf_file_backed_up": "Le fichier de configuration '{conf}' a été sauvegardé sous '{backup}'", @@ -551,5 +551,10 @@ "service_regen_conf_is_deprecated": "« yunohost service regen-conf » est obsolète ! Veuillez plutôt utiliser « yunohost tools regen-conf ».", "tools_upgrade_at_least_one": "Veuillez spécifier --apps OU --system", "tools_upgrade_cant_both": "Impossible de mettre à niveau le système et les applications en même temps", - "tools_upgrade_cant_hold_critical_packages": "Impossibilité de maintenir les paquets critiques..." + "tools_upgrade_cant_hold_critical_packages": "Impossibilité de maintenir les paquets critiques...", + "tools_upgrade_regular_packages": "Mise à jour des paquets « normaux » (non liés a YunoHost) ...", + "tools_upgrade_regular_packages_failed": "Impossible de mettre à jour les paquets suivants : {packages_list}", + "tools_upgrade_special_packages": "Mise à jour des paquets « spéciaux » (liés a YunoHost) ...", + "tools_upgrade_special_packages_completed": "La mise à jour des paquets de YunoHost est finie!\nPressez [Entrée] pour revenir à la ligne de commande", + "updating_app_lists": "Récupération des mises à jour des applications disponibles…" } From 99a98064960740fbd5f5ce9111e32a93780083c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quent=C3=AD?= Date: Wed, 12 Jun 2019 16:20:35 +0000 Subject: [PATCH 46/88] Translated using Weblate (Occitan) Currently translated at 92.0% (483 of 525 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/oc/ --- locales/oc.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/locales/oc.json b/locales/oc.json index f601b2a04..58ff89e7d 100644 --- a/locales/oc.json +++ b/locales/oc.json @@ -502,5 +502,8 @@ "service_reloaded_or_restarted": "Lo servici « {service:s} » es estat recargat o reaviat", "tools_upgrade_regular_packages_failed": "Actualizacion impossibla dels paquets seguents : {packages_list}", "tools_upgrade_special_packages_completed": "L’actualizacion dels paquets de YunoHost es acabada !\nQuichatz [Entrada] per tornar a la linha de comanda", - "updating_app_lists": "Recuperacion de las mesas a jorn disponiblas per las aplicacions…" + "updating_app_lists": "Recuperacion de las mesas a jorn disponiblas per las aplicacions…", + "dpkg_is_broken": "Podètz pas far aquò pel moment perque dpkg/apt (los gestionaris de paquets del sistèma) sembla èsser mal configurat... Podètz ensajar de solucionar aquò en vos connectar via SSH e en executar « sudo dpkg --configure -a ».", + "global_settings_setting_service_ssh_allow_deprecated_dsa_hostkey": "Autorizar l’utilizacion de la clau òst DSA (obsolèta) per la configuracion del servici SSH", + "migration_0008_general_disclaimer": "Per melhorar la seguretat del servidor, es recomandat de daissar YunoHost gerir la configuracion SSH. Vòstra configuracion actuala es diferenta de la configuracion recomandada. Se daissatz YunoHost la reconfigurar, lo biais de vos connectar al servidor via SSH cambiarà coma aquò :" } From 75755ff65b3900d4506be6adf8d52358ae473c0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quent=C3=AD?= Date: Fri, 14 Jun 2019 18:13:44 +0000 Subject: [PATCH 47/88] Translated using Weblate (Occitan) Currently translated at 92.4% (485 of 525 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/oc/ --- locales/oc.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/locales/oc.json b/locales/oc.json index 58ff89e7d..57914d5e5 100644 --- a/locales/oc.json +++ b/locales/oc.json @@ -505,5 +505,7 @@ "updating_app_lists": "Recuperacion de las mesas a jorn disponiblas per las aplicacions…", "dpkg_is_broken": "Podètz pas far aquò pel moment perque dpkg/apt (los gestionaris de paquets del sistèma) sembla èsser mal configurat... Podètz ensajar de solucionar aquò en vos connectar via SSH e en executar « sudo dpkg --configure -a ».", "global_settings_setting_service_ssh_allow_deprecated_dsa_hostkey": "Autorizar l’utilizacion de la clau òst DSA (obsolèta) per la configuracion del servici SSH", - "migration_0008_general_disclaimer": "Per melhorar la seguretat del servidor, es recomandat de daissar YunoHost gerir la configuracion SSH. Vòstra configuracion actuala es diferenta de la configuracion recomandada. Se daissatz YunoHost la reconfigurar, lo biais de vos connectar al servidor via SSH cambiarà coma aquò :" + "migration_0008_general_disclaimer": "Per melhorar la seguretat del servidor, es recomandat de daissar YunoHost gerir la configuracion SSH. Vòstra configuracion actuala es diferenta de la configuracion recomandada. Se daissatz YunoHost la reconfigurar, lo biais de vos connectar al servidor via SSH cambiarà coma aquò :", + "hook_json_return_error": "Fracàs de la lectura del retorn de l’script {path:s}. Error : {msg:s}. Contengut brut : {raw_content}", + "migration_0008_port": " - vos cal vos connectar en utilizar lo pòrt 22 allòc de vòstre pòrt SSH actual personalizat. Esitetz pas a lo reconfigurar ;" } From 279b23b96c96ab18244d3a129f3baeb72b33fd1a Mon Sep 17 00:00:00 2001 From: Weblate Date: Sat, 15 Jun 2019 00:06:53 +0000 Subject: [PATCH 48/88] =?UTF-8?q?Added=20translation=20using=20Weblate=20(?= =?UTF-8?q?Norwegian=20Bokm=C3=A5l)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- locales/nb_NO.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 locales/nb_NO.json diff --git a/locales/nb_NO.json b/locales/nb_NO.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/locales/nb_NO.json @@ -0,0 +1 @@ +{} From 3d5f9c014508f58efc224390c4d82db315d47411 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quent=C3=AD?= Date: Sun, 16 Jun 2019 09:43:36 +0000 Subject: [PATCH 49/88] Translated using Weblate (Occitan) Currently translated at 95.8% (503 of 525 strings) Translation: YunoHost/core Translate-URL: https://translate.yunohost.org/projects/yunohost/core/oc/ --- locales/oc.json | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/locales/oc.json b/locales/oc.json index 57914d5e5..dbad5ac22 100644 --- a/locales/oc.json +++ b/locales/oc.json @@ -507,5 +507,23 @@ "global_settings_setting_service_ssh_allow_deprecated_dsa_hostkey": "Autorizar l’utilizacion de la clau òst DSA (obsolèta) per la configuracion del servici SSH", "migration_0008_general_disclaimer": "Per melhorar la seguretat del servidor, es recomandat de daissar YunoHost gerir la configuracion SSH. Vòstra configuracion actuala es diferenta de la configuracion recomandada. Se daissatz YunoHost la reconfigurar, lo biais de vos connectar al servidor via SSH cambiarà coma aquò :", "hook_json_return_error": "Fracàs de la lectura del retorn de l’script {path:s}. Error : {msg:s}. Contengut brut : {raw_content}", - "migration_0008_port": " - vos cal vos connectar en utilizar lo pòrt 22 allòc de vòstre pòrt SSH actual personalizat. Esitetz pas a lo reconfigurar ;" + "migration_0008_port": " - vos cal vos connectar en utilizar lo pòrt 22 allòc de vòstre pòrt SSH actual personalizat. Esitetz pas a lo reconfigurar ;", + "migration_0009_not_needed": "Sembla qu’i aguèt ja una migracion. Passem.", + "pattern_password_app": "O planhèm, los senhals devon pas conténer los caractèrs seguents : {forbidden_chars}", + "regenconf_file_backed_up": "Lo fichièr de configuracion « {conf} » es estat salvagardat dins « {backup} »", + "regenconf_file_copy_failed": "Còpia impossibla del nòu fichièr de configuracion « {new} » cap a « {conf} »", + "regenconf_file_manually_modified": "Lo fichièr de configuracion « {conf} » es estat modificat manualament e serà pas actualizat", + "regenconf_file_manually_removed": "Lo fichièr de configuracion « {conf} » es estat suprimit manualament e serà pas creat", + "regenconf_file_remove_failed": "Supression impossibla del fichièr de configuracion « {conf} »", + "regenconf_file_removed": "Lo fichièr de configuracion « {conf} » es estat suprimit", + "regenconf_file_updated": "Lo fichièr de configuracion « {conf} » es estat actualizat", + "regenconf_now_managed_by_yunohost": "Lo fichièr de configuracion « {conf} » es ara gerit per YunoHost (categoria {category}).", + "regenconf_up_to_date": "La configuracion es ja a jorn per la categoria « {category} »", + "regenconf_updated": "La configuracion es estada actualizada per la categoria « {category} »", + "regenconf_would_be_updated": "La configuracion seriá estada actualizada per la categoria « {category} »", + "regenconf_dry_pending_applying": "Verificacion de la configuracion que seriá estada aplicada a la categoria « {category} »…", + "regenconf_failed": "Regeneracion impossibla de la configuracion per la(s) categoria(s) : {categories}", + "regenconf_pending_applying": "Aplicacion de la configuracion en espèra per la categoria « {category} »…", + "tools_upgrade_cant_both": "Actualizacion impossibla del sistèma e de las aplicacions a l’encòp", + "tools_upgrade_cant_hold_critical_packages": "Manteniment impossible dels paquets critiques…" } From 46e388460cddf70fdbf1d4fac8f7c5d22ebd3927 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 18 Jun 2019 14:11:53 +0200 Subject: [PATCH 50/88] [fix] Backup delete should delete symlink target --- src/yunohost/backup.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 20dbc8695..835b6979b 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -2319,7 +2319,14 @@ def backup_delete(name): archive_file = '%s/%s.tar.gz' % (ARCHIVES_PATH, name) info_file = "%s/%s.info.json" % (ARCHIVES_PATH, name) - for backup_file in [archive_file, info_file]: + files_to_delete = [archive_file, info_file] + + # To handle the case where archive_file is in fact a symlink + actual_archive = os.path.realpath(archive_file) + if actual_archive != archive_file: + files_to_delete.append(actual_archive) + + for backup_file in files_to_delete: try: os.remove(backup_file) except: From b1634785fb510d3a0def3931845420b3b38a870f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 17 Jun 2019 17:32:47 +0200 Subject: [PATCH 51/88] Add a new 'plain_dict' return type for hook_exec, and use it for app config --- src/yunohost/app.py | 33 +++++++-------------------------- src/yunohost/hook.py | 28 ++++++++++++++++++++-------- 2 files changed, 27 insertions(+), 34 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 1f172a267..c6187a52b 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -362,10 +362,10 @@ def app_info(app, show_status=False, raw=False): ret['upgradable'] = upgradable ret['change_url'] = os.path.exists(os.path.join(app_setting_path, "scripts", "change_url")) - + with open(os.path.join(APPS_SETTING_PATH, app, 'manifest.json')) as json_manifest: manifest = json.load(json_manifest) - + ret['version'] = manifest.get('version', '-') return ret @@ -1606,31 +1606,12 @@ def app_config_show_panel(app): "YNH_APP_INSTANCE_NAME": app, "YNH_APP_INSTANCE_NUMBER": str(app_instance_nb), } - parsed_values = {} - # I need to parse stdout to communicate between scripts because I can't - # read the child environment :( (that would simplify things so much) - # after hours of research this is apparently quite a standard way, another - # option would be to add an explicite pipe or a named pipe for that - # a third option would be to write in a temporary file but I don't like - # that because that could expose sensitive data - def parse_stdout(line): - line = line.rstrip() - logger.info(line) - - if line.strip().startswith("YNH_CONFIG_") and "=" in line: - # XXX error handling? - # XXX this might not work for multilines stuff :( (but echo without - # formatting should do it no?) - key, value = line.strip().split("=", 1) - logger.debug("config script declared: %s -> %s", key, value) - parsed_values[key] = value - - return_code = hook_exec(config_script, - args=["show"], - env=env, - stdout_callback=parse_stdout, - )[0] + return_code, parsed_values = hook_exec(config_script, + args=["show"], + env=env, + return_format="plain_dict" + ) if return_code != 0: raise Exception("script/config show return value code: %s (considered as an error)", return_code) diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index 42807fdf7..f1b7515bc 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -298,7 +298,7 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None, def hook_exec(path, args=None, raise_on_error=False, no_trace=False, chdir=None, env=None, user="root", stdout_callback=None, - stderr_callback=None): + stderr_callback=None, return_format="json"): """ Execute hook from a file with arguments @@ -401,19 +401,31 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False, try: with open(stdreturn, 'r') as f: raw_content = f.read() - if raw_content != '': - returnjson = read_json(stdreturn) + returncontent = {} + + if return_format == "json": + if raw_content != '': + try: + returncontent = read_json(stdreturn) + except Exception as e: + raise YunohostError('hook_json_return_error', + path=path, msg=str(e), + raw_content=raw_content) + + elif return_format == "plain_dict": + for line in raw_content.split("\n"): + if "=" in line: + key, value = line.strip().split("=", 1) + returncontent[key] = value + else: - returnjson = {} - except Exception as e: - raise YunohostError('hook_json_return_error', path=path, msg=str(e), - raw_content=raw_content) + raise YunohostError("Excepted value for return_format is either 'json' or 'plain_dict', got '%s'" % return_format) finally: stdreturndir = os.path.split(stdreturn)[0] os.remove(stdreturn) os.rmdir(stdreturndir) - return returncode, returnjson + return returncode, returncontent def _extract_filename_parts(filename): From 52d1aff2ae6a9a7bbd3ee8b51a2d9f577b710f6e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 18 Jun 2019 15:11:19 +0200 Subject: [PATCH 52/88] Remove unused stdout/stderr_callback arg --- src/yunohost/hook.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index f1b7515bc..67e77f033 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -297,8 +297,7 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None, def hook_exec(path, args=None, raise_on_error=False, no_trace=False, - chdir=None, env=None, user="root", stdout_callback=None, - stderr_callback=None, return_format="json"): + chdir=None, env=None, user="root", return_format="json"): """ Execute hook from a file with arguments @@ -372,8 +371,8 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False, # Define output callbacks and call command callbacks = ( - stdout_callback if stdout_callback else lambda l: logger.debug(l.rstrip()+"\r"), - stderr_callback if stderr_callback else lambda l: logger.warning(l.rstrip()), + lambda l: logger.debug(l.rstrip()+"\r"), + lambda l: logger.warning(l.rstrip()), ) if stdinfo: From 9d63a067be65546f994918b368b22e741dc81f95 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 18 Jun 2019 15:41:27 +0200 Subject: [PATCH 53/88] Improve semantics as suggested by decentral1se --- src/yunohost/backup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 835b6979b..6745df865 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -2322,8 +2322,8 @@ def backup_delete(name): files_to_delete = [archive_file, info_file] # To handle the case where archive_file is in fact a symlink - actual_archive = os.path.realpath(archive_file) - if actual_archive != archive_file: + if os.islink(archive_file): + actual_archive = os.path.realpath(archive_file) files_to_delete.append(actual_archive) for backup_file in files_to_delete: From 5262b52cffd7ed649a69fd62f74fa79a05d88597 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 18 Jun 2019 16:03:00 +0200 Subject: [PATCH 54/88] Add a helper to easily return stuff in config script --- data/helpers.d/logging | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/data/helpers.d/logging b/data/helpers.d/logging index 9e8cc06cb..ed8054622 100644 --- a/data/helpers.d/logging +++ b/data/helpers.d/logging @@ -282,6 +282,16 @@ ynh_script_progression () { ynh_print_info "[$progression_bar] > ${message}${print_exec_time}" } +# Return data to the Yunohost core for later processing +# (to be used by special hooks like app config panel and core diagnosis) +# +# usage: ynh_return somedata +# +# Requires YunoHost version 3.6.0 or higher. +ynh_return () { + echo "$1" >> "$YNH_STDRETURN" +} + # Debugger for app packagers # # usage: ynh_debug [--message=message] [--trace=1/0] From 158aa0802219b9b600cd7ae23f131c6052244013 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jun 2019 17:30:05 +0200 Subject: [PATCH 55/88] Fix self-upgrade mechanism, now using systemd-run which is more robust than '&' --- src/yunohost/tools.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index acde7a8d5..294d9a3fa 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -691,7 +691,7 @@ def tools_upgrade(operation_logger, apps=None, system=False): # command before it ends...) # logfile = operation_logger.log_path - command = dist_upgrade + " 2>&1 | tee -a {}".format(logfile) + dist_upgrade = dist_upgrade + " 2>&1 | tee -a {}".format(logfile) MOULINETTE_LOCK = "/var/run/moulinette_yunohost.lock" wait_until_end_of_yunohost_command = "(while [ -f {} ]; do sleep 2; done)".format(MOULINETTE_LOCK) @@ -708,9 +708,9 @@ def tools_upgrade(operation_logger, apps=None, system=False): operation_logger.ended_at = "notyet" upgrade_completed = "\n" + m18n.n("tools_upgrade_special_packages_completed") - command = "(({wait} && {cmd}) && {mark_success} || {mark_failure}; {update_metadata}; echo '{done}') &".format( + command = "({wait} && {dist_upgrade}) && {mark_success} || {mark_failure}; {update_metadata}; echo '{done}'".format( wait=wait_until_end_of_yunohost_command, - cmd=command, + dist_upgrade=dist_upgrade, mark_success=mark_success, mark_failure=mark_failure, update_metadata=update_log_metadata, @@ -718,7 +718,12 @@ def tools_upgrade(operation_logger, apps=None, system=False): logger.warning(m18n.n("tools_upgrade_special_packages_explanation")) logger.debug("Running command :\n{}".format(command)) - os.system(command) + open("/tmp/yunohost-selfupgrade", "w").write("rm /tmp/yunohost-selfupgrade; " + command) + # Using systemd-run --scope is like nohup/disown and &, but more robust somehow + # (despite using nohup/disown and &, the self-upgrade process was still getting killed...) + # ref: https://unix.stackexchange.com/questions/420594/why-process-killed-with-nohup + # (though I still don't understand it 100%...) + os.system("systemd-run --scope bash /tmp/yunohost-selfupgrade &") return else: From fb815d8f3ebfe4bfac43871804a0540af756a4b8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jun 2019 18:13:15 +0200 Subject: [PATCH 56/88] [fix] Ugh we can't specify the file destination name, gotta put directly the parent folder --- debian/install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/install b/debian/install index 3cb7e356a..df65fe51e 100644 --- a/debian/install +++ b/debian/install @@ -1,7 +1,7 @@ bin/* /usr/bin/ sbin/* /usr/sbin/ data/bash-completion.d/yunohost /etc/bash_completion.d/ -doc/yunohost.8.gz /usr/share/man/man8/yunohost.8.gz +doc/yunohost.8.gz /usr/share/man/man8/ data/actionsmap/* /usr/share/moulinette/actionsmap/ data/hooks/* /usr/share/yunohost/hooks/ data/other/yunoprompt.service /etc/systemd/system/ From 984be7987b1722f1abeccc23de2fc9193469d979 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jun 2019 20:22:54 +0200 Subject: [PATCH 57/88] Update acme_tiny.py to 5350420 --- src/yunohost/vendor/acme_tiny/acme_tiny.py | 263 ++++++++++----------- 1 file changed, 131 insertions(+), 132 deletions(-) diff --git a/src/yunohost/vendor/acme_tiny/acme_tiny.py b/src/yunohost/vendor/acme_tiny/acme_tiny.py index f36aef877..ba04e37ad 100644 --- a/src/yunohost/vendor/acme_tiny/acme_tiny.py +++ b/src/yunohost/vendor/acme_tiny/acme_tiny.py @@ -1,73 +1,94 @@ #!/usr/bin/env python +# Copyright Daniel Roesler, under MIT license, see LICENSE at github.com/diafygi/acme-tiny import argparse, subprocess, json, os, sys, base64, binascii, time, hashlib, re, copy, textwrap, logging try: - from urllib.request import urlopen # Python 3 + from urllib.request import urlopen, Request # Python 3 except ImportError: - from urllib2 import urlopen # Python 2 + from urllib2 import urlopen, Request # Python 2 -#DEFAULT_CA = "https://acme-staging.api.letsencrypt.org" -DEFAULT_CA = "https://acme-v01.api.letsencrypt.org" +DEFAULT_CA = "https://acme-v02.api.letsencrypt.org" # DEPRECATED! USE DEFAULT_DIRECTORY_URL INSTEAD +DEFAULT_DIRECTORY_URL = "https://acme-v02.api.letsencrypt.org/directory" LOGGER = logging.getLogger(__name__) LOGGER.addHandler(logging.StreamHandler()) LOGGER.setLevel(logging.INFO) -def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA, no_checks=False): - # helper function base64 encode for jose spec +def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA, disable_check=False, directory_url=DEFAULT_DIRECTORY_URL, contact=None): + directory, acct_headers, alg, jwk = None, None, None, None # global variables + + # helper functions - base64 encode for jose spec def _b64(b): return base64.urlsafe_b64encode(b).decode('utf8').replace("=", "") + # helper function - run external commands + def _cmd(cmd_list, stdin=None, cmd_input=None, err_msg="Command Line Error"): + proc = subprocess.Popen(cmd_list, stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = proc.communicate(cmd_input) + if proc.returncode != 0: + raise IOError("{0}\n{1}".format(err_msg, err)) + return out + + # helper function - make request and automatically parse json response + def _do_request(url, data=None, err_msg="Error", depth=0): + try: + resp = urlopen(Request(url, data=data, headers={"Content-Type": "application/jose+json", "User-Agent": "acme-tiny"})) + resp_data, code, headers = resp.read().decode("utf8"), resp.getcode(), resp.headers + except IOError as e: + resp_data = e.read().decode("utf8") if hasattr(e, "read") else str(e) + code, headers = getattr(e, "code", None), {} + try: + resp_data = json.loads(resp_data) # try to parse json results + except ValueError: + pass # ignore json parsing errors + if depth < 100 and code == 400 and resp_data['type'] == "urn:ietf:params:acme:error:badNonce": + raise IndexError(resp_data) # allow 100 retrys for bad nonces + if code not in [200, 201, 204]: + raise ValueError("{0}:\nUrl: {1}\nData: {2}\nResponse Code: {3}\nResponse: {4}".format(err_msg, url, data, code, resp_data)) + return resp_data, code, headers + + # helper function - make signed requests + def _send_signed_request(url, payload, err_msg, depth=0): + payload64 = _b64(json.dumps(payload).encode('utf8')) + new_nonce = _do_request(directory['newNonce'])[2]['Replay-Nonce'] + protected = {"url": url, "alg": alg, "nonce": new_nonce} + protected.update({"jwk": jwk} if acct_headers is None else {"kid": acct_headers['Location']}) + protected64 = _b64(json.dumps(protected).encode('utf8')) + protected_input = "{0}.{1}".format(protected64, payload64).encode('utf8') + out = _cmd(["openssl", "dgst", "-sha256", "-sign", account_key], stdin=subprocess.PIPE, cmd_input=protected_input, err_msg="OpenSSL Error") + data = json.dumps({"protected": protected64, "payload": payload64, "signature": _b64(out)}) + try: + return _do_request(url, data=data.encode('utf8'), err_msg=err_msg, depth=depth) + except IndexError: # retry bad nonces (they raise IndexError) + return _send_signed_request(url, payload, err_msg, depth=(depth + 1)) + + # helper function - poll until complete + def _poll_until_not(url, pending_statuses, err_msg): + while True: + result, _, _ = _do_request(url, err_msg=err_msg) + if result['status'] in pending_statuses: + time.sleep(2) + continue + return result + # parse account key to get public key log.info("Parsing account key...") - proc = subprocess.Popen(["openssl", "rsa", "-in", account_key, "-noout", "-text"], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = proc.communicate() - if proc.returncode != 0: - raise IOError("OpenSSL Error: {0}".format(err)) - pub_hex, pub_exp = re.search( - r"modulus:\n\s+00:([a-f0-9\:\s]+?)\npublicExponent: ([0-9]+)", - out.decode('utf8'), re.MULTILINE|re.DOTALL).groups() + out = _cmd(["openssl", "rsa", "-in", account_key, "-noout", "-text"], err_msg="OpenSSL Error") + pub_pattern = r"modulus:\n\s+00:([a-f0-9\:\s]+?)\npublicExponent: ([0-9]+)" + pub_hex, pub_exp = re.search(pub_pattern, out.decode('utf8'), re.MULTILINE|re.DOTALL).groups() pub_exp = "{0:x}".format(int(pub_exp)) pub_exp = "0{0}".format(pub_exp) if len(pub_exp) % 2 else pub_exp - header = { - "alg": "RS256", - "jwk": { - "e": _b64(binascii.unhexlify(pub_exp.encode("utf-8"))), - "kty": "RSA", - "n": _b64(binascii.unhexlify(re.sub(r"(\s|:)", "", pub_hex).encode("utf-8"))), - }, + alg = "RS256" + jwk = { + "e": _b64(binascii.unhexlify(pub_exp.encode("utf-8"))), + "kty": "RSA", + "n": _b64(binascii.unhexlify(re.sub(r"(\s|:)", "", pub_hex).encode("utf-8"))), } - accountkey_json = json.dumps(header['jwk'], sort_keys=True, separators=(',', ':')) + accountkey_json = json.dumps(jwk, sort_keys=True, separators=(',', ':')) thumbprint = _b64(hashlib.sha256(accountkey_json.encode('utf8')).digest()) - # helper function make signed requests - def _send_signed_request(url, payload): - payload64 = _b64(json.dumps(payload).encode('utf8')) - protected = copy.deepcopy(header) - protected["nonce"] = urlopen(CA + "/directory").headers['Replay-Nonce'] - protected64 = _b64(json.dumps(protected).encode('utf8')) - proc = subprocess.Popen(["openssl", "dgst", "-sha256", "-sign", account_key], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = proc.communicate("{0}.{1}".format(protected64, payload64).encode('utf8')) - if proc.returncode != 0: - raise IOError("OpenSSL Error: {0}".format(err)) - data = json.dumps({ - "header": header, "protected": protected64, - "payload": payload64, "signature": _b64(out), - }) - try: - resp = urlopen(url, data.encode('utf8')) - return resp.getcode(), resp.read() - except IOError as e: - return getattr(e, "code", None), getattr(e, "read", e.__str__)() - # find domains log.info("Parsing CSR...") - proc = subprocess.Popen(["openssl", "req", "-in", csr, "-noout", "-text"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = proc.communicate() - if proc.returncode != 0: - raise IOError("Error loading {0}: {1}".format(csr, err)) + out = _cmd(["openssl", "req", "-in", csr, "-noout", "-text"], err_msg="Error loading {0}".format(csr)) domains = set([]) common_name = re.search(r"Subject:.*? CN\s?=\s?([^\s,;/]+)", out.decode('utf8')) if common_name is not None: @@ -77,122 +98,100 @@ def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA, no_checks=Fal for san in subject_alt_names.group(1).split(", "): if san.startswith("DNS:"): domains.add(san[4:]) + log.info("Found domains: {0}".format(", ".join(domains))) - # get the certificate domains and expiration + # get the ACME directory of urls + log.info("Getting directory...") + directory_url = CA + "/directory" if CA != DEFAULT_CA else directory_url # backwards compatibility with deprecated CA kwarg + directory, _, _ = _do_request(directory_url, err_msg="Error getting directory") + log.info("Directory found!") + + # create account, update contact details (if any), and set the global key identifier log.info("Registering account...") - code, result = _send_signed_request(CA + "/acme/new-reg", { - "resource": "new-reg", - "agreement": json.loads(urlopen(CA + "/directory").read().decode('utf8'))['meta']['terms-of-service'], - }) - if code == 201: - log.info("Registered!") - elif code == 409: - log.info("Already registered!") - else: - raise ValueError("Error registering: {0} {1}".format(code, result)) + reg_payload = {"termsOfServiceAgreed": True} + account, code, acct_headers = _send_signed_request(directory['newAccount'], reg_payload, "Error registering") + log.info("Registered!" if code == 201 else "Already registered!") + if contact is not None: + account, _, _ = _send_signed_request(acct_headers['Location'], {"contact": contact}, "Error updating contact details") + log.info("Updated contact details:\n{0}".format("\n".join(account['contact']))) - # verify each domain - for domain in domains: + # create a new order + log.info("Creating new order...") + order_payload = {"identifiers": [{"type": "dns", "value": d} for d in domains]} + order, _, order_headers = _send_signed_request(directory['newOrder'], order_payload, "Error creating new order") + log.info("Order created!") + + # get the authorizations that need to be completed + for auth_url in order['authorizations']: + authorization, _, _ = _do_request(auth_url, err_msg="Error getting challenges") + domain = authorization['identifier']['value'] log.info("Verifying {0}...".format(domain)) - # get new challenge - code, result = _send_signed_request(CA + "/acme/new-authz", { - "resource": "new-authz", - "identifier": {"type": "dns", "value": domain}, - }) - if code != 201: - raise ValueError("Error requesting challenges: {0} {1}".format(code, result)) - - # make the challenge file - challenge = [c for c in json.loads(result.decode('utf8'))['challenges'] if c['type'] == "http-01"][0] + # find the http-01 challenge and write the challenge file + challenge = [c for c in authorization['challenges'] if c['type'] == "http-01"][0] token = re.sub(r"[^A-Za-z0-9_\-]", "_", challenge['token']) keyauthorization = "{0}.{1}".format(token, thumbprint) wellknown_path = os.path.join(acme_dir, token) with open(wellknown_path, "w") as wellknown_file: wellknown_file.write(keyauthorization) - if not no_checks: # sometime the local g - # check that the file is in place + # check that the file is in place + try: wellknown_url = "http://{0}/.well-known/acme-challenge/{1}".format(domain, token) - try: - resp = urlopen(wellknown_url) - resp_data = resp.read().decode('utf8').strip() - assert resp_data == keyauthorization - except (IOError, AssertionError): - os.remove(wellknown_path) - raise ValueError("Wrote file to {0}, but couldn't download {1}".format( - wellknown_path, wellknown_url)) + assert(disable_check or _do_request(wellknown_url)[0] == keyauthorization) + except (AssertionError, ValueError) as e: + os.remove(wellknown_path) + raise ValueError("Wrote file to {0}, but couldn't download {1}: {2}".format(wellknown_path, wellknown_url, e)) - # notify challenge are met - code, result = _send_signed_request(challenge['uri'], { - "resource": "challenge", - "keyAuthorization": keyauthorization, - }) - if code != 202: - raise ValueError("Error triggering challenge: {0} {1}".format(code, result)) + # say the challenge is done + _send_signed_request(challenge['url'], {}, "Error submitting challenges: {0}".format(domain)) + authorization = _poll_until_not(auth_url, ["pending"], "Error checking challenge status for {0}".format(domain)) + if authorization['status'] != "valid": + raise ValueError("Challenge did not pass for {0}: {1}".format(domain, authorization)) + log.info("{0} verified!".format(domain)) - # wait for challenge to be verified - while True: - try: - resp = urlopen(challenge['uri']) - challenge_status = json.loads(resp.read().decode('utf8')) - except IOError as e: - raise ValueError("Error checking challenge: {0} {1}".format( - e.code, json.loads(e.read().decode('utf8')))) - if challenge_status['status'] == "pending": - time.sleep(2) - elif challenge_status['status'] == "valid": - log.info("{0} verified!".format(domain)) - os.remove(wellknown_path) - break - else: - raise ValueError("{0} challenge did not pass: {1}".format( - domain, challenge_status)) - - # get the new certificate + # finalize the order with the csr log.info("Signing certificate...") - proc = subprocess.Popen(["openssl", "req", "-in", csr, "-outform", "DER"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - csr_der, err = proc.communicate() - code, result = _send_signed_request(CA + "/acme/new-cert", { - "resource": "new-cert", - "csr": _b64(csr_der), - }) - if code != 201: - raise ValueError("Error signing certificate: {0} {1}".format(code, result)) + csr_der = _cmd(["openssl", "req", "-in", csr, "-outform", "DER"], err_msg="DER Export Error") + _send_signed_request(order['finalize'], {"csr": _b64(csr_der)}, "Error finalizing order") - # return signed certificate! + # poll the order to monitor when it's done + order = _poll_until_not(order_headers['Location'], ["pending", "processing"], "Error checking order status") + if order['status'] != "valid": + raise ValueError("Order failed: {0}".format(order)) + + # download the certificate + certificate_pem, _, _ = _do_request(order['certificate'], err_msg="Certificate download failed") log.info("Certificate signed!") - return """-----BEGIN CERTIFICATE-----\n{0}\n-----END CERTIFICATE-----\n""".format( - "\n".join(textwrap.wrap(base64.b64encode(result).decode('utf8'), 64))) + return certificate_pem -def main(argv): +def main(argv=None): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent("""\ - This script automates the process of getting a signed TLS certificate from - Let's Encrypt using the ACME protocol. It will need to be run on your server - and have access to your private account key, so PLEASE READ THROUGH IT! It's - only ~200 lines, so it won't take long. + This script automates the process of getting a signed TLS certificate from Let's Encrypt using + the ACME protocol. It will need to be run on your server and have access to your private + account key, so PLEASE READ THROUGH IT! It's only ~200 lines, so it won't take long. - ===Example Usage=== - python acme_tiny.py --account-key ./account.key --csr ./domain.csr --acme-dir /usr/share/nginx/html/.well-known/acme-challenge/ > signed.crt - =================== + Example Usage: + python acme_tiny.py --account-key ./account.key --csr ./domain.csr --acme-dir /usr/share/nginx/html/.well-known/acme-challenge/ > signed_chain.crt - ===Example Crontab Renewal (once per month)=== - 0 0 1 * * python /path/to/acme_tiny.py --account-key /path/to/account.key --csr /path/to/domain.csr --acme-dir /usr/share/nginx/html/.well-known/acme-challenge/ > /path/to/signed.crt 2>> /var/log/acme_tiny.log - ============================================== + Example Crontab Renewal (once per month): + 0 0 1 * * python /path/to/acme_tiny.py --account-key /path/to/account.key --csr /path/to/domain.csr --acme-dir /usr/share/nginx/html/.well-known/acme-challenge/ > /path/to/signed_chain.crt 2>> /var/log/acme_tiny.log """) ) parser.add_argument("--account-key", required=True, help="path to your Let's Encrypt account private key") parser.add_argument("--csr", required=True, help="path to your certificate signing request") parser.add_argument("--acme-dir", required=True, help="path to the .well-known/acme-challenge/ directory") parser.add_argument("--quiet", action="store_const", const=logging.ERROR, help="suppress output except for errors") - parser.add_argument("--ca", default=DEFAULT_CA, help="certificate authority, default is Let's Encrypt") + parser.add_argument("--disable-check", default=False, action="store_true", help="disable checking if the challenge file is hosted correctly before telling the CA") + parser.add_argument("--directory-url", default=DEFAULT_DIRECTORY_URL, help="certificate authority directory url, default is Let's Encrypt") + parser.add_argument("--ca", default=DEFAULT_CA, help="DEPRECATED! USE --directory-url INSTEAD!") + parser.add_argument("--contact", metavar="CONTACT", default=None, nargs="*", help="Contact details (e.g. mailto:aaa@bbb.com) for your account-key") args = parser.parse_args(argv) LOGGER.setLevel(args.quiet or LOGGER.level) - signed_crt = get_crt(args.account_key, args.csr, args.acme_dir, log=LOGGER, CA=args.ca) + signed_crt = get_crt(args.account_key, args.csr, args.acme_dir, log=LOGGER, CA=args.ca, disable_check=args.disable_check, directory_url=args.directory_url, contact=args.contact) sys.stdout.write(signed_crt) if __name__ == "__main__": # pragma: no cover From 6f439c917b88f999a00ba37668dbcbeb1a8a8226 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jun 2019 20:40:19 +0200 Subject: [PATCH 58/88] Use the new disable_check arg in the acme_tiny interface --- src/yunohost/certificate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/certificate.py b/src/yunohost/certificate.py index 4235d522d..a4a31055d 100644 --- a/src/yunohost/certificate.py +++ b/src/yunohost/certificate.py @@ -566,7 +566,7 @@ def _fetch_and_enable_new_certificate(domain, staging=False, no_checks=False): domain_csr_file, WEBROOT_FOLDER, log=logger, - no_checks=no_checks, + disable_check=no_checks, CA=certification_authority) except ValueError as e: if "urn:acme:error:rateLimited" in str(e): From 37274a9e54b828a8d6dc8eca3db99c024167efed Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jun 2019 21:33:25 +0200 Subject: [PATCH 59/88] Add redacting mechanism for secrets, using a custom Formatter --- src/yunohost/log.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/yunohost/log.py b/src/yunohost/log.py index dd3bbd8b3..553822ff3 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -289,6 +289,19 @@ def is_unit_operation(entities=['app', 'domain', 'service', 'user'], return decorate +class RedactingFormatter(Formatter): + + def __init__(self, format_string, data_to_redact): + super(RedactingFormatter, self).__init__(format_string) + self.data_to_redact = data_to_redact + + def format(self, record): + msg = super(RedactingFormatter, self).format(record) + for data in self.data_to_redact: + msg = msg.replace(data, "**********") + return msg + + class OperationLogger(object): """ @@ -309,6 +322,7 @@ class OperationLogger(object): self.ended_at = None self.logger = None self._name = None + self.data_to_redact = [] self.path = OPERATIONS_PATH @@ -345,9 +359,12 @@ class OperationLogger(object): Register log with a handler connected on log system """ - # TODO add a way to not save password on app installation self.file_handler = FileHandler(self.log_path) - self.file_handler.formatter = Formatter('%(asctime)s: %(levelname)s - %(message)s') + # We use a custom formatter that's able to redact all stuff in self.data_to_redact + # N.B. : the stubtle thing here is that the class will remember a pointer to the list, + # so we can directly append stuff to self.data_to_redact and that'll be automatically + # propagated to the RedactingFormatter + self.file_handler.formatter = RedactingFormatter('%(asctime)s: %(levelname)s - %(message)s', self.data_to_redact) # Listen to the root logger self.logger = getLogger('yunohost') From b01bf434750faddabe97b44b063a2a3a8b536af2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jun 2019 22:03:52 +0200 Subject: [PATCH 60/88] Change _parse_action_args to also return the arg type :/ --- src/yunohost/app.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 1f172a267..aac7455dd 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -362,10 +362,10 @@ def app_info(app, show_status=False, raw=False): ret['upgradable'] = upgradable ret['change_url'] = os.path.exists(os.path.join(app_setting_path, "scripts", "change_url")) - + with open(os.path.join(APPS_SETTING_PATH, app, 'manifest.json')) as json_manifest: manifest = json.load(json_manifest) - + ret['version'] = manifest.get('version', '-') return ret @@ -490,7 +490,7 @@ def app_change_url(operation_logger, app, domain, path): # Retrieve arguments list for change_url script # TODO: Allow to specify arguments args_odict = _parse_args_from_manifest(manifest, 'change_url') - args_list = args_odict.values() + args_list = [ value[0] for value in args_odict.values() ] args_list.append(app) # Prepare env. var. to pass to script @@ -639,7 +639,7 @@ def app_upgrade(app=[], url=None, file=None): # Retrieve arguments list for upgrade script # TODO: Allow to specify arguments args_odict = _parse_args_from_manifest(manifest, 'upgrade') - args_list = args_odict.values() + args_list = [ value[0] for value in args_odict.values() ] args_list.append(app_instance_name) # Prepare env. var. to pass to script @@ -797,7 +797,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu args_dict = {} if not args else \ dict(urlparse.parse_qsl(args, keep_blank_values=True)) args_odict = _parse_args_from_manifest(manifest, 'install', args=args_dict) - args_list = args_odict.values() + args_list = [ value[0] for value in args_odict.values() ] args_list.append(app_instance_name) # Prepare env. var. to pass to script @@ -1537,7 +1537,7 @@ def app_action_run(app, action, args=None): # Retrieve arguments list for install script args_dict = dict(urlparse.parse_qsl(args, keep_blank_values=True)) if args else {} args_odict = _parse_args_for_action(actions[action], args=args_dict) - args_list = args_odict.values() + args_list = [ value[0] for value in args_odict.values() ] app_id, app_instance_nb = _parse_app_instance_name(app) @@ -2272,7 +2272,7 @@ def _parse_action_args_in_yunohost_format(args, action_args): if arg.get("optional", False): # Argument is optional, keep an empty value # and that's all for this arg ! - args_dict[arg_name] = '' + args_dict[arg_name] = ('', arg_type) continue else: # The argument is required ! @@ -2310,22 +2310,20 @@ def _parse_action_args_in_yunohost_format(args, action_args): raise YunohostError('pattern_password_app', forbidden_chars=forbidden_chars) from yunohost.utils.password import assert_password_is_strong_enough assert_password_is_strong_enough('user', arg_value) - args_dict[arg_name] = arg_value + args_dict[arg_name] = (arg_value, arg_type) # END loop over action_args... # If there's only one "domain" and "path", validate that domain/path # is an available url and normalize the path. - domain_args = [arg["name"] for arg in action_args - if arg.get("type", "string") == "domain"] - path_args = [arg["name"] for arg in action_args - if arg.get("type", "string") == "path"] + domain_args = [ (name, value[0]) for name, value in args_dict.items() if value[1] == "domain" ] + path_args = [ (name, value[0]) for name, value in args_dict.items() if value[1] == "path" ] if len(domain_args) == 1 and len(path_args) == 1: - domain = args_dict[domain_args[0]] - path = args_dict[path_args[0]] + domain = domain_args[0][1] + path = path_args[0][1] domain, path = _normalize_domain_path(domain, path) # Check the url is available @@ -2344,7 +2342,7 @@ def _parse_action_args_in_yunohost_format(args, action_args): # (We save this normalized path so that the install script have a # standard path format to deal with no matter what the user inputted) - args_dict[path_args[0]] = path + args_dict[path_args[0][0]] = (path, "path") return args_dict @@ -2359,8 +2357,8 @@ def _make_environment_dict(args_dict, prefix="APP_ARG_"): """ env_dict = {} - for arg_name, arg_value in args_dict.items(): - env_dict["YNH_%s%s" % (prefix, arg_name.upper())] = arg_value + for arg_name, arg_value_and_type in args_dict.items(): + env_dict["YNH_%s%s" % (prefix, arg_name.upper())] = arg_value_and_type[0] return env_dict From e42ce996603818438fc026dd7f1009a83982430a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jun 2019 22:24:06 +0200 Subject: [PATCH 61/88] Add all password-type argument as data to redact in app install --- src/yunohost/app.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index aac7455dd..0af562add 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -808,6 +808,9 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu # Start register change on system operation_logger.extra.update({'env': env_dict}) + # Tell the operation_logger to redact all password-type args + data_to_redact = [ value[0] for value in args_odict.values() if value[1] == "password" ] + operation_logger.data_to_redact.extend(data_to_redact) operation_logger.related_to = [s for s in operation_logger.related_to if s[0] != "app"] operation_logger.related_to.append(("app", app_id)) operation_logger.start() From 4269c10b80cdab105725cae870de61daa899b99a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jun 2019 22:24:41 +0200 Subject: [PATCH 62/88] Also redact metadata file --- src/yunohost/log.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/yunohost/log.py b/src/yunohost/log.py index 553822ff3..0caae534f 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -375,8 +375,11 @@ class OperationLogger(object): Write or rewrite the metadata file with all metadata known """ + dump = yaml.safe_dump(self.metadata, default_flow_style=False) + for data in self.data_to_redact: + dump = dump.replace(data, "**********") with open(self.md_path, 'w') as outfile: - yaml.safe_dump(self.metadata, outfile, default_flow_style=False) + outfile.write(dump) @property def name(self): From 8f72860e785b5f19377a828d9ac6e00cfcddd79b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jun 2019 22:39:31 +0200 Subject: [PATCH 63/88] Use set +x / -x to hide the whole ynh_script_progression computation --- data/helpers.d/logging | 3 +++ 1 file changed, 3 insertions(+) diff --git a/data/helpers.d/logging b/data/helpers.d/logging index 9e8cc06cb..582d48428 100644 --- a/data/helpers.d/logging +++ b/data/helpers.d/logging @@ -208,6 +208,7 @@ progress_string0="...................." # Define base_time when the file is sourced base_time=$(date +%s) ynh_script_progression () { + set +x # Declare an array to define the options of this helper. local legacy_args=mwtl declare -Ar args_array=( [m]=message= [w]=weight= [t]=time [l]=last ) @@ -217,6 +218,7 @@ ynh_script_progression () { local last # Manage arguments with getopts ynh_handle_getopts_args "$@" + set +x weight=${weight:-1} time=${time:-0} last=${last:-0} @@ -280,6 +282,7 @@ ynh_script_progression () { fi ynh_print_info "[$progression_bar] > ${message}${print_exec_time}" + set -x } # Debugger for app packagers From b9e4b91f1bbeb93c4dfbe3cd9088212afd685186 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jun 2019 23:11:24 +0200 Subject: [PATCH 64/88] Add /etc/yunohost/mysql as data to redact --- src/yunohost/log.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yunohost/log.py b/src/yunohost/log.py index 0caae534f..3631f6bf0 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -323,6 +323,7 @@ class OperationLogger(object): self.logger = None self._name = None self.data_to_redact = [] + self.data_to_redact.append(open("/etc/yunohost/mysql", "r").read().strip()) self.path = OPERATIONS_PATH From b26ec9c2fc9482b0c97bb3e432c08ccb7529dfa0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 19 Jun 2019 23:14:48 +0200 Subject: [PATCH 65/88] Find data to redact on-the-fly corresponding to common stuff --- src/yunohost/log.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/yunohost/log.py b/src/yunohost/log.py index 3631f6bf0..6bb1a4445 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -25,6 +25,7 @@ """ import os +import re import yaml import collections @@ -297,10 +298,22 @@ class RedactingFormatter(Formatter): def format(self, record): msg = super(RedactingFormatter, self).format(record) + self.identify_data_to_redact(msg) for data in self.data_to_redact: msg = msg.replace(data, "**********") return msg + def identify_data_to_redact(self, record): + + # Wrapping this in a try/except because we don't want this to + # break everything in case it fails miserably for some reason :s + try: + match = re.search(r'(db_pwd|password)=(\S{3,})$', record.strip()) + if match and match.group(2) not in self.data_to_redact: + self.data_to_redact.append(match.group(2)) + except Exception as e: + logger.warning("Failed to parse line to try to identify data to redact ... : %s" % e) + class OperationLogger(object): From 5e1377aec197f0fe57f16f50c3f56d08c1a867f9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 24 Jun 2019 17:57:19 +0200 Subject: [PATCH 66/88] Updating changelog for 3.6.2 --- debian/changelog | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/debian/changelog b/debian/changelog index 6bed53f47..c9699d6cb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,17 @@ +yunohost (3.6.2) testing; urgency=low + + - [fix] Use systemd-run for more robust self-upgrade mechanism (158aa08) + - [enh] Add a do_not_backup_data app setting to avoid backing up data (#731) + - [enh] support config_panel in TOML format (#732) + - [fix] ynh_print_OFF when set -x is used in other helpers (#733) + - [enh] Add current and new version for apps in tools_update output (#735) + - [fix] Backup delete should delete symlink target (#738) + - [i18n] Improve translation for Occitan, French + + Thanks to all contributors <3 ! (Aleks, Bram, kay0u, locness3, Maniack, Quentí) + + -- Alexandre Aubin Mon, 24 Jun 2019 18:00:00 +0000 + yunohost (3.6.1.3) testing; urgency=low - [fix] Missing quotes led to an issue during when upgrading postsrsd From 7db63b82218858ee4c1d27c8971129a1d971d2bb Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Tue, 25 Jun 2019 05:19:30 +0200 Subject: [PATCH 67/88] [enh] handle other kinds of type than boolean in config panel --- src/yunohost/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index a200ecae1..8e047169c 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1647,6 +1647,8 @@ def app_config_show_panel(app): option["default"] = parsed_values[generated_name] else: del option["default"] + else: + option["default"] = parsed_values[generated_name] args_dict = _parse_args_in_yunohost_format( [{option["name"]: parsed_values[generated_name]}], From b23296be3cbb97220e788cd5a46c662b77bb22fc Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Tue, 25 Jun 2019 05:37:12 +0200 Subject: [PATCH 68/88] [fix] remove useless line that could generate bugs --- src/yunohost/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 8e047169c..11d9b735b 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1658,7 +1658,7 @@ def app_config_show_panel(app): option["default"] = args_dict[option["name"]] else: logger.debug("Variable '%s' is not declared by config script, using default", generated_name) - option["default"] = option["default"] + # do nothing, we'll use the default if present return { "app_id": app_id, From b9197b62698790c8e3d5339ff13236a91c3db68e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 27 Jun 2019 00:57:28 +0200 Subject: [PATCH 69/88] Fix 'unable to write random state in SSL init (same patch as for yunohost-admin) --- data/hooks/conf_regen/02-ssl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data/hooks/conf_regen/02-ssl b/data/hooks/conf_regen/02-ssl index 963ec12ef..f74da20af 100755 --- a/data/hooks/conf_regen/02-ssl +++ b/data/hooks/conf_regen/02-ssl @@ -23,8 +23,10 @@ do_init_regen() { mkdir -p "${ssl_dir}/"{ca,certs,crl,newcerts} # initialize some files + # N.B. : the weird RANDFILE thing comes from: + # https://stackoverflow.com/questions/94445/using-openssl-what-does-unable-to-write-random-state-mean [[ -f "${ssl_dir}/serial" ]] \ - || openssl rand -hex 19 > "${ssl_dir}/serial" + || RANDFILE=.rnd openssl rand -hex 19 > "${ssl_dir}/serial" [[ -f "${ssl_dir}/index.txt" ]] \ || touch "${ssl_dir}/index.txt" From 6a00aac13acff6faebe60f55a4c59fcb554093ef Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 27 Jun 2019 03:16:46 +0200 Subject: [PATCH 70/88] Catch pwd, pass and password as patterns for password --- src/yunohost/log.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/yunohost/log.py b/src/yunohost/log.py index 6bb1a4445..32b62cb98 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -308,7 +308,9 @@ class RedactingFormatter(Formatter): # Wrapping this in a try/except because we don't want this to # break everything in case it fails miserably for some reason :s try: - match = re.search(r'(db_pwd|password)=(\S{3,})$', record.strip()) + # This matches stuff like db_pwd=the_secret or admin_password=other_secret + # (the secret part being at least 3 chars to avoid catching some lines like just "db_pwd=") + match = re.search(r'(pwd|pass|password)=(\S{3,})$', record.strip()) if match and match.group(2) not in self.data_to_redact: self.data_to_redact.append(match.group(2)) except Exception as e: From 5c7bad84728d0f9790f7ebbabe592ad46fdafbf4 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 27 Jun 2019 03:18:08 +0200 Subject: [PATCH 71/88] Why not use the global variable here ... --- data/helpers.d/postgresql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/postgresql b/data/helpers.d/postgresql index 2b0a86308..3126639a6 100644 --- a/data/helpers.d/postgresql +++ b/data/helpers.d/postgresql @@ -266,7 +266,7 @@ ynh_psql_test_if_first_run() { echo "PostgreSQL is already installed, no need to create master password" else local pgsql="$(ynh_string_random)" - echo "$pgsql" >/etc/yunohost/psql + echo "$pgsql" >$PSQL_ROOT_PWD_FILE if [ -e /etc/postgresql/9.4/ ]; then local pg_hba=/etc/postgresql/9.4/main/pg_hba.conf From 5f1988c2ca195822ef5d742ab4d4b25ec7f7a574 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 27 Jun 2019 03:31:03 +0200 Subject: [PATCH 72/88] Change the variable name so that it's catched by the redacting mechanism --- data/helpers.d/postgresql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data/helpers.d/postgresql b/data/helpers.d/postgresql index 3126639a6..2c1882b8e 100644 --- a/data/helpers.d/postgresql +++ b/data/helpers.d/postgresql @@ -265,8 +265,8 @@ ynh_psql_test_if_first_run() { if [ -f "$PSQL_ROOT_PWD_FILE" ]; then echo "PostgreSQL is already installed, no need to create master password" else - local pgsql="$(ynh_string_random)" - echo "$pgsql" >$PSQL_ROOT_PWD_FILE + local psql_root_password="$(ynh_string_random)" + echo "$psql_root_password" >$PSQL_ROOT_PWD_FILE if [ -e /etc/postgresql/9.4/ ]; then local pg_hba=/etc/postgresql/9.4/main/pg_hba.conf @@ -280,7 +280,7 @@ ynh_psql_test_if_first_run() { ynh_systemd_action --service_name=postgresql --action=start - sudo --login --user=postgres psql -c"ALTER user postgres WITH PASSWORD '$pgsql'" postgres + sudo --login --user=postgres psql -c"ALTER user postgres WITH PASSWORD '$psql_root_password'" postgres # force all user to connect to local database using passwords # https://www.postgresql.org/docs/current/static/auth-pg-hba-conf.html#EXAMPLE-PG-HBA.CONF From aa621abf5775644d6a8c8714601e456fc6848f0d Mon Sep 17 00:00:00 2001 From: Bram Date: Thu, 27 Jun 2019 05:11:04 +0200 Subject: [PATCH 73/88] [doc] typo Co-Authored-By: decentral1se --- src/yunohost/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/log.py b/src/yunohost/log.py index 32b62cb98..c228c2853 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -377,7 +377,7 @@ class OperationLogger(object): self.file_handler = FileHandler(self.log_path) # We use a custom formatter that's able to redact all stuff in self.data_to_redact - # N.B. : the stubtle thing here is that the class will remember a pointer to the list, + # N.B. : the subtle thing here is that the class will remember a pointer to the list, # so we can directly append stuff to self.data_to_redact and that'll be automatically # propagated to the RedactingFormatter self.file_handler.formatter = RedactingFormatter('%(asctime)s: %(levelname)s - %(message)s', self.data_to_redact) From a18f1f784e215a24f6e99a3da6d1d5675da999d0 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 29 Jun 2019 06:03:05 +0200 Subject: [PATCH 74/88] [mod] remove debug stuff --- src/yunohost/app.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 11d9b735b..cad528795 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1654,7 +1654,6 @@ def app_config_show_panel(app): [{option["name"]: parsed_values[generated_name]}], [option] ) - print("%s ----> %s ----> %s" % (option["name"], repr(option.get("default", "[removed -> false]")), args_dict[option["name"]])) option["default"] = args_dict[option["name"]] else: logger.debug("Variable '%s' is not declared by config script, using default", generated_name) @@ -1691,8 +1690,6 @@ def app_config_apply(app, args): "YNH_APP_INSTANCE_NUMBER": str(app_instance_nb), } args = dict(urlparse.parse_qsl(args, keep_blank_values=True)) if args else {} - print(json.dumps(args, sort_keys=True, indent=4)) - print("------------------------------------------") for tab in config_panel.get("panel", []): tab_id = tab["id"] # this makes things easier to debug on crash @@ -1713,7 +1710,6 @@ def app_config_apply(app, args): if key not in env: logger.warning("Ignore key '%s' from arguments because it is not in the config", key) - print(json.dumps(env, sort_keys=True, indent=4)) return_code = hook_exec(config_script, args=["apply"], env=env, From 78391045d8b75dc679086a8f85931b8d2f6ca726 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 29 Jun 2019 07:38:13 +0200 Subject: [PATCH 75/88] [mod] extract manpage template in another file --- doc/generate_manpages.py | 95 +--------------------------------------- doc/manpage.template | 92 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 94 deletions(-) create mode 100644 doc/manpage.template diff --git a/doc/generate_manpages.py b/doc/generate_manpages.py index 694b82cf3..ba90a4f91 100644 --- a/doc/generate_manpages.py +++ b/doc/generate_manpages.py @@ -16,100 +16,7 @@ from collections import OrderedDict from jinja2 import Template -template = Template('''\ -.TH YunoHost "1" "{{ month }} {{ year }}" "YunoHost Collectif" -.SH NAME -YunoHost \\- yunohost server administration command - -.SH SYNOPSIS -yunohost \\fI\\,CATEGORY\\/\\fR \\fI\\,COMMAND\\/\\fR [\\fI\\,SUBCOMMAND\\/\\fR] [\\fI\\,ARGUMENTS\\/\\fR]... [\\fI\\,OPTIONS\\/\\fR]... - -{# generale command format #} -.SH DESCRIPTION -usage: yunohost -{{ '{' }}{{ ",".join(categories) }}{{ '}' }} -\\&... -[\\-h|\\-\\-help] [\\-\\-no\\-cache] [\\-\\-output\\-as {json,plain,none}] [\\-\\-debug] -[\\-\\-quiet] [\\-\\-timeout ==SUPPRESS==] [\\-\\-admin\\-password PASSWORD] -[\\-v|\\-\\-version] - -.SS "optional arguments:" -.TP -\\fB\\-h\\fR, \\fB\\-\\-help\\fR -show this help message and exit - -.SS "categories:" -.IP -{{ '{' }}{{ ",".join(categories) }}{{ '}' }} -{% for name, value in categories.items() %} -.TP -{{ name }} -{{ value["category_help"] }} -{% endfor %} - -.SS "global arguments:" -.TP -\\fB\\-\\-no\\-cache\\fR -Don't use actions map cache -.TP -\\fB\\-\\-output\\-as\\fR {json,plain,none} -Output result in another format -.TP -\\fB\\-\\-debug\\fR -Log and print debug messages -.TP -\\fB\\-\\-quiet\\fR -Don't produce any output -.TP -\\fB\\-\\-timeout\\fR SECONDS -Number of seconds before this command will timeout -because it can't acquire the lock (meaning that -another command is currently running), by default -there is no timeout and the command will wait until it -can get the lock -.TP -\\fB\\-\\-admin\\-password\\fR PASSWORD -The admin password to use to authenticate -.TP -\\fB\\-v\\fR, \\fB\\-\\-version\\fR -Display YunoHost packages versions - -{# each categories #} -{% for name, value in categories.items() %} -.SH YUNOHOST {{ name.upper() }} -usage: yunohost {{ name }} {{ '{' }}{{ ",".join(value["actions"].keys()) }}{{ '}' }} -\\&... -.SS "description:" -.IP -{{ value["category_help"] }} - -{# each command of each category #} -{% for action, action_value in value["actions"].items() %} -.SS "yunohost {{ name }} {{ action }} \ -{% for argument_name, argument_value in action_value.get("arguments", {}).items() %}\ -{% set required=(not str(argument_name).startswith("-")) or argument_value.get("extra", {}).get("required", False) %}\ -{% if not required %}[{% endif %}\ -\\fI\\,{{ argument_name }}\\/\\fR{% if argument_value.get("full") %}|\\fI\\,{{ argument_value["full"] }}\\fR{% endif %}\ -{% if str(argument_name).startswith("-") and not argument_value.get("action") == "store_true" %} {{ (argument_value.get("full", argument_name)).lstrip("-") }}{% endif %}\ -{% if not required %}]{% endif %} \ -{% endfor %}" - -{# help of the command #} -{{ action_value["action_help"] }} - -{# arguments of the command #} -{% if "arguments" in action_value %} -{% for argument_name, argument_value in action_value["arguments"].items() %} -.TP -\\fB{{ argument_name }}\\fR{% if argument_value.get("full") %}, \\fB{{ argument_value["full"] }}\\fR{% endif %}\ -{% if str(argument_name).startswith("-") and not argument_value.get("action") == "store_true" %} \\fI\\,{{ (argument_value.get("full", argument_name)).lstrip("-") }}\\fR {% if "default" in argument_value %}(default: {{ argument_value["default"] }}){% endif %}{% endif %} -{{ argument_value.get("help", "")}} -{% endfor %} - -{% endif %} -{% endfor %} -{% endfor %} -''') +template = Template(open("./manpage.template").read()) THIS_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/doc/manpage.template b/doc/manpage.template new file mode 100644 index 000000000..15931ae85 --- /dev/null +++ b/doc/manpage.template @@ -0,0 +1,92 @@ +.TH YunoHost "1" "{{ month }} {{ year }}" "YunoHost Collectif" +.SH NAME +YunoHost \- yunohost server administration command + +.SH SYNOPSIS +yunohost \fI\,CATEGORY\/\fR \fI\,COMMAND\/\fR [\fI\,SUBCOMMAND\/\fR] [\fI\,ARGUMENTS\/\fR]... [\fI\,OPTIONS\/\fR]... + +{# generale command format #} +.SH DESCRIPTION +usage: yunohost +{{ '{' }}{{ ",".join(categories) }}{{ '}' }} +\&... +[\-h|\-\-help] [\-\-no\-cache] [\-\-output\-as {json,plain,none}] [\-\-debug] +[\-\-quiet] [\-\-timeout ==SUPPRESS==] [\-\-admin\-password PASSWORD] +[\-v|\-\-version] + +.SS "optional arguments:" +.TP +\fB\-h\fR, \fB\-\-help\fR +show this help message and exit + +.SS "categories:" +.IP +{{ '{' }}{{ ",".join(categories) }}{{ '}' }} +{% for name, value in categories.items() %} +.TP +{{ name }} +{{ value["category_help"] }} +{% endfor %} + +.SS "global arguments:" +.TP +\fB\-\-no\-cache\fR +Don't use actions map cache +.TP +\fB\-\-output\-as\fR {json,plain,none} +Output result in another format +.TP +\fB\-\-debug\fR +Log and print debug messages +.TP +\fB\-\-quiet\fR +Don't produce any output +.TP +\fB\-\-timeout\fR SECONDS +Number of seconds before this command will timeout +because it can't acquire the lock (meaning that +another command is currently running), by default +there is no timeout and the command will wait until it +can get the lock +.TP +\fB\-\-admin\-password\fR PASSWORD +The admin password to use to authenticate +.TP +\fB\-v\fR, \fB\-\-version\fR +Display YunoHost packages versions + +{# each categories #} +{% for name, value in categories.items() %} +.SH YUNOHOST {{ name.upper() }} +usage: yunohost {{ name }} {{ '{' }}{{ ",".join(value["actions"].keys()) }}{{ '}' }} +\&... +.SS "description:" +.IP +{{ value["category_help"] }} + +{# each command of each category #} +{% for action, action_value in value["actions"].items() %} +.SS "yunohost {{ name }} {{ action }} \ +{% for argument_name, argument_value in action_value.get("arguments", {}).items() %}\ +{% set required=(not str(argument_name).startswith("-")) or argument_value.get("extra", {}).get("required", False) %}\ +{% if not required %}[{% endif %}\ +\fI\,{{ argument_name }}\/\fR{% if argument_value.get("full") %}|\fI\,{{ argument_value["full"] }}\fR{% endif %}\ +{% if str(argument_name).startswith("-") and not argument_value.get("action") == "store_true" %} {{ (argument_value.get("full", argument_name)).lstrip("-") }}{% endif %}\ +{% if not required %}]{% endif %} \ +{% endfor %}" + +{# help of the command #} +{{ action_value["action_help"] }} + +{# arguments of the command #} +{% if "arguments" in action_value %} +{% for argument_name, argument_value in action_value["arguments"].items() %} +.TP +\fB{{ argument_name }}\fR{% if argument_value.get("full") %}, \fB{{ argument_value["full"] }}\fR{% endif %}\ +{% if str(argument_name).startswith("-") and not argument_value.get("action") == "store_true" %} \fI\,{{ (argument_value.get("full", argument_name)).lstrip("-") }}\fR {% if "default" in argument_value %}(default: {{ argument_value["default"] }}){% endif %}{% endif %} +{{ argument_value.get("help", "")}} +{% endfor %} + +{% endif %} +{% endfor %} +{% endfor %} From 252e3157b7213d8354b54f0d45ee56f490618a86 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 29 Jun 2019 07:41:03 +0200 Subject: [PATCH 76/88] [enh] add command from subcategories in manpage --- doc/manpage.template | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/doc/manpage.template b/doc/manpage.template index 15931ae85..a246e59ac 100644 --- a/doc/manpage.template +++ b/doc/manpage.template @@ -89,4 +89,33 @@ usage: yunohost {{ name }} {{ '{' }}{{ ",".join(value["actions"].keys()) }}{{ '} {% endif %} {% endfor %} + +{# each subcategory #} +{% for subcategory_name, subcategory in value.get("subcategories", {}).items() %} +{% for action, action_value in subcategory["actions"].items() %} +.SS "yunohost {{ subcategory_name }} {{ name }} {{ action }} \ +{% for argument_name, argument_value in action_value.get("arguments", {}).items() %}\ +{% set required=(not str(argument_name).startswith("-")) or argument_value.get("extra", {}).get("required", False) %}\ +{% if not required %}[{% endif %}\ +\fI\,{{ argument_name }}\/\fR{% if argument_value.get("full") %}|\fI\,{{ argument_value["full"] }}\fR{% endif %}\ +{% if str(argument_name).startswith("-") and not argument_value.get("action") == "store_true" %} {{ (argument_value.get("full", argument_name)).lstrip("-") }}{% endif %}\ +{% if not required %}]{% endif %} \ +{% endfor %}" + +{# help of the command #} +{{ action_value["action_help"] }} + +{# arguments of the command #} +{% if "arguments" in action_value %} +{% for argument_name, argument_value in action_value["arguments"].items() %} +.TP +\fB{{ argument_name }}\fR{% if argument_value.get("full") %}, \fB{{ argument_value["full"] }}\fR{% endif %}\ +{% if str(argument_name).startswith("-") and not argument_value.get("action") == "store_true" %} \fI\,{{ (argument_value.get("full", argument_name)).lstrip("-") }}\fR {% if "default" in argument_value %}(default: {{ argument_value["default"] }}){% endif %}{% endif %} +{{ argument_value.get("help", "")}} +{% endfor %} + +{% endif %} +{% endfor %} +{% endfor %} + {% endfor %} From b4be3389fa5f4d33ddd10510dcfeb2204a207356 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sat, 29 Jun 2019 07:42:32 +0200 Subject: [PATCH 77/88] [fix] open file relatively from script location --- doc/generate_manpages.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/generate_manpages.py b/doc/generate_manpages.py index ba90a4f91..0b1251c28 100644 --- a/doc/generate_manpages.py +++ b/doc/generate_manpages.py @@ -16,7 +16,9 @@ from collections import OrderedDict from jinja2 import Template -template = Template(open("./manpage.template").read()) +base_path = os.path.split(os.path.realpath(__file__))[0] + +template = Template(open(os.path.join(base_path, "manpage.template")).read()) THIS_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) From a083cba14927907b69ae717ee59ff7e5021eaffa Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 30 Jun 2019 07:37:14 +0200 Subject: [PATCH 78/88] [enh] allows actions.toml --- src/yunohost/app.py | 97 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 90 insertions(+), 7 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index b9306eeb8..a85f89367 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -363,10 +363,10 @@ def app_info(app, show_status=False, raw=False): ret['upgradable'] = upgradable ret['change_url'] = os.path.exists(os.path.join(app_setting_path, "scripts", "change_url")) - + with open(os.path.join(APPS_SETTING_PATH, app, 'manifest.json')) as json_manifest: manifest = json.load(json_manifest) - + ret['version'] = manifest.get('version', '-') return ret @@ -686,7 +686,7 @@ def app_upgrade(app=[], url=None, file=None): os.system('rm -rf "%s/scripts" "%s/manifest.json %s/conf"' % (app_setting_path, app_setting_path, app_setting_path)) os.system('mv "%s/manifest.json" "%s/scripts" %s' % (extracted_app_folder, extracted_app_folder, app_setting_path)) - for file_to_copy in ["actions.json", "config_panel.json", "config_panel.toml", "conf"]: + for file_to_copy in ["actions.json", "actions.toml", "config_panel.json", "config_panel.toml", "conf"]: if os.path.exists(os.path.join(extracted_app_folder, file_to_copy)): os.system('cp -R %s/%s %s' % (extracted_app_folder, file_to_copy, app_setting_path)) @@ -841,7 +841,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu os.system('cp %s/manifest.json %s' % (extracted_app_folder, app_setting_path)) os.system('cp -R %s/scripts %s' % (extracted_app_folder, app_setting_path)) - for file_to_copy in ["actions.json", "config_panel.json", "config_panel.toml", "conf"]: + for file_to_copy in ["actions.json", "actions.toml", "config_panel.json", "config_panel.toml", "conf"]: if os.path.exists(os.path.join(extracted_app_folder, file_to_copy)): os.system('cp -R %s/%s %s' % (extracted_app_folder, file_to_copy, app_setting_path)) @@ -1511,12 +1511,10 @@ def app_action_list(app): # this will take care of checking if the app is installed app_info_dict = app_info(app) - actions = os.path.join(APPS_SETTING_PATH, app, 'actions.json') - return { "app": app, "app_name": app_info_dict["name"], - "actions": read_json(actions) if os.path.exists(actions) else [], + "actions": _get_app_actions(app) } @@ -1721,6 +1719,91 @@ def app_config_apply(app, args): logger.success("Config updated as expected") +def _get_app_actions(app_id): + "Get app config panel stored in json or in toml" + actions_toml_path = os.path.join(APPS_SETTING_PATH, app_id, 'actions.toml') + actions_json_path = os.path.join(APPS_SETTING_PATH, app_id, 'actions.json') + + # sample data to get an idea of what is going on + # this toml extract: + # + + # [restart_service] + # name = "Restart service" + # command = "echo pouet $YNH_ACTION_SERVICE" + # user = "root" # optional + # cwd = "/" # optional + # accepted_return_codes = [0, 1, 2, 3] # optional + # description.en = "a dummy stupid exemple or restarting a service" + # + # [restart_service.arguments.service] + # type = "string", + # ask.en = "service to restart" + # example = "nginx" + # + # will be parsed into this: + # + # OrderedDict([(u'restart_service', + # OrderedDict([(u'name', u'Restart service'), + # (u'command', u'echo pouet $YNH_ACTION_SERVICE'), + # (u'user', u'root'), + # (u'cwd', u'/'), + # (u'accepted_return_codes', [0, 1, 2, 3]), + # (u'description', + # OrderedDict([(u'en', + # u'a dummy stupid exemple or restarting a service')])), + # (u'arguments', + # OrderedDict([(u'service', + # OrderedDict([(u'type', u'string'), + # (u'ask', + # OrderedDict([(u'en', + # u'service to restart')])), + # (u'example', + # u'nginx')]))]))])), + # + # + # and needs to be converted into this: + # + # [{u'accepted_return_codes': [0, 1, 2, 3], + # u'arguments': [{u'ask': {u'en': u'service to restart'}, + # u'example': u'nginx', + # u'name': u'service', + # u'type': u'string'}], + # u'command': u'echo pouet $YNH_ACTION_SERVICE', + # u'cwd': u'/', + # u'description': {u'en': u'a dummy stupid exemple or restarting a service'}, + # u'id': u'restart_service', + # u'name': u'Restart service', + # u'user': u'root'}] + + if os.path.exists(actions_toml_path): + toml_actions = toml.load(open(actions_toml_path, "r"), _dict=OrderedDict) + + # transform toml format into json format + actions = [] + + for key, value in toml_actions.items(): + action = dict(**value) + action["id"] = key + + arguments = [] + for argument_name, argument in value.get("arguments", {}).items(): + argument = dict(**argument) + argument["name"] = argument_name + + arguments.append(argument) + + action["arguments"] = arguments + actions.append(action) + + return actions + + elif os.path.exists(actions_json_path): + return json.load(open(actions_json_path)) + + return None + + def _get_app_config_panel(app_id): "Get app config panel stored in json or in toml" config_panel_toml_path = os.path.join(APPS_SETTING_PATH, app_id, 'config_panel.toml') From 552eb957316b24a6242bb911f6dd3ce5d3ad695a Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Tue, 2 Jul 2019 14:37:07 +0200 Subject: [PATCH 79/88] [fix] Missing internetcube list --- src/yunohost/data_migrations/0010_migrate_to_apps_json.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/yunohost/data_migrations/0010_migrate_to_apps_json.py b/src/yunohost/data_migrations/0010_migrate_to_apps_json.py index 78f6d733b..43ae9a86f 100644 --- a/src/yunohost/data_migrations/0010_migrate_to_apps_json.py +++ b/src/yunohost/data_migrations/0010_migrate_to_apps_json.py @@ -25,7 +25,8 @@ class MyMigration(Migration): "app.yunohost.org/list.json", # Old list on old installs, alias to official.json "app.yunohost.org/official.json", "app.yunohost.org/community.json", - "labriqueinter.net/apps/labriqueinternet.json" + "labriqueinter.net/apps/labriqueinternet.json", + "labriqueinter.net/internetcube.json" ] appslists = _read_appslist_list() From 18f3540e2f89bf2631f147a19092efee0030ed9f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 2 Jul 2019 15:55:31 +0200 Subject: [PATCH 80/88] Also redact psql password if it exists --- src/yunohost/log.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/yunohost/log.py b/src/yunohost/log.py index c228c2853..decb546f4 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -338,7 +338,10 @@ class OperationLogger(object): self.logger = None self._name = None self.data_to_redact = [] - self.data_to_redact.append(open("/etc/yunohost/mysql", "r").read().strip()) + + for filename in ["/etc/yunohost/mysql", "/etc/yunohost/psql"]: + if os.path.exists(filename): + self.data_to_redact.append(read_file(filename).strip()) self.path = OPERATIONS_PATH From 028a44acc9fa097c7fb8c3071946aac3f8bbdd28 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 2 Jul 2019 22:25:02 +0200 Subject: [PATCH 81/88] Add missing python-jinja2 build dependency (to build manpages) --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index b17be5148..64c7cd31d 100644 --- a/debian/control +++ b/debian/control @@ -2,7 +2,7 @@ Source: yunohost Section: utils Priority: extra Maintainer: YunoHost Contributors -Build-Depends: debhelper (>=9), dh-systemd, dh-python, python-all (>= 2.7), python-yaml +Build-Depends: debhelper (>=9), dh-systemd, dh-python, python-all (>= 2.7), python-yaml, python-jinja2 Standards-Version: 3.9.6 X-Python-Version: >= 2.7 Homepage: https://yunohost.org/ From 83445599ede53141a9b8b3173c505cda9d4364be Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 2 Jul 2019 22:41:43 +0200 Subject: [PATCH 82/88] [fix] Typo breaking backup_delete --- src/yunohost/backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 6745df865..d374653b3 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -2322,7 +2322,7 @@ def backup_delete(name): files_to_delete = [archive_file, info_file] # To handle the case where archive_file is in fact a symlink - if os.islink(archive_file): + if os.path.islink(archive_file): actual_archive = os.path.realpath(archive_file) files_to_delete.append(actual_archive) From 61f8c9efc46fcb7c168bb32eb389a229bb72632a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 2 Jul 2019 23:11:27 +0200 Subject: [PATCH 83/88] Update changelog for 3.6.3 --- debian/changelog | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/debian/changelog b/debian/changelog index c9699d6cb..e6f54ea04 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,18 @@ +yunohost (3.6.3) testing; urgency=low + + - [fix] Less logging madness due ynh_script_progression building progress bar (#741) + - [fix] Update acme-tiny to 4.0.4 (#740) + - [fix] Missing old internet cube list in migration to unified apps.json (#745) + - [enh] Add manpage for Yunohost ! (#682) + - [enh] Config panel : use manifest.json/actions.json args format for config_panel.toml (#734) + - [enh] Allow to describe actions through toml file instead of json (#744) + - [mod] Proper return interface for app config panel (#739) + - [fix] Add mechanism to automatically detect and redact passwords from operation logs (#742) + + Thanks to all contributors <3 ! (Aleks, Bram, ljf, toitoinebzh) + + -- Alexandre Aubin Tue, 02 Jul 2019 11:10:00 +0000 + yunohost (3.6.2) testing; urgency=low - [fix] Use systemd-run for more robust self-upgrade mechanism (158aa08) From f0ad9440cf3ce1406c6e17ffe53859c9952f8010 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 3 Jul 2019 21:01:39 +0200 Subject: [PATCH 84/88] Be more robust against empty metadata file --- src/yunohost/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/log.py b/src/yunohost/log.py index decb546f4..17a6ff87c 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -108,7 +108,7 @@ def log_list(category=[], limit=None, with_details=False): except yaml.YAMLError: logger.warning(m18n.n('log_corrupted_md_file', file=md_path)) - entry["success"] = metadata.get("success", "?") + entry["success"] = metadata.get("success", "?") if metadata else "?" result[category].append(entry) From 41c0689af3a4afa2529d6ce8db40093dde23e17f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 3 Jul 2019 21:14:16 +0200 Subject: [PATCH 85/88] Add '+' in front of messages so that they're displayed in grey in webadmin logs instead of blue, to being able to better visually differentiate these --- src/yunohost/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 294d9a3fa..b635d408b 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -654,7 +654,7 @@ def tools_upgrade(operation_logger, apps=None, system=False): logger.debug("Running apt command :\n{}".format(dist_upgrade)) callbacks = ( - lambda l: logger.info(l.rstrip() + "\r"), + lambda l: logger.info("+" + l.rstrip() + "\r"), lambda l: logger.warning(l.rstrip()), ) returncode = call_async_output(dist_upgrade, callbacks, shell=True) From a25fe53a70c89483bb097c6b243e721ff719bbc2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 Jul 2019 23:50:17 +0200 Subject: [PATCH 86/88] Update changelog for 3.6.4 release --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index e6f54ea04..b3237fab8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +yunohost (3.6.4) stable; urgency=low + + Minor fixes + bumping version for stable release + + -- Alexandre Aubin Thu, 04 Jul 2019 23:30:00 +0000 + yunohost (3.6.3) testing; urgency=low - [fix] Less logging madness due ynh_script_progression building progress bar (#741) From fbfa249ff5e1a601475da5d9968ebc47cdb190dc Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 5 Jul 2019 20:50:13 +0200 Subject: [PATCH 87/88] Remove binding to IPv6 ::1 as it makes ipv4-only instances crash... (unable to bind to ::1) --- data/templates/slapd/slapd.default | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/templates/slapd/slapd.default b/data/templates/slapd/slapd.default index 0041b30c5..cb6f1b6d0 100644 --- a/data/templates/slapd/slapd.default +++ b/data/templates/slapd/slapd.default @@ -21,7 +21,7 @@ SLAPD_PIDFILE= # sockets. # Example usage: # SLAPD_SERVICES="ldap://127.0.0.1:389/ ldaps:/// ldapi:///" -SLAPD_SERVICES="ldap://127.0.0.1:389/ ldap://[::1]:389/ ldapi:///" +SLAPD_SERVICES="ldap://127.0.0.1:389/ ldaps:/// ldapi:///" # If SLAPD_NO_START is set, the init script will not start or restart # slapd (but stop will still work). Uncomment this if you are From a7f920a30c5263ffbdecb04f8ba0ca996b0c4b1c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 5 Jul 2019 20:52:28 +0200 Subject: [PATCH 88/88] Updating changelog for 3.6.4.1 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index b3237fab8..1d2481513 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +yunohost (3.6.4.1) stable; urgency=low + + - [hotfix] Slapd not being able to start on ipv4-only instances + + -- Alexandre Aubin Fri, 05 Jul 2019 20:50:00 +0000 + yunohost (3.6.4) stable; urgency=low Minor fixes + bumping version for stable release