diff --git a/data/helpers.d/backup b/data/helpers.d/backup index ee524ef7f..cfffe13d0 100644 --- a/data/helpers.d/backup +++ b/data/helpers.d/backup @@ -59,12 +59,18 @@ ynh_backup() { local not_mandatory="${not_mandatory:-0}" BACKUP_CORE_ONLY=${BACKUP_CORE_ONLY:-0} + test -n "${app:-}" && do_not_backup_data=$(ynh_app_setting_get --app=$app --key=do_not_backup_data) # If backing up core only (used by ynh_backup_before_upgrade), # don't backup big data items - if [ "$is_big" == "1" ] && [ "$BACKUP_CORE_ONLY" == "1" ] ; then - ynh_print_info --message="$src_path will not be saved, because backup_core_only is set." - return 0 + if [ $is_big -eq 1 ] && ( [ ${do_not_backup_data:-0} -eq 1 ] || [ $BACKUP_CORE_ONLY -eq 1 ] ) + then + if [ $BACKUP_CORE_ONLY -eq 1 ]; then + ynh_print_warn --message="$src_path will not be saved, because 'BACKUP_CORE_ONLY' is set." + else + ynh_print_warn --message="$src_path will not be saved, because 'do_not_backup_data' is set." + fi + return 0 fi # ============================================================================== diff --git a/data/helpers.d/logging b/data/helpers.d/logging index c46000509..8464a414f 100644 --- a/data/helpers.d/logging +++ b/data/helpers.d/logging @@ -172,7 +172,7 @@ ynh_exec_fully_quiet () { # # Requires YunoHost version 3.2.0 or higher. ynh_print_OFF () { - set +x + exec {BASH_XTRACEFD}>/dev/null } # Restore the logging after ynh_print_OFF @@ -181,7 +181,7 @@ ynh_print_OFF () { # # Requires YunoHost version 3.2.0 or higher. ynh_print_ON () { - set -x + exec {BASH_XTRACEFD}>&1 # Print an echo only for the log, to be able to know that ynh_print_ON has been called. echo ynh_print_ON > /dev/null } @@ -208,6 +208,7 @@ progress_string0="...................." # Define base_time when the file is sourced base_time=$(date +%s) ynh_script_progression () { + set +x # Declare an array to define the options of this helper. local legacy_args=mwtl declare -Ar args_array=( [m]=message= [w]=weight= [t]=time [l]=last ) @@ -217,6 +218,7 @@ ynh_script_progression () { local last # Manage arguments with getopts ynh_handle_getopts_args "$@" + set +x weight=${weight:-1} time=${time:-0} last=${last:-0} @@ -280,6 +282,17 @@ ynh_script_progression () { fi ynh_print_info "[$progression_bar] > ${message}${print_exec_time}" + set -x +} + +# Return data to the Yunohost core for later processing +# (to be used by special hooks like app config panel and core diagnosis) +# +# usage: ynh_return somedata +# +# Requires YunoHost version 3.6.0 or higher. +ynh_return () { + echo "$1" >> "$YNH_STDRETURN" } # Debugger for app packagers diff --git a/data/helpers.d/mysql b/data/helpers.d/mysql index 39f93891c..372819025 100644 --- a/data/helpers.d/mysql +++ b/data/helpers.d/mysql @@ -225,7 +225,6 @@ ynh_mysql_remove_db () { local mysql_root_password=$(sudo cat $MYSQL_ROOT_PWD_FILE) if mysqlshow -u root -p$mysql_root_password | grep -q "^| $db_name"; then # Check if the database exists - ynh_print_info --message="Removing database $db_name" ynh_mysql_drop_db $db_name # Remove the database else ynh_print_warn --message="Database $db_name not found" diff --git a/data/helpers.d/postgresql b/data/helpers.d/postgresql index 8c96c78ac..2c1882b8e 100644 --- a/data/helpers.d/postgresql +++ b/data/helpers.d/postgresql @@ -244,7 +244,6 @@ ynh_psql_remove_db() { local psql_root_password=$(sudo cat $PSQL_ROOT_PWD_FILE) if ynh_psql_database_exists --database=$db_name; then # Check if the database exists - ynh_print_info --message="Removing database $db_name" ynh_psql_drop_db $db_name # Remove the database else ynh_print_warn --message="Database $db_name not found" @@ -252,7 +251,6 @@ ynh_psql_remove_db() { # Remove psql user if it exists if ynh_psql_user_exists --user=$db_user; then - ynh_print_info --message="Removing user $db_user" ynh_psql_drop_user $db_user else ynh_print_warn --message="User $db_user not found" @@ -267,8 +265,8 @@ ynh_psql_test_if_first_run() { if [ -f "$PSQL_ROOT_PWD_FILE" ]; then echo "PostgreSQL is already installed, no need to create master password" else - local pgsql="$(ynh_string_random)" - echo "$pgsql" >/etc/yunohost/psql + local psql_root_password="$(ynh_string_random)" + echo "$psql_root_password" >$PSQL_ROOT_PWD_FILE if [ -e /etc/postgresql/9.4/ ]; then local pg_hba=/etc/postgresql/9.4/main/pg_hba.conf @@ -282,7 +280,7 @@ ynh_psql_test_if_first_run() { ynh_systemd_action --service_name=postgresql --action=start - sudo --login --user=postgres psql -c"ALTER user postgres WITH PASSWORD '$pgsql'" postgres + sudo --login --user=postgres psql -c"ALTER user postgres WITH PASSWORD '$psql_root_password'" postgres # force all user to connect to local database using passwords # https://www.postgresql.org/docs/current/static/auth-pg-hba-conf.html#EXAMPLE-PG-HBA.CONF diff --git a/data/helpers.d/systemd b/data/helpers.d/systemd index c4100bf8a..c3a8da822 100644 --- a/data/helpers.d/systemd +++ b/data/helpers.d/systemd @@ -120,8 +120,6 @@ ynh_systemd_action() { fi fi - ynh_print_info --message="${action^} the service $service_name" - # Use reload-or-restart instead of reload. So it wouldn't fail if the service isn't running. if [ "$action" == "reload" ]; then action="reload-or-restart" diff --git a/data/helpers.d/user b/data/helpers.d/user index 0c4591dcd..e7890ccb2 100644 --- a/data/helpers.d/user +++ b/data/helpers.d/user @@ -145,7 +145,6 @@ ynh_system_user_delete () { # Check if the user exists on the system if ynh_system_user_exists "$username" then - ynh_print_info --message="Remove the user $username" deluser $username else ynh_print_warn --message="The user $username was not found" @@ -154,7 +153,6 @@ ynh_system_user_delete () { # Check if the group exists on the system if ynh_system_group_exists "$username" then - ynh_print_info --message="Remove the group $username" delgroup $username fi } diff --git a/data/hooks/conf_regen/02-ssl b/data/hooks/conf_regen/02-ssl index 963ec12ef..f74da20af 100755 --- a/data/hooks/conf_regen/02-ssl +++ b/data/hooks/conf_regen/02-ssl @@ -23,8 +23,10 @@ do_init_regen() { mkdir -p "${ssl_dir}/"{ca,certs,crl,newcerts} # initialize some files + # N.B. : the weird RANDFILE thing comes from: + # https://stackoverflow.com/questions/94445/using-openssl-what-does-unable-to-write-random-state-mean [[ -f "${ssl_dir}/serial" ]] \ - || openssl rand -hex 19 > "${ssl_dir}/serial" + || RANDFILE=.rnd openssl rand -hex 19 > "${ssl_dir}/serial" [[ -f "${ssl_dir}/index.txt" ]] \ || touch "${ssl_dir}/index.txt" diff --git a/data/hooks/conf_regen/06-slapd b/data/hooks/conf_regen/06-slapd index 9ba223e4c..fdb7a36d1 100755 --- a/data/hooks/conf_regen/06-slapd +++ b/data/hooks/conf_regen/06-slapd @@ -101,6 +101,8 @@ do_post_regen() { sudo chown -R openldap:openldap /etc/ldap/slapd.d/ fi + sudo -u openldap slapindex + sudo service slapd force-reload # on slow hardware/vm this regen conf would exit before the admin user that diff --git a/data/templates/postfix/postsrsd b/data/templates/postfix/postsrsd index 56bfd091e..a0451faf6 100644 --- a/data/templates/postfix/postsrsd +++ b/data/templates/postfix/postsrsd @@ -12,7 +12,7 @@ SRS_DOMAIN={{ main_domain }} # the domain itself. Separate multiple domains by space or comma. # We have to put some "dummy" stuff at start and end... see this comment : # https://github.com/roehling/postsrsd/issues/64#issuecomment-284003762 -SRS_EXCLUDE_DOMAINS=dummy {{ domain_list }} dummy +SRS_EXCLUDE_DOMAINS="dummy {{ domain_list }} dummy" # First separator character after SRS0 or SRS1. # Can be one of: -+= diff --git a/data/templates/slapd/slapd.default b/data/templates/slapd/slapd.default index 0041b30c5..cb6f1b6d0 100644 --- a/data/templates/slapd/slapd.default +++ b/data/templates/slapd/slapd.default @@ -21,7 +21,7 @@ SLAPD_PIDFILE= # sockets. # Example usage: # SLAPD_SERVICES="ldap://127.0.0.1:389/ ldaps:/// ldapi:///" -SLAPD_SERVICES="ldap://127.0.0.1:389/ ldap://[::1]:389/ ldapi:///" +SLAPD_SERVICES="ldap://127.0.0.1:389/ ldaps:/// ldapi:///" # If SLAPD_NO_START is set, the init script will not start or restart # slapd (but stop will still work). Uncomment this if you are diff --git a/debian/changelog b/debian/changelog index 0a92b9e51..1d2481513 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,77 @@ +yunohost (3.6.4.1) stable; urgency=low + + - [hotfix] Slapd not being able to start on ipv4-only instances + + -- Alexandre Aubin Fri, 05 Jul 2019 20:50:00 +0000 + +yunohost (3.6.4) stable; urgency=low + + Minor fixes + bumping version for stable release + + -- Alexandre Aubin Thu, 04 Jul 2019 23:30:00 +0000 + +yunohost (3.6.3) testing; urgency=low + + - [fix] Less logging madness due ynh_script_progression building progress bar (#741) + - [fix] Update acme-tiny to 4.0.4 (#740) + - [fix] Missing old internet cube list in migration to unified apps.json (#745) + - [enh] Add manpage for Yunohost ! (#682) + - [enh] Config panel : use manifest.json/actions.json args format for config_panel.toml (#734) + - [enh] Allow to describe actions through toml file instead of json (#744) + - [mod] Proper return interface for app config panel (#739) + - [fix] Add mechanism to automatically detect and redact passwords from operation logs (#742) + + Thanks to all contributors <3 ! (Aleks, Bram, ljf, toitoinebzh) + + -- Alexandre Aubin Tue, 02 Jul 2019 11:10:00 +0000 + +yunohost (3.6.2) testing; urgency=low + + - [fix] Use systemd-run for more robust self-upgrade mechanism (158aa08) + - [enh] Add a do_not_backup_data app setting to avoid backing up data (#731) + - [enh] support config_panel in TOML format (#732) + - [fix] ynh_print_OFF when set -x is used in other helpers (#733) + - [enh] Add current and new version for apps in tools_update output (#735) + - [fix] Backup delete should delete symlink target (#738) + - [i18n] Improve translation for Occitan, French + + Thanks to all contributors <3 ! (Aleks, Bram, kay0u, locness3, Maniack, Quentí) + + -- Alexandre Aubin Mon, 24 Jun 2019 18:00:00 +0000 + +yunohost (3.6.1.3) testing; urgency=low + + - [fix] Missing quotes led to an issue during when upgrading postsrsd + - [fix] Running slapindex seems to fix the previous issues about LDAP indexing stuff + + -- Alexandre Aubin Fri, 07 Jun 2019 06:38:00 +0000 + +yunohost (3.6.1.2) testing; urgency=low + + - [fix] More weird issues with slapd indexation ... + - [fix] Small issue with operation logging during failed upgrade (success status set to true) + + -- Alexandre Aubin Wed, 05 Jun 2019 16:25:00 +0000 + +yunohost (3.6.1.1) testing; urgency=low + + - [fix] Weird issue in slapd triggered by indexing uidNumber / gidNumber + + -- Alexandre Aubin Tue, 04 Jun 2019 15:10:00 +0000 + +yunohost (3.6.1) testing; urgency=low + + - [fix] current version in app_info (#730) + - [fix] Add indexes for fields listed by slapd in the logs (#729) + - [fix] Allow to display logs when postinstall fails (#728) + - [fix] Stupid issue with files inside tar : foo is not the same as ./foo (#726) + - [enh] Remove unecessary log messages (#724) + - [enh] Check for obvious conflict with already running apt/dpkg commands when running yunohost upgrade (d0c982a) + + Thanks to all contributors <3 ! (Aleks, Kay0u, Bram, L. Murphy, MCMic) + + -- Alexandre Aubin Tue, 04 Jun 2019 13:20:00 +0000 + yunohost (3.6.0) testing; urgency=low ## Major changes diff --git a/debian/control b/debian/control index f19ddd465..64c7cd31d 100644 --- a/debian/control +++ b/debian/control @@ -2,7 +2,7 @@ Source: yunohost Section: utils Priority: extra Maintainer: YunoHost Contributors -Build-Depends: debhelper (>=9), dh-systemd, dh-python, python-all (>= 2.7), python-yaml +Build-Depends: debhelper (>=9), dh-systemd, dh-python, python-all (>= 2.7), python-yaml, python-jinja2 Standards-Version: 3.9.6 X-Python-Version: >= 2.7 Homepage: https://yunohost.org/ @@ -14,6 +14,7 @@ Depends: ${python:Depends}, ${misc:Depends} , moulinette (>= 2.7.1), ssowat (>= 2.7.1) , python-psutil, python-requests, python-dnspython, python-openssl , python-apt, python-miniupnpc, python-dbus, python-jinja2 + , python-toml , glances, apt-transport-https , dnsutils, bind9utils, unzip, git, curl, cron, wget, jq , ca-certificates, netcat-openbsd, iproute diff --git a/debian/install b/debian/install index b540ca749..df65fe51e 100644 --- a/debian/install +++ b/debian/install @@ -1,6 +1,7 @@ bin/* /usr/bin/ sbin/* /usr/sbin/ data/bash-completion.d/yunohost /etc/bash_completion.d/ +doc/yunohost.8.gz /usr/share/man/man8/ data/actionsmap/* /usr/share/moulinette/actionsmap/ data/hooks/* /usr/share/yunohost/hooks/ data/other/yunoprompt.service /etc/systemd/system/ diff --git a/debian/rules b/debian/rules index d012c73f3..8afe372b5 100755 --- a/debian/rules +++ b/debian/rules @@ -10,6 +10,7 @@ override_dh_auto_build: # Generate bash completion file python data/actionsmap/yunohost_completion.py + python doc/generate_manpages.py --gzip --output doc/yunohost.8.gz override_dh_installinit: dh_installinit -pyunohost --name=yunohost-api --restart-after-upgrade diff --git a/doc/generate_manpages.py b/doc/generate_manpages.py new file mode 100644 index 000000000..0b1251c28 --- /dev/null +++ b/doc/generate_manpages.py @@ -0,0 +1,85 @@ +""" +Inspired by yunohost_completion.py (author: Christophe Vuillot) +======= + +This script generates man pages for yunohost. +Pages are stored in OUTPUT_DIR +""" + +import os +import yaml +import gzip +import argparse + +from datetime import date +from collections import OrderedDict + +from jinja2 import Template + +base_path = os.path.split(os.path.realpath(__file__))[0] + +template = Template(open(os.path.join(base_path, "manpage.template")).read()) + + +THIS_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +ACTIONSMAP_FILE = os.path.join(THIS_SCRIPT_DIR, '../data/actionsmap/yunohost.yml') + + +def ordered_yaml_load(stream): + class OrderedLoader(yaml.Loader): + pass + OrderedLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + lambda loader, node: OrderedDict(loader.construct_pairs(node))) + return yaml.load(stream, OrderedLoader) + + +def main(): + parser = argparse.ArgumentParser(description="generate yunohost manpage based on actionsmap.yml") + parser.add_argument("-o", "--output", default="output/yunohost") + parser.add_argument("-z", "--gzip", action="store_true", default=False) + + args = parser.parse_args() + + if os.path.isdir(args.output): + if not os.path.exists(args.output): + os.makedirs(args.output) + + output_path = os.path.join(args.output, "yunohost") + else: + output_dir = os.path.split(args.output)[0] + + if output_dir and not os.path.exists(output_dir): + os.makedirs(output_dir) + + output_path = args.output + + # man pages of "yunohost *" + with open(ACTIONSMAP_FILE, 'r') as actionsmap: + + # Getting the dictionary containning what actions are possible per domain + actionsmap = ordered_yaml_load(actionsmap) + + for i in actionsmap.keys(): + if i.startswith("_"): + del actionsmap[i] + + today = date.today() + + result = template.render( + month=today.strftime("%B"), + year=today.year, + categories=actionsmap, + str=str, + ) + + if not args.gzip: + with open(output_path, "w") as output: + output.write(result) + else: + with gzip.open(output_path, mode="w", compresslevel=9) as output: + output.write(result) + + +if __name__ == '__main__': + main() diff --git a/doc/manpage.template b/doc/manpage.template new file mode 100644 index 000000000..a246e59ac --- /dev/null +++ b/doc/manpage.template @@ -0,0 +1,121 @@ +.TH YunoHost "1" "{{ month }} {{ year }}" "YunoHost Collectif" +.SH NAME +YunoHost \- yunohost server administration command + +.SH SYNOPSIS +yunohost \fI\,CATEGORY\/\fR \fI\,COMMAND\/\fR [\fI\,SUBCOMMAND\/\fR] [\fI\,ARGUMENTS\/\fR]... [\fI\,OPTIONS\/\fR]... + +{# generale command format #} +.SH DESCRIPTION +usage: yunohost +{{ '{' }}{{ ",".join(categories) }}{{ '}' }} +\&... +[\-h|\-\-help] [\-\-no\-cache] [\-\-output\-as {json,plain,none}] [\-\-debug] +[\-\-quiet] [\-\-timeout ==SUPPRESS==] [\-\-admin\-password PASSWORD] +[\-v|\-\-version] + +.SS "optional arguments:" +.TP +\fB\-h\fR, \fB\-\-help\fR +show this help message and exit + +.SS "categories:" +.IP +{{ '{' }}{{ ",".join(categories) }}{{ '}' }} +{% for name, value in categories.items() %} +.TP +{{ name }} +{{ value["category_help"] }} +{% endfor %} + +.SS "global arguments:" +.TP +\fB\-\-no\-cache\fR +Don't use actions map cache +.TP +\fB\-\-output\-as\fR {json,plain,none} +Output result in another format +.TP +\fB\-\-debug\fR +Log and print debug messages +.TP +\fB\-\-quiet\fR +Don't produce any output +.TP +\fB\-\-timeout\fR SECONDS +Number of seconds before this command will timeout +because it can't acquire the lock (meaning that +another command is currently running), by default +there is no timeout and the command will wait until it +can get the lock +.TP +\fB\-\-admin\-password\fR PASSWORD +The admin password to use to authenticate +.TP +\fB\-v\fR, \fB\-\-version\fR +Display YunoHost packages versions + +{# each categories #} +{% for name, value in categories.items() %} +.SH YUNOHOST {{ name.upper() }} +usage: yunohost {{ name }} {{ '{' }}{{ ",".join(value["actions"].keys()) }}{{ '}' }} +\&... +.SS "description:" +.IP +{{ value["category_help"] }} + +{# each command of each category #} +{% for action, action_value in value["actions"].items() %} +.SS "yunohost {{ name }} {{ action }} \ +{% for argument_name, argument_value in action_value.get("arguments", {}).items() %}\ +{% set required=(not str(argument_name).startswith("-")) or argument_value.get("extra", {}).get("required", False) %}\ +{% if not required %}[{% endif %}\ +\fI\,{{ argument_name }}\/\fR{% if argument_value.get("full") %}|\fI\,{{ argument_value["full"] }}\fR{% endif %}\ +{% if str(argument_name).startswith("-") and not argument_value.get("action") == "store_true" %} {{ (argument_value.get("full", argument_name)).lstrip("-") }}{% endif %}\ +{% if not required %}]{% endif %} \ +{% endfor %}" + +{# help of the command #} +{{ action_value["action_help"] }} + +{# arguments of the command #} +{% if "arguments" in action_value %} +{% for argument_name, argument_value in action_value["arguments"].items() %} +.TP +\fB{{ argument_name }}\fR{% if argument_value.get("full") %}, \fB{{ argument_value["full"] }}\fR{% endif %}\ +{% if str(argument_name).startswith("-") and not argument_value.get("action") == "store_true" %} \fI\,{{ (argument_value.get("full", argument_name)).lstrip("-") }}\fR {% if "default" in argument_value %}(default: {{ argument_value["default"] }}){% endif %}{% endif %} +{{ argument_value.get("help", "")}} +{% endfor %} + +{% endif %} +{% endfor %} + +{# each subcategory #} +{% for subcategory_name, subcategory in value.get("subcategories", {}).items() %} +{% for action, action_value in subcategory["actions"].items() %} +.SS "yunohost {{ subcategory_name }} {{ name }} {{ action }} \ +{% for argument_name, argument_value in action_value.get("arguments", {}).items() %}\ +{% set required=(not str(argument_name).startswith("-")) or argument_value.get("extra", {}).get("required", False) %}\ +{% if not required %}[{% endif %}\ +\fI\,{{ argument_name }}\/\fR{% if argument_value.get("full") %}|\fI\,{{ argument_value["full"] }}\fR{% endif %}\ +{% if str(argument_name).startswith("-") and not argument_value.get("action") == "store_true" %} {{ (argument_value.get("full", argument_name)).lstrip("-") }}{% endif %}\ +{% if not required %}]{% endif %} \ +{% endfor %}" + +{# help of the command #} +{{ action_value["action_help"] }} + +{# arguments of the command #} +{% if "arguments" in action_value %} +{% for argument_name, argument_value in action_value["arguments"].items() %} +.TP +\fB{{ argument_name }}\fR{% if argument_value.get("full") %}, \fB{{ argument_value["full"] }}\fR{% endif %}\ +{% if str(argument_name).startswith("-") and not argument_value.get("action") == "store_true" %} \fI\,{{ (argument_value.get("full", argument_name)).lstrip("-") }}\fR {% if "default" in argument_value %}(default: {{ argument_value["default"] }}){% endif %}{% endif %} +{{ argument_value.get("help", "")}} +{% endfor %} + +{% endif %} +{% endfor %} +{% endfor %} + +{% endfor %} diff --git a/locales/en.json b/locales/en.json index eae8621de..b792081a2 100644 --- a/locales/en.json +++ b/locales/en.json @@ -127,7 +127,7 @@ "backup_with_no_restore_script_for_app": "App {app:s} has no restore script, you won't be able to automatically restore the backup of this app.", "certmanager_acme_not_configured_for_domain": "Certificate for domain {domain:s} does not appear to be correctly installed. Please run cert-install for this domain first.", "certmanager_attempt_to_renew_nonLE_cert": "The certificate for domain {domain:s} is not issued by Let's Encrypt. Cannot renew it automatically!", - "certmanager_attempt_to_renew_valid_cert": "The certificate for domain {domain:s} is not about to expire! Use --force to bypass", + "certmanager_attempt_to_renew_valid_cert": "The certificate for domain {domain:s} is not about to expire! (You may use --force if you know what you're doing)", "certmanager_attempt_to_replace_valid_cert": "You are attempting to overwrite a good and valid certificate for domain {domain:s}! (Use --force to bypass)", "certmanager_cannot_read_cert": "Something wrong happened when trying to open current certificate for domain {domain:s} (file: {file:s}), reason: {reason:s}", "certmanager_cert_install_success": "Successfully installed Let's Encrypt certificate for domain {domain:s}!", @@ -447,7 +447,7 @@ "port_already_opened": "Port {port:d} is already opened for {ip_version:s} connections", "port_available": "Port {port:d} is available", "port_unavailable": "Port {port:d} is not available", - "recommend_to_add_first_user": "The post-install is finished but YunoHost needs at least one user to work correctly, you should add one using 'yunohost user create' or the admin interface.", + "recommend_to_add_first_user": "The post-install is finished but YunoHost needs at least one user to work correctly, you should add one using 'yunohost user create $username' or the admin interface.", "remove_main_permission_not_allowed": "Removing the main permission is not allowed", "remove_user_of_group_not_allowed": "You are not allowed to remove the user {user:s} in the group {group:s}", "regenconf_file_backed_up": "The configuration file '{conf}' has been backed up to '{backup}'", diff --git a/locales/fr.json b/locales/fr.json index ef9954a6e..944a0b3a8 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -25,7 +25,7 @@ "app_requirements_checking": "Vérification des paquets requis pour {app} …", "app_requirements_failed": "Impossible de satisfaire les pré-requis pour {app} : {error}", "app_requirements_unmeet": "Les pré-requis de {app} ne sont pas satisfaits, le paquet {pkgname} ({version}) doit être {spec}", - "app_sources_fetch_failed": "Impossible de récupérer les fichiers sources", + "app_sources_fetch_failed": "Impossible de récupérer les fichiers sources, l'URL est-elle correcte ?", "app_unknown": "Application inconnue", "app_unsupported_remote_type": "Ce type de commande à distance utilisé pour cette application n'est pas supporté", "app_upgrade_failed": "Impossible de mettre à jour {app:s}", @@ -111,7 +111,7 @@ "hook_choice_invalid": "Choix incorrect : '{:s}'", "hook_exec_failed": "Échec de l’exécution du script : {path:s}", "hook_exec_not_terminated": "L’exécution du script {path:s} ne s’est pas terminée correctement", - "hook_list_by_invalid": "Propriété invalide pour lister les actions par", + "hook_list_by_invalid": "Propriété invalide pour lister les actions par celle-ci", "hook_name_unknown": "Nom de l'action '{name:s}' inconnu", "installation_complete": "Installation terminée", "installation_failed": "Échec de l’installation", @@ -254,7 +254,7 @@ "certmanager_domain_cert_not_selfsigned": "Le certificat du domaine {domain:s} n’est pas auto-signé. Voulez-vous vraiment le remplacer ? (Utilisez --force pour cela)", "certmanager_certificate_fetching_or_enabling_failed": "Il semble que l’activation du nouveau certificat pour {domain:s} a échoué …", "certmanager_attempt_to_renew_nonLE_cert": "Le certificat pour le domaine {domain:s} n’est pas émis par Let’s Encrypt. Impossible de le renouveler automatiquement !", - "certmanager_attempt_to_renew_valid_cert": "Le certificat pour le domaine {domain:s} est sur le point d’expirer ! Utilisez --force pour contourner cela", + "certmanager_attempt_to_renew_valid_cert": "Le certificat pour le domaine {domain:s} n'est pas sur le point d’expirer ! (Vous pouvez utiliser --force si vous savez ce que vous faites)", "certmanager_domain_http_not_working": "Il semble que le domaine {domain:s} ne soit pas accessible via HTTP. Veuillez vérifier que vos configuration DNS et Nginx sont correctes", "certmanager_error_no_A_record": "Aucun enregistrement DNS 'A' n’a été trouvé pour {domain:s}. Vous devez faire pointer votre nom de domaine vers votre machine pour être en mesure d’installer un certificat Let’s Encrypt ! (Si vous savez ce que vous faites, utilisez --no-checks pour désactiver ces contrôles)", "certmanager_domain_dns_ip_differs_from_public_ip": "L’enregistrement DNS 'A' du domaine {domain:s} est différent de l’adresse IP de ce serveur. Si vous avez récemment modifié votre enregistrement 'A', veuillez attendre sa propagation (quelques vérificateur de propagation DNS sont disponibles en ligne). (Si vous savez ce que vous faites, utilisez --no-checks pour désactiver ces contrôles)", @@ -458,7 +458,7 @@ "log_tools_migrations_migrate_forward": "Migrer vers", "log_tools_migrations_migrate_backward": "Revenir en arrière", "log_tools_postinstall": "Faire la post-installation de votre serveur YunoHost", - "log_tools_upgrade": "Mettre à jour les paquets système", + "log_tools_upgrade": "Mettre à jour les paquets du système", "log_tools_shutdown": "Éteindre votre serveur", "log_tools_reboot": "Redémarrer votre serveur", "mail_unavailable": "Cette adresse de courriel est réservée et doit être automatiquement attribuée au tout premier utilisateur", @@ -467,7 +467,7 @@ "migration_0005_postgresql_94_not_installed": "PostgreSQL n’a pas été installé sur votre système. Rien à faire !", "migration_0005_postgresql_96_not_installed": "PostgreSQL 9.4 a été trouvé et installé, mais pas PostgreSQL 9.6 !? Quelque chose d’étrange a dû arriver à votre système :( …", "migration_0005_not_enough_space": "Il n’y a pas assez d’espace libre de disponible sur {path} pour lancer maintenant la migration :(.", - "recommend_to_add_first_user": "La post-installation est terminée. YunoHost a besoin d’au moins un utilisateur pour fonctionner correctement. Vous devez en ajouter un en utilisant 'yunohost user create' ou bien via l’interface d’administration web.", + "recommend_to_add_first_user": "La post-installation est terminée mais YunoHost a besoin d’au moins un utilisateur pour fonctionner correctement. Vous devez en ajouter un en utilisant 'yunohost user create $nomdutilisateur' ou bien via l’interface d’administration web.", "service_description_php7.0-fpm": "exécute des applications écrites en PHP avec Nginx", "users_available": "Liste des utilisateurs disponibles :", "good_practices_about_admin_password": "Vous êtes maintenant sur le point de définir un nouveau mot de passe d’administration. Le mot de passe doit comporter au moins 8 caractères – bien qu’il soit recommandé d’utiliser un mot de passe plus long (c’est-à-dire une phrase secrète) et/ou d’utiliser différents types de caractères (majuscules, minuscules, chiffres et caractères spéciaux).", @@ -524,7 +524,7 @@ "service_reloaded_or_restarted": "Le service '{service:s}' a été rechargé ou redémarré", "this_action_broke_dpkg": "Cette action a laissé des paquets non configurés par dpkg/apt (les gestionnaires de paquets système). Vous pouvez essayer de résoudre ce problème en vous connectant via SSH et en exécutant `sudo dpkg --configure -a`.", "app_action_cannot_be_ran_because_required_services_down": "Cette application requiert certains services qui sont actuellement en pannes. Avant de continuer, vous devriez essayer de redémarrer les services suivant (et éventuellement rechercher pourquoi ils sont en panne) : {services}", - "admin_password_too_long": "Choisissez un mot de passe de 127 caractères maximum.", + "admin_password_too_long": "Choisissez un mot de passe plus court que 127 caractères", "log_regen_conf": "Régénérer les configurations du système '{}'", "migration_0009_not_needed": "Cette migration semble avoir déjà été jouée ? On l'ignore.", "regenconf_file_backed_up": "Le fichier de configuration '{conf}' a été sauvegardé sous '{backup}'", @@ -551,5 +551,10 @@ "service_regen_conf_is_deprecated": "« yunohost service regen-conf » est obsolète ! Veuillez plutôt utiliser « yunohost tools regen-conf ».", "tools_upgrade_at_least_one": "Veuillez spécifier --apps OU --system", "tools_upgrade_cant_both": "Impossible de mettre à niveau le système et les applications en même temps", - "tools_upgrade_cant_hold_critical_packages": "Impossibilité de maintenir les paquets critiques..." + "tools_upgrade_cant_hold_critical_packages": "Impossibilité de maintenir les paquets critiques...", + "tools_upgrade_regular_packages": "Mise à jour des paquets « normaux » (non liés a YunoHost) ...", + "tools_upgrade_regular_packages_failed": "Impossible de mettre à jour les paquets suivants : {packages_list}", + "tools_upgrade_special_packages": "Mise à jour des paquets « spéciaux » (liés a YunoHost) ...", + "tools_upgrade_special_packages_completed": "La mise à jour des paquets de YunoHost est finie!\nPressez [Entrée] pour revenir à la ligne de commande", + "updating_app_lists": "Récupération des mises à jour des applications disponibles…" } diff --git a/locales/nb_NO.json b/locales/nb_NO.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/locales/nb_NO.json @@ -0,0 +1 @@ +{} diff --git a/locales/oc.json b/locales/oc.json index 90641fabd..dbad5ac22 100644 --- a/locales/oc.json +++ b/locales/oc.json @@ -53,7 +53,7 @@ "app_manifest_invalid": "Manifest d’aplicacion incorrècte : {error}", "app_package_need_update": "Lo paquet de l’aplicacion {app} deu èsser mes a jorn per seguir los cambiaments de YunoHost", "app_requirements_checking": "Verificacion dels paquets requesits per {app}…", - "app_sources_fetch_failed": "Recuperacion dels fichièrs fonts impossibla", + "app_sources_fetch_failed": "Recuperacion dels fichièrs fonts impossibla, l’URL es corrècta ?", "app_unsupported_remote_type": "Lo tipe alonhat utilizat per l’aplicacion es pas suportat", "appslist_retrieve_error": "Impossible de recuperar la lista d’aplicacions alonhadas {appslist:s} : {error:s}", "backup_archive_app_not_found": "L’aplicacion « {app:s} » es pas estada trobada dins l’archiu de la salvagarda", @@ -121,7 +121,7 @@ "backup_with_no_restore_script_for_app": "L’aplicacion {app:s} a pas cap de script de restauracion, poiretz pas restaurar automaticament la salvagarda d’aquesta aplicacion.", "certmanager_acme_not_configured_for_domain": "Lo certificat del domeni {domain:s} sembla pas corrèctament installat. Mercés de lançar d’en primièr cert-install per aqueste domeni.", "certmanager_attempt_to_renew_nonLE_cert": "Lo certificat pel domeni {domain:s} es pas provesit per Let’s Encrypt. Impossible de lo renovar automaticament !", - "certmanager_attempt_to_renew_valid_cert": "Lo certificat pel domeni {domain:s} es a man d’expirar ! Utilizatz --force per cortcircuitar", + "certmanager_attempt_to_renew_valid_cert": "Lo certificat pel domeni {domain:s} es a man d’expirar ! (Podètz utilizar --force se sabètz çò que fasètz)", "certmanager_cannot_read_cert": "Quicòm a trucat en ensajar de dobrir lo certificat actual pel domeni {domain:s} (fichièr : {file:s}), rason : {reason:s}", "certmanager_cert_install_success": "Installacion capitada del certificat Let’s Encrypt pel domeni {domain:s} !", "certmanager_cert_install_success_selfsigned": "Installacion capitada del certificat auto-signat pel domeni {domain:s} !", @@ -175,7 +175,7 @@ "global_settings_key_doesnt_exists": "La clau « {settings_key:s} » existís pas dins las configuracions globalas, podètz veire totas las claus disponiblas en picant « yunohost settings list »", "global_settings_reset_success": "Capitada ! Vòstra configuracion precedenta es estada salvagarda dins {path:s}", "global_settings_setting_example_bool": "Exemple d’opcion booleana", - "global_settings_unknown_setting_from_settings_file": "Clau desconeguda dins los paramètres : {setting_key:s}, apartada e salvagardada dins /etc/yunohost/unkown_settings.json", + "global_settings_unknown_setting_from_settings_file": "Clau desconeguda dins los paramètres : {setting_key:s}, apartada e salvagardada dins /etc/yunohost/settings-unknown.json", "installation_failed": "Fracàs de l’installacion", "invalid_url_format": "Format d’URL pas valid", "ldap_initialized": "L’annuari LDAP es inicializat", @@ -444,7 +444,7 @@ "log_tools_migrations_migrate_forward": "Migrar", "log_tools_migrations_migrate_backward": "Tornar en arrièr", "log_tools_postinstall": "Realizar la post installacion del servidor YunoHost", - "log_tools_upgrade": "Mesa a jorn dels paquets Debian", + "log_tools_upgrade": "Mesa a jorn dels paquets sistèma", "log_tools_shutdown": "Atudar lo servidor", "log_tools_reboot": "Reaviar lo servidor", "mail_unavailable": "Aquesta adreça electronica es reservada e deu èsser automaticament atribuida al tot bèl just primièr utilizaire", @@ -453,7 +453,7 @@ "migration_0005_postgresql_94_not_installed": "Postgresql es pas installat sul sistèma. Pas res de far !", "migration_0005_postgresql_96_not_installed": "Avèm trobat que Postgresql 9.4 es installat, mas cap de version de Postgresql 9.6 pas trobada !? Quicòm d’estranh a degut arribar a vòstre sistèma :( …", "migration_0005_not_enough_space": "I a pas pro d’espaci disponible sus {path} per lançar la migracion d’aquela passa :(.", - "recommend_to_add_first_user": "La post installacion es acabada, mas YunoHost fa besonh d’almens un utilizaire per foncionar coma cal. Vos cal n’ajustar un en utilizant la comanda « yunohost user create » o ben l’interfàcia d’administracion.", + "recommend_to_add_first_user": "La post installacion es acabada, mas YunoHost fa besonh d’almens un utilizaire per foncionar coma cal. Vos cal n’ajustar un en utilizant la comanda « yunohost user create $username » o ben l’interfàcia d’administracion.", "service_description_php7.0-fpm": "executa d’aplicacions escrichas en PHP amb nginx", "users_available": "Lista dels utilizaires disponibles :", "good_practices_about_admin_password": "Sètz per definir un nòu senhal per l’administracion. Lo senhal deu almens conténer 8 caractèrs - encara que siá de bon far d’utilizar un senhal mai long qu’aquò (ex. una passafrasa) e/o d’utilizar mantun tipes de caractèrs (majuscula, minuscula, nombre e caractèrs especials).", @@ -491,5 +491,39 @@ "migration_0007_cannot_restart": "SSH pòt pas èsser reavit aprèp aver ensajat d’anullar la migracion numèro 6.", "migrations_success": "Migracion {number} {name} reüssida !", "service_conf_now_managed_by_yunohost": "Lo fichièr de configuracion « {conf} » es ara gerit per YunoHost.", - "service_reloaded": "Lo servici « {servici:s} » es estat tornat cargar" + "service_reloaded": "Lo servici « {servici:s} » es estat tornat cargar", + "already_up_to_date": "I a pas res a far ! Tot es ja a jorn !", + "app_action_cannot_be_ran_because_required_services_down": "Aquesta aplicacion necessita unes servicis que son actualament encalats. Abans de contunhar deuriatz ensajar de reaviar los servicis seguents (e tanben cercar perque son tombats en pana) : {services}", + "confirm_app_install_warning": "Atencion : aquesta aplicacion fonciona mas non es pas ben integrada amb YunoHost. Unas foncionalitats coma l’autentificacion unica e la còpia de seguretat/restauracion pòdon èsser indisponiblas. volètz l’installar de totas manièras ? [{answers:s}] ", + "confirm_app_install_danger": "ATENCION ! Aquesta aplicacion es encara experimentala (autrament dich, fonciona pas) e es possible que còpe lo sistèma ! Deuriatz PAS l’installar se non sabètz çò que fasètz. Volètz vertadièrament córrer aqueste risc ? [{answers:s}] ", + "confirm_app_install_thirdparty": "ATENCION ! L’installacion d’aplicacions tèrças pòt comprometre l’integralitat e la seguretat del sistèma. Deuriatz PAS l’installar se non sabètz pas çò que fasètz. Volètz vertadièrament córrer aqueste risc ? [{answers:s}] ", + "dpkg_lock_not_available": "Aquesta comanda pòt pas s’executar pel moment perque un autre programa sembla utilizar lo varrolh de dpkg (lo gestionari de paquets del sistèma)", + "log_regen_conf": "Regenerar las configuracions del sistèma « {} »", + "service_reloaded_or_restarted": "Lo servici « {service:s} » es estat recargat o reaviat", + "tools_upgrade_regular_packages_failed": "Actualizacion impossibla dels paquets seguents : {packages_list}", + "tools_upgrade_special_packages_completed": "L’actualizacion dels paquets de YunoHost es acabada !\nQuichatz [Entrada] per tornar a la linha de comanda", + "updating_app_lists": "Recuperacion de las mesas a jorn disponiblas per las aplicacions…", + "dpkg_is_broken": "Podètz pas far aquò pel moment perque dpkg/apt (los gestionaris de paquets del sistèma) sembla èsser mal configurat... Podètz ensajar de solucionar aquò en vos connectar via SSH e en executar « sudo dpkg --configure -a ».", + "global_settings_setting_service_ssh_allow_deprecated_dsa_hostkey": "Autorizar l’utilizacion de la clau òst DSA (obsolèta) per la configuracion del servici SSH", + "migration_0008_general_disclaimer": "Per melhorar la seguretat del servidor, es recomandat de daissar YunoHost gerir la configuracion SSH. Vòstra configuracion actuala es diferenta de la configuracion recomandada. Se daissatz YunoHost la reconfigurar, lo biais de vos connectar al servidor via SSH cambiarà coma aquò :", + "hook_json_return_error": "Fracàs de la lectura del retorn de l’script {path:s}. Error : {msg:s}. Contengut brut : {raw_content}", + "migration_0008_port": " - vos cal vos connectar en utilizar lo pòrt 22 allòc de vòstre pòrt SSH actual personalizat. Esitetz pas a lo reconfigurar ;", + "migration_0009_not_needed": "Sembla qu’i aguèt ja una migracion. Passem.", + "pattern_password_app": "O planhèm, los senhals devon pas conténer los caractèrs seguents : {forbidden_chars}", + "regenconf_file_backed_up": "Lo fichièr de configuracion « {conf} » es estat salvagardat dins « {backup} »", + "regenconf_file_copy_failed": "Còpia impossibla del nòu fichièr de configuracion « {new} » cap a « {conf} »", + "regenconf_file_manually_modified": "Lo fichièr de configuracion « {conf} » es estat modificat manualament e serà pas actualizat", + "regenconf_file_manually_removed": "Lo fichièr de configuracion « {conf} » es estat suprimit manualament e serà pas creat", + "regenconf_file_remove_failed": "Supression impossibla del fichièr de configuracion « {conf} »", + "regenconf_file_removed": "Lo fichièr de configuracion « {conf} » es estat suprimit", + "regenconf_file_updated": "Lo fichièr de configuracion « {conf} » es estat actualizat", + "regenconf_now_managed_by_yunohost": "Lo fichièr de configuracion « {conf} » es ara gerit per YunoHost (categoria {category}).", + "regenconf_up_to_date": "La configuracion es ja a jorn per la categoria « {category} »", + "regenconf_updated": "La configuracion es estada actualizada per la categoria « {category} »", + "regenconf_would_be_updated": "La configuracion seriá estada actualizada per la categoria « {category} »", + "regenconf_dry_pending_applying": "Verificacion de la configuracion que seriá estada aplicada a la categoria « {category} »…", + "regenconf_failed": "Regeneracion impossibla de la configuracion per la(s) categoria(s) : {categories}", + "regenconf_pending_applying": "Aplicacion de la configuracion en espèra per la categoria « {category} »…", + "tools_upgrade_cant_both": "Actualizacion impossibla del sistèma e de las aplicacions a l’encòp", + "tools_upgrade_cant_hold_critical_packages": "Manteniment impossible dels paquets critiques…" } diff --git a/src/yunohost/app.py b/src/yunohost/app.py index aadcf8a0e..f9af38e5b 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -24,6 +24,7 @@ Manage apps """ import os +import toml import json import shutil import yaml @@ -363,6 +364,11 @@ def app_info(app, show_status=False, raw=False): ret['upgradable'] = upgradable ret['change_url'] = os.path.exists(os.path.join(app_setting_path, "scripts", "change_url")) + with open(os.path.join(APPS_SETTING_PATH, app, 'manifest.json')) as json_manifest: + manifest = json.load(json_manifest) + + ret['version'] = manifest.get('version', '-') + return ret # Retrieve manifest and status @@ -488,7 +494,7 @@ def app_change_url(operation_logger, app, domain, path): # Retrieve arguments list for change_url script # TODO: Allow to specify arguments args_odict = _parse_args_from_manifest(manifest, 'change_url') - args_list = args_odict.values() + args_list = [ value[0] for value in args_odict.values() ] args_list.append(app) # Prepare env. var. to pass to script @@ -638,7 +644,7 @@ def app_upgrade(app=[], url=None, file=None): # Retrieve arguments list for upgrade script # TODO: Allow to specify arguments args_odict = _parse_args_from_manifest(manifest, 'upgrade') - args_list = args_odict.values() + args_list = [ value[0] for value in args_odict.values() ] args_list.append(app_instance_name) # Prepare env. var. to pass to script @@ -684,7 +690,7 @@ def app_upgrade(app=[], url=None, file=None): os.system('rm -rf "%s/scripts" "%s/manifest.json %s/conf"' % (app_setting_path, app_setting_path, app_setting_path)) os.system('mv "%s/manifest.json" "%s/scripts" %s' % (extracted_app_folder, extracted_app_folder, app_setting_path)) - for file_to_copy in ["actions.json", "config_panel.json", "conf"]: + for file_to_copy in ["actions.json", "actions.toml", "config_panel.json", "config_panel.toml", "conf"]: if os.path.exists(os.path.join(extracted_app_folder, file_to_copy)): os.system('cp -R %s/%s %s' % (extracted_app_folder, file_to_copy, app_setting_path)) @@ -799,7 +805,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu args_dict = {} if not args else \ dict(urlparse.parse_qsl(args, keep_blank_values=True)) args_odict = _parse_args_from_manifest(manifest, 'install', args=args_dict) - args_list = args_odict.values() + args_list = [ value[0] for value in args_odict.values() ] args_list.append(app_instance_name) # Prepare env. var. to pass to script @@ -810,6 +816,9 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu # Start register change on system operation_logger.extra.update({'env': env_dict}) + # Tell the operation_logger to redact all password-type args + data_to_redact = [ value[0] for value in args_odict.values() if value[1] == "password" ] + operation_logger.data_to_redact.extend(data_to_redact) operation_logger.related_to = [s for s in operation_logger.related_to if s[0] != "app"] operation_logger.related_to.append(("app", app_id)) operation_logger.start() @@ -842,7 +851,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu os.system('cp %s/manifest.json %s' % (extracted_app_folder, app_setting_path)) os.system('cp -R %s/scripts %s' % (extracted_app_folder, app_setting_path)) - for file_to_copy in ["actions.json", "config_panel.json", "conf"]: + for file_to_copy in ["actions.json", "actions.toml", "config_panel.json", "config_panel.toml", "conf"]: if os.path.exists(os.path.join(extracted_app_folder, file_to_copy)): os.system('cp -R %s/%s %s' % (extracted_app_folder, file_to_copy, app_setting_path)) @@ -1444,12 +1453,10 @@ def app_action_list(app): # this will take care of checking if the app is installed app_info_dict = app_info(app) - actions = os.path.join(APPS_SETTING_PATH, app, 'actions.json') - return { "app": app, "app_name": app_info_dict["name"], - "actions": read_json(actions) if os.path.exists(actions) else [], + "actions": _get_app_actions(app) } @@ -1471,7 +1478,7 @@ def app_action_run(app, action, args=None): # Retrieve arguments list for install script args_dict = dict(urlparse.parse_qsl(args, keep_blank_values=True)) if args else {} args_odict = _parse_args_for_action(actions[action], args=args_dict) - args_list = args_odict.values() + args_list = [ value[0] for value in args_odict.values() ] app_id, app_instance_nb = _parse_app_instance_name(app) @@ -1520,12 +1527,12 @@ def app_config_show_panel(app): # this will take care of checking if the app is installed app_info_dict = app_info(app) - config_panel = os.path.join(APPS_SETTING_PATH, app, 'config_panel.json') + config_panel = _get_app_config_panel(app) config_script = os.path.join(APPS_SETTING_PATH, app, 'scripts', 'config') app_id, app_instance_nb = _parse_app_instance_name(app) - if not os.path.exists(config_panel) or not os.path.exists(config_script): + if not config_panel or not os.path.exists(config_script): return { "app_id": app_id, "app": app, @@ -1533,38 +1540,17 @@ def app_config_show_panel(app): "config_panel": [], } - config_panel = read_json(config_panel) - env = { "YNH_APP_ID": app_id, "YNH_APP_INSTANCE_NAME": app, "YNH_APP_INSTANCE_NUMBER": str(app_instance_nb), } - parsed_values = {} - # I need to parse stdout to communicate between scripts because I can't - # read the child environment :( (that would simplify things so much) - # after hours of research this is apparently quite a standard way, another - # option would be to add an explicite pipe or a named pipe for that - # a third option would be to write in a temporary file but I don't like - # that because that could expose sensitive data - def parse_stdout(line): - line = line.rstrip() - logger.info(line) - - if line.strip().startswith("YNH_CONFIG_") and "=" in line: - # XXX error handling? - # XXX this might not work for multilines stuff :( (but echo without - # formatting should do it no?) - key, value = line.strip().split("=", 1) - logger.debug("config script declared: %s -> %s", key, value) - parsed_values[key] = value - - return_code = hook_exec(config_script, - args=["show"], - env=env, - stdout_callback=parse_stdout, - )[0] + return_code, parsed_values = hook_exec(config_script, + args=["show"], + env=env, + return_format="plain_dict" + ) if return_code != 0: raise Exception("script/config show return value code: %s (considered as an error)", return_code) @@ -1575,24 +1561,29 @@ def app_config_show_panel(app): for section in tab.get("sections", []): section_id = section["id"] for option in section.get("options", []): - option_id = option["id"] - generated_id = ("YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_id)).upper() - option["id"] = generated_id - logger.debug(" * '%s'.'%s'.'%s' -> %s", tab.get("name"), section.get("name"), option.get("name"), generated_id) + option_name = option["name"] + generated_name = ("YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_name)).upper() + option["name"] = generated_name + logger.debug(" * '%s'.'%s'.'%s' -> %s", tab.get("name"), section.get("name"), option.get("name"), generated_name) - if generated_id in parsed_values: - # XXX we should probably uses the one of install here but it's at a POC state right now - option_type = option["type"] - if option_type == "bool": - assert parsed_values[generated_id].lower() in ("true", "false") - option["value"] = True if parsed_values[generated_id].lower() == "true" else False - elif option_type == "integer": - option["value"] = int(parsed_values[generated_id]) - elif option_type == "text": - option["value"] = parsed_values[generated_id] + if generated_name in parsed_values: + # code is not adapted for that so we have to mock expected format :/ + if option.get("type") == "boolean": + if parsed_values[generated_name].lower() in ("true", "1", "y"): + option["default"] = parsed_values[generated_name] + else: + del option["default"] + else: + option["default"] = parsed_values[generated_name] + + args_dict = _parse_args_in_yunohost_format( + [{option["name"]: parsed_values[generated_name]}], + [option] + ) + option["default"] = args_dict[option["name"]] else: - logger.debug("Variable '%s' is not declared by config script, using default", generated_id) - option["value"] = option["default"] + logger.debug("Variable '%s' is not declared by config script, using default", generated_name) + # do nothing, we'll use the default if present return { "app_id": app_id, @@ -1611,15 +1602,13 @@ def app_config_apply(app, args): if not installed: raise YunohostError('app_not_installed', app=app) - config_panel = os.path.join(APPS_SETTING_PATH, app, 'config_panel.json') + config_panel = _get_app_config_panel(app) config_script = os.path.join(APPS_SETTING_PATH, app, 'scripts', 'config') - if not os.path.exists(config_panel) or not os.path.exists(config_script): + if not config_panel or not os.path.exists(config_script): # XXX real exception raise Exception("Not config-panel.json nor scripts/config") - config_panel = read_json(config_panel) - app_id, app_instance_nb = _parse_app_instance_name(app) env = { "YNH_APP_ID": app_id, @@ -1633,14 +1622,14 @@ def app_config_apply(app, args): for section in tab.get("sections", []): section_id = section["id"] for option in section.get("options", []): - option_id = option["id"] - generated_id = ("YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_id)).upper() + option_name = option["name"] + generated_name = ("YNH_CONFIG_%s_%s_%s" % (tab_id, section_id, option_name)).upper() - if generated_id in args: - logger.debug("include into env %s=%s", generated_id, args[generated_id]) - env[generated_id] = args[generated_id] + if generated_name in args: + logger.debug("include into env %s=%s", generated_name, args[generated_name]) + env[generated_name] = args[generated_name] else: - logger.debug("no value for key id %s", generated_id) + logger.debug("no value for key id %s", generated_name) # for debug purpose for key in args: @@ -1658,6 +1647,217 @@ def app_config_apply(app, args): logger.success("Config updated as expected") +def _get_app_actions(app_id): + "Get app config panel stored in json or in toml" + actions_toml_path = os.path.join(APPS_SETTING_PATH, app_id, 'actions.toml') + actions_json_path = os.path.join(APPS_SETTING_PATH, app_id, 'actions.json') + + # sample data to get an idea of what is going on + # this toml extract: + # + + # [restart_service] + # name = "Restart service" + # command = "echo pouet $YNH_ACTION_SERVICE" + # user = "root" # optional + # cwd = "/" # optional + # accepted_return_codes = [0, 1, 2, 3] # optional + # description.en = "a dummy stupid exemple or restarting a service" + # + # [restart_service.arguments.service] + # type = "string", + # ask.en = "service to restart" + # example = "nginx" + # + # will be parsed into this: + # + # OrderedDict([(u'restart_service', + # OrderedDict([(u'name', u'Restart service'), + # (u'command', u'echo pouet $YNH_ACTION_SERVICE'), + # (u'user', u'root'), + # (u'cwd', u'/'), + # (u'accepted_return_codes', [0, 1, 2, 3]), + # (u'description', + # OrderedDict([(u'en', + # u'a dummy stupid exemple or restarting a service')])), + # (u'arguments', + # OrderedDict([(u'service', + # OrderedDict([(u'type', u'string'), + # (u'ask', + # OrderedDict([(u'en', + # u'service to restart')])), + # (u'example', + # u'nginx')]))]))])), + # + # + # and needs to be converted into this: + # + # [{u'accepted_return_codes': [0, 1, 2, 3], + # u'arguments': [{u'ask': {u'en': u'service to restart'}, + # u'example': u'nginx', + # u'name': u'service', + # u'type': u'string'}], + # u'command': u'echo pouet $YNH_ACTION_SERVICE', + # u'cwd': u'/', + # u'description': {u'en': u'a dummy stupid exemple or restarting a service'}, + # u'id': u'restart_service', + # u'name': u'Restart service', + # u'user': u'root'}] + + if os.path.exists(actions_toml_path): + toml_actions = toml.load(open(actions_toml_path, "r"), _dict=OrderedDict) + + # transform toml format into json format + actions = [] + + for key, value in toml_actions.items(): + action = dict(**value) + action["id"] = key + + arguments = [] + for argument_name, argument in value.get("arguments", {}).items(): + argument = dict(**argument) + argument["name"] = argument_name + + arguments.append(argument) + + action["arguments"] = arguments + actions.append(action) + + return actions + + elif os.path.exists(actions_json_path): + return json.load(open(actions_json_path)) + + return None + + +def _get_app_config_panel(app_id): + "Get app config panel stored in json or in toml" + config_panel_toml_path = os.path.join(APPS_SETTING_PATH, app_id, 'config_panel.toml') + config_panel_json_path = os.path.join(APPS_SETTING_PATH, app_id, 'config_panel.json') + + # sample data to get an idea of what is going on + # this toml extract: + # + # version = "0.1" + # name = "Unattended-upgrades configuration panel" + # + # [main] + # name = "Unattended-upgrades configuration" + # + # [main.unattended_configuration] + # name = "50unattended-upgrades configuration file" + # + # [main.unattended_configuration.upgrade_level] + # name = "Choose the sources of packages to automatically upgrade." + # default = "Security only" + # type = "text" + # help = "We can't use a choices field for now. In the meantime please choose between one of this values:
Security only, Security and updates." + # # choices = ["Security only", "Security and updates"] + + # [main.unattended_configuration.ynh_update] + # name = "Would you like to update YunoHost packages automatically ?" + # type = "bool" + # default = true + # + # will be parsed into this: + # + # OrderedDict([(u'version', u'0.1'), + # (u'name', u'Unattended-upgrades configuration panel'), + # (u'main', + # OrderedDict([(u'name', u'Unattended-upgrades configuration'), + # (u'unattended_configuration', + # OrderedDict([(u'name', + # u'50unattended-upgrades configuration file'), + # (u'upgrade_level', + # OrderedDict([(u'name', + # u'Choose the sources of packages to automatically upgrade.'), + # (u'default', + # u'Security only'), + # (u'type', u'text'), + # (u'help', + # u"We can't use a choices field for now. In the meantime please choose between one of this values:
Security only, Security and updates.")])), + # (u'ynh_update', + # OrderedDict([(u'name', + # u'Would you like to update YunoHost packages automatically ?'), + # (u'type', u'bool'), + # (u'default', True)])), + # + # and needs to be converted into this: + # + # {u'name': u'Unattended-upgrades configuration panel', + # u'panel': [{u'id': u'main', + # u'name': u'Unattended-upgrades configuration', + # u'sections': [{u'id': u'unattended_configuration', + # u'name': u'50unattended-upgrades configuration file', + # u'options': [{u'//': u'"choices" : ["Security only", "Security and updates"]', + # u'default': u'Security only', + # u'help': u"We can't use a choices field for now. In the meantime please choose between one of this values:
Security only, Security and updates.", + # u'id': u'upgrade_level', + # u'name': u'Choose the sources of packages to automatically upgrade.', + # u'type': u'text'}, + # {u'default': True, + # u'id': u'ynh_update', + # u'name': u'Would you like to update YunoHost packages automatically ?', + # u'type': u'bool'}, + + if os.path.exists(config_panel_toml_path): + toml_config_panel = toml.load(open(config_panel_toml_path, "r"), _dict=OrderedDict) + + # transform toml format into json format + config_panel = { + "name": toml_config_panel["name"], + "version": toml_config_panel["version"], + "panel": [], + } + + panels = filter(lambda (key, value): key not in ("name", "version") + and isinstance(value, OrderedDict), + toml_config_panel.items()) + + for key, value in panels: + panel = { + "id": key, + "name": value["name"], + "sections": [], + } + + sections = filter(lambda (k, v): k not in ("name",) + and isinstance(v, OrderedDict), + value.items()) + + for section_key, section_value in sections: + section = { + "id": section_key, + "name": section_value["name"], + "options": [], + } + + options = filter(lambda (k, v): k not in ("name",) + and isinstance(v, OrderedDict), + section_value.items()) + + for option_key, option_value in options: + option = dict(option_value) + option["name"] = option_key + option["ask"] = {"en": option["ask"]} + if "help" in option: + option["help"] = {"en": option["help"]} + section["options"].append(option) + + panel["sections"].append(section) + + config_panel["panel"].append(panel) + + return config_panel + + elif os.path.exists(config_panel_json_path): + return json.load(open(config_panel_json_path)) + + return None + + def _get_app_settings(app_id): """ Get settings of an installed app @@ -2097,7 +2297,7 @@ def _parse_args_from_manifest(manifest, action, args={}): return OrderedDict() action_args = manifest['arguments'][action] - return _parse_action_args_in_yunohost_format(args, action_args) + return _parse_args_in_yunohost_format(args, action_args) def _parse_args_for_action(action, args={}): @@ -2121,10 +2321,10 @@ def _parse_args_for_action(action, args={}): action_args = action['arguments'] - return _parse_action_args_in_yunohost_format(args, action_args) + return _parse_args_in_yunohost_format(args, action_args) -def _parse_action_args_in_yunohost_format(args, action_args): +def _parse_args_in_yunohost_format(args, action_args): """Parse arguments store in either manifest.json or actions.json """ from yunohost.domain import (domain_list, _get_maindomain, @@ -2206,7 +2406,7 @@ def _parse_action_args_in_yunohost_format(args, action_args): if arg.get("optional", False): # Argument is optional, keep an empty value # and that's all for this arg ! - args_dict[arg_name] = '' + args_dict[arg_name] = ('', arg_type) continue else: # The argument is required ! @@ -2244,22 +2444,20 @@ def _parse_action_args_in_yunohost_format(args, action_args): raise YunohostError('pattern_password_app', forbidden_chars=forbidden_chars) from yunohost.utils.password import assert_password_is_strong_enough assert_password_is_strong_enough('user', arg_value) - args_dict[arg_name] = arg_value + args_dict[arg_name] = (arg_value, arg_type) # END loop over action_args... # If there's only one "domain" and "path", validate that domain/path # is an available url and normalize the path. - domain_args = [arg["name"] for arg in action_args - if arg.get("type", "string") == "domain"] - path_args = [arg["name"] for arg in action_args - if arg.get("type", "string") == "path"] + domain_args = [ (name, value[0]) for name, value in args_dict.items() if value[1] == "domain" ] + path_args = [ (name, value[0]) for name, value in args_dict.items() if value[1] == "path" ] if len(domain_args) == 1 and len(path_args) == 1: - domain = args_dict[domain_args[0]] - path = args_dict[path_args[0]] + domain = domain_args[0][1] + path = path_args[0][1] domain, path = _normalize_domain_path(domain, path) # Check the url is available @@ -2278,7 +2476,7 @@ def _parse_action_args_in_yunohost_format(args, action_args): # (We save this normalized path so that the install script have a # standard path format to deal with no matter what the user inputted) - args_dict[path_args[0]] = path + args_dict[path_args[0][0]] = (path, "path") return args_dict @@ -2293,8 +2491,8 @@ def _make_environment_dict(args_dict, prefix="APP_ARG_"): """ env_dict = {} - for arg_name, arg_value in args_dict.items(): - env_dict["YNH_%s%s" % (prefix, arg_name.upper())] = arg_value + for arg_name, arg_value_and_type in args_dict.items(): + env_dict["YNH_%s%s" % (prefix, arg_name.upper())] = arg_value_and_type[0] return env_dict diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 704f80e14..5a05a0e2a 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -1933,11 +1933,19 @@ class TarBackupMethod(BackupMethod): # Mount the tarball logger.debug(m18n.n("restore_extracting")) tar = tarfile.open(self._archive_file, "r:gz") - tar.extract('info.json', path=self.work_dir) - try: + if "info.json" in tar.getnames(): + leading_dot = "" + tar.extract('info.json', path=self.work_dir) + elif "./info.json" in tar.getnames(): + leading_dot = "./" + tar.extract('./info.json', path=self.work_dir) + + if "backup.csv" in tar.getnames(): tar.extract('backup.csv', path=self.work_dir) - except KeyError: + elif "./backup.csv" in tar.getnames(): + tar.extract('./backup.csv', path=self.work_dir) + else: # Old backup archive have no backup.csv file pass @@ -1959,12 +1967,12 @@ class TarBackupMethod(BackupMethod): system_part = system_part.replace("_", "/") + "/" subdir_and_files = [ tarinfo for tarinfo in tar.getmembers() - if tarinfo.name.startswith(system_part) + if tarinfo.name.startswith(leading_dot+system_part) ] tar.extractall(members=subdir_and_files, path=self.work_dir) subdir_and_files = [ tarinfo for tarinfo in tar.getmembers() - if tarinfo.name.startswith("hooks/restore/") + if tarinfo.name.startswith(leading_dot+"hooks/restore/") ] tar.extractall(members=subdir_and_files, path=self.work_dir) @@ -1972,7 +1980,7 @@ class TarBackupMethod(BackupMethod): for app in apps_targets: subdir_and_files = [ tarinfo for tarinfo in tar.getmembers() - if tarinfo.name.startswith("apps/" + app) + if tarinfo.name.startswith(leading_dot+"apps/" + app) ] tar.extractall(members=subdir_and_files, path=self.work_dir) @@ -2313,7 +2321,12 @@ def backup_info(name, with_details=False, human_readable=False): tar = tarfile.open(archive_file, "r:gz") info_dir = info_file + '.d' try: - tar.extract('info.json', path=info_dir) + if "info.json" in tar.getnames(): + tar.extract('info.json', path=info_dir) + elif "./info.json" in tar.getnames(): + tar.extract('./info.json', path=info_dir) + else: + raise KeyError except KeyError: logger.debug("unable to retrieve '%s' inside the archive", info_file, exc_info=1) @@ -2389,7 +2402,14 @@ def backup_delete(name): archive_file = '%s/%s.tar.gz' % (ARCHIVES_PATH, name) info_file = "%s/%s.info.json" % (ARCHIVES_PATH, name) - for backup_file in [archive_file, info_file]: + files_to_delete = [archive_file, info_file] + + # To handle the case where archive_file is in fact a symlink + if os.path.islink(archive_file): + actual_archive = os.path.realpath(archive_file) + files_to_delete.append(actual_archive) + + for backup_file in files_to_delete: try: os.remove(backup_file) except: diff --git a/src/yunohost/certificate.py b/src/yunohost/certificate.py index 4235d522d..a4a31055d 100644 --- a/src/yunohost/certificate.py +++ b/src/yunohost/certificate.py @@ -566,7 +566,7 @@ def _fetch_and_enable_new_certificate(domain, staging=False, no_checks=False): domain_csr_file, WEBROOT_FOLDER, log=logger, - no_checks=no_checks, + disable_check=no_checks, CA=certification_authority) except ValueError as e: if "urn:acme:error:rateLimited" in str(e): diff --git a/src/yunohost/data_migrations/0010_migrate_to_apps_json.py b/src/yunohost/data_migrations/0010_migrate_to_apps_json.py index 78f6d733b..43ae9a86f 100644 --- a/src/yunohost/data_migrations/0010_migrate_to_apps_json.py +++ b/src/yunohost/data_migrations/0010_migrate_to_apps_json.py @@ -25,7 +25,8 @@ class MyMigration(Migration): "app.yunohost.org/list.json", # Old list on old installs, alias to official.json "app.yunohost.org/official.json", "app.yunohost.org/community.json", - "labriqueinter.net/apps/labriqueinternet.json" + "labriqueinter.net/apps/labriqueinternet.json", + "labriqueinter.net/internetcube.json" ] appslists = _read_appslist_list() diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index 42807fdf7..67e77f033 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -297,8 +297,7 @@ def hook_callback(action, hooks=[], args=None, no_trace=False, chdir=None, def hook_exec(path, args=None, raise_on_error=False, no_trace=False, - chdir=None, env=None, user="root", stdout_callback=None, - stderr_callback=None): + chdir=None, env=None, user="root", return_format="json"): """ Execute hook from a file with arguments @@ -372,8 +371,8 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False, # Define output callbacks and call command callbacks = ( - stdout_callback if stdout_callback else lambda l: logger.debug(l.rstrip()+"\r"), - stderr_callback if stderr_callback else lambda l: logger.warning(l.rstrip()), + lambda l: logger.debug(l.rstrip()+"\r"), + lambda l: logger.warning(l.rstrip()), ) if stdinfo: @@ -401,19 +400,31 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False, try: with open(stdreturn, 'r') as f: raw_content = f.read() - if raw_content != '': - returnjson = read_json(stdreturn) + returncontent = {} + + if return_format == "json": + if raw_content != '': + try: + returncontent = read_json(stdreturn) + except Exception as e: + raise YunohostError('hook_json_return_error', + path=path, msg=str(e), + raw_content=raw_content) + + elif return_format == "plain_dict": + for line in raw_content.split("\n"): + if "=" in line: + key, value = line.strip().split("=", 1) + returncontent[key] = value + else: - returnjson = {} - except Exception as e: - raise YunohostError('hook_json_return_error', path=path, msg=str(e), - raw_content=raw_content) + raise YunohostError("Excepted value for return_format is either 'json' or 'plain_dict', got '%s'" % return_format) finally: stdreturndir = os.path.split(stdreturn)[0] os.remove(stdreturn) os.rmdir(stdreturndir) - return returncode, returnjson + return returncode, returncontent def _extract_filename_parts(filename): diff --git a/src/yunohost/log.py b/src/yunohost/log.py index dd3bbd8b3..17a6ff87c 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -25,6 +25,7 @@ """ import os +import re import yaml import collections @@ -107,7 +108,7 @@ def log_list(category=[], limit=None, with_details=False): except yaml.YAMLError: logger.warning(m18n.n('log_corrupted_md_file', file=md_path)) - entry["success"] = metadata.get("success", "?") + entry["success"] = metadata.get("success", "?") if metadata else "?" result[category].append(entry) @@ -289,6 +290,33 @@ def is_unit_operation(entities=['app', 'domain', 'service', 'user'], return decorate +class RedactingFormatter(Formatter): + + def __init__(self, format_string, data_to_redact): + super(RedactingFormatter, self).__init__(format_string) + self.data_to_redact = data_to_redact + + def format(self, record): + msg = super(RedactingFormatter, self).format(record) + self.identify_data_to_redact(msg) + for data in self.data_to_redact: + msg = msg.replace(data, "**********") + return msg + + def identify_data_to_redact(self, record): + + # Wrapping this in a try/except because we don't want this to + # break everything in case it fails miserably for some reason :s + try: + # This matches stuff like db_pwd=the_secret or admin_password=other_secret + # (the secret part being at least 3 chars to avoid catching some lines like just "db_pwd=") + match = re.search(r'(pwd|pass|password)=(\S{3,})$', record.strip()) + if match and match.group(2) not in self.data_to_redact: + self.data_to_redact.append(match.group(2)) + except Exception as e: + logger.warning("Failed to parse line to try to identify data to redact ... : %s" % e) + + class OperationLogger(object): """ @@ -309,6 +337,11 @@ class OperationLogger(object): self.ended_at = None self.logger = None self._name = None + self.data_to_redact = [] + + for filename in ["/etc/yunohost/mysql", "/etc/yunohost/psql"]: + if os.path.exists(filename): + self.data_to_redact.append(read_file(filename).strip()) self.path = OPERATIONS_PATH @@ -345,9 +378,12 @@ class OperationLogger(object): Register log with a handler connected on log system """ - # TODO add a way to not save password on app installation self.file_handler = FileHandler(self.log_path) - self.file_handler.formatter = Formatter('%(asctime)s: %(levelname)s - %(message)s') + # We use a custom formatter that's able to redact all stuff in self.data_to_redact + # N.B. : the subtle thing here is that the class will remember a pointer to the list, + # so we can directly append stuff to self.data_to_redact and that'll be automatically + # propagated to the RedactingFormatter + self.file_handler.formatter = RedactingFormatter('%(asctime)s: %(levelname)s - %(message)s', self.data_to_redact) # Listen to the root logger self.logger = getLogger('yunohost') @@ -358,8 +394,11 @@ class OperationLogger(object): Write or rewrite the metadata file with all metadata known """ + dump = yaml.safe_dump(self.metadata, default_flow_style=False) + for data in self.data_to_redact: + dump = dump.replace(data, "**********") with open(self.md_path, 'w') as outfile: - yaml.safe_dump(self.metadata, outfile, default_flow_style=False) + outfile.write(dump) @property def name(self): diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index ea74aa790..c00a5cb41 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -540,9 +540,21 @@ def _list_upgradable_apps(): app_dict = app_info(app_id, raw=True) if app_dict["upgradable"] == "yes": + + current_version = app_dict.get("version", "?") + current_commit = app_dict.get("status", {}).get("remote", {}).get("revision", "?")[:7] + new_version = app_dict.get("manifest",{}).get("version","?") + new_commit = app_dict.get("git", {}).get("revision", "?")[:7] + + if current_version == new_version: + current_version += " (" + current_commit + ")" + new_version += " (" + new_commit + ")" + yield { 'id': app_id, - 'label': app_dict['settings']['label'] + 'label': app_dict['settings']['label'], + 'current_version': current_version, + 'new_version': new_version } @@ -648,7 +660,7 @@ def tools_upgrade(operation_logger, apps=None, system=False): logger.debug("Running apt command :\n{}".format(dist_upgrade)) callbacks = ( - lambda l: logger.info(l.rstrip() + "\r"), + lambda l: logger.info("+" + l.rstrip() + "\r"), lambda l: logger.warning(l.rstrip()), ) returncode = call_async_output(dist_upgrade, callbacks, shell=True) @@ -685,7 +697,7 @@ def tools_upgrade(operation_logger, apps=None, system=False): # command before it ends...) # logfile = operation_logger.log_path - command = dist_upgrade + " 2>&1 | tee -a {}".format(logfile) + dist_upgrade = dist_upgrade + " 2>&1 | tee -a {}".format(logfile) MOULINETTE_LOCK = "/var/run/moulinette_yunohost.lock" wait_until_end_of_yunohost_command = "(while [ -f {} ]; do sleep 2; done)".format(MOULINETTE_LOCK) @@ -694,10 +706,17 @@ def tools_upgrade(operation_logger, apps=None, system=False): update_log_metadata = "sed -i \"s/ended_at: .*$/ended_at: $(date -u +'%Y-%m-%d %H:%M:%S.%N')/\" {}" update_log_metadata = update_log_metadata.format(operation_logger.md_path) + # Dirty hack such that the operation_logger does not add ended_at + # and success keys in the log metadata. (c.f. the code of the + # is_unit_operation + operation_logger.close()) We take care of + # this ourselves (c.f. the mark_success and updated_log_metadata in + # the huge command launched by os.system) + operation_logger.ended_at = "notyet" + upgrade_completed = "\n" + m18n.n("tools_upgrade_special_packages_completed") - command = "(({wait} && {cmd}) && {mark_success} || {mark_failure}; {update_metadata}; echo '{done}') &".format( + command = "({wait} && {dist_upgrade}) && {mark_success} || {mark_failure}; {update_metadata}; echo '{done}'".format( wait=wait_until_end_of_yunohost_command, - cmd=command, + dist_upgrade=dist_upgrade, mark_success=mark_success, mark_failure=mark_failure, update_metadata=update_log_metadata, @@ -705,7 +724,12 @@ def tools_upgrade(operation_logger, apps=None, system=False): logger.warning(m18n.n("tools_upgrade_special_packages_explanation")) logger.debug("Running command :\n{}".format(command)) - os.system(command) + open("/tmp/yunohost-selfupgrade", "w").write("rm /tmp/yunohost-selfupgrade; " + command) + # Using systemd-run --scope is like nohup/disown and &, but more robust somehow + # (despite using nohup/disown and &, the self-upgrade process was still getting killed...) + # ref: https://unix.stackexchange.com/questions/420594/why-process-killed-with-nohup + # (though I still don't understand it 100%...) + os.system("systemd-run --scope bash /tmp/yunohost-selfupgrade &") return else: diff --git a/src/yunohost/vendor/acme_tiny/acme_tiny.py b/src/yunohost/vendor/acme_tiny/acme_tiny.py index f36aef877..ba04e37ad 100644 --- a/src/yunohost/vendor/acme_tiny/acme_tiny.py +++ b/src/yunohost/vendor/acme_tiny/acme_tiny.py @@ -1,73 +1,94 @@ #!/usr/bin/env python +# Copyright Daniel Roesler, under MIT license, see LICENSE at github.com/diafygi/acme-tiny import argparse, subprocess, json, os, sys, base64, binascii, time, hashlib, re, copy, textwrap, logging try: - from urllib.request import urlopen # Python 3 + from urllib.request import urlopen, Request # Python 3 except ImportError: - from urllib2 import urlopen # Python 2 + from urllib2 import urlopen, Request # Python 2 -#DEFAULT_CA = "https://acme-staging.api.letsencrypt.org" -DEFAULT_CA = "https://acme-v01.api.letsencrypt.org" +DEFAULT_CA = "https://acme-v02.api.letsencrypt.org" # DEPRECATED! USE DEFAULT_DIRECTORY_URL INSTEAD +DEFAULT_DIRECTORY_URL = "https://acme-v02.api.letsencrypt.org/directory" LOGGER = logging.getLogger(__name__) LOGGER.addHandler(logging.StreamHandler()) LOGGER.setLevel(logging.INFO) -def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA, no_checks=False): - # helper function base64 encode for jose spec +def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA, disable_check=False, directory_url=DEFAULT_DIRECTORY_URL, contact=None): + directory, acct_headers, alg, jwk = None, None, None, None # global variables + + # helper functions - base64 encode for jose spec def _b64(b): return base64.urlsafe_b64encode(b).decode('utf8').replace("=", "") + # helper function - run external commands + def _cmd(cmd_list, stdin=None, cmd_input=None, err_msg="Command Line Error"): + proc = subprocess.Popen(cmd_list, stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = proc.communicate(cmd_input) + if proc.returncode != 0: + raise IOError("{0}\n{1}".format(err_msg, err)) + return out + + # helper function - make request and automatically parse json response + def _do_request(url, data=None, err_msg="Error", depth=0): + try: + resp = urlopen(Request(url, data=data, headers={"Content-Type": "application/jose+json", "User-Agent": "acme-tiny"})) + resp_data, code, headers = resp.read().decode("utf8"), resp.getcode(), resp.headers + except IOError as e: + resp_data = e.read().decode("utf8") if hasattr(e, "read") else str(e) + code, headers = getattr(e, "code", None), {} + try: + resp_data = json.loads(resp_data) # try to parse json results + except ValueError: + pass # ignore json parsing errors + if depth < 100 and code == 400 and resp_data['type'] == "urn:ietf:params:acme:error:badNonce": + raise IndexError(resp_data) # allow 100 retrys for bad nonces + if code not in [200, 201, 204]: + raise ValueError("{0}:\nUrl: {1}\nData: {2}\nResponse Code: {3}\nResponse: {4}".format(err_msg, url, data, code, resp_data)) + return resp_data, code, headers + + # helper function - make signed requests + def _send_signed_request(url, payload, err_msg, depth=0): + payload64 = _b64(json.dumps(payload).encode('utf8')) + new_nonce = _do_request(directory['newNonce'])[2]['Replay-Nonce'] + protected = {"url": url, "alg": alg, "nonce": new_nonce} + protected.update({"jwk": jwk} if acct_headers is None else {"kid": acct_headers['Location']}) + protected64 = _b64(json.dumps(protected).encode('utf8')) + protected_input = "{0}.{1}".format(protected64, payload64).encode('utf8') + out = _cmd(["openssl", "dgst", "-sha256", "-sign", account_key], stdin=subprocess.PIPE, cmd_input=protected_input, err_msg="OpenSSL Error") + data = json.dumps({"protected": protected64, "payload": payload64, "signature": _b64(out)}) + try: + return _do_request(url, data=data.encode('utf8'), err_msg=err_msg, depth=depth) + except IndexError: # retry bad nonces (they raise IndexError) + return _send_signed_request(url, payload, err_msg, depth=(depth + 1)) + + # helper function - poll until complete + def _poll_until_not(url, pending_statuses, err_msg): + while True: + result, _, _ = _do_request(url, err_msg=err_msg) + if result['status'] in pending_statuses: + time.sleep(2) + continue + return result + # parse account key to get public key log.info("Parsing account key...") - proc = subprocess.Popen(["openssl", "rsa", "-in", account_key, "-noout", "-text"], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = proc.communicate() - if proc.returncode != 0: - raise IOError("OpenSSL Error: {0}".format(err)) - pub_hex, pub_exp = re.search( - r"modulus:\n\s+00:([a-f0-9\:\s]+?)\npublicExponent: ([0-9]+)", - out.decode('utf8'), re.MULTILINE|re.DOTALL).groups() + out = _cmd(["openssl", "rsa", "-in", account_key, "-noout", "-text"], err_msg="OpenSSL Error") + pub_pattern = r"modulus:\n\s+00:([a-f0-9\:\s]+?)\npublicExponent: ([0-9]+)" + pub_hex, pub_exp = re.search(pub_pattern, out.decode('utf8'), re.MULTILINE|re.DOTALL).groups() pub_exp = "{0:x}".format(int(pub_exp)) pub_exp = "0{0}".format(pub_exp) if len(pub_exp) % 2 else pub_exp - header = { - "alg": "RS256", - "jwk": { - "e": _b64(binascii.unhexlify(pub_exp.encode("utf-8"))), - "kty": "RSA", - "n": _b64(binascii.unhexlify(re.sub(r"(\s|:)", "", pub_hex).encode("utf-8"))), - }, + alg = "RS256" + jwk = { + "e": _b64(binascii.unhexlify(pub_exp.encode("utf-8"))), + "kty": "RSA", + "n": _b64(binascii.unhexlify(re.sub(r"(\s|:)", "", pub_hex).encode("utf-8"))), } - accountkey_json = json.dumps(header['jwk'], sort_keys=True, separators=(',', ':')) + accountkey_json = json.dumps(jwk, sort_keys=True, separators=(',', ':')) thumbprint = _b64(hashlib.sha256(accountkey_json.encode('utf8')).digest()) - # helper function make signed requests - def _send_signed_request(url, payload): - payload64 = _b64(json.dumps(payload).encode('utf8')) - protected = copy.deepcopy(header) - protected["nonce"] = urlopen(CA + "/directory").headers['Replay-Nonce'] - protected64 = _b64(json.dumps(protected).encode('utf8')) - proc = subprocess.Popen(["openssl", "dgst", "-sha256", "-sign", account_key], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = proc.communicate("{0}.{1}".format(protected64, payload64).encode('utf8')) - if proc.returncode != 0: - raise IOError("OpenSSL Error: {0}".format(err)) - data = json.dumps({ - "header": header, "protected": protected64, - "payload": payload64, "signature": _b64(out), - }) - try: - resp = urlopen(url, data.encode('utf8')) - return resp.getcode(), resp.read() - except IOError as e: - return getattr(e, "code", None), getattr(e, "read", e.__str__)() - # find domains log.info("Parsing CSR...") - proc = subprocess.Popen(["openssl", "req", "-in", csr, "-noout", "-text"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = proc.communicate() - if proc.returncode != 0: - raise IOError("Error loading {0}: {1}".format(csr, err)) + out = _cmd(["openssl", "req", "-in", csr, "-noout", "-text"], err_msg="Error loading {0}".format(csr)) domains = set([]) common_name = re.search(r"Subject:.*? CN\s?=\s?([^\s,;/]+)", out.decode('utf8')) if common_name is not None: @@ -77,122 +98,100 @@ def get_crt(account_key, csr, acme_dir, log=LOGGER, CA=DEFAULT_CA, no_checks=Fal for san in subject_alt_names.group(1).split(", "): if san.startswith("DNS:"): domains.add(san[4:]) + log.info("Found domains: {0}".format(", ".join(domains))) - # get the certificate domains and expiration + # get the ACME directory of urls + log.info("Getting directory...") + directory_url = CA + "/directory" if CA != DEFAULT_CA else directory_url # backwards compatibility with deprecated CA kwarg + directory, _, _ = _do_request(directory_url, err_msg="Error getting directory") + log.info("Directory found!") + + # create account, update contact details (if any), and set the global key identifier log.info("Registering account...") - code, result = _send_signed_request(CA + "/acme/new-reg", { - "resource": "new-reg", - "agreement": json.loads(urlopen(CA + "/directory").read().decode('utf8'))['meta']['terms-of-service'], - }) - if code == 201: - log.info("Registered!") - elif code == 409: - log.info("Already registered!") - else: - raise ValueError("Error registering: {0} {1}".format(code, result)) + reg_payload = {"termsOfServiceAgreed": True} + account, code, acct_headers = _send_signed_request(directory['newAccount'], reg_payload, "Error registering") + log.info("Registered!" if code == 201 else "Already registered!") + if contact is not None: + account, _, _ = _send_signed_request(acct_headers['Location'], {"contact": contact}, "Error updating contact details") + log.info("Updated contact details:\n{0}".format("\n".join(account['contact']))) - # verify each domain - for domain in domains: + # create a new order + log.info("Creating new order...") + order_payload = {"identifiers": [{"type": "dns", "value": d} for d in domains]} + order, _, order_headers = _send_signed_request(directory['newOrder'], order_payload, "Error creating new order") + log.info("Order created!") + + # get the authorizations that need to be completed + for auth_url in order['authorizations']: + authorization, _, _ = _do_request(auth_url, err_msg="Error getting challenges") + domain = authorization['identifier']['value'] log.info("Verifying {0}...".format(domain)) - # get new challenge - code, result = _send_signed_request(CA + "/acme/new-authz", { - "resource": "new-authz", - "identifier": {"type": "dns", "value": domain}, - }) - if code != 201: - raise ValueError("Error requesting challenges: {0} {1}".format(code, result)) - - # make the challenge file - challenge = [c for c in json.loads(result.decode('utf8'))['challenges'] if c['type'] == "http-01"][0] + # find the http-01 challenge and write the challenge file + challenge = [c for c in authorization['challenges'] if c['type'] == "http-01"][0] token = re.sub(r"[^A-Za-z0-9_\-]", "_", challenge['token']) keyauthorization = "{0}.{1}".format(token, thumbprint) wellknown_path = os.path.join(acme_dir, token) with open(wellknown_path, "w") as wellknown_file: wellknown_file.write(keyauthorization) - if not no_checks: # sometime the local g - # check that the file is in place + # check that the file is in place + try: wellknown_url = "http://{0}/.well-known/acme-challenge/{1}".format(domain, token) - try: - resp = urlopen(wellknown_url) - resp_data = resp.read().decode('utf8').strip() - assert resp_data == keyauthorization - except (IOError, AssertionError): - os.remove(wellknown_path) - raise ValueError("Wrote file to {0}, but couldn't download {1}".format( - wellknown_path, wellknown_url)) + assert(disable_check or _do_request(wellknown_url)[0] == keyauthorization) + except (AssertionError, ValueError) as e: + os.remove(wellknown_path) + raise ValueError("Wrote file to {0}, but couldn't download {1}: {2}".format(wellknown_path, wellknown_url, e)) - # notify challenge are met - code, result = _send_signed_request(challenge['uri'], { - "resource": "challenge", - "keyAuthorization": keyauthorization, - }) - if code != 202: - raise ValueError("Error triggering challenge: {0} {1}".format(code, result)) + # say the challenge is done + _send_signed_request(challenge['url'], {}, "Error submitting challenges: {0}".format(domain)) + authorization = _poll_until_not(auth_url, ["pending"], "Error checking challenge status for {0}".format(domain)) + if authorization['status'] != "valid": + raise ValueError("Challenge did not pass for {0}: {1}".format(domain, authorization)) + log.info("{0} verified!".format(domain)) - # wait for challenge to be verified - while True: - try: - resp = urlopen(challenge['uri']) - challenge_status = json.loads(resp.read().decode('utf8')) - except IOError as e: - raise ValueError("Error checking challenge: {0} {1}".format( - e.code, json.loads(e.read().decode('utf8')))) - if challenge_status['status'] == "pending": - time.sleep(2) - elif challenge_status['status'] == "valid": - log.info("{0} verified!".format(domain)) - os.remove(wellknown_path) - break - else: - raise ValueError("{0} challenge did not pass: {1}".format( - domain, challenge_status)) - - # get the new certificate + # finalize the order with the csr log.info("Signing certificate...") - proc = subprocess.Popen(["openssl", "req", "-in", csr, "-outform", "DER"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - csr_der, err = proc.communicate() - code, result = _send_signed_request(CA + "/acme/new-cert", { - "resource": "new-cert", - "csr": _b64(csr_der), - }) - if code != 201: - raise ValueError("Error signing certificate: {0} {1}".format(code, result)) + csr_der = _cmd(["openssl", "req", "-in", csr, "-outform", "DER"], err_msg="DER Export Error") + _send_signed_request(order['finalize'], {"csr": _b64(csr_der)}, "Error finalizing order") - # return signed certificate! + # poll the order to monitor when it's done + order = _poll_until_not(order_headers['Location'], ["pending", "processing"], "Error checking order status") + if order['status'] != "valid": + raise ValueError("Order failed: {0}".format(order)) + + # download the certificate + certificate_pem, _, _ = _do_request(order['certificate'], err_msg="Certificate download failed") log.info("Certificate signed!") - return """-----BEGIN CERTIFICATE-----\n{0}\n-----END CERTIFICATE-----\n""".format( - "\n".join(textwrap.wrap(base64.b64encode(result).decode('utf8'), 64))) + return certificate_pem -def main(argv): +def main(argv=None): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent("""\ - This script automates the process of getting a signed TLS certificate from - Let's Encrypt using the ACME protocol. It will need to be run on your server - and have access to your private account key, so PLEASE READ THROUGH IT! It's - only ~200 lines, so it won't take long. + This script automates the process of getting a signed TLS certificate from Let's Encrypt using + the ACME protocol. It will need to be run on your server and have access to your private + account key, so PLEASE READ THROUGH IT! It's only ~200 lines, so it won't take long. - ===Example Usage=== - python acme_tiny.py --account-key ./account.key --csr ./domain.csr --acme-dir /usr/share/nginx/html/.well-known/acme-challenge/ > signed.crt - =================== + Example Usage: + python acme_tiny.py --account-key ./account.key --csr ./domain.csr --acme-dir /usr/share/nginx/html/.well-known/acme-challenge/ > signed_chain.crt - ===Example Crontab Renewal (once per month)=== - 0 0 1 * * python /path/to/acme_tiny.py --account-key /path/to/account.key --csr /path/to/domain.csr --acme-dir /usr/share/nginx/html/.well-known/acme-challenge/ > /path/to/signed.crt 2>> /var/log/acme_tiny.log - ============================================== + Example Crontab Renewal (once per month): + 0 0 1 * * python /path/to/acme_tiny.py --account-key /path/to/account.key --csr /path/to/domain.csr --acme-dir /usr/share/nginx/html/.well-known/acme-challenge/ > /path/to/signed_chain.crt 2>> /var/log/acme_tiny.log """) ) parser.add_argument("--account-key", required=True, help="path to your Let's Encrypt account private key") parser.add_argument("--csr", required=True, help="path to your certificate signing request") parser.add_argument("--acme-dir", required=True, help="path to the .well-known/acme-challenge/ directory") parser.add_argument("--quiet", action="store_const", const=logging.ERROR, help="suppress output except for errors") - parser.add_argument("--ca", default=DEFAULT_CA, help="certificate authority, default is Let's Encrypt") + parser.add_argument("--disable-check", default=False, action="store_true", help="disable checking if the challenge file is hosted correctly before telling the CA") + parser.add_argument("--directory-url", default=DEFAULT_DIRECTORY_URL, help="certificate authority directory url, default is Let's Encrypt") + parser.add_argument("--ca", default=DEFAULT_CA, help="DEPRECATED! USE --directory-url INSTEAD!") + parser.add_argument("--contact", metavar="CONTACT", default=None, nargs="*", help="Contact details (e.g. mailto:aaa@bbb.com) for your account-key") args = parser.parse_args(argv) LOGGER.setLevel(args.quiet or LOGGER.level) - signed_crt = get_crt(args.account_key, args.csr, args.acme_dir, log=LOGGER, CA=args.ca) + signed_crt = get_crt(args.account_key, args.csr, args.acme_dir, log=LOGGER, CA=args.ca, disable_check=args.disable_check, directory_url=args.directory_url, contact=args.contact) sys.stdout.write(signed_crt) if __name__ == "__main__": # pragma: no cover