From 98c7b60311ee664d06fb451cca016d41cdb761fe Mon Sep 17 00:00:00 2001 From: yunohost-bot Date: Thu, 9 Mar 2023 16:19:40 +0000 Subject: [PATCH 01/43] [CI] Format code with Black --- src/app.py | 4 ++- src/utils/resources.py | 58 ++++++++++++++++++++++++++++++++++-------- 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/src/app.py b/src/app.py index 091dd05d9..b37b680ec 100644 --- a/src/app.py +++ b/src/app.py @@ -1444,7 +1444,9 @@ def app_remove(operation_logger, app, purge=False, force_workdir=None): from yunohost.utils.resources import AppResourceManager AppResourceManager(app, wanted={}, current=manifest).apply( - rollback_and_raise_exception_if_failure=False, purge_data_dir=purge, action="remove" + rollback_and_raise_exception_if_failure=False, + purge_data_dir=purge, + action="remove", ) else: # Remove all permission in LDAP diff --git a/src/utils/resources.py b/src/utils/resources.py index 56ffa9156..87446bdd8 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -258,6 +258,7 @@ ynh_abort_if_errors # print(ret) + class SourcesResource(AppResource): """ Declare what are the sources / assets used by this app. Typically, this corresponds to some tarball published by the upstream project, that needs to be downloaded and extracted in the install dir using the ynh_setup_source helper. @@ -335,7 +336,6 @@ class SourcesResource(AppResource): sources: Dict[str, Dict[str, Any]] = {} def __init__(self, properties: Dict[str, Any], *args, **kwargs): - for source_id, infos in properties.items(): properties[source_id] = copy.copy(self.default_sources_properties) properties[source_id].update(infos) @@ -347,29 +347,37 @@ class SourcesResource(AppResource): rm(f"/var/cache/yunohost/download/{self.app}/", recursive=True) def provision_or_update(self, context: Dict = {}): - # Don't prefetch stuff during restore if context.get("action") == "restore": return for source_id, infos in self.sources.items(): - if not infos["prefetch"]: continue if infos["url"] is None: arch = system_arch() - if arch in infos and isinstance(infos[arch], dict) and isinstance(infos[arch].get("url"), str) and isinstance(infos[arch].get("sha256"), str): + if ( + arch in infos + and isinstance(infos[arch], dict) + and isinstance(infos[arch].get("url"), str) + and isinstance(infos[arch].get("sha256"), str) + ): self.prefetch(source_id, infos[arch]["url"], infos[arch]["sha256"]) else: - raise YunohostError(f"In resources.sources: it looks like you forgot to define url/sha256 or {arch}.url/{arch}.sha256", raw_msg=True) + raise YunohostError( + f"In resources.sources: it looks like you forgot to define url/sha256 or {arch}.url/{arch}.sha256", + raw_msg=True, + ) else: if infos["sha256"] is None: - raise YunohostError(f"In resources.sources: it looks like the sha256 is missing for {source_id}", raw_msg=True) + raise YunohostError( + f"In resources.sources: it looks like the sha256 is missing for {source_id}", + raw_msg=True, + ) self.prefetch(source_id, infos["url"], infos["sha256"]) def prefetch(self, source_id, url, expected_sha256): - logger.debug(f"Prefetching asset {source_id}: {url} ...") if not os.path.isdir(f"/var/cache/yunohost/download/{self.app}/"): @@ -378,21 +386,49 @@ class SourcesResource(AppResource): # NB: we use wget and not requests.get() because we want to output to a file (ie avoid ending up with the full archive in RAM) # AND the nice --tries, --no-dns-cache, --timeout options ... - p = subprocess.Popen(["/usr/bin/wget", "--tries=3", "--no-dns-cache", "--timeout=900", "--no-verbose", "--output-document=" + filename, url], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + p = subprocess.Popen( + [ + "/usr/bin/wget", + "--tries=3", + "--no-dns-cache", + "--timeout=900", + "--no-verbose", + "--output-document=" + filename, + url, + ], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) out, _ = p.communicate() returncode = p.returncode if returncode != 0: if os.path.exists(filename): rm(filename) - raise YunohostError("app_failed_to_download_asset", source_id=source_id, url=url, app=self.app, out=out.decode()) + raise YunohostError( + "app_failed_to_download_asset", + source_id=source_id, + url=url, + app=self.app, + out=out.decode(), + ) - assert os.path.exists(filename), f"For some reason, wget worked but {filename} doesnt exists?" + assert os.path.exists( + filename + ), f"For some reason, wget worked but {filename} doesnt exists?" computed_sha256 = check_output(f"sha256sum {filename}").split()[0] if computed_sha256 != expected_sha256: size = check_output(f"du -hs {filename}").split()[0] rm(filename) - raise YunohostError("app_corrupt_source", source_id=source_id, url=url, app=self.app, expected_sha256=expected_sha256, computed_sha256=computed_sha256, size=size) + raise YunohostError( + "app_corrupt_source", + source_id=source_id, + url=url, + app=self.app, + expected_sha256=expected_sha256, + computed_sha256=computed_sha256, + size=size, + ) class PermissionsResource(AppResource): From 89d139e47ac28d1a87ded2de0f31aa8ecaa39f7f Mon Sep 17 00:00:00 2001 From: yunohost-bot Date: Thu, 9 Mar 2023 16:47:09 +0000 Subject: [PATCH 02/43] [CI] Reformat / remove stale translated strings --- locales/ar.json | 2 +- locales/de.json | 2 +- locales/en.json | 2 +- locales/fr.json | 2 +- locales/gl.json | 2 +- locales/oc.json | 2 +- locales/pl.json | 2 +- locales/uk.json | 6 +++--- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/locales/ar.json b/locales/ar.json index feb375a94..8ff300109 100644 --- a/locales/ar.json +++ b/locales/ar.json @@ -257,4 +257,4 @@ "diagnosis_ip_dnsresolution_working": "تحليل اسم النطاق يعمل!", "diagnosis_mail_blacklist_listed_by": "إن عنوان الـ IP الخص بك أو نطاقك {item} مُدرَج ضمن قائمة سوداء على {blacklist_name}", "diagnosis_mail_outgoing_port_25_ok": "خادم بريد SMTP قادر على إرسال رسائل البريد الإلكتروني (منفذ البريد الصادر 25 غير محظور)." -} +} \ No newline at end of file diff --git a/locales/de.json b/locales/de.json index 2b7ee0456..b61d0a431 100644 --- a/locales/de.json +++ b/locales/de.json @@ -705,4 +705,4 @@ "app_change_url_failed": "Kann die URL für {app} nicht ändern: {error}", "app_change_url_script_failed": "Es ist ein Fehler im URL-Änderungs-Script aufgetreten", "app_resource_failed": "Automatische Ressourcen-Allokation (provisioning), die Unterbindung des Zugriffts auf Ressourcen (deprovisioning) oder die Aktualisierung der Ressourcen für {app} schlug fehl: {error}" -} +} \ No newline at end of file diff --git a/locales/en.json b/locales/en.json index ab606c81c..4dcb00ee6 100644 --- a/locales/en.json +++ b/locales/en.json @@ -26,9 +26,9 @@ "app_change_url_success": "{app} URL is now {domain}{path}", "app_config_unable_to_apply": "Failed to apply config panel values.", "app_config_unable_to_read": "Failed to read config panel values.", - "app_failed_to_download_asset": "Failed to download asset '{source_id}' ({url}) for {app}: {out}", "app_corrupt_source": "YunoHost was able to download the asset '{source_id}' ({url}) for {app}, but the asset doesn't match the expected checksum. This could mean that some temporary network failure happened on your server, OR the asset was somehow changed by the upstream maintainer (or a malicious actor?) and YunoHost packagers need to investigate and update the app manifest to reflect this change.\n Expected sha256 checksum: {expected_sha256}\n Downloaded sha256 checksum: {computed_sha256}\n Downloaded file size: {size}", "app_extraction_failed": "Could not extract the installation files", + "app_failed_to_download_asset": "Failed to download asset '{source_id}' ({url}) for {app}: {out}", "app_failed_to_upgrade_but_continue": "App {failed_app} failed to upgrade, continue to next upgrades as requested. Run 'yunohost log show {operation_logger_name}' to see failure log", "app_full_domain_unavailable": "Sorry, this app must be installed on a domain of its own, but other apps are already installed on the domain '{domain}'. You could use a subdomain dedicated to this app instead.", "app_id_invalid": "Invalid app ID", diff --git a/locales/fr.json b/locales/fr.json index b08b142c5..440fe1144 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -761,4 +761,4 @@ "app_not_upgraded_broken_system": "L'application '{failed_app}' n'a pas réussi à se mettre à jour et a mis le système dans un état alternatif car quelque chose est au moins momentanément \"cassé\". En conséquence, les mises à jour des applications suivantes ont été annulées : {apps}", "apps_failed_to_upgrade": "Ces applications n'ont pas pu être mises à jour : {apps}", "apps_failed_to_upgrade_line": "\n * {app_id} (pour voir le journal/log correspondant, faites un 'yunohost log show {operation_logger_name}')" -} +} \ No newline at end of file diff --git a/locales/gl.json b/locales/gl.json index 2b9e89ffb..065e41686 100644 --- a/locales/gl.json +++ b/locales/gl.json @@ -761,4 +761,4 @@ "invalid_shell": "Intérprete de ordes non válido: {shell}", "log_resource_snippet": "Aprovisionamento/desaprovisionamento/actualización dun recurso", "app_resource_failed": "Fallou o aprovisionamento, desaprovisionamento ou actualización de recursos para {app}: {error}" -} +} \ No newline at end of file diff --git a/locales/oc.json b/locales/oc.json index bdc9f5360..1c13fc6b5 100644 --- a/locales/oc.json +++ b/locales/oc.json @@ -469,4 +469,4 @@ "global_settings_setting_user_strength": "Fòrça del senhal utilizaire", "global_settings_setting_postfix_compatibility_help": "Solucion de compromés entre compatibilitat e seguretat pel servidor Postfix. Afècta los criptografs (e d’autres aspèctes ligats amb la seguretat)", "global_settings_setting_ssh_compatibility_help": "Solucion de compromés entre compatibilitat e seguretat pel servidor SSH. Afècta los criptografs (e d’autres aspèctes ligats amb la seguretat)" -} +} \ No newline at end of file diff --git a/locales/pl.json b/locales/pl.json index 9ce4e0950..c58f7223e 100644 --- a/locales/pl.json +++ b/locales/pl.json @@ -180,4 +180,4 @@ "certmanager_cert_renew_failed": "Nieudane odnowienie certyfikatu Let's Encrypt dla {domains}", "apps_failed_to_upgrade": "Nieudana aktualizacja aplikacji: {apps}", "backup_output_directory_required": "Musisz wybrać katalog dla kopii zapasowej" -} +} \ No newline at end of file diff --git a/locales/uk.json b/locales/uk.json index f1d689e40..fca0ea360 100644 --- a/locales/uk.json +++ b/locales/uk.json @@ -736,7 +736,7 @@ "password_confirmation_not_the_same": "Пароль і його підтвердження не збігаються", "password_too_long": "Будь ласка, виберіть пароль коротший за 127 символів", "pattern_fullname": "Має бути дійсне повне ім’я (принаймні 3 символи)", - "app_failed_to_upgrade_but_continue": "Застосунок {failed_app} не вдалося оновити, продовжуйте наступні оновлення відповідно до запиту. Запустіть 'yunohost log show {назва_логгера_операції}', щоб побачити журнал помилок", + "app_failed_to_upgrade_but_continue": "Застосунок {failed_app} не вдалося оновити, продовжуйте наступні оновлення відповідно до запиту. Запустіть 'yunohost log show {operation_logger_name}', щоб побачити журнал помилок", "app_not_upgraded_broken_system": "Застосунок '{failed_app}' не зміг оновитися і перевів систему в неробочий стан, і як наслідок, оновлення наступних застосунків було скасовано: {apps}", "app_not_upgraded_broken_system_continue": "Застосунок '{failed_app}' не зміг оновитися і перевів систему у неробочий стан (тому --continue-on-failure ігнорується), і як наслідок, оновлення наступних застосунків було скасовано: {apps}", "confirm_app_insufficient_ram": "НЕБЕЗПЕКА! Цей застосунок вимагає {required} оперативної пам'яті для встановлення/оновлення, але зараз доступно лише {current}. Навіть якби цей застосунок можна було б запустити, процес його встановлення/оновлення вимагає великої кількості оперативної пам'яті, тому ваш сервер може зависнути і вийти з ладу. Якщо ви все одно готові піти на цей ризик, введіть '{answers}'", @@ -760,5 +760,5 @@ "app_not_enough_ram": "Для встановлення/оновлення цього застосунку потрібно {required} оперативної пам'яті, але наразі доступно лише {current}.", "app_resource_failed": "Не вдалося надати, позбавити або оновити ресурси для {app}: {error}", "apps_failed_to_upgrade": "Ці застосунки не вдалося оновити:{apps}", - "apps_failed_to_upgrade_line": "\n * {app_id} (щоб побачити відповідний журнал, виконайте 'yunohost log show {назва_логгера_операції}')" -} + "apps_failed_to_upgrade_line": "\n * {app_id} (щоб побачити відповідний журнал, виконайте 'yunohost log show {operation_logger_name}')" +} \ No newline at end of file From 69518b541728d9fdf47ce71b3752248267b97759 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 9 Mar 2023 20:41:29 +0100 Subject: [PATCH 03/43] Bash being bash ~_~ --- helpers/logging | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/logging b/helpers/logging index 82cb2814a..ab5d564aa 100644 --- a/helpers/logging +++ b/helpers/logging @@ -309,7 +309,7 @@ ynh_script_progression() { local print_exec_time="" if [ $time -eq 1 ] && [ "$exec_time" -gt 10 ]; then - print_exec_time=" [$(bc <<< 'scale=1; $exec_time / 60' ) minutes]" + print_exec_time=" [$(bc <<< "scale=1; $exec_time / 60" ) minutes]" fi ynh_print_info "[$progression_bar] > ${message}${print_exec_time}" From 7491dd4c50ff9e99f045c5c6ed9ddb6df1764e9b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 9 Mar 2023 20:57:33 +0100 Subject: [PATCH 04/43] helpers: Fix documentation for ynh_setup_source --- helpers/utils | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/helpers/utils b/helpers/utils index 695b165c0..4a964a14e 100644 --- a/helpers/utils +++ b/helpers/utils @@ -89,7 +89,9 @@ fi # sha256 = "0123456789abcdef" # The sha256 sum of the asset obtained from the URL # ``` # -# # Optional flags: +# ##### Optional flags +# +# ```text # format = "tar.gz"/xz/bz2 # automatically guessed from the extension of the URL, but can be set explicitly. Will use `tar` to extract # "zip" # automatically guessed from the extension of the URL, but can be set explicitly. Will use `unzip` to extract # "docker" # useful to extract files from an already-built docker image (instead of rebuilding them locally). Will use `docker-image-extract` to extract @@ -105,7 +107,7 @@ fi # # rename = "whatever_your_want" # to be used for convenience when `extract` is false and the default name of the file is not practical # platform = "linux/amd64" # (defaults to "linux/$YNH_ARCH") to be used in conjonction with `format = "docker"` to specify which architecture to extract for -# +# ``` # # You may also define assets url and checksum per-architectures such as: # ```toml From 5b58e0e60c2ad231952104298479c30521cf6a46 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 9 Mar 2023 21:17:02 +0100 Subject: [PATCH 05/43] doc: Fix version number in autogenerated resource doc --- doc/generate_resource_doc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/generate_resource_doc.py b/doc/generate_resource_doc.py index 272845104..201d25265 100644 --- a/doc/generate_resource_doc.py +++ b/doc/generate_resource_doc.py @@ -2,7 +2,7 @@ import ast import datetime import subprocess -version = (open("../debian/changelog").readlines()[0].split()[1].strip("()"),) +version = open("../debian/changelog").readlines()[0].split()[1].strip("()") today = datetime.datetime.now().strftime("%d/%m/%Y") From 13ac9dade639cf104b038c45b862e0762e9c518f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 10 Mar 2023 16:00:53 +0100 Subject: [PATCH 06/43] helpers/nodejs: simplify 'n' script install and maintenance --- .github/workflows/n_updater.sh | 78 -- .github/workflows/n_updater.yml | 3 +- helpers/nodejs | 34 +- helpers/vendor/n/LICENSE | 21 + helpers/vendor/n/README.md | 1 + helpers/vendor/n/n | 1621 +++++++++++++++++++++++++++++++ 6 files changed, 1649 insertions(+), 109 deletions(-) delete mode 100644 .github/workflows/n_updater.sh create mode 100644 helpers/vendor/n/LICENSE create mode 100644 helpers/vendor/n/README.md create mode 100755 helpers/vendor/n/n diff --git a/.github/workflows/n_updater.sh b/.github/workflows/n_updater.sh deleted file mode 100644 index a8b0b0eec..000000000 --- a/.github/workflows/n_updater.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -#================================================= -# N UPDATING HELPER -#================================================= - -# This script is meant to be run by GitHub Actions. -# It is derived from the Updater script from the YunoHost-Apps organization. -# It aims to automate the update of `n`, the Node version management system. - -#================================================= -# FETCHING LATEST RELEASE AND ITS ASSETS -#================================================= - -# Fetching information -source helpers/nodejs -current_version="$n_version" -repo="tj/n" -# Some jq magic is needed, because the latest upstream release is not always the latest version (e.g. security patches for older versions) -version=$(curl --silent "https://api.github.com/repos/$repo/releases" | jq -r '.[] | select( .prerelease != true ) | .tag_name' | sort -V | tail -1) - -# Later down the script, we assume the version has only digits and dots -# Sometimes the release name starts with a "v", so let's filter it out. -if [[ ${version:0:1} == "v" || ${version:0:1} == "V" ]]; then - version=${version:1} -fi - -# Setting up the environment variables -echo "Current version: $current_version" -echo "Latest release from upstream: $version" -echo "VERSION=$version" >> $GITHUB_ENV -# For the time being, let's assume the script will fail -echo "PROCEED=false" >> $GITHUB_ENV - -# Proceed only if the retrieved version is greater than the current one -if ! dpkg --compare-versions "$current_version" "lt" "$version" ; then - echo "::warning ::No new version available" - exit 0 -# Proceed only if a PR for this new version does not already exist -elif git ls-remote -q --exit-code --heads https://github.com/${GITHUB_REPOSITORY:-YunoHost/yunohost}.git ci-auto-update-n-v$version ; then - echo "::warning ::A branch already exists for this update" - exit 0 -fi - -#================================================= -# UPDATE SOURCE FILES -#================================================= - -asset_url="https://github.com/tj/n/archive/v${version}.tar.gz" - -echo "Handling asset at $asset_url" - -# Create the temporary directory -tempdir="$(mktemp -d)" - -# Download sources and calculate checksum -filename=${asset_url##*/} -curl --silent -4 -L $asset_url -o "$tempdir/$filename" -checksum=$(sha256sum "$tempdir/$filename" | head -c 64) - -# Delete temporary directory -rm -rf $tempdir - -echo "Calculated checksum for n v${version} is $checksum" - -#================================================= -# GENERIC FINALIZATION -#================================================= - -# Replace new version in helper -sed -i -E "s/^n_version=.*$/n_version=$version/" helpers/nodejs - -# Replace checksum in helper -sed -i -E "s/^n_checksum=.*$/n_checksum=$checksum/" helpers/nodejs - -# The Action will proceed only if the PROCEED environment variable is set to true -echo "PROCEED=true" >> $GITHUB_ENV -exit 0 diff --git a/.github/workflows/n_updater.yml b/.github/workflows/n_updater.yml index 4c422c14c..ce3e9c925 100644 --- a/.github/workflows/n_updater.yml +++ b/.github/workflows/n_updater.yml @@ -21,7 +21,8 @@ jobs: git config --global user.name 'yunohost-bot' git config --global user.email 'yunohost-bot@users.noreply.github.com' # Run the updater script - /bin/bash .github/workflows/n_updater.sh + wget https://raw.githubusercontent.com/tj/n/master/bin/n --output-document=helpers/vendor/n/n + [[ -z "$(git diff helpers/vendor/n/n)" ]] || echo "PROCEED=true" >> $GITHUB_ENV - name: Commit changes id: commit if: ${{ env.PROCEED == 'true' }} diff --git a/helpers/nodejs b/helpers/nodejs index b692bfc70..e3ccf82dd 100644 --- a/helpers/nodejs +++ b/helpers/nodejs @@ -1,32 +1,10 @@ #!/bin/bash -n_version=9.0.1 -n_checksum=ad305e8ee9111aa5b08e6dbde23f01109401ad2d25deecacd880b3f9ea45702b n_install_dir="/opt/node_n" node_version_path="$n_install_dir/n/versions/node" # N_PREFIX is the directory of n, it needs to be loaded as a environment variable. export N_PREFIX="$n_install_dir" -# Install Node version management -# -# [internal] -# -# usage: ynh_install_n -# -# Requires YunoHost version 2.7.12 or higher. -ynh_install_n() { - # Build an app.src for n - echo "SOURCE_URL=https://github.com/tj/n/archive/v${n_version}.tar.gz -SOURCE_SUM=${n_checksum}" >"$YNH_APP_BASEDIR/conf/n.src" - # Download and extract n - ynh_setup_source --dest_dir="$n_install_dir/git" --source_id=n - # Install n - ( - cd "$n_install_dir/git" - PREFIX=$N_PREFIX make install 2>&1 - ) -} - # Load the version of node for an app, and set variables. # # usage: ynh_use_nodejs @@ -133,14 +111,10 @@ ynh_install_nodejs() { test -x /usr/bin/node && mv /usr/bin/node /usr/bin/node_n test -x /usr/bin/npm && mv /usr/bin/npm /usr/bin/npm_n - # If n is not previously setup, install it - if ! $n_install_dir/bin/n --version >/dev/null 2>&1; then - ynh_install_n - elif dpkg --compare-versions "$($n_install_dir/bin/n --version)" lt $n_version; then - ynh_install_n - fi - - # Modify the default N_PREFIX in n script + # Install (or update if YunoHost vendor/ folder updated since last install) n + mkdir -p $n_install_dir/bin/ + cp /usr/share/yunohost/helpers.d/vendor/n/n $n_install_dir/bin/n + # Tweak for n to understand it's installed in $N_PREFIX ynh_replace_string --match_string="^N_PREFIX=\${N_PREFIX-.*}$" --replace_string="N_PREFIX=\${N_PREFIX-$N_PREFIX}" --target_file="$n_install_dir/bin/n" # Restore /usr/local/bin in PATH diff --git a/helpers/vendor/n/LICENSE b/helpers/vendor/n/LICENSE new file mode 100644 index 000000000..8e04e8467 --- /dev/null +++ b/helpers/vendor/n/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 TJ Holowaychuk + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/helpers/vendor/n/README.md b/helpers/vendor/n/README.md new file mode 100644 index 000000000..9a29a3936 --- /dev/null +++ b/helpers/vendor/n/README.md @@ -0,0 +1 @@ +This is taken from https://github.com/tj/n/ diff --git a/helpers/vendor/n/n b/helpers/vendor/n/n new file mode 100755 index 000000000..2739e2d00 --- /dev/null +++ b/helpers/vendor/n/n @@ -0,0 +1,1621 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2155 +# Disabled "Declare and assign separately to avoid masking return values": https://github.com/koalaman/shellcheck/wiki/SC2155 + +# +# log +# + +log() { + printf " ${SGR_CYAN}%10s${SGR_RESET} : ${SGR_FAINT}%s${SGR_RESET}\n" "$1" "$2" +} + +# +# verbose_log +# Can suppress with --quiet. +# Like log but to stderr rather than stdout, so can also be used from "display" routines. +# + +verbose_log() { + if [[ "${SHOW_VERBOSE_LOG}" == "true" ]]; then + >&2 printf " ${SGR_CYAN}%10s${SGR_RESET} : ${SGR_FAINT}%s${SGR_RESET}\n" "$1" "$2" + fi +} + +# +# Exit with the given +# + +abort() { + >&2 printf "\n ${SGR_RED}Error: %s${SGR_RESET}\n\n" "$*" && exit 1 +} + +# +# Synopsis: trace message ... +# Debugging output to stderr, not used in production code. +# + +function trace() { + >&2 printf "trace: %s\n" "$*" +} + +# +# Synopsis: echo_red message ... +# Highlight message in colour (on stdout). +# + +function echo_red() { + printf "${SGR_RED}%s${SGR_RESET}\n" "$*" +} + +# +# Synopsis: n_grep +# grep wrapper to ensure consistent grep options and circumvent aliases. +# + +function n_grep() { + GREP_OPTIONS='' command grep "$@" +} + +# +# Setup and state +# + +VERSION="v9.0.1" + +N_PREFIX="${N_PREFIX-/usr/local}" +N_PREFIX=${N_PREFIX%/} +readonly N_PREFIX + +N_CACHE_PREFIX="${N_CACHE_PREFIX-${N_PREFIX}}" +N_CACHE_PREFIX=${N_CACHE_PREFIX%/} +CACHE_DIR="${N_CACHE_PREFIX}/n/versions" +readonly N_CACHE_PREFIX CACHE_DIR + +N_NODE_MIRROR=${N_NODE_MIRROR:-${NODE_MIRROR:-https://nodejs.org/dist}} +N_NODE_MIRROR=${N_NODE_MIRROR%/} +readonly N_NODE_MIRROR + +N_NODE_DOWNLOAD_MIRROR=${N_NODE_DOWNLOAD_MIRROR:-https://nodejs.org/download} +N_NODE_DOWNLOAD_MIRROR=${N_NODE_DOWNLOAD_MIRROR%/} +readonly N_NODE_DOWNLOAD_MIRROR + +# Using xz instead of gzip is enabled by default, if xz compatibility checks pass. +# User may set N_USE_XZ to 0 to disable, or set to anything else to enable. +# May also be overridden by command line flags. + +# Normalise external values to true/false +if [[ "${N_USE_XZ}" = "0" ]]; then + N_USE_XZ="false" +elif [[ -n "${N_USE_XZ+defined}" ]]; then + N_USE_XZ="true" +fi +# Not setting to readonly. Overriden by CLI flags, and update_xz_settings_for_version. + +N_MAX_REMOTE_MATCHES=${N_MAX_REMOTE_MATCHES:-20} +# modified by update_mirror_settings_for_version +g_mirror_url=${N_NODE_MIRROR} +g_mirror_folder_name="node" + +# Options for curl and wget. +# Defining commands in variables is fraught (https://mywiki.wooledge.org/BashFAQ/050) +# but we can follow the simple case and store arguments in an array. + +GET_SHOWS_PROGRESS="false" +# --location to follow redirects +# --fail to avoid happily downloading error page from web server for 404 et al +# --show-error to show why failed (on stderr) +CURL_OPTIONS=( "--location" "--fail" "--show-error" ) +if [[ -t 1 ]]; then + CURL_OPTIONS+=( "--progress-bar" ) + command -v curl &> /dev/null && GET_SHOWS_PROGRESS="true" +else + CURL_OPTIONS+=( "--silent" ) +fi +WGET_OPTIONS=( "-q" "-O-" ) + +# Legacy support using unprefixed env. No longer documented in README. +if [ -n "$HTTP_USER" ];then + if [ -z "$HTTP_PASSWORD" ]; then + abort "Must specify HTTP_PASSWORD when supplying HTTP_USER" + fi + CURL_OPTIONS+=( "-u $HTTP_USER:$HTTP_PASSWORD" ) + WGET_OPTIONS+=( "--http-password=$HTTP_PASSWORD" + "--http-user=$HTTP_USER" ) +elif [ -n "$HTTP_PASSWORD" ]; then + abort "Must specify HTTP_USER when supplying HTTP_PASSWORD" +fi + +# Set by set_active_node +g_active_node= + +# set by various lookups to allow mixed logging and return value from function, especially for engine and node +g_target_node= + +DOWNLOAD=false # set to opt-out of activate (install), and opt-in to download (run, exec) +ARCH= +SHOW_VERBOSE_LOG="true" + +# ANSI escape codes +# https://en.wikipedia.org/wiki/ANSI_escape_code +# https://no-color.org +# https://bixense.com/clicolors + +USE_COLOR="true" +if [[ -n "${CLICOLOR_FORCE+defined}" && "${CLICOLOR_FORCE}" != "0" ]]; then + USE_COLOR="true" +elif [[ -n "${NO_COLOR+defined}" || "${CLICOLOR}" = "0" || ! -t 1 ]]; then + USE_COLOR="false" +fi +readonly USE_COLOR +# Select Graphic Rendition codes +if [[ "${USE_COLOR}" = "true" ]]; then + # KISS and use codes rather than tput, avoid dealing with missing tput or TERM. + readonly SGR_RESET="\033[0m" + readonly SGR_FAINT="\033[2m" + readonly SGR_RED="\033[31m" + readonly SGR_CYAN="\033[36m" +else + readonly SGR_RESET= + readonly SGR_FAINT= + readonly SGR_RED= + readonly SGR_CYAN= +fi + +# +# set_arch to override $(uname -a) +# + +set_arch() { + if test -n "$1"; then + ARCH="$1" + else + abort "missing -a|--arch value" + fi +} + +# +# Synopsis: set_insecure +# Globals modified: +# - CURL_OPTIONS +# - WGET_OPTIONS +# + +function set_insecure() { + CURL_OPTIONS+=( "--insecure" ) + WGET_OPTIONS+=( "--no-check-certificate" ) +} + +# +# Synposis: display_major_version numeric-version +# +display_major_version() { + local version=$1 + version="${version#v}" + version="${version%%.*}" + echo "${version}" +} + +# +# Synopsis: update_mirror_settings_for_version version +# e.g. means using download mirror and folder is nightly +# Globals modified: +# - g_mirror_url +# - g_mirror_folder_name +# + +function update_mirror_settings_for_version() { + if is_download_folder "$1" ; then + g_mirror_folder_name="$1" + g_mirror_url="${N_NODE_DOWNLOAD_MIRROR}/${g_mirror_folder_name}" + elif is_download_version "$1"; then + [[ "$1" =~ ^([^/]+)/(.*) ]] + local remote_folder="${BASH_REMATCH[1]}" + g_mirror_folder_name="${remote_folder}" + g_mirror_url="${N_NODE_DOWNLOAD_MIRROR}/${g_mirror_folder_name}" + fi +} + +# +# Synopsis: update_xz_settings_for_version numeric-version +# Globals modified: +# - N_USE_XZ +# + +function update_xz_settings_for_version() { + # tarballs in xz format were available in later version of iojs, but KISS and only use xz from v4. + if [[ "${N_USE_XZ}" = "true" ]]; then + local major_version="$(display_major_version "$1")" + if [[ "${major_version}" -lt 4 ]]; then + N_USE_XZ="false" + fi + fi +} + +# +# Synopsis: update_arch_settings_for_version numeric-version +# Globals modified: +# - ARCH +# + +function update_arch_settings_for_version() { + local tarball_platform="$(display_tarball_platform)" + if [[ -z "${ARCH}" && "${tarball_platform}" = "darwin-arm64" ]]; then + # First native builds were for v16, but can use x64 in rosetta for older versions. + local major_version="$(display_major_version "$1")" + if [[ "${major_version}" -lt 16 ]]; then + ARCH=x64 + fi + fi +} + +# +# Synopsis: is_lts_codename version +# + +function is_lts_codename() { + # https://github.com/nodejs/Release/blob/master/CODENAMES.md + # e.g. argon, Boron + [[ "$1" =~ ^([Aa]rgon|[Bb]oron|[Cc]arbon|[Dd]ubnium|[Ee]rbium|[Ff]ermium|[Gg]allium|[Hh]ydrogen|[Ii]ron|[Jj]od)$ ]] +} + +# +# Synopsis: is_download_folder version +# + +function is_download_folder() { + # e.g. nightly + [[ "$1" =~ ^(next-nightly|nightly|rc|release|test|v8-canary)$ ]] +} + +# +# Synopsis: is_download_version version +# + +function is_download_version() { + # e.g. nightly/, nightly/latest, nightly/v11 + if [[ "$1" =~ ^([^/]+)/(.*) ]]; then + local remote_folder="${BASH_REMATCH[1]}" + is_download_folder "${remote_folder}" + return + fi + return 2 +} + +# +# Synopsis: is_numeric_version version +# + +function is_numeric_version() { + # e.g. 6, v7.1, 8.11.3 + [[ "$1" =~ ^[v]{0,1}[0-9]+(\.[0-9]+){0,2}$ ]] +} + +# +# Synopsis: is_exact_numeric_version version +# + +function is_exact_numeric_version() { + # e.g. 6, v7.1, 8.11.3 + [[ "$1" =~ ^[v]{0,1}[0-9]+\.[0-9]+\.[0-9]+$ ]] +} + +# +# Synopsis: is_node_support_version version +# Reference: https://github.com/nodejs/package-maintenance/issues/236#issue-474783582 +# + +function is_node_support_version() { + [[ "$1" =~ ^(active|lts_active|lts_latest|lts|current|supported)$ ]] +} + +# +# Synopsis: display_latest_node_support_alias version +# Map aliases onto existing n aliases, current and lts +# + +function display_latest_node_support_alias() { + case "$1" in + "active") printf "current" ;; + "lts_active") printf "lts" ;; + "lts_latest") printf "lts" ;; + "lts") printf "lts" ;; + "current") printf "current" ;; + "supported") printf "current" ;; + *) printf "unexpected-version" + esac +} + +# +# Functions used when showing versions installed +# + +enter_fullscreen() { + # Set cursor to be invisible + tput civis 2> /dev/null + # Save screen contents + tput smcup 2> /dev/null + stty -echo +} + +leave_fullscreen() { + # Set cursor to normal + tput cnorm 2> /dev/null + # Restore screen contents + tput rmcup 2> /dev/null + stty echo +} + +handle_sigint() { + leave_fullscreen + S="$?" + kill 0 + exit $S +} + +handle_sigtstp() { + leave_fullscreen + kill -s SIGSTOP $$ +} + +# +# Output usage information. +# + +display_help() { + cat <<-EOF + +Usage: n [options] [COMMAND] [args] + +Commands: + + n Display downloaded Node.js versions and install selection + n latest Install the latest Node.js release (downloading if necessary) + n lts Install the latest LTS Node.js release (downloading if necessary) + n Install Node.js (downloading if necessary) + n install Install Node.js (downloading if necessary) + n run [args ...] Execute downloaded Node.js with [args ...] + n which Output path for downloaded node + n exec [args...] Execute command with modified PATH, so downloaded node and npm first + n rm Remove the given downloaded version(s) + n prune Remove all downloaded versions except the installed version + n --latest Output the latest Node.js version available + n --lts Output the latest LTS Node.js version available + n ls Output downloaded versions + n ls-remote [version] Output matching versions available for download + n uninstall Remove the installed Node.js + +Options: + + -V, --version Output version of n + -h, --help Display help information + -p, --preserve Preserve npm and npx during install of Node.js + -q, --quiet Disable curl output. Disable log messages processing "auto" and "engine" labels. + -d, --download Download if necessary, and don't make active + -a, --arch Override system architecture + --all ls-remote displays all matches instead of last 20 + --insecure Turn off certificate checking for https requests (may be needed from behind a proxy server) + --use-xz/--no-use-xz Override automatic detection of xz support and enable/disable use of xz compressed node downloads. + +Aliases: + + install: i + latest: current + ls: list + lsr: ls-remote + lts: stable + rm: - + run: use, as + which: bin + +Versions: + + Numeric version numbers can be complete or incomplete, with an optional leading 'v'. + Versions can also be specified by label, or codename, + and other downloadable releases by / + + 4.9.1, 8, v6.1 Numeric versions + lts Newest Long Term Support official release + latest, current Newest official release + auto Read version from file: .n-node-version, .node-version, .nvmrc, or package.json + engine Read version from package.json + boron, carbon Codenames for release streams + lts_latest Node.js support aliases + + and nightly, rc/10 et al + +EOF +} + +err_no_installed_print_help() { + display_help + abort "no downloaded versions yet, see above help for commands" +} + +# +# Synopsis: next_version_installed selected_version +# Output version after selected (which may be blank under some circumstances). +# + +function next_version_installed() { + display_cache_versions | n_grep "$1" -A 1 | tail -n 1 +} + +# +# Synopsis: prev_version_installed selected_version +# Output version before selected (which may be blank under some circumstances). +# + +function prev_version_installed() { + display_cache_versions | n_grep "$1" -B 1 | head -n 1 +} + +# +# Output n version. +# + +display_n_version() { + echo "$VERSION" && exit 0 +} + +# +# Synopsis: set_active_node +# Checks cached downloads for a binary matching the active node. +# Globals modified: +# - g_active_node +# + +function set_active_node() { + g_active_node= + local node_path="$(command -v node)" + if [[ -x "${node_path}" ]]; then + local installed_version=$(node --version) + installed_version=${installed_version#v} + for dir in "${CACHE_DIR}"/*/ ; do + local folder_name="${dir%/}" + folder_name="${folder_name##*/}" + if diff &> /dev/null \ + "${CACHE_DIR}/${folder_name}/${installed_version}/bin/node" \ + "${node_path}" ; then + g_active_node="${folder_name}/${installed_version}" + break + fi + done + fi +} + +# +# Display sorted versions directories paths. +# + +display_versions_paths() { + find "$CACHE_DIR" -maxdepth 2 -type d \ + | sed 's|'"$CACHE_DIR"'/||g' \ + | n_grep -E "/[0-9]+\.[0-9]+\.[0-9]+" \ + | sed 's|/|.|' \ + | sort -k 1,1 -k 2,2n -k 3,3n -k 4,4n -t . \ + | sed 's|\.|/|' +} + +# +# Display installed versions with +# + +display_versions_with_selected() { + local selected="$1" + echo + for version in $(display_versions_paths); do + if test "$version" = "$selected"; then + printf " ${SGR_CYAN}ο${SGR_RESET} %s\n" "$version" + else + printf " ${SGR_FAINT}%s${SGR_RESET}\n" "$version" + fi + done + echo + printf "Use up/down arrow keys to select a version, return key to install, d to delete, q to quit" +} + +# +# Synopsis: display_cache_versions +# + +function display_cache_versions() { + for folder_and_version in $(display_versions_paths); do + echo "${folder_and_version}" + done +} + +# +# Display current node --version and others installed. +# + +menu_select_cache_versions() { + enter_fullscreen + set_active_node + local selected="${g_active_node}" + + clear + display_versions_with_selected "${selected}" + + trap handle_sigint INT + trap handle_sigtstp SIGTSTP + + ESCAPE_SEQ=$'\033' + UP=$'A' + DOWN=$'B' + CTRL_P=$'\020' + CTRL_N=$'\016' + + while true; do + read -rsn 1 key + case "$key" in + "$ESCAPE_SEQ") + # Handle ESC sequences followed by other characters, i.e. arrow keys + read -rsn 1 -t 1 tmp + # See "[" if terminal in normal mode, and "0" in application mode + if [[ "$tmp" == "[" || "$tmp" == "O" ]]; then + read -rsn 1 -t 1 arrow + case "$arrow" in + "$UP") + clear + selected="$(prev_version_installed "${selected}")" + display_versions_with_selected "${selected}" + ;; + "$DOWN") + clear + selected="$(next_version_installed "${selected}")" + display_versions_with_selected "${selected}" + ;; + esac + fi + ;; + "d") + if [[ -n "${selected}" ]]; then + clear + # Note: prev/next is constrained to min/max + local after_delete_selection="$(next_version_installed "${selected}")" + if [[ "${after_delete_selection}" == "${selected}" ]]; then + after_delete_selection="$(prev_version_installed "${selected}")" + fi + remove_versions "${selected}" + + if [[ "${after_delete_selection}" == "${selected}" ]]; then + clear + leave_fullscreen + echo "All downloaded versions have been deleted from cache." + exit + fi + + selected="${after_delete_selection}" + display_versions_with_selected "${selected}" + fi + ;; + # Vim or Emacs 'up' key + "k"|"$CTRL_P") + clear + selected="$(prev_version_installed "${selected}")" + display_versions_with_selected "${selected}" + ;; + # Vim or Emacs 'down' key + "j"|"$CTRL_N") + clear + selected="$(next_version_installed "${selected}")" + display_versions_with_selected "${selected}" + ;; + "q") + clear + leave_fullscreen + exit + ;; + "") + # enter key returns empty string + leave_fullscreen + [[ -n "${selected}" ]] && activate "${selected}" + exit + ;; + esac + done +} + +# +# Move up a line and erase. +# + +erase_line() { + printf "\033[1A\033[2K" +} + +# +# Disable PaX mprotect for +# + +disable_pax_mprotect() { + test -z "$1" && abort "binary required" + local binary="$1" + + # try to disable mprotect via XATTR_PAX header + local PAXCTL="$(PATH="/sbin:/usr/sbin:$PATH" command -v paxctl-ng 2>&1)" + local PAXCTL_ERROR=1 + if [ -x "$PAXCTL" ]; then + $PAXCTL -l && $PAXCTL -m "$binary" >/dev/null 2>&1 + PAXCTL_ERROR="$?" + fi + + # try to disable mprotect via PT_PAX header + if [ "$PAXCTL_ERROR" != 0 ]; then + PAXCTL="$(PATH="/sbin:/usr/sbin:$PATH" command -v paxctl 2>&1)" + if [ -x "$PAXCTL" ]; then + $PAXCTL -Cm "$binary" >/dev/null 2>&1 + fi + fi +} + +# +# clean_copy_folder +# + +clean_copy_folder() { + local source="$1" + local target="$2" + if [[ -d "${source}" ]]; then + rm -rf "${target}" + cp -fR "${source}" "${target}" + fi +} + +# +# Activate +# + +activate() { + local version="$1" + local dir="$CACHE_DIR/$version" + local original_node="$(command -v node)" + local installed_node="${N_PREFIX}/bin/node" + log "copying" "$version" + + + # Ideally we would just copy from cache to N_PREFIX, but there are some complications + # - various linux versions use symlinks for folders in /usr/local and also error when copy folder onto symlink + # - we have used cp for years, so keep using it for backwards compatibility (instead of say rsync) + # - we allow preserving npm + # - we want to be somewhat robust to changes in tarball contents, so use find instead of hard-code expected subfolders + # + # This code was purist and concise for a long time. + # Now twice as much code, but using same code path for all uses, and supporting more setups. + + # Copy lib before bin so symlink targets exist. + # lib + mkdir -p "$N_PREFIX/lib" + # Copy everything except node_modules. + find "$dir/lib" -mindepth 1 -maxdepth 1 \! -name node_modules -exec cp -fR "{}" "$N_PREFIX/lib" \; + if [[ -z "${N_PRESERVE_NPM}" ]]; then + mkdir -p "$N_PREFIX/lib/node_modules" + # Copy just npm, skipping possible added global modules after download. Clean copy to avoid version change problems. + clean_copy_folder "$dir/lib/node_modules/npm" "$N_PREFIX/lib/node_modules/npm" + fi + # Takes same steps for corepack (experimental in node 16.9.0) as for npm, to avoid version problems. + if [[ -e "$dir/lib/node_modules/corepack" && -z "${N_PRESERVE_COREPACK}" ]]; then + mkdir -p "$N_PREFIX/lib/node_modules" + clean_copy_folder "$dir/lib/node_modules/corepack" "$N_PREFIX/lib/node_modules/corepack" + fi + + # bin + mkdir -p "$N_PREFIX/bin" + # Remove old node to avoid potential problems with firewall getting confused on Darwin by overwrite. + rm -f "$N_PREFIX/bin/node" + # Copy bin items by hand, in case user has installed global npm modules into cache. + cp -f "$dir/bin/node" "$N_PREFIX/bin" + [[ -e "$dir/bin/node-waf" ]] && cp -f "$dir/bin/node-waf" "$N_PREFIX/bin" # v0.8.x + if [[ -z "${N_PRESERVE_COREPACK}" ]]; then + [[ -e "$dir/bin/corepack" ]] && cp -fR "$dir/bin/corepack" "$N_PREFIX/bin" # from 16.9.0 + fi + if [[ -z "${N_PRESERVE_NPM}" ]]; then + [[ -e "$dir/bin/npm" ]] && cp -fR "$dir/bin/npm" "$N_PREFIX/bin" + [[ -e "$dir/bin/npx" ]] && cp -fR "$dir/bin/npx" "$N_PREFIX/bin" + fi + + # include + mkdir -p "$N_PREFIX/include" + find "$dir/include" -mindepth 1 -maxdepth 1 -exec cp -fR "{}" "$N_PREFIX/include" \; + + # share + mkdir -p "$N_PREFIX/share" + # Copy everything except man, at it is a symlink on some Linux (e.g. archlinux). + find "$dir/share" -mindepth 1 -maxdepth 1 \! -name man -exec cp -fR "{}" "$N_PREFIX/share" \; + mkdir -p "$N_PREFIX/share/man" + find "$dir/share/man" -mindepth 1 -maxdepth 1 -exec cp -fR "{}" "$N_PREFIX/share/man" \; + + disable_pax_mprotect "${installed_node}" + + local active_node="$(command -v node)" + if [[ -e "${active_node}" && -e "${installed_node}" && "${active_node}" != "${installed_node}" ]]; then + # Installed and active are different which might be a PATH problem. List both to give user some clues. + log "installed" "$("${installed_node}" --version) to ${installed_node}" + log "active" "$("${active_node}" --version) at ${active_node}" + else + local npm_version_str="" + local installed_npm="${N_PREFIX}/bin/npm" + local active_npm="$(command -v npm)" + if [[ -z "${N_PRESERVE_NPM}" && -e "${active_npm}" && -e "${installed_npm}" && "${active_npm}" = "${installed_npm}" ]]; then + npm_version_str=" (with npm $(npm --version))" + fi + + log "installed" "$("${installed_node}" --version)${npm_version_str}" + + # Extra tips for changed location. + if [[ -e "${active_node}" && -e "${original_node}" && "${active_node}" != "${original_node}" ]]; then + printf '\nNote: the node command changed location and the old location may be remembered in your current shell.\n' + log old "${original_node}" + log new "${active_node}" + printf 'If "node --version" shows the old version then start a new shell, or reset the location hash with:\nhash -r (for bash, zsh, ash, dash, and ksh)\nrehash (for csh and tcsh)\n' + fi + fi +} + +# +# Install +# + +install() { + [[ -z "$1" ]] && abort "version required" + local version + get_latest_resolved_version "$1" || return 2 + version="${g_target_node}" + [[ -n "${version}" ]] || abort "no version found for '$1'" + update_mirror_settings_for_version "$1" + update_xz_settings_for_version "${version}" + update_arch_settings_for_version "${version}" + + local dir="${CACHE_DIR}/${g_mirror_folder_name}/${version}" + + # Note: decompression flags ignored with default Darwin tar which autodetects. + if test "$N_USE_XZ" = "true"; then + local tarflag="-Jx" + else + local tarflag="-zx" + fi + + if test -d "$dir"; then + if [[ ! -e "$dir/n.lock" ]] ; then + if [[ "$DOWNLOAD" == "false" ]] ; then + activate "${g_mirror_folder_name}/${version}" + fi + exit + fi + fi + + log installing "${g_mirror_folder_name}-v$version" + + local url="$(tarball_url "$version")" + is_ok "${url}" || abort "download preflight failed for '$version' (${url})" + + log mkdir "$dir" + mkdir -p "$dir" || abort "sudo required (or change ownership, or define N_PREFIX)" + touch "$dir/n.lock" + + cd "${dir}" || abort "Failed to cd to ${dir}" + + log fetch "$url" + do_get "${url}" | tar "$tarflag" --strip-components=1 --no-same-owner -f - + pipe_results=( "${PIPESTATUS[@]}" ) + if [[ "${pipe_results[0]}" -ne 0 ]]; then + abort "failed to download archive for $version" + fi + if [[ "${pipe_results[1]}" -ne 0 ]]; then + abort "failed to extract archive for $version" + fi + [ "$GET_SHOWS_PROGRESS" = "true" ] && erase_line + rm -f "$dir/n.lock" + + disable_pax_mprotect bin/node + + if [[ "$DOWNLOAD" == "false" ]]; then + activate "${g_mirror_folder_name}/$version" + fi +} + +# +# Be more silent. +# + +set_quiet() { + SHOW_VERBOSE_LOG="false" + command -v curl > /dev/null && CURL_OPTIONS+=( "--silent" ) && GET_SHOWS_PROGRESS="false" +} + +# +# Synopsis: do_get [option...] url +# Call curl or wget with combination of global and passed options. +# + +function do_get() { + if command -v curl &> /dev/null; then + curl "${CURL_OPTIONS[@]}" "$@" + elif command -v wget &> /dev/null; then + wget "${WGET_OPTIONS[@]}" "$@" + else + abort "curl or wget command required" + fi +} + +# +# Synopsis: do_get_index [option...] url +# Call curl or wget with combination of global and passed options, +# with options tweaked to be more suitable for getting index. +# + +function do_get_index() { + if command -v curl &> /dev/null; then + # --silent to suppress progress et al + curl --silent --compressed "${CURL_OPTIONS[@]}" "$@" + elif command -v wget &> /dev/null; then + wget "${WGET_OPTIONS[@]}" "$@" + else + abort "curl or wget command required" + fi +} + +# +# Synopsis: remove_versions version ... +# + +function remove_versions() { + [[ -z "$1" ]] && abort "version(s) required" + while [[ $# -ne 0 ]]; do + local version + get_latest_resolved_version "$1" || break + version="${g_target_node}" + if [[ -n "${version}" ]]; then + update_mirror_settings_for_version "$1" + local dir="${CACHE_DIR}/${g_mirror_folder_name}/${version}" + if [[ -s "${dir}" ]]; then + rm -rf "${dir}" + else + echo "$1 (${version}) not in downloads cache" + fi + else + echo "No version found for '$1'" + fi + shift + done +} + +# +# Synopsis: prune_cache +# + +function prune_cache() { + set_active_node + + for folder_and_version in $(display_versions_paths); do + if [[ "${folder_and_version}" != "${g_active_node}" ]]; then + echo "${folder_and_version}" + rm -rf "${CACHE_DIR:?}/${folder_and_version}" + fi + done +} + +# +# Synopsis: find_cached_version version +# Finds cache directory for resolved version. +# Globals modified: +# - g_cached_version + +function find_cached_version() { + [[ -z "$1" ]] && abort "version required" + local version + get_latest_resolved_version "$1" || exit 1 + version="${g_target_node}" + [[ -n "${version}" ]] || abort "no version found for '$1'" + + update_mirror_settings_for_version "$1" + g_cached_version="${CACHE_DIR}/${g_mirror_folder_name}/${version}" + if [[ ! -d "${g_cached_version}" && "${DOWNLOAD}" == "true" ]]; then + (install "${version}") + fi + [[ -d "${g_cached_version}" ]] || abort "'$1' (${version}) not in downloads cache" +} + + +# +# Synopsis: display_bin_path_for_version version +# + +function display_bin_path_for_version() { + find_cached_version "$1" + echo "${g_cached_version}/bin/node" +} + +# +# Synopsis: run_with_version version [args...] +# Run the given of node with [args ..] +# + +function run_with_version() { + find_cached_version "$1" + shift # remove version from parameters + exec "${g_cached_version}/bin/node" "$@" +} + +# +# Synopsis: exec_with_version command [args...] +# Modify the path to include and execute command. +# + +function exec_with_version() { + find_cached_version "$1" + shift # remove version from parameters + PATH="${g_cached_version}/bin:$PATH" exec "$@" +} + +# +# Synopsis: is_ok url +# Check the HEAD response of . +# + +function is_ok() { + # Note: both curl and wget can follow redirects, as present on some mirrors (e.g. https://npm.taobao.org/mirrors/node). + # The output is complicated with redirects, so keep it simple and use command status rather than parse output. + if command -v curl &> /dev/null; then + do_get --silent --head "$1" > /dev/null || return 1 + else + do_get --spider "$1" > /dev/null || return 1 + fi +} + +# +# Synopsis: can_use_xz +# Test system to see if xz decompression is supported by tar. +# + +function can_use_xz() { + # Be conservative and only enable if xz is likely to work. Unfortunately we can't directly query tar itself. + # For research, see https://github.com/shadowspawn/nvh/issues/8 + local uname_s="$(uname -s)" + if [[ "${uname_s}" = "Linux" ]] && command -v xz &> /dev/null ; then + # tar on linux is likely to support xz if it is available as a command + return 0 + elif [[ "${uname_s}" = "Darwin" ]]; then + local macos_version="$(sw_vers -productVersion)" + local macos_major_version="$(echo "${macos_version}" | cut -d '.' -f 1)" + local macos_minor_version="$(echo "${macos_version}" | cut -d '.' -f 2)" + if [[ "${macos_major_version}" -gt 10 || "${macos_minor_version}" -gt 8 ]]; then + # tar on recent Darwin has xz support built-in + return 0 + fi + fi + return 2 # not supported +} + +# +# Synopsis: display_tarball_platform +# + +function display_tarball_platform() { + # https://en.wikipedia.org/wiki/Uname + + local os="unexpected_os" + local uname_a="$(uname -a)" + case "${uname_a}" in + Linux*) os="linux" ;; + Darwin*) os="darwin" ;; + SunOS*) os="sunos" ;; + AIX*) os="aix" ;; + CYGWIN*) >&2 echo_red "Cygwin is not supported by n" ;; + MINGW*) >&2 echo_red "Git BASH (MSYS) is not supported by n" ;; + esac + + local arch="unexpected_arch" + local uname_m="$(uname -m)" + case "${uname_m}" in + x86_64) arch=x64 ;; + i386 | i686) arch="x86" ;; + aarch64) arch=arm64 ;; + armv8l) arch=arm64 ;; # armv8l probably supports arm64, and there is no specific armv8l build so give it a go + *) + # e.g. armv6l, armv7l, arm64 + arch="${uname_m}" + ;; + esac + # Override from command line, or version specific adjustment. + [ -n "$ARCH" ] && arch="$ARCH" + + echo "${os}-${arch}" +} + +# +# Synopsis: display_compatible_file_field +# display for current platform, as per field in index.tab, which is different than actual download +# + +function display_compatible_file_field { + local compatible_file_field="$(display_tarball_platform)" + if [[ -z "${ARCH}" && "${compatible_file_field}" = "darwin-arm64" ]]; then + # Look for arm64 for native but also x64 for older versions which can run in rosetta. + # (Downside is will get an install error if install version above 16 with x64 and not arm64.) + compatible_file_field="osx-arm64-tar|osx-x64-tar" + elif [[ "${compatible_file_field}" =~ darwin-(.*) ]]; then + compatible_file_field="osx-${BASH_REMATCH[1]}-tar" + fi + echo "${compatible_file_field}" +} + +# +# Synopsis: tarball_url version +# + +function tarball_url() { + local version="$1" + local ext=gz + [ "$N_USE_XZ" = "true" ] && ext="xz" + echo "${g_mirror_url}/v${version}/node-v${version}-$(display_tarball_platform).tar.${ext}" +} + +# +# Synopsis: get_file_node_version filename +# Sets g_target_node +# + +function get_file_node_version() { + g_target_node= + local filepath="$1" + verbose_log "found" "${filepath}" + # read returns a non-zero status but does still work if there is no line ending + local version + <"${filepath}" read -r version + # trim possible trailing \d from a Windows created file + version="${version%%[[:space:]]}" + verbose_log "read" "${version}" + g_target_node="${version}" +} + +# +# Synopsis: get_package_engine_version\ +# Sets g_target_node +# + +function get_package_engine_version() { + g_target_node= + local filepath="$1" + verbose_log "found" "${filepath}" + command -v node &> /dev/null || abort "an active version of node is required to read 'engines' from package.json" + local range + range="$(node -e "package = require('${filepath}'); if (package && package.engines && package.engines.node) console.log(package.engines.node)")" + verbose_log "read" "${range}" + [[ -n "${range}" ]] || return 2 + if [[ "*" == "${range}" ]]; then + verbose_log "target" "current" + g_target_node="current" + return + fi + + local version + if [[ "${range}" =~ ^([>~^=]|\>\=)?v?([0-9]+(\.[0-9]+){0,2})(.[xX*])?$ ]]; then + local operator="${BASH_REMATCH[1]}" + version="${BASH_REMATCH[2]}" + case "${operator}" in + '' | =) ;; + \> | \>=) version="current" ;; + \~) [[ "${version}" =~ ^([0-9]+\.[0-9]+)\.[0-9]+$ ]] && version="${BASH_REMATCH[1]}" ;; + ^) [[ "${version}" =~ ^([0-9]+) ]] && version="${BASH_REMATCH[1]}" ;; + esac + verbose_log "target" "${version}" + else + command -v npx &> /dev/null || abort "an active version of npx is required to use complex 'engine' ranges from package.json" + verbose_log "resolving" "${range}" + local version_per_line="$(n lsr --all)" + local versions_one_line=$(echo "${version_per_line}" | tr '\n' ' ') + # Using semver@7 so works with older versions of node. + # shellcheck disable=SC2086 + version=$(npm_config_yes=true npx --quiet semver@7 -r "${range}" ${versions_one_line} | tail -n 1) + fi + g_target_node="${version}" +} + +# +# Synopsis: get_nvmrc_version +# Sets g_target_node +# + +function get_nvmrc_version() { + g_target_node= + local filepath="$1" + verbose_log "found" "${filepath}" + local version + <"${filepath}" read -r version + verbose_log "read" "${version}" + # Translate from nvm aliases + case "${version}" in + lts/\*) version="lts" ;; + lts/*) version="${version:4}" ;; + node) version="current" ;; + *) ;; + esac + g_target_node="${version}" +} + +# +# Synopsis: get_engine_version [error-message] +# Sets g_target_node +# + +function get_engine_version() { + g_target_node= + local error_message="${1-package.json not found}" + local parent + parent="${PWD}" + while [[ -n "${parent}" ]]; do + if [[ -e "${parent}/package.json" ]]; then + get_package_engine_version "${parent}/package.json" + else + parent=${parent%/*} + continue + fi + break + done + [[ -n "${parent}" ]] || abort "${error_message}" + [[ -n "${g_target_node}" ]] || abort "did not find supported version of node in 'engines' field of package.json" +} + +# +# Synopsis: get_auto_version +# Sets g_target_node +# + +function get_auto_version() { + g_target_node= + # Search for a version control file first + local parent + parent="${PWD}" + while [[ -n "${parent}" ]]; do + if [[ -e "${parent}/.n-node-version" ]]; then + get_file_node_version "${parent}/.n-node-version" + elif [[ -e "${parent}/.node-version" ]]; then + get_file_node_version "${parent}/.node-version" + elif [[ -e "${parent}/.nvmrc" ]]; then + get_nvmrc_version "${parent}/.nvmrc" + else + parent=${parent%/*} + continue + fi + break + done + # Fallback to package.json + [[ -n "${parent}" ]] || get_engine_version "no file found for auto version (.n-node-version, .node-version, .nvmrc, or package.json)" + [[ -n "${g_target_node}" ]] || abort "file found for auto did not contain target version of node" +} + +# +# Synopsis: get_latest_resolved_version version +# Sets g_target_node +# + +function get_latest_resolved_version() { + g_target_node= + local version=${1} + simple_version=${version#node/} # Only place supporting node/ [sic] + if is_exact_numeric_version "${simple_version}"; then + # Just numbers, already resolved, no need to lookup first. + simple_version="${simple_version#v}" + g_target_node="${simple_version}" + else + # Complicated recognising exact version, KISS and lookup. + g_target_node=$(N_MAX_REMOTE_MATCHES=1 display_remote_versions "$version") + fi +} + +# +# Synopsis: display_remote_index +# index.tab reference: https://github.com/nodejs/nodejs-dist-indexer +# Index fields are: version date files npm v8 uv zlib openssl modules lts security +# KISS and just return fields we currently care about: version files lts +# + +display_remote_index() { + local index_url="${g_mirror_url}/index.tab" + # tail to remove header line + do_get_index "${index_url}" | tail -n +2 | cut -f 1,3,10 + if [[ "${PIPESTATUS[0]}" -ne 0 ]]; then + # Reminder: abort will only exit subshell, but consistent error display + abort "failed to download version index (${index_url})" + fi +} + +# +# Synopsis: display_match_limit limit +# + +function display_match_limit(){ + if [[ "$1" -gt 1 && "$1" -lt 32000 ]]; then + echo "Listing remote... Displaying $1 matches (use --all to see all)." + fi +} + +# +# Synopsis: display_remote_versions version +# + +function display_remote_versions() { + local version="$1" + update_mirror_settings_for_version "${version}" + local match='.' + local match_count="${N_MAX_REMOTE_MATCHES}" + + # Transform some labels before processing further. + if is_node_support_version "${version}"; then + version="$(display_latest_node_support_alias "${version}")" + match_count=1 + elif [[ "${version}" = "auto" ]]; then + # suppress stdout logging so lsr layout same as usual for scripting + get_auto_version || return 2 + version="${g_target_node}" + elif [[ "${version}" = "engine" ]]; then + # suppress stdout logging so lsr layout same as usual for scripting + get_engine_version || return 2 + version="${g_target_node}" + fi + + if [[ -z "${version}" ]]; then + match='.' + elif [[ "${version}" = "lts" || "${version}" = "stable" ]]; then + match_count=1 + # Codename is last field, first one with a name is newest lts + match="${TAB_CHAR}[a-zA-Z]+\$" + elif [[ "${version}" = "latest" || "${version}" = "current" ]]; then + match_count=1 + match='.' + elif is_numeric_version "${version}"; then + version="v${version#v}" + # Avoid restriction message if exact version + is_exact_numeric_version "${version}" && match_count=1 + # Quote any dots in version so they are literal for expression + match="${version//\./\.}" + # Avoid 1.2 matching 1.23 + match="^${match}[^0-9]" + elif is_lts_codename "${version}"; then + # Capitalise (could alternatively make grep case insensitive) + codename="$(echo "${version:0:1}" | tr '[:lower:]' '[:upper:]')${version:1}" + # Codename is last field + match="${TAB_CHAR}${codename}\$" + elif is_download_folder "${version}"; then + match='.' + elif is_download_version "${version}"; then + version="${version#"${g_mirror_folder_name}"/}" + if [[ "${version}" = "latest" || "${version}" = "current" ]]; then + match_count=1 + match='.' + else + version="v${version#v}" + match="${version//\./\.}" + match="^${match}" # prefix + if is_numeric_version "${version}"; then + # Exact numeric match + match="${match}[^0-9]" + fi + fi + else + abort "invalid version '$1'" + fi + display_match_limit "${match_count}" + + # Implementation notes: + # - using awk rather than head so do not close pipe early on curl + # - restrict search to compatible files as not always available, or not at same time + # - return status of curl command (i.e. PIPESTATUS[0]) + display_remote_index \ + | n_grep -E "$(display_compatible_file_field)" \ + | n_grep -E "${match}" \ + | awk "NR<=${match_count}" \ + | cut -f 1 \ + | n_grep -E -o '[^v].*' + return "${PIPESTATUS[0]}" +} + +# +# Synopsis: delete_with_echo target +# + +function delete_with_echo() { + if [[ -e "$1" ]]; then + echo "$1" + rm -rf "$1" + fi +} + +# +# Synopsis: uninstall_installed +# Uninstall the installed node and npm (leaving alone the cache), +# so undo install, and may expose possible system installed versions. +# + +uninstall_installed() { + # npm: https://docs.npmjs.com/misc/removing-npm + # rm -rf /usr/local/{lib/node{,/.npm,_modules},bin,share/man}/npm* + # node: https://stackabuse.com/how-to-uninstall-node-js-from-mac-osx/ + # Doing it by hand rather than scanning cache, so still works if cache deleted first. + # This covers tarballs for at least node 4 through 10. + + while true; do + read -r -p "Do you wish to delete node and npm from ${N_PREFIX}? " yn + case $yn in + [Yy]* ) break ;; + [Nn]* ) exit ;; + * ) echo "Please answer yes or no.";; + esac + done + + echo "" + echo "Uninstalling node and npm" + delete_with_echo "${N_PREFIX}/bin/node" + delete_with_echo "${N_PREFIX}/bin/npm" + delete_with_echo "${N_PREFIX}/bin/npx" + delete_with_echo "${N_PREFIX}/bin/corepack" + delete_with_echo "${N_PREFIX}/include/node" + delete_with_echo "${N_PREFIX}/lib/dtrace/node.d" + delete_with_echo "${N_PREFIX}/lib/node_modules/npm" + delete_with_echo "${N_PREFIX}/lib/node_modules/corepack" + delete_with_echo "${N_PREFIX}/share/doc/node" + delete_with_echo "${N_PREFIX}/share/man/man1/node.1" + delete_with_echo "${N_PREFIX}/share/systemtap/tapset/node.stp" +} + +# +# Synopsis: show_permission_suggestions +# + +function show_permission_suggestions() { + echo "Suggestions:" + echo "- run n with sudo, or" + echo "- define N_PREFIX to a writeable location, or" +} + +# +# Synopsis: show_diagnostics +# Show environment and check for common problems. +# + +function show_diagnostics() { + echo "This information is to help you diagnose issues, and useful when reporting an issue." + echo "Note: some output may contain passwords. Redact before sharing." + + printf "\n\nCOMMAND LOCATIONS AND VERSIONS\n" + + printf "\nbash\n" + command -v bash && bash --version + + printf "\nn\n" + command -v n && n --version + + printf "\nnode\n" + if command -v node &> /dev/null; then + command -v node && node --version + node -e 'if (process.versions.v8) console.log("JavaScript engine: v8");' + + printf "\nnpm\n" + command -v npm && npm --version + fi + + printf "\ntar\n" + if command -v tar &> /dev/null; then + command -v tar && tar --version + else + echo_red "tar not found. Needed for extracting downloads." + fi + + printf "\ncurl or wget\n" + if command -v curl &> /dev/null; then + command -v curl && curl --version + elif command -v wget &> /dev/null; then + command -v wget && wget --version + else + echo_red "Neither curl nor wget found. Need one of them for downloads." + fi + + printf "\nuname\n" + uname -a + + printf "\n\nSETTINGS\n" + + printf "\nn\n" + echo "node mirror: ${N_NODE_MIRROR}" + echo "node downloads mirror: ${N_NODE_DOWNLOAD_MIRROR}" + echo "install destination: ${N_PREFIX}" + [[ -n "${N_PREFIX}" ]] && echo "PATH: ${PATH}" + echo "ls-remote max matches: ${N_MAX_REMOTE_MATCHES}" + [[ -n "${N_PRESERVE_NPM}" ]] && echo "installs preserve npm by default" + [[ -n "${N_PRESERVE_COREPACK}" ]] && echo "installs preserve corepack by default" + + printf "\nProxy\n" + # disable "var is referenced but not assigned": https://github.com/koalaman/shellcheck/wiki/SC2154 + # shellcheck disable=SC2154 + [[ -n "${http_proxy}" ]] && echo "http_proxy: ${http_proxy}" + # shellcheck disable=SC2154 + [[ -n "${https_proxy}" ]] && echo "https_proxy: ${https_proxy}" + if command -v curl &> /dev/null; then + # curl supports lower case and upper case! + # shellcheck disable=SC2154 + [[ -n "${all_proxy}" ]] && echo "all_proxy: ${all_proxy}" + [[ -n "${ALL_PROXY}" ]] && echo "ALL_PROXY: ${ALL_PROXY}" + [[ -n "${HTTP_PROXY}" ]] && echo "HTTP_PROXY: ${HTTP_PROXY}" + [[ -n "${HTTPS_PROXY}" ]] && echo "HTTPS_PROXY: ${HTTPS_PROXY}" + if [[ -e "${CURL_HOME}/.curlrc" ]]; then + echo "have \$CURL_HOME/.curlrc" + elif [[ -e "${HOME}/.curlrc" ]]; then + echo "have \$HOME/.curlrc" + fi + elif command -v wget &> /dev/null; then + if [[ -e "${WGETRC}" ]]; then + echo "have \$WGETRC" + elif [[ -e "${HOME}/.wgetrc" ]]; then + echo "have \$HOME/.wgetrc" + fi + fi + + printf "\n\nCHECKS\n" + + printf "\nChecking n install destination is in PATH...\n" + local install_bin="${N_PREFIX}/bin" + local path_wth_guards=":${PATH}:" + if [[ "${path_wth_guards}" =~ :${install_bin}/?: ]]; then + printf "good\n" + else + echo_red "'${install_bin}' is not in PATH" + fi + if command -v node &> /dev/null; then + printf "\nChecking n install destination priority in PATH...\n" + local node_dir="$(dirname "$(command -v node)")" + + local index=0 + local path_entry + local path_entries + local install_bin_index=0 + local node_index=999 + IFS=':' read -ra path_entries <<< "${PATH}" + for path_entry in "${path_entries[@]}"; do + (( index++ )) + [[ "${path_entry}" =~ ^${node_dir}/?$ ]] && node_index="${index}" + [[ "${path_entry}" =~ ^${install_bin}/?$ ]] && install_bin_index="${index}" + done + if [[ "${node_index}" -lt "${install_bin_index}" ]]; then + echo_red "There is a version of node installed which will be found in PATH before the n installed version." + else + printf "good\n" + fi + fi + + printf "\nChecking permissions for cache folder...\n" + # Most likely problem is ownership rather than than permissions as such. + local cache_root="${N_PREFIX}/n" + if [[ -e "${N_PREFIX}" && ! -w "${N_PREFIX}" && ! -e "${cache_root}" ]]; then + echo_red "You do not have write permission to create: ${cache_root}" + show_permission_suggestions + echo "- make a folder you own:" + echo " sudo mkdir -p \"${cache_root}\"" + echo " sudo chown $(whoami) \"${cache_root}\"" + elif [[ -e "${cache_root}" && ! -w "${cache_root}" ]]; then + echo_red "You do not have write permission to: ${cache_root}" + show_permission_suggestions + echo "- change folder ownership to yourself:" + echo " sudo chown -R $(whoami) \"${cache_root}\"" + elif [[ ! -e "${cache_root}" ]]; then + echo "Cache folder does not exist: ${cache_root}" + echo "This is normal if you have not done an install yet, as cache is only created when needed." + elif [[ -e "${CACHE_DIR}" && ! -w "${CACHE_DIR}" ]]; then + echo_red "You do not have write permission to: ${CACHE_DIR}" + show_permission_suggestions + echo "- change folder ownership to yourself:" + echo " sudo chown -R $(whoami) \"${CACHE_DIR}\"" + else + echo "good" + fi + + if [[ -e "${N_PREFIX}" ]]; then + # Most likely problem is ownership rather than than permissions as such. + printf "\nChecking permissions for install folders...\n" + local install_writeable="true" + for subdir in bin lib include share; do + if [[ -e "${N_PREFIX}/${subdir}" && ! -w "${N_PREFIX}/${subdir}" ]]; then + install_writeable="false" + echo_red "You do not have write permission to: ${N_PREFIX}/${subdir}" + break + fi + done + if [[ "${install_writeable}" = "true" ]]; then + echo "good" + else + show_permission_suggestions + echo "- change folder ownerships to yourself:" + echo " (cd \"${N_PREFIX}\" && sudo chown -R $(whoami) bin lib include share)" + fi + fi + + printf "\nChecking mirror is reachable...\n" + if is_ok "${N_NODE_MIRROR}/"; then + printf "good\n" + else + echo_red "mirror not reachable" + printf "Showing failing command and output\n" + if command -v curl &> /dev/null; then + ( set -x; do_get --head "${N_NODE_MIRROR}/" ) + else + ( set -x; do_get --spider "${N_NODE_MIRROR}/" ) + printf "\n" + fi + fi +} + +# +# Handle arguments. +# + +# First pass. Process the options so they can come before or after commands, +# particularly for `n lsr --all` and `n install --arch x686` +# which feel pretty natural. + +unprocessed_args=() +positional_arg="false" + +while [[ $# -ne 0 ]]; do + case "$1" in + --all) N_MAX_REMOTE_MATCHES=32000 ;; + -V|--version) display_n_version ;; + -h|--help|help) display_help; exit ;; + -q|--quiet) set_quiet ;; + -d|--download) DOWNLOAD="true" ;; + --insecure) set_insecure ;; + -p|--preserve) N_PRESERVE_NPM="true" N_PRESERVE_COREPACK="true" ;; + --no-preserve) N_PRESERVE_NPM="" N_PRESERVE_COREPACK="" ;; + --use-xz) N_USE_XZ="true" ;; + --no-use-xz) N_USE_XZ="false" ;; + --latest) display_remote_versions latest; exit ;; + --stable) display_remote_versions lts; exit ;; # [sic] old terminology + --lts) display_remote_versions lts; exit ;; + -a|--arch) shift; set_arch "$1";; # set arch and continue + exec|run|as|use) + unprocessed_args+=( "$1" ) + positional_arg="true" + ;; + *) + if [[ "${positional_arg}" == "true" ]]; then + unprocessed_args+=( "$@" ) + break + fi + unprocessed_args+=( "$1" ) + ;; + esac + shift +done + +if [[ -z "${N_USE_XZ+defined}" ]]; then + N_USE_XZ="true" # Default to using xz + can_use_xz || N_USE_XZ="false" +fi + +set -- "${unprocessed_args[@]}" + +if test $# -eq 0; then + test -z "$(display_versions_paths)" && err_no_installed_print_help + menu_select_cache_versions +else + while test $# -ne 0; do + case "$1" in + bin|which) display_bin_path_for_version "$2"; exit ;; + run|as|use) shift; run_with_version "$@"; exit ;; + exec) shift; exec_with_version "$@"; exit ;; + doctor) show_diagnostics; exit ;; + rm|-) shift; remove_versions "$@"; exit ;; + prune) prune_cache; exit ;; + latest) install latest; exit ;; + stable) install stable; exit ;; + lts) install lts; exit ;; + ls|list) display_versions_paths; exit ;; + lsr|ls-remote|list-remote) shift; display_remote_versions "$1"; exit ;; + uninstall) uninstall_installed; exit ;; + i|install) shift; install "$1"; exit ;; + N_TEST_DISPLAY_LATEST_RESOLVED_VERSION) shift; get_latest_resolved_version "$1" > /dev/null || exit 2; echo "${g_target_node}"; exit ;; + *) install "$1"; exit ;; + esac + shift + done +fi From eaf7a2904c0c609ee37467f45c2a630449461fb1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 11 Mar 2023 14:57:48 +0100 Subject: [PATCH 07/43] helpers: fix ynh_setup_source, 'source_id' may contain slashes x_x --- helpers/utils | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helpers/utils b/helpers/utils index 4a964a14e..167b67d37 100644 --- a/helpers/utils +++ b/helpers/utils @@ -235,7 +235,8 @@ ynh_setup_source() { # (Unused?) mecanism where one can have the file in a special local cache to not have to download it... local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${source_id}" - mkdir -p /var/cache/yunohost/download/${YNH_APP_ID}/ + # Gotta use this trick with 'dirname' because source_id may contain slashes x_x + mkdir -p $(dirname /var/cache/yunohost/download/${YNH_APP_ID}/${source_id}) src_filename="/var/cache/yunohost/download/${YNH_APP_ID}/${source_id}" if [ "$src_format" = "docker" ]; then From f9a7016931de4293d4a7bcce3ff5357040356349 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 11 Mar 2023 16:51:42 +0100 Subject: [PATCH 08/43] Update changelog for 11.1.15 --- debian/changelog | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/debian/changelog b/debian/changelog index a29ba223c..0373a10b8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +yunohost (11.1.15) stable; urgency=low + + - doc: Fix version number in autogenerated resource doc (5b58e0e6) + - helpers: Fix documentation for ynh_setup_source (7491dd4c) + - helpers: fix ynh_setup_source, 'source_id' may contain slashes x_x (eaf7a290) + - helpers/nodejs: simplify 'n' script install and maintenance ([#1627](https://github.com/yunohost/yunohost/pull/1627)) + + -- Alexandre Aubin Sat, 11 Mar 2023 16:50:50 +0100 + yunohost (11.1.14) stable; urgency=low - helpers: simplify --time display option for ynh_script_progression .. we don't care about displaying time when below 10 sc (8731f77a) From a95d10e50c5b60aac7623fa1acc430799686a79d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 13 Mar 2023 18:48:57 +0100 Subject: [PATCH 09/43] backup: fix boring issue where archive is a broken symlink... --- src/backup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backup.py b/src/backup.py index ee218607d..ce1e8ba2c 100644 --- a/src/backup.py +++ b/src/backup.py @@ -2376,6 +2376,7 @@ def backup_list(with_info=False, human_readable=False): # (we do a realpath() to resolve symlinks) archives = glob(f"{ARCHIVES_PATH}/*.tar.gz") + glob(f"{ARCHIVES_PATH}/*.tar") archives = {os.path.realpath(archive) for archive in archives} + archives = {archive for archive in archives if os.path.exists(archive)} archives = sorted(archives, key=lambda x: os.path.getctime(x)) # Extract only filename without the extension From 3656c199186d47d7f07f1bbd8651c77c95cd2fb6 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 14 Mar 2023 18:45:04 +0100 Subject: [PATCH 10/43] helpers/appsv2: don't remove yhh-deps virtual package if ... it doesn't exist. Otherwise when apt fails to install dependency, we end up with another error about failing to remove the ynh-deps package --- helpers/apt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/helpers/apt b/helpers/apt index c36f4aa27..a2f2d3de8 100644 --- a/helpers/apt +++ b/helpers/apt @@ -370,7 +370,13 @@ ynh_remove_app_dependencies() { apt-mark unhold ${dep_app}-ynh-deps fi - ynh_package_autopurge ${dep_app}-ynh-deps # Remove the fake package and its dependencies if they not still used. + # Remove the fake package and its dependencies if they not still used. + # (except if dpkg doesn't know anything about the package, + # which should be symptomatic of a failed install, and we don't want bash to report an error) + if dpkg-query --show ${dep_app}-ynh-deps &>/dev/null + then + ynh_package_autopurge ${dep_app}-ynh-deps + fi } # Install packages from an extra repository properly. From b2596f328751a108852d59acb9677292405c0612 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 14 Mar 2023 19:23:24 +0100 Subject: [PATCH 11/43] appsv2: add validation for expected types for permissions stuff --- src/utils/resources.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/utils/resources.py b/src/utils/resources.py index 87446bdd8..b9bb1fee7 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -497,11 +497,21 @@ class PermissionsResource(AppResource): properties["main"] = self.default_perm_properties for perm, infos in properties.items(): + if "auth_header" in infos and not isinstance(infos.get("auth_header"), bool): + raise YunohostError(f"In manifest, for permission '{perm}', 'auth_header' should be a boolean", raw_msg=True) + if "show_tile" in infos and not isinstance(infos.get("show_tile"), bool): + raise YunohostError(f"In manifest, for permission '{perm}', 'show_tile' should be a boolean", raw_msg=True) + if "protected" in infos and not isinstance(infos.get("protected"), bool): + raise YunohostError(f"In manifest, for permission '{perm}', 'protected' should be a boolean", raw_msg=True) + if "additional_urls" in infos and not isinstance(infos.get("additional_urls"), list): + raise YunohostError(f"In manifest, for permission '{perm}', 'additional_urls' should be a list", raw_msg=True) + properties[perm] = copy.copy(self.default_perm_properties) properties[perm].update(infos) if properties[perm]["show_tile"] is None: properties[perm]["show_tile"] = bool(properties[perm]["url"]) + if properties["main"]["url"] is not None and ( not isinstance(properties["main"].get("url"), str) or properties["main"]["url"] != "/" From 1b2fa91ff02d241f2101fdc30d7e22e78ceacc2d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 20 Mar 2023 15:49:23 +0100 Subject: [PATCH 12/43] ynh_setup_source: fix buggy checksum mismatch handling, can't compute the sha256sum after we delete the file @_@ --- helpers/utils | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/helpers/utils b/helpers/utils index 167b67d37..97bd8e6b5 100644 --- a/helpers/utils +++ b/helpers/utils @@ -267,8 +267,10 @@ ynh_setup_source() { # Check the control sum if ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status then - rm ${src_filename} - ynh_die --message="Corrupt source for ${src_filename}: Expected ${src_sum} but got $(${src_sumprg} ${src_filename} | cut --delimiter=' ' --fields=1) (size: $(du -hs ${src_filename} | cut --delimiter=' ' --fields=1))." + local actual_sum="$(${src_sumprg} ${src_filename} | cut --delimiter=' ' --fields=1)" + local actual_size="$(du -hs ${src_filename} | cut --delimiter=' ' --fields=1)" + rm -f ${src_filename} + ynh_die --message="Corrupt source for ${src_filename}: Expected ${src_sum} but got ${actual_sum} (size: ${actual_size})." fi fi From c211b75279077754a3a5392b22538e3d2a3c8100 Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 21 Mar 2023 18:31:24 +0100 Subject: [PATCH 13/43] options:tests: add base class Test --- src/tests/test_questions.py | 476 +++++++++++++++++++++++++++++++++++- 1 file changed, 475 insertions(+), 1 deletion(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index cf7c3c6e6..e849b6892 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -1,15 +1,22 @@ +import inspect import sys import pytest import os +from contextlib import contextmanager from mock import patch from io import StringIO +from typing import Any, Literal, Sequence, TypedDict, Union + +from _pytest.mark.structures import ParameterSet + from moulinette import Moulinette - from yunohost import domain, user from yunohost.utils.config import ( + ARGUMENTS_TYPE_PARSERS, ask_questions_and_parse_answers, + DisplayTextQuestion, PasswordQuestion, DomainQuestion, PathQuestion, @@ -44,6 +51,473 @@ User answers: """ +# ╭───────────────────────────────────────────────────────╮ +# │ ┌─╮╭─┐╶┬╴╭─╴╷ ╷╶┬╴╭╮╷╭─╮ │ +# │ ├─╯├─┤ │ │ ├─┤ │ ││││╶╮ │ +# │ ╵ ╵ ╵ ╵ ╰─╴╵ ╵╶┴╴╵╰╯╰─╯ │ +# ╰───────────────────────────────────────────────────────╯ + + +@contextmanager +def patch_isatty(isatty): + with patch.object(os, "isatty", return_value=isatty): + yield + + +@contextmanager +def patch_interface(interface: Literal["api", "cli"] = "api"): + with patch.object(Moulinette.interface, "type", interface), patch_isatty( + interface == "cli" + ): + yield + + +@contextmanager +def patch_prompt(return_value): + with patch_interface("cli"), patch.object( + Moulinette, "prompt", return_value=return_value + ) as prompt: + yield prompt + + +@pytest.fixture +def patch_no_tty(): + with patch_isatty(False): + yield + + +@pytest.fixture +def patch_with_tty(): + with patch_isatty(True): + yield + + +# ╭───────────────────────────────────────────────────────╮ +# │ ╭─╴╭─╴┌─╴╭╮╷╭─┐┌─╮╶┬╴╭─╮╭─╴ │ +# │ ╰─╮│ ├─╴│││├─┤├┬╯ │ │ │╰─╮ │ +# │ ╶─╯╰─╴╰─╴╵╰╯╵ ╵╵ ╰╶┴╴╰─╯╶─╯ │ +# ╰───────────────────────────────────────────────────────╯ + + +MinScenario = tuple[Any, Union[Literal["FAIL"], Any]] +PartialScenario = tuple[Any, Union[Literal["FAIL"], Any], dict[str, Any]] +FullScenario = tuple[Any, Union[Literal["FAIL"], Any], dict[str, Any], dict[str, Any]] + +Scenario = Union[ + MinScenario, + PartialScenario, + FullScenario, + "InnerScenario", +] + + +class InnerScenario(TypedDict, total=False): + scenarios: Sequence[Scenario] + raw_options: Sequence[dict[str, Any]] + data: Sequence[dict[str, Any]] + + +# ╭───────────────────────────────────────────────────────╮ +# │ Scenario generators/helpers │ +# ╰───────────────────────────────────────────────────────╯ + + +def get_hydrated_scenarios(raw_options, scenarios, data=[{}]): + """ + Normalize and hydrate a mixed list of scenarios to proper tuple/pytest.param flattened list values. + + Example:: + scenarios = [ + { + "raw_options": [{}, {"optional": True}], + "scenarios": [ + ("", "value", {"default": "value"}), + *unchanged("value", "other"), + ] + }, + *all_fails(-1, 0, 1, raw_options={"optional": True}), + *xfail(scenarios=[(True, "True"), (False, "False)], reason="..."), + ] + # Is exactly the same as + scenarios = [ + ("", "value", {"default": "value"}), + ("", "value", {"optional": True, "default": "value"}), + ("value", "value", {}), + ("value", "value", {"optional": True}), + ("other", "other", {}), + ("other", "other", {"optional": True}), + (-1, FAIL, {"optional": True}), + (0, FAIL, {"optional": True}), + (1, FAIL, {"optional": True}), + pytest.param(True, "True", {}, marks=pytest.mark.xfail(reason="...")), + pytest.param(False, "False", {}, marks=pytest.mark.xfail(reason="...")), + ] + """ + hydrated_scenarios = [] + for raw_option in raw_options: + for mocked_data in data: + for scenario in scenarios: + if isinstance(scenario, dict): + merged_raw_options = [ + {**raw_option, **raw_opt} + for raw_opt in scenario.get("raw_options", [{}]) + ] + hydrated_scenarios += get_hydrated_scenarios( + merged_raw_options, + scenario["scenarios"], + scenario.get("data", [mocked_data]), + ) + elif isinstance(scenario, ParameterSet): + intake, output, custom_raw_option = ( + scenario.values + if len(scenario.values) == 3 + else (*scenario.values, {}) + ) + merged_raw_option = {**raw_option, **custom_raw_option} + hydrated_scenarios.append( + pytest.param( + intake, + output, + merged_raw_option, + mocked_data, + marks=scenario.marks, + ) + ) + elif isinstance(scenario, tuple): + intake, output, custom_raw_option = ( + scenario if len(scenario) == 3 else (*scenario, {}) + ) + merged_raw_option = {**raw_option, **custom_raw_option} + hydrated_scenarios.append( + (intake, output, merged_raw_option, mocked_data) + ) + else: + raise Exception( + "Test scenario should be tuple(intake, output, raw_option), pytest.param(intake, output, raw_option) or dict(raw_options, scenarios, data)" + ) + + return hydrated_scenarios + + +def generate_test_name(intake, output, raw_option, data): + values_as_str = [] + for value in (intake, output): + if isinstance(value, str) and value != FAIL: + values_as_str.append(f"'{value}'") + elif inspect.isclass(value) and issubclass(value, Exception): + values_as_str.append(value.__name__) + else: + values_as_str.append(value) + name = f"{values_as_str[0]} -> {values_as_str[1]}" + + keys = [ + "=".join( + [ + key, + str(raw_option[key]) + if not isinstance(raw_option[key], str) + else f"'{raw_option[key]}'", + ] + ) + for key in raw_option.keys() + if key not in ("id", "type") + ] + if keys: + name += " (" + ",".join(keys) + ")" + return name + + +def pytest_generate_tests(metafunc): + """ + Pytest test factory that, for each `BaseTest` subclasses, parametrize its + methods if it requires it by checking the method's parameters. + For those and based on their `cls.scenarios`, a series of `pytest.param` are + automaticly injected as test values. + """ + if metafunc.cls and issubclass(metafunc.cls, BaseTest): + argnames = [] + argvalues = [] + ids = [] + fn_params = inspect.signature(metafunc.function).parameters + + for params in [ + ["intake", "expected_output", "raw_option", "data"], + ["intake", "expected_normalized", "raw_option", "data"], + ["intake", "expected_humanized", "raw_option", "data"], + ]: + if all(param in fn_params for param in params): + argnames += params + if params[1] == "expected_output": + # Hydrate scenarios with generic raw_option data + argvalues += get_hydrated_scenarios( + [metafunc.cls.raw_option], metafunc.cls.scenarios + ) + ids += [ + generate_test_name(*args.values) + if isinstance(args, ParameterSet) + else generate_test_name(*args) + for args in argvalues + ] + elif params[1] == "expected_normalized": + argvalues += metafunc.cls.normalized + ids += [ + f"{metafunc.cls.raw_option['type']}-normalize-{scenario[0]}" + for scenario in metafunc.cls.normalized + ] + elif params[1] == "expected_humanized": + argvalues += metafunc.cls.humanized + ids += [ + f"{metafunc.cls.raw_option['type']}-normalize-{scenario[0]}" + for scenario in metafunc.cls.humanized + ] + + metafunc.parametrize(argnames, argvalues, ids=ids) + + +# ╭───────────────────────────────────────────────────────╮ +# │ Scenario helpers │ +# ╰───────────────────────────────────────────────────────╯ + +FAIL = YunohostValidationError + + +def nones( + *nones, output, raw_option: dict[str, Any] = {}, fail_if_required: bool = True +) -> list[PartialScenario]: + """ + Returns common scenarios for ~None values. + - required and required + as default -> `FAIL` + - optional and optional + as default -> `expected_output=None` + """ + return [ + (none, FAIL if fail_if_required else output, base_raw_option | raw_option) # type: ignore + for none in nones + for base_raw_option in ({}, {"default": none}) + ] + [ + (none, output, base_raw_option | raw_option) + for none in nones + for base_raw_option in ({"optional": True}, {"optional": True, "default": none}) + ] + + +def unchanged(*args, raw_option: dict[str, Any] = {}) -> list[PartialScenario]: + """ + Returns a series of params for which output is expected to be the same as its intake + + Example:: + # expect `"value"` to output as `"value"`, etc. + unchanged("value", "yes", "none") + + """ + return [(arg, arg, raw_option.copy()) for arg in args] + + +def all_as(*args, output, raw_option: dict[str, Any] = {}) -> list[PartialScenario]: + """ + Returns a series of params for which output is expected to be the same single value + + Example:: + # expect all values to output as `True` + all_as("y", "yes", 1, True, output=True) + """ + return [(arg, output, raw_option.copy()) for arg in args] + + +def all_fails( + *args, raw_option: dict[str, Any] = {}, error=FAIL +) -> list[PartialScenario]: + """ + Returns a series of params for which output is expected to be failing with validation error + """ + return [(arg, error, raw_option.copy()) for arg in args] + + +def xpass(*, scenarios: list[Scenario], reason="unknown") -> list[Scenario]: + """ + Return a pytest param for which test should have fail but currently passes. + """ + return [ + pytest.param( + *scenario, + marks=pytest.mark.xfail( + reason=f"Currently valid but probably shouldn't. details: {reason}." + ), + ) + for scenario in scenarios + ] + + +def xfail(*, scenarios: list[Scenario], reason="unknown") -> list[Scenario]: + """ + Return a pytest param for which test should have passed but currently fails. + """ + return [ + pytest.param( + *scenario, + marks=pytest.mark.xfail( + reason=f"Currently invalid but should probably pass. details: {reason}." + ), + ) + for scenario in scenarios + ] + + +# ╭───────────────────────────────────────────────────────╮ +# │ ╶┬╴┌─╴╭─╴╶┬╴╭─╴ │ +# │ │ ├─╴╰─╮ │ ╰─╮ │ +# │ ╵ ╰─╴╶─╯ ╵ ╶─╯ │ +# ╰───────────────────────────────────────────────────────╯ + + +def _fill_or_prompt_one_option(raw_option, intake): + raw_option = raw_option.copy() + id_ = raw_option.pop("id") + options = {id_: raw_option} + answers = {id_: intake} if intake is not None else {} + + option = ask_questions_and_parse_answers(options, answers)[0] + + return (option, option.value) + + +def _test_value_is_expected_output(value, expected_output): + """ + Properly compares bools and None + """ + if isinstance(expected_output, bool) or expected_output is None: + assert value is expected_output + else: + assert value == expected_output + + +def _test_intake(raw_option, intake, expected_output): + option, value = _fill_or_prompt_one_option(raw_option, intake) + + _test_value_is_expected_output(value, expected_output) + + +def _test_intake_may_fail(raw_option, intake, expected_output): + if inspect.isclass(expected_output) and issubclass(expected_output, Exception): + with pytest.raises(expected_output): + _fill_or_prompt_one_option(raw_option, intake) + else: + _test_intake(raw_option, intake, expected_output) + + +class BaseTest: + raw_option: dict[str, Any] = {} + prefill: dict[Literal["raw_option", "prefill", "intake"], Any] + scenarios: list[Scenario] + + # fmt: off + # scenarios = [ + # *all_fails(False, True, 0, 1, -1, 1337, 13.37, [], ["one"], {}, raw_option={"optional": True}), + # *all_fails("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), + # *nones(None, "", output=""), + # ] + # fmt: on + # TODO + # - pattern (also on Date for example to see if it override the default pattern) + # - example + # - visible + # - redact + # - regex + # - hooks + + @classmethod + def get_raw_option(cls, raw_option={}, **kwargs): + base_raw_option = cls.raw_option.copy() + base_raw_option.update(**raw_option) + base_raw_option.update(**kwargs) + return base_raw_option + + @classmethod + def _test_basic_attrs(self): + raw_option = self.get_raw_option(optional=True) + id_ = raw_option["id"] + option, value = _fill_or_prompt_one_option(raw_option, None) + + is_special_readonly_option = isinstance(option, DisplayTextQuestion) + + assert isinstance(option, ARGUMENTS_TYPE_PARSERS[raw_option["type"]]) + assert option.type == raw_option["type"] + assert option.name == id_ + assert option.ask == {"en": id_} + assert option.readonly is (True if is_special_readonly_option else False) + assert option.visible is None + # assert option.bind is None + + if is_special_readonly_option: + assert value is None + + return (raw_option, option, value) + + @pytest.mark.usefixtures("patch_no_tty") + def test_basic_attrs(self): + """ + Test basic options factories and BaseOption default attributes values. + """ + # Intermediate method since pytest doesn't like tests that returns something. + # This allow a test class to call `_test_basic_attrs` then do additional checks + self._test_basic_attrs() + + def test_options_prompted_with_ask_help(self, prefill_data=None): + """ + Test that assert that moulinette prompt is called with: + - `message` with translated string and possible choices list + - help` with translated string + - `prefill` is the expected string value from a custom default + - `is_password` is true for `password`s only + - `is_multiline` is true for `text`s only + - `autocomplete` is option choices + + Ran only once with `cls.prefill` data + """ + if prefill_data is None: + prefill_data = self.prefill + + base_raw_option = prefill_data["raw_option"] + prefill = prefill_data["prefill"] + + with patch_prompt("") as prompt: + raw_option = self.get_raw_option( + raw_option=base_raw_option, + ask={"en": "Can i haz question?"}, + help={"en": "Here's help!"}, + ) + option, value = _fill_or_prompt_one_option(raw_option, None) + + expected_message = option.ask["en"] + + if option.choices: + choices = ( + option.choices + if isinstance(option.choices, list) + else option.choices.keys() + ) + expected_message += f" [{' | '.join(choices)}]" + if option.type == "boolean": + expected_message += " [yes | no]" + + prompt.assert_called_with( + message=expected_message, + is_password=option.type == "password", + confirm=False, # FIXME no confirm? + prefill=prefill, + is_multiline=option.type == "text", + autocomplete=option.choices or [], + help=option.help["en"], + ) + + def test_scenarios(self, intake, expected_output, raw_option, data): + with patch_interface("api"): + _test_intake_may_fail( + raw_option, + intake, + expected_output, + ) + + def test_question_empty(): ask_questions_and_parse_answers({}, {}) == [] From 26ca9e5c69f7f188e9a9ce2c48572616a1ed64bd Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 21 Mar 2023 18:37:49 +0100 Subject: [PATCH 14/43] options:tests: replace some string tests --- src/tests/test_questions.py | 281 ++++++++++-------------------------- 1 file changed, 78 insertions(+), 203 deletions(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index e849b6892..f8f8f9fef 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -518,40 +518,88 @@ class BaseTest: ) +# ╭───────────────────────────────────────────────────────╮ +# │ STRING │ +# ╰───────────────────────────────────────────────────────╯ + + +class TestString(BaseTest): + raw_option = {"type": "string", "id": "string_id"} + prefill = { + "raw_option": {"default": " custom default"}, + "prefill": " custom default", + } + # fmt: off + scenarios = [ + *nones(None, "", output=""), + # basic typed values + *unchanged(False, True, 0, 1, -1, 1337, 13.37, [], ["one"], {}, raw_option={"optional": True}), # FIXME should output as str? + *unchanged("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", raw_option={"optional": True}), + *xpass(scenarios=[ + ([], []), + ], reason="Should fail"), + # test strip + ("value", "value"), + ("value\n", "value"), + (" \n value\n", "value"), + (" \\n value\\n", "\\n value\\n"), + (" \tvalue\t", "value"), + (r" ##value \n \tvalue\n ", r"##value \n \tvalue\n"), + *xpass(scenarios=[ + ("value\nvalue", "value\nvalue"), + (" ##value \n \tvalue\n ", "##value \n \tvalue"), + ], reason=r"should fail or without `\n`?"), + # readonly + *xfail(scenarios=[ + ("overwrite", "expected value", {"readonly": True, "default": "expected value"}), + ], reason="Should not be overwritten"), + ] + # fmt: on + + +# ╭───────────────────────────────────────────────────────╮ +# │ TEXT │ +# ╰───────────────────────────────────────────────────────╯ + + +class TestText(BaseTest): + raw_option = {"type": "text", "id": "text_id"} + prefill = { + "raw_option": {"default": "some value\nanother line "}, + "prefill": "some value\nanother line ", + } + # fmt: off + scenarios = [ + *nones(None, "", output=""), + # basic typed values + *unchanged(False, True, 0, 1, -1, 1337, 13.37, [], ["one"], {}, raw_option={"optional": True}), # FIXME should fail or output as str? + *unchanged("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", raw_option={"optional": True}), + *xpass(scenarios=[ + ([], []) + ], reason="Should fail"), + ("value", "value"), + ("value\n value", "value\n value"), + # test no strip + *xpass(scenarios=[ + ("value\n", "value"), + (" \n value\n", "value"), + (" \\n value\\n", "\\n value\\n"), + (" \tvalue\t", "value"), + (" ##value \n \tvalue\n ", "##value \n \tvalue"), + (r" ##value \n \tvalue\n ", r"##value \n \tvalue\n"), + ], reason="Should not be stripped"), + # readonly + *xfail(scenarios=[ + ("overwrite", "expected value", {"readonly": True, "default": "expected value"}), + ], reason="Should not be overwritten"), + ] + # fmt: on + + def test_question_empty(): ask_questions_and_parse_answers({}, {}) == [] -def test_question_string(): - questions = { - "some_string": { - "type": "string", - } - } - answers = {"some_string": "some_value"} - - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_string" - assert out.type == "string" - assert out.value == "some_value" - - -def test_question_string_from_query_string(): - questions = { - "some_string": { - "type": "string", - } - } - answers = "foo=bar&some_string=some_value&lorem=ipsum" - - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_string" - assert out.type == "string" - assert out.value == "some_value" - - def test_question_string_default_type(): questions = {"some_string": {}} answers = {"some_string": "some_value"} @@ -563,179 +611,6 @@ def test_question_string_default_type(): assert out.value == "some_value" -def test_question_string_no_input(): - questions = {"some_string": {}} - answers = {} - - with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - ask_questions_and_parse_answers(questions, answers) - - -def test_question_string_input(): - questions = { - "some_string": { - "ask": "some question", - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_string" - assert out.type == "string" - assert out.value == "some_value" - - -def test_question_string_input_no_ask(): - questions = {"some_string": {}} - answers = {} - - with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_string" - assert out.type == "string" - assert out.value == "some_value" - - -def test_question_string_no_input_optional(): - questions = {"some_string": {"optional": True}} - answers = {} - with patch.object(os, "isatty", return_value=False): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_string" - assert out.type == "string" - assert out.value == "" - - -def test_question_string_optional_with_input(): - questions = { - "some_string": { - "ask": "some question", - "optional": True, - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_string" - assert out.type == "string" - assert out.value == "some_value" - - -def test_question_string_optional_with_empty_input(): - questions = { - "some_string": { - "ask": "some question", - "optional": True, - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value=""), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_string" - assert out.type == "string" - assert out.value == "" - - -def test_question_string_optional_with_input_without_ask(): - questions = { - "some_string": { - "optional": True, - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_string" - assert out.type == "string" - assert out.value == "some_value" - - -def test_question_string_no_input_default(): - questions = { - "some_string": { - "ask": "some question", - "default": "some_value", - } - } - answers = {} - with patch.object(os, "isatty", return_value=False): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_string" - assert out.type == "string" - assert out.value == "some_value" - - -def test_question_string_input_test_ask(): - ask_text = "some question" - questions = { - "some_string": { - "ask": ask_text, - } - } - answers = {} - - with patch.object( - Moulinette, "prompt", return_value="some_value" - ) as prompt, patch.object(os, "isatty", return_value=True): - ask_questions_and_parse_answers(questions, answers) - prompt.assert_called_with( - message=ask_text, - is_password=False, - confirm=False, - prefill="", - is_multiline=False, - autocomplete=[], - help=None, - ) - - -def test_question_string_input_test_ask_with_default(): - ask_text = "some question" - default_text = "some example" - questions = { - "some_string": { - "ask": ask_text, - "default": default_text, - } - } - answers = {} - - with patch.object( - Moulinette, "prompt", return_value="some_value" - ) as prompt, patch.object(os, "isatty", return_value=True): - ask_questions_and_parse_answers(questions, answers) - prompt.assert_called_with( - message=ask_text, - is_password=False, - confirm=False, - prefill=default_text, - is_multiline=False, - autocomplete=[], - help=None, - ) - - @pytest.mark.skip # we should do something with this example def test_question_string_input_test_ask_with_example(): ask_text = "some question" From 38381b8149e374cea81063d33c39a5605316a874 Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 21 Mar 2023 18:47:04 +0100 Subject: [PATCH 15/43] options:tests: replace some password tests --- src/tests/test_questions.py | 286 ++++-------------------------------- 1 file changed, 32 insertions(+), 254 deletions(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index f8f8f9fef..a8e55a93d 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -596,6 +596,38 @@ class TestText(BaseTest): # fmt: on +# ╭───────────────────────────────────────────────────────╮ +# │ PASSWORD │ +# ╰───────────────────────────────────────────────────────╯ + + +class TestPassword(BaseTest): + raw_option = {"type": "password", "id": "password_id"} + prefill = { + "raw_option": {"default": None, "optional": True}, + "prefill": "", + } + # fmt: off + scenarios = [ + *all_fails(False, True, 0, 1, -1, 1337, 13.37, raw_option={"optional": True}, error=TypeError), # FIXME those fails with TypeError + *all_fails([], ["one"], {}, raw_option={"optional": True}, error=AttributeError), # FIXME those fails with AttributeError + *all_fails("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), + *nones(None, "", output=""), + *xpass(scenarios=[ + (" value \n moarc0mpl1cat3d\n ", "value \n moarc0mpl1cat3d"), + (" some_ value", "some_ value"), + ], reason="Should output exactly the same"), + ("s3cr3t!!", "s3cr3t!!"), + ("secret", FAIL), + *[("supersecret" + char, FAIL) for char in PasswordQuestion.forbidden_chars], # FIXME maybe add ` \n` to the list? + # readonly + *xpass(scenarios=[ + ("s3cr3t!!", "s3cr3t!!", {"readonly": True}), + ], reason="Should fail since readonly is forbidden"), + ] + # fmt: on + + def test_question_empty(): ask_questions_and_parse_answers({}, {}) == [] @@ -720,210 +752,6 @@ def test_question_string_with_choice_default(): assert out.value == "en" -def test_question_password(): - questions = { - "some_password": { - "type": "password", - } - } - answers = {"some_password": "some_value"} - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_password" - assert out.type == "password" - assert out.value == "some_value" - - -def test_question_password_no_input(): - questions = { - "some_password": { - "type": "password", - } - } - answers = {} - - with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - ask_questions_and_parse_answers(questions, answers) - - -def test_question_password_input(): - questions = { - "some_password": { - "type": "password", - "ask": "some question", - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_password" - assert out.type == "password" - assert out.value == "some_value" - - -def test_question_password_input_no_ask(): - questions = { - "some_password": { - "type": "password", - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_password" - assert out.type == "password" - assert out.value == "some_value" - - -def test_question_password_no_input_optional(): - questions = { - "some_password": { - "type": "password", - "optional": True, - } - } - answers = {} - - with patch.object(os, "isatty", return_value=False): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_password" - assert out.type == "password" - assert out.value == "" - - questions = {"some_password": {"type": "password", "optional": True, "default": ""}} - - with patch.object(os, "isatty", return_value=False): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_password" - assert out.type == "password" - assert out.value == "" - - -def test_question_password_optional_with_input(): - questions = { - "some_password": { - "ask": "some question", - "type": "password", - "optional": True, - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_password" - assert out.type == "password" - assert out.value == "some_value" - - -def test_question_password_optional_with_empty_input(): - questions = { - "some_password": { - "ask": "some question", - "type": "password", - "optional": True, - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value=""), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_password" - assert out.type == "password" - assert out.value == "" - - -def test_question_password_optional_with_input_without_ask(): - questions = { - "some_password": { - "type": "password", - "optional": True, - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value="some_value"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_password" - assert out.type == "password" - assert out.value == "some_value" - - -def test_question_password_no_input_default(): - questions = { - "some_password": { - "type": "password", - "ask": "some question", - "default": "some_value", - } - } - answers = {} - - # no default for password! - with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - ask_questions_and_parse_answers(questions, answers) - - -@pytest.mark.skip # this should raises -def test_question_password_no_input_example(): - questions = { - "some_password": { - "type": "password", - "ask": "some question", - "example": "some_value", - } - } - answers = {"some_password": "some_value"} - - # no example for password! - with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - ask_questions_and_parse_answers(questions, answers) - - -def test_question_password_input_test_ask(): - ask_text = "some question" - questions = { - "some_password": { - "type": "password", - "ask": ask_text, - } - } - answers = {} - - with patch.object( - Moulinette, "prompt", return_value="some_value" - ) as prompt, patch.object(os, "isatty", return_value=True): - ask_questions_and_parse_answers(questions, answers) - prompt.assert_called_with( - message=ask_text, - is_password=True, - confirm=False, - prefill="", - is_multiline=False, - autocomplete=[], - help=None, - ) - - @pytest.mark.skip # we should do something with this example def test_question_password_input_test_ask_with_example(): ask_text = "some question" @@ -966,56 +794,6 @@ def test_question_password_input_test_ask_with_help(): assert help_text in prompt.call_args[1]["message"] -def test_question_password_bad_chars(): - questions = { - "some_password": { - "type": "password", - "ask": "some question", - "example": "some_value", - } - } - - for i in PasswordQuestion.forbidden_chars: - with pytest.raises(YunohostError), patch.object( - os, "isatty", return_value=False - ): - ask_questions_and_parse_answers(questions, {"some_password": i * 8}) - - -def test_question_password_strong_enough(): - questions = { - "some_password": { - "type": "password", - "ask": "some question", - "example": "some_value", - } - } - - with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - # too short - ask_questions_and_parse_answers(questions, {"some_password": "a"}) - - with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - ask_questions_and_parse_answers(questions, {"some_password": "password"}) - - -def test_question_password_optional_strong_enough(): - questions = { - "some_password": { - "ask": "some question", - "type": "password", - "optional": True, - } - } - - with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - # too short - ask_questions_and_parse_answers(questions, {"some_password": "a"}) - - with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - ask_questions_and_parse_answers(questions, {"some_password": "password"}) - - def test_question_path(): questions = { "some_path": { From 70149fe41d2e4cf21cff3a9e86ccfd380d3bb3dc Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 21 Mar 2023 18:50:23 +0100 Subject: [PATCH 16/43] options:tests: replace path tests --- src/tests/test_questions.py | 259 ++++++++---------------------------- 1 file changed, 52 insertions(+), 207 deletions(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index a8e55a93d..910b8b5a0 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -628,6 +628,58 @@ class TestPassword(BaseTest): # fmt: on +# ╭───────────────────────────────────────────────────────╮ +# │ PATH │ +# ╰───────────────────────────────────────────────────────╯ + + +class TestWebPath(BaseTest): + raw_option = {"type": "path", "id": "path_id"} + prefill = { + "raw_option": {"default": "some_path"}, + "prefill": "some_path", + } + # fmt: off + scenarios = [ + *all_fails(False, True, 0, 1, -1, 1337, 13.37, [], ["one"], {}, raw_option={"optional": True}), + + *nones(None, "", output=""), + # custom valid + ("/", "/"), + ("/one/two", "/one/two"), + *[ + (v, "/" + v) + for v in ("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value") + ], + ("value\n", "/value"), + ("//value", "/value"), + ("///value///", "/value"), + *xpass(scenarios=[ + ("value\nvalue", "/value\nvalue"), + ("value value", "/value value"), + ("value//value", "/value//value"), + ], reason="Should fail"), + *xpass(scenarios=[ + ("./here", "/./here"), + ("../here", "/../here"), + ("/somewhere/../here", "/somewhere/../here"), + ], reason="Should fail or flattened"), + + *xpass(scenarios=[ + ("/one?withquery=ah", "/one?withquery=ah"), + ], reason="Should fail or query string removed"), + *xpass(scenarios=[ + ("https://example.com/folder", "/https://example.com/folder") + ], reason="Should fail or scheme+domain removed"), + # readonly + *xfail(scenarios=[ + ("/overwrite", "/value", {"readonly": True, "default": "/value"}), + ], reason="Should not be overwritten"), + # FIXME should path have forbidden_chars? + ] + # fmt: on + + def test_question_empty(): ask_questions_and_parse_answers({}, {}) == [] @@ -794,213 +846,6 @@ def test_question_password_input_test_ask_with_help(): assert help_text in prompt.call_args[1]["message"] -def test_question_path(): - questions = { - "some_path": { - "type": "path", - } - } - answers = {"some_path": "/some_value"} - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_path" - assert out.type == "path" - assert out.value == "/some_value" - - -def test_question_path_no_input(): - questions = { - "some_path": { - "type": "path", - } - } - answers = {} - - with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - ask_questions_and_parse_answers(questions, answers) - - -def test_question_path_input(): - questions = { - "some_path": { - "type": "path", - "ask": "some question", - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value="/some_value"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_path" - assert out.type == "path" - assert out.value == "/some_value" - - -def test_question_path_input_no_ask(): - questions = { - "some_path": { - "type": "path", - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value="/some_value"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_path" - assert out.type == "path" - assert out.value == "/some_value" - - -def test_question_path_no_input_optional(): - questions = { - "some_path": { - "type": "path", - "optional": True, - } - } - answers = {} - with patch.object(os, "isatty", return_value=False): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_path" - assert out.type == "path" - assert out.value == "" - - -def test_question_path_optional_with_input(): - questions = { - "some_path": { - "ask": "some question", - "type": "path", - "optional": True, - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value="/some_value"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_path" - assert out.type == "path" - assert out.value == "/some_value" - - -def test_question_path_optional_with_empty_input(): - questions = { - "some_path": { - "ask": "some question", - "type": "path", - "optional": True, - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value=""), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_path" - assert out.type == "path" - assert out.value == "" - - -def test_question_path_optional_with_input_without_ask(): - questions = { - "some_path": { - "type": "path", - "optional": True, - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value="/some_value"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_path" - assert out.type == "path" - assert out.value == "/some_value" - - -def test_question_path_no_input_default(): - questions = { - "some_path": { - "ask": "some question", - "type": "path", - "default": "some_value", - } - } - answers = {} - with patch.object(os, "isatty", return_value=False): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_path" - assert out.type == "path" - assert out.value == "/some_value" - - -def test_question_path_input_test_ask(): - ask_text = "some question" - questions = { - "some_path": { - "type": "path", - "ask": ask_text, - } - } - answers = {} - - with patch.object( - Moulinette, "prompt", return_value="some_value" - ) as prompt, patch.object(os, "isatty", return_value=True): - ask_questions_and_parse_answers(questions, answers) - prompt.assert_called_with( - message=ask_text, - is_password=False, - confirm=False, - prefill="", - is_multiline=False, - autocomplete=[], - help=None, - ) - - -def test_question_path_input_test_ask_with_default(): - ask_text = "some question" - default_text = "someexample" - questions = { - "some_path": { - "type": "path", - "ask": ask_text, - "default": default_text, - } - } - answers = {} - - with patch.object( - Moulinette, "prompt", return_value="some_value" - ) as prompt, patch.object(os, "isatty", return_value=True): - ask_questions_and_parse_answers(questions, answers) - prompt.assert_called_with( - message=ask_text, - is_password=False, - confirm=False, - prefill=default_text, - is_multiline=False, - autocomplete=[], - help=None, - ) - - @pytest.mark.skip # we should do something with this example def test_question_path_input_test_ask_with_example(): ask_text = "some question" From df6bb228202067332a524e88567de9ed89a00835 Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 21 Mar 2023 18:53:55 +0100 Subject: [PATCH 17/43] options:tests: replace boolean tests --- src/tests/test_questions.py | 318 ++++++++---------------------------- 1 file changed, 66 insertions(+), 252 deletions(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index 910b8b5a0..f8cc5ce98 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -628,6 +628,72 @@ class TestPassword(BaseTest): # fmt: on +# ╭───────────────────────────────────────────────────────╮ +# │ BOOLEAN │ +# ╰───────────────────────────────────────────────────────╯ + + +class TestBoolean(BaseTest): + raw_option = {"type": "boolean", "id": "boolean_id"} + prefill = { + "raw_option": {"default": True}, + "prefill": "yes", + } + # fmt: off + truthy_values = (True, 1, "1", "True", "true", "Yes", "yes", "y", "on") + falsy_values = (False, 0, "0", "False", "false", "No", "no", "n", "off") + scenarios = [ + *all_as(None, "", output=0), + *all_fails("none", "None"), # FIXME should output as `0` (default) like other none values when required? + *all_as(None, "", output=0, raw_option={"optional": True}), # FIXME should output as `None`? + *all_as("none", "None", output=None, raw_option={"optional": True}), + # FIXME even if default is explicity `None|""`, it ends up with class_default `0` + *all_as(None, "", output=0, raw_option={"default": None}), # FIXME this should fail, default is `None` + *all_as(None, "", output=0, raw_option={"optional": True, "default": None}), # FIXME even if default is explicity None, it ends up with class_default + *all_as(None, "", output=0, raw_option={"default": ""}), # FIXME this should fail, default is `""` + *all_as(None, "", output=0, raw_option={"optional": True, "default": ""}), # FIXME even if default is explicity None, it ends up with class_default + # With "none" behavior is ok + *all_fails(None, "", raw_option={"default": "none"}), + *all_as(None, "", output=None, raw_option={"optional": True, "default": "none"}), + # Unhandled types should fail + *all_fails(1337, "1337", "string", [], "[]", ",", "one,two"), + *all_fails(1337, "1337", "string", [], "[]", ",", "one,two", {"optional": True}), + # Required + *all_as(*truthy_values, output=1), + *all_as(*falsy_values, output=0), + # Optional + *all_as(*truthy_values, output=1, raw_option={"optional": True}), + *all_as(*falsy_values, output=0, raw_option={"optional": True}), + # test values as default, as required option without intake + *[(None, 1, {"default": true for true in truthy_values})], + *[(None, 0, {"default": false for false in falsy_values})], + # custom boolean output + ("", "disallow", {"yes": "allow", "no": "disallow"}), # required -> default to False -> `"disallow"` + ("n", "disallow", {"yes": "allow", "no": "disallow"}), + ("y", "allow", {"yes": "allow", "no": "disallow"}), + ("", False, {"yes": True, "no": False}), # required -> default to False -> `False` + ("n", False, {"yes": True, "no": False}), + ("y", True, {"yes": True, "no": False}), + ("", -1, {"yes": 1, "no": -1}), # required -> default to False -> `-1` + ("n", -1, {"yes": 1, "no": -1}), + ("y", 1, {"yes": 1, "no": -1}), + { + "raw_options": [ + {"yes": "no", "no": "yes", "optional": True}, + {"yes": False, "no": True, "optional": True}, + {"yes": "0", "no": "1", "optional": True}, + ], + # "no" for "yes" and "yes" for "no" should fail + "scenarios": all_fails("", "y", "n", error=AssertionError), + }, + # readonly + *xfail(scenarios=[ + (1, 0, {"readonly": True, "default": 0}), + ], reason="Should not be overwritten"), + ] + # fmt: on + + # ╭───────────────────────────────────────────────────────╮ # │ PATH │ # ╰───────────────────────────────────────────────────────╯ @@ -888,258 +954,6 @@ def test_question_path_input_test_ask_with_help(): assert help_text in prompt.call_args[1]["message"] -def test_question_boolean(): - questions = { - "some_boolean": { - "type": "boolean", - } - } - answers = {"some_boolean": "y"} - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_boolean" - assert out.type == "boolean" - assert out.value == 1 - - -def test_question_boolean_all_yes(): - questions = { - "some_boolean": { - "type": "boolean", - } - } - - for value in ["Y", "yes", "Yes", "YES", "1", 1, True, "True", "TRUE", "true"]: - out = ask_questions_and_parse_answers(questions, {"some_boolean": value})[0] - assert out.name == "some_boolean" - assert out.type == "boolean" - assert out.value == 1 - - -def test_question_boolean_all_no(): - questions = { - "some_boolean": { - "type": "boolean", - } - } - - for value in ["n", "N", "no", "No", "No", "0", 0, False, "False", "FALSE", "false"]: - out = ask_questions_and_parse_answers(questions, {"some_boolean": value})[0] - assert out.name == "some_boolean" - assert out.type == "boolean" - assert out.value == 0 - - -# XXX apparently boolean are always False (0) by default, I'm not sure what to think about that -def test_question_boolean_no_input(): - questions = { - "some_boolean": { - "type": "boolean", - } - } - answers = {} - - with patch.object(os, "isatty", return_value=False): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.value == 0 - - -def test_question_boolean_bad_input(): - questions = { - "some_boolean": { - "type": "boolean", - } - } - answers = {"some_boolean": "stuff"} - - with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - ask_questions_and_parse_answers(questions, answers) - - -def test_question_boolean_input(): - questions = { - "some_boolean": { - "type": "boolean", - "ask": "some question", - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value="y"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - assert out.value == 1 - - with patch.object(Moulinette, "prompt", return_value="n"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - assert out.value == 0 - - -def test_question_boolean_input_no_ask(): - questions = { - "some_boolean": { - "type": "boolean", - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value="y"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - assert out.value == 1 - - -def test_question_boolean_no_input_optional(): - questions = { - "some_boolean": { - "type": "boolean", - "optional": True, - } - } - answers = {} - with patch.object(os, "isatty", return_value=False): - out = ask_questions_and_parse_answers(questions, answers)[0] - assert out.value == 0 - - -def test_question_boolean_optional_with_input(): - questions = { - "some_boolean": { - "ask": "some question", - "type": "boolean", - "optional": True, - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value="y"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - assert out.value == 1 - - -def test_question_boolean_optional_with_empty_input(): - questions = { - "some_boolean": { - "ask": "some question", - "type": "boolean", - "optional": True, - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value=""), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.value == 0 - - -def test_question_boolean_optional_with_input_without_ask(): - questions = { - "some_boolean": { - "type": "boolean", - "optional": True, - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value="n"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.value == 0 - - -def test_question_boolean_no_input_default(): - questions = { - "some_boolean": { - "ask": "some question", - "type": "boolean", - "default": 0, - } - } - answers = {} - - with patch.object(os, "isatty", return_value=False): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.value == 0 - - -def test_question_boolean_bad_default(): - questions = { - "some_boolean": { - "ask": "some question", - "type": "boolean", - "default": "bad default", - } - } - answers = {} - with pytest.raises(YunohostError): - ask_questions_and_parse_answers(questions, answers) - - -def test_question_boolean_input_test_ask(): - ask_text = "some question" - questions = { - "some_boolean": { - "type": "boolean", - "ask": ask_text, - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value=0) as prompt, patch.object( - os, "isatty", return_value=True - ): - ask_questions_and_parse_answers(questions, answers) - prompt.assert_called_with( - message=ask_text + " [yes | no]", - is_password=False, - confirm=False, - prefill="no", - is_multiline=False, - autocomplete=[], - help=None, - ) - - -def test_question_boolean_input_test_ask_with_default(): - ask_text = "some question" - default_text = 1 - questions = { - "some_boolean": { - "type": "boolean", - "ask": ask_text, - "default": default_text, - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value=1) as prompt, patch.object( - os, "isatty", return_value=True - ): - ask_questions_and_parse_answers(questions, answers) - prompt.assert_called_with( - message=ask_text + " [yes | no]", - is_password=False, - confirm=False, - prefill="yes", - is_multiline=False, - autocomplete=[], - help=None, - ) - - def test_question_domain_empty(): questions = { "some_domain": { From db1710a0a928affec7ff5be5e1b80330d194171a Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 21 Mar 2023 18:56:02 +0100 Subject: [PATCH 18/43] options:tests: replace domain tests --- src/tests/test_questions.py | 243 ++++++++++-------------------------- 1 file changed, 67 insertions(+), 176 deletions(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index f8cc5ce98..a42b501f7 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -746,6 +746,73 @@ class TestWebPath(BaseTest): # fmt: on +# ╭───────────────────────────────────────────────────────╮ +# │ DOMAIN │ +# ╰───────────────────────────────────────────────────────╯ + +main_domain = "ynh.local" +domains1 = ["ynh.local"] +domains2 = ["another.org", "ynh.local", "yet.another.org"] + + +@contextmanager +def patch_domains(*, domains, main_domain): + """ + Data mocking for DomainOption: + - yunohost.domain.domain_list + """ + with patch.object( + domain, + "domain_list", + return_value={"domains": domains, "main": main_domain}, + ), patch.object(domain, "_get_maindomain", return_value=main_domain): + yield + + +class TestDomain(BaseTest): + raw_option = {"type": "domain", "id": "domain_id"} + prefill = { + "raw_option": { + "default": None, + }, + "prefill": main_domain, + } + # fmt: off + scenarios = [ + # Probably not needed to test common types since those are not available as choices + # Also no scenarios with no domains since it should not be possible + { + "data": [{"main_domain": domains1[0], "domains": domains1}], + "scenarios": [ + *nones(None, "", output=domains1[0], fail_if_required=False), + (domains1[0], domains1[0], {}), + ("doesnt_exist.pouet", FAIL, {}), + ("fake.com", FAIL, {"choices": ["fake.com"]}), + # readonly + *xpass(scenarios=[ + (domains1[0], domains1[0], {"readonly": True}), + ], reason="Should fail since readonly is forbidden"), + ] + }, + { + "data": [{"main_domain": domains2[1], "domains": domains2}], + "scenarios": [ + *nones(None, "", output=domains2[1], fail_if_required=False), + (domains2[1], domains2[1], {}), + (domains2[0], domains2[0], {}), + ("doesnt_exist.pouet", FAIL, {}), + ("fake.com", FAIL, {"choices": ["fake.com"]}), + ] + }, + + ] + # fmt: on + + def test_scenarios(self, intake, expected_output, raw_option, data): + with patch_domains(**data): + super().test_scenarios(intake, expected_output, raw_option, data) + + def test_question_empty(): ask_questions_and_parse_answers({}, {}) == [] @@ -954,182 +1021,6 @@ def test_question_path_input_test_ask_with_help(): assert help_text in prompt.call_args[1]["message"] -def test_question_domain_empty(): - questions = { - "some_domain": { - "type": "domain", - } - } - main_domain = "my_main_domain.com" - answers = {} - - with patch.object( - domain, "_get_maindomain", return_value="my_main_domain.com" - ), patch.object( - domain, "domain_list", return_value={"domains": [main_domain]} - ), patch.object( - os, "isatty", return_value=False - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_domain" - assert out.type == "domain" - assert out.value == main_domain - - -def test_question_domain(): - main_domain = "my_main_domain.com" - domains = [main_domain] - questions = { - "some_domain": { - "type": "domain", - } - } - - answers = {"some_domain": main_domain} - - with patch.object( - domain, "_get_maindomain", return_value=main_domain - ), patch.object(domain, "domain_list", return_value={"domains": domains}): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_domain" - assert out.type == "domain" - assert out.value == main_domain - - -def test_question_domain_two_domains(): - main_domain = "my_main_domain.com" - other_domain = "some_other_domain.tld" - domains = [main_domain, other_domain] - - questions = { - "some_domain": { - "type": "domain", - } - } - answers = {"some_domain": other_domain} - - with patch.object( - domain, "_get_maindomain", return_value=main_domain - ), patch.object(domain, "domain_list", return_value={"domains": domains}): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_domain" - assert out.type == "domain" - assert out.value == other_domain - - answers = {"some_domain": main_domain} - - with patch.object( - domain, "_get_maindomain", return_value=main_domain - ), patch.object(domain, "domain_list", return_value={"domains": domains}): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_domain" - assert out.type == "domain" - assert out.value == main_domain - - -def test_question_domain_two_domains_wrong_answer(): - main_domain = "my_main_domain.com" - other_domain = "some_other_domain.tld" - domains = [main_domain, other_domain] - - questions = { - "some_domain": { - "type": "domain", - } - } - answers = {"some_domain": "doesnt_exist.pouet"} - - with patch.object( - domain, "_get_maindomain", return_value=main_domain - ), patch.object(domain, "domain_list", return_value={"domains": domains}): - with pytest.raises(YunohostError), patch.object( - os, "isatty", return_value=False - ): - ask_questions_and_parse_answers(questions, answers) - - -def test_question_domain_two_domains_default_no_ask(): - main_domain = "my_main_domain.com" - other_domain = "some_other_domain.tld" - domains = [main_domain, other_domain] - - questions = { - "some_domain": { - "type": "domain", - } - } - answers = {} - - with patch.object( - domain, "_get_maindomain", return_value=main_domain - ), patch.object( - domain, "domain_list", return_value={"domains": domains} - ), patch.object( - os, "isatty", return_value=False - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_domain" - assert out.type == "domain" - assert out.value == main_domain - - -def test_question_domain_two_domains_default(): - main_domain = "my_main_domain.com" - other_domain = "some_other_domain.tld" - domains = [main_domain, other_domain] - - questions = {"some_domain": {"type": "domain", "ask": "choose a domain"}} - answers = {} - - with patch.object( - domain, "_get_maindomain", return_value=main_domain - ), patch.object( - domain, "domain_list", return_value={"domains": domains} - ), patch.object( - os, "isatty", return_value=False - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_domain" - assert out.type == "domain" - assert out.value == main_domain - - -def test_question_domain_two_domains_default_input(): - main_domain = "my_main_domain.com" - other_domain = "some_other_domain.tld" - domains = [main_domain, other_domain] - - questions = {"some_domain": {"type": "domain", "ask": "choose a domain"}} - answers = {} - - with patch.object( - domain, "_get_maindomain", return_value=main_domain - ), patch.object( - domain, "domain_list", return_value={"domains": domains} - ), patch.object( - os, "isatty", return_value=True - ): - with patch.object(Moulinette, "prompt", return_value=main_domain): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_domain" - assert out.type == "domain" - assert out.value == main_domain - - with patch.object(Moulinette, "prompt", return_value=other_domain): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_domain" - assert out.type == "domain" - assert out.value == other_domain - - def test_question_user_empty(): users = { "some_user": { From af77e0b62fca9df863dcdaf5d6ac4337d8ad9c48 Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 21 Mar 2023 18:58:25 +0100 Subject: [PATCH 19/43] options:tests: replace user tests --- src/tests/test_questions.py | 314 ++++++++++++------------------------ 1 file changed, 106 insertions(+), 208 deletions(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index a42b501f7..a74dbe2be 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -813,6 +813,112 @@ class TestDomain(BaseTest): super().test_scenarios(intake, expected_output, raw_option, data) +# ╭───────────────────────────────────────────────────────╮ +# │ USER │ +# ╰───────────────────────────────────────────────────────╯ + +admin_username = "admin_user" +admin_user = { + "ssh_allowed": False, + "username": admin_username, + "mailbox-quota": "0", + "mail": "a@ynh.local", + "mail-aliases": [f"root@{main_domain}"], # Faking "admin" + "fullname": "john doe", + "group": [], +} +regular_username = "normal_user" +regular_user = { + "ssh_allowed": False, + "username": regular_username, + "mailbox-quota": "0", + "mail": "z@ynh.local", + "fullname": "john doe", + "group": [], +} + + +@contextmanager +def patch_users( + *, + users, + admin_username, + main_domain, +): + """ + Data mocking for UserOption: + - yunohost.user.user_list + - yunohost.user.user_info + - yunohost.domain._get_maindomain + """ + admin_info = next( + (user for user in users.values() if user["username"] == admin_username), + {"mail-aliases": []}, + ) + with patch.object(user, "user_list", return_value={"users": users}), patch.object( + user, + "user_info", + return_value=admin_info, # Faking admin user + ), patch.object(domain, "_get_maindomain", return_value=main_domain): + yield + + +class TestUser(BaseTest): + raw_option = {"type": "user", "id": "user_id"} + # fmt: off + scenarios = [ + # No tests for empty users since it should not happens + { + "data": [ + {"users": {admin_username: admin_user}, "admin_username": admin_username, "main_domain": main_domain}, + {"users": {admin_username: admin_user, regular_username: regular_user}, "admin_username": admin_username, "main_domain": main_domain}, + ], + "scenarios": [ + # FIXME User option is not really nullable, even if optional + *nones(None, "", output=admin_username, fail_if_required=False), + ("fake_user", FAIL), + ("fake_user", FAIL, {"choices": ["fake_user"]}), + ] + }, + { + "data": [ + {"users": {admin_username: admin_user, regular_username: regular_user}, "admin_username": admin_username, "main_domain": main_domain}, + ], + "scenarios": [ + *xpass(scenarios=[ + ("", regular_username, {"default": regular_username}) + ], reason="Should throw 'no default allowed'"), + # readonly + *xpass(scenarios=[ + (admin_username, admin_username, {"readonly": True}), + ], reason="Should fail since readonly is forbidden"), + ] + }, + ] + # fmt: on + + def test_options_prompted_with_ask_help(self, prefill_data=None): + with patch_users( + users={admin_username: admin_user, regular_username: regular_user}, + admin_username=admin_username, + main_domain=main_domain, + ): + super().test_options_prompted_with_ask_help( + prefill_data={"raw_option": {}, "prefill": admin_username} + ) + # FIXME This should fail, not allowed to set a default + super().test_options_prompted_with_ask_help( + prefill_data={ + "raw_option": {"default": regular_username}, + "prefill": regular_username, + } + ) + + def test_scenarios(self, intake, expected_output, raw_option, data): + with patch_users(**data): + super().test_scenarios(intake, expected_output, raw_option, data) + + def test_question_empty(): ask_questions_and_parse_answers({}, {}) == [] @@ -1021,214 +1127,6 @@ def test_question_path_input_test_ask_with_help(): assert help_text in prompt.call_args[1]["message"] -def test_question_user_empty(): - users = { - "some_user": { - "ssh_allowed": False, - "username": "some_user", - "mailbox-quota": "0", - "mail": "p@ynh.local", - "fullname": "the first name the last name", - } - } - - questions = { - "some_user": { - "type": "user", - } - } - answers = {} - - with patch.object(user, "user_list", return_value={"users": users}): - with pytest.raises(YunohostError), patch.object( - os, "isatty", return_value=False - ): - ask_questions_and_parse_answers(questions, answers) - - -def test_question_user(): - username = "some_user" - users = { - username: { - "ssh_allowed": False, - "username": "some_user", - "mailbox-quota": "0", - "mail": "p@ynh.local", - "fullname": "the first name the last name", - } - } - - questions = { - "some_user": { - "type": "user", - } - } - answers = {"some_user": username} - - with patch.object(user, "user_list", return_value={"users": users}), patch.object( - user, "user_info", return_value={} - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_user" - assert out.type == "user" - assert out.value == username - - -def test_question_user_two_users(): - username = "some_user" - other_user = "some_other_user" - users = { - username: { - "ssh_allowed": False, - "username": "some_user", - "mailbox-quota": "0", - "mail": "p@ynh.local", - "fullname": "the first name the last name", - }, - other_user: { - "ssh_allowed": False, - "username": "some_user", - "mailbox-quota": "0", - "mail": "z@ynh.local", - "fullname": "john doe", - }, - } - - questions = { - "some_user": { - "type": "user", - } - } - answers = {"some_user": other_user} - - with patch.object(user, "user_list", return_value={"users": users}), patch.object( - user, "user_info", return_value={} - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_user" - assert out.type == "user" - assert out.value == other_user - - answers = {"some_user": username} - - with patch.object(user, "user_list", return_value={"users": users}), patch.object( - user, "user_info", return_value={} - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_user" - assert out.type == "user" - assert out.value == username - - -def test_question_user_two_users_wrong_answer(): - username = "my_username.com" - other_user = "some_other_user" - users = { - username: { - "ssh_allowed": False, - "username": "some_user", - "mailbox-quota": "0", - "mail": "p@ynh.local", - "fullname": "the first name the last name", - }, - other_user: { - "ssh_allowed": False, - "username": "some_user", - "mailbox-quota": "0", - "mail": "z@ynh.local", - "fullname": "john doe", - }, - } - - questions = { - "some_user": { - "type": "user", - } - } - answers = {"some_user": "doesnt_exist.pouet"} - - with patch.object(user, "user_list", return_value={"users": users}): - with pytest.raises(YunohostError), patch.object( - os, "isatty", return_value=False - ): - ask_questions_and_parse_answers(questions, answers) - - -def test_question_user_two_users_no_default(): - username = "my_username.com" - other_user = "some_other_user.tld" - users = { - username: { - "ssh_allowed": False, - "username": "some_user", - "mailbox-quota": "0", - "mail": "p@ynh.local", - "fullname": "the first name the last name", - }, - other_user: { - "ssh_allowed": False, - "username": "some_user", - "mailbox-quota": "0", - "mail": "z@ynh.local", - "fullname": "john doe", - }, - } - - questions = {"some_user": {"type": "user", "ask": "choose a user"}} - answers = {} - - with patch.object(user, "user_list", return_value={"users": users}): - with pytest.raises(YunohostError), patch.object( - os, "isatty", return_value=False - ): - ask_questions_and_parse_answers(questions, answers) - - -def test_question_user_two_users_default_input(): - username = "my_username.com" - other_user = "some_other_user.tld" - users = { - username: { - "ssh_allowed": False, - "username": "some_user", - "mailbox-quota": "0", - "mail": "p@ynh.local", - "fullname": "the first name the last name", - }, - other_user: { - "ssh_allowed": False, - "username": "some_user", - "mailbox-quota": "0", - "mail": "z@ynh.local", - "fullname": "john doe", - }, - } - - questions = {"some_user": {"type": "user", "ask": "choose a user"}} - answers = {} - - with patch.object(user, "user_list", return_value={"users": users}), patch.object( - os, "isatty", return_value=True - ): - with patch.object(user, "user_info", return_value={}): - with patch.object(Moulinette, "prompt", return_value=username): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_user" - assert out.type == "user" - assert out.value == username - - with patch.object(Moulinette, "prompt", return_value=other_user): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_user" - assert out.type == "user" - assert out.value == other_user - - def test_question_number(): questions = { "some_number": { From af0cd78fcce86690c3bcf249568265f2ba2fa29f Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 21 Mar 2023 19:01:45 +0100 Subject: [PATCH 20/43] options:tests: replace number tests --- src/tests/test_questions.py | 279 ++++++------------------------------ 1 file changed, 45 insertions(+), 234 deletions(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index a74dbe2be..ac782fc9e 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -628,6 +628,51 @@ class TestPassword(BaseTest): # fmt: on +# ╭───────────────────────────────────────────────────────╮ +# │ NUMBER | RANGE │ +# ╰───────────────────────────────────────────────────────╯ +# Testing only number since "range" is only for webadmin (slider instead of classic intake). + + +class TestNumber(BaseTest): + raw_option = {"type": "number", "id": "number_id"} + prefill = { + "raw_option": {"default": 10}, + "prefill": "10", + } + # fmt: off + scenarios = [ + *all_fails([], ["one"], {}), + *all_fails("none", "_none", "False", "True", "[]", ",", "['one']", "one,two", r"{}", "value"), + + *nones(None, "", output=None), + *unchanged(0, 1, -1, 1337), + *xpass(scenarios=[(False, False)], reason="should fail or output as `0`"), + *xpass(scenarios=[(True, True)], reason="should fail or output as `1`"), + *all_as("0", 0, output=0), + *all_as("1", 1, output=1), + *all_as("1337", 1337, output=1337), + *xfail(scenarios=[ + ("-1", -1) + ], reason="should output as `-1` instead of failing"), + *all_fails(13.37, "13.37"), + + *unchanged(10, 5000, 10000, raw_option={"min": 10, "max": 10000}), + *all_fails(9, 10001, raw_option={"min": 10, "max": 10000}), + + *all_as(None, "", output=0, raw_option={"default": 0}), + *all_as(None, "", output=0, raw_option={"default": 0, "optional": True}), + (-10, -10, {"default": 10}), + (-10, -10, {"default": 10, "optional": True}), + # readonly + *xfail(scenarios=[ + (1337, 10000, {"readonly": True, "default": 10000}), + ], reason="Should not be overwritten"), + ] + # fmt: on + # FIXME should `step` be some kind of "multiple of"? + + # ╭───────────────────────────────────────────────────────╮ # │ BOOLEAN │ # ╰───────────────────────────────────────────────────────╯ @@ -1127,240 +1172,6 @@ def test_question_path_input_test_ask_with_help(): assert help_text in prompt.call_args[1]["message"] -def test_question_number(): - questions = { - "some_number": { - "type": "number", - } - } - answers = {"some_number": 1337} - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_number" - assert out.type == "number" - assert out.value == 1337 - - -def test_question_number_no_input(): - questions = { - "some_number": { - "type": "number", - } - } - answers = {} - - with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - ask_questions_and_parse_answers(questions, answers) - - -def test_question_number_bad_input(): - questions = { - "some_number": { - "type": "number", - } - } - answers = {"some_number": "stuff"} - - with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - ask_questions_and_parse_answers(questions, answers) - - answers = {"some_number": 1.5} - with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - ask_questions_and_parse_answers(questions, answers) - - -def test_question_number_input(): - questions = { - "some_number": { - "type": "number", - "ask": "some question", - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value="1337"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_number" - assert out.type == "number" - assert out.value == 1337 - - with patch.object(Moulinette, "prompt", return_value=1337), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_number" - assert out.type == "number" - assert out.value == 1337 - - with patch.object(Moulinette, "prompt", return_value="0"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_number" - assert out.type == "number" - assert out.value == 0 - - -def test_question_number_input_no_ask(): - questions = { - "some_number": { - "type": "number", - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value="1337"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_number" - assert out.type == "number" - assert out.value == 1337 - - -def test_question_number_no_input_optional(): - questions = { - "some_number": { - "type": "number", - "optional": True, - } - } - answers = {} - with patch.object(os, "isatty", return_value=False): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_number" - assert out.type == "number" - assert out.value is None - - -def test_question_number_optional_with_input(): - questions = { - "some_number": { - "ask": "some question", - "type": "number", - "optional": True, - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value="1337"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_number" - assert out.type == "number" - assert out.value == 1337 - - -def test_question_number_optional_with_input_without_ask(): - questions = { - "some_number": { - "type": "number", - "optional": True, - } - } - answers = {} - - with patch.object(Moulinette, "prompt", return_value="0"), patch.object( - os, "isatty", return_value=True - ): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_number" - assert out.type == "number" - assert out.value == 0 - - -def test_question_number_no_input_default(): - questions = { - "some_number": { - "ask": "some question", - "type": "number", - "default": 1337, - } - } - answers = {} - with patch.object(os, "isatty", return_value=False): - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_number" - assert out.type == "number" - assert out.value == 1337 - - -def test_question_number_bad_default(): - questions = { - "some_number": { - "ask": "some question", - "type": "number", - "default": "bad default", - } - } - answers = {} - with pytest.raises(YunohostError), patch.object(os, "isatty", return_value=False): - ask_questions_and_parse_answers(questions, answers) - - -def test_question_number_input_test_ask(): - ask_text = "some question" - questions = { - "some_number": { - "type": "number", - "ask": ask_text, - } - } - answers = {} - - with patch.object( - Moulinette, "prompt", return_value="1111" - ) as prompt, patch.object(os, "isatty", return_value=True): - ask_questions_and_parse_answers(questions, answers) - prompt.assert_called_with( - message=ask_text, - is_password=False, - confirm=False, - prefill="", - is_multiline=False, - autocomplete=[], - help=None, - ) - - -def test_question_number_input_test_ask_with_default(): - ask_text = "some question" - default_value = 1337 - questions = { - "some_number": { - "type": "number", - "ask": ask_text, - "default": default_value, - } - } - answers = {} - - with patch.object( - Moulinette, "prompt", return_value="1111" - ) as prompt, patch.object(os, "isatty", return_value=True): - ask_questions_and_parse_answers(questions, answers) - prompt.assert_called_with( - message=ask_text, - is_password=False, - confirm=False, - prefill=str(default_value), - is_multiline=False, - autocomplete=[], - help=None, - ) - - @pytest.mark.skip # we should do something with this example def test_question_number_input_test_ask_with_example(): ask_text = "some question" From eacb7016e2fd9d70975dab33a7ee74d5ccd80e8f Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 21 Mar 2023 19:03:09 +0100 Subject: [PATCH 21/43] options:tests: replace display_text tests --- src/tests/test_questions.py | 50 +++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index ac782fc9e..dffa93d14 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -518,6 +518,45 @@ class BaseTest: ) +# ╭───────────────────────────────────────────────────────╮ +# │ DISPLAY_TEXT │ +# ╰───────────────────────────────────────────────────────╯ + + +class TestDisplayText(BaseTest): + raw_option = {"type": "display_text", "id": "display_text_id"} + prefill = { + "raw_option": {}, + "prefill": " custom default", + } + # fmt: off + scenarios = [ + (None, None, {"ask": "Some text\na new line"}), + (None, None, {"ask": {"en": "Some text\na new line", "fr": "Un peu de texte\nune nouvelle ligne"}}), + ] + # fmt: on + + def test_options_prompted_with_ask_help(self, prefill_data=None): + pytest.skip(reason="no prompt for display types") + + def test_scenarios(self, intake, expected_output, raw_option, data): + _id = raw_option.pop("id") + answers = {_id: intake} if intake is not None else {} + options = None + with patch_interface("cli"): + if inspect.isclass(expected_output) and issubclass( + expected_output, Exception + ): + with pytest.raises(expected_output): + ask_questions_and_parse_answers({_id: raw_option}, answers) + else: + with patch.object(sys, "stdout", new_callable=StringIO) as stdout: + options = ask_questions_and_parse_answers( + {_id: raw_option}, answers + ) + assert stdout.getvalue() == f"{options[0].ask['en']}\n" + + # ╭───────────────────────────────────────────────────────╮ # │ STRING │ # ╰───────────────────────────────────────────────────────╯ @@ -1214,17 +1253,6 @@ def test_question_number_input_test_ask_with_help(): assert help_value in prompt.call_args[1]["message"] -def test_question_display_text(): - questions = {"some_app": {"type": "display_text", "ask": "foobar"}} - answers = {} - - with patch.object(sys, "stdout", new_callable=StringIO) as stdout, patch.object( - os, "isatty", return_value=True - ): - ask_questions_and_parse_answers(questions, answers) - assert "foobar" in stdout.getvalue() - - def test_question_file_from_cli(): FileQuestion.clean_upload_dirs() From f4b79068111237edd9c3acadb94de1c5c51eb9a4 Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 21 Mar 2023 21:15:29 +0100 Subject: [PATCH 22/43] options:tests: replace file tests --- src/tests/test_questions.py | 281 +++++++++++++++++------------------- 1 file changed, 136 insertions(+), 145 deletions(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index dffa93d14..cecb59b80 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -2,6 +2,7 @@ import inspect import sys import pytest import os +import tempfile from contextlib import contextmanager from mock import patch @@ -830,6 +831,141 @@ class TestWebPath(BaseTest): # fmt: on +# ╭───────────────────────────────────────────────────────╮ +# │ FILE │ +# ╰───────────────────────────────────────────────────────╯ + + +@pytest.fixture +def file_clean(): + FileQuestion.clean_upload_dirs() + yield + FileQuestion.clean_upload_dirs() + + +@contextmanager +def patch_file_cli(intake): + upload_dir = tempfile.mkdtemp(prefix="ynh_test_option_file") + _, filename = tempfile.mkstemp(dir=upload_dir) + with open(filename, "w") as f: + f.write(intake) + + yield filename + os.system(f"rm -f {filename}") + + +@contextmanager +def patch_file_api(intake): + from base64 import b64encode + + with patch_interface("api"): + yield b64encode(intake.encode()) + + +def _test_file_intake_may_fail(raw_option, intake, expected_output): + if inspect.isclass(expected_output) and issubclass(expected_output, Exception): + with pytest.raises(expected_output): + _fill_or_prompt_one_option(raw_option, intake) + + option, value = _fill_or_prompt_one_option(raw_option, intake) + + # The file is supposed to be copied somewhere else + assert value != intake + assert value.startswith("/tmp/ynh_filequestion_") + assert os.path.exists(value) + with open(value) as f: + assert f.read() == expected_output + + FileQuestion.clean_upload_dirs() + + assert not os.path.exists(value) + + +file_content1 = "helloworld" +file_content2 = """ +{ + "testy": true, + "test": ["one"] +} +""" + + +class TestFile(BaseTest): + raw_option = {"type": "file", "id": "file_id"} + # Prefill data is generated in `cls.test_options_prompted_with_ask_help` + # fmt: off + scenarios = [ + *nones(None, "", output=""), + *unchanged(file_content1, file_content2), + # other type checks are done in `test_wrong_intake` + ] + # fmt: on + # TODO test readonly + # TODO test accept + + @pytest.mark.usefixtures("patch_no_tty") + def test_basic_attrs(self): + raw_option, option, value = self._test_basic_attrs() + + accept = raw_option.get("accept", "") # accept default + assert option.accept == accept + + def test_options_prompted_with_ask_help(self): + with patch_file_cli(file_content1) as default_filename: + super().test_options_prompted_with_ask_help( + prefill_data={ + "raw_option": { + "default": default_filename, + }, + "prefill": default_filename, + } + ) + + @pytest.mark.usefixtures("file_clean") + def test_scenarios(self, intake, expected_output, raw_option, data): + if intake in (None, ""): + with patch_prompt(intake): + _test_intake_may_fail(raw_option, None, expected_output) + with patch_isatty(False): + _test_intake_may_fail(raw_option, intake, expected_output) + else: + with patch_file_cli(intake) as filename: + with patch_prompt(filename): + _test_file_intake_may_fail(raw_option, None, expected_output) + with patch_file_api(intake) as b64content: + with patch_isatty(False): + _test_file_intake_may_fail(raw_option, b64content, expected_output) + + @pytest.mark.parametrize( + "path", + [ + "/tmp/inexistant_file.txt", + "/tmp", + "/tmp/", + ], + ) + def test_wrong_cli_filename(self, path): + with patch_prompt(path): + with pytest.raises(YunohostValidationError): + _fill_or_prompt_one_option(self.raw_option, None) + + @pytest.mark.parametrize( + "intake", + [ + # fmt: off + False, True, 0, 1, -1, 1337, 13.37, [], ["one"], {}, + "none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n" + # fmt: on + ], + ) + def test_wrong_intake(self, intake): + with pytest.raises(YunohostValidationError): + with patch_prompt(intake): + _fill_or_prompt_one_option(self.raw_option, None) + with patch_isatty(False): + _fill_or_prompt_one_option(self.raw_option, intake) + + # ╭───────────────────────────────────────────────────────╮ # │ DOMAIN │ # ╰───────────────────────────────────────────────────────╯ @@ -1038,26 +1174,6 @@ def test_question_string_input_test_ask_with_example(): assert example_text in prompt.call_args[1]["message"] -@pytest.mark.skip # we should do something with this help -def test_question_string_input_test_ask_with_help(): - ask_text = "some question" - help_text = "some_help" - questions = { - "some_string": { - "ask": ask_text, - "help": help_text, - } - } - answers = {} - - with patch.object( - Moulinette, "prompt", return_value="some_value" - ) as prompt, patch.object(os, "isatty", return_value=True): - ask_questions_and_parse_answers(questions, answers) - assert ask_text in prompt.call_args[1]["message"] - assert help_text in prompt.call_args[1]["message"] - - def test_question_string_with_choice(): questions = {"some_string": {"type": "string", "choices": ["fr", "en"]}} answers = {"some_string": "fr"} @@ -1148,27 +1264,6 @@ def test_question_password_input_test_ask_with_example(): assert example_text in prompt.call_args[1]["message"] -@pytest.mark.skip # we should do something with this help -def test_question_password_input_test_ask_with_help(): - ask_text = "some question" - help_text = "some_help" - questions = { - "some_password": { - "type": "password", - "ask": ask_text, - "help": help_text, - } - } - answers = {} - - with patch.object( - Moulinette, "prompt", return_value="some_value" - ) as prompt, patch.object(os, "isatty", return_value=True): - ask_questions_and_parse_answers(questions, answers) - assert ask_text in prompt.call_args[1]["message"] - assert help_text in prompt.call_args[1]["message"] - - @pytest.mark.skip # we should do something with this example def test_question_path_input_test_ask_with_example(): ask_text = "some question" @@ -1190,27 +1285,6 @@ def test_question_path_input_test_ask_with_example(): assert example_text in prompt.call_args[1]["message"] -@pytest.mark.skip # we should do something with this help -def test_question_path_input_test_ask_with_help(): - ask_text = "some question" - help_text = "some_help" - questions = { - "some_path": { - "type": "path", - "ask": ask_text, - "help": help_text, - } - } - answers = {} - - with patch.object( - Moulinette, "prompt", return_value="some_value" - ) as prompt, patch.object(os, "isatty", return_value=True): - ask_questions_and_parse_answers(questions, answers) - assert ask_text in prompt.call_args[1]["message"] - assert help_text in prompt.call_args[1]["message"] - - @pytest.mark.skip # we should do something with this example def test_question_number_input_test_ask_with_example(): ask_text = "some question" @@ -1232,89 +1306,6 @@ def test_question_number_input_test_ask_with_example(): assert example_value in prompt.call_args[1]["message"] -@pytest.mark.skip # we should do something with this help -def test_question_number_input_test_ask_with_help(): - ask_text = "some question" - help_value = 1337 - questions = { - "some_number": { - "type": "number", - "ask": ask_text, - "help": help_value, - } - } - answers = {} - - with patch.object( - Moulinette, "prompt", return_value="1111" - ) as prompt, patch.object(os, "isatty", return_value=True): - ask_questions_and_parse_answers(questions, answers) - assert ask_text in prompt.call_args[1]["message"] - assert help_value in prompt.call_args[1]["message"] - - -def test_question_file_from_cli(): - FileQuestion.clean_upload_dirs() - - filename = "/tmp/ynh_test_question_file" - os.system(f"rm -f {filename}") - os.system(f"echo helloworld > {filename}") - - questions = { - "some_file": { - "type": "file", - } - } - answers = {"some_file": filename} - - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.name == "some_file" - assert out.type == "file" - - # The file is supposed to be copied somewhere else - assert out.value != filename - assert out.value.startswith("/tmp/") - assert os.path.exists(out.value) - assert "helloworld" in open(out.value).read().strip() - - FileQuestion.clean_upload_dirs() - - assert not os.path.exists(out.value) - - -def test_question_file_from_api(): - FileQuestion.clean_upload_dirs() - - from base64 import b64encode - - b64content = b64encode(b"helloworld") - questions = { - "some_file": { - "type": "file", - } - } - answers = {"some_file": b64content} - - interface_type_bkp = Moulinette.interface.type - try: - Moulinette.interface.type = "api" - out = ask_questions_and_parse_answers(questions, answers)[0] - finally: - Moulinette.interface.type = interface_type_bkp - - assert out.name == "some_file" - assert out.type == "file" - - assert out.value.startswith("/tmp/") - assert os.path.exists(out.value) - assert "helloworld" in open(out.value).read().strip() - - FileQuestion.clean_upload_dirs() - - assert not os.path.exists(out.value) - - def test_normalize_boolean_nominal(): assert BooleanQuestion.normalize("yes") == 1 assert BooleanQuestion.normalize("Yes") == 1 From 8e6178a863202e137d7dd5376d0dddbd0ce7b361 Mon Sep 17 00:00:00 2001 From: axolotle Date: Wed, 22 Mar 2023 14:11:00 +0100 Subject: [PATCH 23/43] options:tests: add missing types tests --- src/tests/test_questions.py | 833 +++++++++++++++++++++++++++++++++++- 1 file changed, 832 insertions(+), 1 deletion(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index cecb59b80..4e8133960 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -558,6 +558,77 @@ class TestDisplayText(BaseTest): assert stdout.getvalue() == f"{options[0].ask['en']}\n" +# ╭───────────────────────────────────────────────────────╮ +# │ MARKDOWN │ +# ╰───────────────────────────────────────────────────────╯ + + +class TestMarkdown(TestDisplayText): + raw_option = {"type": "markdown", "id": "markdown_id"} + # in cli this option is exactly the same as "display_text", no markdown support for now + + +# ╭───────────────────────────────────────────────────────╮ +# │ ALERT │ +# ╰───────────────────────────────────────────────────────╯ + + +class TestAlert(TestDisplayText): + raw_option = {"type": "alert", "id": "alert_id"} + prefill = { + "raw_option": {"ask": " Custom info message"}, + "prefill": " custom default", + } + # fmt: off + scenarios = [ + (None, None, {"ask": "Some text\na new line"}), + (None, None, {"ask": {"en": "Some text\na new line", "fr": "Un peu de texte\nune nouvelle ligne"}}), + *[(None, None, {"ask": "question", "style": style}) for style in ("success", "info", "warning", "danger")], + *xpass(scenarios=[ + (None, None, {"ask": "question", "style": "nimp"}), + ], reason="Should fail, wrong style"), + ] + # fmt: on + + def test_scenarios(self, intake, expected_output, raw_option, data): + style = raw_option.get("style", "info") + colors = {"danger": "31", "warning": "33", "info": "36", "success": "32"} + answers = {"alert_id": intake} if intake is not None else {} + + with patch_interface("cli"): + if inspect.isclass(expected_output) and issubclass( + expected_output, Exception + ): + with pytest.raises(expected_output): + ask_questions_and_parse_answers( + {"display_text_id": raw_option}, answers + ) + else: + with patch.object(sys, "stdout", new_callable=StringIO) as stdout: + options = ask_questions_and_parse_answers( + {"display_text_id": raw_option}, answers + ) + ask = options[0].ask["en"] + if style in colors: + color = colors[style] + title = style.title() + (":" if style != "success" else "!") + assert ( + stdout.getvalue() + == f"\x1b[{color}m\x1b[1m{title}\x1b[m {ask}\n" + ) + else: + # FIXME should fail + stdout.getvalue() == f"{ask}\n" + + +# ╭───────────────────────────────────────────────────────╮ +# │ BUTTON │ +# ╰───────────────────────────────────────────────────────╯ + + +# TODO + + # ╭───────────────────────────────────────────────────────╮ # │ STRING │ # ╰───────────────────────────────────────────────────────╯ @@ -653,6 +724,10 @@ class TestPassword(BaseTest): *all_fails([], ["one"], {}, raw_option={"optional": True}, error=AttributeError), # FIXME those fails with AttributeError *all_fails("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), *nones(None, "", output=""), + ("s3cr3t!!", FAIL, {"default": "SUPAs3cr3t!!"}), # default is forbidden + *xpass(scenarios=[ + ("s3cr3t!!", "s3cr3t!!", {"example": "SUPAs3cr3t!!"}), # example is forbidden + ], reason="Should fail; example is forbidden"), *xpass(scenarios=[ (" value \n moarc0mpl1cat3d\n ", "value \n moarc0mpl1cat3d"), (" some_ value", "some_ value"), @@ -668,6 +743,49 @@ class TestPassword(BaseTest): # fmt: on +# ╭───────────────────────────────────────────────────────╮ +# │ COLOR │ +# ╰───────────────────────────────────────────────────────╯ + + +class TestColor(BaseTest): + raw_option = {"type": "color", "id": "color_id"} + prefill = { + "raw_option": {"default": "#ff0000"}, + "prefill": "#ff0000", + # "intake": "#ff00ff", + } + # fmt: off + scenarios = [ + *all_fails(False, True, 0, 1, -1, 1337, 13.37, [], ["one"], {}, raw_option={"optional": True}), + *all_fails("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), + *nones(None, "", output=""), + # custom valid + ("#000000", "#000000"), + ("#000", "#000"), + ("#fe100", "#fe100"), + (" #fe100 ", "#fe100"), + ("#ABCDEF", "#ABCDEF"), + # custom fail + *xpass(scenarios=[ + ("#feaf", "#feaf"), + ], reason="Should fail; not a legal color value"), + ("000000", FAIL), + ("#12", FAIL), + ("#gggggg", FAIL), + ("#01010101af", FAIL), + *xfail(scenarios=[ + ("red", "#ff0000"), + ("yellow", "#ffff00"), + ], reason="Should work with pydantic"), + # readonly + *xfail(scenarios=[ + ("#ffff00", "#fe100", {"readonly": True, "default": "#fe100"}), + ], reason="Should not be overwritten"), + ] + # fmt: on + + # ╭───────────────────────────────────────────────────────╮ # │ NUMBER | RANGE │ # ╰───────────────────────────────────────────────────────╯ @@ -776,6 +894,171 @@ class TestBoolean(BaseTest): (1, 0, {"readonly": True, "default": 0}), ], reason="Should not be overwritten"), ] + + +# ╭───────────────────────────────────────────────────────╮ +# │ DATE │ +# ╰───────────────────────────────────────────────────────╯ + + +class TestDate(BaseTest): + raw_option = {"type": "date", "id": "date_id"} + prefill = { + "raw_option": {"default": "2024-12-29"}, + "prefill": "2024-12-29", + } + # fmt: off + scenarios = [ + *all_fails(False, True, 0, 1, -1, 1337, 13.37, [], ["one"], {}, raw_option={"optional": True}), + *all_fails("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), + *nones(None, "", output=""), + # custom valid + ("2070-12-31", "2070-12-31"), + ("2024-02-29", "2024-02-29"), + *xfail(scenarios=[ + ("2025-06-15T13:45:30", "2025-06-15"), + ("2025-06-15 13:45:30", "2025-06-15") + ], reason="iso date repr should be valid and extra data striped"), + *xfail(scenarios=[ + (1749938400, "2025-06-15"), + (1749938400.0, "2025-06-15"), + ("1749938400", "2025-06-15"), + ("1749938400.0", "2025-06-15"), + ], reason="timestamp could be an accepted value"), + # custom invalid + ("29-12-2070", FAIL), + ("12-01-10", FAIL), + ("2022-02-29", FAIL), + # readonly + *xfail(scenarios=[ + ("2070-12-31", "2024-02-29", {"readonly": True, "default": "2024-02-29"}), + ], reason="Should not be overwritten"), + ] + # fmt: on + + +# ╭───────────────────────────────────────────────────────╮ +# │ TIME │ +# ╰───────────────────────────────────────────────────────╯ + + +class TestTime(BaseTest): + raw_option = {"type": "time", "id": "time_id"} + prefill = { + "raw_option": {"default": "12:26"}, + "prefill": "12:26", + } + # fmt: off + scenarios = [ + *all_fails(False, True, 0, 1, -1, 1337, 13.37, [], ["one"], {}, raw_option={"optional": True}), + *all_fails("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), + *nones(None, "", output=""), + # custom valid + *unchanged("00:00", "08:00", "12:19", "20:59", "23:59"), + ("3:00", "3:00"), # FIXME should fail or output as `"03:00"`? + *xfail(scenarios=[ + ("22:35:05", "22:35"), + ("22:35:03.514", "22:35"), + ], reason="time as iso format could be valid"), + # custom invalid + ("24:00", FAIL), + ("23:1", FAIL), + ("23:005", FAIL), + # readonly + *xfail(scenarios=[ + ("00:00", "08:00", {"readonly": True, "default": "08:00"}), + ], reason="Should not be overwritten"), + ] + # fmt: on + + +# ╭───────────────────────────────────────────────────────╮ +# │ EMAIL │ +# ╰───────────────────────────────────────────────────────╯ + + +class TestEmail(BaseTest): + raw_option = {"type": "email", "id": "email_id"} + prefill = { + "raw_option": {"default": "Abc@example.tld"}, + "prefill": "Abc@example.tld", + } + # fmt: off + scenarios = [ + *all_fails(False, True, 0, 1, 1337, 13.37, [], ["one"], {}, raw_option={"optional": True}), + *all_fails("none", "_none", "False", "True", "0", "1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), + + *nones(None, "", output=""), + ("\n Abc@example.tld ", "Abc@example.tld"), + # readonly + *xfail(scenarios=[ + ("Abc@example.tld", "admin@ynh.local", {"readonly": True, "default": "admin@ynh.local"}), + ], reason="Should not be overwritten"), + + # Next examples are from https://github.com/JoshData/python-email-validator/blob/main/tests/test_syntax.py + # valid email values + ("Abc@example.tld", "Abc@example.tld"), + ("Abc.123@test-example.com", "Abc.123@test-example.com"), + ("user+mailbox/department=shipping@example.tld", "user+mailbox/department=shipping@example.tld"), + ("伊昭傑@郵件.商務", "伊昭傑@郵件.商務"), + ("राम@मोहन.ईन्फो", "राम@मोहन.ईन्फो"), + ("юзер@екзампл.ком", "юзер@екзампл.ком"), + ("θσερ@εχαμπλε.ψομ", "θσερ@εχαμπλε.ψομ"), + ("葉士豪@臺網中心.tw", "葉士豪@臺網中心.tw"), + ("jeff@臺網中心.tw", "jeff@臺網中心.tw"), + ("葉士豪@臺網中心.台灣", "葉士豪@臺網中心.台灣"), + ("jeff葉@臺網中心.tw", "jeff葉@臺網中心.tw"), + ("ñoñó@example.tld", "ñoñó@example.tld"), + ("甲斐黒川日本@example.tld", "甲斐黒川日本@example.tld"), + ("чебурашкаящик-с-апельсинами.рф@example.tld", "чебурашкаящик-с-апельсинами.рф@example.tld"), + ("उदाहरण.परीक्ष@domain.with.idn.tld", "उदाहरण.परीक्ष@domain.with.idn.tld"), + ("ιωάννης@εεττ.gr", "ιωάννης@εεττ.gr"), + # invalid email (Hiding because our current regex is very permissive) + # ("my@localhost", FAIL), + # ("my@.leadingdot.com", FAIL), + # ("my@．leadingfwdot.com", FAIL), + # ("my@twodots..com", FAIL), + # ("my@twofwdots．．.com", FAIL), + # ("my@trailingdot.com.", FAIL), + # ("my@trailingfwdot.com．", FAIL), + # ("me@-leadingdash", FAIL), + # ("me@－leadingdashfw", FAIL), + # ("me@trailingdash-", FAIL), + # ("me@trailingdashfw－", FAIL), + # ("my@baddash.-.com", FAIL), + # ("my@baddash.-a.com", FAIL), + # ("my@baddash.b-.com", FAIL), + # ("my@baddashfw.－.com", FAIL), + # ("my@baddashfw.－a.com", FAIL), + # ("my@baddashfw.b－.com", FAIL), + # ("my@example.com\n", FAIL), + # ("my@example\n.com", FAIL), + # ("me@x!", FAIL), + # ("me@x ", FAIL), + # (".leadingdot@domain.com", FAIL), + # ("twodots..here@domain.com", FAIL), + # ("trailingdot.@domain.email", FAIL), + # ("me@⒈wouldbeinvalid.com", FAIL), + ("@example.com", FAIL), + # ("\nmy@example.com", FAIL), + ("m\ny@example.com", FAIL), + ("my\n@example.com", FAIL), + # ("11111111112222222222333333333344444444445555555555666666666677777@example.com", FAIL), + # ("111111111122222222223333333333444444444455555555556666666666777777@example.com", FAIL), + # ("me@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.111111111122222222223333333333444444444455555555556.com", FAIL), + # ("me@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555566.com", FAIL), + # ("me@中1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555566.com", FAIL), + # ("my.long.address@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.11111111112222222222333333333344444.info", FAIL), + # ("my.long.address@λ111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.11111111112222222222333333.info", FAIL), + # ("my.long.address@λ111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444.info", FAIL), + # ("my.λong.address@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.111111111122222222223333333333444.info", FAIL), + # ("my.λong.address@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444.info", FAIL), + # ("me@bad-tld-1", FAIL), + # ("me@bad.tld-2", FAIL), + # ("me@xn--0.tld", FAIL), + # ("me@yy--0.tld", FAIL), + # ("me@yy－－0.tld", FAIL), + ] # fmt: on @@ -831,6 +1114,110 @@ class TestWebPath(BaseTest): # fmt: on +# ╭───────────────────────────────────────────────────────╮ +# │ URL │ +# ╰───────────────────────────────────────────────────────╯ + + +class TestUrl(BaseTest): + raw_option = {"type": "url", "id": "url_id"} + prefill = { + "raw_option": {"default": "https://domain.tld"}, + "prefill": "https://domain.tld", + } + # fmt: off + scenarios = [ + *all_fails(False, True, 0, 1, -1, 1337, 13.37, [], ["one"], {}, raw_option={"optional": True}), + *all_fails("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), + + *nones(None, "", output=""), + ("http://some.org/folder/file.txt", "http://some.org/folder/file.txt"), + # readonly + *xfail(scenarios=[ + ("https://overwrite.org", "https://example.org", {"readonly": True, "default": "https://example.org"}), + ], reason="Should not be overwritten"), + # rest is taken from https://github.com/pydantic/pydantic/blob/main/tests/test_networks.py + # valid + *unchanged( + # Those are valid but not sure how they will output with pydantic + 'http://example.org', + 'http://test', + 'http://localhost', + 'https://example.org/whatever/next/', + 'https://example.org', + 'http://localhost', + 'http://localhost/', + 'http://localhost:8000', + 'http://localhost:8000/', + 'https://foo_bar.example.com/', + 'http://example.co.jp', + 'http://www.example.com/a%C2%B1b', + 'http://www.example.com/~username/', + 'http://info.example.com?fred', + 'http://info.example.com/?fred', + 'http://xn--mgbh0fb.xn--kgbechtv/', + 'http://example.com/blue/red%3Fand+green', + 'http://www.example.com/?array%5Bkey%5D=value', + 'http://xn--rsum-bpad.example.org/', + 'http://123.45.67.8/', + 'http://123.45.67.8:8329/', + 'http://[2001:db8::ff00:42]:8329', + 'http://[2001::1]:8329', + 'http://[2001:db8::1]/', + 'http://www.example.com:8000/foo', + 'http://www.cwi.nl:80/%7Eguido/Python.html', + 'https://www.python.org/путь', + 'http://андрей@example.com', + 'https://exam_ple.com/', + 'http://twitter.com/@handle/', + 'http://11.11.11.11.example.com/action', + 'http://abc.11.11.11.11.example.com/action', + 'http://example#', + 'http://example/#', + 'http://example/#fragment', + 'http://example/?#', + 'http://example.org/path#', + 'http://example.org/path#fragment', + 'http://example.org/path?query#', + 'http://example.org/path?query#fragment', + ), + # Pydantic default parsing add a final `/` + ('https://foo_bar.example.com/', 'https://foo_bar.example.com/'), + ('https://exam_ple.com/', 'https://exam_ple.com/'), + *xfail(scenarios=[ + (' https://www.example.com \n', 'https://www.example.com/'), + ('HTTP://EXAMPLE.ORG', 'http://example.org/'), + ('https://example.org', 'https://example.org/'), + ('https://example.org?a=1&b=2', 'https://example.org/?a=1&b=2'), + ('https://example.org#a=3;b=3', 'https://example.org/#a=3;b=3'), + ('https://example.xn--p1ai', 'https://example.xn--p1ai/'), + ('https://example.xn--vermgensberatung-pwb', 'https://example.xn--vermgensberatung-pwb/'), + ('https://example.xn--zfr164b', 'https://example.xn--zfr164b/'), + ], reason="pydantic default behavior would append a final `/`"), + + # invalid + *all_fails( + 'ftp://example.com/', + "$https://example.org", + "../icons/logo.gif", + "abc", + "..", + "/", + "+http://example.com/", + "ht*tp://example.com/", + ), + *xpass(scenarios=[ + ("http:///", "http:///"), + ("http://??", "http://??"), + ("https://example.org more", "https://example.org more"), + ("http://2001:db8::ff00:42:8329", "http://2001:db8::ff00:42:8329"), + ("http://[192.168.1.1]:8329", "http://[192.168.1.1]:8329"), + ("http://example.com:99999", "http://example.com:99999"), + ], reason="Should fail"), + ] + # fmt: on + + # ╭───────────────────────────────────────────────────────╮ # │ FILE │ # ╰───────────────────────────────────────────────────────╯ @@ -966,6 +1353,135 @@ class TestFile(BaseTest): _fill_or_prompt_one_option(self.raw_option, intake) +# ╭───────────────────────────────────────────────────────╮ +# │ SELECT │ +# ╰───────────────────────────────────────────────────────╯ + + +class TestSelect(BaseTest): + raw_option = {"type": "select", "id": "select_id"} + prefill = { + "raw_option": {"default": "one", "choices": ["one", "two"]}, + "prefill": "one", + } + # fmt: off + scenarios = [ + { + # ["one", "two"] + "raw_options": [ + {"choices": ["one", "two"]}, + {"choices": {"one": "verbose one", "two": "verbose two"}}, + ], + "scenarios": [ + *nones(None, "", output=""), + *unchanged("one", "two"), + ("three", FAIL), + ] + }, + # custom bash style list as choices (only strings for now) + ("one", "one", {"choices": "one,two"}), + { + # [-1, 0, 1] + "raw_options": [ + {"choices": [-1, 0, 1, 10]}, + {"choices": {-1: "verbose -one", 0: "verbose zero", 1: "verbose one", 10: "verbose ten"}}, + ], + "scenarios": [ + *nones(None, "", output=""), + *unchanged(-1, 0, 1, 10), + *xfail(scenarios=[ + ("-1", -1), + ("0", 0), + ("1", 1), + ("10", 10), + ], reason="str -> int not handled"), + *all_fails("100", 100), + ] + }, + # [True, False, None] + *unchanged(True, False, raw_option={"choices": [True, False, None]}), # FIXME we should probably forbid None in choices + (None, FAIL, {"choices": [True, False, None]}), + { + # mixed types + "raw_options": [{"choices": ["one", 2, True]}], + "scenarios": [ + *xpass(scenarios=[ + ("one", "one"), + (2, 2), + (True, True), + ], reason="mixed choices, should fail"), + *all_fails("2", "True", "y"), + ] + }, + { + "raw_options": [{"choices": ""}, {"choices": []}], + "scenarios": [ + # FIXME those should fail at option level (wrong default, dev error) + *all_fails(None, ""), + *xpass(scenarios=[ + ("", "", {"optional": True}), + (None, "", {"optional": True}), + ], reason="empty choices, should fail at option instantiation"), + ] + }, + # readonly + *xfail(scenarios=[ + ("one", "two", {"readonly": True, "choices": ["one", "two"], "default": "two"}), + ], reason="Should not be overwritten"), + ] + # fmt: on + + +# ╭───────────────────────────────────────────────────────╮ +# │ TAGS │ +# ╰───────────────────────────────────────────────────────╯ + + +class TestTags(BaseTest): + raw_option = {"type": "tags", "id": "tags_id"} + prefill = { + "raw_option": {"default": ["one", "two"]}, + "prefill": "one,two", + } + # fmt: off + scenarios = [ + *nones(None, [], "", output=""), + # FIXME `","` could be considered a none value which kinda already is since it fail when required + (",", FAIL), + *xpass(scenarios=[ + (",", ",", {"optional": True}) + ], reason="Should output as `''`? ie: None"), + { + "raw_options": [ + {}, + {"choices": ["one", "two"]} + ], + "scenarios": [ + *unchanged("one", "one,two"), + (["one"], "one"), + (["one", "two"], "one,two"), + ] + }, + ("three", FAIL, {"choices": ["one", "two"]}), + *unchanged("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", "['one']", "one,two", r"{}", "value"), + (" value\n", "value"), + ([False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}], "False,True,-1,0,1,1337,13.37,[],['one'],{}"), + *(([t], str(t)) for t in (False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {})), + # basic types (not in a list) should fail + *all_fails(True, False, -1, 0, 1, 1337, 13.37, {}), + # Mixed choices should fail + ([False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}], FAIL, {"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}), + ("False,True,-1,0,1,1337,13.37,[],['one'],{}", FAIL, {"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}), + *all_fails(*([t] for t in [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]), raw_option={"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}), + *all_fails(*([str(t)] for t in [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]), raw_option={"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}), + # readonly + *xfail(scenarios=[ + ("one", "one,two", {"readonly": True, "choices": ["one", "two"], "default": "one,two"}), + ], reason="Should not be overwritten"), + ] + # fmt: on + + # ╭───────────────────────────────────────────────────────╮ # │ DOMAIN │ # ╰───────────────────────────────────────────────────────╯ @@ -1033,6 +1549,124 @@ class TestDomain(BaseTest): super().test_scenarios(intake, expected_output, raw_option, data) +# ╭───────────────────────────────────────────────────────╮ +# │ APP │ +# ╰───────────────────────────────────────────────────────╯ + +installed_webapp = { + "is_webapp": True, + "is_default": True, + "label": "My webapp", + "id": "my_webapp", + "domain_path": "/ynh-dev", +} +installed_non_webapp = { + "is_webapp": False, + "is_default": False, + "label": "My non webapp", + "id": "my_non_webapp", +} + + +@contextmanager +def patch_apps(*, apps): + """ + Data mocking for AppOption: + - yunohost.app.app_list + """ + with patch.object(app, "app_list", return_value={"apps": apps}): + yield + + +class TestApp(BaseTest): + raw_option = {"type": "app", "id": "app_id"} + # fmt: off + scenarios = [ + # Probably not needed to test common types since those are not available as choices + { + "data": [ + {"apps": []}, + {"apps": [installed_webapp]}, + {"apps": [installed_webapp, installed_non_webapp]}, + ], + "scenarios": [ + # FIXME there are currently 3 different nones (`None`, `""` and `_none`), choose one? + *nones(None, output=None), # FIXME Should return chosen none? + *nones("", output=""), # FIXME Should return chosen none? + *xpass(scenarios=[ + ("_none", "_none"), + ("_none", "_none", {"default": "_none"}), + ], reason="should fail; is required"), + *xpass(scenarios=[ + ("_none", "_none", {"optional": True}), + ("_none", "_none", {"optional": True, "default": "_none"}) + ], reason="Should output chosen none value"), + ("fake_app", FAIL), + ("fake_app", FAIL, {"choices": ["fake_app"]}), + ] + }, + { + "data": [ + {"apps": [installed_webapp]}, + {"apps": [installed_webapp, installed_non_webapp]}, + ], + "scenarios": [ + (installed_webapp["id"], installed_webapp["id"]), + (installed_webapp["id"], installed_webapp["id"], {"filter": "is_webapp"}), + (installed_webapp["id"], FAIL, {"filter": "is_webapp == false"}), + (installed_webapp["id"], FAIL, {"filter": "id != 'my_webapp'"}), + (None, None, {"filter": "id == 'fake_app'", "optional": True}), + ] + }, + { + "data": [{"apps": [installed_webapp, installed_non_webapp]}], + "scenarios": [ + (installed_non_webapp["id"], installed_non_webapp["id"]), + (installed_non_webapp["id"], FAIL, {"filter": "is_webapp"}), + # readonly + *xpass(scenarios=[ + (installed_non_webapp["id"], installed_non_webapp["id"], {"readonly": True}), + ], reason="Should fail since readonly is forbidden"), + ] + }, + ] + # fmt: on + + @pytest.mark.usefixtures("patch_no_tty") + def test_basic_attrs(self): + with patch_apps(apps=[]): + raw_option, option, value = self._test_basic_attrs() + + assert option.choices == {"_none": "---"} + assert option.filter is None + + with patch_apps(apps=[installed_webapp, installed_non_webapp]): + raw_option, option, value = self._test_basic_attrs() + + assert option.choices == { + "_none": "---", + "my_webapp": "My webapp (/ynh-dev)", + "my_non_webapp": "My non webapp (my_non_webapp)", + } + assert option.filter is None + + def test_options_prompted_with_ask_help(self, prefill_data=None): + with patch_apps(apps=[installed_webapp, installed_non_webapp]): + super().test_options_prompted_with_ask_help( + prefill_data={ + "raw_option": {"default": installed_webapp["id"]}, + "prefill": installed_webapp["id"], + } + ) + super().test_options_prompted_with_ask_help( + prefill_data={"raw_option": {"optional": True}, "prefill": ""} + ) + + def test_scenarios(self, intake, expected_output, raw_option, data): + with patch_apps(**data): + super().test_scenarios(intake, expected_output, raw_option, data) + + # ╭───────────────────────────────────────────────────────╮ # │ USER │ # ╰───────────────────────────────────────────────────────╯ @@ -1139,10 +1773,207 @@ class TestUser(BaseTest): super().test_scenarios(intake, expected_output, raw_option, data) -def test_question_empty(): +# ╭───────────────────────────────────────────────────────╮ +# │ GROUP │ +# ╰───────────────────────────────────────────────────────╯ + +groups1 = ["all_users", "visitors", "admins"] +groups2 = ["all_users", "visitors", "admins", "custom_group"] + + +@contextmanager +def patch_groups(*, groups): + """ + Data mocking for GroupOption: + - yunohost.user.user_group_list + """ + with patch.object(user, "user_group_list", return_value={"groups": groups}): + yield + + +class TestGroup(BaseTest): + raw_option = {"type": "group", "id": "group_id"} + # fmt: off + scenarios = [ + # No tests for empty groups since it should not happens + { + "data": [ + {"groups": groups1}, + {"groups": groups2}, + ], + "scenarios": [ + # FIXME Group option is not really nullable, even if optional + *nones(None, "", output="all_users", fail_if_required=False), + ("admins", "admins"), + ("fake_group", FAIL), + ("fake_group", FAIL, {"choices": ["fake_group"]}), + ] + }, + { + "data": [ + {"groups": groups2}, + ], + "scenarios": [ + ("custom_group", "custom_group"), + *all_as("", None, output="visitors", raw_option={"default": "visitors"}), + *xpass(scenarios=[ + ("", "custom_group", {"default": "custom_group"}), + ], reason="Should throw 'default must be in (None, 'all_users', 'visitors', 'admins')"), + # readonly + *xpass(scenarios=[ + ("admins", "admins", {"readonly": True}), + ], reason="Should fail since readonly is forbidden"), + ] + }, + ] + # fmt: on + + def test_options_prompted_with_ask_help(self, prefill_data=None): + with patch_groups(groups=groups2): + super().test_options_prompted_with_ask_help( + prefill_data={"raw_option": {}, "prefill": "all_users"} + ) + super().test_options_prompted_with_ask_help( + prefill_data={ + "raw_option": {"default": "admins"}, + "prefill": "admins", + } + ) + # FIXME This should fail, not allowed to set a default which is not a default group + super().test_options_prompted_with_ask_help( + prefill_data={ + "raw_option": {"default": "custom_group"}, + "prefill": "custom_group", + } + ) + + def test_scenarios(self, intake, expected_output, raw_option, data): + with patch_groups(**data): + super().test_scenarios(intake, expected_output, raw_option, data) + + +# ╭───────────────────────────────────────────────────────╮ +# │ MULTIPLE │ +# ╰───────────────────────────────────────────────────────╯ + + +@pytest.fixture +def patch_entities(): + with patch_domains(domains=domains2, main_domain=main_domain), patch_apps( + apps=[installed_webapp, installed_non_webapp] + ), patch_users( + users={admin_username: admin_user, regular_username: regular_user}, + admin_username=admin_username, + main_domain=main_domain, + ), patch_groups( + groups=groups2 + ): + yield + + +def test_options_empty(): ask_questions_and_parse_answers({}, {}) == [] +@pytest.mark.usefixtures("patch_entities", "file_clean") +def test_options_query_string(): + raw_options = { + "string_id": {"type": "string"}, + "text_id": {"type": "text"}, + "password_id": {"type": "password"}, + "color_id": {"type": "color"}, + "number_id": {"type": "number"}, + "boolean_id": {"type": "boolean"}, + "date_id": {"type": "date"}, + "time_id": {"type": "time"}, + "email_id": {"type": "email"}, + "path_id": {"type": "path"}, + "url_id": {"type": "url"}, + "file_id": {"type": "file"}, + "select_id": {"type": "select", "choices": ["one", "two"]}, + "tags_id": {"type": "tags", "choices": ["one", "two"]}, + "domain_id": {"type": "domain"}, + "app_id": {"type": "app"}, + "user_id": {"type": "user"}, + "group_id": {"type": "group"}, + } + + results = { + "string_id": "string", + "text_id": "text\ntext", + "password_id": "sUpRSCRT", + "color_id": "#ffff00", + "number_id": 10, + "boolean_id": 1, + "date_id": "2030-03-06", + "time_id": "20:55", + "email_id": "coucou@ynh.local", + "path_id": "/ynh-dev", + "url_id": "https://yunohost.org", + "file_id": file_content1, + "select_id": "one", + "tags_id": "one,two", + "domain_id": main_domain, + "app_id": installed_webapp["id"], + "user_id": regular_username, + "group_id": "admins", + } + + @contextmanager + def patch_query_string(file_repr): + yield ( + "string_id= string" + "&text_id=text\ntext" + "&password_id=sUpRSCRT" + "&color_id=#ffff00" + "&number_id=10" + "&boolean_id=y" + "&date_id=2030-03-06" + "&time_id=20:55" + "&email_id=coucou@ynh.local" + "&path_id=ynh-dev/" + "&url_id=https://yunohost.org" + f"&file_id={file_repr}" + "&select_id=one" + "&tags_id=one,two" + # FIXME We can't test with parse.qs for now, next syntax is available only with config panels + # "&tags_id=one" + # "&tags_id=two" + f"&domain_id={main_domain}" + f"&app_id={installed_webapp['id']}" + f"&user_id={regular_username}" + "&group_id=admins" + # not defined extra values are silently ignored + "&fake_id=fake_value" + ) + + def _assert_correct_values(options, raw_options): + form = {option.name: option.value for option in options} + + for k, v in results.items(): + if k == "file_id": + assert os.path.exists(form["file_id"]) and os.path.isfile( + form["file_id"] + ) + with open(form["file_id"], "r") as f: + assert f.read() == file_content1 + else: + assert form[k] == results[k] + + assert len(options) == len(raw_options.keys()) + assert "fake_id" not in form + + with patch_interface("api"), patch_file_api(file_content1) as b64content: + with patch_query_string(b64content.decode("utf-8")) as query_string: + options = ask_questions_and_parse_answers(raw_options, query_string) + _assert_correct_values(options, raw_options) + + with patch_interface("cli"), patch_file_cli(file_content1) as filepath: + with patch_query_string(filepath) as query_string: + options = ask_questions_and_parse_answers(raw_options, query_string) + _assert_correct_values(options, raw_options) + + def test_question_string_default_type(): questions = {"some_string": {}} answers = {"some_string": "some_value"} From f8c1e7c168b885ea23d6017447b9795b9eb041fd Mon Sep 17 00:00:00 2001 From: axolotle Date: Wed, 22 Mar 2023 14:13:54 +0100 Subject: [PATCH 24/43] options: misc option quick fixes --- src/tests/test_questions.py | 2 +- src/utils/config.py | 42 ++++++++++++++++++++++++++++++------- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index 4e8133960..8ded2e137 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -13,7 +13,7 @@ from _pytest.mark.structures import ParameterSet from moulinette import Moulinette -from yunohost import domain, user +from yunohost import app, domain, user from yunohost.utils.config import ( ARGUMENTS_TYPE_PARSERS, ask_questions_and_parse_answers, diff --git a/src/utils/config.py b/src/utils/config.py index 6f06ed1fb..37f41f8b2 100644 --- a/src/utils/config.py +++ b/src/utils/config.py @@ -856,7 +856,9 @@ class Question: # Don't restrict choices if there's none specified self.choices = question.get("choices", None) self.pattern = question.get("pattern", self.pattern) - self.ask = question.get("ask", {"en": self.name}) + self.ask = question.get("ask", self.name) + if not isinstance(self.ask, dict): + self.ask = {"en": self.ask} self.help = question.get("help") self.redact = question.get("redact", False) self.filter = question.get("filter", None) @@ -962,7 +964,7 @@ class Question: "app_argument_choice_invalid", name=self.name, value=self.value, - choices=", ".join(self.choices), + choices=", ".join(str(choice) for choice in self.choices), ) if self.pattern and not re.match(self.pattern["regexp"], str(self.value)): raise YunohostValidationError( @@ -1085,13 +1087,13 @@ class TagsQuestion(Question): @staticmethod def humanize(value, option={}): if isinstance(value, list): - return ",".join(value) + return ",".join(str(v) for v in value) return value @staticmethod def normalize(value, option={}): if isinstance(value, list): - return ",".join(value) + return ",".join(str(v) for v in value) if isinstance(value, str): value = value.strip() return value @@ -1102,6 +1104,21 @@ class TagsQuestion(Question): values = values.split(",") elif values is None: values = [] + + if not isinstance(values, list): + if self.choices: + raise YunohostValidationError( + "app_argument_choice_invalid", + name=self.name, + value=self.value, + choices=", ".join(str(choice) for choice in self.choices), + ) + raise YunohostValidationError( + "app_argument_invalid", + name=self.name, + error=f"'{str(self.value)}' is not a list", + ) + for value in values: self.value = value super()._prevalidate() @@ -1152,6 +1169,13 @@ class PathQuestion(Question): def normalize(value, option={}): option = option.__dict__ if isinstance(option, Question) else option + if not isinstance(value, str): + raise YunohostValidationError( + "app_argument_invalid", + name=option.get("name"), + error="Argument for path should be a string.", + ) + if not value.strip(): if option.get("optional"): return "" @@ -1399,7 +1423,7 @@ class NumberQuestion(Question): return int(value) if value in [None, ""]: - return value + return None option = option.__dict__ if isinstance(option, Question) else option raise YunohostValidationError( @@ -1481,8 +1505,12 @@ class FileQuestion(Question): super()._prevalidate() + # Validation should have already failed if required + if self.value in (None, ""): + return self.value + if Moulinette.interface.type != "api": - if not self.value or not os.path.exists(str(self.value)): + if not os.path.exists(str(self.value)) or not os.path.isfile(str(self.value)): raise YunohostValidationError( "app_argument_invalid", name=self.name, @@ -1493,7 +1521,7 @@ class FileQuestion(Question): from base64 import b64decode if not self.value: - return self.value + return "" upload_dir = tempfile.mkdtemp(prefix="ynh_filequestion_") _, file_path = tempfile.mkstemp(dir=upload_dir) From 2d03176c7fc5ea29863f0bb2fe1b2878839008ea Mon Sep 17 00:00:00 2001 From: axolotle Date: Wed, 22 Mar 2023 15:37:39 +0100 Subject: [PATCH 25/43] fix i18n panel+section names --- src/utils/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/config.py b/src/utils/config.py index 6f06ed1fb..7b16d6a23 100644 --- a/src/utils/config.py +++ b/src/utils/config.py @@ -581,7 +581,7 @@ class ConfigPanel: logger.warning(f"Unknown key '{key}' found in config panel") # Todo search all i18n keys out[key] = ( - value if key not in ["ask", "help", "name"] else {"en": value} + value if key not in ["ask", "help", "name"] or isinstance(value, dict) else {"en": value} ) return out From 63981aacf9941ac779f437e57844d0bf8d1a0daf Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 Mar 2023 20:34:38 +0200 Subject: [PATCH 26/43] appsv2: Add documentation about the new 'autoupdate' mechanism for app sources --- src/utils/resources.py | 98 +++++++++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 39 deletions(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index b9bb1fee7..4c7c09fd3 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -267,7 +267,7 @@ class SourcesResource(AppResource): Various options are available to accomodate the behavior according to the asset structure - ##### Example: + ##### Example ```toml [resources.sources] @@ -275,6 +275,8 @@ class SourcesResource(AppResource): [resources.sources.main] url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.tar.gz" sha256 = "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b" + + autoupdate.strategy = "latest_github_tag" ``` Or more complex examples with several element, including one with asset that depends on the arch @@ -286,11 +288,16 @@ class SourcesResource(AppResource): in_subdir = false amd64.url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.amd64.tar.gz" amd64.sha256 = "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b" - i386.url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.amd64.tar.gz" + i386.url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.386.tar.gz" i386.sha256 = "53c234e5e8472b6ac51c1ae1cab3fe06fad053beb8ebfd8977b010655bfdd3c3" - armhf.url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.armhf.tar.gz" + armhf.url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.arm.tar.gz" armhf.sha256 = "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865" + autoupdate.strategy = "latest_github_release" + autoupdate.asset.amd64 = ".*\.amd64.tar.gz" + autoupdate.asset.i386 = ".*\.386.tar.gz" + autoupdate.asset.armhf = ".*\.arm.tar.gz" + [resources.sources.zblerg] url = "https://zblerg.com/download/zblerg" sha256 = "1121cfccd5913f0a63fec40a6ffd44ea64f9dc135c66634ba001d10bcf4302a2" @@ -299,7 +306,7 @@ class SourcesResource(AppResource): ``` - ##### Properties (for each source): + ##### Properties (for each source) - `prefetch` : `true` (default) or `false`, wether or not to pre-fetch this asset during the provisioning phase of the resource. If several arch-dependent url are provided, YunoHost will only prefetch the one for the current system architecture. - `url` : the asset's URL @@ -316,11 +323,24 @@ class SourcesResource(AppResource): - `rename`: some string like `whatever_your_want`, to be used for convenience when `extract` is `false` and the default name of the file is not practical - `platform`: for example `linux/amd64` (defaults to `linux/$YNH_ARCH`) to be used in conjonction with `format = "docker"` to specify which architecture to extract for + ###### Regarding `autoupdate` - ##### Provision/Update: + Strictly speaking, this has nothing to do with the actual app install. `autoupdate` is expected to contain metadata for automatic maintenance / update of the app sources info in the manifest. It is meant to be a simpler replacement for "autoupdate" Github workflow mechanism. + + The infos are used by this script : https://github.com/YunoHost/apps/blob/master/tools/autoupdate_app_sources/autoupdate_app_sources.py which is ran by the YunoHost infrastructure periodically and will create the corresponding pull request automatically. + + The script will rely on the code repo specified in the upstream section of the manifest. + + `autoupdate.strategy` is expected to be one of : + - `latest_github_tag` : look for the latest tag (by sorting tags and finding the "largest" version). Then using the corresponding tar.gz url. Tags containing `rc`, `beta`, `alpha`, `start` are ignored, and actually any tag which doesn't look like `x.y.z` or `vx.y.z` + - `latest_github_release` : similar to `latest_github_tags`, but starting from the list of releases. Pre- or draft releases are ignored. Releases may have assets attached to them, in which case you can define: + - `autoupdate.asset = "some regex"` (when there's only one asset to use). The regex is used to find the appropriate asset among the list of all assets + - or several `autoupdate.asset.$arch = "some_regex"` (when the asset is arch-specific). The regex is used to find the appropriate asset for the specific arch among the list of assets + + ##### Provision/Update - For elements with `prefetch = true`, will download the asset (for the appropriate architecture) and store them in `/var/cache/yunohost/download/$app/$source_id`, to be later picked up by `ynh_setup_source`. (NB: this only happens during install and upgrade, not restore) - ##### Deprovision: + ##### Deprovision - Nothing (just cleanup the cache) """ @@ -439,7 +459,7 @@ class PermissionsResource(AppResource): The list of allowed user/groups may be initialized using the content of the `init_{perm}_permission` question from the manifest, hence `init_main_permission` replaces the `is_public` question and shall contain a group name (typically, `all_users` or `visitors`). - ##### Example: + ##### Example ```toml [resources.permissions] main.url = "/" @@ -450,7 +470,7 @@ class PermissionsResource(AppResource): admin.allowed = "admins" # Assuming the "admins" group exists (cf future developments ;)) ``` - ##### Properties (for each perm name): + ##### Properties (for each perm name) - `url`: The relative URI corresponding to this permission. Typically `/` or `/something`. This property may be omitted for non-web permissions. - `show_tile`: (default: `true` if `url` is defined) Wether or not a tile should be displayed for that permission in the user portal - `allowed`: (default: nobody) The group initially allowed to access this perm, if `init_{perm}_permission` is not defined in the manifest questions. Note that the admin may tweak who is allowed/unallowed on that permission later on, this is only meant to **initialize** the permission. @@ -458,14 +478,14 @@ class PermissionsResource(AppResource): - `protected`: (default: `false`) Define if this permission is protected. If it is protected the administrator won't be able to add or remove the visitors group of this permission. Defaults to 'false'. - `additional_urls`: (default: none) List of additional URL for which access will be allowed/forbidden - ##### Provision/Update: + ##### Provision/Update - Delete any permissions that may exist and be related to this app yet is not declared anymore - Loop over the declared permissions and create them if needed or update them with the new values - ##### Deprovision: + ##### Deprovision - Delete all permission related to this app - ##### Legacy management: + ##### Legacy management - Legacy `is_public` setting will be deleted if it exists """ @@ -627,22 +647,22 @@ class SystemuserAppResource(AppResource): """ Provision a system user to be used by the app. The username is exactly equal to the app id - ##### Example: + ##### Example ```toml [resources.system_user] # (empty - defaults are usually okay) ``` - ##### Properties: + ##### Properties - `allow_ssh`: (default: False) Adds the user to the ssh.app group, allowing SSH connection via this user - `allow_sftp`: (default: False) Adds the user to the sftp.app group, allowing SFTP connection via this user - `home`: (default: `/var/www/__APP__`) Defines the home property for this user. NB: unfortunately you can't simply use `__INSTALL_DIR__` or `__DATA_DIR__` for now - ##### Provision/Update: + ##### Provision/Update - will create the system user if it doesn't exists yet - will add/remove the ssh/sftp.app groups - ##### Deprovision: + ##### Deprovision - deletes the user and group """ @@ -735,28 +755,28 @@ class InstalldirAppResource(AppResource): """ Creates a directory to be used by the app as the installation directory, typically where the app sources and assets are located. The corresponding path is stored in the settings as `install_dir` - ##### Example: + ##### Example ```toml [resources.install_dir] # (empty - defaults are usually okay) ``` - ##### Properties: + ##### Properties - `dir`: (default: `/var/www/__APP__`) The full path of the install dir - `owner`: (default: `__APP__:rwx`) The owner (and owner permissions) for the install dir - `group`: (default: `__APP__:rx`) The group (and group permissions) for the install dir - ##### Provision/Update: + ##### Provision/Update - during install, the folder will be deleted if it already exists (FIXME: is this what we want?) - if the dir path changed and a folder exists at the old location, the folder will be `mv`'ed to the new location - otherwise, creates the directory if it doesn't exists yet - (re-)apply permissions (only on the folder itself, not recursively) - save the value of `dir` as `install_dir` in the app's settings, which can be then used by the app scripts (`$install_dir`) and conf templates (`__INSTALL_DIR__`) - ##### Deprovision: + ##### Deprovision - recursively deletes the directory if it exists - ##### Legacy management: + ##### Legacy management - In the past, the setting was called `final_path`. The code will automatically rename it as `install_dir`. - As explained in the 'Provision/Update' section, the folder will also be moved if the location changed @@ -850,28 +870,28 @@ class DatadirAppResource(AppResource): """ Creates a directory to be used by the app as the data store directory, typically where the app multimedia or large assets added by users are located. The corresponding path is stored in the settings as `data_dir`. This resource behaves very similarly to install_dir. - ##### Example: + ##### Example ```toml [resources.data_dir] # (empty - defaults are usually okay) ``` - ##### Properties: + ##### Properties - `dir`: (default: `/home/yunohost.app/__APP__`) The full path of the data dir - `owner`: (default: `__APP__:rwx`) The owner (and owner permissions) for the data dir - `group`: (default: `__APP__:rx`) The group (and group permissions) for the data dir - ##### Provision/Update: + ##### Provision/Update - if the dir path changed and a folder exists at the old location, the folder will be `mv`'ed to the new location - otherwise, creates the directory if it doesn't exists yet - (re-)apply permissions (only on the folder itself, not recursively) - save the value of `dir` as `data_dir` in the app's settings, which can be then used by the app scripts (`$data_dir`) and conf templates (`__DATA_DIR__`) - ##### Deprovision: + ##### Deprovision - (only if the purge option is chosen by the user) recursively deletes the directory if it exists - also delete the corresponding setting - ##### Legacy management: + ##### Legacy management - In the past, the setting may have been called `datadir`. The code will automatically rename it as `data_dir`. - As explained in the 'Provision/Update' section, the folder will also be moved if the location changed @@ -952,7 +972,7 @@ class AptDependenciesAppResource(AppResource): """ Create a virtual package in apt, depending on the list of specified packages that the app needs. The virtual packages is called `$app-ynh-deps` (with `_` being replaced by `-` in the app name, see `ynh_install_app_dependencies`) - ##### Example: + ##### Example ```toml [resources.apt] packages = "nyancat, lolcat, sl" @@ -963,16 +983,16 @@ class AptDependenciesAppResource(AppResource): extras.yarn.packages = "yarn" ``` - ##### Properties: + ##### Properties - `packages`: Comma-separated list of packages to be installed via `apt` - `packages_from_raw_bash`: A multi-line bash snippet (using triple quotes as open/close) which should echo additional packages to be installed. Meant to be used for packages to be conditionally installed depending on architecture, debian version, install questions, or other logic. - `extras`: A dict of (repo, key, packages) corresponding to "extra" repositories to fetch dependencies from - ##### Provision/Update: + ##### Provision/Update - The code literally calls the bash helpers `ynh_install_app_dependencies` and `ynh_install_extra_app_dependencies`, similar to what happens in v1. - Note that when `packages` contains some phpX.Y-foobar dependencies, this will automagically define a `phpversion` setting equal to `X.Y` which can therefore be used in app scripts ($phpversion) or templates (`__PHPVERSION__`) - ##### Deprovision: + ##### Deprovision - The code literally calls the bash helper `ynh_remove_app_dependencies` """ @@ -1031,7 +1051,7 @@ class PortsResource(AppResource): Note that because multiple ports can be booked, each properties is prefixed by the name of the port. `main` is a special name and will correspond to the setting `$port`, whereas for example `xmpp_client` will correspond to the setting `$port_xmpp_client`. - ##### Example: + ##### Example ```toml [resources.ports] # (empty should be fine for most apps... though you can customize stuff if absolutely needed) @@ -1043,21 +1063,21 @@ class PortsResource(AppResource): xmpp_client.exposed = "TCP" # here, we're telling that the port needs to be publicly exposed on TCP on the firewall ``` - ##### Properties (for every port name): + ##### Properties (for every port name) - `default`: The prefered value for the port. If this port is already being used by another process right now, or is booked in another app's setting, the code will increment the value until it finds a free port and store that value as the setting. If no value is specified, a random value between 10000 and 60000 is used. - `exposed`: (default: `false`) Wether this port should be opened on the firewall and be publicly reachable. This should be kept to `false` for the majority of apps than only need a port for internal reverse-proxying! Possible values: `false`, `true`(=`Both`), `Both`, `TCP`, `UDP`. This will result in the port being opened on the firewall, and the diagnosis checking that a program answers on that port. - `fixed`: (default: `false`) Tells that the app absolutely needs the specific value provided in `default`, typically because it's needed for a specific protocol - ##### Provision/Update (for every port name): + ##### Provision/Update (for every port name) - If not already booked, look for a free port, starting with the `default` value (or a random value between 10000 and 60000 if no `default` set) - If `exposed` is not `false`, open the port in the firewall accordingly - otherwise make sure it's closed. - The value of the port is stored in the `$port` setting for the `main` port, or `$port_NAME` for other `NAME`s - ##### Deprovision: + ##### Deprovision - Close the ports on the firewall if relevant - Deletes all the port settings - ##### Legacy management: + ##### Legacy management - In the past, some settings may have been named `NAME_port` instead of `port_NAME`, in which case the code will automatically rename the old setting. """ @@ -1160,25 +1180,25 @@ class DatabaseAppResource(AppResource): NB2: no automagic migration will happen in an suddenly change `type` from `mysql` to `postgresql` or viceversa in its life - ##### Example: + ##### Example ```toml [resources.database] type = "mysql" # or : "postgresql". Only these two values are supported ``` - ##### Properties: + ##### Properties - `type`: The database type, either `mysql` or `postgresql` - ##### Provision/Update: + ##### Provision/Update - (Re)set the `$db_name` and `$db_user` settings with the sanitized app name (replacing `-` and `.` with `_`) - If `$db_pwd` doesn't already exists, pick a random database password and store it in that setting - If the database doesn't exists yet, create the SQL user and DB using `ynh_mysql_create_db` or `ynh_psql_create_db`. - ##### Deprovision: + ##### Deprovision - Drop the DB using `ynh_mysql_remove_db` or `ynh_psql_remove_db` - Deletes the `db_name`, `db_user` and `db_pwd` settings - ##### Legacy management: + ##### Legacy management - In the past, the sql passwords may have been named `mysqlpwd` or `psqlpwd`, in which case it will automatically be renamed as `db_pwd` """ From 306c5e0e102b7eed6eab713bb11de71c5c1054f0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 31 Mar 2023 20:11:25 +0200 Subject: [PATCH 27/43] app resources: add documentation about latest_github_commit strategy for source autoupdate + autoupdate.upstream --- src/utils/resources.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/utils/resources.py b/src/utils/resources.py index 4c7c09fd3..c8e11b990 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -336,6 +336,9 @@ class SourcesResource(AppResource): - `latest_github_release` : similar to `latest_github_tags`, but starting from the list of releases. Pre- or draft releases are ignored. Releases may have assets attached to them, in which case you can define: - `autoupdate.asset = "some regex"` (when there's only one asset to use). The regex is used to find the appropriate asset among the list of all assets - or several `autoupdate.asset.$arch = "some_regex"` (when the asset is arch-specific). The regex is used to find the appropriate asset for the specific arch among the list of assets + - `latest_github_commit` : will use the latest commit on github, and the corresponding tarball. If this is used for the 'main' source, it will also assume that the version is YYYY.MM.DD corresponding to the date of the commit. + + It is also possible to define `autoupdate.upstream` to use a different Git(hub) repository instead of the code repository from the upstream section of the manifest. This can be useful when, for example, the app uses other assets such as plugin from a different repository. ##### Provision/Update - For elements with `prefetch = true`, will download the asset (for the appropriate architecture) and store them in `/var/cache/yunohost/download/$app/$source_id`, to be later picked up by `ynh_setup_source`. (NB: this only happens during install and upgrade, not restore) From 4b46f3220168074598c238de8338c9bdc5478dd0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 31 Mar 2023 20:26:08 +0200 Subject: [PATCH 28/43] appv2: add support for subdirs property in data_dir --- src/utils/resources.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index c8e11b990..3ff3f40d1 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -881,13 +881,15 @@ class DatadirAppResource(AppResource): ##### Properties - `dir`: (default: `/home/yunohost.app/__APP__`) The full path of the data dir + - `subdirs`: (default: empty list) A list of subdirs to initialize inside the data dir. For example, `['foo', 'bar']` - `owner`: (default: `__APP__:rwx`) The owner (and owner permissions) for the data dir - `group`: (default: `__APP__:rx`) The group (and group permissions) for the data dir ##### Provision/Update - if the dir path changed and a folder exists at the old location, the folder will be `mv`'ed to the new location - otherwise, creates the directory if it doesn't exists yet - - (re-)apply permissions (only on the folder itself, not recursively) + - create each subdir declared and which do not exist already + - (re-)apply permissions (only on the folder itself and declared subdirs, not recursively) - save the value of `dir` as `data_dir` in the app's settings, which can be then used by the app scripts (`$data_dir`) and conf templates (`__DATA_DIR__`) ##### Deprovision @@ -910,11 +912,13 @@ class DatadirAppResource(AppResource): default_properties: Dict[str, Any] = { "dir": "/home/yunohost.app/__APP__", + "subdirs": [], "owner": "__APP__:rwx", "group": "__APP__:rx", } dir: str = "" + subdirs: list = [] owner: str = "" group: str = "" @@ -938,6 +942,11 @@ class DatadirAppResource(AppResource): else: mkdir(self.dir) + for subdir in self.subdirs: + full_path = os.path.join(self.dir, subdir) + if not os.path.isdir(full_path): + mkdir(full_path) + owner, owner_perm = self.owner.split(":") group, group_perm = self.group.split(":") owner_perm_octal = ( @@ -956,6 +965,10 @@ class DatadirAppResource(AppResource): # in which case we want to apply the perm to the pointed dir, not to the symlink chmod(os.path.realpath(self.dir), perm_octal) chown(os.path.realpath(self.dir), owner, group) + for subdir in self.subdirs: + full_path = os.path.join(self.dir, subdir) + chmod(os.path.realpath(full_path), perm_octal) + chown(os.path.realpath(full_path), owner, group) self.set_setting("data_dir", self.dir) self.delete_setting("datadir") # Legacy From 821aedefa70ecfdc54378bfd4926633e77dc975f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 31 Mar 2023 20:45:14 +0200 Subject: [PATCH 29/43] users: fix quota parsing being wrong by a factor 1000 ... doveadm returns kilos, not bytes --- src/user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/user.py b/src/user.py index 12f13f75c..f17a60942 100644 --- a/src/user.py +++ b/src/user.py @@ -631,7 +631,7 @@ def user_info(username): has_value = re.search(r"Value=(\d+)", cmd_result) if has_value: - storage_use = int(has_value.group(1)) + storage_use = int(has_value.group(1)) * 1000 storage_use = binary_to_human(storage_use) if is_limited: From 14bf2ee48b113efd66b4c2b91992bd9dd6c978cb Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 2 Apr 2023 20:28:29 +0200 Subject: [PATCH 30/43] appsv2: various fixes regarding sources toml parsing/caching --- helpers/utils | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/helpers/utils b/helpers/utils index 97bd8e6b5..d27b5bca2 100644 --- a/helpers/utils +++ b/helpers/utils @@ -22,7 +22,10 @@ YNH_APP_BASEDIR=${YNH_APP_BASEDIR:-$(realpath ..)} ynh_exit_properly() { local exit_code=$? - rm -rf "/var/cache/yunohost/download/" + if [[ "${YNH_APP_ACTION}" =~ ^install$|^upgrade$|^restore$ ]] + then + rm -rf "/var/cache/yunohost/download/" + fi if [ "$exit_code" -eq 0 ]; then exit 0 # Exit without error if the script ended correctly @@ -164,22 +167,22 @@ ynh_setup_source() { if test -e $YNH_APP_BASEDIR/manifest.toml && cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq -e '.resources.sources' >/dev/null then source_id="${source_id:-main}" - local sources_json=$(cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq '.resources.sources') - if ! echo "$sources_json" | jq -re ".$source_id.url" + local sources_json=$(cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq ".resources.sources[\"$source_id\"]") + if jq -re ".url" <<< "$sources_json" then - local arch_prefix=".$YNH_ARCH" - else local arch_prefix="" + else + local arch_prefix=".$YNH_ARCH" fi - local src_url="$(echo "$sources_json" | jq -r ".$source_id$arch_prefix.url" | sed 's/^null$//')" - local src_sum="$(echo "$sources_json" | jq -r ".$source_id$arch_prefix.sha256" | sed 's/^null$//')" + local src_url="$(jq -r "$arch_prefix.url" <<< "$sources_json" | sed 's/^null$//')" + local src_sum="$(jq -r "$arch_prefix.sha256" <<< "$sources_json" | sed 's/^null$//')" local src_sumprg="sha256sum" - local src_format="$(echo "$sources_json" | jq -r ".$source_id.format" | sed 's/^null$//')" - local src_in_subdir="$(echo "$sources_json" | jq -r ".$source_id.in_subdir" | sed 's/^null$//')" - local src_extract="$(echo "$sources_json" | jq -r ".$source_id.extract" | sed 's/^null$//')" - local src_platform="$(echo "$sources_json" | jq -r ".$source_id.platform" | sed 's/^null$//')" - local src_rename="$(echo "$sources_json" | jq -r ".$source_id.rename" | sed 's/^null$//')" + local src_format="$(jq -r ".format" <<< "$sources_json" | sed 's/^null$//')" + local src_in_subdir="$(jq -r ".in_subdir" <<< "$sources_json" | sed 's/^null$//')" + local src_extract="$(jq -r ".extract" <<< "$sources_json" | sed 's/^null$//')" + local src_platform="$(jq -r ".platform" <<< "$sources_json" | sed 's/^null$//')" + local src_rename="$(jq -r ".rename" <<< "$sources_json" | sed 's/^null$//')" [[ -n "$src_url" ]] || ynh_die "No URL defined for source $source_id$arch_prefix ?" [[ -n "$src_sum" ]] || ynh_die "No sha256 sum defined for source $source_id$arch_prefix ?" @@ -236,8 +239,8 @@ ynh_setup_source() { local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${source_id}" # Gotta use this trick with 'dirname' because source_id may contain slashes x_x - mkdir -p $(dirname /var/cache/yunohost/download/${YNH_APP_ID}/${source_id}) - src_filename="/var/cache/yunohost/download/${YNH_APP_ID}/${source_id}" + mkdir -p $(dirname /var/cache/yunohost/download/${YNH_APP_INSTANCE_NAME}/${source_id}) + src_filename="/var/cache/yunohost/download/${YNH_APP_INSTANCE_NAME}/${source_id}" if [ "$src_format" = "docker" ]; then src_platform="${src_platform:-"linux/$YNH_ARCH"}" From 85a4b78e492306948a9791a020ca0240001be179 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 2 Apr 2023 20:32:17 +0200 Subject: [PATCH 31/43] Update changelog for 11.1.16 --- debian/changelog | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/debian/changelog b/debian/changelog index 0373a10b8..3c0cccbc2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,19 @@ +yunohost (11.1.16) stable; urgency=low + + - apps: fix i18n panel+section names ([#1630](https://github.com/yunohost/yunohost/pull/1630)) + - appsv2: don't remove yhh-deps virtual package if it doesn't exist. Otherwise when apt fails to install dependency, we end up with another error about failing to remove the ynh-deps package (3656c199) + - appsv2: add validation for expected types for permissions stuff (b2596f32) + - appsv2: add support for subdirs property in data_dir (4b46f322) + - appsv2: various fixes regarding sources toml parsing/caching (14bf2ee4) + - appsv2: add documentation about the new 'autoupdate' mechanism for app sources (63981aac) + - ynh_setup_source: fix buggy checksum mismatch handling, can't compute the sha256sum after we delete the file @_@ (1b2fa91f) + - users: fix quota parsing being wrong by a factor 1000 ... doveadm returns kilos, not bytes (821aedef) + - backup: fix boring issue where archive is a broken symlink... (a95d10e5) + + Thanks to all contributors <3 ! (axolotle) + + -- Alexandre Aubin Sun, 02 Apr 2023 20:29:33 +0200 + yunohost (11.1.15) stable; urgency=low - doc: Fix version number in autogenerated resource doc (5b58e0e6) From 4e799bfbc3d73aff82e3c76354630a3b2b4248e7 Mon Sep 17 00:00:00 2001 From: yunohost-bot Date: Sun, 2 Apr 2023 18:52:32 +0000 Subject: [PATCH 32/43] [CI] Format code with Black --- src/utils/config.py | 4 +++- src/utils/resources.py | 29 ++++++++++++++++++++++------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/utils/config.py b/src/utils/config.py index 7b16d6a23..d5bec7731 100644 --- a/src/utils/config.py +++ b/src/utils/config.py @@ -581,7 +581,9 @@ class ConfigPanel: logger.warning(f"Unknown key '{key}' found in config panel") # Todo search all i18n keys out[key] = ( - value if key not in ["ask", "help", "name"] or isinstance(value, dict) else {"en": value} + value + if key not in ["ask", "help", "name"] or isinstance(value, dict) + else {"en": value} ) return out diff --git a/src/utils/resources.py b/src/utils/resources.py index 3ff3f40d1..8f8393e17 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -520,21 +520,36 @@ class PermissionsResource(AppResource): properties["main"] = self.default_perm_properties for perm, infos in properties.items(): - if "auth_header" in infos and not isinstance(infos.get("auth_header"), bool): - raise YunohostError(f"In manifest, for permission '{perm}', 'auth_header' should be a boolean", raw_msg=True) + if "auth_header" in infos and not isinstance( + infos.get("auth_header"), bool + ): + raise YunohostError( + f"In manifest, for permission '{perm}', 'auth_header' should be a boolean", + raw_msg=True, + ) if "show_tile" in infos and not isinstance(infos.get("show_tile"), bool): - raise YunohostError(f"In manifest, for permission '{perm}', 'show_tile' should be a boolean", raw_msg=True) + raise YunohostError( + f"In manifest, for permission '{perm}', 'show_tile' should be a boolean", + raw_msg=True, + ) if "protected" in infos and not isinstance(infos.get("protected"), bool): - raise YunohostError(f"In manifest, for permission '{perm}', 'protected' should be a boolean", raw_msg=True) - if "additional_urls" in infos and not isinstance(infos.get("additional_urls"), list): - raise YunohostError(f"In manifest, for permission '{perm}', 'additional_urls' should be a list", raw_msg=True) + raise YunohostError( + f"In manifest, for permission '{perm}', 'protected' should be a boolean", + raw_msg=True, + ) + if "additional_urls" in infos and not isinstance( + infos.get("additional_urls"), list + ): + raise YunohostError( + f"In manifest, for permission '{perm}', 'additional_urls' should be a list", + raw_msg=True, + ) properties[perm] = copy.copy(self.default_perm_properties) properties[perm].update(infos) if properties[perm]["show_tile"] is None: properties[perm]["show_tile"] = bool(properties[perm]["url"]) - if properties["main"]["url"] is not None and ( not isinstance(properties["main"].get("url"), str) or properties["main"]["url"] != "/" From a16a164e20183584451d35ad6dacdab7b1965c7d Mon Sep 17 00:00:00 2001 From: Kayou Date: Tue, 4 Apr 2023 11:36:35 +0200 Subject: [PATCH 33/43] Fix autodns for gandi root domain --- src/dns.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/dns.py b/src/dns.py index 3a5e654ec..5fa58fb71 100644 --- a/src/dns.py +++ b/src/dns.py @@ -960,6 +960,9 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= f"Pushing {record['type']} records is not properly supported by Lexicon/Godaddy." ) continue + else if registrar == "gandi": + if record["name"] == base_dns_zone: + record["name"] = "@." + record["name"] record["action"] = action query = ( From 74213c6ce9a8f7dea09e281ad19eeb06e5df7832 Mon Sep 17 00:00:00 2001 From: Kayou Date: Tue, 4 Apr 2023 11:40:02 +0200 Subject: [PATCH 34/43] Typo --- src/dns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dns.py b/src/dns.py index 5fa58fb71..e3a26044c 100644 --- a/src/dns.py +++ b/src/dns.py @@ -960,7 +960,7 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= f"Pushing {record['type']} records is not properly supported by Lexicon/Godaddy." ) continue - else if registrar == "gandi": + elif registrar == "gandi": if record["name"] == base_dns_zone: record["name"] = "@." + record["name"] From b5f36626277f40295e2a32b2489ba8ca262d31e9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 4 Apr 2023 13:01:25 +0200 Subject: [PATCH 35/43] Misc syntax --- src/utils/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/config.py b/src/utils/config.py index 37f41f8b2..314f72ce7 100644 --- a/src/utils/config.py +++ b/src/utils/config.py @@ -1506,7 +1506,7 @@ class FileQuestion(Question): super()._prevalidate() # Validation should have already failed if required - if self.value in (None, ""): + if self.value in [None, ""]: return self.value if Moulinette.interface.type != "api": From 9c6a7fdf040e77b1f358c82050f02de4893977a6 Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 4 Apr 2023 15:43:46 +0200 Subject: [PATCH 36/43] mv config.py to form.py --- src/utils/{config.py => form.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/utils/{config.py => form.py} (100%) diff --git a/src/utils/config.py b/src/utils/form.py similarity index 100% rename from src/utils/config.py rename to src/utils/form.py From d8cb2139a9c2bdb9e449d631ac668be4823eda0c Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 4 Apr 2023 15:50:56 +0200 Subject: [PATCH 37/43] remove ConfigPanel code from form.py --- src/utils/form.py | 656 +--------------------------------------------- 1 file changed, 1 insertion(+), 655 deletions(-) diff --git a/src/utils/form.py b/src/utils/form.py index a48883c38..9907dafb1 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -16,7 +16,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . # -import glob import os import re import urllib.parse @@ -24,7 +23,6 @@ import tempfile import shutil import ast import operator as op -from collections import OrderedDict from typing import Optional, Dict, List, Union, Any, Mapping, Callable from moulinette.interfaces.cli import colorize @@ -33,18 +31,13 @@ from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import ( read_file, write_to_file, - read_toml, - read_yaml, - write_to_yaml, - mkdir, ) from yunohost.utils.i18n import _value_for_locale from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.log import OperationLogger -logger = getActionLogger("yunohost.config") -CONFIG_PANEL_VERSION_SUPPORTED = 1.0 +logger = getActionLogger("yunohost.form") # Those js-like evaluate functions are used to eval safely visible attributes @@ -190,653 +183,6 @@ def evaluate_simple_js_expression(expr, context={}): return evaluate_simple_ast(node, context) -class ConfigPanel: - entity_type = "config" - save_path_tpl: Union[str, None] = None - config_path_tpl = "/usr/share/yunohost/config_{entity_type}.toml" - save_mode = "full" - - @classmethod - def list(cls): - """ - List available config panel - """ - try: - entities = [ - re.match( - "^" + cls.save_path_tpl.format(entity="(?p)") + "$", f - ).group("entity") - for f in glob.glob(cls.save_path_tpl.format(entity="*")) - if os.path.isfile(f) - ] - except FileNotFoundError: - entities = [] - return entities - - def __init__(self, entity, config_path=None, save_path=None, creation=False): - self.entity = entity - self.config_path = config_path - if not config_path: - self.config_path = self.config_path_tpl.format( - entity=entity, entity_type=self.entity_type - ) - self.save_path = save_path - if not save_path and self.save_path_tpl: - self.save_path = self.save_path_tpl.format(entity=entity) - self.config = {} - self.values = {} - self.new_values = {} - - if ( - self.save_path - and self.save_mode != "diff" - and not creation - and not os.path.exists(self.save_path) - ): - raise YunohostValidationError( - f"{self.entity_type}_unknown", **{self.entity_type: entity} - ) - if self.save_path and creation and os.path.exists(self.save_path): - raise YunohostValidationError( - f"{self.entity_type}_exists", **{self.entity_type: entity} - ) - - # Search for hooks in the config panel - self.hooks = { - func: getattr(self, func) - for func in dir(self) - if callable(getattr(self, func)) - and re.match("^(validate|post_ask)__", func) - } - - def get(self, key="", mode="classic"): - self.filter_key = key or "" - - # Read config panel toml - self._get_config_panel() - - if not self.config: - raise YunohostValidationError("config_no_panel") - - # Read or get values and hydrate the config - self._load_current_values() - self._hydrate() - - # In 'classic' mode, we display the current value if key refer to an option - if self.filter_key.count(".") == 2 and mode == "classic": - option = self.filter_key.split(".")[-1] - value = self.values.get(option, None) - - option_type = None - for _, _, option_ in self._iterate(): - if option_["id"] == option: - option_type = ARGUMENTS_TYPE_PARSERS[option_["type"]] - break - - return option_type.normalize(value) if option_type else value - - # Format result in 'classic' or 'export' mode - logger.debug(f"Formating result in '{mode}' mode") - result = {} - for panel, section, option in self._iterate(): - if section["is_action_section"] and mode != "full": - continue - - key = f"{panel['id']}.{section['id']}.{option['id']}" - if mode == "export": - result[option["id"]] = option.get("current_value") - continue - - ask = None - if "ask" in option: - ask = _value_for_locale(option["ask"]) - elif "i18n" in self.config: - ask = m18n.n(self.config["i18n"] + "_" + option["id"]) - - if mode == "full": - option["ask"] = ask - question_class = ARGUMENTS_TYPE_PARSERS[option.get("type", "string")] - # FIXME : maybe other properties should be taken from the question, not just choices ?. - option["choices"] = question_class(option).choices - option["default"] = question_class(option).default - option["pattern"] = question_class(option).pattern - else: - result[key] = {"ask": ask} - if "current_value" in option: - question_class = ARGUMENTS_TYPE_PARSERS[ - option.get("type", "string") - ] - result[key]["value"] = question_class.humanize( - option["current_value"], option - ) - # FIXME: semantics, technically here this is not about a prompt... - if question_class.hide_user_input_in_prompt: - result[key][ - "value" - ] = "**************" # Prevent displaying password in `config get` - - if mode == "full": - return self.config - else: - return result - - def list_actions(self): - actions = {} - - # FIXME : meh, loading the entire config panel is again going to cause - # stupid issues for domain (e.g loading registrar stuff when willing to just list available actions ...) - self.filter_key = "" - self._get_config_panel() - for panel, section, option in self._iterate(): - if option["type"] == "button": - key = f"{panel['id']}.{section['id']}.{option['id']}" - actions[key] = _value_for_locale(option["ask"]) - - return actions - - def run_action(self, action=None, args=None, args_file=None, operation_logger=None): - # - # FIXME : this stuff looks a lot like set() ... - # - - self.filter_key = ".".join(action.split(".")[:2]) - action_id = action.split(".")[2] - - # Read config panel toml - self._get_config_panel() - - # FIXME: should also check that there's indeed a key called action - if not self.config: - raise YunohostValidationError(f"No action named {action}", raw_msg=True) - - # Import and parse pre-answered options - logger.debug("Import and parse pre-answered options") - self._parse_pre_answered(args, None, args_file) - - # Read or get values and hydrate the config - self._load_current_values() - self._hydrate() - Question.operation_logger = operation_logger - self._ask(action=action_id) - - # FIXME: here, we could want to check constrains on - # the action's visibility / requirements wrt to the answer to questions ... - - if operation_logger: - operation_logger.start() - - try: - self._run_action(action_id) - except YunohostError: - raise - # Script got manually interrupted ... - # N.B. : KeyboardInterrupt does not inherit from Exception - except (KeyboardInterrupt, EOFError): - error = m18n.n("operation_interrupted") - logger.error(m18n.n("config_action_failed", action=action, error=error)) - raise - # Something wrong happened in Yunohost's code (most probably hook_exec) - except Exception: - import traceback - - error = m18n.n("unexpected_error", error="\n" + traceback.format_exc()) - logger.error(m18n.n("config_action_failed", action=action, error=error)) - raise - finally: - # Delete files uploaded from API - # FIXME : this is currently done in the context of config panels, - # but could also happen in the context of app install ... (or anywhere else - # where we may parse args etc...) - FileQuestion.clean_upload_dirs() - - # FIXME: i18n - logger.success(f"Action {action_id} successful") - operation_logger.success() - - def set( - self, key=None, value=None, args=None, args_file=None, operation_logger=None - ): - self.filter_key = key or "" - - # Read config panel toml - self._get_config_panel() - - if not self.config: - raise YunohostValidationError("config_no_panel") - - if (args is not None or args_file is not None) and value is not None: - raise YunohostValidationError( - "You should either provide a value, or a serie of args/args_file, but not both at the same time", - raw_msg=True, - ) - - if self.filter_key.count(".") != 2 and value is not None: - raise YunohostValidationError("config_cant_set_value_on_section") - - # Import and parse pre-answered options - logger.debug("Import and parse pre-answered options") - self._parse_pre_answered(args, value, args_file) - - # Read or get values and hydrate the config - self._load_current_values() - self._hydrate() - Question.operation_logger = operation_logger - self._ask() - - if operation_logger: - operation_logger.start() - - try: - self._apply() - except YunohostError: - raise - # Script got manually interrupted ... - # N.B. : KeyboardInterrupt does not inherit from Exception - except (KeyboardInterrupt, EOFError): - error = m18n.n("operation_interrupted") - logger.error(m18n.n("config_apply_failed", error=error)) - raise - # Something wrong happened in Yunohost's code (most probably hook_exec) - except Exception: - import traceback - - error = m18n.n("unexpected_error", error="\n" + traceback.format_exc()) - logger.error(m18n.n("config_apply_failed", error=error)) - raise - finally: - # Delete files uploaded from API - # FIXME : this is currently done in the context of config panels, - # but could also happen in the context of app install ... (or anywhere else - # where we may parse args etc...) - FileQuestion.clean_upload_dirs() - - self._reload_services() - - logger.success("Config updated as expected") - operation_logger.success() - - def _get_toml(self): - return read_toml(self.config_path) - - def _get_config_panel(self): - # Split filter_key - filter_key = self.filter_key.split(".") if self.filter_key != "" else [] - if len(filter_key) > 3: - raise YunohostError( - f"The filter key {filter_key} has too many sub-levels, the max is 3.", - raw_msg=True, - ) - - if not os.path.exists(self.config_path): - logger.debug(f"Config panel {self.config_path} doesn't exists") - return None - - toml_config_panel = self._get_toml() - - # Check TOML config panel is in a supported version - if float(toml_config_panel["version"]) < CONFIG_PANEL_VERSION_SUPPORTED: - logger.error( - f"Config panels version {toml_config_panel['version']} are not supported" - ) - return None - - # Transform toml format into internal format - format_description = { - "root": { - "properties": ["version", "i18n"], - "defaults": {"version": 1.0}, - }, - "panels": { - "properties": ["name", "services", "actions", "help"], - "defaults": { - "services": [], - "actions": {"apply": {"en": "Apply"}}, - }, - }, - "sections": { - "properties": ["name", "services", "optional", "help", "visible"], - "defaults": { - "name": "", - "services": [], - "optional": True, - "is_action_section": False, - }, - }, - "options": { - "properties": [ - "ask", - "type", - "bind", - "help", - "example", - "default", - "style", - "icon", - "placeholder", - "visible", - "optional", - "choices", - "yes", - "no", - "pattern", - "limit", - "min", - "max", - "step", - "accept", - "redact", - "filter", - "readonly", - "enabled", - # "confirm", # TODO: to ask confirmation before running an action - ], - "defaults": {}, - }, - } - - def _build_internal_config_panel(raw_infos, level): - """Convert TOML in internal format ('full' mode used by webadmin) - Here are some properties of 1.0 config panel in toml: - - node properties and node children are mixed, - - text are in english only - - some properties have default values - This function detects all children nodes and put them in a list - """ - - defaults = format_description[level]["defaults"] - properties = format_description[level]["properties"] - - # Start building the ouput (merging the raw infos + defaults) - out = {key: raw_infos.get(key, value) for key, value in defaults.items()} - - # Now fill the sublevels (+ apply filter_key) - i = list(format_description).index(level) - sublevel = list(format_description)[i + 1] if level != "options" else None - search_key = filter_key[i] if len(filter_key) > i else False - - for key, value in raw_infos.items(): - # Key/value are a child node - if ( - isinstance(value, OrderedDict) - and key not in properties - and sublevel - ): - # We exclude all nodes not referenced by the filter_key - if search_key and key != search_key: - continue - subnode = _build_internal_config_panel(value, sublevel) - subnode["id"] = key - if level == "root": - subnode.setdefault("name", {"en": key.capitalize()}) - elif level == "sections": - subnode["name"] = key # legacy - subnode.setdefault("optional", raw_infos.get("optional", True)) - # If this section contains at least one button, it becomes an "action" section - if subnode.get("type") == "button": - out["is_action_section"] = True - out.setdefault(sublevel, []).append(subnode) - # Key/value are a property - else: - if key not in properties: - logger.warning(f"Unknown key '{key}' found in config panel") - # Todo search all i18n keys - out[key] = ( - value - if key not in ["ask", "help", "name"] or isinstance(value, dict) - else {"en": value} - ) - return out - - self.config = _build_internal_config_panel(toml_config_panel, "root") - - try: - self.config["panels"][0]["sections"][0]["options"][0] - except (KeyError, IndexError): - raise YunohostValidationError( - "config_unknown_filter_key", filter_key=self.filter_key - ) - - # List forbidden keywords from helpers and sections toml (to avoid conflict) - forbidden_keywords = [ - "old", - "app", - "changed", - "file_hash", - "binds", - "types", - "formats", - "getter", - "setter", - "short_setting", - "type", - "bind", - "nothing_changed", - "changes_validated", - "result", - "max_progression", - ] - forbidden_keywords += format_description["sections"] - forbidden_readonly_types = ["password", "app", "domain", "user", "file"] - - for _, _, option in self._iterate(): - if option["id"] in forbidden_keywords: - raise YunohostError("config_forbidden_keyword", keyword=option["id"]) - if ( - option.get("readonly", False) - and option.get("type", "string") in forbidden_readonly_types - ): - raise YunohostError( - "config_forbidden_readonly_type", - type=option["type"], - id=option["id"], - ) - - return self.config - - def _hydrate(self): - # Hydrating config panel with current value - for _, section, option in self._iterate(): - if option["id"] not in self.values: - allowed_empty_types = [ - "alert", - "display_text", - "markdown", - "file", - "button", - ] - - if section["is_action_section"] and option.get("default") is not None: - self.values[option["id"]] = option["default"] - elif ( - option["type"] in allowed_empty_types - or option.get("bind") == "null" - ): - continue - else: - raise YunohostError( - f"Config panel question '{option['id']}' should be initialized with a value during install or upgrade.", - raw_msg=True, - ) - value = self.values[option["name"]] - - # Allow to use value instead of current_value in app config script. - # e.g. apps may write `echo 'value: "foobar"'` in the config file (which is more intuitive that `echo 'current_value: "foobar"'` - # For example hotspot used it... - # See https://github.com/YunoHost/yunohost/pull/1546 - if ( - isinstance(value, dict) - and "value" in value - and "current_value" not in value - ): - value["current_value"] = value["value"] - - # In general, the value is just a simple value. - # Sometimes it could be a dict used to overwrite the option itself - value = value if isinstance(value, dict) else {"current_value": value} - option.update(value) - - self.values[option["id"]] = value.get("current_value") - - return self.values - - def _ask(self, action=None): - logger.debug("Ask unanswered question and prevalidate data") - - if "i18n" in self.config: - for panel, section, option in self._iterate(): - if "ask" not in option: - option["ask"] = m18n.n(self.config["i18n"] + "_" + option["id"]) - # auto add i18n help text if present in locales - if m18n.key_exists(self.config["i18n"] + "_" + option["id"] + "_help"): - option["help"] = m18n.n( - self.config["i18n"] + "_" + option["id"] + "_help" - ) - - def display_header(message): - """CLI panel/section header display""" - if Moulinette.interface.type == "cli" and self.filter_key.count(".") < 2: - Moulinette.display(colorize(message, "purple")) - - for panel, section, obj in self._iterate(["panel", "section"]): - if ( - section - and section.get("visible") - and not evaluate_simple_js_expression( - section["visible"], context=self.future_values - ) - ): - continue - - # Ugly hack to skip action section ... except when when explicitly running actions - if not action: - if section and section["is_action_section"]: - continue - - if panel == obj: - name = _value_for_locale(panel["name"]) - display_header(f"\n{'='*40}\n>>>> {name}\n{'='*40}") - else: - name = _value_for_locale(section["name"]) - if name: - display_header(f"\n# {name}") - elif section: - # filter action section options in case of multiple buttons - section["options"] = [ - option - for option in section["options"] - if option.get("type", "string") != "button" - or option["id"] == action - ] - - if panel == obj: - continue - - # Check and ask unanswered questions - prefilled_answers = self.args.copy() - prefilled_answers.update(self.new_values) - - questions = ask_questions_and_parse_answers( - {question["name"]: question for question in section["options"]}, - prefilled_answers=prefilled_answers, - current_values=self.values, - hooks=self.hooks, - ) - self.new_values.update( - { - question.name: question.value - for question in questions - if question.value is not None - } - ) - - def _get_default_values(self): - return { - option["id"]: option["default"] - for _, _, option in self._iterate() - if "default" in option - } - - @property - def future_values(self): - return {**self.values, **self.new_values} - - def __getattr__(self, name): - if "new_values" in self.__dict__ and name in self.new_values: - return self.new_values[name] - - if "values" in self.__dict__ and name in self.values: - return self.values[name] - - return self.__dict__[name] - - def _load_current_values(self): - """ - Retrieve entries in YAML file - And set default values if needed - """ - - # Inject defaults if needed (using the magic .update() ;)) - self.values = self._get_default_values() - - # Retrieve entries in the YAML - if os.path.exists(self.save_path) and os.path.isfile(self.save_path): - self.values.update(read_yaml(self.save_path) or {}) - - def _parse_pre_answered(self, args, value, args_file): - args = urllib.parse.parse_qs(args or "", keep_blank_values=True) - self.args = {key: ",".join(value_) for key, value_ in args.items()} - - if args_file: - # Import YAML / JSON file but keep --args values - self.args = {**read_yaml(args_file), **self.args} - - if value is not None: - self.args = {self.filter_key.split(".")[-1]: value} - - def _apply(self): - logger.info("Saving the new configuration...") - dir_path = os.path.dirname(os.path.realpath(self.save_path)) - if not os.path.exists(dir_path): - mkdir(dir_path, mode=0o700) - - values_to_save = self.future_values - if self.save_mode == "diff": - defaults = self._get_default_values() - values_to_save = { - k: v for k, v in values_to_save.items() if defaults.get(k) != v - } - - # Save the settings to the .yaml file - write_to_yaml(self.save_path, values_to_save) - - def _reload_services(self): - from yunohost.service import service_reload_or_restart - - services_to_reload = set() - for panel, section, obj in self._iterate(["panel", "section", "option"]): - services_to_reload |= set(obj.get("services", [])) - - services_to_reload = list(services_to_reload) - services_to_reload.sort(key="nginx".__eq__) - if services_to_reload: - logger.info("Reloading services...") - for service in services_to_reload: - if hasattr(self, "entity"): - service = service.replace("__APP__", self.entity) - service_reload_or_restart(service) - - def _iterate(self, trigger=["option"]): - for panel in self.config.get("panels", []): - if "panel" in trigger: - yield (panel, None, panel) - for section in panel.get("sections", []): - if "section" in trigger: - yield (panel, section, section) - if "option" in trigger: - for option in section.get("options", []): - yield (panel, section, option) - - class Question: hide_user_input_in_prompt = False pattern: Optional[Dict] = None From 478291766e637c6f6c2e3ab50d1fcf7013038575 Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 4 Apr 2023 15:51:55 +0200 Subject: [PATCH 38/43] mv config.py to configpanel.py --- src/utils/{config.py => configpanel.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/utils/{config.py => configpanel.py} (100%) diff --git a/src/utils/config.py b/src/utils/configpanel.py similarity index 100% rename from src/utils/config.py rename to src/utils/configpanel.py From b688944d117fc33e044dba00ae6524875c1b0a0e Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 4 Apr 2023 15:54:28 +0200 Subject: [PATCH 39/43] remove form related code from configpanel.py --- src/utils/configpanel.py | 976 +-------------------------------------- 1 file changed, 2 insertions(+), 974 deletions(-) diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index a48883c38..1f1351bcb 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -20,19 +20,13 @@ import glob import os import re import urllib.parse -import tempfile -import shutil -import ast -import operator as op from collections import OrderedDict -from typing import Optional, Dict, List, Union, Any, Mapping, Callable +from typing import Union from moulinette.interfaces.cli import colorize from moulinette import Moulinette, m18n from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import ( - read_file, - write_to_file, read_toml, read_yaml, write_to_yaml, @@ -41,155 +35,11 @@ from moulinette.utils.filesystem import ( from yunohost.utils.i18n import _value_for_locale from yunohost.utils.error import YunohostError, YunohostValidationError -from yunohost.log import OperationLogger -logger = getActionLogger("yunohost.config") +logger = getActionLogger("yunohost.configpanel") CONFIG_PANEL_VERSION_SUPPORTED = 1.0 -# Those js-like evaluate functions are used to eval safely visible attributes -# The goal is to evaluate in the same way than js simple-evaluate -# https://github.com/shepherdwind/simple-evaluate -def evaluate_simple_ast(node, context=None): - if context is None: - context = {} - - operators = { - ast.Not: op.not_, - ast.Mult: op.mul, - ast.Div: op.truediv, # number - ast.Mod: op.mod, # number - ast.Add: op.add, # str - ast.Sub: op.sub, # number - ast.USub: op.neg, # Negative number - ast.Gt: op.gt, - ast.Lt: op.lt, - ast.GtE: op.ge, - ast.LtE: op.le, - ast.Eq: op.eq, - ast.NotEq: op.ne, - } - context["true"] = True - context["false"] = False - context["null"] = None - - # Variable - if isinstance(node, ast.Name): # Variable - return context[node.id] - - # Python <=3.7 String - elif isinstance(node, ast.Str): - return node.s - - # Python <=3.7 Number - elif isinstance(node, ast.Num): - return node.n - - # Boolean, None and Python 3.8 for Number, Boolean, String and None - elif isinstance(node, (ast.Constant, ast.NameConstant)): - return node.value - - # + - * / % - elif ( - isinstance(node, ast.BinOp) and type(node.op) in operators - ): # - left = evaluate_simple_ast(node.left, context) - right = evaluate_simple_ast(node.right, context) - if type(node.op) == ast.Add: - if isinstance(left, str) or isinstance(right, str): # support 'I am ' + 42 - left = str(left) - right = str(right) - elif type(left) != type(right): # support "111" - "1" -> 110 - left = float(left) - right = float(right) - - return operators[type(node.op)](left, right) - - # Comparison - # JS and Python don't give the same result for multi operators - # like True == 10 > 2. - elif ( - isinstance(node, ast.Compare) and len(node.comparators) == 1 - ): # - left = evaluate_simple_ast(node.left, context) - right = evaluate_simple_ast(node.comparators[0], context) - operator = node.ops[0] - if isinstance(left, (int, float)) or isinstance(right, (int, float)): - try: - left = float(left) - right = float(right) - except ValueError: - return type(operator) == ast.NotEq - try: - return operators[type(operator)](left, right) - except TypeError: # support "e" > 1 -> False like in JS - return False - - # and / or - elif isinstance(node, ast.BoolOp): # - for value in node.values: - value = evaluate_simple_ast(value, context) - if isinstance(node.op, ast.And) and not value: - return False - elif isinstance(node.op, ast.Or) and value: - return True - return isinstance(node.op, ast.And) - - # not / USub (it's negation number -\d) - elif isinstance(node, ast.UnaryOp): # e.g., -1 - return operators[type(node.op)](evaluate_simple_ast(node.operand, context)) - - # match function call - elif isinstance(node, ast.Call) and node.func.__dict__.get("id") == "match": - return re.match( - evaluate_simple_ast(node.args[1], context), context[node.args[0].id] - ) - - # Unauthorized opcode - else: - opcode = str(type(node)) - raise YunohostError( - f"Unauthorize opcode '{opcode}' in visible attribute", raw_msg=True - ) - - -def js_to_python(expr): - in_string = None - py_expr = "" - i = 0 - escaped = False - for char in expr: - if char in r"\"'": - # Start a string - if not in_string: - in_string = char - - # Finish a string - elif in_string == char and not escaped: - in_string = None - - # If we are not in a string, replace operators - elif not in_string: - if char == "!" and expr[i + 1] != "=": - char = "not " - elif char in "|&" and py_expr[-1:] == char: - py_expr = py_expr[:-1] - char = " and " if char == "&" else " or " - - # Determine if next loop will be in escaped mode - escaped = char == "\\" and not escaped - py_expr += char - i += 1 - return py_expr - - -def evaluate_simple_js_expression(expr, context={}): - if not expr.strip(): - return False - node = ast.parse(js_to_python(expr), mode="eval").body - return evaluate_simple_ast(node, context) - - class ConfigPanel: entity_type = "config" save_path_tpl: Union[str, None] = None @@ -835,825 +685,3 @@ class ConfigPanel: if "option" in trigger: for option in section.get("options", []): yield (panel, section, option) - - -class Question: - hide_user_input_in_prompt = False - pattern: Optional[Dict] = None - - def __init__( - self, - question: Dict[str, Any], - context: Mapping[str, Any] = {}, - hooks: Dict[str, Callable] = {}, - ): - self.name = question["name"] - self.context = context - self.hooks = hooks - self.type = question.get("type", "string") - self.default = question.get("default", None) - self.optional = question.get("optional", False) - self.visible = question.get("visible", None) - self.readonly = question.get("readonly", False) - # Don't restrict choices if there's none specified - self.choices = question.get("choices", None) - self.pattern = question.get("pattern", self.pattern) - self.ask = question.get("ask", self.name) - if not isinstance(self.ask, dict): - self.ask = {"en": self.ask} - self.help = question.get("help") - self.redact = question.get("redact", False) - self.filter = question.get("filter", None) - # .current_value is the currently stored value - self.current_value = question.get("current_value") - # .value is the "proposed" value which we got from the user - self.value = question.get("value") - # Use to return several values in case answer is in mutipart - self.values: Dict[str, Any] = {} - - # Empty value is parsed as empty string - if self.default == "": - self.default = None - - @staticmethod - def humanize(value, option={}): - return str(value) - - @staticmethod - def normalize(value, option={}): - if isinstance(value, str): - value = value.strip() - return value - - def _prompt(self, text): - prefill = "" - if self.current_value is not None: - prefill = self.humanize(self.current_value, self) - elif self.default is not None: - prefill = self.humanize(self.default, self) - self.value = Moulinette.prompt( - message=text, - is_password=self.hide_user_input_in_prompt, - confirm=False, - prefill=prefill, - is_multiline=(self.type == "text"), - autocomplete=self.choices or [], - help=_value_for_locale(self.help), - ) - - def ask_if_needed(self): - if self.visible and not evaluate_simple_js_expression( - self.visible, context=self.context - ): - # FIXME There could be several use case if the question is not displayed: - # - we doesn't want to give a specific value - # - we want to keep the previous value - # - we want the default value - self.value = self.values[self.name] = None - return self.values - - for i in range(5): - # Display question if no value filled or if it's a readonly message - if Moulinette.interface.type == "cli" and os.isatty(1): - text_for_user_input_in_cli = self._format_text_for_user_input_in_cli() - if self.readonly: - Moulinette.display(text_for_user_input_in_cli) - self.value = self.values[self.name] = self.current_value - return self.values - elif self.value is None: - self._prompt(text_for_user_input_in_cli) - - # Apply default value - class_default = getattr(self, "default_value", None) - if self.value in [None, ""] and ( - self.default is not None or class_default is not None - ): - self.value = class_default if self.default is None else self.default - - try: - # Normalize and validate - self.value = self.normalize(self.value, self) - self._prevalidate() - except YunohostValidationError as e: - # If in interactive cli, re-ask the current question - if i < 4 and Moulinette.interface.type == "cli" and os.isatty(1): - logger.error(str(e)) - self.value = None - continue - - # Otherwise raise the ValidationError - raise - - break - - self.value = self.values[self.name] = self._post_parse_value() - - # Search for post actions in hooks - post_hook = f"post_ask__{self.name}" - if post_hook in self.hooks: - self.values.update(self.hooks[post_hook](self)) - - return self.values - - def _prevalidate(self): - if self.value in [None, ""] and not self.optional: - raise YunohostValidationError("app_argument_required", name=self.name) - - # we have an answer, do some post checks - if self.value not in [None, ""]: - if self.choices and self.value not in self.choices: - raise YunohostValidationError( - "app_argument_choice_invalid", - name=self.name, - value=self.value, - choices=", ".join(str(choice) for choice in self.choices), - ) - if self.pattern and not re.match(self.pattern["regexp"], str(self.value)): - raise YunohostValidationError( - self.pattern["error"], - name=self.name, - value=self.value, - ) - - def _format_text_for_user_input_in_cli(self): - text_for_user_input_in_cli = _value_for_locale(self.ask) - - if self.readonly: - text_for_user_input_in_cli = colorize(text_for_user_input_in_cli, "purple") - if self.choices: - return ( - text_for_user_input_in_cli + f" {self.choices[self.current_value]}" - ) - return text_for_user_input_in_cli + f" {self.humanize(self.current_value)}" - elif self.choices: - # Prevent displaying a shitload of choices - # (e.g. 100+ available users when choosing an app admin...) - choices = ( - list(self.choices.keys()) - if isinstance(self.choices, dict) - else self.choices - ) - choices_to_display = choices[:20] - remaining_choices = len(choices[20:]) - - if remaining_choices > 0: - choices_to_display += [ - m18n.n("other_available_options", n=remaining_choices) - ] - - choices_to_display = " | ".join(choices_to_display) - - text_for_user_input_in_cli += f" [{choices_to_display}]" - - return text_for_user_input_in_cli - - def _post_parse_value(self): - if not self.redact: - return self.value - - # Tell the operation_logger to redact all password-type / secret args - # Also redact the % escaped version of the password that might appear in - # the 'args' section of metadata (relevant for password with non-alphanumeric char) - data_to_redact = [] - if self.value and isinstance(self.value, str): - data_to_redact.append(self.value) - if self.current_value and isinstance(self.current_value, str): - data_to_redact.append(self.current_value) - data_to_redact += [ - urllib.parse.quote(data) - for data in data_to_redact - if urllib.parse.quote(data) != data - ] - - for operation_logger in OperationLogger._instances: - operation_logger.data_to_redact.extend(data_to_redact) - - return self.value - - -class StringQuestion(Question): - argument_type = "string" - default_value = "" - - -class EmailQuestion(StringQuestion): - pattern = { - "regexp": r"^.+@.+", - "error": "config_validate_email", # i18n: config_validate_email - } - - -class URLQuestion(StringQuestion): - pattern = { - "regexp": r"^https?://.*$", - "error": "config_validate_url", # i18n: config_validate_url - } - - -class DateQuestion(StringQuestion): - pattern = { - "regexp": r"^\d{4}-\d\d-\d\d$", - "error": "config_validate_date", # i18n: config_validate_date - } - - def _prevalidate(self): - from datetime import datetime - - super()._prevalidate() - - if self.value not in [None, ""]: - try: - datetime.strptime(self.value, "%Y-%m-%d") - except ValueError: - raise YunohostValidationError("config_validate_date") - - -class TimeQuestion(StringQuestion): - pattern = { - "regexp": r"^(?:\d|[01]\d|2[0-3]):[0-5]\d$", - "error": "config_validate_time", # i18n: config_validate_time - } - - -class ColorQuestion(StringQuestion): - pattern = { - "regexp": r"^#[ABCDEFabcdef\d]{3,6}$", - "error": "config_validate_color", # i18n: config_validate_color - } - - -class TagsQuestion(Question): - argument_type = "tags" - default_value = "" - - @staticmethod - def humanize(value, option={}): - if isinstance(value, list): - return ",".join(str(v) for v in value) - return value - - @staticmethod - def normalize(value, option={}): - if isinstance(value, list): - return ",".join(str(v) for v in value) - if isinstance(value, str): - value = value.strip() - return value - - def _prevalidate(self): - values = self.value - if isinstance(values, str): - values = values.split(",") - elif values is None: - values = [] - - if not isinstance(values, list): - if self.choices: - raise YunohostValidationError( - "app_argument_choice_invalid", - name=self.name, - value=self.value, - choices=", ".join(str(choice) for choice in self.choices), - ) - raise YunohostValidationError( - "app_argument_invalid", - name=self.name, - error=f"'{str(self.value)}' is not a list", - ) - - for value in values: - self.value = value - super()._prevalidate() - self.value = values - - def _post_parse_value(self): - if isinstance(self.value, list): - self.value = ",".join(self.value) - return super()._post_parse_value() - - -class PasswordQuestion(Question): - hide_user_input_in_prompt = True - argument_type = "password" - default_value = "" - forbidden_chars = "{}" - - def __init__( - self, question, context: Mapping[str, Any] = {}, hooks: Dict[str, Callable] = {} - ): - super().__init__(question, context, hooks) - self.redact = True - if self.default is not None: - raise YunohostValidationError( - "app_argument_password_no_default", name=self.name - ) - - def _prevalidate(self): - super()._prevalidate() - - if self.value not in [None, ""]: - if any(char in self.value for char in self.forbidden_chars): - raise YunohostValidationError( - "pattern_password_app", forbidden_chars=self.forbidden_chars - ) - - # If it's an optional argument the value should be empty or strong enough - from yunohost.utils.password import assert_password_is_strong_enough - - assert_password_is_strong_enough("user", self.value) - - -class PathQuestion(Question): - argument_type = "path" - default_value = "" - - @staticmethod - def normalize(value, option={}): - option = option.__dict__ if isinstance(option, Question) else option - - if not isinstance(value, str): - raise YunohostValidationError( - "app_argument_invalid", - name=option.get("name"), - error="Argument for path should be a string.", - ) - - if not value.strip(): - if option.get("optional"): - return "" - # Hmpf here we could just have a "else" case - # but we also want PathQuestion.normalize("") to return "/" - # (i.e. if no option is provided, hence .get("optional") is None - elif option.get("optional") is False: - raise YunohostValidationError( - "app_argument_invalid", - name=option.get("name"), - error="Question is mandatory", - ) - - return "/" + value.strip().strip(" /") - - -class BooleanQuestion(Question): - argument_type = "boolean" - default_value = 0 - yes_answers = ["1", "yes", "y", "true", "t", "on"] - no_answers = ["0", "no", "n", "false", "f", "off"] - - @staticmethod - def humanize(value, option={}): - option = option.__dict__ if isinstance(option, Question) else option - - yes = option.get("yes", 1) - no = option.get("no", 0) - - value = BooleanQuestion.normalize(value, option) - - if value == yes: - return "yes" - if value == no: - return "no" - if value is None: - return "" - - raise YunohostValidationError( - "app_argument_choice_invalid", - name=option.get("name"), - value=value, - choices="yes/no", - ) - - @staticmethod - def normalize(value, option={}): - option = option.__dict__ if isinstance(option, Question) else option - - if isinstance(value, str): - value = value.strip() - - technical_yes = option.get("yes", 1) - technical_no = option.get("no", 0) - - no_answers = BooleanQuestion.no_answers - yes_answers = BooleanQuestion.yes_answers - - assert ( - str(technical_yes).lower() not in no_answers - ), f"'yes' value can't be in {no_answers}" - assert ( - str(technical_no).lower() not in yes_answers - ), f"'no' value can't be in {yes_answers}" - - no_answers += [str(technical_no).lower()] - yes_answers += [str(technical_yes).lower()] - - strvalue = str(value).lower() - - if strvalue in yes_answers: - return technical_yes - if strvalue in no_answers: - return technical_no - - if strvalue in ["none", ""]: - return None - - raise YunohostValidationError( - "app_argument_choice_invalid", - name=option.get("name"), - value=strvalue, - choices="yes/no", - ) - - def __init__( - self, question, context: Mapping[str, Any] = {}, hooks: Dict[str, Callable] = {} - ): - super().__init__(question, context, hooks) - self.yes = question.get("yes", 1) - self.no = question.get("no", 0) - if self.default is None: - self.default = self.no - - def _format_text_for_user_input_in_cli(self): - text_for_user_input_in_cli = super()._format_text_for_user_input_in_cli() - - if not self.readonly: - text_for_user_input_in_cli += " [yes | no]" - - return text_for_user_input_in_cli - - def get(self, key, default=None): - return getattr(self, key, default) - - -class DomainQuestion(Question): - argument_type = "domain" - - def __init__( - self, question, context: Mapping[str, Any] = {}, hooks: Dict[str, Callable] = {} - ): - from yunohost.domain import domain_list, _get_maindomain - - super().__init__(question, context, hooks) - - if self.default is None: - self.default = _get_maindomain() - - self.choices = { - domain: domain + " ★" if domain == self.default else domain - for domain in domain_list()["domains"] - } - - @staticmethod - def normalize(value, option={}): - if value.startswith("https://"): - value = value[len("https://") :] - elif value.startswith("http://"): - value = value[len("http://") :] - - # Remove trailing slashes - value = value.rstrip("/").lower() - - return value - - -class AppQuestion(Question): - argument_type = "app" - - def __init__( - self, question, context: Mapping[str, Any] = {}, hooks: Dict[str, Callable] = {} - ): - from yunohost.app import app_list - - super().__init__(question, context, hooks) - - apps = app_list(full=True)["apps"] - - if self.filter: - apps = [ - app - for app in apps - if evaluate_simple_js_expression(self.filter, context=app) - ] - - def _app_display(app): - domain_path_or_id = f" ({app.get('domain_path', app['id'])})" - return app["label"] + domain_path_or_id - - self.choices = {"_none": "---"} - self.choices.update({app["id"]: _app_display(app) for app in apps}) - - -class UserQuestion(Question): - argument_type = "user" - - def __init__( - self, question, context: Mapping[str, Any] = {}, hooks: Dict[str, Callable] = {} - ): - from yunohost.user import user_list, user_info - from yunohost.domain import _get_maindomain - - super().__init__(question, context, hooks) - - self.choices = { - username: f"{infos['fullname']} ({infos['mail']})" - for username, infos in user_list()["users"].items() - } - - if not self.choices: - raise YunohostValidationError( - "app_argument_invalid", - name=self.name, - error="You should create a YunoHost user first.", - ) - - if self.default is None: - # FIXME: this code is obsolete with the new admins group - # Should be replaced by something like "any first user we find in the admin group" - root_mail = "root@%s" % _get_maindomain() - for user in self.choices.keys(): - if root_mail in user_info(user).get("mail-aliases", []): - self.default = user - break - - -class GroupQuestion(Question): - argument_type = "group" - - def __init__( - self, question, context: Mapping[str, Any] = {}, hooks: Dict[str, Callable] = {} - ): - from yunohost.user import user_group_list - - super().__init__(question, context) - - self.choices = list( - user_group_list(short=True, include_primary_groups=False)["groups"] - ) - - def _human_readable_group(g): - # i18n: visitors - # i18n: all_users - # i18n: admins - return m18n.n(g) if g in ["visitors", "all_users", "admins"] else g - - self.choices = {g: _human_readable_group(g) for g in self.choices} - - if self.default is None: - self.default = "all_users" - - -class NumberQuestion(Question): - argument_type = "number" - default_value = None - - def __init__( - self, question, context: Mapping[str, Any] = {}, hooks: Dict[str, Callable] = {} - ): - super().__init__(question, context, hooks) - self.min = question.get("min", None) - self.max = question.get("max", None) - self.step = question.get("step", None) - - @staticmethod - def normalize(value, option={}): - if isinstance(value, int): - return value - - if isinstance(value, str): - value = value.strip() - - if isinstance(value, str) and value.isdigit(): - return int(value) - - if value in [None, ""]: - return None - - option = option.__dict__ if isinstance(option, Question) else option - raise YunohostValidationError( - "app_argument_invalid", - name=option.get("name"), - error=m18n.n("invalid_number"), - ) - - def _prevalidate(self): - super()._prevalidate() - if self.value in [None, ""]: - return - - if self.min is not None and int(self.value) < self.min: - raise YunohostValidationError( - "app_argument_invalid", - name=self.name, - error=m18n.n("invalid_number_min", min=self.min), - ) - - if self.max is not None and int(self.value) > self.max: - raise YunohostValidationError( - "app_argument_invalid", - name=self.name, - error=m18n.n("invalid_number_max", max=self.max), - ) - - -class DisplayTextQuestion(Question): - argument_type = "display_text" - - def __init__( - self, question, context: Mapping[str, Any] = {}, hooks: Dict[str, Callable] = {} - ): - super().__init__(question, context, hooks) - - self.optional = True - self.readonly = True - self.style = question.get( - "style", "info" if question["type"] == "alert" else "" - ) - - def _format_text_for_user_input_in_cli(self): - text = _value_for_locale(self.ask) - - if self.style in ["success", "info", "warning", "danger"]: - color = { - "success": "green", - "info": "cyan", - "warning": "yellow", - "danger": "red", - } - prompt = m18n.g(self.style) if self.style != "danger" else m18n.n("danger") - return colorize(prompt, color[self.style]) + f" {text}" - else: - return text - - -class FileQuestion(Question): - argument_type = "file" - upload_dirs: List[str] = [] - - @classmethod - def clean_upload_dirs(cls): - # Delete files uploaded from API - for upload_dir in cls.upload_dirs: - if os.path.exists(upload_dir): - shutil.rmtree(upload_dir) - - def __init__( - self, question, context: Mapping[str, Any] = {}, hooks: Dict[str, Callable] = {} - ): - super().__init__(question, context, hooks) - self.accept = question.get("accept", "") - - def _prevalidate(self): - if self.value is None: - self.value = self.current_value - - super()._prevalidate() - - # Validation should have already failed if required - if self.value in [None, ""]: - return self.value - - if Moulinette.interface.type != "api": - if not os.path.exists(str(self.value)) or not os.path.isfile(str(self.value)): - raise YunohostValidationError( - "app_argument_invalid", - name=self.name, - error=m18n.n("file_does_not_exist", path=str(self.value)), - ) - - def _post_parse_value(self): - from base64 import b64decode - - if not self.value: - return "" - - upload_dir = tempfile.mkdtemp(prefix="ynh_filequestion_") - _, file_path = tempfile.mkstemp(dir=upload_dir) - - FileQuestion.upload_dirs += [upload_dir] - - logger.debug(f"Saving file {self.name} for file question into {file_path}") - - def is_file_path(s): - return isinstance(s, str) and s.startswith("/") and os.path.exists(s) - - if Moulinette.interface.type != "api" or is_file_path(self.value): - content = read_file(str(self.value), file_mode="rb") - else: - content = b64decode(self.value) - - write_to_file(file_path, content, file_mode="wb") - - self.value = file_path - - return self.value - - -class ButtonQuestion(Question): - argument_type = "button" - enabled = None - - def __init__( - self, question, context: Mapping[str, Any] = {}, hooks: Dict[str, Callable] = {} - ): - super().__init__(question, context, hooks) - self.enabled = question.get("enabled", None) - - -ARGUMENTS_TYPE_PARSERS = { - "string": StringQuestion, - "text": StringQuestion, - "select": StringQuestion, - "tags": TagsQuestion, - "email": EmailQuestion, - "url": URLQuestion, - "date": DateQuestion, - "time": TimeQuestion, - "color": ColorQuestion, - "password": PasswordQuestion, - "path": PathQuestion, - "boolean": BooleanQuestion, - "domain": DomainQuestion, - "user": UserQuestion, - "group": GroupQuestion, - "number": NumberQuestion, - "range": NumberQuestion, - "display_text": DisplayTextQuestion, - "alert": DisplayTextQuestion, - "markdown": DisplayTextQuestion, - "file": FileQuestion, - "app": AppQuestion, - "button": ButtonQuestion, -} - - -def ask_questions_and_parse_answers( - raw_questions: Dict, - prefilled_answers: Union[str, Mapping[str, Any]] = {}, - current_values: Mapping[str, Any] = {}, - hooks: Dict[str, Callable[[], None]] = {}, -) -> List[Question]: - """Parse arguments store in either manifest.json or actions.json or from a - config panel against the user answers when they are present. - - Keyword arguments: - raw_questions -- the arguments description store in yunohost - format from actions.json/toml, manifest.json/toml - or config_panel.json/toml - prefilled_answers -- a url "query-string" such as "domain=yolo.test&path=/foobar&admin=sam" - or a dict such as {"domain": "yolo.test", "path": "/foobar", "admin": "sam"} - """ - - if isinstance(prefilled_answers, str): - # FIXME FIXME : this is not uniform with config_set() which uses parse.qs (no l) - # parse_qsl parse single values - # whereas parse.qs return list of values (which is useful for tags, etc) - # For now, let's not migrate this piece of code to parse_qs - # Because Aleks believes some bits of the app CI rely on overriding values (e.g. foo=foo&...&foo=bar) - answers = dict( - urllib.parse.parse_qsl(prefilled_answers or "", keep_blank_values=True) - ) - elif isinstance(prefilled_answers, Mapping): - answers = {**prefilled_answers} - else: - answers = {} - - context = {**current_values, **answers} - out = [] - - for name, raw_question in raw_questions.items(): - raw_question["name"] = name - question_class = ARGUMENTS_TYPE_PARSERS[raw_question.get("type", "string")] - raw_question["value"] = answers.get(name) - question = question_class(raw_question, context=context, hooks=hooks) - if question.type == "button": - if question.enabled is None or evaluate_simple_js_expression( # type: ignore - question.enabled, context=context # type: ignore - ): # type: ignore - continue - else: - raise YunohostValidationError( - "config_action_disabled", - action=question.name, - help=_value_for_locale(question.help), - ) - - new_values = question.ask_if_needed() - answers.update(new_values) - context.update(new_values) - out.append(question) - - return out - - -def hydrate_questions_with_choices(raw_questions: List) -> List: - out = [] - - for raw_question in raw_questions: - question = ARGUMENTS_TYPE_PARSERS[raw_question.get("type", "string")]( - raw_question - ) - if question.choices: - raw_question["choices"] = question.choices - raw_question["default"] = question.default - out.append(raw_question) - - return out From 8c25aa9b9faaf190792738277f71d74822f11088 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 4 Apr 2023 15:55:55 +0200 Subject: [PATCH 40/43] helpers: fix previous change about using YNH_APP_ACTION ... which is not defined in config panel context --- helpers/utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/utils b/helpers/utils index d27b5bca2..a88be38a8 100644 --- a/helpers/utils +++ b/helpers/utils @@ -22,7 +22,7 @@ YNH_APP_BASEDIR=${YNH_APP_BASEDIR:-$(realpath ..)} ynh_exit_properly() { local exit_code=$? - if [[ "${YNH_APP_ACTION}" =~ ^install$|^upgrade$|^restore$ ]] + if [[ "${YNH_APP_ACTION:-}" =~ ^install$|^upgrade$|^restore$ ]] then rm -rf "/var/cache/yunohost/download/" fi From bee218e560374569cd032a4234adcf88b0242f16 Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 4 Apr 2023 16:05:36 +0200 Subject: [PATCH 41/43] fix configpanel.py and form.py imports --- src/app.py | 5 ++--- src/domain.py | 3 ++- src/settings.py | 3 ++- src/tests/test_questions.py | 2 +- src/utils/configpanel.py | 7 +++++++ 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/app.py b/src/app.py index b37b680ec..1daa14d98 100644 --- a/src/app.py +++ b/src/app.py @@ -48,9 +48,8 @@ from moulinette.utils.filesystem import ( chmod, ) -from yunohost.utils.config import ( - ConfigPanel, - ask_questions_and_parse_answers, +from yunohost.utils.configpanel import ConfigPanel, ask_questions_and_parse_answers +from yunohost.utils.form import ( DomainQuestion, PathQuestion, hydrate_questions_with_choices, diff --git a/src/domain.py b/src/domain.py index 7839b988d..9f38d6765 100644 --- a/src/domain.py +++ b/src/domain.py @@ -33,7 +33,8 @@ from yunohost.app import ( _get_conflicting_apps, ) from yunohost.regenconf import regen_conf, _force_clear_hashes, _process_regen_conf -from yunohost.utils.config import ConfigPanel, Question +from yunohost.utils.configpanel import ConfigPanel +from yunohost.utils.form import Question from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.log import is_unit_operation diff --git a/src/settings.py b/src/settings.py index 4905049d6..5d52329b3 100644 --- a/src/settings.py +++ b/src/settings.py @@ -21,7 +21,8 @@ import subprocess from moulinette import m18n from yunohost.utils.error import YunohostError, YunohostValidationError -from yunohost.utils.config import ConfigPanel, Question +from yunohost.utils.configpanel import ConfigPanel +from yunohost.utils.form import Question from moulinette.utils.log import getActionLogger from yunohost.regenconf import regen_conf from yunohost.firewall import firewall_reload diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index 8ded2e137..506fde077 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -14,7 +14,7 @@ from _pytest.mark.structures import ParameterSet from moulinette import Moulinette from yunohost import app, domain, user -from yunohost.utils.config import ( +from yunohost.utils.form import ( ARGUMENTS_TYPE_PARSERS, ask_questions_and_parse_answers, DisplayTextQuestion, diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index 1f1351bcb..e50d0a3ec 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -35,6 +35,13 @@ from moulinette.utils.filesystem import ( from yunohost.utils.i18n import _value_for_locale from yunohost.utils.error import YunohostError, YunohostValidationError +from yunohost.utils.form import ( + ARGUMENTS_TYPE_PARSERS, + FileQuestion, + Question, + ask_questions_and_parse_answers, + evaluate_simple_js_expression, +) logger = getActionLogger("yunohost.configpanel") CONFIG_PANEL_VERSION_SUPPORTED = 1.0 From 9a4267ffa41d53ebd7e137108b4e4a38e863faa1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 5 Apr 2023 15:58:07 +0200 Subject: [PATCH 42/43] appsv2: for the dir/subdirs of data_dir, create parent folders if they don't exist --- src/utils/resources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/resources.py b/src/utils/resources.py index 8f8393e17..bd50cca04 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -955,12 +955,12 @@ class DatadirAppResource(AppResource): ) shutil.move(current_data_dir, self.dir) else: - mkdir(self.dir) + mkdir(self.dir, parents=True) for subdir in self.subdirs: full_path = os.path.join(self.dir, subdir) if not os.path.isdir(full_path): - mkdir(full_path) + mkdir(full_path, parents=True) owner, owner_perm = self.owner.split(":") group, group_perm = self.group.split(":") From 021099aa1e62badb5d5c573a8e521f8d24f9f847 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 5 Apr 2023 16:02:02 +0200 Subject: [PATCH 43/43] Update changelog for 11.1.17 --- debian/changelog | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/debian/changelog b/debian/changelog index 3c0cccbc2..9b61a7b45 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +yunohost (11.1.17) stable; urgency=low + + - domains: fix autodns for gandi root domain ([#1634](https://github.com/yunohost/yunohost/pull/1634)) + - helpers: fix previous change about using YNH_APP_ACTION ... which is not defined in config panel context (8c25aa9b) + - appsv2: for the dir/subdirs of data_dir, create parent folders if they don't exist (9a4267ff) + - quality: Split utils/config.py ([#1635](https://github.com/yunohost/yunohost/pull/1635)) + - quality: Rework questions/options tests ([#1629](https://github.com/yunohost/yunohost/pull/1629)) + + Thanks to all contributors <3 ! (axolotle, Kayou) + + -- Alexandre Aubin Wed, 05 Apr 2023 16:00:09 +0200 + yunohost (11.1.16) stable; urgency=low - apps: fix i18n panel+section names ([#1630](https://github.com/yunohost/yunohost/pull/1630))