Merge remote-tracking branch 'origin/dev' into bookworm

.github/workflows/autoblack.yml

@@ -1,29 +1,24 @@
 name: Check / auto apply Black
+
 on:
   push:
-      branches:
-          - dev
+    branches: [ "dev" ]
+
 jobs:
   black:
     name: Check / auto apply black
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+
       - name: Check files using the black formatter
         uses: psf/black@stable
         id: black
         with:
           options: "."
         continue-on-error: true
-      - shell: pwsh
-        id: check_files_changed
-        run: |
-          # Diff HEAD with the previous commit
-          $diff = git diff
-          $HasDiff = $diff.Length -gt 0
-          Write-Host "::set-output name=files_changed::$HasDiff"
+
       - name: Create Pull Request
-        if: steps.check_files_changed.outputs.files_changed == 'true'
         uses: peter-evans/create-pull-request@v6
         with:
           token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/n_updater.yml

@@ -12,28 +12,18 @@ jobs:
     steps:
       - name: Fetch the source code
         uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Run the updater script
         id: run_updater
         run: |
-          # Setting up Git user
-          git config --global user.name 'yunohost-bot'
-          git config --global user.email 'yunohost-bot@users.noreply.github.com'
           # Download n
           wget https://raw.githubusercontent.com/tj/n/master/bin/n --output-document=helpers/vendor/n/n
-          # Proceed only if there is a change
-          [[ -z "$(git diff helpers/vendor/n/n)" ]] || echo "PROCEED=true" >> $GITHUB_ENV
-          echo "VERSION=$(sed -n 's/^VERSION=\"\(.*\)\"/\1/p' < n)" >> $GITHUB_ENV
-      - name: Commit changes
-        id: commit
-        if: ${{ env.PROCEED == 'true' }}
-        run: |
-          git commit -am "Upgrade n to v$VERSION"
+
+          echo "VERSION=$(sed -n 's/^VERSION=\"\(.*\)\"/\1/p' < helpers/vendor/n/n)" >> $GITHUB_ENV
+
       - name: Create Pull Request
-        id: cpr
-        if: ${{ env.PROCEED == 'true' }}
         uses: peter-evans/create-pull-request@v6
+        id: cpr
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           commit-message: Update n to ${{ env.VERSION }}
@@ -41,7 +31,7 @@ jobs:
           author: 'yunohost-bot <yunohost-bot@users.noreply.github.com>'
           signoff: false
           base: dev
-          branch: ci-auto-update-n-v${{ env.VERSION }}
+          branch: ci-auto-update-n-${{ env.VERSION }}
           delete-branch: true
           title: 'Upgrade n to ${{ env.VERSION }}'
           body: |

debian/control

@@ -28,7 +28,7 @@ Depends: ${python3:Depends}, ${misc:Depends}
  , opendkim-tools, opendkim, postsrsd, procmail, mailutils
  , redis-server
  , acl
- , git, curl, wget, cron, unzip, jq, bc, at, procps
+ , git, curl, wget, cron, unzip, jq, bc, at, procps, j2cli
  , lsb-release, haveged, fake-hwclock, equivs, lsof, whois
 Recommends: yunohost-admin, yunohost-portal (>= 12.0)
  , ntp, inetutils-ping | iputils-ping

debian/install

@@ -1,7 +1,7 @@
 bin/* /usr/bin/
 share/* /usr/share/yunohost/
 hooks/* /usr/share/yunohost/hooks/
-helpers/* /usr/share/yunohost/helpers.d/
+helpers/* /usr/share/yunohost/
 conf/* /usr/share/yunohost/conf/
 locales/* /usr/share/yunohost/locales/
 doc/yunohost.8.gz /usr/share/man/man8/

helpers/helpers

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Entrypoint for the helpers scripts
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+# Helpers version can be specified via an environment variable or default to 1.
+YNH_HELPERS_VERSION=${YNH_HELPERS_VERSION:-1}
+
+# This is a trick to later only restore set -x if it was set when calling this script
+readonly XTRACE_ENABLE=$(set +o | grep xtrace)
+set +x
+
+YNH_HELPERS_DIR="$SCRIPT_DIR/helpers.v${YNH_HELPERS_VERSION}.d"
+case "$YNH_HELPERS_VERSION" in
+    "1" | "2")
+        readarray -t HELPERS < <(find -L "$YNH_HELPERS_DIR" -mindepth 1 -maxdepth 1 -type f)
+        for helper in "${HELPERS[@]}"; do
+            [ -r "$helper" ] && source "$helper"
+        done
+        ;;
+    *)
+        echo "Helpers are not available in version '$YNH_HELPERS_VERSION'." >&2
+        exit 1
+esac
+
+eval "$XTRACE_ENABLE"

helpers/helpers.v1.d/go

@@ -111,43 +111,35 @@ ynh_install_go () {
     test -x /usr/bin/go && mv /usr/bin/go /usr/bin/go_goenv
 
     # Install or update goenv
-    goenv="$(command -v goenv $goenv_install_dir/bin/goenv | head -1)"
-    if [ -n "$goenv" ]; then
-        ynh_print_info --message="goenv already seems installed in \`$goenv'."
-        pushd "${goenv%/*/*}"
-            if git remote -v 2>/dev/null | grep "https://github.com/syndbg/goenv.git"; then
-                echo "Trying to update with Git..."
-                git pull -q --tags origin master
-                cd ..
-                ynh_go_try_bash_extension
-            fi
-        popd
-    else
-        ynh_print_info --message="Installing goenv with Git..."
     mkdir -p $goenv_install_dir
-        pushd $goenv_install_dir
+    pushd "$goenv_install_dir"
+        if ! [ -x "$goenv_install_dir/bin/goenv" ]; then
+            ynh_print_info --message="Downloading goenv..."
             git init -q
-            git remote add -f -t master origin https://github.com/syndbg/goenv.git > /dev/null 2>&1
-            git checkout -q -b master origin/master
+            git remote add origin https://github.com/syndbg/goenv.git 
+        else
+            ynh_print_info --message="Updating goenv..."
+        fi
+        git fetch -q --tags --prune origin
+        local git_latest_tag=$(git describe --tags "$(git rev-list --tags --max-count=1)")
+        git checkout -q "$git_latest_tag"
         ynh_go_try_bash_extension
         goenv=$goenv_install_dir/bin/goenv
     popd
-    fi
 
-    goenv_latest="$(command -v "$goenv_install_dir"/plugins/*/bin/goenv-latest goenv-latest | head -1)"
-    if [ -n "$goenv_latest" ]; then
-        ynh_print_info --message="\`goenv latest' command already available in \`$goenv_latest'."
-        pushd "${goenv_latest%/*/*}"
-            if git remote -v 2>/dev/null | grep "https://github.com/momo-lab/xxenv-latest.git"; then
-                ynh_print_info --message="Trying to update xxenv-latest with git..."
-                git pull -q origin master
-            fi
-        popd
+    # Install or update xxenv-latest
+    mkdir -p "$goenv_install_dir/plugins/xxenv-latest"
+    pushd "$goenv_install_dir/plugins/xxenv-latest"
+        if ! [ -x "$goenv_install_dir/plugins/xxenv-latest/bin/goenv-latest" ]; then
+            ynh_print_info --message="Downloading xxenv-latest..."
+            git init -q
+            git remote add origin https://github.com/momo-lab/xxenv-latest.git 
         else
-        ynh_print_info --message="Installing xxenv-latest with Git..."
-        mkdir -p "${goenv_install_dir}/plugins"
-        git clone -q https://github.com/momo-lab/xxenv-latest.git "${goenv_install_dir}/plugins/xxenv-latest"
+            ynh_print_info --message="Updating xxenv-latest..."
         fi
+        local git_latest_tag=$(git describe --tags "$(git rev-list --tags --max-count=1)")
+        git checkout -q "$git_latest_tag"
+    popd
 
     # Enable caching
     mkdir -p "${goenv_install_dir}/cache"
@@ -162,12 +154,12 @@ ynh_install_go () {
     test -x /usr/bin/go_goenv && mv /usr/bin/go_goenv /usr/bin/go
 
     # Install the requested version of Go
-    local final_go_version=$(goenv latest --print $go_version)
+    local final_go_version=$(goenv latest --print "$go_version")
     ynh_print_info --message="Installation of Go-$final_go_version"
-    goenv install --skip-existing $final_go_version
+    goenv install --skip-existing "$final_go_version"
 
     # Store go_version into the config of this app
-    ynh_app_setting_set --app=$YNH_APP_INSTANCE_NAME --key=go_version --value=$final_go_version
+    ynh_app_setting_set --app="$app" --key="go_version" --value="$final_go_version"
 
     # Cleanup Go versions
     ynh_cleanup_go
@@ -189,7 +181,7 @@ eval \"\$(goenv init -)\"
 #
 # usage: ynh_remove_go
 ynh_remove_go () {
-    local go_version=$(ynh_app_setting_get --app=$YNH_APP_INSTANCE_NAME --key=go_version)
+    local go_version=$(ynh_app_setting_get --app="$app" --key="go_version")
 
     # Load goenv path in PATH
     local CLEAR_PATH="$goenv_install_dir/bin:$PATH"
@@ -198,7 +190,7 @@ ynh_remove_go () {
     PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@')
 
     # Remove the line for this app
-    ynh_app_setting_delete --app=$YNH_APP_INSTANCE_NAME --key=go_version
+    ynh_app_setting_delete --app="$app" --key="go_version"
 
     # Cleanup Go versions
     ynh_cleanup_go
@@ -232,7 +224,7 @@ ynh_cleanup_go () {
         if ! `echo ${required_go_versions} | grep "${installed_go_version}" 1>/dev/null 2>&1`
         then
             ynh_print_info --message="Removing of Go-$installed_go_version"
-            $goenv_install_dir/bin/goenv uninstall --force $installed_go_version
+            $goenv_install_dir/bin/goenv uninstall --force "$installed_go_version"
         fi
     done
 

helpers/helpers.v1.d/nodejs

@@ -115,7 +115,7 @@ ynh_install_nodejs() {
 
     # Install (or update if YunoHost vendor/ folder updated since last install) n
     mkdir -p $n_install_dir/bin/
-    cp /usr/share/yunohost/helpers.d/vendor/n/n $n_install_dir/bin/n
+    cp "$YNH_HELPERS_DIR/vendor/n/n" $n_install_dir/bin/n
     # Tweak for n to understand it's installed in $N_PREFIX
     ynh_replace_string --match_string="^N_PREFIX=\${N_PREFIX-.*}$" --replace_string="N_PREFIX=\${N_PREFIX-$N_PREFIX}" --target_file="$n_install_dir/bin/n"
 
@@ -144,7 +144,7 @@ ynh_install_nodejs() {
     fi
 
     # Store the ID of this app and the version of node requested for it
-    echo "$YNH_APP_INSTANCE_NAME:$nodejs_version" | tee --append "$n_install_dir/ynh_app_version"
+    echo "$app:$nodejs_version" | tee --append "$n_install_dir/ynh_app_version"
 
     # Store nodejs_version into the config of this app
     ynh_app_setting_set --app=$app --key=nodejs_version --value=$nodejs_version
@@ -168,7 +168,7 @@ ynh_remove_nodejs() {
     nodejs_version=$(ynh_app_setting_get --app=$app --key=nodejs_version)
 
     # Remove the line for this app
-    sed --in-place "/$YNH_APP_INSTANCE_NAME:$nodejs_version/d" "$n_install_dir/ynh_app_version"
+    sed --in-place "/$app:$nodejs_version/d" "$n_install_dir/ynh_app_version"
 
     # If no other app uses this version of nodejs, remove it.
     if ! grep --quiet "$nodejs_version" "$n_install_dir/ynh_app_version"; then

helpers/helpers.v1.d/ruby

@@ -54,7 +54,7 @@ ynh_use_ruby () {
     ruby_version=$(ynh_app_setting_get --app=$app --key=ruby_version)
 
     # Get the absolute path of this version of Ruby
-    ruby_path="$ruby_version_path/$YNH_APP_INSTANCE_NAME/bin"
+    ruby_path="$ruby_version_path/$app/bin"
 
     # Allow alias to be used into bash script
     shopt -s expand_aliases
@@ -207,16 +207,16 @@ ynh_install_ruby () {
     RUBY_CONFIGURE_OPTS="--disable-install-doc --with-jemalloc" MAKE_OPTS="-j2" rbenv install --skip-existing $final_ruby_version > /dev/null 2>&1
 
     # Store ruby_version into the config of this app
-    ynh_app_setting_set --app=$YNH_APP_INSTANCE_NAME --key=ruby_version --value=$final_ruby_version
+    ynh_app_setting_set --app=$app --key=ruby_version --value=$final_ruby_version
 
     # Remove app virtualenv
-    if rbenv alias --list | grep --quiet "$YNH_APP_INSTANCE_NAME "
+    if rbenv alias --list | grep --quiet "$app "
     then
-        rbenv alias $YNH_APP_INSTANCE_NAME --remove
+        rbenv alias $app --remove
     fi
 
     # Create app virtualenv
-    rbenv alias $YNH_APP_INSTANCE_NAME $final_ruby_version
+    rbenv alias $app $final_ruby_version
 
     # Cleanup Ruby versions
     ynh_cleanup_ruby
@@ -238,7 +238,7 @@ eval \"\$(rbenv init -)\"
 #
 # usage: ynh_remove_ruby
 ynh_remove_ruby () {
-    local ruby_version=$(ynh_app_setting_get --app=$YNH_APP_INSTANCE_NAME --key=ruby_version)
+    local ruby_version=$(ynh_app_setting_get --app=$app --key=ruby_version)
 
     # Load rbenv path in PATH
     local CLEAR_PATH="$rbenv_install_dir/bin:$PATH"
@@ -246,10 +246,10 @@ ynh_remove_ruby () {
     # Remove /usr/local/bin in PATH in case of Ruby prior installation
     PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@')
 
-    rbenv alias $YNH_APP_INSTANCE_NAME --remove
+    rbenv alias $app --remove
 
     # Remove the line for this app
-    ynh_app_setting_delete --app=$YNH_APP_INSTANCE_NAME --key=ruby_version
+    ynh_app_setting_delete --app=$app --key=ruby_version
 
     # Cleanup Ruby versions
     ynh_cleanup_ruby

helpers/helpers.v1.d/utils

@@ -318,7 +318,7 @@ ynh_setup_source() {
             mv $src_filename $dest_dir/$src_rename
         fi
     elif [[ "$src_format" == "docker" ]]; then
-        /usr/share/yunohost/helpers.d/vendor/docker-image-extract/docker-image-extract -p $src_platform -o $dest_dir $src_url 2>&1
+        "$YNH_HELPERS_DIR/vendor/docker-image-extract/docker-image-extract" -p $src_platform -o $dest_dir $src_url 2>&1
     elif [[ "$src_format" == "zip" ]]; then
         # Zip format
         # Using of a temp directory, because unzip doesn't manage --strip-components
@@ -452,11 +452,16 @@ ynh_local_curl() {
 # usage: ynh_add_config --template="template" --destination="destination"
 # | arg: -t, --template=     - Template config file to use
 # | arg: -d, --destination=  - Destination of the config file
+# | arg: -j, --jinja         - Use jinja template instead of legacy __MY_VAR__
 #
 # examples:
 #    ynh_add_config --template=".env" --destination="$install_dir/.env" use the template file "../conf/.env"
+#    ynh_add_config --jinja --template="config.j2" --destination="$install_dir/config" use the template file "../conf/config.j2"
 #    ynh_add_config --template="/etc/nginx/sites-available/default" --destination="etc/nginx/sites-available/mydomain.conf"
 #
+##
+## How it works in "legacy" mode
+##
 # The template can be by default the name of a file in the conf directory
 # of a YunoHost Package, a relative path or an absolute path.
 #
@@ -480,6 +485,37 @@ ynh_local_curl() {
 #   __VAR_2__    by $var_2
 # ```
 #
+##
+## When --jinja is enabled
+##
+# For a full documentation of the template you can refer to: https://jinja.palletsprojects.com/en/3.1.x/templates/
+# In Yunohost context there are no really some specificity except that all variable passed are of type string.
+# So here are some example of recommended usage:
+#
+# If you need a conditional block
+#
+# {% if should_my_block_be_shown == 'true' %}
+# ...
+# {% endif %}
+#
+# or
+#
+# {% if should_my_block_be_shown == '1' %}
+# ...
+# {% endif %}
+#
+# If you need to iterate with loop:
+#
+# {% for yolo in var_with_multiline_value.splitlines() %}
+# ...
+# {% endfor %}
+#
+# or
+#
+# {% for jail in my_var_with_coma.split(',') %}
+# ...
+# {% endfor %}
+#
 # The helper will verify the checksum and backup the destination file
 # if it's different before applying the new template.
 #
@@ -489,13 +525,15 @@ ynh_local_curl() {
 # Requires YunoHost version 4.1.0 or higher.
 ynh_add_config() {
     # Declare an array to define the options of this helper.
-    local legacy_args=tdv
-    local -A args_array=([t]=template= [d]=destination=)
+    local legacy_args=tdj
+    local -A args_array=([t]=template= [d]=destination= [j]=jinja)
     local template
     local destination
+    local jinja
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
     local template_path
+    jinja="${jinja:-0}"
 
     if [ -f "$YNH_APP_BASEDIR/conf/$template" ]; then
         template_path="$YNH_APP_BASEDIR/conf/$template"
@@ -512,14 +550,20 @@ ynh_add_config() {
     # created a file beforehand to have control over it
     # (cp won't overwrite ownership / modes by default...)
     touch $destination
-    chown root:root $destination
     chmod 640 $destination
-
-    cp -f "$template_path" "$destination"
-
     _ynh_apply_default_permissions $destination
 
+    if [[ "$jinja" == 1 ]]
+    then
+        # This is ran in a subshell such that the "export" does not "contaminate" the main process
+        (
+            export $(compgen -v)
+            j2 "$template_path" -f env -o $destination
+        )
+    else
+        cp -f "$template_path" "$destination"
         ynh_replace_vars --file="$destination"
+    fi
 
     ynh_store_file_checksum --file="$destination"
 }
@@ -1084,7 +1128,7 @@ _ynh_apply_default_permissions() {
     # Crons should be owned by root
     # Also we don't want systemd conf, nginx conf or others stuff to be owned by the app,
     # otherwise they could self-edit their own systemd conf and escalate privilege
-    if echo "$target" | grep -q '^/etc/cron\|/etc/php\|/etc/nginx/conf.d\|/etc/fail2ban\|/etc/systemd/system'
+    if grep -qE '^(/etc/cron|/etc/php|/etc/nginx/conf.d|/etc/fail2ban|/etc/systemd/system)' <<< "$target"
     then
         chmod 400 $target
         chown root:root $target

helpers/helpers.v1.d/vendor

@@ -0,0 +1 @@
+../vendor

helpers/helpers.v2.d

@@ -0,0 +1 @@
+helpers.v1.d

share/helpers

@@ -1,8 +0,0 @@
-# -*- shell-script -*-
-
-readonly XTRACE_ENABLE=$(set +o | grep xtrace) # This is a trick to later only restore set -x if it was set when calling this script
-set +x
-for helper in $(run-parts --list /usr/share/yunohost/helpers.d 2>/dev/null) ; do
-    [ -r $helper ] && . $helper || true
-done
-eval "$XTRACE_ENABLE"

src/app.py

@@ -2870,6 +2870,10 @@ def _make_environment_for_app_script(
         "YNH_APP_INSTANCE_NUMBER": str(app_instance_nb),
         "YNH_APP_MANIFEST_VERSION": manifest.get("version", "?"),
         "YNH_APP_PACKAGING_FORMAT": str(manifest["packaging_format"]),
+        "YNH_HELPERS_VERSION": str(
+            manifest.get("integration", {}).get("helpers_version")
+            or manifest["packaging_format"]
+        ).replace(".0", ""),
         "YNH_ARCH": system_arch(),
         "YNH_DEBIAN_VERSION": debian_version(),
     }

tests/test_helpers.d/ynhtest_templating.sh

@@ -0,0 +1,62 @@
+ynhtest_simple_template_app_config() {
+
+    mkdir -p /etc/yunohost/apps/$app/
+    echo "id: $app" > /etc/yunohost/apps/$app/settings.yml
+
+    template="$(mktemp -d -p $VAR_WWW)/template.txt"
+    cat << EOF > $template 
+app=__APP__
+foo=__FOO__
+EOF
+
+    foo="bar"
+    
+    ynh_add_config --template="$template" --destination="$VAR_WWW/config.txt"
+
+    test "$(cat $VAR_WWW/config.txt)" == "$(echo -ne 'app=ynhtest\nfoo=bar')"
+    test "$(ls -l $VAR_WWW/config.txt | cut -d' ' -f1-4)" == "-rw-r----- 1 ynhtest ynhtest"
+}
+
+ynhtest_simple_template_system_config() {
+
+    mkdir -p /etc/yunohost/apps/$app/
+    echo "id: $app" > /etc/yunohost/apps/$app/settings.yml
+
+    rm -f /etc/cron.d/ynhtest_config
+
+    template="$(mktemp -d -p $VAR_WWW)/template.txt"
+    cat << EOF > $template 
+app=__APP__
+foo=__FOO__
+EOF
+
+    foo="bar"
+    
+    ynh_add_config --template="$template" --destination="/etc/cron.d/ynhtest_config"
+
+    test "$(cat $VAR_WWW/config.txt)" == "$(echo -ne 'app=ynhtest\nfoo=bar')"
+    test "$(ls -l /etc/cron.d/ynhtest_config | cut -d' ' -f1-4)" == "-r-------- 1 root root"
+
+    rm -f /etc/cron.d/ynhtest_config
+}
+
+ynhtest_jinja_template_app_config() {
+
+    mkdir -p /etc/yunohost/apps/$app/
+    echo "id: $app" > /etc/yunohost/apps/$app/settings.yml
+
+    template="$(mktemp -d -p $VAR_WWW)/template.txt"
+    cat << EOF > $template 
+app={{ app }}
+{% if foo == "bar" %}foo=true{% endif %}
+EOF
+
+    foo="bar"
+    
+    ynh_add_config --template="$template" --destination="$VAR_WWW/config.txt" --jinja
+
+    test "$(cat $VAR_WWW/config.txt)" == "$(echo -ne 'app=ynhtest\nfoo=true')"
+    test "$(ls -l $VAR_WWW/config.txt | cut -d' ' -f1-4)" == "-rw-r----- 1 ynhtest ynhtest"
+}
+
+

tests/test_helpers.sh

@@ -41,6 +41,10 @@ popd >/dev/null
 
 VAR_WWW=$(mktemp -d)/var/www
 mkdir -p $VAR_WWW
+
+# Needed to check the permission behavior in ynh_add_config x_x
+getent passwd ynhtest &>/dev/null || useradd --system ynhtest
+
 # =========================================================
 
 for TEST_SUITE in $(ls test_helpers.d/*)