Update changelog for 11.2.30

Fix ynh_restore_everything

.coveragerc

@@ -1,2 +1,2 @@
 [report]
-omit=src/tests/*,src/vendor/*,/usr/lib/moulinette/yunohost/*
+omit=src/tests/*,src/vendor/*,/usr/lib/moulinette/yunohost/*,/usr/lib/python3/dist-packages/yunohost/tests/*,/usr/lib/python3/dist-packages/yunohost/vendor/*

.github/workflows/autoblack.yml

@@ -0,0 +1,30 @@
+name: Check / auto apply Black
+
+on:
+  push:
+    branches: [ "dev" ]
+
+jobs:
+  black:
+    name: Check / auto apply black
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check files using the black formatter
+        uses: psf/black@stable
+        id: black
+        with:
+          options: "."
+        continue-on-error: true
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          title: "Format Python code with Black"
+          commit-message: ":art: Format Python code with Black"
+          body: |
+            This pull request uses the [psf/black](https://github.com/psf/black) formatter.
+          base: ${{ github.head_ref }} # Creates pull request onto pull request or commit branch
+          branch: actions/black

.github/workflows/codeql.yml

@@ -0,0 +1,42 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "dev" ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "dev" ]
+    paths-ignore:
+      - 'src/tests/**'
+  schedule:
+    - cron: '43 12 * * 3'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        queries: security-extended,security-and-quality
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/n_updater.sh

@@ -1,78 +0,0 @@
-#!/bin/bash
-
-#=================================================
-# N UPDATING HELPER
-#=================================================
-
-# This script is meant to be run by GitHub Actions.
-# It is derived from the Updater script from the YunoHost-Apps organization.
-# It aims to automate the update of `n`, the Node version management system.
-
-#=================================================
-# FETCHING LATEST RELEASE AND ITS ASSETS
-#=================================================
-
-# Fetching information
-source helpers/nodejs
-current_version="$n_version"
-repo="tj/n"
-# Some jq magic is needed, because the latest upstream release is not always the latest version (e.g. security patches for older versions)
-version=$(curl --silent "https://api.github.com/repos/$repo/releases" | jq -r '.[] | select( .prerelease != true ) | .tag_name' | sort -V | tail -1)
-
-# Later down the script, we assume the version has only digits and dots
-# Sometimes the release name starts with a "v", so let's filter it out.
-if [[ ${version:0:1} == "v" || ${version:0:1} == "V" ]]; then
-    version=${version:1}
-fi
-
-# Setting up the environment variables
-echo "Current version: $current_version"
-echo "Latest release from upstream: $version"
-echo "VERSION=$version" >> $GITHUB_ENV
-# For the time being, let's assume the script will fail
-echo "PROCEED=false" >> $GITHUB_ENV
-
-# Proceed only if the retrieved version is greater than the current one
-if ! dpkg --compare-versions "$current_version" "lt" "$version" ; then
-    echo "::warning ::No new version available"
-    exit 0
-# Proceed only if a PR for this new version does not already exist
-elif git ls-remote -q --exit-code --heads https://github.com/${GITHUB_REPOSITORY:-YunoHost/yunohost}.git ci-auto-update-n-v$version ; then
-    echo "::warning ::A branch already exists for this update"
-    exit 0
-fi
-
-#=================================================
-# UPDATE SOURCE FILES
-#=================================================
-
-asset_url="https://github.com/tj/n/archive/v${version}.tar.gz"
-
-echo "Handling asset at $asset_url"
-
-# Create the temporary directory
-tempdir="$(mktemp -d)"
-
-# Download sources and calculate checksum
-filename=${asset_url##*/}
-curl --silent -4 -L $asset_url -o "$tempdir/$filename"
-checksum=$(sha256sum "$tempdir/$filename" | head -c 64)
-
-# Delete temporary directory
-rm -rf $tempdir
-
-echo "Calculated checksum for n v${version} is $checksum"
-
-#=================================================
-# GENERIC FINALIZATION
-#=================================================
-
-# Replace new version in helper
-sed -i -E "s/^n_version=.*$/n_version=$version/" helpers/nodejs
-
-# Replace checksum in helper
-sed -i -E "s/^n_checksum=.*$/n_checksum=$checksum/" helpers/nodejs
-
-# The Action will proceed only if the PROCEED environment variable is set to true
-echo "PROCEED=true" >> $GITHUB_ENV
-exit 0

.github/workflows/n_updater.yml

@@ -11,36 +11,29 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Fetch the source code
-        uses: actions/checkout@v2
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
+        uses: actions/checkout@v4
+
       - name: Run the updater script
         id: run_updater
         run: |
-          # Setting up Git user
-          git config --global user.name 'yunohost-bot'
-          git config --global user.email 'yunohost-bot@users.noreply.github.com'
-          # Run the updater script
-          /bin/bash .github/workflows/n_updater.sh
-      - name: Commit changes
-        id: commit
-        if: ${{ env.PROCEED == 'true' }}
-        run: |
-          git commit -am "Upgrade n to v$VERSION"
+          # Download n
+          wget https://raw.githubusercontent.com/tj/n/master/bin/n --output-document=helpers/vendor/n/n
+
+          echo "VERSION=$(sed -n 's/^VERSION=\"\(.*\)\"/\1/p' < helpers/vendor/n/n)" >> $GITHUB_ENV
+
       - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v6
         id: cpr
-        if: ${{ env.PROCEED == 'true' }}
-        uses: peter-evans/create-pull-request@v3
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          commit-message: Update n to version ${{ env.VERSION }}
+          commit-message: Update n to ${{ env.VERSION }}
           committer: 'yunohost-bot <yunohost-bot@users.noreply.github.com>'
           author: 'yunohost-bot <yunohost-bot@users.noreply.github.com>'
           signoff: false
           base: dev
-          branch: ci-auto-update-n-v${{ env.VERSION }}
+          branch: ci-auto-update-n-${{ env.VERSION }}
           delete-branch: true
-          title: 'Upgrade n to version ${{ env.VERSION }}'
+          title: 'Upgrade n to ${{ env.VERSION }}'
           body: |
-            Upgrade `n` to v${{ env.VERSION }}
+            Upgrade `n` to ${{ env.VERSION }}
           draft: false

.gitignore

@@ -35,3 +35,10 @@ src/locales
 
 # Test
 src/tests/apps
+
+# Tmp/local doc stuff
+doc/bash-completion.sh
+doc/bash_completion.d
+doc/openapi.js
+doc/openapi.json
+doc/swagger

.gitlab-ci.yml

@@ -16,6 +16,9 @@ default:
 code_quality:
   tags:
     - docker
+  rules:
+    - if: $CI_COMMIT_TAG # Only for tags
+
 
 code_quality_html:
   extends: code_quality
@@ -23,6 +26,9 @@ code_quality_html:
     REPORT_FORMAT: html
   artifacts:
     paths: [gl-code-quality-report.html]
+  rules:
+    - if: $CI_COMMIT_TAG # Only for tags
+
 
 # see: https://docs.gitlab.com/ee/ci/yaml/#switch-between-branch-pipelines-and-merge-request-pipelines
 workflow:
@@ -32,12 +38,17 @@ workflow:
     - if: $CI_COMMIT_TAG # For tags
     - if: $CI_COMMIT_REF_NAME == "ci-format-$CI_DEFAULT_BRANCH"  # Ignore black formatting branch created by the CI
       when: never
+    - if: $CI_COMMIT_REF_NAME == "actions/black"  # Ignore black formatting branch created by the CI
+      when: never
     - if: $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH  && $CI_PIPELINE_SOURCE == "push" # If it's not the default branch and if it's a push, then do not trigger a build
       when: never
     - when: always
 
 variables:
-  YNH_BUILD_DIR: "ynh-build"
+  GIT_CLONE_PATH: '$CI_BUILDS_DIR/$CI_COMMIT_SHA/$CI_JOB_ID'
+  YNH_SOURCE: "https://github.com/yunohost"
+  YNH_DEBIAN: "bullseye"
+  YNH_SKIP_DIAGNOSIS_DURING_UPGRADE: "true"
 
 include:
   - template: Code-Quality.gitlab-ci.yml

.gitlab/ci/build.gitlab-ci.yml

@@ -1,22 +1,22 @@
 .build-stage:
   stage: build
-  image: "before-install"
+  image: "build-and-lint"
   variables:
-    YNH_SOURCE: "https://github.com/yunohost"
+    YNH_BUILD_DIR: "$GIT_CLONE_PATH/build"
   before_script:
     - mkdir -p $YNH_BUILD_DIR
-    - DEBIAN_FRONTEND=noninteractive apt update
   artifacts:
     paths:
-      - $YNH_BUILD_DIR/*.deb
+      - ./*.deb
 
 .build_script: &build_script
-  - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" install devscripts --no-install-recommends
   - cd $YNH_BUILD_DIR/$PACKAGE
   - VERSION=$(dpkg-parsechangelog -S Version 2>/dev/null)
-  - VERSION_NIGHTLY="${VERSION}+$(date +%Y%m%d%H%M)"
-  - dch --package "${PACKAGE}" --force-bad-version -v "${VERSION_NIGHTLY}" -D "unstable" --force-distribution "Daily build."
+  - VERSION_TIMESTAMPED="${VERSION}+$(date +%Y%m%d%H%M)"
+  - dch --package "${PACKAGE}" --force-bad-version -v "${VERSION_TIMESTAMPED}" -D "unstable" --force-distribution "CI build."
   - debuild --no-lintian -us -uc
+  - cp $YNH_BUILD_DIR/*.deb ${CI_PROJECT_DIR}/
+  - cd ${CI_PROJECT_DIR}
 
 ########################################
 # BUILD DEB
@@ -31,18 +31,16 @@ build-yunohost:
     - mkdir -p $YNH_BUILD_DIR/$PACKAGE
     - cat archive.tar.gz | tar -xz -C $YNH_BUILD_DIR/$PACKAGE
     - rm archive.tar.gz
-    - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $(pwd)/$YNH_BUILD_DIR/$PACKAGE
+    - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $YNH_BUILD_DIR/$PACKAGE
     - *build_script
 
-
 build-ssowat:
   extends: .build-stage
   variables:
     PACKAGE: "ssowat"
   script:
-    - DEBIAN_DEPENDS=$(cat debian/control | tr "," "\n" | grep -Po "ssowat \([>,=,<]+ .*\)" | grep -Po "[0-9\.]+")
-    - git clone $YNH_SOURCE/$PACKAGE -b $CI_COMMIT_REF_NAME $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE -b $DEBIAN_DEPENDS $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE $YNH_BUILD_DIR/$PACKAGE --depth 1
-    - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $(pwd)/$YNH_BUILD_DIR/$PACKAGE
+    - git clone $YNH_SOURCE/$PACKAGE -b $CI_COMMIT_REF_NAME $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE -b $YNH_DEBIAN $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE $YNH_BUILD_DIR/$PACKAGE --depth 1
+    - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $YNH_BUILD_DIR/$PACKAGE
     - *build_script
 
 build-moulinette:
@@ -50,7 +48,6 @@ build-moulinette:
   variables:
     PACKAGE: "moulinette"
   script:
-    - DEBIAN_DEPENDS=$(cat debian/control | tr "," "\n" | grep -Po "moulinette \([>,=,<]+ .*\)" | grep -Po "[0-9\.]+")
-    - git clone $YNH_SOURCE/$PACKAGE -b $CI_COMMIT_REF_NAME $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE -b $DEBIAN_DEPENDS $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE $YNH_BUILD_DIR/$PACKAGE --depth 1
-    - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $(pwd)/$YNH_BUILD_DIR/$PACKAGE
+    - git clone $YNH_SOURCE/$PACKAGE -b $CI_COMMIT_REF_NAME $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE -b $YNH_DEBIAN $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE $YNH_BUILD_DIR/$PACKAGE --depth 1
+    - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $YNH_BUILD_DIR/$PACKAGE
     - *build_script

.gitlab/ci/doc.gitlab-ci.yml

@@ -4,24 +4,28 @@
 
 generate-helpers-doc:
   stage: doc
-  image: "before-install"
+  image: "build-and-lint"
   needs: []
   before_script:
-    - apt-get update -y && apt-get install git hub -y
     - git config --global user.email "yunohost@yunohost.org"
     - git config --global user.name "$GITHUB_USER"
   script:
     - cd doc
-    - python3 generate_helper_doc.py
+    - python3 generate_helper_doc.py 2
+    - python3 generate_helper_doc.py 2.1
+    - python3 generate_resource_doc.py > resources.md
     - hub clone https://$GITHUB_TOKEN:x-oauth-basic@github.com/YunoHost/doc.git doc_repo
-    - cp helpers.md doc_repo/pages/06.contribute/10.packaging_apps/11.helpers/packaging_apps_helpers.md
+    - cp helpers.v2.md doc_repo/pages/06.contribute/10.packaging_apps/20.scripts/10.helpers/packaging_app_scripts_helpers.md
+    - cp helpers.v2.1.md doc_repo/pages/06.contribute/10.packaging_apps/20.scripts/12.helpers21/packaging_app_scripts_helpers_v21.md
+    - cp resources.md doc_repo/pages/06.contribute/10.packaging_apps/10.manifest/10.appresources/packaging_app_manifest_resources.md
     - cd doc_repo
     # replace ${CI_COMMIT_REF_NAME} with ${CI_COMMIT_TAG} ?
     - hub checkout -b "${CI_COMMIT_REF_NAME}"
-    - hub commit -am "[CI] Helper for ${CI_COMMIT_REF_NAME}"
-    - hub pull-request -m "[CI] Helper for ${CI_COMMIT_REF_NAME}" -p # GITHUB_USER and GITHUB_TOKEN registered here https://gitlab.com/yunohost/yunohost/-/settings/ci_cd
+    - hub commit -am "[CI] Update app helpers/resources for ${CI_COMMIT_REF_NAME}"
+    - hub pull-request -m "[CI] Update app helpers/resources for ${CI_COMMIT_REF_NAME}" -p # GITHUB_USER and GITHUB_TOKEN registered here https://gitlab.com/yunohost/yunohost/-/settings/ci_cd
   artifacts:
     paths:
       - doc/helpers.md
+      - doc/resources.md
   only:
     - tags

.gitlab/ci/install.gitlab-ci.yml

@@ -14,16 +14,14 @@
 
 upgrade:
   extends: .install-stage
-  image: "after-install"
+  image: "core-tests"
   script:
-    - apt-get update -o Acquire::Retries=3
-    - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ./$YNH_BUILD_DIR/*.deb
+    - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb
 
 
 install-postinstall:
   extends: .install-stage
   image: "before-install"
   script:
-    - apt-get update -o Acquire::Retries=3
-    - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ./$YNH_BUILD_DIR/*.deb
+    - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb
     - yunohost tools postinstall -d domain.tld -u syssa -F 'Syssa Mine' -p the_password --ignore-dyndns --force-diskspace

.gitlab/ci/lint.gitlab-ci.yml

@@ -5,7 +5,7 @@
 
 lint39:
   stage: lint
-  image: "before-install"
+  image: "build-and-lint"
   needs: []
   allow_failure: true
   script:
@@ -13,36 +13,14 @@ lint39:
 
 invalidcode39:
   stage: lint
-  image: "before-install"
+  image: "build-and-lint"
   needs: []
   script:
     - tox -e py39-invalidcode
 
 mypy:
   stage: lint
-  image: "before-install"
+  image: "build-and-lint"
   needs: []
   script:
     - tox -e py39-mypy
-
-black:
-  stage: lint
-  image: "before-install"
-  needs: []
-  before_script:
-    - apt-get update -y && apt-get install git hub -y
-    - git config --global user.email "yunohost@yunohost.org"
-    - git config --global user.name "$GITHUB_USER"
-    - hub clone --branch ${CI_COMMIT_REF_NAME} "https://$GITHUB_TOKEN:x-oauth-basic@github.com/YunoHost/yunohost.git" github_repo
-    - cd github_repo
-  script:
-    # create a local branch that will overwrite distant one
-    - git checkout -b "ci-format-${CI_COMMIT_REF_NAME}" --no-track
-    - tox -e py39-black-run
-    - '[ $(git diff | wc -l) != 0 ] || exit 0'  # stop if there is nothing to commit
-    - git commit -am "[CI] Format code with Black" || true
-    - git push -f origin "ci-format-${CI_COMMIT_REF_NAME}":"ci-format-${CI_COMMIT_REF_NAME}"
-    - hub pull-request -m "[CI] Format code with Black" -b Yunohost:$CI_COMMIT_REF_NAME -p || true # GITHUB_USER and GITHUB_TOKEN registered here https://gitlab.com/yunohost/yunohost/-/settings/ci_cd
-  only:
-    variables:
-      - $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH

.gitlab/ci/test.gitlab-ci.yml

@@ -1,11 +1,9 @@
 .install_debs: &install_debs
-    - apt-get update -o Acquire::Retries=3
-    - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ./$YNH_BUILD_DIR/*.deb
-    - pip3 install -U mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2
+    - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb
 
 .test-stage:
   stage: test
-  image: "after-install"
+  image: "core-tests"
   variables:
     PYTEST_ADDOPTS: "--color=yes"
   before_script:
@@ -34,11 +32,10 @@ full-tests:
     PYTEST_ADDOPTS: "--color=yes"
   before_script:
     - *install_debs
+    - pip install mock pip pyOpenSSL pytest pytest-cov pytest-mock pytest-sugar requests-mock "packaging<22"
     - yunohost tools postinstall -d domain.tld -u syssa -F 'Syssa Mine' -p the_password --ignore-dyndns --force-diskspace
   script:
-    - python3 -m pytest --cov=yunohost tests/ src/tests/ src/diagnosers/ --junitxml=report.xml
-    - cd tests
-    - bash test_helpers.sh
+    - python3 -m pytest --cov=yunohost tests/ src/tests/ --junitxml=report.xml
   needs:
       - job: build-yunohost
         artifacts: true
@@ -46,6 +43,7 @@ full-tests:
         artifacts: true
       - job: build-moulinette
         artifacts: true
+  coverage: '/TOTAL.*\s+(\d+%)/'
   artifacts:
     reports:
       junit: report.xml
@@ -58,14 +56,17 @@ test-actionmap:
     changes:
       - share/actionsmap.yml
 
-test-helpers:
+test-helpers2:
   extends: .test-stage
   script:
     - cd tests
     - bash test_helpers.sh
-  only:
-    changes:
-      - helpers/*
+
+test-helpers2.1:
+  extends: .test-stage
+  script:
+    - cd tests
+    - bash test_helpers.sh 2.1
 
 test-domains:
   extends: .test-stage

.gitlab/ci/translation.gitlab-ci.yml

@@ -13,10 +13,9 @@ test-i18n-keys:
 
 autofix-translated-strings:
   stage: translation
-  image: "before-install"
+  image: "build-and-lint"
   needs: []
   before_script:
-    - apt-get update -y && apt-get install git hub -y
     - git config --global user.email "yunohost@yunohost.org"
     - git config --global user.name "$GITHUB_USER"
     - hub clone --branch ${CI_COMMIT_REF_NAME} "https://$GITHUB_TOKEN:x-oauth-basic@github.com/YunoHost/yunohost.git" github_repo
@@ -26,7 +25,7 @@ autofix-translated-strings:
     - git checkout -b "ci-autofix-translated-strings-${CI_COMMIT_REF_NAME}" --no-track
     - python3 maintenance/missing_i18n_keys.py --fix
     - python3 maintenance/autofix_locale_format.py
-    - '[ $(git diff | wc -l) != 0 ] || exit 0'  # stop if there is nothing to commit
+    - '[ $(git diff --ignore-blank-lines --ignore-all-space --ignore-space-at-eol --ignore-cr-at-eol | wc -l) != 0 ] || exit 0'  # stop if there is nothing to commit
     - git commit -am "[CI] Reformat / remove stale translated strings" || true
     - git push -f origin "ci-autofix-translated-strings-${CI_COMMIT_REF_NAME}":"ci-remove-stale-translated-strings-${CI_COMMIT_REF_NAME}"
     - hub pull-request -m "[CI] Reformat / remove stale translated strings" -b Yunohost:$CI_COMMIT_REF_NAME -p || true # GITHUB_USER and GITHUB_TOKEN registered here https://gitlab.com/yunohost/yunohost/-/settings/ci_cd

.lgtm.yml

@@ -1,4 +0,0 @@
-extraction:
-  python:
-    python_setup:
-      version: "3"

README.md

@@ -5,12 +5,12 @@
 <h1 align="center">YunoHost</h1>
 
 <div align="center">
-
+    
 ![Version](https://img.shields.io/github/v/tag/yunohost/yunohost?label=version&sort=semver)
-[![Build status](https://shields.io/gitlab/pipeline/yunohost/yunohost/dev)](https://gitlab.com/yunohost/yunohost/-/pipelines)
-![Test coverage](https://img.shields.io/gitlab/coverage/yunohost/yunohost/dev)
-[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/YunoHost/yunohost.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/YunoHost/yunohost/context:python)
-[![GitHub license](https://img.shields.io/github/license/YunoHost/yunohost)](https://github.com/YunoHost/yunohost/blob/dev/LICENSE)
+[![Pipeline status](https://gitlab.com/yunohost/yunohost/badges/dev/pipeline.svg)](https://gitlab.com/yunohost/yunohost/-/pipelines)
+![Test coverage](https://gitlab.com/yunohost/yunohost/badges/dev/coverage.svg)
+[![Project license](https://img.shields.io/gitlab/license/yunohost/yunohost)](https://github.com/YunoHost/yunohost/blob/dev/LICENSE)
+[![CodeQL](https://github.com/yunohost/yunohost/workflows/CodeQL/badge.svg)](https://github.com/YunoHost/yunohost/security/code-scanning)
 [![Mastodon Follow](https://img.shields.io/mastodon/follow/28084)](https://mastodon.social/@yunohost)
 
 </div>
@@ -19,31 +19,31 @@ YunoHost is an operating system aiming to simplify as much as possible the admin
 
 This repository corresponds to the core code of YunoHost, mainly written in Python and Bash.
 
-- [Project features](https://yunohost.org/#/whatsyunohost)
+- [Project features](https://yunohost.org/whatsyunohost)
 - [Project website](https://yunohost.org)
 - [Install documentation](https://yunohost.org/install)
 - [Issue tracker](https://github.com/YunoHost/issues)
 
-# Screenshots
+## Screenshots
 
 Webadmin ([Yunohost-Admin](https://github.com/YunoHost/yunohost-admin)) | Single sign-on user portal ([SSOwat](https://github.com/YunoHost/ssowat))
---- |  ---
-![](https://raw.githubusercontent.com/YunoHost/doc/master/images/webadmin.png) | ![](https://raw.githubusercontent.com/YunoHost/doc/master/images/user_panel.png)
+--- | ---
+![Web admin insterface screenshot](https://raw.githubusercontent.com/YunoHost/doc/master/images/webadmin.png) | ![User portal screenshot](https://raw.githubusercontent.com/YunoHost/doc/master/images/user_panel.png)
 
 
 ## Contributing
 
 - You can learn how to get started with developing on YunoHost by reading [this piece of documentation](https://yunohost.org/dev).
-- Come chat with us on the [dev chatroom](https://yunohost.org/#/chat_rooms) !
-- You can help translate YunoHost on our [translation platform](https://translate.yunohost.org/engage/yunohost/?utm_source=widget)
+- Come chat with us on the [dev chatroom](https://yunohost.org/chat_rooms)!
+- You can help translate YunoHost on our [translation platform](https://translate.yunohost.org/engage/yunohost/?utm_source=widget).
 
 <p align="center">
-<img src="https://translate.yunohost.org/widgets/yunohost/-/core/horizontal-auto.svg" alt="Translation status" />
+<img alt="View of the translation rate for the different languages available in YunoHost" src="https://translate.yunohost.org/widgets/yunohost/-/core/horizontal-auto.svg" alt="Translation status" />
 </p>
 
 ## License
 
-As [other components of YunoHost](https://yunohost.org/#/faq_en), this repository is licensed under GNU AGPL v3.
+As [other components of YunoHost](https://yunohost.org/faq), this repository is licensed under GNU AGPL v3.
 
 ## They support us <3
 
@@ -51,16 +51,16 @@ We are thankful for our sponsors providing us with infrastructure and grants!
 
 <div align="center">
 <p style="margin-left:auto;margin-right:auto;">
-<a style="padding: 5px;" href="https://nlnet.nl"><img src="https://user-images.githubusercontent.com/36127788/198088570-823c40bd-7ac3-44e3-a8ee-e7a9f14b47ac.png" width="150px"/></a>
-<a style="padding: 5px;" href="https://www.ngi.eu"><img src="https://user-images.githubusercontent.com/36127788/198088663-daf587b9-fd09-4c00-aaf2-37c803939c94.png" width="130px"/></a>
-<a style="padding: 5px;" href="https://www.codelutin.com"><img src="https://user-images.githubusercontent.com/36127788/198088737-d37b6674-379c-4be4-9d74-b93b6ad318d1.png" width="100px"/></a>
+<a style="padding: 5px;" href="https://nlnet.nl"><img alt="NLnet Foundation" src="https://user-images.githubusercontent.com/36127788/198088570-823c40bd-7ac3-44e3-a8ee-e7a9f14b47ac.png" width="150px"/></a>
+<a style="padding: 5px;" href="https://www.ngi.eu"><img alt="Next Generation Internet" src="https://user-images.githubusercontent.com/36127788/198088663-daf587b9-fd09-4c00-aaf2-37c803939c94.png" width="130px"/></a>
+<a style="padding: 5px;" href="https://www.codelutin.com"><img alt="Code Lutin" src="https://user-images.githubusercontent.com/36127788/198088737-d37b6674-379c-4be4-9d74-b93b6ad318d1.png" width="100px"/></a>
 </p>
 <p style="margin-left:auto;margin-right:auto;">
-<a style="padding: 5px;" href="https://www.globenet.org"><img src="https://user-images.githubusercontent.com/36127788/198088794-751129ab-737d-4d99-9f35-5e01845dcdfe.png" width="150px"/></a>
-<a style="padding: 5px;" href="https://www.gitoyen.net"><img src="https://user-images.githubusercontent.com/36127788/198088931-f16f4af4-57ae-42e9-8d42-fb3e2d8d7ee3.png" width="150px"/></a>
-<a style="padding: 5px;" href="https://tetaneutral.net"><img src="https://user-images.githubusercontent.com/36127788/198088995-3ad9c34d-9807-4ead-934b-44df97d3c552.png" width="90px"/></a>
-<a style="padding: 5px;" href="https://ldn-fai.net"><img src="https://user-images.githubusercontent.com/36127788/198089086-a4089d51-9173-4081-bd2e-fa1ac3378e49.png" width="120px"/></a>
-<a style="padding: 5px;" href="https://www.nbs-system.com"><img src="https://user-images.githubusercontent.com/36127788/198089161-4cc0b7b7-bf56-4798-892e-a76112497921.png" width="130px"/></a>
+<a style="padding: 5px;" href="https://www.globenet.org"><img alt="Globenet" src="https://user-images.githubusercontent.com/36127788/198088794-751129ab-737d-4d99-9f35-5e01845dcdfe.png" width="150px"/></a>
+<a style="padding: 5px;" href="https://www.gitoyen.net"><img alt="Gitoyen" src="https://user-images.githubusercontent.com/36127788/198088931-f16f4af4-57ae-42e9-8d42-fb3e2d8d7ee3.png" width="150px"/></a>
+<a style="padding: 5px;" href="https://tetaneutral.net"><img alt="tetaneutral.net" src="https://user-images.githubusercontent.com/36127788/198088995-3ad9c34d-9807-4ead-934b-44df97d3c552.png" width="90px"/></a>
+<a style="padding: 5px;" href="https://ldn-fai.net"><img alt="LDN (Lorraine Data Network)" src="https://user-images.githubusercontent.com/36127788/198089086-a4089d51-9173-4081-bd2e-fa1ac3378e49.png" width="120px"/></a>
+<a style="padding: 5px;" href="https://www.nbs-system.com"><img alt="NBS System" src="https://user-images.githubusercontent.com/36127788/198089161-4cc0b7b7-bf56-4798-892e-a76112497921.png" width="130px"/></a>
 </p>
 </div>
 

bin/yunohost

@@ -24,6 +24,9 @@ def _parse_cli_args():
     parser.add_argument(
         "--quiet", action="store_true", default=False, help="Don't produce any output"
     )
+    parser.add_argument(
+        "--version", action="store_true", default=False, help="Display YunoHost packages versions (alias to 'yunohost tools versions')"
+    )
     parser.add_argument(
         "--timeout",
         type=int,
@@ -50,6 +53,7 @@ def _parse_cli_args():
 
 
 # Stupid PATH management because sometimes (e.g. some cron job) PATH is only /usr/bin:/bin ...
+
 default_path = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 if os.environ["PATH"] != default_path:
     os.environ["PATH"] = default_path + ":" + os.environ["PATH"]
@@ -66,6 +70,9 @@ if __name__ == "__main__":
 
     parser, opts, args = _parse_cli_args()
 
+    if opts.version:
+        args = ["tools", "versions"]
+
     # Execute the action
     yunohost.cli(
         debug=opts.debug,

bin/yunopaste

@@ -1,77 +1,34 @@
-#!/bin/bash
+#!/usr/bin/env python3
 
-set -e
-set -u
+import sys
+import requests
+import json
 
-PASTE_URL="https://paste.yunohost.org"
+SERVER_URL = "https://paste.yunohost.org"
+TIMEOUT = 3
 
-_die() {
-  printf "Error: %s\n" "$*"
-  exit 1
-}
+def create_snippet(data):
+    try:
+        url = SERVER_URL + "/documents"
+        response = requests.post(url, data=data.encode('utf-8'), timeout=TIMEOUT)
+        response.raise_for_status()
+        dockey = json.loads(response.text)['key']
+        return SERVER_URL + "/raw/" + dockey
+    except requests.exceptions.RequestException as e:
+        print("\033[31mError: {}\033[0m".format(e))
+        sys.exit(1)
 
-check_dependencies() {
-  curl -V > /dev/null 2>&1 || _die "This script requires curl."
-}
 
-paste_data() {
-  json=$(curl -X POST -s -d "$1" "${PASTE_URL}/documents")
-  [[ -z "$json" ]] && _die "Unable to post the data to the server."
+def main():
+    output = sys.stdin.read()
 
-  key=$(echo "$json" \
-    | python -c 'import json,sys;o=json.load(sys.stdin);print o["key"]' \
-        2>/dev/null)
-  [[ -z "$key" ]] && _die "Unable to parse the server response."
+    if not output:
+        print("\033[31mError: No input received from stdin.\033[0m")
+        sys.exit(1)
 
-  echo "${PASTE_URL}/${key}"
-}
+    url = create_snippet(output)
 
-usage() {
-  printf "Usage: ${0} [OPTION]...
+    print("\033[32mURL: {}\033[0m".format(url))
 
-Read from input stream and paste the data to the YunoHost
-Haste server.
-
-For example, to paste the output of the YunoHost diagnosis, you
-can simply execute the following:
-  yunohost diagnosis show | ${0}
-
-It will return the URL where you can access the pasted data.
-
-Options:
-  -h, --help   show this help message and exit
-"
-}
-
-main() {
-  # parse options
-  while (( ${#} )); do
-    case "${1}" in
-      --help|-h)
-        usage
-        exit 0
-        ;;
-      *)
-        echo "Unknown parameter detected: ${1}" >&2
-        echo >&2
-        usage >&2
-        exit 1
-        ;;
-    esac
-
-    shift 1
-  done
-
-  # check input stream
-  read -t 0 || {
-    echo -e "Invalid usage: No input is provided.\n" >&2
-    usage
-    exit 1
-  }
-
-  paste_data "$(cat)"
-}
-
-check_dependencies
-
-main "${@}"
+if __name__ == "__main__":
+    main()

bin/yunoprompt

@@ -56,7 +56,7 @@ EOF
 
 echo "$LOGO_AND_FINGERPRINTS" > /etc/issue
 
-if [[ ! -f /etc/yunohost/installed ]]
+if ! groups | grep -q all_users && [[ ! -f /etc/yunohost/installed ]]
 then
     chvt 2
 
@@ -69,7 +69,7 @@ then
 You should now proceed with YunoHost post-installation. This is where you will
 be asked for:
   - the main domain of your server;
-  - the administration password.
+  - the username and password for the first admin
 
 You can perform this step:
   - from your web browser, by accessing: https://yunohost.local/ or ${local_ip}

conf/dovecot/dovecot.conf

@@ -10,7 +10,7 @@ mail_uid = 500
 
 protocols = imap sieve {% if pop3_enabled == "True" %}pop3{% endif %}
 
-mail_plugins = $mail_plugins quota
+mail_plugins = $mail_plugins quota notify push_notification
 
 ###############################################################################
 
@@ -38,14 +38,26 @@ ssl_prefer_server_ciphers = no
 ###############################################################################
 
 
+# Regular Yunohost accounts
 passdb {
   args = /etc/dovecot/dovecot-ldap.conf
   driver = ldap
 }
 
+# Internally, allow authentication from apps system user who have "enable_email = true"
+passdb {
+  driver = passwd-file
+  args = /etc/dovecot/app-senders-passwd
+}
+
 userdb {
-  args = /etc/dovecot/dovecot-ldap.conf
   driver = ldap
+  args = /etc/dovecot/dovecot-ldap.conf
+}
+
+userdb {
+  driver = passwd-file
+  args = username_format=%n /etc/dovecot/app-senders-passwd
 }
 
 protocol imap {
@@ -53,13 +65,40 @@ protocol imap {
   mail_plugins = $mail_plugins imap_quota antispam
 }
 
-
 protocol lda {
   auth_socket_path = /var/run/dovecot/auth-master
   mail_plugins = quota sieve
   postmaster_address = postmaster@{{ main_domain }}
 }
 
+namespace inbox {
+  inbox = yes
+
+  mailbox Drafts {
+    special_use = \Drafts
+    auto = subscribe
+  }
+  mailbox Junk {
+    special_use = \Junk
+    auto = subscribe
+  }
+  mailbox Trash {
+    special_use = \Trash
+    auto = subscribe
+  }
+  mailbox Sent {
+    special_use = \Sent
+    auto = subscribe
+  }
+  mailbox "Sent Messages" {
+    special_use = \Sent
+  }
+  mailbox "Archive" {
+    special_use = \Archive
+    auto = subscribe
+  }
+}
+
 protocol sieve {
 }
 
@@ -107,8 +146,8 @@ plugin {
   antispam_debug_target = syslog
   antispam_verbose_debug = 0
   antispam_backend = pipe
-  antispam_spam = Junk;SPAM
-  antispam_trash = Trash
+  antispam_spam_pattern_ignorecase = junk;spam
+  antispam_trash_pattern_ignorecase = trash;papierkorb;deleted messages
   antispam_pipe_program = /usr/bin/rspamc
   antispam_pipe_program_args = -h;localhost:11334;-P;q1
   antispam_pipe_program_spam_arg = learn_spam

conf/dovecot/dovecot.sieve

@@ -1,4 +1,4 @@
 require "fileinto"; 
-    if header :contains "X-Spam-Flag" "YES" { 
+    if header :contains "X-Spam-Flag" "Yes" { 
         fileinto "Junk"; 
     }

conf/fail2ban/postfix-sasl.conf

@@ -0,0 +1,6 @@
+# Fail2Ban filter for postfix authentication failures
+[INCLUDES]
+before = common.conf
+[Definition]
+_daemon = postfix/smtpd
+failregex = ^%(__prefix_line)swarning: [-._\w]+\[<HOST>\]: SASL (?:LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(: [ A-Za-z0-9+/]*={0,2})?\s*$

conf/fail2ban/yunohost-jails.conf

@@ -8,6 +8,13 @@ enabled = true
 [postfix]
 enabled = true
 
+[sasl]
+enabled  = true
+port     = smtp
+filter   = postfix-sasl
+logpath  = /var/log/mail.log
+maxretry = 5
+
 [dovecot]
 enabled = true
 

conf/metronome/domain.tpl.cfg.lua

@@ -1,28 +1,28 @@
 VirtualHost "{{ domain }}"
-  enable = true
-  ssl = {
-        key = "/etc/yunohost/certs/{{ domain }}/key.pem";
-        certificate = "/etc/yunohost/certs/{{ domain }}/crt.pem";
-  }
-  authentication = "ldap2"
-  ldap = {
-     hostname      = "localhost",
-     user = {
-       basedn        = "ou=users,dc=yunohost,dc=org",
-       filter        = "(&(objectClass=posixAccount)(mail=*@{{ domain }})(permission=cn=xmpp.main,ou=permission,dc=yunohost,dc=org))",
-       usernamefield = "mail",
-       namefield     = "cn",
-       },
-  }
+    enable = true
+    ssl = {
+                key = "/etc/yunohost/certs/{{ domain }}/key.pem";
+                certificate = "/etc/yunohost/certs/{{ domain }}/crt.pem";
+    }
+    authentication = "ldap2"
+    ldap = {
+         hostname      = "localhost",
+         user = {
+             basedn        = "ou=users,dc=yunohost,dc=org",
+             filter        = "(&(objectClass=posixAccount)(mail=*@{{ domain }})(permission=cn=xmpp.main,ou=permission,dc=yunohost,dc=org))",
+             usernamefield = "mail",
+             namefield     = "cn",
+             },
+    }
 
-  -- Discovery items
-  disco_items = {
-    { "muc.{{ domain }}" },
-    { "pubsub.{{ domain }}" },
-    { "jabber.{{ domain }}" },
-    { "vjud.{{ domain }}" },
-    { "xmpp-upload.{{ domain }}" },
-  };
+    -- Discovery items
+    disco_items = {
+        { "muc.{{ domain }}" },
+        { "pubsub.{{ domain }}" },
+        { "jabber.{{ domain }}" },
+        { "vjud.{{ domain }}" },
+        { "xmpp-upload.{{ domain }}" },
+    };
 
 --  contact_info = {
 --    abuse = { "mailto:abuse@{{ domain }}", "xmpp:admin@{{ domain }}" };
@@ -35,41 +35,41 @@ VirtualHost "{{ domain }}"
 
 ---Set up a MUC (multi-user chat) room server
 Component "muc.{{ domain }}" "muc"
-  name = "{{ domain }} Chatrooms"
+    name = "{{ domain }} Chatrooms"
 
-  modules_enabled = {
-    "muc_limits";
-    "muc_log";
-    "muc_log_mam";
-    "muc_log_http";
-    "muc_vcard";
-  }
+    modules_enabled = {
+        "muc_limits";
+        "muc_log";
+        "muc_log_mam";
+        "muc_log_http";
+        "muc_vcard";
+    }
 
-  muc_event_rate = 0.5
-  muc_burst_factor = 10
-  room_default_config = {
-    logging = true,
-    persistent = true
-  };
+    muc_event_rate = 0.5
+    muc_burst_factor = 10
+    room_default_config = {
+        logging = true,
+        persistent = true
+    };
 
 ---Set up a PubSub server
 Component "pubsub.{{ domain }}" "pubsub"
-  name = "{{ domain }} Publish/Subscribe"
+    name = "{{ domain }} Publish/Subscribe"
 
-  unrestricted_node_creation = true -- Anyone can create a PubSub node (from any server)
+    unrestricted_node_creation = true -- Anyone can create a PubSub node (from any server)
 
 ---Set up a HTTP Upload service
 Component "xmpp-upload.{{ domain }}" "http_upload"
-  name = "{{ domain }} Sharing Service"
+    name = "{{ domain }} Sharing Service"
 
-  http_file_path = "/var/xmpp-upload/{{ domain }}/upload"
-  http_external_url = "https://xmpp-upload.{{ domain }}:443"
-  http_file_base_path = "/upload"
-  http_file_size_limit = 6*1024*1024
-  http_file_quota = 60*1024*1024
-  http_upload_file_size_limit = 100 * 1024 * 1024 -- bytes
-  http_upload_quota = 10 * 1024 * 1024 * 1024 -- bytes
+    http_file_path = "/var/xmpp-upload/{{ domain }}/upload"
+    http_external_url = "https://xmpp-upload.{{ domain }}:443"
+    http_file_base_path = "/upload"
+    http_file_size_limit = 6*1024*1024
+    http_file_quota = 60*1024*1024
+    http_upload_file_size_limit = 100 * 1024 * 1024 -- bytes
+    http_upload_quota = 10 * 1024 * 1024 * 1024 -- bytes
 
 ---Set up a VJUD service
 Component "vjud.{{ domain }}" "vjud"
-  vjud_disco_name = "{{ domain }} User Directory"
+    vjud_disco_name = "{{ domain }} User Directory"

conf/metronome/metronome.cfg.lua

@@ -1,72 +1,72 @@
 -- ** Metronome's config file example **
--- 
+--
 -- The format is exactly equal to Prosody's:
 --
--- Lists are written { "like", "this", "one" } 
--- Lists can also be of { 1, 2, 3 } numbers, etc. 
+-- Lists are written { "like", "this", "one" }
+-- Lists can also be of { 1, 2, 3 } numbers, etc.
 -- Either commas, or semi-colons; may be used as seperators.
 --
--- A table is a list of values, except each value has a name. An 
+-- A table is a list of values, except each value has a name. An
 -- example would be:
 --
 -- ssl = { key = "keyfile.key", certificate = "certificate.cert" }
 --
 -- Tip: You can check that the syntax of this file is correct when you have finished
 -- by running: luac -p metronome.cfg.lua
--- If there are any errors, it will let you know what and where they are, otherwise it 
+-- If there are any errors, it will let you know what and where they are, otherwise it
 -- will keep quiet.
 
 -- Global settings go in this section
- 
+
 -- This is the list of modules Metronome will load on startup.
 -- It looks for mod_modulename.lua in the plugins folder, so make sure that exists too.
 
 modules_enabled = {
-	-- Generally required
-		"roster"; -- Allow users to have a roster. Recommended.
-		"saslauth"; -- Authentication for clients. Recommended if you want to log in.
-		"tls"; -- Add support for secure TLS on c2s/s2s connections
-		"disco"; -- Service discovery
- 
-	-- Not essential, but recommended
-		"private"; -- Private XML storage (for room bookmarks, etc.)
-		"vcard"; -- Allow users to set vCards
-		"pep"; -- Allows setting of mood, tune, etc.
-		"pubsub";  -- Publish-subscribe XEP-0060
-		"posix"; -- POSIX functionality, sends server to background, enables syslog, etc.
-		"bidi"; -- Enables Bidirectional Server-to-Server Streams.
- 
-	-- Nice to have
-		"version"; -- Replies to server version requests
-		"uptime"; -- Report how long server has been running
-		"time"; -- Let others know the time here on this server
-		"ping"; -- Replies to XMPP pings with pongs
-		"register"; -- Allow users to register on this server using a client and change passwords
-		"stream_management"; -- Allows clients and servers to use Stream Management
-		"stanza_optimizations"; -- Allows clients to use Client State Indication and SIFT
-		"message_carbons"; -- Allows clients to enable carbon copies of messages
-		"mam"; -- Enable server-side message archives using Message Archive Management
-		"push"; -- Enable Push Notifications via PubSub using XEP-0357
-		"lastactivity"; -- Enables clients to know the last presence status of an user
-		"adhoc_cm"; -- Allow to set client certificates to login through SASL External via adhoc
-		"admin_adhoc"; -- administration adhoc commands
-		"bookmarks"; -- XEP-0048 Bookmarks synchronization between PEP and Private Storage
-		"sec_labels"; -- Allows to use a simplified version XEP-0258 Security Labels and related ACDFs.
-		"privacy"; -- Add privacy lists and simple blocking command support
+    -- Generally required
+        "roster"; -- Allow users to have a roster. Recommended.
+        "saslauth"; -- Authentication for clients. Recommended if you want to log in.
+        "tls"; -- Add support for secure TLS on c2s/s2s connections
+        "disco"; -- Service discovery
 
-		-- Other specific functionality
-		--"admin_telnet"; -- administration console, telnet to port 5582
-		--"admin_web"; -- administration web interface
-		"bosh"; -- Enable support for BOSH clients, aka "XMPP over Bidirectional Streams over Synchronous HTTP"
-		--"compression"; -- Allow clients to enable Stream Compression
-		--"spim_block"; -- Require authorization via OOB form for messages from non-contacts and block unsollicited messages
-		--"gate_guard"; -- Enable config-based blacklisting and hit-based auto-banning features
-		--"incidents_handling"; -- Enable Incidents Handling support (can be administered via adhoc commands)
-		--"server_presence"; -- Enables Server Buddies extension support
-		--"service_directory"; -- Enables Service Directories extension support
-		--"public_service"; -- Enables Server vCard support for public services in directories and advertises in features
-		--"register_api"; -- Provides secure API for both Out-Of-Band and In-Band registration for E-Mail verification
-		"websocket"; -- Enable support for WebSocket clients, aka "XMPP over WebSockets"
+    -- Not essential, but recommended
+        "private"; -- Private XML storage (for room bookmarks, etc.)
+        "vcard"; -- Allow users to set vCards
+        "pep"; -- Allows setting of mood, tune, etc.
+        "pubsub";  -- Publish-subscribe XEP-0060
+        "posix"; -- POSIX functionality, sends server to background, enables syslog, etc.
+        "bidi"; -- Enables Bidirectional Server-to-Server Streams.
+
+    -- Nice to have
+        "version"; -- Replies to server version requests
+        "uptime"; -- Report how long server has been running
+        "time"; -- Let others know the time here on this server
+        "ping"; -- Replies to XMPP pings with pongs
+        "register"; -- Allow users to register on this server using a client and change passwords
+        "stream_management"; -- Allows clients and servers to use Stream Management
+        "stanza_optimizations"; -- Allows clients to use Client State Indication and SIFT
+        "message_carbons"; -- Allows clients to enable carbon copies of messages
+        "mam"; -- Enable server-side message archives using Message Archive Management
+        "push"; -- Enable Push Notifications via PubSub using XEP-0357
+        "lastactivity"; -- Enables clients to know the last presence status of an user
+        "adhoc_cm"; -- Allow to set client certificates to login through SASL External via adhoc
+        "admin_adhoc"; -- administration adhoc commands
+        "bookmarks"; -- XEP-0048 Bookmarks synchronization between PEP and Private Storage
+        "sec_labels"; -- Allows to use a simplified version XEP-0258 Security Labels and related ACDFs.
+        "privacy"; -- Add privacy lists and simple blocking command support
+
+        -- Other specific functionality
+        --"admin_telnet"; -- administration console, telnet to port 5582
+        --"admin_web"; -- administration web interface
+        "bosh"; -- Enable support for BOSH clients, aka "XMPP over Bidirectional Streams over Synchronous HTTP"
+        --"compression"; -- Allow clients to enable Stream Compression
+        --"spim_block"; -- Require authorization via OOB form for messages from non-contacts and block unsollicited messages
+        --"gate_guard"; -- Enable config-based blacklisting and hit-based auto-banning features
+        --"incidents_handling"; -- Enable Incidents Handling support (can be administered via adhoc commands)
+        --"server_presence"; -- Enables Server Buddies extension support
+        --"service_directory"; -- Enables Service Directories extension support
+        --"public_service"; -- Enables Server vCard support for public services in directories and advertises in features
+        --"register_api"; -- Provides secure API for both Out-Of-Band and In-Band registration for E-Mail verification
+        "websocket"; -- Enable support for WebSocket clients, aka "XMPP over WebSockets"
 };
 
 -- Server PID
@@ -102,10 +102,10 @@ csi_config_queue_all_muc_messages_but_mentions = false;
 
 -- Logging configuration
 log = {
-	info = "/var/log/metronome/metronome.log"; -- Change 'info' to 'debug' for verbose logging
-	error = "/var/log/metronome/metronome.err";
-	-- "*syslog"; -- Uncomment this for logging to syslog
-	-- "*console"; -- Log to the console, useful for debugging with daemonize=false
+    info = "/var/log/metronome/metronome.log"; -- Change 'info' to 'debug' for verbose logging
+    error = "/var/log/metronome/metronome.err";
+    -- "*syslog"; -- Uncomment this for logging to syslog
+    -- "*console"; -- Log to the console, useful for debugging with daemonize=false
 }
 
 ------ Components ------
@@ -114,7 +114,7 @@ log = {
 
 ---Set up a local BOSH service
 Component "localhost" "http"
-	modules_enabled = { "bosh" }
+    modules_enabled = { "bosh" }
 
 ----------- Virtual hosts -----------
 -- You need to add a VirtualHost entry for each domain you wish Metronome to serve.

conf/metronome/modules/mod_auth_ldap2.lua

@@ -23,68 +23,68 @@ if not ldap then
 end
 
 function new_default_provider(host)
- local provider = { name = "ldap2" };
- log("debug", "initializing ldap2 authentication provider for host '%s'", host);
+    local provider = { name = "ldap2" };
+    log("debug", "initializing ldap2 authentication provider for host '%s'", host);
 
- function provider.test_password(username, password)
-     return ldap.bind(username, password);
- end
+    function provider.test_password(username, password)
+        return ldap.bind(username, password);
+    end
 
- function provider.user_exists(username)
-     local params = ldap.getparams()
+    function provider.user_exists(username)
+        local params = ldap.getparams()
 
-     local filter = ldap.filter.combine_and(params.user.filter, params.user.usernamefield .. '=' .. username);
-     if params.user.usernamefield == 'mail' then
-         filter = ldap.filter.combine_and(params.user.filter, 'mail=' .. username .. '@*');
-     end
+        local filter = ldap.filter.combine_and(params.user.filter, params.user.usernamefield .. '=' .. username);
+        if params.user.usernamefield == 'mail' then
+            filter = ldap.filter.combine_and(params.user.filter, 'mail=' .. username .. '@*');
+        end
 
-     return ldap.singlematch {
-         base   = params.user.basedn,
-         filter = filter,
-     };
- end
+        return ldap.singlematch {
+            base   = params.user.basedn,
+            filter = filter,
+        };
+    end
 
- function provider.get_password(username)
-     return nil, "Passwords unavailable for LDAP.";
- end
+    function provider.get_password(username)
+        return nil, "Passwords unavailable for LDAP.";
+    end
 
- function provider.set_password(username, password)
-     return nil, "Passwords unavailable for LDAP.";
- end
+    function provider.set_password(username, password)
+        return nil, "Passwords unavailable for LDAP.";
+    end
 
- function provider.create_user(username, password)
-     return nil, "Account creation/modification not available with LDAP.";
- end
+    function provider.create_user(username, password)
+        return nil, "Account creation/modification not available with LDAP.";
+    end
 
- function provider.get_sasl_handler(session)
-     local testpass_authentication_profile = {
-         session = session,
-         plain_test = function(sasl, username, password, realm)
-             return provider.test_password(username, password), true;
-         end,
-         order = { "plain_test" },
-     };
-     return new_sasl(module.host, testpass_authentication_profile);
- end
+    function provider.get_sasl_handler(session)
+        local testpass_authentication_profile = {
+            session = session,
+            plain_test = function(sasl, username, password, realm)
+                return provider.test_password(username, password), true;
+            end,
+            order = { "plain_test" },
+        };
+        return new_sasl(module.host, testpass_authentication_profile);
+    end
 
- function provider.is_admin(jid)
-     local admin_config = ldap.getparams().admin;
+    function provider.is_admin(jid)
+        local admin_config = ldap.getparams().admin;
 
-     if not admin_config then
-         return;
-     end
+        if not admin_config then
+            return;
+        end
 
-     local ld       = ldap:getconnection();
-     local username = jsplit(jid);
-     local filter   = ldap.filter.combine_and(admin_config.filter, admin_config.namefield .. '=' .. username);
+        local ld       = ldap:getconnection();
+        local username = jsplit(jid);
+        local filter   = ldap.filter.combine_and(admin_config.filter, admin_config.namefield .. '=' .. username);
 
-     return ldap.singlematch {
-         base   = admin_config.basedn,
-         filter = filter,
-     };
- end
+        return ldap.singlematch {
+            base   = admin_config.basedn,
+            filter = filter,
+        };
+    end
 
- return provider;
+    return provider;
 end
 
 module:add_item("auth-provider", new_default_provider(module.host));

conf/metronome/modules/mod_legacyauth.lua

@@ -1,7 +1,7 @@
 -- Prosody IM
 -- Copyright (C) 2008-2010 Matthew Wild
 -- Copyright (C) 2008-2010 Waqas Hussain
--- 
+--
 -- This project is MIT/X11 licensed. Please see the
 -- COPYING file in the source package for more information.
 --
@@ -12,8 +12,8 @@ local st = require "util.stanza";
 local t_concat = table.concat;
 
 local secure_auth_only = module:get_option("c2s_require_encryption")
-	or module:get_option("require_encryption")
-	or not(module:get_option("allow_unencrypted_plain_auth"));
+    or module:get_option("require_encryption")
+    or not(module:get_option("allow_unencrypted_plain_auth"));
 
 local sessionmanager = require "core.sessionmanager";
 local usermanager = require "core.usermanager";
@@ -22,66 +22,65 @@ local resourceprep = require "util.encodings".stringprep.resourceprep;
 
 module:add_feature("jabber:iq:auth");
 module:hook("stream-features", function(event)
-	local origin, features = event.origin, event.features;
-	if secure_auth_only and not origin.secure then
-		-- Sorry, not offering to insecure streams!
-		return;
-	elseif not origin.username then
-		features:tag("auth", {xmlns='http://jabber.org/features/iq-auth'}):up();
-	end
+    local origin, features = event.origin, event.features;
+    if secure_auth_only and not origin.secure then
+        -- Sorry, not offering to insecure streams!
+        return;
+    elseif not origin.username then
+        features:tag("auth", {xmlns='http://jabber.org/features/iq-auth'}):up();
+    end
 end);
 
 module:hook("stanza/iq/jabber:iq:auth:query", function(event)
-	local session, stanza = event.origin, event.stanza;
+    local session, stanza = event.origin, event.stanza;
 
-	if session.type ~= "c2s_unauthed" then
-		(session.sends2s or session.send)(st.error_reply(stanza, "cancel", "service-unavailable", "Legacy authentication is only allowed for unauthenticated client connections."));
-		return true;
-	end
+    if session.type ~= "c2s_unauthed" then
+        (session.sends2s or session.send)(st.error_reply(stanza, "cancel", "service-unavailable", "Legacy authentication is only allowed for unauthenticated client connections."));
+        return true;
+    end
 
-	if secure_auth_only and not session.secure then
-		session.send(st.error_reply(stanza, "modify", "not-acceptable", "Encryption (SSL or TLS) is required to connect to this server"));
-		return true;
-	end
-	
-	local username = stanza.tags[1]:child_with_name("username");
-	local password = stanza.tags[1]:child_with_name("password");
-	local resource = stanza.tags[1]:child_with_name("resource");
-	if not (username and password and resource) then
-		local reply = st.reply(stanza);
-		session.send(reply:query("jabber:iq:auth")
-			:tag("username"):up()
-			:tag("password"):up()
-			:tag("resource"):up());
-	else
-		username, password, resource = t_concat(username), t_concat(password), t_concat(resource);
-		username = nodeprep(username);
-		resource = resourceprep(resource)
-		if not (username and resource) then
-			session.send(st.error_reply(stanza, "modify", "bad-request"));
-			return true;
-		end
-		if usermanager.test_password(username, session.host, password) then
-			-- Authentication successful!
-			local success, err = sessionmanager.make_authenticated(session, username);
-			if success then
-				local err_type, err_msg;
-				success, err_type, err, err_msg = sessionmanager.bind_resource(session, resource);
-				if not success then
-					session.send(st.error_reply(stanza, err_type, err, err_msg));
-					session.username, session.type = nil, "c2s_unauthed"; -- FIXME should this be placed in sessionmanager?
-					return true;
-				elseif resource ~= session.resource then -- server changed resource, not supported by legacy auth
-					session.send(st.error_reply(stanza, "cancel", "conflict", "The requested resource could not be assigned to this session."));
-					session:close(); -- FIXME undo resource bind and auth instead of closing the session?
-					return true;
-				end
-			end
-			session.send(st.reply(stanza));
-		else
-			session.send(st.error_reply(stanza, "auth", "not-authorized"));
-		end
-	end
-	return true;
+    if secure_auth_only and not session.secure then
+        session.send(st.error_reply(stanza, "modify", "not-acceptable", "Encryption (SSL or TLS) is required to connect to this server"));
+        return true;
+    end
+
+    local username = stanza.tags[1]:child_with_name("username");
+    local password = stanza.tags[1]:child_with_name("password");
+    local resource = stanza.tags[1]:child_with_name("resource");
+    if not (username and password and resource) then
+        local reply = st.reply(stanza);
+        session.send(reply:query("jabber:iq:auth")
+            :tag("username"):up()
+            :tag("password"):up()
+            :tag("resource"):up());
+    else
+        username, password, resource = t_concat(username), t_concat(password), t_concat(resource);
+        username = nodeprep(username);
+        resource = resourceprep(resource)
+        if not (username and resource) then
+            session.send(st.error_reply(stanza, "modify", "bad-request"));
+            return true;
+        end
+        if usermanager.test_password(username, session.host, password) then
+            -- Authentication successful!
+            local success, err = sessionmanager.make_authenticated(session, username);
+            if success then
+                local err_type, err_msg;
+                success, err_type, err, err_msg = sessionmanager.bind_resource(session, resource);
+                if not success then
+                    session.send(st.error_reply(stanza, err_type, err, err_msg));
+                    session.username, session.type = nil, "c2s_unauthed"; -- FIXME should this be placed in sessionmanager?
+                    return true;
+                elseif resource ~= session.resource then -- server changed resource, not supported by legacy auth
+                    session.send(st.error_reply(stanza, "cancel", "conflict", "The requested resource could not be assigned to this session."));
+                    session:close(); -- FIXME undo resource bind and auth instead of closing the session?
+                    return true;
+                end
+            end
+            session.send(st.reply(stanza));
+        else
+            session.send(st.error_reply(stanza, "auth", "not-authorized"));
+        end
+    end
+    return true;
 end);
-

conf/metronome/modules/mod_storage_ldap.lua

@@ -43,35 +43,35 @@ end
 local get_alias_for_user;
 
 do
-  local user_cache;
-  local last_fetch_time;
+    local user_cache;
+    local last_fetch_time;
 
-  local function populate_user_cache()
-      local user_c = get_config(module.host, 'ldap').user;
-      if not user_c then return; end
+    local function populate_user_cache()
+        local user_c = get_config(module.host, 'ldap').user;
+        if not user_c then return; end
 
-      local ld = ldap.getconnection();
+        local ld = ldap.getconnection();
 
-      local usernamefield = user_c.usernamefield;
-      local namefield     = user_c.namefield;
+        local usernamefield = user_c.usernamefield;
+        local namefield     = user_c.namefield;
 
-      user_cache = {};
+        user_cache = {};
 
-      for _, attrs in ld:search { base = user_c.basedn, scope = 'onelevel', filter = user_c.filter } do
-          user_cache[attrs[usernamefield]] = attrs[namefield];
-      end
-      last_fetch_time = gettime();
-  end
+        for _, attrs in ld:search { base = user_c.basedn, scope = 'onelevel', filter = user_c.filter } do
+            user_cache[attrs[usernamefield]] = attrs[namefield];
+        end
+        last_fetch_time = gettime();
+    end
 
-  function get_alias_for_user(user)
-      if last_fetch_time and last_fetch_time + CACHE_EXPIRY < gettime() then
-          user_cache = nil;
-      end
-      if not user_cache then
-          populate_user_cache();
-      end
-      return user_cache[user];
-  end
+    function get_alias_for_user(user)
+        if last_fetch_time and last_fetch_time + CACHE_EXPIRY < gettime() then
+            user_cache = nil;
+        end
+        if not user_cache then
+            populate_user_cache();
+        end
+        return user_cache[user];
+    end
 end
 
 ----------------------------------------
@@ -79,18 +79,18 @@ end
 ----------------------------------------
 
 local function ldap_store(config)
-  local self = {};
-  local config = config;
+    local self = {};
+    local config = config;
 
-  function self:get(username)
-      return nil, "Data getting is not available for this storage backend";
-  end
+    function self:get(username)
+        return nil, "Data getting is not available for this storage backend";
+    end
 
-  function self:set(username, data)
-      return nil, "Data setting is not available for this storage backend";
-  end
+    function self:set(username, data)
+        return nil, "Data setting is not available for this storage backend";
+    end
 
-  return self;
+    return self;
 end
 
 local adapters = {};
@@ -100,60 +100,60 @@ local adapters = {};
 ----------------------------------------
 
 adapters.roster = function (config)
-  -- Validate configuration requirements
-  if not config.groups then return nil; end
+    -- Validate configuration requirements
+    if not config.groups then return nil; end
 
-  local self = ldap_store(config)
+    local self = ldap_store(config)
 
-  function self:get(username)
-      local ld = ldap.getconnection();
-      local contacts = {};
+    function self:get(username)
+        local ld = ldap.getconnection();
+        local contacts = {};
 
-      local memberfield = config.groups.memberfield;
-      local namefield   = config.groups.namefield;
-      local filter      = memberfield .. '=' .. tostring(username);
+        local memberfield = config.groups.memberfield;
+        local namefield   = config.groups.namefield;
+        local filter      = memberfield .. '=' .. tostring(username);
 
-      local groups = {};
-      for _, config in ipairs(config.groups) do
-          groups[ config[namefield] ] = config.name;
-      end
+        local groups = {};
+        for _, config in ipairs(config.groups) do
+            groups[ config[namefield] ] = config.name;
+        end
 
-      log("debug", "Found %d group(s) for user %s", select('#', groups), username)
+        log("debug", "Found %d group(s) for user %s", select('#', groups), username)
 
-      -- XXX this kind of relies on the way we do groups at INOC
-      for _, attrs in ld:search { base = config.groups.basedn, scope = 'onelevel', filter = filter } do
-          if groups[ attrs[namefield] ] then
-              local members = attrs[memberfield];
+        -- XXX this kind of relies on the way we do groups at INOC
+        for _, attrs in ld:search { base = config.groups.basedn, scope = 'onelevel', filter = filter } do
+            if groups[ attrs[namefield] ] then
+                local members = attrs[memberfield];
 
-              for _, user in ipairs(members) do
-                  if user ~= username then
-                      local jid    = user .. '@' .. module.host;
-                      local record = contacts[jid];
+                for _, user in ipairs(members) do
+                    if user ~= username then
+                        local jid    = user .. '@' .. module.host;
+                        local record = contacts[jid];
 
-                      if not record then
-                          record = {
-                              subscription = 'both',
-                              groups       = {},
-                              name         = get_alias_for_user(user),
-                          };
-                          contacts[jid] = record;
-                      end
+                        if not record then
+                            record = {
+                                subscription = 'both',
+                                groups       = {},
+                                name         = get_alias_for_user(user),
+                            };
+                            contacts[jid] = record;
+                        end
 
-                      record.groups[ groups[ attrs[namefield] ] ] = true;
-                  end
-              end
-          end
-      end
+                        record.groups[ groups[ attrs[namefield] ] ] = true;
+                    end
+                end
+            end
+        end
 
-      return contacts;
-  end
+        return contacts;
+    end
 
-  function self:set(username, data)
-      log("warn", "Setting data in Roster LDAP storage is not supported yet")
-      return nil, "not supported";
-  end
+    function self:set(username, data)
+        log("warn", "Setting data in Roster LDAP storage is not supported yet")
+        return nil, "not supported";
+    end
 
-  return self;
+    return self;
 end
 
 ----------------------------------------
@@ -161,35 +161,35 @@ end
 ----------------------------------------
 
 adapters.vcard = function (config)
-  -- Validate configuration requirements
-  if not config.vcard_format or not config.user then return nil; end
+    -- Validate configuration requirements
+    if not config.vcard_format or not config.user then return nil; end
 
-  local self = ldap_store(config)
+    local self = ldap_store(config)
 
-  function self:get(username)
-      local ld     = ldap.getconnection();
-      local filter = config.user.usernamefield .. '=' .. tostring(username);
+    function self:get(username)
+        local ld     = ldap.getconnection();
+        local filter = config.user.usernamefield .. '=' .. tostring(username);
 
-      log("debug", "Retrieving vCard for user '%s'", username);
+        log("debug", "Retrieving vCard for user '%s'", username);
 
-      local match = ldap.singlematch {
-          base   = config.user.basedn,
-          filter = filter,
-      };
-      if match then
-          match.jid = username .. '@' .. module.host
-          return st.preserialize(ldap_record_to_vcard(match, config.vcard_format));
-      else
-          return nil, "username not found";
-      end
-  end
+        local match = ldap.singlematch {
+            base   = config.user.basedn,
+            filter = filter,
+        };
+        if match then
+            match.jid = username .. '@' .. module.host
+            return st.preserialize(ldap_record_to_vcard(match, config.vcard_format));
+        else
+            return nil, "username not found";
+        end
+    end
 
-  function self:set(username, data)
-      log("warn", "Setting data in vCard LDAP storage is not supported yet")
-      return nil, "not supported";
-  end
+    function self:set(username, data)
+        log("warn", "Setting data in vCard LDAP storage is not supported yet")
+        return nil, "not supported";
+    end
 
-  return self;
+    return self;
 end
 
 ----------------------------------------

conf/metronome/modules/vcard.lib.lua

@@ -135,28 +135,28 @@ function builder_methods:build()
 end
 
 local function new_builder(params)
-  local vcard_tag = st.stanza('vCard', { xmlns = VCARD_NS });
+    local vcard_tag = st.stanza('vCard', { xmlns = VCARD_NS });
 
-  local object = {
-      vcard   = vcard_tag,
-      __index = builder_methods,
-  };
+    local object = {
+        vcard   = vcard_tag,
+        __index = builder_methods,
+    };
 
-  for k, v in pairs(params) do
-      object[k] = v;
-  end
+    for k, v in pairs(params) do
+        object[k] = v;
+    end
 
-  setmetatable(object, object);
+    setmetatable(object, object);
 
-  return object;
+    return object;
 end
 
 local _M = {};
 
 function _M.create(params)
-  local builder = new_builder(params);
+    local builder = new_builder(params);
 
-  return builder:build();
+    return builder:build();
 end
 
 return _M;

conf/nginx/plain/acme-challenge.conf.inc

@@ -1,6 +1,6 @@
 location ^~ '/.well-known/acme-challenge/'
 {
         default_type "text/plain";
-        alias /tmp/acme-challenge-public/;
+        alias /var/www/.well-known/acme-challenge-public/;
         gzip off;
 }

conf/nginx/plain/yunohost_http_errors.conf.inc

@@ -0,0 +1,7 @@
+error_page 502 /502.html;
+
+location = /502.html {
+
+        root /usr/share/yunohost/html/;
+
+}

conf/nginx/redirect_to_admin.conf

@@ -1,3 +1,3 @@
 location / {
-    return 302 https://$http_host/yunohost/admin;
+    return 302 https://$host/yunohost/admin;
 }

conf/nginx/security.conf.inc

@@ -26,7 +26,7 @@ ssl_dhparam /usr/share/yunohost/ffdhe2048.pem;
 # https://wiki.mozilla.org/Security/Guidelines/Web_Security
 # https://observatory.mozilla.org/
 {% if experimental == "True" %}
-more_set_headers "Content-Security-Policy : upgrade-insecure-requests; default-src https: data: blob: ; object-src https: data: 'unsafe-inline'; style-src https: data: 'unsafe-inline' ; script-src https: data: 'unsafe-inline' 'unsafe-eval'";
+more_set_headers "Content-Security-Policy : upgrade-insecure-requests; default-src https: data: blob: ; object-src https: data: 'unsafe-inline'; style-src https: data: 'unsafe-inline' ; script-src https: data: 'unsafe-inline' 'unsafe-eval'; worker-src 'self' blob:;";
 {% else %}
 more_set_headers "Content-Security-Policy : upgrade-insecure-requests";
 {% endif %}

conf/nginx/server.tpl.conf

@@ -6,30 +6,34 @@ map $http_upgrade $connection_upgrade {
 server {
     listen 80;
     listen [::]:80;
-    server_name {{ domain }} xmpp-upload.{{ domain }};
+    server_name {{ domain }}{% if xmpp_enabled == "True" %} xmpp-upload.{{ domain }} muc.{{ domain }}{% endif %};
 
     access_by_lua_file /usr/share/ssowat/access.lua;
 
     include /etc/nginx/conf.d/acme-challenge.conf.inc;
 
     location ^~ '/.well-known/ynh-diagnosis/' {
-        alias /tmp/.well-known/ynh-diagnosis/;
+        alias /var/www/.well-known/ynh-diagnosis/;
     }
 
+    {% if mail_enabled == "True" %}
     location ^~ '/.well-known/autoconfig/mail/' {
         alias /var/www/.well-known/{{ domain }}/autoconfig/mail/;
     }
+    {% endif %}
 
     {# Note that this != "False" is meant to be failure-safe, in the case the redrect_to_https would happen to contain empty string or whatever value. We absolutely don't want to disable the HTTPS redirect *except* when it's explicitly being asked to be disabled. #}
     {% if redirect_to_https != "False" %}
     location / {
-        return 301 https://$http_host$request_uri;
+        return 301 https://$host$request_uri;
     }
     {# The app config snippets are not included in the HTTP conf unless HTTPS redirect is disabled, because app's location may blocks will conflict or bypass/ignore the HTTPS redirection. #}
     {% else %}
     include /etc/nginx/conf.d/{{ domain }}.d/*.conf;
     {% endif %}
 
+    include /etc/nginx/conf.d/yunohost_http_errors.conf.inc;
+
     access_log /var/log/nginx/{{ domain }}-access.log;
     error_log /var/log/nginx/{{ domain }}-error.log;
 }
@@ -52,13 +56,15 @@ server {
     ssl_stapling on;
     ssl_stapling_verify on;
     ssl_trusted_certificate /etc/yunohost/certs/{{ domain }}/crt.pem;
-    resolver 127.0.0.1 127.0.1.1 valid=300s;
+    resolver 1.1.1.1 9.9.9.9 valid=300s;
     resolver_timeout 5s;
     {% endif %}
 
+    {% if mail_enabled == "True" %}
     location ^~ '/.well-known/autoconfig/mail/' {
         alias /var/www/.well-known/{{ domain }}/autoconfig/mail/;
     }
+    {% endif %}
 
     access_by_lua_file /usr/share/ssowat/access.lua;
 
@@ -67,11 +73,13 @@ server {
     include /etc/nginx/conf.d/yunohost_sso.conf.inc;
     include /etc/nginx/conf.d/yunohost_admin.conf.inc;
     include /etc/nginx/conf.d/yunohost_api.conf.inc;
+    include /etc/nginx/conf.d/yunohost_http_errors.conf.inc;
 
     access_log /var/log/nginx/{{ domain }}-access.log;
     error_log /var/log/nginx/{{ domain }}-error.log;
 }
 
+{% if xmpp_enabled == "True" %}
 # vhost dedicated to XMPP http_upload
 server {
     listen 443 ssl http2;
@@ -107,10 +115,11 @@ server {
     ssl_stapling on;
     ssl_stapling_verify on;
     ssl_trusted_certificate /etc/yunohost/certs/{{ domain }}/crt.pem;
-    resolver 127.0.0.1 127.0.1.1 valid=300s;
+    resolver 1.1.1.1 9.9.9.9 valid=300s;
     resolver_timeout 5s;
     {% endif %}
 
     access_log /var/log/nginx/xmpp-upload.{{ domain }}-access.log;
     error_log /var/log/nginx/xmpp-upload.{{ domain }}-error.log;
 }
+{% endif %}

conf/nginx/yunohost_admin.conf.inc

@@ -7,9 +7,11 @@ location /yunohost/admin/ {
     index index.html;
 
     {% if webadmin_allowlist_enabled == "True" %}
-    {% for ip in webadmin_allowlist.split(',') %}
-    allow {{ ip }};
-    {% endfor %}
+    {% if webadmin_allowlist.strip() -%}
+    {% for ip in webadmin_allowlist.strip().split(',') -%}
+    allow {{ ip.strip() }};
+    {% endfor -%}
+    {% endif -%}
     deny all;
     {% endif %}
 
@@ -19,6 +21,10 @@ location /yunohost/admin/ {
         more_set_headers "Cache-Control: no-store, no-cache, must-revalidate";
     }
 
+    location /yunohost/admin/applogos/ {
+        alias /usr/share/yunohost/applogos/;
+    }
+
     more_set_headers "Content-Security-Policy: upgrade-insecure-requests; default-src 'self'; connect-src 'self' https://paste.yunohost.org wss://$host; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-eval'; object-src 'none'; img-src 'self' data:;";
     more_set_headers "Content-Security-Policy-Report-Only:";
 }

conf/nginx/yunohost_api.conf.inc

@@ -4,7 +4,7 @@ location /yunohost/api/ {
     proxy_http_version 1.1;
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection "upgrade";
-    proxy_set_header Host $http_host;
+    proxy_set_header Host $host;
 
     {% if webadmin_allowlist_enabled == "True" %}
     {% for ip in webadmin_allowlist.split(',') %}

conf/postfix/main.cf

@@ -93,21 +93,25 @@ recipient_delimiter = +
 inet_interfaces = all
 
 #### Fit to the maximum message size to 25mb, more than allowed by GMail or Yahoo ####
-# /!\ This size is the size of the attachment in base64. 
+# /!\ This size is the size of the attachment in base64.
 # BASE64_SIZE_IN_BYTE = ORIGINAL_SIZE_IN_MEGABYTE * 1,37 *1024*1024 + 980
 # See https://serverfault.com/questions/346895/postfix-mail-size-counting
 message_size_limit = 35914708
 
 # Virtual Domains Control
 virtual_mailbox_domains = ldap:/etc/postfix/ldap-domains.cf
-virtual_mailbox_maps = ldap:/etc/postfix/ldap-accounts.cf
+virtual_mailbox_maps = ldap:/etc/postfix/ldap-accounts.cf,hash:/etc/postfix/app_senders_login_maps
 virtual_mailbox_base =
 virtual_alias_maps = ldap:/etc/postfix/ldap-aliases.cf,ldap:/etc/postfix/ldap-groups.cf
 virtual_alias_domains =
 virtual_minimum_uid = 100
 virtual_uid_maps = static:vmail
 virtual_gid_maps = static:mail
-smtpd_sender_login_maps= ldap:/etc/postfix/ldap-accounts.cf
+smtpd_sender_login_maps = unionmap:{
+   # Regular Yunohost accounts
+   ldap:/etc/postfix/ldap-accounts.cf,
+   # Extra maps for app system users who need to send emails
+   hash:/etc/postfix/app_senders_login_maps }
 
 # Dovecot LDA
 virtual_transport = dovecot
@@ -207,3 +211,11 @@ smtp_sasl_security_options = noanonymous
 # where to find sasl_passwd
 smtp_sasl_password_maps = hash:/etc/postfix/sasl_passwd
 {% endif %}
+
+{% if backup_mx_domains != "" %}
+# Backup MX (secondary MX)
+relay_domains = $mydestination {{backup_mx_domains}}
+relay_recipient_maps = hash:/etc/postfix/relay_recipients
+maximal_queue_lifetime = 20d
+{% endif %}
+

conf/postfix/plain/ldap-groups.cf

@@ -2,8 +2,6 @@ server_host = localhost
 server_port = 389
 search_base = dc=yunohost,dc=org
 query_filter = (&(objectClass=groupOfNamesYnh)(mail=%s))
-exclude_internal = yes
-search_timeout = 30
 scope = sub
 result_attribute = memberUid, mail
 terminal_result_attribute = memberUid

conf/postfix/sni

@@ -1,2 +1,4 @@
+# This maps domain to certificates to properly handle multi-domain context
+# (also we need a comment in this file such that it's never empty to prevent regenconf issues)
 {% for domain in domain_list.split() %}{{ domain }} /etc/yunohost/certs/{{ domain }}/key.pem /etc/yunohost/certs/{{ domain }}/crt.pem
-{% endfor %}
+{% endfor %}

conf/rspamd/redis.conf

@@ -0,0 +1,2 @@
+# set redis server
+servers = "127.0.0.1";

conf/rspamd/rspamd.sieve

@@ -1,4 +1,4 @@
 require ["fileinto"];
-if header :is "X-Spam" "yes" {
+if header :is "X-Spam" "Yes" {
     fileinto "Junk";
 }

conf/slapd/config.ldif

@@ -10,7 +10,7 @@
 # Config database customization:
 # 1. Edit this file as you want.
 # 2. Apply your modifications. For this just run this following command in a shell:
-#    $ /usr/share/yunohost/hooks/conf_regen/06-slapd apply_config
+#    $ /usr/share/yunohost/hooks/conf_regen/06-slapd post true
 #
 # Note that if you customize this file, YunoHost's regen-conf will NOT
 # overwrite this file. But that also means that you should be careful about
@@ -159,6 +159,7 @@ olcAccess: {2}to dn.base=""
 # can read everything.
 olcAccess: {3}to *
     by dn.base="gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth" write
+    by group/groupOfNamesYnh/member.exact="cn=admins,ou=groups,dc=yunohost,dc=org" write
     by * read
 #
 olcAddContentAcl: FALSE

conf/slapd/db_init.ldif

@@ -47,14 +47,7 @@ dn: cn=admins,ou=groups,dc=yunohost,dc=org
 objectClass: posixGroup
 objectClass: top
 objectClass: groupOfNamesYnh
-objectClass: mailGroup
 gidNumber: 4001
-mail: root
-mail: admin
-mail: admins
-mail: webmaster
-mail: postmaster
-mail: abuse
 cn: admins
 
 dn: cn=all_users,ou=groups,dc=yunohost,dc=org

conf/slapd/slapd.default

@@ -21,7 +21,7 @@ SLAPD_PIDFILE=
 # sockets.
 # Example usage:
 # SLAPD_SERVICES="ldap://127.0.0.1:389/ ldaps:/// ldapi:///"
-SLAPD_SERVICES="ldap://127.0.0.1:389/ ldaps:/// ldapi:///"
+SLAPD_SERVICES="ldap://localhost:389/ ldaps:/// ldapi:///"
 
 # If SLAPD_NO_START is set, the init script will not start or restart
 # slapd (but stop will still work).  Uncomment this if you are

conf/ssh/sshd_config

@@ -64,7 +64,7 @@ PasswordAuthentication no
 {% endif %}
 
 # Post-login stuff
-Banner /etc/issue.net
+# Banner none
 PrintMotd no
 PrintLastLog yes
 ClientAliveInterval 60
@@ -84,7 +84,7 @@ Subsystem sftp internal-sftp
 
 # Apply following instructions to user with sftp perm only
 Match Group sftp.main,!ssh.main
-    ForceCommand internal-sftp
+    ForceCommand internal-sftp -u 0002
     # We can't restrict to /home/%u because the chroot base must be owned by root
     # So we chroot only on /home
     # See https://serverfault.com/questions/584986/bad-ownership-or-modes-for-chroot-directory-component
@@ -97,7 +97,7 @@ Match Group sftp.main,!ssh.main
     PermitUserRC no
 
 Match Group sftp.app,!ssh.app
-    ForceCommand internal-sftp
+    ForceCommand internal-sftp -u 0002
     ChrootDirectory %h
     AllowTcpForwarding no
     AllowStreamLocalForwarding no

debian/control

@@ -10,14 +10,14 @@ Package: yunohost
 Essential: yes
 Architecture: all
 Depends: ${python3:Depends}, ${misc:Depends}
- , moulinette (>= 11.0), ssowat (>= 11.0)
+ , moulinette (>= 11.1), moulinette (<< 12.0), ssowat (>= 11.1), ssowat (<< 12.0)
  , python3-psutil, python3-requests, python3-dnspython, python3-openssl
  , python3-miniupnpc, python3-dbus, python3-jinja2
  , python3-toml, python3-packaging, python3-publicsuffix2
  , python3-ldap, python3-zeroconf (>= 0.36), python3-lexicon,
  , python-is-python3
  , nginx, nginx-extras (>=1.18)
- , apt, apt-transport-https, apt-utils, dirmngr
+ , apt, apt-transport-https, apt-utils, aptitude, dirmngr
  , openssh-server, iptables, fail2ban, bind9-dnsutils
  , openssl, ca-certificates, netcat-openbsd, iproute2
  , slapd, ldap-utils, sudo-ldap, libnss-ldapd, unscd, libpam-ldapd
@@ -27,8 +27,8 @@ Depends: ${python3:Depends}, ${misc:Depends}
  , rspamd, opendkim-tools, postsrsd, procmail, mailutils
  , redis-server
  , acl
- , git, curl, wget, cron, unzip, jq, bc, at, procps
- , lsb-release, haveged, fake-hwclock, equivs, lsof, whois
+ , git, curl, wget, cron, unzip, jq, bc, at, procps, j2cli
+ , lsb-release, haveged, fake-hwclock, lsof, whois
 Recommends: yunohost-admin
  , ntp, inetutils-ping | iputils-ping
  , bash-completion, rsyslog
@@ -43,7 +43,7 @@ Conflicts: iptables-persistent
  , apache2
  , bind9
  , nginx-extras (>= 1.19)
- , openssl (>= 1.1.1o-0)
+ , openssl (>= 3.0)
  , slapd (>= 2.4.58)
  , dovecot-core (>= 1:2.3.14)
  , redis-server (>= 5:6.1)

debian/install

@@ -1,7 +1,7 @@
 bin/* /usr/bin/
 share/* /usr/share/yunohost/
 hooks/* /usr/share/yunohost/hooks/
-helpers/* /usr/share/yunohost/helpers.d/
+helpers/* /usr/share/yunohost/
 conf/* /usr/share/yunohost/conf/
 locales/* /usr/share/yunohost/locales/
 doc/yunohost.8.gz /usr/share/man/man8/

debian/postinst

@@ -20,13 +20,17 @@ do_configure() {
         fi
     else
         echo "Regenerating configuration, this might take a while..."
+        yunohost app ssowatconf
         yunohost tools regen-conf --output-as none
 
         echo "Launching migrations..."
         yunohost tools migrations run --auto
 
         echo "Re-diagnosing server health..."
-        yunohost diagnosis run --force
+        [[ -n "${YNH_SKIP_DIAGNOSIS_DURING_UPGRADE:-}" ]] && echo "(Skipping)" || yunohost diagnosis run --force
+
+        echo "Refreshing app catalog..."
+        yunohost tools update apps --output-as none || true
     fi
 
     # Trick to let yunohost handle the restart of the API,
@@ -52,7 +56,6 @@ API_START_TIMESTAMP="\$(date --date="\$(systemctl show yunohost-api | grep ExecM
 
 if [ "\$(( \$(date +%s) - \$API_START_TIMESTAMP ))" -ge 60 ];
 then
-    echo "restart" >> /var/log/testalex
     systemctl restart yunohost-api
 fi
 EOF

doc/api.html

@@ -0,0 +1,42 @@
+<!-- HTML for static distribution bundle build -->
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8">
+    <title>Swagger UI</title>
+    <link rel="stylesheet" type="text/css" href="swagger/swagger-ui.css" />
+    <link rel="stylesheet" type="text/css" href="swagger/index.css" />
+    <link rel="icon" type="image/png" href="swagger/favicon-32x32.png" sizes="32x32" />
+  </head>
+
+  <body>
+    <div id="swagger-ui"></div>
+    <script src="swagger/swagger-ui-bundle.js" charset="UTF-8"> </script>
+    <script src="swagger/swagger-ui-standalone-preset.js" charset="UTF-8"> </script>
+    <script src="openapi.js" type="text/javascript" language="javascript"></script>
+    <script>
+    window.onload = function() {
+  //<editor-fold desc="Changeable Configuration Block">
+  // the following lines will be replaced by docker/configurator, when it runs in a docker-container
+  window.ui = SwaggerUIBundle({
+    spec: openapiJSON,
+    dom_id: '#swagger-ui',
+    deepLinking: true,
+    displayOperationId: true,
+    validatorUrl: null,
+    presets: [
+      SwaggerUIBundle.presets.apis,
+      SwaggerUIStandalonePreset
+    ],
+    layout: "StandaloneLayout"
+  });
+
+  //</editor-fold>
+};
+
+
+
+    </script>
+  </body>
+</html>
+

doc/generate_api_doc.py

@@ -0,0 +1,283 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+""" License
+    Copyright (C) 2013 YunoHost
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+    You should have received a copy of the GNU Affero General Public License
+    along with this program; if not, see http://www.gnu.org/licenses
+"""
+
+"""
+    Generate JSON specification files API
+"""
+import os
+import sys
+import yaml
+import json
+
+
+def main():
+    with open("../share/actionsmap.yml") as f:
+        action_map = yaml.safe_load(f)
+
+    # try:
+    #    with open("/etc/yunohost/current_host", "r") as f:
+    #        domain = f.readline().rstrip()
+    # except IOError:
+    #    domain = requests.get("http://ip.yunohost.org").text
+
+    with open("../debian/changelog") as f:
+        top_changelog = f.readline()
+    api_version = top_changelog[top_changelog.find("(") + 1 : top_changelog.find(")")]
+
+    csrf = {
+        "name": "X-Requested-With",
+        "in": "header",
+        "required": True,
+        "schema": {"type": "string", "default": "Swagger API"},
+    }
+
+    resource_list = {
+        "openapi": "3.0.3",
+        "info": {
+            "title": "YunoHost API",
+            "description": "This is the YunoHost API used on all YunoHost instances. This API is essentially used by YunoHost Webadmin.",
+            "version": api_version,
+        },
+        "servers": [
+            {
+                "url": "https://{domain}/yunohost/api",
+                "variables": {
+                    "domain": {
+                        "default": "demo.yunohost.org",
+                        "description": "Your yunohost domain",
+                    }
+                },
+            }
+        ],
+        "tags": [{"name": "public", "description": "Public route"}],
+        "paths": {
+            "/login": {
+                "post": {
+                    "tags": ["public"],
+                    "summary": "Logs in and returns the authentication cookie",
+                    "parameters": [csrf],
+                    "requestBody": {
+                        "required": True,
+                        "content": {
+                            "multipart/form-data": {
+                                "schema": {
+                                    "type": "object",
+                                    "properties": {
+                                        "credentials": {
+                                            "type": "string",
+                                            "format": "password",
+                                        }
+                                    },
+                                    "required": ["credentials"],
+                                }
+                            }
+                        },
+                    },
+                    "security": [],
+                    "responses": {
+                        "200": {
+                            "description": "Successfully login",
+                            "headers": {"Set-Cookie": {"schema": {"type": "string"}}},
+                        }
+                    },
+                }
+            },
+            "/installed": {
+                "get": {
+                    "tags": ["public"],
+                    "summary": "Test if the API is working",
+                    "parameters": [],
+                    "security": [],
+                    "responses": {
+                        "200": {
+                            "description": "Successfully working",
+                        }
+                    },
+                }
+            },
+        },
+    }
+
+    def convert_categories(categories, parent_category=""):
+        for category, category_params in categories.items():
+            if parent_category:
+                category = f"{parent_category} {category}"
+            if "subcategory_help" in category_params:
+                category_params["category_help"] = category_params["subcategory_help"]
+
+            if "category_help" not in category_params:
+                category_params["category_help"] = ""
+            resource_list["tags"].append(
+                {"name": category, "description": category_params["category_help"]}
+            )
+
+            for action, action_params in category_params["actions"].items():
+                if "action_help" not in action_params:
+                    action_params["action_help"] = ""
+                if "api" not in action_params:
+                    continue
+                if not isinstance(action_params["api"], list):
+                    action_params["api"] = [action_params["api"]]
+
+                for i, api in enumerate(action_params["api"]):
+                    print(api)
+                    method, path = api.split(" ")
+                    method = method.lower()
+                    key_param = ""
+                    if "{" in path:
+                        key_param = path[path.find("{") + 1 : path.find("}")]
+                    resource_list["paths"].setdefault(path, {})
+
+                    notes = ""
+
+                    operationId = f"{category}_{action}"
+                    if i > 0:
+                        operationId += f"_{i}"
+                    operation = {
+                        "tags": [category],
+                        "operationId": operationId,
+                        "summary": action_params["action_help"],
+                        "description": notes,
+                        "responses": {"200": {"description": "successful operation"}},
+                    }
+                    if action_params.get("deprecated"):
+                        operation["deprecated"] = True
+
+                    operation["parameters"] = []
+                    if method == "post":
+                        operation["parameters"] = [csrf]
+
+                    if "arguments" in action_params:
+                        if method in ["put", "post", "patch"]:
+                            operation["requestBody"] = {
+                                "required": True,
+                                "content": {
+                                    "multipart/form-data": {
+                                        "schema": {
+                                            "type": "object",
+                                            "properties": {},
+                                            "required": [],
+                                        }
+                                    }
+                                },
+                            }
+                        for arg_name, arg_params in action_params["arguments"].items():
+                            if "help" not in arg_params:
+                                arg_params["help"] = ""
+                            param_type = "query"
+                            allow_multiple = False
+                            required = True
+                            allowable_values = None
+                            name = str(arg_name).replace("-", "_")
+                            if name[0] == "_":
+                                required = False
+                                if "full" in arg_params:
+                                    name = arg_params["full"][2:]
+                                else:
+                                    name = name[2:]
+                                name = name.replace("-", "_")
+
+                            if "choices" in arg_params:
+                                allowable_values = arg_params["choices"]
+                            _type = "string"
+                            if "type" in arg_params:
+                                types = {"open": "file", "int": "int"}
+                                _type = types[arg_params["type"]]
+                            if (
+                                "action" in arg_params
+                                and arg_params["action"] == "store_true"
+                            ):
+                                _type = "boolean"
+
+                            if "nargs" in arg_params:
+                                if arg_params["nargs"] == "*":
+                                    allow_multiple = True
+                                    required = False
+                                    _type = "array"
+                                if arg_params["nargs"] == "+":
+                                    allow_multiple = True
+                                    required = True
+                                    _type = "array"
+                                if arg_params["nargs"] == "?":
+                                    allow_multiple = False
+                                    required = False
+                            else:
+                                allow_multiple = False
+
+                            if name == key_param:
+                                param_type = "path"
+                                required = True
+                                allow_multiple = False
+
+                            if method in ["put", "post", "patch"]:
+                                schema = operation["requestBody"]["content"][
+                                    "multipart/form-data"
+                                ]["schema"]
+                                schema["properties"][name] = {
+                                    "type": _type,
+                                    "description": arg_params["help"],
+                                }
+                                if required:
+                                    schema["required"].append(name)
+                                prop_schema = schema["properties"][name]
+                            else:
+                                parameters = {
+                                    "name": name,
+                                    "in": param_type,
+                                    "description": arg_params["help"],
+                                    "required": required,
+                                    "schema": {
+                                        "type": _type,
+                                    },
+                                    "explode": allow_multiple,
+                                }
+                                prop_schema = parameters["schema"]
+                                operation["parameters"].append(parameters)
+
+                            if allowable_values is not None:
+                                prop_schema["enum"] = allowable_values
+                            if "default" in arg_params:
+                                prop_schema["default"] = arg_params["default"]
+                            if arg_params.get("metavar") == "PASSWORD":
+                                prop_schema["format"] = "password"
+                            if arg_params.get("metavar") == "MAIL":
+                                prop_schema["format"] = "mail"
+                            # Those lines seems to slow swagger ui too much
+                            # if 'pattern' in arg_params.get('extra', {}):
+                            #    prop_schema['pattern'] = arg_params['extra']['pattern'][0]
+
+                    resource_list["paths"][path][method.lower()] = operation
+
+            # Includes subcategories
+            if "subcategories" in category_params:
+                convert_categories(category_params["subcategories"], category)
+
+    del action_map["_global"]
+    convert_categories(action_map)
+
+    openapi_json = json.dumps(resource_list)
+    # Save the OpenAPI json
+    with open(os.getcwd() + "/openapi.json", "w") as f:
+        f.write(openapi_json)
+
+    openapi_js = f"var openapiJSON = {openapi_json}"
+    with open(os.getcwd() + "/openapi.js", "w") as f:
+        f.write(openapi_js)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

doc/generate_bash_completion.py

@@ -8,6 +8,7 @@ adds `--help` at the end if one presses [tab] again.
 
 author: Christophe Vuillot
 """
+
 import os
 import yaml
 
@@ -31,7 +32,6 @@ def get_dict_actions(OPTION_SUBTREE, category):
 
 
 with open(ACTIONSMAP_FILE, "r") as stream:
-
     # Getting the dictionary containning what actions are possible per category
     OPTION_TREE = yaml.safe_load(stream)
 
@@ -65,7 +65,6 @@ with open(ACTIONSMAP_FILE, "r") as stream:
     os.makedirs(BASH_COMPLETION_FOLDER, exist_ok=True)
 
     with open(BASH_COMPLETION_FILE, "w") as generated_file:
-
         # header of the file
         generated_file.write("#\n")
         generated_file.write("# completion for yunohost\n")

doc/generate_helper_doc.py

@@ -1,10 +1,55 @@
 #!/usr/env/python3
 
+import sys
 import os
 import glob
 import datetime
 import subprocess
 
+tree = {
+    "sources": {
+        "title": "Sources",
+        "notes": "This is coupled to the 'sources' resource in the manifest.toml",
+        "subsections": ["sources"],
+    },
+    "tech": {
+        "title": "App technologies",
+        "notes": "These allow to install specific version of the technology required to run some apps",
+        "subsections": ["nodejs", "ruby", "go", "composer"],
+    },
+    "db": {
+        "title": "Databases",
+        "notes": "This is coupled to the 'database' resource in the manifest.toml - at least for mysql/postgresql. Mongodb/redis may have better integration in the future.",
+        "subsections": ["mysql", "postgresql", "mongodb", "redis"],
+    },
+    "conf": {
+        "title": "Configurations / templating",
+        "subsections": [
+            "templating",
+            "nginx",
+            "php",
+            "systemd",
+            "fail2ban",
+            "logrotate",
+        ],
+    },
+    "misc": {
+        "title": "Misc tools",
+        "subsections": [
+            "utils",
+            "setting",
+            "string",
+            "backup",
+            "logging",
+            "multimedia",
+        ],
+    },
+    "meh": {
+        "title": "Deprecated or handled by the core / app resources since v2",
+        "subsections": ["permission", "apt", "systemuser"],
+    },
+}
+
 
 def get_current_commit():
     p = subprocess.Popen(
@@ -19,15 +64,7 @@ def get_current_commit():
     return current_commit
 
 
-def render(helpers):
-
-    current_commit = get_current_commit()
-
-    data = {
-        "helpers": helpers,
-        "date": datetime.datetime.now().strftime("%m/%d/%Y"),
-        "version": open("../debian/changelog").readlines()[0].split()[1].strip("()"),
-    }
+def render(tree, helpers_version):
 
     from jinja2 import Template
     from ansi2html import Ansi2HTMLConverter
@@ -43,12 +80,15 @@ def render(helpers):
     t = Template(template)
     t.globals["now"] = datetime.datetime.utcnow
     result = t.render(
-        current_commit=current_commit,
-        data=data,
+        tree=tree,
+        date=datetime.datetime.now().strftime("%d/%m/%Y"),
+        version=open("../debian/changelog").readlines()[0].split()[1].strip("()"),
+        helpers_version=helpers_version,
+        current_commit=get_current_commit(),
         convert=shell_to_html,
         shell_css=shell_css,
     )
-    open("helpers.md", "w").write(result)
+    open(f"helpers.v{helpers_version}.md", "w").write(result)
 
 
 ##############################################################################
@@ -56,20 +96,17 @@ def render(helpers):
 
 class Parser:
     def __init__(self, filename):
-
         self.filename = filename
         self.file = open(filename, "r").readlines()
         self.blocks = None
 
     def parse_blocks(self):
-
         self.blocks = []
 
         current_reading = "void"
         current_block = {"name": None, "line": -1, "comments": [], "code": []}
 
         for i, line in enumerate(self.file):
-
             if line.startswith("#!/bin/bash"):
                 continue
 
@@ -91,7 +128,7 @@ class Parser:
                     # We're still in a comment bloc
                     assert line.startswith("# ") or line == "#", malformed_error(i)
                     current_block["comments"].append(line[2:])
-                elif line.strip() == "":
+                elif line.strip() == "" or line.startswith("_ynh"):
                     # Well eh that was not an actual helper definition ... start over ?
                     current_reading = "void"
                     current_block = {
@@ -117,14 +154,20 @@ class Parser:
                     current_reading = "code"
 
             elif current_reading == "code":
-
                 if line == "}":
                     # We're getting out of the function
                     current_reading = "void"
 
                     # Then we keep this bloc and start a new one
                     # (we ignore helpers containing [internal] ...)
-                    if "[internal]" not in current_block["comments"]:
+                    if (
+                        "[packagingv1]" not in current_block["comments"]
+                        and not any(
+                            line.startswith("[internal]")
+                            for line in current_block["comments"]
+                        )
+                        and not current_block["name"].startswith("_")
+                    ):
                         self.blocks.append(current_block)
                     current_block = {
                         "name": None,
@@ -138,7 +181,6 @@ class Parser:
                 continue
 
     def parse_block(self, b):
-
         b["brief"] = ""
         b["details"] = ""
         b["usage"] = ""
@@ -164,7 +206,6 @@ class Parser:
 
             elif subblock.startswith("usage"):
                 for line in subblock.split("\n"):
-
                     if line.startswith("| arg"):
                         linesplit = line.split()
                         argname = linesplit[2]
@@ -217,20 +258,26 @@ def malformed_error(line_number):
 
 def main():
 
-    helper_files = sorted(glob.glob("../helpers/*"))
-    helpers = []
+    if len(sys.argv) == 1:
+        print("This script needs the helper version (1, 2, 2.1) as an argument")
+        sys.exit(1)
 
-    for helper_file in helper_files:
-        category_name = os.path.basename(helper_file)
-        print("Parsing %s ..." % category_name)
-        p = Parser(helper_file)
-        p.parse_blocks()
-        for b in p.blocks:
-            p.parse_block(b)
+    version = sys.argv[1]
 
-        helpers.append((category_name, p.blocks))
+    for section in tree.values():
+        section["helpers"] = {}
+        for subsection in section["subsections"]:
+            print(f"Parsing {subsection} ...")
+            helper_file = f"../helpers/helpers.v{version}.d/{subsection}"
+            assert os.path.isfile(helper_file), f"Uhoh, {file} doesn't exists?"
+            p = Parser(helper_file)
+            p.parse_blocks()
+            for b in p.blocks:
+                p.parse_block(b)
 
-    render(helpers)
+            section["helpers"][subsection] = p.blocks
+
+    render(tree, version)
 
 
 main()

doc/generate_manpages.py

@@ -60,7 +60,6 @@ def main():
 
     # man pages of "yunohost *"
     with open(ACTIONSMAP_FILE, "r") as actionsmap:
-
         # Getting the dictionary containning what actions are possible per domain
         actionsmap = ordered_yaml_load(actionsmap)
 

doc/generate_resource_doc.py

@@ -1,12 +1,70 @@
-from yunohost.utils.resources import AppResourceClassesByType
+import ast
+import datetime
+import subprocess
 
-resources = sorted(AppResourceClassesByType.values(), key=lambda r: r.priority)
+version = open("../debian/changelog").readlines()[0].split()[1].strip("()")
+today = datetime.datetime.now().strftime("%d/%m/%Y")
 
-for klass in resources:
 
-    doc = klass.__doc__.replace("\n    ", "\n")
+def get_current_commit():
+    p = subprocess.Popen(
+        "git rev-parse --verify HEAD",
+        shell=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    )
+    stdout, stderr = p.communicate()
 
+    current_commit = stdout.strip().decode("utf-8")
+    return current_commit
+
+
+current_commit = get_current_commit()
+
+
+print(
+    f"""---
+title: App resources
+template: docs
+taxonomy:
+    category: docs
+routes:
+  default: '/packaging_apps_resources'
+---
+
+Doc auto-generated by [this script](https://github.com/YunoHost/yunohost/blob/{current_commit}/doc/generate_resource_doc.py) on {today} (YunoHost version {version})
+
+"""
+)
+
+
+fname = "../src/utils/resources.py"
+content = open(fname).read()
+
+# NB: This magic is because we want to be able to run this script outside of a YunoHost context,
+# in which we cant really 'import' the file because it will trigger a bunch of moulinette/yunohost imports...
+tree = ast.parse(content)
+
+ResourceClasses = [
+    c
+    for c in tree.body
+    if isinstance(c, ast.ClassDef) and c.bases and c.bases[0].id == "AppResource"
+]
+
+ResourceDocString = {}
+
+for c in ResourceClasses:
+    assert c.body[1].targets[0].id == "type"
+    resource_id = c.body[1].value.value
+    docstring = ast.get_docstring(c)
+
+    ResourceDocString[resource_id] = docstring
+
+
+for resource_id, doc in sorted(ResourceDocString.items()):
+    print("---")
     print("")
-    print(f"## {klass.type.replace('_', ' ').title()}")
+    print(f"## {resource_id.replace('_', ' ').title()}")
     print("")
     print(doc)
+    print("")

doc/helper_doc_template.md

@@ -1,18 +1,26 @@
 ---
-title: App helpers
+title: App helpers (v{{ helpers_version }})
 template: docs
 taxonomy:
     category: docs
 routes:
-  default: '/packaging_apps_helpers'
+  default: '/packaging_apps_helpers{% if helpers_version not in ["1", "2"] %}_v{{ helpers_version }}{% endif %}'
 ---
 
-Doc auto-generated by [this script](https://github.com/YunoHost/yunohost/blob/{{ current_commit }}/doc/generate_helper_doc.py) on {{data.date}} (YunoHost version {{data.version}})
+Doc auto-generated by [this script](https://github.com/YunoHost/yunohost/blob/{{ current_commit }}/doc/generate_helper_doc.py) on {{date}} (YunoHost version {{version}})
 
-{% for category, helpers in data.helpers %}
-## {{ category.upper() }}
-{% for h in helpers %}
+
+{% for section_id, section in tree.items() %}
+## {{ section["title"].title() }}
+
+{% if section['notes'] %}<p>{{ section['notes'] }}</p>{% endif %}
+
+    {% for subsection, helpers in section["helpers"].items() %}
+
+### {{ subsection.upper() }}
+        {% for h in helpers %}
 #### {{ h.name }}
+
 [details summary="<i>{{ h.brief }}</i>" class="helper-card-subtitle text-muted"]
 
 **Usage**: `{{ h.usage }}`
@@ -48,12 +56,12 @@ Doc auto-generated by [this script](https://github.com/YunoHost/yunohost/blob/{{
 {%- endif %}
 {%- if h.details %}
 
-**Details**:<br/>
+**Details**:  
 {{ h.details }}
 {%- endif %}
-
-[Dude, show me the code!](https://github.com/YunoHost/yunohost/blob/{{ current_commit }}/helpers/{{ category }}#L{{ h.line + 1 }})
+[Dude, show me the code!](https://github.com/YunoHost/yunohost/blob/{{ current_commit }}/helpers/helpers.v{{ helpers_version if helpers_version != "2" else "1" }}.d/{{ subsection }}#L{{ h.line + 1 }})
 [/details]
-----------------
-{% endfor %}
+        {% endfor %}
+---
+    {% endfor %}
 {% endfor %}

helpers/apps

@@ -1,113 +0,0 @@
-#!/bin/bash
-
-# Install others YunoHost apps
-#
-# usage: ynh_install_apps --apps="appfoo?domain=domain.foo&path=/foo appbar?domain=domain.bar&path=/bar&admin=USER&language=fr&is_public=1&pass?word=pass&port=666"
-# | arg: -a, --apps=     - apps to install
-#
-# Requires YunoHost version *.*.* or higher.
-ynh_install_apps() {
-	# Declare an array to define the options of this helper.
-	local legacy_args=a
-	local -A args_array=([a]=apps=)
-	local apps
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
-
-	# Split the list of apps in an array
-	local apps_list=($(echo $apps | tr " " "\n"))
-	local apps_dependencies=""
-	
-	# For each app
-	for one_app_and_its_args in "${apps_list[@]}"
-	do
-		# Retrieve the name of the app (part before ?)
-		local one_app=$(cut -d "?" -f1 <<< "$one_app_and_its_args")
-		[ -z "$one_app" ] && ynh_die --message="You didn't provided a YunoHost app to install"
-
-		yunohost tools update apps
-		
-		# Installing or upgrading the app depending if it's installed or not
-		if ! yunohost app list --output-as json --quiet | jq -e --arg id $one_app '.apps[] | select(.id == $id)' >/dev/null
-		then
-			# Retrieve the arguments of the app (part after ?)
-			local one_argument=""
-			if [[ "$one_app_and_its_args" == *"?"* ]]; then
-				one_argument=$(cut -d "?" -f2- <<< "$one_app_and_its_args")
-				one_argument="--args $one_argument"
-			fi
-			
-			# Install the app with its arguments
-			yunohost app install $one_app $one_argument
-		else
-			# Upgrade the app
-			yunohost app upgrade $one_app
-		fi
-
-		if [ ! -z "$apps_dependencies" ]
-		then
-			apps_dependencies="$apps_dependencies, $one_app"
-		else
-			apps_dependencies="$one_app"
-		fi
-	done
-
-	ynh_app_setting_set --app=$app --key=apps_dependencies --value="$apps_dependencies"
-}
-
-# Remove other YunoHost apps
-#
-# Other YunoHost apps will be removed only if no other apps need them.
-#
-# usage: ynh_remove_apps
-#
-# Requires YunoHost version *.*.* or higher.
-ynh_remove_apps() {
-	# Retrieve the apps dependencies of the app
-	local apps_dependencies=$(ynh_app_setting_get --app=$app --key=apps_dependencies)
-	ynh_app_setting_delete --app=$app --key=apps_dependencies
-
-	if [ ! -z "$apps_dependencies" ]
-	then
-		# Split the list of apps dependencies in an array
-		local apps_dependencies_list=($(echo $apps_dependencies | tr ", " "\n"))
-		
-		# For each apps dependencies
-		for one_app in "${apps_dependencies_list[@]}"
-		do
-			# Retrieve the list of installed apps
-			local installed_apps_list=$(yunohost app list --output-as json --quiet | jq -r .apps[].id)
-			local required_by=""
-			local installed_app_required_by=""
-
-			# For each other installed app
-			for one_installed_app in $installed_apps_list
-			do
-				# Retrieve the other apps dependencies
-				one_installed_apps_dependencies=$(ynh_app_setting_get --app=$one_installed_app --key=apps_dependencies)
-				if [ ! -z "$one_installed_apps_dependencies" ]
-				then
-					one_installed_apps_dependencies_list=($(echo $one_installed_apps_dependencies | tr ", " "\n"))
-
-					# For each dependency of the other apps
-					for one_installed_app_dependency in "${one_installed_apps_dependencies_list[@]}"
-					do
-						if [[ $one_installed_app_dependency == $one_app ]]; then
-							required_by="$required_by $one_installed_app"
-						fi
-					done
-				fi
-			done
-
-			# If $one_app is no more required
-			if [[ -z "$required_by" ]]
-			then
-				# Remove $one_app
-				ynh_print_info --message="Removing of $one_app"
-				yunohost app remove $one_app --purge
-			else
-				ynh_print_info --message="$one_app was not removed because it's still required by${required_by}"
-			fi
-		done
-	fi
-}

helpers/helpers

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Entrypoint for the helpers scripts
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)
+
+# Helpers version can be specified via an environment variable or default to 1.
+YNH_HELPERS_VERSION=${YNH_HELPERS_VERSION:-1}
+
+# This is a trick to later only restore set -x if it was set when calling this script
+readonly XTRACE_ENABLE=$(set +o | grep xtrace)
+set +x
+
+YNH_HELPERS_DIR="$SCRIPT_DIR/helpers.v${YNH_HELPERS_VERSION}.d"
+case "$YNH_HELPERS_VERSION" in
+    "1" | "2" | "2.1")
+        readarray -t HELPERS < <(find -L "$YNH_HELPERS_DIR" -mindepth 1 -maxdepth 1 -type f)
+        source $YNH_HELPERS_DIR/getopts
+        for helper in "${HELPERS[@]}"; do
+            [ -r "$helper" ] && source "$helper"
+        done
+        ;;
+    *)
+        echo "Helpers are not available in version '$YNH_HELPERS_VERSION'." >&2
+        exit 1
+        ;;
+esac
+
+eval "$XTRACE_ENABLE"

helpers/helpers.v1.d/apps

@@ -0,0 +1,199 @@
+#!/bin/bash
+
+# Install others YunoHost apps
+#
+# usage: ynh_install_apps --apps="appfoo?domain=domain.foo&path=/foo appbar?domain=domain.bar&path=/bar&admin=USER&language=fr&is_public=1&pass?word=pass&port=666"
+# | arg: -a, --apps=     - apps to install
+#
+# Requires YunoHost version *.*.* or higher.
+ynh_install_apps() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=a
+    local -A args_array=([a]=apps=)
+    local apps
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    # Split the list of apps in an array
+    local apps_list=($(echo $apps | tr " " "\n"))
+    local apps_dependencies=""
+
+    # For each app
+    for one_app_and_its_args in "${apps_list[@]}"; do
+        # Retrieve the name of the app (part before ?)
+        local one_app=$(cut -d "?" -f1 <<< "$one_app_and_its_args")
+        [ -z "$one_app" ] && ynh_die --message="You didn't provided a YunoHost app to install"
+
+        yunohost tools update apps
+
+        # Installing or upgrading the app depending if it's installed or not
+        if ! yunohost app list --output-as json --quiet | jq -e --arg id $one_app '.apps[] | select(.id == $id)' > /dev/null; then
+            # Retrieve the arguments of the app (part after ?)
+            local one_argument=""
+            if [[ "$one_app_and_its_args" == *"?"* ]]; then
+                one_argument=$(cut -d "?" -f2- <<< "$one_app_and_its_args")
+                one_argument="--args $one_argument"
+            fi
+
+            # Install the app with its arguments
+            yunohost app install $one_app $one_argument
+        else
+            # Upgrade the app
+            yunohost app upgrade $one_app
+        fi
+
+        if [ ! -z "$apps_dependencies" ]; then
+            apps_dependencies="$apps_dependencies, $one_app"
+        else
+            apps_dependencies="$one_app"
+        fi
+    done
+
+    ynh_app_setting_set --app=$app --key=apps_dependencies --value="$apps_dependencies"
+}
+
+# Remove other YunoHost apps
+#
+# Other YunoHost apps will be removed only if no other apps need them.
+#
+# usage: ynh_remove_apps
+#
+# Requires YunoHost version *.*.* or higher.
+ynh_remove_apps() {
+    # Retrieve the apps dependencies of the app
+    local apps_dependencies=$(ynh_app_setting_get --app=$app --key=apps_dependencies)
+    ynh_app_setting_delete --app=$app --key=apps_dependencies
+
+    if [ ! -z "$apps_dependencies" ]; then
+        # Split the list of apps dependencies in an array
+        local apps_dependencies_list=($(echo $apps_dependencies | tr ", " "\n"))
+
+        # For each apps dependencies
+        for one_app in "${apps_dependencies_list[@]}"; do
+            # Retrieve the list of installed apps
+            local installed_apps_list=$(yunohost app list --output-as json --quiet | jq -r .apps[].id)
+            local required_by=""
+            local installed_app_required_by=""
+
+            # For each other installed app
+            for one_installed_app in $installed_apps_list; do
+                # Retrieve the other apps dependencies
+                one_installed_apps_dependencies=$(ynh_app_setting_get --app=$one_installed_app --key=apps_dependencies)
+                if [ ! -z "$one_installed_apps_dependencies" ]; then
+                    one_installed_apps_dependencies_list=($(echo $one_installed_apps_dependencies | tr ", " "\n"))
+
+                    # For each dependency of the other apps
+                    for one_installed_app_dependency in "${one_installed_apps_dependencies_list[@]}"; do
+                        if [[ $one_installed_app_dependency == $one_app ]]; then
+                            required_by="$required_by $one_installed_app"
+                        fi
+                    done
+                fi
+            done
+
+            # If $one_app is no more required
+            if [[ -z "$required_by" ]]; then
+                # Remove $one_app
+                ynh_print_info --message="Removing of $one_app"
+                yunohost app remove $one_app --purge
+            else
+                ynh_print_info --message="$one_app was not removed because it's still required by${required_by}"
+            fi
+        done
+    fi
+}
+
+# Spawn a Bash shell with the app environment loaded
+#
+# usage: ynh_spawn_app_shell --app="app"
+#     | arg: -a, --app=     - the app ID
+#
+# examples:
+#   ynh_spawn_app_shell --app="APP" <<< 'echo "$USER"'
+#   ynh_spawn_app_shell --app="APP" < /tmp/some_script.bash
+#
+# Requires YunoHost version 11.0.* or higher, and that the app relies on packaging v2 or higher.
+# The spawned shell will have environment variables loaded and environment files sourced
+# from the app's service configuration file (defaults to $app.service, overridable by the packager with `service` setting).
+# If the app relies on a specific PHP version, then `php` will be aliased that version. The PHP command will also be appended with the `phpflags` settings.
+ynh_spawn_app_shell() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=a
+    local -A args_array=([a]=app=)
+    local app
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    # Force Bash to be used to run this helper
+    if [[ ! $0 =~ \/?bash$ ]]; then
+        ynh_print_err --message="Please use Bash as shell"
+        exit 1
+    fi
+
+    # Make sure the app is installed
+    local installed_apps_list=($(yunohost app list --output-as json --quiet | jq -r .apps[].id))
+    if [[ " ${installed_apps_list[*]} " != *" ${app} "* ]]; then
+        ynh_print_err --message="$app is not in the apps list"
+        exit 1
+    fi
+
+    # Make sure the app has its own user
+    if ! id -u "$app" &> /dev/null; then
+        ynh_print_err --message="There is no \"$app\" system user"
+        exit 1
+    fi
+
+    # Make sure the app has an install_dir setting
+    local install_dir=$(ynh_app_setting_get --app=$app --key=install_dir)
+    if [ -z "$install_dir" ]; then
+        ynh_print_err --message="$app has no install_dir setting (does it use packaging format >=2?)"
+        exit 1
+    fi
+
+    # Load the app's service name, or default to $app
+    local service=$(ynh_app_setting_get --app=$app --key=service)
+    [ -z "$service" ] && service=$app
+
+    # Export HOME variable
+    export HOME=$install_dir
+
+    # Load the Environment variables from the app's service
+    local env_var=$(systemctl show $service.service -p "Environment" --value)
+    [ -n "$env_var" ] && export $env_var
+
+    # Force `php` to its intended version
+    # We use `eval`+`export` since `alias` is not propagated to subshells, even with `export`
+    local phpversion=$(ynh_app_setting_get --app=$app --key=phpversion)
+    local phpflags=$(ynh_app_setting_get --app=$app --key=phpflags)
+    if [ -n "$phpversion" ]; then
+        eval "php() { php${phpversion} ${phpflags} \"\$@\"; }"
+        export -f php
+    fi
+
+    # Source the EnvironmentFiles from the app's service
+    local env_files=($(systemctl show $service.service -p "EnvironmentFiles" --value))
+    if [ ${#env_files[*]} -gt 0 ]; then
+        # set -/+a enables and disables new variables being automatically exported. Needed when using `source`.
+        set -a
+        for file in ${env_files[*]}; do
+            [[ $file = /* ]] && source $file
+        done
+        set +a
+    fi
+
+    # Activate the Python environment, if it exists
+    if [ -f $install_dir/venv/bin/activate ]; then
+        # set -/+a enables and disables new variables being automatically exported. Needed when using `source`.
+        set -a
+        source $install_dir/venv/bin/activate
+        set +a
+    fi
+
+    # cd into the WorkingDirectory set in the service, or default to the install_dir
+    local env_dir=$(systemctl show $service.service -p "WorkingDirectory" --value)
+    [ -z $env_dir ] && env_dir=$install_dir
+    cd $env_dir
+
+    # Spawn the app shell
+    su -s /bin/bash $app
+}

helpers/helpers.v1.d/apt

@@ -14,7 +14,7 @@ ynh_wait_dpkg_free() {
     # With seq 1 17, timeout will be almost 30 minutes
     for try in $(seq 1 17); do
         # Check if /var/lib/dpkg/lock is used by another process
-        if lsof /var/lib/dpkg/lock >/dev/null; then
+        if lsof /var/lib/dpkg/lock > /dev/null; then
             echo "apt is already in use..."
             # Sleep an exponential time at each round
             sleep $((try * try))
@@ -32,7 +32,7 @@ ynh_wait_dpkg_free() {
                     set -o xtrace # set -x
                     return 1
                 fi
-            done 9<<<"$(ls -1 $dpkg_dir)"
+            done 9<<< "$(ls -1 $dpkg_dir)"
             set -o xtrace # set -x
             return 0
         fi
@@ -58,15 +58,16 @@ ynh_package_is_installed() {
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    ynh_wait_dpkg_free
-    dpkg-query --show --showformat='${Status}' "$package" 2>/dev/null \
-        | grep --count "ok installed" &>/dev/null
+    dpkg-query --show --showformat='${Status}' "$package" 2> /dev/null \
+        | grep --count "ok installed" &> /dev/null
 }
 
 # Get the version of an installed package
 #
 # example: version=$(ynh_package_version --package=yunohost)
 #
+# [internal]
+#
 # usage: ynh_package_version --package=name
 # | arg: -p, --package=     - the package name to get version
 # | ret: the version or an empty string
@@ -81,7 +82,7 @@ ynh_package_version() {
     ynh_handle_getopts_args "$@"
 
     if ynh_package_is_installed "$package"; then
-        dpkg-query --show --showformat='${Version}' "$package" 2>/dev/null
+        dpkg-query --show --showformat='${Version}' "$package" 2> /dev/null
     else
         echo ''
     fi
@@ -101,6 +102,8 @@ ynh_apt() {
 
 # Update package index files
 #
+# [internal]
+#
 # usage: ynh_package_update
 #
 # Requires YunoHost version 2.2.4 or higher.
@@ -110,6 +113,8 @@ ynh_package_update() {
 
 # Install package(s)
 #
+# [internal]
+#
 # usage: ynh_package_install name [name [...]]
 # | arg: name - the package name to install
 #
@@ -121,6 +126,8 @@ ynh_package_install() {
 
 # Remove package(s)
 #
+# [internal]
+#
 # usage: ynh_package_remove name [name [...]]
 # | arg: name - the package name to remove
 #
@@ -131,6 +138,8 @@ ynh_package_remove() {
 
 # Remove package(s) and their uneeded dependencies
 #
+# [internal]
+#
 # usage: ynh_package_autoremove name [name [...]]
 # | arg: name - the package name to remove
 #
@@ -141,6 +150,8 @@ ynh_package_autoremove() {
 
 # Purge package(s) and their uneeded dependencies
 #
+# [internal]
+#
 # usage: ynh_package_autopurge name [name [...]]
 # | arg: name - the package name to autoremove and purge
 #
@@ -175,22 +186,25 @@ ynh_package_install_from_equivs() {
 
     # Build and install the package
     local TMPDIR=$(mktemp --directory)
-
-    # Make sure to delete the legacy compat file
-    # It's now handle somewhat magically through the control file
-    rm -f /usr/share/equivs/template/debian/compat
+    mkdir -p ${TMPDIR}/${pkgname}/DEBIAN/
+    # For some reason, dpkg-deb insists for folder perm to be 755 and sometimes it's 777 o_O?
+    chmod -R 755 ${TMPDIR}/${pkgname}
 
     # Note that the cd executes into a sub shell
     # Create a fake deb package with equivs-build and the given control file
     # Install the fake package without its dependencies with dpkg
     # Install missing dependencies with ynh_package_install
     ynh_wait_dpkg_free
-    cp "$controlfile" "${TMPDIR}/control"
-    (
-        cd "$TMPDIR"
-        LC_ALL=C equivs-build ./control 2>&1
-        LC_ALL=C dpkg --force-depends --install "./${pkgname}_${pkgversion}_all.deb" 2>&1 | tee ./dpkg_log
-    )
+
+    cp "$controlfile" "${TMPDIR}/${pkgname}/DEBIAN/control"
+
+    # Install the fake package without its dependencies with dpkg --force-depends
+    if ! LC_ALL=C dpkg-deb --build "${TMPDIR}/${pkgname}" "${TMPDIR}/${pkgname}.deb" > "${TMPDIR}/dpkg_log" 2>&1; then
+        cat "${TMPDIR}/dpkg_log" >&2
+        ynh_die --message="Unable to install dependencies"
+    fi
+    # Don't crash in case of error, because is nicely covered by the following line
+    LC_ALL=C dpkg --force-depends --install "${TMPDIR}/${pkgname}.deb" 2>&1 | tee "${TMPDIR}/dpkg_log" || true
 
     ynh_package_install --fix-broken \
         || { # If the installation failed
@@ -250,10 +264,9 @@ ynh_install_app_dependencies() {
     # Check for specific php dependencies which requires sury
     # This grep will for example return "7.4" if dependencies is "foo bar php7.4-pwet php-gni"
     # The (?<=php) syntax corresponds to lookbehind ;)
-    local specific_php_version=$(echo $dependencies | grep -oP '(?<=php)[0-9.]+(?=-|\>)' | sort -u)
+    local specific_php_version=$(echo $dependencies | grep -oP '(?<=php)[0-9.]+(?=-|\>|)' | sort -u)
 
-    if [[ -n "$specific_php_version" ]]
-    then
+    if [[ -n "$specific_php_version" ]]; then
         # Cover a small edge case where a packager could have specified "php7.4-pwet php5-gni" which is confusing
         [[ $(echo $specific_php_version | wc -l) -eq 1 ]] \
             || ynh_die --message="Inconsistent php versions in dependencies ... found : $specific_php_version"
@@ -267,8 +280,7 @@ ynh_install_app_dependencies() {
             local old_php_fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir)
             local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf"
 
-            if [[ -f "$old_php_finalphpconf" ]]
-            then
+            if [[ -f "$old_php_finalphpconf" ]]; then
                 ynh_backup_if_checksum_is_different --file="$old_php_finalphpconf"
                 ynh_remove_fpm_config
             fi
@@ -277,7 +289,9 @@ ynh_install_app_dependencies() {
         ynh_app_setting_set --app=$app --key=phpversion --value=$specific_php_version
 
         # Set the default php version back as the default version for php-cli.
-        update-alternatives --set php /usr/bin/php$YNH_DEFAULT_PHP_VERSION
+        if test -e /usr/bin/php$YNH_DEFAULT_PHP_VERSION; then
+            update-alternatives --set php /usr/bin/php$YNH_DEFAULT_PHP_VERSION
+        fi
     elif grep --quiet 'php' <<< "$dependencies"; then
         ynh_app_setting_set --app=$app --key=phpversion --value=$YNH_DEFAULT_PHP_VERSION
     fi
@@ -289,26 +303,25 @@ ynh_install_app_dependencies() {
     # upgrade script where ynh_install_app_dependencies is called with this
     # expected effect) Otherwise, any subsequent call will add dependencies
     # to those already present in the equivs control file.
-    if [[ $YNH_INSTALL_APP_DEPENDENCIES_REPLACE == "true" ]]
-    then
+    if [[ $YNH_INSTALL_APP_DEPENDENCIES_REPLACE == "true" ]]; then
         YNH_INSTALL_APP_DEPENDENCIES_REPLACE="false"
     else
         local current_dependencies=""
-        if ynh_package_is_installed --package="${dep_app}-ynh-deps"
-        then
+        if ynh_package_is_installed --package="${dep_app}-ynh-deps"; then
             current_dependencies="$(dpkg-query --show --showformat='${Depends}' ${dep_app}-ynh-deps) "
             current_dependencies=${current_dependencies// | /|}
         fi
         dependencies="$current_dependencies, $dependencies"
     fi
 
-    cat >/tmp/${dep_app}-ynh-deps.control <<EOF # Make a control file for equivs-build
+    cat > /tmp/${dep_app}-ynh-deps.control << EOF # Make a control file for equivs-build
 Section: misc
 Priority: optional
 Package: ${dep_app}-ynh-deps
 Version: ${version}
-Depends: ${dependencies}
+Depends: ${dependencies//,,/,}
 Architecture: all
+Maintainer: root@localhost
 Description: Fake package for ${app} (YunoHost app) dependencies
  This meta-package is only responsible of installing its dependencies.
 EOF
@@ -319,8 +332,7 @@ EOF
 
     # Trigger postgresql regenconf if we may have just installed postgresql
     local psql_installed2="$(ynh_package_is_installed "postgresql-$PSQL_VERSION" && echo yes || echo no)"
-    if [[ "$psql_installed" != "$psql_installed2" ]]
-    then
+    if [[ "$psql_installed" != "$psql_installed2" ]]; then
         yunohost tools regen-conf postgresql
     fi
 
@@ -328,6 +340,8 @@ EOF
 
 # Add dependencies to install with ynh_install_app_dependencies
 #
+# [packagingv1]
+#
 # usage: ynh_add_app_dependencies --package=phpversion [--replace]
 # | arg: -p, --package=     - Packages to add as dependencies for the app.
 #
@@ -352,7 +366,7 @@ ynh_add_app_dependencies() {
 #
 # Requires YunoHost version 2.6.4 or higher.
 ynh_remove_app_dependencies() {
-    local dep_app=${app//_/-}	# Replace all '_' by '-'
+    local dep_app=${app//_/-} # Replace all '_' by '-'
 
     local current_dependencies=""
     if ynh_package_is_installed --package="${dep_app}-ynh-deps"; then
@@ -362,12 +376,16 @@ ynh_remove_app_dependencies() {
 
     # Edge case where the app dep may be on hold,
     # cf https://forum.yunohost.org/t/migration-error-cause-of-ffsync/20675/4
-    if apt-mark showhold | grep -q -w ${dep_app}-ynh-deps
-    then
+    if apt-mark showhold | grep -q -w ${dep_app}-ynh-deps; then
         apt-mark unhold ${dep_app}-ynh-deps
     fi
 
-    ynh_package_autopurge ${dep_app}-ynh-deps	# Remove the fake package and its dependencies if they not still used.
+    # Remove the fake package and its dependencies if they not still used.
+    # (except if dpkg doesn't know anything about the package,
+    # which should be symptomatic of a failed install, and we don't want bash to report an error)
+    if dpkg-query --show ${dep_app}-ynh-deps &> /dev/null; then
+        ynh_package_autopurge ${dep_app}-ynh-deps
+    fi
 }
 
 # Install packages from an extra repository properly.
@@ -409,7 +427,7 @@ ynh_install_extra_app_dependencies() {
     [ -z "$apps_auto_installed" ] || apt-mark auto $apps_auto_installed
 
     # Remove this extra repository after packages are installed
-    ynh_remove_extra_repo --name=$app
+    ynh_remove_extra_repo --name=$name
 }
 
 # Add an extra repository correctly, pin it and get the key.
@@ -448,21 +466,31 @@ ynh_install_extra_repo() {
         wget_append="tee"
     fi
 
-    # Split the repository into uri, suite and components.
+    if [[ "$key" == "trusted=yes" ]]; then
+        trusted="--trusted"
+    else
+        trusted=""
+    fi
+
+    IFS=', ' read -r -a repo_parts <<< "$repo"
+    index=0
+
     # Remove "deb " at the beginning of the repo.
-    repo="${repo#deb }"
-
-    # Get the uri
-    local uri="$(echo "$repo" | awk '{ print $1 }')"
-
-    # Get the suite
-    local suite="$(echo "$repo" | awk '{ print $2 }')"
+    if [[ "${repo_parts[0]}" == "deb" ]]; then
+        index=1
+    fi
+    uri="${repo_parts[$index]}"
+    index=$((index + 1))
+    suite="${repo_parts[$index]}"
+    index=$((index + 1))
 
     # Get the components
-    local component="${repo##$uri $suite }"
+    if (("${#repo_parts[@]}" > 0)); then
+        component="${repo_parts[*]:$index}"
+    fi
 
     # Add the repository into sources.list.d
-    ynh_add_repo --uri="$uri" --suite="$suite" --component="$component" --name="$name" $append
+    ynh_add_repo --uri="$uri" --suite="$suite" --component="$component" --name="$name" $append $trusted
 
     # Pin the new repo with the default priority, so it won't be used for upgrades.
     # Build $pin from the uri without http and any sub path
@@ -475,10 +503,10 @@ ynh_install_extra_repo() {
     ynh_pin_repo --package="*" --pin="origin \"$pin\"" $priority --name="$name" $append
 
     # Get the public key for the repo
-    if [ -n "$key" ]; then
+    if [ -n "$key" ] && [[ "$key" != "trusted=yes" ]]; then
         mkdir --parents "/etc/apt/trusted.gpg.d"
         # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget)
-        wget --timeout 900 --quiet "$key" --output-document=- | gpg --dearmor | $wget_append /etc/apt/trusted.gpg.d/$name.gpg >/dev/null
+        wget --timeout 900 --quiet "$key" --output-document=- | gpg --dearmor | $wget_append /etc/apt/trusted.gpg.d/$name.gpg > /dev/null
     fi
 
     # Update the list of package with the new repo
@@ -528,6 +556,7 @@ ynh_remove_extra_repo() {
 # | arg: -c, --component=   - Component of the repository.
 # | arg: -n, --name=        - Name for the files for this repo, $app as default value.
 # | arg: -a, --append       - Do not overwrite existing files.
+# | arg: -t, --trusted      - Add trusted=yes to the repository (not recommended)
 #
 # Example for a repo like deb http://forge.yunohost.org/debian/ stretch stable
 #                             uri                               suite   component
@@ -536,27 +565,34 @@ ynh_remove_extra_repo() {
 # Requires YunoHost version 3.8.1 or higher.
 ynh_add_repo() {
     # Declare an array to define the options of this helper.
-    local legacy_args=uscna
-    local -A args_array=([u]=uri= [s]=suite= [c]=component= [n]=name= [a]=append)
+    local legacy_args=uscnat
+    local -A args_array=([u]=uri= [s]=suite= [c]=component= [n]=name= [a]=append [t]=trusted)
     local uri
     local suite
     local component
     local name
     local append
+    local trusted
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
     name="${name:-$app}"
     append=${append:-0}
+    trusted=${trusted:-0}
 
     if [ $append -eq 1 ]; then
         append="tee --append"
     else
         append="tee"
     fi
+    if [[ "$trusted" -eq 1 ]]; then
+        trust="[trusted=yes]"
+    else
+        trust=""
+    fi
 
     mkdir --parents "/etc/apt/sources.list.d"
     # Add the new repo in sources.list.d
-    echo "deb $uri $suite $component" \
+    echo "deb $trust $uri $suite $component" \
         | $append "/etc/apt/sources.list.d/$name.list"
 }
 

helpers/helpers.v1.d/backup

@@ -162,7 +162,7 @@ ynh_backup() {
     # ==============================================================================
     local src=$(echo "${src_path}" | sed --regexp-extended 's/"/\"\"/g')
     local dest=$(echo "${dest_path}" | sed --regexp-extended 's/"/\"\"/g')
-    echo "\"${src}\",\"${dest}\"" >>"${YNH_BACKUP_CSV}"
+    echo "\"${src}\",\"${dest}\"" >> "${YNH_BACKUP_CSV}"
 
     # ==============================================================================
 
@@ -289,8 +289,7 @@ ynh_restore_file() {
     # Boring hack for nginx conf file mapped to php7.3
     # Note that there's no need to patch the fpm config because most php apps
     # will call "ynh_add_fpm_config" during restore, effectively recreating the file from scratch
-    if [[ "${dest_path}" == "/etc/nginx/conf.d/"* ]] && grep 'php7.3.*sock' "${dest_path}"
-    then
+    if [[ "${dest_path}" == "/etc/nginx/conf.d/"* ]] && grep 'php7.3.*sock' "${dest_path}"; then
         sed -i 's/php7.3/php7.4/g' "${dest_path}"
     fi
 }
@@ -327,6 +326,13 @@ ynh_store_file_checksum() {
 
     ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(md5sum "$file" | cut --delimiter=' ' --fields=1)
 
+    if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then
+        # Using a base64 is in fact more reversible than "replace / and space by _" ... So we can in fact obtain the original file path in an easy reliable way ...
+        local file_path_base64=$(echo "$file" | base64 -w0)
+        mkdir -p /var/cache/yunohost/appconfbackup/
+        cat $file > /var/cache/yunohost/appconfbackup/original_${file_path_base64}
+    fi
+
     # If backup_file_checksum isn't empty, ynh_backup_if_checksum_is_different has made a backup
     if [ -n "${backup_file_checksum-}" ]; then
         # Print the diff between the previous file and the new one.
@@ -361,11 +367,19 @@ ynh_backup_if_checksum_is_different() {
     backup_file_checksum=""
     if [ -n "$checksum_value" ]; then                                                     # Proceed only if a value was stored into the app settings
         if [ -e $file ] && ! echo "$checksum_value $file" | md5sum --check --status; then # If the checksum is now different
+
             backup_file_checksum="/var/cache/yunohost/appconfbackup/$file.backup.$(date '+%Y%m%d.%H%M%S')"
             mkdir --parents "$(dirname "$backup_file_checksum")"
             cp --archive "$file" "$backup_file_checksum" # Backup the current file
             ynh_print_warn "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum"
             echo "$backup_file_checksum" # Return the name of the backup file
+            if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then
+                local file_path_base64=$(echo "$file" | base64 -w0)
+                if test -e /var/cache/yunohost/appconfbackup/original_${file_path_base64}; then
+                    ynh_print_warn "Diff with the original file:"
+                    diff --report-identical-files --unified --color=always /var/cache/yunohost/appconfbackup/original_${file_path_base64} $file >&2 || true
+                fi
+            fi
         fi
     fi
 }
@@ -396,11 +410,13 @@ ynh_delete_file_checksum() {
 #
 ynh_backup_archive_exists() {
     yunohost backup list --output-as json --quiet \
-        | jq -e --arg archive "$1" '.archives | index($archive)' >/dev/null
+        | jq -e --arg archive "$1" '.archives | index($archive)' > /dev/null
 }
 
 # Make a backup in case of failed upgrade
 #
+# [packagingv1]
+#
 # usage: ynh_backup_before_upgrade
 #
 # Usage in a package script:
@@ -437,7 +453,7 @@ ynh_backup_before_upgrade() {
             # If the backup succeeded, remove the previous backup
             if ynh_backup_archive_exists "$app_bck-pre-upgrade$old_backup_number"; then
                 # Remove the previous backup only if it exists
-                yunohost backup delete $app_bck-pre-upgrade$old_backup_number >/dev/null
+                yunohost backup delete $app_bck-pre-upgrade$old_backup_number > /dev/null
             fi
         else
             ynh_die --message="Backup failed, the upgrade process was aborted."
@@ -449,6 +465,8 @@ ynh_backup_before_upgrade() {
 
 # Restore a previous backup if the upgrade process failed
 #
+# [packagingv1]
+#
 # usage: ynh_restore_upgradebackup
 #
 # Usage in a package script:
@@ -474,8 +492,7 @@ ynh_restore_upgradebackup() {
             yunohost app remove $app
             # Restore the backup
             yunohost backup restore $app_bck-pre-upgrade$backup_number --apps $app --force --debug
-            if [[ -d /etc/yunohost/apps/$app ]]
-            then
+            if [[ -d /etc/yunohost/apps/$app ]]; then
                 ynh_die --message="The app was restored to the way it was before the failed upgrade."
             else
                 ynh_die --message="Uhoh ... Yunohost failed to restore the app to the way it was before the failed upgrade :|"

helpers/helpers.v1.d/composer

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+readonly YNH_DEFAULT_COMPOSER_VERSION=1.10.17
+# Declare the actual composer version to use.
+# A packager willing to use another version of composer can override the variable into its _common.sh.
+YNH_COMPOSER_VERSION=${YNH_COMPOSER_VERSION:-$YNH_DEFAULT_COMPOSER_VERSION}
+
+# Execute a command with Composer
+#
+# usage: ynh_composer_exec [--phpversion=phpversion] [--workdir=$install_dir] --commands="commands"
+# | arg: -v, --phpversion - PHP version to use with composer
+# | arg: -w, --workdir - The directory from where the command will be executed. Default $install_dir or $final_path
+# | arg: -c, --commands - Commands to execute.
+#
+# Requires YunoHost version 4.2 or higher.
+ynh_composer_exec() {
+    local _globalphpversion=${phpversion-:}
+    # Declare an array to define the options of this helper.
+    local legacy_args=vwc
+    declare -Ar args_array=([v]=phpversion= [w]=workdir= [c]=commands=)
+    local phpversion
+    local workdir
+    local commands
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    workdir="${workdir:-${install_dir:-$final_path}}"
+
+    if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then
+        phpversion="${phpversion:-$YNH_PHP_VERSION}"
+    else
+        phpversion="${phpversion:-$_globalphpversion}"
+    fi
+
+    COMPOSER_HOME="$workdir/.composer" COMPOSER_MEMORY_LIMIT=-1 \
+        php${phpversion} "$workdir/composer.phar" $commands \
+        -d "$workdir" --no-interaction --no-ansi 2>&1
+}
+
+# Install and initialize Composer in the given directory
+#
+# usage: ynh_install_composer [--phpversion=phpversion] [--workdir=$install_dir] [--install_args="--optimize-autoloader"] [--composerversion=composerversion]
+# | arg: -v, --phpversion - PHP version to use with composer
+# | arg: -w, --workdir - The directory from where the command will be executed. Default $install_dir.
+# | arg: -a, --install_args - Additional arguments provided to the composer install. Argument --no-dev already include
+# | arg: -c, --composerversion - Composer version to install
+#
+# Requires YunoHost version 4.2 or higher.
+ynh_install_composer() {
+    local _globalphpversion=${phpversion-:}
+    # Declare an array to define the options of this helper.
+    local legacy_args=vwac
+    declare -Ar args_array=([v]=phpversion= [w]=workdir= [a]=install_args= [c]=composerversion=)
+    local phpversion
+    local workdir
+    local install_args
+    local composerversion
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then
+        workdir="${workdir:-$final_path}"
+    else
+        workdir="${workdir:-$install_dir}"
+    fi
+
+    if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then
+        phpversion="${phpversion:-$YNH_PHP_VERSION}"
+    else
+        phpversion="${phpversion:-$_globalphpversion}"
+    fi
+
+    install_args="${install_args:-}"
+    composerversion="${composerversion:-$YNH_COMPOSER_VERSION}"
+
+    curl -sS https://getcomposer.org/installer \
+        | COMPOSER_HOME="$workdir/.composer" \
+            php${phpversion} -- --quiet --install-dir="$workdir" --version=$composerversion \
+        || ynh_die --message="Unable to install Composer."
+
+    # install dependencies
+    ynh_composer_exec --phpversion="${phpversion}" --workdir="$workdir" --commands="install --no-dev $install_args" \
+        || ynh_die --message="Unable to install core dependencies with Composer."
+}

helpers/helpers.v1.d/config

@@ -6,11 +6,11 @@ _ynh_app_config_get_one() {
     local bind="$3"
     local getter="get__${short_setting}"
     # Get value from getter if exists
-    if type -t $getter 2>/dev/null | grep -q '^function$' 2>/dev/null; then
+    if type -t $getter 2> /dev/null | grep -q '^function$' 2> /dev/null; then
         old[$short_setting]="$($getter)"
         formats[${short_setting}]="yaml"
 
-    elif [[ "$bind" == *"("* ]] && type -t "get__${bind%%(*}" 2>/dev/null | grep -q '^function$' 2>/dev/null; then
+    elif [[ "$bind" == *"("* ]] && type -t "get__${bind%%(*}" 2> /dev/null | grep -q '^function$' 2> /dev/null; then
         old[$short_setting]="$("get__${bind%%(*}" $short_setting $type $bind)"
         formats[${short_setting}]="yaml"
 
@@ -22,7 +22,7 @@ _ynh_app_config_get_one() {
         if [[ "$bind" == "settings" ]]; then
             ynh_die --message="File '${short_setting}' can't be stored in settings"
         fi
-        old[$short_setting]="$(ls "$(echo $bind | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" 2>/dev/null || echo YNH_NULL)"
+        old[$short_setting]="$(ls "$bind" 2> /dev/null || echo YNH_NULL)"
         file_hash[$short_setting]="true"
 
     # Get multiline text from settings or from a full file
@@ -32,7 +32,7 @@ _ynh_app_config_get_one() {
         elif [[ "$bind" == *":"* ]]; then
             ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically  in a variable file, you have to create custom getter/setter"
         else
-            old[$short_setting]="$(cat $(echo $bind | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2>/dev/null || echo YNH_NULL)"
+            old[$short_setting]="$(cat "$bind" 2> /dev/null || echo YNH_NULL)"
         fi
 
     # Get value from a kind of key/value file
@@ -47,7 +47,7 @@ _ynh_app_config_get_one() {
             bind_after="$(echo "${bind_key_}" | cut -d'>' -f1)"
             bind_key_="$(echo "${bind_key_}" | cut -d'>' -f2)"
         fi
-        local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
+        local bind_file="$(echo "$bind" | cut -d: -f2)"
         old[$short_setting]="$(ynh_read_var_in_file --file="${bind_file}" --key="${bind_key_}" --after="${bind_after}")"
 
     fi
@@ -59,10 +59,10 @@ _ynh_app_config_apply_one() {
     local type="${types[$short_setting]}"
     if [ "${changed[$short_setting]}" == "true" ]; then
         # Apply setter if exists
-        if type -t $setter 2>/dev/null | grep -q '^function$' 2>/dev/null; then
+        if type -t $setter 2> /dev/null | grep -q '^function$' 2> /dev/null; then
             $setter
 
-        elif [[ "$bind" == *"("* ]] && type -t "set__${bind%%(*}" 2>/dev/null | grep -q '^function$' 2>/dev/null; then
+        elif [[ "$bind" == *"("* ]] && type -t "set__${bind%%(*}" 2> /dev/null | grep -q '^function$' 2> /dev/null; then
             "set__${bind%%(*}" $short_setting $type $bind
 
         elif [[ "$bind" == "null" ]]; then
@@ -73,7 +73,7 @@ _ynh_app_config_apply_one() {
             if [[ "$bind" == "settings" ]]; then
                 ynh_die --message="File '${short_setting}' can't be stored in settings"
             fi
-            local bind_file="$(echo "$bind" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
+            local bind_file="$bind"
             if [[ "${!short_setting}" == "" ]]; then
                 ynh_backup_if_checksum_is_different --file="$bind_file"
                 ynh_secure_remove --file="$bind_file"
@@ -98,9 +98,9 @@ _ynh_app_config_apply_one() {
             if [[ "$bind" == *":"* ]]; then
                 ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically  in a variable file, you have to create custom getter/setter"
             fi
-            local bind_file="$(echo "$bind" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
+            local bind_file="$bind"
             ynh_backup_if_checksum_is_different --file="$bind_file"
-            echo "${!short_setting}" >"$bind_file"
+            echo "${!short_setting}" > "$bind_file"
             ynh_store_file_checksum --file="$bind_file" --update_only
             ynh_print_info --message="File '$bind_file' overwritten with the content provided in question '${short_setting}'"
 
@@ -108,12 +108,12 @@ _ynh_app_config_apply_one() {
         else
             local bind_after=""
             local bind_key_="$(echo "$bind" | cut -d: -f1)"
-            bind_key_=${bind_key_:-$short_setting}
             if [[ "$bind_key_" == *">"* ]]; then
                 bind_after="$(echo "${bind_key_}" | cut -d'>' -f1)"
                 bind_key_="$(echo "${bind_key_}" | cut -d'>' -f2)"
             fi
-            local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
+            bind_key_=${bind_key_:-$short_setting}
+            local bind_file="$(echo "$bind" | cut -d: -f2)"
 
             ynh_backup_if_checksum_is_different --file="$bind_file"
             ynh_write_var_in_file --file="${bind_file}" --key="${bind_key_}" --value="${!short_setting}" --after="${bind_after}"
@@ -126,41 +126,17 @@ _ynh_app_config_apply_one() {
         fi
     fi
 }
-_ynh_app_config_get() {
-    # From settings
-    local lines
-    lines=$(
-        python3 <<EOL
-import toml
-from collections import OrderedDict
-with open("../config_panel.toml", "r") as f:
-    file_content = f.read()
-loaded_toml = toml.loads(file_content, _dict=OrderedDict)
 
-for panel_name, panel in loaded_toml.items():
-    if not isinstance(panel, dict): continue
-    for section_name, section in panel.items():
-        if not isinstance(section, dict): continue
-        for name, param in section.items():
-            if not isinstance(param, dict):
-                continue
-            print(';'.join([
-                name,
-                param.get('type', 'string'),
-                param.get('bind', 'settings' if param.get('type', 'string') != 'file' else 'null')
-                ]))
-EOL
-    )
-    for line in $lines; do
+_ynh_app_config_get() {
+    for line in $YNH_APP_CONFIG_PANEL_OPTIONS_TYPES_AND_BINDS; do
         # Split line into short_setting, type and bind
-        IFS=';' read short_setting type bind <<<"$line"
+        IFS='|' read short_setting type bind <<< "$line"
         binds[${short_setting}]="$bind"
         types[${short_setting}]="$type"
         file_hash[${short_setting}]=""
         formats[${short_setting}]=""
         ynh_app_config_get_one $short_setting $type $bind
     done
-
 }
 
 _ynh_app_config_apply() {
@@ -176,8 +152,7 @@ _ynh_app_config_show() {
                 ynh_return "${short_setting}:"
                 ynh_return "$(echo "${old[$short_setting]}" | sed 's/^/  /g')"
             else
-                ynh_return "${short_setting}: "'"'"$(echo "${old[$short_setting]}" | sed 's/"/\\"/g' | sed ':a;N;$!ba;s/\n/\n\n/g')"'"'
-
+                ynh_return "${short_setting}: '$(echo "${old[$short_setting]}" | sed "s/'/''/g" | sed ':a;N;$!ba;s/\n/\n\n/g')'"
             fi
         fi
     done
@@ -231,9 +206,9 @@ _ynh_app_config_validate() {
     for short_setting in "${!old[@]}"; do
         [[ "${changed[$short_setting]}" == "false" ]] && continue
         local result=""
-        if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null; then
+        if type -t validate__$short_setting | grep -q '^function$' 2> /dev/null; then
             result="$(validate__$short_setting)"
-        elif [[ "$bind" == *"("* ]] && type -t "validate__${bind%%(*}" 2>/dev/null | grep -q '^function$' 2>/dev/null; then
+        elif [[ "$bind" == *"("* ]] && type -t "validate__${bind%%(*}" 2> /dev/null | grep -q '^function$' 2> /dev/null; then
             "validate__${bind%%(*}" $short_setting
         fi
         if [ -n "$result" ]; then
@@ -288,7 +263,7 @@ ynh_app_config_apply() {
 ynh_app_action_run() {
     local runner="run__$1"
     # Get value from getter if exists
-    if type -t "$runner" 2>/dev/null | grep -q '^function$' 2>/dev/null; then
+    if type -t "$runner" 2> /dev/null | grep -q '^function$' 2> /dev/null; then
         $runner
         #ynh_return "result:"
         #ynh_return "$(echo "${result}" | sed 's/^/  /g')"
@@ -306,22 +281,23 @@ ynh_app_config_run() {
     declare -Ag formats=()
 
     case $1 in
-    show)
-        ynh_app_config_get
-        ynh_app_config_show
-        ;;
-    apply)
-        max_progression=4
-        ynh_script_progression --message="Reading config panel description and current configuration..."
-        ynh_app_config_get
+        show)
+            ynh_app_config_get
+            ynh_app_config_show
+            ;;
+        apply)
+            max_progression=4
+            ynh_script_progression --message="Reading config panel description and current configuration..."
+            ynh_app_config_get
 
-        ynh_app_config_validate
+            ynh_app_config_validate
 
-        ynh_script_progression --message="Applying the new configuration..."
-        ynh_app_config_apply
-        ynh_script_progression --message="Configuration of $app completed" --last
-        ;;
-    *)
-        ynh_app_action_run $1
+            ynh_script_progression --message="Applying the new configuration..."
+            ynh_app_config_apply
+            ynh_script_progression --message="Configuration of $app completed" --last
+            ;;
+        *)
+            ynh_app_action_run $1
+            ;;
     esac
 }

helpers/helpers.v1.d/fail2ban

@@ -8,8 +8,6 @@
 # | arg: -m, --max_retry= - Maximum number of retries allowed before banning IP address - default: 3
 # | arg: -p, --ports=     - Ports blocked for a banned IP address - default: http,https
 #
-# -----------------------------------------------------------------------------
-#
 # usage 2: ynh_add_fail2ban_config --use_template
 # | arg: -t, --use_template - Use this helper in template mode
 #
@@ -42,9 +40,7 @@
 #     ignoreregex =
 # ```
 #
-# -----------------------------------------------------------------------------
-#
-# Note about the "failregex" option:
+# ##### Note about the "failregex" option:
 #
 # regex to match the password failure messages in the logfile. The host must be
 # matched by a group named "`host`". The tag "`<HOST>`" can be used for standard
@@ -53,8 +49,6 @@
 # You can find some more explainations about how to make a regex here :
 # https://www.fail2ban.org/wiki/index.php/MANUAL_0_8#Filters
 #
-# Note that the logfile need to exist before to call this helper !!
-#
 # To validate your regex you can test with this command:
 # ```
 # fail2ban-regex /var/log/YOUR_LOG_FILE_PATH /etc/fail2ban/filter.d/YOUR_APP.conf
@@ -76,7 +70,7 @@ ynh_add_fail2ban_config() {
     ports=${ports:-http,https}
     use_template="${use_template:-0}"
 
-    if [ $use_template -ne 1 ]; then
+    if [ "$use_template" -ne 1 ]; then
         # Usage 1, no template. Build a config file from scratch.
         test -n "$logpath" || ynh_die --message="ynh_add_fail2ban_config expects a logfile path as first argument and received nothing."
         test -n "$failregex" || ynh_die --message="ynh_add_fail2ban_config expects a failure regex as second argument and received nothing."
@@ -88,7 +82,7 @@ port = __PORTS__
 filter = __APP__
 logpath = __LOGPATH__
 maxretry = __MAX_RETRY__
-" >$YNH_APP_BASEDIR/conf/f2b_jail.conf
+" > "$YNH_APP_BASEDIR/conf/f2b_jail.conf"
 
         echo "
 [INCLUDES]
@@ -96,12 +90,29 @@ before = common.conf
 [Definition]
 failregex = __FAILREGEX__
 ignoreregex =
-" >$YNH_APP_BASEDIR/conf/f2b_filter.conf
+" > "$YNH_APP_BASEDIR/conf/f2b_filter.conf"
     fi
 
     ynh_add_config --template="f2b_jail.conf" --destination="/etc/fail2ban/jail.d/$app.conf"
     ynh_add_config --template="f2b_filter.conf" --destination="/etc/fail2ban/filter.d/$app.conf"
 
+    # if "$logpath" doesn't exist (as if using --use_template argument), assign
+    # "$logpath" using the one in the previously generated fail2ban conf file
+    if [ -z "${logpath:-}" ]; then
+        # the first sed deletes possibles spaces and the second one extract the path
+        logpath=$(grep "^logpath" "/etc/fail2ban/jail.d/$app.conf" | sed "s/ //g" | sed "s/logpath=//g")
+    fi
+
+    # Create the folder and logfile if they doesn't exist,
+    # as fail2ban require an existing logfile before configuration
+    mkdir -p "/var/log/$app"
+    if [ ! -f "$logpath" ]; then
+        touch "$logpath"
+    fi
+    # Make sure log folder's permissions are correct
+    chown -R "$app:$app" "/var/log/$app"
+    chmod -R u=rwX,g=rX,o= "/var/log/$app"
+
     ynh_systemd_action --service_name=fail2ban --action=reload --line_match="(Started|Reloaded) Fail2Ban Service" --log_path=systemd
 
     local fail2ban_error="$(journalctl --no-hostname --unit=fail2ban | tail --lines=50 | grep "WARNING.*$app.*")"

helpers/helpers.v1.d/getopts

@@ -20,7 +20,7 @@
 # | arg: $@    - Simply "$@" to tranfert all the positionnal arguments to the function
 #
 # This helper need an array, named "args_array" with all the arguments used by the helper
-# 	that want to use ynh_handle_getopts_args
+# that want to use ynh_handle_getopts_args
 # Be carreful, this array has to be an associative array, as the following example:
 # local -A args_array=( [a]=arg1 [b]=arg2= [c]=arg3 )
 # Let's explain this array:
@@ -77,9 +77,9 @@ ynh_handle_getopts_args() {
                 # And replace long option (value of the option_flag) by the short option, the option_flag itself
                 # (e.g. for [u]=user, --user will be -u)
                 # Replace long option with = (match the beginning of the argument)
-                arguments[arg]="$(echo "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]}/-${option_flag} /")"
+                arguments[arg]="$(printf '%s\n' "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]}/-${option_flag} /")"
                 # And long option without = (match the whole line)
-                arguments[arg]="$(echo "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]%=}$/-${option_flag} /")"
+                arguments[arg]="$(printf '%s\n' "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]%=}$/-${option_flag} /")"
             done
         done
 
@@ -180,7 +180,7 @@ ynh_handle_getopts_args() {
             # If not, enter in legacy mode and manage the arguments as positionnal ones..
             # Dot not echo, to prevent to go through a helper output. But print only in the log.
             set -x
-            echo "! Helper used in legacy mode !" >/dev/null
+            echo "! Helper used in legacy mode !" > /dev/null
             set +x
             local i
             for i in $(seq 0 $((${#arguments[@]} - 1))); do

helpers/helpers.v1.d/go

@@ -0,0 +1,236 @@
+#!/bin/bash
+
+ynh_go_try_bash_extension() {
+    if [ -x src/configure ]; then
+        src/configure && make -C src || {
+            ynh_print_info --message="Optional bash extension failed to build, but things will still work normally."
+        }
+    fi
+}
+
+goenv_install_dir="/opt/goenv"
+go_version_path="$goenv_install_dir/versions"
+# goenv_ROOT is the directory of goenv, it needs to be loaded as a environment variable.
+export GOENV_ROOT="$goenv_install_dir"
+
+# Load the version of Go for an app, and set variables.
+#
+# ynh_use_go has to be used in any app scripts before using Go for the first time.
+# This helper will provide alias and variables to use in your scripts.
+#
+# To use gem or Go, use the alias `ynh_gem` and `ynh_go`
+# Those alias will use the correct version installed for the app
+# For example: use `ynh_gem install` instead of `gem install`
+#
+# With `sudo` or `ynh_exec_as`, use instead the fallback variables `$ynh_gem` and `$ynh_go`
+# And propagate $PATH to sudo with $ynh_go_load_path
+# Exemple: `ynh_exec_as $app $ynh_go_load_path $ynh_gem install`
+#
+# $PATH contains the path of the requested version of Go.
+# However, $PATH is duplicated into $go_path to outlast any manipulation of $PATH
+# You can use the variable `$ynh_go_load_path` to quickly load your Go version
+#  in $PATH for an usage into a separate script.
+# Exemple: `$ynh_go_load_path $install_dir/script_that_use_gem.sh`
+#
+#
+# Finally, to start a Go service with the correct version, 2 solutions
+#  Either the app is dependent of Go or gem, but does not called it directly.
+#  In such situation, you need to load PATH
+#    `Environment="__YNH_GO_LOAD_PATH__"`
+#    `ExecStart=__INSTALL_DIR__/my_app`
+#     You will replace __YNH_GO_LOAD_PATH__ with $ynh_go_load_path
+#
+#  Or Go start the app directly, then you don't need to load the PATH variable
+#    `ExecStart=__YNH_GO__ my_app run`
+#     You will replace __YNH_GO__ with $ynh_go
+#
+#
+# one other variable is also available
+#   - $go_path: The absolute path to Go binaries for the chosen version.
+#
+# usage: ynh_use_go
+#
+# Requires YunoHost version 3.2.2 or higher.
+ynh_use_go() {
+    go_version=$(ynh_app_setting_get --app=$app --key=go_version)
+
+    # Get the absolute path of this version of Go
+    go_path="$go_version_path/$go_version/bin"
+
+    # Allow alias to be used into bash script
+    shopt -s expand_aliases
+
+    # Create an alias for the specific version of Go and a variable as fallback
+    ynh_go="$go_path/go"
+    alias ynh_go="$ynh_go"
+
+    # Load the path of this version of Go in $PATH
+    if [[ :$PATH: != *":$go_path"* ]]; then
+        PATH="$go_path:$PATH"
+    fi
+    # Create an alias to easily load the PATH
+    ynh_go_load_path="PATH=$PATH"
+
+    # Sets the local application-specific Go version
+    pushd $install_dir
+    $goenv_install_dir/bin/goenv local $go_version
+    popd
+}
+
+# Install a specific version of Go
+#
+# ynh_install_go will install the version of Go provided as argument by using goenv.
+#
+# This helper creates a /etc/profile.d/goenv.sh that configures PATH environment for goenv
+# for every LOGIN user, hence your user must have a defined shell (as opposed to /usr/sbin/nologin)
+#
+# Don't forget to execute go-dependent command in a login environment
+# (e.g. sudo --login option)
+# When not possible (e.g. in systemd service definition), please use direct path
+# to goenv shims (e.g. $goenv_ROOT/shims/bundle)
+#
+# usage: ynh_install_go --go_version=go_version
+# | arg: -v, --go_version= - Version of go to install.
+#
+# Requires YunoHost version 3.2.2 or higher.
+ynh_install_go() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=v
+    local -A args_array=([v]=go_version=)
+    local go_version
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    # Load goenv path in PATH
+    local CLEAR_PATH="$goenv_install_dir/bin:$PATH"
+
+    # Remove /usr/local/bin in PATH in case of Go prior installation
+    PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@')
+
+    # Move an existing Go binary, to avoid to block goenv
+    test -x /usr/bin/go && mv /usr/bin/go /usr/bin/go_goenv
+
+    # Install or update goenv
+    mkdir -p $goenv_install_dir
+    pushd "$goenv_install_dir"
+    if ! [ -x "$goenv_install_dir/bin/goenv" ]; then
+        ynh_print_info --message="Downloading goenv..."
+        git init -q
+        git remote add origin https://github.com/syndbg/goenv.git
+    else
+        ynh_print_info --message="Updating goenv..."
+    fi
+    git fetch -q --tags --prune origin
+    local git_latest_tag=$(git describe --tags "$(git rev-list --tags --max-count=1)")
+    git checkout -q "$git_latest_tag"
+    ynh_go_try_bash_extension
+    goenv=$goenv_install_dir/bin/goenv
+    popd
+
+    # Install or update xxenv-latest
+    goenv_latest_dir="$goenv_install_dir/plugins/xxenv-latest"
+    mkdir -p "$goenv_latest_dir"
+    pushd "$goenv_latest_dir"
+    if ! [ -x "$goenv_latest_dir/bin/goenv-latest" ]; then
+        ynh_print_info --message="Downloading xxenv-latest..."
+        git init -q
+        git remote add origin https://github.com/momo-lab/xxenv-latest.git
+    else
+        ynh_print_info --message="Updating xxenv-latest..."
+    fi
+    git fetch -q --tags --prune origin
+    local git_latest_tag=$(git describe --tags "$(git rev-list --tags --max-count=1)")
+    git checkout -q "$git_latest_tag"
+    popd
+
+    # Enable caching
+    mkdir -p "${goenv_install_dir}/cache"
+
+    # Create shims directory if needed
+    mkdir -p "${goenv_install_dir}/shims"
+
+    # Restore /usr/local/bin in PATH
+    PATH=$CLEAR_PATH
+
+    # And replace the old Go binary
+    test -x /usr/bin/go_goenv && mv /usr/bin/go_goenv /usr/bin/go
+
+    # Install the requested version of Go
+    local final_go_version=$("$goenv_latest_dir/bin/goenv-latest" --print "$go_version")
+    ynh_print_info --message="Installation of Go-$final_go_version"
+    goenv install --skip-existing "$final_go_version"
+
+    # Store go_version into the config of this app
+    ynh_app_setting_set --app="$app" --key="go_version" --value="$final_go_version"
+
+    # Cleanup Go versions
+    ynh_cleanup_go
+
+    # Set environment for Go users
+    echo "#goenv
+export GOENV_ROOT=$goenv_install_dir
+export PATH=\"$goenv_install_dir/bin:$PATH\"
+eval \"\$(goenv init -)\"
+#goenv" > /etc/profile.d/goenv.sh
+
+    # Load the environment
+    eval "$(goenv init -)"
+}
+
+# Remove the version of Go used by the app.
+#
+# This helper will also cleanup Go versions
+#
+# usage: ynh_remove_go
+ynh_remove_go() {
+    local go_version=$(ynh_app_setting_get --app="$app" --key="go_version")
+
+    # Load goenv path in PATH
+    local CLEAR_PATH="$goenv_install_dir/bin:$PATH"
+
+    # Remove /usr/local/bin in PATH in case of Go prior installation
+    PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@')
+
+    # Remove the line for this app
+    ynh_app_setting_delete --app="$app" --key="go_version"
+
+    # Cleanup Go versions
+    ynh_cleanup_go
+}
+
+# Remove no more needed versions of Go used by the app.
+#
+# This helper will check what Go version are no more required,
+# and uninstall them
+# If no app uses Go, goenv will be also removed.
+#
+# usage: ynh_cleanup_go
+ynh_cleanup_go() {
+
+    # List required Go versions
+    local installed_apps=$(yunohost app list --output-as json --quiet | jq -r .apps[].id)
+    local required_go_versions=""
+    for installed_app in $installed_apps; do
+        local installed_app_go_version=$(ynh_app_setting_get --app=$installed_app --key="go_version")
+        if [[ $installed_app_go_version ]]; then
+            required_go_versions="${installed_app_go_version}\n${required_go_versions}"
+        fi
+    done
+
+    # Remove no more needed Go versions
+    local installed_go_versions=$(goenv versions --bare --skip-aliases | grep -Ev '/')
+    for installed_go_version in $installed_go_versions; do
+        if ! $(echo ${required_go_versions} | grep "${installed_go_version}" 1> /dev/null 2>&1); then
+            ynh_print_info --message="Removing of Go-$installed_go_version"
+            $goenv_install_dir/bin/goenv uninstall --force "$installed_go_version"
+        fi
+    done
+
+    # If none Go version is required
+    if [[ ! $required_go_versions ]]; then
+        # Remove goenv environment configuration
+        ynh_print_info --message="Removing of goenv"
+        ynh_secure_remove --file="$goenv_install_dir"
+        ynh_secure_remove --file="/etc/profile.d/goenv.sh"
+    fi
+}

helpers/helpers.v1.d/hardware

@@ -2,6 +2,8 @@
 
 # Get the total or free amount of RAM+swap on the system
 #
+# [packagingv1]
+#
 # usage: ynh_get_ram [--free|--total] [--ignore_swap|--only_swap]
 # | arg: -f, --free         - Count free RAM+swap
 # | arg: -t, --total        - Count total RAM+swap
@@ -63,6 +65,8 @@ ynh_get_ram() {
 
 # Return 0 or 1 depending if the system has a given amount of RAM+swap free or total
 #
+# [packagingv1]
+#
 # usage: ynh_require_ram --required=RAM [--free|--total] [--ignore_swap|--only_swap]
 # | arg: -r, --required=    - The amount to require, in MB
 # | arg: -f, --free         - Count free RAM+swap

helpers/helpers.v1.d/logging

@@ -93,8 +93,7 @@ ynh_exec_err() {
     # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes,
     # (because in the past eval was used) ...
     # we detect this by checking that there's no 2nd arg, and $1 contains a space
-    if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]]
-    then
+    if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]]; then
         ynh_print_err --message="$(eval $@)"
     else
         # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077
@@ -114,8 +113,7 @@ ynh_exec_warn() {
     # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes,
     # (because in the past eval was used) ...
     # we detect this by checking that there's no 2nd arg, and $1 contains a space
-    if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]]
-    then
+    if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]]; then
         ynh_print_warn --message="$(eval $@)"
     else
         # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077
@@ -135,8 +133,7 @@ ynh_exec_warn_less() {
     # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes,
     # (because in the past eval was used) ...
     # we detect this by checking that there's no 2nd arg, and $1 contains a space
-    if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]]
-    then
+    if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]]; then
         eval $@ 2>&1
     else
         # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077
@@ -156,8 +153,7 @@ ynh_exec_quiet() {
     # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes,
     # (because in the past eval was used) ...
     # we detect this by checking that there's no 2nd arg, and $1 contains a space
-    if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]]
-    then
+    if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]]; then
         eval $@ > /dev/null
     else
         # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077
@@ -177,8 +173,7 @@ ynh_exec_fully_quiet() {
     # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes,
     # (because in the past eval was used) ...
     # we detect this by checking that there's no 2nd arg, and $1 contains a space
-    if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]]
-    then
+    if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]]; then
         eval $@ > /dev/null 2>&1
     else
         # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077
@@ -186,6 +181,26 @@ ynh_exec_fully_quiet() {
     fi
 }
 
+# Execute a command and redirect stderr in /dev/null. Print stderr on error.
+#
+# usage: ynh_exec_and_print_stderr_only_if_error your command and args
+# | arg: command - command to execute
+#
+# Note that you should NOT quote the command but only prefix it with ynh_exec_and_print_stderr_only_if_error
+#
+# Requires YunoHost version 11.2 or higher.
+ynh_exec_and_print_stderr_only_if_error() {
+    logfile="$(mktemp)"
+    rc=0
+    # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077
+    "$@" 2> "$logfile" || rc="$?"
+    if ((rc != 0)); then
+        ynh_exec_warn cat "$logfile"
+        ynh_secure_remove "$logfile"
+        return "$rc"
+    fi
+}
+
 # Remove any logs for all the following commands.
 #
 # usage: ynh_print_OFF
@@ -196,7 +211,7 @@ ynh_exec_fully_quiet() {
 #
 # Requires YunoHost version 3.2.0 or higher.
 ynh_print_OFF() {
-    exec {BASH_XTRACEFD}>/dev/null
+    exec {BASH_XTRACEFD}> /dev/null
 }
 
 # Restore the logging after ynh_print_OFF
@@ -209,7 +224,7 @@ ynh_print_OFF() {
 ynh_print_ON() {
     exec {BASH_XTRACEFD}>&1
     # Print an echo only for the log, to be able to know that ynh_print_ON has been called.
-    echo ynh_print_ON >/dev/null
+    echo ynh_print_ON > /dev/null
 }
 
 # Initial definitions for ynh_script_progression
@@ -248,12 +263,12 @@ ynh_script_progression() {
     # Re-disable xtrace, ynh_handle_getopts_args set it back
     set +o xtrace # set +x
     weight=${weight:-1}
-    
+
     # Always activate time when running inside CI tests
     if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then
-    	time=${time:-1}
+        time=${time:-1}
     else
-    	time=${time:-0}
+        time=${time:-0}
     fi
 
     last=${last:-0}
@@ -278,8 +293,8 @@ ynh_script_progression() {
         local weight_values=$(($(echo "$weight_valuesA" "$weight_valuesB" | grep -v -E '^\s*$' | tr '\n' '+' | sed 's/+$/+0/g')))
 
         # max_progression is a total number of calls to this helper.
-        # 	Less the number of calls with a weight value.
-        # 	Plus the total of weight values
+        #  Less the number of calls with a weight value.
+        #  Plus the total of weight values
         max_progression=$(($helper_calls - $weight_calls + $weight_values))
     fi
 
@@ -308,8 +323,8 @@ ynh_script_progression() {
     local progression_bar="${progress_string2:0:$effective_progression}${progress_string1:0:$expected_progression}${progress_string0:0:$left_progression}"
 
     local print_exec_time=""
-    if [ $time -eq 1 ]; then
-        print_exec_time=" [$(date +%Hh%Mm,%Ss --date="0 + $exec_time sec")]"
+    if [ $time -eq 1 ] && [ "$exec_time" -gt 10 ]; then
+        print_exec_time=" [$(bc <<< "scale=1; $exec_time / 60") minutes]"
     fi
 
     ynh_print_info "[$progression_bar] > ${message}${print_exec_time}"
@@ -323,5 +338,5 @@ ynh_script_progression() {
 #
 # Requires YunoHost version 3.6.0 or higher.
 ynh_return() {
-    echo "$1" >>"$YNH_STDRETURN"
+    echo "$1" >> "$YNH_STDRETURN"
 }

helpers/helpers.v1.d/logrotate

@@ -0,0 +1,99 @@
+#!/bin/bash
+
+FIRST_CALL_TO_LOGROTATE="true"
+
+# Use logrotate to manage the logfile
+#
+# usage: ynh_use_logrotate [--logfile=/log/file] [--specific_user=user/group]
+# | arg: -l, --logfile=         - absolute path of logfile
+# | arg: -u, --specific_user=   - run logrotate as the specified user and group. If not specified logrotate is runned as root.
+#
+# If no `--logfile` is provided, `/var/log/$app` will be used as default.
+# `logfile` can point to a directory or a file.
+#
+# Requires YunoHost version 2.6.4 or higher.
+ynh_use_logrotate() {
+
+    # Stupid patch to ignore legacy --non-append and --nonappend
+    # which was never properly understood and improperly used and kind of bullshit
+    local all_args=(${@})
+    for I in $(seq 0 $(($# - 1))); do
+        if [[ "${all_args[$I]}" == "--non-append" ]] || [[ "${all_args[$I]}" == "--nonappend" ]]; then
+            unset all_args[$I]
+        fi
+    done
+    set -- "${all_args[@]}"
+
+    # Argument parsing
+    local legacy_args=lu
+    local -A args_array=([l]=logfile= [u]=specific_user=)
+    local logfile
+    local specific_user
+    ynh_handle_getopts_args "$@"
+    logfile="${logfile:-}"
+    specific_user="${specific_user:-}"
+
+    set -o noglob
+    if [[ -z "$logfile" ]]; then
+        logfile="/var/log/${app}/*.log"
+    elif [[ "${logfile##*.}" != "log" ]] && [[ "${logfile##*.}" != "txt" ]]; then
+        logfile="$logfile/*.log"
+    fi
+    set +o noglob
+
+    for stuff in $logfile; do
+        mkdir --parents $(dirname "$stuff")
+    done
+
+    local su_directive=""
+    if [[ -n "$specific_user" ]]; then
+        su_directive="su ${specific_user%/*} ${specific_user#*/}"
+    fi
+
+    local tempconf="$(mktemp)"
+    cat << EOF > $tempconf
+$logfile {
+    # Rotate if the logfile exceeds 100Mo
+    size 100M
+    # Keep 12 old log maximum
+    rotate 12
+    # Compress the logs with gzip
+    compress
+    # Compress the log at the next cycle. So keep always 2 non compressed logs
+    delaycompress
+    # Copy and truncate the log to allow to continue write on it. Instead of moving the log.
+    copytruncate
+    # Do not trigger an error if the log is missing
+    missingok
+    # Do not rotate if the log is empty
+    notifempty
+    # Keep old logs in the same dir
+    noolddir
+    $su_directive
+}
+EOF
+
+    if [[ "$FIRST_CALL_TO_LOGROTATE" == "true" ]]; then
+        cat $tempconf > /etc/logrotate.d/$app
+    else
+        cat $tempconf >> /etc/logrotate.d/$app
+    fi
+
+    FIRST_CALL_TO_LOGROTATE="false"
+
+    # Make sure permissions are correct (otherwise the config file could be ignored and the corresponding logs never rotated)
+    chmod 644 "/etc/logrotate.d/$app"
+    mkdir -p "/var/log/$app"
+    chmod 750 "/var/log/$app"
+}
+
+# Remove the app's logrotate config.
+#
+# usage: ynh_remove_logrotate
+#
+# Requires YunoHost version 2.6.4 or higher.
+ynh_remove_logrotate() {
+    if [ -e "/etc/logrotate.d/$app" ]; then
+        rm "/etc/logrotate.d/$app"
+    fi
+}

helpers/helpers.v1.d/mongodb

@@ -0,0 +1,345 @@
+#!/bin/bash
+
+# Execute a mongo command
+#
+# example: ynh_mongo_exec --command='db.getMongo().getDBNames().indexOf("wekan")'
+# example: ynh_mongo_exec --command="db.getMongo().getDBNames().indexOf(\"wekan\")"
+#
+# usage: ynh_mongo_exec [--user=user] [--password=password] [--authenticationdatabase=authenticationdatabase] [--database=database] [--host=host] [--port=port] --command="command" [--eval]
+# | arg: -u, --user=                        - The user name to connect as
+# | arg: -p, --password=                    - The user password
+# | arg: -d, --authenticationdatabase=      - The authenticationdatabase to connect to
+# | arg: -d, --database=                    - The database to connect to
+# | arg: -h, --host=                        - The host to connect to
+# | arg: -P, --port=                        - The port to connect to
+# | arg: -c, --command=                     - The command to evaluate
+# | arg: -e, --eval                         - Evaluate instead of execute the command.
+#
+#
+ynh_mongo_exec() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=upadhPce
+    local -A args_array=([u]=user= [p]=password= [a]=authenticationdatabase= [d]=database= [h]=host= [P]=port= [c]=command= [e]=eval)
+    local user
+    local password
+    local authenticationdatabase
+    local database
+    local host
+    local port
+    local command
+    local eval
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    user="${user:-}"
+    password="${password:-}"
+    authenticationdatabase="${authenticationdatabase:-}"
+    database="${database:-}"
+    host="${host:-}"
+    port="${port:-}"
+    eval=${eval:-0}
+
+    # If user is provided
+    if [ -n "$user" ]; then
+        user="--username=$user"
+
+        # If password is provided
+        if [ -n "$password" ]; then
+            password="--password=$password"
+        fi
+
+        # If authenticationdatabase is provided
+        if [ -n "$authenticationdatabase" ]; then
+            authenticationdatabase="--authenticationDatabase=$authenticationdatabase"
+        else
+            authenticationdatabase="--authenticationDatabase=admin"
+        fi
+    else
+        password=""
+        authenticationdatabase=""
+    fi
+
+    # If host is provided
+    if [ -n "$host" ]; then
+        host="--host=$host"
+    fi
+
+    # If port is provided
+    if [ -n "$port" ]; then
+        port="--port=$port"
+    fi
+
+    # If eval is not provided
+    if [ $eval -eq 0 ]; then
+        # If database is provided
+        if [ -n "$database" ]; then
+            database="use $database"
+        else
+            database=""
+        fi
+
+        mongosh --quiet --username $user --password $password --authenticationDatabase $authenticationdatabase --host $host --port $port << EOF
+$database
+${command}
+quit()
+EOF
+    else
+        # If database is provided
+        if [ -n "$database" ]; then
+            database="$database"
+        else
+            database=""
+        fi
+
+        mongosh --quiet $database --username $user --password $password --authenticationDatabase $authenticationdatabase --host $host --port $port --eval="$command"
+    fi
+}
+
+# Drop a database
+#
+# [internal]
+#
+# If you intend to drop the database *and* the associated user,
+# consider using ynh_mongo_remove_db instead.
+#
+# usage: ynh_mongo_drop_db --database=database
+# | arg: -d, --database=    - The database name to drop
+#
+#
+ynh_mongo_drop_db() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=d
+    local -A args_array=([d]=database=)
+    local database
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    ynh_mongo_exec --database="$database" --command='db.runCommand({dropDatabase: 1})'
+}
+
+# Dump a database
+#
+# example: ynh_mongo_dump_db --database=wekan > ./dump.bson
+#
+# usage: ynh_mongo_dump_db --database=database
+# | arg: -d, --database=    - The database name to dump
+# | ret: the mongodump output
+#
+#
+ynh_mongo_dump_db() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=d
+    local -A args_array=([d]=database=)
+    local database
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    mongodump --quiet --db="$database" --archive
+}
+
+# Create a user
+#
+# [internal]
+#
+# usage: ynh_mongo_create_user --db_user=user --db_pwd=pwd --db_name=name
+# | arg: -u, --db_user=     - The user name to create
+# | arg: -p, --db_pwd=      - The password to identify user by
+# | arg: -n, --db_name=     - Name of the database to grant privilegies
+#
+#
+ynh_mongo_create_user() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=unp
+    local -A args_array=([u]=db_user= [n]=db_name= [p]=db_pwd=)
+    local db_user
+    local db_name
+    local db_pwd
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    # Create the user and set the user as admin of the db
+    ynh_mongo_exec --database="$db_name" --command='db.createUser( { user: "'${db_user}'", pwd: "'${db_pwd}'", roles: [ { role: "readWrite", db: "'${db_name}'" } ] } );'
+
+    # Add clustermonitoring rights
+    ynh_mongo_exec --database="$db_name" --command='db.grantRolesToUser("'${db_user}'",[{ role: "clusterMonitor", db: "admin" }]);'
+}
+
+# Check if a mongo database exists
+#
+# usage: ynh_mongo_database_exists --database=database
+# | arg: -d, --database=    - The database for which to check existence
+# | exit: Return 1 if the database doesn't exist, 0 otherwise
+#
+#
+ynh_mongo_database_exists() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=d
+    local -A args_array=([d]=database=)
+    local database
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    if [ $(ynh_mongo_exec --command='db.getMongo().getDBNames().indexOf("'${database}'")' --eval) -lt 0 ]; then
+        return 1
+    else
+        return 0
+    fi
+}
+
+# Restore a database
+#
+# example: ynh_mongo_restore_db --database=wekan < ./dump.bson
+#
+# usage: ynh_mongo_restore_db --database=database
+# | arg: -d, --database=    - The database name to restore
+#
+#
+ynh_mongo_restore_db() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=d
+    local -A args_array=([d]=database=)
+    local database
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    mongorestore --quiet --db="$database" --archive
+}
+
+# Drop a user
+#
+# [internal]
+#
+# usage: ynh_mongo_drop_user --db_user=user --db_name=name
+# | arg: -u, --db_user=     - The user to drop
+# | arg: -n, --db_name=     - Name of the database
+#
+#
+ynh_mongo_drop_user() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=un
+    local -A args_array=([u]=db_user= [n]=db_name=)
+    local db_user
+    local db_name
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    ynh_mongo_exec --database="$db_name" --command='db.dropUser("'$db_user'", {w: "majority", wtimeout: 5000})'
+}
+
+# Create a database, an user and its password. Then store the password in the app's config
+#
+# usage: ynh_mongo_setup_db --db_user=user --db_name=name [--db_pwd=pwd]
+# | arg: -u, --db_user=     - Owner of the database
+# | arg: -n, --db_name=     - Name of the database
+# | arg: -p, --db_pwd=      - Password of the database. If not provided, a password will be generated
+#
+# After executing this helper, the password of the created database will be available in $db_pwd
+# It will also be stored as "mongopwd" into the app settings.
+#
+#
+ynh_mongo_setup_db() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=unp
+    local -A args_array=([u]=db_user= [n]=db_name= [p]=db_pwd=)
+    local db_user
+    local db_name
+    db_pwd=""
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    local new_db_pwd=$(ynh_string_random) # Generate a random password
+    # If $db_pwd is not provided, use new_db_pwd instead for db_pwd
+    db_pwd="${db_pwd:-$new_db_pwd}"
+
+    # Create the user and grant access to the database
+    ynh_mongo_create_user --db_user="$db_user" --db_pwd="$db_pwd" --db_name="$db_name"
+
+    # Store the password in the app's config
+    ynh_app_setting_set --app=$app --key=db_pwd --value=$db_pwd
+}
+
+# Remove a database if it exists, and the associated user
+#
+# usage: ynh_mongo_remove_db --db_user=user --db_name=name
+# | arg: -u, --db_user=     - Owner of the database
+# | arg: -n, --db_name=     - Name of the database
+#
+#
+ynh_mongo_remove_db() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=un
+    local -A args_array=([u]=db_user= [n]=db_name=)
+    local db_user
+    local db_name
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    if ynh_mongo_database_exists --database=$db_name; then # Check if the database exists
+        ynh_mongo_drop_db --database=$db_name              # Remove the database
+    else
+        ynh_print_warn --message="Database $db_name not found"
+    fi
+
+    # Remove mongo user if it exists
+    ynh_mongo_drop_user --db_user=$db_user --db_name=$db_name
+}
+
+# Install MongoDB and integrate MongoDB service in YunoHost
+#
+# usage: ynh_install_mongo [--mongo_version=mongo_version]
+# | arg: -m, --mongo_version=   - Version of MongoDB to install
+#
+#
+ynh_install_mongo() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=m
+    local -A args_array=([m]=mongo_version=)
+    local mongo_version
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    mongo_version="${mongo_version:-$YNH_MONGO_VERSION}"
+
+    ynh_print_info --message="Installing MongoDB Community Edition ..."
+    local mongo_debian_release=$(ynh_get_debian_release)
+
+    if [[ "$(grep '^flags' /proc/cpuinfo | uniq)" != *"avx"* && "$mongo_version" != "4.4" ]]; then
+        ynh_print_warn --message="Installing Mongo 4.4 as $mongo_version is not compatible with your cpu (see https://docs.mongodb.com/manual/administration/production-notes/#x86_64)."
+        mongo_version="4.4"
+    fi
+    if [[ "$mongo_version" == "4.4" ]]; then
+        ynh_print_warn --message="Switched to buster install as Mongo 4.4 is not compatible with $mongo_debian_release."
+        mongo_debian_release=buster
+    fi
+
+    ynh_install_extra_app_dependencies --repo="deb http://repo.mongodb.org/apt/debian $mongo_debian_release/mongodb-org/$mongo_version main" --package="mongodb-org mongodb-org-server mongodb-org-tools mongodb-mongosh" --key="https://www.mongodb.org/static/pgp/server-$mongo_version.asc"
+    mongodb_servicename=mongod
+
+    # Make sure MongoDB is started and enabled
+    systemctl enable $mongodb_servicename --quiet
+    systemctl daemon-reload --quiet
+    ynh_systemd_action --service_name=$mongodb_servicename --action=restart --line_match="aiting for connections" --log_path="/var/log/mongodb/$mongodb_servicename.log"
+
+    # Integrate MongoDB service in YunoHost
+    yunohost service add $mongodb_servicename --description="MongoDB daemon" --log="/var/log/mongodb/$mongodb_servicename.log"
+
+    # Store mongo_version into the config of this app
+    ynh_app_setting_set --app=$app --key=mongo_version --value=$mongo_version
+}
+
+# Remove MongoDB
+# Only remove the MongoDB service integration in YunoHost for now
+# if MongoDB package as been removed
+#
+# usage: ynh_remove_mongo
+#
+#
+ynh_remove_mongo() {
+    # Only remove the mongodb service if it is not installed.
+    if ! ynh_package_is_installed --package="mongodb*"; then
+        ynh_print_info --message="Removing MongoDB service..."
+        mongodb_servicename=mongod
+        # Remove the mongodb service
+        yunohost service remove $mongodb_servicename
+        ynh_secure_remove --file="/var/lib/mongodb"
+        ynh_secure_remove --file="/var/log/mongodb"
+    fi
+}

helpers/helpers.v1.d/multimedia

@@ -44,11 +44,11 @@ ynh_multimedia_build_main_dir() {
 
     ## Application des droits étendus sur le dossier multimedia.
     # Droit d'écriture pour le groupe et le groupe multimedia en acl et droit de lecture pour other:
-    setfacl -RnL -m g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$MEDIA_DIRECTORY"
+    setfacl -RnL -m g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$MEDIA_DIRECTORY" || true
     # Application de la même règle que précédemment, mais par défaut pour les nouveaux fichiers.
-    setfacl -RnL -m d:g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$MEDIA_DIRECTORY"
+    setfacl -RnL -m d:g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$MEDIA_DIRECTORY" || true
     # Réglage du masque par défaut. Qui garantie (en principe...) un droit maximal à rwx. Donc pas de restriction de droits par l'acl.
-    setfacl -RL -m m::rwx "$MEDIA_DIRECTORY"
+    setfacl -RL -m m::rwx "$MEDIA_DIRECTORY" || true
 }
 
 # Add a directory in yunohost.multimedia

helpers/helpers.v1.d/mysql

@@ -47,7 +47,7 @@ ynh_mysql_execute_as_root() {
         database="--database=$database"
     fi
 
-    mysql -B "$database" <<<"$sql"
+    mysql -B "$database" <<< "$sql"
 }
 
 # Execute a command from a file as root user
@@ -71,7 +71,7 @@ ynh_mysql_execute_file_as_root() {
         database="--database=$database"
     fi
 
-    mysql -B "$database" <"$file"
+    mysql -B "$database" < "$file"
 }
 
 # Create a database and grant optionnaly privilegies to a user
@@ -133,7 +133,7 @@ ynh_mysql_dump_db() {
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    mysqldump --single-transaction --skip-dump-date "$database"
+    mysqldump --single-transaction --skip-dump-date --routines "$database"
 }
 
 # Create a user
@@ -152,6 +152,8 @@ ynh_mysql_create_user() {
 
 # Check if a mysql user exists
 #
+# [internal]
+#
 # usage: ynh_mysql_user_exists --user=user
 # | arg: -u, --user=    - the user for which to check existence
 # | ret: 0 if the user exists, 1 otherwise.
@@ -172,6 +174,19 @@ ynh_mysql_user_exists() {
     fi
 }
 
+# Check if a mysql database exists
+#
+# [internal]
+#
+# usage: ynh_mysql_database_exists database
+# | arg: database    - the database for which to check existence
+# | exit: Return 1 if the database doesn't exist, 0 otherwise
+#
+ynh_mysql_database_exists() {
+    local database=$1
+    mysqlshow | grep -qE "^|\s+$database\s+|"
+}
+
 # Drop a user
 #
 # [internal]
@@ -186,6 +201,8 @@ ynh_mysql_drop_user() {
 
 # Create a database, an user and its password. Then store the password in the app's config
 #
+# [packagingv1]
+#
 # usage: ynh_mysql_setup_db --db_user=user --db_name=name [--db_pwd=pwd]
 # | arg: -u, --db_user=     - Owner of the database
 # | arg: -n, --db_name=     - Name of the database
@@ -216,6 +233,8 @@ ynh_mysql_setup_db() {
 
 # Remove a database if it exists, and the associated user
 #
+# [packagingv1]
+#
 # usage: ynh_mysql_remove_db --db_user=user --db_name=name
 # | arg: -u, --db_user=     - Owner of the database
 # | arg: -n, --db_name=     - Name of the database
@@ -230,7 +249,7 @@ ynh_mysql_remove_db() {
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    if mysqlshow | grep -q "^| $db_name "; then
+    if ynh_mysql_database_exists "$db_name"; then
         ynh_mysql_drop_db $db_name
     else
         ynh_print_warn --message="Database $db_name not found"

helpers/helpers.v1.d/network

@@ -2,6 +2,8 @@
 
 # Find a free port and return it
 #
+# [packagingv1]
+#
 # usage: ynh_find_port --port=begin_port
 # | arg: -p, --port=    - port to start to search
 # | ret: the port number
@@ -26,6 +28,8 @@ ynh_find_port() {
 
 # Test if a port is available
 #
+# [packagingv1]
+#
 # usage: ynh_find_port --port=XYZ
 # | arg: -p, --port=    - port to check
 # | ret: 0 if the port is available, 1 if it is already used by another process.
@@ -75,7 +79,7 @@ ynh_validate_ip() {
 
     [ "$family" == "4" ] || [ "$family" == "6" ] || return 1
 
-    python3 /dev/stdin <<EOF
+    python3 /dev/stdin << EOF
 import socket
 import sys
 family = { "4" : socket.AF_INET, "6" : socket.AF_INET6 }

helpers/helpers.v1.d/nginx

@@ -42,3 +42,23 @@ ynh_remove_nginx_config() {
     ynh_secure_remove --file="/etc/nginx/conf.d/$domain.d/$app.conf"
     ynh_systemd_action --service_name=nginx --action=reload
 }
+
+# Regen the nginx config in a change url context
+#
+# usage: ynh_change_url_nginx_config
+#
+# Requires YunoHost version 11.1.9 or higher.
+ynh_change_url_nginx_config() {
+
+    # Make a backup of the original NGINX config file if manually modified
+    # (nb: this is possibly different from the same instruction called by
+    # ynh_add_config inside ynh_add_nginx_config because the path may have
+    # changed if we're changing the domain too...)
+    local old_nginx_conf_path=/etc/nginx/conf.d/$old_domain.d/$app.conf
+    ynh_backup_if_checksum_is_different --file="$old_nginx_conf_path"
+    ynh_delete_file_checksum --file="$old_nginx_conf_path"
+    ynh_secure_remove --file="$old_nginx_conf_path"
+
+    # Regen the nginx conf
+    ynh_add_nginx_config
+}

helpers/helpers.v1.d/nodejs

@@ -1,32 +1,10 @@
 #!/bin/bash
 
-n_version=9.0.0
-n_checksum=37a987230d1ed0392a83f9c02c1e535a524977c00c64a4adb771ab60237be1c6
 n_install_dir="/opt/node_n"
 node_version_path="$n_install_dir/n/versions/node"
 # N_PREFIX is the directory of n, it needs to be loaded as a environment variable.
 export N_PREFIX="$n_install_dir"
 
-# Install Node version management
-#
-# [internal]
-#
-# usage: ynh_install_n
-#
-# Requires YunoHost version 2.7.12 or higher.
-ynh_install_n() {
-    # Build an app.src for n
-    echo "SOURCE_URL=https://github.com/tj/n/archive/v${n_version}.tar.gz
-SOURCE_SUM=${n_checksum}" >"$YNH_APP_BASEDIR/conf/n.src"
-    # Download and extract n
-    ynh_setup_source --dest_dir="$n_install_dir/git" --source_id=n
-    # Install n
-    (
-        cd "$n_install_dir/git"
-        PREFIX=$N_PREFIX make install 2>&1
-    )
-}
-
 # Load the version of node for an app, and set variables.
 #
 # usage: ynh_use_nodejs
@@ -96,6 +74,8 @@ ynh_use_nodejs() {
     ynh_node_load_PATH="PATH=$node_PATH"
     # Same var but in lower case to be compatible with ynh_replace_vars...
     ynh_node_load_path="PATH=$node_PATH"
+    # Prevent yet another Node and Corepack madness, with Corepack wanting the user to confirm download of Yarn
+    export COREPACK_ENABLE_DOWNLOAD_PROMPT=0
 }
 
 # Install a specific version of nodejs
@@ -103,7 +83,7 @@ ynh_use_nodejs() {
 # ynh_install_nodejs will install the version of node provided as argument by using n.
 #
 # usage: ynh_install_nodejs --nodejs_version=nodejs_version
-# | arg: -n, --nodejs_version=      - Version of node to install. When possible, your should prefer to use major version number (e.g. 8 instead of 8.10.0). The crontab will then handle the update of minor versions when needed.
+# | arg: -n, --nodejs_version=      - Version of node to install. When possible, your should prefer to use major version number (e.g. 8 instead of 8.10.0).
 #
 # `n` (Node version management) uses the `PATH` variable to store the path of the version of node it is going to use.
 # That's how it changes the version
@@ -133,14 +113,10 @@ ynh_install_nodejs() {
     test -x /usr/bin/node && mv /usr/bin/node /usr/bin/node_n
     test -x /usr/bin/npm && mv /usr/bin/npm /usr/bin/npm_n
 
-    # If n is not previously setup, install it
-    if ! $n_install_dir/bin/n --version >/dev/null 2>&1; then
-        ynh_install_n
-    elif dpkg --compare-versions "$($n_install_dir/bin/n --version)" lt $n_version; then
-        ynh_install_n
-    fi
-
-    # Modify the default N_PREFIX in n script
+    # Install (or update if YunoHost vendor/ folder updated since last install) n
+    mkdir -p $n_install_dir/bin/
+    cp "$YNH_HELPERS_DIR/vendor/n/n" $n_install_dir/bin/n
+    # Tweak for n to understand it's installed in $N_PREFIX
     ynh_replace_string --match_string="^N_PREFIX=\${N_PREFIX-.*}$" --replace_string="N_PREFIX=\${N_PREFIX-$N_PREFIX}" --target_file="$n_install_dir/bin/n"
 
     # Restore /usr/local/bin in PATH
@@ -168,14 +144,11 @@ ynh_install_nodejs() {
     fi
 
     # Store the ID of this app and the version of node requested for it
-    echo "$YNH_APP_INSTANCE_NAME:$nodejs_version" | tee --append "$n_install_dir/ynh_app_version"
+    echo "$app:$nodejs_version" | tee --append "$n_install_dir/ynh_app_version"
 
     # Store nodejs_version into the config of this app
     ynh_app_setting_set --app=$app --key=nodejs_version --value=$nodejs_version
 
-    # Build the update script and set the cronjob
-    ynh_cron_upgrade_node
-
     ynh_use_nodejs
 }
 
@@ -192,7 +165,7 @@ ynh_remove_nodejs() {
     nodejs_version=$(ynh_app_setting_get --app=$app --key=nodejs_version)
 
     # Remove the line for this app
-    sed --in-place "/$YNH_APP_INSTANCE_NAME:$nodejs_version/d" "$n_install_dir/ynh_app_version"
+    sed --in-place "/$app:$nodejs_version/d" "$n_install_dir/ynh_app_version"
 
     # If no other app uses this version of nodejs, remove it.
     if ! grep --quiet "$nodejs_version" "$n_install_dir/ynh_app_version"; then
@@ -204,62 +177,5 @@ ynh_remove_nodejs() {
         ynh_secure_remove --file="$n_install_dir"
         ynh_secure_remove --file="/usr/local/n"
         sed --in-place "/N_PREFIX/d" /root/.bashrc
-        rm --force /etc/cron.daily/node_update
     fi
 }
-
-# Set a cron design to update your node versions
-#
-# [internal]
-#
-# This cron will check and update all minor node versions used by your apps.
-#
-# usage: ynh_cron_upgrade_node
-#
-# Requires YunoHost version 2.7.12 or higher.
-ynh_cron_upgrade_node() {
-    # Build the update script
-    cat >"$n_install_dir/node_update.sh" <<EOF
-#!/bin/bash
-
-version_path="$node_version_path"
-n_install_dir="$n_install_dir"
-
-# Log the date
-date
-
-# List all real installed version of node
-all_real_version="\$(find \$version_path/* -maxdepth 0 -type d | sed "s@\$version_path/@@g")"
-
-# Keep only the major version number of each line
-all_real_version=\$(echo "\$all_real_version" | sed 's/\..*\$//')
-
-# Remove double entries
-all_real_version=\$(echo "\$all_real_version" | sort --unique)
-
-# Read each major version
-while read version
-do
-    echo "Update of the version \$version"
-    sudo \$n_install_dir/bin/n \$version
-
-    # Find the last "real" version for this major version of node.
-    real_nodejs_version=\$(find \$version_path/\$version* -maxdepth 0 | sort --version-sort | tail --lines=1)
-    real_nodejs_version=\$(basename \$real_nodejs_version)
-
-    # Update the symbolic link for this version
-    sudo ln --symbolic --force --no-target-directory \$version_path/\$real_nodejs_version \$version_path/\$version
-done <<< "\$(echo "\$all_real_version")"
-EOF
-
-    chmod +x "$n_install_dir/node_update.sh"
-
-    # Build the cronjob
-    cat >"/etc/cron.daily/node_update" <<EOF
-#!/bin/bash
-
-$n_install_dir/node_update.sh >> $n_install_dir/node_update.log
-EOF
-
-    chmod +x "/etc/cron.daily/node_update"
-}

helpers/helpers.v1.d/permission

@@ -174,7 +174,7 @@ ynh_permission_exists() {
     ynh_handle_getopts_args "$@"
 
     yunohost user permission list "$app" --output-as json --quiet \
-        | jq -e --arg perm "$app.$permission" '.permissions[$perm]' >/dev/null
+        | jq -e --arg perm "$app.$permission" '.permissions[$perm]' > /dev/null
 }
 
 # Redefine the url associated to a permission
@@ -342,7 +342,7 @@ ynh_permission_has_user() {
     # Check both allowed and corresponding_users sections in the json
     for section in "allowed" "corresponding_users"; do
         if yunohost user permission info "$app.$permission" --output-as json --quiet \
-            | jq -e --arg user $user --arg section $section '.[$section] | index($user)' >/dev/null; then
+            | jq -e --arg user $user --arg section $section '.[$section] | index($user)' > /dev/null; then
             return 0
         fi
     done

helpers/helpers.v1.d/php

@@ -7,33 +7,44 @@ YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION}
 
 # Create a dedicated PHP-FPM config
 #
-# usage 1: ynh_add_fpm_config [--phpversion=7.X] [--use_template] [--package=packages] [--dedicated_service]
-# | arg: -v, --phpversion=          - Version of PHP to use.
-# | arg: -t, --use_template         - Use this helper in template mode.
-# | arg: -p, --package=             - Additionnal PHP packages to install
-# | arg: -d, --dedicated_service    - Use a dedicated PHP-FPM service instead of the common one.
+# usage: ynh_add_fpm_config
 #
-# -----------------------------------------------------------------------------
+# Case 1 (recommended) : your provided a snippet conf/extra_php-fpm.conf
 #
-# usage 2: ynh_add_fpm_config [--phpversion=7.X] --usage=usage --footprint=footprint [--package=packages] [--dedicated_service]
-# | arg: -v, --phpversion=          - Version of PHP to use.
-# | arg: -f, --footprint=           - Memory footprint of the service (low/medium/high).
+# The actual PHP configuration will be automatically generated,
+# and your extra_php-fpm.conf will be appended (typically contains PHP upload limits)
+#
+# The resulting configuration will be deployed to the appropriate place, /etc/php/$phpversion/fpm/pool.d/$app.conf
+#
+# Performance-related options in the PHP conf, such as :
+# pm.max_children, pm.start_servers, pm.min_spare_servers pm.max_spare_servers
+# are computed from two parameters called "usage" and "footprint" which can be set to low/medium/high. (cf details below)
+#
+# If you wish to tweak those, please initialize the settings `fpm_usage` and `fpm_footprint`
+# *prior* to calling this helper. Otherwise, "low" will be used as a default for both values.
+#
+# Otherwise, if you want the user to have control over these, we encourage to create a config panel
+# (which should ultimately be standardized by the core ...)
+#
+# Case 2 (deprecate) : you provided an entire conf/php-fpm.conf
+#
+# The configuration will be hydrated, replacing __FOOBAR__ placeholders with $foobar values, etc.
+#
+# The resulting configuration will be deployed to the appropriate place, /etc/php/$phpversion/fpm/pool.d/$app.conf
+#
+# ----------------------
+#
+# fpm_footprint: Memory footprint of the service (low/medium/high).
 # low    - Less than 20 MB of RAM by pool.
 # medium - Between 20 MB and 40 MB of RAM by pool.
 # high   - More than 40 MB of RAM by pool.
-# Or specify exactly the footprint, the load of the service as MB by pool instead of having a standard value.
-# To have this value, use the following command and stress the service.
-# watch -n0.5 ps -o user,cmd,%cpu,rss -u APP
+# N      - Or you can specify a quantitative footprint as MB by pool (use watch -n0.5 ps -o user,cmd,%cpu,rss -u APP)
 #
-# | arg: -u, --usage=               - Expected usage of the service (low/medium/high).
+# fpm_usage: Expected usage of the service (low/medium/high).
 # low    - Personal usage, behind the SSO.
 # medium - Low usage, few people or/and publicly accessible.
 # high   - High usage, frequently visited website.
 #
-# | arg: -p, --package=             - Additionnal PHP packages to install for a specific version of PHP
-# | arg: -d, --dedicated_service    - Use a dedicated PHP-FPM service instead of the common one.
-#
-#
 # The footprint of the service will be used to defined the maximum footprint we can allow, which is half the maximum RAM.
 # So it will be used to defined 'pm.max_children'
 # A lower value for the footprint will allow more children for 'pm.max_children'. And so for
@@ -57,11 +68,12 @@ YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION}
 #
 # Requires YunoHost version 4.1.0 or higher.
 ynh_add_fpm_config() {
+    local _globalphpversion=${phpversion-:}
     # Declare an array to define the options of this helper.
-    local legacy_args=vtufpd
-    local -A args_array=([v]=phpversion= [t]=use_template [u]=usage= [f]=footprint= [p]=package= [d]=dedicated_service)
+    local legacy_args=vufpdg
+    local -A args_array=([v]=phpversion= [u]=usage= [f]=footprint= [p]=package= [d]=dedicated_service [g]=group=)
+    local group
     local phpversion
-    local use_template
     local usage
     local footprint
     local package
@@ -69,29 +81,49 @@ ynh_add_fpm_config() {
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
     package=${package:-}
+    group=${group:-}
 
     # The default behaviour is to use the template.
-    use_template="${use_template:-1}"
+    local autogenconf=false
     usage="${usage:-}"
     footprint="${footprint:-}"
-    if [ -n "$usage" ] || [ -n "$footprint" ]; then
-        use_template=0
+    if [ -n "$usage" ] || [ -n "$footprint" ] || [[ -e $YNH_APP_BASEDIR/conf/extra_php-fpm.conf ]]; then
+        autogenconf=true
+
+        # If no usage provided, default to the value existing in setting ... or to low
+        local fpm_usage_in_setting=$(ynh_app_setting_get --app=$app --key=fpm_usage)
+        if [ -z "$usage" ]; then
+            usage=${fpm_usage_in_setting:-low}
+            ynh_app_setting_set --app=$app --key=fpm_usage --value=$usage
+        fi
+
+        # If no footprint provided, default to the value existing in setting ... or to low
+        local fpm_footprint_in_setting=$(ynh_app_setting_get --app=$app --key=fpm_footprint)
+        if [ -z "$footprint" ]; then
+            footprint=${fpm_footprint_in_setting:-low}
+            ynh_app_setting_set --app=$app --key=fpm_footprint --value=$footprint
+        fi
+
     fi
     # Do not use a dedicated service by default
     dedicated_service=${dedicated_service:-0}
 
     # Set the default PHP-FPM version by default
-    phpversion="${phpversion:-$YNH_PHP_VERSION}"
+    if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then
+        phpversion="${phpversion:-$YNH_PHP_VERSION}"
+    else
+        phpversion="${phpversion:-$_globalphpversion}"
+    fi
 
     local old_phpversion=$(ynh_app_setting_get --app=$app --key=phpversion)
 
     # If the PHP version changed, remove the old fpm conf
+    # (NB: This stuff is also handled by the apt helper, which is usually triggered before this helper)
     if [ -n "$old_phpversion" ] && [ "$old_phpversion" != "$phpversion" ]; then
         local old_php_fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir)
         local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf"
 
-        if [[ -f "$old_php_finalphpconf" ]]
-        then
+        if [[ -f "$old_php_finalphpconf" ]]; then
             ynh_backup_if_checksum_is_different --file="$old_php_finalphpconf"
             ynh_remove_fpm_config
         fi
@@ -100,10 +132,12 @@ ynh_add_fpm_config() {
     # Legacy args (packager should just list their php dependency as regular apt dependencies...
     if [ -n "$package" ]; then
         # Install the additionnal packages from the default repository
+        ynh_print_warn --message "Argument --package of ynh_add_fpm_config is deprecated and to be removed in the future"
         ynh_install_app_dependencies "$package"
     fi
 
     if [ $dedicated_service -eq 1 ]; then
+        ynh_print_warn --message "Argument --dedicated_service of ynh_add_fpm_config is deprecated and to be removed in the future"
         local fpm_service="${app}-phpfpm"
         local fpm_config_dir="/etc/php/$phpversion/dedicated-fpm"
     else
@@ -134,7 +168,7 @@ ynh_add_fpm_config() {
         fi
     fi
 
-    if [ $use_template -eq 1 ]; then
+    if [ $autogenconf == "false" ]; then
         # Usage 1, use the template in conf/php-fpm.conf
         local phpfpm_path="$YNH_APP_BASEDIR/conf/php-fpm.conf"
         # Make sure now that the template indeed exists
@@ -142,21 +176,18 @@ ynh_add_fpm_config() {
     else
         # Usage 2, generate a PHP-FPM config file with ynh_get_scalable_phpfpm
 
-        # Store settings
-        ynh_app_setting_set --app=$app --key=fpm_footprint --value=$footprint
-        ynh_app_setting_set --app=$app --key=fpm_usage --value=$usage
-
         # Define the values to use for the configuration of PHP.
         ynh_get_scalable_phpfpm --usage=$usage --footprint=$footprint
 
+        local phpfpm_group=$([[ -n "$group" ]] && echo "$group" || echo "$app")
         local phpfpm_path="$YNH_APP_BASEDIR/conf/php-fpm.conf"
         echo "
 [__APP__]
 
 user = __APP__
-group = __APP__
+group = __PHPFPM_GROUP__
 
-chdir = __FINALPATH__
+chdir = __INSTALL_DIR__
 
 listen = /var/run/php/php__PHPVERSION__-fpm-__APP__.sock
 listen.owner = www-data
@@ -166,24 +197,24 @@ pm = __PHP_PM__
 pm.max_children = __PHP_MAX_CHILDREN__
 pm.max_requests = 500
 request_terminate_timeout = 1d
-" >$phpfpm_path
+" > "$phpfpm_path"
 
         if [ "$php_pm" = "dynamic" ]; then
             echo "
 pm.start_servers = __PHP_START_SERVERS__
 pm.min_spare_servers = __PHP_MIN_SPARE_SERVERS__
 pm.max_spare_servers = __PHP_MAX_SPARE_SERVERS__
-" >>$phpfpm_path
+" >> "$phpfpm_path"
 
         elif [ "$php_pm" = "ondemand" ]; then
             echo "
 pm.process_idle_timeout = 10s
-" >>$phpfpm_path
+" >> "$phpfpm_path"
         fi
 
         # Concatene the extra config.
         if [ -e $YNH_APP_BASEDIR/conf/extra_php-fpm.conf ]; then
-            cat $YNH_APP_BASEDIR/conf/extra_php-fpm.conf >>"$phpfpm_path"
+            cat $YNH_APP_BASEDIR/conf/extra_php-fpm.conf >> "$phpfpm_path"
         fi
     fi
 
@@ -204,7 +235,7 @@ pid = /run/php/php__PHPVERSION__-fpm-__APP__.pid
 error_log = /var/log/php/fpm-php.__APP__.log
 syslog.ident = php-fpm-__APP__
 include = __FINALPHPCONF__
-" >$YNH_APP_BASEDIR/conf/php-fpm-$app.conf
+" > $YNH_APP_BASEDIR/conf/php-fpm-$app.conf
 
         ynh_add_config --template="php-fpm-$app.conf" --destination="$globalphpconf"
 
@@ -221,7 +252,7 @@ ExecReload=/bin/kill -USR2 \$MAINPID
 
 [Install]
 WantedBy=multi-user.target
-" >$YNH_APP_BASEDIR/conf/$fpm_service
+" > $YNH_APP_BASEDIR/conf/$fpm_service
 
         # Create this dedicated PHP-FPM service
         ynh_add_systemd_config --service=$fpm_service --template=$fpm_service
@@ -233,7 +264,7 @@ WantedBy=multi-user.target
         ynh_systemd_action --service_name=$fpm_service --action=restart
     else
         # Validate that the new php conf doesn't break php-fpm entirely
-        if ! php-fpm${phpversion} --test 2>/dev/null; then
+        if ! php-fpm${phpversion} --test 2> /dev/null; then
             php-fpm${phpversion} --test || true
             ynh_secure_remove --file="$finalphpconf"
             ynh_die --message="The new configuration broke php-fpm?"
@@ -283,7 +314,7 @@ ynh_remove_fpm_config() {
     # If the PHP version used is not the default version for YunoHost
     # The second part with YNH_APP_PURGE is an ugly hack to guess that we're inside the remove script
     # (we don't actually care about its value, we just check its not empty hence it exists)
-    if [ "$phpversion" != "$YNH_DEFAULT_PHP_VERSION" ] && [ -n "${YNH_APP_PURGE:-}" ]; then
+    if [ "$phpversion" != "$YNH_DEFAULT_PHP_VERSION" ] && [ -n "${YNH_APP_PURGE:-}" ] && dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then
         # Remove app dependencies ... but ideally should happen via an explicit call from packager
         ynh_remove_app_dependencies
     fi
@@ -326,7 +357,7 @@ ynh_install_php() {
 # usage: ynh_remove_php
 #
 # Requires YunoHost version 3.8.1 or higher.
-ynh_remove_php () {
+ynh_remove_php() {
     ynh_remove_app_dependencies
 }
 
@@ -466,67 +497,3 @@ ynh_get_scalable_phpfpm() {
         fi
     fi
 }
-
-readonly YNH_DEFAULT_COMPOSER_VERSION=1.10.17
-# Declare the actual composer version to use.
-# A packager willing to use another version of composer can override the variable into its _common.sh.
-YNH_COMPOSER_VERSION=${YNH_COMPOSER_VERSION:-$YNH_DEFAULT_COMPOSER_VERSION}
-
-# Execute a command with Composer
-#
-# usage: ynh_composer_exec [--phpversion=phpversion] [--workdir=$final_path] --commands="commands"
-# | arg: -v, --phpversion - PHP version to use with composer
-# | arg: -w, --workdir - The directory from where the command will be executed. Default $final_path.
-# | arg: -c, --commands - Commands to execute.
-#
-# Requires YunoHost version 4.2 or higher.
-ynh_composer_exec() {
-    # Declare an array to define the options of this helper.
-    local legacy_args=vwc
-    declare -Ar args_array=([v]=phpversion= [w]=workdir= [c]=commands=)
-    local phpversion
-    local workdir
-    local commands
-    # Manage arguments with getopts
-    ynh_handle_getopts_args "$@"
-    workdir="${workdir:-$final_path}"
-    phpversion="${phpversion:-$YNH_PHP_VERSION}"
-
-    COMPOSER_HOME="$workdir/.composer" COMPOSER_MEMORY_LIMIT=-1 \
-        php${phpversion} "$workdir/composer.phar" $commands \
-        -d "$workdir" --no-interaction --no-ansi 2>&1
-}
-
-# Install and initialize Composer in the given directory
-#
-# usage: ynh_install_composer [--phpversion=phpversion] [--workdir=$final_path] [--install_args="--optimize-autoloader"] [--composerversion=composerversion]
-# | arg: -v, --phpversion - PHP version to use with composer
-# | arg: -w, --workdir - The directory from where the command will be executed. Default $final_path.
-# | arg: -a, --install_args - Additional arguments provided to the composer install. Argument --no-dev already include
-# | arg: -c, --composerversion - Composer version to install
-#
-# Requires YunoHost version 4.2 or higher.
-ynh_install_composer() {
-    # Declare an array to define the options of this helper.
-    local legacy_args=vwac
-    declare -Ar args_array=([v]=phpversion= [w]=workdir= [a]=install_args= [c]=composerversion=)
-    local phpversion
-    local workdir
-    local install_args
-    local composerversion
-    # Manage arguments with getopts
-    ynh_handle_getopts_args "$@"
-    workdir="${workdir:-$final_path}"
-    phpversion="${phpversion:-$YNH_PHP_VERSION}"
-    install_args="${install_args:-}"
-    composerversion="${composerversion:-$YNH_COMPOSER_VERSION}"
-
-    curl -sS https://getcomposer.org/installer \
-        | COMPOSER_HOME="$workdir/.composer" \
-            php${phpversion} -- --quiet --install-dir="$workdir" --version=$composerversion \
-        || ynh_die --message="Unable to install Composer."
-
-    # install dependencies
-    ynh_composer_exec --phpversion="${phpversion}" --workdir="$workdir" --commands="install --no-dev $install_args" \
-        || ynh_die --message="Unable to install core dependencies with Composer."
-}

helpers/helpers.v1.d/postgresql

@@ -51,7 +51,7 @@ ynh_psql_execute_as_root() {
     fi
 
     ynh_psql_connect_as --user="postgres" --password="$(cat $PSQL_ROOT_PWD_FILE)" \
-        $database <<<"$sql"
+        $database <<< "$sql"
 }
 
 # Execute a command from a file as root user
@@ -76,7 +76,7 @@ ynh_psql_execute_file_as_root() {
     fi
 
     ynh_psql_connect_as --user="postgres" --password="$(cat $PSQL_ROOT_PWD_FILE)" \
-        $database <"$file"
+        $database < "$file"
 }
 
 # Create a database and grant optionnaly privilegies to a user
@@ -160,6 +160,8 @@ ynh_psql_create_user() {
 
 # Check if a psql user exists
 #
+# [packagingv1]
+#
 # usage: ynh_psql_user_exists --user=user
 # | arg: -u, --user=    - the user for which to check existence
 # | exit: Return 1 if the user doesn't exist, 0 otherwise
@@ -195,7 +197,12 @@ ynh_psql_database_exists() {
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    if ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT datname FROM pg_database WHERE datname='$database';" | grep --quiet "$database"; then
+    # if psql is not there, we cannot check the db
+    # though it could exists.
+    if ! command -v psql; then
+        ynh_print_err -m "PostgreSQL is not installed, impossible to check for db existence."
+        return 1
+    elif ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT datname FROM pg_database WHERE datname='$database';" | grep --quiet "$database"; then
         return 1
     else
         return 0
@@ -216,6 +223,8 @@ ynh_psql_drop_user() {
 
 # Create a database, an user and its password. Then store the password in the app's config
 #
+# [packagingv1]
+#
 # usage: ynh_psql_setup_db --db_user=user --db_name=name [--db_pwd=pwd]
 # | arg: -u, --db_user=     - Owner of the database
 # | arg: -n, --db_name=     - Name of the database
@@ -251,6 +260,8 @@ ynh_psql_setup_db() {
 
 # Remove a database if it exists, and the associated user
 #
+# [packagingv1]
+#
 # usage: ynh_psql_remove_db --db_user=user --db_name=name
 # | arg: -u, --db_user=     - Owner of the database
 # | arg: -n, --db_name=     - Name of the database

helpers/helpers.v1.d/redis

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# get the first available redis database
+#
+# usage: ynh_redis_get_free_db
+# | returns: the database number to use
+ynh_redis_get_free_db() {
+    local result max db
+    result=$(redis-cli INFO keyspace)
+
+    # get the num
+    max=$(cat /etc/redis/redis.conf | grep ^databases | grep -Eow "[0-9]+")
+
+    db=0
+    # default Debian setting is 15 databases
+    for i in $(seq 0 "$max"); do
+        if ! echo "$result" | grep -q "db$i"; then
+            db=$i
+            break 1
+        fi
+        db=-1
+    done
+
+    test "$db" -eq -1 && ynh_die --message="No available Redis databases..."
+
+    echo "$db"
+}
+
+# Create a master password and set up global settings
+# Please always call this script in install and restore scripts
+#
+# usage: ynh_redis_remove_db database
+# | arg: database - the database to erase
+ynh_redis_remove_db() {
+    local db=$1
+    redis-cli -n "$db" flushdb
+}

helpers/helpers.v1.d/ruby

@@ -0,0 +1,300 @@
+#!/bin/bash
+
+rbenv_install_dir="/opt/rbenv"
+ruby_version_path="$rbenv_install_dir/versions"
+
+# RBENV_ROOT is the directory of rbenv, it needs to be loaded as a environment variable.
+export RBENV_ROOT="$rbenv_install_dir"
+export rbenv_root="$rbenv_install_dir"
+
+if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then
+    build_ruby_dependencies="libjemalloc-dev curl build-essential libreadline-dev zlib1g-dev libsqlite3-dev libssl-dev libxml2-dev libxslt-dev autoconf automake bison libtool"
+    build_pkg_dependencies="${build_pkg_dependencies:-} $build_ruby_dependencies"
+fi
+
+# Load the version of Ruby for an app, and set variables.
+#
+# ynh_use_ruby has to be used in any app scripts before using Ruby for the first time.
+# This helper will provide alias and variables to use in your scripts.
+#
+# To use gem or Ruby, use the alias `ynh_gem` and `ynh_ruby`
+# Those alias will use the correct version installed for the app
+# For example: use `ynh_gem install` instead of `gem install`
+#
+# With `sudo` or `ynh_exec_as`, use instead the fallback variables `$ynh_gem` and `$ynh_ruby`
+# And propagate $PATH to sudo with $ynh_ruby_load_path
+# Exemple: `ynh_exec_as $app $ynh_ruby_load_path $ynh_gem install`
+#
+# $PATH contains the path of the requested version of Ruby.
+# However, $PATH is duplicated into $ruby_path to outlast any manipulation of $PATH
+# You can use the variable `$ynh_ruby_load_path` to quickly load your Ruby version
+#  in $PATH for an usage into a separate script.
+# Exemple: $ynh_ruby_load_path $final_path/script_that_use_gem.sh`
+#
+#
+# Finally, to start a Ruby service with the correct version, 2 solutions
+#  Either the app is dependent of Ruby or gem, but does not called it directly.
+#  In such situation, you need to load PATH
+#    `Environment="__YNH_RUBY_LOAD_PATH__"`
+#    `ExecStart=__FINALPATH__/my_app`
+#     You will replace __YNH_RUBY_LOAD_PATH__ with $ynh_ruby_load_path
+#
+#  Or Ruby start the app directly, then you don't need to load the PATH variable
+#    `ExecStart=__YNH_RUBY__ my_app run`
+#     You will replace __YNH_RUBY__ with $ynh_ruby
+#
+#
+# one other variable is also available
+#   - $ruby_path: The absolute path to Ruby binaries for the chosen version.
+#
+# usage: ynh_use_ruby
+#
+# Requires YunoHost version 3.2.2 or higher.
+ynh_use_ruby() {
+    ruby_version=$(ynh_app_setting_get --app=$app --key=ruby_version)
+
+    # Get the absolute path of this version of Ruby
+    ruby_path="$ruby_version_path/$app/bin"
+
+    # Allow alias to be used into bash script
+    shopt -s expand_aliases
+
+    # Create an alias for the specific version of Ruby and a variable as fallback
+    ynh_ruby="$ruby_path/ruby"
+    alias ynh_ruby="$ynh_ruby"
+    # And gem
+    ynh_gem="$ruby_path/gem"
+    alias ynh_gem="$ynh_gem"
+
+    # Load the path of this version of Ruby in $PATH
+    if [[ :$PATH: != *":$ruby_path"* ]]; then
+        PATH="$ruby_path:$PATH"
+    fi
+    # Create an alias to easily load the PATH
+    ynh_ruby_load_path="PATH=$PATH"
+
+    # Sets the local application-specific Ruby version
+    pushd ${install_dir:-$final_path}
+    $rbenv_install_dir/bin/rbenv local $ruby_version
+    popd
+}
+
+# Install a specific version of Ruby
+#
+# ynh_install_ruby will install the version of Ruby provided as argument by using rbenv.
+#
+# This helper creates a /etc/profile.d/rbenv.sh that configures PATH environment for rbenv
+# for every LOGIN user, hence your user must have a defined shell (as opposed to /usr/sbin/nologin)
+#
+# Don't forget to execute ruby-dependent command in a login environment
+# (e.g. sudo --login option)
+# When not possible (e.g. in systemd service definition), please use direct path
+# to rbenv shims (e.g. $RBENV_ROOT/shims/bundle)
+#
+# usage: ynh_install_ruby --ruby_version=ruby_version
+# | arg: -v, --ruby_version= - Version of ruby to install.
+#
+# Requires YunoHost version 3.2.2 or higher.
+ynh_install_ruby() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=v
+    local -A args_array=([v]=ruby_version=)
+    local ruby_version
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    # Load rbenv path in PATH
+    local CLEAR_PATH="$rbenv_install_dir/bin:$PATH"
+
+    # Remove /usr/local/bin in PATH in case of Ruby prior installation
+    PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@')
+
+    # Move an existing Ruby binary, to avoid to block rbenv
+    test -x /usr/bin/ruby && mv /usr/bin/ruby /usr/bin/ruby_rbenv
+
+    # Install or update rbenv
+    mkdir -p $rbenv_install_dir
+    rbenv="$(command -v rbenv $rbenv_install_dir/bin/rbenv | grep "$rbenv_install_dir/bin/rbenv" | head -1)"
+    if [ -n "$rbenv" ]; then
+        pushd "${rbenv%/*/*}"
+        if git remote -v 2> /dev/null | grep "https://github.com/rbenv/rbenv.git"; then
+            ynh_print_info --message="Updating rbenv..."
+            git pull -q --tags origin master
+            ynh_ruby_try_bash_extension
+        else
+            ynh_print_info --message="Reinstalling rbenv..."
+            cd ..
+            ynh_secure_remove --file=$rbenv_install_dir
+            mkdir -p $rbenv_install_dir
+            cd $rbenv_install_dir
+            git init -q
+            git remote add -f -t master origin https://github.com/rbenv/rbenv.git > /dev/null 2>&1
+            git checkout -q -b master origin/master
+            ynh_ruby_try_bash_extension
+            rbenv=$rbenv_install_dir/bin/rbenv
+        fi
+        popd
+    else
+        ynh_print_info --message="Installing rbenv..."
+        pushd $rbenv_install_dir
+        git init -q
+        git remote add -f -t master origin https://github.com/rbenv/rbenv.git > /dev/null 2>&1
+        git checkout -q -b master origin/master
+        ynh_ruby_try_bash_extension
+        rbenv=$rbenv_install_dir/bin/rbenv
+        popd
+    fi
+
+    mkdir -p "${rbenv_install_dir}/plugins"
+
+    ruby_build="$(command -v "$rbenv_install_dir"/plugins/*/bin/rbenv-install rbenv-install | head -1)"
+    if [ -n "$ruby_build" ]; then
+        pushd "${ruby_build%/*/*}"
+        if git remote -v 2> /dev/null | grep "https://github.com/rbenv/ruby-build.git"; then
+            ynh_print_info --message="Updating ruby-build..."
+            git pull -q origin master
+        fi
+        popd
+    else
+        ynh_print_info --message="Installing ruby-build..."
+        git clone -q https://github.com/rbenv/ruby-build.git "${rbenv_install_dir}/plugins/ruby-build"
+    fi
+
+    rbenv_alias="$(command -v "$rbenv_install_dir"/plugins/*/bin/rbenv-alias rbenv-alias | head -1)"
+    if [ -n "$rbenv_alias" ]; then
+        pushd "${rbenv_alias%/*/*}"
+        if git remote -v 2> /dev/null | grep "https://github.com/tpope/rbenv-aliases.git"; then
+            ynh_print_info --message="Updating rbenv-aliases..."
+            git pull -q origin master
+        fi
+        popd
+    else
+        ynh_print_info --message="Installing rbenv-aliases..."
+        git clone -q https://github.com/tpope/rbenv-aliases.git "${rbenv_install_dir}/plugins/rbenv-aliase"
+    fi
+
+    rbenv_latest="$(command -v "$rbenv_install_dir"/plugins/*/bin/rbenv-latest rbenv-latest | head -1)"
+    if [ -n "$rbenv_latest" ]; then
+        pushd "${rbenv_latest%/*/*}"
+        if git remote -v 2> /dev/null | grep "https://github.com/momo-lab/xxenv-latest.git"; then
+            ynh_print_info --message="Updating xxenv-latest..."
+            git pull -q origin master
+        fi
+        popd
+    else
+        ynh_print_info --message="Installing xxenv-latest..."
+        git clone -q https://github.com/momo-lab/xxenv-latest.git "${rbenv_install_dir}/plugins/xxenv-latest"
+    fi
+
+    # Enable caching
+    mkdir -p "${rbenv_install_dir}/cache"
+
+    # Create shims directory if needed
+    mkdir -p "${rbenv_install_dir}/shims"
+
+    # Restore /usr/local/bin in PATH
+    PATH=$CLEAR_PATH
+
+    # And replace the old Ruby binary
+    test -x /usr/bin/ruby_rbenv && mv /usr/bin/ruby_rbenv /usr/bin/ruby
+
+    # Install the requested version of Ruby
+    local final_ruby_version=$(rbenv latest --print $ruby_version)
+    if ! [ -n "$final_ruby_version" ]; then
+        final_ruby_version=$ruby_version
+    fi
+    ynh_print_info --message="Installing Ruby $final_ruby_version"
+    RUBY_CONFIGURE_OPTS="--disable-install-doc --with-jemalloc" MAKE_OPTS="-j2" rbenv install --skip-existing $final_ruby_version > /dev/null 2>&1
+
+    # Store ruby_version into the config of this app
+    ynh_app_setting_set --app=$app --key=ruby_version --value=$final_ruby_version
+
+    # Remove app virtualenv
+    if rbenv alias --list | grep --quiet "$app "; then
+        rbenv alias $app --remove
+    fi
+
+    # Create app virtualenv
+    rbenv alias $app $final_ruby_version
+
+    # Cleanup Ruby versions
+    ynh_cleanup_ruby
+
+    # Set environment for Ruby users
+    echo "#rbenv
+export RBENV_ROOT=$rbenv_install_dir
+export PATH=\"$rbenv_install_dir/bin:$PATH\"
+eval \"\$(rbenv init -)\"
+#rbenv" > /etc/profile.d/rbenv.sh
+
+    # Load the environment
+    eval "$(rbenv init -)"
+}
+
+# Remove the version of Ruby used by the app.
+#
+# This helper will also cleanup Ruby versions
+#
+# usage: ynh_remove_ruby
+ynh_remove_ruby() {
+    local ruby_version=$(ynh_app_setting_get --app=$app --key=ruby_version)
+
+    # Load rbenv path in PATH
+    local CLEAR_PATH="$rbenv_install_dir/bin:$PATH"
+
+    # Remove /usr/local/bin in PATH in case of Ruby prior installation
+    PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@')
+
+    rbenv alias $app --remove
+
+    # Remove the line for this app
+    ynh_app_setting_delete --app=$app --key=ruby_version
+
+    # Cleanup Ruby versions
+    ynh_cleanup_ruby
+}
+
+# Remove no more needed versions of Ruby used by the app.
+#
+# This helper will check what Ruby version are no more required,
+# and uninstall them
+# If no app uses Ruby, rbenv will be also removed.
+#
+# usage: ynh_cleanup_ruby
+ynh_cleanup_ruby() {
+
+    # List required Ruby versions
+    local installed_apps=$(yunohost app list | grep -oP 'id: \K.*$')
+    local required_ruby_versions=""
+    for installed_app in $installed_apps; do
+        local installed_app_ruby_version=$(ynh_app_setting_get --app=$installed_app --key="ruby_version")
+        if [[ -n "$installed_app_ruby_version" ]]; then
+            required_ruby_versions="${installed_app_ruby_version}\n${required_ruby_versions}"
+        fi
+    done
+
+    # Remove no more needed Ruby versions
+    local installed_ruby_versions=$(rbenv versions --bare --skip-aliases | grep -Ev '/')
+    for installed_ruby_version in $installed_ruby_versions; do
+        if ! echo ${required_ruby_versions} | grep -q "${installed_ruby_version}"; then
+            ynh_print_info --message="Removing Ruby-$installed_ruby_version"
+            $rbenv_install_dir/bin/rbenv uninstall --force $installed_ruby_version
+        fi
+    done
+
+    # If none Ruby version is required
+    if [[ -z "$required_ruby_versions" ]]; then
+        # Remove rbenv environment configuration
+        ynh_print_info --message="Removing rbenv"
+        ynh_secure_remove --file="$rbenv_install_dir"
+        ynh_secure_remove --file="/etc/profile.d/rbenv.sh"
+    fi
+}
+
+ynh_ruby_try_bash_extension() {
+    if [ -x src/configure ]; then
+        src/configure && make -C src || {
+            ynh_print_info --message="Optional bash extension failed to build, but things will still work normally."
+        }
+    fi
+}

helpers/helpers.v1.d/setting

@@ -52,6 +52,42 @@ ynh_app_setting_set() {
     fi
 }
 
+# Set an application setting but only if the "$key" variable ain't set yet
+#
+# Note that it doesn't just define the setting but ALSO define the $foobar variable
+#
+# Hence it's meant as a replacement for this legacy overly complex syntax:
+#
+# if [ -z "${foo:-}" ]
+# then
+#     foo="bar"
+#     ynh_app_setting_set --key="foo" --value="$foo"
+# fi
+#
+# usage: ynh_app_setting_set_default --app=app --key=key --value=value
+# | arg: -a, --app=     - the application id
+# | arg: -k, --key=     - the setting name to set
+# | arg: -v, --value=   - the default setting value to set
+#
+# Requires YunoHost version 11.1.16 or higher.
+ynh_app_setting_set_default() {
+    local _globalapp=${app-:}
+    # Declare an array to define the options of this helper.
+    local legacy_args=akv
+    local -A args_array=([a]=app= [k]=key= [v]=value=)
+    local app
+    local key
+    local value
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    app="${app:-$_globalapp}"
+
+    if [ -z "${!key:-}" ]; then
+        eval $key=\$value
+        ynh_app_setting "set" "$app" "$key" "$value"
+    fi
+}
+
 # Delete an application setting
 #
 # usage: ynh_app_setting_delete --app=app --key=key
@@ -84,7 +120,7 @@ ynh_app_setting_delete() {
 #
 ynh_app_setting() {
     set +o xtrace # set +x
-    ACTION="$1" APP="$2" KEY="$3" VALUE="${4:-}" python3 - <<EOF
+    ACTION="$1" APP="$2" KEY="$3" VALUE="${4:-}" python3 - << EOF
 import os, yaml, sys
 app, action = os.environ['APP'], os.environ['ACTION'].lower()
 key, value = os.environ['KEY'], os.environ.get('VALUE', None)
@@ -113,6 +149,8 @@ EOF
 
 # Check availability of a web path
 #
+# [packagingv1]
+#
 # usage: ynh_webpath_available --domain=domain --path_url=path
 # | arg: -d, --domain=      - the domain/host of the url
 # | arg: -p, --path_url=    - the web path to check the availability of
@@ -134,6 +172,8 @@ ynh_webpath_available() {
 
 # Register/book a web path for an app
 #
+# [packagingv1]
+#
 # usage: ynh_webpath_register --app=app --domain=domain --path_url=path
 # | arg: -a, --app=         - the app for which the domain should be registered
 # | arg: -d, --domain=      - the domain/host of the web path

helpers/helpers.v1.d/sources

@@ -0,0 +1,286 @@
+#!/bin/bash
+
+# Download, check integrity, uncompress and patch the source from app.src
+#
+# usage: ynh_setup_source --dest_dir=dest_dir [--source_id=source_id] [--keep="file1 file2"] [--full_replace]
+# | arg: -d, --dest_dir=     - Directory where to setup sources
+# | arg: -s, --source_id=    - Name of the source, defaults to `main` (when the sources resource exists in manifest.toml) or (legacy) `app` otherwise
+# | arg: -k, --keep=         - Space-separated list of files/folders that will be backup/restored in $dest_dir, such as a config file you don't want to overwrite. For example 'conf.json secrets.json logs' (no trailing `/` for folders)
+# | arg: -r, --full_replace= - Remove previous sources before installing new sources  (can be 1 or 0, default to 0)
+#
+# ##### New 'sources' resources
+#
+# (See also the resources documentation which may be more complete?)
+#
+# This helper will read infos from the 'sources' resources in the manifest.toml of the app
+# and expect a structure like:
+#
+# ```toml
+# [resources.sources]
+#     [resources.sources.main]
+#     url = "https://some.address.to/download/the/app/archive"
+#     sha256 = "0123456789abcdef"    # The sha256 sum of the asset obtained from the URL
+# ```
+#
+# ##### Optional flags
+#
+# ```text
+# format    = "tar.gz"/xz/bz2    # automatically guessed from the extension of the URL, but can be set explicitly. Will use `tar` to extract
+#             "zip"              # automatically guessed from the extension of the URL, but can be set explicitly. Will use `unzip` to extract
+#             "docker"           # useful to extract files from an already-built docker image (instead of rebuilding them locally). Will use `docker-image-extract` to extract
+#             "whatever"         # an arbitrary value, not really meaningful except to imply that the file won't be extracted
+#
+# in_subdir = true    # default, there's an intermediate subdir in the archive before accessing the actual files
+#             false   # sources are directly in the archive root
+#             n       # (special cases) an integer representing a number of subdirs levels to get rid of
+#
+# extract   = true    # default if file is indeed an archive such as .zip, .tar.gz, .tar.bz2, ...
+#           = false   # default if file 'format' is not set and the file is not to be extracted because it is not an archive but a script or binary or whatever asset.
+#                     #    in which case the file will only be `mv`ed to the location possibly renamed using the `rename` value
+#
+# rename    = "whatever_your_want"   # to be used for convenience when `extract` is false and the default name of the file is not practical
+# platform  = "linux/amd64"          # (defaults to "linux/$YNH_ARCH") to be used in conjonction with `format = "docker"` to specify which architecture to extract for
+# ```
+#
+# You may also define assets url and checksum per-architectures such as:
+# ```toml
+# [resources.sources]
+#     [resources.sources.main]
+#     amd64.url = "https://some.address.to/download/the/app/archive/when/amd64"
+#     amd64.sha256 = "0123456789abcdef"
+#     armhf.url = "https://some.address.to/download/the/app/archive/when/armhf"
+#     armhf.sha256 = "fedcba9876543210"
+# ```
+#
+# In which case `ynh_setup_source --dest_dir="$install_dir"` will automatically pick the appropriate source depending on the arch
+#
+# The helper will:
+# - Download the specific URL if there is no local archive
+# - Check the integrity with the specific sha256 sum
+# - Uncompress the archive to `$dest_dir`.
+#   - If `in_subdir` is true, the first level directory of the archive will be removed.
+#   - If `in_subdir` is a numeric value, the N first level directories will be removed.
+# - Patches named `sources/patches/${src_id}-*.patch` will be applied to `$dest_dir`
+# - Extra files in `sources/extra_files/$src_id` will be copied to dest_dir
+#
+# Requires YunoHost version 2.6.4 or higher.
+ynh_setup_source() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=dsk
+    local -A args_array=([d]=dest_dir= [s]=source_id= [k]=keep= [r]=full_replace=)
+    local dest_dir
+    local source_id
+    local keep
+    local full_replace
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    keep="${keep:-}"
+    full_replace="${full_replace:-0}"
+
+    if test -e $YNH_APP_BASEDIR/manifest.toml && cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq -e '.resources.sources' > /dev/null; then
+        source_id="${source_id:-main}"
+        local sources_json=$(cat $YNH_APP_BASEDIR/manifest.toml | toml_to_json | jq ".resources.sources[\"$source_id\"]")
+        if jq -re ".url" <<< "$sources_json"; then
+            local arch_prefix=""
+        else
+            local arch_prefix=".$YNH_ARCH"
+        fi
+
+        local src_url="$(jq -r "$arch_prefix.url" <<< "$sources_json" | sed 's/^null$//')"
+        local src_sum="$(jq -r "$arch_prefix.sha256" <<< "$sources_json" | sed 's/^null$//')"
+        local src_sumprg="sha256sum"
+        local src_format="$(jq -r ".format" <<< "$sources_json" | sed 's/^null$//')"
+        local src_in_subdir="$(jq -r ".in_subdir" <<< "$sources_json" | sed 's/^null$//')"
+        local src_extract="$(jq -r ".extract" <<< "$sources_json" | sed 's/^null$//')"
+        local src_platform="$(jq -r ".platform" <<< "$sources_json" | sed 's/^null$//')"
+        local src_rename="$(jq -r ".rename" <<< "$sources_json" | sed 's/^null$//')"
+
+        [[ -n "$src_url" ]] || ynh_die "No URL defined for source $source_id$arch_prefix ?"
+        [[ -n "$src_sum" ]] || ynh_die "No sha256 sum defined for source $source_id$arch_prefix ?"
+
+        if [[ -z "$src_format" ]]; then
+            if [[ "$src_url" =~ ^.*\.zip$ ]] || [[ "$src_url" =~ ^.*/zipball/.*$ ]]; then
+                src_format="zip"
+            elif [[ "$src_url" =~ ^.*\.tar\.gz$ ]] || [[ "$src_url" =~ ^.*\.tgz$ ]] || [[ "$src_url" =~ ^.*/tar\.gz/.*$ ]] || [[ "$src_url" =~ ^.*/tarball/.*$ ]]; then
+                src_format="tar.gz"
+            elif [[ "$src_url" =~ ^.*\.tar\.xz$ ]]; then
+                src_format="tar.xz"
+            elif [[ "$src_url" =~ ^.*\.tar\.bz2$ ]]; then
+                src_format="tar.bz2"
+            elif [[ -z "$src_extract" ]]; then
+                src_extract="false"
+            fi
+        fi
+    else
+        source_id="${source_id:-app}"
+        local src_file_path="$YNH_APP_BASEDIR/conf/${source_id}.src"
+
+        # Load value from configuration file (see above for a small doc about this file
+        # format)
+        local src_url=$(grep 'SOURCE_URL=' "$src_file_path" | cut --delimiter='=' --fields=2-)
+        local src_sum=$(grep 'SOURCE_SUM=' "$src_file_path" | cut --delimiter='=' --fields=2-)
+        local src_sumprg=$(grep 'SOURCE_SUM_PRG=' "$src_file_path" | cut --delimiter='=' --fields=2-)
+        local src_format=$(grep 'SOURCE_FORMAT=' "$src_file_path" | cut --delimiter='=' --fields=2-)
+        local src_in_subdir=$(grep 'SOURCE_IN_SUBDIR=' "$src_file_path" | cut --delimiter='=' --fields=2-)
+        local src_rename=$(grep 'SOURCE_FILENAME=' "$src_file_path" | cut --delimiter='=' --fields=2-)
+        local src_extract=$(grep 'SOURCE_EXTRACT=' "$src_file_path" | cut --delimiter='=' --fields=2-)
+        local src_platform=$(grep 'SOURCE_PLATFORM=' "$src_file_path" | cut --delimiter='=' --fields=2-)
+    fi
+
+    # Default value
+    src_sumprg=${src_sumprg:-sha256sum}
+    src_in_subdir=${src_in_subdir:-true}
+    src_format=${src_format:-tar.gz}
+    src_format=$(echo "$src_format" | tr '[:upper:]' '[:lower:]')
+    src_extract=${src_extract:-true}
+
+    if [[ "$src_extract" != "true" ]] && [[ "$src_extract" != "false" ]]; then
+        ynh_die "For source $source_id, expected either 'true' or 'false' for the extract parameter"
+    fi
+
+    # (Unused?) mecanism where one can have the file in a special local cache to not have to download it...
+    local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${source_id}"
+
+    # Gotta use this trick with 'dirname' because source_id may contain slashes x_x
+    mkdir -p $(dirname /var/cache/yunohost/download/${YNH_APP_ID}/${source_id})
+    src_filename="/var/cache/yunohost/download/${YNH_APP_ID}/${source_id}"
+
+    if [ "$src_format" = "docker" ]; then
+        src_platform="${src_platform:-"linux/$YNH_ARCH"}"
+    else
+        if test -e "$local_src"; then
+            cp $local_src $src_filename
+        fi
+
+        [ -n "$src_url" ] || ynh_die "Couldn't parse SOURCE_URL from $src_file_path ?"
+
+        # If the file was prefetched but somehow doesn't match the sum, rm and redownload it
+        if [ -e "$src_filename" ] && ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status; then
+            rm -f "$src_filename"
+        fi
+
+        # Only redownload the file if it wasnt prefetched
+        if [ ! -e "$src_filename" ]; then
+            # NB. we have to declare the var as local first,
+            # otherwise 'local foo=$(false) || echo 'pwet'" does'nt work
+            # because local always return 0 ...
+            local out
+            # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget)
+            out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$src_filename $src_url 2>&1) \
+                || ynh_die --message="$out"
+        fi
+
+        # Check the control sum
+        if ! echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status; then
+            local actual_sum="$(${src_sumprg} ${src_filename} | cut --delimiter=' ' --fields=1)"
+            local actual_size="$(du -hs ${src_filename} | cut --fields=1)"
+            rm -f ${src_filename}
+            ynh_die --message="Corrupt source for ${src_url}: Expected sha256sum to be ${src_sum} but got ${actual_sum} (size: ${actual_size})."
+        fi
+    fi
+
+    # Keep files to be backup/restored at the end of the helper
+    # Assuming $dest_dir already exists
+    rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/
+    if [ -n "$keep" ] && [ -e "$dest_dir" ]; then
+        local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID}
+        mkdir -p $keep_dir
+        local stuff_to_keep
+        for stuff_to_keep in $keep; do
+            if [ -e "$dest_dir/$stuff_to_keep" ]; then
+                mkdir --parents "$(dirname "$keep_dir/$stuff_to_keep")"
+                cp --archive "$dest_dir/$stuff_to_keep" "$keep_dir/$stuff_to_keep"
+            fi
+        done
+    fi
+
+    if [ "$full_replace" -eq 1 ]; then
+        ynh_secure_remove --file="$dest_dir"
+    fi
+
+    # Extract source into the app dir
+    mkdir --parents "$dest_dir"
+
+    if [ -n "${install_dir:-}" ] && [ "$dest_dir" == "$install_dir" ]; then
+        _ynh_apply_default_permissions $dest_dir
+    fi
+    if [ -n "${final_path:-}" ] && [ "$dest_dir" == "$final_path" ]; then
+        _ynh_apply_default_permissions $dest_dir
+    fi
+
+    if [[ "$src_extract" == "false" ]]; then
+        if [[ -z "$src_rename" ]]; then
+            mv $src_filename $dest_dir
+        else
+            mv $src_filename $dest_dir/$src_rename
+        fi
+    elif [[ "$src_format" == "docker" ]]; then
+        "$YNH_HELPERS_DIR/vendor/docker-image-extract/docker-image-extract" -p $src_platform -o $dest_dir $src_url 2>&1
+    elif [[ "$src_format" == "zip" ]]; then
+        # Zip format
+        # Using of a temp directory, because unzip doesn't manage --strip-components
+        if $src_in_subdir; then
+            local tmp_dir=$(mktemp --directory)
+            unzip -quo $src_filename -d "$tmp_dir"
+            cp --archive $tmp_dir/*/. "$dest_dir"
+            ynh_secure_remove --file="$tmp_dir"
+        else
+            unzip -quo $src_filename -d "$dest_dir"
+        fi
+        ynh_secure_remove --file="$src_filename"
+    else
+        local strip=""
+        if [ "$src_in_subdir" != "false" ]; then
+            if [ "$src_in_subdir" == "true" ]; then
+                local sub_dirs=1
+            else
+                local sub_dirs="$src_in_subdir"
+            fi
+            strip="--strip-components $sub_dirs"
+        fi
+        if [[ "$src_format" =~ ^tar.gz|tar.bz2|tar.xz$ ]]; then
+            tar --extract --file=$src_filename --directory="$dest_dir" $strip
+        else
+            ynh_die --message="Archive format unrecognized."
+        fi
+        ynh_secure_remove --file="$src_filename"
+    fi
+
+    # Apply patches
+    if [ -d "$YNH_APP_BASEDIR/sources/patches/" ]; then
+        local patches_folder=$(realpath $YNH_APP_BASEDIR/sources/patches/)
+        if (($(find $patches_folder -type f -name "${source_id}-*.patch" 2> /dev/null | wc --lines) > "0")); then
+            pushd "$dest_dir"
+            for p in $patches_folder/${source_id}-*.patch; do
+                echo $p
+                patch --strip=1 < $p || ynh_print_warn --message="Packagers /!\\ patch $p failed to apply"
+            done
+            popd
+        fi
+    fi
+
+    # Add supplementary files
+    if test -e "$YNH_APP_BASEDIR/sources/extra_files/${source_id}"; then
+        cp --archive $YNH_APP_BASEDIR/sources/extra_files/$source_id/. "$dest_dir"
+    fi
+
+    # Keep files to be backup/restored at the end of the helper
+    # Assuming $dest_dir already exists
+    if [ -n "$keep" ]; then
+        local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID}
+        local stuff_to_keep
+        for stuff_to_keep in $keep; do
+            if [ -e "$keep_dir/$stuff_to_keep" ]; then
+                mkdir --parents "$(dirname "$dest_dir/$stuff_to_keep")"
+
+                # We add "--no-target-directory" (short option is -T) to handle the special case
+                # when we "keep" a folder, but then the new setup already contains the same dir (but possibly empty)
+                # in which case a regular "cp" will create a copy of the directory inside the directory ...
+                # resulting in something like /var/www/$app/data/data instead of /var/www/$app/data
+                # cf https://unix.stackexchange.com/q/94831 for a more elaborate explanation on the option
+                cp --archive --no-target-directory "$keep_dir/$stuff_to_keep" "$dest_dir/$stuff_to_keep"
+            fi
+        done
+    fi
+    rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/
+}

helpers/helpers.v1.d/string

@@ -21,7 +21,7 @@ ynh_string_random() {
     length=${length:-24}
     filter=${filter:-'A-Za-z0-9'}
 
-    dd if=/dev/urandom bs=1 count=1000 2>/dev/null \
+    dd if=/dev/urandom bs=1 count=1000 2> /dev/null \
         | tr --complement --delete "$filter" \
         | sed --quiet 's/\(.\{'"$length"'\}\).*/\1/p'
 }
@@ -48,7 +48,7 @@ ynh_replace_string() {
     ynh_handle_getopts_args "$@"
     set +o xtrace # set +x
 
-    local delimit=@
+    local delimit=$'\001'
     # Escape the delimiter if it's in the string.
     match_string=${match_string//${delimit}/"\\${delimit}"}
     replace_string=${replace_string//${delimit}/"\\${delimit}"}
@@ -91,6 +91,8 @@ ynh_replace_special_string() {
 
 # Sanitize a string intended to be the name of a database
 #
+# [packagingv1]
+#
 # usage: ynh_sanitize_dbid --db_name=name
 # | arg: -n, --db_name=     - name to correct/sanitize
 # | ret: the corrected name

helpers/helpers.v1.d/systemd

@@ -61,7 +61,7 @@ ynh_remove_systemd_config() {
 # | arg: -l, --line_match=   - Line to match - The line to find in the log to attest the service have finished to boot. If not defined it don't wait until the service is completely started.
 # | arg: -p, --log_path=     - Log file - Path to the log file. Default : `/var/log/$app/$app.log`
 # | arg: -t, --timeout=      - Timeout - The maximum time to wait before ending the watching. Default : 300 seconds.
-# | arg: -e, --length=       - Length of the error log : Default : 20
+# | arg: -e, --length=       - Length of the error log displayed for debugging : Default : 20
 #
 # Requires YunoHost version 3.5.0 or higher.
 ynh_systemd_action() {
@@ -94,12 +94,12 @@ ynh_systemd_action() {
         # Following the starting of the app in its log
         if [ "$log_path" == "systemd" ]; then
             # Read the systemd journal
-            journalctl --unit=$service_name --follow --since=-0 --quiet >"$templog" &
+            journalctl --unit=$service_name --follow --since=-0 --quiet > "$templog" &
             # Get the PID of the journalctl command
             local pid_tail=$!
         else
             # Read the specified log file
-            tail --follow=name --retry --lines=0 "$log_path" >"$templog" 2>&1 &
+            tail --follow=name --retry --lines=0 "$log_path" > "$templog" 2>&1 &
             # Get the PID of the tail command
             local pid_tail=$!
         fi
@@ -110,6 +110,8 @@ ynh_systemd_action() {
         action="reload-or-restart"
     fi
 
+    local time_start="$(date --utc --rfc-3339=seconds | cut -d+ -f1) UTC"
+
     # If the service fails to perform the action
     if ! systemctl $action $service_name; then
         # Show syslog for this service
@@ -126,15 +128,31 @@ ynh_systemd_action() {
     if [[ -n "${line_match:-}" ]]; then
         set +x
         local i=0
+        local starttime=$(date +%s)
         for i in $(seq 1 $timeout); do
             # Read the log until the sentence is found, that means the app finished to start. Or run until the timeout
-            if grep --extended-regexp --quiet "$line_match" "$templog"; then
-                ynh_print_info --message="The service $service_name has correctly executed the action ${action}."
-                break
+            if [ "$log_path" == "systemd" ]; then
+                # For systemd services, we in fact dont rely on the templog, which for some reason is not reliable, but instead re-read journalctl every iteration, starting at the timestamp where we triggered the action
+                if journalctl --unit=$service_name --since="$time_start" --quiet --no-pager --no-hostname | grep --extended-regexp --quiet "$line_match"; then
+                    ynh_print_info --message="The service $service_name has correctly executed the action ${action}."
+                    break
+                fi
+            else
+                if grep --extended-regexp --quiet "$line_match" "$templog"; then
+                    ynh_print_info --message="The service $service_name has correctly executed the action ${action}."
+                    break
+                fi
             fi
             if [ $i -eq 30 ]; then
                 echo "(this may take some time)" >&2
             fi
+            # Also check the timeout using actual timestamp, because sometimes for some reason,
+            # journalctl may take a huge time to run, and we end up waiting literally an entire hour
+            # instead of 5 min ...
+            if [[ "$(($(date +%s) - $starttime))" -gt "$timeout" ]]; then
+                i=$timeout
+                break
+            fi
             sleep 1
         done
         set -x

helpers/helpers.v1.d/systemuser

@@ -1,61 +1,9 @@
 #!/bin/bash
 
-# Check if a YunoHost user exists
-#
-# usage: ynh_user_exists --username=username
-# | arg: -u, --username=    - the username to check
-# | ret: 0 if the user exists, 1 otherwise.
-#
-# example: ynh_user_exists 'toto' || echo "User does not exist"
-#
-# Requires YunoHost version 2.2.4 or higher.
-ynh_user_exists() {
-    # Declare an array to define the options of this helper.
-    local legacy_args=u
-    local -A args_array=([u]=username=)
-    local username
-    # Manage arguments with getopts
-    ynh_handle_getopts_args "$@"
-
-    yunohost user list --output-as json --quiet | jq -e ".users.\"${username}\"" >/dev/null
-}
-
-# Retrieve a YunoHost user information
-#
-# usage: ynh_user_get_info --username=username --key=key
-# | arg: -u, --username=    - the username to retrieve info from
-# | arg: -k, --key=         - the key to retrieve
-# | ret: the value associate to that key
-#
-# example: mail=$(ynh_user_get_info --username="toto" --key=mail)
-#
-# Requires YunoHost version 2.2.4 or higher.
-ynh_user_get_info() {
-    # Declare an array to define the options of this helper.
-    local legacy_args=uk
-    local -A args_array=([u]=username= [k]=key=)
-    local username
-    local key
-    # Manage arguments with getopts
-    ynh_handle_getopts_args "$@"
-
-    yunohost user info "$username" --output-as json --quiet | jq -r ".$key"
-}
-
-# Get the list of YunoHost users
-#
-# usage: ynh_user_list
-# | ret: one username per line as strings
-#
-# example: for u in $(ynh_user_list); do ... ; done
-#
-# Requires YunoHost version 2.4.0 or higher.
-ynh_user_list() {
-    yunohost user list --output-as json --quiet | jq -r ".users | keys[]"
-}
-
 # Check if a user exists on the system
 #
+# [packagingv1]
+#
 # usage: ynh_system_user_exists --username=username
 # | arg: -u, --username=    - the username to check
 # | ret: 0 if the user exists, 1 otherwise.
@@ -69,11 +17,13 @@ ynh_system_user_exists() {
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    getent passwd "$username" &>/dev/null
+    getent passwd "$username" &> /dev/null
 }
 
 # Check if a group exists on the system
 #
+# [packagingv1]
+#
 # usage: ynh_system_group_exists --group=group
 # | arg: -g, --group=   - the group to check
 # | ret: 0 if the group exists, 1 otherwise.
@@ -87,7 +37,7 @@ ynh_system_group_exists() {
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    getent group "$group" &>/dev/null
+    getent group "$group" &> /dev/null
 }
 
 # Create a system user

helpers/helpers.v1.d/templating

@@ -0,0 +1,406 @@
+#!/bin/bash
+
+# Create a dedicated config file from a template
+#
+# usage: ynh_add_config --template="template" --destination="destination"
+# | arg: -t, --template=     - Template config file to use
+# | arg: -d, --destination=  - Destination of the config file
+# | arg: -j, --jinja         - Use jinja template instead of the simple `__MY_VAR__` templating format
+#
+# examples:
+# ynh_add_config --template=".env" --destination="$install_dir/.env"   # (use the template file "conf/.env" from the app's package)
+# ynh_add_config --jinja --template="config.j2" --destination="$install_dir/config"    # (use the template file "conf/config.j2" from the app's package)
+#
+# The template can be by default the name of a file in the conf directory
+# of a YunoHost Package, a relative path or an absolute path.
+#
+# The helper will use the template `template` to generate a config file
+# `destination` by replacing the following keywords with global variables
+# that should be defined before calling this helper :
+# ```
+#   __PATH__                by $path_url
+#   __NAME__                by $app
+#   __NAMETOCHANGE__        by $app
+#   __USER__                by $app
+#   __FINALPATH__           by $final_path
+#   __PHPVERSION__          by $YNH_PHP_VERSION (packaging v1 only, packaging v2 uses phpversion setting implicitly set by apt resource)
+#   __YNH_NODE_LOAD_PATH__  by $ynh_node_load_PATH
+# ```
+# And any dynamic variables that should be defined before calling this helper like:
+# ```
+#   __DOMAIN__   by $domain
+#   __APP__      by $app
+#   __VAR_1__    by $var_1
+#   __VAR_2__    by $var_2
+# ```
+#
+# ##### When --jinja is enabled
+#
+# This option is meant for advanced use-cases where the "simple" templating
+# mode ain't enough because you need conditional blocks or loops.
+#
+# For a full documentation of jinja's syntax you can refer to:
+# https://jinja.palletsprojects.com/en/3.1.x/templates/
+#
+# Note that in YunoHost context, all variables are from shell variables and therefore are strings
+#
+# ##### Keeping track of manual changes by the admin
+#
+# The helper will verify the checksum and backup the destination file
+# if it's different before applying the new template.
+#
+# And it will calculate and store the destination file checksum
+# into the app settings when configuration is done.
+#
+# Requires YunoHost version 4.1.0 or higher.
+ynh_add_config() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=tdj
+    local -A args_array=([t]=template= [d]=destination= [j]=jinja)
+    local template
+    local destination
+    local jinja
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    local template_path
+    jinja="${jinja:-0}"
+
+    if [ -f "$YNH_APP_BASEDIR/conf/$template" ]; then
+        template_path="$YNH_APP_BASEDIR/conf/$template"
+    elif [ -f "$template" ]; then
+        template_path=$template
+    else
+        ynh_die --message="The provided template $template doesn't exist"
+    fi
+
+    ynh_backup_if_checksum_is_different --file="$destination"
+
+    # Make sure to set the permissions before we copy the file
+    # This is to cover a case where an attacker could have
+    # created a file beforehand to have control over it
+    # (cp won't overwrite ownership / modes by default...)
+    touch $destination
+    chmod 640 $destination
+    _ynh_apply_default_permissions $destination
+
+    if [[ "$jinja" == 1 ]]; then
+        # This is ran in a subshell such that the "export" does not "contaminate" the main process
+        (   
+            export $(compgen -v)
+            j2 "$template_path" -f env -o $destination
+        )
+    else
+        cp -f "$template_path" "$destination"
+        ynh_replace_vars --file="$destination"
+    fi
+
+    ynh_store_file_checksum --file="$destination"
+}
+
+# Replace variables in a file
+#
+# [internal]
+#
+# usage: ynh_replace_vars --file="file"
+# | arg: -f, --file=     - File where to replace variables
+#
+# The helper will replace the following keywords with global variables
+# that should be defined before calling this helper :
+#   __PATH__                by $path_url
+#   __NAME__                by $app
+#   __NAMETOCHANGE__        by $app
+#   __USER__                by $app
+#   __FINALPATH__           by $final_path
+#   __PHPVERSION__          by $YNH_PHP_VERSION (packaging v1 only, packaging v2 uses phpversion setting implicitly set by apt resource)
+#   __YNH_NODE_LOAD_PATH__  by $ynh_node_load_PATH
+#
+# And any dynamic variables that should be defined before calling this helper like:
+#   __DOMAIN__   by $domain
+#   __APP__      by $app
+#   __VAR_1__    by $var_1
+#   __VAR_2__    by $var_2
+#
+# Requires YunoHost version 4.1.0 or higher.
+ynh_replace_vars() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=f
+    local -A args_array=([f]=file=)
+    local file
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    # Replace specific YunoHost variables
+    if test -n "${path_url:-}"; then
+        # path_url_slash_less is path_url, or a blank value if path_url is only '/'
+        local path_url_slash_less=${path_url%/}
+        ynh_replace_string --match_string="__PATH__/" --replace_string="$path_url_slash_less/" --target_file="$file"
+        ynh_replace_string --match_string="__PATH__" --replace_string="$path_url" --target_file="$file"
+    fi
+    if test -n "${app:-}"; then
+        ynh_replace_string --match_string="__NAME__" --replace_string="$app" --target_file="$file"
+        ynh_replace_string --match_string="__NAMETOCHANGE__" --replace_string="$app" --target_file="$file"
+        ynh_replace_string --match_string="__USER__" --replace_string="$app" --target_file="$file"
+    fi
+    # Legacy
+    if test -n "${final_path:-}"; then
+        ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$file"
+        ynh_replace_string --match_string="__INSTALL_DIR__" --replace_string="$final_path" --target_file="$file"
+    fi
+    # Legacy / Packaging v1 only
+    if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2 && test -n "${YNH_PHP_VERSION:-}"; then
+        ynh_replace_string --match_string="__PHPVERSION__" --replace_string="$YNH_PHP_VERSION" --target_file="$file"
+    fi
+    if test -n "${ynh_node_load_PATH:-}"; then
+        ynh_replace_string --match_string="__YNH_NODE_LOAD_PATH__" --replace_string="$ynh_node_load_PATH" --target_file="$file"
+    fi
+
+    # Replace others variables
+
+    # List other unique (__ __) variables in $file
+    local uniques_vars=($(grep -oP '__[A-Z0-9]+?[A-Z0-9_]*?[A-Z0-9]*?__' $file | sort --unique | sed "s@__\([^.]*\)__@\L\1@g"))
+
+    set +o xtrace # set +x
+
+    # Do the replacement
+    local delimit=@
+    for one_var in "${uniques_vars[@]}"; do
+        # Validate that one_var is indeed defined
+        # -v checks if the variable is defined, for example:
+        #     -v FOO  tests if $FOO is defined
+        #     -v $FOO tests if ${!FOO} is defined
+        # More info: https://stackoverflow.com/questions/3601515/how-to-check-if-a-variable-is-set-in-bash/17538964#comment96392525_17538964
+        [[ -v "${one_var:-}" ]] || ynh_die --message="Variable \$$one_var wasn't initialized when trying to replace __${one_var^^}__ in $file"
+
+        # Escape delimiter in match/replace string
+        match_string="__${one_var^^}__"
+        match_string=${match_string//${delimit}/"\\${delimit}"}
+        replace_string="${!one_var}"
+        replace_string=${replace_string//\\/\\\\}
+        replace_string=${replace_string//${delimit}/"\\${delimit}"}
+
+        # Actually replace (sed is used instead of ynh_replace_string to avoid triggering an epic amount of debug logs)
+        sed --in-place "s${delimit}${match_string}${delimit}${replace_string}${delimit}g" "$file"
+    done
+    set -o xtrace # set -x
+}
+
+# Get a value from heterogeneous file (yaml, json, php, python...)
+#
+# usage: ynh_read_var_in_file --file=PATH --key=KEY
+# | arg: -f, --file=     - the path to the file
+# | arg: -k, --key=     - the key to get
+# | arg: -a, --after=     - the line just before the key (in case of multiple lines with the name of the key in the file)
+#
+# This helpers match several var affectation use case in several languages
+# We don't use jq or equivalent to keep comments and blank space in files
+# This helpers work line by line, it is not able to work correctly
+# if you have several identical keys in your files
+#
+# Example of line this helpers can managed correctly
+# .yml
+#     title: YunoHost documentation
+#     email: 'yunohost@yunohost.org'
+# .json
+#     "theme": "colib'ris",
+#     "port": 8102
+#     "some_boolean":     false,
+#     "user": null
+# .ini
+#     some_boolean = On
+#     action = "Clear"
+#     port = 20
+# .php
+#     $user=
+#     user => 20
+# .py
+#     USER = 8102
+#     user = 'https://donate.local'
+#     CUSTOM['user'] = 'YunoHost'
+#
+# Requires YunoHost version 4.3 or higher.
+ynh_read_var_in_file() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=fka
+    local -A args_array=([f]=file= [k]=key= [a]=after=)
+    local file
+    local key
+    local after
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    after="${after:-}"
+
+    [[ -f $file ]] || ynh_die --message="File $file does not exists"
+
+    set +o xtrace # set +x
+
+    # Get the line number after which we search for the variable
+    local line_number=1
+    if [[ -n "$after" ]]; then
+        line_number=$(grep -m1 -n $after $file | cut -d: -f1)
+        if [[ -z "$line_number" ]]; then
+            set -o xtrace # set -x
+            return 1
+        fi
+    fi
+
+    local filename="$(basename -- "$file")"
+    local ext="${filename##*.}"
+    local endline=',;'
+    local assign="=>|:|="
+    local comments="#"
+    local string="\"'"
+    if [[ "$ext" =~ ^ini|env|toml|yml|yaml$ ]]; then
+        endline='#'
+    fi
+    if [[ "$ext" =~ ^ini|env$ ]]; then
+        comments="[;#]"
+    fi
+    if [[ "php" == "$ext" ]] || [[ "$ext" == "js" ]]; then
+        comments="//"
+    fi
+    local list='\[\s*['$string']?\w+['$string']?\]'
+    local var_part='^\s*((const|var|let)\s+)?\$?(\w+('$list')*(->|\.|\[))*\s*'
+    var_part+="[$string]?${key}[$string]?"
+    var_part+='\s*\]?\s*'
+    var_part+="($assign)"
+    var_part+='\s*'
+
+    # Extract the part after assignation sign
+    local expression_with_comment="$( (tail +$line_number ${file} | grep -i -o -P $var_part'\K.*$' || echo YNH_NULL) | head -n1)"
+    if [[ "$expression_with_comment" == "YNH_NULL" ]]; then
+        set -o xtrace # set -x
+        echo YNH_NULL
+        return 0
+    fi
+
+    # Remove comments if needed
+    local expression="$(echo "$expression_with_comment" | sed "s@${comments}[^$string]*\$@@g" | sed "s@\s*[$endline]*\s*]*\$@@")"
+
+    local first_char="${expression:0:1}"
+    if [[ "$first_char" == '"' ]]; then
+        echo "$expression" | grep -m1 -o -P '"\K([^"](\\")?)*[^\\](?=")' | head -n1 | sed 's/\\"/"/g'
+    elif [[ "$first_char" == "'" ]]; then
+        echo "$expression" | grep -m1 -o -P "'\K([^'](\\\\')?)*[^\\\\](?=')" | head -n1 | sed "s/\\\\'/'/g"
+    else
+        echo "$expression"
+    fi
+    set -o xtrace # set -x
+}
+
+# Set a value into heterogeneous file (yaml, json, php, python...)
+#
+# usage: ynh_write_var_in_file --file=PATH --key=KEY --value=VALUE
+# | arg: -f, --file=     - the path to the file
+# | arg: -k, --key=     - the key to set
+# | arg: -v, --value=     - the value to set
+# | arg: -a, --after=     - the line just before the key (in case of multiple lines with the name of the key in the file)
+#
+# Requires YunoHost version 4.3 or higher.
+ynh_write_var_in_file() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=fkva
+    local -A args_array=([f]=file= [k]=key= [v]=value= [a]=after=)
+    local file
+    local key
+    local value
+    local after
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    after="${after:-}"
+
+    [[ -f $file ]] || ynh_die --message="File $file does not exists"
+
+    set +o xtrace # set +x
+
+    # Get the line number after which we search for the variable
+    local after_line_number=1
+    if [[ -n "$after" ]]; then
+        after_line_number=$(grep -m1 -n $after $file | cut -d: -f1)
+        if [[ -z "$after_line_number" ]]; then
+            set -o xtrace # set -x
+            return 1
+        fi
+    fi
+
+    local filename="$(basename -- "$file")"
+    local ext="${filename##*.}"
+    local endline=',;'
+    local assign="=>|:|="
+    local comments="#"
+    local string="\"'"
+    if [[ "$ext" =~ ^ini|env|toml|yml|yaml$ ]]; then
+        endline='#'
+    fi
+    if [[ "$ext" =~ ^ini|env$ ]]; then
+        comments="[;#]"
+    fi
+    if [[ "php" == "$ext" ]] || [[ "$ext" == "js" ]]; then
+        comments="//"
+    fi
+    local list='\[\s*['$string']?\w+['$string']?\]'
+    local var_part='^\s*((const|var|let)\s+)?\$?(\w+('$list')*(->|\.|\[))*\s*'
+    var_part+="[$string]?${key}[$string]?"
+    var_part+='\s*\]?\s*'
+    var_part+="($assign)"
+    var_part+='\s*'
+
+    # Extract the part after assignation sign
+    local expression_with_comment="$( (tail +$after_line_number ${file} | grep -i -o -P $var_part'\K.*$' || echo YNH_NULL) | head -n1)"
+    if [[ "$expression_with_comment" == "YNH_NULL" ]]; then
+        set -o xtrace # set -x
+        return 1
+    fi
+    local value_line_number="$(tail +$after_line_number ${file} | grep -m1 -n -i -P $var_part'\K.*$' | cut -d: -f1)"
+    value_line_number=$((after_line_number + value_line_number))
+    local range="${after_line_number},${value_line_number} "
+
+    # Remove comments if needed
+    local expression="$(echo "$expression_with_comment" | sed "s@${comments}[^$string]*\$@@g" | sed "s@\s*[$endline]*\s*]*\$@@")"
+    endline=${expression_with_comment#"$expression"}
+    endline="$(echo "$endline" | sed 's/\\/\\\\/g')"
+    value="$(echo "$value" | sed 's/\\/\\\\/g')"
+    value=${value//&/"\&"}
+    local first_char="${expression:0:1}"
+    delimiter=$'\001'
+    if [[ "$first_char" == '"' ]]; then
+        # \ and sed is quite complex you need 2 \\ to get one in a sed
+        # So we need \\\\ to go through 2 sed
+        value="$(echo "$value" | sed 's/"/\\\\"/g')"
+        sed -ri "${range}s$delimiter"'(^'"${var_part}"'")([^"]|\\")*("[\s;,]*)(\s*'$comments'.*)?$'$delimiter'\1'"${value}"'"'"${endline}${delimiter}i" ${file}
+    elif [[ "$first_char" == "'" ]]; then
+        # \ and sed is quite complex you need 2 \\ to get one in a sed
+        # However double quotes implies to double \\ to
+        # So we need \\\\\\\\ to go through 2 sed and 1 double quotes str
+        value="$(echo "$value" | sed "s/'/\\\\\\\\'/g")"
+        sed -ri "${range}s$delimiter(^${var_part}')([^']|\\')*('"'[\s,;]*)(\s*'$comments'.*)?$'$delimiter'\1'"${value}'${endline}${delimiter}i" ${file}
+    else
+        if [[ "$value" == *"'"* ]] || [[ "$value" == *'"'* ]] || [[ "$ext" =~ ^php|py|json|js$ ]]; then
+            value='\"'"$(echo "$value" | sed 's/"/\\\\"/g')"'\"'
+        fi
+        if [[ "$ext" =~ ^yaml|yml$ ]]; then
+            value=" $value"
+        fi
+        sed -ri "${range}s$delimiter(^${var_part}).*\$$delimiter\1${value}${endline}${delimiter}i" ${file}
+    fi
+    set -o xtrace # set -x
+}
+
+# Render templates with Jinja2
+#
+# [internal]
+#
+# Attention : Variables should be exported before calling this helper to be
+# accessible inside templates.
+#
+# usage: ynh_render_template some_template output_path
+# | arg: some_template - Template file to be rendered
+# | arg: output_path   - The path where the output will be redirected to
+ynh_render_template() {
+    local template_path=$1
+    local output_path=$2
+    mkdir -p "$(dirname $output_path)"
+    # Taken from https://stackoverflow.com/a/35009576
+    python3 -c 'import os, sys, jinja2; sys.stdout.write(
+                    jinja2.Template(sys.stdin.read()
+                    ).render(os.environ));' < $template_path > $output_path
+}

helpers/helpers.v1.d/utils

@@ -0,0 +1,450 @@
+#!/bin/bash
+
+YNH_APP_BASEDIR=${YNH_APP_BASEDIR:-$(realpath ..)}
+
+# Handle script crashes / failures
+#
+# [internal]
+#
+# usage:
+# ynh_exit_properly is used only by the helper ynh_abort_if_errors.
+# You should not use it directly.
+# Instead, add to your script:
+# ynh_clean_setup () {
+#        instructions...
+# }
+#
+# This function provide a way to clean some residual of installation that not managed by remove script.
+#
+# It prints a warning to inform that the script was failed, and execute the ynh_clean_setup function if used in the app script
+#
+# Requires YunoHost version 2.6.4 or higher.
+ynh_exit_properly() {
+    local exit_code=$?
+
+    if [[ "${YNH_APP_ACTION:-}" =~ ^install$|^upgrade$|^restore$ ]]; then
+        rm -rf "/var/cache/yunohost/download/"
+    fi
+
+    if [ "$exit_code" -eq 0 ]; then
+        exit 0 # Exit without error if the script ended correctly
+    fi
+
+    trap '' EXIT # Ignore new exit signals
+    # Do not exit anymore if a command fail or if a variable is empty
+    set +o errexit # set +e
+    set +o nounset # set +u
+
+    # Small tempo to avoid the next message being mixed up with other DEBUG messages
+    sleep 0.5
+
+    if type -t ynh_clean_setup > /dev/null; then # Check if the function exist in the app script.
+        ynh_clean_setup                         # Call the function to do specific cleaning for the app.
+    fi
+
+    # Exit with error status
+    # We don't call ynh_die basically to avoid unecessary 10-ish
+    # debug lines about parsing args and stuff just to exit 1..
+    exit 1
+}
+
+# Exits if an error occurs during the execution of the script.
+#
+# [packagingv1]
+#
+# usage: ynh_abort_if_errors
+#
+# This configure the rest of the script execution such that, if an error occurs
+# or if an empty variable is used, the execution of the script stops immediately
+# and a call to `ynh_clean_setup` is triggered if it has been defined by your script.
+#
+# Requires YunoHost version 2.6.4 or higher.
+ynh_abort_if_errors() {
+    set -o errexit              # set -e; Exit if a command fail
+    set -o nounset              # set -u; And if a variable is used unset
+    trap ynh_exit_properly EXIT # Capturing exit signals on shell script
+}
+
+# When running an app script with packaging format >= 2, auto-enable ynh_abort_if_errors except for remove script
+if [[ "${YNH_CONTEXT:-}" != "regenconf" ]] && dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} ge 2 && [[ ${YNH_APP_ACTION} != "remove" ]]; then
+    ynh_abort_if_errors
+fi
+
+# Curl abstraction to help with POST requests to local pages (such as installation forms)
+#
+# usage: ynh_local_curl "page_uri" "key1=value1" "key2=value2" ...
+# | arg: page_uri    - Path (relative to `$path_url`) of the page where POST data will be sent
+# | arg: key1=value1 - (Optionnal) POST key and corresponding value
+# | arg: key2=value2 - (Optionnal) Another POST key and corresponding value
+# | arg: ...         - (Optionnal) More POST keys and values
+#
+# example: ynh_local_curl "/install.php?installButton" "foo=$var1" "bar=$var2"
+#
+# For multiple calls, cookies are persisted between each call for the same app
+#
+# `$domain` and `$path_url` should be defined externally (and correspond to the domain.tld and the /path (of the app?))
+#
+# Requires YunoHost version 2.6.4 or higher.
+ynh_local_curl() {
+    # Define url of page to curl
+    local local_page=$(ynh_normalize_url_path $1)
+    local full_path=$path_url$local_page
+
+    if [ "${path_url}" == "/" ]; then
+        full_path=$local_page
+    fi
+
+    local full_page_url=https://localhost$full_path
+
+    # Concatenate all other arguments with '&' to prepare POST data
+    local POST_data=""
+    local arg=""
+    for arg in "${@:2}"; do
+        POST_data="${POST_data}${arg}&"
+    done
+    if [ -n "$POST_data" ]; then
+        # Add --data arg and remove the last character, which is an unecessary '&'
+        POST_data="--data ${POST_data::-1}"
+    fi
+
+    # Wait untils nginx has fully reloaded (avoid curl fail with http2)
+    sleep 2
+
+    local cookiefile=/tmp/ynh-$app-cookie.txt
+    touch $cookiefile
+    chown root $cookiefile
+    chmod 700 $cookiefile
+
+    # Temporarily enable visitors if needed...
+    local visitors_enabled=$(ynh_permission_has_user "main" "visitors" && echo yes || echo no)
+    if [[ $visitors_enabled == "no" ]]; then
+        ynh_permission_update --permission "main" --add "visitors"
+    fi
+
+    # Curl the URL
+    curl --silent --show-error --insecure --location --header "Host: $domain" --resolve $domain:443:127.0.0.1 $POST_data "$full_page_url" --cookie-jar $cookiefile --cookie $cookiefile
+
+    if [[ $visitors_enabled == "no" ]]; then
+        ynh_permission_update --permission "main" --remove "visitors"
+    fi
+}
+
+# Fetch the Debian release codename
+#
+# [packagingv1]
+#
+# usage: ynh_get_debian_release
+# | ret: The Debian release codename (i.e. jessie, stretch, ...)
+#
+# Requires YunoHost version 2.7.12 or higher.
+ynh_get_debian_release() {
+    echo $(lsb_release --codename --short)
+}
+
+_acceptable_path_to_delete() {
+    local file=$1
+
+    local forbidden_paths=$(ls -d / /* /{var,home,usr}/* /etc/{default,sudoers.d,yunohost,cron*} /etc/yunohost/{apps,domains,hooks.d} /opt/yunohost 2> /dev/null)
+
+    # Legacy : A couple apps still have data in /home/$app ...
+    if [[ -n "${app:-}" ]]; then
+        forbidden_paths=$(echo "$forbidden_paths" | grep -v "/home/$app")
+    fi
+
+    # Use realpath to normalize the path ..
+    # i.e convert ///foo//bar//..///baz//// to /foo/baz
+    file=$(realpath --no-symlinks "$file")
+    if [ -z "$file" ] || grep -q -x -F "$file" <<< "$forbidden_paths"; then
+        return 1
+    else
+        return 0
+    fi
+}
+
+# Remove a file or a directory securely
+#
+# usage: ynh_secure_remove --file=path_to_remove
+# | arg: -f, --file=    - File or directory to remove
+#
+# Requires YunoHost version 2.6.4 or higher.
+ynh_secure_remove() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=f
+    local -A args_array=([f]=file=)
+    local file
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    set +o xtrace # set +x
+
+    if [ $# -ge 2 ]; then
+        ynh_print_warn --message="/!\ Packager ! You provided more than one argument to ynh_secure_remove but it will be ignored... Use this helper with one argument at time."
+    fi
+
+    if [[ -z "$file" ]]; then
+        ynh_print_warn --message="ynh_secure_remove called with empty argument, ignoring."
+    elif [[ ! -e $file ]]; then
+        ynh_print_info --message="'$file' wasn't deleted because it doesn't exist."
+    elif ! _acceptable_path_to_delete "$file"; then
+        ynh_print_warn --message="Not deleting '$file' because it is not an acceptable path to delete."
+    else
+        rm --recursive "$file"
+    fi
+
+    set -o xtrace # set -x
+}
+
+# Read the value of a key in a ynh manifest file
+#
+# usage: ynh_read_manifest --manifest="manifest.json" --manifest_key="key"
+# | arg: -m, --manifest=      - Path of the manifest to read
+# | arg: -k, --manifest_key=  - Name of the key to find
+# | ret: the value associate to that key
+#
+# Requires YunoHost version 3.5.0 or higher.
+ynh_read_manifest() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=mk
+    local -A args_array=([m]=manifest= [k]=manifest_key=)
+    local manifest
+    local manifest_key
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    if [ ! -e "${manifest:-}" ]; then
+        # If the manifest isn't found, try the common place for backup and restore script.
+        if [ -e "$YNH_APP_BASEDIR/manifest.json" ]; then
+            manifest="$YNH_APP_BASEDIR/manifest.json"
+        elif [ -e "$YNH_APP_BASEDIR/manifest.toml" ]; then
+            manifest="$YNH_APP_BASEDIR/manifest.toml"
+        else
+            ynh_die --message "No manifest found !?"
+        fi
+    fi
+
+    if echo "$manifest" | grep -q '\.json$'; then
+        jq ".$manifest_key" "$manifest" --raw-output
+    else
+        cat "$manifest" | python3 -c 'import json, toml, sys; print(json.dumps(toml.load(sys.stdin)))' | jq ".$manifest_key" --raw-output
+    fi
+}
+
+# Read the upstream version from the manifest or `$YNH_APP_MANIFEST_VERSION`
+#
+# usage: ynh_app_upstream_version [--manifest="manifest.json"]
+# | arg: -m, --manifest=    - Path of the manifest to read
+# | ret: the version number of the upstream app
+#
+# If the `manifest` is not specified, the envvar `$YNH_APP_MANIFEST_VERSION` will be used.
+#
+# The version number in the manifest is defined by `<upstreamversion>~ynh<packageversion>`.
+#
+# For example, if the manifest contains `4.3-2~ynh3` the function will return `4.3-2`
+#
+# Requires YunoHost version 3.5.0 or higher.
+ynh_app_upstream_version() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=m
+    local -A args_array=([m]=manifest=)
+    local manifest
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    manifest="${manifest:-}"
+
+    if [[ "$manifest" != "" ]] && [[ -e "$manifest" ]]; then
+        version_key_=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version")
+    else
+        version_key_=$YNH_APP_MANIFEST_VERSION
+    fi
+
+    echo "${version_key_/~ynh*/}"
+}
+
+# Read package version from the manifest
+#
+# [internal]
+#
+# usage: ynh_app_package_version [--manifest="manifest.json"]
+# | arg: -m, --manifest=    - Path of the manifest to read
+# | ret: the version number of the package
+#
+# The version number in the manifest is defined by `<upstreamversion>~ynh<packageversion>`.
+#
+# For example, if the manifest contains `4.3-2~ynh3` the function will return `3`
+#
+# Requires YunoHost version 3.5.0 or higher.
+ynh_app_package_version() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=m
+    local -A args_array=([m]=manifest=)
+    local manifest
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    version_key_=$YNH_APP_MANIFEST_VERSION
+    echo "${version_key_/*~ynh/}"
+}
+
+# Checks the app version to upgrade with the existing app version and returns:
+#
+# usage: ynh_check_app_version_changed
+# | ret: `UPGRADE_APP` if the upstream version changed, `UPGRADE_PACKAGE` otherwise.
+#
+# This helper should be used to avoid an upgrade of an app, or the upstream part
+# of it, when it's not needed
+#
+# Requires YunoHost version 3.5.0 or higher.
+ynh_check_app_version_changed() {
+    local return_value=${YNH_APP_UPGRADE_TYPE}
+
+    if [ "$return_value" == "UPGRADE_SAME" ] || [ "$return_value" == "DOWNGRADE" ]; then
+        return_value="UPGRADE_APP"
+    fi
+
+    echo $return_value
+}
+
+# Compare the current package version against another version given as an argument.
+#
+# usage: ynh_compare_current_package_version --comparison (lt|le|eq|ne|ge|gt) --version <X~ynhY>
+# | arg: --comparison - Comparison type. Could be : `lt` (lower than), `le` (lower or equal), `eq` (equal), `ne` (not equal), `ge` (greater or equal), `gt` (greater than)
+# | arg: --version - The version to compare. Need to be a version in the yunohost package version type (like `2.3.1~ynh4`)
+# | ret: 0 if the evaluation is true, 1 if false.
+#
+# example: ynh_compare_current_package_version --comparison lt --version 2.3.2~ynh1
+#
+# This helper is usually used when we need to do some actions only for some old package versions.
+#
+# Generally you might probably use it as follow in the upgrade script :
+# ```
+# if ynh_compare_current_package_version --comparison lt --version 2.3.2~ynh1
+# then
+#     # Do something that is needed for the package version older than 2.3.2~ynh1
+# fi
+# ```
+#
+# Requires YunoHost version 3.8.0 or higher.
+ynh_compare_current_package_version() {
+    local legacy_args=cv
+    declare -Ar args_array=([c]=comparison= [v]=version=)
+    local version
+    local comparison
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    local current_version=$YNH_APP_CURRENT_VERSION
+
+    # Check the syntax of the versions
+    if [[ ! $version =~ '~ynh' ]] || [[ ! $current_version =~ '~ynh' ]]; then
+        ynh_die --message="Invalid argument for version."
+    fi
+
+    # Check validity of the comparator
+    if [[ ! $comparison =~ (lt|le|eq|ne|ge|gt) ]]; then
+        ynh_die --message="Invalid comparator must be : lt, le, eq, ne, ge, gt"
+    fi
+
+    # Return the return value of dpkg --compare-versions
+    dpkg --compare-versions $current_version $comparison $version
+}
+
+# Check if we should enforce sane default permissions (= disable rwx for 'others')
+# on file/folders handled with ynh_setup_source and ynh_add_config
+#
+# [internal]
+#
+# Having a file others-readable or a folder others-executable(=enterable)
+# is a security risk comparable to "chmod 777"
+#
+# Configuration files may contain secrets. Or even just being able to enter a
+# folder may allow an attacker to do nasty stuff (maybe a file or subfolder has
+# some write permission enabled for 'other' and the attacker may edit the
+# content or create files as leverage for priviledge escalation ...)
+#
+# The sane default should be to set ownership to $app:$app.
+# In specific case, you may want to set the ownership to $app:www-data
+# for example if nginx needs access to static files.
+#
+_ynh_apply_default_permissions() {
+    local target=$1
+
+    local ynh_requirement=$(ynh_read_manifest --manifest_key="requirements.yunohost" | tr -d '<>= ')
+
+    if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} ge 2 || [ -z "$ynh_requirement" ] || [ "$ynh_requirement" == "null" ] || dpkg --compare-versions $ynh_requirement ge 4.2; then
+        chmod o-rwx $target
+        chmod g-w $target
+        chown -R root:root $target
+        if ynh_system_user_exists $app; then
+            chown $app:$app $target
+        fi
+    fi
+
+    # Crons should be owned by root
+    # Also we don't want systemd conf, nginx conf or others stuff to be owned by the app,
+    # otherwise they could self-edit their own systemd conf and escalate privilege
+    if grep -qE '^(/etc/cron|/etc/php|/etc/nginx/conf.d|/etc/fail2ban|/etc/systemd/system)' <<< "$target"; then
+        chmod 400 $target
+        chown root:root $target
+    fi
+}
+
+int_to_bool() {
+    sed -e 's/^1$/True/g' -e 's/^0$/False/g'
+}
+
+toml_to_json() {
+    python3 -c 'import toml, json, sys; print(json.dumps(toml.load(sys.stdin)))'
+}
+
+# Check if a YunoHost user exists
+#
+# usage: ynh_user_exists --username=username
+# | arg: -u, --username=    - the username to check
+# | ret: 0 if the user exists, 1 otherwise.
+#
+# example: ynh_user_exists 'toto' || echo "User does not exist"
+#
+# Requires YunoHost version 2.2.4 or higher.
+ynh_user_exists() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=u
+    local -A args_array=([u]=username=)
+    local username
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    yunohost user list --output-as json --quiet | jq -e ".users.\"${username}\"" > /dev/null
+}
+
+# Retrieve a YunoHost user information
+#
+# usage: ynh_user_get_info --username=username --key=key
+# | arg: -u, --username=    - the username to retrieve info from
+# | arg: -k, --key=         - the key to retrieve
+# | ret: the value associate to that key
+#
+# example: mail=$(ynh_user_get_info --username="toto" --key=mail)
+#
+# Requires YunoHost version 2.2.4 or higher.
+ynh_user_get_info() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=uk
+    local -A args_array=([u]=username= [k]=key=)
+    local username
+    local key
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    yunohost user info "$username" --output-as json --quiet | jq -r ".$key"
+}
+
+# Get the list of YunoHost users
+#
+# usage: ynh_user_list
+# | ret: one username per line as strings
+#
+# example: for u in $(ynh_user_list); do ... ; done
+#
+# Requires YunoHost version 2.4.0 or higher.
+ynh_user_list() {
+    yunohost user list --output-as json --quiet | jq -r ".users | keys[]"
+}

helpers/helpers.v1.d/vendor

@@ -0,0 +1 @@
+../vendor

helpers/helpers.v2.1.d/apt

@@ -0,0 +1,330 @@
+#!/bin/bash
+
+YNH_APT_INSTALL_DEPENDENCIES_REPLACE="true"
+
+# Define and install dependencies with a equivs control file
+#
+# example : ynh_apt_install_dependencies dep1 dep2 "dep3|dep4|dep5"
+#
+# usage: ynh_apt_install_dependencies dep [dep [...]]
+# | arg: dep - the package name to install in dependence.
+# | arg: "dep1|dep2|…" - You can specify alternatives. It will require to install (dep1 or dep2, etc).
+#
+ynh_apt_install_dependencies() {
+
+    # Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See below)
+    local dependencies="$(sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g' <<< "$@" | sed 's/|/ | /')"
+    local version=$(ynh_read_manifest "version")
+    local app_ynh_deps="${app//_/-}-ynh-deps" # Replace all '_' by '-', and append -ynh-deps
+
+    # Handle specific versions
+    if grep '[<=>]' <<< "$dependencies"; then
+        # Replace version specifications by relationships syntax
+        # https://www.debian.org/doc/debian-policy/ch-relationships.html
+        # Sed clarification
+        # [^(\<=\>] ignore if it begins by ( or < = >. To not apply twice.
+        # [\<=\>] matches < = or >
+        # \+ matches one or more occurence of the previous characters, for >= or >>.
+        # [^,]\+ matches all characters except ','
+        # Ex: 'package>=1.0' will be replaced by 'package (>= 1.0)'
+        dependencies="$(sed 's/\([^(\<=\>]\)\([\<=\>]\+\)\([^,]\+\)/\1 (\2 \3)/g' <<< "$dependencies")"
+    fi
+
+    # ############################## #
+    # Specific tweaks related to PHP #
+    # ############################## #
+
+    # Check for specific php dependencies which requires sury
+    # This grep will for example return "7.4" if dependencies is "foo bar php7.4-pwet php-gni"
+    # The (?<=php) syntax corresponds to lookbehind ;)
+    local specific_php_version=$(grep -oP '(?<=php)[0-9.]+(?=-|\>|)' <<< "$dependencies" | sort -u)
+
+    if [[ -n "$specific_php_version" ]]; then
+        # Cover a small edge case where a packager could have specified "php7.4-pwet php5-gni" which is confusing
+        [[ $(echo $specific_php_version | wc -l) -eq 1 ]] \
+            || ynh_die "Inconsistent php versions in dependencies ... found : $specific_php_version"
+
+        dependencies+=", php${specific_php_version}, php${specific_php_version}-fpm, php${specific_php_version}-common"
+
+        local old_php_version=$(ynh_app_setting_get --key=php_version)
+
+        # If the PHP version changed, remove the old fpm conf
+        if [ -n "$old_php_version" ] && [ "$old_php_version" != "$specific_php_version" ]; then
+            if [[ -f "/etc/php/$php_version/fpm/pool.d/$app.conf" ]]; then
+                ynh_backup_if_checksum_is_different "/etc/php/$php_version/fpm/pool.d/$app.conf"
+                ynh_config_remove_phpfpm
+            fi
+        fi
+        # Store php_version into the config of this app
+        ynh_app_setting_set --key=php_version --value=$specific_php_version
+
+        # Set the default php version back as the default version for php-cli.
+        if test -e /usr/bin/php$YNH_DEFAULT_PHP_VERSION; then
+            update-alternatives --set php /usr/bin/php$YNH_DEFAULT_PHP_VERSION
+        fi
+    elif grep --quiet 'php' <<< "$dependencies"; then
+        ynh_app_setting_set --key=php_version --value=$YNH_DEFAULT_PHP_VERSION
+    fi
+
+    # Specific tweak related to Postgresql (cf end of the helper)
+    local psql_installed="$(_ynh_apt_package_is_installed "postgresql-$PSQL_VERSION" && echo yes || echo no)"
+
+    # The first time we run ynh_apt_install_dependencies, we will replace the
+    # entire control file (This is in particular meant to cover the case of
+    # upgrade script where ynh_apt_install_dependencies is called with this
+    # expected effect) Otherwise, any subsequent call will add dependencies
+    # to those already present in the equivs control file.
+    if [[ $YNH_APT_INSTALL_DEPENDENCIES_REPLACE == "true" ]]; then
+        YNH_APT_INSTALL_DEPENDENCIES_REPLACE="false"
+    else
+        local current_dependencies=""
+        if _ynh_apt_package_is_installed "${app_ynh_deps}"; then
+            current_dependencies="$(dpkg-query --show --showformat='${Depends}' ${app_ynh_deps}) "
+            current_dependencies=${current_dependencies// | /|}
+        fi
+        dependencies="$current_dependencies, $dependencies"
+    fi
+
+    # ################
+    # Actual install #
+    # ################
+
+    # Prepare the virtual-dependency control file for dpkg-deb --build
+    local TMPDIR=$(mktemp --directory)
+    mkdir -p ${TMPDIR}/${app_ynh_deps}/DEBIAN
+    # For some reason, dpkg-deb insists for folder perm to be 755 and sometimes it's 777 o_O?
+    chmod -R 755 ${TMPDIR}/${app_ynh_deps}
+
+    cat > ${TMPDIR}/${app_ynh_deps}/DEBIAN/control << EOF
+Section: misc
+Priority: optional
+Package: ${app_ynh_deps}
+Version: ${version}
+Depends: ${dependencies//,,/,}
+Architecture: all
+Maintainer: root@localhost
+Description: Fake package for ${app} (YunoHost app) dependencies
+ This meta-package is only responsible of installing its dependencies.
+EOF
+
+    _ynh_apt update
+
+    _ynh_wait_dpkg_free
+
+    # Install the fake package without its dependencies with dpkg --force-depends
+    if ! LC_ALL=C dpkg-deb --build "${TMPDIR}/${app_ynh_deps}" "${TMPDIR}/${app_ynh_deps}.deb" > "${TMPDIR}/dpkg_log" 2>&1; then
+        cat "${TMPDIR}/dpkg_log" >&2
+        ynh_die --message="Unable to install dependencies"
+    fi
+    # Don't crash in case of error, because is nicely covered by the following line
+    LC_ALL=C dpkg --force-depends --install "${TMPDIR}/${app_ynh_deps}.deb" 2>&1 | tee "${TMPDIR}/dpkg_log" || true
+
+    # Then install the missing dependencies with apt install
+    _ynh_apt_install --fix-broken || {
+        # If the installation failed
+        # (the following is ran inside { } to not start a subshell otherwise ynh_die wouldnt exit the original process)
+        # Parse the list of problematic dependencies from dpkg's log ...
+        # (relevant lines look like: "foo-ynh-deps depends on bar; however:")
+        cat $TMPDIR/dpkg_log
+        local problematic_dependencies="$(grep -oP '(?<=-ynh-deps depends on ).*(?=; however)' $TMPDIR/dpkg_log | tr '\n' ' ')"
+        # Fake an install of those dependencies to see the errors
+        # The sed command here is, Print only from 'Reading state info' to the end.
+        [[ -n "$problematic_dependencies" ]] && _ynh_apt_install $problematic_dependencies --dry-run 2>&1 | sed --quiet '/Reading state info/,$p' | grep -v "fix-broken\|Reading state info" >&2
+        ynh_die "Unable to install apt dependencies"
+    }
+    rm --recursive --force "$TMPDIR" # Remove the temp dir.
+
+    # check if the package is actually installed
+    _ynh_apt_package_is_installed "${app_ynh_deps}" || ynh_die "Unable to install apt dependencies"
+
+    # Specific tweak related to Postgresql
+    # -> trigger postgresql regenconf if we may have just installed postgresql
+    local psql_installed2="$(_ynh_apt_package_is_installed "postgresql-$PSQL_VERSION" && echo yes || echo no)"
+    if [[ "$psql_installed" != "$psql_installed2" ]]; then
+        yunohost tools regen-conf postgresql
+    fi
+
+}
+
+# Remove fake package and its dependencies
+#
+# Dependencies will removed only if no other package need them.
+#
+# usage: ynh_apt_remove_dependencies
+ynh_apt_remove_dependencies() {
+    local app_ynh_deps="${app//_/-}-ynh-deps" # Replace all '_' by '-', and append -ynh-deps
+
+    local current_dependencies=""
+    if _ynh_apt_package_is_installed "${app_ynh_deps}"; then
+        current_dependencies="$(dpkg-query --show --showformat='${Depends}' ${app_ynh_deps}) "
+        current_dependencies=${current_dependencies// | /|}
+    fi
+
+    # Edge case where the app dep may be on hold,
+    # cf https://forum.yunohost.org/t/migration-error-cause-of-ffsync/20675/4
+    if apt-mark showhold | grep -q -w ${app_ynh_deps}; then
+        apt-mark unhold ${app_ynh_deps}
+    fi
+
+    # Remove the fake package and its dependencies if they not still used.
+    # (except if dpkg doesn't know anything about the package,
+    # which should be symptomatic of a failed install, and we don't want bash to report an error)
+    if dpkg-query --show ${app_ynh_deps} &> /dev/null; then
+        _ynh_apt autoremove --purge ${app_ynh_deps}
+    fi
+}
+
+# Install packages from an extra repository properly.
+#
+# usage: ynh_apt_install_dependencies_from_extra_repository --repo="repo" --package="dep1 dep2" --key=key_url
+# | arg: --repo=    - Complete url of the extra repository.
+# | arg: --package= - The packages to install from this extra repository
+# | arg: --key=     - url to get the public key.
+#
+ynh_apt_install_dependencies_from_extra_repository() {
+    # ============ Argument parsing =============
+    local -A args_array=([r]=repo= [p]=package= [k]=key=)
+    local repo
+    local package
+    local key
+    ynh_handle_getopts_args "$@"
+    # ===========================================
+
+    # Split the repository into uri, suite and components.
+    IFS=', ' read -r -a repo_parts <<< "$repo"
+    index=0
+
+    # Remove "deb " at the beginning of the repo.
+    if [[ "${repo_parts[0]}" == "deb" ]]; then
+        index=1
+    fi
+    uri="${repo_parts[$index]}"
+    index=$((index + 1))
+    suite="${repo_parts[$index]}"
+    index=$((index + 1))
+
+    # Get the components
+    if (("${#repo_parts[@]}" > 0)); then
+        component="${repo_parts[*]:$index}"
+    fi
+
+    if [[ "$key" == "trusted=yes" ]]; then
+        trust="[trusted=yes]"
+    else
+        trust=""
+    fi
+
+    # Add the new repo in sources.list.d
+    mkdir --parents "/etc/apt/sources.list.d"
+    echo "deb $trust $uri $suite $component" > "/etc/apt/sources.list.d/$app.list"
+
+    # Pin the new repo with the default priority, so it won't be used for upgrades.
+    # Build $pin from the uri without http and any sub path
+    local pin="${uri#*://}"
+    pin="${pin%%/*}"
+
+    # Pin repository
+    mkdir --parents "/etc/apt/preferences.d"
+    cat << EOF > "/etc/apt/preferences.d/$app"
+Package: *
+Pin: origin $pin
+Pin-Priority: 995
+EOF
+
+    if [ -n "$key" ] && [[ "$key" != "trusted=yes" ]]; then
+        mkdir --parents "/etc/apt/trusted.gpg.d"
+        # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget)
+        wget --timeout 900 --quiet "$key" --output-document=- | gpg --dearmor > /etc/apt/trusted.gpg.d/$app.gpg
+    fi
+
+    # Update the list of package with the new repo NB: we use -o
+    # Dir::Etc::sourcelist to only refresh this repo, because
+    # ynh_apt_install_dependencies will also call an ynh_apt update on its own
+    # and it's good to limit unecessary requests ...  Here we mainly want to
+    # validate that the url+key is correct before going further
+    _ynh_apt update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/$app.list"
+
+    # Install requested dependencies from this extra repository.
+    # NB: because of the mechanism with $ynh_apt_install_DEPENDENCIES_REPLACE,
+    # this will usually only *append* to the existing list of dependency, not
+    # replace the existing $app-ynh-deps
+    ynh_apt_install_dependencies "$package"
+
+    # Force to upgrade to the last version...
+    # Without doing apt install, an already installed dep is not upgraded
+    local apps_auto_installed="$(apt-mark showauto $package)"
+    _ynh_apt_install "$package"
+    [ -z "$apps_auto_installed" ] || apt-mark auto $apps_auto_installed
+
+    # Remove this extra repository after packages are installed
+    ynh_safe_rm "/etc/apt/sources.list.d/$app.list"
+    ynh_safe_rm "/etc/apt/preferences.d/$app"
+    ynh_safe_rm "/etc/apt/trusted.gpg.d/$app.gpg"
+    _ynh_apt update
+}
+
+# #####################
+# Internal misc utils #
+# #####################
+
+# Check if apt is free to use, or wait, until timeout.
+_ynh_wait_dpkg_free() {
+    local try
+    set +o xtrace # set +x
+    # With seq 1 17, timeout will be almost 30 minutes
+    for try in $(seq 1 17); do
+        # Check if /var/lib/dpkg/lock is used by another process
+        if lsof /var/lib/dpkg/lock > /dev/null; then
+            echo "apt is already in use..."
+            # Sleep an exponential time at each round
+            sleep $((try * try))
+        else
+            # Check if dpkg hasn't been interrupted and is fully available.
+            # See this for more information: https://sources.debian.org/src/apt/1.4.9/apt-pkg/deb/debsystem.cc/#L141-L174
+            local dpkg_dir="/var/lib/dpkg/updates/"
+
+            # For each file in $dpkg_dir
+            while read dpkg_file <&9; do
+                # Check if the name of this file contains only numbers.
+                if echo "$dpkg_file" | grep --perl-regexp --quiet "^[[:digit:]]+$"; then
+                    # If so, that a remaining of dpkg.
+                    ynh_print_warn "dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem."
+                    set -o xtrace # set -x
+                    return 1
+                fi
+            done 9<<< "$(ls -1 $dpkg_dir)"
+            set -o xtrace # set -x
+            return 0
+        fi
+    done
+    echo "apt still used, but timeout reached !"
+    set -o xtrace # set -x
+}
+
+# Check either a package is installed or not
+_ynh_apt_package_is_installed() {
+    local package=$1
+    dpkg-query --show --showformat='${db:Status-Status}' "$package" 2> /dev/null \
+        | grep --quiet "^installed$" &> /dev/null
+}
+
+# Return the installed version of an apt package, if installed
+_ynh_apt_package_version() {
+    if _ynh_apt_package_is_installed "$package"; then
+        dpkg-query --show --showformat='${Version}' "$package" 2> /dev/null
+    else
+        echo ''
+    fi
+}
+
+# APT wrapper for non-interactive operation
+_ynh_apt() {
+    _ynh_wait_dpkg_free
+    LC_ALL=C DEBIAN_FRONTEND=noninteractive apt-get --assume-yes --quiet -o=Acquire::Retries=3 -o=Dpkg::Use-Pty=0 $@
+}
+
+# Wrapper around "apt install" with the appropriate options
+_ynh_apt_install() {
+    _ynh_apt --no-remove --option Dpkg::Options::=--force-confdef \
+        --option Dpkg::Options::=--force-confold install $@
+}

helpers/helpers.v2.1.d/backup

@@ -0,0 +1,272 @@
+#!/bin/bash
+
+CAN_BIND=${CAN_BIND:-1}
+
+# Add a file or a directory to the list of paths to backup
+#
+# usage: ynh_backup /path/to/stuff
+#
+# NB : note that this helper does *NOT* perform any copy in itself, it only
+# declares stuff to be backuped via a CSV which is later picked up by the core
+#
+# NB 2 : there is a specific behavior for $data_dir (or childs of $data_dir) and
+# /var/log/$app which are *NOT* backedup during safety-backup-before-upgrade,
+# OR if the setting "do_not_backup_data" is equals 1 for that app
+#
+# The rationale is that these directories are usually too heavy to be integrated in every backup
+# (think for example about Nextcloud with quite a lot of data, or an app with a lot of media files...)
+#
+# This is coupled to the fact that $data_dir and the log dir won't be (and
+# should NOT) be deleted during remove, unless --purge is used. Hence, if the
+# upgrade fails and the script is removed prior to restoring the backup, the
+# data/logs are not destroyed.
+#
+ynh_backup() {
+
+    local target="$1"
+    local is_data=false
+
+    # If the path starts with /var/log/$app or $data_dir
+    if ([[ -n "${app:-}" ]] && [[ "$target" == "/var/log/$app*" ]]) || ([[ -n "${data_dir:-}" ]] && [[ "$target" == "$data_dir*" ]]); then
+        is_data=true
+    fi
+
+    if [[ -n "${app:-}" ]]; then
+        local do_not_backup_data=$(ynh_app_setting_get --key=do_not_backup_data)
+    fi
+
+    # If backing up core only (used by ynh_backup_before_upgrade),
+    # don't backup big data items
+    if [[ "$is_data" == true ]] && ([[ ${do_not_backup_data:-0} -eq 1 ]] || [[ ${BACKUP_CORE_ONLY:-0} -eq 1 ]]); then
+        if [ $BACKUP_CORE_ONLY -eq 1 ]; then
+            ynh_print_info "$target will not be saved, because 'BACKUP_CORE_ONLY' is set."
+        else
+            ynh_print_info "$target will not be saved, because 'do_not_backup_data' is set."
+        fi
+        return 1
+    fi
+
+    # ==============================================================================
+    # Format correctly source and destination paths
+    # ==============================================================================
+    # Be sure the source path is not empty
+    if [ ! -e "$target" ]; then
+        ynh_print_warn "File or folder '${target}' to be backed up does not exist"
+        return 1
+    fi
+
+    # Transform the source path as an absolute path
+    # If it's a dir remove the ending /
+    src_path=$(realpath "$target")
+
+    # Initialize the dest path with the source path relative to "/".
+    # eg: src_path=/etc/yunohost -> dest_path=etc/yunohost
+    dest_path="${src_path#/}"
+
+    # Check if dest_path already exists in tmp archive
+    if [[ -e "${dest_path}" ]]; then
+        ynh_print_warn "Destination path '${dest_path}' already exist"
+        return 1
+    fi
+
+    # Add the relative current working directory to the destination path
+    local rel_dir="${YNH_CWD#$YNH_BACKUP_DIR}"
+    rel_dir="${rel_dir%/}/"
+    dest_path="${rel_dir}${dest_path}"
+    dest_path="${dest_path#/}"
+    # ==============================================================================
+
+    # ==============================================================================
+    # Write file to backup into backup_list
+    # ==============================================================================
+    local src=$(echo "${src_path}" | sed --regexp-extended 's/"/\"\"/g')
+    local dest=$(echo "${dest_path}" | sed --regexp-extended 's/"/\"\"/g')
+    echo "\"${src}\",\"${dest}\"" >> "${YNH_BACKUP_CSV}"
+
+    # ==============================================================================
+
+    # Create the parent dir of the destination path
+    # It's for retro compatibility, some script consider ynh_backup creates this dir
+    mkdir --parents $(dirname "$YNH_BACKUP_DIR/${dest_path}")
+}
+
+# Return the path in the archive where has been stocked the origin path
+#
+# [internal]
+#
+# usage: _get_archive_path ORIGIN_PATH
+_get_archive_path() {
+    # For security reasons we use csv python library to read the CSV
+    python3 -c "
+import sys
+import csv
+with open(sys.argv[1], 'r') as backup_file:
+    backup_csv = csv.DictReader(backup_file, fieldnames=['source', 'dest'])
+    for row in backup_csv:
+        if row['source']==sys.argv[2].strip('\"'):
+            print(row['dest'])
+            sys.exit(0)
+    raise Exception('Original path for %s not found' % sys.argv[2])
+    " "${YNH_BACKUP_CSV}" "$1"
+    return $?
+}
+
+# Restore a file or a directory from the backup archive
+#
+# usage: ynh_restore /path/to/stuff
+#
+# examples:
+#     ynh_restore "/etc/nginx/conf.d/$domain.d/$app.conf"
+#
+# If the file or dir to be restored already exists on the system and is lighter
+# than 500 Mo, it is backed up in `/var/cache/yunohost/appconfbackup/`.
+# Otherwise, the existing file or dir is removed.
+#
+# if `apps/$app/etc/nginx/conf.d/$domain.d/$app.conf` exists, restore it into
+# `/etc/nginx/conf.d/$domain.d/$app.conf`
+# otheriwse, search for a match in the csv (eg: conf/nginx.conf) and restore it into
+# `/etc/nginx/conf.d/$domain.d/$app.conf`
+ynh_restore() {
+    target="$1"
+
+    local archive_path="$YNH_CWD${target}"
+
+    # If the path starts with /var/log/$app or $data_dir
+    local is_data=false
+    if ([[ -n "${app:-}" ]] && [[ "$target" == "/var/log/$app*" ]]) || ([[ -n "${data_dir:-}" ]] && [[ "$target" == "$data_dir*" ]]); then
+        is_data=true
+    fi
+
+    # If archive_path doesn't exist, search for a corresponding path in CSV
+    if [ ! -d "$archive_path" ] && [ ! -f "$archive_path" ] && [ ! -L "$archive_path" ]; then
+        if [[ "$is_data" == true ]]; then
+            ynh_print_info "Skipping $target which doesn't exists in the archive, probably because restoring from a safety-backup-before-upgrade"
+            # Assume it's not a big deal, we may be restoring a safety-backup-before-upgrade which doesnt contain those
+            return 0
+        else
+            # (get_archive_path will raise an exception if no match found)
+            archive_path="$YNH_BACKUP_DIR/$(_get_archive_path \"$target\")"
+        fi
+    fi
+
+    # Move the old directory if it already exists
+    if [[ -e "${target}" ]]; then
+        # Check if the file/dir size is less than 500 Mo
+        if [[ $(du --summarize --bytes ${target} | cut --delimiter="/" --fields=1) -le "500000000" ]]; then
+            local backup_file="/var/cache/yunohost/appconfbackup/${target}.backup.$(date '+%Y%m%d.%H%M%S')"
+            mkdir --parents "$(dirname "$backup_file")"
+            mv "${target}" "$backup_file" # Move the current file or directory
+        else
+            ynh_safe_rm "${target}"
+        fi
+    fi
+
+    # Restore target into target
+    mkdir --parents $(dirname "$target")
+
+    # Do a copy if it's just a mounting point
+    if mountpoint --quiet $YNH_BACKUP_DIR; then
+        if [[ -d "${archive_path}" ]]; then
+            archive_path="${archive_path}/."
+            mkdir --parents "$target"
+        fi
+        cp --archive "$archive_path" "${target}"
+    # Do a move if YNH_BACKUP_DIR is already a copy
+    else
+        mv "$archive_path" "${target}"
+    fi
+
+    _ynh_apply_default_permissions "$target"
+}
+
+# Restore all files that were previously backuped in an app backup script
+#
+# usage: ynh_restore_everything
+ynh_restore_everything() {
+    # Deduce the relative path of $YNH_CWD
+    local REL_DIR="${YNH_CWD#$YNH_BACKUP_DIR/}"
+    REL_DIR="${REL_DIR%/}/"
+
+    # For each destination path begining by $REL_DIR
+    cat ${YNH_BACKUP_CSV} | tr --delete $'\r' | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR.*\"$" \
+        | while read line; do
+            local ARCHIVE_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\"\K.*(?=\",\"$REL_DIR.*\"$)")
+            ynh_restore "$ARCHIVE_PATH"
+        done
+}
+
+_ynh_file_checksum_exists() {
+    local file=$1
+    local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
+    [[ -n "$(ynh_app_setting_get --key=$checksum_setting_name)" ]]
+}
+
+# Calculate and store a file checksum into the app settings
+#
+# usage: ynh_store_file_checksum /path/to/file
+ynh_store_file_checksum() {
+    set +o xtrace # set +x
+    local file=$1
+    local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
+
+    ynh_app_setting_set --key=$checksum_setting_name --value=$(md5sum "$file" | cut --delimiter=' ' --fields=1)
+
+    if ynh_in_ci_tests; then
+        # Using a base64 is in fact more reversible than "replace / and space by _" ... So we can in fact obtain the original file path in an easy reliable way ...
+        local file_path_base64=$(echo "$file" | base64 -w0)
+        mkdir -p /var/cache/yunohost/appconfbackup/
+        cat $file > /var/cache/yunohost/appconfbackup/original_${file_path_base64}
+    fi
+
+    # If backup_file_checksum isn't empty, ynh_backup_if_checksum_is_different has made a backup
+    if [ -n "${backup_file_checksum-}" ]; then
+        # Print the diff between the previous file and the new one.
+        # diff return 1 if the files are different, so the || true
+        diff --report-identical-files --unified --color=always $backup_file_checksum $file >&2 || true
+    fi
+    # Unset the variable, so it wouldn't trig a ynh_store_file_checksum without a ynh_backup_if_checksum_is_different before it.
+    unset backup_file_checksum
+    set -o xtrace # set -x
+}
+
+# Verify the checksum and backup the file if it's different
+#
+# usage: ynh_backup_if_checksum_is_different /path/to/file
+#
+# This helper is primarily meant to allow to easily backup personalised/manually
+# modified config files.
+ynh_backup_if_checksum_is_different() {
+    set +o xtrace # set +x
+    local file=$1
+    local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
+    local checksum_value=$(ynh_app_setting_get --key=$checksum_setting_name)
+    # backup_file_checksum isn't declare as local, so it can be reuse by ynh_store_file_checksum
+    backup_file_checksum=""
+    if [ -n "$checksum_value" ]; then                                                     # Proceed only if a value was stored into the app settings
+        if [ -e $file ] && ! echo "$checksum_value $file" | md5sum --check --status; then # If the checksum is now different
+
+            backup_file_checksum="/var/cache/yunohost/appconfbackup/$file.backup.$(date '+%Y%m%d.%H%M%S')"
+            mkdir --parents "$(dirname "$backup_file_checksum")"
+            cp --archive "$file" "$backup_file_checksum" # Backup the current file
+            ynh_print_warn "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum"
+            echo "$backup_file_checksum" # Return the name of the backup file
+            if ynh_in_ci_tests; then
+                local file_path_base64=$(echo "$file" | base64 -w0)
+                if test -e /var/cache/yunohost/appconfbackup/original_${file_path_base64}; then
+                    ynh_print_warn "Diff with the original file:"
+                    diff --report-identical-files --unified --color=always /var/cache/yunohost/appconfbackup/original_${file_path_base64} $file >&2 || true
+                fi
+            fi
+        fi
+    fi
+    set -o xtrace # set -x
+}
+
+# Delete a file checksum from the app settings
+#
+# usage: ynh_delete_file_checksum /path/to/file
+ynh_delete_file_checksum() {
+    local file=$1
+    local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
+    ynh_app_setting_delete --key=$checksum_setting_name
+}

helpers/helpers.v2.1.d/composer

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Install and initialize Composer in the given directory
+#
+# The installed version is defined by `$composer_version` which should be defined
+# as global prior to calling this helper.
+#
+# Will use `$install_dir` as workdir unless `$composer_workdir` exists (but that shouldnt be necessary)
+#
+# usage: ynh_composer_install
+ynh_composer_install() {
+    local workdir="${composer_workdir:-$install_dir}"
+
+    [[ -n "${composer_version}" ]] || ynh_die "\$composer_version should be defined before calling ynh_composer_install. (In the past, this was called \$YNH_COMPOSER_VERSION)"
+
+    [[ ! -e "$workdir/composer.phar" ]] || ynh_safe_rm $workdir/composer.phar
+
+    local composer_url="https://getcomposer.org/download/$composer_version/composer.phar"
+
+    # NB. we have to declare the var as local first,
+    # otherwise 'local foo=$(false) || echo 'pwet'" does'nt work
+    # because local always return 0 ...
+    local out
+    # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget)
+    out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$workdir/composer.phar $composer_url 2>&1) \
+        || ynh_die "$out"
+}
+
+# Execute a command with Composer
+#
+# Will use `$install_dir` as workdir unless `$composer_workdir` exists (but that shouldnt be necessary)
+#
+# You may also define `composer_user=root` prior to call this helper if you
+# absolutely need composer to run as root, but this is discouraged...
+#
+# usage: ynh_composer_exec commands
+ynh_composer_exec() {
+    local workdir="${composer_workdir:-$install_dir}"
+
+    COMPOSER_HOME="$workdir/.composer" \
+        COMPOSER_MEMORY_LIMIT=-1 \
+        sudo -E -u "${composer_user:-$app}" \
+        php${php_version} "$workdir/composer.phar" $@ \
+        -d "$workdir" --no-interaction --no-ansi 2>&1
+}

helpers/helpers.v2.1.d/config

@@ -0,0 +1,309 @@
+#!/bin/bash
+
+_ynh_app_config_get_one() {
+    local short_setting="$1"
+    local type="$2"
+    local bind="$3"
+    local getter="get__${short_setting}"
+    # Get value from getter if exists
+    if type -t $getter 2> /dev/null | grep -q '^function$' 2> /dev/null; then
+        old[$short_setting]="$($getter)"
+        formats[${short_setting}]="yaml"
+
+    elif [[ "$bind" == *"("* ]] && type -t "get__${bind%%(*}" 2> /dev/null | grep -q '^function$' 2> /dev/null; then
+        old[$short_setting]="$("get__${bind%%(*}" $short_setting $type $bind)"
+        formats[${short_setting}]="yaml"
+
+    elif [[ "$bind" == "null" ]]; then
+        old[$short_setting]="YNH_NULL"
+
+    # Get value from app settings or from another file
+    elif [[ "$type" == "file" ]]; then
+        if [[ "$bind" == "settings" ]]; then
+            ynh_die "File '${short_setting}' can't be stored in settings"
+        fi
+        old[$short_setting]="$(ls "$bind" 2> /dev/null || echo YNH_NULL)"
+        file_hash[$short_setting]="true"
+
+    # Get multiline text from settings or from a full file
+    elif [[ "$type" == "text" ]]; then
+        if [[ "$bind" == "settings" ]]; then
+            old[$short_setting]="$(ynh_app_setting_get $app $short_setting)"
+        elif [[ "$bind" == *":"* ]]; then
+            ynh_die "For technical reasons, multiline text '${short_setting}' can't be stored automatically  in a variable file, you have to create custom getter/setter"
+        else
+            old[$short_setting]="$(cat "$bind" 2> /dev/null || echo YNH_NULL)"
+        fi
+
+    # Get value from a kind of key/value file
+    else
+        local bind_after=""
+        if [[ "$bind" == "settings" ]]; then
+            bind=":/etc/yunohost/apps/$app/settings.yml"
+        fi
+        local bind_key_="$(echo "$bind" | cut -d: -f1)"
+        bind_key_=${bind_key_:-$short_setting}
+        if [[ "$bind_key_" == *">"* ]]; then
+            bind_after="$(echo "${bind_key_}" | cut -d'>' -f1)"
+            bind_key_="$(echo "${bind_key_}" | cut -d'>' -f2)"
+        fi
+        local bind_file="$(echo "$bind" | cut -d: -f2)"
+        old[$short_setting]="$(ynh_read_var_in_file --file="${bind_file}" --key="${bind_key_}" --after="${bind_after}")"
+
+    fi
+}
+_ynh_app_config_apply_one() {
+    local short_setting="$1"
+    local setter="set__${short_setting}"
+    local bind="${binds[$short_setting]}"
+    local type="${types[$short_setting]}"
+    if [ "${changed[$short_setting]}" == "true" ]; then
+        # Apply setter if exists
+        if type -t $setter 2> /dev/null | grep -q '^function$' 2> /dev/null; then
+            $setter
+
+        elif [[ "$bind" == *"("* ]] && type -t "set__${bind%%(*}" 2> /dev/null | grep -q '^function$' 2> /dev/null; then
+            "set__${bind%%(*}" $short_setting $type $bind
+
+        elif [[ "$bind" == "null" ]]; then
+            return
+
+        # Save in a file
+        elif [[ "$type" == "file" ]]; then
+            if [[ "$bind" == "settings" ]]; then
+                ynh_die "File '${short_setting}' can't be stored in settings"
+            fi
+            local bind_file="$bind"
+            if [[ "${!short_setting}" == "" ]]; then
+                ynh_backup_if_checksum_is_different "$bind_file"
+                ynh_safe_rm "$bind_file"
+                ynh_delete_file_checksum "$bind_file"
+                ynh_print_info "File '$bind_file' removed"
+            else
+                ynh_backup_if_checksum_is_different "$bind_file"
+                if [[ "${!short_setting}" != "$bind_file" ]]; then
+                    cp "${!short_setting}" "$bind_file"
+                fi
+                if _ynh_file_checksum_exists "$bind_file"; then
+                    ynh_store_file_checksum "$bind_file"
+                fi
+                ynh_print_info "File '$bind_file' overwritten with ${!short_setting}"
+            fi
+
+        # Save value in app settings
+        elif [[ "$bind" == "settings" ]]; then
+            ynh_app_setting_set --key=$short_setting --value="${!short_setting}"
+            ynh_print_info "Configuration key '$short_setting' edited in app settings"
+
+        # Save multiline text in a file
+        elif [[ "$type" == "text" ]]; then
+            if [[ "$bind" == *":"* ]]; then
+                ynh_die "For technical reasons, multiline text '${short_setting}' can't be stored automatically  in a variable file, you have to create custom getter/setter"
+            fi
+            local bind_file="$bind"
+            ynh_backup_if_checksum_is_different "$bind_file"
+            echo "${!short_setting}" > "$bind_file"
+            if _ynh_file_checksum_exists "$bind_file"; then
+                ynh_store_file_checksum "$bind_file"
+            fi
+            ynh_print_info "File '$bind_file' overwritten with the content provided in question '${short_setting}'"
+
+        # Set value into a kind of key/value file
+        else
+            local bind_after=""
+            local bind_key_="$(echo "$bind" | cut -d: -f1)"
+            if [[ "$bind_key_" == *">"* ]]; then
+                bind_after="$(echo "${bind_key_}" | cut -d'>' -f1)"
+                bind_key_="$(echo "${bind_key_}" | cut -d'>' -f2)"
+            fi
+            bind_key_=${bind_key_:-$short_setting}
+            local bind_file="$(echo "$bind" | cut -d: -f2)"
+
+            ynh_backup_if_checksum_is_different "$bind_file"
+            ynh_write_var_in_file --file="${bind_file}" --key="${bind_key_}" --value="${!short_setting}" --after="${bind_after}"
+            if _ynh_file_checksum_exists "$bind_file"; then
+                ynh_store_file_checksum "$bind_file"
+            fi
+
+            # We stored the info in settings in order to be able to upgrade the app
+            ynh_app_setting_set --key=$short_setting --value="${!short_setting}"
+            ynh_print_info "Configuration key '$bind_key_' edited into $bind_file"
+
+        fi
+    fi
+}
+
+_ynh_app_config_get() {
+    for line in $YNH_APP_CONFIG_PANEL_OPTIONS_TYPES_AND_BINDS; do
+        # Split line into short_setting, type and bind
+        IFS='|' read short_setting type bind <<< "$line"
+        binds[${short_setting}]="$bind"
+        types[${short_setting}]="$type"
+        file_hash[${short_setting}]=""
+        formats[${short_setting}]=""
+        ynh_app_config_get_one $short_setting $type $bind
+    done
+}
+
+_ynh_app_config_apply() {
+    for short_setting in "${!old[@]}"; do
+        ynh_app_config_apply_one $short_setting
+    done
+}
+
+_ynh_app_config_show() {
+    for short_setting in "${!old[@]}"; do
+        if [[ "${old[$short_setting]}" != YNH_NULL ]]; then
+            if [[ "${formats[$short_setting]}" == "yaml" ]]; then
+                ynh_return "${short_setting}:"
+                ynh_return "$(echo "${old[$short_setting]}" | sed 's/^/  /g')"
+            else
+                ynh_return "${short_setting}: '$(echo "${old[$short_setting]}" | sed "s/'/''/g" | sed ':a;N;$!ba;s/\n/\n\n/g')'"
+            fi
+        fi
+    done
+}
+
+_ynh_app_config_validate() {
+    # Change detection
+    ynh_script_progression "Checking what changed in the new configuration..."
+    local nothing_changed=true
+    local changes_validated=true
+    for short_setting in "${!old[@]}"; do
+        changed[$short_setting]=false
+        if [ -z ${!short_setting+x} ]; then
+            # Assign the var with the old value in order to allows multiple
+            # args validation
+            declare -g "$short_setting"="${old[$short_setting]}"
+            continue
+        fi
+        if [ ! -z "${file_hash[${short_setting}]}" ]; then
+            file_hash[old__$short_setting]=""
+            file_hash[new__$short_setting]=""
+            if [ -f "${old[$short_setting]}" ]; then
+                file_hash[old__$short_setting]=$(sha256sum "${old[$short_setting]}" | cut -d' ' -f1)
+                if [ -z "${!short_setting}" ]; then
+                    changed[$short_setting]=true
+                    nothing_changed=false
+                fi
+            fi
+            if [ -f "${!short_setting}" ]; then
+                file_hash[new__$short_setting]=$(sha256sum "${!short_setting}" | cut -d' ' -f1)
+                if [[ "${file_hash[old__$short_setting]}" != "${file_hash[new__$short_setting]}" ]]; then
+                    changed[$short_setting]=true
+                    nothing_changed=false
+                fi
+            fi
+        else
+            if [[ "${!short_setting}" != "${old[$short_setting]}" ]]; then
+                changed[$short_setting]=true
+                nothing_changed=false
+            fi
+        fi
+    done
+    if [[ "$nothing_changed" == "true" ]]; then
+        ynh_print_info "Nothing has changed"
+        exit 0
+    fi
+
+    # Run validation if something is changed
+    ynh_script_progression "Validating the new configuration..."
+
+    for short_setting in "${!old[@]}"; do
+        [[ "${changed[$short_setting]}" == "false" ]] && continue
+        local result=""
+        if type -t validate__$short_setting | grep -q '^function$' 2> /dev/null; then
+            result="$(validate__$short_setting)"
+        elif [[ "$bind" == *"("* ]] && type -t "validate__${bind%%(*}" 2> /dev/null | grep -q '^function$' 2> /dev/null; then
+            "validate__${bind%%(*}" $short_setting
+        fi
+        if [ -n "$result" ]; then
+            #
+            # Return a yaml such as:
+            #
+            # validation_errors:
+            #   some_key: "An error message"
+            #   some_other_key: "Another error message"
+            #
+            # We use changes_validated to know if this is
+            # the first validation error
+            if [[ "$changes_validated" == true ]]; then
+                ynh_return "validation_errors:"
+            fi
+            ynh_return "  ${short_setting}: \"$result\""
+            changes_validated=false
+        fi
+    done
+
+    # If validation failed, exit the script right now (instead of going into apply)
+    # Yunohost core will pick up the errors returned via ynh_return previously
+    if [[ "$changes_validated" == "false" ]]; then
+        exit 0
+    fi
+
+}
+
+ynh_app_config_get_one() {
+    _ynh_app_config_get_one $1 $2 $3
+}
+
+ynh_app_config_get() {
+    _ynh_app_config_get
+}
+
+ynh_app_config_show() {
+    _ynh_app_config_show
+}
+
+ynh_app_config_validate() {
+    _ynh_app_config_validate
+}
+
+ynh_app_config_apply_one() {
+    _ynh_app_config_apply_one $1
+}
+ynh_app_config_apply() {
+    _ynh_app_config_apply
+}
+
+ynh_app_action_run() {
+    local runner="run__$1"
+    # Get value from getter if exists
+    if type -t "$runner" 2> /dev/null | grep -q '^function$' 2> /dev/null; then
+        $runner
+        #ynh_return "result:"
+        #ynh_return "$(echo "${result}" | sed 's/^/  /g')"
+    else
+        ynh_die "No handler defined in app's script for action $1. If you are the maintainer of this app, you should define '$runner'"
+    fi
+}
+
+ynh_app_config_run() {
+    declare -Ag old=()
+    declare -Ag changed=()
+    declare -Ag file_hash=()
+    declare -Ag binds=()
+    declare -Ag types=()
+    declare -Ag formats=()
+
+    case $1 in
+        show)
+            ynh_app_config_get
+            ynh_app_config_show
+            ;;
+        apply)
+            max_progression=4
+            ynh_script_progression "Reading config panel description and current configuration..."
+            ynh_app_config_get
+
+            ynh_app_config_validate
+
+            ynh_script_progression "Applying the new configuration..."
+            ynh_app_config_apply
+            ynh_script_progression "Configuration of $app completed"
+            ;;
+        *)
+            ynh_app_action_run $1
+            ;;
+    esac
+}

helpers/helpers.v2.1.d/fail2ban

@@ -0,0 +1,118 @@
+#!/bin/bash
+
+# Create a dedicated fail2ban config (jail and filter conf files)
+#
+# usage: ynh_config_add_fail2ban --logpath=log_file --failregex=filter
+# | arg: --logpath=   - Log file to be checked by fail2ban
+# | arg: --failregex= - Failregex to be looked for by fail2ban
+#
+# If --logpath / --failregex are provided, the helper will generate the appropriate conf using these.
+#
+# Otherwise, it will assume that the app provided templates, namely
+# `../conf/f2b_jail.conf` and `../conf/f2b_filter.conf`
+#
+# They will typically look like (for example here for synapse):
+# ```
+# f2b_jail.conf:
+#     [__APP__]
+#     enabled = true
+#     port = http,https
+#     filter = __APP__
+#     logpath = /var/log/__APP__/logfile.log
+#     maxretry = 5
+# ```
+# ```
+# f2b_filter.conf:
+#     [INCLUDES]
+#     before = common.conf
+#     [Definition]
+#
+#     # Part of regex definition (just used to make more easy to make the global regex)
+#     __synapse_start_line = .? \- synapse\..+ \-
+#
+#    # Regex definition.
+#    failregex = ^%(__synapse_start_line)s INFO \- POST\-(\d+)\- <HOST> \- \d+ \- Received request\: POST /_matrix/client/r0/login\??<SKIPLINES>%(__synapse_start_line)s INFO \- POST\-\1\- Got login request with identifier: \{u'type': u'm.id.user', u'user'\: u'(.+?)'\}, medium\: None, address: None, user\: u'\5'<SKIPLINES>%(__synapse_start_line)s WARNING \- \- (Attempted to login as @\5\:.+ but they do not exist|Failed password login for user @\5\:.+)$
+#
+#     ignoreregex =
+# ```
+#
+# ##### Regarding the the `failregex` option:
+#
+# regex to match the password failure messages in the logfile. The host must be
+# matched by a group named "`host`". The tag "`<HOST>`" can be used for standard
+# IP/hostname matching and is only an alias for `(?:::f{4,6}:)?(?P<host>[\w\-.^_]+)`
+#
+# You can find some more explainations about how to make a regex here :
+# https://www.fail2ban.org/wiki/index.php/MANUAL_0_8#Filters
+#
+# To validate your regex you can test with this command:
+# ```
+# fail2ban-regex /var/log/YOUR_LOG_FILE_PATH /etc/fail2ban/filter.d/YOUR_APP.conf
+# ```
+ynh_config_add_fail2ban() {
+    # ============ Argument parsing =============
+    local -A args_array=([l]=logpath= [r]=failregex=)
+    local logpath
+    local failregex
+    ynh_handle_getopts_args "$@"
+    # ===========================================
+
+    # If failregex is provided, Build a config file on-the-fly using $logpath and $failregex
+    if [[ -n "${failregex:-}" ]]; then
+        test -n "$logpath" || ynh_die "ynh_config_add_fail2ban expects a logfile path as first argument and received nothing."
+
+        echo "
+[__APP__]
+enabled = true
+port = http,https
+filter = __APP__
+logpath = __LOGPATH__
+maxretry = 5
+" > "$YNH_APP_BASEDIR/conf/f2b_jail.conf"
+
+        echo "
+[INCLUDES]
+before = common.conf
+[Definition]
+failregex = __FAILREGEX__
+ignoreregex =
+" > "$YNH_APP_BASEDIR/conf/f2b_filter.conf"
+    fi
+
+    ynh_config_add --template="f2b_jail.conf" --destination="/etc/fail2ban/jail.d/$app.conf"
+    ynh_config_add --template="f2b_filter.conf" --destination="/etc/fail2ban/filter.d/$app.conf"
+
+    # if "$logpath" doesn't exist (as if using --use_template argument), assign
+    # "$logpath" using the one in the previously generated fail2ban conf file
+    if [ -z "${logpath:-}" ]; then
+        # the first sed deletes possibles spaces and the second one extract the path
+        logpath=$(grep "^logpath" "/etc/fail2ban/jail.d/$app.conf" | sed "s/ //g" | sed "s/logpath=//g")
+    fi
+
+    # Create the folder and logfile if they doesn't exist,
+    # as fail2ban require an existing logfile before configuration
+    mkdir -p "/var/log/$app"
+    if [ ! -f "$logpath" ]; then
+        touch "$logpath"
+    fi
+    # Make sure log folder's permissions are correct
+    chown -R "$app:$app" "/var/log/$app"
+    chmod -R u=rwX,g=rX,o= "/var/log/$app"
+
+    ynh_systemctl --service=fail2ban --action=reload --wait_until="(Started|Reloaded) Fail2Ban Service" --log_path=systemd
+
+    local fail2ban_error="$(journalctl --no-hostname --unit=fail2ban | tail --lines=50 | grep "WARNING.*$app.*")"
+    if [[ -n "$fail2ban_error" ]]; then
+        ynh_print_warn "Fail2ban failed to load the jail for $app"
+        ynh_print_warn "${fail2ban_error#*WARNING}"
+    fi
+}
+
+# Remove the dedicated fail2ban config (jail and filter conf files)
+#
+# usage: ynh_config_remove_fail2ban
+ynh_config_remove_fail2ban() {
+    ynh_safe_rm "/etc/fail2ban/jail.d/$app.conf"
+    ynh_safe_rm "/etc/fail2ban/filter.d/$app.conf"
+    ynh_systemctl --service=fail2ban --action=reload
+}

helpers/helpers.v2.1.d/getopts

@@ -0,0 +1,186 @@
+#!/bin/bash
+
+# Internal helper design to allow helpers to use getopts to manage their arguments
+#
+# [internal]
+#
+# example: function my_helper()
+# {
+#     local -A args_array=( [a]=arg1= [b]=arg2= [c]=arg3 )
+#     local arg1
+#     local arg2
+#     local arg3
+#     ynh_handle_getopts_args "$@"
+#
+#     [...]
+# }
+# my_helper --arg1 "val1" -b val2 -c
+#
+# usage: ynh_handle_getopts_args "$@"
+# | arg: $@    - Simply "$@" to tranfert all the positionnal arguments to the function
+#
+# This helper need an array, named "args_array" with all the arguments used by the helper
+# that want to use ynh_handle_getopts_args
+# Be carreful, this array has to be an associative array, as the following example:
+# local -A args_array=( [a]=arg1 [b]=arg2= [c]=arg3 )
+# Let's explain this array:
+# a, b and c are short options, -a, -b and -c
+# arg1, arg2 and arg3 are the long options associated to the previous short ones. --arg1, --arg2 and --arg3
+# For each option, a short and long version has to be defined.
+# Let's see something more significant
+# local -A args_array=( [u]=user [f]=finalpath= [d]=database )
+#
+# NB: Because we're using 'declare' without -g, the array will be declared as a local variable.
+#
+# Please keep in mind that the long option will be used as a variable to store the values for this option.
+# For the previous example, that means that $finalpath will be fill with the value given as argument for this option.
+#
+# Also, in the previous example, finalpath has a '=' at the end. That means this option need a value.
+# So, the helper has to be call with --finalpath /final/path, --finalpath=/final/path or -f /final/path, the variable $finalpath will get the value /final/path
+# If there's many values for an option, -f /final /path, the value will be separated by a ';' $finalpath=/final;/path
+# For an option without value, like --user in the example, the helper can be called only with --user or -u. $user will then get the value 1.
+#
+ynh_handle_getopts_args() {
+    # Trick to only re-enable debugging if it was set before
+    local xtrace_enable=$(set +o | grep xtrace)
+
+    # Manage arguments only if there's some provided
+    set +o xtrace # set +x
+    if [ $# -eq 0 ]; then
+        eval "$xtrace_enable"
+        return
+    # Validate that the first char is - because it should be something like --option=value or -o ...
+    elif [[ "${1:0:1}" != "-" ]]; then
+        ynh_die "It looks like you called the helper using positional arguments instead of keyword arguments ?"
+    fi
+
+    # Store arguments in an array to keep each argument separated
+    local arguments=("$@")
+
+    # For each option in the array, reduce to short options for getopts (e.g. for [u]=user, --user will be -u)
+    # And built parameters string for getopts
+    # ${!args_array[@]} is the list of all option_flags in the array (An option_flag is 'u' in [u]=user, user is a value)
+    local getopts_parameters=""
+    local option_flag=""
+    for option_flag in "${!args_array[@]}"; do
+        # Concatenate each option_flags of the array to build the string of arguments for getopts
+        # Will looks like 'abcd' for -a -b -c -d
+        # If the value of an option_flag finish by =, it's an option with additionnal values. (e.g. --user bob or -u bob)
+        # Check the last character of the value associate to the option_flag
+        if [ "${args_array[$option_flag]: -1}" = "=" ]; then
+            # For an option with additionnal values, add a ':' after the letter for getopts.
+            getopts_parameters="${getopts_parameters}${option_flag}:"
+        else
+            getopts_parameters="${getopts_parameters}${option_flag}"
+        fi
+        # Check each argument given to the function
+        local arg=""
+        # ${#arguments[@]} is the size of the array
+        for arg in $(seq 0 $((${#arguments[@]} - 1))); do
+            # Escape options' values starting with -. Otherwise the - will be considered as another option.
+            arguments[arg]="${arguments[arg]//--${args_array[$option_flag]}-/--${args_array[$option_flag]}\\TOBEREMOVED\\-}"
+            # And replace long option (value of the option_flag) by the short option, the option_flag itself
+            # (e.g. for [u]=user, --user will be -u)
+            # Replace long option with = (match the beginning of the argument)
+            arguments[arg]="$(printf '%s\n' "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]}/-${option_flag} /")"
+            # And long option without = (match the whole line)
+            arguments[arg]="$(printf '%s\n' "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]%=}$/-${option_flag} /")"
+        done
+    done
+
+    # Read and parse all the arguments
+    # Use a function here, to use standart arguments $@ and be able to use shift.
+    parse_arg() {
+        # Read all arguments, until no arguments are left
+        while [ $# -ne 0 ]; do
+            # Initialize the index of getopts
+            OPTIND=1
+            # Parse with getopts only if the argument begin by -, that means the argument is an option
+            # getopts will fill $parameter with the letter of the option it has read.
+            local parameter=""
+            getopts ":$getopts_parameters" parameter || true
+
+            if [ "$parameter" = "?" ]; then
+                ynh_die "Invalid argument: ${1:-}"
+            elif [ "$parameter" = ":" ]; then
+                ynh_die "${1:-} parameter requires an argument."
+            else
+                local shift_value=1
+                # Use the long option, corresponding to the short option read by getopts, as a variable
+                # (e.g. for [u]=user, 'user' will be used as a variable)
+                # Also, remove '=' at the end of the long option
+                # The variable name will be stored in 'option_var'
+                local option_var="${args_array[$parameter]%=}"
+                # If this option doesn't take values
+                # if there's a '=' at the end of the long option name, this option takes values
+                if [ "${args_array[$parameter]: -1}" != "=" ]; then
+                    # 'eval ${option_var}' will use the content of 'option_var'
+                    eval ${option_var}=1
+                else
+                    # Read all other arguments to find multiple value for this option.
+                    # Load args in a array
+                    local all_args=("$@")
+
+                    # If the first argument is longer than 2 characters,
+                    # There's a value attached to the option, in the same array cell
+                    if [ ${#all_args[0]} -gt 2 ]; then
+                        # Remove the option and the space, so keep only the value itself.
+                        all_args[0]="${all_args[0]#-${parameter} }"
+
+                        # At this point, if all_args[0] start with "-", then the argument is not well formed
+                        if [ "${all_args[0]:0:1}" == "-" ]; then
+                            ynh_die "Argument \"${all_args[0]}\" not valid! Did you use a single \"-\" instead of two?"
+                        fi
+                        # Reduce the value of shift, because the option has been removed manually
+                        shift_value=$((shift_value - 1))
+                    fi
+
+                    # Declare the content of option_var as a variable.
+                    eval ${option_var}=""
+                    # Then read the array value per value
+                    local i
+                    for i in $(seq 0 $((${#all_args[@]} - 1))); do
+                        # If this argument is an option, end here.
+                        if [ "${all_args[$i]:0:1}" == "-" ]; then
+                            # Ignore the first value of the array, which is the option itself
+                            if [ "$i" -ne 0 ]; then
+                                break
+                            fi
+                        else
+                            # Ignore empty parameters
+                            if [ -n "${all_args[$i]}" ]; then
+                                # Else, add this value to this option
+                                # Each value will be separated by ';'
+                                if [ -n "${!option_var}" ]; then
+                                    # If there's already another value for this option, add a ; before adding the new value
+                                    eval ${option_var}+="\;"
+                                fi
+
+                                # Remove the \ that escape - at beginning of values.
+                                all_args[i]="${all_args[i]//\\TOBEREMOVED\\/}"
+
+                                # For the record.
+                                # We're using eval here to get the content of the variable stored itself as simple text in $option_var...
+                                # Other ways to get that content would be to use either ${!option_var} or declare -g ${option_var}
+                                # But... ${!option_var} can't be used as left part of an assignation.
+                                # declare -g ${option_var} will create a local variable (despite -g !) and will not be available for the helper itself.
+                                # So... Stop fucking arguing each time that eval is evil... Go find an other working solution if you can find one!
+
+                                eval ${option_var}+='"${all_args[$i]}"'
+                            fi
+                            shift_value=$((shift_value + 1))
+                        fi
+                    done
+                fi
+            fi
+
+            # Shift the parameter and its argument(s)
+            shift $shift_value
+        done
+    }
+
+    # Call parse_arg and pass the modified list of args as an array of arguments.
+    parse_arg "${arguments[@]}"
+
+    eval "$xtrace_enable"
+}

helpers/helpers.v2.1.d/go

@@ -0,0 +1,190 @@
+#!/bin/bash
+
+readonly GOENV_INSTALL_DIR="/opt/goenv"
+# goenv_ROOT is the directory of goenv, it needs to be loaded as a environment variable.
+export GOENV_ROOT="$GOENV_INSTALL_DIR"
+
+_ynh_load_go_in_path_and_other_tweaks() {
+
+    # Get the absolute path of this version of go
+    go_dir="$GOENV_INSTALL_DIR/versions/$app/bin"
+
+    # Load the path of this version of go in $PATH
+    if [[ :$PATH: != *":$go_dir"* ]]; then
+        PATH="$go_dir:$PATH"
+    fi
+
+    # Export PATH such that it's available through sudo -E / ynh_exec_as $app
+    export PATH
+
+    # This is in full lowercase such that it gets replaced in templates
+    path_with_go="$PATH"
+    PATH_with_go="$PATH"
+
+    # Sets the local application-specific go version
+    pushd ${install_dir}
+    $GOENV_INSTALL_DIR/bin/goenv local $go_version
+    popd
+}
+
+# Install a specific version of Go using goenv
+#
+# The installed version is defined by `$go_version` which should be defined as global prior to calling this helper
+#
+# usage: ynh_go_install
+#
+# The helper adds the appropriate, specific version of go to the `$PATH` variable (which
+# is preserved when calling `ynh_exec_as_app`). Also defines:
+# - `$path_with_go` (the value of the modified `$PATH`, but you dont really need it?)
+# - `$go_dir` (the directory containing the specific go version)
+#
+# This helper also creates a /etc/profile.d/goenv.sh that configures PATH environment for goenv
+ynh_go_install() {
+
+    [[ -n "${go_version:-}" ]] || ynh_die "\$go_version should be defined prior to calling ynh_go_install"
+
+    # Load goenv path in PATH
+    local CLEAR_PATH="$GOENV_INSTALL_DIR/bin:$PATH"
+
+    # Remove /usr/local/bin in PATH in case of Go prior installation
+    PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@')
+
+    # Move an existing Go binary, to avoid to block goenv
+    test -x /usr/bin/go && mv /usr/bin/go /usr/bin/go_goenv
+
+    # Install or update goenv
+    mkdir -p $GOENV_INSTALL_DIR
+    pushd "$GOENV_INSTALL_DIR"
+    if ! [ -x "$GOENV_INSTALL_DIR/bin/goenv" ]; then
+        ynh_print_info "Downloading goenv..."
+        git init -q
+        git remote add origin https://github.com/syndbg/goenv.git
+    else
+        ynh_print_info "Updating goenv..."
+    fi
+    git fetch -q --tags --prune origin
+    local git_latest_tag=$(git describe --tags "$(git rev-list --tags --max-count=1)")
+    git checkout -q "$git_latest_tag"
+    _ynh_go_try_bash_extension
+    goenv=$GOENV_INSTALL_DIR/bin/goenv
+    popd
+
+    # Install or update xxenv-latest
+    mkdir -p "$GOENV_INSTALL_DIR/plugins/xxenv-latest"
+    pushd "$GOENV_INSTALL_DIR/plugins/xxenv-latest"
+    if ! [ -x "$GOENV_INSTALL_DIR/plugins/xxenv-latest/bin/goenv-latest" ]; then
+        ynh_print_info "Downloading xxenv-latest..."
+        git init -q
+        git remote add origin https://github.com/momo-lab/xxenv-latest.git
+    else
+        ynh_print_info "Updating xxenv-latest..."
+    fi
+    git fetch -q --tags --prune origin
+    local git_latest_tag=$(git describe --tags "$(git rev-list --tags --max-count=1)")
+    git checkout -q "$git_latest_tag"
+    popd
+
+    # Enable caching
+    mkdir -p "${GOENV_INSTALL_DIR}/cache"
+
+    # Create shims directory if needed
+    mkdir -p "${GOENV_INSTALL_DIR}/shims"
+
+    # Restore /usr/local/bin in PATH
+    PATH=$CLEAR_PATH
+
+    # And replace the old Go binary
+    test -x /usr/bin/go_goenv && mv /usr/bin/go_goenv /usr/bin/go
+
+    # Install the requested version of Go
+    local final_go_version=$("$GOENV_INSTALL_DIR/plugins/xxenv-latest/bin/goenv-latest" --print "$go_version")
+    ynh_print_info "Installation of Go-$final_go_version"
+    goenv install --quiet --skip-existing "$final_go_version" 2>&1
+
+    # Store go_version into the config of this app
+    ynh_app_setting_set --app="$app" --key="go_version" --value="$final_go_version"
+    go_version=$final_go_version
+
+    # Cleanup Go versions
+    _ynh_go_cleanup
+
+    # Set environment for Go users
+    echo "#goenv
+export GOENV_ROOT=$GOENV_INSTALL_DIR
+export PATH=\"$GOENV_INSTALL_DIR/bin:$PATH\"
+eval \"\$(goenv init -)\"
+#goenv" > /etc/profile.d/goenv.sh
+
+    # Load the environment
+    eval "$(goenv init -)"
+
+    _ynh_load_go_in_path_and_other_tweaks
+}
+
+# Remove the version of Go used by the app.
+#
+# This helper will also cleanup Go versions
+#
+# usage: ynh_go_remove
+ynh_go_remove() {
+    local go_version=$(ynh_app_setting_get --key="go_version")
+
+    # Load goenv path in PATH
+    local CLEAR_PATH="$GOENV_INSTALL_DIR/bin:$PATH"
+
+    # Remove /usr/local/bin in PATH in case of Go prior installation
+    PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@')
+
+    # Remove the line for this app
+    ynh_app_setting_delete --key="go_version"
+
+    # Cleanup Go versions
+    _ynh_go_cleanup
+}
+
+# Remove no more needed versions of Go used by the app.
+#
+# [internal]
+#
+# This helper will check what Go version are no more required,
+# and uninstall them
+# If no app uses Go, goenv will be also removed.
+#
+# usage: _ynh_go_cleanup
+_ynh_go_cleanup() {
+
+    # List required Go versions
+    local installed_apps=$(yunohost app list --output-as json --quiet | jq -r .apps[].id)
+    local required_go_versions=""
+    for installed_app in $installed_apps; do
+        local installed_app_go_version=$(ynh_app_setting_get --app=$installed_app --key="go_version")
+        if [[ $installed_app_go_version ]]; then
+            required_go_versions="${installed_app_go_version}\n${required_go_versions}"
+        fi
+    done
+
+    # Remove no more needed Go versions
+    local installed_go_versions=$(goenv versions --bare --skip-aliases | grep -Ev '/')
+    for installed_go_version in $installed_go_versions; do
+        if ! $(echo ${required_go_versions} | grep "${installed_go_version}" 1> /dev/null 2>&1); then
+            ynh_print_info "Removing of Go-$installed_go_version"
+            $GOENV_INSTALL_DIR/bin/goenv uninstall --force "$installed_go_version"
+        fi
+    done
+
+    # If none Go version is required
+    if [[ ! $required_go_versions ]]; then
+        # Remove goenv environment configuration
+        ynh_print_info "Removing of goenv"
+        ynh_safe_rm "$GOENV_INSTALL_DIR"
+        ynh_safe_rm "/etc/profile.d/goenv.sh"
+    fi
+}
+
+_ynh_go_try_bash_extension() {
+    if [ -x src/configure ]; then
+        src/configure && make -C src || {
+            ynh_print_info "Optional bash extension failed to build, but things will still work normally."
+        }
+    fi
+}

helpers/helpers.v2.1.d/logging

@@ -0,0 +1,117 @@
+#!/bin/bash
+
+# Print a message to stderr and terminate the current script
+#
+# usage: ynh_die "Some message"
+ynh_die() {
+    set +o xtrace # set +x
+    if [[ -n "${1:-}" ]]; then
+        if [[ -n "${YNH_STDRETURN:-}" ]]; then
+            python3 -c 'import yaml, sys; print(yaml.dump({"error": sys.stdin.read()}))' <<< "${1:-}" >> "$YNH_STDRETURN"
+        fi
+        echo "${1:-}" 1>&2
+    fi
+    exit 1
+}
+
+# Print an "INFO" message
+#
+# usage: ynh_print_info "Some message"
+ynh_print_info() {
+    echo "$1" >&$YNH_STDINFO
+}
+
+# Print a warning on stderr
+#
+# usage: ynh_print_warn "Some message"
+ynh_print_warn() {
+    echo "$1" >&2
+}
+
+# Execute a command and redirect stderr to stdout
+#
+# usage: ynh_hide_warnings your command and args
+# | arg: command - command to execute
+#
+ynh_hide_warnings() {
+    # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077
+    "$@" 2>&1
+}
+
+# Execute a command and redirect stderr in /dev/null. Print stderr on error.
+#
+# usage: ynh_exec_and_print_stderr_only_if_error your command and args
+# | arg: command - command to execute
+#
+# Note that you should NOT quote the command but only prefix it with ynh_exec_and_print_stderr_only_if_error
+ynh_exec_and_print_stderr_only_if_error() {
+    logfile="$(mktemp)"
+    rc=0
+    # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077
+    "$@" 2> "$logfile" || rc="$?"
+    if ((rc != 0)); then
+        cat "$logfile" >&2
+        ynh_safe_rm "$logfile"
+        return "$rc"
+    fi
+}
+
+# Return data to the YunoHost core for later processing
+# (to be used by special hooks like app config panel and core diagnosis)
+#
+# usage: ynh_return somedata
+ynh_return() {
+    echo "$1" >> "$YNH_STDRETURN"
+}
+
+# Initial definitions for ynh_script_progression
+increment_progression=0
+previous_weight=0
+max_progression=-1
+# Set the scale of the progression bar
+# progress_string(0,1,2) should have the size of the scale.
+progress_scale=20
+progress_string2="####################"
+progress_string1="++++++++++++++++++++"
+progress_string0="...................."
+
+# Print a progress bar showing the progression of an app script
+#
+# usage: ynh_script_progression "Some message"
+ynh_script_progression() {
+    set +o xtrace # set +x
+
+    # Compute $max_progression (if we didn't already)
+    if [ "$max_progression" = -1 ]; then
+        # Get the number of occurrences of 'ynh_script_progression' in the script. Except those are commented.
+        local helper_calls=
+        max_progression="$(grep --count "^[^#]*ynh_script_progression" $0)"
+    fi
+
+    # Increment each execution of ynh_script_progression in this script by the weight of the previous call.
+    increment_progression=$(($increment_progression + $previous_weight))
+    # Store the weight of the current call in $previous_weight for next call
+    previous_weight=1
+
+    # Reduce $increment_progression to the size of the scale
+    local effective_progression=$(($increment_progression * $progress_scale / $max_progression))
+
+    # If last is specified, fill immediately the progression_bar
+
+    # Build $progression_bar from progress_string(0,1,2) according to $effective_progression and the weight of the current task
+    # expected_progression is the progression expected after the current task
+    local expected_progression="$((($increment_progression + 1) * $progress_scale / $max_progression - $effective_progression))"
+
+    # Hack for the "--last" message
+    if grep -qw 'completed' <<< "$1"; then
+        effective_progression=$progress_scale
+        expected_progression=0
+    fi
+    # left_progression is the progression not yet done
+    local left_progression="$(($progress_scale - $effective_progression - $expected_progression))"
+    # Build the progression bar with $effective_progression, work done, $expected_progression, current work and $left_progression, work to be done.
+    local progression_bar="${progress_string2:0:$effective_progression}${progress_string1:0:$expected_progression}${progress_string0:0:$left_progression}"
+
+    echo "[$progression_bar] > ${1}" >&$YNH_STDINFO
+    set -o xtrace # set -x
+}

helpers/helpers.v2.1.d/logrotate

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+FIRST_CALL_TO_LOGROTATE="true"
+
+# Add a logrotate configuration to manage log files / log directory
+#
+# usage: ynh_config_add_logrotate [/path/to/log/file/or/folder]
+#
+# If not argument is provided, `/var/log/$app/*.log` is used as default.
+#
+# The configuration is autogenerated by YunoHost
+# (ie it doesnt come from a specific app template like nginx or systemd conf)
+ynh_config_add_logrotate() {
+
+    local logfile="${1:-}"
+
+    set -o noglob
+    if [[ -z "$logfile" ]]; then
+        logfile="/var/log/${app}/*.log"
+    elif [[ "${logfile##*.}" != "log" ]] && [[ "${logfile##*.}" != "txt" ]]; then
+        logfile="$logfile/*.log"
+    fi
+    set +o noglob
+
+    for stuff in $logfile; do
+        # Make sure the permissions of the parent dir are correct (otherwise the config file could be ignored and the corresponding logs never rotated)
+        local dir=$(dirname "$stuff")
+        mkdir --parents $dir
+        chmod 750 $dir
+        chown $app:$app $dir
+    done
+
+    local tempconf="$(mktemp)"
+    cat << EOF > $tempconf
+$logfile {
+    # Rotate if the logfile exceeds 100Mo
+    size 100M
+    # Keep 12 old log maximum
+    rotate 12
+    # Compress the logs with gzip
+    compress
+    # Compress the log at the next cycle. So keep always 2 non compressed logs
+    delaycompress
+    # Copy and truncate the log to allow to continue write on it. Instead of moving the log.
+    copytruncate
+    # Do not trigger an error if the log is missing
+    missingok
+    # Do not rotate if the log is empty
+    notifempty
+    # Keep old logs in the same dir
+    noolddir
+}
+EOF
+
+    if [[ "$FIRST_CALL_TO_LOGROTATE" == "true" ]]; then
+        cat $tempconf > /etc/logrotate.d/$app
+    else
+        cat $tempconf >> /etc/logrotate.d/$app
+    fi
+
+    FIRST_CALL_TO_LOGROTATE="false"
+
+    chmod 644 "/etc/logrotate.d/$app"
+}
+
+# Remove the app's logrotate config.
+#
+# usage:ynh_config_remove_logrotate
+ynh_config_remove_logrotate() {
+    if [ -e "/etc/logrotate.d/$app" ]; then
+        rm "/etc/logrotate.d/$app"
+    fi
+}

helpers/helpers.v2.1.d/mongodb

@@ -0,0 +1,271 @@
+#!/bin/bash
+
+# Execute a mongo command
+#
+# example: ynh_mongo_exec --command='db.getMongo().getDBNames().indexOf("wekan")'
+# example: ynh_mongo_exec --command="db.getMongo().getDBNames().indexOf(\"wekan\")"
+#
+# usage: ynh_mongo_exec [--database=database] --command="command"
+# | arg: --database=    - The database to connect to
+# | arg: --command=     - The command to evaluate
+#
+#
+ynh_mongo_exec() {
+    # ============ Argument parsing =============
+    local -A args_array=([d]=database= [c]=command=)
+    local database
+    local command
+    ynh_handle_getopts_args "$@"
+    database="${database:-}"
+    # ===========================================
+
+    if [ -n "$database" ]; then
+        mongosh --quiet << EOF
+use $database
+${command}
+quit()
+EOF
+    else
+        mongosh --quiet --eval="$command"
+    fi
+}
+
+# Drop a database
+#
+# [internal]
+#
+# If you intend to drop the database *and* the associated user,
+# consider using ynh_mongo_remove_db instead.
+#
+# usage: ynh_mongo_drop_db --database=database
+# | arg: --database=    - The database name to drop
+#
+#
+ynh_mongo_drop_db() {
+    # ============ Argument parsing =============
+    local -A args_array=([d]=database=)
+    local database
+    ynh_handle_getopts_args "$@"
+    # ===========================================
+
+    ynh_mongo_exec --database="$database" --command='db.runCommand({dropDatabase: 1})'
+}
+
+# Dump a database
+#
+# example: ynh_mongo_dump_db --database=wekan > ./dump.bson
+#
+# usage: ynh_mongo_dump_db --database=database
+# | arg: --database=    - The database name to dump
+# | ret: the mongodump output
+#
+#
+ynh_mongo_dump_db() {
+    # ============ Argument parsing =============
+    local -A args_array=([d]=database=)
+    local database
+    ynh_handle_getopts_args "$@"
+    # ===========================================
+
+    mongodump --quiet --db="$database" --archive
+}
+
+# Create a user
+#
+# [internal]
+#
+# usage: ynh_mongo_create_user --db_user=user --db_pwd=pwd --db_name=name
+# | arg: --db_user=       - The user name to create
+# | arg: --db_pwd=        - The password to identify user by
+# | arg: --db_name=       - Name of the database to grant privilegies
+#
+#
+ynh_mongo_create_user() {
+    # ============ Argument parsing =============
+    local -A args_array=([u]=db_user= [n]=db_name= [p]=db_pwd=)
+    local db_user
+    local db_name
+    local db_pwd
+    ynh_handle_getopts_args "$@"
+    # ===========================================
+
+    # Create the user and set the user as admin of the db
+    ynh_mongo_exec --database="$db_name" --command='db.createUser( { user: "'${db_user}'", pwd: "'${db_pwd}'", roles: [ { role: "readWrite", db: "'${db_name}'" } ] } );'
+
+    # Add clustermonitoring rights
+    ynh_mongo_exec --database="$db_name" --command='db.grantRolesToUser("'${db_user}'",[{ role: "clusterMonitor", db: "admin" }]);'
+}
+
+# Check if a mongo database exists
+#
+# usage: ynh_mongo_database_exists --database=database
+# | arg: --database=        - The database for which to check existence
+# | exit: Return 1 if the database doesn't exist, 0 otherwise
+#
+#
+ynh_mongo_database_exists() {
+    # ============ Argument parsing =============
+    local -A args_array=([d]=database=)
+    local database
+    ynh_handle_getopts_args "$@"
+    # ===========================================
+
+    if [ $(ynh_mongo_exec --command='db.getMongo().getDBNames().indexOf("'${database}'")') -lt 0 ]; then
+        return 1
+    else
+        return 0
+    fi
+}
+
+# Restore a database
+#
+# example: ynh_mongo_restore_db --database=wekan < ./dump.bson
+#
+# usage: ynh_mongo_restore_db --database=database
+# | arg: --database=    - The database name to restore
+#
+#
+ynh_mongo_restore_db() {
+    # ============ Argument parsing =============
+    local -A args_array=([d]=database=)
+    local database
+    ynh_handle_getopts_args "$@"
+    # ===========================================
+
+    mongorestore --quiet --db="$database" --archive
+}
+
+# Drop a user
+#
+# [internal]
+#
+# usage: ynh_mongo_drop_user --db_user=user --db_name=name
+# | arg: --db_user=        - The user to drop
+# | arg: --db_name=        - Name of the database
+#
+#
+ynh_mongo_drop_user() {
+    # ============ Argument parsing =============
+    local -A args_array=([u]=db_user= [n]=db_name=)
+    local db_user
+    local db_name
+    ynh_handle_getopts_args "$@"
+    # ===========================================
+
+    ynh_mongo_exec --database="$db_name" --command='db.dropUser("'$db_user'", {w: "majority", wtimeout: 5000})'
+}
+
+# Create a database, an user and its password. Then store the password in the app's config
+#
+# usage: ynh_mongo_setup_db --db_user=user --db_name=name [--db_pwd=pwd]
+# | arg: --db_user=        - Owner of the database
+# | arg: --db_name=        - Name of the database
+# | arg: --db_pwd=        - Password of the database. If not provided, a password will be generated
+#
+# After executing this helper, the password of the created database will be available in $db_pwd
+# It will also be stored as "mongopwd" into the app settings.
+#
+#
+ynh_mongo_setup_db() {
+    # ============ Argument parsing =============
+    local -A args_array=([u]=db_user= [n]=db_name= [p]=db_pwd=)
+    local db_user
+    local db_name
+    db_pwd=""
+    ynh_handle_getopts_args "$@"
+    # ===========================================
+
+    local new_db_pwd=$(ynh_string_random) # Generate a random password
+    # If $db_pwd is not provided, use new_db_pwd instead for db_pwd
+    db_pwd="${db_pwd:-$new_db_pwd}"
+
+    # Create the user and grant access to the database
+    ynh_mongo_create_user --db_user="$db_user" --db_pwd="$db_pwd" --db_name="$db_name"
+
+    # Store the password in the app's config
+    ynh_app_setting_set --key=db_pwd --value=$db_pwd
+}
+
+# Remove a database if it exists, and the associated user
+#
+# usage: ynh_mongo_remove_db --db_user=user --db_name=name
+# | arg: --db_user=        - Owner of the database
+# | arg: --db_name=        - Name of the database
+#
+#
+ynh_mongo_remove_db() {
+    # ============ Argument parsing =============
+    local -A args_array=([u]=db_user= [n]=db_name=)
+    local db_user
+    local db_name
+    ynh_handle_getopts_args "$@"
+    # ===========================================
+
+    if ynh_mongo_database_exists --database=$db_name; then # Check if the database exists
+        ynh_mongo_drop_db --database=$db_name              # Remove the database
+    else
+        ynh_print_warn "Database $db_name not found"
+    fi
+
+    # Remove mongo user if it exists
+    ynh_mongo_drop_user --db_user=$db_user --db_name=$db_name
+}
+
+# Install MongoDB and integrate MongoDB service in YunoHost
+#
+# The installed version is defined by $mongo_version which should be defined as global prior to calling this helper
+#
+# usage: ynh_install_mongo
+#
+ynh_install_mongo() {
+
+    [[ -n "${mongo_version:-}" ]] || ynh_die "\$mongo_version should be defined prior to calling ynh_install_mongo"
+
+    ynh_print_info "Installing MongoDB Community Edition ..."
+    local mongo_debian_release=$YNH_DEBIAN_VERSION
+
+    if [[ "$(grep '^flags' /proc/cpuinfo | uniq)" != *"avx"* && "$mongo_version" != "4.4" ]]; then
+        ynh_print_warn "Installing Mongo 4.4 as $mongo_version is not compatible with your cpu (see https://docs.mongodb.com/manual/administration/production-notes/#x86_64)."
+        mongo_version="4.4"
+    fi
+    if [[ "$mongo_version" == "4.4" ]]; then
+        ynh_print_warn "Switched to buster install as Mongo 4.4 is not compatible with $mongo_debian_release."
+        mongo_debian_release=buster
+    fi
+
+    ynh_apt_install_dependencies_from_extra_repository \
+        --repo="deb http://repo.mongodb.org/apt/debian $mongo_debian_release/mongodb-org/$mongo_version main" \
+        --package="mongodb-org mongodb-org-server mongodb-org-tools mongodb-mongosh" \
+        --key="https://www.mongodb.org/static/pgp/server-$mongo_version.asc"
+    mongodb_servicename=mongod
+
+    # Make sure MongoDB is started and enabled
+    systemctl enable $mongodb_servicename --quiet
+    systemctl daemon-reload --quiet
+    ynh_systemctl --service=$mongodb_servicename --action=restart --wait_until="aiting for connections" --log_path="/var/log/mongodb/$mongodb_servicename.log"
+
+    # Integrate MongoDB service in YunoHost
+    yunohost service add $mongodb_servicename --description="MongoDB daemon" --log="/var/log/mongodb/$mongodb_servicename.log"
+
+    # Store mongo_version into the config of this app
+    ynh_app_setting_set --key=mongo_version --value=$mongo_version
+}
+
+# Remove MongoDB
+# Only remove the MongoDB service integration in YunoHost for now
+# if MongoDB package as been removed
+#
+# usage: ynh_remove_mongo
+#
+#
+ynh_remove_mongo() {
+    # Only remove the mongodb service if it is not installed.
+    if ! _ynh_apt_package_is_installed "mongodb*"; then
+        ynh_print_info "Removing MongoDB service..."
+        mongodb_servicename=mongod
+        # Remove the mongodb service
+        yunohost service remove $mongodb_servicename
+        ynh_safe_rm "/var/lib/mongodb"
+        ynh_safe_rm "/var/log/mongodb"
+    fi
+}

helpers/helpers.v2.1.d/multimedia

@@ -0,0 +1,89 @@
+#!/bin/bash
+
+readonly MEDIA_GROUP=multimedia
+readonly MEDIA_DIRECTORY=/home/yunohost.multimedia
+
+# Initialize the multimedia directory system
+#
+# usage: ynh_multimedia_build_main_dir
+ynh_multimedia_build_main_dir() {
+
+    ## Création du groupe multimedia
+    groupadd -f $MEDIA_GROUP
+
+    ## Création des dossiers génériques
+    mkdir -p "$MEDIA_DIRECTORY"
+    mkdir -p "$MEDIA_DIRECTORY/share"
+    mkdir -p "$MEDIA_DIRECTORY/share/Music"
+    mkdir -p "$MEDIA_DIRECTORY/share/Picture"
+    mkdir -p "$MEDIA_DIRECTORY/share/Video"
+    mkdir -p "$MEDIA_DIRECTORY/share/eBook"
+
+    ## Création des dossiers utilisateurs
+    for user in $(yunohost user list --output-as json | jq -r '.users | keys[]'); do
+        mkdir -p "$MEDIA_DIRECTORY/$user"
+        mkdir -p "$MEDIA_DIRECTORY/$user/Music"
+        mkdir -p "$MEDIA_DIRECTORY/$user/Picture"
+        mkdir -p "$MEDIA_DIRECTORY/$user/Video"
+        mkdir -p "$MEDIA_DIRECTORY/$user/eBook"
+        ln -sfn "$MEDIA_DIRECTORY/share" "$MEDIA_DIRECTORY/$user/Share"
+        # Création du lien symbolique dans le home de l'utilisateur.
+        #link will only be created if the home directory of the user exists and if it's located in '/home' folder
+        local user_home="$(getent passwd $user | cut -d: -f6 | grep '^/home/')"
+        if [[ -d "$user_home" ]]; then
+            ln -sfn "$MEDIA_DIRECTORY/$user" "$user_home/Multimedia"
+        fi
+        # Propriétaires des dossiers utilisateurs.
+        chown -R $user "$MEDIA_DIRECTORY/$user"
+    done
+    # Default yunohost hooks for post_user_create,delete will take care
+    # of creating/deleting corresponding multimedia folders when users
+    # are created/deleted in the future...
+
+    ## Application des droits étendus sur le dossier multimedia.
+    # Droit d'écriture pour le groupe et le groupe multimedia en acl et droit de lecture pour other:
+    setfacl -RnL -m g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$MEDIA_DIRECTORY" || true
+    # Application de la même règle que précédemment, mais par défaut pour les nouveaux fichiers.
+    setfacl -RnL -m d:g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$MEDIA_DIRECTORY" || true
+    # Réglage du masque par défaut. Qui garantie (en principe...) un droit maximal à rwx. Donc pas de restriction de droits par l'acl.
+    setfacl -RL -m m::rwx "$MEDIA_DIRECTORY" || true
+}
+
+# Add a directory in yunohost.multimedia
+#
+# usage: ynh_multimedia_addfolder --source_dir="source_dir" --dest_dir="dest_dir"
+#
+# | arg: --source_dir= - Source directory - The real directory which contains your medias.
+# | arg: --dest_dir= - Destination directory - The name and the place of the symbolic link, relative to "/home/yunohost.multimedia"
+#
+# This "directory" will be a symbolic link to a existing directory.
+ynh_multimedia_addfolder() {
+
+    # ============ Argument parsing =============
+    local -A args_array=([s]=source_dir= [d]=dest_dir=)
+    local source_dir
+    local dest_dir
+    ynh_handle_getopts_args "$@"
+    # ===========================================
+
+    # Ajout d'un lien symbolique vers le dossier à partager
+    ln -sfn "$source_dir" "$MEDIA_DIRECTORY/$dest_dir"
+
+    ## Application des droits étendus sur le dossier ajouté
+    # Droit d'écriture pour le groupe et le groupe multimedia en acl et droit de lecture pour other:
+    setfacl -RnL -m g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$source_dir"
+    # Application de la même règle que précédemment, mais par défaut pour les nouveaux fichiers.
+    setfacl -RnL -m d:g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$source_dir"
+    # Réglage du masque par défaut. Qui garantie (en principe...) un droit maximal à rwx. Donc pas de restriction de droits par l'acl.
+    setfacl -RL -m m::rwx "$source_dir"
+}
+
+# Add an user to the multimedia group, in turn having write permission in multimedia directories
+#
+# usage: ynh_multimedia_addaccess user_name
+#
+# | arg: user_name   - The name of the user which gain this access.
+ynh_multimedia_addaccess() {
+    groupadd -f $MEDIA_GROUP
+    usermod -a -G $MEDIA_GROUP $1
+}

helpers/helpers.v2.1.d/mysql

@@ -0,0 +1,116 @@
+#!/bin/bash
+
+# Run SQL instructions in a database ($db_name by default)
+#
+# usage: ynh_mysql_db_shell [database] <<< "instructions"
+# | arg: database=    - the database to connect to (by default, $db_name)
+#
+# examples:
+#    ynh_mysql_db_shell $db_name <<< "UPDATE ...;"
+#    ynh_mysql_db_shell < /path/to/file.sql
+#
+ynh_mysql_db_shell() {
+    local database=${1:-$db_name}
+    mysql -B $database
+}
+
+# Create a database and grant optionnaly privilegies to a user
+#
+# [internal] ... handled by the core / "database resource"
+#
+# usage: ynh_mysql_create_db db [user [pwd]]
+# | arg: db - the database name to create
+# | arg: user - the user to grant privilegies
+# | arg: pwd - the password to identify user by
+#
+ynh_mysql_create_db() {
+    local db=$1
+
+    local sql="CREATE DATABASE ${db};"
+
+    # grant all privilegies to user
+    if [[ $# -gt 1 ]]; then
+        sql+=" GRANT ALL PRIVILEGES ON ${db}.* TO '${2}'@'localhost'"
+        if [[ -n ${3:-} ]]; then
+            sql+=" IDENTIFIED BY '${3}'"
+        fi
+        sql+=" WITH GRANT OPTION;"
+    fi
+
+    mysql -B <<< "$sql"
+}
+
+# Drop a database
+#
+# [internal] ... handled by the core / "database resource"
+#
+# If you intend to drop the database *and* the associated user,
+# consider using ynh_mysql_remove_db instead.
+#
+# usage: ynh_mysql_drop_db db
+# | arg: db - the database name to drop
+#
+ynh_mysql_drop_db() {
+    mysql -B <<< "DROP DATABASE ${1};"
+}
+
+# Dump a database
+#
+# usage: ynh_mysql_dump_db database
+# | arg: database   - the database name to dump (by default, $db_name)
+# | ret: The mysqldump output
+#
+# example: ynh_mysql_dump_db "roundcube" > ./dump.sql
+#
+ynh_mysql_dump_db() {
+    local database=${1:-$db_name}
+    mysqldump --single-transaction --skip-dump-date --routines "$database"
+}
+
+# Create a user
+#
+# [internal] ... handled by the core / "database resource"
+#
+# usage: ynh_mysql_create_user user pwd [host]
+# | arg: user - the user name to create
+# | arg: pwd - the password to identify user by
+#
+ynh_mysql_create_user() {
+    mysql -B <<< "CREATE USER '${1}'@'localhost' IDENTIFIED BY '${2}';"
+}
+
+# Check if a mysql user exists
+#
+# [internal]
+#
+# usage: ynh_mysql_user_exists user
+# | arg: user   - the user for which to check existence
+# | ret: 0 if the user exists, 1 otherwise.
+ynh_mysql_user_exists() {
+    local user=$1
+    [[ -n "$(mysql -B <<< "SELECT User from mysql.user WHERE User = '$user';")" ]]
+}
+
+# Check if a mysql database exists
+#
+# [internal]
+#
+# usage: ynh_mysql_database_exists database
+# | arg: database    - the database for which to check existence
+# | exit: Return 1 if the database doesn't exist, 0 otherwise
+#
+ynh_mysql_database_exists() {
+    local database=$1
+    mysqlshow | grep -q "^| $database "
+}
+
+# Drop a user
+#
+# [internal] ... handled by the core / "database resource"
+#
+# usage: ynh_mysql_drop_user user
+# | arg: user - the user name to drop
+#
+ynh_mysql_drop_user() {
+    mysql -B <<< "DROP USER '${1}'@'localhost';"
+}

helpers/helpers.v2.1.d/nginx

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# Create a dedicated nginx config
+#
+# usage: ynh_config_add_nginx
+#
+# This will use a template in `../conf/nginx.conf`
+# See the documentation of `ynh_config_add` for a description of the template
+# format and how placeholders are replaced with actual variables.
+#
+# Additionally, ynh_config_add_nginx will replace:
+# - `#sub_path_only`      by empty string if `path` is not `'/'`
+# - `#root_path_only`     by empty string if `path`  *is*  `'/'`
+#
+# This allows to enable/disable specific behaviors dependenging on the install
+# location
+ynh_config_add_nginx() {
+
+    local finalnginxconf="/etc/nginx/conf.d/$domain.d/$app.conf"
+
+    ynh_config_add --template="nginx.conf" --destination="$finalnginxconf"
+
+    if [ "${path:-}" != "/" ]; then
+        ynh_replace --match="^#sub_path_only" --replace="" --file="$finalnginxconf"
+    else
+        ynh_replace --match="^#root_path_only" --replace="" --file="$finalnginxconf"
+    fi
+
+    ynh_store_file_checksum "$finalnginxconf"
+
+    ynh_systemctl --service=nginx --action=reload
+}
+
+# Remove the dedicated nginx config
+#
+# usage: ynh_config_remove_nginx
+ynh_config_remove_nginx() {
+    ynh_safe_rm "/etc/nginx/conf.d/$domain.d/$app.conf"
+    ynh_systemctl --service=nginx --action=reload
+}
+
+# Regen the nginx config in a change url context
+#
+# usage: ynh_config_change_url_nginx
+ynh_config_change_url_nginx() {
+
+    # Make a backup of the original NGINX config file if manually modified
+    # (nb: this is possibly different from the same instruction called by
+    # ynh_config_add inside ynh_config_add_nginx because the path may have
+    # changed if we're changing the domain too...)
+    local old_nginx_conf_path=/etc/nginx/conf.d/$old_domain.d/$app.conf
+    ynh_backup_if_checksum_is_different "$old_nginx_conf_path"
+    ynh_delete_file_checksum "$old_nginx_conf_path"
+    ynh_safe_rm "$old_nginx_conf_path"
+
+    # Regen the nginx conf
+    ynh_config_add_nginx
+}