.codeclimate.yml

@@ -1,24 +0,0 @@
----
-version: "2"
-plugins:
-  duplication:
-    enabled: true
-    config:
-      languages:
-        python:
-          python_version: 3
-  shellcheck:
-    enabled: true
-  pep8:
-    enabled: true
-  fixme:
-    enabled: true
-  sonar-python:
-    enabled: true
-    config:
-      tests_patterns:
-        - bin/*
-        - data/**
-        - doc/*
-        - src/**
-        - tests/**

.coveragerc

@@ -1,2 +0,0 @@
-[report]
-omit=src/tests/*,src/vendor/*,/usr/lib/moulinette/yunohost/*,/usr/lib/python3/dist-packages/yunohost/tests/*,/usr/lib/python3/dist-packages/yunohost/vendor/*

.github/workflows/autoblack.yml

@@ -1,30 +0,0 @@
-name: Check / auto apply Black
-
-on:
-  push:
-    branches: [ "dev" ]
-
-jobs:
-  black:
-    name: Check / auto apply black
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Check files using the black formatter
-        uses: psf/black@stable
-        id: black
-        with:
-          options: "."
-        continue-on-error: true
-
-      - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v6
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          title: "Format Python code with Black"
-          commit-message: ":art: Format Python code with Black"
-          body: |
-            This pull request uses the [psf/black](https://github.com/psf/black) formatter.
-          base: ${{ github.head_ref }} # Creates pull request onto pull request or commit branch
-          branch: actions/black

.github/workflows/codeql.yml

@@ -1,42 +0,0 @@
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ "dev" ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ "dev" ]
-    paths-ignore:
-      - 'src/tests/**'
-  schedule:
-    - cron: '43 12 * * 3'
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'python' ]
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: ${{ matrix.language }}
-        queries: security-extended,security-and-quality
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
-      with:
-        category: "/language:${{matrix.language}}"

.github/workflows/n_updater.yml

@@ -1,39 +0,0 @@
-# This workflow allows GitHub Actions to automagically update YunoHost NodeJS helper whenever a new release of n is detected.
-name: Check for new n releases
-on:
-  # Allow to manually trigger the workflow
-  workflow_dispatch:
-  # Run it every day at 5:00 UTC
-  schedule:
-    - cron:  '0 5 * * *'
-jobs:
-  updater:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Fetch the source code
-        uses: actions/checkout@v4
-
-      - name: Run the updater script
-        id: run_updater
-        run: |
-          # Download n
-          wget https://raw.githubusercontent.com/tj/n/master/bin/n --output-document=helpers/vendor/n/n
-
-          echo "VERSION=$(sed -n 's/^VERSION=\"\(.*\)\"/\1/p' < helpers/vendor/n/n)" >> $GITHUB_ENV
-
-      - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v6
-        id: cpr
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          commit-message: Update n to ${{ env.VERSION }}
-          committer: 'yunohost-bot <yunohost-bot@users.noreply.github.com>'
-          author: 'yunohost-bot <yunohost-bot@users.noreply.github.com>'
-          signoff: false
-          base: dev
-          branch: ci-auto-update-n-${{ env.VERSION }}
-          delete-branch: true
-          title: 'Upgrade n to ${{ env.VERSION }}'
-          body: |
-            Upgrade `n` to ${{ env.VERSION }}
-          draft: false

.gitignore

@@ -31,14 +31,7 @@ pip-log.txt
 .mr.developer.cfg
 
 # moulinette lib
-src/locales
+src/yunohost/locales
 
 # Test
-src/tests/apps
-
-# Tmp/local doc stuff
-doc/bash-completion.sh
-doc/bash_completion.d
-doc/openapi.js
-doc/openapi.json
-doc/swagger
+src/yunohost/tests/apps

.gitlab-ci.yml

@@ -1,11 +1,9 @@
----
 stages:
   - build
   - install
-  - test
+  - tests
   - lint
   - doc
-  - translation
 
 default:
   tags:
@@ -13,43 +11,12 @@ default:
   # All jobs are interruptible by default
   interruptible: true
 
-code_quality:
-  tags:
-    - docker
-  rules:
-    - if: $CI_COMMIT_TAG # Only for tags
-
-
-code_quality_html:
-  extends: code_quality
-  variables:
-    REPORT_FORMAT: html
-  artifacts:
-    paths: [gl-code-quality-report.html]
-  rules:
-    - if: $CI_COMMIT_TAG # Only for tags
-
-
-# see: https://docs.gitlab.com/ee/ci/yaml/#switch-between-branch-pipelines-and-merge-request-pipelines
-workflow:
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event" # If we move to gitlab one day
-    - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" # For github PR
-    - if: $CI_COMMIT_TAG # For tags
-    - if: $CI_COMMIT_REF_NAME == "ci-format-$CI_DEFAULT_BRANCH"  # Ignore black formatting branch created by the CI
-      when: never
-    - if: $CI_COMMIT_REF_NAME == "actions/black"  # Ignore black formatting branch created by the CI
-      when: never
-    - if: $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH  && $CI_PIPELINE_SOURCE == "push" # If it's not the default branch and if it's a push, then do not trigger a build
-      when: never
-    - when: always
-
 variables:
-  GIT_CLONE_PATH: '$CI_BUILDS_DIR/$CI_COMMIT_SHA/$CI_JOB_ID'
-  YNH_SOURCE: "https://github.com/yunohost"
-  YNH_DEBIAN: "bullseye"
-  YNH_SKIP_DIAGNOSIS_DURING_UPGRADE: "true"
+    YNH_BUILD_DIR: "ynh-build"
 
 include:
-  - template: Code-Quality.gitlab-ci.yml
-  - local: .gitlab/ci/*.gitlab-ci.yml
+  - local: .gitlab/ci/build.gitlab-ci.yml
+  - local: .gitlab/ci/install.gitlab-ci.yml
+  - local: .gitlab/ci/test.gitlab-ci.yml
+  - local: .gitlab/ci/lint.gitlab-ci.yml
+  - local: .gitlab/ci/doc.gitlab-ci.yml

.gitlab/ci/build.gitlab-ci.yml

@@ -1,22 +1,20 @@
 .build-stage:
   stage: build
-  image: "build-and-lint"
+  image: "before-install"
   variables:
-    YNH_BUILD_DIR: "$GIT_CLONE_PATH/build"
+    YNH_SOURCE: "https://github.com/yunohost"
   before_script:
     - mkdir -p $YNH_BUILD_DIR
   artifacts:
     paths:
-      - ./*.deb
+      - $YNH_BUILD_DIR/*.deb
 
 .build_script: &build_script
   - cd $YNH_BUILD_DIR/$PACKAGE
   - VERSION=$(dpkg-parsechangelog -S Version 2>/dev/null)
-  - VERSION_TIMESTAMPED="${VERSION}+$(date +%Y%m%d%H%M)"
-  - dch --package "${PACKAGE}" --force-bad-version -v "${VERSION_TIMESTAMPED}" -D "unstable" --force-distribution "CI build."
+  - VERSION_NIGHTLY="${VERSION}+$(date +%Y%m%d%H%M)"
+  - dch --package "${PACKAGE}" --force-bad-version -v "${VERSION_NIGHTLY}" -D "unstable" --force-distribution "Daily build."
   - debuild --no-lintian -us -uc
-  - cp $YNH_BUILD_DIR/*.deb ${CI_PROJECT_DIR}/
-  - cd ${CI_PROJECT_DIR}
 
 ########################################
 # BUILD DEB
@@ -31,16 +29,18 @@ build-yunohost:
     - mkdir -p $YNH_BUILD_DIR/$PACKAGE
     - cat archive.tar.gz | tar -xz -C $YNH_BUILD_DIR/$PACKAGE
     - rm archive.tar.gz
-    - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $YNH_BUILD_DIR/$PACKAGE
+    - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $(pwd)/$YNH_BUILD_DIR/$PACKAGE
     - *build_script
 
+
 build-ssowat:
   extends: .build-stage
   variables:
     PACKAGE: "ssowat"
   script:
-    - git clone $YNH_SOURCE/$PACKAGE -b $CI_COMMIT_REF_NAME $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE -b $YNH_DEBIAN $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE $YNH_BUILD_DIR/$PACKAGE --depth 1
-    - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $YNH_BUILD_DIR/$PACKAGE
+    - DEBIAN_DEPENDS=$(cat debian/control | tr "," "\n" | grep -Po "ssowat \([>,=,<]+ .*\)" | grep -Po "[0-9\.]+")
+    - git clone $YNH_SOURCE/$PACKAGE -b $CI_COMMIT_REF_NAME $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE -b $DEBIAN_DEPENDS $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE $YNH_BUILD_DIR/$PACKAGE --depth 1
+    - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $(pwd)/$YNH_BUILD_DIR/$PACKAGE
     - *build_script
 
 build-moulinette:
@@ -48,6 +48,7 @@ build-moulinette:
   variables:
     PACKAGE: "moulinette"
   script:
-    - git clone $YNH_SOURCE/$PACKAGE -b $CI_COMMIT_REF_NAME $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE -b $YNH_DEBIAN $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE $YNH_BUILD_DIR/$PACKAGE --depth 1
-    - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $YNH_BUILD_DIR/$PACKAGE
+    - DEBIAN_DEPENDS=$(cat debian/control | tr "," "\n" | grep -Po "moulinette \([>,=,<]+ .*\)" | grep -Po "[0-9\.]+")
+    - git clone $YNH_SOURCE/$PACKAGE -b $CI_COMMIT_REF_NAME $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE -b $DEBIAN_DEPENDS $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE $YNH_BUILD_DIR/$PACKAGE --depth 1
+    - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $(pwd)/$YNH_BUILD_DIR/$PACKAGE
     - *build_script

.gitlab/ci/doc.gitlab-ci.yml

@@ -4,28 +4,24 @@
 
 generate-helpers-doc:
   stage: doc
-  image: "build-and-lint"
+  image: "before-install"
   needs: []
   before_script:
+    - apt-get update -y && apt-get install git hub -y
     - git config --global user.email "yunohost@yunohost.org"
     - git config --global user.name "$GITHUB_USER"
   script:
     - cd doc
-    - python3 generate_helper_doc.py 2
-    - python3 generate_helper_doc.py 2.1
-    - python3 generate_resource_doc.py > resources.md
+    - python generate_helper_doc.py
     - hub clone https://$GITHUB_TOKEN:x-oauth-basic@github.com/YunoHost/doc.git doc_repo
-    - cp helpers.v2.md doc_repo/pages/06.contribute/10.packaging_apps/20.scripts/10.helpers/packaging_app_scripts_helpers.md
-    - cp helpers.v2.1.md doc_repo/pages/06.contribute/10.packaging_apps/20.scripts/12.helpers21/packaging_app_scripts_helpers_v21.md
-    - cp resources.md doc_repo/pages/06.contribute/10.packaging_apps/10.manifest/10.appresources/packaging_app_manifest_resources.md
+    - cp helpers.html doc_repo/packaging_apps_helpers.md
     - cd doc_repo
     # replace ${CI_COMMIT_REF_NAME} with ${CI_COMMIT_TAG} ?
     - hub checkout -b "${CI_COMMIT_REF_NAME}"
-    - hub commit -am "[CI] Update app helpers/resources for ${CI_COMMIT_REF_NAME}"
-    - hub pull-request -m "[CI] Update app helpers/resources for ${CI_COMMIT_REF_NAME}" -p # GITHUB_USER and GITHUB_TOKEN registered here https://gitlab.com/yunohost/yunohost/-/settings/ci_cd
+    - hub commit -am "[CI] Helper for ${CI_COMMIT_REF_NAME}"
+    - hub pull-request -m "[CI] Helper for ${CI_COMMIT_REF_NAME}" -p # GITHUB_USER and GITHUB_TOKEN registered here https://gitlab.com/yunohost/yunohost/-/settings/ci_cd
   artifacts:
     paths:
-      - doc/helpers.md
-      - doc/resources.md
+      - doc/helpers.html
   only:
     - tags

.gitlab/ci/install.gitlab-ci.yml

@@ -14,14 +14,16 @@
 
 upgrade:
   extends: .install-stage
-  image: "core-tests"
+  image: "after-install"
   script:
-    - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb
+    - apt-get update -o Acquire::Retries=3
+    - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ./$YNH_BUILD_DIR/*.deb
 
 
 install-postinstall:
   extends: .install-stage
   image: "before-install"
   script:
-    - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb
-    - yunohost tools postinstall -d domain.tld -u syssa -F 'Syssa Mine' -p the_password --ignore-dyndns --force-diskspace
+    - apt-get update -o Acquire::Retries=3
+    - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ./$YNH_BUILD_DIR/*.deb
+    - yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns

.gitlab/ci/lint.gitlab-ci.yml

@@ -3,24 +3,41 @@
 ########################################
 # later we must fix lint and format-check jobs and remove "allow_failure"
 
-lint39:
+lint27:
   stage: lint
-  image: "build-and-lint"
+  image: "before-install"
   needs: []
   allow_failure: true
   script:
-    - tox -e py39-lint
+    - tox -e py27-lint
 
-invalidcode39:
+lint37:
   stage: lint
-  image: "build-and-lint"
+  image: "before-install"
+  needs: []
+  allow_failure: true
+  script:
+    - tox -e py37-lint
+
+invalidcode27:
+  stage: lint
+  image: "before-install"
   needs: []
   script:
-    - tox -e py39-invalidcode
+    - tox -e py27-invalidcode
 
-mypy:
+invalidcode37:
   stage: lint
-  image: "build-and-lint"
+  image: "before-install"
+  allow_failure: true
   needs: []
   script:
-    - tox -e py39-mypy
+    - tox -e py37-invalidcode
+
+format-check:
+  stage: lint
+  image: "before-install"
+  needs: []
+  allow_failure: true
+  script:
+    - tox -e py37-black

.gitlab/ci/test.gitlab-ci.yml

@@ -1,16 +1,17 @@
 .install_debs: &install_debs
-    - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb
+    - apt-get update -o Acquire::Retries=3
+    - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ./$YNH_BUILD_DIR/*.deb
 
 .test-stage:
-  stage: test
-  image: "core-tests"
+  stage: tests
+  image: "after-install"
   variables:
     PYTEST_ADDOPTS: "--color=yes"
   before_script:
     - *install_debs
   cache:
     paths: 
-      - src/tests/apps
+      - src/yunohost/tests/apps
     key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG"
   needs:
       - job: build-yunohost
@@ -21,21 +22,21 @@
         artifacts: true
       - job: upgrade
 
+
 ########################################
 # TESTS
 ########################################
 
 full-tests:
-  stage: test
+  stage: tests
   image: "before-install"
   variables:
     PYTEST_ADDOPTS: "--color=yes"
   before_script:
     - *install_debs
-    - pip install mock pip pyOpenSSL pytest pytest-cov pytest-mock pytest-sugar requests-mock "packaging<22"
-    - yunohost tools postinstall -d domain.tld -u syssa -F 'Syssa Mine' -p the_password --ignore-dyndns --force-diskspace
+    - yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns
   script:
-    - python3 -m pytest --cov=yunohost tests/ src/tests/ --junitxml=report.xml
+    - python -m pytest --cov=yunohost tests/ src/yunohost/tests/ --junitxml=report.xml
   needs:
       - job: build-yunohost
         artifacts: true
@@ -43,158 +44,77 @@ full-tests:
         artifacts: true
       - job: build-moulinette
         artifacts: true
-  coverage: '/TOTAL.*\s+(\d+%)/'
   artifacts:
     reports:
       junit: report.xml
 
-test-actionmap:
+root-tests:
   extends: .test-stage
   script:
-    - python3 -m pytest tests/test_actionmap.py
-  only:
-    changes:
-      - share/actionsmap.yml
-
-test-helpers2:
-  extends: .test-stage
-  script:
-    - cd tests
-    - bash test_helpers.sh
-
-test-helpers2.1:
-  extends: .test-stage
-  script:
-    - cd tests
-    - bash test_helpers.sh 2.1
-
-test-domains:
-  extends: .test-stage
-  script:
-    - python3 -m pytest src/tests/test_domains.py
-  only:
-    changes:
-      - src/domain.py
-
-test-dns:
-  extends: .test-stage
-  script:
-    - python3 -m pytest src/tests/test_dns.py
-  only:
-    changes:
-      - src/dns.py
-      - src/utils/dns.py
+    - python -m pytest tests
 
 test-apps:
   extends: .test-stage
   script:
-    - python3 -m pytest src/tests/test_apps.py
-  only:
-    changes:
-      - src/app.py
+    - cd src/yunohost
+    - python -m pytest tests/test_apps.py
 
 test-appscatalog:
   extends: .test-stage
   script:
-    - python3 -m pytest src/tests/test_app_catalog.py
-  only:
-    changes:
-      - src/app_calalog.py
+    - cd src/yunohost
+    - python -m pytest tests/test_appscatalog.py
 
 test-appurl:
   extends: .test-stage
   script:
-    - python3 -m pytest src/tests/test_appurl.py
-  only:
-    changes:
-      - src/app.py
+    - cd src/yunohost
+    - python -m pytest tests/test_appurl.py
 
-test-questions:
+test-apps-arguments-parsing:
   extends: .test-stage
   script:
-    - python3 -m pytest src/tests/test_questions.py
-  only:
-    changes:
-      - src/utils/config.py
-
-test-app-config:
-  extends: .test-stage
-  script:
-    - python3 -m pytest src/tests/test_app_config.py
-  only:
-    changes:
-      - src/app.py
-      - src/utils/config.py
-
-test-app-resources:
-  extends: .test-stage
-  script:
-    - python3 -m pytest src/tests/test_app_resources.py
-  only:
-    changes:
-      - src/app.py
-      - src/utils/resources.py
-
-test-changeurl:
-  extends: .test-stage
-  script:
-    - python3 -m pytest src/tests/test_changeurl.py
-  only:
-    changes:
-      - src/app.py
+    - cd src/yunohost
+    - python -m pytest tests/test_apps_arguments_parsing.py
 
 test-backuprestore:
   extends: .test-stage
   script:
-    - python3 -m pytest src/tests/test_backuprestore.py
-  only:
-    changes:
-      - src/backup.py
+    - cd src/yunohost
+    - python -m pytest tests/test_backuprestore.py
+
+test-changeurl:
+  extends: .test-stage
+  script:
+    - cd src/yunohost
+    - python -m pytest tests/test_changeurl.py
 
 test-permission:
   extends: .test-stage
   script:
-    - python3 -m pytest src/tests/test_permission.py
-  only:
-    changes:
-      - src/permission.py
+    - cd src/yunohost
+    - python -m pytest tests/test_permission.py
 
 test-settings:
   extends: .test-stage
   script:
-    - python3 -m pytest src/tests/test_settings.py
-  only:
-    changes:
-      - src/settings.py
+    - cd src/yunohost
+    - python -m pytest tests/test_settings.py
 
 test-user-group:
   extends: .test-stage
   script:
-    - python3 -m pytest src/tests/test_user-group.py
-  only:
-    changes:
-      - src/user.py
-
+    - cd src/yunohost
+    - python -m pytest tests/test_user-group.py
+    
 test-regenconf:
   extends: .test-stage
   script:
-    - python3 -m pytest src/tests/test_regenconf.py
-  only:
-    changes:
-      - src/regenconf.py
+    - cd src/yunohost
+    - python -m pytest tests/test_regenconf.py
 
 test-service:
   extends: .test-stage
   script:
-    - python3 -m pytest src/tests/test_service.py
-  only:
-    changes:
-      - src/service.py
-
-test-ldapauth:
-  extends: .test-stage
-  script:
-    - python3 -m pytest src/tests/test_ldapauth.py
-  only:
-    changes:
-      - src/authenticators/*.py
+    - cd src/yunohost
+    - python -m pytest tests/test_service.py

.gitlab/ci/translation.gitlab-ci.yml

@@ -1,36 +0,0 @@
-########################################
-# TRANSLATION
-########################################
-test-i18n-keys:
-  stage: translation
-  script:
-    - python3 maintenance/missing_i18n_keys.py --check
-  only:
-    changes:
-      - locales/en.json
-      - src/*.py
-      - src/diagnosers/*.py
-
-autofix-translated-strings:
-  stage: translation
-  image: "build-and-lint"
-  needs: []
-  before_script:
-    - git config --global user.email "yunohost@yunohost.org"
-    - git config --global user.name "$GITHUB_USER"
-    - hub clone --branch ${CI_COMMIT_REF_NAME} "https://$GITHUB_TOKEN:x-oauth-basic@github.com/YunoHost/yunohost.git" github_repo
-    - cd github_repo
-  script:
-    # create a local branch that will overwrite distant one
-    - git checkout -b "ci-autofix-translated-strings-${CI_COMMIT_REF_NAME}" --no-track
-    - python3 maintenance/missing_i18n_keys.py --fix
-    - python3 maintenance/autofix_locale_format.py
-    - '[ $(git diff --ignore-blank-lines --ignore-all-space --ignore-space-at-eol --ignore-cr-at-eol | wc -l) != 0 ] || exit 0'  # stop if there is nothing to commit
-    - git commit -am "[CI] Reformat / remove stale translated strings" || true
-    - git push -f origin "ci-autofix-translated-strings-${CI_COMMIT_REF_NAME}":"ci-remove-stale-translated-strings-${CI_COMMIT_REF_NAME}"
-    - hub pull-request -m "[CI] Reformat / remove stale translated strings" -b Yunohost:$CI_COMMIT_REF_NAME -p || true # GITHUB_USER and GITHUB_TOKEN registered here https://gitlab.com/yunohost/yunohost/-/settings/ci_cd
-  only:
-    variables:
-      - $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
-    changes:
-      - locales/*

.travis.yml

@@ -0,0 +1,22 @@
+language: python
+
+matrix:
+  allow_failures:
+    - env: TOXENV=py27-lint
+    - env: TOXENV=py37-lint
+    - env: TOXENV=py37-invalidcode
+  include:
+    - python: 2.7
+      env: TOXENV=py27-lint
+    - python: 2.7
+      env: TOXENV=py27-invalidcode
+    - python: 3.7
+      env: TOXENV=py37-lint
+    - python: 3.7
+      env: TOXENV=py37-invalidcode
+
+install:
+  - pip install tox
+
+script:
+  - tox

CONTRIBUTORS.md

@@ -0,0 +1,101 @@
+YunoHost core contributors
+==========================
+
+YunoHost is built and maintained by the YunoHost project community.
+Everyone is encouraged to submit issues and changes, and to contribute in other ways -- see https://yunohost.org/contribute to find out how.
+
+--
+
+Initial YunoHost core was built by Kload & beudbeud, for YunoHost v2.
+
+Most of code was written by Kload and jerome, with help of numerous contributors.
+
+Translation is made by a bunch of lovely people all over the world.
+
+We would like to thank anyone who ever helped the YunoHost project <3
+
+
+YunoHost core Contributors
+--------------------------
+
+- Jérôme Lebleu
+- Kload
+- Laurent 'Bram' Peuch
+- Julien 'ju' Malik
+- opi
+- Aleks
+- Adrien 'beudbeud' Beudin
+- M5oul
+- Valentin 'zamentur' / 'ljf' Grimaud
+- Jocelyn Delalande
+- infertux
+- Taziden
+- ZeHiro
+- Josue-T
+- nahoj
+- a1ex
+- JimboJoe
+- vetetix
+- jellium
+- Sebastien 'sebian' Badia
+- lmangani
+- Julien Vaubourg
+- thardev
+- zimo2001
+
+
+YunoHost core Translators
+-------------------------
+
+If you want to help translation, please visit https://translate.yunohost.org/projects/yunohost/yunohost/
+
+
+### Dutch
+
+- DUBWiSE
+- Jeroen Keerl
+- marut
+
+### English
+
+- Bugsbane
+- rokaz
+
+### French
+
+- aoz roon
+- Genma
+- Jean-Baptiste Holcroft
+- Jean P.
+- Jérôme Lebleu
+- Lapineige
+- paddy
+
+
+### German
+
+- david.bartke
+- Fabian Gruber
+- Felix Bartels
+- Jeroen Keerl
+- martin kistner
+- Philip Gatzka
+
+### Hindi
+
+- Anmol
+
+### Italian
+
+- bricabrac
+- Thomas Bille
+
+### Portuguese
+
+- Deleted User
+- Trollken
+
+### Spanish
+
+- Juanu
+

README.md

@@ -5,12 +5,9 @@
 <h1 align="center">YunoHost</h1>
 
 <div align="center">
-    
-![Version](https://img.shields.io/github/v/tag/yunohost/yunohost?label=version&sort=semver)
-[![Pipeline status](https://gitlab.com/yunohost/yunohost/badges/dev/pipeline.svg)](https://gitlab.com/yunohost/yunohost/-/pipelines)
-![Test coverage](https://gitlab.com/yunohost/yunohost/badges/dev/coverage.svg)
-[![Project license](https://img.shields.io/gitlab/license/yunohost/yunohost)](https://github.com/YunoHost/yunohost/blob/dev/LICENSE)
-[![CodeQL](https://github.com/yunohost/yunohost/workflows/CodeQL/badge.svg)](https://github.com/YunoHost/yunohost/security/code-scanning)
+
+[![Build status](https://travis-ci.org/YunoHost/yunohost.svg?branch=stretch-unstable)](https://travis-ci.org/YunoHost/yunohost)
+[![GitHub license](https://img.shields.io/github/license/YunoHost/yunohost)](https://github.com/YunoHost/yunohost/blob/stretch-unstable/LICENSE)
 [![Mastodon Follow](https://img.shields.io/mastodon/follow/28084)](https://mastodon.social/@yunohost)
 
 </div>
@@ -19,49 +16,28 @@ YunoHost is an operating system aiming to simplify as much as possible the admin
 
 This repository corresponds to the core code of YunoHost, mainly written in Python and Bash.
 
-- [Project features](https://yunohost.org/whatsyunohost)
+- [Project features](https://yunohost.org/#/whatsyunohost)
 - [Project website](https://yunohost.org)
 - [Install documentation](https://yunohost.org/install)
 - [Issue tracker](https://github.com/YunoHost/issues)
 
-## Screenshots
+# Screenshots
 
 Webadmin ([Yunohost-Admin](https://github.com/YunoHost/yunohost-admin)) | Single sign-on user portal ([SSOwat](https://github.com/YunoHost/ssowat))
---- | ---
-![Web admin insterface screenshot](https://raw.githubusercontent.com/YunoHost/doc/master/images/webadmin.png) | ![User portal screenshot](https://raw.githubusercontent.com/YunoHost/doc/master/images/user_panel.png)
+--- |  ---
+![](https://raw.githubusercontent.com/YunoHost/doc/master/images/webadmin.png) | ![](https://raw.githubusercontent.com/YunoHost/doc/master/images/user_panel.png)
 
 
 ## Contributing
 
 - You can learn how to get started with developing on YunoHost by reading [this piece of documentation](https://yunohost.org/dev).
-- Come chat with us on the [dev chatroom](https://yunohost.org/chat_rooms)!
-- You can help translate YunoHost on our [translation platform](https://translate.yunohost.org/engage/yunohost/?utm_source=widget).
+- Come chat with us on the [dev chatroom](https://yunohost.org/#/chat_rooms) !
+- You can help translate YunoHost on our [translation platform](https://translate.yunohost.org/engage/yunohost/?utm_source=widget)
 
 <p align="center">
-<img alt="View of the translation rate for the different languages available in YunoHost" src="https://translate.yunohost.org/widgets/yunohost/-/core/horizontal-auto.svg" alt="Translation status" />
+<img src="https://translate.yunohost.org/widgets/yunohost/-/multi-auto.svg" alt="Translation status" />
 </p>
 
 ## License
 
-As [other components of YunoHost](https://yunohost.org/faq), this repository is licensed under GNU AGPL v3.
-
-## They support us <3
-
-We are thankful for our sponsors providing us with infrastructure and grants!
-
-<div align="center">
-<p style="margin-left:auto;margin-right:auto;">
-<a style="padding: 5px;" href="https://nlnet.nl"><img alt="NLnet Foundation" src="https://user-images.githubusercontent.com/36127788/198088570-823c40bd-7ac3-44e3-a8ee-e7a9f14b47ac.png" width="150px"/></a>
-<a style="padding: 5px;" href="https://www.ngi.eu"><img alt="Next Generation Internet" src="https://user-images.githubusercontent.com/36127788/198088663-daf587b9-fd09-4c00-aaf2-37c803939c94.png" width="130px"/></a>
-<a style="padding: 5px;" href="https://www.codelutin.com"><img alt="Code Lutin" src="https://user-images.githubusercontent.com/36127788/198088737-d37b6674-379c-4be4-9d74-b93b6ad318d1.png" width="100px"/></a>
-</p>
-<p style="margin-left:auto;margin-right:auto;">
-<a style="padding: 5px;" href="https://www.globenet.org"><img alt="Globenet" src="https://user-images.githubusercontent.com/36127788/198088794-751129ab-737d-4d99-9f35-5e01845dcdfe.png" width="150px"/></a>
-<a style="padding: 5px;" href="https://www.gitoyen.net"><img alt="Gitoyen" src="https://user-images.githubusercontent.com/36127788/198088931-f16f4af4-57ae-42e9-8d42-fb3e2d8d7ee3.png" width="150px"/></a>
-<a style="padding: 5px;" href="https://tetaneutral.net"><img alt="tetaneutral.net" src="https://user-images.githubusercontent.com/36127788/198088995-3ad9c34d-9807-4ead-934b-44df97d3c552.png" width="90px"/></a>
-<a style="padding: 5px;" href="https://ldn-fai.net"><img alt="LDN (Lorraine Data Network)" src="https://user-images.githubusercontent.com/36127788/198089086-a4089d51-9173-4081-bd2e-fa1ac3378e49.png" width="120px"/></a>
-<a style="padding: 5px;" href="https://www.nbs-system.com"><img alt="NBS System" src="https://user-images.githubusercontent.com/36127788/198089161-4cc0b7b7-bf56-4798-892e-a76112497921.png" width="130px"/></a>
-</p>
-</div>
-
-This project was funded through the [NGI0 PET](https://nlnet.nl/PET) Fund, a fund established by NLnet with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 825310. If you're interested, [check out how to apply in this video](https://media.ccc.de/v/36c3-10795-ngi_zero_a_treasure_trove_of_it_innovation)! 
+As [other components of YunoHost](https://yunohost.org/#/faq_en), this repository is licensed under GNU AGPL v3.

bin/yunohost

@@ -1,78 +1,67 @@
-#! /usr/bin/python3
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
 
 import os
 import sys
 import argparse
+
+sys.path.insert(0, "/usr/lib/moulinette/")
 import yunohost
 
 
 def _parse_cli_args():
     """Parse additional arguments for the cli"""
     parser = argparse.ArgumentParser(add_help=False)
-    parser.add_argument(
-        "--output-as",
-        choices=["json", "plain", "none"],
-        default=None,
-        help="Output result in another format",
+    parser.add_argument('--output-as',
+        choices=['json', 'plain', 'none'], default=None,
+        help="Output result in another format"
     )
-    parser.add_argument(
-        "--debug",
-        action="store_true",
-        default=False,
-        help="Log and print debug messages",
+    parser.add_argument('--debug',
+        action='store_true', default=False,
+        help="Log and print debug messages"
     )
-    parser.add_argument(
-        "--quiet", action="store_true", default=False, help="Don't produce any output"
+    parser.add_argument('--quiet',
+        action='store_true', default=False,
+        help="Don't produce any output"
     )
-    parser.add_argument(
-        "--version", action="store_true", default=False, help="Display YunoHost packages versions (alias to 'yunohost tools versions')"
-    )
-    parser.add_argument(
-        "--timeout",
-        type=int,
-        default=None,
-        help="Number of seconds before this command will timeout because it can't acquire the lock (meaning that another command is currently running), by default there is no timeout and the command will wait until it can get the lock",
+    parser.add_argument('--timeout',
+        type=int, default=None,
+        help="Number of seconds before this command will timeout because it can't acquire the lock (meaning that another command is currently running), by default there is no timeout and the command will wait until it can get the lock"
     )
     # deprecated arguments
-    parser.add_argument(
-        "--plain", action="store_true", default=False, help=argparse.SUPPRESS
+    parser.add_argument('--plain',
+        action='store_true', default=False, help=argparse.SUPPRESS
     )
-    parser.add_argument(
-        "--json", action="store_true", default=False, help=argparse.SUPPRESS
+    parser.add_argument('--json',
+        action='store_true', default=False, help=argparse.SUPPRESS
     )
 
     opts, args = parser.parse_known_args()
 
     # output compatibility
     if opts.plain:
-        opts.output_as = "plain"
+        opts.output_as = 'plain'
     elif opts.json:
-        opts.output_as = "json"
+        opts.output_as = 'json'
 
     return (parser, opts, args)
 
 
 # Stupid PATH management because sometimes (e.g. some cron job) PATH is only /usr/bin:/bin ...
-
 default_path = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 if os.environ["PATH"] != default_path:
     os.environ["PATH"] = default_path + ":" + os.environ["PATH"]
 
 # Main action ----------------------------------------------------------
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     if os.geteuid() != 0:
-        sys.stderr.write(
-            "\033[1;31mError:\033[0m yunohost command must be "
-            "run as root or with sudo.\n"
-        )
+        sys.stderr.write("\033[1;31mError:\033[0m yunohost command must be "
+                         "run as root or with sudo.\n")
         sys.exit(1)
 
     parser, opts, args = _parse_cli_args()
 
-    if opts.version:
-        args = ["tools", "versions"]
-
     # Execute the action
     yunohost.cli(
         debug=opts.debug,
@@ -80,5 +69,5 @@ if __name__ == "__main__":
         output_as=opts.output_as,
         timeout=opts.timeout,
         args=args,
-        parser=parser,
+        parser=parser
     )

bin/yunohost-api

@@ -1,52 +1,44 @@
-#! /usr/bin/python3
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
 
+import sys
 import argparse
+
+sys.path.insert(0, "/usr/lib/moulinette/")
 import yunohost
 
 # Default server configuration
-DEFAULT_HOST = "localhost"
+DEFAULT_HOST = 'localhost'
 DEFAULT_PORT = 6787
 
 
 def _parse_api_args():
     """Parse main arguments for the api"""
-    parser = argparse.ArgumentParser(
-        add_help=False,
+    parser = argparse.ArgumentParser(add_help=False,
         description="Run the YunoHost API to manage your server.",
     )
-    srv_group = parser.add_argument_group("server configuration")
-    srv_group.add_argument(
-        "-h",
-        "--host",
-        action="store",
-        default=DEFAULT_HOST,
+    srv_group = parser.add_argument_group('server configuration')
+    srv_group.add_argument('-h', '--host',
+        action='store', default=DEFAULT_HOST,
         help="Host to listen on (default: %s)" % DEFAULT_HOST,
     )
-    srv_group.add_argument(
-        "-p",
-        "--port",
-        action="store",
-        default=DEFAULT_PORT,
-        type=int,
+    srv_group.add_argument('-p', '--port',
+        action='store', default=DEFAULT_PORT, type=int,
         help="Port to listen on (default: %d)" % DEFAULT_PORT,
     )
-    glob_group = parser.add_argument_group("global arguments")
-    glob_group.add_argument(
-        "--debug",
-        action="store_true",
-        default=False,
+    glob_group = parser.add_argument_group('global arguments')
+    glob_group.add_argument('--debug',
+        action='store_true', default=False,
         help="Set log level to DEBUG",
     )
-    glob_group.add_argument(
-        "--help",
-        action="help",
-        help="Show this help message and exit",
+    glob_group.add_argument('--help',
+        action='help', help="Show this help message and exit",
     )
 
     return parser.parse_args()
 
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     opts = _parse_api_args()
     # Run the server
     yunohost.api(debug=opts.debug, host=opts.host, port=opts.port)

bin/yunomdns

@@ -1,191 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-Pythonic declaration of mDNS .local domains for YunoHost
-"""
-
-import sys
-import yaml
-from time import sleep
-from typing import List, Dict
-
-import ifaddr
-from ipaddress import ip_address
-from zeroconf import Zeroconf, ServiceInfo, ServiceBrowser
-
-
-def get_network_local_interfaces() -> Dict[str, Dict[str, List[str]]]:
-    """
-    Returns interfaces with their associated local IPs
-    """
-
-    interfaces = {
-        adapter.name: {
-            "ipv4": [
-                ip.ip
-                for ip in adapter.ips
-                if ip.is_IPv4
-                and ip_address(ip.ip).is_private
-                and not ip_address(ip.ip).is_link_local
-            ],
-            "ipv6": [
-                ip.ip[0]
-                for ip in adapter.ips
-                if ip.is_IPv6
-                and ip_address(ip.ip[0]).is_private
-                and not ip_address(ip.ip[0]).is_link_local
-            ],
-        }
-        for adapter in ifaddr.get_adapters()
-        if adapter.name != "lo"
-    }
-    return interfaces
-
-
-# Listener class, to detect duplicates on the network
-# Stores the list of servers in its list property
-class Listener:
-    def __init__(self):
-        self.list = []
-
-    def remove_service(self, zeroconf, type, name):
-        info = zeroconf.get_service_info(type, name)
-        self.list.remove(info.server)
-
-    def update_service(self, zeroconf, type, name):
-        pass
-
-    def add_service(self, zeroconf, type, name):
-        info = zeroconf.get_service_info(type, name)
-        self.list.append(info.server[:-1])
-
-
-def main() -> bool:
-    ###
-    #  CONFIG
-    ###
-
-    with open("/etc/yunohost/mdns.yml", "r") as f:
-        config = yaml.safe_load(f) or {}
-
-    required_fields = ["domains"]
-    missing_fields = [field for field in required_fields if field not in config]
-    interfaces = get_network_local_interfaces()
-
-    if missing_fields:
-        print(f"The fields {missing_fields} are required in mdns.yml")
-        return False
-
-    if "interfaces" not in config:
-        config["interfaces"] = [
-            interface
-            for interface, local_ips in interfaces.items()
-            if local_ips["ipv4"]
-        ]
-
-    if "ban_interfaces" in config:
-        config["interfaces"] = [
-            interface
-            for interface in config["interfaces"]
-            if interface not in config["ban_interfaces"]
-        ]
-
-    # Let's discover currently published .local domains accross the network
-    zc = Zeroconf()
-    listener = Listener()
-    browser = ServiceBrowser(zc, "_device-info._tcp.local.", listener)
-    sleep(2)
-    browser.cancel()
-    zc.close()
-
-    # Always attempt to publish yunohost.local
-    if "yunohost.local" not in config["domains"]:
-        config["domains"].append("yunohost.local")
-
-    def find_domain_not_already_published(domain):
-
-        # Try domain.local ... but if it's already published by another entity,
-        # try domain-2.local, domain-3.local, ...
-
-        i = 1
-        domain_i = domain
-
-        while domain_i in listener.list:
-            print(f"Uh oh, {domain_i} already exists on the network...")
-
-            i += 1
-            domain_i = domain.replace(".local", f"-{i}.local")
-
-        return domain_i
-
-    config["domains"] = [
-        find_domain_not_already_published(domain) for domain in config["domains"]
-    ]
-
-    zcs: Dict[Zeroconf, List[ServiceInfo]] = {}
-
-    for interface in config["interfaces"]:
-
-        if interface not in interfaces:
-            print(
-                f"Interface {interface} listed in config file is not present on system."
-            )
-            continue
-
-        # Only broadcast IPv4 because IPv6 is buggy ... because we ain't using python3-ifaddr >= 0.1.7
-        # Buster only ships 0.1.6
-        # Bullseye ships 0.1.7
-        # To be re-enabled once we're on bullseye...
-        # ips: List[str] = interfaces[interface]["ipv4"] + interfaces[interface]["ipv6"]
-        ips: List[str] = interfaces[interface]["ipv4"]
-
-        # If at least one IP is listed
-        if not ips:
-            continue
-
-        # Create a Zeroconf object, and store the ServiceInfos
-        zc = Zeroconf(interfaces=ips)  # type: ignore
-        zcs[zc] = []
-
-        for d in config["domains"]:
-            d_domain = d.replace(".local", "")
-            if "." in d_domain:
-                print(f"{d_domain}.local: subdomains are not supported.")
-                continue
-            # Create a ServiceInfo object for each .local domain
-            zcs[zc].append(
-                ServiceInfo(
-                    type_="_device-info._tcp.local.",
-                    name=f"{interface}: {d_domain}._device-info._tcp.local.",
-                    parsed_addresses=ips,
-                    port=80,
-                    server=f"{d}.",
-                )
-            )
-            print(f"Adding {d} with addresses {ips} on interface {interface}")
-
-    # Run registration
-    print("Registering...")
-    for zc, infos in zcs.items():
-        for info in infos:
-            zc.register_service(
-                info, allow_name_change=True, cooperating_responders=True
-            )
-
-    try:
-        print("Registered. Press Ctrl+C or stop service to stop.")
-        while True:
-            sleep(1)
-    except KeyboardInterrupt:
-        pass
-    finally:
-        print("Unregistering...")
-        for zc, infos in zcs.items():
-            zc.unregister_all_services()
-            zc.close()
-
-    return True
-
-
-if __name__ == "__main__":
-    sys.exit(0 if main() else 1)

bin/yunopaste

@@ -1,34 +1,77 @@
-#!/usr/bin/env python3
+#!/bin/bash
 
-import sys
-import requests
-import json
+set -e
+set -u
 
-SERVER_URL = "https://paste.yunohost.org"
-TIMEOUT = 3
+PASTE_URL="https://paste.yunohost.org"
 
-def create_snippet(data):
-    try:
-        url = SERVER_URL + "/documents"
-        response = requests.post(url, data=data.encode('utf-8'), timeout=TIMEOUT)
-        response.raise_for_status()
-        dockey = json.loads(response.text)['key']
-        return SERVER_URL + "/raw/" + dockey
-    except requests.exceptions.RequestException as e:
-        print("\033[31mError: {}\033[0m".format(e))
-        sys.exit(1)
+_die() {
+  printf "Error: %s\n" "$*"
+  exit 1
+}
 
+check_dependencies() {
+  curl -V > /dev/null 2>&1 || _die "This script requires curl."
+}
 
-def main():
-    output = sys.stdin.read()
+paste_data() {
+  json=$(curl -X POST -s -d "$1" "${PASTE_URL}/documents")
+  [[ -z "$json" ]] && _die "Unable to post the data to the server."
 
-    if not output:
-        print("\033[31mError: No input received from stdin.\033[0m")
-        sys.exit(1)
+  key=$(echo "$json" \
+    | python -c 'import json,sys;o=json.load(sys.stdin);print o["key"]' \
+        2>/dev/null)
+  [[ -z "$key" ]] && _die "Unable to parse the server response."
 
-    url = create_snippet(output)
+  echo "${PASTE_URL}/${key}"
+}
 
-    print("\033[32mURL: {}\033[0m".format(url))
+usage() {
+  printf "Usage: ${0} [OPTION]...
 
-if __name__ == "__main__":
-    main()
+Read from input stream and paste the data to the YunoHost
+Haste server.
+
+For example, to paste the output of the YunoHost diagnosis, you
+can simply execute the following:
+  yunohost tools diagnosis | ${0}
+
+It will return the URL where you can access the pasted data.
+
+Options:
+  -h, --help   show this help message and exit
+"
+}
+
+main() {
+  # parse options
+  while (( ${#} )); do
+    case "${1}" in
+      --help|-h)
+        usage
+        exit 0
+        ;;
+      *)
+        echo "Unknown parameter detected: ${1}" >&2
+        echo >&2
+        usage >&2
+        exit 1
+        ;;
+    esac
+
+    shift 1
+  done
+
+  # check input stream
+  read -t 0 || {
+    echo -e "Invalid usage: No input is provided.\n" >&2
+    usage
+    exit 1
+  }
+
+  paste_data "$(cat)"
+}
+
+check_dependencies
+
+main "${@}"

bin/yunoprompt

@@ -56,7 +56,7 @@ EOF
 
 echo "$LOGO_AND_FINGERPRINTS" > /etc/issue
 
-if ! groups | grep -q all_users && [[ ! -f /etc/yunohost/installed ]]
+if [[ ! -f /etc/yunohost/installed ]]
 then
     chvt 2
 
@@ -66,19 +66,19 @@ then
     echo "$LOGO_AND_FINGERPRINTS"
     cat << EOF
 ===============================================================================
-You should now proceed with YunoHost post-installation. This is where you will
-be asked for:
-  - the main domain of your server;
-  - the username and password for the first admin
+You should now proceed with Yunohost post-installation. This is where you will
+be asked for :
+  - the main domain of your server ;
+  - the administration password.
 
-You can perform this step:
-  - from your web browser, by accessing: https://yunohost.local/ or ${local_ip}
+You can perform this step :
+  - from your web browser, by accessing : https://yunohost.local/ or ${local_ip}
   - or in this terminal by answering 'yes' to the following question
 
 If this is your first time with YunoHost, it is strongly recommended to take
 time to read the administator documentation and in particular the sections
 'Finalizing your setup' and 'Getting to know YunoHost'. It is available at
-the following URL: https://yunohost.org/admindoc
+the following URL : https://yunohost.org/admindoc
 ===============================================================================
 EOF
 

conf/dnsmasq/dnsmasq.conf.tpl

@@ -1,10 +0,0 @@
-domain-needed
-expand-hosts
-localise-queries
-
-{% set interfaces = wireless_interfaces.strip().split(' ') %}
-{% for interface in interfaces %}
-interface={{ interface }}
-{% endfor %}
-resolv-file=/etc/resolv.dnsmasq.conf
-cache-size=256

conf/fail2ban/postfix-sasl.conf

@@ -1,6 +0,0 @@
-# Fail2Ban filter for postfix authentication failures
-[INCLUDES]
-before = common.conf
-[Definition]
-_daemon = postfix/smtpd
-failregex = ^%(__prefix_line)swarning: [-._\w]+\[<HOST>\]: SASL (?:LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(: [ A-Za-z0-9+/]*={0,2})?\s*$

conf/mdns/yunomdns.service

@@ -1,15 +0,0 @@
-[Unit]
-Description=YunoHost mDNS service
-Wants=network-online.target
-After=network-online.target
-
-[Service]
-User=mdns
-Group=mdns
-Type=simple
-Environment=PYTHONUNBUFFERED=1
-ExecStart=/usr/bin/yunomdns
-StandardOutput=journal
-
-[Install]
-WantedBy=default.target

conf/metronome/domain.tpl.cfg.lua

@@ -1,75 +0,0 @@
-VirtualHost "{{ domain }}"
-    enable = true
-    ssl = {
-                key = "/etc/yunohost/certs/{{ domain }}/key.pem";
-                certificate = "/etc/yunohost/certs/{{ domain }}/crt.pem";
-    }
-    authentication = "ldap2"
-    ldap = {
-         hostname      = "localhost",
-         user = {
-             basedn        = "ou=users,dc=yunohost,dc=org",
-             filter        = "(&(objectClass=posixAccount)(mail=*@{{ domain }})(permission=cn=xmpp.main,ou=permission,dc=yunohost,dc=org))",
-             usernamefield = "mail",
-             namefield     = "cn",
-             },
-    }
-
-    -- Discovery items
-    disco_items = {
-        { "muc.{{ domain }}" },
-        { "pubsub.{{ domain }}" },
-        { "jabber.{{ domain }}" },
-        { "vjud.{{ domain }}" },
-        { "xmpp-upload.{{ domain }}" },
-    };
-
---  contact_info = {
---    abuse = { "mailto:abuse@{{ domain }}", "xmpp:admin@{{ domain }}" };
---    admin = { "mailto:root@{{ domain }}", "xmpp:admin@{{ domain }}" };
---  };
-
------- Components ------
--- You can specify components to add hosts that provide special services,
--- like multi-user conferences, and transports.
-
----Set up a MUC (multi-user chat) room server
-Component "muc.{{ domain }}" "muc"
-    name = "{{ domain }} Chatrooms"
-
-    modules_enabled = {
-        "muc_limits";
-        "muc_log";
-        "muc_log_mam";
-        "muc_log_http";
-        "muc_vcard";
-    }
-
-    muc_event_rate = 0.5
-    muc_burst_factor = 10
-    room_default_config = {
-        logging = true,
-        persistent = true
-    };
-
----Set up a PubSub server
-Component "pubsub.{{ domain }}" "pubsub"
-    name = "{{ domain }} Publish/Subscribe"
-
-    unrestricted_node_creation = true -- Anyone can create a PubSub node (from any server)
-
----Set up a HTTP Upload service
-Component "xmpp-upload.{{ domain }}" "http_upload"
-    name = "{{ domain }} Sharing Service"
-
-    http_file_path = "/var/xmpp-upload/{{ domain }}/upload"
-    http_external_url = "https://xmpp-upload.{{ domain }}:443"
-    http_file_base_path = "/upload"
-    http_file_size_limit = 6*1024*1024
-    http_file_quota = 60*1024*1024
-    http_upload_file_size_limit = 100 * 1024 * 1024 -- bytes
-    http_upload_quota = 10 * 1024 * 1024 * 1024 -- bytes
-
----Set up a VJUD service
-Component "vjud.{{ domain }}" "vjud"
-    vjud_disco_name = "{{ domain }} User Directory"

conf/metronome/metronome.cfg.lua

@@ -1,123 +0,0 @@
--- ** Metronome's config file example **
---
--- The format is exactly equal to Prosody's:
---
--- Lists are written { "like", "this", "one" }
--- Lists can also be of { 1, 2, 3 } numbers, etc.
--- Either commas, or semi-colons; may be used as seperators.
---
--- A table is a list of values, except each value has a name. An
--- example would be:
---
--- ssl = { key = "keyfile.key", certificate = "certificate.cert" }
---
--- Tip: You can check that the syntax of this file is correct when you have finished
--- by running: luac -p metronome.cfg.lua
--- If there are any errors, it will let you know what and where they are, otherwise it
--- will keep quiet.
-
--- Global settings go in this section
-
--- This is the list of modules Metronome will load on startup.
--- It looks for mod_modulename.lua in the plugins folder, so make sure that exists too.
-
-modules_enabled = {
-    -- Generally required
-        "roster"; -- Allow users to have a roster. Recommended.
-        "saslauth"; -- Authentication for clients. Recommended if you want to log in.
-        "tls"; -- Add support for secure TLS on c2s/s2s connections
-        "disco"; -- Service discovery
-
-    -- Not essential, but recommended
-        "private"; -- Private XML storage (for room bookmarks, etc.)
-        "vcard"; -- Allow users to set vCards
-        "pep"; -- Allows setting of mood, tune, etc.
-        "pubsub";  -- Publish-subscribe XEP-0060
-        "posix"; -- POSIX functionality, sends server to background, enables syslog, etc.
-        "bidi"; -- Enables Bidirectional Server-to-Server Streams.
-
-    -- Nice to have
-        "version"; -- Replies to server version requests
-        "uptime"; -- Report how long server has been running
-        "time"; -- Let others know the time here on this server
-        "ping"; -- Replies to XMPP pings with pongs
-        "register"; -- Allow users to register on this server using a client and change passwords
-        "stream_management"; -- Allows clients and servers to use Stream Management
-        "stanza_optimizations"; -- Allows clients to use Client State Indication and SIFT
-        "message_carbons"; -- Allows clients to enable carbon copies of messages
-        "mam"; -- Enable server-side message archives using Message Archive Management
-        "push"; -- Enable Push Notifications via PubSub using XEP-0357
-        "lastactivity"; -- Enables clients to know the last presence status of an user
-        "adhoc_cm"; -- Allow to set client certificates to login through SASL External via adhoc
-        "admin_adhoc"; -- administration adhoc commands
-        "bookmarks"; -- XEP-0048 Bookmarks synchronization between PEP and Private Storage
-        "sec_labels"; -- Allows to use a simplified version XEP-0258 Security Labels and related ACDFs.
-        "privacy"; -- Add privacy lists and simple blocking command support
-
-        -- Other specific functionality
-        --"admin_telnet"; -- administration console, telnet to port 5582
-        --"admin_web"; -- administration web interface
-        "bosh"; -- Enable support for BOSH clients, aka "XMPP over Bidirectional Streams over Synchronous HTTP"
-        --"compression"; -- Allow clients to enable Stream Compression
-        --"spim_block"; -- Require authorization via OOB form for messages from non-contacts and block unsollicited messages
-        --"gate_guard"; -- Enable config-based blacklisting and hit-based auto-banning features
-        --"incidents_handling"; -- Enable Incidents Handling support (can be administered via adhoc commands)
-        --"server_presence"; -- Enables Server Buddies extension support
-        --"service_directory"; -- Enables Service Directories extension support
-        --"public_service"; -- Enables Server vCard support for public services in directories and advertises in features
-        --"register_api"; -- Provides secure API for both Out-Of-Band and In-Band registration for E-Mail verification
-        "websocket"; -- Enable support for WebSocket clients, aka "XMPP over WebSockets"
-};
-
--- Server PID
-pidfile = "/var/run/metronome/metronome.pid"
-
--- HTTP server
-http_ports = { 5290 }
-http_interfaces = { "127.0.0.1", "::1" }
-
---https_ports = { 5291 }
---https_interfaces = { "127.0.0.1", "::1" }
-
--- Enable IPv6
-use_ipv6 = true
-
--- BOSH configuration (mod_bosh)
-consider_bosh_secure = true
-cross_domain_bosh = true
-
--- WebSocket configuration (mod_websocket)
-consider_websocket_secure = true
-cross_domain_websocket = true
-
--- Disable account creation by default, for security
-allow_registration = false
-
--- Use LDAP storage backend for all stores
-storage = "ldap"
-
--- stanza optimization
-csi_config_queue_all_muc_messages_but_mentions = false;
-
-
--- Logging configuration
-log = {
-    info = "/var/log/metronome/metronome.log"; -- Change 'info' to 'debug' for verbose logging
-    error = "/var/log/metronome/metronome.err";
-    -- "*syslog"; -- Uncomment this for logging to syslog
-    -- "*console"; -- Log to the console, useful for debugging with daemonize=false
-}
-
------- Components ------
--- You can specify components to add hosts that provide special services,
--- like multi-user conferences, and transports.
-
----Set up a local BOSH service
-Component "localhost" "http"
-    modules_enabled = { "bosh" }
-
------------ Virtual hosts -----------
--- You need to add a VirtualHost entry for each domain you wish Metronome to serve.
--- Settings under each VirtualHost entry apply *only* to that host.
-
-Include "conf.d/*.cfg.lua"

conf/metronome/modules/mod_auth_ldap2.lua

@@ -1,90 +0,0 @@
--- vim:sts=4 sw=4
-
--- Metronome IM
--- Copyright (C) 2008-2010 Matthew Wild
--- Copyright (C) 2008-2010 Waqas Hussain
--- Copyright (C) 2012 Rob Hoelz
--- Copyright (C) 2015 YUNOHOST.ORG
---
--- This project is MIT/X11 licensed. Please see the
--- COPYING file in the source package for more information.
---
--- https://github.com/YunoHost/yunohost-config-metronome/blob/unstable/lib/modules/mod_auth_ldap2.lua
--- adapted to use common LDAP store on Metronome
-
-local ldap     = module:require 'ldap';
-local new_sasl = require 'util.sasl'.new;
-local jsplit   = require 'util.jid'.split;
-
-local log = module._log
-
-if not ldap then
-    return;
-end
-
-function new_default_provider(host)
-    local provider = { name = "ldap2" };
-    log("debug", "initializing ldap2 authentication provider for host '%s'", host);
-
-    function provider.test_password(username, password)
-        return ldap.bind(username, password);
-    end
-
-    function provider.user_exists(username)
-        local params = ldap.getparams()
-
-        local filter = ldap.filter.combine_and(params.user.filter, params.user.usernamefield .. '=' .. username);
-        if params.user.usernamefield == 'mail' then
-            filter = ldap.filter.combine_and(params.user.filter, 'mail=' .. username .. '@*');
-        end
-
-        return ldap.singlematch {
-            base   = params.user.basedn,
-            filter = filter,
-        };
-    end
-
-    function provider.get_password(username)
-        return nil, "Passwords unavailable for LDAP.";
-    end
-
-    function provider.set_password(username, password)
-        return nil, "Passwords unavailable for LDAP.";
-    end
-
-    function provider.create_user(username, password)
-        return nil, "Account creation/modification not available with LDAP.";
-    end
-
-    function provider.get_sasl_handler(session)
-        local testpass_authentication_profile = {
-            session = session,
-            plain_test = function(sasl, username, password, realm)
-                return provider.test_password(username, password), true;
-            end,
-            order = { "plain_test" },
-        };
-        return new_sasl(module.host, testpass_authentication_profile);
-    end
-
-    function provider.is_admin(jid)
-        local admin_config = ldap.getparams().admin;
-
-        if not admin_config then
-            return;
-        end
-
-        local ld       = ldap:getconnection();
-        local username = jsplit(jid);
-        local filter   = ldap.filter.combine_and(admin_config.filter, admin_config.namefield .. '=' .. username);
-
-        return ldap.singlematch {
-            base   = admin_config.basedn,
-            filter = filter,
-        };
-    end
-
-    return provider;
-end
-
-module:add_item("auth-provider", new_default_provider(module.host));

conf/metronome/modules/mod_legacyauth.lua

@@ -1,86 +0,0 @@
--- Prosody IM
--- Copyright (C) 2008-2010 Matthew Wild
--- Copyright (C) 2008-2010 Waqas Hussain
---
--- This project is MIT/X11 licensed. Please see the
--- COPYING file in the source package for more information.
---
-
-
-
-local st = require "util.stanza";
-local t_concat = table.concat;
-
-local secure_auth_only = module:get_option("c2s_require_encryption")
-    or module:get_option("require_encryption")
-    or not(module:get_option("allow_unencrypted_plain_auth"));
-
-local sessionmanager = require "core.sessionmanager";
-local usermanager = require "core.usermanager";
-local nodeprep = require "util.encodings".stringprep.nodeprep;
-local resourceprep = require "util.encodings".stringprep.resourceprep;
-
-module:add_feature("jabber:iq:auth");
-module:hook("stream-features", function(event)
-    local origin, features = event.origin, event.features;
-    if secure_auth_only and not origin.secure then
-        -- Sorry, not offering to insecure streams!
-        return;
-    elseif not origin.username then
-        features:tag("auth", {xmlns='http://jabber.org/features/iq-auth'}):up();
-    end
-end);
-
-module:hook("stanza/iq/jabber:iq:auth:query", function(event)
-    local session, stanza = event.origin, event.stanza;
-
-    if session.type ~= "c2s_unauthed" then
-        (session.sends2s or session.send)(st.error_reply(stanza, "cancel", "service-unavailable", "Legacy authentication is only allowed for unauthenticated client connections."));
-        return true;
-    end
-
-    if secure_auth_only and not session.secure then
-        session.send(st.error_reply(stanza, "modify", "not-acceptable", "Encryption (SSL or TLS) is required to connect to this server"));
-        return true;
-    end
-
-    local username = stanza.tags[1]:child_with_name("username");
-    local password = stanza.tags[1]:child_with_name("password");
-    local resource = stanza.tags[1]:child_with_name("resource");
-    if not (username and password and resource) then
-        local reply = st.reply(stanza);
-        session.send(reply:query("jabber:iq:auth")
-            :tag("username"):up()
-            :tag("password"):up()
-            :tag("resource"):up());
-    else
-        username, password, resource = t_concat(username), t_concat(password), t_concat(resource);
-        username = nodeprep(username);
-        resource = resourceprep(resource)
-        if not (username and resource) then
-            session.send(st.error_reply(stanza, "modify", "bad-request"));
-            return true;
-        end
-        if usermanager.test_password(username, session.host, password) then
-            -- Authentication successful!
-            local success, err = sessionmanager.make_authenticated(session, username);
-            if success then
-                local err_type, err_msg;
-                success, err_type, err, err_msg = sessionmanager.bind_resource(session, resource);
-                if not success then
-                    session.send(st.error_reply(stanza, err_type, err, err_msg));
-                    session.username, session.type = nil, "c2s_unauthed"; -- FIXME should this be placed in sessionmanager?
-                    return true;
-                elseif resource ~= session.resource then -- server changed resource, not supported by legacy auth
-                    session.send(st.error_reply(stanza, "cancel", "conflict", "The requested resource could not be assigned to this session."));
-                    session:close(); -- FIXME undo resource bind and auth instead of closing the session?
-                    return true;
-                end
-            end
-            session.send(st.reply(stanza));
-        else
-            session.send(st.error_reply(stanza, "auth", "not-authorized"));
-        end
-    end
-    return true;
-end);

conf/metronome/modules/mod_storage_ldap.lua

@@ -1,243 +0,0 @@
--- vim:sts=4 sw=4
-
--- Metronome IM
--- Copyright (C) 2008-2010 Matthew Wild
--- Copyright (C) 2008-2010 Waqas Hussain
--- Copyright (C) 2012 Rob Hoelz
--- Copyright (C) 2015 YUNOHOST.ORG
---
--- This project is MIT/X11 licensed. Please see the
--- COPYING file in the source package for more information.
-
-----------------------------------------
--- Constants and such --
-----------------------------------------
-
-local setmetatable = setmetatable;
-
-local get_config = require "core.configmanager".get;
-local ldap       = module:require 'ldap';
-local vcardlib   = module:require 'vcard';
-local st         = require 'util.stanza';
-local gettime    = require 'socket'.gettime;
-
-local log = module._log
-
-if not ldap then
-    return;
-end
-
-local CACHE_EXPIRY = 300;
-
-----------------------------------------
--- Utility Functions --
-----------------------------------------
-
-local function ldap_record_to_vcard(record, format)
-    return vcardlib.create {
-        record = record,
-        format = format,
-    }
-end
-
-local get_alias_for_user;
-
-do
-    local user_cache;
-    local last_fetch_time;
-
-    local function populate_user_cache()
-        local user_c = get_config(module.host, 'ldap').user;
-        if not user_c then return; end
-
-        local ld = ldap.getconnection();
-
-        local usernamefield = user_c.usernamefield;
-        local namefield     = user_c.namefield;
-
-        user_cache = {};
-
-        for _, attrs in ld:search { base = user_c.basedn, scope = 'onelevel', filter = user_c.filter } do
-            user_cache[attrs[usernamefield]] = attrs[namefield];
-        end
-        last_fetch_time = gettime();
-    end
-
-    function get_alias_for_user(user)
-        if last_fetch_time and last_fetch_time + CACHE_EXPIRY < gettime() then
-            user_cache = nil;
-        end
-        if not user_cache then
-            populate_user_cache();
-        end
-        return user_cache[user];
-    end
-end
-
-----------------------------------------
--- Base LDAP store class --
-----------------------------------------
-
-local function ldap_store(config)
-    local self = {};
-    local config = config;
-
-    function self:get(username)
-        return nil, "Data getting is not available for this storage backend";
-    end
-
-    function self:set(username, data)
-        return nil, "Data setting is not available for this storage backend";
-    end
-
-    return self;
-end
-
-local adapters = {};
-
-----------------------------------------
--- Roster Storage Implementation --
-----------------------------------------
-
-adapters.roster = function (config)
-    -- Validate configuration requirements
-    if not config.groups then return nil; end
-
-    local self = ldap_store(config)
-
-    function self:get(username)
-        local ld = ldap.getconnection();
-        local contacts = {};
-
-        local memberfield = config.groups.memberfield;
-        local namefield   = config.groups.namefield;
-        local filter      = memberfield .. '=' .. tostring(username);
-
-        local groups = {};
-        for _, config in ipairs(config.groups) do
-            groups[ config[namefield] ] = config.name;
-        end
-
-        log("debug", "Found %d group(s) for user %s", select('#', groups), username)
-
-        -- XXX this kind of relies on the way we do groups at INOC
-        for _, attrs in ld:search { base = config.groups.basedn, scope = 'onelevel', filter = filter } do
-            if groups[ attrs[namefield] ] then
-                local members = attrs[memberfield];
-
-                for _, user in ipairs(members) do
-                    if user ~= username then
-                        local jid    = user .. '@' .. module.host;
-                        local record = contacts[jid];
-
-                        if not record then
-                            record = {
-                                subscription = 'both',
-                                groups       = {},
-                                name         = get_alias_for_user(user),
-                            };
-                            contacts[jid] = record;
-                        end
-
-                        record.groups[ groups[ attrs[namefield] ] ] = true;
-                    end
-                end
-            end
-        end
-
-        return contacts;
-    end
-
-    function self:set(username, data)
-        log("warn", "Setting data in Roster LDAP storage is not supported yet")
-        return nil, "not supported";
-    end
-
-    return self;
-end
-
-----------------------------------------
--- vCard Storage Implementation --
-----------------------------------------
-
-adapters.vcard = function (config)
-    -- Validate configuration requirements
-    if not config.vcard_format or not config.user then return nil; end
-
-    local self = ldap_store(config)
-
-    function self:get(username)
-        local ld     = ldap.getconnection();
-        local filter = config.user.usernamefield .. '=' .. tostring(username);
-
-        log("debug", "Retrieving vCard for user '%s'", username);
-
-        local match = ldap.singlematch {
-            base   = config.user.basedn,
-            filter = filter,
-        };
-        if match then
-            match.jid = username .. '@' .. module.host
-            return st.preserialize(ldap_record_to_vcard(match, config.vcard_format));
-        else
-            return nil, "username not found";
-        end
-    end
-
-    function self:set(username, data)
-        log("warn", "Setting data in vCard LDAP storage is not supported yet")
-        return nil, "not supported";
-    end
-
-    return self;
-end
-
-----------------------------------------
--- Driver Definition --
-----------------------------------------
-
-cache = {};
-
-local driver = { name = "ldap" };
-
-function driver:open(store)
-    log("debug", "Opening ldap storage backend for host '%s' and store '%s'", module.host, store);
-
-    if not cache[module.host] then
-        log("debug", "Caching adapters for the host '%s'", module.host);
-
-        local ad_config = get_config(module.host, "ldap");
-        local ad_cache  = {};
-        for k, v in pairs(adapters) do
-            ad_cache[k] = v(ad_config);
-        end
-
-        cache[module.host] = ad_cache;
-    end
-
-    local adapter = cache[module.host][store];
-
-    if not adapter then
-        log("info", "Unavailable adapter for store '%s'", store);
-        return nil, "unsupported-store";
-    end
-    return adapter;
-end
-
-function driver:stores(username, type, pattern)
-    return nil, "not implemented";
-end
-
-function driver:store_exists(username, type)
-    return nil, "not implemented";
-end
-
-function driver:purge(username)
-    return nil, "not implemented";
-end
-
-function driver:nodes(type)
-    return nil, "not implemented";
-end
-
-module:add_item("data-driver", driver);

conf/nginx/plain/yunohost_http_errors.conf.inc

@@ -1,7 +0,0 @@
-error_page 502 /502.html;
-
-location = /502.html {
-
-        root /usr/share/yunohost/html/;
-
-}

conf/nginx/redirect_to_admin.conf

@@ -1,3 +0,0 @@
-location / {
-    return 302 https://$host/yunohost/admin;
-}

conf/nginx/yunohost_admin.conf.inc

@@ -1,30 +0,0 @@
-# Avoid the nginx path/alias traversal weakness ( #1037 )
-rewrite ^/yunohost/admin$ /yunohost/admin/ permanent;
-
-location /yunohost/admin/ {
-    alias /usr/share/yunohost/admin/;
-    default_type text/html;
-    index index.html;
-
-    {% if webadmin_allowlist_enabled == "True" %}
-    {% if webadmin_allowlist.strip() -%}
-    {% for ip in webadmin_allowlist.strip().split(',') -%}
-    allow {{ ip.strip() }};
-    {% endfor -%}
-    {% endif -%}
-    deny all;
-    {% endif %}
-
-    location = /yunohost/admin/index.html {
-        etag off;
-        expires off;
-        more_set_headers "Cache-Control: no-store, no-cache, must-revalidate";
-    }
-
-    location /yunohost/admin/applogos/ {
-        alias /usr/share/yunohost/applogos/;
-    }
-
-    more_set_headers "Content-Security-Policy: upgrade-insecure-requests; default-src 'self'; connect-src 'self' https://paste.yunohost.org wss://$host; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-eval'; object-src 'none'; img-src 'self' data:;";
-    more_set_headers "Content-Security-Policy-Report-Only:";
-}

conf/postfix/plain/ldap-groups.cf

@@ -1,7 +0,0 @@
-server_host = localhost
-server_port = 389
-search_base = dc=yunohost,dc=org
-query_filter = (&(objectClass=groupOfNamesYnh)(mail=%s))
-scope = sub
-result_attribute = memberUid, mail
-terminal_result_attribute = memberUid

conf/postfix/sni

@@ -1,4 +0,0 @@
-# This maps domain to certificates to properly handle multi-domain context
-# (also we need a comment in this file such that it's never empty to prevent regenconf issues)
-{% for domain in domain_list.split() %}{{ domain }} /etc/yunohost/certs/{{ domain }}/key.pem /etc/yunohost/certs/{{ domain }}/crt.pem
-{% endfor %}

conf/rspamd/redis.conf

@@ -1,2 +0,0 @@
-# set redis server
-servers = "127.0.0.1";

conf/slapd/db_init.ldif

@@ -1,107 +0,0 @@
-dn: dc=yunohost,dc=org
-objectClass: top
-objectClass: dcObject
-objectClass: organization
-o: yunohost.org
-dc: yunohost
-
-dn: ou=users,dc=yunohost,dc=org
-objectClass: organizationalUnit
-objectClass: top
-ou: users
-
-dn: ou=domains,dc=yunohost,dc=org
-objectClass: organizationalUnit
-objectClass: top
-ou: domains
-
-dn: ou=apps,dc=yunohost,dc=org
-objectClass: organizationalUnit
-objectClass: top
-ou: apps
-
-dn: ou=permission,dc=yunohost,dc=org
-objectClass: organizationalUnit
-objectClass: top
-ou: permission
-
-dn: ou=groups,dc=yunohost,dc=org
-objectClass: organizationalUnit
-objectClass: top
-ou: groups
-
-dn: cn=admins,ou=sudo,dc=yunohost,dc=org
-cn: admins
-objectClass: sudoRole
-objectClass: top
-sudoCommand: ALL
-sudoUser: %admins
-sudoHost: ALL
-
-dn: ou=sudo,dc=yunohost,dc=org
-objectClass: organizationalUnit
-objectClass: top
-ou: sudo
-
-dn: cn=admins,ou=groups,dc=yunohost,dc=org
-objectClass: posixGroup
-objectClass: top
-objectClass: groupOfNamesYnh
-gidNumber: 4001
-cn: admins
-
-dn: cn=all_users,ou=groups,dc=yunohost,dc=org
-objectClass: posixGroup
-objectClass: groupOfNamesYnh
-gidNumber: 4002
-cn: all_users
-permission: cn=mail.main,ou=permission,dc=yunohost,dc=org
-permission: cn=xmpp.main,ou=permission,dc=yunohost,dc=org
-
-dn: cn=visitors,ou=groups,dc=yunohost,dc=org
-objectClass: posixGroup
-objectClass: groupOfNamesYnh
-gidNumber: 4003
-cn: visitors
-
-dn: cn=mail.main,ou=permission,dc=yunohost,dc=org
-groupPermission: cn=all_users,ou=groups,dc=yunohost,dc=org
-cn: mail.main
-objectClass: posixGroup
-objectClass: permissionYnh
-isProtected: TRUE
-label: E-mail
-gidNumber: 5001
-showTile: FALSE
-authHeader: FALSE
-
-dn: cn=xmpp.main,ou=permission,dc=yunohost,dc=org
-groupPermission: cn=all_users,ou=groups,dc=yunohost,dc=org
-cn: xmpp.main
-objectClass: posixGroup
-objectClass: permissionYnh
-isProtected: TRUE
-label: XMPP
-gidNumber: 5002
-showTile: FALSE
-authHeader: FALSE
-
-dn: cn=ssh.main,ou=permission,dc=yunohost,dc=org
-cn: ssh.main
-objectClass: posixGroup
-objectClass: permissionYnh
-isProtected: TRUE
-label: SSH
-gidNumber: 5003
-showTile: FALSE
-authHeader: FALSE
-
-dn: cn=sftp.main,ou=permission,dc=yunohost,dc=org
-cn: sftp.main
-objectClass: posixGroup
-objectClass: permissionYnh
-isProtected: TRUE
-label: SFTP
-gidNumber: 5004
-showTile: FALSE
-authHeader: FALSE

conf/yunohost/proc-hidepid.service

@@ -1,14 +0,0 @@
-[Unit]
-Description=Mounts /proc with hidepid=2
-DefaultDependencies=no
-Before=sysinit.target
-Requires=local-fs.target
-After=local-fs.target
-
-[Service]
-Type=oneshot
-ExecStart=/bin/mount -o remount,nosuid,nodev,noexec,hidepid=2 /proc
-RemainAfterExit=yes
-
-[Install]
-WantedBy=sysinit.target

conf/yunohost/yunohost-api.service

@@ -1,13 +0,0 @@
-[Unit]
-Description=YunoHost API Server
-After=network.target
-
-[Service]
-Type=simple
-ExecStart=/usr/bin/yunohost-api
-Restart=always
-RestartSec=5
-TimeoutStopSec=30
-
-[Install]
-WantedBy=multi-user.target

data/actionsmap/yunohost_completion.py

@@ -0,0 +1,127 @@
+"""
+Simple automated generation of a bash_completion file
+for yunohost command from the actionsmap.
+
+Generates a bash completion file assuming the structure
+`yunohost category action`
+adds `--help` at the end if one presses [tab] again.
+
+author: Christophe Vuillot
+"""
+import os
+import yaml
+
+THIS_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+ACTIONSMAP_FILE = THIS_SCRIPT_DIR + '/yunohost.yml'
+BASH_COMPLETION_FILE = THIS_SCRIPT_DIR + '/../bash-completion.d/yunohost'
+
+
+def get_dict_actions(OPTION_SUBTREE, category):
+    ACTIONS = [action for action in OPTION_SUBTREE[category]["actions"].keys()
+               if not action.startswith('_')]
+    ACTIONS_STR = '{}'.format(' '.join(ACTIONS))
+
+    DICT = {"actions_str": ACTIONS_STR}
+
+    return DICT
+
+
+with open(ACTIONSMAP_FILE, 'r') as stream:
+
+    # Getting the dictionary containning what actions are possible per category
+    OPTION_TREE = yaml.load(stream)
+
+    CATEGORY = [category for category in OPTION_TREE.keys() if not category.startswith('_')]
+
+    CATEGORY_STR = '{}'.format(' '.join(CATEGORY))
+    ACTIONS_DICT = {}
+    for category in CATEGORY:
+        ACTIONS_DICT[category] = get_dict_actions(OPTION_TREE, category)
+
+        ACTIONS_DICT[category]["subcategories"] = {}
+        ACTIONS_DICT[category]["subcategories_str"] = ""
+
+        if "subcategories" in OPTION_TREE[category].keys():
+            SUBCATEGORIES = [subcategory for subcategory in OPTION_TREE[category]["subcategories"].keys()]
+
+            SUBCATEGORIES_STR = '{}'.format(' '.join(SUBCATEGORIES))
+
+            ACTIONS_DICT[category]["subcategories_str"] = SUBCATEGORIES_STR
+
+            for subcategory in SUBCATEGORIES:
+                ACTIONS_DICT[category]["subcategories"][subcategory] = get_dict_actions(OPTION_TREE[category]["subcategories"], subcategory)
+
+    with open(BASH_COMPLETION_FILE, 'w') as generated_file:
+
+        # header of the file
+        generated_file.write('#\n')
+        generated_file.write('# completion for yunohost\n')
+        generated_file.write('# automatically generated from the actionsmap\n')
+        generated_file.write('#\n\n')
+
+        # Start of the completion function
+        generated_file.write('_yunohost()\n')
+        generated_file.write('{\n')
+
+        # Defining local variable for previously and currently typed words
+        generated_file.write('\tlocal cur prev opts narg\n')
+        generated_file.write('\tCOMPREPLY=()\n\n')
+        generated_file.write('\t# the number of words already typed\n')
+        generated_file.write('\tnarg=${#COMP_WORDS[@]}\n\n')
+        generated_file.write('\t# the current word being typed\n')
+        generated_file.write('\tcur="${COMP_WORDS[COMP_CWORD]}"\n\n')
+
+        # If one is currently typing a category then match with the category list
+        generated_file.write('\t# If one is currently typing a category,\n')
+        generated_file.write('\t# match with categorys\n')
+        generated_file.write('\tif [[ $narg == 2 ]]; then\n')
+        generated_file.write('\t\topts="{}"\n'.format(CATEGORY_STR))
+        generated_file.write('\tfi\n\n')
+
+        # If one is currently typing an action then match with the action list
+        # of the previously typed category
+        generated_file.write('\t# If one already typed a category,\n')
+        generated_file.write('\t# match the actions or the subcategories of that category\n')
+        generated_file.write('\tif [[ $narg == 3 ]]; then\n')
+        generated_file.write('\t\t# the category typed\n')
+        generated_file.write('\t\tcategory="${COMP_WORDS[1]}"\n\n')
+        for category in CATEGORY:
+            generated_file.write('\t\tif [[ $category == "{}" ]]; then\n'.format(category))
+            generated_file.write('\t\t\topts="{} {}"\n'.format(ACTIONS_DICT[category]["actions_str"], ACTIONS_DICT[category]["subcategories_str"]))
+            generated_file.write('\t\tfi\n')
+        generated_file.write('\tfi\n\n')
+
+        generated_file.write('\t# If one already typed an action or a subcategory,\n')
+        generated_file.write('\t# match the actions of that subcategory\n')
+        generated_file.write('\tif [[ $narg == 4 ]]; then\n')
+        generated_file.write('\t\t# the category typed\n')
+        generated_file.write('\t\tcategory="${COMP_WORDS[1]}"\n\n')
+        generated_file.write('\t\t# the action or the subcategory typed\n')
+        generated_file.write('\t\taction_or_subcategory="${COMP_WORDS[2]}"\n\n')
+        for category in CATEGORY:
+            if len(ACTIONS_DICT[category]["subcategories"]):
+                generated_file.write('\t\tif [[ $category == "{}" ]]; then\n'.format(category))
+                for subcategory in ACTIONS_DICT[category]["subcategories"]:
+                    generated_file.write('\t\t\tif [[ $action_or_subcategory == "{}" ]]; then\n'.format(subcategory))
+                    generated_file.write('\t\t\t\topts="{}"\n'.format(ACTIONS_DICT[category]["subcategories"][subcategory]["actions_str"]))
+                    generated_file.write('\t\t\tfi\n')
+                generated_file.write('\t\tfi\n')
+        generated_file.write('\tfi\n\n')
+
+        # If both category and action have been typed or the category
+        # was not recognized propose --help (only once)
+        generated_file.write('\t# If no options were found propose --help\n')
+        generated_file.write('\tif [ -z "$opts" ]; then\n')
+        generated_file.write('\t\tprev="${COMP_WORDS[COMP_CWORD-1]}"\n\n')
+        generated_file.write('\t\tif [[ $prev != "--help" ]]; then\n')
+        generated_file.write('\t\t\topts=( --help )\n')
+        generated_file.write('\t\tfi\n')
+        generated_file.write('\tfi\n')
+
+        # generate the completion list from the possible options
+        generated_file.write('\tCOMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )\n')
+        generated_file.write('\treturn 0\n')
+        generated_file.write('}\n\n')
+
+        # Add the function to bash completion
+        generated_file.write('complete -F _yunohost yunohost')

data/bash-completion.d/yunohost

@@ -0,0 +1,3 @@
+# This file is automatically generated
+# during Debian's package build by the script
+# data/actionsmap/yunohost_completion.py

data/helpers

@@ -0,0 +1,8 @@
+# -*- shell-script -*-
+
+readonly XTRACE_ENABLE=$(set +o | grep xtrace) # This is a trick to later only restore set -x if it was set when calling this script
+set +x
+for helper in $(run-parts --list /usr/share/yunohost/helpers.d 2>/dev/null) ; do
+    [ -r $helper ] && . $helper || true
+done
+eval "$XTRACE_ENABLE"

data/helpers.d/apt

@@ -12,21 +12,25 @@ ynh_wait_dpkg_free() {
     local try
     set +o xtrace # set +x
     # With seq 1 17, timeout will be almost 30 minutes
-    for try in $(seq 1 17); do
+    for try in `seq 1 17`
+    do
         # Check if /var/lib/dpkg/lock is used by another process
-        if lsof /var/lib/dpkg/lock > /dev/null; then
+        if lsof /var/lib/dpkg/lock > /dev/null
+        then
             echo "apt is already in use..."
             # Sleep an exponential time at each round
-            sleep $((try * try))
+            sleep $(( try * try ))
         else
             # Check if dpkg hasn't been interrupted and is fully available.
             # See this for more information: https://sources.debian.org/src/apt/1.4.9/apt-pkg/deb/debsystem.cc/#L141-L174
             local dpkg_dir="/var/lib/dpkg/updates/"
 
             # For each file in $dpkg_dir
-            while read dpkg_file <&9; do
+            while read dpkg_file <&9
+            do
                 # Check if the name of this file contains only numbers.
-                if echo "$dpkg_file" | grep --perl-regexp --quiet "^[[:digit:]]+$"; then
+                if echo "$dpkg_file" | grep --perl-regexp --quiet "^[[:digit:]]+$"
+                then
                     # If so, that a remaining of dpkg.
                     ynh_print_err "dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem."
                     set -o xtrace # set -x
@@ -43,31 +47,29 @@ ynh_wait_dpkg_free() {
 
 # Check either a package is installed or not
 #
-# example: ynh_package_is_installed --package=yunohost && echo "installed"
+# example: ynh_package_is_installed --package=yunohost && echo "ok"
 #
 # usage: ynh_package_is_installed --package=name
 # | arg: -p, --package=     - the package name to check
-# | ret: 0 if the package is installed, 1 else.
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_package_is_installed() {
     # Declare an array to define the options of this helper.
     local legacy_args=p
-    local -A args_array=([p]=package=)
+    local -A args_array=( [p]=package= )
     local package
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    dpkg-query --show --showformat='${Status}' "$package" 2> /dev/null \
-        | grep --count "ok installed" &> /dev/null
+    ynh_wait_dpkg_free
+    dpkg-query --show --showformat='${Status}' "$package" 2>/dev/null \
+        | grep --count "ok installed" &>/dev/null
 }
 
 # Get the version of an installed package
 #
 # example: version=$(ynh_package_version --package=yunohost)
 #
-# [internal]
-#
 # usage: ynh_package_version --package=name
 # | arg: -p, --package=     - the package name to get version
 # | ret: the version or an empty string
@@ -76,13 +78,14 @@ ynh_package_is_installed() {
 ynh_package_version() {
     # Declare an array to define the options of this helper.
     local legacy_args=p
-    local -A args_array=([p]=package=)
+    local -A args_array=( [p]=package= )
     local package
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    if ynh_package_is_installed "$package"; then
-        dpkg-query --show --showformat='${Version}' "$package" 2> /dev/null
+    if ynh_package_is_installed "$package"
+    then
+        dpkg-query --show --showformat='${Version}' "$package" 2>/dev/null
     else
         echo ''
     fi
@@ -102,8 +105,6 @@ ynh_apt() {
 
 # Update package index files
 #
-# [internal]
-#
 # usage: ynh_package_update
 #
 # Requires YunoHost version 2.2.4 or higher.
@@ -113,8 +114,6 @@ ynh_package_update() {
 
 # Install package(s)
 #
-# [internal]
-#
 # usage: ynh_package_install name [name [...]]
 # | arg: name - the package name to install
 #
@@ -126,8 +125,6 @@ ynh_package_install() {
 
 # Remove package(s)
 #
-# [internal]
-#
 # usage: ynh_package_remove name [name [...]]
 # | arg: name - the package name to remove
 #
@@ -138,8 +135,6 @@ ynh_package_remove() {
 
 # Remove package(s) and their uneeded dependencies
 #
-# [internal]
-#
 # usage: ynh_package_autoremove name [name [...]]
 # | arg: name - the package name to remove
 #
@@ -150,8 +145,6 @@ ynh_package_autoremove() {
 
 # Purge package(s) and their uneeded dependencies
 #
-# [internal]
-#
 # usage: ynh_package_autopurge name [name [...]]
 # | arg: name - the package name to autoremove and purge
 #
@@ -172,59 +165,50 @@ ynh_package_autopurge() {
 # | arg: controlfile - path of the equivs control file
 #
 # Requires YunoHost version 2.2.4 or higher.
-ynh_package_install_from_equivs() {
+ynh_package_install_from_equivs () {
     local controlfile=$1
 
     # retrieve package information
-    local pkgname=$(grep '^Package: ' $controlfile | cut --delimiter=' ' --fields=2)    # Retrieve the name of the debian package
-    local pkgversion=$(grep '^Version: ' $controlfile | cut --delimiter=' ' --fields=2) # And its version number
+    local pkgname=$(grep '^Package: ' $controlfile | cut --delimiter=' ' --fields=2)	# Retrieve the name of the debian package
+    local pkgversion=$(grep '^Version: ' $controlfile | cut --delimiter=' ' --fields=2)	# And its version number
     [[ -z "$pkgname" || -z "$pkgversion" ]] \
-        && ynh_die --message="Invalid control file" # Check if this 2 variables aren't empty.
+        && ynh_die --message="Invalid control file"	# Check if this 2 variables aren't empty.
 
     # Update packages cache
     ynh_package_update
 
     # Build and install the package
     local TMPDIR=$(mktemp --directory)
-    mkdir -p ${TMPDIR}/${pkgname}/DEBIAN/
-    # For some reason, dpkg-deb insists for folder perm to be 755 and sometimes it's 777 o_O?
-    chmod -R 755 ${TMPDIR}/${pkgname}
+
+    # Force the compatibility level at 10, levels below are deprecated 
+    echo 10 > /usr/share/equivs/template/debian/compat
 
     # Note that the cd executes into a sub shell
     # Create a fake deb package with equivs-build and the given control file
     # Install the fake package without its dependencies with dpkg
     # Install missing dependencies with ynh_package_install
     ynh_wait_dpkg_free
+    cp "$controlfile" "${TMPDIR}/control"
+    (cd "$TMPDIR"
+    LC_ALL=C equivs-build ./control 1> /dev/null
+    LC_ALL=C dpkg --force-depends --install "./${pkgname}_${pkgversion}_all.deb" 2>&1 | tee ./dpkg_log)
 
-    cp "$controlfile" "${TMPDIR}/${pkgname}/DEBIAN/control"
-
-    # Install the fake package without its dependencies with dpkg --force-depends
-    if ! LC_ALL=C dpkg-deb --build "${TMPDIR}/${pkgname}" "${TMPDIR}/${pkgname}.deb" > "${TMPDIR}/dpkg_log" 2>&1; then
-        cat "${TMPDIR}/dpkg_log" >&2
-        ynh_die --message="Unable to install dependencies"
-    fi
-    # Don't crash in case of error, because is nicely covered by the following line
-    LC_ALL=C dpkg --force-depends --install "${TMPDIR}/${pkgname}.deb" 2>&1 | tee "${TMPDIR}/dpkg_log" || true
-
-    ynh_package_install --fix-broken \
-        || { # If the installation failed
-            # (the following is ran inside { } to not start a subshell otherwise ynh_die wouldnt exit the original process)
-            # Parse the list of problematic dependencies from dpkg's log ...
-            # (relevant lines look like: "foo-ynh-deps depends on bar; however:")
-            local problematic_dependencies="$(cat $TMPDIR/dpkg_log | grep -oP '(?<=-ynh-deps depends on ).*(?=; however)' | tr '\n' ' ')"
-            # Fake an install of those dependencies to see the errors
-            # The sed command here is, Print only from 'Reading state info' to the end.
-            [[ -n "$problematic_dependencies" ]] && ynh_package_install $problematic_dependencies --dry-run 2>&1 | sed --quiet '/Reading state info/,$p' | grep -v "fix-broken\|Reading state info" >&2
-            ynh_die --message="Unable to install dependencies"
-        }
-    [[ -n "$TMPDIR" ]] && rm --recursive --force $TMPDIR # Remove the temp dir.
+    ynh_package_install --fix-broken || \
+        { # If the installation failed 
+        # (the following is ran inside { } to not start a subshell otherwise ynh_die wouldnt exit the original process)
+        # Parse the list of problematic dependencies from dpkg's log ...
+        # (relevant lines look like: "foo-ynh-deps depends on bar; however:")
+        local problematic_dependencies="$(cat $TMPDIR/dpkg_log | grep -oP '(?<=-ynh-deps depends on ).*(?=; however)' | tr '\n' ' ')"
+        # Fake an install of those dependencies to see the errors
+        # The sed command here is, Print only from 'Reading state info' to the end.
+        [[ -n "$problematic_dependencies" ]] && ynh_package_install $problematic_dependencies --dry-run 2>&1 | sed --quiet '/Reading state info/,$p' | grep -v "fix-broken\|Reading state info" >&2
+        ynh_die --message="Unable to install dependencies"; }
+    [[ -n "$TMPDIR" ]] && rm --recursive --force $TMPDIR	# Remove the temp dir.
 
     # check if the package is actually installed
     ynh_package_is_installed "$pkgname"
 }
 
-YNH_INSTALL_APP_DEPENDENCIES_REPLACE="true"
-
 # Define and install dependencies with a equivs control file
 #
 # This helper can/should only be called once per app
@@ -232,24 +216,28 @@ YNH_INSTALL_APP_DEPENDENCIES_REPLACE="true"
 # example : ynh_install_app_dependencies dep1 dep2 "dep3|dep4|dep5"
 #
 # usage: ynh_install_app_dependencies dep [dep [...]]
-# | arg: dep - the package name to install in dependence.
-# | arg: "dep1|dep2|…" - You can specify alternatives. It will require to install (dep1 or dep2, etc).
+# | arg: dep - the package name to install in dependence. Writing "dep3|dep4|dep5" can be used to specify alternatives. For example : dep1 dep2 "dep3|dep4|dep5" will require to install dep1 and dep 2 and (dep3 or dep4 or dep5).
 #
 # Requires YunoHost version 2.6.4 or higher.
-ynh_install_app_dependencies() {
+ynh_install_app_dependencies () {
     local dependencies=$@
     # Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See  below)
     dependencies="$(echo "$dependencies" | sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g')"
     local dependencies=${dependencies//|/ | }
+    local manifest_path="../manifest.json"
+    if [ ! -e "$manifest_path" ]; then
+        manifest_path="../settings/manifest.json"	# Into the restore script, the manifest is not at the same place
+    fi
 
-    local version=$(ynh_read_manifest --manifest_key="version")
-    if [ -z "${version}" ] || [ "$version" == "null" ]; then
+    local version=$(grep '\"version\": ' "$manifest_path" | cut --delimiter='"' --fields=4)	# Retrieve the version number in the manifest file.
+    if [ ${#version} -eq 0 ]; then
         version="1.0"
     fi
-    local dep_app=${app//_/-} # Replace all '_' by '-'
+    local dep_app=${app//_/-}	# Replace all '_' by '-'
 
     # Handle specific versions
-    if [[ "$dependencies" =~ [\<=\>] ]]; then
+    if [[ "$dependencies" =~ [\<=\>] ]]
+    then
         # Replace version specifications by relationships syntax
         # https://www.debian.org/doc/debian-policy/ch-relationships.html
         # Sed clarification
@@ -261,101 +249,72 @@ ynh_install_app_dependencies() {
         dependencies="$(echo "$dependencies" | sed 's/\([^(\<=\>]\)\([\<=\>]\+\)\([^,]\+\)/\1 (\2 \3)/g')"
     fi
 
-    # Check for specific php dependencies which requires sury
-    # This grep will for example return "7.4" if dependencies is "foo bar php7.4-pwet php-gni"
-    # The (?<=php) syntax corresponds to lookbehind ;)
-    local specific_php_version=$(echo $dependencies | grep -oP '(?<=php)[0-9.]+(?=-|\>|)' | sort -u)
-
-    if [[ -n "$specific_php_version" ]]; then
-        # Cover a small edge case where a packager could have specified "php7.4-pwet php5-gni" which is confusing
-        [[ $(echo $specific_php_version | wc -l) -eq 1 ]] \
-            || ynh_die --message="Inconsistent php versions in dependencies ... found : $specific_php_version"
-
-        dependencies+=", php${specific_php_version}, php${specific_php_version}-fpm, php${specific_php_version}-common"
-
-        local old_phpversion=$(ynh_app_setting_get --app=$app --key=phpversion)
-
-        # If the PHP version changed, remove the old fpm conf
-        if [ -n "$old_phpversion" ] && [ "$old_phpversion" != "$specific_php_version" ]; then
-            local old_php_fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir)
-            local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf"
-
-            if [[ -f "$old_php_finalphpconf" ]]; then
-                ynh_backup_if_checksum_is_different --file="$old_php_finalphpconf"
-                ynh_remove_fpm_config
+    #
+    # Epic ugly hack to fix the goddamn dependency nightmare of sury
+    # Sponsored by the "Djeezusse Fokin Kraiste Why Do Adminsys Has To Be So Fucking Complicated I Should Go Grow Potatoes Instead Of This Shit" collective
+    # https://github.com/YunoHost/issues/issues/1407
+    # 
+    # If we require to install php dependency
+    if echo $dependencies | grep --quiet 'php'
+    then
+        # And we have packages from sury installed (7.0.33-10+weirdshiftafter instead of 7.0.33-0 on debian)
+        if dpkg --list | grep "php7.0" | grep --quiet --invert-match "7.0.33-0+deb9"
+        then
+            # And sury ain't already in sources.lists
+            if ! grep --recursive --quiet "^ *deb.*sury" /etc/apt/sources.list*
+            then
+                # Re-add sury
+                ynh_install_extra_repo --repo="https://packages.sury.org/php/ $(ynh_get_debian_release) main" --key="https://packages.sury.org/php/apt.gpg" --name=extra_php_version --priority=600
             fi
         fi
-        # Store phpversion into the config of this app
-        ynh_app_setting_set --app=$app --key=phpversion --value=$specific_php_version
-
-        # Set the default php version back as the default version for php-cli.
-        if test -e /usr/bin/php$YNH_DEFAULT_PHP_VERSION; then
-            update-alternatives --set php /usr/bin/php$YNH_DEFAULT_PHP_VERSION
-        fi
-    elif grep --quiet 'php' <<< "$dependencies"; then
-        ynh_app_setting_set --app=$app --key=phpversion --value=$YNH_DEFAULT_PHP_VERSION
     fi
 
-    local psql_installed="$(ynh_package_is_installed "postgresql-$PSQL_VERSION" && echo yes || echo no)"
-
-    # The first time we run ynh_install_app_dependencies, we will replace the
-    # entire control file (This is in particular meant to cover the case of
-    # upgrade script where ynh_install_app_dependencies is called with this
-    # expected effect) Otherwise, any subsequent call will add dependencies
-    # to those already present in the equivs control file.
-    if [[ $YNH_INSTALL_APP_DEPENDENCIES_REPLACE == "true" ]]; then
-        YNH_INSTALL_APP_DEPENDENCIES_REPLACE="false"
-    else
-        local current_dependencies=""
-        if ynh_package_is_installed --package="${dep_app}-ynh-deps"; then
-            current_dependencies="$(dpkg-query --show --showformat='${Depends}' ${dep_app}-ynh-deps) "
-            current_dependencies=${current_dependencies// | /|}
-        fi
-        dependencies="$current_dependencies, $dependencies"
-    fi
-
-    cat > /tmp/${dep_app}-ynh-deps.control << EOF # Make a control file for equivs-build
+    cat > /tmp/${dep_app}-ynh-deps.control << EOF	# Make a control file for equivs-build
 Section: misc
 Priority: optional
 Package: ${dep_app}-ynh-deps
 Version: ${version}
-Depends: ${dependencies//,,/,}
+Depends: ${dependencies}
 Architecture: all
-Maintainer: root@localhost
 Description: Fake package for ${app} (YunoHost app) dependencies
  This meta-package is only responsible of installing its dependencies.
 EOF
-
     ynh_package_install_from_equivs /tmp/${dep_app}-ynh-deps.control \
-        || ynh_die --message="Unable to install dependencies" # Install the fake package and its dependencies
+        || ynh_die --message="Unable to install dependencies"	# Install the fake package and its dependencies
     rm /tmp/${dep_app}-ynh-deps.control
-
-    # Trigger postgresql regenconf if we may have just installed postgresql
-    local psql_installed2="$(ynh_package_is_installed "postgresql-$PSQL_VERSION" && echo yes || echo no)"
-    if [[ "$psql_installed" != "$psql_installed2" ]]; then
-        yunohost tools regen-conf postgresql
-    fi
-
+    ynh_app_setting_set --app=$app --key=apt_dependencies --value="$dependencies"
 }
 
 # Add dependencies to install with ynh_install_app_dependencies
 #
-# [packagingv1]
-#
 # usage: ynh_add_app_dependencies --package=phpversion [--replace]
 # | arg: -p, --package=     - Packages to add as dependencies for the app.
+# | arg: -r, --replace      - Replace dependencies instead of adding to existing ones.
 #
 # Requires YunoHost version 3.8.1 or higher.
-ynh_add_app_dependencies() {
+ynh_add_app_dependencies () {
     # Declare an array to define the options of this helper.
     local legacy_args=pr
-    local -A args_array=([p]=package= [r]=replace)
+    local -A args_array=( [p]=package= [r]=replace)
     local package
+    local replace
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
+    replace=${replace:-0}
 
-    ynh_print_warn --message="Packagers: ynh_add_app_dependencies is deprecated and is now only an alias to ynh_install_app_dependencies"
-    ynh_install_app_dependencies "${package}"
+    local current_dependencies=""
+    if [ $replace -eq 0 ]
+    then
+        local dep_app=${app//_/-}	# Replace all '_' by '-'
+        if ynh_package_is_installed --package="${dep_app}-ynh-deps"
+        then
+            current_dependencies="$(dpkg-query --show --showformat='${Depends}' ${dep_app}-ynh-deps) "
+        fi
+
+        current_dependencies=${current_dependencies// | /|}
+    fi
+
+    ynh_install_app_dependencies "${current_dependencies}${package}"
 }
 
 # Remove fake package and its dependencies
@@ -365,27 +324,9 @@ ynh_add_app_dependencies() {
 # usage: ynh_remove_app_dependencies
 #
 # Requires YunoHost version 2.6.4 or higher.
-ynh_remove_app_dependencies() {
-    local dep_app=${app//_/-} # Replace all '_' by '-'
-
-    local current_dependencies=""
-    if ynh_package_is_installed --package="${dep_app}-ynh-deps"; then
-        current_dependencies="$(dpkg-query --show --showformat='${Depends}' ${dep_app}-ynh-deps) "
-        current_dependencies=${current_dependencies// | /|}
-    fi
-
-    # Edge case where the app dep may be on hold,
-    # cf https://forum.yunohost.org/t/migration-error-cause-of-ffsync/20675/4
-    if apt-mark showhold | grep -q -w ${dep_app}-ynh-deps; then
-        apt-mark unhold ${dep_app}-ynh-deps
-    fi
-
-    # Remove the fake package and its dependencies if they not still used.
-    # (except if dpkg doesn't know anything about the package,
-    # which should be symptomatic of a failed install, and we don't want bash to report an error)
-    if dpkg-query --show ${dep_app}-ynh-deps &> /dev/null; then
-        ynh_package_autopurge ${dep_app}-ynh-deps
-    fi
+ynh_remove_app_dependencies () {
+    local dep_app=${app//_/-}	# Replace all '_' by '-'
+    ynh_package_autopurge ${dep_app}-ynh-deps	# Remove the fake package and its dependencies if they not still used.
 }
 
 # Install packages from an extra repository properly.
@@ -397,10 +338,10 @@ ynh_remove_app_dependencies() {
 # | arg: -n, --name=    - Name for the files for this repo, $app as default value.
 #
 # Requires YunoHost version 3.8.1 or higher.
-ynh_install_extra_app_dependencies() {
+ynh_install_extra_app_dependencies () {
     # Declare an array to define the options of this helper.
     local legacy_args=rpkn
-    local -A args_array=([r]=repo= [p]=package= [k]=key= [n]=name=)
+    local -A args_array=( [r]=repo= [p]=package= [k]=key= [n]=name= )
     local repo
     local package
     local key
@@ -411,23 +352,18 @@ ynh_install_extra_app_dependencies() {
     key=${key:-}
 
     # Set a key only if asked
-    if [ -n "$key" ]; then
+    if [ -n "$key" ]
+    then
         key="--key=$key"
     fi
     # Add an extra repository for those packages
     ynh_install_extra_repo --repo="$repo" $key --priority=995 --name=$name
 
     # Install requested dependencies from this extra repository.
-    ynh_install_app_dependencies "$package"
-
-    # Force to upgrade to the last version...
-    # Without doing apt install, an already installed dep is not upgraded
-    local apps_auto_installed="$(apt-mark showauto $package)"
-    ynh_package_install "$package"
-    [ -z "$apps_auto_installed" ] || apt-mark auto $apps_auto_installed
+    ynh_add_app_dependencies --package="$package"
 
     # Remove this extra repository after packages are installed
-    ynh_remove_extra_repo --name=$name
+    ynh_remove_extra_repo --name=$app
 }
 
 # Add an extra repository correctly, pin it and get the key.
@@ -442,10 +378,10 @@ ynh_install_extra_app_dependencies() {
 # | arg: -a, --append       - Do not overwrite existing files.
 #
 # Requires YunoHost version 3.8.1 or higher.
-ynh_install_extra_repo() {
+ynh_install_extra_repo () {
     # Declare an array to define the options of this helper.
     local legacy_args=rkpna
-    local -A args_array=([r]=repo= [k]=key= [p]=priority= [n]=name= [a]=append)
+    local -A args_array=( [r]=repo= [k]=key= [p]=priority= [n]=name= [a]=append )
     local repo
     local key
     local priority
@@ -458,7 +394,8 @@ ynh_install_extra_repo() {
     key=${key:-}
     priority=${priority:-}
 
-    if [ $append -eq 1 ]; then
+    if [ $append -eq 1 ]
+    then
         append="--append"
         wget_append="tee --append"
     else
@@ -466,44 +403,36 @@ ynh_install_extra_repo() {
         wget_append="tee"
     fi
 
-    if [[ "$key" == "trusted=yes" ]]; then
-        trusted="--trusted"
-    else
-        trusted=""
-    fi
-
-    IFS=', ' read -r -a repo_parts <<< "$repo"
-    index=0
-
+    # Split the repository into uri, suite and components.
     # Remove "deb " at the beginning of the repo.
-    if [[ "${repo_parts[0]}" == "deb" ]]; then
-        index=1
-    fi
-    uri="${repo_parts[$index]}"
-    index=$((index + 1))
-    suite="${repo_parts[$index]}"
-    index=$((index + 1))
+    repo="${repo#deb }"
+
+    # Get the uri
+    local uri="$(echo "$repo" | awk '{ print $1 }')"
+
+    # Get the suite
+    local suite="$(echo "$repo" | awk '{ print $2 }')"
 
     # Get the components
-    if (("${#repo_parts[@]}" > 0)); then
-        component="${repo_parts[*]:$index}"
-    fi
+    local component="${repo##$uri $suite }"
 
     # Add the repository into sources.list.d
-    ynh_add_repo --uri="$uri" --suite="$suite" --component="$component" --name="$name" $append $trusted
+    ynh_add_repo --uri="$uri" --suite="$suite" --component="$component" --name="$name" $append
 
     # Pin the new repo with the default priority, so it won't be used for upgrades.
     # Build $pin from the uri without http and any sub path
     local pin="${uri#*://}"
     pin="${pin%%/*}"
     # Set a priority only if asked
-    if [ -n "$priority" ]; then
+    if [ -n "$priority" ]
+    then
         priority="--priority=$priority"
     fi
     ynh_pin_repo --package="*" --pin="origin \"$pin\"" $priority --name="$name" $append
 
     # Get the public key for the repo
-    if [ -n "$key" ] && [[ "$key" != "trusted=yes" ]]; then
+    if [ -n "$key" ]
+    then
         mkdir --parents "/etc/apt/trusted.gpg.d"
         # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget)
         wget --timeout 900 --quiet "$key" --output-document=- | gpg --dearmor | $wget_append /etc/apt/trusted.gpg.d/$name.gpg > /dev/null
@@ -521,26 +450,20 @@ ynh_install_extra_repo() {
 # | arg: -n, --name=    - Name for the files for this repo, $app as default value.
 #
 # Requires YunoHost version 3.8.1 or higher.
-ynh_remove_extra_repo() {
+ynh_remove_extra_repo () {
     # Declare an array to define the options of this helper.
     local legacy_args=n
-    local -A args_array=([n]=name=)
+    local -A args_array=( [n]=name= )
     local name
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
     name="${name:-$app}"
 
-    ynh_secure_remove --file="/etc/apt/sources.list.d/$name.list"
+    ynh_secure_remove "/etc/apt/sources.list.d/$name.list"
     # Sury pinning is managed by the regenconf in the core...
     [[ "$name" == "extra_php_version" ]] || ynh_secure_remove "/etc/apt/preferences.d/$name"
-    if [ -e /etc/apt/trusted.gpg.d/$name.gpg ]; then
-        ynh_secure_remove --file="/etc/apt/trusted.gpg.d/$name.gpg"
-    fi
-
-    # (Do we even create a .asc file anywhere ...?)
-    if [ -e /etc/apt/trusted.gpg.d/$name.asc ]; then
-        ynh_secure_remove --file="/etc/apt/trusted.gpg.d/$name.asc"
-    fi
+    ynh_secure_remove "/etc/apt/trusted.gpg.d/$name.gpg" > /dev/null
+    ynh_secure_remove "/etc/apt/trusted.gpg.d/$name.asc" > /dev/null
 
     # Update the list of package to exclude the old repo
     ynh_package_update
@@ -556,43 +479,36 @@ ynh_remove_extra_repo() {
 # | arg: -c, --component=   - Component of the repository.
 # | arg: -n, --name=        - Name for the files for this repo, $app as default value.
 # | arg: -a, --append       - Do not overwrite existing files.
-# | arg: -t, --trusted      - Add trusted=yes to the repository (not recommended)
 #
 # Example for a repo like deb http://forge.yunohost.org/debian/ stretch stable
 #                             uri                               suite   component
 # ynh_add_repo --uri=http://forge.yunohost.org/debian/ --suite=stretch --component=stable
 #
 # Requires YunoHost version 3.8.1 or higher.
-ynh_add_repo() {
+ynh_add_repo () {
     # Declare an array to define the options of this helper.
-    local legacy_args=uscnat
-    local -A args_array=([u]=uri= [s]=suite= [c]=component= [n]=name= [a]=append [t]=trusted)
+    local legacy_args=uscna
+    local -A args_array=( [u]=uri= [s]=suite= [c]=component= [n]=name= [a]=append )
     local uri
     local suite
     local component
     local name
     local append
-    local trusted
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
     name="${name:-$app}"
     append=${append:-0}
-    trusted=${trusted:-0}
 
-    if [ $append -eq 1 ]; then
+    if [ $append -eq 1 ]
+    then
         append="tee --append"
     else
         append="tee"
     fi
-    if [[ "$trusted" -eq 1 ]]; then
-        trust="[trusted=yes]"
-    else
-        trust=""
-    fi
 
     mkdir --parents "/etc/apt/sources.list.d"
     # Add the new repo in sources.list.d
-    echo "deb $trust $uri $suite $component" \
+    echo "deb $uri $suite $component" \
         | $append "/etc/apt/sources.list.d/$name.list"
 }
 
@@ -610,10 +526,10 @@ ynh_add_repo() {
 # See https://manpages.debian.org/stretch/apt/apt_preferences.5.en.html#How_APT_Interprets_Priorities for information about pinning.
 #
 # Requires YunoHost version 3.8.1 or higher.
-ynh_pin_repo() {
+ynh_pin_repo () {
     # Declare an array to define the options of this helper.
     local legacy_args=pirna
-    local -A args_array=([p]=package= [i]=pin= [r]=priority= [n]=name= [a]=append)
+    local -A args_array=( [p]=package= [i]=pin= [r]=priority= [n]=name= [a]=append )
     local package
     local pin
     local priority
@@ -626,7 +542,8 @@ ynh_pin_repo() {
     name="${name:-$app}"
     append=${append:-0}
 
-    if [ $append -eq 1 ]; then
+    if [ $append -eq 1 ]
+    then
         append="tee --append"
     else
         append="tee"
@@ -640,5 +557,5 @@ ynh_pin_repo() {
 Pin: $pin
 Pin-Priority: $priority
 " \
-        | $append "/etc/apt/preferences.d/$name"
+    | $append "/etc/apt/preferences.d/$name"
 }

data/helpers.d/backup

@@ -9,16 +9,17 @@ CAN_BIND=${CAN_BIND:-1}
 # | arg: -d, --dest_path=       - destination file or directory inside the backup dir
 # | arg: -b, --is_big           - Indicate data are big (mail, video, image ...)
 # | arg: -m, --not_mandatory    - Indicate that if the file is missing, the backup can ignore it.
+# | arg: arg - Deprecated arg
 #
 # This helper can be used both in a system backup hook, and in an app backup script
 #
-# `ynh_backup` writes `src_path` and the relative `dest_path` into a CSV file, and it
+# Details: ynh_backup writes SRC and the relative DEST into a CSV file. And it
 # creates the parent destination directory
 #
-# If `dest_path` is ended by a slash it complete this path with the basename of `src_path`.
+# If DEST is ended by a slash it complete this path with the basename of SRC.
+#
+# Example in the context of a wordpress app
 #
-# Example in the context of a wordpress app :
-# ```
 # ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf"
 # # => This line will be added into CSV file
 # # "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/etc/nginx/conf.d/$domain.d/$app.conf"
@@ -39,34 +40,32 @@ CAN_BIND=${CAN_BIND:-1}
 # ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "/conf/"
 # # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/$app.conf"
 #
-# ```
 #
-# How to use `--is_big`:
-#
-# `--is_big` is used to specify that this part of the backup can be quite huge.
+# How to use --is_big:
+# --is_big is used to specify that this part of the backup can be quite huge.
 # So, you don't want that your package does backup that part during ynh_backup_before_upgrade.
 # In the same way, an user may doesn't want to backup this big part of the app for
-# each of his backup. And so handle that part differently.
-#
+#   each of his backup. And so handle that part differently.
+# 
 # As this part of your backup may not be done, your restore script has to handle it.
-# In your restore script, use `--not_mandatory` with `ynh_restore_file`
+# In your restore script, use --not_mandatory with ynh_restore_file
 # As well in your remove script, you should not remove those data ! Or an user may end up with
-# a failed upgrade restoring an app without data anymore !
+#   a failed upgrade restoring an app without data anymore !
 #
-# To have the benefit of `--is_big` while doing a backup, you can whether set the environement
-# variable `BACKUP_CORE_ONLY` to 1 (`BACKUP_CORE_ONLY=1`) before the backup command. It will affect
-# only that backup command.
-# Or set the config `do_not_backup_data` to 1 into the `settings.yml` of the app. This will affect
-# all backups for this app until the setting is removed.
+# To have the benefit of --is_big while doing a backup, you can whether set the environement
+#   variable BACKUP_CORE_ONLY to 1 (BACKUP_CORE_ONLY=1) before the backup command. It will affect
+#   only that backup command.
+# Or set the config do_not_backup_data to 1 into the settings.yml of the app. This will affect
+#   all backups for this app until the setting is removed.
 #
 # Requires YunoHost version 2.4.0 or higher.
-# Requires YunoHost version 3.5.0 or higher for the argument `--not_mandatory`
+# Requires YunoHost version 3.5.0 or higher for the argument --not_mandatory
 ynh_backup() {
     # TODO find a way to avoid injection by file strange naming !
 
     # Declare an array to define the options of this helper.
     local legacy_args=sdbm
-    local -A args_array=([s]=src_path= [d]=dest_path= [b]=is_big [m]=not_mandatory)
+    local -A args_array=( [s]=src_path= [d]=dest_path= [b]=is_big [m]=not_mandatory )
     local src_path
     local dest_path
     local is_big
@@ -82,8 +81,10 @@ ynh_backup() {
 
     # If backing up core only (used by ynh_backup_before_upgrade),
     # don't backup big data items
-    if [ $is_big -eq 1 ] && ([ ${do_not_backup_data:-0} -eq 1 ] || [ $BACKUP_CORE_ONLY -eq 1 ]); then
-        if [ $BACKUP_CORE_ONLY -eq 1 ]; then
+    if [ $is_big -eq 1 ] && ( [ ${do_not_backup_data:-0} -eq 1 ] || [ $BACKUP_CORE_ONLY -eq 1 ] ) 
+    then
+        if [ $BACKUP_CORE_ONLY -eq 1 ]
+        then
             ynh_print_info --message="$src_path will not be saved, because 'BACKUP_CORE_ONLY' is set."
         else
             ynh_print_info --message="$src_path will not be saved, because 'do_not_backup_data' is set."
@@ -95,11 +96,14 @@ ynh_backup() {
     # Format correctly source and destination paths
     # ==============================================================================
     # Be sure the source path is not empty
-    if [ ! -e "$src_path" ]; then
+    if [ ! -e "$src_path" ]
+    then
         ynh_print_warn --message="Source path '${src_path}' does not exist"
-        if [ "$not_mandatory" == "0" ]; then
+        if [ "$not_mandatory" == "0" ]
+        then
             # This is a temporary fix for fail2ban config files missing after the migration to stretch.
-            if echo "${src_path}" | grep --quiet "/etc/fail2ban"; then
+            if echo "${src_path}" | grep --quiet "/etc/fail2ban"
+            then
                 touch "${src_path}"
                 ynh_print_info --message="The missing file will be replaced by a dummy one for the backup !!!"
             else
@@ -117,11 +121,13 @@ ynh_backup() {
     # If there is no destination path, initialize it with the source path
     # relative to "/".
     # eg: src_path=/etc/yunohost -> dest_path=etc/yunohost
-    if [[ -z "$dest_path" ]]; then
+    if [[ -z "$dest_path" ]]
+    then
         dest_path="${src_path#/}"
 
     else
-        if [[ "${dest_path:0:1}" == "/" ]]; then
+        if [[ "${dest_path:0:1}" == "/" ]]
+        then
 
             # If the destination path is an absolute path, transform it as a path
             # relative to the current working directory ($YNH_CWD)
@@ -145,7 +151,8 @@ ynh_backup() {
     fi
 
     # Check if dest_path already exists in tmp archive
-    if [[ -e "${dest_path}" ]]; then
+    if [[ -e "${dest_path}" ]]
+    then
         ynh_print_err --message="Destination path '${dest_path}' already exist"
         return 1
     fi
@@ -176,18 +183,19 @@ ynh_backup() {
 # usage: ynh_restore
 #
 # Requires YunoHost version 2.6.4 or higher.
-ynh_restore() {
+ynh_restore () {
     # Deduce the relative path of $YNH_CWD
     local REL_DIR="${YNH_CWD#$YNH_BACKUP_DIR/}"
     REL_DIR="${REL_DIR%/}/"
 
     # For each destination path begining by $REL_DIR
-    cat ${YNH_BACKUP_CSV} | tr --delete $'\r' | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR.*\"$" \
-        | while read line; do
-            local ORIGIN_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\"\K.*(?=\",\".*\"$)")
-            local ARCHIVE_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR\K.*(?=\"$)")
-            ynh_restore_file --origin_path="$ARCHIVE_PATH" --dest_path="$ORIGIN_PATH"
-        done
+    cat ${YNH_BACKUP_CSV} | tr --delete $'\r' | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR.*\"$" | \
+    while read line
+    do
+        local ORIGIN_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\"\K.*(?=\",\".*\"$)")
+        local ARCHIVE_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR\K.*(?=\"$)")
+        ynh_restore_file --origin_path="$ARCHIVE_PATH" --dest_path="$ORIGIN_PATH"
+    done
 }
 
 # Return the path in the archive where has been stocked the origin path
@@ -195,50 +203,51 @@ ynh_restore() {
 # [internal]
 #
 # usage: _get_archive_path ORIGIN_PATH
-_get_archive_path() {
+_get_archive_path () {
     # For security reasons we use csv python library to read the CSV
-    python3 -c "
+    python -c "
 import sys
 import csv
 with open(sys.argv[1], 'r') as backup_file:
     backup_csv = csv.DictReader(backup_file, fieldnames=['source', 'dest'])
     for row in backup_csv:
         if row['source']==sys.argv[2].strip('\"'):
-            print(row['dest'])
+            print row['dest']
             sys.exit(0)
     raise Exception('Original path for %s not found' % sys.argv[2])
-    " "${YNH_BACKUP_CSV}" "$1"
+    "  "${YNH_BACKUP_CSV}" "$1"
     return $?
 }
 
 # Restore a file or a directory
 #
+# Use the registered path in backup_list by ynh_backup to restore the file at
+# the right place.
+#
 # usage: ynh_restore_file --origin_path=origin_path [--dest_path=dest_path] [--not_mandatory]
 # | arg: -o, --origin_path=     - Path where was located the file or the directory before to be backuped or relative path to $YNH_CWD where it is located in the backup archive
-# | arg: -d, --dest_path=       - Path where restore the file or the dir. If unspecified, the destination will be `ORIGIN_PATH` or if the `ORIGIN_PATH` doesn't exist in the archive, the destination will be searched into `backup.csv`
+# | arg: -d, --dest_path=       - Path where restore the file or the dir, if unspecified, the destination will be ORIGIN_PATH or if the ORIGIN_PATH doesn't exist in the archive, the destination will be searched into backup.csv
 # | arg: -m, --not_mandatory    - Indicate that if the file is missing, the restore process can ignore it.
 #
-# Use the registered path in backup_list by ynh_backup to restore the file at the right place.
-#
 # examples:
-#     ynh_restore_file -o "/etc/nginx/conf.d/$domain.d/$app.conf"
+#     ynh_restore_file "/etc/nginx/conf.d/$domain.d/$app.conf"
 #     # You can also use relative paths:
-#     ynh_restore_file -o "conf/nginx.conf"
+#     ynh_restore_file "conf/nginx.conf"
 #
-# If `DEST_PATH` already exists and is lighter than 500 Mo, a backup will be made in
-# `/var/cache/yunohost/appconfbackup/`. Otherwise, the existing file is removed.
+# If DEST_PATH already exists and is lighter than 500 Mo, a backup will be made in
+# /home/yunohost.conf/backup/. Otherwise, the existing file is removed.
 #
-# if `apps/$app/etc/nginx/conf.d/$domain.d/$app.conf` exists, restore it into
-# `/etc/nginx/conf.d/$domain.d/$app.conf`
+# if apps/wordpress/etc/nginx/conf.d/$domain.d/$app.conf exists, restore it into
+# /etc/nginx/conf.d/$domain.d/$app.conf
 # if no, search for a match in the csv (eg: conf/nginx.conf) and restore it into
-# `/etc/nginx/conf.d/$domain.d/$app.conf`
+# /etc/nginx/conf.d/$domain.d/$app.conf
 #
 # Requires YunoHost version 2.6.4 or higher.
 # Requires YunoHost version 3.5.0 or higher for the argument --not_mandatory
-ynh_restore_file() {
+ynh_restore_file () {
     # Declare an array to define the options of this helper.
     local legacy_args=odm
-    local -A args_array=([o]=origin_path= [d]=dest_path= [m]=not_mandatory)
+    local -A args_array=( [o]=origin_path= [d]=dest_path= [m]=not_mandatory )
     local origin_path
     local dest_path
     local not_mandatory
@@ -251,8 +260,10 @@ ynh_restore_file() {
 
     local archive_path="$YNH_CWD${origin_path}"
     # If archive_path doesn't exist, search for a corresponding path in CSV
-    if [ ! -d "$archive_path" ] && [ ! -f "$archive_path" ] && [ ! -L "$archive_path" ]; then
-        if [ "$not_mandatory" == "0" ]; then
+    if [ ! -d "$archive_path" ] && [ ! -f "$archive_path" ] && [ ! -L "$archive_path" ]
+    then
+        if [ "$not_mandatory" == "0" ]
+        then
             archive_path="$YNH_BACKUP_DIR/$(_get_archive_path \"$origin_path\")"
         else
             return 0
@@ -260,12 +271,14 @@ ynh_restore_file() {
     fi
 
     # Move the old directory if it already exists
-    if [[ -e "${dest_path}" ]]; then
+    if [[ -e "${dest_path}" ]]
+    then
         # Check if the file/dir size is less than 500 Mo
-        if [[ $(du --summarize --bytes ${dest_path} | cut --delimiter="/" --fields=1) -le "500000000" ]]; then
-            local backup_file="/var/cache/yunohost/appconfbackup/${dest_path}.backup.$(date '+%Y%m%d.%H%M%S')"
+        if [[ $(du --summarize --bytes ${dest_path} | cut --delimiter="/" --fields=1) -le "500000000" ]]
+        then
+            local backup_file="/home/yunohost.conf/backup/${dest_path}.backup.$(date '+%Y%m%d.%H%M%S')"
             mkdir --parents "$(dirname "$backup_file")"
-            mv "${dest_path}" "$backup_file" # Move the current file or directory
+            mv "${dest_path}" "$backup_file"    # Move the current file or directory
         else
             ynh_secure_remove --file=${dest_path}
         fi
@@ -275,8 +288,10 @@ ynh_restore_file() {
     mkdir --parents $(dirname "$dest_path")
 
     # Do a copy if it's just a mounting point
-    if mountpoint --quiet $YNH_BACKUP_DIR; then
-        if [[ -d "${archive_path}" ]]; then
+    if mountpoint --quiet $YNH_BACKUP_DIR
+    then
+        if [[ -d "${archive_path}" ]]
+        then
             archive_path="${archive_path}/."
             mkdir --parents "$dest_path"
         fi
@@ -285,13 +300,18 @@ ynh_restore_file() {
     else
         mv "$archive_path" "${dest_path}"
     fi
+}
 
-    # Boring hack for nginx conf file mapped to php7.3
-    # Note that there's no need to patch the fpm config because most php apps
-    # will call "ynh_add_fpm_config" during restore, effectively recreating the file from scratch
-    if [[ "${dest_path}" == "/etc/nginx/conf.d/"* ]] && grep 'php7.3.*sock' "${dest_path}"; then
-        sed -i 's/php7.3/php7.4/g' "${dest_path}"
-    fi
+# Deprecated helper since it's a dangerous one!
+#
+# [internal]
+#
+ynh_bind_or_cp() {
+    local AS_ROOT=${3:-0}
+    local NO_ROOT=0
+    [[ "${AS_ROOT}" = "1" ]] || NO_ROOT=1
+    ynh_print_warn --message="This helper is deprecated, you should use ynh_backup instead"
+    ynh_backup "$1" "$2" 1
 }
 
 # Calculate and store a file checksum into the app settings
@@ -302,39 +322,20 @@ ynh_restore_file() {
 # $app should be defined when calling this helper
 #
 # Requires YunoHost version 2.6.4 or higher.
-ynh_store_file_checksum() {
+ynh_store_file_checksum () {
     # Declare an array to define the options of this helper.
     local legacy_args=f
-    local -A args_array=([f]=file= [u]=update_only)
+    local -A args_array=( [f]=file= )
     local file
-    local update_only
-    update_only="${update_only:-0}"
-
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
-
-    # If update only, we don't save the new checksum if no old checksum exist
-    if [ $update_only -eq 1 ]; then
-        local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name)
-        if [ -z "${checksum_value}" ]; then
-            unset backup_file_checksum
-            return 0
-        fi
-    fi
-
+    local checksum_setting_name=checksum_${file//[\/ ]/_}    # Replace all '/' and ' ' by '_'
     ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(md5sum "$file" | cut --delimiter=' ' --fields=1)
 
-    if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then
-        # Using a base64 is in fact more reversible than "replace / and space by _" ... So we can in fact obtain the original file path in an easy reliable way ...
-        local file_path_base64=$(echo "$file" | base64 -w0)
-        mkdir -p /var/cache/yunohost/appconfbackup/
-        cat $file > /var/cache/yunohost/appconfbackup/original_${file_path_base64}
-    fi
-
     # If backup_file_checksum isn't empty, ynh_backup_if_checksum_is_different has made a backup
-    if [ -n "${backup_file_checksum-}" ]; then
+    if [ -n "${backup_file_checksum-}" ]
+    then
         # Print the diff between the previous file and the new one.
         # diff return 1 if the files are different, so the || true
         diff --report-identical-files --unified --color=always $backup_file_checksum $file >&2 || true
@@ -344,42 +345,36 @@ ynh_store_file_checksum() {
 }
 
 # Verify the checksum and backup the file if it's different
+# 
+# This helper is primarily meant to allow to easily backup personalised/manually
+# modified config files.
 #
 # usage: ynh_backup_if_checksum_is_different --file=file
 # | arg: -f, --file=    - The file on which the checksum test will be perfomed.
 # | ret: the name of a backup file, or nothing
 #
-# This helper is primarily meant to allow to easily backup personalised/manually
-# modified config files.
-#
 # Requires YunoHost version 2.6.4 or higher.
-ynh_backup_if_checksum_is_different() {
+ynh_backup_if_checksum_is_different () {
     # Declare an array to define the options of this helper.
     local legacy_args=f
-    local -A args_array=([f]=file=)
+    local -A args_array=( [f]=file= )
     local file
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
+    local checksum_setting_name=checksum_${file//[\/ ]/_}    # Replace all '/' and ' ' by '_'
     local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name)
     # backup_file_checksum isn't declare as local, so it can be reuse by ynh_store_file_checksum
     backup_file_checksum=""
-    if [ -n "$checksum_value" ]; then                                                     # Proceed only if a value was stored into the app settings
-        if [ -e $file ] && ! echo "$checksum_value $file" | md5sum --check --status; then # If the checksum is now different
-
-            backup_file_checksum="/var/cache/yunohost/appconfbackup/$file.backup.$(date '+%Y%m%d.%H%M%S')"
+    if [ -n "$checksum_value" ]
+    then    # Proceed only if a value was stored into the app settings
+        if [ -e $file ] && ! echo "$checksum_value $file" | md5sum --check --status
+        then    # If the checksum is now different
+            backup_file_checksum="/home/yunohost.conf/backup/$file.backup.$(date '+%Y%m%d.%H%M%S')"
             mkdir --parents "$(dirname "$backup_file_checksum")"
-            cp --archive "$file" "$backup_file_checksum" # Backup the current file
+            cp --archive "$file" "$backup_file_checksum"    # Backup the current file
             ynh_print_warn "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum"
-            echo "$backup_file_checksum" # Return the name of the backup file
-            if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then
-                local file_path_base64=$(echo "$file" | base64 -w0)
-                if test -e /var/cache/yunohost/appconfbackup/original_${file_path_base64}; then
-                    ynh_print_warn "Diff with the original file:"
-                    diff --report-identical-files --unified --color=always /var/cache/yunohost/appconfbackup/original_${file_path_base64} $file >&2 || true
-                fi
-            fi
+            echo "$backup_file_checksum"    # Return the name of the backup file
         fi
     fi
 }
@@ -392,56 +387,44 @@ ynh_backup_if_checksum_is_different() {
 # $app should be defined when calling this helper
 #
 # Requires YunoHost version 3.3.1 or higher.
-ynh_delete_file_checksum() {
+ynh_delete_file_checksum () {
     # Declare an array to define the options of this helper.
     local legacy_args=f
-    local -A args_array=([f]=file=)
+    local -A args_array=( [f]=file= )
     local file
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    local checksum_setting_name=checksum_${file//[\/ ]/_} # Replace all '/' and ' ' by '_'
+    local checksum_setting_name=checksum_${file//[\/ ]/_}    # Replace all '/' and ' ' by '_'
     ynh_app_setting_delete --app=$app --key=$checksum_setting_name
 }
 
-# Checks a backup archive exists
-#
-# [internal]
-#
-ynh_backup_archive_exists() {
-    yunohost backup list --output-as json --quiet \
-        | jq -e --arg archive "$1" '.archives | index($archive)' > /dev/null
-}
-
 # Make a backup in case of failed upgrade
 #
-# [packagingv1]
-#
-# usage: ynh_backup_before_upgrade
-#
-# Usage in a package script:
-# ```
+# usage:
 #   ynh_backup_before_upgrade
 #   ynh_clean_setup () {
 #       ynh_restore_upgradebackup
 #   }
 #   ynh_abort_if_errors
-# ```
 #
 # Requires YunoHost version 2.7.2 or higher.
-ynh_backup_before_upgrade() {
-    if [ ! -e "/etc/yunohost/apps/$app/scripts/backup" ]; then
+ynh_backup_before_upgrade () {
+    if [ ! -e "/etc/yunohost/apps/$app/scripts/backup" ]
+    then
         ynh_print_warn --message="This app doesn't have any backup script."
         return
     fi
     backup_number=1
     local old_backup_number=2
-    local app_bck=${app//_/-} # Replace all '_' by '-'
+    local app_bck=${app//_/-}    # Replace all '_' by '-'
     NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0}
 
-    if [ "$NO_BACKUP_UPGRADE" -eq 0 ]; then
+    if [ "$NO_BACKUP_UPGRADE" -eq 0 ]
+    then
         # Check if a backup already exists with the prefix 1
-        if ynh_backup_archive_exists "$app_bck-pre-upgrade1"; then
+        if yunohost backup list | grep --quiet $app_bck-pre-upgrade1
+        then
             # Prefix becomes 2 to preserve the previous backup
             backup_number=2
             old_backup_number=1
@@ -449,9 +432,11 @@ ynh_backup_before_upgrade() {
 
         # Create backup
         BACKUP_CORE_ONLY=1 yunohost backup create --apps $app --name $app_bck-pre-upgrade$backup_number --debug
-        if [ "$?" -eq 0 ]; then
+        if [ "$?" -eq 0 ]
+        then
             # If the backup succeeded, remove the previous backup
-            if ynh_backup_archive_exists "$app_bck-pre-upgrade$old_backup_number"; then
+            if yunohost backup list | grep --quiet $app_bck-pre-upgrade$old_backup_number
+            then
                 # Remove the previous backup only if it exists
                 yunohost backup delete $app_bck-pre-upgrade$old_backup_number > /dev/null
             fi
@@ -465,38 +450,30 @@ ynh_backup_before_upgrade() {
 
 # Restore a previous backup if the upgrade process failed
 #
-# [packagingv1]
-#
-# usage: ynh_restore_upgradebackup
-#
-# Usage in a package script:
-# ```
+# usage:
 #   ynh_backup_before_upgrade
 #   ynh_clean_setup () {
 #       ynh_restore_upgradebackup
 #   }
 #   ynh_abort_if_errors
-# ```
 #
 # Requires YunoHost version 2.7.2 or higher.
-ynh_restore_upgradebackup() {
+ynh_restore_upgradebackup () {
     ynh_print_err --message="Upgrade failed."
-    local app_bck=${app//_/-} # Replace all '_' by '-'
+    local app_bck=${app//_/-}    # Replace all '_' by '-'
 
     NO_BACKUP_UPGRADE=${NO_BACKUP_UPGRADE:-0}
 
-    if [ "$NO_BACKUP_UPGRADE" -eq 0 ]; then
+    if [ "$NO_BACKUP_UPGRADE" -eq 0 ]
+    then
         # Check if an existing backup can be found before removing and restoring the application.
-        if ynh_backup_archive_exists "$app_bck-pre-upgrade$backup_number"; then
+        if yunohost backup list | grep --quiet $app_bck-pre-upgrade$backup_number
+        then
             # Remove the application then restore it
             yunohost app remove $app
             # Restore the backup
             yunohost backup restore $app_bck-pre-upgrade$backup_number --apps $app --force --debug
-            if [[ -d /etc/yunohost/apps/$app ]]; then
-                ynh_die --message="The app was restored to the way it was before the failed upgrade."
-            else
-                ynh_die --message="Uhoh ... Yunohost failed to restore the app to the way it was before the failed upgrade :|"
-            fi
+            ynh_die --message="The app was restored to the way it was before the failed upgrade."
         fi
     else
         ynh_print_warn --message="\$NO_BACKUP_UPGRADE is set, that means there's no backup to restore. You have to fix this upgrade by yourself !"

data/helpers.d/fail2ban

@@ -0,0 +1,154 @@
+#!/bin/bash
+
+# Create a dedicated fail2ban config (jail and filter conf files)
+#
+# usage 1: ynh_add_fail2ban_config --logpath=log_file --failregex=filter [--max_retry=max_retry] [--ports=ports]
+# | arg: -l, --logpath=   - Log file to be checked by fail2ban
+# | arg: -r, --failregex= - Failregex to be looked for by fail2ban
+# | arg: -m, --max_retry= - Maximum number of retries allowed before banning IP address - default: 3
+# | arg: -p, --ports=     - Ports blocked for a banned IP address - default: http,https
+#
+# -----------------------------------------------------------------------------
+#
+# usage 2: ynh_add_fail2ban_config --use_template [--others_var="list of others variables to replace"]
+# | arg: -t, --use_template - Use this helper in template mode
+# | arg: -v, --others_var=  - List of others variables to replace separeted by a space
+# |                           for example : 'var_1 var_2 ...'
+#
+# This will use a template in ../conf/f2b_jail.conf and ../conf/f2b_filter.conf
+#   __APP__      by  $app
+#
+#  You can dynamically replace others variables by example :
+#   __VAR_1__    by $var_1
+#   __VAR_2__    by $var_2
+#
+# Generally your template will look like that by example (for synapse):
+#
+# f2b_jail.conf:
+#     [__APP__]
+#     enabled = true
+#     port = http,https
+#     filter = __APP__
+#     logpath = /var/log/__APP__/logfile.log
+#     maxretry = 3
+#
+# f2b_filter.conf:
+#     [INCLUDES]
+#     before = common.conf
+#     [Definition]
+#
+#     # Part of regex definition (just used to make more easy to make the global regex)
+#     __synapse_start_line = .? \- synapse\..+ \-
+#
+#    # Regex definition.
+#    failregex = ^%(__synapse_start_line)s INFO \- POST\-(\d+)\- <HOST> \- \d+ \- Received request\: POST /_matrix/client/r0/login\??<SKIPLINES>%(__synapse_start_line)s INFO \- POST\-\1\- Got login request with identifier: \{u'type': u'm.id.user', u'user'\: u'(.+?)'\}, medium\: None, address: None, user\: u'\5'<SKIPLINES>%(__synapse_start_line)s WARNING \- \- (Attempted to login as @\5\:.+ but they do not exist|Failed password login for user @\5\:.+)$
+#
+#     ignoreregex =
+#
+# -----------------------------------------------------------------------------
+#
+# Note about the "failregex" option:
+#          regex to match the password failure messages in the logfile. The
+#          host must be matched by a group named "host". The tag "<HOST>" can
+#          be used for standard IP/hostname matching and is only an alias for
+#          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
+#
+#          You can find some more explainations about how to make a regex here :
+#          https://www.fail2ban.org/wiki/index.php/MANUAL_0_8#Filters
+#
+# Note that the logfile need to exist before to call this helper !!
+#
+# To validate your regex you can test with this command:
+# fail2ban-regex /var/log/YOUR_LOG_FILE_PATH /etc/fail2ban/filter.d/YOUR_APP.conf
+#
+# Requires YunoHost version 3.5.0 or higher.
+ynh_add_fail2ban_config () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=lrmptv
+    local -A args_array=( [l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= [t]=use_template [v]=others_var=)
+    local logpath
+    local failregex
+    local max_retry
+    local ports
+    local others_var
+    local use_template
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    max_retry=${max_retry:-3}
+    ports=${ports:-http,https}
+    others_var=${others_var:-}
+    use_template="${use_template:-0}"
+
+    finalfail2banjailconf="/etc/fail2ban/jail.d/$app.conf"
+    finalfail2banfilterconf="/etc/fail2ban/filter.d/$app.conf"
+    ynh_backup_if_checksum_is_different "$finalfail2banjailconf"
+    ynh_backup_if_checksum_is_different "$finalfail2banfilterconf"
+
+    if [ $use_template -eq 1 ]
+    then
+        # Usage 2, templates
+        cp ../conf/f2b_jail.conf $finalfail2banjailconf
+        cp ../conf/f2b_filter.conf $finalfail2banfilterconf
+
+        if [ -n "${app:-}" ]
+        then
+            ynh_replace_string "__APP__" "$app" "$finalfail2banjailconf"
+            ynh_replace_string "__APP__" "$app" "$finalfail2banfilterconf"
+        fi
+
+        # Replace all other variable given as arguments
+        for var_to_replace in $others_var
+        do
+            # ${var_to_replace^^} make the content of the variable on upper-cases
+            # ${!var_to_replace} get the content of the variable named $var_to_replace
+            ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banjailconf"
+            ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banfilterconf"
+        done
+
+    else
+        # Usage 1, no template. Build a config file from scratch.
+        test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing."
+        test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing."
+
+        tee $finalfail2banjailconf <<EOF
+[$app]
+enabled = true
+port = $ports
+filter = $app
+logpath = $logpath
+maxretry = $max_retry
+EOF
+
+    tee $finalfail2banfilterconf <<EOF
+[INCLUDES]
+before = common.conf
+[Definition]
+failregex = $failregex
+ignoreregex =
+EOF
+    fi
+
+    # Common to usage 1 and 2.
+    ynh_store_file_checksum "$finalfail2banjailconf"
+    ynh_store_file_checksum "$finalfail2banfilterconf"
+
+    ynh_systemd_action --service_name=fail2ban --action=reload --line_match="(Started|Reloaded) Fail2Ban Service" --log_path=systemd
+
+    local fail2ban_error="$(journalctl --no-hostname --unit=fail2ban | tail --lines=50 | grep "WARNING.*$app.*")"
+    if [[ -n "$fail2ban_error" ]]
+    then
+        ynh_print_err --message="Fail2ban failed to load the jail for $app"
+        ynh_print_warn --message="${fail2ban_error#*WARNING}"
+    fi
+}
+
+# Remove the dedicated fail2ban config (jail and filter conf files)
+#
+# usage: ynh_remove_fail2ban_config
+#
+# Requires YunoHost version 3.5.0 or higher.
+ynh_remove_fail2ban_config () {
+    ynh_secure_remove "/etc/fail2ban/jail.d/$app.conf"
+    ynh_secure_remove "/etc/fail2ban/filter.d/$app.conf"
+    ynh_systemd_action --service_name=fail2ban --action=reload
+}

data/helpers.d/getopts

@@ -20,7 +20,7 @@
 # | arg: $@    - Simply "$@" to tranfert all the positionnal arguments to the function
 #
 # This helper need an array, named "args_array" with all the arguments used by the helper
-# that want to use ynh_handle_getopts_args
+# 	that want to use ynh_handle_getopts_args
 # Be carreful, this array has to be an associative array, as the following example:
 # local -A args_array=( [a]=arg1 [b]=arg2= [c]=arg3 )
 # Let's explain this array:
@@ -45,10 +45,11 @@
 # e.g. for `my_helper "val1" val2`, arg1 will be filled with val1, and arg2 with val2.
 #
 # Requires YunoHost version 3.2.2 or higher.
-ynh_handle_getopts_args() {
+ynh_handle_getopts_args () {
     # Manage arguments only if there's some provided
     set +o xtrace # set +x
-    if [ $# -ne 0 ]; then
+    if [ $# -ne 0 ]
+    then
         # Store arguments in an array to keep each argument separated
         local arguments=("$@")
 
@@ -57,12 +58,14 @@ ynh_handle_getopts_args() {
         # ${!args_array[@]} is the list of all option_flags in the array (An option_flag is 'u' in [u]=user, user is a value)
         local getopts_parameters=""
         local option_flag=""
-        for option_flag in "${!args_array[@]}"; do
+        for option_flag in "${!args_array[@]}"
+        do
             # Concatenate each option_flags of the array to build the string of arguments for getopts
             # Will looks like 'abcd' for -a -b -c -d
             # If the value of an option_flag finish by =, it's an option with additionnal values. (e.g. --user bob or -u bob)
             # Check the last character of the value associate to the option_flag
-            if [ "${args_array[$option_flag]: -1}" = "=" ]; then
+            if [ "${args_array[$option_flag]: -1}" = "=" ]
+            then
                 # For an option with additionnal values, add a ':' after the letter for getopts.
                 getopts_parameters="${getopts_parameters}${option_flag}:"
             else
@@ -71,23 +74,25 @@ ynh_handle_getopts_args() {
             # Check each argument given to the function
             local arg=""
             # ${#arguments[@]} is the size of the array
-            for arg in $(seq 0 $((${#arguments[@]} - 1))); do
+            for arg in `seq 0 $(( ${#arguments[@]} - 1 ))`
+            do
                 # Escape options' values starting with -. Otherwise the - will be considered as another option.
                 arguments[arg]="${arguments[arg]//--${args_array[$option_flag]}-/--${args_array[$option_flag]}\\TOBEREMOVED\\-}"
                 # And replace long option (value of the option_flag) by the short option, the option_flag itself
                 # (e.g. for [u]=user, --user will be -u)
                 # Replace long option with = (match the beginning of the argument)
-                arguments[arg]="$(printf '%s\n' "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]}/-${option_flag} /")"
+                arguments[arg]="$(echo "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]}/-${option_flag} /")"
                 # And long option without = (match the whole line)
-                arguments[arg]="$(printf '%s\n' "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]%=}$/-${option_flag} /")"
+                arguments[arg]="$(echo "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]%=}$/-${option_flag} /")"
             done
         done
 
         # Read and parse all the arguments
         # Use a function here, to use standart arguments $@ and be able to use shift.
-        parse_arg() {
+        parse_arg () {
             # Read all arguments, until no arguments are left
-            while [ $# -ne 0 ]; do
+            while [ $# -ne 0 ]
+            do
                 # Initialize the index of getopts
                 OPTIND=1
                 # Parse with getopts only if the argument begin by -, that means the argument is an option
@@ -95,9 +100,11 @@ ynh_handle_getopts_args() {
                 local parameter=""
                 getopts ":$getopts_parameters" parameter || true
 
-                if [ "$parameter" = "?" ]; then
+                if [ "$parameter" = "?" ]
+                then
                     ynh_die --message="Invalid argument: -${OPTARG:-}"
-                elif [ "$parameter" = ":" ]; then
+                elif [ "$parameter" = ":" ]
+                then
                     ynh_die --message="-$OPTARG parameter requires an argument."
                 else
                     local shift_value=1
@@ -108,7 +115,8 @@ ynh_handle_getopts_args() {
                     local option_var="${args_array[$parameter]%=}"
                     # If this option doesn't take values
                     # if there's a '=' at the end of the long option name, this option takes values
-                    if [ "${args_array[$parameter]: -1}" != "=" ]; then
+                    if [ "${args_array[$parameter]: -1}" != "=" ]
+                    then
                         # 'eval ${option_var}' will use the content of 'option_var'
                         eval ${option_var}=1
                     else
@@ -118,35 +126,41 @@ ynh_handle_getopts_args() {
 
                         # If the first argument is longer than 2 characters,
                         # There's a value attached to the option, in the same array cell
-                        if [ ${#all_args[0]} -gt 2 ]; then
+                        if [ ${#all_args[0]} -gt 2 ]
+                        then
                             # Remove the option and the space, so keep only the value itself.
                             all_args[0]="${all_args[0]#-${parameter} }"
 
                             # At this point, if all_args[0] start with "-", then the argument is not well formed
-                            if [ "${all_args[0]:0:1}" == "-" ]; then
+                            if [ "${all_args[0]:0:1}" == "-" ]
+                            then
                                 ynh_die --message="Argument \"${all_args[0]}\" not valid! Did you use a single \"-\" instead of two?"
                             fi
                             # Reduce the value of shift, because the option has been removed manually
-                            shift_value=$((shift_value - 1))
+                            shift_value=$(( shift_value - 1 ))
                         fi
 
                         # Declare the content of option_var as a variable.
                         eval ${option_var}=""
                         # Then read the array value per value
                         local i
-                        for i in $(seq 0 $((${#all_args[@]} - 1))); do
+                        for i in `seq 0 $(( ${#all_args[@]} - 1 ))`
+                        do
                             # If this argument is an option, end here.
-                            if [ "${all_args[$i]:0:1}" == "-" ]; then
+                            if [ "${all_args[$i]:0:1}" == "-" ]
+                            then
                                 # Ignore the first value of the array, which is the option itself
                                 if [ "$i" -ne 0 ]; then
                                     break
                                 fi
                             else
                                 # Ignore empty parameters
-                                if [ -n "${all_args[$i]}" ]; then
+                                if [ -n "${all_args[$i]}" ]
+                                then
                                     # Else, add this value to this option
                                     # Each value will be separated by ';'
-                                    if [ -n "${!option_var}" ]; then
+                                    if [ -n "${!option_var}" ]
+                                    then
                                         # If there's already another value for this option, add a ; before adding the new value
                                         eval ${option_var}+="\;"
                                     fi
@@ -163,7 +177,7 @@ ynh_handle_getopts_args() {
 
                                     eval ${option_var}+='"${all_args[$i]}"'
                                 fi
-                                shift_value=$((shift_value + 1))
+                                shift_value=$(( shift_value + 1 ))
                             fi
                         done
                     fi
@@ -176,23 +190,24 @@ ynh_handle_getopts_args() {
 
         # LEGACY MODE
         # Check if there's getopts arguments
-        if [ "${arguments[0]:0:1}" != "-" ]; then
+        if [ "${arguments[0]:0:1}" != "-" ]
+        then
             # If not, enter in legacy mode and manage the arguments as positionnal ones..
             # Dot not echo, to prevent to go through a helper output. But print only in the log.
-            set -x
-            echo "! Helper used in legacy mode !" > /dev/null
-            set +x
+            set -x; echo "! Helper used in legacy mode !" > /dev/null; set +x
             local i
-            for i in $(seq 0 $((${#arguments[@]} - 1))); do
+            for i in `seq 0 $(( ${#arguments[@]} -1 ))`
+            do
                 # Try to use legacy_args as a list of option_flag of the array args_array
                 # Otherwise, fallback to getopts_parameters to get the option_flag. But an associative arrays isn't always sorted in the correct order...
                 # Remove all ':' in getopts_parameters
-                getopts_parameters=${legacy_args:-${getopts_parameters//:/}}
+                getopts_parameters=${legacy_args:-${getopts_parameters//:}}
                 # Get the option_flag from getopts_parameters, by using the option_flag according to the position of the argument.
                 option_flag=${getopts_parameters:$i:1}
-                if [ -z "$option_flag" ]; then
-                    ynh_print_warn --message="Too many arguments ! \"${arguments[$i]}\" will be ignored."
-                    continue
+                if [ -z "$option_flag" ]
+                then
+                        ynh_print_warn --message="Too many arguments ! \"${arguments[$i]}\" will be ignored."
+                        continue
                 fi
                 # Use the long option, corresponding to the option_flag, as a variable
                 # (e.g. for [u]=user, 'user' will be used as a variable)

data/helpers.d/hardware

@@ -2,20 +2,18 @@
 
 # Get the total or free amount of RAM+swap on the system
 #
-# [packagingv1]
-#
 # usage: ynh_get_ram [--free|--total] [--ignore_swap|--only_swap]
 # | arg: -f, --free         - Count free RAM+swap
 # | arg: -t, --total        - Count total RAM+swap
 # | arg: -s, --ignore_swap  - Ignore swap, consider only real RAM
 # | arg: -o, --only_swap    - Ignore real RAM, consider only swap
-# | ret: the amount of free ram, in MB (MegaBytes)
+# | ret: the amount of free ram
 #
 # Requires YunoHost version 3.8.1 or higher.
-ynh_get_ram() {
+ynh_get_ram () {
     # Declare an array to define the options of this helper.
     local legacy_args=ftso
-    local -A args_array=([f]=free [t]=total [s]=ignore_swap [o]=only_swap)
+    local -A args_array=( [f]=free [t]=total [s]=ignore_swap [o]=only_swap )
     local free
     local total
     local ignore_swap
@@ -27,34 +25,41 @@ ynh_get_ram() {
     free=${free:-0}
     total=${total:-0}
 
-    if [ $free -eq $total ]; then
+    if [ $free -eq $total ]
+    then
         ynh_print_warn --message="You have to choose --free or --total when using ynh_get_ram"
         ram=0
     # Use the total amount of ram
-    elif [ $free -eq 1 ]; then
-        local free_ram=$(LC_ALL=C vmstat --stats --unit M | grep "free memory" | awk '{print $1}')
-        local free_swap=$(LC_ALL=C vmstat --stats --unit M | grep "free swap" | awk '{print $1}')
-        local free_ram_swap=$((free_ram + free_swap))
-
+    elif [ $free -eq 1 ]
+    then
+        local free_ram=$(vmstat --stats --unit M | grep "free memory" | awk '{print $1}')
+        local free_swap=$(vmstat --stats --unit M | grep "free swap" | awk '{print $1}')
+        local free_ram_swap=$(( free_ram + free_swap ))
+        
         # Use the total amount of free ram
         local ram=$free_ram_swap
-        if [ $ignore_swap -eq 1 ]; then
+        if [ $ignore_swap -eq 1 ]
+        then
             # Use only the amount of free ram
             ram=$free_ram
-        elif [ $only_swap -eq 1 ]; then
+        elif [ $only_swap -eq 1 ]
+        then
             # Use only the amount of free swap
             ram=$free_swap
         fi
-    elif [ $total -eq 1 ]; then
-        local total_ram=$(LC_ALL=C vmstat --stats --unit M | grep "total memory" | awk '{print $1}')
-        local total_swap=$(LC_ALL=C vmstat --stats --unit M | grep "total swap" | awk '{print $1}')
-        local total_ram_swap=$((total_ram + total_swap))
-
+    elif [ $total -eq 1 ]
+    then
+        local total_ram=$(vmstat --stats --unit M | grep "total memory" | awk '{print $1}')
+        local total_swap=$(vmstat --stats --unit M | grep "total swap" | awk '{print $1}')
+        local total_ram_swap=$(( total_ram + total_swap ))
+    
         local ram=$total_ram_swap
-        if [ $ignore_swap -eq 1 ]; then
+        if [ $ignore_swap -eq 1 ]
+        then
             # Use only the amount of free ram
             ram=$total_ram
-        elif [ $only_swap -eq 1 ]; then
+        elif [ $only_swap -eq 1 ]
+        then
             # Use only the amount of free swap
             ram=$total_swap
         fi
@@ -65,21 +70,19 @@ ynh_get_ram() {
 
 # Return 0 or 1 depending if the system has a given amount of RAM+swap free or total
 #
-# [packagingv1]
-#
-# usage: ynh_require_ram --required=RAM [--free|--total] [--ignore_swap|--only_swap]
-# | arg: -r, --required=    - The amount to require, in MB
+# usage: ynh_require_ram --required=RAM required in Mb [--free|--total] [--ignore_swap|--only_swap]
+# | arg: -r, --required=    - The amount to require, in Mb
 # | arg: -f, --free         - Count free RAM+swap
 # | arg: -t, --total        - Count total RAM+swap
 # | arg: -s, --ignore_swap  - Ignore swap, consider only real RAM
 # | arg: -o, --only_swap    - Ignore real RAM, consider only swap
-# | ret: 1 if the ram is under the requirement, 0 otherwise.
+# | exit: Return 1 if the ram is under the requirement, 0 otherwise.
 #
 # Requires YunoHost version 3.8.1 or higher.
-ynh_require_ram() {
+ynh_require_ram () {
     # Declare an array to define the options of this helper.
     local legacy_args=rftso
-    local -A args_array=([r]=required= [f]=free [t]=total [s]=ignore_swap [o]=only_swap)
+    local -A args_array=( [r]=required= [f]=free [t]=total [s]=ignore_swap [o]=only_swap )
     local required
     local free
     local total
@@ -97,7 +100,8 @@ ynh_require_ram() {
 
     local ram=$(ynh_get_ram $free $total $ignore_swap $only_swap)
 
-    if [ $ram -lt $required ]; then
+    if [ $ram -lt $required ]
+    then
         return 1
     else
         return 0

data/helpers.d/logging

@@ -10,7 +10,7 @@
 ynh_die() {
     # Declare an array to define the options of this helper.
     local legacy_args=mc
-    local -A args_array=([m]=message= [c]=ret_code=)
+    local -A args_array=( [m]=message= [c]=ret_code= )
     local message
     local ret_code
     # Manage arguments with getopts
@@ -30,12 +30,31 @@ ynh_die() {
 ynh_print_info() {
     # Declare an array to define the options of this helper.
     local legacy_args=m
-    local -A args_array=([m]=message=)
+    local -A args_array=( [m]=message= )
     local message
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    echo "$message" >&$YNH_STDINFO
+    echo "$message" >> "$YNH_STDINFO"
+}
+
+# Ignore the yunohost-cli log to prevent errors with conditional commands
+#
+# [internal]
+#
+# usage: ynh_no_log COMMAND
+#
+# Simply duplicate the log, execute the yunohost command and replace the log without the result of this command
+# It's a very badly hack...
+#
+# Requires YunoHost version 2.6.4 or higher.
+ynh_no_log() {
+    local ynh_cli_log=/var/log/yunohost/yunohost-cli.log
+    cp --archive ${ynh_cli_log} ${ynh_cli_log}-move
+    eval $@
+    local exit_code=$?
+    mv ${ynh_cli_log}-move ${ynh_cli_log}
+    return $exit_code
 }
 
 # Main printer, just in case in the future we have to change anything about that.
@@ -43,7 +62,7 @@ ynh_print_info() {
 # [internal]
 #
 # Requires YunoHost version 3.2.0 or higher.
-ynh_print_log() {
+ynh_print_log () {
     echo -e "${1}"
 }
 
@@ -53,10 +72,10 @@ ynh_print_log() {
 # | arg: -m, --message=     - The text to print
 #
 # Requires YunoHost version 3.2.0 or higher.
-ynh_print_warn() {
+ynh_print_warn () {
     # Declare an array to define the options of this helper.
     local legacy_args=m
-    local -A args_array=([m]=message=)
+    local -A args_array=( [m]=message= )
     local message
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
@@ -70,10 +89,10 @@ ynh_print_warn() {
 # | arg: -m, --message=     - The text to print
 #
 # Requires YunoHost version 3.2.0 or higher.
-ynh_print_err() {
+ynh_print_err () {
     # Declare an array to define the options of this helper.
     local legacy_args=m
-    local -A args_array=([m]=message=)
+    local -A args_array=( [m]=message= )
     local message
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
@@ -83,145 +102,96 @@ ynh_print_err() {
 
 # Execute a command and print the result as an error
 #
-# usage: ynh_exec_err your command and args
+# usage: ynh_exec_err your_command
+# usage: ynh_exec_err "your_command | other_command"
 # | arg: command - command to execute
 #
-# Note that you should NOT quote the command but only prefix it with ynh_exec_err
+# When using pipes, double quotes are required - otherwise, this helper will run the first command, and the whole output will be sent through the next pipe.
+#
+# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed.
 #
 # Requires YunoHost version 3.2.0 or higher.
-ynh_exec_err() {
-    # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes,
-    # (because in the past eval was used) ...
-    # we detect this by checking that there's no 2nd arg, and $1 contains a space
-    if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]]; then
-        ynh_print_err --message="$(eval $@)"
-    else
-        # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077
-        ynh_print_err --message="$("$@")"
-    fi
+ynh_exec_err () {
+    ynh_print_err "$(eval $@)"
 }
 
 # Execute a command and print the result as a warning
 #
-# usage: ynh_exec_warn your command and args
+# usage: ynh_exec_warn your_command
+# usage: ynh_exec_warn "your_command | other_command"
 # | arg: command - command to execute
 #
-# Note that you should NOT quote the command but only prefix it with ynh_exec_warn
+# When using pipes, double quotes are required - otherwise, this helper will run the first command, and the whole output will be sent through the next pipe.
+#
+# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed.
 #
 # Requires YunoHost version 3.2.0 or higher.
-ynh_exec_warn() {
-    # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes,
-    # (because in the past eval was used) ...
-    # we detect this by checking that there's no 2nd arg, and $1 contains a space
-    if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]]; then
-        ynh_print_warn --message="$(eval $@)"
-    else
-        # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077
-        ynh_print_warn --message="$("$@")"
-    fi
+ynh_exec_warn () {
+    ynh_print_warn "$(eval $@)"
 }
 
 # Execute a command and force the result to be printed on stdout
 #
-# usage: ynh_exec_warn_less your command and args
+# usage: ynh_exec_warn_less your_command
+# usage: ynh_exec_warn_less "your_command | other_command"
 # | arg: command - command to execute
 #
-# Note that you should NOT quote the command but only prefix it with ynh_exec_warn
+# When using pipes, double quotes are required - otherwise, this helper will run the first command, and the whole output will be sent through the next pipe.
+#
+# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed.
 #
 # Requires YunoHost version 3.2.0 or higher.
-ynh_exec_warn_less() {
-    # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes,
-    # (because in the past eval was used) ...
-    # we detect this by checking that there's no 2nd arg, and $1 contains a space
-    if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]]; then
-        eval $@ 2>&1
-    else
-        # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077
-        "$@" 2>&1
-    fi
+ynh_exec_warn_less () {
+    eval $@ 2>&1
 }
 
 # Execute a command and redirect stdout in /dev/null
 #
-# usage: ynh_exec_quiet your command and args
+# usage: ynh_exec_quiet your_command
+# usage: ynh_exec_quiet "your_command | other_command"
 # | arg: command - command to execute
 #
-# Note that you should NOT quote the command but only prefix it with ynh_exec_warn
+# When using pipes, double quotes are required - otherwise, this helper will run the first command, and the whole output will be sent through the next pipe.
+#
+# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed.
 #
 # Requires YunoHost version 3.2.0 or higher.
-ynh_exec_quiet() {
-    # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes,
-    # (because in the past eval was used) ...
-    # we detect this by checking that there's no 2nd arg, and $1 contains a space
-    if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]]; then
-        eval $@ > /dev/null
-    else
-        # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077
-        "$@" > /dev/null
-    fi
+ynh_exec_quiet () {
+    eval $@ > /dev/null
 }
 
 # Execute a command and redirect stdout and stderr in /dev/null
 #
-# usage: ynh_exec_quiet your command and args
+# usage: ynh_exec_fully_quiet your_command
+# usage: ynh_exec_fully_quiet "your_command | other_command"
 # | arg: command - command to execute
 #
-# Note that you should NOT quote the command but only prefix it with ynh_exec_quiet
+# When using pipes, double quotes are required - otherwise, this helper will run the first command, and the whole output will be sent through the next pipe.
+#
+# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed.
 #
 # Requires YunoHost version 3.2.0 or higher.
-ynh_exec_fully_quiet() {
-    # Boring legacy handling for when people calls ynh_exec_* wrapping the command in quotes,
-    # (because in the past eval was used) ...
-    # we detect this by checking that there's no 2nd arg, and $1 contains a space
-    if [[ "$#" -eq 1 ]] && [[ "$1" == *" "* ]]; then
-        eval $@ > /dev/null 2>&1
-    else
-        # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077
-        "$@" > /dev/null 2>&1
-    fi
-}
-
-# Execute a command and redirect stderr in /dev/null. Print stderr on error.
-#
-# usage: ynh_exec_and_print_stderr_only_if_error your command and args
-# | arg: command - command to execute
-#
-# Note that you should NOT quote the command but only prefix it with ynh_exec_and_print_stderr_only_if_error
-#
-# Requires YunoHost version 11.2 or higher.
-ynh_exec_and_print_stderr_only_if_error() {
-    logfile="$(mktemp)"
-    rc=0
-    # Note that "$@" is used and not $@, c.f. https://unix.stackexchange.com/a/129077
-    "$@" 2> "$logfile" || rc="$?"
-    if ((rc != 0)); then
-        ynh_exec_warn cat "$logfile"
-        ynh_secure_remove "$logfile"
-        return "$rc"
-    fi
+ynh_exec_fully_quiet () {
+    eval $@ > /dev/null 2>&1
 }
 
 # Remove any logs for all the following commands.
 #
 # usage: ynh_print_OFF
 #
-# [internal]
-#
 # WARNING: You should be careful with this helper, and never forget to use ynh_print_ON as soon as possible to restore the logging.
 #
 # Requires YunoHost version 3.2.0 or higher.
-ynh_print_OFF() {
-    exec {BASH_XTRACEFD}> /dev/null
+ynh_print_OFF () {
+    exec {BASH_XTRACEFD}>/dev/null
 }
 
 # Restore the logging after ynh_print_OFF
 #
 # usage: ynh_print_ON
 #
-# [internal]
-#
 # Requires YunoHost version 3.2.0 or higher.
-ynh_print_ON() {
+ynh_print_ON () {
     exec {BASH_XTRACEFD}>&1
     # Print an echo only for the log, to be able to know that ynh_print_ON has been called.
     echo ynh_print_ON > /dev/null
@@ -249,11 +219,11 @@ base_time=$(date +%s)
 # | arg: -l, --last     - Use for the last call of the helper, to fill the progression bar.
 #
 # Requires YunoHost version 3.5.0 or higher.
-ynh_script_progression() {
+ynh_script_progression () {
     set +o xtrace # set +x
     # Declare an array to define the options of this helper.
     local legacy_args=mwtl
-    local -A args_array=([m]=message= [w]=weight= [t]=time [l]=last)
+    local -A args_array=( [m]=message= [w]=weight= [t]=time [l]=last )
     local message
     local weight
     local time
@@ -263,22 +233,16 @@ ynh_script_progression() {
     # Re-disable xtrace, ynh_handle_getopts_args set it back
     set +o xtrace # set +x
     weight=${weight:-1}
-
-    # Always activate time when running inside CI tests
-    if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then
-        time=${time:-1}
-    else
-        time=${time:-0}
-    fi
-
+    time=${time:-0}
     last=${last:-0}
 
     # Get execution time since the last $base_time
-    local exec_time=$(($(date +%s) - $base_time))
+    local exec_time=$(( $(date +%s) - $base_time ))
     base_time=$(date +%s)
 
     # Compute $max_progression (if we didn't already)
-    if [ "$max_progression" = -1 ]; then
+    if [ "$max_progression" = -1 ]
+    then
         # Get the number of occurrences of 'ynh_script_progression' in the script. Except those are commented.
         local helper_calls="$(grep --count "^[^#]*ynh_script_progression" $0)"
         # Get the number of call with a weight value
@@ -290,22 +254,23 @@ ynh_script_progression() {
         local weight_valuesB="$(grep --perl-regexp "^[^#]*ynh_script_progression.*-w " $0 | sed 's/.*-w[= ]\([[:digit:]]*\).*/\1/g')"
         # Each value will be on a different line.
         # Remove each 'end of line' and replace it by a '+' to sum the values.
-        local weight_values=$(($(echo "$weight_valuesA" "$weight_valuesB" | grep -v -E '^\s*$' | tr '\n' '+' | sed 's/+$/+0/g')))
+        local weight_values=$(( $(echo "$weight_valuesA" | tr '\n' '+') + $(echo "$weight_valuesB" | tr '\n' '+') 0 ))
 
         # max_progression is a total number of calls to this helper.
-        #  Less the number of calls with a weight value.
-        #  Plus the total of weight values
-        max_progression=$(($helper_calls - $weight_calls + $weight_values))
+        # 	Less the number of calls with a weight value.
+        # 	Plus the total of weight values
+        max_progression=$(( $helper_calls - $weight_calls + $weight_values ))
     fi
 
     # Increment each execution of ynh_script_progression in this script by the weight of the previous call.
-    increment_progression=$(($increment_progression + $previous_weight))
+    increment_progression=$(( $increment_progression + $previous_weight ))
     # Store the weight of the current call in $previous_weight for next call
     previous_weight=$weight
 
     # Reduce $increment_progression to the size of the scale
-    if [ $last -eq 0 ]; then
-        local effective_progression=$(($increment_progression * $progress_scale / $max_progression))
+    if [ $last -eq 0 ]
+    then
+        local effective_progression=$(( $increment_progression * $progress_scale / $max_progression ))
     # If last is specified, fill immediately the progression_bar
     else
         local effective_progression=$progress_scale
@@ -313,30 +278,100 @@ ynh_script_progression() {
 
     # Build $progression_bar from progress_string(0,1,2) according to $effective_progression and the weight of the current task
     # expected_progression is the progression expected after the current task
-    local expected_progression="$((($increment_progression + $weight) * $progress_scale / $max_progression - $effective_progression))"
-    if [ $last -eq 1 ]; then
+    local expected_progression="$(( ( $increment_progression + $weight ) * $progress_scale / $max_progression - $effective_progression ))"
+    if [ $last -eq 1 ]
+    then
         expected_progression=0
     fi
     # left_progression is the progression not yet done
-    local left_progression="$(($progress_scale - $effective_progression - $expected_progression))"
+    local left_progression="$(( $progress_scale - $effective_progression - $expected_progression ))"
     # Build the progression bar with $effective_progression, work done, $expected_progression, current work and $left_progression, work to be done.
     local progression_bar="${progress_string2:0:$effective_progression}${progress_string1:0:$expected_progression}${progress_string0:0:$left_progression}"
 
     local print_exec_time=""
-    if [ $time -eq 1 ] && [ "$exec_time" -gt 10 ]; then
-        print_exec_time=" [$(bc <<< "scale=1; $exec_time / 60") minutes]"
+    if [ $time -eq 1 ]
+    then
+        print_exec_time=" [$(date +%Hh%Mm,%Ss --date="0 + $exec_time sec")]"
     fi
 
     ynh_print_info "[$progression_bar] > ${message}${print_exec_time}"
     set -o xtrace # set -x
 }
 
-# Return data to the YunoHost core for later processing
+# Return data to the Yunohost core for later processing
 # (to be used by special hooks like app config panel and core diagnosis)
 #
 # usage: ynh_return somedata
 #
 # Requires YunoHost version 3.6.0 or higher.
-ynh_return() {
+ynh_return () {
     echo "$1" >> "$YNH_STDRETURN"
 }
+
+# Debugger for app packagers
+#
+# usage: ynh_debug [--message=message] [--trace=1/0]
+# | arg: -m, --message= - The text to print
+# | arg: -t, --trace=   - Turn on or off the trace of the script. Usefull to trace nonly a small part of a script.
+#
+# Requires YunoHost version 3.5.0 or higher.
+ynh_debug () {
+    # Disable set xtrace for the helper itself, to not pollute the debug log
+    set +o xtrace # set +x
+    # Declare an array to define the options of this helper.
+    local legacy_args=mt
+    local -A args_array=( [m]=message= [t]=trace= )
+    local message
+    local trace
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    # Re-disable xtrace, ynh_handle_getopts_args set it back
+    set +o xtrace # set +x
+    message=${message:-}
+    trace=${trace:-}
+
+    if [ -n "$message" ]
+    then
+        ynh_print_log "[Debug] ${message}" >&2
+    fi
+
+    if [ "$trace" == "1" ]
+    then
+        ynh_debug --message="Enable debugging"
+        set +o xtrace # set +x
+        # Get the current file descriptor of xtrace
+        old_bash_xtracefd=$BASH_XTRACEFD
+        # Add the current file name and the line number of any command currently running while tracing.
+        PS4='$(basename ${BASH_SOURCE[0]})-L${LINENO}: '
+        # Force xtrace to stderr
+        BASH_XTRACEFD=2
+        # Force stdout to stderr
+        exec 1>&2
+    fi
+    if [ "$trace" == "0" ]
+    then
+        ynh_debug --message="Disable debugging"
+        set +o xtrace # set +x
+        # Put xtrace back to its original fild descriptor
+        BASH_XTRACEFD=$old_bash_xtracefd
+        # Restore stdout
+        exec 1>&1
+    fi
+    # Renable set xtrace
+    set -o xtrace # set -x
+}
+
+# Execute a command and print the result as debug
+#
+# usage: ynh_debug_exec your_command
+# usage: ynh_debug_exec "your_command | other_command"
+# | arg: command - command to execute
+#
+# When using pipes, double quotes are required - otherwise, this helper will run the first command, and the whole output will be sent through the next pipe.
+#
+# If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed.
+#
+# Requires YunoHost version 3.5.0 or higher.
+ynh_debug_exec () {
+    ynh_debug --message="$(eval $@)"
+}

data/helpers.d/logrotate

@@ -0,0 +1,112 @@
+#!/bin/bash
+
+# Use logrotate to manage the logfile
+#
+# usage: ynh_use_logrotate [--logfile=/log/file] [--nonappend] [--specific_user=user/group]
+# | arg: -l, --logfile=         - absolute path of logfile
+# | arg: -n, --nonappend        - (optional) Replace the config file instead of appending this new config.
+# | arg: -u, --specific_user=   - run logrotate as the specified user and group. If not specified logrotate is runned as root.
+#
+# If no --logfile is provided, /var/log/${app} will be used as default.
+# logfile can be just a directory, or a full path to a logfile :
+# /parentdir/logdir
+# /parentdir/logdir/logfile.log
+#
+# It's possible to use this helper multiple times, each config will be added to
+# the same logrotate config file.  Unless you use the option --non-append
+#
+# Requires YunoHost version 2.6.4 or higher.
+# Requires YunoHost version 3.2.0 or higher for the argument --specific_user
+ynh_use_logrotate () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=lnuya
+    local -A args_array=( [l]=logfile= [n]=nonappend [u]=specific_user= [y]=non [a]=append )
+    # [y]=non [a]=append are only for legacy purpose, to not fail on the old option '--non-append'
+    local logfile
+    local nonappend
+    local specific_user
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    logfile="${logfile:-}"
+    nonappend="${nonappend:-0}"
+    specific_user="${specific_user:-}"
+
+    # LEGACY CODE - PRE GETOPTS
+    if [ $# -gt 0 ] && [ "$1" == "--non-append" ]
+    then
+        nonappend=1
+        # Destroy this argument for the next command.
+        shift
+    elif [ $# -gt 1 ] && [ "$2" == "--non-append" ]
+    then
+        nonappend=1
+    fi
+
+    if [ $# -gt 0 ] && [ "$(echo ${1:0:1})" != "-" ]
+    then
+        # If the given logfile parameter already exists as a file, or if it ends up with ".log",
+        # we just want to manage a single file
+        if [ -f "$1" ] || [ "$(echo ${1##*.})" == "log" ]
+        then
+            local logfile=$1
+        # Otherwise we assume we want to manage a directory and all its .log file inside
+        else
+            local logfile=$1/*.log
+        fi
+    fi
+    # LEGACY CODE
+
+    local customtee="tee --append"
+    if [ "$nonappend" -eq 1 ]; then
+        customtee="tee"
+    fi
+    if [ -n "$logfile" ]
+    then
+        if [ ! -f "$1" ] && [ "$(echo ${logfile##*.})" != "log" ]; then	# Keep only the extension to check if it's a logfile
+            local logfile="$logfile/*.log"	# Else, uses the directory and all logfile into it.
+        fi
+    else
+        logfile="/var/log/${app}/*.log" # Without argument, use a defaut directory in /var/log
+    fi
+    local su_directive=""
+    if [[ -n $specific_user ]]
+    then
+        su_directive="	# Run logorotate as specific user - group
+    su ${specific_user%/*} ${specific_user#*/}"
+    fi
+
+    cat > ./${app}-logrotate << EOF	# Build a config file for logrotate
+$logfile {
+        # Rotate if the logfile exceeds 100Mo
+    size 100M
+        # Keep 12 old log maximum
+    rotate 12
+        # Compress the logs with gzip
+    compress
+        # Compress the log at the next cycle. So keep always 2 non compressed logs
+    delaycompress
+        # Copy and truncate the log to allow to continue write on it. Instead of move the log.
+    copytruncate
+        # Do not do an error if the log is missing
+    missingok
+        # Not rotate if the log is empty
+    notifempty
+        # Keep old logs in the same dir
+    noolddir
+    $su_directive
+}
+EOF
+    mkdir --parents $(dirname "$logfile")	# Create the log directory, if not exist
+    cat ${app}-logrotate | $customtee /etc/logrotate.d/$app > /dev/null	# Append this config to the existing config file, or replace the whole config file (depending on $customtee)
+}
+
+# Remove the app's logrotate config.
+#
+# usage: ynh_remove_logrotate
+#
+# Requires YunoHost version 2.6.4 or higher.
+ynh_remove_logrotate () {
+    if [ -e "/etc/logrotate.d/$app" ]; then
+        rm "/etc/logrotate.d/$app"
+    fi
+}

data/helpers.d/mysql

@@ -1,21 +1,22 @@
 #!/bin/bash
 
+MYSQL_ROOT_PWD_FILE=/etc/yunohost/mysql
+
 # Open a connection as a user
 #
+# example: ynh_mysql_connect_as --user="user" --password="pass" <<< "UPDATE ...;"
+# example: ynh_mysql_connect_as --user="user" --password="pass" < /path/to/file.sql
+#
 # usage: ynh_mysql_connect_as --user=user --password=password [--database=database]
 # | arg: -u, --user=        - the user name to connect as
 # | arg: -p, --password=    - the user password
 # | arg: -d, --database=    - the database to connect to
 #
-# examples:
-#    ynh_mysql_connect_as --user="user" --password="pass" <<< "UPDATE ...;"
-#    ynh_mysql_connect_as --user="user" --password="pass" < /path/to/file.sql
-#
 # Requires YunoHost version 2.2.4 or higher.
 ynh_mysql_connect_as() {
     # Declare an array to define the options of this helper.
     local legacy_args=upd
-    local -A args_array=([u]=user= [p]=password= [d]=database=)
+    local -A args_array=( [u]=user= [p]=password= [d]=database= )
     local user
     local password
     local database
@@ -36,18 +37,20 @@ ynh_mysql_connect_as() {
 ynh_mysql_execute_as_root() {
     # Declare an array to define the options of this helper.
     local legacy_args=sd
-    local -A args_array=([s]=sql= [d]=database=)
+    local -A args_array=( [s]=sql= [d]=database= )
     local sql
     local database
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
     database="${database:-}"
 
-    if [ -n "$database" ]; then
+    if [ -n "$database" ]
+    then
         database="--database=$database"
     fi
 
-    mysql -B "$database" <<< "$sql"
+    ynh_mysql_connect_as --user="root" --password="$(cat $MYSQL_ROOT_PWD_FILE)" \
+        $database <<< "$sql"
 }
 
 # Execute a command from a file as root user
@@ -60,18 +63,21 @@ ynh_mysql_execute_as_root() {
 ynh_mysql_execute_file_as_root() {
     # Declare an array to define the options of this helper.
     local legacy_args=fd
-    local -A args_array=([f]=file= [d]=database=)
+    local -A args_array=( [f]=file= [d]=database= )
     local file
     local database
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
     database="${database:-}"
 
-    if [ -n "$database" ]; then
+    if [ -n "$database" ]
+    then
         database="--database=$database"
     fi
 
-    mysql -B "$database" < "$file"
+
+    ynh_mysql_connect_as --user="root" --password="$(cat $MYSQL_ROOT_PWD_FILE)" \
+        $database < "$file"
 }
 
 # Create a database and grant optionnaly privilegies to a user
@@ -90,7 +96,8 @@ ynh_mysql_create_db() {
     local sql="CREATE DATABASE ${db};"
 
     # grant all privilegies to user
-    if [[ $# -gt 1 ]]; then
+    if [[ $# -gt 1 ]]
+    then
         sql+=" GRANT ALL PRIVILEGES ON ${db}.* TO '${2}'@'localhost'"
         if [[ -n ${3:-} ]]; then
             sql+=" IDENTIFIED BY '${3}'"
@@ -118,22 +125,22 @@ ynh_mysql_drop_db() {
 
 # Dump a database
 #
+# example: ynh_mysql_dump_db --database=roundcube > ./dump.sql
+#
 # usage: ynh_mysql_dump_db --database=database
 # | arg: -d, --database=    - the database name to dump
-# | ret: The mysqldump output
-#
-# example: ynh_mysql_dump_db --database=roundcube > ./dump.sql
+# | ret: the mysqldump output
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_mysql_dump_db() {
     # Declare an array to define the options of this helper.
     local legacy_args=d
-    local -A args_array=([d]=database=)
+    local -A args_array=( [d]=database= )
     local database
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    mysqldump --single-transaction --skip-dump-date --routines "$database"
+    mysqldump --user="root" --password="$(cat $MYSQL_ROOT_PWD_FILE)" --single-transaction --skip-dump-date "$database"
 }
 
 # Create a user
@@ -152,41 +159,28 @@ ynh_mysql_create_user() {
 
 # Check if a mysql user exists
 #
-# [internal]
-#
 # usage: ynh_mysql_user_exists --user=user
 # | arg: -u, --user=    - the user for which to check existence
-# | ret: 0 if the user exists, 1 otherwise.
+# | exit: Return 1 if the user doesn't exist, 0 otherwise.
 #
 # Requires YunoHost version 2.2.4 or higher.
-ynh_mysql_user_exists() {
+ynh_mysql_user_exists()
+{
     # Declare an array to define the options of this helper.
     local legacy_args=u
-    local -A args_array=([u]=user=)
+    local -A args_array=( [u]=user= )
     local user
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    if [[ -z $(ynh_mysql_execute_as_root --sql="SELECT User from mysql.user WHERE User = '$user';") ]]; then
+    if [[ -z $(ynh_mysql_execute_as_root --sql="SELECT User from mysql.user WHERE User = '$user';") ]]
+    then
         return 1
     else
         return 0
     fi
 }
 
-# Check if a mysql database exists
-#
-# [internal]
-#
-# usage: ynh_mysql_database_exists database
-# | arg: database    - the database for which to check existence
-# | exit: Return 1 if the database doesn't exist, 0 otherwise
-#
-ynh_mysql_database_exists() {
-    local database=$1
-    mysqlshow | grep -qE "^|\s+$database\s+|"
-}
-
 # Drop a user
 #
 # [internal]
@@ -201,56 +195,53 @@ ynh_mysql_drop_user() {
 
 # Create a database, an user and its password. Then store the password in the app's config
 #
-# [packagingv1]
-#
 # usage: ynh_mysql_setup_db --db_user=user --db_name=name [--db_pwd=pwd]
 # | arg: -u, --db_user=     - Owner of the database
 # | arg: -n, --db_name=     - Name of the database
 # | arg: -p, --db_pwd=      - Password of the database. If not provided, a password will be generated
 #
-# After executing this helper, the password of the created database will be available in `$db_pwd`
-# It will also be stored as "`mysqlpwd`" into the app settings.
+# After executing this helper, the password of the created database will be available in $db_pwd
+# It will also be stored as "mysqlpwd" into the app settings.
 #
 # Requires YunoHost version 2.6.4 or higher.
-ynh_mysql_setup_db() {
+ynh_mysql_setup_db () {
     # Declare an array to define the options of this helper.
     local legacy_args=unp
-    local -A args_array=([u]=db_user= [n]=db_name= [p]=db_pwd=)
+    local -A args_array=( [u]=db_user= [n]=db_name= [p]=db_pwd= )
     local db_user
     local db_name
     db_pwd=""
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    # Generate a random password
-    local new_db_pwd=$(ynh_string_random)
+    local new_db_pwd=$(ynh_string_random)	# Generate a random password
     # If $db_pwd is not provided, use new_db_pwd instead for db_pwd
     db_pwd="${db_pwd:-$new_db_pwd}"
 
-    ynh_mysql_create_db "$db_name" "$db_user" "$db_pwd"
-    ynh_app_setting_set --app=$app --key=mysqlpwd --value=$db_pwd
+    ynh_mysql_create_db "$db_name" "$db_user" "$db_pwd"	# Create the database
+    ynh_app_setting_set --app=$app --key=mysqlpwd --value=$db_pwd	# Store the password in the app's config
 }
 
 # Remove a database if it exists, and the associated user
 #
-# [packagingv1]
-#
 # usage: ynh_mysql_remove_db --db_user=user --db_name=name
 # | arg: -u, --db_user=     - Owner of the database
 # | arg: -n, --db_name=     - Name of the database
 #
 # Requires YunoHost version 2.6.4 or higher.
-ynh_mysql_remove_db() {
+ynh_mysql_remove_db () {
     # Declare an array to define the options of this helper.
     local legacy_args=un
-    local -Ar args_array=([u]=db_user= [n]=db_name=)
+    local -A args_array=( [u]=db_user= [n]=db_name= )
     local db_user
     local db_name
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    if ynh_mysql_database_exists "$db_name"; then
-        ynh_mysql_drop_db $db_name
+    local mysql_root_password=$(cat $MYSQL_ROOT_PWD_FILE)
+    if mysqlshow --user=root --password=$mysql_root_password | grep --quiet "^| $db_name"
+    then	# Check if the database exists
+        ynh_mysql_drop_db $db_name	# Remove the database	
     else
         ynh_print_warn --message="Database $db_name not found"
     fi

data/helpers.d/network

@@ -2,60 +2,55 @@
 
 # Find a free port and return it
 #
-# [packagingv1]
+# example: port=$(ynh_find_port --port=8080)
 #
 # usage: ynh_find_port --port=begin_port
 # | arg: -p, --port=    - port to start to search
 # | ret: the port number
 #
-# example: port=$(ynh_find_port --port=8080)
-#
 # Requires YunoHost version 2.6.4 or higher.
-ynh_find_port() {
+ynh_find_port () {
     # Declare an array to define the options of this helper.
     local legacy_args=p
-    local -A args_array=([p]=port=)
+    local -A args_array=( [p]=port= )
     local port
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
     test -n "$port" || ynh_die --message="The argument of ynh_find_port must be a valid port."
-    while ! ynh_port_available --port=$port; do
-        port=$((port + 1))
+    while ! ynh_port_available --port=$port
+    do
+        port=$((port+1))	# Else, pass to next port
     done
     echo $port
 }
 
 # Test if a port is available
 #
-# [packagingv1]
+# example: ynh_port_available --port=1234 || ynh_die "Port 1234 is needs to be available for this app"
 #
 # usage: ynh_find_port --port=XYZ
 # | arg: -p, --port=    - port to check
-# | ret: 0 if the port is available, 1 if it is already used by another process.
-#
-# example: ynh_port_available --port=1234 || ynh_die --message="Port 1234 is needs to be available for this app"
+# | exit: Return 1 if the port is already used by another process.
 #
 # Requires YunoHost version 3.8.0 or higher.
-ynh_port_available() {
+ynh_port_available () {
     # Declare an array to define the options of this helper.
     local legacy_args=p
-    local -A args_array=([p]=port=)
+    local -A args_array=( [p]=port= )
     local port
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    # Check if the port is free
-    if ss --numeric --listening --tcp --udp | awk '{print$5}' | grep --quiet --extended-regexp ":$port$"; then
-        return 1
-    # This is to cover (most) case where an app is using a port yet ain't currently using it for some reason (typically service ain't up)
-    elif grep -q "port: '$port'" /etc/yunohost/apps/*/settings.yml; then
+    if ss --numeric --listening --tcp --udp | awk '{print$5}' | grep --quiet --extended-regexp ":$port$"       # Check if the port is free
+    then
         return 1
     else
         return 0
     fi
 }
 
+
 # Validate an IP address
 #
 # [internal]
@@ -66,12 +61,13 @@ ynh_port_available() {
 # example: ynh_validate_ip 4 111.222.333.444
 #
 # Requires YunoHost version 2.2.4 or higher.
-ynh_validate_ip() {
+ynh_validate_ip()
+{
     # http://stackoverflow.com/questions/319279/how-to-validate-ip-address-in-python#319298
 
     # Declare an array to define the options of this helper.
     local legacy_args=fi
-    local -A args_array=([f]=family= [i]=ip_address=)
+    local -A args_array=( [f]=family= [i]=ip_address= )
     local family
     local ip_address
     # Manage arguments with getopts
@@ -79,7 +75,7 @@ ynh_validate_ip() {
 
     [ "$family" == "4" ] || [ "$family" == "6" ] || return 1
 
-    python3 /dev/stdin << EOF
+  python /dev/stdin << EOF
 import socket
 import sys
 family = { "4" : socket.AF_INET, "6" : socket.AF_INET6 }
@@ -93,17 +89,18 @@ EOF
 
 # Validate an IPv4 address
 #
+# example: ynh_validate_ip4 111.222.333.444
+#
 # usage: ynh_validate_ip4 --ip_address=ip_address
 # | arg: -i, --ip_address=     - the ipv4 address to check
 # | ret: 0 for valid ipv4 addresses, 1 otherwise
 #
-# example: ynh_validate_ip4 111.222.333.444
-#
 # Requires YunoHost version 2.2.4 or higher.
-ynh_validate_ip4() {
+ynh_validate_ip4()
+{
     # Declare an array to define the options of this helper.
     local legacy_args=i
-    local -A args_array=([i]=ip_address=)
+    local -A args_array=( [i]=ip_address= )
     local ip_address
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
@@ -111,19 +108,21 @@ ynh_validate_ip4() {
     ynh_validate_ip --family=4 --ip_address=$ip_address
 }
 
+
 # Validate an IPv6 address
 #
-# usage: ynh_validate_ip6 --ip_address=ip_address
-# | arg: -i, --ip_address=  - the ipv6 address to check
-# | ret: 0 for valid ipv6 addresses, 1 otherwise
-#
 # example: ynh_validate_ip6 2000:dead:beef::1
 #
+# usage: ynh_validate_ip6 --ip_address=ip_address
+# | arg: -i, --ip_address=     - the ipv6 address to check
+# | ret: 0 for valid ipv6 addresses, 1 otherwise
+#
 # Requires YunoHost version 2.2.4 or higher.
-ynh_validate_ip6() {
+ynh_validate_ip6()
+{
     # Declare an array to define the options of this helper.
     local legacy_args=i
-    local -A args_array=([i]=ip_address=)
+    local -A args_array=( [i]=ip_address= )
     local ip_address
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"

data/helpers.d/nginx

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# Create a dedicated nginx config
+#
+# usage: ynh_add_nginx_config "list of others variables to replace"
+#
+# | arg: list - (Optional) list of others variables to replace separated by spaces. For example : 'path_2 port_2 ...'
+#
+# This will use a template in ../conf/nginx.conf
+#   __PATH__      by  $path_url
+#   __DOMAIN__    by  $domain
+#   __PORT__      by  $port
+#   __NAME__      by  $app
+#   __FINALPATH__ by  $final_path
+#   __PHPVERSION__ by  $YNH_PHP_VERSION ($YNH_PHP_VERSION is either the default php version or the version defined for the app)
+#
+#  And dynamic variables (from the last example) :
+#   __PATH_2__    by $path_2
+#   __PORT_2__    by $port_2
+#
+# Requires YunoHost version 2.7.2 or higher.
+# Requires YunoHost version 2.7.13 or higher for dynamic variables
+ynh_add_nginx_config () {
+    finalnginxconf="/etc/nginx/conf.d/$domain.d/$app.conf"
+    local others_var=${1:-}
+    ynh_backup_if_checksum_is_different --file="$finalnginxconf"
+    cp ../conf/nginx.conf "$finalnginxconf"
+
+    # To avoid a break by set -u, use a void substitution ${var:-}. If the variable is not set, it's simply set with an empty variable.
+    # Substitute in a nginx config file only if the variable is not empty
+    if test -n "${path_url:-}"
+    then
+        # path_url_slash_less is path_url, or a blank value if path_url is only '/'
+        local path_url_slash_less=${path_url%/}
+        ynh_replace_string --match_string="__PATH__/" --replace_string="$path_url_slash_less/" --target_file="$finalnginxconf"
+        ynh_replace_string --match_string="__PATH__" --replace_string="$path_url" --target_file="$finalnginxconf"
+    fi
+    if test -n "${domain:-}"; then
+        ynh_replace_string --match_string="__DOMAIN__" --replace_string="$domain" --target_file="$finalnginxconf"
+    fi
+    if test -n "${port:-}"; then
+        ynh_replace_string --match_string="__PORT__" --replace_string="$port" --target_file="$finalnginxconf"
+    fi
+    if test -n "${app:-}"; then
+        ynh_replace_string --match_string="__NAME__" --replace_string="$app" --target_file="$finalnginxconf"
+    fi
+    if test -n "${final_path:-}"; then
+        ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalnginxconf"
+    fi
+    ynh_replace_string --match_string="__PHPVERSION__" --replace_string="$YNH_PHP_VERSION" --target_file="$finalnginxconf"
+
+    # Replace all other variable given as arguments
+    for var_to_replace in $others_var
+    do
+        # ${var_to_replace^^} make the content of the variable on upper-cases
+        # ${!var_to_replace} get the content of the variable named $var_to_replace 
+        ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalnginxconf"
+    done
+
+    if [ "${path_url:-}" != "/" ]
+    then
+        ynh_replace_string --match_string="^#sub_path_only" --replace_string="" --target_file="$finalnginxconf"
+    else
+        ynh_replace_string --match_string="^#root_path_only" --replace_string="" --target_file="$finalnginxconf"
+    fi
+
+    ynh_store_file_checksum --file="$finalnginxconf"
+
+    ynh_systemd_action --service_name=nginx --action=reload
+}
+
+# Remove the dedicated nginx config
+#
+# usage: ynh_remove_nginx_config
+#
+# Requires YunoHost version 2.7.2 or higher.
+ynh_remove_nginx_config () {
+    ynh_secure_remove --file="/etc/nginx/conf.d/$domain.d/$app.conf"
+    ynh_systemd_action --service_name=nginx --action=reload
+}

data/helpers.d/nodejs

@@ -1,20 +1,38 @@
 #!/bin/bash
 
+n_version=6.7.0
 n_install_dir="/opt/node_n"
 node_version_path="$n_install_dir/n/versions/node"
 # N_PREFIX is the directory of n, it needs to be loaded as a environment variable.
 export N_PREFIX="$n_install_dir"
 
+# Install Node version management
+#
+# [internal]
+#
+# usage: ynh_install_n
+#
+# Requires YunoHost version 2.7.12 or higher.
+ynh_install_n () {
+    ynh_print_info --message="Installation of N - Node.js version management"
+    # Build an app.src for n
+    mkdir --parents "../conf"
+    echo "SOURCE_URL=https://github.com/tj/n/archive/v${n_version}.tar.gz
+SOURCE_SUM=92e00fa86d1c4e8dc6ca8df7e75fc93afe8f71949890ef67c40555df4efc4abe" > "../conf/n.src"
+    # Download and extract n
+    ynh_setup_source --dest_dir="$n_install_dir/git" --source_id=n
+    # Install n
+    (cd "$n_install_dir/git"
+    PREFIX=$N_PREFIX make install 2>&1)
+}
+
 # Load the version of node for an app, and set variables.
 #
-# usage: ynh_use_nodejs
-#
-# `ynh_use_nodejs` has to be used in any app scripts before using node for the first time.
+# ynh_use_nodejs has to be used in any app scripts before using node for the first time.
 # This helper will provide alias and variables to use in your scripts.
 #
-# To use npm or node, use the alias `ynh_npm` and `ynh_node`.
-#
-# Those alias will use the correct version installed for the app.
+# To use npm or node, use the alias `ynh_npm` and `ynh_node`
+# Those alias will use the correct version installed for the app
 # For example: use `ynh_npm install` instead of `npm install`
 #
 # With `sudo` or `ynh_exec_as`, use instead the fallback variables `$ynh_npm` and `$ynh_node`
@@ -22,34 +40,32 @@ export N_PREFIX="$n_install_dir"
 # Exemple: `ynh_exec_as $app $ynh_node_load_PATH $ynh_npm install`
 #
 # $PATH contains the path of the requested version of node.
-# However, $PATH is duplicated into $node_PATH to outlast any manipulation of `$PATH`
+# However, $PATH is duplicated into $node_PATH to outlast any manipulation of $PATH
 # You can use the variable `$ynh_node_load_PATH` to quickly load your node version
-# in $PATH for an usage into a separate script.
+#  in $PATH for an usage into a separate script.
 # Exemple: $ynh_node_load_PATH $final_path/script_that_use_npm.sh`
 #
 #
 # Finally, to start a nodejs service with the correct version, 2 solutions
 #  Either the app is dependent of node or npm, but does not called it directly.
-#  In such situation, you need to load PATH :
-# ```
-# Environment="__NODE_ENV_PATH__"
-# ExecStart=__FINALPATH__/my_app
-# ```
-# You will replace __NODE_ENV_PATH__ with $ynh_node_load_PATH.
+#  In such situation, you need to load PATH
+#    `Environment="__NODE_ENV_PATH__"`
+#    `ExecStart=__FINALPATH__/my_app`
+#     You will replace __NODE_ENV_PATH__ with $ynh_node_load_PATH
 #
 #  Or node start the app directly, then you don't need to load the PATH variable
-# ```
-# ExecStart=__YNH_NODE__ my_app run
-# ```
-# You will replace __YNH_NODE__ with $ynh_node
+#    `ExecStart=__YNH_NODE__ my_app run`
+#     You will replace __YNH_NODE__ with $ynh_node
 #
 #
 # 2 other variables are also available
 #   - $nodejs_path: The absolute path to node binaries for the chosen version.
 #   - $nodejs_version: Just the version number of node for this app. Stored as 'nodejs_version' in settings.yml.
 #
+# usage: ynh_use_nodejs
+#
 # Requires YunoHost version 2.7.12 or higher.
-ynh_use_nodejs() {
+ynh_use_nodejs () {
     nodejs_version=$(ynh_app_setting_get --app=$app --key=nodejs_version)
 
     # Get the absolute path of this version of node
@@ -72,10 +88,6 @@ ynh_use_nodejs() {
     node_PATH="$PATH"
     # Create an alias to easily load the PATH
     ynh_node_load_PATH="PATH=$node_PATH"
-    # Same var but in lower case to be compatible with ynh_replace_vars...
-    ynh_node_load_path="PATH=$node_PATH"
-    # Prevent yet another Node and Corepack madness, with Corepack wanting the user to confirm download of Yarn
-    export COREPACK_ENABLE_DOWNLOAD_PROMPT=0
 }
 
 # Install a specific version of nodejs
@@ -83,20 +95,20 @@ ynh_use_nodejs() {
 # ynh_install_nodejs will install the version of node provided as argument by using n.
 #
 # usage: ynh_install_nodejs --nodejs_version=nodejs_version
-# | arg: -n, --nodejs_version=      - Version of node to install. When possible, your should prefer to use major version number (e.g. 8 instead of 8.10.0).
+# | arg: -n, --nodejs_version=      - Version of node to install. When possible, your should prefer to use major version number (e.g. 8 instead of 8.10.0). The crontab will then handle the update of minor versions when needed.
 #
-# `n` (Node version management) uses the `PATH` variable to store the path of the version of node it is going to use.
+# n (Node version management) uses the PATH variable to store the path of the version of node it is going to use.
 # That's how it changes the version
 #
-# Refer to `ynh_use_nodejs` for more information about available commands and variables
+# Refer to ynh_use_nodejs for more information about available commands and variables
 #
 # Requires YunoHost version 2.7.12 or higher.
-ynh_install_nodejs() {
+ynh_install_nodejs () {
     # Use n, https://github.com/tj/n to manage the nodejs versions
 
     # Declare an array to define the options of this helper.
     local legacy_args=n
-    local -A args_array=([n]=nodejs_version=)
+    local -A args_array=( [n]=nodejs_version= )
     local nodejs_version
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
@@ -113,10 +125,16 @@ ynh_install_nodejs() {
     test -x /usr/bin/node && mv /usr/bin/node /usr/bin/node_n
     test -x /usr/bin/npm && mv /usr/bin/npm /usr/bin/npm_n
 
-    # Install (or update if YunoHost vendor/ folder updated since last install) n
-    mkdir -p $n_install_dir/bin/
-    cp "$YNH_HELPERS_DIR/vendor/n/n" $n_install_dir/bin/n
-    # Tweak for n to understand it's installed in $N_PREFIX
+    # If n is not previously setup, install it
+    if ! $n_install_dir/bin/n --version > /dev/null 2>&1
+    then
+        ynh_install_n
+    elif dpkg --compare-versions "$($n_install_dir/bin/n --version)" lt $n_version
+    then
+        ynh_install_n
+    fi
+
+    # Modify the default N_PREFIX in n script
     ynh_replace_string --match_string="^N_PREFIX=\${N_PREFIX-.*}$" --replace_string="N_PREFIX=\${N_PREFIX-$N_PREFIX}" --target_file="$n_install_dir/bin/n"
 
     # Restore /usr/local/bin in PATH
@@ -128,7 +146,8 @@ ynh_install_nodejs() {
 
     # Install the requested version of nodejs
     uname=$(uname --machine)
-    if [[ $uname =~ aarch64 || $uname =~ arm64 ]]; then
+    if [[ $uname =~ aarch64 || $uname =~ arm64 ]]
+    then
         n $nodejs_version --arch=arm64
     else
         n $nodejs_version
@@ -139,43 +158,106 @@ ynh_install_nodejs() {
     real_nodejs_version=$(basename $real_nodejs_version)
 
     # Create a symbolic link for this major version if the file doesn't already exist
-    if [ ! -e "$node_version_path/$nodejs_version" ]; then
+    if [ ! -e "$node_version_path/$nodejs_version" ]
+    then
         ln --symbolic --force --no-target-directory $node_version_path/$real_nodejs_version $node_version_path/$nodejs_version
     fi
 
     # Store the ID of this app and the version of node requested for it
-    echo "$app:$nodejs_version" | tee --append "$n_install_dir/ynh_app_version"
+    echo "$YNH_APP_INSTANCE_NAME:$nodejs_version" | tee --append "$n_install_dir/ynh_app_version"
 
     # Store nodejs_version into the config of this app
     ynh_app_setting_set --app=$app --key=nodejs_version --value=$nodejs_version
 
+    # Build the update script and set the cronjob
+    ynh_cron_upgrade_node
+
     ynh_use_nodejs
 }
 
 # Remove the version of node used by the app.
 #
+# This helper will check if another app uses the same version of node,
+# if not, this version of node will be removed.
+# If no other app uses node, n will be also removed.
+#
 # usage: ynh_remove_nodejs
 #
-# This helper will check if another app uses the same version of node.
-# - If not, this version of node will be removed.
-# - If no other app uses node, n will be also removed.
-#
 # Requires YunoHost version 2.7.12 or higher.
-ynh_remove_nodejs() {
+ynh_remove_nodejs () {
     nodejs_version=$(ynh_app_setting_get --app=$app --key=nodejs_version)
 
     # Remove the line for this app
-    sed --in-place "/$app:$nodejs_version/d" "$n_install_dir/ynh_app_version"
+    sed --in-place "/$YNH_APP_INSTANCE_NAME:$nodejs_version/d" "$n_install_dir/ynh_app_version"
 
     # If no other app uses this version of nodejs, remove it.
-    if ! grep --quiet "$nodejs_version" "$n_install_dir/ynh_app_version"; then
+    if ! grep --quiet "$nodejs_version" "$n_install_dir/ynh_app_version"
+    then
         $n_install_dir/bin/n rm $nodejs_version
     fi
 
     # If no other app uses n, remove n
-    if [ ! -s "$n_install_dir/ynh_app_version" ]; then
+    if [ ! -s "$n_install_dir/ynh_app_version" ]
+    then
         ynh_secure_remove --file="$n_install_dir"
         ynh_secure_remove --file="/usr/local/n"
         sed --in-place "/N_PREFIX/d" /root/.bashrc
+        rm --force /etc/cron.daily/node_update
     fi
 }
+
+# Set a cron design to update your node versions
+#
+# [internal]
+#
+# This cron will check and update all minor node versions used by your apps.
+#
+# usage: ynh_cron_upgrade_node
+#
+# Requires YunoHost version 2.7.12 or higher.
+ynh_cron_upgrade_node () {
+    # Build the update script
+    cat > "$n_install_dir/node_update.sh" << EOF
+#!/bin/bash
+
+version_path="$node_version_path"
+n_install_dir="$n_install_dir"
+
+# Log the date
+date
+
+# List all real installed version of node
+all_real_version="\$(find \$version_path/* -maxdepth 0 -type d | sed "s@\$version_path/@@g")"
+
+# Keep only the major version number of each line
+all_real_version=\$(echo "\$all_real_version" | sed 's/\..*\$//')
+
+# Remove double entries
+all_real_version=\$(echo "\$all_real_version" | sort --unique)
+
+# Read each major version
+while read version
+do
+    echo "Update of the version \$version"
+    sudo \$n_install_dir/bin/n \$version
+
+    # Find the last "real" version for this major version of node.
+    real_nodejs_version=\$(find \$version_path/\$version* -maxdepth 0 | sort --version-sort | tail --lines=1)
+    real_nodejs_version=\$(basename \$real_nodejs_version)
+
+    # Update the symbolic link for this version
+    sudo ln --symbolic --force --no-target-directory \$version_path/\$real_nodejs_version \$version_path/\$version
+done <<< "\$(echo "\$all_real_version")"
+EOF
+
+    chmod +x "$n_install_dir/node_update.sh"
+
+    # Build the cronjob
+    cat > "/etc/cron.daily/node_update" << EOF
+#!/bin/bash
+
+$n_install_dir/node_update.sh >> $n_install_dir/node_update.log
+EOF
+
+    chmod +x "/etc/cron.daily/node_update"
+}

data/helpers.d/permission

@@ -2,17 +2,15 @@
 
 # Create a new permission for the app
 #
-# Example 1: `ynh_permission_create --permission=admin --url=/admin --additional_urls=domain.tld/admin /superadmin --allowed=alice bob \
-#                                  --label="My app admin" --show_tile=true`
+# example 1: ynh_permission_create --permission=admin --url=/admin --additional_urls=domain.tld/admin /superadmin --allowed=alice bob \
+#                                  --label="My app admin" --show_tile=true
 #
 # This example will create a new permission permission with this following effect:
 # - A tile named "My app admin" in the SSO will be available for the users alice and bob. This tile will point to the relative url '/admin'.
 # - Only the user alice and bob will have the access to theses following url: /admin, domain.tld/admin, /superadmin
 #
 #
-# Example 2:
-#
-#     ynh_permission_create --permission=api --url=domain.tld/api --auth_header=false --allowed=visitors \
+# example 2: ynh_permission_create --permission=api --url=domain.tld/api --auth_header=false --allowed=visitors \
 #                                  --label="MyApp API" --protected=true
 #
 # This example will create a new protected permission. So the admin won't be able to add/remove the visitors group of this permission.
@@ -20,7 +18,7 @@
 # With this permission all client will be allowed to access to the url 'domain.tld/api'.
 # Note that in this case no tile will be show on the SSO.
 # Note that the auth_header parameter is to 'false'. So no authentication header will be passed to the application.
-# Generally the API is requested by an application and enabling the auth_header has no advantage and could bring some issues in some case.
+# Generally the API is requested by an application and enabling the auth_header has no advantage and could bring some issues in some case. 
 # So in this case it's better to disable this option for all API.
 #
 #
@@ -38,14 +36,14 @@
 #
 # If provided, 'url' or 'additional_urls' is assumed to be relative to the app domain/path if they
 # start with '/'.  For example:
-#     /                             -> domain.tld/app
-#     /admin                        -> domain.tld/app/admin
-#     domain.tld/app/api            -> domain.tld/app/api
+#    /                             -> domain.tld/app
+#    /admin                        -> domain.tld/app/admin
+#    domain.tld/app/api            -> domain.tld/app/api
 #
 # 'url' or 'additional_urls' can be treated as a PCRE (not lua) regex if it starts with "re:".
 # For example:
-#     re:/api/[A-Z]*$               -> domain.tld/app/api/[A-Z]*$
-#     re:domain.tld/app/api/[A-Z]*$ -> domain.tld/app/api/[A-Z]*$
+#    re:/api/[A-Z]*$               -> domain.tld/app/api/[A-Z]*$
+#    re:domain.tld/app/api/[A-Z]*$ -> domain.tld/app/api/[A-Z]*$
 #
 # Note that globally the parameter 'url' and 'additional_urls' are same. The only difference is:
 # - 'url' is only one url, 'additional_urls' can be a list of urls. There are no limitation of 'additional_urls'
@@ -58,7 +56,7 @@
 #        - "Remote-User": username
 #        - "Email": user email
 #
-# Generally this feature is usefull to authenticate automatically the user in the application but in some case the application don't work with theses header and theses header need to be disabled to have the application to work correctly.
+# Generally this feature is usefull to authenticate automatically the user in the application but in some case the application don't work with theses header and theses header need to be disabled to have the application to work correctly. 
 # See https://github.com/YunoHost/issues/issues/1420 for more informations
 #
 #
@@ -66,7 +64,7 @@
 ynh_permission_create() {
     # Declare an array to define the options of this helper.
     local legacy_args=puAhaltP
-    local -A args_array=([p]=permission= [u]=url= [A]=additional_urls= [h]=auth_header= [a]=allowed= [l]=label= [t]=show_tile= [P]=protected=)
+    local -A args_array=( [p]=permission= [u]=url= [A]=additional_urls= [h]=auth_header= [a]=allowed= [l]=label= [t]=show_tile= [P]=protected= )
     local permission
     local url
     local additional_urls
@@ -84,11 +82,13 @@ ynh_permission_create() {
     show_tile=${show_tile:-}
     protected=${protected:-}
 
-    if [[ -n $url ]]; then
+    if [[ -n $url ]]
+    then
         url=",url='$url'"
     fi
 
-    if [[ -n $additional_urls ]]; then
+    if [[ -n $additional_urls ]]
+    then
         # Convert a list from getopts to python list
         # Note that getopts separate the args with ';'
         # By example:
@@ -98,15 +98,18 @@ ynh_permission_create() {
         additional_urls=",additional_urls=['${additional_urls//;/\',\'}']"
     fi
 
-    if [[ -n $auth_header ]]; then
-        if [ $auth_header == "true" ]; then
+    if [[ -n $auth_header ]]
+    then
+        if [ $auth_header == "true" ]
+        then
             auth_header=",auth_header=True"
         else
             auth_header=",auth_header=False"
         fi
     fi
 
-    if [[ -n $allowed ]]; then
+    if [[ -n $allowed ]]
+    then
         # Convert a list from getopts to python list
         # Note that getopts separate the args with ';'
         # By example:
@@ -122,16 +125,20 @@ ynh_permission_create() {
         label=",label='$permission'"
     fi
 
-    if [[ -n ${show_tile:-} ]]; then
-        if [ $show_tile == "true" ]; then
+    if [[ -n ${show_tile:-} ]]
+    then
+        if [ $show_tile == "true" ]
+        then
             show_tile=",show_tile=True"
         else
             show_tile=",show_tile=False"
         fi
     fi
 
-    if [[ -n ${protected:-} ]]; then
-        if [ $protected == "true" ]; then
+    if [[ -n ${protected:-} ]]
+    then
+        if [ $protected == "true" ]
+        then
             protected=",protected=True"
         else
             protected=",protected=False"
@@ -152,7 +159,7 @@ ynh_permission_create() {
 ynh_permission_delete() {
     # Declare an array to define the options of this helper.
     local legacy_args=p
-    local -A args_array=([p]=permission=)
+    local -A args_array=( [p]=permission= )
     local permission
     ynh_handle_getopts_args "$@"
 
@@ -169,17 +176,16 @@ ynh_permission_delete() {
 ynh_permission_exists() {
     # Declare an array to define the options of this helper.
     local legacy_args=p
-    local -A args_array=([p]=permission=)
+    local -A args_array=( [p]=permission= )
     local permission
     ynh_handle_getopts_args "$@"
 
-    yunohost user permission list "$app" --output-as json --quiet \
-        | jq -e --arg perm "$app.$permission" '.permissions[$perm]' > /dev/null
+    yunohost user permission list --short | grep --word-regexp --quiet "$app.$permission"
 }
 
 # Redefine the url associated to a permission
 #
-# usage: ynh_permission_url --permission "permission" [--url="url"] [--add_url="new-url" [ "other-new-url" ]] [--remove_url="old-url" [ "other-old-url" ]]
+# usage: ynh_permission_url --permission "permission" [--url="url"] [--add_url="new-url" [ "other-new-url" ]] [--remove_url="old-url" [ "other-old-url" ]] 
 #                                                     [--auth_header=true|false] [--clear_urls]
 # | arg: -p, --permission=   - the name for the permission (by default a permission named "main" is removed automatically when the app is removed)
 # | arg: -u, --url=          - (optional) URL for which access will be allowed/forbidden. Note that if you want to remove url you can pass an empty sting as arguments ("").
@@ -192,7 +198,7 @@ ynh_permission_exists() {
 ynh_permission_url() {
     # Declare an array to define the options of this helper.
     local legacy_args=puarhc
-    local -A args_array=([p]=permission= [u]=url= [a]=add_url= [r]=remove_url= [h]=auth_header= [c]=clear_urls)
+    local -A args_array=( [p]=permission= [u]=url= [a]=add_url= [r]=remove_url= [h]=auth_header= [c]=clear_urls )
     local permission
     local url
     local add_url
@@ -206,11 +212,13 @@ ynh_permission_url() {
     auth_header=${auth_header:-}
     clear_urls=${clear_urls:-}
 
-    if [[ -n $url ]]; then
+    if [[ -n $url ]]
+    then
         url=",url='$url'"
     fi
 
-    if [[ -n $add_url ]]; then
+    if [[ -n $add_url ]]
+    then
         # Convert a list from getopts to python list
         # Note that getopts separate the args with ';'
         # For example:
@@ -220,7 +228,8 @@ ynh_permission_url() {
         add_url=",add_url=['${add_url//;/\',\'}']"
     fi
 
-    if [[ -n $remove_url ]]; then
+    if [[ -n $remove_url ]]
+    then
         # Convert a list from getopts to python list
         # Note that getopts separate the args with ';'
         # For example:
@@ -230,21 +239,25 @@ ynh_permission_url() {
         remove_url=",remove_url=['${remove_url//;/\',\'}']"
     fi
 
-    if [[ -n $auth_header ]]; then
-        if [ $auth_header == "true" ]; then
+    if [[ -n $auth_header ]]
+    then
+        if [ $auth_header == "true" ]
+        then
             auth_header=",auth_header=True"
         else
             auth_header=",auth_header=False"
         fi
     fi
 
-    if [[ -n $clear_urls ]] && [ $clear_urls -eq 1 ]; then
+    if [[ -n $clear_urls ]] && [ $clear_urls -eq 1 ]
+    then
         clear_urls=",clear_urls=True"
     fi
 
     yunohost tools shell -c "from yunohost.permission import permission_url; permission_url('$app.$permission' $url $add_url $remove_url $auth_header $clear_urls)"
 }
 
+
 # Update a permission for the app
 #
 # usage: ynh_permission_update --permission "permission" [--add="group" ["group" ...]] [--remove="group" ["group" ...]]
@@ -260,7 +273,7 @@ ynh_permission_url() {
 ynh_permission_update() {
     # Declare an array to define the options of this helper.
     local legacy_args=parltP
-    local -A args_array=([p]=permission= [a]=add= [r]=remove= [l]=label= [t]=show_tile= [P]=protected=)
+    local -A args_array=( [p]=permission= [a]=add= [r]=remove= [l]=label= [t]=show_tile= [P]=protected= )
     local permission
     local add
     local remove
@@ -274,7 +287,8 @@ ynh_permission_update() {
     show_tile=${show_tile:-}
     protected=${protected:-}
 
-    if [[ -n $add ]]; then
+    if [[ -n $add ]]
+    then
         # Convert a list from getopts to python list
         # Note that getopts separate the args with ';'
         # For example:
@@ -283,7 +297,8 @@ ynh_permission_update() {
         # add=['alice', 'bob']
         add=",add=['${add//';'/"','"}']"
     fi
-    if [[ -n $remove ]]; then
+    if [[ -n $remove ]]
+    then
         # Convert a list from getopts to python list
         # Note that getopts separate the args with ';'
         # For example:
@@ -293,12 +308,15 @@ ynh_permission_update() {
         remove=",remove=['${remove//';'/"','"}']"
     fi
 
-    if [[ -n $label ]]; then
+    if [[ -n $label ]]
+    then
         label=",label='$label'"
     fi
 
-    if [[ -n $show_tile ]]; then
-        if [ $show_tile == "true" ]; then
+    if [[ -n $show_tile ]]
+    then
+        if [ $show_tile == "true" ]
+        then
             show_tile=",show_tile=True"
         else
             show_tile=",show_tile=False"
@@ -306,7 +324,8 @@ ynh_permission_update() {
     fi
 
     if [[ -n $protected ]]; then
-        if [ $protected == "true" ]; then
+        if [ $protected == "true" ]
+        then
             protected=",protected=True"
         else
             protected=",protected=False"
@@ -329,25 +348,18 @@ ynh_permission_update() {
 ynh_permission_has_user() {
     local legacy_args=pu
     # Declare an array to define the options of this helper.
-    local -A args_array=([p]=permission= [u]=user=)
+    local -A args_array=( [p]=permission= [u]=user= )
     local permission
     local user
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    if ! ynh_permission_exists --permission=$permission; then
+    if ! ynh_permission_exists --permission=$permission
+    then
         return 1
     fi
 
-    # Check both allowed and corresponding_users sections in the json
-    for section in "allowed" "corresponding_users"; do
-        if yunohost user permission info "$app.$permission" --output-as json --quiet \
-            | jq -e --arg user $user --arg section $section '.[$section] | index($user)' > /dev/null; then
-            return 0
-        fi
-    done
-
-    return 1
+    yunohost user permission info "$app.$permission" | grep --word-regexp --quiet "$user"
 }
 
 # Check if a legacy permissions exist
@@ -356,8 +368,9 @@ ynh_permission_has_user() {
 # | exit: Return 1 if the permission doesn't exist, 0 otherwise
 #
 # Requires YunoHost version 4.1.2 or higher.
-ynh_legacy_permissions_exists() {
-    for permission in "skipped" "unprotected" "protected"; do
+ynh_legacy_permissions_exists () {
+    for permission in "skipped" "unprotected" "protected"
+    do
         if ynh_permission_exists --permission="legacy_${permission}_uris"; then
             return 0
         fi
@@ -376,8 +389,9 @@ ynh_legacy_permissions_exists() {
 #       # You can recreate the required permissions here with ynh_permission_create
 #   fi
 # Requires YunoHost version 4.1.2 or higher.
-ynh_legacy_permissions_delete_all() {
-    for permission in "skipped" "unprotected" "protected"; do
+ynh_legacy_permissions_delete_all () {
+    for permission in "skipped" "unprotected" "protected"
+    do
         if ynh_permission_exists --permission="legacy_${permission}_uris"; then
             ynh_permission_delete --permission="legacy_${permission}_uris"
         fi

data/helpers.d/php

@@ -0,0 +1,575 @@
+#!/bin/bash
+
+readonly YNH_DEFAULT_PHP_VERSION=7.3
+# Declare the actual PHP version to use.
+# A packager willing to use another version of PHP can override the variable into its _common.sh.
+YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION}
+
+# Create a dedicated PHP-FPM config
+#
+# usage 1: ynh_add_fpm_config [--phpversion=7.X] [--use_template] [--package=packages] [--dedicated_service]
+# | arg: -v, --phpversion=          - Version of PHP to use.
+# | arg: -t, --use_template         - Use this helper in template mode.
+# | arg: -p, --package=             - Additionnal PHP packages to install
+# | arg: -d, --dedicated_service    - Use a dedicated PHP-FPM service instead of the common one.
+#
+# -----------------------------------------------------------------------------
+#
+# usage 2: ynh_add_fpm_config [--phpversion=7.X] --usage=usage --footprint=footprint [--package=packages] [--dedicated_service]
+# | arg: -v, --phpversion=          - Version of PHP to use.
+# | arg: -f, --footprint=           - Memory footprint of the service (low/medium/high).
+# low    - Less than 20 MB of RAM by pool.
+# medium - Between 20 MB and 40 MB of RAM by pool.
+# high   - More than 40 MB of RAM by pool.
+# Or specify exactly the footprint, the load of the service as MB by pool instead of having a standard value.
+# To have this value, use the following command and stress the service.
+# watch -n0.5 ps -o user,cmd,%cpu,rss -u APP
+#
+# | arg: -u, --usage=               - Expected usage of the service (low/medium/high).
+# low    - Personal usage, behind the SSO.
+# medium - Low usage, few people or/and publicly accessible.
+# high   - High usage, frequently visited website.
+#
+# | arg: -p, --package=             - Additionnal PHP packages to install for a specific version of PHP
+# | arg: -d, --dedicated_service    - Use a dedicated PHP-FPM service instead of the common one.
+#
+#
+# The footprint of the service will be used to defined the maximum footprint we can allow, which is half the maximum RAM.
+# So it will be used to defined 'pm.max_children'
+# A lower value for the footprint will allow more children for 'pm.max_children'. And so for
+#    'pm.start_servers', 'pm.min_spare_servers' and 'pm.max_spare_servers' which are defined from the
+#    value of 'pm.max_children'
+# NOTE: 'pm.max_children' can't exceed 4 times the number of processor's cores.
+#
+# The usage value will defined the way php will handle the children for the pool.
+# A value set as 'low' will set the process manager to 'ondemand'. Children will start only if the
+#   service is used, otherwise no child will stay alive. This config gives the lower footprint when the
+#   service is idle. But will use more proc since it has to start a child as soon it's used.
+# Set as 'medium', the process manager will be at dynamic. If the service is idle, a number of children
+#   equal to pm.min_spare_servers will stay alive. So the service can be quick to answer to any request.
+#   The number of children can grow if needed. The footprint can stay low if the service is idle, but
+#   not null. The impact on the proc is a little bit less than 'ondemand' as there's always a few
+#   children already available.
+# Set as 'high', the process manager will be set at 'static'. There will be always as many children as
+#   'pm.max_children', the footprint is important (but will be set as maximum a quarter of the maximum
+#   RAM) but the impact on the proc is lower. The service will be quick to answer as there's always many
+#   children ready to answer.
+#
+# Requires YunoHost version 2.7.2 or higher.
+# Requires YunoHost version 3.5.1 or higher for the argument --phpversion
+# Requires YunoHost version 3.8.1 or higher for the arguments --use_template, --usage, --footprint, --package and --dedicated_service
+ynh_add_fpm_config () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=vtufpd
+    local -A args_array=( [v]=phpversion= [t]=use_template [u]=usage= [f]=footprint= [p]=package= [d]=dedicated_service )
+    local phpversion
+    local use_template
+    local usage
+    local footprint
+    local package
+    local dedicated_service
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    package=${package:-}
+
+    # The default behaviour is to use the template.
+    use_template="${use_template:-1}"
+    usage="${usage:-}"
+    footprint="${footprint:-}"
+    if [ -n "$usage" ] || [ -n "$footprint" ]; then
+        use_template=0
+    fi
+    # Do not use a dedicated service by default
+    dedicated_service=${dedicated_service:-0}
+
+    # Set the default PHP-FPM version by default
+    phpversion="${phpversion:-$YNH_PHP_VERSION}"
+
+    local old_phpversion=$(ynh_app_setting_get --app=$app --key=phpversion)
+
+    # If the PHP version changed, remove the old fpm conf
+    if [ -n "$old_phpversion" ] && [ "$old_phpversion" != "$phpversion" ]
+    then
+        local old_php_fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir)
+        local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf"
+
+        ynh_backup_if_checksum_is_different --file="$old_php_finalphpconf"
+
+        ynh_remove_fpm_config
+    fi
+
+    # If the requested PHP version is not the default version for YunoHost
+    if [ "$phpversion" != "$YNH_DEFAULT_PHP_VERSION" ]
+    then
+        # If the argument --package is used, add the packages to ynh_install_php to install them from sury
+        if [ -n "$package" ]
+        then
+            local additionnal_packages="--package=$package"
+        else
+            local additionnal_packages=""
+        fi
+        # Install this specific version of PHP.
+        ynh_install_php --phpversion="$phpversion" "$additionnal_packages"
+    elif [ -n "$package" ]
+    then
+        # Install the additionnal packages from the default repository
+        ynh_add_app_dependencies --package="$package"
+    fi
+
+    if [ $dedicated_service -eq 1 ]
+    then
+        local fpm_service="${app}-phpfpm"
+        local fpm_config_dir="/etc/php/$phpversion/dedicated-fpm"
+    else
+        local fpm_service="php${phpversion}-fpm"
+        local fpm_config_dir="/etc/php/$phpversion/fpm"
+    fi
+
+    # Create the directory for FPM pools
+    mkdir --parents "$fpm_config_dir/pool.d"
+
+    ynh_app_setting_set --app=$app --key=fpm_config_dir --value="$fpm_config_dir"
+    ynh_app_setting_set --app=$app --key=fpm_service --value="$fpm_service"
+    ynh_app_setting_set --app=$app --key=fpm_dedicated_service --value="$dedicated_service"
+    ynh_app_setting_set --app=$app --key=phpversion --value=$phpversion
+    finalphpconf="$fpm_config_dir/pool.d/$app.conf"
+
+    # Migrate from mutual PHP service to dedicated one.
+    if [ $dedicated_service -eq 1 ]
+    then
+        local old_fpm_config_dir="/etc/php/$phpversion/fpm"
+        # If a config file exist in the common pool, move it.
+        if [ -e "$old_fpm_config_dir/pool.d/$app.conf" ]
+        then
+            ynh_print_info --message="Migrate to a dedicated php-fpm service for $app."
+            # Create a backup of the old file before migration
+            ynh_backup_if_checksum_is_different --file="$old_fpm_config_dir/pool.d/$app.conf"
+            # Remove the old PHP config file
+            ynh_secure_remove --file="$old_fpm_config_dir/pool.d/$app.conf"
+            # Reload PHP to release the socket and allow the dedicated service to use it
+            ynh_systemd_action --service_name=php${phpversion}-fpm --action=reload
+        fi
+    fi
+
+    ynh_backup_if_checksum_is_different --file="$finalphpconf"
+
+    if [ $use_template -eq 1 ]
+    then
+        # Usage 1, use the template in conf/php-fpm.conf
+        local phpfpm_path="../conf/php-fpm.conf"
+        if [ ! -e "$phpfpm_path" ]; then
+            phpfpm_path="../settings/conf/php-fpm.conf"	# Into the restore script, the PHP-FPM template is not at the same place
+        fi
+        # Make sure now that the template indeed exists
+        [ -e "$phpfpm_path" ] || ynh_die --message="Unable to find template to configure PHP-FPM."
+        cp "$phpfpm_path" "$finalphpconf"
+        ynh_replace_string --match_string="__NAMETOCHANGE__" --replace_string="$app" --target_file="$finalphpconf"
+        ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalphpconf"
+        ynh_replace_string --match_string="__USER__" --replace_string="$app" --target_file="$finalphpconf"
+        ynh_replace_string --match_string="__PHPVERSION__" --replace_string="$phpversion" --target_file="$finalphpconf"
+
+    else
+        # Usage 2, generate a PHP-FPM config file with ynh_get_scalable_phpfpm
+
+        # Store settings
+        ynh_app_setting_set --app=$app --key=fpm_footprint --value=$footprint
+        ynh_app_setting_set --app=$app --key=fpm_usage --value=$usage
+
+        # Define the values to use for the configuration of PHP.
+        ynh_get_scalable_phpfpm --usage=$usage --footprint=$footprint
+
+        # Copy the default file
+        cp "/etc/php/$phpversion/fpm/pool.d/www.conf" "$finalphpconf"
+
+        # Replace standard variables into the default file
+        ynh_replace_string --match_string="^\[www\]" --replace_string="[$app]" --target_file="$finalphpconf"
+        ynh_replace_string --match_string=".*listen = .*" --replace_string="listen = /var/run/php/php$phpversion-fpm-$app.sock" --target_file="$finalphpconf"
+        ynh_replace_string --match_string="^user = .*" --replace_string="user = $app" --target_file="$finalphpconf"
+        ynh_replace_string --match_string="^group = .*" --replace_string="group = $app" --target_file="$finalphpconf"
+        ynh_replace_string --match_string=".*chdir = .*" --replace_string="chdir = $final_path" --target_file="$finalphpconf"
+
+        # Configure FPM children
+        ynh_replace_string --match_string=".*pm = .*" --replace_string="pm = $php_pm" --target_file="$finalphpconf"
+        ynh_replace_string --match_string=".*pm.max_children = .*" --replace_string="pm.max_children = $php_max_children" --target_file="$finalphpconf"
+        ynh_replace_string --match_string=".*pm.max_requests = .*" --replace_string="pm.max_requests = 500" --target_file="$finalphpconf"
+        ynh_replace_string --match_string=".*request_terminate_timeout = .*" --replace_string="request_terminate_timeout = 1d" --target_file="$finalphpconf"
+        if [ "$php_pm" = "dynamic" ]
+        then
+            ynh_replace_string --match_string=".*pm.start_servers = .*" --replace_string="pm.start_servers = $php_start_servers" --target_file="$finalphpconf"
+            ynh_replace_string --match_string=".*pm.min_spare_servers = .*" --replace_string="pm.min_spare_servers = $php_min_spare_servers" --target_file="$finalphpconf"
+            ynh_replace_string --match_string=".*pm.max_spare_servers = .*" --replace_string="pm.max_spare_servers = $php_max_spare_servers" --target_file="$finalphpconf"
+        elif [ "$php_pm" = "ondemand" ]
+        then
+            ynh_replace_string --match_string=".*pm.process_idle_timeout = .*" --replace_string="pm.process_idle_timeout = 10s" --target_file="$finalphpconf"
+        fi
+
+        # Comment unused parameters
+        if [ "$php_pm" != "dynamic" ]
+        then
+            ynh_replace_string --match_string=".*\(pm.start_servers = .*\)" --replace_string=";\1" --target_file="$finalphpconf"
+            ynh_replace_string --match_string=".*\(pm.min_spare_servers = .*\)" --replace_string=";\1" --target_file="$finalphpconf"
+            ynh_replace_string --match_string=".*\(pm.max_spare_servers = .*\)" --replace_string=";\1" --target_file="$finalphpconf"
+        fi
+        if [ "$php_pm" != "ondemand" ]
+        then
+            ynh_replace_string --match_string=".*\(pm.process_idle_timeout = .*\)" --replace_string=";\1" --target_file="$finalphpconf"
+        fi
+
+        # Concatene the extra config.
+        if [ -e ../conf/extra_php-fpm.conf ]; then
+            cat ../conf/extra_php-fpm.conf >> "$finalphpconf"
+        fi
+    fi
+
+    chown root: "$finalphpconf"
+    ynh_store_file_checksum --file="$finalphpconf"
+
+    if [ -e "../conf/php-fpm.ini" ]
+    then
+        ynh_print_warn --message="Packagers ! Please do not use a separate php ini file, merge your directives in the pool file instead."
+        finalphpini="$fpm_config_dir/conf.d/20-$app.ini"
+        ynh_backup_if_checksum_is_different "$finalphpini"
+        cp ../conf/php-fpm.ini "$finalphpini"
+        chown root: "$finalphpini"
+        ynh_store_file_checksum "$finalphpini"
+    fi
+
+    if [ $dedicated_service -eq 1 ]
+    then
+        # Create a dedicated php-fpm.conf for the service
+        local globalphpconf=$fpm_config_dir/php-fpm-$app.conf
+        cp /etc/php/${phpversion}/fpm/php-fpm.conf $globalphpconf
+
+        ynh_replace_string --match_string="^[; ]*pid *=.*" --replace_string="pid = /run/php/php${phpversion}-fpm-$app.pid" --target_file="$globalphpconf"
+        ynh_replace_string --match_string="^[; ]*error_log *=.*" --replace_string="error_log = /var/log/php/fpm-php.$app.log" --target_file="$globalphpconf"
+        ynh_replace_string --match_string="^[; ]*syslog.ident *=.*" --replace_string="syslog.ident = php-fpm-$app" --target_file="$globalphpconf"
+        ynh_replace_string --match_string="^[; ]*include *=.*" --replace_string="include = $finalphpconf" --target_file="$globalphpconf"
+
+        # Create a config for a dedicated PHP-FPM service for the app
+        echo "[Unit]
+Description=PHP $phpversion FastCGI Process Manager for $app
+After=network.target
+
+[Service] 
+Type=notify
+PIDFile=/run/php/php${phpversion}-fpm-$app.pid
+ExecStart=/usr/sbin/php-fpm$phpversion --nodaemonize --fpm-config $globalphpconf
+ExecReload=/bin/kill -USR2 \$MAINPID
+
+[Install]
+WantedBy=multi-user.target
+" > ../conf/$fpm_service
+
+        # Create this dedicated PHP-FPM service
+        ynh_add_systemd_config --service=$fpm_service --template=$fpm_service
+        # Integrate the service in YunoHost admin panel
+        yunohost service add $fpm_service --log /var/log/php/fpm-php.$app.log --log_type file --description "Php-fpm dedicated to $app"
+        # Configure log rotate
+        ynh_use_logrotate --logfile=/var/log/php
+        # Restart the service, as this service is either stopped or only for this app
+        ynh_systemd_action --service_name=$fpm_service --action=restart
+    else
+        # Validate that the new php conf doesn't break php-fpm entirely
+        if ! php-fpm${phpversion} --test 2>/dev/null
+        then
+            php-fpm${phpversion} --test || true
+            ynh_secure_remove --file="$finalphpconf"
+            ynh_die --message="The new configuration broke php-fpm?"
+        fi
+        ynh_systemd_action --service_name=$fpm_service --action=reload
+    fi
+}
+
+# Remove the dedicated PHP-FPM config
+#
+# usage: ynh_remove_fpm_config
+#
+# Requires YunoHost version 2.7.2 or higher.
+ynh_remove_fpm_config () {
+    local fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir)
+    local fpm_service=$(ynh_app_setting_get --app=$app --key=fpm_service)
+    local dedicated_service=$(ynh_app_setting_get --app=$app --key=fpm_dedicated_service)
+    dedicated_service=${dedicated_service:-0}
+    # Get the version of PHP used by this app
+    local phpversion=$(ynh_app_setting_get --app=$app --key=phpversion)
+
+    # Assume default PHP-FPM version by default
+    phpversion="${phpversion:-$YNH_DEFAULT_PHP_VERSION}"
+
+    # Assume default PHP files if not set
+    if [ -z "$fpm_config_dir" ]
+    then
+        fpm_config_dir="/etc/php/$YNH_DEFAULT_PHP_VERSION/fpm"
+        fpm_service="php$YNH_DEFAULT_PHP_VERSION-fpm"
+    fi
+
+    if [ $dedicated_service -eq 1 ]
+    then
+        # Remove the dedicated service PHP-FPM service for the app
+        ynh_remove_systemd_config --service=$fpm_service
+        # Remove the global PHP-FPM conf
+        ynh_secure_remove --file="$fpm_config_dir/php-fpm-$app.conf"
+        # Remove the service from the list of services known by YunoHost
+        yunohost service remove $fpm_service
+    elif ynh_package_is_installed --package="php${phpversion}-fpm"; then
+        ynh_systemd_action --service_name=$fpm_service --action=reload
+    fi
+
+    ynh_secure_remove --file="$fpm_config_dir/pool.d/$app.conf"
+    if [ -e $fpm_config_dir/conf.d/20-$app.ini ]
+    then
+        ynh_secure_remove --file="$fpm_config_dir/conf.d/20-$app.ini"
+    fi
+
+    # If the PHP version used is not the default version for YunoHost
+    if [ "$phpversion" != "$YNH_DEFAULT_PHP_VERSION" ]
+    then
+        # Remove this specific version of PHP
+        ynh_remove_php
+    fi
+}
+
+# Install another version of PHP.
+#
+# [internal]
+#
+# usage: ynh_install_php --phpversion=phpversion [--package=packages]
+# | arg: -v, --phpversion=  - Version of PHP to install.
+# | arg: -p, --package=     - Additionnal PHP packages to install
+#
+# Requires YunoHost version 3.8.1 or higher.
+ynh_install_php () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=vp
+    local -A args_array=( [v]=phpversion= [p]=package= )
+    local phpversion
+    local package
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    package=${package:-}
+
+    # Store phpversion into the config of this app
+    ynh_app_setting_set $app phpversion $phpversion
+
+    if [ "$phpversion" == "$YNH_DEFAULT_PHP_VERSION" ]
+    then
+        ynh_die "Do not use ynh_install_php to install php$YNH_DEFAULT_PHP_VERSION"
+    fi
+
+    # Create the file if doesn't exist already
+    touch /etc/php/ynh_app_version
+
+    # Do not add twice the same line
+    if ! grep --quiet "$YNH_APP_INSTANCE_NAME:" "/etc/php/ynh_app_version"
+    then
+        # Store the ID of this app and the version of PHP requested for it
+        echo "$YNH_APP_INSTANCE_NAME:$phpversion" | tee --append "/etc/php/ynh_app_version"
+    fi
+
+    # Add an extra repository for those packages
+    ynh_install_extra_repo --repo="https://packages.sury.org/php/ $(ynh_get_debian_release) main" --key="https://packages.sury.org/php/apt.gpg" --name=extra_php_version --priority=600
+
+    # Install requested dependencies from this extra repository.
+    # Install PHP-FPM first, otherwise PHP will install apache as a dependency.
+    ynh_add_app_dependencies --package="php${phpversion}-fpm"
+    ynh_add_app_dependencies --package="php$phpversion php${phpversion}-common $package"
+
+    # Set the default PHP version back as the default version for php-cli.
+    update-alternatives --set php /usr/bin/php$YNH_DEFAULT_PHP_VERSION
+
+    # Advertise service in admin panel
+    yunohost service add php${phpversion}-fpm --log "/var/log/php${phpversion}-fpm.log"
+}
+
+# Remove the specific version of PHP used by the app.
+#
+# [internal]
+#
+# usage: ynh_install_php
+#
+# Requires YunoHost version 3.8.1 or higher.
+ynh_remove_php () {
+    # Get the version of PHP used by this app
+    local phpversion=$(ynh_app_setting_get --app=$app --key=phpversion)
+
+    if [ "$phpversion" == "$YNH_DEFAULT_PHP_VERSION" ] || [ -z "$phpversion" ]
+    then
+        if [ "$phpversion" == "$YNH_DEFAULT_PHP_VERSION" ]
+        then
+            ynh_print_err "Do not use ynh_remove_php to remove php$YNH_DEFAULT_PHP_VERSION !"
+        fi
+        return 0
+    fi
+
+    # Create the file if doesn't exist already
+    touch /etc/php/ynh_app_version
+
+    # Remove the line for this app
+    sed --in-place "/$YNH_APP_INSTANCE_NAME:$phpversion/d" "/etc/php/ynh_app_version"
+
+    # If no other app uses this version of PHP, remove it.
+    if ! grep --quiet "$phpversion" "/etc/php/ynh_app_version"
+    then
+        # Remove the service from the admin panel
+        if ynh_package_is_installed --package="php${phpversion}-fpm"; then
+            yunohost service remove php${phpversion}-fpm
+        fi
+
+        # Purge PHP dependencies for this version.
+        ynh_package_autopurge "php$phpversion php${phpversion}-fpm php${phpversion}-common"
+    fi
+}
+
+# Define the values to configure PHP-FPM
+#
+# [internal]
+#
+# usage: ynh_get_scalable_phpfpm --usage=usage --footprint=footprint [--print]
+# | arg: -f, --footprint=       - Memory footprint of the service (low/medium/high).
+# low    - Less than 20 MB of RAM by pool.
+# medium - Between 20 MB and 40 MB of RAM by pool.
+# high   - More than 40 MB of RAM by pool.
+# Or specify exactly the footprint, the load of the service as MB by pool instead of having a standard value.
+# To have this value, use the following command and stress the service.
+# watch -n0.5 ps -o user,cmd,%cpu,rss -u APP
+#
+# | arg: -u, --usage=           - Expected usage of the service (low/medium/high).
+# low    - Personal usage, behind the SSO.
+# medium - Low usage, few people or/and publicly accessible.
+# high   - High usage, frequently visited website.
+#
+# | arg: -p, --print            - Print the result (intended for debug purpose only when packaging the app)
+ynh_get_scalable_phpfpm () {
+    local legacy_args=ufp
+    # Declare an array to define the options of this helper.
+    local -A args_array=( [u]=usage= [f]=footprint= [p]=print )
+    local usage
+    local footprint
+    local print
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    # Set all characters as lowercase
+    footprint=${footprint,,}
+    usage=${usage,,}
+    print=${print:-0}
+
+    if [ "$footprint" = "low" ]
+    then
+        footprint=20
+    elif [ "$footprint" = "medium" ]
+    then
+        footprint=35
+    elif [ "$footprint" = "high" ]
+    then
+        footprint=50
+    fi
+
+    # Define the factor to determine min_spare_servers
+    # to avoid having too few children ready to start for heavy apps
+    if [ $footprint -le 20 ]
+    then
+        min_spare_servers_factor=8
+    elif [ $footprint -le 35 ]
+    then
+        min_spare_servers_factor=5
+    else
+        min_spare_servers_factor=3
+    fi
+
+    # Define the way the process manager handle child processes.
+    if [ "$usage" = "low" ]
+    then
+        php_pm=ondemand
+    elif [ "$usage" = "medium" ]
+    then
+        php_pm=dynamic
+    elif [ "$usage" = "high" ]
+    then
+        php_pm=static
+    else
+        ynh_die --message="Does not recognize '$usage' as an usage value."
+    fi
+
+    # Get the total of RAM available, except swap.
+    local max_ram=$(ynh_get_ram --total --ignore_swap)
+
+    at_least_one() {
+        # Do not allow value below 1
+        if [ $1 -le 0 ]
+        then
+            echo 1
+        else
+            echo $1
+        fi
+    }
+
+    # Define pm.max_children
+    # The value of pm.max_children is the total amount of ram divide by 2 and divide again by the footprint of a pool for this app.
+    # So if PHP-FPM start the maximum of children, it won't exceed half of the ram.
+    php_max_children=$(( $max_ram / 2 / $footprint ))
+    # If process manager is set as static, use half less children.
+    # Used as static, there's always as many children as the value of pm.max_children
+    if [ "$php_pm" = "static" ]
+    then
+        php_max_children=$(( $php_max_children / 2 ))
+    fi
+    php_max_children=$(at_least_one $php_max_children)
+
+    # To not overload the proc, limit the number of children to 4 times the number of cores.
+    local core_number=$(nproc)
+    local max_proc=$(( $core_number * 4 ))
+    if [ $php_max_children -gt $max_proc ]
+    then
+        php_max_children=$max_proc
+    fi
+
+    # Get a potential forced value for php_max_children
+    local php_forced_max_children=$(ynh_app_setting_get --app=$app --key=php_forced_max_children)
+    if [ -n "$php_forced_max_children" ]; then
+        php_max_children=$php_forced_max_children
+    fi
+
+    if [ "$php_pm" = "dynamic" ]
+    then
+        # Define pm.start_servers, pm.min_spare_servers and pm.max_spare_servers for a dynamic process manager
+        php_min_spare_servers=$(( $php_max_children / $min_spare_servers_factor ))
+        php_min_spare_servers=$(at_least_one $php_min_spare_servers)
+
+        php_max_spare_servers=$(( $php_max_children / 2 ))
+        php_max_spare_servers=$(at_least_one $php_max_spare_servers)
+
+        php_start_servers=$(( $php_min_spare_servers + ( $php_max_spare_servers - $php_min_spare_servers ) /2 ))
+        php_start_servers=$(at_least_one $php_start_servers)
+    else
+        php_min_spare_servers=0
+        php_max_spare_servers=0
+        php_start_servers=0
+    fi
+
+    if [ $print -eq 1 ]
+    then
+        ynh_debug --message="Footprint=${footprint}Mb by pool."
+        ynh_debug --message="Process manager=$php_pm"
+        ynh_debug --message="Max RAM=${max_ram}Mb"
+        if [ "$php_pm" != "static" ]
+        then
+            ynh_debug --message="\nMax estimated footprint=$(( $php_max_children * $footprint ))"
+            ynh_debug --message="Min estimated footprint=$(( $php_min_spare_servers * $footprint ))"
+        fi
+        if [ "$php_pm" = "dynamic" ]
+        then
+            ynh_debug --message="Estimated average footprint=$(( $php_max_spare_servers * $footprint ))"
+        elif [ "$php_pm" = "static" ]
+        then
+            ynh_debug --message="Estimated footprint=$(( $php_max_children * $footprint ))"
+        fi
+        ynh_debug --message="\nRaw php-fpm values:"
+        ynh_debug --message="pm.max_children = $php_max_children"
+        if [ "$php_pm" = "dynamic" ]
+        then
+            ynh_debug --message="pm.start_servers = $php_start_servers"
+            ynh_debug --message="pm.min_spare_servers = $php_min_spare_servers"
+            ynh_debug --message="pm.max_spare_servers = $php_max_spare_servers"
+        fi
+    fi
+}

data/helpers.d/postgresql

@@ -1,19 +1,19 @@
 #!/bin/bash
 
 PSQL_ROOT_PWD_FILE=/etc/yunohost/psql
-PSQL_VERSION=13
+PSQL_VERSION=11
 
 # Open a connection as a user
 #
+# examples: 
+#    ynh_psql_connect_as 'user' 'pass' <<< "UPDATE ...;"
+#    ynh_psql_connect_as 'user' 'pass' < /path/to/file.sql
+#
 # usage: ynh_psql_connect_as --user=user --password=password [--database=database]
 # | arg: -u, --user=        - the user name to connect as
 # | arg: -p, --password=    - the user password
 # | arg: -d, --database=    - the database to connect to
 #
-# examples:
-#    ynh_psql_connect_as 'user' 'pass' <<< "UPDATE ...;"
-#    ynh_psql_connect_as 'user' 'pass' < /path/to/file.sql
-#
 # Requires YunoHost version 3.5.0 or higher.
 ynh_psql_connect_as() {
     # Declare an array to define the options of this helper.
@@ -46,12 +46,13 @@ ynh_psql_execute_as_root() {
     ynh_handle_getopts_args "$@"
     database="${database:-}"
 
-    if [ -n "$database" ]; then
+    if [ -n "$database" ]
+    then
         database="--database=$database"
     fi
 
     ynh_psql_connect_as --user="postgres" --password="$(cat $PSQL_ROOT_PWD_FILE)" \
-        $database <<< "$sql"
+        $database <<<"$sql"
 }
 
 # Execute a command from a file as root user
@@ -71,12 +72,13 @@ ynh_psql_execute_file_as_root() {
     ynh_handle_getopts_args "$@"
     database="${database:-}"
 
-    if [ -n "$database" ]; then
+    if [ -n "$database" ]
+    then
         database="--database=$database"
     fi
 
     ynh_psql_connect_as --user="postgres" --password="$(cat $PSQL_ROOT_PWD_FILE)" \
-        $database < "$file"
+        $database <"$file"
 }
 
 # Create a database and grant optionnaly privilegies to a user
@@ -125,12 +127,12 @@ ynh_psql_drop_db() {
 
 # Dump a database
 #
+# example: ynh_psql_dump_db 'roundcube' > ./dump.sql
+#
 # usage: ynh_psql_dump_db --database=database
 # | arg: -d, --database=    - the database name to dump
 # | ret: the psqldump output
 #
-# example: ynh_psql_dump_db 'roundcube' > ./dump.sql
-#
 # Requires YunoHost version 3.5.0 or higher.
 ynh_psql_dump_db() {
     # Declare an array to define the options of this helper.
@@ -160,8 +162,6 @@ ynh_psql_create_user() {
 
 # Check if a psql user exists
 #
-# [packagingv1]
-#
 # usage: ynh_psql_user_exists --user=user
 # | arg: -u, --user=    - the user for which to check existence
 # | exit: Return 1 if the user doesn't exist, 0 otherwise
@@ -175,7 +175,8 @@ ynh_psql_user_exists() {
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    if ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT rolname FROM pg_roles WHERE rolname='$user';" | grep --quiet "$user"; then
+    if ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT rolname FROM pg_roles WHERE rolname='$user';" | grep --quiet "$user"
+    then
         return 1
     else
         return 0
@@ -197,12 +198,8 @@ ynh_psql_database_exists() {
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    # if psql is not there, we cannot check the db
-    # though it could exists.
-    if ! command -v psql; then
-        ynh_print_err -m "PostgreSQL is not installed, impossible to check for db existence."
-        return 1
-    elif ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT datname FROM pg_database WHERE datname='$database';" | grep --quiet "$database"; then
+    if ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT datname FROM pg_database WHERE datname='$database';" | grep --quiet "$database"
+    then
         return 1
     else
         return 0
@@ -223,8 +220,6 @@ ynh_psql_drop_user() {
 
 # Create a database, an user and its password. Then store the password in the app's config
 #
-# [packagingv1]
-#
 # usage: ynh_psql_setup_db --db_user=user --db_name=name [--db_pwd=pwd]
 # | arg: -u, --db_user=     - Owner of the database
 # | arg: -n, --db_name=     - Name of the database
@@ -248,7 +243,7 @@ ynh_psql_setup_db() {
         local new_db_pwd=$(ynh_string_random) # Generate a random password
         # If $db_pwd is not provided, use new_db_pwd instead for db_pwd
         db_pwd="${db_pwd:-$new_db_pwd}"
-
+        
         ynh_psql_create_user "$db_user" "$db_pwd"
     elif [ -z $db_pwd ]; then
         ynh_die --message="The user $db_user exists, please provide his password"
@@ -260,8 +255,6 @@ ynh_psql_setup_db() {
 
 # Remove a database if it exists, and the associated user
 #
-# [packagingv1]
-#
 # usage: ynh_psql_remove_db --db_user=user --db_name=name
 # | arg: -u, --db_user=     - Owner of the database
 # | arg: -n, --db_name=     - Name of the database
@@ -276,14 +269,16 @@ ynh_psql_remove_db() {
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    if ynh_psql_database_exists --database=$db_name; then # Check if the database exists
-        ynh_psql_drop_db $db_name                         # Remove the database
+    if ynh_psql_database_exists --database=$db_name
+    then # Check if the database exists
+        ynh_psql_drop_db $db_name # Remove the database
     else
         ynh_print_warn --message="Database $db_name not found"
     fi
 
     # Remove psql user if it exists
-    if ynh_psql_user_exists --user=$db_user; then
+    if ynh_psql_user_exists --user=$db_user
+    then
         ynh_psql_drop_user $db_user
     else
         ynh_print_warn --message="User $db_user not found"
@@ -291,19 +286,46 @@ ynh_psql_remove_db() {
 }
 
 # Create a master password and set up global settings
-#
-# [internal]
-#
-# usage: ynh_psql_test_if_first_run
-#
 # It also make sure that postgresql is installed and running
 # Please always call this script in install and restore scripts
 #
+# usage: ynh_psql_test_if_first_run
+#
 # Requires YunoHost version 2.7.13 or higher.
 ynh_psql_test_if_first_run() {
 
     # Make sure postgresql is indeed installed
-    dpkg --list | grep -q "ii  postgresql-$PSQL_VERSION" || ynh_die --message="postgresql-$PSQL_VERSION is not installed !?"
+    dpkg --list | grep -q "ii  postgresql-$PSQL_VERSION" || ynh_die "postgresql-$PSQL_VERSION is not installed !?"
 
-    yunohost tools regen-conf postgresql
+    # Check for some weird issue where postgresql could be installed but etc folder would not exist ...
+    [ -e "/etc/postgresql/$PSQL_VERSION" ] || ynh_die "It looks like postgresql was not properly configured ? /etc/postgresql/$PSQL_VERSION is missing ... Could be due to a locale issue, c.f.https://serverfault.com/questions/426989/postgresql-etc-postgresql-doesnt-exist"
+
+    # Make sure postgresql is started and enabled
+    # (N.B. : to check the active state, we check the cluster state because
+    # postgresql could be flagged as active even though the cluster is in
+    # failed state because of how the service is configured..)
+    systemctl is-active postgresql@$PSQL_VERSION-main -q || ynh_systemd_action --service_name=postgresql --action=restart
+    systemctl is-enabled postgresql -q || systemctl enable postgresql --quiet
+
+    # If this is the very first time, we define the root password
+    # and configure a few things
+    if [ ! -f "$PSQL_ROOT_PWD_FILE" ]
+    then
+        local pg_hba=/etc/postgresql/$PSQL_VERSION/main/pg_hba.conf
+
+        local psql_root_password="$(ynh_string_random)"
+        echo "$psql_root_password" >$PSQL_ROOT_PWD_FILE
+        sudo --login --user=postgres psql -c"ALTER user postgres WITH PASSWORD '$psql_root_password'" postgres
+
+        # force all user to connect to local databases using hashed passwords
+        # https://www.postgresql.org/docs/current/static/auth-pg-hba-conf.html#EXAMPLE-PG-HBA.CONF
+        # Note: we can't use peer since YunoHost create users with nologin
+        #  See: https://github.com/YunoHost/yunohost/blob/unstable/data/helpers.d/user
+        ynh_replace_string --match_string="local\(\s*\)all\(\s*\)all\(\s*\)peer" --replace_string="local\1all\2all\3md5" --target_file="$pg_hba"
+
+        # Integrate postgresql service in yunohost
+        yunohost service add postgresql --log "/var/log/postgresql/"
+
+        ynh_systemd_action --service_name=postgresql --action=reload
+    fi
 }

data/helpers.d/setting

@@ -8,15 +8,13 @@
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_app_setting_get() {
-    local _globalapp=${app-:}
     # Declare an array to define the options of this helper.
     local legacy_args=ak
-    local -A args_array=([a]=app= [k]=key=)
+    local -A args_array=( [a]=app= [k]=key= )
     local app
     local key
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
-    app="${app:-$_globalapp}"
 
     if [[ $key =~ (unprotected|protected|skipped)_ ]]; then
         yunohost app setting $app $key
@@ -34,16 +32,14 @@ ynh_app_setting_get() {
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_app_setting_set() {
-    local _globalapp=${app-:}
     # Declare an array to define the options of this helper.
     local legacy_args=akv
-    local -A args_array=([a]=app= [k]=key= [v]=value=)
+    local -A args_array=( [a]=app= [k]=key= [v]=value= )
     local app
     local key
     local value
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
-    app="${app:-$_globalapp}"
 
     if [[ $key =~ (unprotected|protected|skipped)_ ]]; then
         yunohost app setting $app $key -v $value
@@ -52,42 +48,6 @@ ynh_app_setting_set() {
     fi
 }
 
-# Set an application setting but only if the "$key" variable ain't set yet
-#
-# Note that it doesn't just define the setting but ALSO define the $foobar variable
-#
-# Hence it's meant as a replacement for this legacy overly complex syntax:
-#
-# if [ -z "${foo:-}" ]
-# then
-#     foo="bar"
-#     ynh_app_setting_set --key="foo" --value="$foo"
-# fi
-#
-# usage: ynh_app_setting_set_default --app=app --key=key --value=value
-# | arg: -a, --app=     - the application id
-# | arg: -k, --key=     - the setting name to set
-# | arg: -v, --value=   - the default setting value to set
-#
-# Requires YunoHost version 11.1.16 or higher.
-ynh_app_setting_set_default() {
-    local _globalapp=${app-:}
-    # Declare an array to define the options of this helper.
-    local legacy_args=akv
-    local -A args_array=([a]=app= [k]=key= [v]=value=)
-    local app
-    local key
-    local value
-    # Manage arguments with getopts
-    ynh_handle_getopts_args "$@"
-    app="${app:-$_globalapp}"
-
-    if [ -z "${!key:-}" ]; then
-        eval $key=\$value
-        ynh_app_setting "set" "$app" "$key" "$value"
-    fi
-}
-
 # Delete an application setting
 #
 # usage: ynh_app_setting_delete --app=app --key=key
@@ -96,15 +56,13 @@ ynh_app_setting_set_default() {
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_app_setting_delete() {
-    local _globalapp=${app-:}
     # Declare an array to define the options of this helper.
     local legacy_args=ak
-    local -A args_array=([a]=app= [k]=key=)
+    local -A args_array=( [a]=app= [k]=key= )
     local app
     local key
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
-    app="${app:-$_globalapp}"
 
     if [[ "$key" =~ (unprotected|skipped|protected)_ ]]; then
         yunohost app setting $app $key -d
@@ -118,16 +76,17 @@ ynh_app_setting_delete() {
 #
 # [internal]
 #
-ynh_app_setting() {
+ynh_app_setting()
+{
     set +o xtrace # set +x
-    ACTION="$1" APP="$2" KEY="$3" VALUE="${4:-}" python3 - << EOF
+    ACTION="$1" APP="$2" KEY="$3" VALUE="${4:-}" python2.7 - <<EOF
 import os, yaml, sys
 app, action = os.environ['APP'], os.environ['ACTION'].lower()
 key, value = os.environ['KEY'], os.environ.get('VALUE', None)
 setting_file = "/etc/yunohost/apps/%s/settings.yml" % app
 assert os.path.exists(setting_file), "Setting file %s does not exists ?" % setting_file
 with open(setting_file) as f:
-    settings = yaml.safe_load(f)
+    settings = yaml.load(f)
 if action == "get":
     if key in settings:
         print(settings[key])
@@ -137,7 +96,7 @@ else:
             del settings[key]
     elif action == "set":
         if key in ['redirected_urls', 'redirected_regex']:
-            value = yaml.safe_load(value)
+            value = yaml.load(value)
         settings[key] = value
     else:
         raise ValueError("action should either be get, set or delete")
@@ -149,19 +108,17 @@ EOF
 
 # Check availability of a web path
 #
-# [packagingv1]
+# example: ynh_webpath_available --domain=some.domain.tld --path_url=/coffee
 #
 # usage: ynh_webpath_available --domain=domain --path_url=path
 # | arg: -d, --domain=      - the domain/host of the url
 # | arg: -p, --path_url=    - the web path to check the availability of
 #
-# example: ynh_webpath_available --domain=some.domain.tld --path_url=/coffee
-#
 # Requires YunoHost version 2.6.4 or higher.
-ynh_webpath_available() {
+ynh_webpath_available () {
     # Declare an array to define the options of this helper.
     local legacy_args=dp
-    local -A args_array=([d]=domain= [p]=path_url=)
+    local -A args_array=( [d]=domain= [p]=path_url= )
     local domain
     local path_url
     # Manage arguments with getopts
@@ -172,20 +129,18 @@ ynh_webpath_available() {
 
 # Register/book a web path for an app
 #
-# [packagingv1]
+# example: ynh_webpath_register --app=wordpress --domain=some.domain.tld --path_url=/coffee
 #
 # usage: ynh_webpath_register --app=app --domain=domain --path_url=path
 # | arg: -a, --app=         - the app for which the domain should be registered
 # | arg: -d, --domain=      - the domain/host of the web path
 # | arg: -p, --path_url=    - the web path to be registered
 #
-# example: ynh_webpath_register --app=wordpress --domain=some.domain.tld --path_url=/coffee
-#
 # Requires YunoHost version 2.6.4 or higher.
-ynh_webpath_register() {
+ynh_webpath_register () {
     # Declare an array to define the options of this helper.
     local legacy_args=adp
-    local -A args_array=([a]=app= [d]=domain= [p]=path_url=)
+    local -A args_array=( [a]=app= [d]=domain= [p]=path_url= )
     local app
     local domain
     local path_url

data/helpers.d/string

@@ -2,27 +2,24 @@
 
 # Generate a random string
 #
+# example: pwd=$(ynh_string_random --length=8)
+#
 # usage: ynh_string_random [--length=string_length]
 # | arg: -l, --length=  - the string length to generate (default: 24)
-# | arg: -f, --filter=  - the kind of characters accepted in the output (default: 'A-Za-z0-9')
 # | ret: the generated string
 #
-# example: pwd=$(ynh_string_random --length=8)
-#
 # Requires YunoHost version 2.2.4 or higher.
 ynh_string_random() {
     # Declare an array to define the options of this helper.
-    local legacy_args=lf
-    local -A args_array=([l]=length= [f]=filter=)
+    local legacy_args=l
+    local -A args_array=( [l]=length= )
     local length
-    local filter
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
     length=${length:-24}
-    filter=${filter:-'A-Za-z0-9'}
 
     dd if=/dev/urandom bs=1 count=1000 2> /dev/null \
-        | tr --complement --delete "$filter" \
+        | tr --complement --delete 'A-Za-z0-9' \
         | sed --quiet 's/\(.\{'"$length"'\}\).*/\1/p'
 }
 
@@ -33,27 +30,26 @@ ynh_string_random() {
 # | arg: -r, --replace_string=  - String that will replace matches
 # | arg: -f, --target_file=     - File in which the string will be replaced.
 #
-# As this helper is based on sed command, regular expressions and references to
-# sub-expressions can be used (see sed manual page for more information)
+# As this helper is based on sed command, regular expressions and
+# references to sub-expressions can be used
+# (see sed manual page for more information)
 #
 # Requires YunoHost version 2.6.4 or higher.
-ynh_replace_string() {
+ynh_replace_string () {
     # Declare an array to define the options of this helper.
     local legacy_args=mrf
-    local -A args_array=([m]=match_string= [r]=replace_string= [f]=target_file=)
+    local -A args_array=( [m]=match_string= [r]=replace_string= [f]=target_file= )
     local match_string
     local replace_string
     local target_file
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
-    set +o xtrace # set +x
 
-    local delimit=$'\001'
+    local delimit=@
     # Escape the delimiter if it's in the string.
     match_string=${match_string//${delimit}/"\\${delimit}"}
     replace_string=${replace_string//${delimit}/"\\${delimit}"}
 
-    set -o xtrace # set -x
     sed --in-place "s${delimit}${match_string}${delimit}${replace_string}${delimit}g" "$target_file"
 }
 
@@ -68,10 +64,10 @@ ynh_replace_string() {
 # characters, you can't use some regular expressions and sub-expressions.
 #
 # Requires YunoHost version 2.7.7 or higher.
-ynh_replace_special_string() {
+ynh_replace_special_string () {
     # Declare an array to define the options of this helper.
     local legacy_args=mrf
-    local -A args_array=([m]=match_string= [r]=replace_string= [f]=target_file=)
+    local -A args_array=( [m]=match_string= [r]=replace_string= [f]=target_file= )
     local match_string
     local replace_string
     local target_file
@@ -90,22 +86,19 @@ ynh_replace_special_string() {
 }
 
 # Sanitize a string intended to be the name of a database
+# (More specifically : replace - and . by _)
 #
-# [packagingv1]
+# example: dbname=$(ynh_sanitize_dbid $app)
 #
 # usage: ynh_sanitize_dbid --db_name=name
 # | arg: -n, --db_name=     - name to correct/sanitize
 # | ret: the corrected name
 #
-# example: dbname=$(ynh_sanitize_dbid $app)
-#
-# Underscorify the string (replace - and . by _)
-#
 # Requires YunoHost version 2.2.4 or higher.
-ynh_sanitize_dbid() {
+ynh_sanitize_dbid () {
     # Declare an array to define the options of this helper.
     local legacy_args=n
-    local -A args_array=([n]=db_name=)
+    local -A args_array=( [n]=db_name= )
     local db_name
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
@@ -132,20 +125,20 @@ ynh_sanitize_dbid() {
 # | arg: -p, --path_url=    - URL path to normalize before using it
 #
 # Requires YunoHost version 2.6.4 or higher.
-ynh_normalize_url_path() {
+ynh_normalize_url_path () {
     # Declare an array to define the options of this helper.
     local legacy_args=p
-    local -A args_array=([p]=path_url=)
+    local -A args_array=( [p]=path_url= )
     local path_url
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
     test -n "$path_url" || ynh_die --message="ynh_normalize_url_path expect a URL path as first argument and received nothing."
-    if [ "${path_url:0:1}" != "/" ]; then # If the first character is not a /
-        path_url="/$path_url"             # Add / at begin of path variable
+    if [ "${path_url:0:1}" != "/" ]; then    # If the first character is not a /
+        path_url="/$path_url"    # Add / at begin of path variable
     fi
-    if [ "${path_url:${#path_url}-1}" == "/" ] && [ ${#path_url} -gt 1 ]; then # If the last character is a / and that not the only character.
-        path_url="${path_url:0:${#path_url}-1}"                                # Delete the last character
+    if [ "${path_url:${#path_url}-1}" == "/" ] && [ ${#path_url} -gt 1 ]; then    # If the last character is a / and that not the only character.
+        path_url="${path_url:0:${#path_url}-1}"	# Delete the last character
     fi
     echo $path_url
 }

data/helpers.d/systemd

@@ -3,28 +3,61 @@
 # Create a dedicated systemd config
 #
 # usage: ynh_add_systemd_config [--service=service] [--template=template]
-# | arg: -s, --service=     - Service name (optionnal, `$app` by default)
-# | arg: -t, --template=    - Name of template file (optionnal, this is 'systemd' by default, meaning `../conf/systemd.service` will be used as template)
+# usage: ynh_add_systemd_config [--service=service] [--template=template]  [--others_var="list of others variables to replace"]
+# | arg: -s, --service=     - Service name (optionnal, $app by default)
+# | arg: -t, --template=    - Name of template file (optionnal, this is 'systemd' by default, meaning ./conf/systemd.service will be used as template)
+# | arg: -v, --others_var=  - List of others variables to replace separated by a space. For example: 'var_1 var_2 ...'
 #
-# This will use the template `../conf/<templatename>.service`.
+# This will use the template ../conf/<templatename>.service
+# to generate a systemd config, by replacing the following keywords
+# with global variables that should be defined before calling
+# this helper :
 #
-# See the documentation of `ynh_add_config` for a description of the template
-# format and how placeholders are replaced with actual variables.
+#   __APP__       by  $app
+#   __FINALPATH__ by  $final_path
 #
-# Requires YunoHost version 4.1.0 or higher.
-ynh_add_systemd_config() {
+# And dynamic variables (from the last example) :
+#   __VAR_1__    by $var_1
+#   __VAR_2__    by $var_2
+#
+# Requires YunoHost version 2.7.11 or higher.
+ynh_add_systemd_config () {
     # Declare an array to define the options of this helper.
     local legacy_args=stv
-    local -A args_array=([s]=service= [t]=template=)
+    local -A args_array=( [s]=service= [t]=template= [v]=others_var= )
     local service
     local template
+    local others_var
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
-    service="${service:-$app}"
-    template="${template:-systemd.service}"
+    local service="${service:-$app}"
+    local template="${template:-systemd.service}"
+    others_var="${others_var:-}"
 
-    ynh_add_config --template="$template" --destination="/etc/systemd/system/$service.service"
+    finalsystemdconf="/etc/systemd/system/$service.service"
+    ynh_backup_if_checksum_is_different --file="$finalsystemdconf"
+    cp ../conf/$template "$finalsystemdconf"
 
+    # To avoid a break by set -u, use a void substitution ${var:-}. If the variable is not set, it's simply set with an empty variable.
+    # Substitute in a nginx config file only if the variable is not empty
+    if [ -n "${final_path:-}" ]; then
+        ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalsystemdconf"
+    fi
+    if [ -n "${app:-}" ]; then
+        ynh_replace_string --match_string="__APP__" --replace_string="$app" --target_file="$finalsystemdconf"
+    fi
+
+    # Replace all other variables given as arguments
+    for var_to_replace in $others_var
+    do
+        # ${var_to_replace^^} make the content of the variable on upper-cases
+        # ${!var_to_replace} get the content of the variable named $var_to_replace 
+        ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalsystemdconf"
+    done
+
+    ynh_store_file_checksum --file="$finalsystemdconf"
+
+    chown root: "$finalsystemdconf"
     systemctl enable $service --quiet
     systemctl daemon-reload
 }
@@ -35,17 +68,18 @@ ynh_add_systemd_config() {
 # | arg: -s, --service=     - Service name (optionnal, $app by default)
 #
 # Requires YunoHost version 2.7.2 or higher.
-ynh_remove_systemd_config() {
+ynh_remove_systemd_config () {
     # Declare an array to define the options of this helper.
     local legacy_args=s
-    local -A args_array=([s]=service=)
+    local -A args_array=( [s]=service= )
     local service
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
     local service="${service:-$app}"
 
     local finalsystemdconf="/etc/systemd/system/$service.service"
-    if [ -e "$finalsystemdconf" ]; then
+    if [ -e "$finalsystemdconf" ]
+    then
         ynh_systemd_action --service_name=$service --action=stop
         systemctl disable $service --quiet
         ynh_secure_remove --file="$finalsystemdconf"
@@ -56,18 +90,18 @@ ynh_remove_systemd_config() {
 # Start (or other actions) a service,  print a log in case of failure and optionnaly wait until the service is completely started
 #
 # usage: ynh_systemd_action [--service_name=service_name] [--action=action] [ [--line_match="line to match"] [--log_path=log_path] [--timeout=300] [--length=20] ]
-# | arg: -n, --service_name= - Name of the service to start. Default : `$app`
+# | arg: -n, --service_name= - Name of the service to start. Default : $app
 # | arg: -a, --action=       - Action to perform with systemctl. Default: start
 # | arg: -l, --line_match=   - Line to match - The line to find in the log to attest the service have finished to boot. If not defined it don't wait until the service is completely started.
-# | arg: -p, --log_path=     - Log file - Path to the log file. Default : `/var/log/$app/$app.log`
+# | arg: -p, --log_path=     - Log file - Path to the log file. Default : /var/log/$app/$app.log
 # | arg: -t, --timeout=      - Timeout - The maximum time to wait before ending the watching. Default : 300 seconds.
-# | arg: -e, --length=       - Length of the error log displayed for debugging : Default : 20
+# | arg: -e, --length=       - Length of the error log : Default : 20
 #
 # Requires YunoHost version 3.5.0 or higher.
 ynh_systemd_action() {
     # Declare an array to define the options of this helper.
     local legacy_args=nalpte
-    local -A args_array=([n]=service_name= [a]=action= [l]=line_match= [p]=log_path= [t]=timeout= [e]=length=)
+    local -A args_array=( [n]=service_name= [a]=action= [l]=line_match= [p]=log_path= [t]=timeout= [e]=length= )
     local service_name
     local action
     local line_match
@@ -83,16 +117,13 @@ ynh_systemd_action() {
     log_path="${log_path:-/var/log/$service_name/$service_name.log}"
     timeout=${timeout:-300}
 
-    # Manage case of service already stopped
-    if [ "$action" == "stop" ] && ! systemctl is-active --quiet $service_name; then
-        return 0
-    fi
-
     # Start to read the log
-    if [[ -n "$line_match" ]]; then
+    if [[ -n "$line_match" ]]
+    then
         local templog="$(mktemp)"
         # Following the starting of the app in its log
-        if [ "$log_path" == "systemd" ]; then
+        if [ "$log_path" == "systemd" ]
+        then
             # Read the systemd journal
             journalctl --unit=$service_name --follow --since=-0 --quiet > "$templog" &
             # Get the PID of the journalctl command
@@ -110,14 +141,14 @@ ynh_systemd_action() {
         action="reload-or-restart"
     fi
 
-    local time_start="$(date --utc --rfc-3339=seconds | cut -d+ -f1) UTC"
-
     # If the service fails to perform the action
-    if ! systemctl $action $service_name; then
+    if ! systemctl $action $service_name
+    then
         # Show syslog for this service
         ynh_exec_err journalctl --quiet --no-hostname --no-pager --lines=$length --unit=$service_name
         # If a log is specified for this service, show also the content of this log
-        if [ -e "$log_path" ]; then
+        if [ -e "$log_path" ]
+        then
             ynh_exec_err tail --lines=$length "$log_path"
         fi
         ynh_clean_check_starting
@@ -125,45 +156,37 @@ ynh_systemd_action() {
     fi
 
     # Start the timeout and try to find line_match
-    if [[ -n "${line_match:-}" ]]; then
+    if [[ -n "${line_match:-}" ]]
+    then
         set +x
         local i=0
-        local starttime=$(date +%s)
-        for i in $(seq 1 $timeout); do
+        for i in $(seq 1 $timeout)
+        do
             # Read the log until the sentence is found, that means the app finished to start. Or run until the timeout
-            if [ "$log_path" == "systemd" ]; then
-                # For systemd services, we in fact dont rely on the templog, which for some reason is not reliable, but instead re-read journalctl every iteration, starting at the timestamp where we triggered the action
-                if journalctl --unit=$service_name --since="$time_start" --quiet --no-pager --no-hostname | grep --extended-regexp --quiet "$line_match"; then
-                    ynh_print_info --message="The service $service_name has correctly executed the action ${action}."
-                    break
-                fi
-            else
-                if grep --extended-regexp --quiet "$line_match" "$templog"; then
-                    ynh_print_info --message="The service $service_name has correctly executed the action ${action}."
-                    break
-                fi
-            fi
-            if [ $i -eq 30 ]; then
-                echo "(this may take some time)" >&2
-            fi
-            # Also check the timeout using actual timestamp, because sometimes for some reason,
-            # journalctl may take a huge time to run, and we end up waiting literally an entire hour
-            # instead of 5 min ...
-            if [[ "$(($(date +%s) - $starttime))" -gt "$timeout" ]]; then
-                i=$timeout
+            if grep --extended-regexp --quiet "$line_match" "$templog"
+            then
+                ynh_print_info --message="The service $service_name has correctly executed the action ${action}."
                 break
             fi
+            if [ $i -eq 3 ]; then
+                echo -n "Please wait, the service $service_name is ${action}ing" >&2
+            fi
+            if [ $i -ge 3 ]; then
+                echo -n "." >&2
+            fi
             sleep 1
         done
         set -x
         if [ $i -ge 3 ]; then
             echo "" >&2
         fi
-        if [ $i -eq $timeout ]; then
+        if [ $i -eq $timeout ]
+        then
             ynh_print_warn --message="The service $service_name didn't fully executed the action ${action} before the timeout."
             ynh_print_warn --message="Please find here an extract of the end of the log of the service $service_name:"
             ynh_exec_warn journalctl --quiet --no-hostname --no-pager --lines=$length --unit=$service_name
-            if [ -e "$log_path" ]; then
+            if [ -e "$log_path" ]
+            then
                 ynh_print_warn --message="\-\-\-"
                 ynh_exec_warn tail --lines=$length "$log_path"
             fi
@@ -177,12 +200,15 @@ ynh_systemd_action() {
 # [internal]
 #
 # Requires YunoHost version 3.5.0 or higher.
-ynh_clean_check_starting() {
-    if [ -n "${pid_tail:-}" ]; then
+ynh_clean_check_starting () {
+    if [ -n "$pid_tail" ]
+    then
         # Stop the execution of tail.
         kill -SIGTERM $pid_tail 2>&1
     fi
-    if [ -n "${templog:-}" ]; then
+    if [ -n "$templog" ]
+    then
         ynh_secure_remove --file="$templog" 2>&1
     fi
 }
+

data/helpers.d/user

@@ -1,97 +1,138 @@
 #!/bin/bash
 
+# Check if a YunoHost user exists
+#
+# example: ynh_user_exists 'toto' || exit 1
+#
+# usage: ynh_user_exists --username=username
+# | arg: -u, --username=    - the username to check
+# | exit: Return 1 if the user doesn't exist, 0 otherwise
+#
+# Requires YunoHost version 2.2.4 or higher.
+ynh_user_exists() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=u
+    local -A args_array=( [u]=username= )
+    local username
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    yunohost user list --output-as json | grep --quiet "\"username\": \"${username}\""
+}
+
+# Retrieve a YunoHost user information
+#
+# example: mail=$(ynh_user_get_info 'toto' 'mail')
+#
+# usage: ynh_user_get_info --username=username --key=key
+# | arg: -u, --username=    - the username to retrieve info from
+# | arg: -k, --key=         - the key to retrieve
+# | ret: string - the key's value
+#
+# Requires YunoHost version 2.2.4 or higher.
+ynh_user_get_info() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=uk
+    local -A args_array=( [u]=username= [k]=key= )
+    local username
+    local key
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    yunohost user info "$username" --output-as plain | ynh_get_plain_key "$key"
+}
+
+# Get the list of YunoHost users
+#
+# example: for u in $(ynh_user_list); do ...
+#
+# usage: ynh_user_list
+# | ret: string - one username per line
+#
+# Requires YunoHost version 2.4.0 or higher.
+ynh_user_list() {
+    yunohost user list --output-as plain --quiet \
+        | awk '/^##username$/{getline; print}'
+}
+
 # Check if a user exists on the system
 #
-# [packagingv1]
-#
 # usage: ynh_system_user_exists --username=username
 # | arg: -u, --username=    - the username to check
-# | ret: 0 if the user exists, 1 otherwise.
+# | exit: Return 1 if the user doesn't exist, 0 otherwise
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_system_user_exists() {
     # Declare an array to define the options of this helper.
     local legacy_args=u
-    local -A args_array=([u]=username=)
+    local -A args_array=( [u]=username= )
     local username
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    getent passwd "$username" &> /dev/null
+    getent passwd "$username" &>/dev/null
 }
 
 # Check if a group exists on the system
 #
-# [packagingv1]
-#
 # usage: ynh_system_group_exists --group=group
 # | arg: -g, --group=   - the group to check
-# | ret: 0 if the group exists, 1 otherwise.
+# | exit: Return 1 if the group doesn't exist, 0 otherwise
 #
 # Requires YunoHost version 3.5.0.2 or higher.
 ynh_system_group_exists() {
     # Declare an array to define the options of this helper.
     local legacy_args=g
-    local -A args_array=([g]=group=)
+    local -A args_array=( [g]=group= )
     local group
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    getent group "$group" &> /dev/null
+    getent group "$group" &>/dev/null
 }
 
 # Create a system user
 #
-# usage: ynh_system_user_create --username=user_name [--home_dir=home_dir] [--use_shell] [--groups="group1 group2"]
+# examples:
+#   # Create a nextcloud user with no home directory and /usr/sbin/nologin login shell (hence no login capability)
+#   ynh_system_user_create --username=nextcloud
+#   # Create a discourse user using /var/www/discourse as home directory and the default login shell
+#   ynh_system_user_create --username=discourse --home_dir=/var/www/discourse --use_shell 
+#
+# usage: ynh_system_user_create --username=user_name [--home_dir=home_dir] [--use_shell]
 # | arg: -u, --username=    - Name of the system user that will be create
 # | arg: -h, --home_dir=    - Path of the home dir for the user. Usually the final path of the app. If this argument is omitted, the user will be created without home
 # | arg: -s, --use_shell    - Create a user using the default login shell if present. If this argument is omitted, the user will be created with /usr/sbin/nologin shell
-# | arg: -g, --groups       - Add the user to system groups. Typically meant to add the user to the ssh.app / sftp.app group (e.g. for borgserver, my_webapp)
-#
-# Create a nextcloud user with no home directory and /usr/sbin/nologin login shell (hence no login capability) :
-# ```
-# ynh_system_user_create --username=nextcloud
-# ```
-# Create a discourse user using /var/www/discourse as home directory and the default login shell :
-# ```
-# ynh_system_user_create --username=discourse --home_dir=/var/www/discourse --use_shell
-# ```
 #
 # Requires YunoHost version 2.6.4 or higher.
-ynh_system_user_create() {
+ynh_system_user_create () {
     # Declare an array to define the options of this helper.
     local legacy_args=uhs
-    local -A args_array=([u]=username= [h]=home_dir= [s]=use_shell [g]=groups=)
+    local -A args_array=( [u]=username= [h]=home_dir= [s]=use_shell )
     local username
     local home_dir
     local use_shell
-    local groups
-
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
     use_shell="${use_shell:-0}"
     home_dir="${home_dir:-}"
-    groups="${groups:-}"
 
-    if ! ynh_system_user_exists "$username"; then # Check if the user exists on the system
-        # If the user doesn't exist
-        if [ -n "$home_dir" ]; then # If a home dir is mentioned
+    if ! ynh_system_user_exists "$username"	# Check if the user exists on the system
+    then	# If the user doesn't exist
+        if [ -n "$home_dir" ]
+        then	# If a home dir is mentioned
             local user_home_dir="--home-dir $home_dir"
         else
             local user_home_dir="--no-create-home"
         fi
-        if [ $use_shell -eq 1 ]; then # If we want a shell for the user
-            local shell=""            # Use default shell
+        if [ $use_shell -eq 1 ]
+        then	# If we want a shell for the user
+            local shell="" # Use default shell
         else
             local shell="--shell /usr/sbin/nologin"
         fi
         useradd $user_home_dir --system --user-group $username $shell || ynh_die --message="Unable to create $username system account"
     fi
-
-    local group
-    for group in $groups; do
-        usermod -a -G "$group" "$username"
-    done
 }
 
 # Delete a system user
@@ -100,23 +141,25 @@ ynh_system_user_create() {
 # | arg: -u, --username=    - Name of the system user that will be create
 #
 # Requires YunoHost version 2.6.4 or higher.
-ynh_system_user_delete() {
+ynh_system_user_delete () {
     # Declare an array to define the options of this helper.
     local legacy_args=u
-    local -A args_array=([u]=username=)
+    local -A args_array=( [u]=username= )
     local username
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
     # Check if the user exists on the system
-    if ynh_system_user_exists "$username"; then
+    if ynh_system_user_exists "$username"
+    then
         deluser $username
     else
         ynh_print_warn --message="The user $username was not found"
     fi
 
     # Check if the group exists on the system
-    if ynh_system_group_exists "$username"; then
+    if ynh_system_group_exists "$username"
+    then
         delgroup $username
     fi
 }

data/helpers.d/utils

@@ -0,0 +1,693 @@
+#!/bin/bash
+
+# Handle script crashes / failures
+#
+# [internal]
+#
+# usage:
+# ynh_exit_properly is used only by the helper ynh_abort_if_errors.
+# You should not use it directly.
+# Instead, add to your script:
+# ynh_clean_setup () {
+#        instructions...
+# }
+#
+# This function provide a way to clean some residual of installation that not managed by remove script.
+#
+# It prints a warning to inform that the script was failed, and execute the ynh_clean_setup function if used in the app script
+#
+# Requires YunoHost version 2.6.4 or higher.
+ynh_exit_properly () {
+    local exit_code=$?
+    if [ "$exit_code" -eq 0 ]; then
+        exit 0	# Exit without error if the script ended correctly
+    fi
+
+    trap '' EXIT	# Ignore new exit signals
+    # Do not exit anymore if a command fail or if a variable is empty
+    set +o errexit  # set +e
+    set +o nounset  # set +u
+
+    # Small tempo to avoid the next message being mixed up with other DEBUG messages
+    sleep 0.5
+
+    if type -t ynh_clean_setup > /dev/null; then	# Check if the function exist in the app script.
+        ynh_clean_setup	# Call the function to do specific cleaning for the app.
+    fi
+
+    # Exit with error status
+    # We don't call ynh_die basically to avoid unecessary 10-ish
+    # debug lines about parsing args and stuff just to exit 1..
+    exit 1
+}
+
+# Exits if an error occurs during the execution of the script.
+#
+# usage: ynh_abort_if_errors
+#
+# This configure the rest of the script execution such that, if an error occurs
+# or if an empty variable is used, the execution of the script stops
+# immediately and a call to `ynh_clean_setup` is triggered if it has been
+# defined by your script.
+#
+# Requires YunoHost version 2.6.4 or higher.
+ynh_abort_if_errors () {
+    set -o errexit  # set -e; Exit if a command fail
+    set -o nounset  # set -u; And if a variable is used unset
+    trap ynh_exit_properly EXIT	# Capturing exit signals on shell script
+}
+
+# Download, check integrity, uncompress and patch the source from app.src
+#
+# usage: ynh_setup_source --dest_dir=dest_dir [--source_id=source_id]
+# | arg: -d, --dest_dir=    - Directory where to setup sources
+# | arg: -s, --source_id=   - Name of the app, if the package contains more than one app
+#
+# The file conf/app.src need to contains:
+#
+# SOURCE_URL=Address to download the app archive
+# SOURCE_SUM=Control sum
+# # (Optional) Program to check the integrity (sha256sum, md5sum...)
+# # default: sha256
+# SOURCE_SUM_PRG=sha256
+# # (Optional) Archive format
+# # default: tar.gz
+# SOURCE_FORMAT=tar.gz
+# # (Optional) Put false if sources are directly in the archive root
+# # default: true
+# # Instead of true, SOURCE_IN_SUBDIR could be the number of sub directories
+# # to remove.
+# SOURCE_IN_SUBDIR=false
+# # (Optionnal) Name of the local archive (offline setup support)
+# # default: ${src_id}.${src_format}
+# SOURCE_FILENAME=example.tar.gz
+# # (Optional) If it set as false don't extract the source.
+# # (Useful to get a debian package or a python wheel.)
+# # default: true
+# SOURCE_EXTRACT=(true|false)
+#
+# Details:
+# This helper downloads sources from SOURCE_URL if there is no local source
+# archive in /opt/yunohost-apps-src/APP_ID/SOURCE_FILENAME
+#
+# Next, it checks the integrity with "SOURCE_SUM_PRG -c --status" command.
+#
+# If it's ok, the source archive will be uncompressed in $dest_dir. If the
+# SOURCE_IN_SUBDIR is true, the first level directory of the archive will be
+# removed.
+# If SOURCE_IN_SUBDIR is a numeric value, 2 for example, the 2 first level
+# directories will be removed
+#
+# Finally, patches named sources/patches/${src_id}-*.patch and extra files in
+# sources/extra_files/$src_id will be applied to dest_dir
+#
+# Requires YunoHost version 2.6.4 or higher.
+ynh_setup_source () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=ds
+    local -A args_array=( [d]=dest_dir= [s]=source_id= )
+    local dest_dir
+    local source_id
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    source_id="${source_id:-app}" # If the argument is not given, source_id equals "app"
+
+    local src_file_path="$YNH_CWD/../conf/${source_id}.src"
+    # In case of restore script the src file is in an other path.
+    # So try to use the restore path if the general path point to no file.
+    if [ ! -e "$src_file_path" ]; then
+        src_file_path="$YNH_CWD/../settings/conf/${source_id}.src"
+    fi
+
+    # Load value from configuration file (see above for a small doc about this file
+    # format)
+    local src_url=$(grep 'SOURCE_URL=' "$src_file_path" | cut --delimiter='=' --fields=2-)
+    local src_sum=$(grep 'SOURCE_SUM=' "$src_file_path" | cut --delimiter='=' --fields=2-)
+    local src_sumprg=$(grep 'SOURCE_SUM_PRG=' "$src_file_path" | cut --delimiter='=' --fields=2-)
+    local src_format=$(grep 'SOURCE_FORMAT=' "$src_file_path" | cut --delimiter='=' --fields=2-)
+    local src_extract=$(grep 'SOURCE_EXTRACT=' "$src_file_path" | cut --delimiter='=' --fields=2-)
+    local src_in_subdir=$(grep 'SOURCE_IN_SUBDIR=' "$src_file_path" | cut --delimiter='=' --fields=2-)
+    local src_filename=$(grep 'SOURCE_FILENAME=' "$src_file_path" | cut --delimiter='=' --fields=2-)
+
+    # Default value
+    src_sumprg=${src_sumprg:-sha256sum}
+    src_in_subdir=${src_in_subdir:-true}
+    src_format=${src_format:-tar.gz}
+    src_format=$(echo "$src_format" | tr '[:upper:]' '[:lower:]')
+    src_extract=${src_extract:-true}
+    if [ "$src_filename" = "" ]; then
+        src_filename="${source_id}.${src_format}"
+    fi
+    local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${src_filename}"
+
+    if test -e "$local_src"
+    then    # Use the local source file if it is present
+        cp $local_src $src_filename
+    else    # If not, download the source
+        # NB. we have to declare the var as local first,
+        # otherwise 'local foo=$(false) || echo 'pwet'" does'nt work
+        # because local always return 0 ...
+        local out
+        # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget)
+        out=$(wget --tries 3 --no-dns-cache --timeout 900 --no-verbose --output-document=$src_filename $src_url 2>&1) \
+            || ynh_die --message="$out"
+    fi
+
+    # Check the control sum
+    echo "${src_sum} ${src_filename}" | ${src_sumprg} --check --status \
+        || ynh_die --message="Corrupt source"
+
+    # Extract source into the app dir
+    mkdir --parents "$dest_dir"
+
+    if ! "$src_extract"
+    then
+        mv $src_filename $dest_dir
+    elif [ "$src_format" = "zip" ]
+    then
+        # Zip format
+        # Using of a temp directory, because unzip doesn't manage --strip-components
+        if $src_in_subdir
+        then
+            local tmp_dir=$(mktemp --directory)
+            unzip -quo $src_filename -d "$tmp_dir"
+            cp --archive $tmp_dir/*/. "$dest_dir"
+            ynh_secure_remove --file="$tmp_dir"
+        else
+            unzip -quo $src_filename -d "$dest_dir"
+        fi
+    else
+        local strip=""
+        if [ "$src_in_subdir" != "false" ]
+        then
+            if [ "$src_in_subdir" == "true" ]
+            then
+                local sub_dirs=1
+            else
+                local sub_dirs="$src_in_subdir"
+            fi
+            strip="--strip-components $sub_dirs"
+        fi
+        if [[ "$src_format" =~ ^tar.gz|tar.bz2|tar.xz$ ]]
+        then
+            tar --extract --file=$src_filename --directory="$dest_dir" $strip
+        else
+            ynh_die --message="Archive format unrecognized."
+        fi
+    fi
+
+    # Apply patches
+    if (( $(find $YNH_CWD/../sources/patches/ -type f -name "${source_id}-*.patch" 2> /dev/null | wc --lines) > "0" ))
+    then
+        (cd "$dest_dir"
+        for p in $YNH_CWD/../sources/patches/${source_id}-*.patch
+        do
+            patch --strip=1 < $p
+        done) || ynh_die --message="Unable to apply patches"
+    fi
+
+    # Add supplementary files
+    if test -e "$YNH_CWD/../sources/extra_files/${source_id}"; then
+        cp --archive $YNH_CWD/../sources/extra_files/$source_id/. "$dest_dir"
+    fi
+}
+
+# Curl abstraction to help with POST requests to local pages (such as installation forms)
+#
+# example: ynh_local_curl "/install.php?installButton" "foo=$var1" "bar=$var2"
+#
+# usage: ynh_local_curl "page_uri" "key1=value1" "key2=value2" ...
+# | arg: page_uri    - Path (relative to $path_url) of the page where POST data will be sent
+# | arg: key1=value1 - (Optionnal) POST key and corresponding value
+# | arg: key2=value2 - (Optionnal) Another POST key and corresponding value
+# | arg: ...         - (Optionnal) More POST keys and values
+#
+# For multiple calls, cookies are persisted between each call for the same app
+#
+# $domain and $path_url should be defined externally (and correspond to the domain.tld and the /path (of the app?))
+#
+# Requires YunoHost version 2.6.4 or higher.
+ynh_local_curl () {
+    # Define url of page to curl
+    local local_page=$(ynh_normalize_url_path $1)
+    local full_path=$path_url$local_page
+
+    if [ "${path_url}" == "/" ]; then
+        full_path=$local_page
+    fi
+
+    local full_page_url=https://localhost$full_path
+
+    # Concatenate all other arguments with '&' to prepare POST data
+    local POST_data=""
+    local arg=""
+    for arg in "${@:2}"
+    do
+        POST_data="${POST_data}${arg}&"
+    done
+    if [ -n "$POST_data" ]
+    then
+        # Add --data arg and remove the last character, which is an unecessary '&'
+        POST_data="--data ${POST_data::-1}"
+    fi
+
+    # Wait untils nginx has fully reloaded (avoid curl fail with http2)
+    sleep 2
+    
+    local cookiefile=/tmp/ynh-$app-cookie.txt
+    touch $cookiefile
+    chown root $cookiefile
+    chmod 700 $cookiefile
+
+    # Curl the URL
+    curl --silent --show-error --insecure --location --header "Host: $domain" --resolve $domain:443:127.0.0.1 $POST_data "$full_page_url" --cookie-jar $cookiefile --cookie $cookiefile
+}
+
+# Create a dedicated config file from a template
+#
+# examples:
+#    ynh_add_config --template=".env" --destination="$final_path/.env"
+#    ynh_add_config --template="../conf/.env" --destination="$final_path/.env"
+#    ynh_add_config --template="/etc/nginx/sites-available/default" --destination="etc/nginx/sites-available/mydomain.conf"
+#
+# usage: ynh_add_config --template="template" --destination="destination"
+# | arg: -t, --template=     - Template config file to use
+# | arg: -d, --destination=    - Destination of the config file
+#
+# The template can be by default the name of a file in the conf directory
+# of a YunoHost Package, a relative path or an absolute path
+# The helper will use the template $template to generate a config file
+# $destination by replacing the following keywords with global variables
+# that should be defined before calling this helper :
+#   __PATH__                by $path_url
+#   __NAME__                by $app
+#   __NAMETOCHANGE__        by $app
+#   __USER__                by $app
+#   __FINALPATH__           by $final_path
+#   __PHPVERSION__          by $YNH_PHP_VERSION
+#   __YNH_NODE_LOAD_PATH__  by $ynh_node_load_PATH
+#
+# And any dynamic variables that should be defined before calling this helper like:
+#   __DOMAIN__   by $domain
+#   __APP__      by $app
+#   __VAR_1__    by $var_1
+#   __VAR_2__    by $var_2
+#
+# The helper will verify the checksum and backup the destination file
+# if it's different before applying the new template.
+# And it will calculate and store the destination file checksum
+# into the app settings when configuration is done.
+#
+# Requires YunoHost version 4.1.0 or higher.
+ynh_add_config () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=tdv
+    local -A args_array=( [t]=template= [d]=destination= )
+    local template
+    local destination
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    local template_path
+
+    if [ -f "../conf/$template" ]; then
+        template_path="../conf/$template"
+    elif [ -f "../settings/conf/$template" ]; then
+        template_path="../settings/conf/$template"
+    elif [ -f "$template" ]; then
+        template_path=$template
+    else
+        ynh_die --message="The provided template $template doesn't exist"
+    fi
+
+    ynh_backup_if_checksum_is_different --file="$destination"
+
+    cp -f "$template_path" "$destination"
+
+    ynh_replace_vars --file="$destination"
+
+    ynh_store_file_checksum --file="$destination"
+}
+
+# Replace variables in a file
+#
+# [internal]
+#
+# usage: ynh_replace_vars --file="file"
+# | arg: -f, --file=     - File where to replace variables
+#
+# The helper will replace the following keywords with global variables
+# that should be defined before calling this helper :
+#   __PATH__                by $path_url
+#   __NAME__                by $app
+#   __NAMETOCHANGE__        by $app
+#   __USER__                by $app
+#   __FINALPATH__           by $final_path
+#   __PHPVERSION__          by $YNH_PHP_VERSION
+#   __YNH_NODE_LOAD_PATH__  by $ynh_node_load_PATH
+#
+# And any dynamic variables that should be defined before calling this helper like:
+#   __DOMAIN__   by $domain
+#   __APP__      by $app
+#   __VAR_1__    by $var_1
+#   __VAR_2__    by $var_2
+#
+# Requires YunoHost version 4.1.0 or higher.
+ynh_replace_vars () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=f
+    local -A args_array=( [f]=file= )
+    local file
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    # Replace specific YunoHost variables
+    if test -n "${path_url:-}"
+    then
+        # path_url_slash_less is path_url, or a blank value if path_url is only '/'
+        local path_url_slash_less=${path_url%/}
+        ynh_replace_string --match_string="__PATH__/" --replace_string="$path_url_slash_less/" --target_file="$file"
+        ynh_replace_string --match_string="__PATH__" --replace_string="$path_url" --target_file="$file"
+    fi
+    if test -n "${app:-}"; then
+        ynh_replace_string --match_string="__NAME__" --replace_string="$app" --target_file="$file"
+        ynh_replace_string --match_string="__NAMETOCHANGE__" --replace_string="$app" --target_file="$file"
+        ynh_replace_string --match_string="__USER__" --replace_string="$app" --target_file="$file"
+    fi
+    if test -n "${final_path:-}"; then
+        ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$file"
+    fi
+    if test -n "${YNH_PHP_VERSION:-}"; then
+        ynh_replace_string --match_string="__PHPVERSION__" --replace_string="$YNH_PHP_VERSION" --target_file="$file"
+    fi
+    if test -n "${ynh_node_load_PATH:-}"; then
+        ynh_replace_string --match_string="__YNH_NODE_LOAD_PATH__" --replace_string="$ynh_node_load_PATH" --target_file="$file"
+    fi
+
+    # Replace others variables
+
+    # List other unique (__ __) variables in $file
+    local uniques_vars=( $(grep -oP '__[A-Z0-9]+?[A-Z0-9_]*?[A-Z0-9]*?__' $file | sort --unique | sed "s@__\([^.]*\)__@\L\1@g" ))
+
+    # Do the replacement
+    local delimit=@
+    for one_var in "${uniques_vars[@]}"
+    do
+        # Validate that one_var is indeed defined
+        # -v checks if the variable is defined, for example:
+        #     -v FOO  tests if $FOO is defined
+        #     -v $FOO tests if ${!FOO} is defined
+        # More info: https://stackoverflow.com/questions/3601515/how-to-check-if-a-variable-is-set-in-bash/17538964#comment96392525_17538964
+        [[ -v "${one_var:-}" ]] || ynh_die --message="Variable \$$one_var wasn't initialized when trying to replace __${one_var^^}__ in $file"
+
+        # Escape delimiter in match/replace string
+        match_string="__${one_var^^}__"
+        match_string=${match_string//${delimit}/"\\${delimit}"}
+        replace_string="${!one_var}"
+        replace_string=${replace_string//\\/\\\\}
+        replace_string=${replace_string//${delimit}/"\\${delimit}"}
+
+        # Actually replace (sed is used instead of ynh_replace_string to avoid triggering an epic amount of debug logs)
+        sed --in-place "s${delimit}${match_string}${delimit}${replace_string}${delimit}g" "$file"
+    done
+}
+
+# Render templates with Jinja2
+#
+# [internal]
+#
+# Attention : Variables should be exported before calling this helper to be
+# accessible inside templates.
+#
+# usage: ynh_render_template some_template output_path
+# | arg: some_template - Template file to be rendered
+# | arg: output_path   - The path where the output will be redirected to
+ynh_render_template() {
+    local template_path=$1
+    local output_path=$2
+    mkdir -p "$(dirname $output_path)"
+    # Taken from https://stackoverflow.com/a/35009576
+    python2.7 -c 'import os, sys, jinja2; sys.stdout.write(
+                    jinja2.Template(sys.stdin.read()
+                    ).render(os.environ));' < $template_path > $output_path
+}
+
+# Fetch the Debian release codename
+#
+# usage: ynh_get_debian_release
+# | ret: The Debian release codename (i.e. jessie, stretch, ...)
+#
+# Requires YunoHost version 2.7.12 or higher.
+ynh_get_debian_release () {
+    echo $(lsb_release --codename --short)
+}
+
+# Create a directory under /tmp
+#
+# [internal]
+#
+# Deprecated helper
+#
+# usage: ynh_mkdir_tmp
+# | ret: the created directory path
+ynh_mkdir_tmp() {
+    ynh_print_warn --message="The helper ynh_mkdir_tmp is deprecated."
+    ynh_print_warn --message="You should use 'mktemp -d' instead and manage permissions \
+properly with chmod/chown."
+    local TMP_DIR=$(mktemp --directory)
+
+    # Give rights to other users could be a security risk.
+    # But for retrocompatibility we need it. (This helpers is deprecated)
+    chmod 755 $TMP_DIR
+    echo $TMP_DIR
+}
+
+# Remove a file or a directory securely
+#
+# usage: ynh_secure_remove --file=path_to_remove
+# | arg: -f, --file=    - File or directory to remove
+#
+# Requires YunoHost version 2.6.4 or higher.
+ynh_secure_remove () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=f
+    local -A args_array=( [f]=file= )
+    local file
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    local forbidden_path=" \
+    /var/www \
+    /home/yunohost.app"
+
+    if [ $# -ge 2 ]
+    then
+        ynh_print_warn --message="/!\ Packager ! You provided more than one argument to ynh_secure_remove but it will be ignored... Use this helper with one argument at time."
+    fi
+
+    if [[ -z "$file" ]]
+    then
+        ynh_print_warn --message="ynh_secure_remove called with empty argument, ignoring."
+    elif [[ "$forbidden_path" =~ "$file" \
+        # Match all paths or subpaths in $forbidden_path
+        || "$file" =~ ^/[[:alnum:]]+$ \
+        # Match all first level paths from / (Like /var, /root, etc...)
+        || "${file:${#file}-1}" = "/" ]]
+        # Match if the path finishes by /. Because it seems there is an empty variable
+    then
+        ynh_print_warn --message="Not deleting '$file' because it is not an acceptable path to delete."
+    elif [ -e "$file" ]
+    then
+        rm --recursive "$file"
+    else
+        ynh_print_info --message="'$file' wasn't deleted because it doesn't exist."
+    fi
+}
+
+# Extract a key from a plain command output
+#
+# [internal]
+#
+# example: yunohost user info tata --output-as plain | ynh_get_plain_key mail
+#
+# usage: ynh_get_plain_key key [subkey [subsubkey ...]]
+# | ret: string - the key's value
+#
+# Requires YunoHost version 2.2.4 or higher.
+ynh_get_plain_key() {
+    local prefix="#"
+    local founded=0
+    # We call this key_ so that it's not caught as
+    # an info to be redacted by the core
+    local key_=$1
+    shift
+    while read line
+    do
+        if [[ "$founded" == "1" ]]
+        then
+            [[ "$line" =~ ^${prefix}[^#] ]] && return
+            echo $line
+        elif [[ "$line" =~ ^${prefix}${key_}$ ]]
+        then
+            if [[ -n "${1:-}" ]]
+            then
+                prefix+="#"
+                key_=$1
+                shift
+            else
+                founded=1
+            fi
+        fi
+    done
+}
+
+# Read the value of a key in a ynh manifest file
+#
+# usage: ynh_read_manifest --manifest="manifest.json" --key="key"
+# | arg: -m, --manifest=    - Path of the manifest to read
+# | arg: -k, --key=         - Name of the key to find
+# | ret: the value associate to that key
+#
+# Requires YunoHost version 3.5.0 or higher.
+ynh_read_manifest () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=mk
+    local -A args_array=( [m]=manifest= [k]=manifest_key= )
+    local manifest
+    local manifest_key
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    if [ ! -e "$manifest" ]; then
+        # If the manifest isn't found, try the common place for backup and restore script.
+        manifest="../settings/manifest.json"
+    fi
+
+    jq ".$manifest_key" "$manifest" --raw-output
+}
+
+# Read the upstream version from the manifest, or from the env variable $YNH_APP_MANIFEST_VERSION if not given
+#
+# usage: ynh_app_upstream_version [--manifest="manifest.json"]
+# | arg: -m, --manifest=    - Path of the manifest to read
+# | ret: the version number of the upstream app
+#
+# The version number in the manifest is defined by <upstreamversion>~ynh<packageversion>
+# For example : 4.3-2~ynh3
+# This include the number before ~ynh
+# In the last example it return 4.3-2
+#
+# Requires YunoHost version 3.5.0 or higher.
+ynh_app_upstream_version () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=m
+    local -A args_array=( [m]=manifest= )
+    local manifest
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    manifest="${manifest:-}"
+
+    if [[ "$manifest" != "" ]] && [[ -e "$manifest" ]];
+    then
+      version_key_=$(ynh_read_manifest --manifest="$manifest" --manifest_key="version")
+    else
+      version_key_=$YNH_APP_MANIFEST_VERSION
+    fi
+
+    echo "${version_key_/~ynh*/}"
+}
+
+# Read package version from the manifest
+#
+# usage: ynh_app_package_version [--manifest="manifest.json"]
+# | arg: -m, --manifest=    - Path of the manifest to read
+# | ret: the version number of the package
+#
+# The version number in the manifest is defined by <upstreamversion>~ynh<packageversion>
+# For example : 4.3-2~ynh3
+# This include the number after ~ynh
+# In the last example it return 3
+#
+# Requires YunoHost version 3.5.0 or higher.
+ynh_app_package_version () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=m
+    local -A args_array=( [m]=manifest= )
+    local manifest
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    version_key_=$YNH_APP_MANIFEST_VERSION
+    echo "${version_key_/*~ynh/}"
+}
+
+# Checks the app version to upgrade with the existing app version and returns:
+#
+# - UPGRADE_PACKAGE if only the YunoHost package has changed
+# - UPGRADE_APP otherwise
+#
+# This helper should be used to avoid an upgrade of an app, or the upstream part
+# of it, when it's not needed
+#
+# To force an upgrade, even if the package is up to date,
+# you have to use the parameter --force (or -F).
+# example: sudo yunohost app upgrade MyApp --force
+#
+# usage: ynh_check_app_version_changed
+#
+# Requires YunoHost version 3.5.0 or higher.
+ynh_check_app_version_changed () {
+    local return_value=${YNH_APP_UPGRADE_TYPE}
+
+    if [ "$return_value" == "UPGRADE_FULL" ] || [ "$return_value" == "UPGRADE_FORCED" ] || [ "$return_value" == "DOWNGRADE_FORCED" ]
+    then
+        return_value="UPGRADE_APP"
+    fi
+
+    echo $return_value
+}
+
+# Compare the current package version against another version given as an argument.
+# This is really useful when we need to do some actions only for some old package versions.
+#
+# example: ynh_compare_current_package_version --comparison lt --version 2.3.2~ynh1
+# This example will check if the installed version is lower than (lt) the version 2.3.2~ynh1
+#
+# Generally you might probably use it as follow in the upgrade script
+#
+# if ynh_compare_current_package_version --comparison lt --version 2.3.2~ynh1
+# then
+#     # Do something that is needed for the package version older than 2.3.2~ynh1
+# fi
+#
+# usage: ynh_compare_current_package_version --comparison lt|le|eq|ne|ge|gt
+# | arg: --comparison - Comparison type. Could be : lt (lower than), le (lower or equal),
+# | eq (equal), ne (not equal), ge (greater or equal), gt (greater than)
+# | arg: --version - The version to compare. Need to be a version in the yunohost package version type (like 2.3.1~ynh4)
+#
+# Return 0 if the evaluation is true. 1 if false.
+#
+# Requires YunoHost version 3.8.0 or higher.
+ynh_compare_current_package_version() {
+    local legacy_args=cv
+    declare -Ar args_array=( [c]=comparison= [v]=version= )
+    local version
+    local comparison
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    local current_version=$YNH_APP_CURRENT_VERSION
+
+    # Check the syntax of the versions
+    if [[ ! $version =~ '~ynh' ]] || [[ ! $current_version =~ '~ynh' ]]
+    then
+        ynh_die --message="Invalid argument for version."
+    fi
+
+    # Check validity of the comparator
+    if [[ ! $comparison =~ (lt|le|eq|ne|ge|gt) ]]; then
+        ynh_die --message="Invialid comparator must be : lt, le, eq, ne, ge, gt"
+    fi
+
+    # Return the return value of dpkg --compare-versions
+    dpkg --compare-versions $current_version $comparison $version
+}

data/hooks/backup/05-conf_ldap

@@ -11,6 +11,7 @@ backup_dir="${1}/conf/ldap"
 
 # Backup the configuration
 ynh_backup "/etc/ldap/ldap.conf" "${backup_dir}/ldap.conf"
+ynh_backup "/etc/ldap/slapd.ldif" "${backup_dir}/slapd.ldif"
 slapcat -b cn=config -l "${backup_dir}/cn=config.master.ldif"
 
 # Backup the database

data/hooks/backup/08-conf_ssh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Exit hook on subcommand error or unset variable
+set -eu
+
+# Source YNH helpers
+source /usr/share/yunohost/helpers
+
+# Backup destination
+backup_dir="${1}/conf/ssh"
+
+# Backup the configuration
+if [ -d /etc/ssh/ ]; then
+    ynh_backup "/etc/ssh" "$backup_dir"
+else
+    echo "SSH is not installed"
+fi

data/hooks/backup/14-conf_ssowat

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Exit hook on subcommand error or unset variable
+set -eu
+
+# Source YNH helpers
+source /usr/share/yunohost/helpers
+
+# Backup destination
+backup_dir="${1}/conf/ssowat"
+
+# Backup the configuration
+ynh_backup "/etc/ssowat" "$backup_dir"

data/hooks/backup/20-conf_ynh_firewall

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Exit hook on subcommand error or unset variable
+set -eu
+
+# Source YNH helpers
+source /usr/share/yunohost/helpers
+
+# Backup destination
+backup_dir="${1}/conf/ynh/firewall"
+
+# Backup the configuration
+ynh_backup "/etc/yunohost/firewall.yml" "${backup_dir}/firewall.yml"

data/hooks/backup/22-conf_mail

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+source /usr/share/yunohost/helpers
+ynh_abort_if_errors
+YNH_CWD="${YNH_BACKUP_DIR%/}/conf/dkim"
+mkdir -p "$YNH_CWD"
+cd "$YNH_CWD"
+
+ynh_backup --src_path="/etc/dkim"

data/hooks/backup/26-conf_xmpp

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# Exit hook on subcommand error or unset variable
+set -eu
+
+# Source YNH helpers
+source /usr/share/yunohost/helpers
+
+# Backup destination
+backup_dir="${1}/conf/xmpp"
+
+# Backup the configuration
+ynh_backup /etc/metronome "${backup_dir}/etc"
+ynh_backup /var/lib/metronome "${backup_dir}/var"

data/hooks/backup/29-conf_nginx

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Exit hook on subcommand error or unset variable
+set -eu
+
+# Source YNH helpers
+source /usr/share/yunohost/helpers
+
+# Backup destination
+backup_dir="${1}/conf/nginx"
+
+# Backup the configuration
+ynh_backup "/etc/nginx/conf.d" "$backup_dir"

data/hooks/backup/32-conf_cron

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Exit hook on subcommand error or unset variable
+set -eu
+
+# Source YNH helpers
+source /usr/share/yunohost/helpers
+
+# Backup destination
+backup_dir="${1}/conf/cron"
+
+# Backup the configuration
+for f in $(ls -1B /etc/cron.d/yunohost* 2> /dev/null); do
+  ynh_backup "$f" "${backup_dir}/${f##*/}"
+done

data/hooks/backup/40-conf_ynh_currenthost

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Exit hook on subcommand error or unset variable
+set -eu
+
+# Source YNH helpers
+source /usr/share/yunohost/helpers
+
+# Backup destination
+backup_dir="${1}/conf/ynh"
+
+# Backup the configuration
+ynh_backup "/etc/yunohost/current_host" "${backup_dir}/current_host"

data/hooks/backup/42-conf_ynh_dyndns

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+source /usr/share/yunohost/helpers
+ynh_abort_if_errors
+YNH_CWD="${YNH_BACKUP_DIR%/}/conf/ynh/dyndns"
+mkdir -p $YNH_CWD
+cd "$YNH_CWD"
+
+# Backup the configuration
+ynh_exec_warn_less ynh_backup --src_path="/etc/yunohost/dyndns" --not_mandatory
+ynh_exec_warn_less ynh_backup --src_path="/etc/cron.d/yunohost-dyndns" --not_mandatory

data/hooks/conf_regen/01-yunohost

@@ -0,0 +1,195 @@
+#!/bin/bash
+
+set -e
+
+services_path="/etc/yunohost/services.yml"
+
+do_init_regen() {
+  if [[ $EUID -ne 0 ]]; then
+    echo "You must be root to run this script" 1>&2
+    exit 1
+  fi
+
+  cd /usr/share/yunohost/templates/yunohost
+
+  [[ -d /etc/yunohost ]] || mkdir -p /etc/yunohost
+
+  # set default current_host
+  [[ -f /etc/yunohost/current_host ]] \
+    || echo "yunohost.org" > /etc/yunohost/current_host
+
+  # copy default services and firewall
+  [[ -f $services_path ]] \
+    || cp services.yml "$services_path"
+  [[ -f /etc/yunohost/firewall.yml ]] \
+    || cp firewall.yml /etc/yunohost/firewall.yml
+
+  # allow users to access /media directory
+  [[ -d /etc/skel/media ]] \
+    || (mkdir -p /media && ln -s /media /etc/skel/media)
+}
+
+do_pre_regen() {
+  pending_dir=$1
+
+  cd /usr/share/yunohost/templates/yunohost
+
+  # update services.yml
+  if [[ -f $services_path ]]; then
+      tmp_services_path="${services_path}-tmp"
+      new_services_path="${services_path}-new"
+      cp "$services_path" "$tmp_services_path"
+      _update_services "$new_services_path" || {
+        mv "$tmp_services_path" "$services_path"
+        exit 1
+      }
+      if [[ -f $new_services_path ]]; then
+          # replace services.yml with new one
+          mv "$new_services_path" "$services_path"
+          mv "$tmp_services_path" "${services_path}-old"
+      else
+          rm -f "$tmp_services_path"
+      fi
+  else
+      cp services.yml /etc/yunohost/services.yml
+  fi
+
+  # add cron job for diagnosis to be ran at 7h and 19h + a random delay between
+  # 0 and 20min, meant to avoid every instances running their diagnosis at
+  # exactly the same time, which may overload the diagnosis server.
+  mkdir -p $pending_dir/etc/cron.d/
+  cat > $pending_dir/etc/cron.d/yunohost-diagnosis << EOF
+SHELL=/bin/bash
+0 7,19 * * * root : YunoHost Automatic Diagnosis; sleep \$((RANDOM\\%1200)); yunohost diagnosis run --email > /dev/null 2>/dev/null || echo "Running the automatic diagnosis failed miserably"
+EOF
+
+  # legacy stuff to avoid yunohost reporting etckeeper as manually modified
+  # (this make sure that the hash is null / file is flagged as to-delete)
+  mkdir -p $pending_dir/etc/etckeeper
+  touch $pending_dir/etc/etckeeper/etckeeper.conf
+  
+  # Skip ntp if inside a container (inspired from the conf of systemd-timesyncd)
+  mkdir -p ${pending_dir}/etc/systemd/system/ntp.service.d/
+  echo "
+[Unit]
+ConditionCapability=CAP_SYS_TIME
+ConditionVirtualization=!container
+" > ${pending_dir}/etc/systemd/system/ntp.service.d/ynh-override.conf
+  
+  # Make nftable conflict with yunohost-firewall
+  mkdir -p ${pending_dir}/etc/systemd/system/nftables.service.d/
+  cat > ${pending_dir}/etc/systemd/system/nftables.service.d/ynh-override.conf << EOF
+[Unit]
+# yunohost-firewall and nftables conflict with each other
+Conflicts=yunohost-firewall.service
+ConditionFileIsExecutable=!/etc/init.d/yunohost-firewall
+ConditionPathExists=!/etc/systemd/system/multi-user.target.wants/yunohost-firewall.service
+EOF
+}
+
+do_post_regen() {
+  regen_conf_files=$1
+
+  ######################
+  # Enfore permissions #
+  ######################
+
+  # Certs
+  # We do this with find because there could be a lot of them...
+  chown -R root:ssl-cert /etc/yunohost/certs
+  chmod 750 /etc/yunohost/certs
+  find /etc/yunohost/certs/ -type f -exec chmod 640 {} \;
+  find /etc/yunohost/certs/ -type d -exec chmod 750 {} \;
+
+  # Misc configuration / state files
+  chown root:root $(ls /etc/yunohost/{*.yml,*.yaml,*.json,mysql,psql} 2>/dev/null)
+  chmod 600 $(ls /etc/yunohost/{*.yml,*.yaml,*.json,mysql,psql} 2>/dev/null)
+
+  # Apps folder, custom hooks folder
+  [[ ! -e /etc/yunohost/hooks.d ]] || (chown root /etc/yunohost/hooks.d && chmod 700 /etc/yunohost/hooks.d)
+  [[ ! -e /etc/yunohost/apps ]] || (chown root /etc/yunohost/apps && chmod 700 /etc/yunohost/apps)
+
+  # Propagates changes in systemd service config overrides
+  [[ ! "$regen_conf_files" =~ "ntp.service.d/ynh-override.conf" ]] || { systemctl daemon-reload; systemctl restart ntp; }
+  [[ ! "$regen_conf_files" =~ "nftables.service.d/ynh-override.conf" ]] || systemctl daemon-reload
+}
+
+_update_services() {
+  python2 - << EOF
+import yaml
+
+
+with open('services.yml') as f:
+    new_services = yaml.load(f)
+
+with open('/etc/yunohost/services.yml') as f:
+    services = yaml.load(f) or {}
+
+updated = False
+
+
+for service, conf in new_services.items():
+    # remove service with empty conf
+    if conf is None:
+        if service in services:
+            print("removing '{0}' from services".format(service))
+            del services[service]
+            updated = True
+
+    # add new service
+    elif not services.get(service, None):
+        print("adding '{0}' to services".format(service))
+        services[service] = conf
+        updated = True
+
+    # update service conf
+    else:
+        conffiles = services[service].pop('conffiles', {})
+
+        # status need to be removed
+        if "status" not in conf and "status" in services[service]:
+            print("update '{0}' service status access".format(service))
+            del services[service]["status"]
+            updated = True
+
+        if services[service] != conf:
+            print("update '{0}' service".format(service))
+            services[service].update(conf)
+            updated = True
+
+        if conffiles:
+            services[service]['conffiles'] = conffiles
+
+        # Remove legacy /var/log/daemon.log and /var/log/syslog from log entries
+        # because they are too general. Instead, now the journalctl log is
+        # returned by default which is more relevant.
+        if "log" in services[service]:
+            if services[service]["log"] in ["/var/log/syslog", "/var/log/daemon.log"]:
+                del services[service]["log"]
+
+if updated:
+    with open('/etc/yunohost/services.yml-new', 'w') as f:
+        yaml.safe_dump(services, f, default_flow_style=False)
+EOF
+}
+
+FORCE=${2:-0}
+DRY_RUN=${3:-0}
+
+case "$1" in
+  pre)
+    do_pre_regen $4
+    ;;
+  post)
+    do_post_regen $4
+    ;;
+  init)
+    do_init_regen
+    ;;
+  *)
+    echo "hook called with unknown argument \`$1'" >&2
+    exit 1
+    ;;
+esac
+
+exit 0

data/hooks/conf_regen/02-ssl

@@ -0,0 +1,133 @@
+#!/bin/bash
+
+set -e
+
+ssl_dir="/usr/share/yunohost/yunohost-config/ssl/yunoCA"
+
+do_init_regen() {
+  if [[ $EUID -ne 0 ]]; then
+    echo "You must be root to run this script" 1>&2
+    exit 1
+  fi
+
+  LOGFILE="/tmp/yunohost-ssl-init"
+
+  echo "Initializing a local SSL certification authority ..."
+  echo "(logs available in $LOGFILE)"
+
+  rm -f $LOGFILE
+  touch $LOGFILE
+
+  # create certs and SSL directories
+  mkdir -p "/etc/yunohost/certs/yunohost.org"
+  mkdir -p "${ssl_dir}/"{ca,certs,crl,newcerts}
+
+  # initialize some files
+  # N.B. : the weird RANDFILE thing comes from:
+  # https://stackoverflow.com/questions/94445/using-openssl-what-does-unable-to-write-random-state-mean
+  [[ -f "${ssl_dir}/serial" ]] \
+    || RANDFILE=.rnd openssl rand -hex 19 > "${ssl_dir}/serial"
+  [[ -f "${ssl_dir}/index.txt" ]] \
+    || touch "${ssl_dir}/index.txt"
+
+  openssl_conf="/usr/share/yunohost/templates/ssl/openssl.cnf"
+  ynh_ca="/etc/yunohost/certs/yunohost.org/ca.pem"
+  ynh_crt="/etc/yunohost/certs/yunohost.org/crt.pem"
+  ynh_key="/etc/yunohost/certs/yunohost.org/key.pem"
+
+  # create default certificates
+  if [[ ! -f "$ynh_ca" ]]; then
+      echo -e "\n# Creating the CA key (?)\n" >>$LOGFILE
+
+      openssl req -x509 \
+        -new \
+        -config "$openssl_conf" \
+        -days 3650 \
+        -out "${ssl_dir}/ca/cacert.pem" \
+        -keyout "${ssl_dir}/ca/cakey.pem" \
+        -nodes -batch >>$LOGFILE 2>&1
+
+      cp "${ssl_dir}/ca/cacert.pem" "$ynh_ca"
+      ln -sf "$ynh_ca" /etc/ssl/certs/ca-yunohost_crt.pem
+      update-ca-certificates
+  fi
+
+  if [[ ! -f "$ynh_crt" ]]; then
+      echo -e "\n# Creating initial key and certificate (?)\n" >>$LOGFILE
+
+      openssl req -new \
+        -config "$openssl_conf" \
+        -days 730 \
+        -out "${ssl_dir}/certs/yunohost_csr.pem" \
+        -keyout "${ssl_dir}/certs/yunohost_key.pem" \
+        -nodes -batch >>$LOGFILE 2>&1
+
+      openssl ca \
+        -config "$openssl_conf" \
+        -days 730 \
+        -in "${ssl_dir}/certs/yunohost_csr.pem" \
+        -out "${ssl_dir}/certs/yunohost_crt.pem" \
+        -batch >>$LOGFILE 2>&1
+
+      chmod 640 "${ssl_dir}/certs/yunohost_key.pem"
+      chmod 640 "${ssl_dir}/certs/yunohost_crt.pem"
+
+      cp "${ssl_dir}/certs/yunohost_key.pem" "$ynh_key"
+      cp "${ssl_dir}/certs/yunohost_crt.pem" "$ynh_crt"
+      ln -sf "$ynh_crt" /etc/ssl/certs/yunohost_crt.pem
+      ln -sf "$ynh_key" /etc/ssl/private/yunohost_key.pem
+  fi
+
+  chown -R root:ssl-cert /etc/yunohost/certs/yunohost.org/
+  chmod o-rwx /etc/yunohost/certs/yunohost.org/
+}
+
+do_pre_regen() {
+  pending_dir=$1
+
+  cd /usr/share/yunohost/templates/ssl
+
+  install -D -m 644 openssl.cnf "${pending_dir}/${ssl_dir}/openssl.cnf"
+}
+
+do_post_regen() {
+  regen_conf_files=$1
+
+  # Ensure that index.txt exists
+  index_txt=/usr/share/yunohost/yunohost-config/ssl/yunoCA/index.txt
+  [[ -f "${index_txt}" ]] || {
+    if [[ -f "${index_txt}.saved" ]]; then
+      # use saved database from 2.2
+      cp "${index_txt}.saved" "${index_txt}"
+    elif [[ -f "${index_txt}.old" ]]; then
+      # ... or use the state-1 database
+      cp "${index_txt}.old" "${index_txt}"
+    else
+      # ... or create an empty one
+      touch "${index_txt}"
+    fi
+  }
+
+  # TODO: regenerate certificates if conf changed?
+}
+
+FORCE=${2:-0}
+DRY_RUN=${3:-0}
+
+case "$1" in
+  pre)
+    do_pre_regen $4
+    ;;
+  post)
+    do_post_regen $4
+    ;;
+  init)
+    do_init_regen
+    ;;
+  *)
+    echo "hook called with unknown argument \`$1'" >&2
+    exit 1
+    ;;
+esac
+
+exit 0

data/hooks/conf_regen/03-ssh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+set -e
+
+. /usr/share/yunohost/helpers
+
+do_pre_regen() {
+    pending_dir=$1
+
+    # If the (legacy) 'from_script' flag is here,
+    # we won't touch anything in the ssh config.
+    [[ ! -f /etc/yunohost/from_script ]] || return 0
+
+    cd /usr/share/yunohost/templates/ssh
+
+    # do not listen to IPv6 if unavailable
+    [[ -f /proc/net/if_inet6 ]] && ipv6_enabled=true || ipv6_enabled=false
+
+    ssh_keys=$(ls /etc/ssh/ssh_host_{ed25519,rsa,ecdsa}_key 2>/dev/null || true)
+
+    # Support legacy setting (this setting might be disabled by a user during a migration)
+    if [[ "$(yunohost settings get 'service.ssh.allow_deprecated_dsa_hostkey')" == "True" ]]; then
+        ssh_keys="$ssh_keys $(ls /etc/ssh/ssh_host_dsa_key 2>/dev/null || true)"
+    fi
+
+    # Support different strategy for security configurations
+    export compatibility="$(yunohost settings get 'security.ssh.compatibility')"
+
+    export ssh_keys
+    export ipv6_enabled
+    ynh_render_template "sshd_config" "${pending_dir}/etc/ssh/sshd_config"
+}
+
+do_post_regen() {
+    regen_conf_files=$1
+
+    # If the (legacy) 'from_script' flag is here,
+    # we won't touch anything in the ssh config.
+    [[ ! -f /etc/yunohost/from_script ]] || return 0
+
+    # If no file changed, there's nothing to do
+    [[ -n "$regen_conf_files" ]] || return 0
+
+    # Enforce permissions for /etc/ssh/sshd_config
+    chown root:root "/etc/ssh/sshd_config"
+    chmod 644 "/etc/ssh/sshd_config"
+
+    systemctl restart ssh
+}
+
+FORCE=${2:-0}
+DRY_RUN=${3:-0}
+
+case "$1" in
+  pre)
+    do_pre_regen $4
+    ;;
+  post)
+    do_post_regen $4
+    ;;
+  *)
+    echo "hook called with unknown argument \`$1'" >&2
+    exit 1
+    ;;
+esac
+
+exit 0

data/hooks/conf_regen/06-slapd

@@ -0,0 +1,163 @@
+#!/bin/bash
+
+set -e
+
+tmp_backup_dir_file="/tmp/slapd-backup-dir.txt"
+
+do_init_regen() {
+  if [[ $EUID -ne 0 ]]; then
+      echo "You must be root to run this script" 1>&2
+      exit 1
+  fi
+
+  do_pre_regen ""
+
+  systemctl daemon-reload
+  
+  _regenerate_slapd_conf
+
+   # Enforce permissions
+  chown root:openldap /etc/ldap/slapd.ldif
+  chown -R openldap:openldap /etc/ldap/schema/
+  usermod -aG ssl-cert openldap
+
+  service slapd restart
+}
+
+_regenerate_slapd_conf() {
+
+  # Validate the new slapd config
+  # To do so, we have to use the .ldif to generate the config directory
+  # so we use a temporary directory slapd_new.d
+  rm -Rf /etc/ldap/slapd_new.d
+  mkdir /etc/ldap/slapd_new.d
+  slapadd -n0 -l /etc/ldap/slapd.ldif -F /etc/ldap/slapd_new.d/ 2>&1
+  # Actual validation (-Q is for quiet, -u is for dry-run)
+  slaptest -Q -u -F /etc/ldap/slapd_new.d
+
+  # "Commit" / apply the new config (meaning we delete the old one and replace
+  # it with the new one)
+  rm -Rf /etc/ldap/slapd.d
+  mv /etc/ldap/slapd_new.d /etc/ldap/slapd.d
+
+  chown -R openldap:openldap /etc/ldap/slapd.d/
+}
+
+do_pre_regen() {
+  pending_dir=$1
+
+  # remove temporary backup file
+  rm -f "$tmp_backup_dir_file"
+
+  # Define if we need to migrate from hdb to mdb
+  curr_backend=$(grep '^database' /etc/ldap/slapd.conf 2>/dev/null | awk '{print $2}')
+  if [ -e /etc/ldap/slapd.conf ] && [ -n "$curr_backend" ] && \
+     [ $curr_backend != 'mdb' ]; then
+    backup_dir="/var/backups/dc=yunohost,dc=org-${curr_backend}-$(date +%s)"
+    mkdir -p "$backup_dir"
+    slapcat -b dc=yunohost,dc=org -l "${backup_dir}/dc=yunohost-dc=org.ldif"
+    echo "$backup_dir" > "$tmp_backup_dir_file"
+  fi
+
+  # create needed directories
+  ldap_dir="${pending_dir}/etc/ldap"
+  schema_dir="${ldap_dir}/schema"
+  mkdir -p "$ldap_dir" "$schema_dir"
+
+  # remove legacy configuration file
+  [ ! -f /etc/ldap/slapd-yuno.conf ] || touch "${ldap_dir}/slapd-yuno.conf"
+  [ ! -f /etc/ldap/slapd.conf ] || touch "${ldap_dir}/slapd.conf"
+  [ ! -f /etc/ldap/schema/yunohost.schema ] || touch "${schema_dir}/yunohost.schema"
+
+  cd /usr/share/yunohost/templates/slapd
+
+  # copy configuration files
+  cp -a ldap.conf slapd.ldif "$ldap_dir"
+  cp -a sudo.ldif mailserver.ldif permission.ldif "$schema_dir"
+
+  mkdir -p ${pending_dir}/etc/systemd/system/slapd.service.d/
+  cp systemd-override.conf ${pending_dir}/etc/systemd/system/slapd.service.d/ynh-override.conf
+
+  install -D -m 644 slapd.default "${pending_dir}/etc/default/slapd"
+}
+
+do_post_regen() {
+  regen_conf_files=$1
+
+  # fix some permissions
+  echo "Enforce permissions on ldap/slapd directories and certs ..."
+  # penldap user should be in the ssl-cert group to let it access the certificate for TLS
+  usermod -aG ssl-cert openldap
+  chown root:openldap /etc/ldap/slapd.ldif
+  chown -R openldap:openldap /etc/ldap/schema/
+  chown -R openldap:openldap /etc/ldap/slapd.d/
+
+  # If we changed the systemd ynh-override conf
+  if echo "$regen_conf_files" | sed 's/,/\n/g' | grep -q "^/etc/systemd/system/slapd.service.d/ynh-override.conf$"
+  then
+      systemctl daemon-reload
+      systemctl restart slapd
+  fi
+
+  [ -z "$regen_conf_files" ] && exit 0
+
+  # regenerate LDAP config directory from slapd.conf
+  echo "Regenerate LDAP config directory from slapd.ldif"
+  _regenerate_slapd_conf
+
+  # If there's a backup, re-import its data
+  backup_dir=$(cat "$tmp_backup_dir_file" 2>/dev/null || true)
+  if [[ -n "$backup_dir" && -f "${backup_dir}/dc=yunohost-dc=org.ldif" ]]; then
+      # regenerate LDAP config directory and import database as root
+    echo "Import the database using slapadd"
+    slapadd -F /etc/ldap/slapd.d -b dc=yunohost,dc=org -l "${backup_dir}/dc=yunohost-dc=org.ldif"
+    chown -R openldap:openldap /var/lib/ldap 2>&1
+  fi
+
+  echo "Running slapdindex"
+  su openldap -s "/bin/bash" -c "/usr/sbin/slapindex"
+
+  echo "Reloading slapd"
+  service slapd force-reload
+
+  # on slow hardware/vm this regen conf would exit before the admin user that
+  # is stored in ldap is available because ldap seems to slow to restart
+  # so we'll wait either until we are able to log as admin or until a timeout
+  # is reached
+  # we need to do this because the next hooks executed after this one during
+  # postinstall requires to run as admin thus breaking postinstall on slow
+  # hardware which mean yunohost can't be correctly installed on those hardware
+  # and this sucks
+  # wait a maximum time of 5 minutes
+  # yes, force-reload behave like a restart
+  number_of_wait=0
+  while ! su admin -c '' && ((number_of_wait < 60))
+  do
+      sleep 5
+      ((number_of_wait += 1))
+  done
+}
+
+FORCE=${2:-0}
+DRY_RUN=${3:-0}
+
+case "$1" in
+  pre)
+    do_pre_regen $4
+    ;;
+  post)
+    do_post_regen $4
+    ;;
+  init)
+    do_init_regen
+    ;;
+  apply_config)
+    do_post_regen /etc/ldap/slapd.ldif
+    ;;
+  *)
+    echo "hook called with unknown argument \`$1'" >&2
+    exit 1
+    ;;
+esac
+
+exit 0

data/hooks/conf_regen/09-nslcd

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+
+do_pre_regen() {
+  pending_dir=$1
+
+  cd /usr/share/yunohost/templates/nslcd
+
+  install -D -m 644 nslcd.conf "${pending_dir}/etc/nslcd.conf"
+}
+
+do_post_regen() {
+  regen_conf_files=$1
+
+  [[ -z "$regen_conf_files" ]] \
+    || service nslcd restart
+}
+
+FORCE=${2:-0}
+DRY_RUN=${3:-0}
+
+case "$1" in
+  pre)
+    do_pre_regen $4
+    ;;
+  post)
+    do_post_regen $4
+    ;;
+  *)
+    echo "hook called with unknown argument \`$1'" >&2
+    exit 1
+    ;;
+esac
+
+exit 0

data/hooks/conf_regen/10-apt

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -e
+
+do_pre_regen() {
+    pending_dir=$1
+
+    mkdir --parents "${pending_dir}/etc/apt/preferences.d"
+
+    packages_to_refuse_from_sury="php php-fpm php-mysql php-xml php-zip php-mbstring php-ldap php-gd php-curl php-bz2 php-json php-sqlite3 php-intl openssl libssl1.1 libssl-dev"
+    for package in $packages_to_refuse_from_sury
+    do
+        echo "
+Package: $package
+Pin: origin \"packages.sury.org\" 
+Pin-Priority: -1" >> "${pending_dir}/etc/apt/preferences.d/extra_php_version"
+    done
+}
+
+do_post_regen() {
+  regen_conf_files=$1
+
+  # Make sure php7.3 is the default version when using php in cli
+  update-alternatives --set php /usr/bin/php7.3
+}
+
+FORCE=${2:-0}
+DRY_RUN=${3:-0}
+
+case "$1" in
+  pre)
+    do_pre_regen $4
+    ;;
+  post)
+    do_post_regen $4
+    ;;
+  *)
+    echo "hook called with unknown argument \`$1'" >&2
+    exit 1
+    ;;
+esac
+
+exit 0

data/hooks/conf_regen/12-metronome

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+set -e
+
+do_pre_regen() {
+  pending_dir=$1
+
+  cd /usr/share/yunohost/templates/metronome
+
+  # create directories for pending conf
+  metronome_dir="${pending_dir}/etc/metronome"
+  metronome_conf_dir="${metronome_dir}/conf.d"
+  mkdir -p "$metronome_conf_dir"
+
+  # retrieve variables
+  main_domain=$(cat /etc/yunohost/current_host)
+
+  # install main conf file
+  cat metronome.cfg.lua \
+    | sed "s/{{ main_domain }}/${main_domain}/g" \
+    > "${metronome_dir}/metronome.cfg.lua"
+
+  # add domain conf files
+  for domain in $YNH_DOMAINS; do
+    cat domain.tpl.cfg.lua \
+      | sed "s/{{ domain }}/${domain}/g" \
+      > "${metronome_conf_dir}/${domain}.cfg.lua"
+  done
+
+  # remove old domain conf files
+  conf_files=$(ls -1 /etc/metronome/conf.d \
+                 | awk '/^[^\.]+\.[^\.]+.*\.cfg\.lua$/ { print $1 }')
+  for file in $conf_files; do
+    domain=${file%.cfg.lua}
+    [[ $YNH_DOMAINS =~ $domain ]] \
+      || touch "${metronome_conf_dir}/${file}"
+  done
+}
+
+do_post_regen() {
+  regen_conf_files=$1
+
+  # retrieve variables
+  main_domain=$(cat /etc/yunohost/current_host)
+  
+  # FIXME : small optimization to do to avoid calling a yunohost command ... 
+  # maybe another env variable like YNH_MAIN_DOMAINS idk
+  domain_list=$(yunohost domain list --exclude-subdomains --output-as plain --quiet)
+
+  # create metronome directories for domains
+  for domain in $domain_list; do
+    mkdir -p "/var/lib/metronome/${domain//./%2e}/pep"
+    # http_upload directory must be writable by metronome and readable by nginx
+    mkdir -p  "/var/xmpp-upload/${domain}/upload"
+    chmod g+s "/var/xmpp-upload/${domain}/upload"
+    chown -R metronome:www-data "/var/xmpp-upload/${domain}"
+  done
+
+  # fix some permissions
+
+  # metronome should be in ssl-cert group to let it access SSL certificates
+  usermod -aG ssl-cert metronome
+  chown -R metronome: /var/lib/metronome/
+  chown -R metronome: /etc/metronome/conf.d/
+
+  [[ -z "$regen_conf_files" ]] \
+    || service metronome restart
+}
+
+FORCE=${2:-0}
+DRY_RUN=${3:-0}
+
+case "$1" in
+  pre)
+    do_pre_regen $4
+    ;;
+  post)
+    do_post_regen $4
+    ;;
+  *)
+    echo "hook called with unknown argument \`$1'" >&2
+    exit 1
+    ;;
+esac
+
+exit 0

data/hooks/conf_regen/15-nginx

@@ -0,0 +1,154 @@
+#!/bin/bash
+
+set -e
+
+. /usr/share/yunohost/helpers
+
+do_init_regen() {
+  if [[ $EUID -ne 0 ]]; then
+    echo "You must be root to run this script" 1>&2
+    exit 1
+  fi
+
+  cd /usr/share/yunohost/templates/nginx
+
+  nginx_dir="/etc/nginx"
+  nginx_conf_dir="${nginx_dir}/conf.d"
+  mkdir -p "$nginx_conf_dir"
+
+  # install plain conf files
+  cp plain/* "$nginx_conf_dir"
+
+  # probably run with init: just disable default site, restart NGINX and exit
+  rm -f "${nginx_dir}/sites-enabled/default"
+
+  export compatibility="intermediate"
+  ynh_render_template "security.conf.inc" "${nginx_conf_dir}/security.conf.inc"
+  ynh_render_template "yunohost_admin.conf" "${nginx_conf_dir}/yunohost_admin.conf"
+
+  mkdir -p $nginx_conf_dir/default.d/
+  cp "redirect_to_admin.conf" $nginx_conf_dir/default.d/
+
+  # Restart nginx if conf looks good, otherwise display error and exit unhappy
+  nginx -t 2>/dev/null || { nginx -t; exit 1; }
+  systemctl restart nginx || { journalctl --no-pager --lines=10 -u nginx >&2; exit 1; }
+
+  exit 0
+}
+
+do_pre_regen() {
+  pending_dir=$1
+
+  cd /usr/share/yunohost/templates/nginx
+
+  nginx_dir="${pending_dir}/etc/nginx"
+  nginx_conf_dir="${nginx_dir}/conf.d"
+  mkdir -p "$nginx_conf_dir"
+
+  # install / update plain conf files
+  cp plain/* "$nginx_conf_dir"
+
+  # retrieve variables
+  main_domain=$(cat /etc/yunohost/current_host)
+
+  # Support different strategy for security configurations
+  export compatibility="$(yunohost settings get 'security.nginx.compatibility')"
+  ynh_render_template "security.conf.inc" "${nginx_conf_dir}/security.conf.inc"
+
+  cert_status=$(yunohost domain cert-status --json)
+
+  # add domain conf files
+  for domain in $YNH_DOMAINS; do
+    domain_conf_dir="${nginx_conf_dir}/${domain}.d"
+    mkdir -p "$domain_conf_dir"
+    mail_autoconfig_dir="${pending_dir}/var/www/.well-known/${domain}/autoconfig/mail/"
+    mkdir -p "$mail_autoconfig_dir"
+
+    # NGINX server configuration
+    export domain
+    export domain_cert_ca=$(echo $cert_status \
+                            | jq ".certificates.\"$domain\".CA_type" \
+                            | tr -d '"')
+
+    ynh_render_template "server.tpl.conf" "${nginx_conf_dir}/${domain}.conf"
+    ynh_render_template "autoconfig.tpl.xml" "${mail_autoconfig_dir}/config-v1.1.xml"
+
+    touch "${domain_conf_dir}/yunohost_local.conf"  # Clean legacy conf files
+
+  done
+
+  ynh_render_template "yunohost_admin.conf" "${nginx_conf_dir}/yunohost_admin.conf"
+  mkdir -p $nginx_conf_dir/default.d/
+  cp "redirect_to_admin.conf" $nginx_conf_dir/default.d/
+
+  # remove old domain conf files
+  conf_files=$(ls -1 /etc/nginx/conf.d \
+                 | awk '/^[^\.]+\.[^\.]+.*\.conf$/ { print $1 }')
+  for file in $conf_files; do
+    domain=${file%.conf}
+    [[ $YNH_DOMAINS =~ $domain ]] \
+      || touch "${nginx_conf_dir}/${file}"
+  done
+
+  # remove old mail-autoconfig files
+  autoconfig_files=$(ls -1 /var/www/.well-known/*/autoconfig/mail/config-v1.1.xml 2>/dev/null || true)
+  for file in $autoconfig_files; do
+    domain=$(basename $(readlink -f $(dirname $file)/../..))
+    [[ $YNH_DOMAINS =~ $domain ]] \
+      || (mkdir -p "$(dirname ${pending_dir}/${file})" && touch "${pending_dir}/${file}")
+  done
+
+  # disable default site
+  mkdir -p "${nginx_dir}/sites-enabled"
+  touch "${nginx_dir}/sites-enabled/default"
+}
+
+do_post_regen() {
+  regen_conf_files=$1
+
+  [ -z "$regen_conf_files" ] && exit 0
+
+  # create NGINX conf directories for domains
+  for domain in $YNH_DOMAINS; do
+    mkdir -p "/etc/nginx/conf.d/${domain}.d"
+  done
+
+  # Get rid of legacy lets encrypt snippets
+  for domain in $YNH_DOMAINS; do
+      # If the legacy letsencrypt / acme-challenge domain-specific snippet is still there
+      if [ -e /etc/nginx/conf.d/${domain}.d/000-acmechallenge.conf ]
+      then
+          # And if we're effectively including the new domain-independant snippet now
+          if grep -q "include /etc/nginx/conf.d/acme-challenge.conf.inc;" /etc/nginx/conf.d/${domain}.conf
+          then
+              # Delete the old domain-specific snippet
+              rm /etc/nginx/conf.d/${domain}.d/000-acmechallenge.conf
+          fi
+      fi
+  done
+
+  # Reload nginx if conf looks good, otherwise display error and exit unhappy
+  nginx -t 2>/dev/null || { nginx -t; exit 1; }
+  pgrep nginx && systemctl reload nginx || { journalctl --no-pager --lines=10 -u nginx >&2; exit 1; }
+}
+
+FORCE=${2:-0}
+DRY_RUN=${3:-0}
+
+case "$1" in
+  pre)
+    do_pre_regen $4
+    ;;
+  post)
+    do_post_regen $4
+    ;;
+  init)
+    do_init_regen
+    ;;
+  *)
+    echo "hook called with unknown argument \`$1'" >&2
+    exit 1
+    ;;
+esac
+
+exit 0

data/hooks/conf_regen/19-postfix

@@ -0,0 +1,99 @@
+#!/bin/bash
+
+set -e
+
+. /usr/share/yunohost/helpers
+
+do_pre_regen() {
+  pending_dir=$1
+
+  cd /usr/share/yunohost/templates/postfix
+
+  postfix_dir="${pending_dir}/etc/postfix"
+  mkdir -p "$postfix_dir"
+
+  default_dir="${pending_dir}/etc/default/"
+  mkdir -p "$default_dir"
+
+  # install plain conf files
+  cp plain/* "$postfix_dir"
+
+  # prepare main.cf conf file
+  main_domain=$(cat /etc/yunohost/current_host)
+
+  # Support different strategy for security configurations
+  export compatibility="$(yunohost settings get 'security.postfix.compatibility')"
+  
+  # Add possibility to specify a relay
+  # Could be useful with some isp with no 25 port open or more complex setup
+  export relay_port=""
+  export relay_user=""
+  export relay_host="$(yunohost settings get 'smtp.relay.host')"
+  if [ -n "${relay_host}" ]
+  then
+    relay_port="$(yunohost settings get 'smtp.relay.port')"
+    relay_user="$(yunohost settings get 'smtp.relay.user')"
+    relay_password="$(yunohost settings get 'smtp.relay.password')"
+    
+    # Avoid to display "Relay account paswword" to other users
+    touch ${postfix_dir}/sasl_passwd
+    chmod 750 ${postfix_dir}/sasl_passwd
+    # Avoid "postmap: warning: removing zero-length database file"
+    chown postfix ${pending_dir}/etc/postfix
+    chown postfix ${pending_dir}/etc/postfix/sasl_passwd
+
+    cat <<< "[${relay_host}]:${relay_port} ${relay_user}:${relay_password}" > ${postfix_dir}/sasl_passwd
+    postmap ${postfix_dir}/sasl_passwd
+  fi
+  export main_domain
+  export domain_list="$YNH_DOMAINS"
+  ynh_render_template "main.cf" "${postfix_dir}/main.cf"
+
+  cat postsrsd \
+    | sed "s/{{ main_domain }}/${main_domain}/g" \
+    | sed "s/{{ domain_list }}/${YNH_DOMAINS}/g" \
+    > "${default_dir}/postsrsd"
+
+  # adapt it for IPv4-only hosts
+  ipv6="$(yunohost settings get 'smtp.allow_ipv6')"
+  if [ "$ipv6" == "False" ] || [ ! -f /proc/net/if_inet6 ]; then
+    sed -i \
+      's/ \[::ffff:127.0.0.0\]\/104 \[::1\]\/128//g' \
+      "${postfix_dir}/main.cf"
+    sed -i \
+      's/inet_interfaces = all/&\ninet_protocols = ipv4/' \
+      "${postfix_dir}/main.cf"
+  fi
+}
+
+do_post_regen() {
+  regen_conf_files=$1
+
+  if [ -e /etc/postfix/sasl_passwd ]
+  then
+      chmod 750 /etc/postfix/sasl_passwd*
+      chown postfix:root /etc/postfix/sasl_passwd*
+  fi
+
+  [[ -z "$regen_conf_files" ]] \
+    || { service postfix restart && service postsrsd restart; }
+
+}
+
+FORCE=${2:-0}
+DRY_RUN=${3:-0}
+
+case "$1" in
+  pre)
+    do_pre_regen $4
+    ;;
+  post)
+    do_post_regen $4
+    ;;
+  *)
+    echo "hook called with unknown argument \`$1'" >&2
+    exit 1
+    ;;
+esac
+
+exit 0

data/hooks/conf_regen/25-dovecot

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+set -e
+
+. /usr/share/yunohost/helpers
+
+do_pre_regen() {
+  pending_dir=$1
+
+  cd /usr/share/yunohost/templates/dovecot
+
+  dovecot_dir="${pending_dir}/etc/dovecot"
+  mkdir -p "${dovecot_dir}/global_script"
+
+  # copy simple conf files
+  cp dovecot-ldap.conf "${dovecot_dir}/dovecot-ldap.conf"
+  cp dovecot.sieve "${dovecot_dir}/global_script/dovecot.sieve"
+
+  export pop3_enabled="$(yunohost settings get 'pop3.enabled')"
+  export main_domain=$(cat /etc/yunohost/current_host)
+
+  ynh_render_template "dovecot.conf" "${dovecot_dir}/dovecot.conf"
+
+  # adapt it for IPv4-only hosts
+  if [ ! -f /proc/net/if_inet6 ]; then
+    sed -i \
+      's/^\(listen =\).*/\1 */' \
+      "${dovecot_dir}/dovecot.conf"
+  fi
+
+  mkdir -p "${dovecot_dir}/yunohost.d"
+  cp pre-ext.conf "${dovecot_dir}/yunohost.d"
+  cp post-ext.conf "${dovecot_dir}/yunohost.d"
+}
+
+do_post_regen() {
+  regen_conf_files=$1
+
+  mkdir -p "/etc/dovecot/yunohost.d/pre-ext.d"
+  mkdir -p "/etc/dovecot/yunohost.d/post-ext.d"
+
+  # create vmail user
+  id vmail > /dev/null 2>&1 \
+    || adduser --system --ingroup mail --uid 500 vmail
+
+  # fix permissions
+  chown -R vmail:mail /etc/dovecot/global_script
+  chmod 770 /etc/dovecot/global_script
+  chown root:mail /var/mail
+  chmod 1775 /var/mail
+
+  [ -z "$regen_conf_files" ] && exit 0
+
+  # compile sieve script
+  [[ "$regen_conf_files" =~ dovecot\.sieve ]] && {
+    sievec /etc/dovecot/global_script/dovecot.sieve
+    chown -R vmail:mail /etc/dovecot/global_script
+  }
+
+  service dovecot restart
+}
+
+FORCE=${2:-0}
+DRY_RUN=${3:-0}
+
+case "$1" in
+  pre)
+    do_pre_regen $4
+    ;;
+  post)
+    do_post_regen $4
+    ;;
+  *)
+    echo "hook called with unknown argument \`$1'" >&2
+    exit 1
+    ;;
+esac
+
+exit 0

data/hooks/conf_regen/31-rspamd

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+set -e
+
+do_pre_regen() {
+  pending_dir=$1
+
+  cd /usr/share/yunohost/templates/rspamd
+
+  install -D -m 644 metrics.local.conf \
+      "${pending_dir}/etc/rspamd/local.d/metrics.conf"
+  install -D -m 644 dkim_signing.conf \
+      "${pending_dir}/etc/rspamd/local.d/dkim_signing.conf"
+  install -D -m 644 rspamd.sieve \
+      "${pending_dir}/etc/dovecot/global_script/rspamd.sieve"
+}
+
+do_post_regen() {
+
+  ##
+  ## DKIM key generation
+  ##
+
+  # create DKIM directory with proper permission
+  mkdir -p /etc/dkim
+  chown _rspamd /etc/dkim
+
+  # create DKIM key for domains
+  for domain in $YNH_DOMAINS; do
+    domain_key="/etc/dkim/${domain}.mail.key"
+    [ ! -f "$domain_key" ] && {
+      # We use a 1024 bit size because nsupdate doesn't seem to be able to
+      # handle 2048...
+      opendkim-genkey --domain="$domain" \
+          --selector=mail --directory=/etc/dkim -b 1024
+      mv /etc/dkim/mail.private "$domain_key"
+      mv /etc/dkim/mail.txt "/etc/dkim/${domain}.mail.txt"
+    }
+  done
+
+  # fix DKIM keys permissions
+  chown _rspamd /etc/dkim/*.mail.key
+  chmod 400 /etc/dkim/*.mail.key
+
+  [ ! -e /var/log/rspamd ] || chown -R _rspamd:_rspamd /var/log/rspamd
+
+  regen_conf_files=$1
+  [ -z "$regen_conf_files" ] && exit 0
+
+  # compile sieve script
+  [[ "$regen_conf_files" =~ rspamd\.sieve ]] && {
+    sievec /etc/dovecot/global_script/rspamd.sieve
+    chown -R vmail:mail /etc/dovecot/global_script
+    systemctl restart dovecot
+  }
+
+  # Restart rspamd due to the upgrade
+  # https://rspamd.com/announce/2016/08/01/rspamd-1.3.1.html 
+  systemctl -q restart rspamd.service 
+}
+
+FORCE=${2:-0}
+DRY_RUN=${3:-0}
+
+case "$1" in
+  pre)
+    do_pre_regen $4
+    ;;
+  post)
+    do_post_regen $4
+    ;;
+  *)
+    echo "hook called with unknown argument \`$1'" >&2
+    exit 1
+    ;;
+esac
+
+exit 0

data/hooks/conf_regen/34-mysql

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+set -e
+MYSQL_PKG="$(dpkg --list | sed -ne 's/^ii  \(mariadb-server-[[:digit:].]\+\) .*$/\1/p')"
+. /usr/share/yunohost/helpers
+
+do_pre_regen() {
+  pending_dir=$1
+
+  cd /usr/share/yunohost/templates/mysql
+
+  install -D -m 644 my.cnf "${pending_dir}/etc/mysql/my.cnf"
+}
+
+do_post_regen() {
+  regen_conf_files=$1
+
+  if [[ ! -d /var/lib/mysql/mysql ]]
+  then
+      # dpkg-reconfigure will initialize mysql (if it ain't already)
+      # It enabled auth_socket for root, so no need to define any root password...
+      # c.f. : cat /var/lib/dpkg/info/mariadb-server-10.3.postinst | grep install_db -C3
+      dpkg-reconfigure -freadline -u "$MYSQL_PKG" 2>&1
+
+      systemctl -q is-active mariadb.service \
+        || systemctl start mariadb
+
+      sleep 5
+
+      echo "" | mysql && echo "Can't connect to mysql using unix_socket auth ... something went wrong during initial configuration of mysql !?"
+  fi
+
+  if [ ! -e /etc/yunohost/mysql ]
+  then
+      # Dummy password that's not actually used nor meaningful ...
+      # (because mysql is supposed to be configured to use unix_socket on new setups)
+      # but keeping it for legacy
+      # until we merge https://github.com/YunoHost/yunohost/pull/912 ...
+      ynh_string_random 10 > /etc/yunohost/mysql
+      chmod 400 /etc/yunohost/mysql
+  fi
+
+  # mysql is supposed to be an alias to mariadb... but in some weird case is not
+  # c.f. https://forum.yunohost.org/t/mysql-ne-fonctionne-pas/11661
+  # Playing with enable/disable allows to recreate the proper symlinks.
+  if [ ! -e /etc/systemd/system/mysql.service ]
+  then
+      systemctl stop mysql -q
+      systemctl disable mysql -q
+      systemctl disable mariadb -q
+      systemctl enable mariadb -q
+      systemctl is-active mariadb -q || systemctl start mariadb
+  fi
+
+  [[ -z "$regen_conf_files" ]] \
+    || service mysql restart
+}
+
+FORCE=${2:-0}
+DRY_RUN=${3:-0}
+
+case "$1" in
+  pre)
+    do_pre_regen $4
+    ;;
+  post)
+    do_post_regen $4
+    ;;
+  *)
+    echo "hook called with unknown argument \`$1'" >&2
+    exit 1
+    ;;
+esac
+
+exit 0

data/hooks/conf_regen/35-redis

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+do_pre_regen() {
+  :
+}
+
+do_post_regen() {
+  # Enforce these damn permissions because for some reason in some weird cases
+  # they are spontaneously replaced by root:root -_-
+  chown -R redis:adm /var/log/redis
+}
+
+FORCE=${2:-0}
+DRY_RUN=${3:-0}
+
+case "$1" in
+  pre)
+    do_pre_regen $4
+    ;;
+  post)
+    do_post_regen $4
+    ;;
+  *)
+    echo "hook called with unknown argument \`$1'" >&2
+    exit 1
+    ;;
+esac
+
+exit 0

data/hooks/conf_regen/37-avahi-daemon

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+set -e
+
+do_pre_regen() {
+  pending_dir=$1
+
+  cd /usr/share/yunohost/templates/avahi-daemon
+
+  install -D -m 644 avahi-daemon.conf \
+    "${pending_dir}/etc/avahi/avahi-daemon.conf"
+}
+
+do_post_regen() {
+  regen_conf_files=$1
+
+  [[ -z "$regen_conf_files" ]] \
+    || service avahi-daemon restart
+}
+
+FORCE=${2:-0}
+DRY_RUN=${3:-0}
+
+case "$1" in
+  pre)
+    do_pre_regen $4
+    ;;
+  post)
+    do_post_regen $4
+    ;;
+  *)
+    echo "hook called with unknown argument \`$1'" >&2
+    exit 1
+    ;;
+esac
+
+exit 0

data/hooks/conf_regen/43-dnsmasq

@@ -0,0 +1,99 @@
+#!/bin/bash
+
+set -e
+. /usr/share/yunohost/helpers
+
+do_pre_regen() {
+  pending_dir=$1
+
+  cd /usr/share/yunohost/templates/dnsmasq
+
+  # create directory for pending conf
+  dnsmasq_dir="${pending_dir}/etc/dnsmasq.d"
+  mkdir -p "$dnsmasq_dir"
+  etcdefault_dir="${pending_dir}/etc/default"
+  mkdir -p "$etcdefault_dir"
+
+  # add general conf files
+  cp plain/etcdefault ${pending_dir}/etc/default/dnsmasq
+  cp plain/dnsmasq.conf ${pending_dir}/etc/dnsmasq.conf
+
+  # add resolver file
+  cat plain/resolv.dnsmasq.conf | grep "^nameserver" | shuf > ${pending_dir}/etc/resolv.dnsmasq.conf
+
+  # retrieve variables
+  ipv4=$(curl -s -4 https://ip.yunohost.org 2>/dev/null || true)
+  ynh_validate_ip4 "$ipv4" || ipv4='127.0.0.1'
+  ipv6=$(curl -s -6 https://ip6.yunohost.org 2>/dev/null || true)
+  ynh_validate_ip6 "$ipv6" || ipv6=''
+
+  export ipv4
+  export ipv6
+
+  # add domain conf files
+  for domain in $YNH_DOMAINS; do
+      export domain
+      ynh_render_template "domain.tpl" "${dnsmasq_dir}/${domain}"
+  done
+
+  # remove old domain conf files
+  conf_files=$(ls -1 /etc/dnsmasq.d \
+                 | awk '/^[^\.]+\.[^\.]+.*$/ { print $1 }')
+  for domain in $conf_files; do
+    [[ $YNH_DOMAINS =~ $domain ]] \
+      || touch "${dnsmasq_dir}/${domain}"
+  done
+}
+
+do_post_regen() {
+  regen_conf_files=$1
+
+  # Fuck it, those domain/search entries from dhclient are usually annoying
+  # lying shit from the ISP trying to MiTM
+  if grep -q -E "^ *(domain|search)" /run/resolvconf/resolv.conf
+  then
+      if grep -q -E "^ *(domain|search)" /run/resolvconf/interface/*.dhclient 2>/dev/null
+      then
+        sed -E "s/^(domain|search)/#\1/g" -i /run/resolvconf/interface/*.dhclient
+      fi
+
+      grep -q '^supersede domain-name "";' /etc/dhcp/dhclient.conf 2>/dev/null || echo 'supersede domain-name "";' >> /etc/dhcp/dhclient.conf
+      grep -q '^supersede domain-search "";' /etc/dhcp/dhclient.conf 2>/dev/null || echo 'supersede domain-search "";' >> /etc/dhcp/dhclient.conf
+      grep -q '^supersede name "";' /etc/dhcp/dhclient.conf 2>/dev/null || echo 'supersede name "";' >> /etc/dhcp/dhclient.conf
+      systemctl restart resolvconf
+  fi
+
+  # Some stupid things like rabbitmq-server used by onlyoffice won't work if
+  # the *short* hostname doesn't exists in /etc/hosts -_-
+  short_hostname=$(hostname -s)
+  grep -q "127.0.0.1.*$short_hostname" /etc/hosts || echo -e "\n127.0.0.1\t$short_hostname" >>/etc/hosts
+
+  [[ -n "$regen_conf_files" ]] || return
+
+  # Remove / disable services likely to conflict with dnsmasq
+  for SERVICE in systemd-resolved bind9
+  do
+      systemctl is-enabled $SERVICE &>/dev/null && systemctl disable $SERVICE 2>/dev/null
+      systemctl is-active $SERVICE &>/dev/null && systemctl stop $SERVICE
+  done
+
+  systemctl restart dnsmasq
+}
+
+FORCE=${2:-0}
+DRY_RUN=${3:-0}
+
+case "$1" in
+  pre)
+    do_pre_regen $4
+    ;;
+  post)
+    do_post_regen $4
+    ;;
+  *)
+    echo "hook called with unknown argument \`$1'" >&2
+    exit 1
+    ;;
+esac
+
+exit 0

data/hooks/conf_regen/46-nsswitch

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+
+do_pre_regen() {
+  pending_dir=$1
+
+  cd /usr/share/yunohost/templates/nsswitch
+
+  install -D -m 644 nsswitch.conf "${pending_dir}/etc/nsswitch.conf"
+}
+
+do_post_regen() {
+  regen_conf_files=$1
+
+  [[ -z "$regen_conf_files" ]] \
+    || service unscd restart
+}
+
+FORCE=${2:-0}
+DRY_RUN=${3:-0}
+
+case "$1" in
+  pre)
+    do_pre_regen $4
+    ;;
+  post)
+    do_post_regen $4
+    ;;
+  *)
+    echo "hook called with unknown argument \`$1'" >&2
+    exit 1
+    ;;
+esac
+
+exit 0

data/hooks/conf_regen/52-fail2ban

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -e
+
+do_pre_regen() {
+  pending_dir=$1
+
+  cd /usr/share/yunohost/templates/fail2ban
+
+  fail2ban_dir="${pending_dir}/etc/fail2ban"
+  mkdir -p "${fail2ban_dir}/filter.d"
+  mkdir -p "${fail2ban_dir}/jail.d"
+
+  cp yunohost.conf "${fail2ban_dir}/filter.d/yunohost.conf"
+  cp jail.conf "${fail2ban_dir}/jail.conf"
+  cp yunohost-jails.conf "${fail2ban_dir}/jail.d/"
+}
+
+do_post_regen() {
+  regen_conf_files=$1
+
+  [[ -z "$regen_conf_files" ]] \
+    || service fail2ban restart
+}
+
+FORCE=${2:-0}
+DRY_RUN=${3:-0}
+
+case "$1" in
+  pre)
+    do_pre_regen $4
+    ;;
+  post)
+    do_post_regen $4
+    ;;
+  *)
+    echo "hook called with unknown argument \`$1'" >&2
+    exit 1
+    ;;
+esac
+
+exit 0

data/hooks/diagnosis/00-basesystem.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+
+import os
+import json
+import subprocess
+
+from moulinette.utils.process import check_output
+from moulinette.utils.filesystem import read_file, read_json, write_to_json
+from yunohost.diagnosis import Diagnoser
+from yunohost.utils.packages import ynh_packages_version
+
+
+class BaseSystemDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 600
+    dependencies = []
+
+    def run(self):
+
+        # Detect virt technology (if not bare metal) and arch
+        # Gotta have this "|| true" because it systemd-detect-virt return 'none'
+        # with an error code on bare metal ~.~
+        virt = check_output("systemd-detect-virt || true", shell=True)
+        if virt.lower() == "none":
+            virt = "bare-metal"
+
+        # Detect arch
+        arch = check_output("dpkg --print-architecture")
+        hardware = dict(meta={"test": "hardware"},
+                        status="INFO",
+                        data={"virt": virt, "arch": arch},
+                        summary="diagnosis_basesystem_hardware")
+
+        # Also possibly the board / hardware name
+        if os.path.exists("/proc/device-tree/model"):
+            model = read_file('/proc/device-tree/model').strip().replace('\x00', '')
+            hardware["data"]["model"] = model
+            hardware["details"] = ["diagnosis_basesystem_hardware_model"]
+        elif os.path.exists("/sys/devices/virtual/dmi/id/sys_vendor"):
+            model = read_file("/sys/devices/virtual/dmi/id/sys_vendor").strip()
+            if os.path.exists("/sys/devices/virtual/dmi/id/product_name"):
+                model = "%s %s" % (model, read_file("/sys/devices/virtual/dmi/id/product_name").strip())
+            hardware["data"]["model"] = model
+            hardware["details"] = ["diagnosis_basesystem_hardware_model"]
+
+        yield hardware
+
+        # Kernel version
+        kernel_version = read_file('/proc/sys/kernel/osrelease').strip()
+        yield dict(meta={"test": "kernel"},
+                   data={"kernel_version": kernel_version},
+                   status="INFO",
+                   summary="diagnosis_basesystem_kernel")
+
+        # Debian release
+        debian_version = read_file("/etc/debian_version").strip()
+        yield dict(meta={"test": "host"},
+                   data={"debian_version": debian_version},
+                   status="INFO",
+                   summary="diagnosis_basesystem_host")
+
+        # Yunohost packages versions
+        # We check if versions are consistent (e.g. all 3.6 and not 3 packages with 3.6 and the other with 3.5)
+        # This is a classical issue for upgrades that failed in the middle
+        # (or people upgrading half of the package because they did 'apt upgrade' instead of 'dist-upgrade')
+        # Here, ynh_core_version is for example "3.5.4.12", so [:3] is "3.5" and we check it's the same for all packages
+        ynh_packages = ynh_packages_version()
+        ynh_core_version = ynh_packages["yunohost"]["version"]
+        consistent_versions = all(infos["version"][:3] == ynh_core_version[:3] for infos in ynh_packages.values())
+        ynh_version_details = [("diagnosis_basesystem_ynh_single_version",
+                                {"package": package,
+                                 "version": infos["version"],
+                                 "repo": infos["repo"]}
+                                )
+                               for package, infos in ynh_packages.items()]
+
+        yield dict(meta={"test": "ynh_versions"},
+                   data={"main_version": ynh_core_version, "repo": ynh_packages["yunohost"]["repo"]},
+                   status="INFO" if consistent_versions else "ERROR",
+                   summary="diagnosis_basesystem_ynh_main_version" if consistent_versions else "diagnosis_basesystem_ynh_inconsistent_versions",
+                   details=ynh_version_details)
+
+        if self.is_vulnerable_to_meltdown():
+            yield dict(meta={"test": "meltdown"},
+                       status="ERROR",
+                       summary="diagnosis_security_vulnerable_to_meltdown",
+                       details=["diagnosis_security_vulnerable_to_meltdown_details"]
+                       )
+
+        bad_sury_packages = list(self.bad_sury_packages())
+        if bad_sury_packages:
+            cmd_to_fix = "apt install --allow-downgrades " \
+                         + " ".join(["%s=%s" % (package, version) for package, version in bad_sury_packages])
+            yield dict(meta={"test": "packages_from_sury"},
+                       data={"cmd_to_fix": cmd_to_fix},
+                       status="WARNING",
+                       summary="diagnosis_package_installed_from_sury",
+                       details=["diagnosis_package_installed_from_sury_details"])
+
+        if self.backports_in_sources_list():
+            yield dict(meta={"test": "backports_in_sources_list"},
+                       status="WARNING",
+                       summary="diagnosis_backports_in_sources_list")
+
+    def bad_sury_packages(self):
+
+        packages_to_check = ["openssl", "libssl1.1", "libssl-dev"]
+        for package in packages_to_check:
+            cmd = "dpkg --list | grep '^ii' | grep gbp | grep -q -w %s" % package
+            # If version currently installed is not from sury, nothing to report
+            if os.system(cmd) != 0:
+                continue
+
+            cmd = "LC_ALL=C apt policy %s 2>&1 | grep http -B1 | tr -d '*' | grep '+deb' | grep -v 'gbp' | head -n 1 | awk '{print $1}'" % package
+            version_to_downgrade_to = check_output(cmd)
+            yield (package, version_to_downgrade_to)
+
+    def backports_in_sources_list(self):
+
+        cmd = "grep -q -nr '^ *deb .*-backports' /etc/apt/sources.list*"
+        return os.system(cmd) == 0
+
+    def is_vulnerable_to_meltdown(self):
+        # meltdown CVE: https://security-tracker.debian.org/tracker/CVE-2017-5754
+
+        # We use a cache file to avoid re-running the script so many times,
+        # which can be expensive (up to around 5 seconds on ARM)
+        # and make the admin appear to be slow (c.f. the calls to diagnosis
+        # from the webadmin)
+        #
+        # The cache is in /tmp and shall disappear upon reboot
+        # *or* we compare it to dpkg.log modification time
+        # such that it's re-ran if there was package upgrades
+        # (e.g. from yunohost)
+        cache_file = "/tmp/yunohost-meltdown-diagnosis"
+        dpkg_log = "/var/log/dpkg.log"
+        if os.path.exists(cache_file):
+            if not os.path.exists(dpkg_log) or os.path.getmtime(cache_file) > os.path.getmtime(dpkg_log):
+                self.logger_debug("Using cached results for meltdown checker, from %s" % cache_file)
+                return read_json(cache_file)[0]["VULNERABLE"]
+
+        # script taken from https://github.com/speed47/spectre-meltdown-checker
+        # script commit id is store directly in the script
+        SCRIPT_PATH = "/usr/lib/moulinette/yunohost/vendor/spectre-meltdown-checker/spectre-meltdown-checker.sh"
+
+        # '--variant 3' corresponds to Meltdown
+        # example output from the script:
+        # [{"NAME":"MELTDOWN","CVE":"CVE-2017-5754","VULNERABLE":false,"INFOS":"PTI mitigates the vulnerability"}]
+        try:
+            self.logger_debug("Running meltdown vulnerability checker")
+            call = subprocess.Popen("bash %s --batch json --variant 3" %
+                                    SCRIPT_PATH, shell=True,
+                                    stdout=subprocess.PIPE,
+                                    stderr=subprocess.PIPE)
+
+            # TODO / FIXME : here we are ignoring error messages ...
+            # in particular on RPi2 and other hardware, the script complains about
+            # "missing some kernel info (see -v), accuracy might be reduced"
+            # Dunno what to do about that but we probably don't want to harass
+            # users with this warning ...
+            output, err = call.communicate()
+            assert call.returncode in (0, 2, 3), "Return code: %s" % call.returncode
+
+            # If there are multiple lines, sounds like there was some messages
+            # in stdout that are not json >.> ... Try to get the actual json
+            # stuff which should be the last line
+            output = output.strip()
+            if "\n" in output:
+                self.logger_debug("Original meltdown checker output : %s" % output)
+                output = output.split("\n")[-1]
+
+            CVEs = json.loads(output)
+            assert len(CVEs) == 1
+            assert CVEs[0]["NAME"] == "MELTDOWN"
+        except Exception as e:
+            import traceback
+            traceback.print_exc()
+            self.logger_warning("Something wrong happened when trying to diagnose Meltdown vunerability, exception: %s" % e)
+            raise Exception("Command output for failed meltdown check: '%s'" % output)
+
+        self.logger_debug("Writing results from meltdown checker to cache file, %s" % cache_file)
+        write_to_json(cache_file, CVEs)
+        return CVEs[0]["VULNERABLE"]
+
+
+def main(args, env, loggers):
+    return BaseSystemDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/10-ip.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python
+
+import re
+import os
+import random
+
+from moulinette.utils.network import download_text
+from moulinette.utils.process import check_output
+from moulinette.utils.filesystem import read_file
+
+from yunohost.diagnosis import Diagnoser
+from yunohost.utils.network import get_network_interfaces
+
+
+class IPDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 600
+    dependencies = []
+
+    def run(self):
+
+        # ############################################################ #
+        # PING : Check that we can ping outside at least in ipv4 or v6 #
+        # ############################################################ #
+
+        can_ping_ipv4 = self.can_ping_outside(4)
+        can_ping_ipv6 = self.can_ping_outside(6)
+
+        if not can_ping_ipv4 and not can_ping_ipv6:
+            yield dict(meta={"test": "ping"},
+                       status="ERROR",
+                       summary="diagnosis_ip_not_connected_at_all")
+            # Not much else we can do if there's no internet at all
+            return
+
+        # ###################################################### #
+        # DNS RESOLUTION : Check that we can resolve domain name #
+        # (later needed to talk to ip. and ip6.yunohost.org)     #
+        # ###################################################### #
+
+        can_resolve_dns = self.can_resolve_dns()
+
+        # In every case, we can check that resolvconf seems to be okay
+        # (symlink managed by resolvconf service + pointing to dnsmasq)
+        good_resolvconf = self.good_resolvconf()
+
+        # If we can't resolve domain names at all, that's a pretty big issue ...
+        # If it turns out that at the same time, resolvconf is bad, that's probably
+        # the cause of this, so we use a different message in that case
+        if not can_resolve_dns:
+            yield dict(meta={"test": "dnsresolv"},
+                       status="ERROR",
+                       summary="diagnosis_ip_broken_dnsresolution" if good_resolvconf else "diagnosis_ip_broken_resolvconf")
+            return
+        # Otherwise, if the resolv conf is bad but we were able to resolve domain name,
+        # still warn that we're using a weird resolv conf ...
+        elif not good_resolvconf:
+            yield dict(meta={"test": "dnsresolv"},
+                       status="WARNING",
+                       summary="diagnosis_ip_weird_resolvconf",
+                       details=["diagnosis_ip_weird_resolvconf_details"])
+        else:
+            yield dict(meta={"test": "dnsresolv"},
+                       status="SUCCESS",
+                       summary="diagnosis_ip_dnsresolution_working")
+
+        # ##################################################### #
+        # IP DIAGNOSIS : Check that we're actually able to talk #
+        # to a web server to fetch current IPv4 and v6          #
+        # ##################################################### #
+
+        ipv4 = self.get_public_ip(4) if can_ping_ipv4 else None
+        ipv6 = self.get_public_ip(6) if can_ping_ipv6 else None
+
+        network_interfaces = get_network_interfaces()
+
+        def get_local_ip(version):
+            local_ip = {iface: addr[version].split("/")[0]
+                        for iface, addr in network_interfaces.items() if version in addr}
+            if not local_ip:
+                return None
+            elif len(local_ip):
+                return next(iter(local_ip.values()))
+            else:
+                return local_ip
+
+        yield dict(meta={"test": "ipv4"},
+                   data={"global": ipv4, "local": get_local_ip("ipv4")},
+                   status="SUCCESS" if ipv4 else "ERROR",
+                   summary="diagnosis_ip_connected_ipv4" if ipv4 else "diagnosis_ip_no_ipv4",
+                   details=["diagnosis_ip_global", "diagnosis_ip_local"] if ipv4 else None)
+
+        yield dict(meta={"test": "ipv6"},
+                   data={"global": ipv6, "local": get_local_ip("ipv6")},
+                   status="SUCCESS" if ipv6 else "WARNING",
+                   summary="diagnosis_ip_connected_ipv6" if ipv6 else "diagnosis_ip_no_ipv6",
+                   details=["diagnosis_ip_global", "diagnosis_ip_local"] if ipv6 else ["diagnosis_ip_no_ipv6_tip"])
+
+        # TODO / FIXME : add some attempt to detect ISP (using whois ?) ?
+
+    def can_ping_outside(self, protocol=4):
+
+        assert protocol in [4, 6], "Invalid protocol version, it should be either 4 or 6 and was '%s'" % repr(protocol)
+
+        # We can know that ipv6 is not available directly if this file does not exists
+        if protocol == 6 and not os.path.exists("/proc/net/if_inet6"):
+            return False
+
+        # If we are indeed connected in ipv4 or ipv6, we should find a default route
+        routes = check_output("ip -%s route show table all" % protocol).split("\n")
+
+        def is_default_route(r):
+            # Typically the default route starts with "default"
+            # But of course IPv6 is more complex ... e.g. on internet cube there's
+            # no default route but a /3 which acts as a default-like route...
+            # e.g. 2000:/3 dev tun0 ...
+            return r.startswith("default") or (":" in r and re.match(r".*/[0-3]$", r.split()[0]))
+        if not any(is_default_route(r) for r in routes):
+            self.logger_debug("No default route for IPv%s, so assuming there's no IP address for that version" % protocol)
+            return None
+
+        # We use the resolver file as a list of well-known, trustable (ie not google ;)) IPs that we can ping
+        resolver_file = "/usr/share/yunohost/templates/dnsmasq/plain/resolv.dnsmasq.conf"
+        resolvers = [r.split(" ")[1] for r in read_file(resolver_file).split("\n") if r.startswith("nameserver")]
+
+        if protocol == 4:
+            resolvers = [r for r in resolvers if ":" not in r]
+        if protocol == 6:
+            resolvers = [r for r in resolvers if ":" in r]
+
+        assert resolvers != [], "Uhoh, need at least one IPv%s DNS resolver in %s ..." % (protocol, resolver_file)
+
+        # So let's try to ping the first 4~5 resolvers (shuffled)
+        # If we succesfully ping any of them, we conclude that we are indeed connected
+        def ping(protocol, target):
+            return os.system("ping%s -c1 -W 3 %s >/dev/null 2>/dev/null" % ("" if protocol == 4 else "6", target)) == 0
+
+        random.shuffle(resolvers)
+        return any(ping(protocol, resolver) for resolver in resolvers[:5])
+
+    def can_resolve_dns(self):
+        return os.system("dig +short ip.yunohost.org >/dev/null 2>/dev/null") == 0
+
+    def good_resolvconf(self):
+        content = read_file("/etc/resolv.conf").strip().split("\n")
+        # Ignore comments and empty lines
+        content = [l.strip() for l in content if l.strip() and not l.strip().startswith("#") and not l.strip().startswith("search")]
+        # We should only find a "nameserver 127.0.0.1"
+        return len(content) == 1 and content[0].split() == ["nameserver", "127.0.0.1"]
+
+    def get_public_ip(self, protocol=4):
+
+        # FIXME - TODO : here we assume that DNS resolution for ip.yunohost.org is working
+        # but if we want to be able to diagnose DNS resolution issues independently from
+        # internet connectivity, we gotta rely on fixed IPs first....
+
+        assert protocol in [4, 6], "Invalid protocol version, it should be either 4 or 6 and was '%s'" % repr(protocol)
+
+        url = 'https://ip%s.yunohost.org' % ('6' if protocol == 6 else '')
+
+        try:
+            return download_text(url, timeout=30).strip()
+        except Exception as e:
+            self.logger_debug("Could not get public IPv%s : %s" % (str(protocol), str(e)))
+            return None
+
+
+def main(args, env, loggers):
+    return IPDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/12-dnsrecords.py

@@ -0,0 +1,239 @@
+#!/usr/bin/env python
+
+import os
+import re
+
+from datetime import datetime, timedelta
+from publicsuffix import PublicSuffixList
+
+from moulinette.utils.process import check_output
+
+from yunohost.utils.network import dig
+from yunohost.diagnosis import Diagnoser
+from yunohost.domain import domain_list, _build_dns_conf, _get_maindomain
+
+YNH_DYNDNS_DOMAINS = ['nohost.me', 'noho.st', 'ynh.fr']
+
+
+class DNSRecordsDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 600
+    dependencies = ["ip"]
+
+    def run(self):
+
+        main_domain = _get_maindomain()
+
+        all_domains = domain_list()["domains"]
+        for domain in all_domains:
+            self.logger_debug("Diagnosing DNS conf for %s" % domain)
+            is_subdomain = domain.split(".", 1)[1] in all_domains
+            for report in self.check_domain(domain, domain == main_domain, is_subdomain=is_subdomain):
+                yield report
+
+        # Check if a domain buy by the user will expire soon
+        psl = PublicSuffixList()
+        domains_from_registrar = [psl.get_public_suffix(domain) for domain in all_domains]
+        domains_from_registrar = [domain for domain in domains_from_registrar if "." in domain]
+        domains_from_registrar = set(domains_from_registrar) - set(YNH_DYNDNS_DOMAINS + ["netlib.re"])
+        for report in self.check_expiration_date(domains_from_registrar):
+            yield report
+
+    def check_domain(self, domain, is_main_domain, is_subdomain):
+
+        expected_configuration = _build_dns_conf(domain, include_empty_AAAA_if_no_ipv6=True)
+
+        categories = ["basic", "mail", "xmpp", "extra"]
+        # For subdomains, we only diagnosis A and AAAA records
+        if is_subdomain:
+            categories = ["basic"]
+
+        for category in categories:
+
+            records = expected_configuration[category]
+            discrepancies = []
+            results = {}
+
+            for r in records:
+                id_ = r["type"] + ":" + r["name"]
+                r["current"] = self.get_current_record(domain, r["name"], r["type"])
+                if r["value"] == "@":
+                    r["value"] = domain + "."
+
+                if self.current_record_match_expected(r):
+                    results[id_] = "OK"
+                else:
+                    if r["current"] is None:
+                        results[id_] = "MISSING"
+                        discrepancies.append(("diagnosis_dns_missing_record", r))
+                    else:
+                        results[id_] = "WRONG"
+                        discrepancies.append(("diagnosis_dns_discrepancy", r))
+
+            def its_important():
+                # Every mail DNS records are important for main domain
+                # For other domain, we only report it as a warning for now...
+                if is_main_domain and category == "mail":
+                    return True
+                elif category == "basic":
+                    # A bad or missing A record is critical ...
+                    # And so is a wrong AAAA record
+                    # (However, a missing AAAA record is acceptable)
+                    if results["A:@"] != "OK" or results["AAAA:@"] == "WRONG":
+                        return True
+
+                return False
+
+            if discrepancies:
+                status = "ERROR" if its_important() else "WARNING"
+                summary = "diagnosis_dns_bad_conf"
+            else:
+                status = "SUCCESS"
+                summary = "diagnosis_dns_good_conf"
+
+            output = dict(meta={"domain": domain, "category": category},
+                          data=results,
+                          status=status,
+                          summary=summary)
+
+            if discrepancies:
+                # For ynh-managed domains (nohost.me etc...), tell people to try to "yunohost dyndns update --force"
+                if any(domain.endswith(ynh_dyndns_domain) for ynh_dyndns_domain in YNH_DYNDNS_DOMAINS):
+                    output["details"] = ["diagnosis_dns_try_dyndns_update_force"]
+                # Otherwise point to the documentation
+                else:
+                    output["details"] = ["diagnosis_dns_point_to_doc"]
+                output["details"] += discrepancies
+
+            yield output
+
+    def get_current_record(self, domain, name, type_):
+
+        query = "%s.%s" % (name, domain) if name != "@" else domain
+        success, answers = dig(query, type_, resolvers="force_external")
+
+        if success != "ok":
+            return None
+        else:
+            return answers[0] if len(answers) == 1 else answers
+
+    def current_record_match_expected(self, r):
+        if r["value"] is not None and r["current"] is None:
+            return False
+        if r["value"] is None and r["current"] is not None:
+            return False
+        elif isinstance(r["current"], list):
+            return False
+
+        if r["type"] == "TXT":
+            # Split expected/current
+            #  from  "v=DKIM1; k=rsa; p=hugekey;"
+            #  to a set like {'v=DKIM1', 'k=rsa', 'p=...'}
+            # Additionally, for DKIM, because the key is pretty long,
+            # some DNS registrar sometime split it into several pieces like this:
+            # "p=foo" "bar" (with a space and quotes in the middle)...
+            expected = set(r["value"].strip(';" ').replace(";", " ").split())
+            current = set(r["current"].replace('" "', '').strip(';" ').replace(";", " ").split())
+
+            # For SPF, ignore parts starting by ip4: or ip6:
+            if r["name"] == "@":
+                current = {part for part in current if not part.startswith("ip4:") and not part.startswith("ip6:")}
+            return expected == current
+        elif r["type"] == "MX":
+            # For MX, we want to ignore the priority
+            expected = r["value"].split()[-1]
+            current = r["current"].split()[-1]
+            return expected == current
+        else:
+            return r["current"] == r["value"]
+
+    def check_expiration_date(self, domains):
+        """
+        Alert if expiration date of a domain is soon
+        """
+
+        details = {
+            "not_found": [],
+            "error": [],
+            "warning": [],
+            "success": []
+        }
+
+        for domain in domains:
+            expire_date = self.get_domain_expiration(domain)
+
+            if isinstance(expire_date, str):
+                status_ns, _ = dig(domain, "NS", resolvers="force_external")
+                status_a, _ = dig(domain, "A", resolvers="force_external")
+                if "ok" not in [status_ns, status_a]:
+                    details["not_found"].append((
+                        "diagnosis_domain_%s_details" % (expire_date),
+                        {"domain": domain}))
+                else:
+                    self.logger_debug("Dyndns domain: %s" % (domain))
+                continue
+
+            expire_in = expire_date - datetime.now()
+
+            alert_type = "success"
+            if expire_in <= timedelta(15):
+                alert_type = "error"
+            elif expire_in <= timedelta(45):
+                alert_type = "warning"
+
+            args = {
+                "domain": domain,
+                "days": expire_in.days - 1,
+                "expire_date": str(expire_date)
+            }
+            details[alert_type].append(("diagnosis_domain_expires_in", args))
+
+        for alert_type in ["success", "error", "warning", "not_found"]:
+            if details[alert_type]:
+                if alert_type == "not_found":
+                    meta = {"test": "domain_not_found"}
+                else:
+                    meta = {"test": "domain_expiration"}
+                # Allow to ignore specifically a single domain
+                if len(details[alert_type]) == 1:
+                    meta["domain"] = details[alert_type][0][1]["domain"]
+                yield dict(meta=meta,
+                           data={},
+                           status=alert_type.upper() if alert_type != "not_found" else "WARNING",
+                           summary="diagnosis_domain_expiration_" + alert_type,
+                           details=details[alert_type])
+
+    def get_domain_expiration(self, domain):
+        """
+        Return the expiration datetime of a domain or None
+        """
+        command = "whois -H %s || echo failed" % (domain)
+        out = check_output(command).split("\n")
+
+        # Reduce output to determine if whois answer is equivalent to NOT FOUND
+        filtered_out = [line for line in out
+               if re.search(r'^[a-zA-Z0-9 ]{4,25}:', line, re.IGNORECASE) and
+               not re.match(r'>>> Last update of whois', line, re.IGNORECASE) and
+               not re.match(r'^NOTICE:', line, re.IGNORECASE) and
+               not re.match(r'^%%', line, re.IGNORECASE) and
+               not re.match(r'"https?:"', line, re.IGNORECASE)]
+
+        # If there is less than 7 lines, it's NOT FOUND response
+        if len(filtered_out) <= 6:
+            return "not_found"
+
+        for line in out:
+            match = re.search(r'Expir.+(\d{4}-\d{2}-\d{2})', line, re.IGNORECASE)
+            if match is not None:
+                return datetime.strptime(match.group(1), '%Y-%m-%d')
+
+            match = re.search(r'Expir.+(\d{2}-\w{3}-\d{4})', line, re.IGNORECASE)
+            if match is not None:
+                return datetime.strptime(match.group(1), '%d-%b-%Y')
+
+        return "expiration_not_found"
+
+
+def main(args, env, loggers):
+    return DNSRecordsDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/14-ports.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+
+import os
+
+from yunohost.diagnosis import Diagnoser
+from yunohost.service import _get_services
+
+
+class PortsDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 600
+    dependencies = ["ip"]
+
+    def run(self):
+
+        # TODO: report a warning if port 53 or 5353 is exposed to the outside world...
+
+        # This dict is something like :
+        #   {   80: "nginx",
+        #       25: "postfix",
+        #       443: "nginx"
+        #       ... }
+        ports = {}
+        services = _get_services()
+        for service, infos in services.items():
+            for port in infos.get("needs_exposed_ports", []):
+                ports[port] = service
+
+        ipversions = []
+        ipv4 = Diagnoser.get_cached_report("ip", item={"test": "ipv4"}) or {}
+        if ipv4.get("status") == "SUCCESS":
+            ipversions.append(4)
+
+        # To be discussed: we could also make this check dependent on the
+        # existence of an AAAA record...
+        ipv6 = Diagnoser.get_cached_report("ip", item={"test": "ipv6"}) or {}
+        if ipv6.get("status") == "SUCCESS":
+            ipversions.append(6)
+
+        # Fetch test result for each relevant IP version
+        results = {}
+        for ipversion in ipversions:
+            try:
+                r = Diagnoser.remote_diagnosis('check-ports',
+                                               data={'ports': ports.keys()},
+                                               ipversion=ipversion)
+                results[ipversion] = r["ports"]
+            except Exception as e:
+                yield dict(meta={"reason": "remote_diagnosis_failed", "ipversion": ipversion},
+                           data={"error": str(e)},
+                           status="WARNING",
+                           summary="diagnosis_ports_could_not_diagnose",
+                           details=["diagnosis_ports_could_not_diagnose_details"])
+                continue
+
+        ipversions = results.keys()
+        if not ipversions:
+            return
+
+        for port, service in sorted(ports.items()):
+            port = str(port)
+            category = services[service].get("category", "[?]")
+
+            # If both IPv4 and IPv6 (if applicable) are good
+            if all(results[ipversion].get(port) is True for ipversion in ipversions):
+                yield dict(meta={"port": port},
+                           data={"service": service, "category": category},
+                           status="SUCCESS",
+                           summary="diagnosis_ports_ok",
+                           details=["diagnosis_ports_needed_by"])
+            # If both IPv4 and IPv6 (if applicable) are failed
+            elif all(results[ipversion].get(port) is not True for ipversion in ipversions):
+                yield dict(meta={"port": port},
+                           data={"service": service, "category": category},
+                           status="ERROR",
+                           summary="diagnosis_ports_unreachable",
+                           details=["diagnosis_ports_needed_by", "diagnosis_ports_forwarding_tip"])
+            # If only IPv4 is failed or only IPv6 is failed (if applicable)
+            else:
+                passed, failed = (4, 6) if results[4].get(port) is True else (6, 4)
+
+                # Failing in ipv4 is critical.
+                # If we failed in IPv6 but there's in fact no AAAA record
+                # It's an acceptable situation and we shall not report an
+                # error
+                # If any AAAA record is set, IPv6 is important...
+                def ipv6_is_important():
+                    dnsrecords = Diagnoser.get_cached_report("dnsrecords") or {}
+                    return any(record["data"].get("AAAA:@") in ["OK", "WRONG"] for record in dnsrecords.get("items", []))
+
+                if failed == 4 or ipv6_is_important():
+                    yield dict(meta={"port": port},
+                               data={"service": service, "category": category, "passed": passed, "failed": failed},
+                               status="ERROR",
+                               summary="diagnosis_ports_partially_unreachable",
+                               details=["diagnosis_ports_needed_by", "diagnosis_ports_forwarding_tip"])
+                # So otherwise we report a success
+                # And in addition we report an info about the failure in IPv6
+                # *with a different meta* (important to avoid conflicts when
+                # fetching the other info...)
+                else:
+                    yield dict(meta={"port": port},
+                               data={"service": service, "category": category},
+                               status="SUCCESS",
+                               summary="diagnosis_ports_ok",
+                               details=["diagnosis_ports_needed_by"])
+                    yield dict(meta={"test": "ipv6", "port": port},
+                               data={"service": service, "category": category, "passed": passed, "failed": failed},
+                               status="INFO",
+                               summary="diagnosis_ports_partially_unreachable",
+                               details=["diagnosis_ports_needed_by", "diagnosis_ports_forwarding_tip"])
+
+
+def main(args, env, loggers):
+    return PortsDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/21-web.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+
+import os
+import random
+import requests
+
+from moulinette.utils.filesystem import read_file
+
+from yunohost.diagnosis import Diagnoser
+from yunohost.domain import domain_list
+
+DIAGNOSIS_SERVER = "diagnosis.yunohost.org"
+
+
+class WebDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 600
+    dependencies = ["ip"]
+
+    def run(self):
+
+        all_domains = domain_list()["domains"]
+        domains_to_check = []
+        for domain in all_domains:
+
+            # If the diagnosis location ain't defined, can't do diagnosis,
+            # probably because nginx conf manually modified...
+            nginx_conf = "/etc/nginx/conf.d/%s.conf" % domain
+            if ".well-known/ynh-diagnosis/" not in read_file(nginx_conf):
+                yield dict(meta={"domain": domain},
+                           status="WARNING",
+                           summary="diagnosis_http_nginx_conf_not_up_to_date",
+                           details=["diagnosis_http_nginx_conf_not_up_to_date_details"])
+            else:
+                domains_to_check.append(domain)
+
+        self.nonce = ''.join(random.choice("0123456789abcedf") for i in range(16))
+        os.system("rm -rf /tmp/.well-known/ynh-diagnosis/")
+        os.system("mkdir -p /tmp/.well-known/ynh-diagnosis/")
+        os.system("touch /tmp/.well-known/ynh-diagnosis/%s" % self.nonce)
+
+        if not domains_to_check:
+            return
+
+        # To perform hairpinning test, we gotta make sure that port forwarding
+        # is working and therefore we'll do it only if at least one ipv4 domain
+        # works.
+        self.do_hairpinning_test = False
+
+        ipversions = []
+        ipv4 = Diagnoser.get_cached_report("ip", item={"test": "ipv4"}) or {}
+        if ipv4.get("status") == "SUCCESS":
+            ipversions.append(4)
+
+        # To be discussed: we could also make this check dependent on the
+        # existence of an AAAA record...
+        ipv6 = Diagnoser.get_cached_report("ip", item={"test": "ipv6"}) or {}
+        if ipv6.get("status") == "SUCCESS":
+            ipversions.append(6)
+
+        for item in self.test_http(domains_to_check, ipversions):
+            yield item
+
+        # If at least one domain is correctly exposed to the outside,
+        # attempt to diagnose hairpinning situations. On network with
+        # hairpinning issues, the server may be correctly exposed on the
+        # outside, but from the outside, it will be as if the port forwarding
+        # was not configured... Hence, calling for example
+        # "curl --head the.global.ip" will simply timeout...
+        if self.do_hairpinning_test:
+            global_ipv4 = ipv4.get("data", {}).get("global", None)
+            if global_ipv4:
+                try:
+                    requests.head("http://" + global_ipv4, timeout=5)
+                except requests.exceptions.Timeout:
+                    yield dict(meta={"test": "hairpinning"},
+                               status="WARNING",
+                               summary="diagnosis_http_hairpinning_issue",
+                               details=["diagnosis_http_hairpinning_issue_details"])
+                except:
+                    # Well I dunno what to do if that's another exception
+                    # type... That'll most probably *not* be an hairpinning
+                    # issue but something else super weird ...
+                    pass
+
+    def test_http(self, domains, ipversions):
+
+        results = {}
+        for ipversion in ipversions:
+            try:
+                r = Diagnoser.remote_diagnosis('check-http',
+                                               data={'domains': domains,
+                                                     "nonce": self.nonce},
+                                               ipversion=ipversion)
+                results[ipversion] = r["http"]
+            except Exception as e:
+                yield dict(meta={"reason": "remote_diagnosis_failed", "ipversion": ipversion},
+                           data={"error": str(e)},
+                           status="WARNING",
+                           summary="diagnosis_http_could_not_diagnose",
+                           details=["diagnosis_http_could_not_diagnose_details"])
+                continue
+
+        ipversions = results.keys()
+        if not ipversions:
+            return
+
+        for domain in domains:
+
+            # If both IPv4 and IPv6 (if applicable) are good
+            if all(results[ipversion][domain]["status"] == "ok" for ipversion in ipversions):
+                if 4 in ipversions:
+                    self.do_hairpinning_test = True
+                yield dict(meta={"domain": domain},
+                           status="SUCCESS",
+                           summary="diagnosis_http_ok")
+            # If both IPv4 and IPv6 (if applicable) are failed
+            elif all(results[ipversion][domain]["status"] != "ok" for ipversion in ipversions):
+                detail = results[4 if 4 in ipversions else 6][domain]["status"]
+                yield dict(meta={"domain": domain},
+                           status="ERROR",
+                           summary="diagnosis_http_unreachable",
+                           details=[detail.replace("error_http_check", "diagnosis_http")])
+            # If only IPv4 is failed or only IPv6 is failed (if applicable)
+            else:
+                passed, failed = (4, 6) if results[4][domain]["status"] == "ok" else (6, 4)
+                detail = results[failed][domain]["status"]
+
+                # Failing in ipv4 is critical.
+                # If we failed in IPv6 but there's in fact no AAAA record
+                # It's an acceptable situation and we shall not report an
+                # error
+                def ipv6_is_important_for_this_domain():
+                    dnsrecords = Diagnoser.get_cached_report("dnsrecords", item={"domain": domain, "category": "basic"}) or {}
+                    AAAA_status = dnsrecords.get("data", {}).get("AAAA:@")
+
+                    return AAAA_status in ["OK", "WRONG"]
+
+                if failed == 4 or ipv6_is_important_for_this_domain():
+                    yield dict(meta={"domain": domain},
+                               data={"passed": passed, "failed": failed},
+                               status="ERROR",
+                               summary="diagnosis_http_partially_unreachable",
+                               details=[detail.replace("error_http_check", "diagnosis_http")])
+                # So otherwise we report a success (note that this info is
+                # later used to know that ACME challenge is doable)
+                #
+                # And in addition we report an info about the failure in IPv6
+                # *with a different meta* (important to avoid conflicts when
+                # fetching the other info...)
+                else:
+                    self.do_hairpinning_test = True
+                    yield dict(meta={"domain": domain},
+                               status="SUCCESS",
+                               summary="diagnosis_http_ok")
+                    yield dict(meta={"test": "ipv6", "domain": domain},
+                               data={"passed": passed, "failed": failed},
+                               status="INFO",
+                               summary="diagnosis_http_partially_unreachable",
+                               details=[detail.replace("error_http_check", "diagnosis_http")])
+
+
+def main(args, env, loggers):
+    return WebDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/24-mail.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python
+
+import os
+import dns.resolver
+import re
+
+from subprocess import CalledProcessError
+
+from moulinette.utils.process import check_output
+from moulinette.utils.filesystem import read_yaml
+
+from yunohost.diagnosis import Diagnoser
+from yunohost.domain import _get_maindomain, domain_list
+from yunohost.settings import settings_get
+from yunohost.utils.network import dig
+
+DEFAULT_DNS_BLACKLIST = "/usr/share/yunohost/other/dnsbl_list.yml"
+
+
+class MailDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 600
+    dependencies = ["ip"]
+
+    def run(self):
+
+        self.ehlo_domain = _get_maindomain()
+        self.mail_domains = domain_list()["domains"]
+        self.ipversions, self.ips = self.get_ips_checked()
+
+        # TODO Is a A/AAAA and MX Record ?
+        # TODO Are outgoing public IPs authorized to send mail by SPF ?
+        # TODO Validate DKIM and dmarc ?
+        # TODO check that the recent mail logs are not filled with thousand of email sending (unusual number of mail sent)
+        # TODO check for unusual failed sending attempt being refused in the logs ?
+        checks = ["check_outgoing_port_25", "check_ehlo", "check_fcrdns",
+                  "check_blacklist", "check_queue"]
+        for check in checks:
+            self.logger_debug("Running " + check)
+            reports = list(getattr(self, check)())
+            for report in reports:
+                yield report
+            if not reports:
+                name = check[6:]
+                yield dict(meta={"test": "mail_" + name},
+                        status="SUCCESS",
+                        summary="diagnosis_mail_" + name + "_ok")
+
+    def check_outgoing_port_25(self):
+        """
+        Check outgoing port 25 is open and not blocked by router
+        This check is ran on IPs we could used to send mail.
+        """
+
+        for ipversion in self.ipversions:
+            cmd = '/bin/nc -{ipversion} -z -w2 yunohost.org 25'.format(ipversion=ipversion)
+            if os.system(cmd) != 0:
+                yield dict(meta={"test": "outgoing_port_25", "ipversion": ipversion},
+                           data={},
+                           status="ERROR",
+                           summary="diagnosis_mail_outgoing_port_25_blocked",
+                           details=["diagnosis_mail_outgoing_port_25_blocked_details",
+                                    "diagnosis_mail_outgoing_port_25_blocked_relay_vpn"])
+
+    def check_ehlo(self):
+        """
+        Check the server is reachable from outside and it's the good one
+        This check is ran on IPs we could used to send mail.
+        """
+
+        for ipversion in self.ipversions:
+            try:
+                r = Diagnoser.remote_diagnosis('check-smtp',
+                                               data={},
+                                               ipversion=ipversion)
+            except Exception as e:
+                yield dict(meta={"test": "mail_ehlo", "reason": "remote_server_failed",
+                                 "ipversion": ipversion},
+                           data={"error": str(e)},
+                           status="WARNING",
+                           summary="diagnosis_mail_ehlo_could_not_diagnose",
+                           details=["diagnosis_mail_ehlo_could_not_diagnose_details"])
+                continue
+
+            if r["status"] != "ok":
+                summary = r["status"].replace("error_smtp_", "diagnosis_mail_ehlo_")
+                yield dict(meta={"test": "mail_ehlo", "ipversion": ipversion},
+                           data={},
+                           status="ERROR",
+                           summary=summary,
+                           details=[summary + "_details"])
+            elif r["helo"] != self.ehlo_domain:
+                yield dict(meta={"test": "mail_ehlo", "ipversion": ipversion},
+                           data={"wrong_ehlo": r["helo"], "right_ehlo": self.ehlo_domain},
+                           status="ERROR",
+                           summary="diagnosis_mail_ehlo_wrong",
+                           details=["diagnosis_mail_ehlo_wrong_details"])
+
+    def check_fcrdns(self):
+        """
+        Check the reverse DNS is well defined by doing a Forward-confirmed
+        reverse DNS check
+        This check is ran on IPs we could used to send mail.
+        """
+
+        for ip in self.ips:
+            if ":" in ip:
+                ipversion = 6
+                details = ["diagnosis_mail_fcrdns_nok_details",
+                           "diagnosis_mail_fcrdns_nok_alternatives_6"]
+            else:
+                ipversion = 4
+                details = ["diagnosis_mail_fcrdns_nok_details",
+                           "diagnosis_mail_fcrdns_nok_alternatives_4"]
+
+            rev = dns.reversename.from_address(ip)
+            subdomain = str(rev.split(3)[0])
+            query = subdomain
+            if ipversion == 4:
+                query += '.in-addr.arpa'
+            else:
+                query += '.ip6.arpa'
+
+            # Do the DNS Query
+            status, value = dig(query, 'PTR', resolvers="force_external")
+            if status == "nok":
+                yield dict(meta={"test": "mail_fcrdns", "ipversion": ipversion},
+                           data={"ip": ip, "ehlo_domain": self.ehlo_domain},
+                           status="ERROR",
+                           summary="diagnosis_mail_fcrdns_dns_missing",
+                           details=details)
+                continue
+
+            rdns_domain = ''
+            if len(value) > 0:
+                rdns_domain = value[0][:-1] if value[0].endswith('.') else value[0]
+            if rdns_domain != self.ehlo_domain:
+                details = ["diagnosis_mail_fcrdns_different_from_ehlo_domain_details"] + details
+                yield dict(meta={"test": "mail_fcrdns", "ipversion": ipversion},
+                           data={"ip": ip,
+                                 "ehlo_domain": self.ehlo_domain,
+                                 "rdns_domain": rdns_domain},
+                           status="ERROR",
+                           summary="diagnosis_mail_fcrdns_different_from_ehlo_domain",
+                           details=details)
+
+    def check_blacklist(self):
+        """
+        Check with dig onto blacklist DNS server
+        This check is ran on IPs and domains we could used to send mail.
+        """
+
+        dns_blacklists = read_yaml(DEFAULT_DNS_BLACKLIST)
+        for item in self.ips + self.mail_domains:
+            for blacklist in dns_blacklists:
+                item_type = "domain"
+                if ":" in item:
+                    item_type = 'ipv6'
+                elif re.match(r'^\d+\.\d+\.\d+\.\d+$', item):
+                    item_type = 'ipv4'
+
+                if not blacklist[item_type]:
+                    continue
+
+                # Build the query for DNSBL
+                subdomain = item
+                if item_type != "domain":
+                    rev = dns.reversename.from_address(item)
+                    subdomain = str(rev.split(3)[0])
+                query = subdomain + '.' + blacklist['dns_server']
+
+                # Do the DNS Query
+                status, _ = dig(query, 'A')
+                if status != 'ok':
+                    continue
+
+                # Try to get the reason
+                details = []
+                status, answers = dig(query, 'TXT')
+                reason = "-"
+                if status == 'ok':
+                    reason = ', '.join(answers)
+                    details.append("diagnosis_mail_blacklist_reason")
+
+                details.append("diagnosis_mail_blacklist_website")
+
+                yield dict(meta={"test": "mail_blacklist", "item": item,
+                                 "blacklist": blacklist["dns_server"]},
+                           data={'blacklist_name': blacklist['name'],
+                                 'blacklist_website': blacklist['website'],
+                                 'reason': reason},
+                           status="ERROR",
+                           summary='diagnosis_mail_blacklist_listed_by',
+                           details=details)
+
+    def check_queue(self):
+        """
+        Check mail queue is not filled with hundreds of email pending
+        """
+
+        command = 'postqueue -p | grep -v "Mail queue is empty" | grep -c "^[A-Z0-9]" || true'
+        try:
+            output = check_output(command)
+            pending_emails = int(output)
+        except (ValueError, CalledProcessError) as e:
+            yield dict(meta={"test": "mail_queue"},
+                       data={"error": str(e)},
+                       status="ERROR",
+                       summary="diagnosis_mail_queue_unavailable",
+                       details="diagnosis_mail_queue_unavailable_details")
+        else:
+            if pending_emails > 100:
+                yield dict(meta={"test": "mail_queue"},
+                           data={'nb_pending': pending_emails},
+                       status="WARNING",
+                       summary="diagnosis_mail_queue_too_big")
+            else:
+                yield dict(meta={"test": "mail_queue"},
+                           data={'nb_pending': pending_emails},
+                           status="SUCCESS",
+                           summary="diagnosis_mail_queue_ok")
+
+    def get_ips_checked(self):
+        outgoing_ipversions = []
+        outgoing_ips = []
+        ipv4 = Diagnoser.get_cached_report("ip", {"test": "ipv4"}) or {}
+        if ipv4.get("status") == "SUCCESS":
+            outgoing_ipversions.append(4)
+            global_ipv4 = ipv4.get("data", {}).get("global", {})
+            if global_ipv4:
+                outgoing_ips.append(global_ipv4)
+
+        if settings_get("smtp.allow_ipv6"):
+            ipv6 = Diagnoser.get_cached_report("ip", {"test": "ipv6"}) or {}
+            if ipv6.get("status") == "SUCCESS":
+                outgoing_ipversions.append(6)
+                global_ipv6 = ipv6.get("data", {}).get("global", {})
+                if global_ipv6:
+                    outgoing_ips.append(global_ipv6)
+        return (outgoing_ipversions, outgoing_ips)
+
+
+def main(args, env, loggers):
+    return MailDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/30-services.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+
+import os
+
+from yunohost.diagnosis import Diagnoser
+from yunohost.service import service_status
+
+
+class ServicesDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 300
+    dependencies = []
+
+    def run(self):
+
+        all_result = service_status()
+
+        for service, result in sorted(all_result.items()):
+
+            item = dict(meta={"service": service},
+                        data={"status": result["status"], "configuration": result["configuration"]})
+
+            if result["status"] != "running":
+                item["status"] = "ERROR" if result["status"] != "unknown" else "WARNING"
+                item["summary"] = "diagnosis_services_bad_status"
+                item["details"] = ["diagnosis_services_bad_status_tip"]
+
+            elif result["configuration"] == "broken":
+                item["status"] = "WARNING"
+                item["summary"] = "diagnosis_services_conf_broken"
+                item["details"] = result["configuration-details"]
+
+            else:
+                item["status"] = "SUCCESS"
+                item["summary"] = "diagnosis_services_running"
+
+            yield item
+
+
+def main(args, env, loggers):
+    return ServicesDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/50-systemresources.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python
+import os
+import psutil
+import subprocess
+import datetime
+import re
+
+from yunohost.diagnosis import Diagnoser
+
+
+class SystemResourcesDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 300
+    dependencies = []
+
+    def run(self):
+
+        MB = 1024**2
+        GB = MB * 1024
+
+        #
+        # RAM
+        #
+
+        ram = psutil.virtual_memory()
+        ram_available_percent = 100 * ram.available / ram.total
+        item = dict(meta={"test": "ram"},
+                    data={"total": human_size(ram.total),
+                          "available": human_size(ram.available),
+                          "available_percent": round_(ram_available_percent)})
+
+        if ram.available < 100 * MB or ram_available_percent < 5:
+            item["status"] = "ERROR"
+            item["summary"] = "diagnosis_ram_verylow"
+        elif ram.available < 200 * MB or ram_available_percent < 10:
+            item["status"] = "WARNING"
+            item["summary"] = "diagnosis_ram_low"
+        else:
+            item["status"] = "SUCCESS"
+            item["summary"] = "diagnosis_ram_ok"
+        yield item
+
+        #
+        # Swap
+        #
+
+        swap = psutil.swap_memory()
+        item = dict(meta={"test": "swap"},
+                    data={"total": human_size(swap.total), "recommended": "512 MiB"})
+        if swap.total <= 1 * MB:
+            item["status"] = "INFO"
+            item["summary"] = "diagnosis_swap_none"
+        elif swap.total < 450 * MB:
+            item["status"] = "INFO"
+            item["summary"] = "diagnosis_swap_notsomuch"
+        else:
+            item["status"] = "SUCCESS"
+            item["summary"] = "diagnosis_swap_ok"
+        item["details"] = ["diagnosis_swap_tip"]
+        yield item
+
+        # FIXME : add a check that swapiness is low if swap is on a sdcard...
+
+        #
+        # Disks usage
+        #
+
+        disk_partitions = sorted(psutil.disk_partitions(), key=lambda k: k.mountpoint)
+
+        # Ignore /dev/loop stuff which are ~virtual partitions ? (e.g. mounted to /snap/)
+        disk_partitions = [d for d in disk_partitions if not d.device.startswith("/dev/loop")]
+
+        for disk_partition in disk_partitions:
+            device = disk_partition.device
+            mountpoint = disk_partition.mountpoint
+
+            usage = psutil.disk_usage(mountpoint)
+            free_percent = 100 - round_(usage.percent)
+
+            item = dict(meta={"test": "diskusage", "mountpoint": mountpoint},
+                        data={"device": device,
+                              # N.B.: we do not use usage.total because we want
+                              # to take into account the 5% security margin
+                              # correctly (c.f. the doc of psutil ...)
+                              "total": human_size(usage.used + usage.free),
+                              "free": human_size(usage.free),
+                              "free_percent": free_percent})
+
+            # We have an additional absolute constrain on / and /var because
+            # system partitions are critical, having them full may prevent
+            # upgrades etc...
+            if free_percent < 2.5 or (mountpoint in ["/", "/var"] and usage.free < 1 * GB):
+                item["status"] = "ERROR"
+                item["summary"] = "diagnosis_diskusage_verylow"
+            elif free_percent < 5 or (mountpoint in ["/", "/var"] and usage.free < 2 * GB):
+                item["status"] = "WARNING"
+                item["summary"] = "diagnosis_diskusage_low"
+            else:
+                item["status"] = "SUCCESS"
+                item["summary"] = "diagnosis_diskusage_ok"
+
+            yield item
+
+        #
+        # Recent kills by oom_reaper
+        #
+
+        kills_count = self.recent_kills_by_oom_reaper()
+        if kills_count:
+            kills_summary = "\n".join(["%s (x%s)" % (proc, count) for proc, count in kills_count])
+
+            yield dict(meta={"test": "oom_reaper"},
+                       status="WARNING",
+                       summary="diagnosis_processes_killed_by_oom_reaper",
+                       data={"kills_summary": kills_summary})
+
+    def recent_kills_by_oom_reaper(self):
+        if not os.path.exists("/var/log/kern.log"):
+            return []
+
+        def analyzed_kern_log():
+
+            cmd = 'tail -n 10000 /var/log/kern.log | grep "oom_reaper: reaped process" || true'
+            out = subprocess.check_output(cmd, shell=True).strip()
+            lines = out.split("\n") if out else []
+
+            now = datetime.datetime.now()
+
+            for line in reversed(lines):
+                # Lines look like :
+                # Aug 25 18:48:21 yolo kernel: [ 9623.613667] oom_reaper: reaped process 11509 (uwsgi), now anon-rss:0kB, file-rss:0kB, shmem-rss:328kB
+                date_str = str(now.year) + " " + " ".join(line.split()[:3])
+                date = datetime.datetime.strptime(date_str, '%Y %b %d %H:%M:%S')
+                diff = now - date
+                if diff.days >= 1:
+                    break
+                process_killed = re.search(r"\(.*\)", line).group().strip("()")
+                yield process_killed
+
+        processes = list(analyzed_kern_log())
+        kills_count = [(p, len([p_ for p_ in processes if p_ == p])) for p in set(processes)]
+        kills_count = sorted(kills_count, key=lambda p: p[1], reverse=True)
+
+        return kills_count
+
+
+def human_size(bytes_):
+    # Adapted from https://stackoverflow.com/a/1094933
+    for unit in ['', 'ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
+        if abs(bytes_) < 1024.0:
+            return "%s %sB" % (round_(bytes_), unit)
+        bytes_ /= 1024.0
+    return "%s %sB" % (round_(bytes_), 'Yi')
+
+
+def round_(n):
+    # round_(22.124) -> 22
+    # round_(9.45) -> 9.4
+    n = round(n, 1)
+    if n > 10:
+        n = int(round(n))
+    return n
+
+
+def main(args, env, loggers):
+    return SystemResourcesDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/70-regenconf.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+
+import os
+
+from yunohost.diagnosis import Diagnoser
+from yunohost.regenconf import _get_regenconf_infos, _calculate_hash
+
+
+class RegenconfDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 300
+    dependencies = []
+
+    def run(self):
+
+        regenconf_modified_files = list(self.manually_modified_files())
+
+        if not regenconf_modified_files:
+            yield dict(meta={"test": "regenconf"},
+                       status="SUCCESS",
+                       summary="diagnosis_regenconf_allgood"
+                       )
+        else:
+            for f in regenconf_modified_files:
+                yield dict(meta={"test": "regenconf", "category": f['category'], "file": f['path']},
+                           status="WARNING",
+                           summary="diagnosis_regenconf_manually_modified",
+                           details=["diagnosis_regenconf_manually_modified_details"]
+                           )
+
+    def manually_modified_files(self):
+
+        for category, infos in _get_regenconf_infos().items():
+            for path, hash_ in infos["conffiles"].items():
+                if hash_ != _calculate_hash(path):
+                    yield {"path": path, "category": category}
+
+
+def main(args, env, loggers):
+    return RegenconfDiagnoser(args, env, loggers).diagnose()