Merge branch 'dev' into enh-backup-repo

.coveragerc

@@ -0,0 +1,2 @@
+[report]
+omit=src/yunohost/tests/*,src/yunohost/vendor/*,/usr/lib/moulinette/yunohost/*

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# These are supported funding model platforms
+
+custom: https://donate.yunohost.org
+liberapay: YunoHost

.github/PULL_REQUEST_TEMPLATE.md

@@ -13,10 +13,3 @@
 ## How to test
 
 ...
-
-## Validation
-
-- [ ] Principle agreement 0/2 : 
-- [ ] Quick review 0/1 : 
-- [ ] Simple test 0/1 : 
-- [ ] Deep review 0/1 : 

.gitlab-ci.yml

@@ -0,0 +1,30 @@
+---
+stages:
+  - build
+  - install
+  - tests
+  - lint
+  - doc
+  - translation
+
+default:
+  tags:
+    - yunohost-ci
+  # All jobs are interruptible by default
+  interruptible: true
+
+# see: https://docs.gitlab.com/ee/ci/yaml/#switch-between-branch-pipelines-and-merge-request-pipelines
+workflow:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event" # If we move to gitlab one day
+    - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" # For github PR
+    - if: $CI_COMMIT_TAG # For tags
+    - if: $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH  && $CI_PIPELINE_SOURCE == "push" # If it's not the default branch and if it's a push, then do not trigger a build
+      when: never
+    - when: always
+
+variables:
+  YNH_BUILD_DIR: "ynh-build"
+
+include:
+  - local: .gitlab/ci/*.gitlab-ci.yml

.gitlab/ci/build.gitlab-ci.yml

@@ -0,0 +1,54 @@
+.build-stage:
+  stage: build
+  image: "before-install"
+  variables:
+    YNH_SOURCE: "https://github.com/yunohost"
+  before_script:
+    - mkdir -p $YNH_BUILD_DIR
+  artifacts:
+    paths:
+      - $YNH_BUILD_DIR/*.deb
+
+.build_script: &build_script
+  - cd $YNH_BUILD_DIR/$PACKAGE
+  - VERSION=$(dpkg-parsechangelog -S Version 2>/dev/null)
+  - VERSION_NIGHTLY="${VERSION}+$(date +%Y%m%d%H%M)"
+  - dch --package "${PACKAGE}" --force-bad-version -v "${VERSION_NIGHTLY}" -D "unstable" --force-distribution "Daily build."
+  - debuild --no-lintian -us -uc
+
+########################################
+# BUILD DEB
+########################################
+
+build-yunohost:
+  extends: .build-stage
+  variables:
+    PACKAGE: "yunohost"
+  script:
+    - git ls-files | xargs tar -czf archive.tar.gz
+    - mkdir -p $YNH_BUILD_DIR/$PACKAGE
+    - cat archive.tar.gz | tar -xz -C $YNH_BUILD_DIR/$PACKAGE
+    - rm archive.tar.gz
+    - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $(pwd)/$YNH_BUILD_DIR/$PACKAGE
+    - *build_script
+
+
+build-ssowat:
+  extends: .build-stage
+  variables:
+    PACKAGE: "ssowat"
+  script:
+    - DEBIAN_DEPENDS=$(cat debian/control | tr "," "\n" | grep -Po "ssowat \([>,=,<]+ .*\)" | grep -Po "[0-9\.]+")
+    - git clone $YNH_SOURCE/$PACKAGE -b $CI_COMMIT_REF_NAME $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE -b $DEBIAN_DEPENDS $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE $YNH_BUILD_DIR/$PACKAGE --depth 1
+    - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $(pwd)/$YNH_BUILD_DIR/$PACKAGE
+    - *build_script
+
+build-moulinette:
+  extends: .build-stage
+  variables:
+    PACKAGE: "moulinette"
+  script:
+    - DEBIAN_DEPENDS=$(cat debian/control | tr "," "\n" | grep -Po "moulinette \([>,=,<]+ .*\)" | grep -Po "[0-9\.]+")
+    - git clone $YNH_SOURCE/$PACKAGE -b $CI_COMMIT_REF_NAME $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE -b $DEBIAN_DEPENDS $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE $YNH_BUILD_DIR/$PACKAGE --depth 1
+    - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $(pwd)/$YNH_BUILD_DIR/$PACKAGE
+    - *build_script

.gitlab/ci/doc.gitlab-ci.yml

@@ -0,0 +1,27 @@
+########################################
+# DOC
+########################################
+
+generate-helpers-doc:
+  stage: doc
+  image: "before-install"
+  needs: []
+  before_script:
+    - apt-get update -y && apt-get install git hub -y
+    - git config --global user.email "yunohost@yunohost.org"
+    - git config --global user.name "$GITHUB_USER"
+  script:
+    - cd doc
+    - python3 generate_helper_doc.py
+    - hub clone https://$GITHUB_TOKEN:x-oauth-basic@github.com/YunoHost/doc.git doc_repo
+    - cp helpers.md doc_repo/pages/04.contribute/04.packaging_apps/11.helpers/packaging_apps_helpers.md
+    - cd doc_repo
+    # replace ${CI_COMMIT_REF_NAME} with ${CI_COMMIT_TAG} ?
+    - hub checkout -b "${CI_COMMIT_REF_NAME}"
+    - hub commit -am "[CI] Helper for ${CI_COMMIT_REF_NAME}"
+    - hub pull-request -m "[CI] Helper for ${CI_COMMIT_REF_NAME}" -p # GITHUB_USER and GITHUB_TOKEN registered here https://gitlab.com/yunohost/yunohost/-/settings/ci_cd
+  artifacts:
+    paths:
+      - doc/helpers.md
+  only:
+    - tags

.gitlab/ci/install.gitlab-ci.yml

@@ -0,0 +1,29 @@
+.install-stage:
+  stage: install
+  needs:
+      - job: build-yunohost
+        artifacts: true
+      - job: build-ssowat
+        artifacts: true
+      - job: build-moulinette
+        artifacts: true
+
+########################################
+# INSTALL DEB
+########################################
+
+upgrade:
+  extends: .install-stage
+  image: "after-install"
+  script:
+    - apt-get update -o Acquire::Retries=3
+    - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ./$YNH_BUILD_DIR/*.deb
+
+
+install-postinstall:
+  extends: .install-stage
+  image: "before-install"
+  script:
+    - apt-get update -o Acquire::Retries=3
+    - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ./$YNH_BUILD_DIR/*.deb
+    - yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns --force-diskspace

.gitlab/ci/lint.gitlab-ci.yml

@@ -0,0 +1,57 @@
+########################################
+# LINTER
+########################################
+# later we must fix lint and format-check jobs and remove "allow_failure"
+
+---
+lint37:
+  stage: lint
+  image: "before-install"
+  needs: []
+  allow_failure: true
+  script:
+    - tox -e py37-lint
+
+invalidcode37:
+  stage: lint
+  image: "before-install"
+  needs: []
+  script:
+    - tox -e py37-invalidcode
+
+mypy:
+  stage: lint
+  image: "before-install"
+  needs: []
+  script:
+    - tox -e py37-mypy
+
+format-check:
+  stage: lint
+  image: "before-install"
+  allow_failure: true
+  needs: []
+  script:
+    - tox -e py37-black-check
+
+format-run:
+  stage: lint
+  image: "before-install"
+  needs: []
+  before_script:
+    - apt-get update -y && apt-get install git hub -y
+    - git config --global user.email "yunohost@yunohost.org"
+    - git config --global user.name "$GITHUB_USER"
+    - hub clone --branch ${CI_COMMIT_REF_NAME} "https://$GITHUB_TOKEN:x-oauth-basic@github.com/YunoHost/yunohost.git" github_repo
+    - cd github_repo
+  script:
+    # create a local branch that will overwrite distant one
+    - git checkout -b "ci-format-${CI_COMMIT_REF_NAME}" --no-track
+    - tox -e py37-black-run
+    - '[ $(git diff | wc -l) != 0 ] || exit 0'  # stop if there is nothing to commit
+    - git commit -am "[CI] Format code" || true
+    - git push -f origin "ci-format-${CI_COMMIT_REF_NAME}":"ci-format-${CI_COMMIT_REF_NAME}"
+    - hub pull-request -m "[CI] Format code" -b Yunohost:dev -p || true # GITHUB_USER and GITHUB_TOKEN registered here https://gitlab.com/yunohost/yunohost/-/settings/ci_cd
+  only:
+    refs:
+      - dev

.gitlab/ci/test.gitlab-ci.yml

@@ -0,0 +1,208 @@
+.install_debs: &install_debs
+    - apt-get update -o Acquire::Retries=3
+    - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ./$YNH_BUILD_DIR/*.deb
+
+.test-stage:
+  stage: tests
+  image: "after-install"
+  variables:
+    PYTEST_ADDOPTS: "--color=yes"
+  before_script:
+    - *install_debs
+  cache:
+    paths: 
+      - src/yunohost/tests/apps
+    key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG"
+  needs:
+      - job: build-yunohost
+        artifacts: true
+      - job: build-ssowat
+        artifacts: true
+      - job: build-moulinette
+        artifacts: true
+      - job: upgrade
+
+
+########################################
+# TESTS
+########################################
+
+full-tests:
+  stage: tests
+  image: "before-install"
+  variables:
+    PYTEST_ADDOPTS: "--color=yes"
+  before_script:
+    - *install_debs
+    - yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns --force-diskspace
+  script:
+    - python3 -m pytest --cov=yunohost tests/ src/yunohost/tests/ --junitxml=report.xml
+    - cd tests
+    - bash test_helpers.sh
+  needs:
+      - job: build-yunohost
+        artifacts: true
+      - job: build-ssowat
+        artifacts: true
+      - job: build-moulinette
+        artifacts: true
+  artifacts:
+    reports:
+      junit: report.xml
+
+test-i18n-keys:
+  extends: .test-stage
+  script:
+    - python3 -m pytest tests/test_i18n_keys.py
+  only:
+    changes:
+      - locales/en.json
+      - src/yunohost/*.py
+      - data/hooks/diagnosis/*.py
+
+test-translation-format-consistency:
+  extends: .test-stage
+  script:
+    - python3 -m pytest tests/test_translation_format_consistency.py
+  only:
+    changes:
+      - locales/*
+
+test-actionmap:
+  extends: .test-stage
+  script:
+    - python3 -m pytest tests/test_actionmap.py
+  only:
+    changes:
+      - data/actionsmap/*.yml
+
+test-helpers:
+  extends: .test-stage
+  script:
+    - cd tests
+    - bash test_helpers.sh
+  only:
+    changes:
+      - data/helpers.d/*
+
+test-domains:
+  extends: .test-stage
+  script:
+    - python3 -m pytest src/yunohost/tests/test_domains.py
+  only:
+    changes:
+      - src/yunohost/domain.py
+
+test-dns:
+  extends: .test-stage
+  script:
+    - python3 -m pytest src/yunohost/tests/test_dns.py
+  only:
+    changes:
+      - src/yunohost/dns.py
+      - src/yunohost/utils/dns.py
+
+test-apps:
+  extends: .test-stage
+  script:
+    - python3 -m pytest src/yunohost/tests/test_apps.py
+  only:
+    changes:
+      - src/yunohost/app.py
+
+test-appscatalog:
+  extends: .test-stage
+  script:
+    - python3 -m pytest src/yunohost/tests/test_appscatalog.py
+  only:
+    changes:
+      - src/yunohost/app.py
+
+test-appurl:
+  extends: .test-stage
+  script:
+    - python3 -m pytest src/yunohost/tests/test_appurl.py
+  only:
+    changes:
+      - src/yunohost/app.py
+
+test-questions:
+  extends: .test-stage
+  script:
+    - python3 -m pytest src/yunohost/tests/test_questions.py
+  only:
+    changes:
+      - src/yunohost/utils/config.py
+
+test-app-config:
+  extends: .test-stage
+  script:
+    - python3 -m pytest src/yunohost/tests/test_app_config.py
+  only:
+    changes:
+      - src/yunohost/app.py
+      - src/yunohost/utils/config.py
+
+test-changeurl:
+  extends: .test-stage
+  script:
+    - python3 -m pytest src/yunohost/tests/test_changeurl.py
+  only:
+    changes:
+      - src/yunohost/app.py
+
+test-backuprestore:
+  extends: .test-stage
+  script:
+    - python3 -m pytest src/yunohost/tests/test_backuprestore.py
+  only:
+    changes:
+      - src/yunohost/backup.py
+
+test-permission:
+  extends: .test-stage
+  script:
+    - python3 -m pytest src/yunohost/tests/test_permission.py
+  only:
+    changes:
+      - src/yunohost/permission.py
+
+test-settings:
+  extends: .test-stage
+  script:
+    - python3 -m pytest src/yunohost/tests/test_settings.py
+  only:
+    changes:
+      - src/yunohost/settings.py
+
+test-user-group:
+  extends: .test-stage
+  script:
+    - python3 -m pytest src/yunohost/tests/test_user-group.py
+  only:
+    changes:
+      - src/yunohost/user.py
+
+test-regenconf:
+  extends: .test-stage
+  script:
+    - python3 -m pytest src/yunohost/tests/test_regenconf.py
+  only:
+    changes:
+      - src/yunohost/regenconf.py
+
+test-service:
+  extends: .test-stage
+  script:
+    - python3 -m pytest src/yunohost/tests/test_service.py
+  only:
+    changes:
+      - src/yunohost/service.py
+
+test-ldapauth:
+  extends: .test-stage
+  script:
+    - python3 -m pytest src/yunohost/tests/test_ldapauth.py
+  only:
+    changes:
+      - src/yunohost/authenticators/*.py

.gitlab/ci/translation.gitlab-ci.yml

@@ -0,0 +1,29 @@
+########################################
+# TRANSLATION
+########################################
+
+autofix-translated-strings:
+  stage: translation
+  image: "before-install"
+  needs: []
+  before_script:
+    - apt-get update -y && apt-get install git hub -y
+    - git config --global user.email "yunohost@yunohost.org"
+    - git config --global user.name "$GITHUB_USER"
+    - git remote set-url origin https://$GITHUB_TOKEN:x-oauth-basic@github.com/YunoHost/yunohost.git
+  script:
+    - cd tests # Maybe move this script location to another folder?
+    # create a local branch that will overwrite distant one
+    - git checkout -b "ci-autofix-translated-strings-${CI_COMMIT_REF_NAME}" --no-track
+    - python3 remove_stale_translated_strings.py
+    - python3 autofix_locale_format.py
+    - python3 reformat_locales.py
+    - '[ $(git diff -w | wc -l) != 0 ] || exit 0'  # stop if there is nothing to commit
+    - git commit -am "[CI] Reformat / remove stale translated strings" || true
+    - git push -f origin "HEAD":"ci-remove-stale-translated-strings-${CI_COMMIT_REF_NAME}"
+    - hub pull-request -m "[CI] Reformat / remove stale translated strings" -b Yunohost:dev -p || true # GITHUB_USER and GITHUB_TOKEN registered here https://gitlab.com/yunohost/yunohost/-/settings/ci_cd
+  only:
+    variables:
+      - $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
+    changes:
+      - locales/*

.travis.yml

@@ -1,5 +0,0 @@
-language: python
-install: "pip install pytest pyyaml"
-python:
-    - "2.7"
-script: "py.test tests"

README.md

@@ -1,42 +1,43 @@
-# YunoHost core
+<p align="center">
+    <img alt="YunoHost" src="https://raw.githubusercontent.com/YunoHost/doc/master/images/logo_roundcorner.png" width="100px" />
+</p>
 
-- [YunoHost project website](https://yunohost.org)
+<h1 align="center">YunoHost</h1>
 
-This repository is the core of YunoHost code.
+<div align="center">
 
-<a href="https://translate.yunohost.org/engage/yunohost/?utm_source=widget">
-<img src="https://translate.yunohost.org/widgets/yunohost/-/287x66-white.png" alt="Translation status" />
-</a>
+[![Build status](https://shields.io/gitlab/pipeline/yunohost/yunohost/dev)](https://gitlab.com/yunohost/yunohost/-/pipelines)
+[![GitHub license](https://img.shields.io/github/license/YunoHost/yunohost)](https://github.com/YunoHost/yunohost/blob/dev/LICENSE)
+[![Mastodon Follow](https://img.shields.io/mastodon/follow/28084)](https://mastodon.social/@yunohost)
 
-## Issues
-- [Please report issues on YunoHost bugtracker](https://github.com/YunoHost/issues).
+</div>
 
-## Contribute
-- You can develop on this repository using [ynh-dev tool](https://github.com/YunoHost/ynh-dev) with `use-git`  sub-command.
-- On this repository we are [following this workflow](https://yunohost.org/#/build_system_en): `stable <— testing <— branch`.
-- Note: if you modify python scripts, you will have to modifiy the actions map.
+YunoHost is an operating system aiming to simplify as much as possible the administration of a server.
 
-## Repository content
-- [YunoHost core Python 2.7 scripts](https://github.com/YunoHost/yunohost/tree/stable/src/yunohost).
-- [An actionsmap](https://github.com/YunoHost/yunohost/blob/stable/data/actionsmap/yunohost.yml) used by moulinette.
-- [Services configuration templates](https://github.com/YunoHost/yunohost/tree/stable/data/templates).
-- [Hooks](https://github.com/YunoHost/yunohost/tree/stable/data/hooks).
-- [Locales](https://github.com/YunoHost/yunohost/tree/stable/locales) for translations of `yunohost` command.
-- [Shell helpers](https://github.com/YunoHost/yunohost/tree/stable/data/helpers.d) for [application packaging](https://yunohost.org/#/packaging_apps_helpers_en).
-- [Modules for the XMPP server Metronome](https://github.com/YunoHost/yunohost/tree/stable/lib/metronome/modules).
-- [Debian files](https://github.com/YunoHost/yunohost/tree/stable/debian) for package creation.
+This repository corresponds to the core code of YunoHost, mainly written in Python and Bash.
 
-## How does it work?
-- Python core scripts are accessible through two interfaces thanks to the [moulinette framework](https://github.com/YunoHost/moulinette):
- - [CLI](https://en.wikipedia.org/wiki/Command-line_interface) for `yunohost` command.
- - [API](https://en.wikipedia.org/wiki/Application_programming_interface) for [web administration module](https://github.com/YunoHost/yunohost-admin) (other modules could be implemented).
-- You can find more details about how YunoHost works on this [documentation (in french)](https://yunohost.org/#/package_list_fr).
+- [Project features](https://yunohost.org/#/whatsyunohost)
+- [Project website](https://yunohost.org)
+- [Install documentation](https://yunohost.org/install)
+- [Issue tracker](https://github.com/YunoHost/issues)
 
-## Dependencies
-- [Python 2.7](https://www.python.org/download/releases/2.7)
-- [Moulinette](https://github.com/YunoHost/moulinette)
-- [Bash](https://www.gnu.org/software/bash/bash.html)
-- [Debian Jessie](https://www.debian.org/releases/jessie)
+# Screenshots
+
+Webadmin ([Yunohost-Admin](https://github.com/YunoHost/yunohost-admin)) | Single sign-on user portal ([SSOwat](https://github.com/YunoHost/ssowat))
+--- |  ---
+![](https://raw.githubusercontent.com/YunoHost/doc/master/images/webadmin.png) | ![](https://raw.githubusercontent.com/YunoHost/doc/master/images/user_panel.png)
+
+
+## Contributing
+
+- You can learn how to get started with developing on YunoHost by reading [this piece of documentation](https://yunohost.org/dev).
+- Come chat with us on the [dev chatroom](https://yunohost.org/#/chat_rooms) !
+- You can help translate YunoHost on our [translation platform](https://translate.yunohost.org/engage/yunohost/?utm_source=widget)
+
+<p align="center">
+<img src="https://translate.yunohost.org/widgets/yunohost/-/core/horizontal-auto.svg" alt="Translation status" />
+</p>
 
 ## License
-As [other components of YunoHost core code](https://yunohost.org/#/faq_en), this repository is under GNU AGPL v.3 license.
+
+As [other components of YunoHost](https://yunohost.org/#/faq_en), this repository is licensed under GNU AGPL v3.

bin/yunohost

@@ -1,74 +1,32 @@
-#! /usr/bin/python
+#! /usr/bin/python3
 # -*- coding: utf-8 -*-
 
 import os
 import sys
 import argparse
 
-# Either we are in a development environment or not
-IN_DEVEL = False
+sys.path.insert(0, "/usr/lib/moulinette/")
+import yunohost
 
-# Level for which loggers will log
-LOGGERS_LEVEL = 'DEBUG'
-TTY_LOG_LEVEL = 'INFO'
-
-# Handlers that will be used by loggers
-#  - file: log to the file LOG_DIR/LOG_FILE
-#  - tty: log to current tty
-LOGGERS_HANDLERS = ['file', 'tty']
-
-# Directory and file to be used by logging
-LOG_DIR = '/var/log/yunohost'
-LOG_FILE = 'yunohost-cli.log'
-
-# Check and load - as needed - development environment
-if not __file__.startswith('/usr/'):
-    IN_DEVEL = True
-if IN_DEVEL:
-    basedir = os.path.abspath('%s/../' % os.path.dirname(__file__))
-    if os.path.isdir(os.path.join(basedir, 'moulinette')):
-        sys.path.insert(0, basedir)
-    LOG_DIR = os.path.join(basedir, 'log')
-
-
-import moulinette
-from moulinette.actionsmap import ActionsMap
-from moulinette.interfaces.cli import colorize, get_locale
-
-
-# Initialization & helpers functions -----------------------------------
-
-def _die(message, title='Error:'):
-    """Print error message and exit"""
-    print('%s %s' % (colorize(title, 'red'), message))
-    sys.exit(1)
 
 def _parse_cli_args():
     """Parse additional arguments for the cli"""
     parser = argparse.ArgumentParser(add_help=False)
-    parser.add_argument('--no-cache',
-        action='store_false', default=True, dest='use_cache',
-        help="Don't use actions map cache",
-    )
     parser.add_argument('--output-as',
         choices=['json', 'plain', 'none'], default=None,
-        help="Output result in another format",
+        help="Output result in another format"
     )
     parser.add_argument('--debug',
         action='store_true', default=False,
-        help="Log and print debug messages",
+        help="Log and print debug messages"
     )
     parser.add_argument('--quiet',
         action='store_true', default=False,
-        help="Don't produce any output",
+        help="Don't produce any output"
     )
     parser.add_argument('--timeout',
         type=int, default=None,
-        help="Number of seconds before this command will timeout because it can't acquire the lock (meaning that another command is currently running), by default there is no timeout and the command will wait until it can get the lock",
-    )
-    parser.add_argument('--admin-password',
-        default=None, dest='password', metavar='PASSWORD',
-        help="The admin password to use to authenticate",
+        help="Number of seconds before this command will timeout because it can't acquire the lock (meaning that another command is currently running), by default there is no timeout and the command will wait until it can get the lock"
     )
     # deprecated arguments
     parser.add_argument('--plain',
@@ -88,129 +46,28 @@ def _parse_cli_args():
 
     return (parser, opts, args)
 
-def _init_moulinette(debug=False, quiet=False):
-    """Configure logging and initialize the moulinette"""
-    # Define loggers handlers
-    handlers = set(LOGGERS_HANDLERS)
-    if quiet and 'tty' in handlers:
-        handlers.remove('tty')
-    elif 'tty' not in handlers:
-        handlers.append('tty')
-
-    root_handlers = set(handlers)
-    if not debug and 'tty' in root_handlers:
-        root_handlers.remove('tty')
-
-    # Define loggers level
-    level = LOGGERS_LEVEL
-    tty_level = TTY_LOG_LEVEL
-    if debug:
-        tty_level = 'DEBUG'
-
-    # Custom logging configuration
-    logging = {
-        'version': 1,
-        'disable_existing_loggers': True,
-        'formatters': {
-            'tty-debug': {
-                'format': '%(relativeCreated)-4d %(fmessage)s'
-            },
-            'precise': {
-                'format': '%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s'
-            },
-        },
-        'filters': {
-            'action': {
-                '()': 'moulinette.utils.log.ActionFilter',
-            },
-        },
-        'handlers': {
-            'tty': {
-                'level': tty_level,
-                'class': 'moulinette.interfaces.cli.TTYHandler',
-                'formatter': 'tty-debug' if debug else '',
-            },
-            'file': {
-                'class': 'logging.FileHandler',
-                'formatter': 'precise',
-                'filename': '%s/%s' % (LOG_DIR, LOG_FILE),
-                'filters': ['action'],
-            },
-        },
-        'loggers': {
-            'yunohost': {
-                'level': level,
-                'handlers': handlers,
-                'propagate': False,
-            },
-            'moulinette': {
-                'level': level,
-                'handlers': [],
-                'propagate': True,
-            },
-            'moulinette.interface': {
-                'level': level,
-                'handlers': handlers,
-                'propagate': False,
-            },
-        },
-        'root': {
-            'level': level,
-            'handlers': root_handlers,
-        },
-    }
-
-    # Create log directory
-    if not os.path.isdir(LOG_DIR):
-        try:
-            os.makedirs(LOG_DIR, 0750)
-        except os.error as e:
-            _die(str(e))
-
-    # Initialize moulinette
-    moulinette.init(logging_config=logging, _from_source=IN_DEVEL)
-
-def _retrieve_namespaces():
-    """Return the list of namespaces to load"""
-    ret = ['yunohost']
-    for n in ActionsMap.get_namespaces():
-        # Append YunoHost modules
-        if n.startswith('ynh_'):
-            ret.append(n)
-    return ret
 
+# Stupid PATH management because sometimes (e.g. some cron job) PATH is only /usr/bin:/bin ...
+default_path = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+if os.environ["PATH"] != default_path:
+    os.environ["PATH"] = default_path + ":" + os.environ["PATH"]
 
 # Main action ----------------------------------------------------------
 
 if __name__ == '__main__':
     if os.geteuid() != 0:
-        # since moulinette isn't initialized, we can't use m18n here
-        sys.stderr.write("\033[1;31mError:\033[0m yunohost command must be " \
+        sys.stderr.write("\033[1;31mError:\033[0m yunohost command must be "
                          "run as root or with sudo.\n")
         sys.exit(1)
 
     parser, opts, args = _parse_cli_args()
-    _init_moulinette(opts.debug, opts.quiet)
-
-    # Check that YunoHost is installed
-    if not os.path.isfile('/etc/yunohost/installed') and \
-       (len(args) < 2 or (args[0] +' '+ args[1] != 'tools postinstall' and \
-                          args[0] +' '+ args[1] != 'backup restore' and \
-                          args[0] +' '+ args[1] != 'log display')):
-
-        from moulinette import m18n
-        # Init i18n
-        m18n.load_namespace('yunohost')
-        m18n.set_locale(get_locale())
-
-        # Print error and exit
-        _die(m18n.n('yunohost_not_installed'), m18n.g('error'))
 
     # Execute the action
-    ret = moulinette.cli(
-        _retrieve_namespaces(), args,
-        use_cache=opts.use_cache, output_as=opts.output_as,
-        password=opts.password, parser_kwargs={'top_parser': parser},
+    yunohost.cli(
+        debug=opts.debug,
+        quiet=opts.quiet,
+        output_as=opts.output_as,
         timeout=opts.timeout,
+        args=args,
+        parser=parser
     )
-    sys.exit(ret)

bin/yunohost-api

@@ -1,52 +1,16 @@
-#! /usr/bin/python
+#! /usr/bin/python3
 # -*- coding: utf-8 -*-
 
-import os
 import sys
 import argparse
 
-# Either we are in a development environment or not
-IN_DEVEL = False
+sys.path.insert(0, "/usr/lib/moulinette/")
+import yunohost
 
 # Default server configuration
 DEFAULT_HOST = 'localhost'
 DEFAULT_PORT = 6787
 
-# Level for which loggers will log
-LOGGERS_LEVEL = 'DEBUG'
-API_LOGGER_LEVEL = 'INFO'
-
-# Handlers that will be used by loggers
-#  - file: log to the file LOG_DIR/LOG_FILE
-#  - api: serve logs through the api
-#  - console: log to stderr
-LOGGERS_HANDLERS = ['file', 'api']
-
-# Directory and file to be used by logging
-LOG_DIR = '/var/log/yunohost'
-LOG_FILE = 'yunohost-api.log'
-
-# Check and load - as needed - development environment
-if not __file__.startswith('/usr/'):
-    IN_DEVEL = True
-if IN_DEVEL:
-    basedir = os.path.abspath('%s/../' % os.path.dirname(__file__))
-    if os.path.isdir(os.path.join(basedir, 'moulinette')):
-        sys.path.insert(0, basedir)
-    LOG_DIR = os.path.join(basedir, 'log')
-
-
-import moulinette
-from moulinette.actionsmap import ActionsMap
-from moulinette.interfaces.cli import colorize
-
-
-# Initialization & helpers functions -----------------------------------
-
-def _die(message, title='Error:'):
-    """Print error message and exit"""
-    print('%s %s' % (colorize(title, 'red'), message))
-    sys.exit(1)
 
 def _parse_api_args():
     """Parse main arguments for the api"""
@@ -62,149 +26,19 @@ def _parse_api_args():
         action='store', default=DEFAULT_PORT, type=int,
         help="Port to listen on (default: %d)" % DEFAULT_PORT,
     )
-    srv_group.add_argument('--no-websocket',
-        action='store_true', default=True, dest='use_websocket',
-        help="Serve without WebSocket support, used to handle "
-        "asynchronous responses such as the messages",
-    )
     glob_group = parser.add_argument_group('global arguments')
-    glob_group.add_argument('--no-cache',
-        action='store_false', default=True, dest='use_cache',
-        help="Don't use actions map cache",
-    )
     glob_group.add_argument('--debug',
         action='store_true', default=False,
         help="Set log level to DEBUG",
     )
-    glob_group.add_argument('--verbose',
-        action='store_true', default=False,
-        help="Be verbose in the output",
-    )
     glob_group.add_argument('--help',
         action='help', help="Show this help message and exit",
     )
 
     return parser.parse_args()
 
-def _init_moulinette(use_websocket=True, debug=False, verbose=False):
-    """Configure logging and initialize the moulinette"""
-    # Define loggers handlers
-    handlers = set(LOGGERS_HANDLERS)
-    if not use_websocket and 'api' in handlers:
-        handlers.remove('api')
-    if verbose and 'console' not in handlers:
-        handlers.add('console')
-    root_handlers = handlers - set(['api'])
-
-    # Define loggers level
-    level = LOGGERS_LEVEL
-    api_level = API_LOGGER_LEVEL
-    if debug:
-        level = 'DEBUG'
-        api_level = 'DEBUG'
-
-    # Custom logging configuration
-    logging = {
-        'version': 1,
-        'disable_existing_loggers': True,
-        'formatters': {
-            'console': {
-                'format': '%(relativeCreated)-5d %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s'
-            },
-            'precise': {
-                'format': '%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s'
-            },
-        },
-        'filters': {
-            'action': {
-                '()': 'moulinette.utils.log.ActionFilter',
-            },
-        },
-        'handlers': {
-            'api': {
-                'level': api_level,
-                'class': 'moulinette.interfaces.api.APIQueueHandler',
-            },
-            'file': {
-                'class': 'logging.handlers.WatchedFileHandler',
-                'formatter': 'precise',
-                'filename': '%s/%s' % (LOG_DIR, LOG_FILE),
-                'filters': ['action'],
-            },
-            'console': {
-                'class': 'logging.StreamHandler',
-                'formatter': 'console',
-                'stream': 'ext://sys.stdout',
-                'filters': ['action'],
-            },
-        },
-        'loggers': {
-            'yunohost': {
-                'level': level,
-                'handlers': handlers,
-                'propagate': False,
-            },
-            'moulinette': {
-                'level': level,
-                'handlers': [],
-                'propagate': True,
-            },
-            'gnupg': {
-                'level': 'INFO',
-                'handlers': [],
-                'propagate': False,
-            },
-        },
-        'root': {
-            'level': level,
-            'handlers': root_handlers,
-        },
-    }
-
-    # Create log directory
-    if not os.path.isdir(LOG_DIR):
-        try:
-            os.makedirs(LOG_DIR, 0750)
-        except os.error as e:
-            _die(str(e))
-
-    # Initialize moulinette
-    moulinette.init(logging_config=logging, _from_source=IN_DEVEL)
-
-def _retrieve_namespaces():
-    """Return the list of namespaces to load"""
-    ret = ['yunohost']
-    for n in ActionsMap.get_namespaces():
-        # Append YunoHost modules
-        if n.startswith('ynh_'):
-            ret.append(n)
-    return ret
-
-
-# Callbacks for additional routes --------------------------------------
-
-def is_installed():
-    """
-    Check whether YunoHost is installed or not
-
-    """
-    installed = False
-    if os.path.isfile('/etc/yunohost/installed'):
-        installed = True
-    return { 'installed': installed }
-
-
-# Main action ----------------------------------------------------------
 
 if __name__ == '__main__':
     opts = _parse_api_args()
-    _init_moulinette(opts.use_websocket, opts.debug, opts.verbose)
-
     # Run the server
-    ret = moulinette.api(
-        _retrieve_namespaces(),
-        host=opts.host, port=opts.port, routes={
-            ('GET', '/installed'): is_installed,
-        }, use_cache=opts.use_cache, use_websocket=opts.use_websocket
-    )
-    sys.exit(ret)
+    yunohost.api(debug=opts.debug, host=opts.host, port=opts.port)

bin/yunomdns

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+
+"""
+Pythonic declaration of mDNS .local domains for YunoHost
+"""
+
+import subprocess
+import re
+import sys
+import yaml
+
+import socket
+from time import sleep
+from typing import List, Dict
+
+from zeroconf import Zeroconf, ServiceInfo
+
+# Helper command taken from Moulinette
+def check_output(args, stderr=subprocess.STDOUT, shell=True, **kwargs):
+    """Run command with arguments and return its output as a byte string
+    Overwrite some of the arguments to capture standard error in the result
+    and use shell by default before calling subprocess.check_output.
+    """
+    return (
+        subprocess.check_output(args, stderr=stderr, shell=shell, **kwargs)
+        .decode("utf-8")
+        .strip()
+    )
+
+# Helper command taken from Moulinette
+def _extract_inet(string, skip_netmask=False, skip_loopback=True):
+    """
+    Extract IP addresses (v4 and/or v6) from a string limited to one
+    address by protocol
+
+    Keyword argument:
+        string -- String to search in
+        skip_netmask -- True to skip subnet mask extraction
+        skip_loopback -- False to include addresses reserved for the
+            loopback interface
+
+    Returns:
+        A dict of {protocol: address} with protocol one of 'ipv4' or 'ipv6'
+
+    """
+    ip4_pattern = (
+        r"((25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}"
+    )
+    ip6_pattern = r"(((?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?)::?((?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?)"
+    ip4_pattern += r"/[0-9]{1,2})" if not skip_netmask else ")"
+    ip6_pattern += r"/[0-9]{1,3})" if not skip_netmask else ")"
+    result = {}
+
+    for m in re.finditer(ip4_pattern, string):
+        addr = m.group(1)
+        if skip_loopback and addr.startswith("127."):
+            continue
+
+        # Limit to only one result
+        result["ipv4"] = addr
+        break
+
+    for m in re.finditer(ip6_pattern, string):
+        addr = m.group(1)
+        if skip_loopback and addr == "::1":
+            continue
+
+        # Limit to only one result
+        result["ipv6"] = addr
+        break
+
+    return result
+
+# Helper command taken from Moulinette
+def get_network_interfaces():
+
+    # Get network devices and their addresses (raw infos from 'ip addr')
+    devices_raw = {}
+    output = check_output("ip --brief a").split("\n")
+    for line in output:
+        line = line.split()
+        iname = line[0]
+        ips = ' '.join(line[2:])
+
+        devices_raw[iname] = ips
+
+    # Parse relevant informations for each of them
+    devices = {
+        name: _extract_inet(addrs)
+        for name, addrs in devices_raw.items()
+        if name != "lo"
+    }
+
+    return devices
+
+if __name__ == '__main__':
+
+    ###
+    #  CONFIG
+    ###
+
+    with open('/etc/yunohost/mdns.yml', 'r') as f:
+        config = yaml.safe_load(f) or {}
+    updated = False
+
+    required_fields = ["interfaces", "domains"]
+    missing_fields = [field for field in required_fields if field not in config]
+
+    if missing_fields:
+        print("The fields %s are required" % ', '.join(missing_fields))
+
+    if config['interfaces'] is None:
+        print('No interface listed for broadcast.')
+        sys.exit(0)
+
+    if 'yunohost.local' not in config['domains']:
+        config['domains'].append('yunohost.local')
+
+    zcs = {}
+    interfaces = get_network_interfaces()
+    for interface in config['interfaces']:
+        infos = [] # List of ServiceInfo objects, to feed Zeroconf
+        ips = [] # Human-readable IPs
+        b_ips = [] # Binary-convered IPs
+
+        ipv4 = interfaces[interface]['ipv4'].split('/')[0]
+        if ipv4:
+            ips.append(ipv4)
+            b_ips.append(socket.inet_pton(socket.AF_INET, ipv4))
+
+        ipv6 = interfaces[interface]['ipv6'].split('/')[0]
+        if ipv6:
+            ips.append(ipv6)
+            b_ips.append(socket.inet_pton(socket.AF_INET6, ipv6))
+
+        # If at least one IP is listed
+        if ips:
+            # Create a Zeroconf object, and store the ServiceInfos
+            zc = Zeroconf(interfaces=ips)
+            zcs[zc]=[]
+            for d in config['domains']:
+                d_domain=d.replace('.local','')
+                if '.' in d_domain:
+                    print(d_domain+'.local: subdomains are not supported.')
+                else:
+                    # Create a ServiceInfo object for each .local domain
+                    zcs[zc].append(ServiceInfo(
+                                   type_='_device-info._tcp.local.',
+                                   name=interface+': '+d_domain+'._device-info._tcp.local.',
+                                   addresses=b_ips,
+                                   port=80,
+                                   server=d+'.',
+                              ))
+                    print('Adding '+d+' with addresses '+str(ips)+' on interface '+interface)
+
+    # Run registration
+    print("Registering...")
+    for zc, infos in zcs.items():
+        for info in infos:
+            zc.register_service(info)
+
+    try:
+        print("Registered. Press Ctrl+C or stop service to stop.")
+        while True:
+            sleep(1)
+    except KeyboardInterrupt:
+        pass
+    finally:
+        print("Unregistering...")
+        for zc, infos in zcs.items():
+            for info in infos:
+                zc.unregister_service(info)
+            zc.close()

bin/yunopaste

@@ -34,7 +34,7 @@ Haste server.
 
 For example, to paste the output of the YunoHost diagnosis, you
 can simply execute the following:
-  yunohost tools diagnosis | ${0}
+  yunohost diagnosis show | ${0}
 
 It will return the URL where you can access the pasted data.
 

bin/yunoprompt

@@ -1,5 +1,9 @@
 #!/bin/bash
 
+# Fetch x509 fingerprint
+x509_fingerprint=$(openssl x509  -in /etc/yunohost/certs/yunohost.org/crt.pem -noout -fingerprint -sha256 | cut -d= -f2)
+
+
 # Fetch SSH fingerprints
 i=0
 for key in $(ls /etc/ssh/ssh_host_{ed25519,rsa,ecdsa}_key.pub 2> /dev/null) ; do
@@ -39,21 +43,21 @@ LOGO_AND_FINGERPRINTS=$(cat << EOF
 
 $LOGO
 
- IP: ${local_ip}
+ Local IP: ${local_ip:-(no ip detected?)}
+ Local SSL CA X509 fingerprint:
+   ${x509_fingerprint}
  SSH fingerprints:
  ${fingerprint[0]}
  ${fingerprint[1]}
  ${fingerprint[2]}
- ${fingerprint[3]}
- ${fingerprint[4]}
 
 EOF
 )
 
-if [[ -f /etc/yunohost/installed ]]
+echo "$LOGO_AND_FINGERPRINTS" > /etc/issue
+
+if [[ ! -f /etc/yunohost/installed ]]
 then
-    echo "$LOGO_AND_FINGERPRINTS" > /etc/issue
-else
     chvt 2
 
     # Formatting
@@ -62,19 +66,19 @@ else
     echo "$LOGO_AND_FINGERPRINTS"
     cat << EOF
 ===============================================================================
-You should now proceed with Yunohost post-installation. This is where you will
-be asked for :
-  - the main domain of your server ;
+You should now proceed with YunoHost post-installation. This is where you will
+be asked for:
+  - the main domain of your server;
   - the administration password.
 
-You can perform this step :
-  - from your web browser, by accessing : ${local_ip}
+You can perform this step:
+  - from your web browser, by accessing: https://yunohost.local/ or ${local_ip}
   - or in this terminal by answering 'yes' to the following question
 
 If this is your first time with YunoHost, it is strongly recommended to take
 time to read the administator documentation and in particular the sections
 'Finalizing your setup' and 'Getting to know YunoHost'. It is available at
-the following URL : https://yunohost.org/admindoc
+the following URL: https://yunohost.org/admindoc
 ===============================================================================
 EOF
 

data/actionsmap/yunohost_completion.py

@@ -3,7 +3,7 @@ Simple automated generation of a bash_completion file
 for yunohost command from the actionsmap.
 
 Generates a bash completion file assuming the structure
-`yunohost domain action`
+`yunohost category action`
 adds `--help` at the end if one presses [tab] again.
 
 author: Christophe Vuillot
@@ -12,75 +12,148 @@ import os
 import yaml
 
 THIS_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-ACTIONSMAP_FILE = THIS_SCRIPT_DIR + '/yunohost.yml'
-BASH_COMPLETION_FILE = THIS_SCRIPT_DIR + '/../bash-completion.d/yunohost'
+ACTIONSMAP_FILE = THIS_SCRIPT_DIR + "/yunohost.yml"
+os.system(f"mkdir {THIS_SCRIPT_DIR}/../bash-completion.d")
+BASH_COMPLETION_FILE = THIS_SCRIPT_DIR + "/../bash-completion.d/yunohost"
 
-with open(ACTIONSMAP_FILE, 'r') as stream:
 
-    # Getting the dictionary containning what actions are possible per domain
-    OPTION_TREE = yaml.load(stream)
-    DOMAINS = [str for str in OPTION_TREE.keys() if not str.startswith('_')]
-    DOMAINS_STR = '"{}"'.format(' '.join(DOMAINS))
+def get_dict_actions(OPTION_SUBTREE, category):
+    ACTIONS = [
+        action
+        for action in OPTION_SUBTREE[category]["actions"].keys()
+        if not action.startswith("_")
+    ]
+    ACTIONS_STR = "{}".format(" ".join(ACTIONS))
+
+    DICT = {"actions_str": ACTIONS_STR}
+
+    return DICT
+
+
+with open(ACTIONSMAP_FILE, "r") as stream:
+
+    # Getting the dictionary containning what actions are possible per category
+    OPTION_TREE = yaml.safe_load(stream)
+
+    CATEGORY = [
+        category for category in OPTION_TREE.keys() if not category.startswith("_")
+    ]
+
+    CATEGORY_STR = "{}".format(" ".join(CATEGORY))
     ACTIONS_DICT = {}
-    for domain in DOMAINS:
-        ACTIONS = [str for str in OPTION_TREE[domain]['actions'].keys()
-                   if not str.startswith('_')]
-        ACTIONS_STR = '"{}"'.format(' '.join(ACTIONS))
-        ACTIONS_DICT[domain] = ACTIONS_STR
+    for category in CATEGORY:
+        ACTIONS_DICT[category] = get_dict_actions(OPTION_TREE, category)
 
-    with open(BASH_COMPLETION_FILE, 'w') as generated_file:
+        ACTIONS_DICT[category]["subcategories"] = {}
+        ACTIONS_DICT[category]["subcategories_str"] = ""
+
+        if "subcategories" in OPTION_TREE[category].keys():
+            SUBCATEGORIES = [
+                subcategory
+                for subcategory in OPTION_TREE[category]["subcategories"].keys()
+            ]
+
+            SUBCATEGORIES_STR = "{}".format(" ".join(SUBCATEGORIES))
+
+            ACTIONS_DICT[category]["subcategories_str"] = SUBCATEGORIES_STR
+
+            for subcategory in SUBCATEGORIES:
+                ACTIONS_DICT[category]["subcategories"][subcategory] = get_dict_actions(
+                    OPTION_TREE[category]["subcategories"], subcategory
+                )
+
+    with open(BASH_COMPLETION_FILE, "w") as generated_file:
 
         # header of the file
-        generated_file.write('#\n')
-        generated_file.write('# completion for yunohost\n')
-        generated_file.write('# automatically generated from the actionsmap\n')
-        generated_file.write('#\n\n')
+        generated_file.write("#\n")
+        generated_file.write("# completion for yunohost\n")
+        generated_file.write("# automatically generated from the actionsmap\n")
+        generated_file.write("#\n\n")
 
         # Start of the completion function
-        generated_file.write('_yunohost()\n')
-        generated_file.write('{\n')
+        generated_file.write("_yunohost()\n")
+        generated_file.write("{\n")
 
         # Defining local variable for previously and currently typed words
-        generated_file.write('\tlocal cur prev opts narg\n')
-        generated_file.write('\tCOMPREPLY=()\n\n')
-        generated_file.write('\t# the number of words already typed\n')
-        generated_file.write('\tnarg=${#COMP_WORDS[@]}\n\n')
-        generated_file.write('\t# the current word being typed\n')
+        generated_file.write("\tlocal cur prev opts narg\n")
+        generated_file.write("\tCOMPREPLY=()\n\n")
+        generated_file.write("\t# the number of words already typed\n")
+        generated_file.write("\tnarg=${#COMP_WORDS[@]}\n\n")
+        generated_file.write("\t# the current word being typed\n")
         generated_file.write('\tcur="${COMP_WORDS[COMP_CWORD]}"\n\n')
-        generated_file.write('\t# the last typed word\n')
-        generated_file.write('\tprev="${COMP_WORDS[COMP_CWORD-1]}"\n\n')
 
-        # If one is currently typing a domain then match with the domain list
-        generated_file.write('\t# If one is currently typing a domain,\n')
-        generated_file.write('\t# match with domains\n')
-        generated_file.write('\tif [[ $narg == 2 ]]; then\n')
-        generated_file.write('\t\topts={}\n'.format(DOMAINS_STR))
-        generated_file.write('\tfi\n\n')
+        # If one is currently typing a category then match with the category list
+        generated_file.write("\t# If one is currently typing a category,\n")
+        generated_file.write("\t# match with categorys\n")
+        generated_file.write("\tif [[ $narg == 2 ]]; then\n")
+        generated_file.write('\t\topts="{}"\n'.format(CATEGORY_STR))
+        generated_file.write("\tfi\n\n")
 
         # If one is currently typing an action then match with the action list
-        # of the previously typed domain
-        generated_file.write('\t# If one already typed a domain,\n')
-        generated_file.write('\t# match the actions of that domain\n')
-        generated_file.write('\tif [[ $narg == 3 ]]; then\n')
-        for domain in DOMAINS:
-            generated_file.write('\t\tif [[ $prev == "{}" ]]; then\n'.format(domain))
-            generated_file.write('\t\t\topts={}\n'.format(ACTIONS_DICT[domain]))
-            generated_file.write('\t\tfi\n')
-        generated_file.write('\tfi\n\n')
+        # of the previously typed category
+        generated_file.write("\t# If one already typed a category,\n")
+        generated_file.write(
+            "\t# match the actions or the subcategories of that category\n"
+        )
+        generated_file.write("\tif [[ $narg == 3 ]]; then\n")
+        generated_file.write("\t\t# the category typed\n")
+        generated_file.write('\t\tcategory="${COMP_WORDS[1]}"\n\n')
+        for category in CATEGORY:
+            generated_file.write(
+                '\t\tif [[ $category == "{}" ]]; then\n'.format(category)
+            )
+            generated_file.write(
+                '\t\t\topts="{} {}"\n'.format(
+                    ACTIONS_DICT[category]["actions_str"],
+                    ACTIONS_DICT[category]["subcategories_str"],
+                )
+            )
+            generated_file.write("\t\tfi\n")
+        generated_file.write("\tfi\n\n")
 
-        # If both domain and action have been typed or the domain
+        generated_file.write("\t# If one already typed an action or a subcategory,\n")
+        generated_file.write("\t# match the actions of that subcategory\n")
+        generated_file.write("\tif [[ $narg == 4 ]]; then\n")
+        generated_file.write("\t\t# the category typed\n")
+        generated_file.write('\t\tcategory="${COMP_WORDS[1]}"\n\n')
+        generated_file.write("\t\t# the action or the subcategory typed\n")
+        generated_file.write('\t\taction_or_subcategory="${COMP_WORDS[2]}"\n\n')
+        for category in CATEGORY:
+            if len(ACTIONS_DICT[category]["subcategories"]):
+                generated_file.write(
+                    '\t\tif [[ $category == "{}" ]]; then\n'.format(category)
+                )
+                for subcategory in ACTIONS_DICT[category]["subcategories"]:
+                    generated_file.write(
+                        '\t\t\tif [[ $action_or_subcategory == "{}" ]]; then\n'.format(
+                            subcategory
+                        )
+                    )
+                    generated_file.write(
+                        '\t\t\t\topts="{}"\n'.format(
+                            ACTIONS_DICT[category]["subcategories"][subcategory][
+                                "actions_str"
+                            ]
+                        )
+                    )
+                    generated_file.write("\t\t\tfi\n")
+                generated_file.write("\t\tfi\n")
+        generated_file.write("\tfi\n\n")
+
+        # If both category and action have been typed or the category
         # was not recognized propose --help (only once)
-        generated_file.write('\t# If no options were found propose --help\n')
+        generated_file.write("\t# If no options were found propose --help\n")
         generated_file.write('\tif [ -z "$opts" ]; then\n')
+        generated_file.write('\t\tprev="${COMP_WORDS[COMP_CWORD-1]}"\n\n')
         generated_file.write('\t\tif [[ $prev != "--help" ]]; then\n')
-        generated_file.write('\t\t\topts=( --help )\n')
-        generated_file.write('\t\tfi\n')
-        generated_file.write('\tfi\n')
+        generated_file.write("\t\t\topts=( --help )\n")
+        generated_file.write("\t\tfi\n")
+        generated_file.write("\tfi\n")
 
         # generate the completion list from the possible options
         generated_file.write('\tCOMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )\n')
-        generated_file.write('\treturn 0\n')
-        generated_file.write('}\n\n')
+        generated_file.write("\treturn 0\n")
+        generated_file.write("}\n\n")
 
         # Add the function to bash completion
-        generated_file.write('complete -F _yunohost yunohost')
+        generated_file.write("complete -F _yunohost yunohost")

data/bash-completion.d/yunohost

@@ -1,3 +0,0 @@
-# This file is automatically generated
-# during Debian's package build by the script
-# data/actionsmap/yunohost_completion.py

data/helpers

@@ -1,7 +1,8 @@
 # -*- shell-script -*-
 
-# TODO : use --regex to validate against a namespace
+readonly XTRACE_ENABLE=$(set +o | grep xtrace) # This is a trick to later only restore set -x if it was set when calling this script
+set +x
 for helper in $(run-parts --list /usr/share/yunohost/helpers.d 2>/dev/null) ; do
     [ -r $helper ] && . $helper || true
 done
-
+eval "$XTRACE_ENABLE"

data/helpers.d/apt

@@ -5,15 +5,17 @@
 # [internal]
 #
 # usage: ynh_wait_dpkg_free
+# | exit: Return 1 if dpkg is broken
 #
 # Requires YunoHost version 3.3.1 or higher.
 ynh_wait_dpkg_free() {
     local try
+    set +o xtrace # set +x
     # With seq 1 17, timeout will be almost 30 minutes
     for try in `seq 1 17`
     do
         # Check if /var/lib/dpkg/lock is used by another process
-        if sudo lsof /var/lib/dpkg/lock > /dev/null
+        if lsof /var/lib/dpkg/lock > /dev/null
         then
             echo "apt is already in use..."
             # Sleep an exponential time at each round
@@ -27,38 +29,42 @@ ynh_wait_dpkg_free() {
             while read dpkg_file <&9
             do
                 # Check if the name of this file contains only numbers.
-                if echo "$dpkg_file" | grep -Pq "^[[:digit:]]+$"
+                if echo "$dpkg_file" | grep --perl-regexp --quiet "^[[:digit:]]+$"
                 then
                     # If so, that a remaining of dpkg.
-                    ynh_print_err "E: dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem."
+                    ynh_print_err "dpkg was interrupted, you must manually run 'sudo dpkg --configure -a' to correct the problem."
+                    set -o xtrace # set -x
                     return 1
                 fi
             done 9<<< "$(ls -1 $dpkg_dir)"
+            set -o xtrace # set -x
             return 0
         fi
     done
     echo "apt still used, but timeout reached !"
+    set -o xtrace # set -x
 }
 
 # Check either a package is installed or not
 #
-# example: ynh_package_is_installed --package=yunohost && echo "ok"
+# example: ynh_package_is_installed --package=yunohost && echo "installed"
 #
 # usage: ynh_package_is_installed --package=name
-# | arg: -p, --package - the package name to check
+# | arg: -p, --package=     - the package name to check
+# | ret: 0 if the package is installed, 1 else.
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_package_is_installed() {
     # Declare an array to define the options of this helper.
     local legacy_args=p
-    declare -Ar args_array=( [p]=package= )
+    local -A args_array=( [p]=package= )
     local package
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
     ynh_wait_dpkg_free
-    dpkg-query -W -f '${Status}' "$package" 2>/dev/null \
-        | grep -c "ok installed" &>/dev/null
+    dpkg-query --show --showformat='${Status}' "$package" 2>/dev/null \
+        | grep --count "ok installed" &>/dev/null
 }
 
 # Get the version of an installed package
@@ -66,20 +72,21 @@ ynh_package_is_installed() {
 # example: version=$(ynh_package_version --package=yunohost)
 #
 # usage: ynh_package_version --package=name
-# | arg: -p, --package - the package name to get version
+# | arg: -p, --package=     - the package name to get version
 # | ret: the version or an empty string
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_package_version() {
     # Declare an array to define the options of this helper.
     local legacy_args=p
-    declare -Ar args_array=( [p]=package= )
+    local -A args_array=( [p]=package= )
     local package
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    if ynh_package_is_installed "$package"; then
-        dpkg-query -W -f '${Version}' "$package" 2>/dev/null
+    if ynh_package_is_installed "$package"
+    then
+        dpkg-query --show --showformat='${Version}' "$package" 2>/dev/null
     else
         echo ''
     fi
@@ -94,7 +101,7 @@ ynh_package_version() {
 # Requires YunoHost version 2.4.0.3 or higher.
 ynh_apt() {
     ynh_wait_dpkg_free
-    DEBIAN_FRONTEND=noninteractive apt-get -y $@
+    LC_ALL=C DEBIAN_FRONTEND=noninteractive apt-get --assume-yes --quiet -o=Acquire::Retries=3 -o=Dpkg::Use-Pty=0 $@
 }
 
 # Update package index files
@@ -113,8 +120,8 @@ ynh_package_update() {
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_package_install() {
-    ynh_apt --no-remove -o Dpkg::Options::=--force-confdef \
-            -o Dpkg::Options::=--force-confold install $@
+    ynh_apt --no-remove --option Dpkg::Options::=--force-confdef \
+        --option Dpkg::Options::=--force-confold install $@
 }
 
 # Remove package(s)
@@ -163,8 +170,8 @@ ynh_package_install_from_equivs () {
     local controlfile=$1
 
     # retrieve package information
-    local pkgname=$(grep '^Package: ' $controlfile | cut -d' ' -f 2)	# Retrieve the name of the debian package
-    local pkgversion=$(grep '^Version: ' $controlfile | cut -d' ' -f 2)	# And its version number
+    local pkgname=$(grep '^Package: ' $controlfile | cut --delimiter=' ' --fields=2)	# Retrieve the name of the debian package
+    local pkgversion=$(grep '^Version: ' $controlfile | cut --delimiter=' ' --fields=2)	# And its version number
     [[ -z "$pkgname" || -z "$pkgversion" ]] \
         && ynh_die --message="Invalid control file"	# Check if this 2 variables aren't empty.
 
@@ -172,7 +179,7 @@ ynh_package_install_from_equivs () {
     ynh_package_update
 
     # Build and install the package
-    local TMPDIR=$(mktemp -d)
+    local TMPDIR=$(mktemp --directory)
 
     # Force the compatibility level at 10, levels below are deprecated
     echo 10 > /usr/share/equivs/template/debian/compat
@@ -184,10 +191,20 @@ ynh_package_install_from_equivs () {
     ynh_wait_dpkg_free
     cp "$controlfile" "${TMPDIR}/control"
     (cd "$TMPDIR"
-    equivs-build ./control 1> /dev/null
-    dpkg --force-depends -i "./${pkgname}_${pkgversion}_all.deb" 2>&1)
-    ynh_package_install -f || ynh_die --message="Unable to install dependencies"
-    [[ -n "$TMPDIR" ]] && rm -rf $TMPDIR	# Remove the temp dir.
+    LC_ALL=C equivs-build ./control 1> /dev/null
+    LC_ALL=C dpkg --force-depends --install "./${pkgname}_${pkgversion}_all.deb" 2>&1 | tee ./dpkg_log)
+
+    ynh_package_install --fix-broken || \
+        { # If the installation failed
+        # (the following is ran inside { } to not start a subshell otherwise ynh_die wouldnt exit the original process)
+        # Parse the list of problematic dependencies from dpkg's log ...
+        # (relevant lines look like: "foo-ynh-deps depends on bar; however:")
+        local problematic_dependencies="$(cat $TMPDIR/dpkg_log | grep -oP '(?<=-ynh-deps depends on ).*(?=; however)' | tr '\n' ' ')"
+        # Fake an install of those dependencies to see the errors
+        # The sed command here is, Print only from 'Reading state info' to the end.
+        [[ -n "$problematic_dependencies" ]] && ynh_package_install $problematic_dependencies --dry-run 2>&1 | sed --quiet '/Reading state info/,$p' | grep -v "fix-broken\|Reading state info" >&2
+        ynh_die --message="Unable to install dependencies"; }
+    [[ -n "$TMPDIR" ]] && rm --recursive --force $TMPDIR	# Remove the temp dir.
 
     # check if the package is actually installed
     ynh_package_is_installed "$pkgname"
@@ -200,24 +217,57 @@ ynh_package_install_from_equivs () {
 # example : ynh_install_app_dependencies dep1 dep2 "dep3|dep4|dep5"
 #
 # usage: ynh_install_app_dependencies dep [dep [...]]
-# | arg: dep - the package name to install in dependence. Writing "dep3|dep4|dep5" can be used to specify alternatives. For example : dep1 dep2 "dep3|dep4|dep5" will require to install dep1 and dep 2 and (dep3 or dep4 or dep5).
+# | arg: dep - the package name to install in dependence.
+# | arg: "dep1|dep2|…" - You can specify alternatives. It will require to install (dep1 or dep2, etc).
 #
 # Requires YunoHost version 2.6.4 or higher.
 ynh_install_app_dependencies () {
     local dependencies=$@
-    local dependencies=${dependencies// /, }
+    # Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See  below)
+    dependencies="$(echo "$dependencies" | sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g')"
     local dependencies=${dependencies//|/ | }
-    local manifest_path="../manifest.json"
-    if [ ! -e "$manifest_path" ]; then
-    	manifest_path="../settings/manifest.json"	# Into the restore script, the manifest is not at the same place
-    fi
+    local manifest_path="$YNH_APP_BASEDIR/manifest.json"
 
-    local version=$(grep '\"version\": ' "$manifest_path" | cut -d '"' -f 4)	# Retrieve the version number in the manifest file.
-    if [ ${#version} -eq 0 ]; then
+    local version=$(jq -r '.version' "$manifest_path")
+    if [ -z "${version}" ] || [ "$version" == "null" ]; then
         version="1.0"
     fi
     local dep_app=${app//_/-}	# Replace all '_' by '-'
 
+    # Handle specific versions
+    if [[ "$dependencies" =~ [\<=\>] ]]
+    then
+        # Replace version specifications by relationships syntax
+        # https://www.debian.org/doc/debian-policy/ch-relationships.html
+        # Sed clarification
+        # [^(\<=\>] ignore if it begins by ( or < = >. To not apply twice.
+        # [\<=\>] matches < = or >
+        # \+ matches one or more occurence of the previous characters, for >= or >>.
+        # [^,]\+ matches all characters except ','
+        # Ex: 'package>=1.0' will be replaced by 'package (>= 1.0)'
+        dependencies="$(echo "$dependencies" | sed 's/\([^(\<=\>]\)\([\<=\>]\+\)\([^,]\+\)/\1 (\2 \3)/g')"
+    fi
+
+    #
+    # Epic ugly hack to fix the goddamn dependency nightmare of sury
+    # Sponsored by the "Djeezusse Fokin Kraiste Why Do Adminsys Has To Be So Fucking Complicated I Should Go Grow Potatoes Instead Of This Shit" collective
+    # https://github.com/YunoHost/issues/issues/1407
+    #
+    # If we require to install php dependency
+    if echo $dependencies | grep --quiet 'php'
+    then
+        # And we have packages from sury installed (7.0.33-10+weirdshiftafter instead of 7.0.33-0 on debian)
+        if dpkg --list | grep "php7.0" | grep --quiet --invert-match "7.0.33-0+deb9"
+        then
+            # And sury ain't already in sources.lists
+            if ! grep --recursive --quiet "^ *deb.*sury" /etc/apt/sources.list*
+            then
+                # Re-add sury
+                ynh_install_extra_repo --repo="https://packages.sury.org/php/ $(ynh_get_debian_release) main" --key="https://packages.sury.org/php/apt.gpg" --name=extra_php_version --priority=600
+            fi
+        fi
+    fi
+
     cat > /tmp/${dep_app}-ynh-deps.control << EOF	# Make a control file for equivs-build
 Section: misc
 Priority: optional
@@ -234,6 +284,38 @@ EOF
     ynh_app_setting_set --app=$app --key=apt_dependencies --value="$dependencies"
 }
 
+# Add dependencies to install with ynh_install_app_dependencies
+#
+# usage: ynh_add_app_dependencies --package=phpversion [--replace]
+# | arg: -p, --package=     - Packages to add as dependencies for the app.
+# | arg: -r, --replace      - Replace dependencies instead of adding to existing ones.
+#
+# Requires YunoHost version 3.8.1 or higher.
+ynh_add_app_dependencies () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=pr
+    local -A args_array=( [p]=package= [r]=replace)
+    local package
+    local replace
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    replace=${replace:-0}
+
+    local current_dependencies=""
+    if [ $replace -eq 0 ]
+    then
+        local dep_app=${app//_/-}	# Replace all '_' by '-'
+        if ynh_package_is_installed --package="${dep_app}-ynh-deps"
+        then
+            current_dependencies="$(dpkg-query --show --showformat='${Depends}' ${dep_app}-ynh-deps) "
+        fi
+
+        current_dependencies=${current_dependencies// | /|}
+    fi
+
+    ynh_install_app_dependencies "${current_dependencies}${package}"
+}
+
 # Remove fake package and its dependencies
 #
 # Dependencies will removed only if no other package need them.
@@ -245,3 +327,234 @@ ynh_remove_app_dependencies () {
     local dep_app=${app//_/-}	# Replace all '_' by '-'
     ynh_package_autopurge ${dep_app}-ynh-deps	# Remove the fake package and its dependencies if they not still used.
 }
+
+# Install packages from an extra repository properly.
+#
+# usage: ynh_install_extra_app_dependencies --repo="repo" --package="dep1 dep2" [--key=key_url] [--name=name]
+# | arg: -r, --repo=    - Complete url of the extra repository.
+# | arg: -p, --package= - The packages to install from this extra repository
+# | arg: -k, --key=     - url to get the public key.
+# | arg: -n, --name=    - Name for the files for this repo, $app as default value.
+#
+# Requires YunoHost version 3.8.1 or higher.
+ynh_install_extra_app_dependencies () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=rpkn
+    local -A args_array=( [r]=repo= [p]=package= [k]=key= [n]=name= )
+    local repo
+    local package
+    local key
+    local name
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    name="${name:-$app}"
+    key=${key:-}
+
+    # Set a key only if asked
+    if [ -n "$key" ]
+    then
+        key="--key=$key"
+    fi
+    # Add an extra repository for those packages
+    ynh_install_extra_repo --repo="$repo" $key --priority=995 --name=$name
+
+    # Install requested dependencies from this extra repository.
+    ynh_add_app_dependencies --package="$package"
+
+    # Remove this extra repository after packages are installed
+    ynh_remove_extra_repo --name=$app
+}
+
+# Add an extra repository correctly, pin it and get the key.
+#
+# [internal]
+#
+# usage: ynh_install_extra_repo --repo="repo" [--key=key_url] [--priority=priority_value] [--name=name] [--append]
+# | arg: -r, --repo=        - Complete url of the extra repository.
+# | arg: -k, --key=         - url to get the public key.
+# | arg: -p, --priority=    - Priority for the pin
+# | arg: -n, --name=        - Name for the files for this repo, $app as default value.
+# | arg: -a, --append       - Do not overwrite existing files.
+#
+# Requires YunoHost version 3.8.1 or higher.
+ynh_install_extra_repo () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=rkpna
+    local -A args_array=( [r]=repo= [k]=key= [p]=priority= [n]=name= [a]=append )
+    local repo
+    local key
+    local priority
+    local name
+    local append
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    name="${name:-$app}"
+    append=${append:-0}
+    key=${key:-}
+    priority=${priority:-}
+
+    if [ $append -eq 1 ]
+    then
+        append="--append"
+        wget_append="tee --append"
+    else
+        append=""
+        wget_append="tee"
+    fi
+
+    # Split the repository into uri, suite and components.
+    # Remove "deb " at the beginning of the repo.
+    repo="${repo#deb }"
+
+    # Get the uri
+    local uri="$(echo "$repo" | awk '{ print $1 }')"
+
+    # Get the suite
+    local suite="$(echo "$repo" | awk '{ print $2 }')"
+
+    # Get the components
+    local component="${repo##$uri $suite }"
+
+    # Add the repository into sources.list.d
+    ynh_add_repo --uri="$uri" --suite="$suite" --component="$component" --name="$name" $append
+
+    # Pin the new repo with the default priority, so it won't be used for upgrades.
+    # Build $pin from the uri without http and any sub path
+    local pin="${uri#*://}"
+    pin="${pin%%/*}"
+    # Set a priority only if asked
+    if [ -n "$priority" ]
+    then
+        priority="--priority=$priority"
+    fi
+    ynh_pin_repo --package="*" --pin="origin \"$pin\"" $priority --name="$name" $append
+
+    # Get the public key for the repo
+    if [ -n "$key" ]
+    then
+        mkdir --parents "/etc/apt/trusted.gpg.d"
+        # Timeout option is here to enforce the timeout on dns query and tcp connect (c.f. man wget)
+        wget --timeout 900 --quiet "$key" --output-document=- | gpg --dearmor | $wget_append /etc/apt/trusted.gpg.d/$name.gpg > /dev/null
+    fi
+
+    # Update the list of package with the new repo
+    ynh_package_update
+}
+
+# Remove an extra repository and the assiociated configuration.
+#
+# [internal]
+#
+# usage: ynh_remove_extra_repo [--name=name]
+# | arg: -n, --name=    - Name for the files for this repo, $app as default value.
+#
+# Requires YunoHost version 3.8.1 or higher.
+ynh_remove_extra_repo () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=n
+    local -A args_array=( [n]=name= )
+    local name
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    name="${name:-$app}"
+
+    ynh_secure_remove --file="/etc/apt/sources.list.d/$name.list"
+    # Sury pinning is managed by the regenconf in the core...
+    [[ "$name" == "extra_php_version" ]] || ynh_secure_remove "/etc/apt/preferences.d/$name"
+    ynh_secure_remove --file="/etc/apt/trusted.gpg.d/$name.gpg" > /dev/null
+    ynh_secure_remove --file="/etc/apt/trusted.gpg.d/$name.asc" > /dev/null
+
+    # Update the list of package to exclude the old repo
+    ynh_package_update
+}
+
+# Add a repository.
+#
+# [internal]
+#
+# usage: ynh_add_repo --uri=uri --suite=suite --component=component [--name=name] [--append]
+# | arg: -u, --uri=         - Uri of the repository.
+# | arg: -s, --suite=       - Suite of the repository.
+# | arg: -c, --component=   - Component of the repository.
+# | arg: -n, --name=        - Name for the files for this repo, $app as default value.
+# | arg: -a, --append       - Do not overwrite existing files.
+#
+# Example for a repo like deb http://forge.yunohost.org/debian/ stretch stable
+#                             uri                               suite   component
+# ynh_add_repo --uri=http://forge.yunohost.org/debian/ --suite=stretch --component=stable
+#
+# Requires YunoHost version 3.8.1 or higher.
+ynh_add_repo () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=uscna
+    local -A args_array=( [u]=uri= [s]=suite= [c]=component= [n]=name= [a]=append )
+    local uri
+    local suite
+    local component
+    local name
+    local append
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    name="${name:-$app}"
+    append=${append:-0}
+
+    if [ $append -eq 1 ]
+    then
+        append="tee --append"
+    else
+        append="tee"
+    fi
+
+    mkdir --parents "/etc/apt/sources.list.d"
+    # Add the new repo in sources.list.d
+    echo "deb $uri $suite $component" \
+        | $append "/etc/apt/sources.list.d/$name.list"
+}
+
+# Pin a repository.
+#
+# [internal]
+#
+# usage: ynh_pin_repo --package=packages --pin=pin_filter [--priority=priority_value] [--name=name] [--append]
+# | arg: -p, --package=     - Packages concerned by the pin. Or all, *.
+# | arg: -i, --pin=         - Filter for the pin.
+# | arg: -p, --priority=    - Priority for the pin
+# | arg: -n, --name=        - Name for the files for this repo, $app as default value.
+# | arg: -a, --append       - Do not overwrite existing files.
+#
+# See https://manpages.debian.org/stretch/apt/apt_preferences.5.en.html#How_APT_Interprets_Priorities for information about pinning.
+#
+# Requires YunoHost version 3.8.1 or higher.
+ynh_pin_repo () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=pirna
+    local -A args_array=( [p]=package= [i]=pin= [r]=priority= [n]=name= [a]=append )
+    local package
+    local pin
+    local priority
+    local name
+    local append
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    package="${package:-*}"
+    priority=${priority:-50}
+    name="${name:-$app}"
+    append=${append:-0}
+
+    if [ $append -eq 1 ]
+    then
+        append="tee --append"
+    else
+        append="tee"
+    fi
+
+    # Sury pinning is managed by the regenconf in the core...
+    [[ "$name" != "extra_php_version" ]] || return 0
+
+    mkdir --parents "/etc/apt/preferences.d"
+    echo "Package: $package
+Pin: $pin
+Pin-Priority: $priority
+" \
+    | $append "/etc/apt/preferences.d/$name"
+}

data/helpers.d/backup

@@ -4,22 +4,22 @@ CAN_BIND=${CAN_BIND:-1}
 
 # Add a file or a directory to the list of paths to backup
 #
-# This helper can be used both in a system backup hook, and in an app backup script
-#
-# Details: ynh_backup writes SRC and the relative DEST into a CSV file. And it
-# creates the parent destination directory
-#
-# If DEST is ended by a slash it complete this path with the basename of SRC.
-#
 # usage: ynh_backup --src_path=src_path [--dest_path=dest_path] [--is_big] [--not_mandatory]
-# | arg: -s, --src_path - file or directory to bind or symlink or copy. it shouldn't be in the backup dir.
-# | arg: -d, --dest_path - destination file or directory inside the backup dir
-# | arg: -b, --is_big - Indicate data are big (mail, video, image ...)
-# | arg: -m, --not_mandatory - Indicate that if the file is missing, the backup can ignore it.
+# | arg: -s, --src_path=        - file or directory to bind or symlink or copy. it shouldn't be in the backup dir.
+# | arg: -d, --dest_path=       - destination file or directory inside the backup dir
+# | arg: -b, --is_big           - Indicate data are big (mail, video, image ...)
+# | arg: -m, --not_mandatory    - Indicate that if the file is missing, the backup can ignore it.
 # | arg: arg - Deprecated arg
 #
-# Example in the context of a wordpress app
+# This helper can be used both in a system backup hook, and in an app backup script
 #
+# `ynh_backup` writes `src_path` and the relative `dest_path` into a CSV file, and it
+# creates the parent destination directory
+#
+# If `dest_path` is ended by a slash it complete this path with the basename of `src_path`.
+#
+# Example in the context of a wordpress app :
+# ```
 # ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf"
 # # => This line will be added into CSV file
 # # "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/etc/nginx/conf.d/$domain.d/$app.conf"
@@ -40,22 +40,43 @@ CAN_BIND=${CAN_BIND:-1}
 # ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" "/conf/"
 # # => "/etc/nginx/conf.d/$domain.d/$app.conf","apps/wordpress/conf/$app.conf"
 #
+# ```
+#
+# How to use `--is_big`:
+#
+# `--is_big` is used to specify that this part of the backup can be quite huge.
+# So, you don't want that your package does backup that part during ynh_backup_before_upgrade.
+# In the same way, an user may doesn't want to backup this big part of the app for
+# each of his backup. And so handle that part differently.
+#
+# As this part of your backup may not be done, your restore script has to handle it.
+# In your restore script, use `--not_mandatory` with `ynh_restore_file`
+# As well in your remove script, you should not remove those data ! Or an user may end up with
+# a failed upgrade restoring an app without data anymore !
+#
+# To have the benefit of `--is_big` while doing a backup, you can whether set the environement
+# variable `BACKUP_CORE_ONLY` to 1 (`BACKUP_CORE_ONLY=1`) before the backup command. It will affect
+# only that backup command.
+# Or set the config `do_not_backup_data` to 1 into the `settings.yml` of the app. This will affect
+# all backups for this app until the setting is removed.
+#
 # Requires YunoHost version 2.4.0 or higher.
+# Requires YunoHost version 3.5.0 or higher for the argument `--not_mandatory`
 ynh_backup() {
     # TODO find a way to avoid injection by file strange naming !
 
     # Declare an array to define the options of this helper.
     local legacy_args=sdbm
-    declare -Ar args_array=( [s]=src_path= [d]=dest_path= [b]=is_big [m]=not_mandatory )
+    local -A args_array=( [s]=src_path= [d]=dest_path= [b]=is_big [m]=not_mandatory )
     local src_path
     local dest_path
     local is_big
     local not_mandatory
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
-    local dest_path="${dest_path:-}"
-    local is_big="${is_big:-0}"
-    local not_mandatory="${not_mandatory:-0}"
+    dest_path="${dest_path:-}"
+    is_big="${is_big:-0}"
+    not_mandatory="${not_mandatory:-0}"
 
     BACKUP_CORE_ONLY=${BACKUP_CORE_ONLY:-0}
     test -n "${app:-}" && do_not_backup_data=$(ynh_app_setting_get --app=$app --key=do_not_backup_data)
@@ -64,10 +85,11 @@ ynh_backup() {
     # don't backup big data items
     if [ $is_big -eq 1 ] && ( [ ${do_not_backup_data:-0} -eq 1 ] || [ $BACKUP_CORE_ONLY -eq 1 ] )
     then
-        if [ $BACKUP_CORE_ONLY -eq 1 ]; then
-            ynh_print_warn --message="$src_path will not be saved, because 'BACKUP_CORE_ONLY' is set."
+        if [ $BACKUP_CORE_ONLY -eq 1 ]
+        then
+            ynh_print_info --message="$src_path will not be saved, because 'BACKUP_CORE_ONLY' is set."
         else
-            ynh_print_warn --message="$src_path will not be saved, because 'do_not_backup_data' is set."
+            ynh_print_info --message="$src_path will not be saved, because 'do_not_backup_data' is set."
         fi
         return 0
     fi
@@ -76,22 +98,23 @@ ynh_backup() {
     # Format correctly source and destination paths
     # ==============================================================================
     # Be sure the source path is not empty
-    [[ -e "${src_path}" ]] || {
+    if [ ! -e "$src_path" ]
+    then
         ynh_print_warn --message="Source path '${src_path}' does not exist"
         if [ "$not_mandatory" == "0" ]
         then
-                # This is a temporary fix for fail2ban config files missing after the migration to stretch.
-                if echo "${src_path}" | grep --quiet "/etc/fail2ban"
-                then
-                        touch "${src_path}"
-                        ynh_print_info --message="The missing file will be replaced by a dummy one for the backup !!!"
-                else
-                        return 1
-                fi
+            # This is a temporary fix for fail2ban config files missing after the migration to stretch.
+            if echo "${src_path}" | grep --quiet "/etc/fail2ban"
+            then
+                touch "${src_path}"
+                ynh_print_info --message="The missing file will be replaced by a dummy one for the backup !!!"
+            else
+                return 1
+            fi
         else
-                return 0
+            return 0
         fi
-    }
+    fi
 
     # Transform the source path as an absolute path
     # If it's a dir remove the ending /
@@ -100,12 +123,13 @@ ynh_backup() {
     # If there is no destination path, initialize it with the source path
     # relative to "/".
     # eg: src_path=/etc/yunohost -> dest_path=etc/yunohost
-    if [[ -z "$dest_path" ]]; then
-
+    if [[ -z "$dest_path" ]]
+    then
         dest_path="${src_path#/}"
 
     else
-        if [[ "${dest_path:0:1}" == "/" ]]; then
+        if [[ "${dest_path:0:1}" == "/" ]]
+        then
 
             # If the destination path is an absolute path, transform it as a path
             # relative to the current working directory ($YNH_CWD)
@@ -117,20 +141,23 @@ ynh_backup() {
             dest_path="${dest_path#$YNH_CWD/}"
 
             # Case where $2 is an absolute dir but doesn't begin with $YNH_CWD
-            [[ "${dest_path:0:1}" == "/" ]] \
-                && dest_path="${dest_path#/}"
+            if [[ "${dest_path:0:1}" == "/" ]]; then
+                dest_path="${dest_path#/}"
+            fi
         fi
 
         # Complete dest_path if ended by a /
-        [[ "${dest_path: -1}" == "/" ]] \
-            &&  dest_path="${dest_path}/$(basename $src_path)"
+        if [[ "${dest_path: -1}" == "/" ]]; then
+            dest_path="${dest_path}/$(basename $src_path)"
+        fi
     fi
 
     # Check if dest_path already exists in tmp archive
-    [[ ! -e "${dest_path}" ]] || {
+    if [[ -e "${dest_path}" ]]
+    then
         ynh_print_err --message="Destination path '${dest_path}' already exist"
         return 1
-    }
+    fi
 
     # Add the relative current working directory to the destination path
     local rel_dir="${YNH_CWD#$YNH_BACKUP_DIR}"
@@ -142,15 +169,15 @@ ynh_backup() {
     # ==============================================================================
     # Write file to backup into backup_list
     # ==============================================================================
-    local src=$(echo "${src_path}" | sed -r 's/"/\"\"/g')
-    local dest=$(echo "${dest_path}" | sed -r 's/"/\"\"/g')
+    local src=$(echo "${src_path}" | sed --regexp-extended 's/"/\"\"/g')
+    local dest=$(echo "${dest_path}" | sed --regexp-extended 's/"/\"\"/g')
     echo "\"${src}\",\"${dest}\"" >> "${YNH_BACKUP_CSV}"
 
     # ==============================================================================
 
     # Create the parent dir of the destination path
     # It's for retro compatibility, some script consider ynh_backup creates this dir
-    mkdir -p $(dirname "$YNH_BACKUP_DIR/${dest_path}")
+    mkdir --parents $(dirname "$YNH_BACKUP_DIR/${dest_path}")
 }
 
 # Restore all files that were previously backuped in a core backup script or app backup script
@@ -164,10 +191,11 @@ ynh_restore () {
     REL_DIR="${REL_DIR%/}/"
 
     # For each destination path begining by $REL_DIR
-    cat ${YNH_BACKUP_CSV} | tr -d $'\r' | grep -ohP "^\".*\",\"$REL_DIR.*\"$" | \
-    while read line; do
-        local ORIGIN_PATH=$(echo "$line" | grep -ohP "^\"\K.*(?=\",\".*\"$)")
-        local ARCHIVE_PATH=$(echo "$line" | grep -ohP "^\".*\",\"$REL_DIR\K.*(?=\"$)")
+    cat ${YNH_BACKUP_CSV} | tr --delete $'\r' | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR.*\"$" | \
+    while read line
+    do
+        local ORIGIN_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\"\K.*(?=\",\".*\"$)")
+        local ARCHIVE_PATH=$(echo "$line" | grep --only-matching --no-filename --perl-regexp "^\".*\",\"$REL_DIR\K.*(?=\"$)")
         ynh_restore_file --origin_path="$ARCHIVE_PATH" --dest_path="$ORIGIN_PATH"
     done
 }
@@ -179,14 +207,14 @@ ynh_restore () {
 # usage: _get_archive_path ORIGIN_PATH
 _get_archive_path () {
     # For security reasons we use csv python library to read the CSV
-    sudo python -c "
+    python3 -c "
 import sys
 import csv
 with open(sys.argv[1], 'r') as backup_file:
     backup_csv = csv.DictReader(backup_file, fieldnames=['source', 'dest'])
     for row in backup_csv:
         if row['source']==sys.argv[2].strip('\"'):
-            print row['dest']
+            print(row['dest'])
             sys.exit(0)
     raise Exception('Original path for %s not found' % sys.argv[2])
     "  "${YNH_BACKUP_CSV}" "$1"
@@ -195,46 +223,46 @@ with open(sys.argv[1], 'r') as backup_file:
 
 # Restore a file or a directory
 #
-# Use the registered path in backup_list by ynh_backup to restore the file at
-# the right place.
-#
 # usage: ynh_restore_file --origin_path=origin_path [--dest_path=dest_path] [--not_mandatory]
-# | arg: -o, --origin_path - Path where was located the file or the directory before to be backuped or relative path to $YNH_CWD where it is located in the backup archive
-# | arg: -d, --dest_path - Path where restore the file or the dir, if unspecified, the destination will be ORIGIN_PATH or if the ORIGIN_PATH doesn't exist in the archive, the destination will be searched into backup.csv
-# | arg: -m, --not_mandatory - Indicate that if the file is missing, the restore process can ignore it.
+# | arg: -o, --origin_path=     - Path where was located the file or the directory before to be backuped or relative path to $YNH_CWD where it is located in the backup archive
+# | arg: -d, --dest_path=       - Path where restore the file or the dir. If unspecified, the destination will be `ORIGIN_PATH` or if the `ORIGIN_PATH` doesn't exist in the archive, the destination will be searched into `backup.csv`
+# | arg: -m, --not_mandatory    - Indicate that if the file is missing, the restore process can ignore it.
+#
+# Use the registered path in backup_list by ynh_backup to restore the file at the right place.
 #
 # examples:
-#     ynh_restore_file "/etc/nginx/conf.d/$domain.d/$app.conf"
+#     ynh_restore_file -o "/etc/nginx/conf.d/$domain.d/$app.conf"
 #     # You can also use relative paths:
-#     ynh_restore_file "conf/nginx.conf"
+#     ynh_restore_file -o "conf/nginx.conf"
 #
-# If DEST_PATH already exists and is lighter than 500 Mo, a backup will be made in
-# /home/yunohost.conf/backup/. Otherwise, the existing file is removed.
+# If `DEST_PATH` already exists and is lighter than 500 Mo, a backup will be made in
+# `/home/yunohost.conf/backup/`. Otherwise, the existing file is removed.
 #
-# if apps/wordpress/etc/nginx/conf.d/$domain.d/$app.conf exists, restore it into
-# /etc/nginx/conf.d/$domain.d/$app.conf
+# if `apps/$app/etc/nginx/conf.d/$domain.d/$app.conf` exists, restore it into
+# `/etc/nginx/conf.d/$domain.d/$app.conf`
 # if no, search for a match in the csv (eg: conf/nginx.conf) and restore it into
-# /etc/nginx/conf.d/$domain.d/$app.conf
+# `/etc/nginx/conf.d/$domain.d/$app.conf`
 #
 # Requires YunoHost version 2.6.4 or higher.
+# Requires YunoHost version 3.5.0 or higher for the argument --not_mandatory
 ynh_restore_file () {
     # Declare an array to define the options of this helper.
     local legacy_args=odm
-    declare -Ar args_array=( [o]=origin_path= [d]=dest_path= [m]=not_mandatory )
+    local -A args_array=( [o]=origin_path= [d]=dest_path= [m]=not_mandatory )
     local origin_path
-    local archive_path
     local dest_path
     local not_mandatory
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
-    local origin_path="/${origin_path#/}"
-    local archive_path="$YNH_CWD${origin_path}"
+    origin_path="/${origin_path#/}"
     # Default value for dest_path = /$origin_path
-    local dest_path="${dest_path:-$origin_path}"
-    local not_mandatory="${not_mandatory:-0}"
+    dest_path="${dest_path:-$origin_path}"
+    not_mandatory="${not_mandatory:-0}"
 
+    local archive_path="$YNH_CWD${origin_path}"
     # If archive_path doesn't exist, search for a corresponding path in CSV
-    if [ ! -d "$archive_path" ] && [ ! -f "$archive_path" ] && [ ! -L "$archive_path" ]; then
+    if [ ! -d "$archive_path" ] && [ ! -f "$archive_path" ] && [ ! -L "$archive_path" ]
+    then
         if [ "$not_mandatory" == "0" ]
         then
             archive_path="$YNH_BACKUP_DIR/$(_get_archive_path \"$origin_path\")"
@@ -247,10 +275,10 @@ ynh_restore_file () {
     if [[ -e "${dest_path}" ]]
     then
         # Check if the file/dir size is less than 500 Mo
-        if [[ $(du -sb ${dest_path} | cut -d"/" -f1) -le "500000000" ]]
+        if [[ $(du --summarize --bytes ${dest_path} | cut --delimiter="/" --fields=1) -le "500000000" ]]
         then
             local backup_file="/home/yunohost.conf/backup/${dest_path}.backup.$(date '+%Y%m%d.%H%M%S')"
-            mkdir -p "$(dirname "$backup_file")"
+            mkdir --parents "$(dirname "$backup_file")"
             mv "${dest_path}" "$backup_file"    # Move the current file or directory
         else
             ynh_secure_remove --file=${dest_path}
@@ -258,15 +286,17 @@ ynh_restore_file () {
     fi
 
     # Restore origin_path into dest_path
-    mkdir -p $(dirname "$dest_path")
+    mkdir --parents $(dirname "$dest_path")
 
     # Do a copy if it's just a mounting point
-    if mountpoint -q $YNH_BACKUP_DIR; then
-        if [[ -d "${archive_path}" ]]; then
+    if mountpoint --quiet $YNH_BACKUP_DIR
+    then
+        if [[ -d "${archive_path}" ]]
+        then
             archive_path="${archive_path}/."
-            mkdir -p "$dest_path"
+            mkdir --parents "$dest_path"
         fi
-        cp -a "$archive_path" "${dest_path}"
+        cp --archive "$archive_path" "${dest_path}"
     # Do a move if YNH_BACKUP_DIR is already a copy
     else
         mv "$archive_path" "${dest_path}"
@@ -287,22 +317,35 @@ ynh_bind_or_cp() {
 
 # Calculate and store a file checksum into the app settings
 #
-# $app should be defined when calling this helper
-#
 # usage: ynh_store_file_checksum --file=file
-# | arg: -f, --file - The file on which the checksum will performed, then stored.
+# | arg: -f, --file=    - The file on which the checksum will performed, then stored.
+#
+# $app should be defined when calling this helper
 #
 # Requires YunoHost version 2.6.4 or higher.
 ynh_store_file_checksum () {
     # Declare an array to define the options of this helper.
     local legacy_args=f
-    declare -Ar args_array=( [f]=file= )
+    local -A args_array=( [f]=file= [u]=update_only )
     local file
+    local update_only
+    update_only="${update_only:-0}"
+
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
     local checksum_setting_name=checksum_${file//[\/ ]/_}    # Replace all '/' and ' ' by '_'
-    ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(sudo md5sum "$file" | cut -d' ' -f1)
+    
+    # If update only, we don't save the new checksum if no old checksum exist
+    if [ $update_only -eq 1 ] ; then
+        local checksum_value=$(ynh_app_setting_get --app=$app --key=$checksum_setting_name)
+        if [ -z "${checksum_value}" ] ; then
+            unset backup_file_checksum
+            return 0
+        fi
+    fi
+    
+    ynh_app_setting_set --app=$app --key=$checksum_setting_name --value=$(md5sum "$file" | cut --delimiter=' ' --fields=1)
 
     # If backup_file_checksum isn't empty, ynh_backup_if_checksum_is_different has made a backup
     if [ -n "${backup_file_checksum-}" ]
@@ -317,18 +360,18 @@ ynh_store_file_checksum () {
 
 # Verify the checksum and backup the file if it's different
 #
+# usage: ynh_backup_if_checksum_is_different --file=file
+# | arg: -f, --file=    - The file on which the checksum test will be perfomed.
+# | ret: the name of a backup file, or nothing
+#
 # This helper is primarily meant to allow to easily backup personalised/manually
 # modified config files.
 #
-# usage: ynh_backup_if_checksum_is_different --file=file
-# | arg: -f, --file - The file on which the checksum test will be perfomed.
-# | ret: the name of a backup file, or nothing
-#
 # Requires YunoHost version 2.6.4 or higher.
 ynh_backup_if_checksum_is_different () {
     # Declare an array to define the options of this helper.
     local legacy_args=f
-    declare -Ar args_array=( [f]=file= )
+    local -A args_array=( [f]=file= )
     local file
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
@@ -339,11 +382,11 @@ ynh_backup_if_checksum_is_different () {
     backup_file_checksum=""
     if [ -n "$checksum_value" ]
     then    # Proceed only if a value was stored into the app settings
-        if ! echo "$checksum_value $file" | sudo md5sum -c --status
+        if [ -e $file ] && ! echo "$checksum_value $file" | md5sum --check --status
         then    # If the checksum is now different
             backup_file_checksum="/home/yunohost.conf/backup/$file.backup.$(date '+%Y%m%d.%H%M%S')"
-            sudo mkdir -p "$(dirname "$backup_file_checksum")"
-            sudo cp -a "$file" "$backup_file_checksum"    # Backup the current file
+            mkdir --parents "$(dirname "$backup_file_checksum")"
+            cp --archive "$file" "$backup_file_checksum"    # Backup the current file
             ynh_print_warn "File $file has been manually modified since the installation or last upgrade. So it has been duplicated in $backup_file_checksum"
             echo "$backup_file_checksum"    # Return the name of the backup file
         fi
@@ -352,16 +395,16 @@ ynh_backup_if_checksum_is_different () {
 
 # Delete a file checksum from the app settings
 #
-# $app should be defined when calling this helper
+# usage: ynh_delete_file_checksum --file=file
+# | arg: -f, --file=    - The file for which the checksum will be deleted
 #
-# usage: ynh_remove_file_checksum file
-# | arg: -f, --file= - The file for which the checksum will be deleted
+# $app should be defined when calling this helper
 #
 # Requires YunoHost version 3.3.1 or higher.
 ynh_delete_file_checksum () {
     # Declare an array to define the options of this helper.
     local legacy_args=f
-    declare -Ar args_array=( [f]=file= )
+    local -A args_array=( [f]=file= )
     local file
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
@@ -370,14 +413,27 @@ ynh_delete_file_checksum () {
     ynh_app_setting_delete --app=$app --key=$checksum_setting_name
 }
 
+# Checks a backup archive exists
+#
+# [internal]
+#
+ynh_backup_archive_exists () {
+    yunohost backup list --output-as json --quiet \
+        | jq -e --arg archive "$1" '.archives | index($archive)' >/dev/null
+}
+
 # Make a backup in case of failed upgrade
 #
-# usage:
-# ynh_backup_before_upgrade
-# ynh_clean_setup () {
-#     ynh_restore_upgradebackup
-# }
-# ynh_abort_if_errors
+# usage: ynh_backup_before_upgrade
+#
+# Usage in a package script:
+# ```
+#   ynh_backup_before_upgrade
+#   ynh_clean_setup () {
+#       ynh_restore_upgradebackup
+#   }
+#   ynh_abort_if_errors
+# ```
 #
 # Requires YunoHost version 2.7.2 or higher.
 ynh_backup_before_upgrade () {
@@ -394,7 +450,7 @@ ynh_backup_before_upgrade () {
     if [ "$NO_BACKUP_UPGRADE" -eq 0 ]
     then
         # Check if a backup already exists with the prefix 1
-        if sudo yunohost backup list | grep -q $app_bck-pre-upgrade1
+        if ynh_backup_archive_exists "$app_bck-pre-upgrade1"
         then
             # Prefix becomes 2 to preserve the previous backup
             backup_number=2
@@ -402,14 +458,14 @@ ynh_backup_before_upgrade () {
         fi
 
         # Create backup
-        sudo BACKUP_CORE_ONLY=1 yunohost backup create --apps $app --name $app_bck-pre-upgrade$backup_number --debug
+        BACKUP_CORE_ONLY=1 yunohost backup create --apps $app --name $app_bck-pre-upgrade$backup_number --debug
         if [ "$?" -eq 0 ]
         then
             # If the backup succeeded, remove the previous backup
-            if sudo yunohost backup list | grep -q $app_bck-pre-upgrade$old_backup_number
+            if ynh_backup_archive_exists "$app_bck-pre-upgrade$old_backup_number"
             then
                 # Remove the previous backup only if it exists
-                sudo yunohost backup delete $app_bck-pre-upgrade$old_backup_number > /dev/null
+                yunohost backup delete $app_bck-pre-upgrade$old_backup_number > /dev/null
             fi
         else
             ynh_die --message="Backup failed, the upgrade process was aborted."
@@ -421,12 +477,16 @@ ynh_backup_before_upgrade () {
 
 # Restore a previous backup if the upgrade process failed
 #
-# usage:
-# ynh_backup_before_upgrade
-# ynh_clean_setup () {
-#     ynh_restore_upgradebackup
-# }
-# ynh_abort_if_errors
+# usage: ynh_restore_upgradebackup
+#
+# Usage in a package script:
+# ```
+#   ynh_backup_before_upgrade
+#   ynh_clean_setup () {
+#       ynh_restore_upgradebackup
+#   }
+#   ynh_abort_if_errors
+# ```
 #
 # Requires YunoHost version 2.7.2 or higher.
 ynh_restore_upgradebackup () {
@@ -438,12 +498,12 @@ ynh_restore_upgradebackup () {
     if [ "$NO_BACKUP_UPGRADE" -eq 0 ]
     then
         # Check if an existing backup can be found before removing and restoring the application.
-        if sudo yunohost backup list | grep -q $app_bck-pre-upgrade$backup_number
+        if ynh_backup_archive_exists "$app_bck-pre-upgrade$backup_number"
         then
             # Remove the application then restore it
-            sudo yunohost app remove $app
+            yunohost app remove $app
             # Restore the backup
-            sudo yunohost backup restore $app_bck-pre-upgrade$backup_number --apps $app --force --debug
+            yunohost backup restore $app_bck-pre-upgrade$backup_number --apps $app --force --debug
             ynh_die --message="The app was restored to the way it was before the failed upgrade."
         fi
     else

data/helpers.d/config

@@ -0,0 +1,357 @@
+#!/bin/bash
+
+
+_ynh_app_config_get_one() {
+    local short_setting="$1"
+    local type="$2"
+    local bind="$3"
+    local getter="get__${short_setting}"
+    # Get value from getter if exists
+    if type -t $getter 2>/dev/null | grep -q '^function$' 2>/dev/null;
+    then
+        old[$short_setting]="$($getter)"
+        formats[${short_setting}]="yaml"
+
+    elif [[ "$bind" == *"("* ]] && type -t "get__${bind%%(*}" 2>/dev/null | grep -q '^function$' 2>/dev/null;
+    then
+        old[$short_setting]="$("get__${bind%%(*}" $short_setting $type $bind)"
+        formats[${short_setting}]="yaml"
+    
+    elif [[ "$bind" == "null" ]]
+    then
+        old[$short_setting]="YNH_NULL"
+
+    # Get value from app settings or from another file
+    elif [[ "$type" == "file" ]]
+    then
+        if [[ "$bind" == "settings" ]]
+        then
+            ynh_die --message="File '${short_setting}' can't be stored in settings"
+        fi
+        old[$short_setting]="$(ls "$(echo $bind | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)" 2> /dev/null || echo YNH_NULL)"
+        file_hash[$short_setting]="true"
+
+    # Get multiline text from settings or from a full file
+    elif [[ "$type" == "text" ]]
+    then
+        if [[ "$bind" == "settings" ]]
+        then
+            old[$short_setting]="$(ynh_app_setting_get $app $short_setting)"
+        elif [[ "$bind" == *":"* ]]
+        then
+            ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically  in a variable file, you have to create custom getter/setter"
+        else
+            old[$short_setting]="$(cat $(echo $bind | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/) 2> /dev/null || echo YNH_NULL)"
+        fi
+
+    # Get value from a kind of key/value file
+    else
+        local bind_after=""
+        if [[ "$bind" == "settings" ]]
+        then
+            bind=":/etc/yunohost/apps/$app/settings.yml"
+        fi
+        local bind_key="$(echo "$bind" | cut -d: -f1)"
+        bind_key=${bind_key:-$short_setting}
+        if [[ "$bind_key" == *">"* ]];
+        then
+            bind_after="$(echo "${bind_key}" | cut -d'>' -f1)"
+            bind_key="$(echo "${bind_key}" | cut -d'>' -f2)"
+        fi
+        local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
+        old[$short_setting]="$(ynh_read_var_in_file --file="${bind_file}" --key="${bind_key}" --after="${bind_after}")"
+
+    fi
+}
+_ynh_app_config_apply_one() {
+    local short_setting="$1"
+    local setter="set__${short_setting}"
+    local bind="${binds[$short_setting]}"
+    local type="${types[$short_setting]}"
+    if [ "${changed[$short_setting]}" == "true" ]
+    then
+        # Apply setter if exists
+        if type -t $setter 2>/dev/null | grep -q '^function$' 2>/dev/null;
+        then
+            $setter
+
+        elif [[ "$bind" == *"("* ]] && type -t "set__${bind%%(*}" 2>/dev/null | grep -q '^function$' 2>/dev/null;
+        then
+            "set__${bind%%(*}" $short_setting $type $bind
+    
+        elif [[ "$bind" == "null" ]]
+        then
+            continue
+
+        # Save in a file
+        elif [[ "$type" == "file" ]]
+        then
+            if [[ "$bind" == "settings" ]]
+            then
+                ynh_die --message="File '${short_setting}' can't be stored in settings"
+            fi
+            local bind_file="$(echo "$bind" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
+            if [[ "${!short_setting}" == "" ]]
+            then
+                ynh_backup_if_checksum_is_different --file="$bind_file"
+                ynh_secure_remove --file="$bind_file"
+                ynh_delete_file_checksum --file="$bind_file" --update_only
+                ynh_print_info --message="File '$bind_file' removed"
+            else
+                ynh_backup_if_checksum_is_different --file="$bind_file"
+                if [[ "${!short_setting}" != "$bind_file" ]]
+                then
+                    cp "${!short_setting}" "$bind_file"
+                fi
+                ynh_store_file_checksum --file="$bind_file" --update_only
+                ynh_print_info --message="File '$bind_file' overwrited with ${!short_setting}"
+            fi
+
+        # Save value in app settings
+        elif [[ "$bind" == "settings" ]]
+        then
+            ynh_app_setting_set --app=$app --key=$short_setting --value="${!short_setting}"
+            ynh_print_info --message="Configuration key '$short_setting' edited in app settings"
+
+        # Save multiline text in a file
+        elif [[ "$type" == "text" ]]
+        then
+            if [[ "$bind" == *":"* ]]
+            then
+                ynh_die --message="For technical reasons, multiline text '${short_setting}' can't be stored automatically  in a variable file, you have to create custom getter/setter"
+            fi
+            local bind_file="$(echo "$bind" | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
+            ynh_backup_if_checksum_is_different --file="$bind_file"
+            echo "${!short_setting}" > "$bind_file"
+            ynh_store_file_checksum --file="$bind_file" --update_only
+            ynh_print_info --message="File '$bind_file' overwrited with the content you provieded in '${short_setting}' question"
+
+        # Set value into a kind of key/value file
+        else
+            local bind_after=""
+            local bind_key="$(echo "$bind" | cut -d: -f1)"
+            bind_key=${bind_key:-$short_setting}
+            if [[ "$bind_key" == *">"* ]];
+            then
+                bind_after="$(echo "${bind_key}" | cut -d'>' -f1)"
+                bind_key="$(echo "${bind_key}" | cut -d'>' -f2)"
+            fi
+            local bind_file="$(echo "$bind" | cut -d: -f2 | sed s@__FINALPATH__@$final_path@ | sed s/__APP__/$app/)"
+
+            ynh_backup_if_checksum_is_different --file="$bind_file"
+            ynh_write_var_in_file --file="${bind_file}" --key="${bind_key}" --value="${!short_setting}" --after="${bind_after}"
+            ynh_store_file_checksum --file="$bind_file" --update_only
+
+            # We stored the info in settings in order to be able to upgrade the app
+            ynh_app_setting_set --app=$app --key=$short_setting --value="${!short_setting}"
+            ynh_print_info --message="Configuration key '$bind_key' edited into $bind_file"
+
+        fi
+    fi
+}
+_ynh_app_config_get() {
+    # From settings
+    local lines
+    lines=$(python3 << EOL
+import toml
+from collections import OrderedDict
+with open("../config_panel.toml", "r") as f:
+    file_content = f.read()
+loaded_toml = toml.loads(file_content, _dict=OrderedDict)
+
+for panel_name, panel in loaded_toml.items():
+    if not isinstance(panel, dict): continue
+    for section_name, section in panel.items():
+        if not isinstance(section, dict): continue
+        for name, param in section.items():
+            if not isinstance(param, dict):
+                continue
+            print(';'.join([
+                name,
+                param.get('type', 'string'),
+                param.get('bind', 'settings' if param.get('type', 'string') != 'file' else 'null')
+                ]))
+EOL
+)
+    for line in $lines
+    do
+        # Split line into short_setting, type and bind
+        IFS=';' read short_setting type bind <<< "$line"
+        binds[${short_setting}]="$bind"
+        types[${short_setting}]="$type"
+        file_hash[${short_setting}]=""
+        formats[${short_setting}]=""
+        ynh_app_config_get_one $short_setting $type $bind
+    done
+
+
+}
+
+_ynh_app_config_apply() {
+    for short_setting in "${!old[@]}"
+    do
+        ynh_app_config_apply_one $short_setting
+    done
+}
+
+_ynh_app_config_show() {
+    for short_setting in "${!old[@]}"
+    do
+        if [[ "${old[$short_setting]}" != YNH_NULL ]]
+        then
+            if [[ "${formats[$short_setting]}" == "yaml" ]]
+            then
+                ynh_return "${short_setting}:"
+                ynh_return "$(echo "${old[$short_setting]}" | sed 's/^/  /g')"
+            else
+                ynh_return "${short_setting}: "'"'"$(echo "${old[$short_setting]}" | sed 's/"/\\"/g' | sed ':a;N;$!ba;s/\n/\n\n/g')"'"'
+
+            fi
+        fi
+    done
+}
+
+_ynh_app_config_validate() {
+    # Change detection
+    ynh_script_progression --message="Checking what changed in the new configuration..." --weight=1
+    local nothing_changed=true
+    local changes_validated=true
+    for short_setting in "${!old[@]}"
+    do
+        changed[$short_setting]=false
+        if [ -z ${!short_setting+x} ]
+        then
+            # Assign the var with the old value in order to allows multiple
+            # args validation
+            declare "$short_setting"="${old[$short_setting]}"
+            continue
+        fi
+        if [ ! -z "${file_hash[${short_setting}]}" ]
+        then
+            file_hash[old__$short_setting]=""
+            file_hash[new__$short_setting]=""
+            if [ -f "${old[$short_setting]}" ]
+            then
+                file_hash[old__$short_setting]=$(sha256sum "${old[$short_setting]}" | cut -d' ' -f1)
+                if [ -z "${!short_setting}" ]
+                then
+                    changed[$short_setting]=true
+                    nothing_changed=false
+                fi
+            fi
+            if [ -f "${!short_setting}" ]
+            then
+                file_hash[new__$short_setting]=$(sha256sum "${!short_setting}" | cut -d' ' -f1)
+                if [[ "${file_hash[old__$short_setting]}" != "${file_hash[new__$short_setting]}" ]]
+                then
+                    changed[$short_setting]=true
+                    nothing_changed=false
+                fi
+            fi
+        else
+            if [[ "${!short_setting}" != "${old[$short_setting]}" ]]
+            then
+                changed[$short_setting]=true
+                nothing_changed=false
+            fi
+        fi
+    done
+    if [[ "$nothing_changed" == "true" ]]
+    then
+        ynh_print_info --message="Nothing has changed"
+        exit 0
+    fi
+
+    # Run validation if something is changed
+    ynh_script_progression --message="Validating the new configuration..." --weight=1
+
+    for short_setting in "${!old[@]}"
+    do
+        [[ "${changed[$short_setting]}" == "false" ]] && continue
+        local result=""
+        if type -t validate__$short_setting | grep -q '^function$' 2>/dev/null;
+        then
+            result="$(validate__$short_setting)"
+        elif [[ "$bind" == *"("* ]] && type -t "validate__${bind%%(*}" 2>/dev/null | grep -q '^function$' 2>/dev/null;
+        then
+            "validate__${bind%%(*}" $short_setting
+        fi
+        if [ -n "$result" ]
+        then
+            #
+            # Return a yaml such as:
+            #
+            # validation_errors:
+            #   some_key: "An error message"
+            #   some_other_key: "Another error message"
+            #
+            # We use changes_validated to know if this is
+            # the first validation error
+            if [[ "$changes_validated" == true ]]
+            then
+                ynh_return "validation_errors:"
+            fi
+            ynh_return "  ${short_setting}: \"$result\""
+            changes_validated=false
+        fi
+    done
+
+    # If validation failed, exit the script right now (instead of going into apply)
+    # Yunohost core will pick up the errors returned via ynh_return previously
+    if [[ "$changes_validated" == "false" ]]
+    then
+        exit 0
+    fi
+
+}
+
+ynh_app_config_get_one() {
+    _ynh_app_config_get_one $1 $2 $3
+}
+
+ynh_app_config_get() {
+    _ynh_app_config_get
+}
+
+ynh_app_config_show() {
+    _ynh_app_config_show
+}
+
+ynh_app_config_validate() {
+    _ynh_app_config_validate
+}
+
+ynh_app_config_apply_one() {
+    _ynh_app_config_apply_one $1
+}
+ynh_app_config_apply() {
+    _ynh_app_config_apply
+}
+
+ynh_app_config_run() {
+    declare -Ag old=()
+    declare -Ag changed=()
+    declare -Ag file_hash=()
+    declare -Ag binds=()
+    declare -Ag types=()
+    declare -Ag formats=()
+
+    case $1 in
+        show)
+            ynh_app_config_get
+            ynh_app_config_show
+            ;;
+        apply)
+            max_progression=4
+            ynh_script_progression --message="Reading config panel description and current configuration..."
+            ynh_app_config_get
+
+            ynh_app_config_validate
+
+            ynh_script_progression --message="Applying the new configuration..."
+            ynh_app_config_apply
+            ynh_script_progression --message="Configuration of $app completed" --last
+            ;;
+    esac
+}
+

data/helpers.d/fail2ban

@@ -12,18 +12,14 @@
 #
 # usage 2: ynh_add_fail2ban_config --use_template [--others_var="list of others variables to replace"]
 # | arg: -t, --use_template - Use this helper in template mode
-# | arg: -v, --others_var=  - List of others variables to replace separeted by a space
-# |                           for example : 'var_1 var_2 ...'
+# | arg: -v, --others_var=  - List of others variables to replace separeted by a space for example : 'var_1 var_2 ...'
 #
-# This will use a template in ../conf/f2b_jail.conf and ../conf/f2b_filter.conf
-#   __APP__      by  $app
-#
-#  You can dynamically replace others variables by example :
-#   __VAR_1__    by $var_1
-#   __VAR_2__    by $var_2
+# This will use a template in `../conf/f2b_jail.conf` and `../conf/f2b_filter.conf`
+# See the documentation of `ynh_add_config` for a description of the template
+# format and how placeholders are replaced with actual variables.
 #
 # Generally your template will look like that by example (for synapse):
-#
+# ```
 # f2b_jail.conf:
 #     [__APP__]
 #     enabled = true
@@ -31,7 +27,8 @@
 #     filter = __APP__
 #     logpath = /var/log/__APP__/logfile.log
 #     maxretry = 3
-#
+# ```
+# ```
 # f2b_filter.conf:
 #     [INCLUDES]
 #     before = common.conf
@@ -44,99 +41,81 @@
 #    failregex = ^%(__synapse_start_line)s INFO \- POST\-(\d+)\- <HOST> \- \d+ \- Received request\: POST /_matrix/client/r0/login\??<SKIPLINES>%(__synapse_start_line)s INFO \- POST\-\1\- Got login request with identifier: \{u'type': u'm.id.user', u'user'\: u'(.+?)'\}, medium\: None, address: None, user\: u'\5'<SKIPLINES>%(__synapse_start_line)s WARNING \- \- (Attempted to login as @\5\:.+ but they do not exist|Failed password login for user @\5\:.+)$
 #
 #     ignoreregex =
+# ```
 #
 # -----------------------------------------------------------------------------
 #
 # Note about the "failregex" option:
-#          regex to match the password failure messages in the logfile. The
-#          host must be matched by a group named "host". The tag "<HOST>" can
-#          be used for standard IP/hostname matching and is only an alias for
-#          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
 #
-#          You can find some more explainations about how to make a regex here :
-#          https://www.fail2ban.org/wiki/index.php/MANUAL_0_8#Filters
+# regex to match the password failure messages in the logfile. The host must be
+# matched by a group named "`host`". The tag "`<HOST>`" can be used for standard
+# IP/hostname matching and is only an alias for `(?:::f{4,6}:)?(?P<host>[\w\-.^_]+)`
+#
+# You can find some more explainations about how to make a regex here :
+# https://www.fail2ban.org/wiki/index.php/MANUAL_0_8#Filters
 #
 # Note that the logfile need to exist before to call this helper !!
 #
 # To validate your regex you can test with this command:
+# ```
 # fail2ban-regex /var/log/YOUR_LOG_FILE_PATH /etc/fail2ban/filter.d/YOUR_APP.conf
+# ```
 #
-# Requires YunoHost version 3.5.0 or higher.
+# Requires YunoHost version 4.1.0 or higher.
 ynh_add_fail2ban_config () {
-  # Declare an array to define the options of this helper.
-  local legacy_args=lrmptv
-  declare -Ar args_array=( [l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= [t]=use_template [v]=others_var=)
-  local logpath
-  local failregex
-  local max_retry
-  local ports
-  local others_var
-  local use_template
-  # Manage arguments with getopts
-  ynh_handle_getopts_args "$@"
-  use_template="${use_template:-0}"
-  max_retry=${max_retry:-3}
-  ports=${ports:-http,https}
+    # Declare an array to define the options of this helper.
+    local legacy_args=lrmptv
+    local -A args_array=( [l]=logpath= [r]=failregex= [m]=max_retry= [p]=ports= [t]=use_template  [v]=others_var=)
+    local logpath
+    local failregex
+    local max_retry
+    local ports
+    local others_var
+    local use_template
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    max_retry=${max_retry:-3}
+    ports=${ports:-http,https}
+    others_var="${others_var:-}"
+    use_template="${use_template:-0}"
 
-  finalfail2banjailconf="/etc/fail2ban/jail.d/$app.conf"
-  finalfail2banfilterconf="/etc/fail2ban/filter.d/$app.conf"
-  ynh_backup_if_checksum_is_different "$finalfail2banjailconf"
-  ynh_backup_if_checksum_is_different "$finalfail2banfilterconf"
+    [[ -z "$others_var" ]] || ynh_print_warn --message="Packagers: using --others_var is unecessary since YunoHost 4.2"
 
-  if [ $use_template -eq 1 ]
-  then
-    # Usage 2, templates
-    cp ../conf/f2b_jail.conf $finalfail2banjailconf
-    cp ../conf/f2b_filter.conf $finalfail2banfilterconf
-
-    if [ -n "${app:-}" ]
+    if [ $use_template -ne 1 ]
     then
-      ynh_replace_string "__APP__" "$app" "$finalfail2banjailconf"
-      ynh_replace_string "__APP__" "$app" "$finalfail2banfilterconf"
-    fi
+        # Usage 1, no template. Build a config file from scratch.
+        test -n "$logpath" || ynh_die --message="ynh_add_fail2ban_config expects a logfile path as first argument and received nothing."
+        test -n "$failregex" || ynh_die --message="ynh_add_fail2ban_config expects a failure regex as second argument and received nothing."
 
-    # Replace all other variable given as arguments
-    for var_to_replace in ${others_var:-}; do
-      # ${var_to_replace^^} make the content of the variable on upper-cases
-      # ${!var_to_replace} get the content of the variable named $var_to_replace
-      ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banjailconf"
-      ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalfail2banfilterconf"
-    done
-
-  else
-    # Usage 1, no template. Build a config file from scratch.
-    test -n "$logpath" || ynh_die "ynh_add_fail2ban_config expects a logfile path as first argument and received nothing."
-    test -n "$failregex" || ynh_die "ynh_add_fail2ban_config expects a failure regex as second argument and received nothing."
-
-    tee $finalfail2banjailconf <<EOF
-[$app]
+        echo "
+[__APP__]
 enabled = true
-port = $ports
-filter = $app
-logpath = $logpath
-maxretry = $max_retry
-EOF
+port = __PORTS__
+filter = __APP__
+logpath = __LOGPATH__
+maxretry = __MAX_RETRY__
+" > $YNH_APP_BASEDIR/conf/f2b_jail.conf
 
-    tee $finalfail2banfilterconf <<EOF
+       echo "
 [INCLUDES]
 before = common.conf
 [Definition]
-failregex = $failregex
+failregex = __FAILREGEX__
 ignoreregex =
-EOF
-  fi
+" > $YNH_APP_BASEDIR/conf/f2b_filter.conf
+    fi
 
-  # Common to usage 1 and 2.
-  ynh_store_file_checksum "$finalfail2banjailconf"
-  ynh_store_file_checksum "$finalfail2banfilterconf"
+    ynh_add_config --template="$YNH_APP_BASEDIR/conf/f2b_jail.conf" --destination="/etc/fail2ban/jail.d/$app.conf"
+    ynh_add_config --template="$YNH_APP_BASEDIR/conf/f2b_filter.conf" --destination="/etc/fail2ban/filter.d/$app.conf"
 
-  ynh_systemd_action --service_name=fail2ban --action=reload
+    ynh_systemd_action --service_name=fail2ban --action=reload --line_match="(Started|Reloaded) Fail2Ban Service" --log_path=systemd
 
-  local fail2ban_error="$(journalctl -u fail2ban | tail -n50 | grep "WARNING.*$app.*")"
-  if [[ -n "$fail2ban_error" ]]; then
-    ynh_print_err --message="Fail2ban failed to load the jail for $app"
-    ynh_print_warn --message="${fail2ban_error#*WARNING}"
-  fi
+    local fail2ban_error="$(journalctl --no-hostname --unit=fail2ban | tail --lines=50 | grep "WARNING.*$app.*")"
+    if [[ -n "$fail2ban_error" ]]
+    then
+        ynh_print_err --message="Fail2ban failed to load the jail for $app"
+        ynh_print_warn --message="${fail2ban_error#*WARNING}"
+    fi
 }
 
 # Remove the dedicated fail2ban config (jail and filter conf files)
@@ -145,7 +124,7 @@ EOF
 #
 # Requires YunoHost version 3.5.0 or higher.
 ynh_remove_fail2ban_config () {
-  ynh_secure_remove "/etc/fail2ban/jail.d/$app.conf"
-  ynh_secure_remove "/etc/fail2ban/filter.d/$app.conf"
-  ynh_systemd_action --service_name=fail2ban --action=reload
+    ynh_secure_remove --file="/etc/fail2ban/jail.d/$app.conf"
+    ynh_secure_remove --file="/etc/fail2ban/filter.d/$app.conf"
+    ynh_systemd_action --service_name=fail2ban --action=reload
 }

data/helpers.d/getopts

@@ -6,7 +6,7 @@
 #
 # example: function my_helper()
 # {
-#     declare -Ar args_array=( [a]=arg1= [b]=arg2= [c]=arg3 )
+#     local -A args_array=( [a]=arg1= [b]=arg2= [c]=arg3 )
 #     local arg1
 #     local arg2
 #     local arg3
@@ -22,13 +22,13 @@
 # This helper need an array, named "args_array" with all the arguments used by the helper
 # 	that want to use ynh_handle_getopts_args
 # Be carreful, this array has to be an associative array, as the following example:
-# declare -Ar args_array=( [a]=arg1 [b]=arg2= [c]=arg3 )
+# local -A args_array=( [a]=arg1 [b]=arg2= [c]=arg3 )
 # Let's explain this array:
 # a, b and c are short options, -a, -b and -c
 # arg1, arg2 and arg3 are the long options associated to the previous short ones. --arg1, --arg2 and --arg3
 # For each option, a short and long version has to be defined.
 # Let's see something more significant
-# declare -Ar args_array=( [u]=user [f]=finalpath= [d]=database )
+# local -A args_array=( [u]=user [f]=finalpath= [d]=database )
 #
 # NB: Because we're using 'declare' without -g, the array will be declared as a local variable.
 #
@@ -46,173 +46,185 @@
 #
 # Requires YunoHost version 3.2.2 or higher.
 ynh_handle_getopts_args () {
-	# Manage arguments only if there's some provided
-	set +x
-	if [ $# -ne 0 ]
-	then
-		# Store arguments in an array to keep each argument separated
-		local arguments=("$@")
+    # Manage arguments only if there's some provided
+    set +o xtrace # set +x
+    if [ $# -ne 0 ]
+    then
+        # Store arguments in an array to keep each argument separated
+        local arguments=("$@")
 
-		# For each option in the array, reduce to short options for getopts (e.g. for [u]=user, --user will be -u)
-		# And built parameters string for getopts
-		# ${!args_array[@]} is the list of all option_flags in the array (An option_flag is 'u' in [u]=user, user is a value)
-		local getopts_parameters=""
-		local option_flag=""
-		for option_flag in "${!args_array[@]}"
-		do
-			# Concatenate each option_flags of the array to build the string of arguments for getopts
-			# Will looks like 'abcd' for -a -b -c -d
-			# If the value of an option_flag finish by =, it's an option with additionnal values. (e.g. --user bob or -u bob)
-			# Check the last character of the value associate to the option_flag
-			if [ "${args_array[$option_flag]: -1}" = "=" ]
-			then
-				# For an option with additionnal values, add a ':' after the letter for getopts.
-				getopts_parameters="${getopts_parameters}${option_flag}:"
-			else
-				getopts_parameters="${getopts_parameters}${option_flag}"
-			fi
-			# Check each argument given to the function
-			local arg=""
-			# ${#arguments[@]} is the size of the array
-			for arg in `seq 0 $(( ${#arguments[@]} - 1 ))`
-			do
-				# Escape options' values starting with -. Otherwise the - will be considered as another option.
-				arguments[arg]="${arguments[arg]//--${args_array[$option_flag]}-/--${args_array[$option_flag]}\\TOBEREMOVED\\-}"
-				# And replace long option (value of the option_flag) by the short option, the option_flag itself
-				# (e.g. for [u]=user, --user will be -u)
-				# Replace long option with =
-				arguments[arg]="${arguments[arg]//--${args_array[$option_flag]}/-${option_flag} }"
-				# And long option without =
-				arguments[arg]="${arguments[arg]//--${args_array[$option_flag]%=}/-${option_flag}}"
-			done
-		done
+        # For each option in the array, reduce to short options for getopts (e.g. for [u]=user, --user will be -u)
+        # And built parameters string for getopts
+        # ${!args_array[@]} is the list of all option_flags in the array (An option_flag is 'u' in [u]=user, user is a value)
+        local getopts_parameters=""
+        local option_flag=""
+        for option_flag in "${!args_array[@]}"
+        do
+            # Concatenate each option_flags of the array to build the string of arguments for getopts
+            # Will looks like 'abcd' for -a -b -c -d
+            # If the value of an option_flag finish by =, it's an option with additionnal values. (e.g. --user bob or -u bob)
+            # Check the last character of the value associate to the option_flag
+            if [ "${args_array[$option_flag]: -1}" = "=" ]
+            then
+                # For an option with additionnal values, add a ':' after the letter for getopts.
+                getopts_parameters="${getopts_parameters}${option_flag}:"
+            else
+                getopts_parameters="${getopts_parameters}${option_flag}"
+            fi
+            # Check each argument given to the function
+            local arg=""
+            # ${#arguments[@]} is the size of the array
+            for arg in `seq 0 $(( ${#arguments[@]} - 1 ))`
+            do
+                # Escape options' values starting with -. Otherwise the - will be considered as another option.
+                arguments[arg]="${arguments[arg]//--${args_array[$option_flag]}-/--${args_array[$option_flag]}\\TOBEREMOVED\\-}"
+                # And replace long option (value of the option_flag) by the short option, the option_flag itself
+                # (e.g. for [u]=user, --user will be -u)
+                # Replace long option with = (match the beginning of the argument)
+                arguments[arg]="$(echo "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]}/-${option_flag} /")"
+                # And long option without = (match the whole line)
+                arguments[arg]="$(echo "${arguments[arg]}" | sed "s/^--${args_array[$option_flag]%=}$/-${option_flag} /")"
+            done
+        done
 
-		# Read and parse all the arguments
-		# Use a function here, to use standart arguments $@ and be able to use shift.
-		parse_arg () {
-			# Read all arguments, until no arguments are left
-			while [ $# -ne 0 ]
-			do
-				# Initialize the index of getopts
-				OPTIND=1
-				# Parse with getopts only if the argument begin by -, that means the argument is an option
-				# getopts will fill $parameter with the letter of the option it has read.
-				local parameter=""
-				getopts ":$getopts_parameters" parameter || true
+        # Read and parse all the arguments
+        # Use a function here, to use standart arguments $@ and be able to use shift.
+        parse_arg () {
+            # Read all arguments, until no arguments are left
+            while [ $# -ne 0 ]
+            do
+                # Initialize the index of getopts
+                OPTIND=1
+                # Parse with getopts only if the argument begin by -, that means the argument is an option
+                # getopts will fill $parameter with the letter of the option it has read.
+                local parameter=""
+                getopts ":$getopts_parameters" parameter || true
 
-				if [ "$parameter" = "?" ]
-				then
-					ynh_die --message="Invalid argument: -${OPTARG:-}"
-				elif [ "$parameter" = ":" ]
-				then
-					ynh_die --message="-$OPTARG parameter requires an argument."
-				else
-					local shift_value=1
-					# Use the long option, corresponding to the short option read by getopts, as a variable
-					# (e.g. for [u]=user, 'user' will be used as a variable)
-					# Also, remove '=' at the end of the long option
-					# The variable name will be stored in 'option_var'
-					local option_var="${args_array[$parameter]%=}"
-					# If this option doesn't take values
-					# if there's a '=' at the end of the long option name, this option takes values
-					if [ "${args_array[$parameter]: -1}" != "=" ]
-					then
-						# 'eval ${option_var}' will use the content of 'option_var'
-						eval ${option_var}=1
-					else
-						# Read all other arguments to find multiple value for this option.
-						# Load args in a array
-						local all_args=("$@")
+                if [ "$parameter" = "?" ]
+                then
+                    ynh_die --message="Invalid argument: -${OPTARG:-}"
+                elif [ "$parameter" = ":" ]
+                then
+                    ynh_die --message="-$OPTARG parameter requires an argument."
+                else
+                    local shift_value=1
+                    # Use the long option, corresponding to the short option read by getopts, as a variable
+                    # (e.g. for [u]=user, 'user' will be used as a variable)
+                    # Also, remove '=' at the end of the long option
+                    # The variable name will be stored in 'option_var'
+                    local option_var="${args_array[$parameter]%=}"
+                    # If this option doesn't take values
+                    # if there's a '=' at the end of the long option name, this option takes values
+                    if [ "${args_array[$parameter]: -1}" != "=" ]
+                    then
+                        # 'eval ${option_var}' will use the content of 'option_var'
+                        eval ${option_var}=1
+                    else
+                        # Read all other arguments to find multiple value for this option.
+                        # Load args in a array
+                        local all_args=("$@")
 
-						# If the first argument is longer than 2 characters,
-						# There's a value attached to the option, in the same array cell
-						if [ ${#all_args[0]} -gt 2 ]; then
-							# Remove the option and the space, so keep only the value itself.
-							all_args[0]="${all_args[0]#-${parameter} }"
-							# Reduce the value of shift, because the option has been removed manually
-							shift_value=$(( shift_value - 1 ))
-						fi
+                        # If the first argument is longer than 2 characters,
+                        # There's a value attached to the option, in the same array cell
+                        if [ ${#all_args[0]} -gt 2 ]
+                        then
+                            # Remove the option and the space, so keep only the value itself.
+                            all_args[0]="${all_args[0]#-${parameter} }"
 
-						# Declare the content of option_var as a variable.
-						eval ${option_var}=""
-						# Then read the array value per value
-						local i
-						for i in `seq 0 $(( ${#all_args[@]} - 1 ))`
-						do
-							# If this argument is an option, end here.
-							if [ "${all_args[$i]:0:1}" == "-" ]
-							then
-								# Ignore the first value of the array, which is the option itself
-								if [ "$i" -ne 0 ]; then
-									break
-								fi
-							else
-								# Else, add this value to this option
-								# Each value will be separated by ';'
-								if [ -n "${!option_var}" ]
-								then
-									# If there's already another value for this option, add a ; before adding the new value
-									eval ${option_var}+="\;"
-								fi
+                            # At this point, if all_args[0] start with "-", then the argument is not well formed
+                            if [ "${all_args[0]:0:1}" == "-" ]
+                            then
+                                ynh_die --message="Argument \"${all_args[0]}\" not valid! Did you use a single \"-\" instead of two?"
+                            fi
+                            # Reduce the value of shift, because the option has been removed manually
+                            shift_value=$(( shift_value - 1 ))
+                        fi
 
-								# Remove the \ that escape - at beginning of values.
-								all_args[i]="${all_args[i]//\\TOBEREMOVED\\/}"
+                        # Declare the content of option_var as a variable.
+                        eval ${option_var}=""
+                        # Then read the array value per value
+                        local i
+                        for i in `seq 0 $(( ${#all_args[@]} - 1 ))`
+                        do
+                            # If this argument is an option, end here.
+                            if [ "${all_args[$i]:0:1}" == "-" ]
+                            then
+                                # Ignore the first value of the array, which is the option itself
+                                if [ "$i" -ne 0 ]; then
+                                    break
+                                fi
+                            else
+                                # Ignore empty parameters
+                                if [ -n "${all_args[$i]}" ]
+                                then
+                                    # Else, add this value to this option
+                                    # Each value will be separated by ';'
+                                    if [ -n "${!option_var}" ]
+                                    then
+                                        # If there's already another value for this option, add a ; before adding the new value
+                                        eval ${option_var}+="\;"
+                                    fi
 
-								# For the record.
-								# We're using eval here to get the content of the variable stored itself as simple text in $option_var...
-								# Other ways to get that content would be to use either ${!option_var} or declare -g ${option_var}
-								# But... ${!option_var} can't be used as left part of an assignation.
-								# declare -g ${option_var} will create a local variable (despite -g !) and will not be available for the helper itself.
-								# So... Stop fucking arguing each time that eval is evil... Go find an other working solution if you can find one!
+                                    # Remove the \ that escape - at beginning of values.
+                                    all_args[i]="${all_args[i]//\\TOBEREMOVED\\/}"
 
-								eval ${option_var}+='"${all_args[$i]}"'
-								shift_value=$(( shift_value + 1 ))
-							fi
-						done
-					fi
-				fi
+                                    # For the record.
+                                    # We're using eval here to get the content of the variable stored itself as simple text in $option_var...
+                                    # Other ways to get that content would be to use either ${!option_var} or declare -g ${option_var}
+                                    # But... ${!option_var} can't be used as left part of an assignation.
+                                    # declare -g ${option_var} will create a local variable (despite -g !) and will not be available for the helper itself.
+                                    # So... Stop fucking arguing each time that eval is evil... Go find an other working solution if you can find one!
 
-				# Shift the parameter and its argument(s)
-				shift $shift_value
-			done
-		}
+                                    eval ${option_var}+='"${all_args[$i]}"'
+                                fi
+                                shift_value=$(( shift_value + 1 ))
+                            fi
+                        done
+                    fi
+                fi
 
-		# LEGACY MODE
-		# Check if there's getopts arguments
-		if [ "${arguments[0]:0:1}" != "-" ]
-		then
-			# If not, enter in legacy mode and manage the arguments as positionnal ones..
-			# Dot not echo, to prevent to go through a helper output. But print only in the log.
-			set -x; echo "! Helper used in legacy mode !" > /dev/null; set +x
-			local i
-			for i in `seq 0 $(( ${#arguments[@]} -1 ))`
-			do
-				# Try to use legacy_args as a list of option_flag of the array args_array
-				# Otherwise, fallback to getopts_parameters to get the option_flag. But an associative arrays isn't always sorted in the correct order...
-				# Remove all ':' in getopts_parameters
-				getopts_parameters=${legacy_args:-${getopts_parameters//:}}
-				# Get the option_flag from getopts_parameters, by using the option_flag according to the position of the argument.
-				option_flag=${getopts_parameters:$i:1}
-				if [ -z "$option_flag" ]; then
-						ynh_print_warn --message="Too many arguments ! \"${arguments[$i]}\" will be ignored."
-						continue
-				fi
-				# Use the long option, corresponding to the option_flag, as a variable
-				# (e.g. for [u]=user, 'user' will be used as a variable)
-				# Also, remove '=' at the end of the long option
-				# The variable name will be stored in 'option_var'
-				local option_var="${args_array[$option_flag]%=}"
+                # Shift the parameter and its argument(s)
+                shift $shift_value
+            done
+        }
 
-				# Store each value given as argument in the corresponding variable
-				# The values will be stored in the same order than $args_array
-				eval ${option_var}+='"${arguments[$i]}"'
-			done
-			unset legacy_args
-		else
-			# END LEGACY MODE
-			# Call parse_arg and pass the modified list of args as an array of arguments.
-			parse_arg "${arguments[@]}"
-		fi
-	fi
-	set -x
+        # LEGACY MODE
+        # Check if there's getopts arguments
+        if [ "${arguments[0]:0:1}" != "-" ]
+        then
+            # If not, enter in legacy mode and manage the arguments as positionnal ones..
+            # Dot not echo, to prevent to go through a helper output. But print only in the log.
+            set -x; echo "! Helper used in legacy mode !" > /dev/null; set +x
+            local i
+            for i in `seq 0 $(( ${#arguments[@]} -1 ))`
+            do
+                # Try to use legacy_args as a list of option_flag of the array args_array
+                # Otherwise, fallback to getopts_parameters to get the option_flag. But an associative arrays isn't always sorted in the correct order...
+                # Remove all ':' in getopts_parameters
+                getopts_parameters=${legacy_args:-${getopts_parameters//:}}
+                # Get the option_flag from getopts_parameters, by using the option_flag according to the position of the argument.
+                option_flag=${getopts_parameters:$i:1}
+                if [ -z "$option_flag" ]
+                then
+                        ynh_print_warn --message="Too many arguments ! \"${arguments[$i]}\" will be ignored."
+                        continue
+                fi
+                # Use the long option, corresponding to the option_flag, as a variable
+                # (e.g. for [u]=user, 'user' will be used as a variable)
+                # Also, remove '=' at the end of the long option
+                # The variable name will be stored in 'option_var'
+                local option_var="${args_array[$option_flag]%=}"
+
+                # Store each value given as argument in the corresponding variable
+                # The values will be stored in the same order than $args_array
+                eval ${option_var}+='"${arguments[$i]}"'
+            done
+            unset legacy_args
+        else
+            # END LEGACY MODE
+            # Call parse_arg and pass the modified list of args as an array of arguments.
+            parse_arg "${arguments[@]}"
+        fi
+    fi
+    set -o xtrace # set -x
 }

data/helpers.d/hardware

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+# Get the total or free amount of RAM+swap on the system
+#
+# usage: ynh_get_ram [--free|--total] [--ignore_swap|--only_swap]
+# | arg: -f, --free         - Count free RAM+swap
+# | arg: -t, --total        - Count total RAM+swap
+# | arg: -s, --ignore_swap  - Ignore swap, consider only real RAM
+# | arg: -o, --only_swap    - Ignore real RAM, consider only swap
+# | ret: the amount of free ram, in MB (MegaBytes)
+#
+# Requires YunoHost version 3.8.1 or higher.
+ynh_get_ram () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=ftso
+    local -A args_array=( [f]=free [t]=total [s]=ignore_swap [o]=only_swap )
+    local free
+    local total
+    local ignore_swap
+    local only_swap
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    ignore_swap=${ignore_swap:-0}
+    only_swap=${only_swap:-0}
+    free=${free:-0}
+    total=${total:-0}
+
+    if [ $free -eq $total ]
+    then
+        ynh_print_warn --message="You have to choose --free or --total when using ynh_get_ram"
+        ram=0
+    # Use the total amount of ram
+    elif [ $free -eq 1 ]
+    then
+        local free_ram=$(vmstat --stats --unit M | grep "free memory" | awk '{print $1}')
+        local free_swap=$(vmstat --stats --unit M | grep "free swap" | awk '{print $1}')
+        local free_ram_swap=$(( free_ram + free_swap ))
+
+        # Use the total amount of free ram
+        local ram=$free_ram_swap
+        if [ $ignore_swap -eq 1 ]
+        then
+            # Use only the amount of free ram
+            ram=$free_ram
+        elif [ $only_swap -eq 1 ]
+        then
+            # Use only the amount of free swap
+            ram=$free_swap
+        fi
+    elif [ $total -eq 1 ]
+    then
+        local total_ram=$(vmstat --stats --unit M | grep "total memory" | awk '{print $1}')
+        local total_swap=$(vmstat --stats --unit M | grep "total swap" | awk '{print $1}')
+        local total_ram_swap=$(( total_ram + total_swap ))
+
+        local ram=$total_ram_swap
+        if [ $ignore_swap -eq 1 ]
+        then
+            # Use only the amount of free ram
+            ram=$total_ram
+        elif [ $only_swap -eq 1 ]
+        then
+            # Use only the amount of free swap
+            ram=$total_swap
+        fi
+    fi
+
+    echo $ram
+}
+
+# Return 0 or 1 depending if the system has a given amount of RAM+swap free or total
+#
+# usage: ynh_require_ram --required=RAM [--free|--total] [--ignore_swap|--only_swap]
+# | arg: -r, --required=    - The amount to require, in MB
+# | arg: -f, --free         - Count free RAM+swap
+# | arg: -t, --total        - Count total RAM+swap
+# | arg: -s, --ignore_swap  - Ignore swap, consider only real RAM
+# | arg: -o, --only_swap    - Ignore real RAM, consider only swap
+# | ret: 1 if the ram is under the requirement, 0 otherwise.
+#
+# Requires YunoHost version 3.8.1 or higher.
+ynh_require_ram () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=rftso
+    local -A args_array=( [r]=required= [f]=free [t]=total [s]=ignore_swap [o]=only_swap )
+    local required
+    local free
+    local total
+    local ignore_swap
+    local only_swap
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    # Dunno if that's the right way to do, but that's some black magic to be able to
+    # forward the bool args to ynh_get_ram easily?
+    # If the variable $free is not empty, set it to '--free'
+    free=${free:+--free}
+    total=${total:+--total}
+    ignore_swap=${ignore_swap:+--ignore_swap}
+    only_swap=${only_swap:+--only_swap}
+
+    local ram=$(ynh_get_ram $free $total $ignore_swap $only_swap)
+
+    if [ $ram -lt $required ]
+    then
+        return 1
+    else
+        return 0
+    fi
+}

data/helpers.d/logging

@@ -3,35 +3,39 @@
 # Print a message to stderr and exit
 #
 # usage: ynh_die --message=MSG [--ret_code=RETCODE]
+# | arg: -m, --message=     - Message to display
+# | arg: -c, --ret_code=    - Exit code to exit with
 #
 # Requires YunoHost version 2.4.0 or higher.
 ynh_die() {
-  # Declare an array to define the options of this helper.
-  local legacy_args=mc
-  declare -Ar args_array=( [m]=message= [c]=ret_code= )
-  local message
-  local ret_code
-  # Manage arguments with getopts
-  ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=mc
+    local -A args_array=( [m]=message= [c]=ret_code= )
+    local message
+    local ret_code
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    ret_code=${ret_code:-1}
 
-  echo "$message" 1>&2
-  exit "${ret_code:-1}"
+    echo "$message" 1>&2
+    exit "$ret_code"
 }
 
 # Display a message in the 'INFO' logging category
 #
 # usage: ynh_print_info --message="Some message"
+# | arg: -m, --message=     - Message to display
 #
 # Requires YunoHost version 3.2.0 or higher.
 ynh_print_info() {
     # Declare an array to define the options of this helper.
     local legacy_args=m
-    declare -Ar args_array=( [m]=message= )
+    local -A args_array=( [m]=message= )
     local message
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    echo "$message" >> "$YNH_STDINFO"
+    echo "$message" >&$YNH_STDINFO
 }
 
 # Ignore the yunohost-cli log to prevent errors with conditional commands
@@ -45,12 +49,12 @@ ynh_print_info() {
 #
 # Requires YunoHost version 2.6.4 or higher.
 ynh_no_log() {
-  local ynh_cli_log=/var/log/yunohost/yunohost-cli.log
-  sudo cp -a ${ynh_cli_log} ${ynh_cli_log}-move
-  eval $@
-  local exit_code=$?
-  sudo mv ${ynh_cli_log}-move ${ynh_cli_log}
-  return $?
+    local ynh_cli_log=/var/log/yunohost/yunohost-cli.log
+    cp --archive ${ynh_cli_log} ${ynh_cli_log}-move
+    eval $@
+    local exit_code=$?
+    mv ${ynh_cli_log}-move ${ynh_cli_log}
+    return $exit_code
 }
 
 # Main printer, just in case in the future we have to change anything about that.
@@ -59,121 +63,111 @@ ynh_no_log() {
 #
 # Requires YunoHost version 3.2.0 or higher.
 ynh_print_log () {
-  echo -e "${1}"
+    echo -e "${1}"
 }
 
 # Print a warning on stderr
 #
 # usage: ynh_print_warn --message="Text to print"
-# | arg: -m, --message - The text to print
+# | arg: -m, --message=     - The text to print
 #
 # Requires YunoHost version 3.2.0 or higher.
 ynh_print_warn () {
-  # Declare an array to define the options of this helper.
-  local legacy_args=m
-  declare -Ar args_array=( [m]=message= )
-  local message
-  # Manage arguments with getopts
-  ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=m
+    local -A args_array=( [m]=message= )
+    local message
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
 
-  ynh_print_log "\e[93m\e[1m[WARN]\e[0m ${message}" >&2
+    ynh_print_log "${message}" >&2
 }
 
 # Print an error on stderr
 #
 # usage: ynh_print_err --message="Text to print"
-# | arg: -m, --message - The text to print
+# | arg: -m, --message=     - The text to print
 #
 # Requires YunoHost version 3.2.0 or higher.
 ynh_print_err () {
-  # Declare an array to define the options of this helper.
-  local legacy_args=m
-  declare -Ar args_array=( [m]=message= )
-  local message
-  # Manage arguments with getopts
-  ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=m
+    local -A args_array=( [m]=message= )
+    local message
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
 
-  ynh_print_log "\e[91m\e[1m[ERR]\e[0m ${message}" >&2
+    ynh_print_log "[Error] ${message}" >&2
 }
 
 # Execute a command and print the result as an error
 #
-# usage: ynh_exec_err your_command
-# usage: ynh_exec_err "your_command | other_command"
+# usage: ynh_exec_err "your_command [ | other_command ]"
+# | arg: command - command to execute
 #
 # When using pipes, double quotes are required - otherwise, this helper will run the first command, and the whole output will be sent through the next pipe.
 #
 # If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed.
 #
-# | arg: command - command to execute
-#
 # Requires YunoHost version 3.2.0 or higher.
 ynh_exec_err () {
-	ynh_print_err "$(eval $@)"
+    ynh_print_err "$(eval $@)"
 }
 
 # Execute a command and print the result as a warning
 #
-# usage: ynh_exec_warn your_command
-# usage: ynh_exec_warn "your_command | other_command"
+# usage: ynh_exec_warn "your_command [ | other_command ]"
+# | arg: command - command to execute
 #
 # When using pipes, double quotes are required - otherwise, this helper will run the first command, and the whole output will be sent through the next pipe.
 #
 # If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed.
 #
-# | arg: command - command to execute
-#
 # Requires YunoHost version 3.2.0 or higher.
 ynh_exec_warn () {
-	ynh_print_warn "$(eval $@)"
+    ynh_print_warn "$(eval $@)"
 }
 
 # Execute a command and force the result to be printed on stdout
 #
-# usage: ynh_exec_warn_less your_command
-# usage: ynh_exec_warn_less "your_command | other_command"
+# usage: ynh_exec_warn_less "your_command [ | other_command ]"
+# | arg: command - command to execute
 #
 # When using pipes, double quotes are required - otherwise, this helper will run the first command, and the whole output will be sent through the next pipe.
 #
 # If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed.
 #
-# | arg: command - command to execute
-#
 # Requires YunoHost version 3.2.0 or higher.
 ynh_exec_warn_less () {
-	eval $@ 2>&1
+    eval $@ 2>&1
 }
 
 # Execute a command and redirect stdout in /dev/null
 #
-# usage: ynh_exec_quiet your_command
-# usage: ynh_exec_quiet "your_command | other_command"
+# usage: ynh_exec_quiet "your_command [ | other_command ]"
+# | arg: command - command to execute
 #
 # When using pipes, double quotes are required - otherwise, this helper will run the first command, and the whole output will be sent through the next pipe.
 #
 # If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed.
 #
-# | arg: command - command to execute
-#
 # Requires YunoHost version 3.2.0 or higher.
 ynh_exec_quiet () {
-	eval $@ > /dev/null
+    eval $@ > /dev/null
 }
 
 # Execute a command and redirect stdout and stderr in /dev/null
 #
-# usage: ynh_exec_fully_quiet your_command
-# usage: ynh_exec_fully_quiet "your_command | other_command"
+# usage: ynh_exec_fully_quiet "your_command [ | other_command ]"
+# | arg: command - command to execute
 #
 # When using pipes, double quotes are required - otherwise, this helper will run the first command, and the whole output will be sent through the next pipe.
 #
 # If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed.
 #
-# | arg: command - command to execute
-#
 # Requires YunoHost version 3.2.0 or higher.
 ynh_exec_fully_quiet () {
-	eval $@ > /dev/null 2>&1
+    eval $@ > /dev/null 2>&1
 }
 
 # Remove any logs for all the following commands.
@@ -184,7 +178,7 @@ ynh_exec_fully_quiet () {
 #
 # Requires YunoHost version 3.2.0 or higher.
 ynh_print_OFF () {
-	exec {BASH_XTRACEFD}>/dev/null
+    exec {BASH_XTRACEFD}>/dev/null
 }
 
 # Restore the logging after ynh_print_OFF
@@ -193,9 +187,9 @@ ynh_print_OFF () {
 #
 # Requires YunoHost version 3.2.0 or higher.
 ynh_print_ON () {
-	exec {BASH_XTRACEFD}>&1
-	# Print an echo only for the log, to be able to know that ynh_print_ON has been called.
-	echo ynh_print_ON > /dev/null
+    exec {BASH_XTRACEFD}>&1
+    # Print an echo only for the log, to be able to know that ynh_print_ON has been called.
+    echo ynh_print_ON > /dev/null
 }
 
 # Initial definitions for ynh_script_progression
@@ -216,89 +210,90 @@ base_time=$(date +%s)
 # usage: ynh_script_progression --message=message [--weight=weight] [--time]
 # | arg: -m, --message= - The text to print
 # | arg: -w, --weight=  - The weight for this progression. This value is 1 by default. Use a bigger value for a longer part of the script.
-# | arg: -t, --time=    - Print the execution time since the last call to this helper. Especially usefull to define weights. The execution time is given for the duration since the previous call. So the weight should be applied to this previous call.
-# | arg: -l, --last=    - Use for the last call of the helper, to fill te progression bar.
+# | arg: -t, --time     - Print the execution time since the last call to this helper. Especially usefull to define weights. The execution time is given for the duration since the previous call. So the weight should be applied to this previous call.
+# | arg: -l, --last     - Use for the last call of the helper, to fill the progression bar.
 #
 # Requires YunoHost version 3.5.0 or higher.
 ynh_script_progression () {
-	set +x
-	# Declare an array to define the options of this helper.
-	local legacy_args=mwtl
-	declare -Ar args_array=( [m]=message= [w]=weight= [t]=time [l]=last )
-	local message
-	local weight
-	local time
-	local last
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
-	set +x
-	weight=${weight:-1}
-	time=${time:-0}
-	last=${last:-0}
+    set +o xtrace # set +x
+    # Declare an array to define the options of this helper.
+    local legacy_args=mwtl
+    local -A args_array=( [m]=message= [w]=weight= [t]=time [l]=last )
+    local message
+    local weight
+    local time
+    local last
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    # Re-disable xtrace, ynh_handle_getopts_args set it back
+    set +o xtrace # set +x
+    weight=${weight:-1}
+    time=${time:-0}
+    last=${last:-0}
 
-	# Get execution time since the last $base_time
-	local exec_time=$(( $(date +%s) - $base_time ))
-	base_time=$(date +%s)
+    # Get execution time since the last $base_time
+    local exec_time=$(( $(date +%s) - $base_time ))
+    base_time=$(date +%s)
 
-	# Compute $max_progression (if we didn't already)
-	if [ "$max_progression" = -1 ]
-	then
-		# Get the number of occurrences of 'ynh_script_progression' in the script. Except those are commented.
-		local helper_calls="$(grep --count "^[^#]*ynh_script_progression" $0)"
-		# Get the number of call with a weight value
-		local weight_calls=$(grep --perl-regexp --count "^[^#]*ynh_script_progression.*(--weight|-w )" $0)
+    # Compute $max_progression (if we didn't already)
+    if [ "$max_progression" = -1 ]
+    then
+        # Get the number of occurrences of 'ynh_script_progression' in the script. Except those are commented.
+        local helper_calls="$(grep --count "^[^#]*ynh_script_progression" $0)"
+        # Get the number of call with a weight value
+        local weight_calls=$(grep --perl-regexp --count "^[^#]*ynh_script_progression.*(--weight|-w )" $0)
 
-		# Get the weight of each occurrences of 'ynh_script_progression' in the script using --weight
-		local weight_valuesA="$(grep --perl-regexp "^[^#]*ynh_script_progression.*--weight" $0 | sed 's/.*--weight[= ]\([[:digit:]]*\).*/\1/g')"
-		# Get the weight of each occurrences of 'ynh_script_progression' in the script using -w
-		local weight_valuesB="$(grep --perl-regexp "^[^#]*ynh_script_progression.*-w " $0 | sed 's/.*-w[= ]\([[:digit:]]*\).*/\1/g')"
-		# Each value will be on a different line.
-		# Remove each 'end of line' and replace it by a '+' to sum the values.
-		local weight_values=$(( $(echo "$weight_valuesA" | tr '\n' '+') + $(echo "$weight_valuesB" | tr '\n' '+') 0 ))
+        # Get the weight of each occurrences of 'ynh_script_progression' in the script using --weight
+        local weight_valuesA="$(grep --perl-regexp "^[^#]*ynh_script_progression.*--weight" $0 | sed 's/.*--weight[= ]\([[:digit:]]*\).*/\1/g')"
+        # Get the weight of each occurrences of 'ynh_script_progression' in the script using -w
+        local weight_valuesB="$(grep --perl-regexp "^[^#]*ynh_script_progression.*-w " $0 | sed 's/.*-w[= ]\([[:digit:]]*\).*/\1/g')"
+        # Each value will be on a different line.
+        # Remove each 'end of line' and replace it by a '+' to sum the values.
+        local weight_values=$(( $(echo "$weight_valuesA" | tr '\n' '+') + $(echo "$weight_valuesB" | tr '\n' '+') 0 ))
 
-		# max_progression is a total number of calls to this helper.
-		# 	Less the number of calls with a weight value.
-		# 	Plus the total of weight values
-		max_progression=$(( $helper_calls - $weight_calls + $weight_values ))
-	fi
+        # max_progression is a total number of calls to this helper.
+        # 	Less the number of calls with a weight value.
+        # 	Plus the total of weight values
+        max_progression=$(( $helper_calls - $weight_calls + $weight_values ))
+    fi
 
-	# Increment each execution of ynh_script_progression in this script by the weight of the previous call.
-	increment_progression=$(( $increment_progression + $previous_weight ))
-	# Store the weight of the current call in $previous_weight for next call
-	previous_weight=$weight
+    # Increment each execution of ynh_script_progression in this script by the weight of the previous call.
+    increment_progression=$(( $increment_progression + $previous_weight ))
+    # Store the weight of the current call in $previous_weight for next call
+    previous_weight=$weight
 
-	# Reduce $increment_progression to the size of the scale
-	if [ $last -eq 0 ]
-	then
-		local effective_progression=$(( $increment_progression * $progress_scale / $max_progression ))
-	# If last is specified, fill immediately the progression_bar
-	else
-		local effective_progression=$progress_scale
-	fi
+    # Reduce $increment_progression to the size of the scale
+    if [ $last -eq 0 ]
+    then
+        local effective_progression=$(( $increment_progression * $progress_scale / $max_progression ))
+    # If last is specified, fill immediately the progression_bar
+    else
+        local effective_progression=$progress_scale
+    fi
 
-	# Build $progression_bar from progress_string(0,1,2) according to $effective_progression and the weight of the current task
-	# expected_progression is the progression expected after the current task
-	local expected_progression="$(( ( $increment_progression + $weight ) * $progress_scale / $max_progression - $effective_progression ))"
-	if [ $last -eq 1 ]
-	then
-		expected_progression=0
-	fi
-	# left_progression is the progression not yet done
-	local left_progression="$(( $progress_scale - $effective_progression - $expected_progression ))"
-	# Build the progression bar with $effective_progression, work done, $expected_progression, current work and $left_progression, work to be done.
-	local progression_bar="${progress_string2:0:$effective_progression}${progress_string1:0:$expected_progression}${progress_string0:0:$left_progression}"
+    # Build $progression_bar from progress_string(0,1,2) according to $effective_progression and the weight of the current task
+    # expected_progression is the progression expected after the current task
+    local expected_progression="$(( ( $increment_progression + $weight ) * $progress_scale / $max_progression - $effective_progression ))"
+    if [ $last -eq 1 ]
+    then
+        expected_progression=0
+    fi
+    # left_progression is the progression not yet done
+    local left_progression="$(( $progress_scale - $effective_progression - $expected_progression ))"
+    # Build the progression bar with $effective_progression, work done, $expected_progression, current work and $left_progression, work to be done.
+    local progression_bar="${progress_string2:0:$effective_progression}${progress_string1:0:$expected_progression}${progress_string0:0:$left_progression}"
 
-	local print_exec_time=""
-	if [ $time -eq 1 ]
-	then
-		print_exec_time=" [$(date +%Hh%Mm,%Ss --date="0 + $exec_time sec")]"
-	fi
+    local print_exec_time=""
+    if [ $time -eq 1 ]
+    then
+        print_exec_time=" [$(date +%Hh%Mm,%Ss --date="0 + $exec_time sec")]"
+    fi
 
-	ynh_print_info "[$progression_bar] > ${message}${print_exec_time}"
-	set -x
+    ynh_print_info "[$progression_bar] > ${message}${print_exec_time}"
+    set -o xtrace # set -x
 }
 
-# Return data to the Yunohost core for later processing
+# Return data to the YunoHost core for later processing
 # (to be used by special hooks like app config panel and core diagnosis)
 #
 # usage: ynh_return somedata
@@ -316,63 +311,61 @@ ynh_return () {
 #
 # Requires YunoHost version 3.5.0 or higher.
 ynh_debug () {
-	# Disable set xtrace for the helper itself, to not pollute the debug log
-	set +x
-	# Declare an array to define the options of this helper.
-	local legacy_args=mt
-	declare -Ar args_array=( [m]=message= [t]=trace= )
-	local message
-	local trace
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
-	# Redisable xtrace, ynh_handle_getopts_args set it back
-	set +x
-	message=${message:-}
-	trace=${trace:-}
+    # Disable set xtrace for the helper itself, to not pollute the debug log
+    set +o xtrace # set +x
+    # Declare an array to define the options of this helper.
+    local legacy_args=mt
+    local -A args_array=( [m]=message= [t]=trace= )
+    local message
+    local trace
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    # Re-disable xtrace, ynh_handle_getopts_args set it back
+    set +o xtrace # set +x
+    message=${message:-}
+    trace=${trace:-}
 
-	if [ -n "$message" ]
-	then
-		ynh_print_log "\e[34m\e[1m[DEBUG]\e[0m ${message}" >&2
-	fi
+    if [ -n "$message" ]
+    then
+        ynh_print_log "[Debug] ${message}" >&2
+    fi
 
-	if [ "$trace" == "1" ]
-	then
-		ynh_debug --message="Enable debugging"
-		set +x
-		# Get the current file descriptor of xtrace
-		old_bash_xtracefd=$BASH_XTRACEFD
-		# Add the current file name and the line number of any command currently running while tracing.
-		PS4='$(basename ${BASH_SOURCE[0]})-L${LINENO}: '
-		# Force xtrace to stderr
-		BASH_XTRACEFD=2
-		# Force stdout to stderr
-		exec 1>&2
-	fi
-	if [ "$trace" == "0" ]
-	then
-		ynh_debug --message="Disable debugging"
-		set +x
-		# Put xtrace back to its original fild descriptor
-		BASH_XTRACEFD=$old_bash_xtracefd
-		# Restore stdout
-		exec 1>&1
-	fi
-	# Renable set xtrace
-	set -x
+    if [ "$trace" == "1" ]
+    then
+        ynh_debug --message="Enable debugging"
+        set +o xtrace # set +x
+        # Get the current file descriptor of xtrace
+        old_bash_xtracefd=$BASH_XTRACEFD
+        # Add the current file name and the line number of any command currently running while tracing.
+        PS4='$(basename ${BASH_SOURCE[0]})-L${LINENO}: '
+        # Force xtrace to stderr
+        BASH_XTRACEFD=2
+        # Force stdout to stderr
+        exec 1>&2
+    fi
+    if [ "$trace" == "0" ]
+    then
+        ynh_debug --message="Disable debugging"
+        set +o xtrace # set +x
+        # Put xtrace back to its original fild descriptor
+        BASH_XTRACEFD=$old_bash_xtracefd
+        # Restore stdout
+        exec 1>&1
+    fi
+    # Renable set xtrace
+    set -o xtrace # set -x
 }
 
 # Execute a command and print the result as debug
 #
-# usage: ynh_debug_exec your_command
-# usage: ynh_debug_exec "your_command | other_command"
+# usage: ynh_debug_exec "your_command [ | other_command ]"
+# | arg: command - command to execute
 #
 # When using pipes, double quotes are required - otherwise, this helper will run the first command, and the whole output will be sent through the next pipe.
 #
 # If the command to execute uses double quotes, they have to be escaped or they will be interpreted and removed.
 #
-# | arg: command - command to execute
-#
 # Requires YunoHost version 3.5.0 or higher.
 ynh_debug_exec () {
-	ynh_debug --message="$(eval $@)"
+    ynh_debug --message="$(eval $@)"
 }

data/helpers.d/logrotate

@@ -3,92 +3,99 @@
 # Use logrotate to manage the logfile
 #
 # usage: ynh_use_logrotate [--logfile=/log/file] [--nonappend] [--specific_user=user/group]
-# | arg: -l, --logfile - absolute path of logfile
-# | arg: -n, --nonappend - (optional) Replace the config file instead of appending this new config.
-# | arg: -u, --specific_user : run logrotate as the specified user and group. If not specified logrotate is runned as root.
+# | arg: -l, --logfile=         - absolute path of logfile
+# | arg: -n, --nonappend        - (optional) Replace the config file instead of appending this new config.
+# | arg: -u, --specific_user=   - run logrotate as the specified user and group. If not specified logrotate is runned as root.
 #
-# If no --logfile is provided, /var/log/${app} will be used as default.
-# logfile can be just a directory, or a full path to a logfile :
-# /parentdir/logdir
-# /parentdir/logdir/logfile.log
+# If no `--logfile` is provided, `/var/log/$app` will be used as default.
+# `logfile` can point to a directory or a file.
 #
 # It's possible to use this helper multiple times, each config will be added to
-# the same logrotate config file.  Unless you use the option --non-append
+# the same logrotate config file. Unless you use the option `--non-append`
 #
 # Requires YunoHost version 2.6.4 or higher.
+# Requires YunoHost version 3.2.0 or higher for the argument `--specific_user`
 ynh_use_logrotate () {
-	# Declare an array to define the options of this helper.
-	local legacy_args=lnuya
-	declare -Ar args_array=( [l]=logfile= [n]=nonappend [u]=specific_user= [y]=non [a]=append )
-	# [y]=non [a]=append are only for legacy purpose, to not fail on the old option '--non-append'
-	local logfile
-	local nonappend
-	local specific_user
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
-	local logfile="${logfile:-}"
-	local nonappend="${nonappend:-0}"
-	local specific_user="${specific_user:-}"
+    # Declare an array to define the options of this helper.
+    local legacy_args=lnuya
+    local -A args_array=( [l]=logfile= [n]=nonappend [u]=specific_user= [y]=non [a]=append )
+    # [y]=non [a]=append are only for legacy purpose, to not fail on the old option '--non-append'
+    local logfile
+    local nonappend
+    local specific_user
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    logfile="${logfile:-}"
+    nonappend="${nonappend:-0}"
+    specific_user="${specific_user:-}"
 
-	# LEGACY CODE - PRE GETOPTS
-	if [ $# -gt 0 ] && [ "$1" == "--non-append" ]; then
-		nonappend=1
-		# Destroy this argument for the next command.
-		shift
-	elif [ $# -gt 1 ] && [ "$2" == "--non-append" ]; then
-		nonappend=1
-	fi
+    # LEGACY CODE - PRE GETOPTS
+    if [ $# -gt 0 ] && [ "$1" == "--non-append" ]
+    then
+        nonappend=1
+        # Destroy this argument for the next command.
+        shift
+    elif [ $# -gt 1 ] && [ "$2" == "--non-append" ]
+    then
+        nonappend=1
+    fi
 
-	if [ $# -gt 0 ] && [ "$(echo ${1:0:1})" != "-" ]; then
-		if [ "$(echo ${1##*.})" == "log" ]; then	# Keep only the extension to check if it's a logfile
-			local logfile=$1	# In this case, focus logrotate on the logfile
-		else
-			local logfile=$1/*.log	# Else, uses the directory and all logfile into it.
-		fi
-	fi
-	# LEGACY CODE
+    if [ $# -gt 0 ] && [ "$(echo ${1:0:1})" != "-" ]
+    then
+        # If the given logfile parameter already exists as a file, or if it ends up with ".log",
+        # we just want to manage a single file
+        if [ -f "$1" ] || [ "$(echo ${1##*.})" == "log" ]
+        then
+            local logfile=$1
+        # Otherwise we assume we want to manage a directory and all its .log file inside
+        else
+            local logfile=$1/*.log
+        fi
+    fi
+    # LEGACY CODE
 
-	local customtee="tee -a"
-	if [ "$nonappend" -eq 1 ]; then
-		customtee="tee"
-	fi
-	if [ -n "$logfile" ]
-	then
-		if [ "$(echo ${logfile##*.})" != "log" ]; then	# Keep only the extension to check if it's a logfile
-			local logfile="$logfile/*.log"	# Else, uses the directory and all logfile into it.
-		fi
-	else
-		logfile="/var/log/${app}/*.log" # Without argument, use a defaut directory in /var/log
-	fi
-	local su_directive=""
-	if [[ -n $specific_user ]]; then
-		su_directive="	# Run logorotate as specific user - group
-	su ${specific_user%/*} ${specific_user#*/}"
-	fi
+    local customtee="tee --append"
+    if [ "$nonappend" -eq 1 ]; then
+        customtee="tee"
+    fi
+    if [ -n "$logfile" ]
+    then
+        if [ ! -f "$1" ] && [ "$(echo ${logfile##*.})" != "log" ]; then	# Keep only the extension to check if it's a logfile
+            local logfile="$logfile/*.log"	# Else, uses the directory and all logfile into it.
+        fi
+    else
+        logfile="/var/log/${app}/*.log" # Without argument, use a defaut directory in /var/log
+    fi
+    local su_directive=""
+    if [[ -n $specific_user ]]
+    then
+        su_directive="	# Run logorotate as specific user - group
+    su ${specific_user%/*} ${specific_user#*/}"
+    fi
 
-	cat > ./${app}-logrotate << EOF	# Build a config file for logrotate
+    cat > ./${app}-logrotate << EOF	# Build a config file for logrotate
 $logfile {
-		# Rotate if the logfile exceeds 100Mo
-	size 100M
-		# Keep 12 old log maximum
-	rotate 12
-		# Compress the logs with gzip
-	compress
-		# Compress the log at the next cycle. So keep always 2 non compressed logs
-	delaycompress
-		# Copy and truncate the log to allow to continue write on it. Instead of move the log.
-	copytruncate
-		# Do not do an error if the log is missing
-	missingok
-		# Not rotate if the log is empty
-	notifempty
-		# Keep old logs in the same dir
-	noolddir
-	$su_directive
+        # Rotate if the logfile exceeds 100Mo
+    size 100M
+        # Keep 12 old log maximum
+    rotate 12
+        # Compress the logs with gzip
+    compress
+        # Compress the log at the next cycle. So keep always 2 non compressed logs
+    delaycompress
+        # Copy and truncate the log to allow to continue write on it. Instead of move the log.
+    copytruncate
+        # Do not do an error if the log is missing
+    missingok
+        # Not rotate if the log is empty
+    notifempty
+        # Keep old logs in the same dir
+    noolddir
+    $su_directive
 }
 EOF
-	sudo mkdir -p $(dirname "$logfile")	# Create the log directory, if not exist
-	cat ${app}-logrotate | sudo $customtee /etc/logrotate.d/$app > /dev/null	# Append this config to the existing config file, or replace the whole config file (depending on $customtee)
+    mkdir --parents $(dirname "$logfile")	# Create the log directory, if not exist
+    cat ${app}-logrotate | $customtee /etc/logrotate.d/$app > /dev/null	# Append this config to the existing config file, or replace the whole config file (depending on $customtee)
 }
 
 # Remove the app's logrotate config.
@@ -97,7 +104,7 @@ EOF
 #
 # Requires YunoHost version 2.6.4 or higher.
 ynh_remove_logrotate () {
-	if [ -e "/etc/logrotate.d/$app" ]; then
-		sudo rm "/etc/logrotate.d/$app"
-	fi
+    if [ -e "/etc/logrotate.d/$app" ]; then
+        rm "/etc/logrotate.d/$app"
+    fi
 }

data/helpers.d/multimedia

@@ -0,0 +1,104 @@
+#!/bin/bash
+
+readonly MEDIA_GROUP=multimedia
+readonly MEDIA_DIRECTORY=/home/yunohost.multimedia
+
+# Initialize the multimedia directory system
+#
+# usage: ynh_multimedia_build_main_dir
+#
+# Requires YunoHost version 4.2 or higher.
+ynh_multimedia_build_main_dir() {
+
+    ## Création du groupe multimedia
+    groupadd -f $MEDIA_GROUP
+
+    ## Création des dossiers génériques
+    mkdir -p "$MEDIA_DIRECTORY"
+    mkdir -p "$MEDIA_DIRECTORY/share"
+    mkdir -p "$MEDIA_DIRECTORY/share/Music"
+    mkdir -p "$MEDIA_DIRECTORY/share/Picture"
+    mkdir -p "$MEDIA_DIRECTORY/share/Video"
+    mkdir -p "$MEDIA_DIRECTORY/share/eBook"
+
+    ## Création des dossiers utilisateurs
+    for user in $(yunohost user list --output-as json | jq -r '.users | keys[]')
+    do
+        mkdir -p "$MEDIA_DIRECTORY/$user"
+        mkdir -p "$MEDIA_DIRECTORY/$user/Music"
+        mkdir -p "$MEDIA_DIRECTORY/$user/Picture"
+        mkdir -p "$MEDIA_DIRECTORY/$user/Video"
+        mkdir -p "$MEDIA_DIRECTORY/$user/eBook"
+        ln -sfn "$MEDIA_DIRECTORY/share" "$MEDIA_DIRECTORY/$user/Share"
+        # Création du lien symbolique dans le home de l'utilisateur.
+        #link will only be created if the home directory of the user exists and if it's located in '/home' folder
+        local user_home="$(getent passwd $user | cut -d: -f6 | grep '^/home/')"
+        if [[ -d "$user_home" ]]; then
+            ln -sfn "$MEDIA_DIRECTORY/$user" "$user_home/Multimedia"
+        fi
+        # Propriétaires des dossiers utilisateurs.
+        chown -R $user "$MEDIA_DIRECTORY/$user"
+    done
+    # Default yunohost hooks for post_user_create,delete will take care
+    # of creating/deleting corresponding multimedia folders when users
+    # are created/deleted in the future...
+
+    ## Application des droits étendus sur le dossier multimedia.
+    # Droit d'écriture pour le groupe et le groupe multimedia en acl et droit de lecture pour other:
+    setfacl -RnL -m g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$MEDIA_DIRECTORY"
+    # Application de la même règle que précédemment, mais par défaut pour les nouveaux fichiers.
+    setfacl -RnL -m d:g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$MEDIA_DIRECTORY"
+    # Réglage du masque par défaut. Qui garantie (en principe...) un droit maximal à rwx. Donc pas de restriction de droits par l'acl.
+    setfacl -RL -m m::rwx "$MEDIA_DIRECTORY"
+}
+
+# Add a directory in yunohost.multimedia
+#
+# usage: ynh_multimedia_addfolder --source_dir="source_dir" --dest_dir="dest_dir"
+#
+# | arg: -s, --source_dir= - Source directory - The real directory which contains your medias.
+# | arg: -d, --dest_dir= - Destination directory - The name and the place of the symbolic link, relative to "/home/yunohost.multimedia"
+#
+# This "directory" will be a symbolic link to a existing directory.
+#
+# Requires YunoHost version 4.2 or higher.
+ynh_multimedia_addfolder() {
+
+    # Declare an array to define the options of this helper.
+    local legacy_args=sd
+    local -A args_array=( [s]=source_dir= [d]=dest_dir= )
+	local source_dir
+	local dest_dir
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    # Ajout d'un lien symbolique vers le dossier à partager
+	ln -sfn "$source_dir" "$MEDIA_DIRECTORY/$dest_dir"
+
+	## Application des droits étendus sur le dossier ajouté
+	# Droit d'écriture pour le groupe et le groupe multimedia en acl et droit de lecture pour other:
+	setfacl -RnL -m g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$source_dir"
+	# Application de la même règle que précédemment, mais par défaut pour les nouveaux fichiers.
+	setfacl -RnL -m d:g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$source_dir"
+	# Réglage du masque par défaut. Qui garantie (en principe...) un droit maximal à rwx. Donc pas de restriction de droits par l'acl.
+	setfacl -RL -m m::rwx "$source_dir"
+}
+
+# Allow an user to have an write authorisation in multimedia directories
+#
+# usage: ynh_multimedia_addaccess user_name
+#
+# | arg: -u, --user_name= - The name of the user which gain this access.
+#
+# Requires YunoHost version 4.2 or higher.
+ynh_multimedia_addaccess () {
+	# Declare an array to define the options of this helper.
+    local legacy_args=u
+	declare -Ar args_array=( [u]=user_name=)
+	local user_name
+	# Manage arguments with getopts
+	ynh_handle_getopts_args "$@"
+
+	groupadd -f multimedia
+	usermod -a -G multimedia $user_name
+}

data/helpers.d/mysql

@@ -1,22 +1,21 @@
 #!/bin/bash
 
-MYSQL_ROOT_PWD_FILE=/etc/yunohost/mysql
-
 # Open a connection as a user
 #
-# example: ynh_mysql_connect_as 'user' 'pass' <<< "UPDATE ...;"
-# example: ynh_mysql_connect_as 'user' 'pass' < /path/to/file.sql
-#
 # usage: ynh_mysql_connect_as --user=user --password=password [--database=database]
-# | arg: -u, --user - the user name to connect as
-# | arg: -p, --password - the user password
-# | arg: -d, --database - the database to connect to
+# | arg: -u, --user=        - the user name to connect as
+# | arg: -p, --password=    - the user password
+# | arg: -d, --database=    - the database to connect to
+#
+# examples:
+#    ynh_mysql_connect_as --user="user" --password="pass" <<< "UPDATE ...;"
+#    ynh_mysql_connect_as --user="user" --password="pass" < /path/to/file.sql
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_mysql_connect_as() {
     # Declare an array to define the options of this helper.
     local legacy_args=upd
-    declare -Ar args_array=( [u]=user= [p]=password= [d]=database= )
+    local -A args_array=( [u]=user= [p]=password= [d]=database= )
     local user
     local password
     local database
@@ -24,49 +23,57 @@ ynh_mysql_connect_as() {
     ynh_handle_getopts_args "$@"
     database="${database:-}"
 
-    mysql -u "$user" --password="$password" -B "$database"
+    mysql --user="$user" --password="$password" --batch "$database"
 }
 
 # Execute a command as root user
 #
 # usage: ynh_mysql_execute_as_root --sql=sql [--database=database]
-# | arg: -s, --sql - the SQL command to execute
-# | arg: -d, --database - the database to connect to
+# | arg: -s, --sql=         - the SQL command to execute
+# | arg: -d, --database=    - the database to connect to
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_mysql_execute_as_root() {
     # Declare an array to define the options of this helper.
     local legacy_args=sd
-    declare -Ar args_array=( [s]=sql= [d]=database= )
+    local -A args_array=( [s]=sql= [d]=database= )
     local sql
     local database
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
     database="${database:-}"
 
-    ynh_mysql_connect_as --user="root" --password="$(sudo cat $MYSQL_ROOT_PWD_FILE)" \
-        --database="$database" <<< "$sql"
+    if [ -n "$database" ]
+    then
+        database="--database=$database"
+    fi
+
+    mysql -B "$database" <<< "$sql"
 }
 
 # Execute a command from a file as root user
 #
 # usage: ynh_mysql_execute_file_as_root --file=file [--database=database]
-# | arg: -f, --file - the file containing SQL commands
-# | arg: -d, --database - the database to connect to
+# | arg: -f, --file=        - the file containing SQL commands
+# | arg: -d, --database=    - the database to connect to
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_mysql_execute_file_as_root() {
     # Declare an array to define the options of this helper.
     local legacy_args=fd
-    declare -Ar args_array=( [f]=file= [d]=database= )
+    local -A args_array=( [f]=file= [d]=database= )
     local file
     local database
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
     database="${database:-}"
 
-    ynh_mysql_connect_as --user="root" --password="$(sudo cat $MYSQL_ROOT_PWD_FILE)" \
-        --database="$database" < "$file"
+    if [ -n "$database" ]
+    then
+        database="--database=$database"
+    fi
+
+    mysql -B "$database" < "$file"
 }
 
 # Create a database and grant optionnaly privilegies to a user
@@ -85,9 +92,12 @@ ynh_mysql_create_db() {
     local sql="CREATE DATABASE ${db};"
 
     # grant all privilegies to user
-    if [[ $# -gt 1 ]]; then
+    if [[ $# -gt 1 ]]
+    then
         sql+=" GRANT ALL PRIVILEGES ON ${db}.* TO '${2}'@'localhost'"
-        [[ -n ${3:-} ]] && sql+=" IDENTIFIED BY '${3}'"
+        if [[ -n ${3:-} ]]; then
+            sql+=" IDENTIFIED BY '${3}'"
+        fi
         sql+=" WITH GRANT OPTION;"
     fi
 
@@ -111,22 +121,22 @@ ynh_mysql_drop_db() {
 
 # Dump a database
 #
-# example: ynh_mysql_dump_db 'roundcube' > ./dump.sql
-#
 # usage: ynh_mysql_dump_db --database=database
-# | arg: -d, --database - the database name to dump
-# | ret: the mysqldump output
+# | arg: -d, --database=    - the database name to dump
+# | ret: The mysqldump output
+#
+# example: ynh_mysql_dump_db --database=roundcube > ./dump.sql
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_mysql_dump_db() {
     # Declare an array to define the options of this helper.
     local legacy_args=d
-    declare -Ar args_array=( [d]=database= )
+    local -A args_array=( [d]=database= )
     local database
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    mysqldump -u "root" -p"$(sudo cat $MYSQL_ROOT_PWD_FILE)" --single-transaction --skip-dump-date "$database"
+    mysqldump --single-transaction --skip-dump-date "$database"
 }
 
 # Create a user
@@ -146,24 +156,25 @@ ynh_mysql_create_user() {
 # Check if a mysql user exists
 #
 # usage: ynh_mysql_user_exists --user=user
-# | arg: -u, --user - the user for which to check existence
+# | arg: -u, --user=    - the user for which to check existence
+# | ret: 0 if the user exists, 1 otherwise.
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_mysql_user_exists()
 {
-   # Declare an array to define the options of this helper.
-   local legacy_args=u
-   declare -Ar args_array=( [u]=user= )
-   local user
-   # Manage arguments with getopts
-   ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=u
+    local -A args_array=( [u]=user= )
+    local user
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
 
-   if [[ -z $(ynh_mysql_execute_as_root --sql="SELECT User from mysql.user WHERE User = '$user';") ]]
-   then
-      return 1
-   else
-      return 0
-   fi
+    if [[ -z $(ynh_mysql_execute_as_root --sql="SELECT User from mysql.user WHERE User = '$user';") ]]
+    then
+        return 1
+    else
+        return 0
+    fi
 }
 
 # Drop a user
@@ -180,59 +191,58 @@ ynh_mysql_drop_user() {
 
 # Create a database, an user and its password. Then store the password in the app's config
 #
-# After executing this helper, the password of the created database will be available in $db_pwd
-# It will also be stored as "mysqlpwd" into the app settings.
-#
 # usage: ynh_mysql_setup_db --db_user=user --db_name=name [--db_pwd=pwd]
-# | arg: -u, --db_user - Owner of the database
-# | arg: -n, --db_name - Name of the database
-# | arg: -p, --db_pwd - Password of the database. If not provided, a password will be generated
+# | arg: -u, --db_user=     - Owner of the database
+# | arg: -n, --db_name=     - Name of the database
+# | arg: -p, --db_pwd=      - Password of the database. If not provided, a password will be generated
+#
+# After executing this helper, the password of the created database will be available in `$db_pwd`
+# It will also be stored as "`mysqlpwd`" into the app settings.
 #
 # Requires YunoHost version 2.6.4 or higher.
 ynh_mysql_setup_db () {
-	# Declare an array to define the options of this helper.
-	local legacy_args=unp
-	declare -Ar args_array=( [u]=db_user= [n]=db_name= [p]=db_pwd= )
-	local db_user
-	local db_name
-	db_pwd=""
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=unp
+    local -A args_array=( [u]=db_user= [n]=db_name= [p]=db_pwd= )
+    local db_user
+    local db_name
+    db_pwd=""
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
 
-	local new_db_pwd=$(ynh_string_random)	# Generate a random password
-	# If $db_pwd is not provided, use new_db_pwd instead for db_pwd
-	db_pwd="${db_pwd:-$new_db_pwd}"
+    # Generate a random password
+    local new_db_pwd=$(ynh_string_random)
+    # If $db_pwd is not provided, use new_db_pwd instead for db_pwd
+    db_pwd="${db_pwd:-$new_db_pwd}"
 
-	ynh_mysql_create_db "$db_name" "$db_user" "$db_pwd"	# Create the database
-	ynh_app_setting_set --app=$app --key=mysqlpwd --value=$db_pwd	# Store the password in the app's config
+    ynh_mysql_create_db "$db_name" "$db_user" "$db_pwd"
+    ynh_app_setting_set --app=$app --key=mysqlpwd --value=$db_pwd
 }
 
 # Remove a database if it exists, and the associated user
 #
 # usage: ynh_mysql_remove_db --db_user=user --db_name=name
-# | arg: -u, --db_user - Owner of the database
-# | arg: -n, --db_name - Name of the database
+# | arg: -u, --db_user=     - Owner of the database
+# | arg: -n, --db_name=     - Name of the database
 #
 # Requires YunoHost version 2.6.4 or higher.
 ynh_mysql_remove_db () {
-	# Declare an array to define the options of this helper.
-	local legacy_args=un
-	declare -Ar args_array=( [u]=db_user= [n]=db_name= )
-	local db_user
-	local db_name
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=un
+    local -Ar args_array=( [u]=db_user= [n]=db_name= )
+    local db_user
+    local db_name
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
 
-	local mysql_root_password=$(sudo cat $MYSQL_ROOT_PWD_FILE)
-	if mysqlshow -u root -p$mysql_root_password | grep -q "^| $db_name"; then	# Check if the database exists
-		ynh_mysql_drop_db $db_name	# Remove the database	
-	else
-		ynh_print_warn --message="Database $db_name not found"
-	fi
+    if mysqlshow | grep -q "^| $db_name "; then
+        ynh_mysql_drop_db $db_name
+    else
+        ynh_print_warn --message="Database $db_name not found"
+    fi
 
-	# Remove mysql user if it exists
-        if $(ynh_mysql_user_exists --user=$db_user); then
-		ynh_mysql_drop_user $db_user
-	fi
+    # Remove mysql user if it exists
+    if ynh_mysql_user_exists --user=$db_user; then
+        ynh_mysql_drop_user $db_user
+    fi
 }
-

data/helpers.d/network

@@ -2,30 +2,64 @@
 
 # Find a free port and return it
 #
-# example: port=$(ynh_find_port --port=8080)
-#
 # usage: ynh_find_port --port=begin_port
-# | arg: -p, --port - port to start to search
+# | arg: -p, --port=    - port to start to search
+# | ret: the port number
+#
+# example: port=$(ynh_find_port --port=8080)
 #
 # Requires YunoHost version 2.6.4 or higher.
 ynh_find_port () {
-	# Declare an array to define the options of this helper.
-	local legacy_args=p
-	declare -Ar args_array=( [p]=port= )
-	local port
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=p
+    local -A args_array=( [p]=port= )
+    local port
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
 
-	test -n "$port" || ynh_die --message="The argument of ynh_find_port must be a valid port."
-	while netcat -z 127.0.0.1 $port       # Check if the port is free
-	do
-		port=$((port+1))	# Else, pass to next port
-	done
-	echo $port
+    test -n "$port" || ynh_die --message="The argument of ynh_find_port must be a valid port."
+    while ! ynh_port_available --port=$port
+    do
+        port=$((port+1))
+    done
+    echo $port
 }
 
+# Test if a port is available
+#
+# usage: ynh_find_port --port=XYZ
+# | arg: -p, --port=    - port to check
+# | ret: 0 if the port is available, 1 if it is already used by another process.
+#
+# example: ynh_port_available --port=1234 || ynh_die --message="Port 1234 is needs to be available for this app"
+#
+# Requires YunoHost version 3.8.0 or higher.
+ynh_port_available () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=p
+    local -A args_array=( [p]=port= )
+    local port
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    # Check if the port is free
+    if ss --numeric --listening --tcp --udp | awk '{print$5}' | grep --quiet --extended-regexp ":$port$"
+    then
+        return 1
+    # This is to cover (most) case where an app is using a port yet ain't currently using it for some reason (typically service ain't up)
+    elif grep -q "port: '$port'" /etc/yunohost/apps/*/settings.yml
+    then
+        return 1
+    else
+        return 0
+    fi
+}
+
+
 # Validate an IP address
 #
+# [internal]
+#
 # usage: ynh_validate_ip --family=family --ip_address=ip_address
 # | ret: 0 for valid ip addresses, 1 otherwise
 #
@@ -34,19 +68,19 @@ ynh_find_port () {
 # Requires YunoHost version 2.2.4 or higher.
 ynh_validate_ip()
 {
-  # http://stackoverflow.com/questions/319279/how-to-validate-ip-address-in-python#319298
+    # http://stackoverflow.com/questions/319279/how-to-validate-ip-address-in-python#319298
 
-  # Declare an array to define the options of this helper.
-  local legacy_args=fi
-  declare -Ar args_array=( [f]=family= [i]=ip_address= )
-  local family
-  local ip_address
-  # Manage arguments with getopts
-  ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=fi
+    local -A args_array=( [f]=family= [i]=ip_address= )
+    local family
+    local ip_address
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
 
-  [ "$family" == "4" ] || [ "$family" == "6" ] || return 1
+    [ "$family" == "4" ] || [ "$family" == "6" ] || return 1
 
-  python /dev/stdin << EOF
+    python3 /dev/stdin << EOF
 import socket
 import sys
 family = { "4" : socket.AF_INET, "6" : socket.AF_INET6 }
@@ -60,41 +94,43 @@ EOF
 
 # Validate an IPv4 address
 #
-# example: ynh_validate_ip4 111.222.333.444
-#
 # usage: ynh_validate_ip4 --ip_address=ip_address
+# | arg: -i, --ip_address=     - the ipv4 address to check
 # | ret: 0 for valid ipv4 addresses, 1 otherwise
 #
+# example: ynh_validate_ip4 111.222.333.444
+#
 # Requires YunoHost version 2.2.4 or higher.
 ynh_validate_ip4()
 {
-  # Declare an array to define the options of this helper.
-  local legacy_args=i
-  declare -Ar args_array=( [i]=ip_address= )
-  local ip_address
-  # Manage arguments with getopts
-  ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=i
+    local -A args_array=( [i]=ip_address= )
+    local ip_address
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
 
-  ynh_validate_ip 4 $ip_address
+    ynh_validate_ip --family=4 --ip_address=$ip_address
 }
 
 
 # Validate an IPv6 address
 #
-# example: ynh_validate_ip6 2000:dead:beef::1
-#
 # usage: ynh_validate_ip6 --ip_address=ip_address
+# | arg: -i, --ip_address=  - the ipv6 address to check
 # | ret: 0 for valid ipv6 addresses, 1 otherwise
 #
+# example: ynh_validate_ip6 2000:dead:beef::1
+#
 # Requires YunoHost version 2.2.4 or higher.
 ynh_validate_ip6()
 {
-  # Declare an array to define the options of this helper.
-  local legacy_args=i
-  declare -Ar args_array=( [i]=ip_address= )
-  local ip_address
-  # Manage arguments with getopts
-  ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=i
+    local -A args_array=( [i]=ip_address= )
+    local ip_address
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
 
-  ynh_validate_ip 6 $ip_address
+    ynh_validate_ip --family=6 --ip_address=$ip_address
 }

data/helpers.d/nginx

@@ -2,67 +2,35 @@
 
 # Create a dedicated nginx config
 #
-# usage: ynh_add_nginx_config "list of others variables to replace"
+# usage: ynh_add_nginx_config
 #
-# | arg: list - (Optional) list of others variables to replace separated by spaces. For example : 'path_2 port_2 ...'
+# This will use a template in `../conf/nginx.conf`
+# See the documentation of `ynh_add_config` for a description of the template
+# format and how placeholders are replaced with actual variables.
 #
-# This will use a template in ../conf/nginx.conf
-#   __PATH__      by  $path_url
-#   __DOMAIN__    by  $domain
-#   __PORT__      by  $port
-#   __NAME__      by  $app
-#   __FINALPATH__ by  $final_path
+# Additionally, ynh_add_nginx_config will replace:
+# - `#sub_path_only`      by empty string if `path_url` is not `'/'`
+# - `#root_path_only`     by empty string if `path_url`  *is*  `'/'`
 #
-#  And dynamic variables (from the last example) :
-#   __PATH_2__    by $path_2
-#   __PORT_2__    by $port_2
+# This allows to enable/disable specific behaviors dependenging on the install
+# location
 #
-# Requires YunoHost version 2.7.2 or higher.
+# Requires YunoHost version 4.1.0 or higher.
 ynh_add_nginx_config () {
-	finalnginxconf="/etc/nginx/conf.d/$domain.d/$app.conf"
-	local others_var=${1:-}
-	ynh_backup_if_checksum_is_different --file="$finalnginxconf"
-	sudo cp ../conf/nginx.conf "$finalnginxconf"
 
-	# To avoid a break by set -u, use a void substitution ${var:-}. If the variable is not set, it's simply set with an empty variable.
-	# Substitute in a nginx config file only if the variable is not empty
-	if test -n "${path_url:-}"; then
-		# path_url_slash_less is path_url, or a blank value if path_url is only '/'
-		local path_url_slash_less=${path_url%/}
-		ynh_replace_string --match_string="__PATH__/" --replace_string="$path_url_slash_less/" --target_file="$finalnginxconf"
-		ynh_replace_string --match_string="__PATH__" --replace_string="$path_url" --target_file="$finalnginxconf"
-	fi
-	if test -n "${domain:-}"; then
-		ynh_replace_string --match_string="__DOMAIN__" --replace_string="$domain" --target_file="$finalnginxconf"
-	fi
-	if test -n "${port:-}"; then
-		ynh_replace_string --match_string="__PORT__" --replace_string="$port" --target_file="$finalnginxconf"
-	fi
-	if test -n "${app:-}"; then
-		ynh_replace_string --match_string="__NAME__" --replace_string="$app" --target_file="$finalnginxconf"
-	fi
-	if test -n "${final_path:-}"; then
-		ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalnginxconf"
-	fi
+    local finalnginxconf="/etc/nginx/conf.d/$domain.d/$app.conf"
 
-	# Replace all other variable given as arguments
-	for var_to_replace in $others_var
-	do
-		# ${var_to_replace^^} make the content of the variable on upper-cases
-		# ${!var_to_replace} get the content of the variable named $var_to_replace 
-		ynh_replace_string --match_string="__${var_to_replace^^}__" --replace_string="${!var_to_replace}" --target_file="$finalnginxconf"
-	done
+    if [ "${path_url:-}" != "/" ]
+    then
+        ynh_replace_string --match_string="^#sub_path_only" --replace_string="" --target_file="$YNH_APP_BASEDIR/conf/nginx.conf"
+    else
+        ynh_replace_string --match_string="^#root_path_only" --replace_string="" --target_file="$YNH_APP_BASEDIR/conf/nginx.conf"
+    fi
 
-	if [ "${path_url:-}" != "/" ]
-	then
-		ynh_replace_string --match_string="^#sub_path_only" --replace_string="" --target_file="$finalnginxconf"
-	else
-		ynh_replace_string --match_string="^#root_path_only" --replace_string="" --target_file="$finalnginxconf"
-	fi
+    ynh_add_config --template="$YNH_APP_BASEDIR/conf/nginx.conf" --destination="$finalnginxconf"
 
-	ynh_store_file_checksum --file="$finalnginxconf"
 
-	ynh_systemd_action --service_name=nginx --action=reload
+    ynh_systemd_action --service_name=nginx --action=reload
 }
 
 # Remove the dedicated nginx config
@@ -71,6 +39,6 @@ ynh_add_nginx_config () {
 #
 # Requires YunoHost version 2.7.2 or higher.
 ynh_remove_nginx_config () {
-	ynh_secure_remove --file="/etc/nginx/conf.d/$domain.d/$app.conf"
-	ynh_systemd_action --service_name=nginx --action=reload
+    ynh_secure_remove --file="/etc/nginx/conf.d/$domain.d/$app.conf"
+    ynh_systemd_action --service_name=nginx --action=reload
 }

data/helpers.d/nodejs

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+n_version=7.3.0
 n_install_dir="/opt/node_n"
 node_version_path="$n_install_dir/n/versions/node"
 # N_PREFIX is the directory of n, it needs to be loaded as a environment variable.
@@ -13,154 +14,202 @@ export N_PREFIX="$n_install_dir"
 #
 # Requires YunoHost version 2.7.12 or higher.
 ynh_install_n () {
-	ynh_print_info --message="Installation of N - Node.js version management"
-	# Build an app.src for n
-	mkdir -p "../conf"
-	echo "SOURCE_URL=https://github.com/tj/n/archive/v2.1.7.tar.gz
-SOURCE_SUM=2ba3c9d4dd3c7e38885b37e02337906a1ee91febe6d5c9159d89a9050f2eea8f" > "../conf/n.src"
-	# Download and extract n
-	ynh_setup_source --dest_dir="$n_install_dir/git" --source_id=n
-	# Install n
-	(cd "$n_install_dir/git"
-	PREFIX=$N_PREFIX make install 2>&1)
+    ynh_print_info --message="Installation of N - Node.js version management"
+    # Build an app.src for n
+    echo "SOURCE_URL=https://github.com/tj/n/archive/v${n_version}.tar.gz
+SOURCE_SUM=b908b0fc86922ede37e89d1030191285209d7d521507bf136e62895e5797847f" > "$YNH_APP_BASEDIR/conf/n.src"
+    # Download and extract n
+    ynh_setup_source --dest_dir="$n_install_dir/git" --source_id=n
+    # Install n
+    (cd "$n_install_dir/git"
+    PREFIX=$N_PREFIX make install 2>&1)
 }
 
 # Load the version of node for an app, and set variables.
 #
-# ynh_use_nodejs has to be used in any app scripts before using node for the first time.
-#
-# 2 variables are available:
-#   - $nodejs_path: The absolute path of node for the chosen version.
-#   - $nodejs_version: Just the version number of node for this app. Stored as 'nodejs_version' in settings.yml.
-# And 2 alias stored in variables:
-#   - $nodejs_use_version: An old variable, not used anymore. Keep here to not break old apps
-#     NB: $PATH will contain the path to node, it has to be propagated to any other shell which needs to use it.
-#     That's means it has to be added to any systemd script.
-#
 # usage: ynh_use_nodejs
 #
+# `ynh_use_nodejs` has to be used in any app scripts before using node for the first time.
+# This helper will provide alias and variables to use in your scripts.
+#
+# To use npm or node, use the alias `ynh_npm` and `ynh_node`.
+#
+# Those alias will use the correct version installed for the app.
+# For example: use `ynh_npm install` instead of `npm install`
+#
+# With `sudo` or `ynh_exec_as`, use instead the fallback variables `$ynh_npm` and `$ynh_node`
+# And propagate $PATH to sudo with $ynh_node_load_PATH
+# Exemple: `ynh_exec_as $app $ynh_node_load_PATH $ynh_npm install`
+#
+# $PATH contains the path of the requested version of node.
+# However, $PATH is duplicated into $node_PATH to outlast any manipulation of `$PATH`
+# You can use the variable `$ynh_node_load_PATH` to quickly load your node version
+# in $PATH for an usage into a separate script.
+# Exemple: $ynh_node_load_PATH $final_path/script_that_use_npm.sh`
+#
+#
+# Finally, to start a nodejs service with the correct version, 2 solutions
+#  Either the app is dependent of node or npm, but does not called it directly.
+#  In such situation, you need to load PATH :
+# ```
+# Environment="__NODE_ENV_PATH__"
+# ExecStart=__FINALPATH__/my_app
+# ```
+# You will replace __NODE_ENV_PATH__ with $ynh_node_load_PATH.
+#
+#  Or node start the app directly, then you don't need to load the PATH variable
+# ```
+# ExecStart=__YNH_NODE__ my_app run
+# ```
+# You will replace __YNH_NODE__ with $ynh_node
+#
+#
+# 2 other variables are also available
+#   - $nodejs_path: The absolute path to node binaries for the chosen version.
+#   - $nodejs_version: Just the version number of node for this app. Stored as 'nodejs_version' in settings.yml.
+#
 # Requires YunoHost version 2.7.12 or higher.
 ynh_use_nodejs () {
-	nodejs_version=$(ynh_app_setting_get --app=$app --key=nodejs_version)
+    nodejs_version=$(ynh_app_setting_get --app=$app --key=nodejs_version)
 
-	nodejs_use_version="echo \"Deprecated command, should be removed\""
+    # Get the absolute path of this version of node
+    nodejs_path="$node_version_path/$nodejs_version/bin"
 
-	# Get the absolute path of this version of node
-	nodejs_path="$node_version_path/$nodejs_version/bin"
+    # Allow alias to be used into bash script
+    shopt -s expand_aliases
 
-	# Load the path of this version of node in $PATH
-	[[ :$PATH: == *":$nodejs_path"* ]] || PATH="$nodejs_path:$PATH"
+    # Create an alias for the specific version of node and a variable as fallback
+    ynh_node="$nodejs_path/node"
+    alias ynh_node="$ynh_node"
+    # And npm
+    ynh_npm="$nodejs_path/npm"
+    alias ynh_npm="$ynh_npm"
+
+    # Load the path of this version of node in $PATH
+    if [[ :$PATH: != *":$nodejs_path"* ]]; then
+        PATH="$nodejs_path:$PATH"
+    fi
+    node_PATH="$PATH"
+    # Create an alias to easily load the PATH
+    ynh_node_load_PATH="PATH=$node_PATH"
+    # Same var but in lower case to be compatible with ynh_replace_vars...
+    ynh_node_load_path="PATH=$node_PATH"
 }
 
 # Install a specific version of nodejs
 #
-# n (Node version management) uses the PATH variable to store the path of the version of node it is going to use.
-# That's how it changes the version
-#
 # ynh_install_nodejs will install the version of node provided as argument by using n.
 #
 # usage: ynh_install_nodejs --nodejs_version=nodejs_version
-# | arg: -n, --nodejs_version - Version of node to install. When possible, your should prefer to use major version number (e.g. 8 instead of 8.10.0). The crontab will then handle the update of minor versions when needed.
+# | arg: -n, --nodejs_version=      - Version of node to install. When possible, your should prefer to use major version number (e.g. 8 instead of 8.10.0). The crontab will then handle the update of minor versions when needed.
+#
+# `n` (Node version management) uses the `PATH` variable to store the path of the version of node it is going to use.
+# That's how it changes the version
+#
+# Refer to `ynh_use_nodejs` for more information about available commands and variables
 #
 # Requires YunoHost version 2.7.12 or higher.
 ynh_install_nodejs () {
-	# Use n, https://github.com/tj/n to manage the nodejs versions
+    # Use n, https://github.com/tj/n to manage the nodejs versions
 
-	# Declare an array to define the options of this helper.
-	local legacy_args=n
-	declare -Ar args_array=( [n]=nodejs_version= )
-	local nodejs_version
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=n
+    local -A args_array=( [n]=nodejs_version= )
+    local nodejs_version
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
 
-	# Create $n_install_dir
-	mkdir -p "$n_install_dir"
+    # Create $n_install_dir
+    mkdir --parents "$n_install_dir"
 
-	# Load n path in PATH
-	CLEAR_PATH="$n_install_dir/bin:$PATH"
-	# Remove /usr/local/bin in PATH in case of node prior installation
-	PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@')
+    # Load n path in PATH
+    CLEAR_PATH="$n_install_dir/bin:$PATH"
+    # Remove /usr/local/bin in PATH in case of node prior installation
+    PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@')
 
-	# Move an existing node binary, to avoid to block n.
-	test -x /usr/bin/node && mv /usr/bin/node /usr/bin/node_n
-	test -x /usr/bin/npm && mv /usr/bin/npm /usr/bin/npm_n
+    # Move an existing node binary, to avoid to block n.
+    test -x /usr/bin/node && mv /usr/bin/node /usr/bin/node_n
+    test -x /usr/bin/npm && mv /usr/bin/npm /usr/bin/npm_n
 
-	# If n is not previously setup, install it
-	if ! test $(n --version > /dev/null 2>&1)
-	then
-		ynh_install_n
-	fi
+    # If n is not previously setup, install it
+    if ! $n_install_dir/bin/n --version > /dev/null 2>&1
+    then
+        ynh_install_n
+    elif dpkg --compare-versions "$($n_install_dir/bin/n --version)" lt $n_version
+    then
+        ynh_install_n
+    fi
 
-	# Modify the default N_PREFIX in n script
-	ynh_replace_string --match_string="^N_PREFIX=\${N_PREFIX-.*}$" --replace_string="N_PREFIX=\${N_PREFIX-$N_PREFIX}" --target_file="$n_install_dir/bin/n"
+    # Modify the default N_PREFIX in n script
+    ynh_replace_string --match_string="^N_PREFIX=\${N_PREFIX-.*}$" --replace_string="N_PREFIX=\${N_PREFIX-$N_PREFIX}" --target_file="$n_install_dir/bin/n"
 
-	# Restore /usr/local/bin in PATH
-	PATH=$CLEAR_PATH
+    # Restore /usr/local/bin in PATH
+    PATH=$CLEAR_PATH
 
-	# And replace the old node binary.
-	test -x /usr/bin/node_n && mv /usr/bin/node_n /usr/bin/node
-	test -x /usr/bin/npm_n && mv /usr/bin/npm_n /usr/bin/npm
+    # And replace the old node binary.
+    test -x /usr/bin/node_n && mv /usr/bin/node_n /usr/bin/node
+    test -x /usr/bin/npm_n && mv /usr/bin/npm_n /usr/bin/npm
 
-	# Install the requested version of nodejs
-	uname=$(uname -m)
-	if [[ $uname =~ aarch64 || $uname =~ arm64 ]]
-	then
-		n $nodejs_version --arch=arm64
-	else
-		n $nodejs_version
-	fi
+    # Install the requested version of nodejs
+    uname=$(uname --machine)
+    if [[ $uname =~ aarch64 || $uname =~ arm64 ]]
+    then
+        n $nodejs_version --arch=arm64
+    else
+        n $nodejs_version
+    fi
 
-	# Find the last "real" version for this major version of node.
-	real_nodejs_version=$(find $node_version_path/$nodejs_version* -maxdepth 0 | sort --version-sort | tail --lines=1)
-	real_nodejs_version=$(basename $real_nodejs_version)
+    # Find the last "real" version for this major version of node.
+    real_nodejs_version=$(find $node_version_path/$nodejs_version* -maxdepth 0 | sort --version-sort | tail --lines=1)
+    real_nodejs_version=$(basename $real_nodejs_version)
 
-	# Create a symbolic link for this major version if the file doesn't already exist
-	if [ ! -e "$node_version_path/$nodejs_version" ]
-	then
-		ln --symbolic --force --no-target-directory $node_version_path/$real_nodejs_version $node_version_path/$nodejs_version
-	fi
+    # Create a symbolic link for this major version if the file doesn't already exist
+    if [ ! -e "$node_version_path/$nodejs_version" ]
+    then
+        ln --symbolic --force --no-target-directory $node_version_path/$real_nodejs_version $node_version_path/$nodejs_version
+    fi
 
-	# Store the ID of this app and the version of node requested for it
-	echo "$YNH_APP_INSTANCE_NAME:$nodejs_version" | tee --append "$n_install_dir/ynh_app_version"
+    # Store the ID of this app and the version of node requested for it
+    echo "$YNH_APP_INSTANCE_NAME:$nodejs_version" | tee --append "$n_install_dir/ynh_app_version"
 
-	# Store nodejs_version into the config of this app
-	ynh_app_setting_set --app=$app --key=nodejs_version --value=$nodejs_version
+    # Store nodejs_version into the config of this app
+    ynh_app_setting_set --app=$app --key=nodejs_version --value=$nodejs_version
 
-	# Build the update script and set the cronjob
-	ynh_cron_upgrade_node
+    # Build the update script and set the cronjob
+    ynh_cron_upgrade_node
 
-	ynh_use_nodejs
+    ynh_use_nodejs
 }
 
 # Remove the version of node used by the app.
 #
-# This helper will check if another app uses the same version of node,
-# if not, this version of node will be removed.
-# If no other app uses node, n will be also removed.
-#
 # usage: ynh_remove_nodejs
 #
+# This helper will check if another app uses the same version of node.
+# - If not, this version of node will be removed.
+# - If no other app uses node, n will be also removed.
+#
 # Requires YunoHost version 2.7.12 or higher.
 ynh_remove_nodejs () {
-	nodejs_version=$(ynh_app_setting_get --app=$app --key=nodejs_version)
+    nodejs_version=$(ynh_app_setting_get --app=$app --key=nodejs_version)
 
-	# Remove the line for this app
-	sed --in-place "/$YNH_APP_INSTANCE_NAME:$nodejs_version/d" "$n_install_dir/ynh_app_version"
+    # Remove the line for this app
+    sed --in-place "/$YNH_APP_INSTANCE_NAME:$nodejs_version/d" "$n_install_dir/ynh_app_version"
 
-	# If no other app uses this version of nodejs, remove it.
-	if ! grep --quiet "$nodejs_version" "$n_install_dir/ynh_app_version"
-	then
-		$n_install_dir/bin/n rm $nodejs_version
-	fi
+    # If no other app uses this version of nodejs, remove it.
+    if ! grep --quiet "$nodejs_version" "$n_install_dir/ynh_app_version"
+    then
+        $n_install_dir/bin/n rm $nodejs_version
+    fi
 
-	# If no other app uses n, remove n
-	if [ ! -s "$n_install_dir/ynh_app_version" ]
-	then
-		ynh_secure_remove --file="$n_install_dir"
-		ynh_secure_remove --file="/usr/local/n"
-		sed --in-place "/N_PREFIX/d" /root/.bashrc
-		rm -f /etc/cron.daily/node_update
-	fi
+    # If no other app uses n, remove n
+    if [ ! -s "$n_install_dir/ynh_app_version" ]
+    then
+        ynh_secure_remove --file="$n_install_dir"
+        ynh_secure_remove --file="/usr/local/n"
+        sed --in-place "/N_PREFIX/d" /root/.bashrc
+        rm --force /etc/cron.daily/node_update
+    fi
 }
 
 # Set a cron design to update your node versions
@@ -173,8 +222,8 @@ ynh_remove_nodejs () {
 #
 # Requires YunoHost version 2.7.12 or higher.
 ynh_cron_upgrade_node () {
-	# Build the update script
-	cat > "$n_install_dir/node_update.sh" << EOF
+    # Build the update script
+    cat > "$n_install_dir/node_update.sh" << EOF
 #!/bin/bash
 
 version_path="$node_version_path"
@@ -195,26 +244,26 @@ all_real_version=\$(echo "\$all_real_version" | sort --unique)
 # Read each major version
 while read version
 do
-	echo "Update of the version \$version"
-	sudo \$n_install_dir/bin/n \$version
+    echo "Update of the version \$version"
+    sudo \$n_install_dir/bin/n \$version
 
-	# Find the last "real" version for this major version of node.
-	real_nodejs_version=\$(find \$version_path/\$version* -maxdepth 0 | sort --version-sort | tail --lines=1)
-	real_nodejs_version=\$(basename \$real_nodejs_version)
+    # Find the last "real" version for this major version of node.
+    real_nodejs_version=\$(find \$version_path/\$version* -maxdepth 0 | sort --version-sort | tail --lines=1)
+    real_nodejs_version=\$(basename \$real_nodejs_version)
 
-	# Update the symbolic link for this version
-	sudo ln --symbolic --force --no-target-directory \$version_path/\$real_nodejs_version \$version_path/\$version
+    # Update the symbolic link for this version
+    sudo ln --symbolic --force --no-target-directory \$version_path/\$real_nodejs_version \$version_path/\$version
 done <<< "\$(echo "\$all_real_version")"
 EOF
 
-	chmod +x "$n_install_dir/node_update.sh"
+    chmod +x "$n_install_dir/node_update.sh"
 
-	# Build the cronjob
-	cat > "/etc/cron.daily/node_update" << EOF
+    # Build the cronjob
+    cat > "/etc/cron.daily/node_update" << EOF
 #!/bin/bash
 
 $n_install_dir/node_update.sh >> $n_install_dir/node_update.log
 EOF
 
-	chmod +x "/etc/cron.daily/node_update"
+    chmod +x "/etc/cron.daily/node_update"
 }

data/helpers.d/permission

@@ -0,0 +1,412 @@
+#!/bin/bash
+
+# Create a new permission for the app
+#
+# Example 1: `ynh_permission_create --permission=admin --url=/admin --additional_urls=domain.tld/admin /superadmin --allowed=alice bob \
+#                                  --label="My app admin" --show_tile=true`
+#
+# This example will create a new permission permission with this following effect:
+# - A tile named "My app admin" in the SSO will be available for the users alice and bob. This tile will point to the relative url '/admin'.
+# - Only the user alice and bob will have the access to theses following url: /admin, domain.tld/admin, /superadmin
+#
+#
+# Example 2:
+#
+#     ynh_permission_create --permission=api --url=domain.tld/api --auth_header=false --allowed=visitors \
+#                                  --label="MyApp API" --protected=true
+#
+# This example will create a new protected permission. So the admin won't be able to add/remove the visitors group of this permission.
+# In case of an API with need to be always public it avoid that the admin break anything.
+# With this permission all client will be allowed to access to the url 'domain.tld/api'.
+# Note that in this case no tile will be show on the SSO.
+# Note that the auth_header parameter is to 'false'. So no authentication header will be passed to the application.
+# Generally the API is requested by an application and enabling the auth_header has no advantage and could bring some issues in some case.
+# So in this case it's better to disable this option for all API.
+#
+#
+# usage: ynh_permission_create --permission="permission" [--url="url"] [--additional_urls="second-url" [ "third-url" ]] [--auth_header=true|false]
+#                                                        [--allowed=group1 [ group2 ]] [--label="label"] [--show_tile=true|false]
+#                                                        [--protected=true|false]
+# | arg: -p, --permission=       - the name for the permission (by default a permission named "main" already exist)
+# | arg: -u, --url=              - (optional) URL for which access will be allowed/forbidden. Note that if 'show_tile' is enabled, this URL will be the URL of the tile.
+# | arg: -A, --additional_urls=  - (optional) List of additional URL for which access will be allowed/forbidden
+# | arg: -h, --auth_header=      - (optional) Define for the URL of this permission, if SSOwat pass the authentication header to the application. Default is true
+# | arg: -a, --allowed=          - (optional) A list of group/user to allow for the permission
+# | arg: -l, --label=            - (optional) Define a name for the permission. This label will be shown on the SSO and in the admin. Default is "APP_LABEL (permission name)".
+# | arg: -t, --show_tile=        - (optional) Define if a tile will be shown in the SSO. If yes the name of the tile will be the 'label' parameter. Defaults to false for the permission different than 'main'.
+# | arg: -P, --protected=        - (optional) Define if this permission is protected. If it is protected the administrator won't be able to add or remove the visitors group of this permission. Defaults to 'false'.
+#
+# If provided, 'url' or 'additional_urls' is assumed to be relative to the app domain/path if they
+# start with '/'.  For example:
+#     /                             -> domain.tld/app
+#     /admin                        -> domain.tld/app/admin
+#     domain.tld/app/api            -> domain.tld/app/api
+#
+# 'url' or 'additional_urls' can be treated as a PCRE (not lua) regex if it starts with "re:".
+# For example:
+#     re:/api/[A-Z]*$               -> domain.tld/app/api/[A-Z]*$
+#     re:domain.tld/app/api/[A-Z]*$ -> domain.tld/app/api/[A-Z]*$
+#
+# Note that globally the parameter 'url' and 'additional_urls' are same. The only difference is:
+# - 'url' is only one url, 'additional_urls' can be a list of urls. There are no limitation of 'additional_urls'
+# - 'url' is used for the url of tile in the SSO (if enabled with the 'show_tile' parameter)
+#
+#
+# About the authentication header (auth_header parameter).
+# The SSO pass (by default) to the application theses following HTTP header (linked to the authenticated user) to the application:
+#        - "Auth-User": username
+#        - "Remote-User": username
+#        - "Email": user email
+#
+# Generally this feature is usefull to authenticate automatically the user in the application but in some case the application don't work with theses header and theses header need to be disabled to have the application to work correctly.
+# See https://github.com/YunoHost/issues/issues/1420 for more informations
+#
+#
+# Requires YunoHost version 3.7.0 or higher.
+ynh_permission_create() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=puAhaltP
+    local -A args_array=( [p]=permission= [u]=url= [A]=additional_urls= [h]=auth_header= [a]=allowed= [l]=label= [t]=show_tile= [P]=protected= )
+    local permission
+    local url
+    local additional_urls
+    local auth_header
+    local allowed
+    local label
+    local show_tile
+    local protected
+    ynh_handle_getopts_args "$@"
+    url=${url:-}
+    additional_urls=${additional_urls:-}
+    auth_header=${auth_header:-}
+    allowed=${allowed:-}
+    label=${label:-}
+    show_tile=${show_tile:-}
+    protected=${protected:-}
+
+    if [[ -n $url ]]
+    then
+        url=",url='$url'"
+    fi
+
+    if [[ -n $additional_urls ]]
+    then
+        # Convert a list from getopts to python list
+        # Note that getopts separate the args with ';'
+        # By example:
+        # --additional_urls /urlA /urlB
+        # will be:
+        # additional_urls=['/urlA', '/urlB']
+        additional_urls=",additional_urls=['${additional_urls//;/\',\'}']"
+    fi
+
+    if [[ -n $auth_header ]]
+    then
+        if [ $auth_header == "true" ]
+        then
+            auth_header=",auth_header=True"
+        else
+            auth_header=",auth_header=False"
+        fi
+    fi
+
+    if [[ -n $allowed ]]
+    then
+        # Convert a list from getopts to python list
+        # Note that getopts separate the args with ';'
+        # By example:
+        # --allowed alice bob
+        # will be:
+        # allowed=['alice', 'bob']
+        allowed=",allowed=['${allowed//;/\',\'}']"
+    fi
+
+    if [[ -n ${label:-} ]]; then
+        label=",label='$label'"
+    else
+        label=",label='$permission'"
+    fi
+
+    if [[ -n ${show_tile:-} ]]
+    then
+        if [ $show_tile == "true" ]
+        then
+            show_tile=",show_tile=True"
+        else
+            show_tile=",show_tile=False"
+        fi
+    fi
+
+    if [[ -n ${protected:-} ]]
+    then
+        if [ $protected == "true" ]
+        then
+            protected=",protected=True"
+        else
+            protected=",protected=False"
+        fi
+    fi
+
+    yunohost tools shell -c "from yunohost.permission import permission_create; permission_create('$app.$permission' $url $additional_urls $auth_header $allowed $label $show_tile $protected)"
+}
+
+# Remove a permission for the app (note that when the app is removed all permission is automatically removed)
+#
+# example: ynh_permission_delete --permission=editors
+#
+# usage: ynh_permission_delete --permission="permission"
+# | arg: -p, --permission=      - the name for the permission (by default a permission named "main" is removed automatically when the app is removed)
+#
+# Requires YunoHost version 3.7.0 or higher.
+ynh_permission_delete() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=p
+    local -A args_array=( [p]=permission= )
+    local permission
+    ynh_handle_getopts_args "$@"
+
+    yunohost tools shell -c "from yunohost.permission import permission_delete; permission_delete('$app.$permission')"
+}
+
+# Check if a permission exists
+#
+# usage: ynh_permission_exists --permission=permission
+# | arg: -p, --permission=      - the permission to check
+# | exit: Return 1 if the permission doesn't exist, 0 otherwise
+#
+# Requires YunoHost version 3.7.0 or higher.
+ynh_permission_exists() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=p
+    local -A args_array=( [p]=permission= )
+    local permission
+    ynh_handle_getopts_args "$@"
+
+    yunohost user permission list "$app" --output-as json --quiet \
+        | jq -e --arg perm "$app.$permission" '.permissions[$perm]' >/dev/null
+}
+
+# Redefine the url associated to a permission
+#
+# usage: ynh_permission_url --permission "permission" [--url="url"] [--add_url="new-url" [ "other-new-url" ]] [--remove_url="old-url" [ "other-old-url" ]]
+#                                                     [--auth_header=true|false] [--clear_urls]
+# | arg: -p, --permission=   - the name for the permission (by default a permission named "main" is removed automatically when the app is removed)
+# | arg: -u, --url=          - (optional) URL for which access will be allowed/forbidden. Note that if you want to remove url you can pass an empty sting as arguments ("").
+# | arg: -a, --add_url=      - (optional) List of additional url to add for which access will be allowed/forbidden.
+# | arg: -r, --remove_url=   - (optional) List of additional url to remove for which access will be allowed/forbidden
+# | arg: -h, --auth_header=  - (optional) Define for the URL of this permission, if SSOwat pass the authentication header to the application
+# | arg: -c, --clear_urls    - (optional) Clean all urls (url and additional_urls)
+#
+# Requires YunoHost version 3.7.0 or higher.
+ynh_permission_url() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=puarhc
+    local -A args_array=( [p]=permission= [u]=url= [a]=add_url= [r]=remove_url= [h]=auth_header= [c]=clear_urls )
+    local permission
+    local url
+    local add_url
+    local remove_url
+    local auth_header
+    local clear_urls
+    ynh_handle_getopts_args "$@"
+    url=${url:-}
+    add_url=${add_url:-}
+    remove_url=${remove_url:-}
+    auth_header=${auth_header:-}
+    clear_urls=${clear_urls:-}
+
+    if [[ -n $url ]]
+    then
+        url=",url='$url'"
+    fi
+
+    if [[ -n $add_url ]]
+    then
+        # Convert a list from getopts to python list
+        # Note that getopts separate the args with ';'
+        # For example:
+        # --add_url /urlA /urlB
+        # will be:
+        # add_url=['/urlA', '/urlB']
+        add_url=",add_url=['${add_url//;/\',\'}']"
+    fi
+
+    if [[ -n $remove_url ]]
+    then
+        # Convert a list from getopts to python list
+        # Note that getopts separate the args with ';'
+        # For example:
+        # --remove_url /urlA /urlB
+        # will be:
+        # remove_url=['/urlA', '/urlB']
+        remove_url=",remove_url=['${remove_url//;/\',\'}']"
+    fi
+
+    if [[ -n $auth_header ]]
+    then
+        if [ $auth_header == "true" ]
+        then
+            auth_header=",auth_header=True"
+        else
+            auth_header=",auth_header=False"
+        fi
+    fi
+
+    if [[ -n $clear_urls ]] && [ $clear_urls -eq 1 ]
+    then
+        clear_urls=",clear_urls=True"
+    fi
+
+    yunohost tools shell -c "from yunohost.permission import permission_url; permission_url('$app.$permission' $url $add_url $remove_url $auth_header $clear_urls)"
+}
+
+
+# Update a permission for the app
+#
+# usage: ynh_permission_update --permission "permission" [--add="group" ["group" ...]] [--remove="group" ["group" ...]]
+#                                                        [--label="label"] [--show_tile=true|false] [--protected=true|false]
+# | arg: -p, --permission= - the name for the permission (by default a permission named "main" already exist)
+# | arg: -a, --add=        - the list of group or users to enable add to the permission
+# | arg: -r, --remove=     - the list of group or users to remove from the permission
+# | arg: -l, --label=      - (optional) Define a name for the permission. This label will be shown on the SSO and in the admin.
+# | arg: -t, --show_tile=  - (optional) Define if a tile will be shown in the SSO
+# | arg: -P, --protected=  - (optional) Define if this permission is protected. If it is protected the administrator won't be able to add or remove the visitors group of this permission.
+#
+# Requires YunoHost version 3.7.0 or higher.
+ynh_permission_update() {
+    # Declare an array to define the options of this helper.
+    local legacy_args=parltP
+    local -A args_array=( [p]=permission= [a]=add= [r]=remove= [l]=label= [t]=show_tile= [P]=protected= )
+    local permission
+    local add
+    local remove
+    local label
+    local show_tile
+    local protected
+    ynh_handle_getopts_args "$@"
+    add=${add:-}
+    remove=${remove:-}
+    label=${label:-}
+    show_tile=${show_tile:-}
+    protected=${protected:-}
+
+    if [[ -n $add ]]
+    then
+        # Convert a list from getopts to python list
+        # Note that getopts separate the args with ';'
+        # For example:
+        # --add alice bob
+        # will be:
+        # add=['alice', 'bob']
+        add=",add=['${add//';'/"','"}']"
+    fi
+    if [[ -n $remove ]]
+    then
+        # Convert a list from getopts to python list
+        # Note that getopts separate the args with ';'
+        # For example:
+        # --remove alice bob
+        # will be:
+        # remove=['alice', 'bob']
+        remove=",remove=['${remove//';'/"','"}']"
+    fi
+
+    if [[ -n $label ]]
+    then
+        label=",label='$label'"
+    fi
+
+    if [[ -n $show_tile ]]
+    then
+        if [ $show_tile == "true" ]
+        then
+            show_tile=",show_tile=True"
+        else
+            show_tile=",show_tile=False"
+        fi
+    fi
+
+    if [[ -n $protected ]]; then
+        if [ $protected == "true" ]
+        then
+            protected=",protected=True"
+        else
+            protected=",protected=False"
+        fi
+    fi
+
+    yunohost tools shell -c "from yunohost.permission import user_permission_update; user_permission_update('$app.$permission' $add $remove $label $show_tile $protected , force=True)"
+}
+
+# Check if a permission has an user
+#
+# example: ynh_permission_has_user --permission=main --user=visitors
+#
+# usage: ynh_permission_has_user --permission=permission --user=user
+# | arg: -p, --permission=  - the permission to check
+# | arg: -u, --user=        - the user seek in the permission
+# | exit: Return 1 if the permission doesn't have that user or doesn't exist, 0 otherwise
+#
+# Requires YunoHost version 3.7.1 or higher.
+ynh_permission_has_user() {
+    local legacy_args=pu
+    # Declare an array to define the options of this helper.
+    local -A args_array=( [p]=permission= [u]=user= )
+    local permission
+    local user
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+
+    if ! ynh_permission_exists --permission=$permission
+    then
+        return 1
+    fi
+
+    # Check both allowed and corresponding_users sections in the json
+    for section in "allowed" "corresponding_users"
+    do
+        if yunohost user permission info "$app.$permission" --output-as json --quiet \
+            | jq -e --arg user $user --arg section $section '.[$section] | index($user)' >/dev/null
+        then
+            return 0
+        fi
+    done
+
+    return 1
+}
+
+# Check if a legacy permissions exist
+#
+# usage: ynh_legacy_permissions_exists
+# | exit: Return 1 if the permission doesn't exist, 0 otherwise
+#
+# Requires YunoHost version 4.1.2 or higher.
+ynh_legacy_permissions_exists () {
+    for permission in "skipped" "unprotected" "protected"
+    do
+        if ynh_permission_exists --permission="legacy_${permission}_uris"; then
+            return 0
+        fi
+    done
+    return 1
+}
+
+# Remove all legacy permissions
+#
+# usage: ynh_legacy_permissions_delete_all
+#
+# example:
+#   if ynh_legacy_permissions_exists
+#   then
+#       ynh_legacy_permissions_delete_all
+#       # You can recreate the required permissions here with ynh_permission_create
+#   fi
+# Requires YunoHost version 4.1.2 or higher.
+ynh_legacy_permissions_delete_all () {
+    for permission in "skipped" "unprotected" "protected"
+    do
+        if ynh_permission_exists --permission="legacy_${permission}_uris"; then
+            ynh_permission_delete --permission="legacy_${permission}_uris"
+        fi
+    done
+}

data/helpers.d/php

@@ -1,67 +1,621 @@
 #!/bin/bash
 
-# Create a dedicated php-fpm config
+readonly YNH_DEFAULT_PHP_VERSION=7.3
+# Declare the actual PHP version to use.
+# A packager willing to use another version of PHP can override the variable into its _common.sh.
+YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION}
+
+# Create a dedicated PHP-FPM config
 #
-# usage: ynh_add_fpm_config [--phpversion=7.X]
-# | arg: -v, --phpversion - Version of php to use.
+# usage 1: ynh_add_fpm_config [--phpversion=7.X] [--use_template] [--package=packages] [--dedicated_service]
+# | arg: -v, --phpversion=          - Version of PHP to use.
+# | arg: -t, --use_template         - Use this helper in template mode.
+# | arg: -p, --package=             - Additionnal PHP packages to install
+# | arg: -d, --dedicated_service    - Use a dedicated PHP-FPM service instead of the common one.
 #
-# Requires YunoHost version 2.7.2 or higher.
+# -----------------------------------------------------------------------------
+#
+# usage 2: ynh_add_fpm_config [--phpversion=7.X] --usage=usage --footprint=footprint [--package=packages] [--dedicated_service]
+# | arg: -v, --phpversion=          - Version of PHP to use.
+# | arg: -f, --footprint=           - Memory footprint of the service (low/medium/high).
+# low    - Less than 20 MB of RAM by pool.
+# medium - Between 20 MB and 40 MB of RAM by pool.
+# high   - More than 40 MB of RAM by pool.
+# Or specify exactly the footprint, the load of the service as MB by pool instead of having a standard value.
+# To have this value, use the following command and stress the service.
+# watch -n0.5 ps -o user,cmd,%cpu,rss -u APP
+#
+# | arg: -u, --usage=               - Expected usage of the service (low/medium/high).
+# low    - Personal usage, behind the SSO.
+# medium - Low usage, few people or/and publicly accessible.
+# high   - High usage, frequently visited website.
+#
+# | arg: -p, --package=             - Additionnal PHP packages to install for a specific version of PHP
+# | arg: -d, --dedicated_service    - Use a dedicated PHP-FPM service instead of the common one.
+#
+#
+# The footprint of the service will be used to defined the maximum footprint we can allow, which is half the maximum RAM.
+# So it will be used to defined 'pm.max_children'
+# A lower value for the footprint will allow more children for 'pm.max_children'. And so for
+#    'pm.start_servers', 'pm.min_spare_servers' and 'pm.max_spare_servers' which are defined from the
+#    value of 'pm.max_children'
+# NOTE: 'pm.max_children' can't exceed 4 times the number of processor's cores.
+#
+# The usage value will defined the way php will handle the children for the pool.
+# A value set as 'low' will set the process manager to 'ondemand'. Children will start only if the
+#   service is used, otherwise no child will stay alive. This config gives the lower footprint when the
+#   service is idle. But will use more proc since it has to start a child as soon it's used.
+# Set as 'medium', the process manager will be at dynamic. If the service is idle, a number of children
+#   equal to pm.min_spare_servers will stay alive. So the service can be quick to answer to any request.
+#   The number of children can grow if needed. The footprint can stay low if the service is idle, but
+#   not null. The impact on the proc is a little bit less than 'ondemand' as there's always a few
+#   children already available.
+# Set as 'high', the process manager will be set at 'static'. There will be always as many children as
+#   'pm.max_children', the footprint is important (but will be set as maximum a quarter of the maximum
+#   RAM) but the impact on the proc is lower. The service will be quick to answer as there's always many
+#   children ready to answer.
+#
+# Requires YunoHost version 4.1.0 or higher.
 ynh_add_fpm_config () {
-	# Declare an array to define the options of this helper.
-	local legacy_args=v
-	declare -Ar args_array=( [v]=phpversion= )
-	local phpversion
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=vtufpd
+    local -A args_array=( [v]=phpversion= [t]=use_template [u]=usage= [f]=footprint= [p]=package= [d]=dedicated_service )
+    local phpversion
+    local use_template
+    local usage
+    local footprint
+    local package
+    local dedicated_service
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    package=${package:-}
 
-	# Configure PHP-FPM 7.0 by default
-	phpversion="${phpversion:-7.0}"
+    # The default behaviour is to use the template.
+    use_template="${use_template:-1}"
+    usage="${usage:-}"
+    footprint="${footprint:-}"
+    if [ -n "$usage" ] || [ -n "$footprint" ]; then
+        use_template=0
+    fi
+    # Do not use a dedicated service by default
+    dedicated_service=${dedicated_service:-0}
 
-	local fpm_config_dir="/etc/php/$phpversion/fpm"
-	local fpm_service="php${phpversion}-fpm"
-	# Configure PHP-FPM 5 on Debian Jessie
-	if [ "$(ynh_get_debian_release)" == "jessie" ]; then
-		fpm_config_dir="/etc/php5/fpm"
-		fpm_service="php5-fpm"
-	fi
-	ynh_app_setting_set --app=$app --key=fpm_config_dir --value="$fpm_config_dir"
-	ynh_app_setting_set --app=$app --key=fpm_service --value="$fpm_service"
-	finalphpconf="$fpm_config_dir/pool.d/$app.conf"
-	ynh_backup_if_checksum_is_different --file="$finalphpconf"
-	sudo cp ../conf/php-fpm.conf "$finalphpconf"
-	ynh_replace_string --match_string="__NAMETOCHANGE__" --replace_string="$app" --target_file="$finalphpconf"
-	ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalphpconf"
-	ynh_replace_string --match_string="__USER__" --replace_string="$app" --target_file="$finalphpconf"
-	ynh_replace_string --match_string="__PHPVERSION__" --replace_string="$phpversion" --target_file="$finalphpconf"
-	sudo chown root: "$finalphpconf"
-	ynh_store_file_checksum --file="$finalphpconf"
+    # Set the default PHP-FPM version by default
+    phpversion="${phpversion:-$YNH_PHP_VERSION}"
 
-	if [ -e "../conf/php-fpm.ini" ]
-	then
-		echo "Packagers ! Please do not use a separate php ini file, merge your directives in the pool file instead." >&2
-		finalphpini="$fpm_config_dir/conf.d/20-$app.ini"
-		ynh_backup_if_checksum_is_different "$finalphpini"
-		sudo cp ../conf/php-fpm.ini "$finalphpini"
-		sudo chown root: "$finalphpini"
-		ynh_store_file_checksum "$finalphpini"
-	fi
-	ynh_systemd_action --service_name=$fpm_service --action=reload
+    local old_phpversion=$(ynh_app_setting_get --app=$app --key=phpversion)
+
+    # If the PHP version changed, remove the old fpm conf
+    if [ -n "$old_phpversion" ] && [ "$old_phpversion" != "$phpversion" ]
+    then
+        local old_php_fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir)
+        local old_php_finalphpconf="$old_php_fpm_config_dir/pool.d/$app.conf"
+
+        ynh_backup_if_checksum_is_different --file="$old_php_finalphpconf"
+
+        ynh_remove_fpm_config
+    fi
+
+    # If the requested PHP version is not the default version for YunoHost
+    if [ "$phpversion" != "$YNH_DEFAULT_PHP_VERSION" ]
+    then
+        # If the argument --package is used, add the packages to ynh_install_php to install them from sury
+        if [ -n "$package" ]
+        then
+            local additionnal_packages="--package=$package"
+        else
+            local additionnal_packages=""
+        fi
+        # Install this specific version of PHP.
+        ynh_install_php --phpversion="$phpversion" "$additionnal_packages"
+    elif [ -n "$package" ]
+    then
+        # Install the additionnal packages from the default repository
+        ynh_add_app_dependencies --package="$package"
+    fi
+
+    if [ $dedicated_service -eq 1 ]
+    then
+        local fpm_service="${app}-phpfpm"
+        local fpm_config_dir="/etc/php/$phpversion/dedicated-fpm"
+    else
+        local fpm_service="php${phpversion}-fpm"
+        local fpm_config_dir="/etc/php/$phpversion/fpm"
+    fi
+
+    # Create the directory for FPM pools
+    mkdir --parents "$fpm_config_dir/pool.d"
+
+    ynh_app_setting_set --app=$app --key=fpm_config_dir --value="$fpm_config_dir"
+    ynh_app_setting_set --app=$app --key=fpm_service --value="$fpm_service"
+    ynh_app_setting_set --app=$app --key=fpm_dedicated_service --value="$dedicated_service"
+    ynh_app_setting_set --app=$app --key=phpversion --value=$phpversion
+
+    # Migrate from mutual PHP service to dedicated one.
+    if [ $dedicated_service -eq 1 ]
+    then
+        local old_fpm_config_dir="/etc/php/$phpversion/fpm"
+        # If a config file exist in the common pool, move it.
+        if [ -e "$old_fpm_config_dir/pool.d/$app.conf" ]
+        then
+            ynh_print_info --message="Migrate to a dedicated php-fpm service for $app."
+            # Create a backup of the old file before migration
+            ynh_backup_if_checksum_is_different --file="$old_fpm_config_dir/pool.d/$app.conf"
+            # Remove the old PHP config file
+            ynh_secure_remove --file="$old_fpm_config_dir/pool.d/$app.conf"
+            # Reload PHP to release the socket and allow the dedicated service to use it
+            ynh_systemd_action --service_name=php${phpversion}-fpm --action=reload
+        fi
+    fi
+
+    if [ $use_template -eq 1 ]
+    then
+        # Usage 1, use the template in conf/php-fpm.conf
+        local phpfpm_path="$YNH_APP_BASEDIR/conf/php-fpm.conf"
+        # Make sure now that the template indeed exists
+        [ -e "$phpfpm_path" ] || ynh_die --message="Unable to find template to configure PHP-FPM."
+    else
+        # Usage 2, generate a PHP-FPM config file with ynh_get_scalable_phpfpm
+
+        # Store settings
+        ynh_app_setting_set --app=$app --key=fpm_footprint --value=$footprint
+        ynh_app_setting_set --app=$app --key=fpm_usage --value=$usage
+
+        # Define the values to use for the configuration of PHP.
+        ynh_get_scalable_phpfpm --usage=$usage --footprint=$footprint
+
+        local phpfpm_path="$YNH_APP_BASEDIR/conf/php-fpm.conf"
+        echo "
+[__APP__]
+
+user = __APP__
+group = __APP__
+
+chdir = __FINALPATH__
+
+listen = /var/run/php/php__PHPVERSION__-fpm-__APP__.sock
+listen.owner = www-data
+listen.group = www-data
+
+pm = __PHP_PM__
+pm.max_children = __PHP_MAX_CHILDREN__
+pm.max_requests = 500
+request_terminate_timeout = 1d
+" > $phpfpm_path
+
+        if [ "$php_pm" = "dynamic" ]
+        then
+            echo "
+pm.start_servers = __PHP_START_SERVERS__
+pm.min_spare_servers = __PHP_MIN_SPARE_SERVERS__
+pm.max_spare_servers = __PHP_MAX_SPARE_SERVERS__
+" >> $phpfpm_path
+
+        elif [ "$php_pm" = "ondemand" ]
+        then
+            echo "
+pm.process_idle_timeout = 10s
+" >> $phpfpm_path
+        fi
+
+        # Concatene the extra config.
+        if [ -e $YNH_APP_BASEDIR/conf/extra_php-fpm.conf ]; then
+            cat $YNH_APP_BASEDIR/conf/extra_php-fpm.conf >> "$phpfpm_path"
+        fi
+    fi
+
+    local finalphpconf="$fpm_config_dir/pool.d/$app.conf"
+    ynh_add_config --template="$phpfpm_path" --destination="$finalphpconf"
+
+    if [ -e "$YNH_APP_BASEDIR/conf/php-fpm.ini" ]
+    then
+        ynh_print_warn --message="Packagers ! Please do not use a separate php ini file, merge your directives in the pool file instead."
+        ynh_add_config --template="$YNH_APP_BASEDIR/conf/php-fpm.ini" --destination="$fpm_config_dir/conf.d/20-$app.ini"
+    fi
+
+    if [ $dedicated_service -eq 1 ]
+    then
+        # Create a dedicated php-fpm.conf for the service
+        local globalphpconf=$fpm_config_dir/php-fpm-$app.conf
+
+echo "[global]
+pid = /run/php/php__PHPVERSION__-fpm-__APP__.pid
+error_log = /var/log/php/fpm-php.__APP__.log
+syslog.ident = php-fpm-__APP__
+include = __FINALPHPCONF__
+" > $YNH_APP_BASEDIR/conf/php-fpm-$app.conf
+
+        ynh_add_config --template="$YNH_APP_BASEDIR/conf/php-fpm-$app.conf" --destination="$globalphpconf"
+
+        # Create a config for a dedicated PHP-FPM service for the app
+        echo "[Unit]
+Description=PHP __PHPVERSION__ FastCGI Process Manager for __APP__
+After=network.target
+
+[Service]
+Type=notify
+PIDFile=/run/php/php__PHPVERSION__-fpm-__APP__.pid
+ExecStart=/usr/sbin/php-fpm__PHPVERSION__ --nodaemonize --fpm-config __GLOBALPHPCONF__
+ExecReload=/bin/kill -USR2 \$MAINPID
+
+[Install]
+WantedBy=multi-user.target
+" > $YNH_APP_BASEDIR/conf/$fpm_service
+
+        # Create this dedicated PHP-FPM service
+        ynh_add_systemd_config --service=$fpm_service --template=$fpm_service
+        # Integrate the service in YunoHost admin panel
+        yunohost service add $fpm_service --log /var/log/php/fpm-php.$app.log --description "Php-fpm dedicated to $app"
+        # Configure log rotate
+        ynh_use_logrotate --logfile=/var/log/php
+        # Restart the service, as this service is either stopped or only for this app
+        ynh_systemd_action --service_name=$fpm_service --action=restart
+    else
+        # Validate that the new php conf doesn't break php-fpm entirely
+        if ! php-fpm${phpversion} --test 2>/dev/null
+        then
+            php-fpm${phpversion} --test || true
+            ynh_secure_remove --file="$finalphpconf"
+            ynh_die --message="The new configuration broke php-fpm?"
+        fi
+        ynh_systemd_action --service_name=$fpm_service --action=reload
+    fi
 }
 
-# Remove the dedicated php-fpm config
+# Remove the dedicated PHP-FPM config
 #
 # usage: ynh_remove_fpm_config
 #
 # Requires YunoHost version 2.7.2 or higher.
 ynh_remove_fpm_config () {
-	local fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir)
-	local fpm_service=$(ynh_app_setting_get --app=$app --key=fpm_service)
-	# Assume php version 7 if not set
-	if [ -z "$fpm_config_dir" ]; then
-		fpm_config_dir="/etc/php/7.0/fpm"
-		fpm_service="php7.0-fpm"
-	fi
-	ynh_secure_remove --file="$fpm_config_dir/pool.d/$app.conf"
-	ynh_secure_remove --file="$fpm_config_dir/conf.d/20-$app.ini" 2>&1
-	ynh_systemd_action --service_name=$fpm_service --action=reload
+    local fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir)
+    local fpm_service=$(ynh_app_setting_get --app=$app --key=fpm_service)
+    local dedicated_service=$(ynh_app_setting_get --app=$app --key=fpm_dedicated_service)
+    dedicated_service=${dedicated_service:-0}
+    # Get the version of PHP used by this app
+    local phpversion=$(ynh_app_setting_get --app=$app --key=phpversion)
+
+    # Assume default PHP-FPM version by default
+    phpversion="${phpversion:-$YNH_DEFAULT_PHP_VERSION}"
+
+    # Assume default PHP files if not set
+    if [ -z "$fpm_config_dir" ]
+    then
+        fpm_config_dir="/etc/php/$YNH_DEFAULT_PHP_VERSION/fpm"
+        fpm_service="php$YNH_DEFAULT_PHP_VERSION-fpm"
+    fi
+
+    ynh_secure_remove --file="$fpm_config_dir/pool.d/$app.conf"
+    if [ -e $fpm_config_dir/conf.d/20-$app.ini ]
+    then
+        ynh_secure_remove --file="$fpm_config_dir/conf.d/20-$app.ini"
+    fi
+
+    if [ $dedicated_service -eq 1 ]
+    then
+        # Remove the dedicated service PHP-FPM service for the app
+        ynh_remove_systemd_config --service=$fpm_service
+        # Remove the global PHP-FPM conf
+        ynh_secure_remove --file="$fpm_config_dir/php-fpm-$app.conf"
+        # Remove the service from the list of services known by YunoHost
+        yunohost service remove $fpm_service
+    elif ynh_package_is_installed --package="php${phpversion}-fpm"; then
+        ynh_systemd_action --service_name=$fpm_service --action=reload
+    fi
+
+    # If the PHP version used is not the default version for YunoHost
+    if [ "$phpversion" != "$YNH_DEFAULT_PHP_VERSION" ]
+    then
+        # Remove this specific version of PHP
+        ynh_remove_php
+    fi
+}
+
+# Install another version of PHP.
+#
+# [internal]
+#
+# usage: ynh_install_php --phpversion=phpversion [--package=packages]
+# | arg: -v, --phpversion=  - Version of PHP to install.
+# | arg: -p, --package=     - Additionnal PHP packages to install
+#
+# Requires YunoHost version 3.8.1 or higher.
+ynh_install_php () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=vp
+    local -A args_array=( [v]=phpversion= [p]=package= )
+    local phpversion
+    local package
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    package=${package:-}
+
+    # Store phpversion into the config of this app
+    ynh_app_setting_set $app phpversion $phpversion
+
+    if [ "$phpversion" == "$YNH_DEFAULT_PHP_VERSION" ]
+    then
+        ynh_die --message="Do not use ynh_install_php to install php$YNH_DEFAULT_PHP_VERSION"
+    fi
+
+    # Create the file if doesn't exist already
+    touch /etc/php/ynh_app_version
+
+    # Do not add twice the same line
+    if ! grep --quiet "$YNH_APP_INSTANCE_NAME:" "/etc/php/ynh_app_version"
+    then
+        # Store the ID of this app and the version of PHP requested for it
+        echo "$YNH_APP_INSTANCE_NAME:$phpversion" | tee --append "/etc/php/ynh_app_version"
+    fi
+
+    # Add an extra repository for those packages
+    ynh_install_extra_repo --repo="https://packages.sury.org/php/ $(ynh_get_debian_release) main" --key="https://packages.sury.org/php/apt.gpg" --name=extra_php_version --priority=600
+
+    # Install requested dependencies from this extra repository.
+    # Install PHP-FPM first, otherwise PHP will install apache as a dependency.
+    ynh_add_app_dependencies --package="php${phpversion}-fpm"
+    ynh_add_app_dependencies --package="php$phpversion php${phpversion}-common $package"
+
+    # Set the default PHP version back as the default version for php-cli.
+    update-alternatives --set php /usr/bin/php$YNH_DEFAULT_PHP_VERSION
+
+    # Advertise service in admin panel
+    yunohost service add php${phpversion}-fpm --log "/var/log/php${phpversion}-fpm.log"
+}
+
+# Remove the specific version of PHP used by the app.
+#
+# [internal]
+#
+# usage: ynh_install_php
+#
+# Requires YunoHost version 3.8.1 or higher.
+ynh_remove_php () {
+    # Get the version of PHP used by this app
+    local phpversion=$(ynh_app_setting_get --app=$app --key=phpversion)
+
+    if [ "$phpversion" == "$YNH_DEFAULT_PHP_VERSION" ] || [ -z "$phpversion" ]
+    then
+        if [ "$phpversion" == "$YNH_DEFAULT_PHP_VERSION" ]
+        then
+            ynh_print_err "Do not use ynh_remove_php to remove php$YNH_DEFAULT_PHP_VERSION !"
+        fi
+        return 0
+    fi
+
+    # Create the file if doesn't exist already
+    touch /etc/php/ynh_app_version
+
+    # Remove the line for this app
+    sed --in-place "/$YNH_APP_INSTANCE_NAME:$phpversion/d" "/etc/php/ynh_app_version"
+
+    # If no other app uses this version of PHP, remove it.
+    if ! grep --quiet "$phpversion" "/etc/php/ynh_app_version"
+    then
+        # Remove the service from the admin panel
+        if ynh_package_is_installed --package="php${phpversion}-fpm"; then
+            yunohost service remove php${phpversion}-fpm
+        fi
+
+        # Purge PHP dependencies for this version.
+        ynh_package_autopurge "php$phpversion php${phpversion}-fpm php${phpversion}-common"
+    fi
+}
+
+# Define the values to configure PHP-FPM
+#
+# [internal]
+#
+# usage: ynh_get_scalable_phpfpm --usage=usage --footprint=footprint [--print]
+# | arg: -f, --footprint=       - Memory footprint of the service (low/medium/high).
+# low    - Less than 20 MB of RAM by pool.
+# medium - Between 20 MB and 40 MB of RAM by pool.
+# high   - More than 40 MB of RAM by pool.
+# Or specify exactly the footprint, the load of the service as MB by pool instead of having a standard value.
+# To have this value, use the following command and stress the service.
+# watch -n0.5 ps -o user,cmd,%cpu,rss -u APP
+#
+# | arg: -u, --usage=           - Expected usage of the service (low/medium/high).
+# low    - Personal usage, behind the SSO.
+# medium - Low usage, few people or/and publicly accessible.
+# high   - High usage, frequently visited website.
+#
+# | arg: -p, --print            - Print the result (intended for debug purpose only when packaging the app)
+ynh_get_scalable_phpfpm () {
+    local legacy_args=ufp
+    # Declare an array to define the options of this helper.
+    local -A args_array=( [u]=usage= [f]=footprint= [p]=print )
+    local usage
+    local footprint
+    local print
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    # Set all characters as lowercase
+    footprint=${footprint,,}
+    usage=${usage,,}
+    print=${print:-0}
+
+    if [ "$footprint" = "low" ]
+    then
+        footprint=20
+    elif [ "$footprint" = "medium" ]
+    then
+        footprint=35
+    elif [ "$footprint" = "high" ]
+    then
+        footprint=50
+    fi
+
+    # Define the factor to determine min_spare_servers
+    # to avoid having too few children ready to start for heavy apps
+    if [ $footprint -le 20 ]
+    then
+        min_spare_servers_factor=8
+    elif [ $footprint -le 35 ]
+    then
+        min_spare_servers_factor=5
+    else
+        min_spare_servers_factor=3
+    fi
+
+    # Define the way the process manager handle child processes.
+    if [ "$usage" = "low" ]
+    then
+        php_pm=ondemand
+    elif [ "$usage" = "medium" ]
+    then
+        php_pm=dynamic
+    elif [ "$usage" = "high" ]
+    then
+        php_pm=static
+    else
+        ynh_die --message="Does not recognize '$usage' as an usage value."
+    fi
+
+    # Get the total of RAM available, except swap.
+    local max_ram=$(ynh_get_ram --total --ignore_swap)
+
+    at_least_one() {
+        # Do not allow value below 1
+        if [ $1 -le 0 ]
+        then
+            echo 1
+        else
+            echo $1
+        fi
+    }
+
+    # Define pm.max_children
+    # The value of pm.max_children is the total amount of ram divide by 2 and divide again by the footprint of a pool for this app.
+    # So if PHP-FPM start the maximum of children, it won't exceed half of the ram.
+    php_max_children=$(( $max_ram / 2 / $footprint ))
+    # If process manager is set as static, use half less children.
+    # Used as static, there's always as many children as the value of pm.max_children
+    if [ "$php_pm" = "static" ]
+    then
+        php_max_children=$(( $php_max_children / 2 ))
+    fi
+    php_max_children=$(at_least_one $php_max_children)
+
+    # To not overload the proc, limit the number of children to 4 times the number of cores.
+    local core_number=$(nproc)
+    local max_proc=$(( $core_number * 4 ))
+    if [ $php_max_children -gt $max_proc ]
+    then
+        php_max_children=$max_proc
+    fi
+
+    # Get a potential forced value for php_max_children
+    local php_forced_max_children=$(ynh_app_setting_get --app=$app --key=php_forced_max_children)
+    if [ -n "$php_forced_max_children" ]; then
+        php_max_children=$php_forced_max_children
+    fi
+
+    if [ "$php_pm" = "dynamic" ]
+    then
+        # Define pm.start_servers, pm.min_spare_servers and pm.max_spare_servers for a dynamic process manager
+        php_min_spare_servers=$(( $php_max_children / $min_spare_servers_factor ))
+        php_min_spare_servers=$(at_least_one $php_min_spare_servers)
+
+        php_max_spare_servers=$(( $php_max_children / 2 ))
+        php_max_spare_servers=$(at_least_one $php_max_spare_servers)
+
+        php_start_servers=$(( $php_min_spare_servers + ( $php_max_spare_servers - $php_min_spare_servers ) /2 ))
+        php_start_servers=$(at_least_one $php_start_servers)
+    else
+        php_min_spare_servers=0
+        php_max_spare_servers=0
+        php_start_servers=0
+    fi
+
+    if [ $print -eq 1 ]
+    then
+        ynh_debug --message="Footprint=${footprint}Mb by pool."
+        ynh_debug --message="Process manager=$php_pm"
+        ynh_debug --message="Max RAM=${max_ram}Mb"
+        if [ "$php_pm" != "static" ]
+        then
+            ynh_debug --message="\nMax estimated footprint=$(( $php_max_children * $footprint ))"
+            ynh_debug --message="Min estimated footprint=$(( $php_min_spare_servers * $footprint ))"
+        fi
+        if [ "$php_pm" = "dynamic" ]
+        then
+            ynh_debug --message="Estimated average footprint=$(( $php_max_spare_servers * $footprint ))"
+        elif [ "$php_pm" = "static" ]
+        then
+            ynh_debug --message="Estimated footprint=$(( $php_max_children * $footprint ))"
+        fi
+        ynh_debug --message="\nRaw php-fpm values:"
+        ynh_debug --message="pm.max_children = $php_max_children"
+        if [ "$php_pm" = "dynamic" ]
+        then
+            ynh_debug --message="pm.start_servers = $php_start_servers"
+            ynh_debug --message="pm.min_spare_servers = $php_min_spare_servers"
+            ynh_debug --message="pm.max_spare_servers = $php_max_spare_servers"
+        fi
+    fi
+}
+
+readonly YNH_DEFAULT_COMPOSER_VERSION=1.10.17
+# Declare the actual composer version to use.
+# A packager willing to use another version of composer can override the variable into its _common.sh.
+YNH_COMPOSER_VERSION=${YNH_COMPOSER_VERSION:-$YNH_DEFAULT_COMPOSER_VERSION}
+
+# Execute a command with Composer
+#
+# usage: ynh_composer_exec [--phpversion=phpversion] [--workdir=$final_path] --commands="commands"
+# | arg: -v, --phpversion - PHP version to use with composer
+# | arg: -w, --workdir - The directory from where the command will be executed. Default $final_path.
+# | arg: -c, --commands - Commands to execute.
+#
+# Requires YunoHost version 4.2 or higher.
+ynh_composer_exec () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=vwc
+    declare -Ar args_array=( [v]=phpversion= [w]=workdir= [c]=commands= )
+    local phpversion
+    local workdir
+    local commands
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    workdir="${workdir:-$final_path}"
+    phpversion="${phpversion:-$YNH_PHP_VERSION}"
+
+    COMPOSER_HOME="$workdir/.composer" COMPOSER_MEMORY_LIMIT=-1 \
+        php${phpversion} "$workdir/composer.phar" $commands \
+        -d "$workdir" --quiet --no-interaction
+}
+
+# Install and initialize Composer in the given directory
+#
+# usage: ynh_install_composer [--phpversion=phpversion] [--workdir=$final_path] [--install_args="--optimize-autoloader"] [--composerversion=composerversion]
+# | arg: -v, --phpversion - PHP version to use with composer
+# | arg: -w, --workdir - The directory from where the command will be executed. Default $final_path.
+# | arg: -a, --install_args - Additional arguments provided to the composer install. Argument --no-dev already include
+# | arg: -c, --composerversion - Composer version to install
+#
+# Requires YunoHost version 4.2 or higher.
+ynh_install_composer () {
+    # Declare an array to define the options of this helper.
+    local legacy_args=vwac
+    declare -Ar args_array=( [v]=phpversion= [w]=workdir= [a]=install_args= [c]=composerversion=)
+    local phpversion
+    local workdir
+    local install_args
+    local composerversion
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    workdir="${workdir:-$final_path}"
+    phpversion="${phpversion:-$YNH_PHP_VERSION}"
+    install_args="${install_args:-}"
+    composerversion="${composerversion:-$YNH_COMPOSER_VERSION}"
+
+    curl -sS https://getcomposer.org/installer \
+        | COMPOSER_HOME="$workdir/.composer" \
+        php${phpversion} -- --quiet --install-dir="$workdir" --version=$composerversion \
+        || ynh_die --message="Unable to install Composer."
+
+    # install dependencies
+    ynh_composer_exec --phpversion="${phpversion}" --workdir="$workdir" --commands="install --no-dev $install_args" \
+        || ynh_die --message="Unable to install core dependencies with Composer."
 }

data/helpers.d/postgresql

@@ -1,73 +1,84 @@
 #!/bin/bash
 
 PSQL_ROOT_PWD_FILE=/etc/yunohost/psql
+PSQL_VERSION=11
 
 # Open a connection as a user
 #
+# usage: ynh_psql_connect_as --user=user --password=password [--database=database]
+# | arg: -u, --user=        - the user name to connect as
+# | arg: -p, --password=    - the user password
+# | arg: -d, --database=    - the database to connect to
+#
 # examples:
 #    ynh_psql_connect_as 'user' 'pass' <<< "UPDATE ...;"
 #    ynh_psql_connect_as 'user' 'pass' < /path/to/file.sql
 #
-# usage: ynh_psql_connect_as --user=user --password=password [--database=database]
-# | arg: -u, --user - the user name to connect as
-# | arg: -p, --password - the user password
-# | arg: -d, --database - the database to connect to
-#
 # Requires YunoHost version 3.5.0 or higher.
 ynh_psql_connect_as() {
-	# Declare an array to define the options of this helper.
-	local legacy_args=upd
-	declare -Ar args_array=([u]=user= [p]=password= [d]=database=)
-	local user
-	local password
-	local database
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
-	database="${database:-}"
+    # Declare an array to define the options of this helper.
+    local legacy_args=upd
+    local -A args_array=([u]=user= [p]=password= [d]=database=)
+    local user
+    local password
+    local database
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    database="${database:-}"
 
-	sudo --login --user=postgres PGUSER="$user" PGPASSWORD="$password" psql "$database"
+    sudo --login --user=postgres PGUSER="$user" PGPASSWORD="$password" psql "$database"
 }
 
 # Execute a command as root user
 #
 # usage: ynh_psql_execute_as_root --sql=sql [--database=database]
-# | arg: -s, --sql - the SQL command to execute
-# | arg: -d, --database - the database to connect to
+# | arg: -s, --sql=         - the SQL command to execute
+# | arg: -d, --database=    - the database to connect to
 #
 # Requires YunoHost version 3.5.0 or higher.
 ynh_psql_execute_as_root() {
-	# Declare an array to define the options of this helper.
-	local legacy_args=sd
-	declare -Ar args_array=([s]=sql= [d]=database=)
-	local sql
-	local database
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
-	database="${database:-}"
+    # Declare an array to define the options of this helper.
+    local legacy_args=sd
+    local -A args_array=([s]=sql= [d]=database=)
+    local sql
+    local database
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    database="${database:-}"
 
-	ynh_psql_connect_as --user="postgres" --password="$(sudo cat $PSQL_ROOT_PWD_FILE)" \
-		--database="$database" <<<"$sql"
+    if [ -n "$database" ]
+    then
+        database="--database=$database"
+    fi
+
+    ynh_psql_connect_as --user="postgres" --password="$(cat $PSQL_ROOT_PWD_FILE)" \
+        $database <<<"$sql"
 }
 
 # Execute a command from a file as root user
 #
 # usage: ynh_psql_execute_file_as_root --file=file [--database=database]
-# | arg: -f, --file - the file containing SQL commands
-# | arg: -d, --database - the database to connect to
+# | arg: -f, --file=        - the file containing SQL commands
+# | arg: -d, --database=    - the database to connect to
 #
 # Requires YunoHost version 3.5.0 or higher.
 ynh_psql_execute_file_as_root() {
-	# Declare an array to define the options of this helper.
-	local legacy_args=fd
-	declare -Ar args_array=([f]=file= [d]=database=)
-	local file
-	local database
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
-	database="${database:-}"
+    # Declare an array to define the options of this helper.
+    local legacy_args=fd
+    local -A args_array=([f]=file= [d]=database=)
+    local file
+    local database
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    database="${database:-}"
 
-	ynh_psql_connect_as --user="postgres" --password="$(sudo cat $PSQL_ROOT_PWD_FILE)" \
-		--database="$database" <"$file"
+    if [ -n "$database" ]
+    then
+        database="--database=$database"
+    fi
+
+    ynh_psql_connect_as --user="postgres" --password="$(cat $PSQL_ROOT_PWD_FILE)" \
+        $database <"$file"
 }
 
 # Create a database and grant optionnaly privilegies to a user
@@ -80,17 +91,18 @@ ynh_psql_execute_file_as_root() {
 #
 # Requires YunoHost version 3.5.0 or higher.
 ynh_psql_create_db() {
-	local db=$1
-	local user=${2:-}
+    local db=$1
+    local user=${2:-}
 
-	local sql="CREATE DATABASE ${db};"
+    local sql="CREATE DATABASE ${db};"
 
-	# grant all privilegies to user
-	if [ -n "$user" ]; then
-		sql+="GRANT ALL PRIVILEGES ON DATABASE ${db} TO ${user} WITH GRANT OPTION;"
-	fi
+    # grant all privilegies to user
+    if [ -n "$user" ]; then
+        sql+="ALTER DATABASE ${db} OWNER TO ${user};"
+        sql+="GRANT ALL PRIVILEGES ON DATABASE ${db} TO ${user} WITH GRANT OPTION;"
+    fi
 
-	ynh_psql_execute_as_root --sql="$sql"
+    ynh_psql_execute_as_root --sql="$sql"
 }
 
 # Drop a database
@@ -105,32 +117,32 @@ ynh_psql_create_db() {
 #
 # Requires YunoHost version 3.5.0 or higher.
 ynh_psql_drop_db() {
-	local db=$1
-	# First, force disconnection of all clients connected to the database
-	# https://stackoverflow.com/questions/5408156/how-to-drop-a-postgresql-database-if-there-are-active-connections-to-it
-	# https://dba.stackexchange.com/questions/16426/how-to-drop-all-connections-to-a-specific-database-without-stopping-the-server
-	ynh_psql_execute_as_root --sql="SELECT pg_terminate_backend (pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$db';" --database="$db"
-	sudo --login --user=postgres dropdb $db
+    local db=$1
+    # First, force disconnection of all clients connected to the database
+    # https://stackoverflow.com/questions/17449420/postgresql-unable-to-drop-database-because-of-some-auto-connections-to-db
+    ynh_psql_execute_as_root --sql="REVOKE CONNECT ON DATABASE $db FROM public;" --database="$db"
+    ynh_psql_execute_as_root --sql="SELECT pg_terminate_backend (pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$db' AND pid <> pg_backend_pid();" --database="$db"
+    sudo --login --user=postgres dropdb $db
 }
 
 # Dump a database
 #
-# example: ynh_psql_dump_db 'roundcube' > ./dump.sql
-#
 # usage: ynh_psql_dump_db --database=database
-# | arg: -d, --database - the database name to dump
+# | arg: -d, --database=    - the database name to dump
 # | ret: the psqldump output
 #
+# example: ynh_psql_dump_db 'roundcube' > ./dump.sql
+#
 # Requires YunoHost version 3.5.0 or higher.
 ynh_psql_dump_db() {
-	# Declare an array to define the options of this helper.
-	local legacy_args=d
-	declare -Ar args_array=([d]=database=)
-	local database
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=d
+    local -A args_array=([d]=database=)
+    local database
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
 
-	sudo --login --user=postgres pg_dump "$database"
+    sudo --login --user=postgres pg_dump "$database"
 }
 
 # Create a user
@@ -143,47 +155,55 @@ ynh_psql_dump_db() {
 #
 # Requires YunoHost version 3.5.0 or higher.
 ynh_psql_create_user() {
-	local user=$1
-	local pwd=$2
-	ynh_psql_execute_as_root --sql="CREATE USER $user WITH ENCRYPTED PASSWORD '$pwd'"
+    local user=$1
+    local pwd=$2
+    ynh_psql_execute_as_root --sql="CREATE USER $user WITH ENCRYPTED PASSWORD '$pwd'"
 }
 
 # Check if a psql user exists
 #
 # usage: ynh_psql_user_exists --user=user
-# | arg: -u, --user - the user for which to check existence
+# | arg: -u, --user=    - the user for which to check existence
+# | exit: Return 1 if the user doesn't exist, 0 otherwise
+#
+# Requires YunoHost version 3.5.0 or higher.
 ynh_psql_user_exists() {
-	# Declare an array to define the options of this helper.
-	local legacy_args=u
-	declare -Ar args_array=([u]=user=)
-	local user
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=u
+    local -A args_array=([u]=user=)
+    local user
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
 
-	if ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(sudo cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT rolname FROM pg_roles WHERE rolname='$user';" | grep --quiet "$user" ; then
-		return 1
-	else
-		return 0
-	fi
+    if ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT rolname FROM pg_roles WHERE rolname='$user';" | grep --quiet "$user"
+    then
+        return 1
+    else
+        return 0
+    fi
 }
 
 # Check if a psql database exists
 #
 # usage: ynh_psql_database_exists --database=database
-# | arg: -d, --database - the database for which to check existence
+# | arg: -d, --database=    - the database for which to check existence
+# | exit: Return 1 if the database doesn't exist, 0 otherwise
+#
+# Requires YunoHost version 3.5.0 or higher.
 ynh_psql_database_exists() {
-	# Declare an array to define the options of this helper.
-	local legacy_args=d
-	declare -Ar args_array=([d]=database=)
-	local database
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=d
+    local -A args_array=([d]=database=)
+    local database
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
 
-	if ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(sudo cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT datname FROM pg_database WHERE datname='$database';" | grep --quiet "$database"; then
-		return 1
-	else
-		return 0
-	fi
+    if ! sudo --login --user=postgres PGUSER="postgres" PGPASSWORD="$(cat $PSQL_ROOT_PWD_FILE)" psql -tAc "SELECT datname FROM pg_database WHERE datname='$database';" | grep --quiet "$database"
+    then
+        return 1
+    else
+        return 0
+    fi
 }
 
 # Drop a user
@@ -195,104 +215,118 @@ ynh_psql_database_exists() {
 #
 # Requires YunoHost version 3.5.0 or higher.
 ynh_psql_drop_user() {
-	ynh_psql_execute_as_root --sql="DROP USER ${1};"
+    ynh_psql_execute_as_root --sql="DROP USER ${1};"
 }
 
 # Create a database, an user and its password. Then store the password in the app's config
 #
+# usage: ynh_psql_setup_db --db_user=user --db_name=name [--db_pwd=pwd]
+# | arg: -u, --db_user=     - Owner of the database
+# | arg: -n, --db_name=     - Name of the database
+# | arg: -p, --db_pwd=      - Password of the database. If not provided, a password will be generated
+#
 # After executing this helper, the password of the created database will be available in $db_pwd
 # It will also be stored as "psqlpwd" into the app settings.
 #
-# usage: ynh_psql_setup_db --db_user=user --db_name=name [--db_pwd=pwd]
-# | arg: -u, --db_user - Owner of the database
-# | arg: -n, --db_name - Name of the database
-# | arg: -p, --db_pwd - Password of the database. If not given, a password will be generated
+# Requires YunoHost version 2.7.13 or higher.
 ynh_psql_setup_db() {
-	# Declare an array to define the options of this helper.
-	local legacy_args=unp
-	declare -Ar args_array=([u]=db_user= [n]=db_name= [p]=db_pwd=)
-	local db_user
-	local db_name
-	db_pwd=""
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=unp
+    local -A args_array=([u]=db_user= [n]=db_name= [p]=db_pwd=)
+    local db_user
+    local db_name
+    db_pwd=""
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
 
-	local new_db_pwd=$(ynh_string_random) # Generate a random password
-	# If $db_pwd is not given, use new_db_pwd instead for db_pwd
-	db_pwd="${db_pwd:-$new_db_pwd}"
+    if ! ynh_psql_user_exists --user=$db_user; then
+        local new_db_pwd=$(ynh_string_random) # Generate a random password
+        # If $db_pwd is not provided, use new_db_pwd instead for db_pwd
+        db_pwd="${db_pwd:-$new_db_pwd}"
 
-	if ! ynh_psql_user_exists --user=$db_user; then
-		ynh_psql_create_user "$db_user" "$db_pwd"
-	fi
+        ynh_psql_create_user "$db_user" "$db_pwd"
+    elif [ -z $db_pwd ]; then
+        ynh_die --message="The user $db_user exists, please provide his password"
+    fi
 
-	ynh_psql_create_db "$db_name" "$db_user"                     # Create the database
-	ynh_app_setting_set --app=$app --key=psqlpwd --value=$db_pwd # Store the password in the app's config
+    ynh_psql_create_db "$db_name" "$db_user"                     # Create the database
+    ynh_app_setting_set --app=$app --key=psqlpwd --value=$db_pwd # Store the password in the app's config
 }
 
 # Remove a database if it exists, and the associated user
 #
 # usage: ynh_psql_remove_db --db_user=user --db_name=name
-# | arg: -u, --db_user - Owner of the database
-# | arg: -n, --db_name - Name of the database
+# | arg: -u, --db_user=     - Owner of the database
+# | arg: -n, --db_name=     - Name of the database
+#
+# Requires YunoHost version 2.7.13 or higher.
 ynh_psql_remove_db() {
-	# Declare an array to define the options of this helper.
-	local legacy_args=un
-	declare -Ar args_array=([u]=db_user= [n]=db_name=)
-	local db_user
-	local db_name
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=un
+    local -A args_array=([u]=db_user= [n]=db_name=)
+    local db_user
+    local db_name
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
 
-	local psql_root_password=$(sudo cat $PSQL_ROOT_PWD_FILE)
-	if ynh_psql_database_exists --database=$db_name; then # Check if the database exists
-		ynh_psql_drop_db $db_name # Remove the database
-	else
-		ynh_print_warn --message="Database $db_name not found"
-	fi
+    if ynh_psql_database_exists --database=$db_name
+    then # Check if the database exists
+        ynh_psql_drop_db $db_name # Remove the database
+    else
+        ynh_print_warn --message="Database $db_name not found"
+    fi
 
-	# Remove psql user if it exists
-	if ynh_psql_user_exists --user=$db_user; then
-		ynh_psql_drop_user $db_user
-	else
-		ynh_print_warn --message="User $db_user not found"
-	fi
+    # Remove psql user if it exists
+    if ynh_psql_user_exists --user=$db_user
+    then
+        ynh_psql_drop_user $db_user
+    else
+        ynh_print_warn --message="User $db_user not found"
+    fi
 }
 
 # Create a master password and set up global settings
-# Please always call this script in install and restore scripts
 #
 # usage: ynh_psql_test_if_first_run
+#
+# It also make sure that postgresql is installed and running
+# Please always call this script in install and restore scripts
+#
+# Requires YunoHost version 2.7.13 or higher.
 ynh_psql_test_if_first_run() {
-	if [ -f "$PSQL_ROOT_PWD_FILE" ]; then
-		echo "PostgreSQL is already installed, no need to create master password"
-	else
-		local psql_root_password="$(ynh_string_random)"
-		echo "$psql_root_password" >$PSQL_ROOT_PWD_FILE
 
-		if [ -e /etc/postgresql/9.4/ ]; then
-			local pg_hba=/etc/postgresql/9.4/main/pg_hba.conf
-			local logfile=/var/log/postgresql/postgresql-9.4-main.log
-		elif [ -e /etc/postgresql/9.6/ ]; then
-			local pg_hba=/etc/postgresql/9.6/main/pg_hba.conf
-			local logfile=/var/log/postgresql/postgresql-9.6-main.log
-		else
-			ynh_die "postgresql shoud be 9.4 or 9.6"
-		fi
+    # Make sure postgresql is indeed installed
+    dpkg --list | grep -q "ii  postgresql-$PSQL_VERSION" || ynh_die --message="postgresql-$PSQL_VERSION is not installed !?"
 
-		ynh_systemd_action --service_name=postgresql --action=start
+    # Check for some weird issue where postgresql could be installed but etc folder would not exist ...
+    [ -e "/etc/postgresql/$PSQL_VERSION" ] || ynh_die --message="It looks like postgresql was not properly configured ? /etc/postgresql/$PSQL_VERSION is missing ... Could be due to a locale issue, c.f.https://serverfault.com/questions/426989/postgresql-etc-postgresql-doesnt-exist"
 
-		sudo --login --user=postgres psql -c"ALTER user postgres WITH PASSWORD '$psql_root_password'" postgres
+    # Make sure postgresql is started and enabled
+    # (N.B. : to check the active state, we check the cluster state because
+    # postgresql could be flagged as active even though the cluster is in
+    # failed state because of how the service is configured..)
+    systemctl is-active postgresql@$PSQL_VERSION-main -q || ynh_systemd_action --service_name=postgresql --action=restart
+    systemctl is-enabled postgresql -q || systemctl enable postgresql --quiet
 
-		# force all user to connect to local database using passwords
-		# https://www.postgresql.org/docs/current/static/auth-pg-hba-conf.html#EXAMPLE-PG-HBA.CONF
-		# Note: we can't use peer since YunoHost create users with nologin
-		#  See: https://github.com/YunoHost/yunohost/blob/unstable/data/helpers.d/user
-		ynh_replace_string --match_string="local\(\s*\)all\(\s*\)all\(\s*\)peer" --replace_string="local\1all\2all\3password" --target_file="$pg_hba"
+    # If this is the very first time, we define the root password
+    # and configure a few things
+    if [ ! -f "$PSQL_ROOT_PWD_FILE" ]
+    then
+        local pg_hba=/etc/postgresql/$PSQL_VERSION/main/pg_hba.conf
 
-		# Advertise service in admin panel
-		yunohost service add postgresql --log "$logfile"
+        local psql_root_password="$(ynh_string_random)"
+        echo "$psql_root_password" >$PSQL_ROOT_PWD_FILE
+        sudo --login --user=postgres psql -c"ALTER user postgres WITH PASSWORD '$psql_root_password'" postgres
 
-		systemctl enable postgresql
-		ynh_systemd_action --service_name=postgresql --action=reload
-	fi
+        # force all user to connect to local databases using hashed passwords
+        # https://www.postgresql.org/docs/current/static/auth-pg-hba-conf.html#EXAMPLE-PG-HBA.CONF
+        # Note: we can't use peer since YunoHost create users with nologin
+        #  See: https://github.com/YunoHost/yunohost/blob/unstable/data/helpers.d/user
+        ynh_replace_string --match_string="local\(\s*\)all\(\s*\)all\(\s*\)peer" --replace_string="local\1all\2all\3md5" --target_file="$pg_hba"
+
+        # Integrate postgresql service in yunohost
+        yunohost service add postgresql --log "/var/log/postgresql/"
+
+        ynh_systemd_action --service_name=postgresql --action=reload
+    fi
 }

data/helpers.d/setting

@@ -3,152 +3,72 @@
 # Get an application setting
 #
 # usage: ynh_app_setting_get --app=app --key=key
-# | arg: -a, --app - the application id
-# | arg: -k, --key - the setting to get
+# | arg: -a, --app=     - the application id
+# | arg: -k, --key=     - the setting to get
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_app_setting_get() {
     # Declare an array to define the options of this helper.
     local legacy_args=ak
-    declare -Ar args_array=( [a]=app= [k]=key= )
+    local -A args_array=( [a]=app= [k]=key= )
     local app
     local key
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    ynh_app_setting "get" "$app" "$key"
+    if [[ $key =~ (unprotected|protected|skipped)_ ]]; then
+        yunohost app setting $app $key
+    else
+        ynh_app_setting "get" "$app" "$key"
+    fi
 }
 
 # Set an application setting
 #
 # usage: ynh_app_setting_set --app=app --key=key --value=value
-# | arg: -a, --app - the application id
-# | arg: -k, --key - the setting name to set
-# | arg: -v, --value - the setting value to set
+# | arg: -a, --app=     - the application id
+# | arg: -k, --key=     - the setting name to set
+# | arg: -v, --value=   - the setting value to set
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_app_setting_set() {
     # Declare an array to define the options of this helper.
     local legacy_args=akv
-    declare -Ar args_array=( [a]=app= [k]=key= [v]=value= )
+    local -A args_array=( [a]=app= [k]=key= [v]=value= )
     local app
     local key
     local value
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    ynh_app_setting "set" "$app" "$key" "$value"
+    if [[ $key =~ (unprotected|protected|skipped)_ ]]; then
+        yunohost app setting $app $key -v $value
+    else
+        ynh_app_setting "set" "$app" "$key" "$value"
+    fi
 }
 
 # Delete an application setting
 #
 # usage: ynh_app_setting_delete --app=app --key=key
-# | arg: -a, --app - the application id
-# | arg: -k, --key - the setting to delete
+# | arg: -a, --app=     - the application id
+# | arg: -k, --key=     - the setting to delete
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_app_setting_delete() {
     # Declare an array to define the options of this helper.
     local legacy_args=ak
-    declare -Ar args_array=( [a]=app= [k]=key= )
+    local -A args_array=( [a]=app= [k]=key= )
     local app
     local key
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    ynh_app_setting "delete" "$app" "$key"
-}
-
-# Add skipped_uris urls into the config
-#
-# usage: ynh_add_skipped_uris [--appid=app] --url=url1,url2 [--regex]
-# | arg: -a, --appid - the application id
-# | arg: -u, --url - the urls to add to the sso for this app
-# | arg: -r, --regex - Use the key 'skipped_regex' instead of 'skipped_uris'
-#
-# An URL set with 'skipped_uris' key will be totally ignored by the SSO,
-# which means that the access will be public and the logged-in user information will not be passed to the app.
-#
-# Requires YunoHost version 3.6.0 or higher.
-ynh_add_skipped_uris() {
-    # Declare an array to define the options of this helper.
-    local legacy_args=aur
-    declare -Ar args_array=( [a]=appid= [u]=url= [r]=regex )
-    local appid
-    local url
-    local regex
-    # Manage arguments with getopts
-    ynh_handle_getopts_args "$@"
-    appid={appid:-$app}
-    regex={regex:-0}
-
-    local key=skipped_uris
-    if [ $regex -eq 1 ]; then
-        key=skipped_regex
+    if [[ "$key" =~ (unprotected|skipped|protected)_ ]]; then
+        yunohost app setting $app $key -d
+    else
+        ynh_app_setting "delete" "$app" "$key"
     fi
-
-    ynh_app_setting_set --app=$appid --key=$key --value="$url"
-}
-
-# Add unprotected_uris urls into the config
-#
-# usage: ynh_add_unprotected_uris [--appid=app] --url=url1,url2 [--regex]
-# | arg: -a, --appid - the application id
-# | arg: -u, --url - the urls to add to the sso for this app
-# | arg: -r, --regex - Use the key 'unprotected_regex' instead of 'unprotected_uris'
-#
-# An URL set with unprotected_uris key will be accessible publicly, but if an user is logged in,
-# his information will be accessible (through HTTP headers) to the app.
-#
-# Requires YunoHost version 3.6.0 or higher.
-ynh_add_unprotected_uris() {
-    # Declare an array to define the options of this helper.
-    local legacy_args=aur
-    declare -Ar args_array=( [a]=appid= [u]=url= [r]=regex )
-    local appid
-    local url
-    local regex
-    # Manage arguments with getopts
-    ynh_handle_getopts_args "$@"
-    appid={appid:-$app}
-    regex={regex:-0}
-
-    local key=unprotected_uris
-    if [ $regex -eq 1 ]; then
-        key=unprotected_regex
-    fi
-
-    ynh_app_setting_set --app=$appid --key=$key --value="$url"
-}
-
-# Add protected_uris urls into the config
-#
-# usage: ynh_add_protected_uris [--appid=app] --url=url1,url2 [--regex]
-# | arg: -a, --appid - the application id
-# | arg: -u, --url - the urls to add to the sso for this app
-# | arg: -r, --regex - Use the key 'protected_regex' instead of 'protected_uris'
-#
-# An URL set with protected_uris will be blocked by the SSO and accessible only to authenticated and authorized users.
-#
-# Requires YunoHost version 3.6.0 or higher.
-ynh_add_protected_uris() {
-    # Declare an array to define the options of this helper.
-    local legacy_args=aur
-    declare -Ar args_array=( [a]=appid= [u]=url= [r]=regex )
-    local appid
-    local url
-    local regex
-    # Manage arguments with getopts
-    ynh_handle_getopts_args "$@"
-    appid={appid:-$app}
-    regex={regex:-0}
-
-    local key=protected_uris
-    if [ $regex -eq 1 ]; then
-        key=protected_regex
-    fi
-
-    ynh_app_setting_set --app=$appid --key=$key --value="$url"
 }
 
 # Small "hard-coded" interface to avoid calling "yunohost app" directly each
@@ -158,14 +78,15 @@ ynh_add_protected_uris() {
 #
 ynh_app_setting()
 {
-    ACTION="$1" APP="$2" KEY="$3" VALUE="${4:-}" python - <<EOF
-import os, yaml
+    set +o xtrace # set +x
+    ACTION="$1" APP="$2" KEY="$3" VALUE="${4:-}" python3 - <<EOF
+import os, yaml, sys
 app, action = os.environ['APP'], os.environ['ACTION'].lower()
 key, value = os.environ['KEY'], os.environ.get('VALUE', None)
 setting_file = "/etc/yunohost/apps/%s/settings.yml" % app
 assert os.path.exists(setting_file), "Setting file %s does not exists ?" % setting_file
 with open(setting_file) as f:
-    settings = yaml.load(f)
+    settings = yaml.safe_load(f)
 if action == "get":
     if key in settings:
         print(settings[key])
@@ -175,125 +96,56 @@ else:
             del settings[key]
     elif action == "set":
         if key in ['redirected_urls', 'redirected_regex']:
-            value = yaml.load(value)
+            value = yaml.safe_load(value)
         settings[key] = value
     else:
         raise ValueError("action should either be get, set or delete")
     with open(setting_file, "w") as f:
         yaml.safe_dump(settings, f, default_flow_style=False)
 EOF
+    set -o xtrace # set -x
 }
 
 # Check availability of a web path
 #
-# example: ynh_webpath_available --domain=some.domain.tld --path_url=/coffee
-#
 # usage: ynh_webpath_available --domain=domain --path_url=path
-# | arg: -d, --domain - the domain/host of the url
-# | arg: -p, --path_url - the web path to check the availability of
+# | arg: -d, --domain=      - the domain/host of the url
+# | arg: -p, --path_url=    - the web path to check the availability of
+#
+# example: ynh_webpath_available --domain=some.domain.tld --path_url=/coffee
 #
 # Requires YunoHost version 2.6.4 or higher.
 ynh_webpath_available () {
-	# Declare an array to define the options of this helper.
-	local legacy_args=dp
-	declare -Ar args_array=( [d]=domain= [p]=path_url= )
-	local domain
-	local path_url
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=dp
+    local -A args_array=( [d]=domain= [p]=path_url= )
+    local domain
+    local path_url
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
 
-	sudo yunohost domain url-available $domain $path_url
+    yunohost domain url-available $domain $path_url
 }
 
 # Register/book a web path for an app
 #
-# example: ynh_webpath_register --app=wordpress --domain=some.domain.tld --path_url=/coffee
-#
 # usage: ynh_webpath_register --app=app --domain=domain --path_url=path
-# | arg: -a, --app - the app for which the domain should be registered
-# | arg: -d, --domain - the domain/host of the web path
-# | arg: -p, --path_url - the web path to be registered
+# | arg: -a, --app=         - the app for which the domain should be registered
+# | arg: -d, --domain=      - the domain/host of the web path
+# | arg: -p, --path_url=    - the web path to be registered
+#
+# example: ynh_webpath_register --app=wordpress --domain=some.domain.tld --path_url=/coffee
 #
 # Requires YunoHost version 2.6.4 or higher.
 ynh_webpath_register () {
-	# Declare an array to define the options of this helper.
-	local legacy_args=adp
-	declare -Ar args_array=( [a]=app= [d]=domain= [p]=path_url= )
-	local app
-	local domain
-	local path_url
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
-
-	sudo yunohost app register-url $app $domain $path_url
-}
-
-# Create a new permission for the app
-#
-# usage: ynh_permission_create --app "app" --permission "permission" --defaultdisallow [--urls "url" ["url" ...]]
-# | arg: app - the application id
-# | arg: permission - the name for the permission (by default a permission named "main" already exist)
-# | arg: defaultdisallow - define if all user will be allowed by default
-# | arg: urls - the list of urls for the the permission
-ynh_permission_create() {
-    declare -Ar args_array=( [a]=app= [p]=permission= [d]=defaultdisallow [u]=urls= )
+    # Declare an array to define the options of this helper.
+    local legacy_args=adp
+    local -A args_array=( [a]=app= [d]=domain= [p]=path_url= )
     local app
-    local permission
-    local defaultdisallow
-    local urls
-    ynh_handle_getopts_args "$@"
-    if [[ -n ${defaultdisallow:-} ]]; then
-        defaultdisallow=",default_allow=False"
-    fi
-
-    if [[ -n ${urls:-} ]]; then
-        urls=",urls=['${urls//';'/"','"}']"
-    fi
-    yunohost tools shell -c "from yunohost.permission import permission_add; permission_add('$app', '$permission' ${defaultdisallow:-} ${urls:-}, sync_perm=False)"
-}
-
-# Remove a permission for the app (note that when the app is removed all permission is automatically removed)
-#
-# usage: ynh_permission_remove --app "app" --permission "permission"
-# | arg: app - the application id
-# | arg: permission - the name for the permission (by default a permission named "main" is removed automatically when the app is removed)
-ynh_permission_remove() {
-    declare -Ar args_array=( [a]=app= [p]=permission= )
-    local app
-    local permission
+    local domain
+    local path_url
+    # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    yunohost tools shell -c "from yunohost.permission import permission_remove; permission_remove('$app', '$permission', sync_perm=False)"
-}
-
-# Add a path managed by the SSO
-#
-# usage: ynh_permission_add_path --app "app" --permission "permission" --url "url" ["url" ...]
-# | arg: app - the application id
-# | arg: permission - the name for the permission
-# | arg: url - the FULL url for the the permission (ex domain.tld/apps/admin)
-ynh_permission_add_path() {
-    declare -Ar args_array=( [a]=app= [p]=permission= [u]=url= )
-    local app
-    local permission
-    local url
-    ynh_handle_getopts_args "$@"
-
-    yunohost tools shell -c "from yunohost.permission import permission_update; permission_update('$app', '$permission', add_url=['${url//';'/"','"}'], sync_perm=False)"
-}
-
-# Remove a path managed by the SSO
-#
-# usage: ynh_permission_del_path --app "app" --permission "permission" --url "url" ["url" ...]
-# | arg: app - the application id
-# | arg: permission - the name for the permission
-# | arg: url - the FULL url for the the permission (ex domain.tld/apps/admin)
-ynh_permission_del_path() {
-    declare -Ar args_array=( [a]=app= [p]=permission= [u]=url= )
-    local app
-    local permission
-    local url
-    ynh_handle_getopts_args "$@"
-
-    yunohost tools shell -c "from yunohost.permission import permission_update; permission_update('$app', '$permission', remove_url=['${url//';'/"','"}'], sync_perm=False)"
+    yunohost app register-url $app $domain $path_url
 }

data/helpers.d/string

@@ -2,112 +2,117 @@
 
 # Generate a random string
 #
-# example: pwd=$(ynh_string_random --length=8)
-#
 # usage: ynh_string_random [--length=string_length]
-# | arg: -l, --length - the string length to generate (default: 24)
+# | arg: -l, --length=  - the string length to generate (default: 24)
+# | ret: the generated string
+#
+# example: pwd=$(ynh_string_random --length=8)
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_string_random() {
     # Declare an array to define the options of this helper.
     local legacy_args=l
-    declare -Ar args_array=( [l]=length= )
+    local -A args_array=( [l]=length= )
     local length
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
     length=${length:-24}
 
     dd if=/dev/urandom bs=1 count=1000 2> /dev/null \
-      | tr -c -d 'A-Za-z0-9' \
-      | sed -n 's/\(.\{'"$length"'\}\).*/\1/p'
+        | tr --complement --delete 'A-Za-z0-9' \
+        | sed --quiet 's/\(.\{'"$length"'\}\).*/\1/p'
 }
 
 # Substitute/replace a string (or expression) by another in a file
 #
 # usage: ynh_replace_string --match_string=match_string --replace_string=replace_string --target_file=target_file
-# | arg: -m, --match_string - String to be searched and replaced in the file
-# | arg: -r, --replace_string - String that will replace matches
-# | arg: -f, --target_file - File in which the string will be replaced.
+# | arg: -m, --match_string=    - String to be searched and replaced in the file
+# | arg: -r, --replace_string=  - String that will replace matches
+# | arg: -f, --target_file=     - File in which the string will be replaced.
 #
-# As this helper is based on sed command, regular expressions and
-# references to sub-expressions can be used
-# (see sed manual page for more information)
+# As this helper is based on sed command, regular expressions and references to
+# sub-expressions can be used (see sed manual page for more information)
 #
 # Requires YunoHost version 2.6.4 or higher.
 ynh_replace_string () {
-	# Declare an array to define the options of this helper.
-	local legacy_args=mrf
-	declare -Ar args_array=( [m]=match_string= [r]=replace_string= [f]=target_file= )
-	local match_string
-	local replace_string
-	local target_file
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=mrf
+    local -A args_array=( [m]=match_string= [r]=replace_string= [f]=target_file= )
+    local match_string
+    local replace_string
+    local target_file
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    set +o xtrace # set +x
 
-	local delimit=@
-	# Escape the delimiter if it's in the string.
-	match_string=${match_string//${delimit}/"\\${delimit}"}
-	replace_string=${replace_string//${delimit}/"\\${delimit}"}
+    local delimit=@
+    # Escape the delimiter if it's in the string.
+    match_string=${match_string//${delimit}/"\\${delimit}"}
+    replace_string=${replace_string//${delimit}/"\\${delimit}"}
 
-	sudo sed --in-place "s${delimit}${match_string}${delimit}${replace_string}${delimit}g" "$target_file"
+    set -o xtrace # set -x
+    sed --in-place "s${delimit}${match_string}${delimit}${replace_string}${delimit}g" "$target_file"
 }
 
 # Substitute/replace a special string by another in a file
 #
 # usage: ynh_replace_special_string --match_string=match_string --replace_string=replace_string --target_file=target_file
-# | arg: -m, --match_string - String to be searched and replaced in the file
-# | arg: -r, --replace_string - String that will replace matches
-# | arg: -t, --target_file - File in which the string will be replaced.
+# | arg: -m, --match_string=    - String to be searched and replaced in the file
+# | arg: -r, --replace_string=  - String that will replace matches
+# | arg: -t, --target_file=     - File in which the string will be replaced.
 #
 # This helper will use ynh_replace_string, but as you can use special
 # characters, you can't use some regular expressions and sub-expressions.
 #
 # Requires YunoHost version 2.7.7 or higher.
 ynh_replace_special_string () {
-	# Declare an array to define the options of this helper.
-	local legacy_args=mrf
-	declare -Ar args_array=( [m]=match_string= [r]=replace_string= [f]=target_file= )
-	local match_string
-	local replace_string
-	local target_file
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=mrf
+    local -A args_array=( [m]=match_string= [r]=replace_string= [f]=target_file= )
+    local match_string
+    local replace_string
+    local target_file
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
 
-	# Escape any backslash to preserve them as simple backslash.
-	match_string=${match_string//\\/"\\\\"}
-	replace_string=${replace_string//\\/"\\\\"}
+    # Escape any backslash to preserve them as simple backslash.
+    match_string=${match_string//\\/"\\\\"}
+    replace_string=${replace_string//\\/"\\\\"}
 
-	# Escape the & character, who has a special function in sed.
-	match_string=${match_string//&/"\&"}
-	replace_string=${replace_string//&/"\&"}
+    # Escape the & character, who has a special function in sed.
+    match_string=${match_string//&/"\&"}
+    replace_string=${replace_string//&/"\&"}
 
-	ynh_replace_string --match_string="$match_string" --replace_string="$replace_string" --target_file="$target_file"
+    ynh_replace_string --match_string="$match_string" --replace_string="$replace_string" --target_file="$target_file"
 }
 
 # Sanitize a string intended to be the name of a database
-# (More specifically : replace - and . by _)
+#
+# usage: ynh_sanitize_dbid --db_name=name
+# | arg: -n, --db_name=     - name to correct/sanitize
+# | ret: the corrected name
 #
 # example: dbname=$(ynh_sanitize_dbid $app)
 #
-# usage: ynh_sanitize_dbid --db_name=name
-# | arg: -n, --db_name - name to correct/sanitize
-# | ret: the corrected name
+# Underscorify the string (replace - and . by _)
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_sanitize_dbid () {
-	# Declare an array to define the options of this helper.
-	local legacy_args=n
-	declare -Ar args_array=( [n]=db_name= )
-	local db_name
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=n
+    local -A args_array=( [n]=db_name= )
+    local db_name
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
 
-	# We should avoid having - and . in the name of databases. They are replaced by _
-	echo ${db_name//[-.]/_}
+    # We should avoid having - and . in the name of databases. They are replaced by _
+    echo ${db_name//[-.]/_}
 }
 
 # Normalize the url path syntax
 #
+# [internal]
+#
 # Handle the slash at the beginning of path and its absence at ending
 # Return a normalized url path
 #
@@ -119,23 +124,23 @@ ynh_sanitize_dbid () {
 #     ynh_normalize_url_path /          # -> /
 #
 # usage: ynh_normalize_url_path --path_url=path_to_normalize
-# | arg: -p, --path_url - URL path to normalize before using it
+# | arg: -p, --path_url=    - URL path to normalize before using it
 #
 # Requires YunoHost version 2.6.4 or higher.
 ynh_normalize_url_path () {
-	# Declare an array to define the options of this helper.
-	local legacy_args=p
-	declare -Ar args_array=( [p]=path_url= )
-	local path_url
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
+    # Declare an array to define the options of this helper.
+    local legacy_args=p
+    local -A args_array=( [p]=path_url= )
+    local path_url
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
 
-	test -n "$path_url" || ynh_die --message="ynh_normalize_url_path expect a URL path as first argument and received nothing."
-	if [ "${path_url:0:1}" != "/" ]; then    # If the first character is not a /
-		path_url="/$path_url"    # Add / at begin of path variable
-	fi
-	if [ "${path_url:${#path_url}-1}" == "/" ] && [ ${#path_url} -gt 1 ]; then    # If the last character is a / and that not the only character.
-		path_url="${path_url:0:${#path_url}-1}"	# Delete the last character
-	fi
-	echo $path_url
+    test -n "$path_url" || ynh_die --message="ynh_normalize_url_path expect a URL path as first argument and received nothing."
+    if [ "${path_url:0:1}" != "/" ]; then    # If the first character is not a /
+        path_url="/$path_url"    # Add / at begin of path variable
+    fi
+    if [ "${path_url:${#path_url}-1}" == "/" ] && [ ${#path_url} -gt 1 ]; then    # If the last character is a / and that not the only character.
+        path_url="${path_url:0:${#path_url}-1}"	# Delete the last character
+    fi
+    echo $path_url
 }

data/helpers.d/systemd

@@ -3,114 +3,111 @@
 # Create a dedicated systemd config
 #
 # usage: ynh_add_systemd_config [--service=service] [--template=template]
-# | arg: -s, --service - Service name (optionnal, $app by default)
-# | arg: -t, --template - Name of template file (optionnal, this is 'systemd' by default, meaning ./conf/systemd.service will be used as template)
+# | arg: -s, --service=     - Service name (optionnal, `$app` by default)
+# | arg: -t, --template=    - Name of template file (optionnal, this is 'systemd' by default, meaning `../conf/systemd.service` will be used as template)
 #
-# This will use the template ../conf/<templatename>.service
-# to generate a systemd config, by replacing the following keywords
-# with global variables that should be defined before calling
-# this helper :
+# This will use the template `../conf/<templatename>.service`.
 #
-#   __APP__       by  $app
-#   __FINALPATH__ by  $final_path
+# See the documentation of `ynh_add_config` for a description of the template
+# format and how placeholders are replaced with actual variables.
 #
-# Requires YunoHost version 2.7.2 or higher.
+# Requires YunoHost version 4.1.0 or higher.
 ynh_add_systemd_config () {
-	# Declare an array to define the options of this helper.
-	local legacy_args=st
-	declare -Ar args_array=( [s]=service= [t]=template= )
-	local service
-	local template
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
-	local service="${service:-$app}"
-	local template="${template:-systemd.service}"
+    # Declare an array to define the options of this helper.
+    local legacy_args=stv
+    local -A args_array=( [s]=service= [t]=template=  [v]=others_var=)
+    local service
+    local template
+    local others_var
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    service="${service:-$app}"
+    template="${template:-systemd.service}"
+    others_var="${others_var:-}"
 
-	finalsystemdconf="/etc/systemd/system/$service.service"
-	ynh_backup_if_checksum_is_different --file="$finalsystemdconf"
-	sudo cp ../conf/$template "$finalsystemdconf"
+    [[ -z "$others_var" ]] || ynh_print_warn --message="Packagers: using --others_var is unecessary since YunoHost 4.2"
 
-	# To avoid a break by set -u, use a void substitution ${var:-}. If the variable is not set, it's simply set with an empty variable.
-	# Substitute in a nginx config file only if the variable is not empty
-	if test -n "${final_path:-}"; then
-		ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalsystemdconf"
-	fi
-	if test -n "${app:-}"; then
-		ynh_replace_string --match_string="__APP__" --replace_string="$app" --target_file="$finalsystemdconf"
-	fi
-	ynh_store_file_checksum --file="$finalsystemdconf"
+    ynh_add_config --template="$YNH_APP_BASEDIR/conf/$template" --destination="/etc/systemd/system/$service.service"
 
-	sudo chown root: "$finalsystemdconf"
-	sudo systemctl enable $service
-	sudo systemctl daemon-reload
+    systemctl enable $service --quiet
+    systemctl daemon-reload
 }
 
 # Remove the dedicated systemd config
 #
 # usage: ynh_remove_systemd_config [--service=service]
-# | arg: -s, --service - Service name (optionnal, $app by default)
+# | arg: -s, --service=     - Service name (optionnal, $app by default)
 #
 # Requires YunoHost version 2.7.2 or higher.
 ynh_remove_systemd_config () {
-	# Declare an array to define the options of this helper.
-	local legacy_args=s
-	declare -Ar args_array=( [s]=service= )
-	local service
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
-	local service="${service:-$app}"
+    # Declare an array to define the options of this helper.
+    local legacy_args=s
+    local -A args_array=( [s]=service= )
+    local service
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    local service="${service:-$app}"
 
-	local finalsystemdconf="/etc/systemd/system/$service.service"
-	if [ -e "$finalsystemdconf" ]; then
-		ynh_systemd_action --service_name=$service --action=stop
-		systemctl disable $service
-		ynh_secure_remove --file="$finalsystemdconf"
-		systemctl daemon-reload
-	fi
+    local finalsystemdconf="/etc/systemd/system/$service.service"
+    if [ -e "$finalsystemdconf" ]
+    then
+        ynh_systemd_action --service_name=$service --action=stop
+        systemctl disable $service --quiet
+        ynh_secure_remove --file="$finalsystemdconf"
+        systemctl daemon-reload
+    fi
 }
 
 # Start (or other actions) a service,  print a log in case of failure and optionnaly wait until the service is completely started
 #
-# usage: ynh_systemd_action [-n service_name] [-a action] [ [-l "line to match"] [-p log_path] [-t timeout] [-e length] ]
-# | arg: -n, --service_name= - Name of the service to start. Default : $app
+# usage: ynh_systemd_action [--service_name=service_name] [--action=action] [ [--line_match="line to match"] [--log_path=log_path] [--timeout=300] [--length=20] ]
+# | arg: -n, --service_name= - Name of the service to start. Default : `$app`
 # | arg: -a, --action=       - Action to perform with systemctl. Default: start
-# | arg: -l, --line_match=   - Line to match - The line to find in the log to attest the service have finished to boot. If not defined it don't wait until the service is completely started. WARNING: When using --line_match, you should always add `ynh_clean_check_starting` into your `ynh_clean_setup` at the beginning of the script. Otherwise, tail will not stop in case of failure of the script. The script will then hang forever.
-# | arg: -p, --log_path=     - Log file - Path to the log file. Default : /var/log/$app/$app.log
+# | arg: -l, --line_match=   - Line to match - The line to find in the log to attest the service have finished to boot. If not defined it don't wait until the service is completely started.
+# | arg: -p, --log_path=     - Log file - Path to the log file. Default : `/var/log/$app/$app.log`
 # | arg: -t, --timeout=      - Timeout - The maximum time to wait before ending the watching. Default : 300 seconds.
 # | arg: -e, --length=       - Length of the error log : Default : 20
+#
+# Requires YunoHost version 3.5.0 or higher.
 ynh_systemd_action() {
     # Declare an array to define the options of this helper.
     local legacy_args=nalpte
-    declare -Ar args_array=( [n]=service_name= [a]=action= [l]=line_match= [p]=log_path= [t]=timeout= [e]=length= )
+    local -A args_array=( [n]=service_name= [a]=action= [l]=line_match= [p]=log_path= [t]=timeout= [e]=length= )
     local service_name
     local action
     local line_match
     local length
     local log_path
     local timeout
-
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
+    service_name="${service_name:-$app}"
+    action=${action:-start}
+    line_match=${line_match:-}
+    length=${length:-20}
+    log_path="${log_path:-/var/log/$service_name/$service_name.log}"
+    timeout=${timeout:-300}
 
-    local service_name="${service_name:-$app}"
-    local action=${action:-start}
-    local log_path="${log_path:-/var/log/$service_name/$service_name.log}"
-    local length=${length:-20}
-    local timeout=${timeout:-300}
+    # Manage case of service already stopped
+    if [ "$action" == "stop" ] && ! systemctl is-active --quiet $service_name
+    then
+        return 0
+    fi
 
     # Start to read the log
-    if [[ -n "${line_match:-}" ]]
+    if [[ -n "$line_match" ]]
     then
         local templog="$(mktemp)"
         # Following the starting of the app in its log
-        if [ "$log_path" == "systemd" ] ; then
+        if [ "$log_path" == "systemd" ]
+        then
             # Read the systemd journal
             journalctl --unit=$service_name --follow --since=-0 --quiet > "$templog" &
             # Get the PID of the journalctl command
             local pid_tail=$!
         else
             # Read the specified log file
-            tail -F -n0 "$log_path" > "$templog" 2>&1 &
+            tail --follow=name --retry --lines=0 "$log_path" > "$templog" 2>&1 &
             # Get the PID of the tail command
             local pid_tail=$!
         fi
@@ -121,53 +118,70 @@ ynh_systemd_action() {
         action="reload-or-restart"
     fi
 
-    systemctl $action $service_name \
-        || ( journalctl --no-pager --lines=$length -u $service_name >&2 \
-        ; test -e "$log_path" && echo "--" >&2 && tail --lines=$length "$log_path" >&2 \
-        ; false )
+    # If the service fails to perform the action
+    if ! systemctl $action $service_name
+    then
+        # Show syslog for this service
+        ynh_exec_err journalctl --quiet --no-hostname --no-pager --lines=$length --unit=$service_name
+        # If a log is specified for this service, show also the content of this log
+        if [ -e "$log_path" ]
+        then
+            ynh_exec_err tail --lines=$length "$log_path"
+        fi
+        ynh_clean_check_starting
+        return 1
+    fi
 
     # Start the timeout and try to find line_match
     if [[ -n "${line_match:-}" ]]
     then
+        set +x
         local i=0
         for i in $(seq 1 $timeout)
         do
             # Read the log until the sentence is found, that means the app finished to start. Or run until the timeout
-            if grep --quiet "$line_match" "$templog"
+            if grep --extended-regexp --quiet "$line_match" "$templog"
             then
-                ynh_print_info --message="The service $service_name has correctly started."
+                ynh_print_info --message="The service $service_name has correctly executed the action ${action}."
                 break
             fi
-            if [ $i -eq 3 ]; then
-                echo -n "Please wait, the service $service_name is ${action}ing" >&2
-            fi
-            if [ $i -ge 3 ]; then
-                echo -n "." >&2
+            if [ $i -eq 30 ]; then
+                echo "(this may take some time)" >&2
             fi
             sleep 1
         done
+        set -x
         if [ $i -ge 3 ]; then
             echo "" >&2
         fi
         if [ $i -eq $timeout ]
         then
-            ynh_print_warn --message="The service $service_name didn't fully started before the timeout."
+            ynh_print_warn --message="The service $service_name didn't fully executed the action ${action} before the timeout."
             ynh_print_warn --message="Please find here an extract of the end of the log of the service $service_name:"
-            journalctl --no-pager --lines=$length -u $service_name >&2
-            test -e "$log_path" && echo "--" >&2 && tail --lines=$length "$log_path" >&2
+            ynh_exec_warn journalctl --quiet --no-hostname --no-pager --lines=$length --unit=$service_name
+            if [ -e "$log_path" ]
+            then
+                ynh_print_warn --message="\-\-\-"
+                ynh_exec_warn tail --lines=$length "$log_path"
+            fi
         fi
         ynh_clean_check_starting
     fi
 }
 
 # Clean temporary process and file used by ynh_check_starting
-# (usually used in ynh_clean_setup scripts)
 #
-# usage: ynh_clean_check_starting
+# [internal]
+#
+# Requires YunoHost version 3.5.0 or higher.
 ynh_clean_check_starting () {
-	# Stop the execution of tail.
-	kill -s 15 $pid_tail 2>&1
-	ynh_secure_remove "$templog" 2>&1
+    if [ -n "${pid_tail:-}" ]
+    then
+        # Stop the execution of tail.
+        kill -SIGTERM $pid_tail 2>&1
+    fi
+    if [ -n "${templog:-}" ]
+    then
+        ynh_secure_remove --file="$templog" 2>&1
+    fi
 }
-
-

data/helpers.d/user

@@ -2,68 +2,69 @@
 
 # Check if a YunoHost user exists
 #
-# example: ynh_user_exists 'toto' || exit 1
-#
 # usage: ynh_user_exists --username=username
-# | arg: -u, --username - the username to check
+# | arg: -u, --username=    - the username to check
+# | ret: 0 if the user exists, 1 otherwise.
+#
+# example: ynh_user_exists 'toto' || echo "User does not exist"
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_user_exists() {
     # Declare an array to define the options of this helper.
     local legacy_args=u
-    declare -Ar args_array=( [u]=username= )
+    local -A args_array=( [u]=username= )
     local username
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    sudo yunohost user list --output-as json | grep -q "\"username\": \"${username}\""
+    yunohost user list --output-as json --quiet | jq -e ".users.${username}" >/dev/null
 }
 
 # Retrieve a YunoHost user information
 #
-# example: mail=$(ynh_user_get_info 'toto' 'mail')
-#
 # usage: ynh_user_get_info --username=username --key=key
-# | arg: -u, --username - the username to retrieve info from
-# | arg: -k, --key - the key to retrieve
-# | ret: string - the key's value
+# | arg: -u, --username=    - the username to retrieve info from
+# | arg: -k, --key=         - the key to retrieve
+# | ret: the value associate to that key
+#
+# example: mail=$(ynh_user_get_info 'toto' 'mail')
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_user_get_info() {
     # Declare an array to define the options of this helper.
     local legacy_args=uk
-    declare -Ar args_array=( [u]=username= [k]=key= )
+    local -A args_array=( [u]=username= [k]=key= )
     local username
     local key
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    sudo yunohost user info "$username" --output-as plain | ynh_get_plain_key "$key"
+    yunohost user info "$username" --output-as json --quiet | jq -r ".$key"
 }
 
 # Get the list of YunoHost users
 #
-# example: for u in $(ynh_user_list); do ...
-#
 # usage: ynh_user_list
-# | ret: string - one username per line
+# | ret: one username per line as strings
+#
+# example: for u in $(ynh_user_list); do ... ; done
 #
 # Requires YunoHost version 2.4.0 or higher.
 ynh_user_list() {
-    sudo yunohost user list --output-as plain --quiet \
-      | awk '/^##username$/{getline; print}'
+    yunohost user list --output-as json --quiet | jq -r ".users | keys[]"
 }
 
 # Check if a user exists on the system
 #
 # usage: ynh_system_user_exists --username=username
-# | arg: -u, --username - the username to check
+# | arg: -u, --username=    - the username to check
+# | ret: 0 if the user exists, 1 otherwise.
 #
 # Requires YunoHost version 2.2.4 or higher.
 ynh_system_user_exists() {
     # Declare an array to define the options of this helper.
     local legacy_args=u
-    declare -Ar args_array=( [u]=username= )
+    local -A args_array=( [u]=username= )
     local username
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
@@ -74,11 +75,14 @@ ynh_system_user_exists() {
 # Check if a group exists on the system
 #
 # usage: ynh_system_group_exists --group=group
-# | arg: -g, --group - the group to check
+# | arg: -g, --group=   - the group to check
+# | ret: 0 if the group exists, 1 otherwise.
+#
+# Requires YunoHost version 3.5.0.2 or higher.
 ynh_system_group_exists() {
     # Declare an array to define the options of this helper.
     local legacy_args=g
-    declare -Ar args_array=( [g]=group= )
+    local -A args_array=( [g]=group= )
     local group
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
@@ -88,56 +92,71 @@ ynh_system_group_exists() {
 
 # Create a system user
 #
-# examples:
-#   # Create a nextcloud user with no home directory and /usr/sbin/nologin login shell (hence no login capability)
-#   ynh_system_user_create --username=nextcloud
-#   # Create a discourse user using /var/www/discourse as home directory and the default login shell
-#   ynh_system_user_create --username=discourse --home_dir=/var/www/discourse --use_shell 
+# usage: ynh_system_user_create --username=user_name [--home_dir=home_dir] [--use_shell] [--groups="group1 group2"]
+# | arg: -u, --username=    - Name of the system user that will be create
+# | arg: -h, --home_dir=    - Path of the home dir for the user. Usually the final path of the app. If this argument is omitted, the user will be created without home
+# | arg: -s, --use_shell    - Create a user using the default login shell if present. If this argument is omitted, the user will be created with /usr/sbin/nologin shell
+# | arg: -g, --groups       - Add the user to system groups. Typically meant to add the user to the ssh.app / sftp.app group (e.g. for borgserver, my_webapp)
 #
-# usage: ynh_system_user_create --username=user_name [--home_dir=home_dir] [--use_shell]
-# | arg: -u, --username - Name of the system user that will be create
-# | arg: -h, --home_dir - Path of the home dir for the user. Usually the final path of the app. If this argument is omitted, the user will be created without home
-# | arg: -s, --use_shell - Create a user using the default login shell if present. If this argument is omitted, the user will be created with /usr/sbin/nologin shell
+# Create a nextcloud user with no home directory and /usr/sbin/nologin login shell (hence no login capability) :
+# ```
+# ynh_system_user_create --username=nextcloud
+# ```
+# Create a discourse user using /var/www/discourse as home directory and the default login shell :
+# ```
+# ynh_system_user_create --username=discourse --home_dir=/var/www/discourse --use_shell
+# ```
 #
 # Requires YunoHost version 2.6.4 or higher.
 ynh_system_user_create () {
-	# Declare an array to define the options of this helper.
-	local legacy_args=uhs
-	declare -Ar args_array=( [u]=username= [h]=home_dir= [s]=use_shell )
-	local username
-	local home_dir
-	local use_shell
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
- 	use_shell="${use_shell:-0}"
- 	home_dir="${home_dir:-}"
+    # Declare an array to define the options of this helper.
+    local legacy_args=uhs
+    local -A args_array=( [u]=username= [h]=home_dir= [s]=use_shell [g]=groups= )
+    local username
+    local home_dir
+    local use_shell
+    local groups
 
-	if ! ynh_system_user_exists "$username"	# Check if the user exists on the system
-	then	# If the user doesn't exist
-		if [ -n "$home_dir" ]; then	# If a home dir is mentioned
-			local user_home_dir="-d $home_dir"
-		else
-			local user_home_dir="--no-create-home"
-		fi
-		if [ $use_shell -eq 1 ]; then	# If we want a shell for the user
-			local shell="" # Use default shell
-		else
-			local shell="--shell /usr/sbin/nologin"
-		fi
-		useradd $user_home_dir --system --user-group $username $shell || ynh_die "Unable to create $username system account"
-	fi
+    # Manage arguments with getopts
+    ynh_handle_getopts_args "$@"
+    use_shell="${use_shell:-0}"
+    home_dir="${home_dir:-}"
+    groups="${groups:-}"
+
+    if ! ynh_system_user_exists "$username"	# Check if the user exists on the system
+    then	# If the user doesn't exist
+        if [ -n "$home_dir" ]
+        then	# If a home dir is mentioned
+            local user_home_dir="--home-dir $home_dir"
+        else
+            local user_home_dir="--no-create-home"
+        fi
+        if [ $use_shell -eq 1 ]
+        then	# If we want a shell for the user
+            local shell="" # Use default shell
+        else
+            local shell="--shell /usr/sbin/nologin"
+        fi
+        useradd $user_home_dir --system --user-group $username $shell || ynh_die --message="Unable to create $username system account"
+    fi
+
+    local group
+    for group in $groups
+    do
+        usermod -a -G "$group" "$username"
+    done
 }
 
 # Delete a system user
 #
 # usage: ynh_system_user_delete --username=user_name
-# | arg: -u, --username - Name of the system user that will be create
+# | arg: -u, --username=    - Name of the system user that will be create
 #
 # Requires YunoHost version 2.6.4 or higher.
 ynh_system_user_delete () {
     # Declare an array to define the options of this helper.
     local legacy_args=u
-    declare -Ar args_array=( [u]=username= )
+    local -A args_array=( [u]=username= )
     local username
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
@@ -145,14 +164,30 @@ ynh_system_user_delete () {
     # Check if the user exists on the system
     if ynh_system_user_exists "$username"
     then
-		deluser $username
-	else
-		ynh_print_warn --message="The user $username was not found"
+        deluser $username
+    else
+        ynh_print_warn --message="The user $username was not found"
     fi
 
     # Check if the group exists on the system
     if ynh_system_group_exists "$username"
     then
-		delgroup $username
+        delgroup $username
+    fi
+}
+
+# Execute a command as another user
+#
+# usage: ynh_exec_as $USER COMMAND [ARG ...]
+#
+# Requires YunoHost version 4.1.7 or higher.
+ynh_exec_as() {
+    local user=$1
+    shift 1
+
+    if [[ $user = $(whoami) ]]; then
+        eval "$@"
+    else
+        sudo -u "$user" "$@"
     fi
 }

data/hooks/backup/05-conf_ldap

@@ -10,8 +10,8 @@ source /usr/share/yunohost/helpers
 backup_dir="${1}/conf/ldap"
 
 # Backup the configuration
-ynh_backup "/etc/ldap/slapd.conf" "${backup_dir}/slapd.conf"
-sudo slapcat -b cn=config -l "${backup_dir}/cn=config.master.ldif"
+ynh_backup "/etc/ldap/ldap.conf" "${backup_dir}/ldap.conf"
+slapcat -b cn=config -l "${backup_dir}/cn=config.master.ldif"
 
 # Backup the database
-sudo slapcat -b dc=yunohost,dc=org -l "${backup_dir}/dc=yunohost-dc=org.ldif"
+slapcat -b dc=yunohost,dc=org -l "${backup_dir}/dc=yunohost-dc=org.ldif"

data/hooks/backup/08-conf_ssh

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-# Exit hook on subcommand error or unset variable
-set -eu
-
-# Source YNH helpers
-source /usr/share/yunohost/helpers
-
-# Backup destination
-backup_dir="${1}/conf/ssh"
-
-# Backup the configuration
-if [ -d /etc/ssh/ ]; then
-    ynh_backup "/etc/ssh" "$backup_dir"
-else
-    echo "SSH is not installed"
-fi

data/hooks/backup/11-conf_ynh_mysql

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-# Exit hook on subcommand error or unset variable
-set -eu
-
-# Source YNH helpers
-source /usr/share/yunohost/helpers
-
-# Backup destination
-backup_dir="${1}/conf/ynh/mysql"
-
-# Save MySQL root password
-ynh_backup "/etc/yunohost/mysql" "${backup_dir}/root_pwd"

data/hooks/backup/18-data_multimedia

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Exit hook on subcommand error or unset variable
+set -eu
+
+# Source YNH helpers
+source /usr/share/yunohost/helpers
+
+# Backup destination
+backup_dir="${1}/data/multimedia"
+
+if [ -e "/home/yunohost.multimedia/.nobackup" ]; then
+    exit 0
+fi
+
+# Backup multimedia directory
+ynh_backup --src_path="/home/yunohost.multimedia" --dest_path="${backup_dir}" --is_big --not_mandatory

data/hooks/backup/20-conf_ynh_firewall

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-# Exit hook on subcommand error or unset variable
-set -eu
-
-# Source YNH helpers
-source /usr/share/yunohost/helpers
-
-# Backup destination
-backup_dir="${1}/conf/ynh/firewall"
-
-# Backup the configuration
-ynh_backup "/etc/yunohost/firewall.yml" "${backup_dir}/firewall.yml"

data/hooks/backup/20-conf_ynh_settings

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Exit hook on subcommand error or unset variable
+set -eu
+
+# Source YNH helpers
+source /usr/share/yunohost/helpers
+
+# Backup destination
+backup_dir="${1}/conf/ynh"
+
+# Backup the configuration
+ynh_backup "/etc/yunohost/firewall.yml" "${backup_dir}/firewall.yml"
+ynh_backup "/etc/yunohost/current_host" "${backup_dir}/current_host"
+ynh_backup "/etc/yunohost/domains" "${backup_dir}/domains"
+[ ! -e "/etc/yunohost/settings.json" ] || ynh_backup "/etc/yunohost/settings.json" "${backup_dir}/settings.json"
+[ ! -d "/etc/yunohost/dyndns" ] || ynh_backup "/etc/yunohost/dyndns" "${backup_dir}/dyndns"
+[ ! -d "/etc/dkim" ] || ynh_backup "/etc/dkim" "${backup_dir}/dkim"

data/hooks/backup/26-conf_xmpp

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-# Exit hook on subcommand error or unset variable
-set -eu
-
-# Source YNH helpers
-source /usr/share/yunohost/helpers
-
-# Backup destination
-backup_dir="${1}/conf/xmpp"
-
-# Backup the configuration
-ynh_backup /etc/metronome "${backup_dir}/etc"
-ynh_backup /var/lib/metronome "${backup_dir}/var"

data/hooks/backup/27-data_xmpp

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Exit hook on subcommand error or unset variable
+set -eu
+
+# Source YNH helpers
+source /usr/share/yunohost/helpers
+
+# Backup destination
+backup_dir="${1}/data/xmpp"
+
+ynh_backup /var/lib/metronome "${backup_dir}/var_lib_metronome"
+ynh_backup /var/xmpp-upload/ "${backup_dir}/var_xmpp-upload"

data/hooks/backup/29-conf_nginx

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-# Exit hook on subcommand error or unset variable
-set -eu
-
-# Source YNH helpers
-source /usr/share/yunohost/helpers
-
-# Backup destination
-backup_dir="${1}/conf/nginx"
-
-# Backup the configuration
-ynh_backup "/etc/nginx/conf.d" "$backup_dir"

data/hooks/backup/32-conf_cron

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-# Exit hook on subcommand error or unset variable
-set -eu
-
-# Source YNH helpers
-source /usr/share/yunohost/helpers
-
-# Backup destination
-backup_dir="${1}/conf/cron"
-
-# Backup the configuration
-for f in $(ls -1B /etc/cron.d/yunohost* 2> /dev/null); do
-  ynh_backup "$f" "${backup_dir}/${f##*/}"
-done

data/hooks/backup/40-conf_ynh_currenthost

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-# Exit hook on subcommand error or unset variable
-set -eu
-
-# Source YNH helpers
-source /usr/share/yunohost/helpers
-
-# Backup destination
-backup_dir="${1}/conf/ynh"
-
-# Backup the configuration
-ynh_backup "/etc/yunohost/current_host" "${backup_dir}/current_host"

data/hooks/backup/50-conf_manually_modified_files

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+source /usr/share/yunohost/helpers
+ynh_abort_if_errors
+YNH_CWD="${YNH_BACKUP_DIR%/}/conf/manually_modified_files"
+mkdir -p "$YNH_CWD"
+cd "$YNH_CWD"
+
+yunohost tools shell -c "from yunohost.regenconf import manually_modified_files; print('\n'.join(manually_modified_files()))" > ./manually_modified_files_list
+
+ynh_backup --src_path="./manually_modified_files_list"
+
+for file in $(cat ./manually_modified_files_list)
+do
+    [[ -e $file ]] && ynh_backup --src_path="$file"
+done
+    
+ynh_backup --src_path="/etc/ssowat/conf.json.persistent"

data/hooks/conf_regen/01-yunohost

@@ -2,8 +2,6 @@
 
 set -e
 
-services_path="/etc/yunohost/services.yml"
-
 do_init_regen() {
   if [[ $EUID -ne 0 ]]; then
     echo "You must be root to run this script" 1>&2
@@ -19,14 +17,53 @@ do_init_regen() {
     || echo "yunohost.org" > /etc/yunohost/current_host
 
   # copy default services and firewall
-  [[ -f $services_path ]] \
-    || cp services.yml "$services_path"
   [[ -f /etc/yunohost/firewall.yml ]] \
     || cp firewall.yml /etc/yunohost/firewall.yml
 
   # allow users to access /media directory
   [[ -d /etc/skel/media ]] \
     || (mkdir -p /media && ln -s /media /etc/skel/media)
+
+  # Cert folders
+  mkdir -p /etc/yunohost/certs
+  chown -R root:ssl-cert /etc/yunohost/certs
+  chmod 750 /etc/yunohost/certs
+
+  # App folders
+  mkdir -p /etc/yunohost/apps
+  chmod 700 /etc/yunohost/apps
+  mkdir -p /home/yunohost.app
+  chmod 755 /home/yunohost.app
+
+  # Domain settings
+  mkdir -p /etc/yunohost/domains
+  chmod 700 /etc/yunohost/domains
+
+  # Backup folders
+  mkdir -p /home/yunohost.backup/archives
+  chmod 750 /home/yunohost.backup/archives
+  chown root:root /home/yunohost.backup/archives # This is later changed to admin:root once admin user exists
+
+  # Empty ssowat json persistent conf
+  echo "{}" > '/etc/ssowat/conf.json.persistent'
+  chmod 644 /etc/ssowat/conf.json.persistent
+  chown root:root /etc/ssowat/conf.json.persistent
+
+  # Empty service conf
+  touch /etc/yunohost/services.yml
+
+  mkdir -p /var/cache/yunohost/repo
+  chown root:root /var/cache/yunohost
+  chmod 700 /var/cache/yunohost
+
+  cp yunoprompt.service /etc/systemd/system/yunoprompt.service
+  cp dpkg-origins /etc/dpkg/origins/yunohost
+
+  # Change dpkg vendor
+  # see https://wiki.debian.org/Derivatives/Guidelines#Vendor
+  readlink -f /etc/dpkg/origins/default | grep -q debian \
+    && rm -f /etc/dpkg/origins/default \
+    && ln -s /etc/dpkg/origins/yunohost /etc/dpkg/origins/default
 }
 
 do_pre_regen() {
@@ -34,99 +71,170 @@ do_pre_regen() {
 
   cd /usr/share/yunohost/templates/yunohost
 
-  # update services.yml
-  if [[ -f $services_path ]]; then
-      tmp_services_path="${services_path}-tmp"
-      new_services_path="${services_path}-new"
-      sudo cp "$services_path" "$tmp_services_path"
-      _update_services "$new_services_path" || {
-        sudo mv "$tmp_services_path" "$services_path"
-        exit 1
-      }
-      if [[ -f $new_services_path ]]; then
-          # replace services.yml with new one
-          sudo mv "$new_services_path" "$services_path"
-          sudo mv "$tmp_services_path" "${services_path}-old"
-      else
-          sudo rm -f "$tmp_services_path"
-      fi
-  else
-      sudo cp services.yml /etc/yunohost/services.yml
+  # Legacy code that can be removed once on bullseye
+  touch /etc/yunohost/services.yml
+  yunohost tools shell -c "from yunohost.service import _get_services, _save_services; _save_services(_get_services())"
+
+  mkdir -p $pending_dir/etc/systemd/system
+  mkdir -p $pending_dir/etc/cron.d/
+  mkdir -p $pending_dir/etc/cron.daily/
+
+  # add cron job for diagnosis to be ran at 7h and 19h + a random delay between
+  # 0 and 20min, meant to avoid every instances running their diagnosis at
+  # exactly the same time, which may overload the diagnosis server.
+  cat > $pending_dir/etc/cron.d/yunohost-diagnosis << EOF
+SHELL=/bin/bash
+0 7,19 * * * root : YunoHost Automatic Diagnosis; sleep \$((RANDOM\\%1200)); yunohost diagnosis run --email > /dev/null 2>/dev/null || echo "Running the automatic diagnosis failed miserably"
+EOF
+
+  # Cron job that upgrade the app list everyday
+  cat > $pending_dir/etc/cron.daily/yunohost-fetch-apps-catalog << EOF
+#!/bin/bash
+(sleep \$((RANDOM%3600)); yunohost tools update --apps > /dev/null) &
+EOF
+
+  # Cron job that renew lets encrypt certificates if there's any that needs renewal
+  cat > $pending_dir/etc/cron.daily/yunohost-certificate-renew << EOF
+#!/bin/bash
+yunohost domain cert renew --email
+EOF
+
+  # If we subscribed to a dyndns domain, add the corresponding cron
+  # - delay between 0 and 60 secs to spread the check over a 1 min window
+  # - do not run the command if some process already has the lock, to avoid queuing hundreds of commands...
+  if ls -l /etc/yunohost/dyndns/K*.private 2>/dev/null
+  then
+    cat > $pending_dir/etc/cron.d/yunohost-dyndns << EOF
+SHELL=/bin/bash
+*/10 * * * * root : YunoHost DynDNS update; sleep \$((RANDOM\\%60)); test -e /var/run/moulinette_yunohost.lock || yunohost dyndns update >> /dev/null
+EOF
   fi
 
-  mkdir -p "$pending_dir"/etc/etckeeper/
-  cp etckeeper.conf "$pending_dir"/etc/etckeeper/
-}
+  # legacy stuff to avoid yunohost reporting etckeeper as manually modified
+  # (this make sure that the hash is null / file is flagged as to-delete)
+  mkdir -p $pending_dir/etc/etckeeper
+  touch $pending_dir/etc/etckeeper/etckeeper.conf
 
-_update_services() {
-  sudo python2 - << EOF
-import yaml
+  # Skip ntp if inside a container (inspired from the conf of systemd-timesyncd)
+  mkdir -p ${pending_dir}/etc/systemd/system/ntp.service.d/
+  echo "
+[Unit]
+ConditionCapability=CAP_SYS_TIME
+ConditionVirtualization=!container
+" > ${pending_dir}/etc/systemd/system/ntp.service.d/ynh-override.conf
 
-
-with open('services.yml') as f:
-    new_services = yaml.load(f)
-
-with open('/etc/yunohost/services.yml') as f:
-    services = yaml.load(f)
-
-updated = False
-
-
-for service, conf in new_services.items():
-    # remove service with empty conf
-    if conf is None:
-        if service in services:
-            print("removing '{0}' from services".format(service))
-            del services[service]
-            updated = True
-
-    # add new service
-    elif not services.get(service, None):
-        print("adding '{0}' to services".format(service))
-        services[service] = conf
-        updated = True
-
-    # update service conf
-    else:
-        conffiles = services[service].pop('conffiles', {})
-
-        # status need to be removed
-        if "status" not in conf and "status" in services[service]:
-            print("update '{0}' service status access".format(service))
-            del services[service]["status"]
-            updated = True
-
-        if services[service] != conf:
-            print("update '{0}' service".format(service))
-            services[service].update(conf)
-            updated = True
-
-        if conffiles:
-            services[service]['conffiles'] = conffiles
-
-
-if updated:
-    with open('/etc/yunohost/services.yml-new', 'w') as f:
-        yaml.safe_dump(services, f, default_flow_style=False)
+  # Make nftable conflict with yunohost-firewall
+  mkdir -p ${pending_dir}/etc/systemd/system/nftables.service.d/
+  cat > ${pending_dir}/etc/systemd/system/nftables.service.d/ynh-override.conf << EOF
+[Unit]
+# yunohost-firewall and nftables conflict with each other
+Conflicts=yunohost-firewall.service
+ConditionFileIsExecutable=!/etc/init.d/yunohost-firewall
+ConditionPathExists=!/etc/systemd/system/multi-user.target.wants/yunohost-firewall.service
 EOF
+
+  # Don't suspend computer on LidSwitch
+  mkdir -p ${pending_dir}/etc/systemd/logind.conf.d/
+  cat > ${pending_dir}/etc/systemd/logind.conf.d/ynh-override.conf << EOF
+[Login]
+HandleLidSwitch=ignore
+HandleLidSwitchDocked=ignore
+HandleLidSwitchExternalPower=ignore
+EOF
+
+  cp yunoprompt.service ${pending_dir}/etc/systemd/system/yunoprompt.service
+
+  if [[ "$(yunohost settings get 'security.experimental.enabled')" == "True" ]]
+  then
+      cp proc-hidepid.service ${pending_dir}/etc/systemd/system/proc-hidepid.service
+  else
+      touch ${pending_dir}/etc/systemd/system/proc-hidepid.service
+  fi
+
+  mkdir -p ${pending_dir}/etc/dpkg/origins/
+  cp dpkg-origins ${pending_dir}/etc/dpkg/origins/yunohost
+
 }
 
-FORCE=${2:-0}
-DRY_RUN=${3:-0}
+do_post_regen() {
+  regen_conf_files=$1
 
-case "$1" in
-  pre)
-    do_pre_regen $4
-    ;;
-  post)
-    ;;
-  init)
-    do_init_regen
-    ;;
-  *)
-    echo "hook called with unknown argument \`$1'" >&2
-    exit 1
-    ;;
-esac
+  ######################
+  # Enfore permissions #
+  ######################
 
-exit 0
+  chmod 750 /home/admin
+  chmod 750 /home/yunohost.conf
+  chmod 750 /home/yunohost.backup
+  chmod 750 /home/yunohost.backup/archives
+  chown root:root /home/yunohost.conf
+  chown admin:root /home/yunohost.backup
+  chown admin:root /home/yunohost.backup/archives
+
+  # Certs
+  # We do this with find because there could be a lot of them...
+  chown -R root:ssl-cert /etc/yunohost/certs
+  chmod 750 /etc/yunohost/certs
+  find /etc/yunohost/certs/ -type f -exec chmod 640 {} \;
+  find /etc/yunohost/certs/ -type d -exec chmod 750 {} \;
+
+  find /etc/cron.*/yunohost-* -type f -exec chmod 755 {} \;
+  find /etc/cron.d/yunohost-* -type f -exec chmod 644 {} \;
+  find /etc/cron.*/yunohost-* -type f -exec chown root:root {} \;
+
+  chown root:root /var/cache/yunohost
+  chmod 700 /var/cache/yunohost
+  chown root:root /var/cache/moulinette
+  chmod 700 /var/cache/moulinette
+
+  setfacl -m g:all_users:--- /var/www
+  setfacl -m g:all_users:--- /var/log/nginx
+  setfacl -m g:all_users:--- /etc/yunohost
+  setfacl -m g:all_users:--- /etc/ssowat
+
+  for USER in $(yunohost user list --quiet --output-as json | jq -r '.users | .[] | .username')
+  do
+    [ ! -e "/home/$USER" ] || setfacl -m g:all_users:--- /home/$USER
+  done
+
+  # Domain settings
+  mkdir -p /etc/yunohost/domains
+
+  # Misc configuration / state files
+  chown root:root $(ls /etc/yunohost/{*.yml,*.yaml,*.json,mysql,psql} 2>/dev/null)
+  chmod 600 $(ls /etc/yunohost/{*.yml,*.yaml,*.json,mysql,psql} 2>/dev/null)
+
+  # Apps folder, custom hooks folder
+  [[ ! -e /etc/yunohost/hooks.d ]] || (chown root /etc/yunohost/hooks.d && chmod 700 /etc/yunohost/hooks.d)
+  [[ ! -e /etc/yunohost/apps ]] || (chown root /etc/yunohost/apps && chmod 700 /etc/yunohost/apps)
+  [[ ! -e /etc/yunohost/domains ]] || (chown root /etc/yunohost/domains && chmod 700 /etc/yunohost/domains)
+
+  # Create ssh.app and sftp.app groups if they don't exist yet
+  grep -q '^ssh.app:' /etc/group || groupadd ssh.app
+  grep -q '^sftp.app:' /etc/group || groupadd sftp.app
+
+  # Propagates changes in systemd service config overrides
+  [[ ! "$regen_conf_files" =~ "ntp.service.d/ynh-override.conf" ]] || { systemctl daemon-reload; systemctl restart ntp; }
+  [[ ! "$regen_conf_files" =~ "nftables.service.d/ynh-override.conf" ]] || systemctl daemon-reload
+  [[ ! "$regen_conf_files" =~ "login.conf.d/ynh-override.conf" ]] || systemctl daemon-reload
+  if [[ "$regen_conf_files" =~ "yunoprompt.service" ]]
+  then
+    systemctl daemon-reload
+    action=$([[ -e /etc/systemd/system/yunoprompt.service ]] && echo 'enable' || echo 'disable')
+    systemctl $action yunoprompt --quiet --now
+  fi
+  if [[ "$regen_conf_files" =~ "proc-hidepid.service" ]]
+  then
+    systemctl daemon-reload
+    action=$([[ -e /etc/systemd/system/proc-hidepid.service ]] && echo 'enable' || echo 'disable')
+    systemctl $action proc-hidepid --quiet --now
+  fi
+
+  # Change dpkg vendor
+  # see https://wiki.debian.org/Derivatives/Guidelines#Vendor
+  readlink -f /etc/dpkg/origins/default | grep -q debian \
+    && rm -f /etc/dpkg/origins/default \
+    && ln -s /etc/dpkg/origins/yunohost /etc/dpkg/origins/default
+}
+
+do_$1_regen ${@:2}

data/hooks/conf_regen/02-ssl

@@ -3,84 +3,97 @@
 set -e
 
 ssl_dir="/usr/share/yunohost/yunohost-config/ssl/yunoCA"
+ynh_ca="/etc/yunohost/certs/yunohost.org/ca.pem"
+ynh_crt="/etc/yunohost/certs/yunohost.org/crt.pem"
+ynh_key="/etc/yunohost/certs/yunohost.org/key.pem"
+openssl_conf="/usr/share/yunohost/templates/ssl/openssl.cnf"
+
+regen_local_ca() {
+
+    domain="$1"
+
+    echo -e "\n# Creating local certification authority with domain=$domain\n"
+
+    # create certs and SSL directories
+    mkdir -p "/etc/yunohost/certs/yunohost.org"
+    mkdir -p "${ssl_dir}/"{ca,certs,crl,newcerts}
+
+    pushd ${ssl_dir}
+
+    # (Update the serial so that it's specific to this very instance)
+    # N.B. : the weird RANDFILE thing comes from:
+    # https://stackoverflow.com/questions/94445/using-openssl-what-does-unable-to-write-random-state-mean
+    RANDFILE=.rnd openssl rand -hex 19 > serial
+    rm -f index.txt
+    touch index.txt
+    cp /usr/share/yunohost/templates/ssl/openssl.cnf openssl.ca.cnf
+    sed -i "s/yunohost.org/${domain}/g" openssl.ca.cnf
+    openssl req -x509 \
+        -new \
+        -config openssl.ca.cnf \
+        -days 3650 \
+        -out ca/cacert.pem \
+        -keyout ca/cakey.pem \
+        -nodes \
+        -batch \
+        -subj /CN=${domain}/O=${domain%.*} 2>&1
+
+    chmod 640 ca/cacert.pem
+    chmod 640 ca/cakey.pem
+
+    cp ca/cacert.pem $ynh_ca
+    ln -sf "$ynh_ca" /etc/ssl/certs/ca-yunohost_crt.pem
+    update-ca-certificates
+
+    popd
+}
 
 do_init_regen() {
-  if [[ $EUID -ne 0 ]]; then
-    echo "You must be root to run this script" 1>&2
-    exit 1
-  fi
 
-  LOGFILE="/tmp/yunohost-ssl-init"
+  LOGFILE=/tmp/yunohost-ssl-init
+  echo "" > $LOGFILE
+  chown root:root $LOGFILE
+  chmod 640 $LOGFILE
 
-  echo "Initializing a local SSL certification authority ..."
-  echo "(logs available in $LOGFILE)"
-
-  rm -f $LOGFILE
-  touch $LOGFILE
-
-  # create certs and SSL directories
-  mkdir -p "/etc/yunohost/certs/yunohost.org"
-  mkdir -p "${ssl_dir}/"{ca,certs,crl,newcerts}
-
-  # initialize some files
-  # N.B. : the weird RANDFILE thing comes from:
-  # https://stackoverflow.com/questions/94445/using-openssl-what-does-unable-to-write-random-state-mean
-  [[ -f "${ssl_dir}/serial" ]] \
-    || RANDFILE=.rnd openssl rand -hex 19 > "${ssl_dir}/serial"
-  [[ -f "${ssl_dir}/index.txt" ]] \
-    || touch "${ssl_dir}/index.txt"
-
-  openssl_conf="/usr/share/yunohost/templates/ssl/openssl.cnf"
-  ynh_ca="/etc/yunohost/certs/yunohost.org/ca.pem"
-  ynh_crt="/etc/yunohost/certs/yunohost.org/crt.pem"
-  ynh_key="/etc/yunohost/certs/yunohost.org/key.pem"
+  # Make sure this conf exists
+  mkdir -p ${ssl_dir}
+  cp /usr/share/yunohost/templates/ssl/openssl.cnf ${ssl_dir}/openssl.ca.cnf
 
   # create default certificates
   if [[ ! -f "$ynh_ca" ]]; then
-      echo -e "\n# Creating the CA key (?)\n" >>$LOGFILE
-
-      openssl req -x509 \
-        -new \
-        -config "$openssl_conf" \
-        -days 3650 \
-        -out "${ssl_dir}/ca/cacert.pem" \
-        -keyout "${ssl_dir}/ca/cakey.pem" \
-        -nodes -batch >>$LOGFILE 2>&1
-
-      cp "${ssl_dir}/ca/cacert.pem" "$ynh_ca"
-      ln -sf "$ynh_ca" /etc/ssl/certs/ca-yunohost_crt.pem
-      update-ca-certificates
+      regen_local_ca yunohost.org >>$LOGFILE
   fi
 
   if [[ ! -f "$ynh_crt" ]]; then
-      echo -e "\n# Creating initial key and certificate (?)\n" >>$LOGFILE
+      echo -e "\n# Creating initial key and certificate \n" >>$LOGFILE
 
       openssl req -new \
         -config "$openssl_conf" \
         -days 730 \
         -out "${ssl_dir}/certs/yunohost_csr.pem" \
         -keyout "${ssl_dir}/certs/yunohost_key.pem" \
-        -nodes -batch >>$LOGFILE 2>&1
+        -nodes -batch &>>$LOGFILE
 
       openssl ca \
         -config "$openssl_conf" \
         -days 730 \
         -in "${ssl_dir}/certs/yunohost_csr.pem" \
         -out "${ssl_dir}/certs/yunohost_crt.pem" \
-        -batch >>$LOGFILE 2>&1
+        -batch &>>$LOGFILE
 
-      last_cert=$(ls $ssl_dir/newcerts/*.pem | sort -V | tail -n 1)
       chmod 640 "${ssl_dir}/certs/yunohost_key.pem"
-      chmod 640 "$last_cert"
+      chmod 640 "${ssl_dir}/certs/yunohost_crt.pem"
 
       cp "${ssl_dir}/certs/yunohost_key.pem" "$ynh_key"
-      cp "$last_cert" "$ynh_crt"
+      cp "${ssl_dir}/certs/yunohost_crt.pem" "$ynh_crt"
       ln -sf "$ynh_crt" /etc/ssl/certs/yunohost_crt.pem
       ln -sf "$ynh_key" /etc/ssl/private/yunohost_key.pem
   fi
 
   chown -R root:ssl-cert /etc/yunohost/certs/yunohost.org/
   chmod o-rwx /etc/yunohost/certs/yunohost.org/
+
+  install -D -m 644 $openssl_conf "${ssl_dir}/openssl.cnf"
 }
 
 do_pre_regen() {
@@ -94,41 +107,16 @@ do_pre_regen() {
 do_post_regen() {
   regen_conf_files=$1
 
-  # Ensure that index.txt exists
-  index_txt=/usr/share/yunohost/yunohost-config/ssl/yunoCA/index.txt
-  [[ -f "${index_txt}" ]] || {
-    if [[ -f "${index_txt}.saved" ]]; then
-      # use saved database from 2.2
-      sudo cp "${index_txt}.saved" "${index_txt}"
-    elif [[ -f "${index_txt}.old" ]]; then
-      # ... or use the state-1 database
-      sudo cp "${index_txt}.old" "${index_txt}"
-    else
-      # ... or create an empty one
-      sudo touch "${index_txt}"
-    fi
-  }
+  current_local_ca_domain=$(openssl x509 -in $ynh_ca -text | tr ',' '\n' | grep Issuer | awk '{print $4}')
+  main_domain=$(cat /etc/yunohost/current_host)
 
-  # TODO: regenerate certificates if conf changed?
+  if [[ "$current_local_ca_domain" != "$main_domain" ]]
+  then
+      regen_local_ca $main_domain
+      # Idk how useful this is, but this was in the previous python code (domain.main_domain())
+      ln -sf /etc/yunohost/certs/$domain/crt.pem /etc/ssl/certs/yunohost_crt.pem
+      ln -sf /etc/yunohost/certs/$domain/key.pem /etc/ssl/private/yunohost_key.pem
+  fi
 }
 
-FORCE=${2:-0}
-DRY_RUN=${3:-0}
-
-case "$1" in
-  pre)
-    do_pre_regen $4
-    ;;
-  post)
-    do_post_regen $4
-    ;;
-  init)
-    do_init_regen
-    ;;
-  *)
-    echo "hook called with unknown argument \`$1'" >&2
-    exit 1
-    ;;
-esac
-
-exit 0
+do_$1_regen ${@:2}

data/hooks/conf_regen/03-ssh

@@ -25,7 +25,7 @@ do_pre_regen() {
 
     # Support different strategy for security configurations
     export compatibility="$(yunohost settings get 'security.ssh.compatibility')"
-
+    export port="$(yunohost settings get 'security.ssh.port')"
     export ssh_keys
     export ipv6_enabled
     ynh_render_template "sshd_config" "${pending_dir}/etc/ssh/sshd_config"
@@ -48,20 +48,4 @@ do_post_regen() {
     systemctl restart ssh
 }
 
-FORCE=${2:-0}
-DRY_RUN=${3:-0}
-
-case "$1" in
-  pre)
-    do_pre_regen $4
-    ;;
-  post)
-    do_post_regen $4
-    ;;
-  *)
-    echo "hook called with unknown argument \`$1'" >&2
-    exit 1
-    ;;
-esac
-
-exit 0
+do_$1_regen ${@:2}

data/hooks/conf_regen/06-slapd

@@ -2,7 +2,10 @@
 
 set -e
 
-tmp_backup_dir_file="/tmp/slapd-backup-dir.txt"
+tmp_backup_dir_file="/root/slapd-backup-dir.txt"
+
+config="/usr/share/yunohost/templates/slapd/config.ldif"
+db_init="/usr/share/yunohost/templates/slapd/db_init.ldif"
 
 do_init_regen() {
   if [[ $EUID -ne 0 ]]; then
@@ -12,27 +15,95 @@ do_init_regen() {
 
   do_pre_regen ""
 
-  # fix some permissions
-  chown root:openldap /etc/ldap/slapd.conf
+  # Drop current existing slapd data
+
+  rm -rf /var/backups/*.ldapdb
+  rm -rf /var/backups/slapd-*
+
+  debconf-set-selections << EOF
+slapd slapd/password1 password yunohost
+slapd slapd/password2 password yunohost
+slapd slapd/domain string yunohost.org
+slapd shared/organization  string yunohost.org
+slapd slapd/allow_ldap_v2 boolean false
+slapd slapd/invalid_config boolean true
+slapd slapd/backend select MDB
+slapd slapd/move_old_database boolean true
+slapd slapd/no_configuration boolean false
+slapd slapd/purge_database boolean false
+EOF
+
+  DEBIAN_FRONTEND=noninteractive dpkg-reconfigure slapd -u
+
+  # Enforce permissions
   chown -R openldap:openldap /etc/ldap/schema/
   usermod -aG ssl-cert openldap
 
-  # check the slapd config file at first
-  slaptest -Q -u -f /etc/ldap/slapd.conf
+  # (Re-)init data according to default ldap entries
+  echo '  Initializing LDAP with YunoHost DB structure'
 
-  # regenerate LDAP config directory from slapd.conf
+  rm -rf /etc/ldap/slapd.d
+  mkdir -p /etc/ldap/slapd.d
+  slapadd -F /etc/ldap/slapd.d -b cn=config -l "$config" 2>&1 \
+      | grep -v "none elapsed\|Closing DB" || true
+  chown -R openldap: /etc/ldap/slapd.d
+
+  rm -rf /var/lib/ldap
+  mkdir -p /var/lib/ldap
+  slapadd -F /etc/ldap/slapd.d -b dc=yunohost,dc=org -l "$db_init" 2>&1 \
+      | grep -v "none elapsed\|Closing DB" || true
+  chown -R openldap: /var/lib/ldap
+
+  nscd -i group || true
+  nscd -i passwd || true
+
+  systemctl restart slapd
+
+  # We don't use mkhomedir_helper because 'admin' may not be recognized
+  # when this script is ran in a chroot (e.g. ISO install)
+  # We also refer to admin as uid 1007 for the same reason
+  if [ ! -d /home/admin ]
+  then
+      cp -r /etc/skel /home/admin
+      chown -R 1007:1007 /home/admin
+  fi
+}
+
+_regenerate_slapd_conf() {
+
+  # Validate the new slapd config
+  # To do so, we have to use the .ldif to generate the config directory
+  # so we use a temporary directory slapd_new.d
+  rm -Rf /etc/ldap/slapd_new.d
+  mkdir /etc/ldap/slapd_new.d
+  slapadd -b cn=config -l "$config" -F /etc/ldap/slapd_new.d/ 2>&1 \
+      | grep -v "none elapsed\|Closing DB" || true
+  # Actual validation (-Q is for quiet, -u is for dry-run)
+  slaptest -Q -u -F /etc/ldap/slapd_new.d
+
+  # "Commit" / apply the new config (meaning we delete the old one and replace
+  # it with the new one)
   rm -Rf /etc/ldap/slapd.d
-  mkdir /etc/ldap/slapd.d
-  slaptest -f /etc/ldap/slapd.conf -F /etc/ldap/slapd.d/ 2>&1
-  chown -R openldap:openldap /etc/ldap/slapd.d/
+  mv /etc/ldap/slapd_new.d /etc/ldap/slapd.d
 
-  service slapd restart
+  chown -R openldap:openldap /etc/ldap/slapd.d/
 }
 
 do_pre_regen() {
   pending_dir=$1
 
-  cd /usr/share/yunohost/templates/slapd
+  # remove temporary backup file
+  rm -f "$tmp_backup_dir_file"
+
+  # Define if we need to migrate from hdb to mdb
+  curr_backend=$(grep '^database' /etc/ldap/slapd.conf 2>/dev/null | awk '{print $2}')
+  if [ -e /etc/ldap/slapd.conf ] && [ -n "$curr_backend" ] && \
+     [ $curr_backend != 'mdb' ]; then
+    backup_dir="/var/backups/dc=yunohost,dc=org-${curr_backend}-$(date +%s)"
+    mkdir -p "$backup_dir"
+    slapcat -b dc=yunohost,dc=org -l "${backup_dir}/dc=yunohost-dc=org.ldif"
+    echo "$backup_dir" > "$tmp_backup_dir_file"
+  fi
 
   # create needed directories
   ldap_dir="${pending_dir}/etc/ldap"
@@ -40,28 +111,18 @@ do_pre_regen() {
   mkdir -p "$ldap_dir" "$schema_dir"
 
   # remove legacy configuration file
-  [ ! -f /etc/ldap/slapd-yuno.conf ] \
-    || touch "${pending_dir}/etc/ldap/slapd-yuno.conf"
+  [ ! -f /etc/ldap/slapd-yuno.conf ] || touch "${ldap_dir}/slapd-yuno.conf"
+  [ ! -f /etc/ldap/slapd.conf ] || touch "${ldap_dir}/slapd.conf"
+  [ ! -f /etc/ldap/schema/yunohost.schema ] || touch "${schema_dir}/yunohost.schema"
 
-  # remove temporary backup file
-  rm -f "$tmp_backup_dir_file"
-
-  # retrieve current and new backends
-  curr_backend=$(grep '^database' /etc/ldap/slapd.conf 2>/dev/null | awk '{print $2}')
-  new_backend=$(grep '^database' slapd.conf | awk '{print $2}')
-
-  # save current database before any conf changes
-  if [[ -n "$curr_backend" && "$curr_backend" != "$new_backend" ]]; then
-    backup_dir="/var/backups/dc=yunohost,dc=org-${curr_backend}-$(date +%s)"
-    mkdir -p "$backup_dir"
-    slapcat -b dc=yunohost,dc=org \
-      -l "${backup_dir}/dc=yunohost-dc=org.ldif"
-    echo "$backup_dir" > "$tmp_backup_dir_file"
-  fi
+  cd /usr/share/yunohost/templates/slapd
 
   # copy configuration files
-  cp -a ldap.conf slapd.conf "$ldap_dir"
-  cp -a sudo.schema mailserver.schema yunohost.schema "$schema_dir"
+  cp -a ldap.conf "$ldap_dir"
+  cp -a sudo.ldif mailserver.ldif permission.ldif "$schema_dir"
+
+  mkdir -p ${pending_dir}/etc/systemd/system/slapd.service.d/
+  cp systemd-override.conf ${pending_dir}/etc/systemd/system/slapd.service.d/ynh-override.conf
 
   install -D -m 644 slapd.default "${pending_dir}/etc/default/slapd"
 }
@@ -69,51 +130,56 @@ do_pre_regen() {
 do_post_regen() {
   regen_conf_files=$1
 
-  # ensure that slapd.d exists
-  mkdir -p /etc/ldap/slapd.d
-
   # fix some permissions
-  echo "Making sure we have the right permissions needed ..."
+  echo "Enforce permissions on ldap/slapd directories and certs ..."
   # penldap user should be in the ssl-cert group to let it access the certificate for TLS
   usermod -aG ssl-cert openldap
-  chown root:openldap /etc/ldap/slapd.conf
   chown -R openldap:openldap /etc/ldap/schema/
   chown -R openldap:openldap /etc/ldap/slapd.d/
-  chown -R root:ssl-cert /etc/yunohost/certs/yunohost.org/
-  chmod o-rwx /etc/yunohost/certs/yunohost.org/
+
+  # If we changed the systemd ynh-override conf
+  if echo "$regen_conf_files" | sed 's/,/\n/g' | grep -q "^/etc/systemd/system/slapd.service.d/ynh-override.conf$"
+  then
+      systemctl daemon-reload
+      systemctl restart slapd
+      sleep 3
+  fi
+
+  # For some reason, old setups don't have the admins group defined...
+  if ! slapcat | grep -q 'cn=admins,ou=groups,dc=yunohost,dc=org'
+  then
+      slapadd -F /etc/ldap/slapd.d -b dc=yunohost,dc=org <<< \
+"dn: cn=admins,ou=groups,dc=yunohost,dc=org
+cn: admins
+gidNumber: 4001
+memberUid: admin
+objectClass: posixGroup
+objectClass: top"
+      chown -R openldap: /var/lib/ldap
+      systemctl restart slapd
+      nscd -i group
+  fi
 
   [ -z "$regen_conf_files" ] && exit 0
 
-  # check the slapd config file at first
-  slaptest -Q -u -f /etc/ldap/slapd.conf
+  # regenerate LDAP config directory from slapd.conf
+  echo "Regenerate LDAP config directory from config.ldif"
+  _regenerate_slapd_conf
 
-  # check if a backup should be restored
+  # If there's a backup, re-import its data
   backup_dir=$(cat "$tmp_backup_dir_file" 2>/dev/null || true)
   if [[ -n "$backup_dir" && -f "${backup_dir}/dc=yunohost-dc=org.ldif" ]]; then
       # regenerate LDAP config directory and import database as root
-      # since the admin user may be unavailable
-      echo "Regenerate LDAP config directory and import the database using slapadd"
-      sh -c "rm -Rf /etc/ldap/slapd.d;
-  mkdir /etc/ldap/slapd.d;
-  slaptest -f /etc/ldap/slapd.conf -F /etc/ldap/slapd.d;
-  chown -R openldap:openldap /etc/ldap/slapd.d;
-  slapadd -F /etc/ldap/slapd.d -b dc=yunohost,dc=org \
-    -l '${backup_dir}/dc=yunohost-dc=org.ldif';
-  chown -R openldap:openldap /var/lib/ldap" 2>&1
-  else
-      # regenerate LDAP config directory from slapd.conf
-      echo "Regenerate LDAP config directory from slapd.conf"
-      rm -Rf /etc/ldap/slapd.d
-      mkdir /etc/ldap/slapd.d
-      slaptest -f /etc/ldap/slapd.conf -F /etc/ldap/slapd.d/ 2>&1
-      chown -R openldap:openldap /etc/ldap/slapd.d/
+    echo "Import the database using slapadd"
+    slapadd -F /etc/ldap/slapd.d -b dc=yunohost,dc=org -l "${backup_dir}/dc=yunohost-dc=org.ldif"
+    chown -R openldap:openldap /var/lib/ldap 2>&1
   fi
 
   echo "Running slapdindex"
   su openldap -s "/bin/bash" -c "/usr/sbin/slapindex"
 
   echo "Reloading slapd"
-  service slapd force-reload
+  systemctl force-reload slapd
 
   # on slow hardware/vm this regen conf would exit before the admin user that
   # is stored in ldap is available because ldap seems to slow to restart
@@ -126,30 +192,11 @@ do_post_regen() {
   # wait a maximum time of 5 minutes
   # yes, force-reload behave like a restart
   number_of_wait=0
-  while ! sudo su admin -c '' && ((number_of_wait < 60))
+  while ! su admin -c '' && ((number_of_wait < 60))
   do
       sleep 5
       ((number_of_wait += 1))
   done
 }
 
-FORCE=${2:-0}
-DRY_RUN=${3:-0}
-
-case "$1" in
-  pre)
-    do_pre_regen $4
-    ;;
-  post)
-    do_post_regen $4
-    ;;
-  init)
-    do_init_regen
-    ;;
-  *)
-    echo "hook called with unknown argument \`$1'" >&2
-    exit 1
-    ;;
-esac
-
-exit 0
+do_$1_regen ${@:2}

data/hooks/conf_regen/09-nslcd

@@ -2,6 +2,11 @@
 
 set -e
 
+do_init_regen() {
+  do_pre_regen ""
+  systemctl restart nslcd
+}
+
 do_pre_regen() {
   pending_dir=$1
 
@@ -14,23 +19,7 @@ do_post_regen() {
   regen_conf_files=$1
 
   [[ -z "$regen_conf_files" ]] \
-    || sudo service nslcd restart
+    || systemctl restart nslcd
 }
 
-FORCE=${2:-0}
-DRY_RUN=${3:-0}
-
-case "$1" in
-  pre)
-    do_pre_regen $4
-    ;;
-  post)
-    do_post_regen $4
-    ;;
-  *)
-    echo "hook called with unknown argument \`$1'" >&2
-    exit 1
-    ;;
-esac
-
-exit 0
+do_$1_regen ${@:2}

data/hooks/conf_regen/10-apt

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+set -e
+
+do_pre_regen() {
+    pending_dir=$1
+
+    mkdir --parents "${pending_dir}/etc/apt/preferences.d"
+
+    packages_to_refuse_from_sury="php php-fpm php-mysql php-xml php-zip php-mbstring php-ldap php-gd php-curl php-bz2 php-json php-sqlite3 php-intl openssl libssl1.1 libssl-dev"
+    for package in $packages_to_refuse_from_sury
+    do
+        echo "
+Package: $package
+Pin: origin \"packages.sury.org\" 
+Pin-Priority: -1" >> "${pending_dir}/etc/apt/preferences.d/extra_php_version"
+    done
+
+        echo "
+
+# PLEASE READ THIS WARNING AND DON'T EDIT THIS FILE
+
+# You are probably reading this file because you tried to install apache2 or 
+# bind9. These 2 packages conflict with YunoHost.
+
+# Installing apache2 will break nginx and break the entire YunoHost ecosystem
+# on your server, therefore don't remove those lines! 
+
+# You have been warned.
+
+Package: apache2
+Pin: release *
+Pin-Priority: -1
+
+Package: apache2-bin
+Pin: release *
+Pin-Priority: -1
+
+# Also bind9 will conflict with dnsmasq.
+# Same story as for apache2.
+# Don't install it, don't remove those lines.
+
+Package: bind9
+Pin: release *
+Pin-Priority: -1
+" >> "${pending_dir}/etc/apt/preferences.d/ban_packages"
+
+}
+
+do_post_regen() {
+  regen_conf_files=$1
+
+  # Make sure php7.3 is the default version when using php in cli
+  update-alternatives --set php /usr/bin/php7.3
+}
+
+do_$1_regen ${@:2}

data/hooks/conf_regen/12-metronome

@@ -14,7 +14,6 @@ do_pre_regen() {
 
   # retrieve variables
   main_domain=$(cat /etc/yunohost/current_host)
-  domain_list=$(sudo yunohost domain list --output-as plain --quiet)
 
   # install main conf file
   cat metronome.cfg.lua \
@@ -22,7 +21,7 @@ do_pre_regen() {
     > "${metronome_dir}/metronome.cfg.lua"
 
   # add domain conf files
-  for domain in $domain_list; do
+  for domain in $YNH_DOMAINS; do
     cat domain.tpl.cfg.lua \
       | sed "s/{{ domain }}/${domain}/g" \
       > "${metronome_conf_dir}/${domain}.cfg.lua"
@@ -33,7 +32,7 @@ do_pre_regen() {
                  | awk '/^[^\.]+\.[^\.]+.*\.cfg\.lua$/ { print $1 }')
   for file in $conf_files; do
     domain=${file%.cfg.lua}
-    [[ $domain_list =~ $domain ]] \
+    [[ $YNH_DOMAINS =~ $domain ]] \
       || touch "${metronome_conf_dir}/${file}"
   done
 }
@@ -41,36 +40,34 @@ do_pre_regen() {
 do_post_regen() {
   regen_conf_files=$1
 
-  # fix some permissions
-  sudo chown -R metronome: /var/lib/metronome/
-  sudo chown -R metronome: /etc/metronome/conf.d/
-
   # retrieve variables
-  domain_list=$(sudo yunohost domain list --output-as plain --quiet)
+  main_domain=$(cat /etc/yunohost/current_host)
+  
+  # FIXME : small optimization to do to avoid calling a yunohost command ... 
+  # maybe another env variable like YNH_MAIN_DOMAINS idk
+  domain_list=$(yunohost domain list --exclude-subdomains --output-as plain --quiet)
 
   # create metronome directories for domains
   for domain in $domain_list; do
-    sudo mkdir -p "/var/lib/metronome/${domain//./%2e}/pep"
+    mkdir -p "/var/lib/metronome/${domain//./%2e}/pep"
+    # http_upload directory must be writable by metronome and readable by nginx
+    mkdir -p  "/var/xmpp-upload/${domain}/upload"
+    # sgid bit allows that file created in that dir will be owned by www-data
+    # despite the fact that metronome ain't in the www-data group
+    chmod g+s "/var/xmpp-upload/${domain}/upload"
   done
 
+  # fix some permissions
+  [ ! -e '/var/xmpp-upload' ] || chown -R metronome:www-data "/var/xmpp-upload/"
+  [ ! -e '/var/xmpp-upload' ] || chmod 750 "/var/xmpp-upload/"
+
+  # metronome should be in ssl-cert group to let it access SSL certificates
+  usermod -aG ssl-cert metronome
+  chown -R metronome: /var/lib/metronome/
+  chown -R metronome: /etc/metronome/conf.d/
+
   [[ -z "$regen_conf_files" ]] \
-    || sudo service metronome restart
+    || systemctl restart metronome
 }
 
-FORCE=${2:-0}
-DRY_RUN=${3:-0}
-
-case "$1" in
-  pre)
-    do_pre_regen $4
-    ;;
-  post)
-    do_post_regen $4
-    ;;
-  *)
-    echo "hook called with unknown argument \`$1'" >&2
-    exit 1
-    ;;
-esac
-
-exit 0
+do_$1_regen ${@:2}

data/hooks/conf_regen/15-nginx

@@ -23,10 +23,17 @@ do_init_regen() {
   rm -f "${nginx_dir}/sites-enabled/default"
 
   export compatibility="intermediate"
+  ynh_render_template "security.conf.inc" "${nginx_conf_dir}/security.conf.inc"
   ynh_render_template "yunohost_admin.conf" "${nginx_conf_dir}/yunohost_admin.conf"
+  ynh_render_template "yunohost_admin.conf.inc" "${nginx_conf_dir}/yunohost_admin.conf.inc"
+  ynh_render_template "yunohost_api.conf.inc" "${nginx_conf_dir}/yunohost_api.conf.inc"
+
+  mkdir -p $nginx_conf_dir/default.d/
+  cp "redirect_to_admin.conf" $nginx_conf_dir/default.d/
 
   # Restart nginx if conf looks good, otherwise display error and exit unhappy
-  nginx -t 2>/dev/null && service nginx restart || (nginx -t && exit 1)
+  nginx -t 2>/dev/null || { nginx -t; exit 1; }
+  systemctl restart nginx || { journalctl --no-pager --lines=10 -u nginx >&2; exit 1; }
 
   exit 0
 }
@@ -42,16 +49,26 @@ do_pre_regen() {
 
   # install / update plain conf files
   cp plain/* "$nginx_conf_dir"
+  # remove the panel overlay if this is specified in settings
+  panel_overlay=$(yunohost settings get 'ssowat.panel_overlay.enabled')
+  if [ "$panel_overlay" == "false" ] || [ "$panel_overlay" == "False" ]
+  then
+    echo "#" > "${nginx_conf_dir}/yunohost_panel.conf.inc"
+  fi
 
   # retrieve variables
   main_domain=$(cat /etc/yunohost/current_host)
-  domain_list=$(sudo yunohost domain list --output-as plain --quiet)
 
   # Support different strategy for security configurations
+  export redirect_to_https="$(yunohost settings get 'security.nginx.redirect_to_https')"
   export compatibility="$(yunohost settings get 'security.nginx.compatibility')"
+  export experimental="$(yunohost settings get 'security.experimental.enabled')"
+  ynh_render_template "security.conf.inc" "${nginx_conf_dir}/security.conf.inc"
+
+  cert_status=$(yunohost domain cert status --json)
 
   # add domain conf files
-  for domain in $domain_list; do
+  for domain in $YNH_DOMAINS; do
     domain_conf_dir="${nginx_conf_dir}/${domain}.d"
     mkdir -p "$domain_conf_dir"
     mail_autoconfig_dir="${pending_dir}/var/www/.well-known/${domain}/autoconfig/mail/"
@@ -59,27 +76,34 @@ do_pre_regen() {
 
     # NGINX server configuration
     export domain
-    export domain_cert_ca=$(yunohost domain cert-status $domain --json \
+    export domain_cert_ca=$(echo $cert_status \
                             | jq ".certificates.\"$domain\".CA_type" \
                             | tr -d '"')
 
     ynh_render_template "server.tpl.conf" "${nginx_conf_dir}/${domain}.conf"
     ynh_render_template "autoconfig.tpl.xml" "${mail_autoconfig_dir}/config-v1.1.xml"
 
-    [[ $main_domain != $domain ]] \
-      && touch "${domain_conf_dir}/yunohost_local.conf" \
-      || cp yunohost_local.conf "${domain_conf_dir}/yunohost_local.conf"
+    touch "${domain_conf_dir}/yunohost_local.conf"  # Clean legacy conf files
 
   done
 
+  export webadmin_allowlist_enabled=$(yunohost settings get security.webadmin.allowlist.enabled)
+  if [ "$webadmin_allowlist_enabled" == "True" ]
+  then
+    export webadmin_allowlist=$(yunohost settings get security.webadmin.allowlist)
+  fi
+  ynh_render_template "yunohost_admin.conf.inc" "${nginx_conf_dir}/yunohost_admin.conf.inc"
+  ynh_render_template "yunohost_api.conf.inc" "${nginx_conf_dir}/yunohost_api.conf.inc"
   ynh_render_template "yunohost_admin.conf" "${nginx_conf_dir}/yunohost_admin.conf"
+  mkdir -p $nginx_conf_dir/default.d/
+  cp "redirect_to_admin.conf" $nginx_conf_dir/default.d/
 
   # remove old domain conf files
   conf_files=$(ls -1 /etc/nginx/conf.d \
                  | awk '/^[^\.]+\.[^\.]+.*\.conf$/ { print $1 }')
   for file in $conf_files; do
     domain=${file%.conf}
-    [[ $domain_list =~ $domain ]] \
+    [[ $YNH_DOMAINS =~ $domain ]] \
       || touch "${nginx_conf_dir}/${file}"
   done
 
@@ -87,7 +111,7 @@ do_pre_regen() {
   autoconfig_files=$(ls -1 /var/www/.well-known/*/autoconfig/mail/config-v1.1.xml 2>/dev/null || true)
   for file in $autoconfig_files; do
     domain=$(basename $(readlink -f $(dirname $file)/../..))
-    [[ $domain_list =~ $domain ]] \
+    [[ $YNH_DOMAINS =~ $domain ]] \
       || (mkdir -p "$(dirname ${pending_dir}/${file})" && touch "${pending_dir}/${file}")
   done
 
@@ -101,35 +125,28 @@ do_post_regen() {
 
   [ -z "$regen_conf_files" ] && exit 0
 
-  # retrieve variables
-  domain_list=$(sudo yunohost domain list --output-as plain --quiet)
-
   # create NGINX conf directories for domains
-  for domain in $domain_list; do
-    sudo mkdir -p "/etc/nginx/conf.d/${domain}.d"
+  for domain in $YNH_DOMAINS; do
+    mkdir -p "/etc/nginx/conf.d/${domain}.d"
   done
 
-  # Reload nginx configuration
-  pgrep nginx && sudo service nginx reload
+  # Get rid of legacy lets encrypt snippets
+  for domain in $YNH_DOMAINS; do
+      # If the legacy letsencrypt / acme-challenge domain-specific snippet is still there
+      if [ -e /etc/nginx/conf.d/${domain}.d/000-acmechallenge.conf ]
+      then
+          # And if we're effectively including the new domain-independant snippet now
+          if grep -q "include /etc/nginx/conf.d/acme-challenge.conf.inc;" /etc/nginx/conf.d/${domain}.conf
+          then
+              # Delete the old domain-specific snippet
+              rm /etc/nginx/conf.d/${domain}.d/000-acmechallenge.conf
+          fi
+      fi
+  done
+
+  # Reload nginx if conf looks good, otherwise display error and exit unhappy
+  nginx -t 2>/dev/null || { nginx -t; exit 1; }
+  pgrep nginx && systemctl reload nginx || { journalctl --no-pager --lines=10 -u nginx >&2; exit 1; }
 }
 
-FORCE=${2:-0}
-DRY_RUN=${3:-0}
-
-case "$1" in
-  pre)
-    do_pre_regen $4
-    ;;
-  post)
-    do_post_regen $4
-    ;;
-  init)
-    do_init_regen
-    ;;
-  *)
-    echo "hook called with unknown argument \`$1'" >&2
-    exit 1
-    ;;
-esac
-
-exit 0
+do_$1_regen ${@:2}

data/hooks/conf_regen/19-postfix

@@ -20,22 +20,43 @@ do_pre_regen() {
 
   # prepare main.cf conf file
   main_domain=$(cat /etc/yunohost/current_host)
-  domain_list=$(sudo yunohost domain list --output-as plain --quiet | tr '\n' ' ')
 
   # Support different strategy for security configurations
   export compatibility="$(yunohost settings get 'security.postfix.compatibility')"
   
+  # Add possibility to specify a relay
+  # Could be useful with some isp with no 25 port open or more complex setup
+  export relay_port=""
+  export relay_user=""
+  export relay_host="$(yunohost settings get 'smtp.relay.host')"
+  if [ -n "${relay_host}" ]
+  then
+    relay_port="$(yunohost settings get 'smtp.relay.port')"
+    relay_user="$(yunohost settings get 'smtp.relay.user')"
+    relay_password="$(yunohost settings get 'smtp.relay.password')"
+    
+    # Avoid to display "Relay account paswword" to other users
+    touch ${postfix_dir}/sasl_passwd
+    chmod 750 ${postfix_dir}/sasl_passwd
+    # Avoid "postmap: warning: removing zero-length database file"
+    chown postfix ${pending_dir}/etc/postfix
+    chown postfix ${pending_dir}/etc/postfix/sasl_passwd
+
+    cat <<< "[${relay_host}]:${relay_port} ${relay_user}:${relay_password}" > ${postfix_dir}/sasl_passwd
+    postmap ${postfix_dir}/sasl_passwd
+  fi
   export main_domain
-  export domain_list
+  export domain_list="$YNH_DOMAINS"
   ynh_render_template "main.cf" "${postfix_dir}/main.cf"
 
   cat postsrsd \
     | sed "s/{{ main_domain }}/${main_domain}/g" \
-    | sed "s/{{ domain_list }}/${domain_list}/g" \
+    | sed "s/{{ domain_list }}/${YNH_DOMAINS}/g" \
     > "${default_dir}/postsrsd"
 
   # adapt it for IPv4-only hosts
-  if [ ! -f /proc/net/if_inet6 ]; then
+  ipv6="$(yunohost settings get 'smtp.allow_ipv6')"
+  if [ "$ipv6" == "False" ] || [ ! -f /proc/net/if_inet6 ]; then
     sed -i \
       's/ \[::ffff:127.0.0.0\]\/104 \[::1\]\/128//g' \
       "${postfix_dir}/main.cf"
@@ -48,25 +69,15 @@ do_pre_regen() {
 do_post_regen() {
   regen_conf_files=$1
 
+  if [ -e /etc/postfix/sasl_passwd ]
+  then
+      chmod 750 /etc/postfix/sasl_passwd*
+      chown postfix:root /etc/postfix/sasl_passwd*
+  fi
+
   [[ -z "$regen_conf_files" ]] \
-    || { sudo service postfix restart && sudo service postsrsd restart; }
+    || { systemctl restart postfix && systemctl restart postsrsd; }
 
 }
 
-FORCE=${2:-0}
-DRY_RUN=${3:-0}
-
-case "$1" in
-  pre)
-    do_pre_regen $4
-    ;;
-  post)
-    do_post_regen $4
-    ;;
-  *)
-    echo "hook called with unknown argument \`$1'" >&2
-    exit 1
-    ;;
-esac
-
-exit 0
+do_$1_regen ${@:2}

data/hooks/conf_regen/25-dovecot

@@ -2,6 +2,8 @@
 
 set -e
 
+. /usr/share/yunohost/helpers
+
 do_pre_regen() {
   pending_dir=$1
 
@@ -14,11 +16,10 @@ do_pre_regen() {
   cp dovecot-ldap.conf "${dovecot_dir}/dovecot-ldap.conf"
   cp dovecot.sieve "${dovecot_dir}/global_script/dovecot.sieve"
 
-  # prepare dovecot.conf conf file
-  main_domain=$(cat /etc/yunohost/current_host)
-  cat dovecot.conf \
-    | sed "s/{{ main_domain }}/${main_domain}/g" \
-    > "${dovecot_dir}/dovecot.conf"
+  export pop3_enabled="$(yunohost settings get 'pop3.enabled')"
+  export main_domain=$(cat /etc/yunohost/current_host)
+
+  ynh_render_template "dovecot.conf" "${dovecot_dir}/dovecot.conf"
 
   # adapt it for IPv4-only hosts
   if [ ! -f /proc/net/if_inet6 ]; then
@@ -35,44 +36,31 @@ do_pre_regen() {
 do_post_regen() {
   regen_conf_files=$1
 
-  sudo mkdir -p "/etc/dovecot/yunohost.d/pre-ext.d"
-  sudo mkdir -p "/etc/dovecot/yunohost.d/post-ext.d"
+  mkdir -p "/etc/dovecot/yunohost.d/pre-ext.d"
+  mkdir -p "/etc/dovecot/yunohost.d/post-ext.d"
 
   # create vmail user
   id vmail > /dev/null 2>&1 \
-    || sudo adduser --system --ingroup mail --uid 500 vmail
+    || adduser --system --ingroup mail --uid 500 vmail --home /var/vmail --no-create-home
+
+  # Delete legacy home for vmail that existed in the past but was empty, poluting /home/
+  [ ! -e /home/vmail ] || rmdir --ignore-fail-on-non-empty /home/vmail
 
   # fix permissions
-  sudo chown -R vmail:mail /etc/dovecot/global_script
-  sudo chmod 770 /etc/dovecot/global_script
-  sudo chown root:mail /var/mail
-  sudo chmod 1775 /var/mail
+  chown -R vmail:mail /etc/dovecot/global_script
+  chmod 770 /etc/dovecot/global_script
+  chown root:mail /var/mail
+  chmod 1775 /var/mail
 
   [ -z "$regen_conf_files" ] && exit 0
 
   # compile sieve script
   [[ "$regen_conf_files" =~ dovecot\.sieve ]] && {
-    sudo sievec /etc/dovecot/global_script/dovecot.sieve
-    sudo chown -R vmail:mail /etc/dovecot/global_script
+    sievec /etc/dovecot/global_script/dovecot.sieve
+    chown -R vmail:mail /etc/dovecot/global_script
   }
 
-  sudo service dovecot restart
+  systemctl restart dovecot
 }
 
-FORCE=${2:-0}
-DRY_RUN=${3:-0}
-
-case "$1" in
-  pre)
-    do_pre_regen $4
-    ;;
-  post)
-    do_post_regen $4
-    ;;
-  *)
-    echo "hook called with unknown argument \`$1'" >&2
-    exit 1
-    ;;
-esac
-
-exit 0
+do_$1_regen ${@:2}

data/hooks/conf_regen/31-rspamd

@@ -22,58 +22,41 @@ do_post_regen() {
   ##
 
   # create DKIM directory with proper permission
-  sudo mkdir -p /etc/dkim
-  sudo chown _rspamd /etc/dkim
-
-  # retrieve domain list
-  domain_list=$(sudo yunohost domain list --output-as plain --quiet)
+  mkdir -p /etc/dkim
+  chown _rspamd /etc/dkim
 
   # create DKIM key for domains
-  for domain in $domain_list; do
+  for domain in $YNH_DOMAINS; do
     domain_key="/etc/dkim/${domain}.mail.key"
     [ ! -f "$domain_key" ] && {
       # We use a 1024 bit size because nsupdate doesn't seem to be able to
       # handle 2048...
-      sudo opendkim-genkey --domain="$domain" \
+      opendkim-genkey --domain="$domain" \
           --selector=mail --directory=/etc/dkim -b 1024
-      sudo mv /etc/dkim/mail.private "$domain_key"
-      sudo mv /etc/dkim/mail.txt "/etc/dkim/${domain}.mail.txt"
+      mv /etc/dkim/mail.private "$domain_key"
+      mv /etc/dkim/mail.txt "/etc/dkim/${domain}.mail.txt"
     }
   done
 
   # fix DKIM keys permissions
-  sudo chown _rspamd /etc/dkim/*.mail.key
-  sudo chmod 400 /etc/dkim/*.mail.key
+  chown _rspamd /etc/dkim/*.mail.key
+  chmod 400 /etc/dkim/*.mail.key
+
+  [ ! -e /var/log/rspamd ] || chown -R _rspamd:_rspamd /var/log/rspamd
 
   regen_conf_files=$1
   [ -z "$regen_conf_files" ] && exit 0
 
   # compile sieve script
   [[ "$regen_conf_files" =~ rspamd\.sieve ]] && {
-    sudo sievec /etc/dovecot/global_script/rspamd.sieve
-    sudo chown -R vmail:mail /etc/dovecot/global_script
-    sudo systemctl restart dovecot
+    sievec /etc/dovecot/global_script/rspamd.sieve
+    chown -R vmail:mail /etc/dovecot/global_script
+    systemctl restart dovecot
   }
 
   # Restart rspamd due to the upgrade
   # https://rspamd.com/announce/2016/08/01/rspamd-1.3.1.html 
-  sudo systemctl -q restart rspamd.service 
+  systemctl -q restart rspamd.service 
 }
 
-FORCE=${2:-0}
-DRY_RUN=${3:-0}
-
-case "$1" in
-  pre)
-    do_pre_regen $4
-    ;;
-  post)
-    do_post_regen $4
-    ;;
-  *)
-    echo "hook called with unknown argument \`$1'" >&2
-    exit 1
-    ;;
-esac
-
-exit 0
+do_$1_regen ${@:2}

data/hooks/conf_regen/34-mysql

@@ -1,7 +1,6 @@
 #!/bin/bash
 
 set -e
-MYSQL_PKG="$(dpkg --list | sed -ne 's/^ii  \(mariadb-server-[[:digit:].]\+\) .*$/\1/p')"
 . /usr/share/yunohost/helpers
 
 do_pre_regen() {
@@ -15,62 +14,59 @@ do_pre_regen() {
 do_post_regen() {
   regen_conf_files=$1
 
-  if [ ! -f /etc/yunohost/mysql ]; then
+  if [[ ! -d /var/lib/mysql/mysql ]]
+  then
+      # dpkg-reconfigure will initialize mysql (if it ain't already)
+      # It enabled auth_socket for root, so no need to define any root password...
+      # c.f. : cat /var/lib/dpkg/info/mariadb-server-10.3.postinst | grep install_db -C3
+      MYSQL_PKG="$(dpkg --list | sed -ne 's/^ii  \(mariadb-server-[[:digit:].]\+\) .*$/\1/p')"
+      dpkg-reconfigure -freadline -u "$MYSQL_PKG" 2>&1
 
-      # ensure that mysql is running
-      sudo systemctl -q is-active mysql.service \
-        || sudo service mysql start
+      systemctl -q is-active mariadb.service \
+        || systemctl start mariadb
 
-      # generate and set new root password
-      mysql_password=$(ynh_string_random 10)
-      sudo mysqladmin -s -u root -pyunohost password "$mysql_password" || {
-        if [ $FORCE -eq 1 ]; then
-            echo "It seems that you have already configured MySQL." \
-              "YunoHost needs to have a root access to MySQL to runs its" \
-              "applications, and is going to reset the MySQL root password." \
-              "You can find this new password in /etc/yunohost/mysql." >&2
+      sleep 5
 
-            # set new password with debconf
-            sudo debconf-set-selections << EOF
-$MYSQL_PKG mysql-server/root_password password $mysql_password
-$MYSQL_PKG mysql-server/root_password_again password $mysql_password
-EOF
+      echo "" | mysql && echo "Can't connect to mysql using unix_socket auth ... something went wrong during initial configuration of mysql !?" >&2
+  fi
 
-            # reconfigure Debian package
-            sudo dpkg-reconfigure -freadline -u "$MYSQL_PKG" 2>&1
-        else
-            echo "It seems that you have already configured MySQL." \
-              "YunoHost needs to have a root access to MySQL to runs its" \
-              "applications, but the MySQL root password is unknown." \
-              "You must either pass --force to reset the password or" \
-              "put the current one into the file /etc/yunohost/mysql." >&2
-            exit 1
-        fi
-      }
+  # Legacy code to get rid of /etc/yunohost/mysql ...
+  # Nowadays, we can simply run mysql while being run as root of unix_socket/auth_socket is enabled...
+  if [ -f /etc/yunohost/mysql ]; then
 
-      # store new root password
-      echo "$mysql_password" | sudo tee /etc/yunohost/mysql
-      sudo chmod 400 /etc/yunohost/mysql
+      # This is a trick to check if we're able to use mysql without password
+      # Expect instances installed in stretch to already have unix_socket
+      #configured, but not old instances from the jessie/wheezy era
+      if ! echo "" | mysql 2>/dev/null
+      then
+          password="$(cat /etc/yunohost/mysql)"
+          # Enable plugin unix_socket for root on localhost
+          mysql -u root -p"$password" <<< "GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost' IDENTIFIED WITH unix_socket WITH GRANT OPTION;"
+      fi
+
+      # If now we're able to login without password, drop the mysql password
+      if echo "" | mysql 2>/dev/null
+      then
+          rm /etc/yunohost/mysql
+      else
+          echo "Can't connect to mysql using unix_socket auth ... something went wrong while trying to get rid of mysql password !?" >&2
+      fi
+  fi
+
+  # mysql is supposed to be an alias to mariadb... but in some weird case is not
+  # c.f. https://forum.yunohost.org/t/mysql-ne-fonctionne-pas/11661
+  # Playing with enable/disable allows to recreate the proper symlinks.
+  if [ ! -e /etc/systemd/system/mysql.service ]
+  then
+      systemctl stop mysql -q
+      systemctl disable mysql -q
+      systemctl disable mariadb -q
+      systemctl enable mariadb -q
+      systemctl is-active mariadb -q || systemctl start mariadb
   fi
 
   [[ -z "$regen_conf_files" ]] \
-    || sudo service mysql restart
+    || systemctl restart mysql
 }
 
-FORCE=${2:-0}
-DRY_RUN=${3:-0}
-
-case "$1" in
-  pre)
-    do_pre_regen $4
-    ;;
-  post)
-    do_post_regen $4
-    ;;
-  *)
-    echo "hook called with unknown argument \`$1'" >&2
-    exit 1
-    ;;
-esac
-
-exit 0
+do_$1_regen ${@:2}

data/hooks/conf_regen/35-redis

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+do_pre_regen() {
+  :
+}
+
+do_post_regen() {
+  # Enforce these damn permissions because for some reason in some weird cases
+  # they are spontaneously replaced by root:root -_-
+  chown -R redis:adm /var/log/redis
+}
+
+do_$1_regen ${@:2}

data/hooks/conf_regen/37-avahi-daemon

@@ -1,37 +0,0 @@
-#!/bin/bash
-
-set -e
-
-do_pre_regen() {
-  pending_dir=$1
-
-  cd /usr/share/yunohost/templates/avahi-daemon
-
-  install -D -m 644 avahi-daemon.conf \
-    "${pending_dir}/etc/avahi/avahi-daemon.conf"
-}
-
-do_post_regen() {
-  regen_conf_files=$1
-
-  [[ -z "$regen_conf_files" ]] \
-    || sudo service avahi-daemon restart
-}
-
-FORCE=${2:-0}
-DRY_RUN=${3:-0}
-
-case "$1" in
-  pre)
-    do_pre_regen $4
-    ;;
-  post)
-    do_post_regen $4
-    ;;
-  *)
-    echo "hook called with unknown argument \`$1'" >&2
-    exit 1
-    ;;
-esac
-
-exit 0

data/hooks/conf_regen/37-mdns

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+set -e
+
+_generate_config() {
+  echo "domains:"
+  echo "    - yunohost.local"
+  for domain in $YNH_DOMAINS
+  do
+      # Only keep .local domains (don't keep
+      [[ "$domain" =~ [^.]+\.[^.]+\.local$ ]] && echo "Subdomain $domain cannot be handled by Bonjour/Zeroconf/mDNS" >&2
+      [[ "$domain" =~ ^[^.]+\.local$ ]] || continue
+      echo "    - $domain"
+  done
+
+  echo "interfaces:"
+  local_network_interfaces="$(ip --brief a | grep ' 10\.\| 192\.168\.' | awk '{print $1}')"
+  for interface in $local_network_interfaces
+  do
+      echo "    - $interface"
+  done
+}
+
+do_init_regen() {
+   do_pre_regen
+   do_post_regen /etc/systemd/system/yunomdns.service
+   systemctl enable yunomdns
+}
+
+do_pre_regen() {
+  pending_dir="$1"
+
+  cd /usr/share/yunohost/templates/mdns
+  mkdir -p ${pending_dir}/etc/systemd/system/
+  cp yunomdns.service ${pending_dir}/etc/systemd/system/
+
+  getent passwd mdns &>/dev/null || useradd --no-create-home --shell /usr/sbin/nologin --system --user-group mdns
+
+  mkdir -p ${pending_dir}/etc/yunohost
+  _generate_config > ${pending_dir}/etc/yunohost/mdns.yml
+}
+
+do_post_regen() {
+  regen_conf_files="$1"
+
+  chown mdns:mdns /etc/yunohost/mdns.yml
+
+  # If we changed the systemd ynh-override conf
+  if echo "$regen_conf_files" | sed 's/,/\n/g' | grep -q "^/etc/systemd/system/yunomdns.service$"
+  then
+      systemctl daemon-reload
+  fi
+
+  # Legacy stuff to enable the new yunomdns service on legacy systems
+  if [[ -e /etc/avahi/avahi-daemon.conf ]] && grep -q 'yunohost' /etc/avahi/avahi-daemon.conf
+  then
+      systemctl enable yunomdns
+  fi
+
+  [[ -z "$regen_conf_files" ]] \
+    || systemctl restart yunomdns
+}
+
+do_$1_regen ${@:2}

data/hooks/conf_regen/40-glances

@@ -1,36 +0,0 @@
-#!/bin/bash
-
-set -e
-
-do_pre_regen() {
-  pending_dir=$1
-
-  cd /usr/share/yunohost/templates/glances
-
-  install -D -m 644 glances.default "${pending_dir}/etc/default/glances"
-}
-
-do_post_regen() {
-  regen_conf_files=$1
-
-  [[ -z "$regen_conf_files" ]] \
-    || sudo service glances restart
-}
-
-FORCE=${2:-0}
-DRY_RUN=${3:-0}
-
-case "$1" in
-  pre)
-    do_pre_regen $4
-    ;;
-  post)
-    do_post_regen $4
-    ;;
-  *)
-    echo "hook called with unknown argument \`$1'" >&2
-    exit 1
-    ;;
-esac
-
-exit 0

data/hooks/conf_regen/43-dnsmasq

@@ -26,23 +26,21 @@ do_pre_regen() {
   ynh_validate_ip4 "$ipv4" || ipv4='127.0.0.1'
   ipv6=$(curl -s -6 https://ip6.yunohost.org 2>/dev/null || true)
   ynh_validate_ip6 "$ipv6" || ipv6=''
-  domain_list=$(sudo yunohost domain list --output-as plain --quiet)
+
+  export ipv4
+  export ipv6
 
   # add domain conf files
-  for domain in $domain_list; do
-    cat domain.tpl \
-      | sed "s/{{ domain }}/${domain}/g" \
-      | sed "s/{{ ip }}/${ipv4}/g" \
-      > "${dnsmasq_dir}/${domain}"
-    [[ -n $ipv6 ]] \
-      && echo "address=/${domain}/${ipv6}" >> "${dnsmasq_dir}/${domain}"
+  for domain in $YNH_DOMAINS; do
+      export domain
+      ynh_render_template "domain.tpl" "${dnsmasq_dir}/${domain}"
   done
 
   # remove old domain conf files
   conf_files=$(ls -1 /etc/dnsmasq.d \
                  | awk '/^[^\.]+\.[^\.]+.*$/ { print $1 }')
   for domain in $conf_files; do
-    [[ $domain_list =~ $domain ]] \
+    [[ $YNH_DOMAINS =~ $domain ]] \
       || touch "${dnsmasq_dir}/${domain}"
   done
 }
@@ -50,24 +48,36 @@ do_pre_regen() {
 do_post_regen() {
   regen_conf_files=$1
 
-  [[ -z "$regen_conf_files" ]] \
-    || sudo service dnsmasq restart
+  # Fuck it, those domain/search entries from dhclient are usually annoying
+  # lying shit from the ISP trying to MiTM
+  if grep -q -E "^ *(domain|search)" /run/resolvconf/resolv.conf
+  then
+      if grep -q -E "^ *(domain|search)" /run/resolvconf/interface/*.dhclient 2>/dev/null
+      then
+        sed -E "s/^(domain|search)/#\1/g" -i /run/resolvconf/interface/*.dhclient
+      fi
+
+      grep -q '^supersede domain-name "";' /etc/dhcp/dhclient.conf 2>/dev/null || echo 'supersede domain-name "";' >> /etc/dhcp/dhclient.conf
+      grep -q '^supersede domain-search "";' /etc/dhcp/dhclient.conf 2>/dev/null || echo 'supersede domain-search "";' >> /etc/dhcp/dhclient.conf
+      grep -q '^supersede name "";' /etc/dhcp/dhclient.conf 2>/dev/null || echo 'supersede name "";' >> /etc/dhcp/dhclient.conf
+      systemctl restart resolvconf
+  fi
+
+  # Some stupid things like rabbitmq-server used by onlyoffice won't work if
+  # the *short* hostname doesn't exists in /etc/hosts -_-
+  short_hostname=$(hostname -s)
+  grep -q "127.0.0.1.*$short_hostname" /etc/hosts || echo -e "\n127.0.0.1\t$short_hostname" >>/etc/hosts
+
+  [[ -n "$regen_conf_files" ]] || return
+
+  # Remove / disable services likely to conflict with dnsmasq
+  for SERVICE in systemd-resolved bind9
+  do
+      systemctl is-enabled $SERVICE &>/dev/null && systemctl disable $SERVICE 2>/dev/null
+      systemctl is-active $SERVICE &>/dev/null && systemctl stop $SERVICE
+  done
+
+  systemctl restart dnsmasq
 }
 
-FORCE=${2:-0}
-DRY_RUN=${3:-0}
-
-case "$1" in
-  pre)
-    do_pre_regen $4
-    ;;
-  post)
-    do_post_regen $4
-    ;;
-  *)
-    echo "hook called with unknown argument \`$1'" >&2
-    exit 1
-    ;;
-esac
-
-exit 0
+do_$1_regen ${@:2}

data/hooks/conf_regen/46-nsswitch

@@ -2,6 +2,11 @@
 
 set -e
 
+do_init_regen() {
+  do_pre_regen ""
+  systemctl restart unscd
+}
+
 do_pre_regen() {
   pending_dir=$1
 
@@ -14,23 +19,7 @@ do_post_regen() {
   regen_conf_files=$1
 
   [[ -z "$regen_conf_files" ]] \
-    || sudo service unscd restart
+    || systemctl restart unscd
 }
 
-FORCE=${2:-0}
-DRY_RUN=${3:-0}
-
-case "$1" in
-  pre)
-    do_pre_regen $4
-    ;;
-  post)
-    do_post_regen $4
-    ;;
-  *)
-    echo "hook called with unknown argument \`$1'" >&2
-    exit 1
-    ;;
-esac
-
-exit 0
+do_$1_regen ${@:2}

data/hooks/conf_regen/52-fail2ban

@@ -2,6 +2,8 @@
 
 set -e
 
+. /usr/share/yunohost/helpers
+
 do_pre_regen() {
   pending_dir=$1
 
@@ -13,30 +15,16 @@ do_pre_regen() {
 
   cp yunohost.conf "${fail2ban_dir}/filter.d/yunohost.conf"
   cp jail.conf "${fail2ban_dir}/jail.conf"
-  cp yunohost-jails.conf "${fail2ban_dir}/jail.d/"
+
+  export ssh_port="$(yunohost settings get 'security.ssh.port')"
+  ynh_render_template "yunohost-jails.conf" "${fail2ban_dir}/jail.d/yunohost-jails.conf"
 }
 
 do_post_regen() {
   regen_conf_files=$1
 
   [[ -z "$regen_conf_files" ]] \
-    || sudo service fail2ban restart
+    || systemctl reload fail2ban
 }
 
-FORCE=${2:-0}
-DRY_RUN=${3:-0}
-
-case "$1" in
-  pre)
-    do_pre_regen $4
-    ;;
-  post)
-    do_post_regen $4
-    ;;
-  *)
-    echo "hook called with unknown argument \`$1'" >&2
-    exit 1
-    ;;
-esac
-
-exit 0
+do_$1_regen ${@:2}

data/hooks/diagnosis/00-basesystem.py

@@ -0,0 +1,258 @@
+#!/usr/bin/env python
+
+import os
+import json
+import subprocess
+
+from moulinette.utils.process import check_output
+from moulinette.utils.filesystem import read_file, read_json, write_to_json
+from yunohost.diagnosis import Diagnoser
+from yunohost.utils.packages import ynh_packages_version
+
+
+class BaseSystemDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 600
+    dependencies = []
+
+    def run(self):
+
+        # Detect virt technology (if not bare metal) and arch
+        # Gotta have this "|| true" because it systemd-detect-virt return 'none'
+        # with an error code on bare metal ~.~
+        virt = check_output("systemd-detect-virt || true", shell=True)
+        if virt.lower() == "none":
+            virt = "bare-metal"
+
+        # Detect arch
+        arch = check_output("dpkg --print-architecture")
+        hardware = dict(
+            meta={"test": "hardware"},
+            status="INFO",
+            data={"virt": virt, "arch": arch},
+            summary="diagnosis_basesystem_hardware",
+        )
+
+        # Also possibly the board / hardware name
+        if os.path.exists("/proc/device-tree/model"):
+            model = read_file("/proc/device-tree/model").strip().replace("\x00", "")
+            hardware["data"]["model"] = model
+            hardware["details"] = ["diagnosis_basesystem_hardware_model"]
+        elif os.path.exists("/sys/devices/virtual/dmi/id/sys_vendor"):
+            model = read_file("/sys/devices/virtual/dmi/id/sys_vendor").strip()
+            if os.path.exists("/sys/devices/virtual/dmi/id/product_name"):
+                model = "%s %s" % (
+                    model,
+                    read_file("/sys/devices/virtual/dmi/id/product_name").strip(),
+                )
+            hardware["data"]["model"] = model
+            hardware["details"] = ["diagnosis_basesystem_hardware_model"]
+
+        yield hardware
+
+        # Kernel version
+        kernel_version = read_file("/proc/sys/kernel/osrelease").strip()
+        yield dict(
+            meta={"test": "kernel"},
+            data={"kernel_version": kernel_version},
+            status="INFO",
+            summary="diagnosis_basesystem_kernel",
+        )
+
+        # Debian release
+        debian_version = read_file("/etc/debian_version").strip()
+        yield dict(
+            meta={"test": "host"},
+            data={"debian_version": debian_version},
+            status="INFO",
+            summary="diagnosis_basesystem_host",
+        )
+
+        # Yunohost packages versions
+        # We check if versions are consistent (e.g. all 3.6 and not 3 packages with 3.6 and the other with 3.5)
+        # This is a classical issue for upgrades that failed in the middle
+        # (or people upgrading half of the package because they did 'apt upgrade' instead of 'dist-upgrade')
+        # Here, ynh_core_version is for example "3.5.4.12", so [:3] is "3.5" and we check it's the same for all packages
+        ynh_packages = ynh_packages_version()
+        ynh_core_version = ynh_packages["yunohost"]["version"]
+        consistent_versions = all(
+            infos["version"][:3] == ynh_core_version[:3]
+            for infos in ynh_packages.values()
+        )
+        ynh_version_details = [
+            (
+                "diagnosis_basesystem_ynh_single_version",
+                {
+                    "package": package,
+                    "version": infos["version"],
+                    "repo": infos["repo"],
+                },
+            )
+            for package, infos in ynh_packages.items()
+        ]
+
+        yield dict(
+            meta={"test": "ynh_versions"},
+            data={
+                "main_version": ynh_core_version,
+                "repo": ynh_packages["yunohost"]["repo"],
+            },
+            status="INFO" if consistent_versions else "ERROR",
+            summary="diagnosis_basesystem_ynh_main_version"
+            if consistent_versions
+            else "diagnosis_basesystem_ynh_inconsistent_versions",
+            details=ynh_version_details,
+        )
+
+        if self.is_vulnerable_to_meltdown():
+            yield dict(
+                meta={"test": "meltdown"},
+                status="ERROR",
+                summary="diagnosis_security_vulnerable_to_meltdown",
+                details=["diagnosis_security_vulnerable_to_meltdown_details"],
+            )
+
+        bad_sury_packages = list(self.bad_sury_packages())
+        if bad_sury_packages:
+            cmd_to_fix = "apt install --allow-downgrades " + " ".join(
+                ["%s=%s" % (package, version) for package, version in bad_sury_packages]
+            )
+            yield dict(
+                meta={"test": "packages_from_sury"},
+                data={"cmd_to_fix": cmd_to_fix},
+                status="WARNING",
+                summary="diagnosis_package_installed_from_sury",
+                details=["diagnosis_package_installed_from_sury_details"],
+            )
+
+        if self.backports_in_sources_list():
+            yield dict(
+                meta={"test": "backports_in_sources_list"},
+                status="WARNING",
+                summary="diagnosis_backports_in_sources_list",
+            )
+
+        if self.number_of_recent_auth_failure() > 500:
+            yield dict(
+                meta={"test": "high_number_auth_failure"},
+                status="WARNING",
+                summary="diagnosis_high_number_auth_failures",
+            )
+
+    def bad_sury_packages(self):
+
+        packages_to_check = ["openssl", "libssl1.1", "libssl-dev"]
+        for package in packages_to_check:
+            cmd = "dpkg --list | grep '^ii' | grep gbp | grep -q -w %s" % package
+            # If version currently installed is not from sury, nothing to report
+            if os.system(cmd) != 0:
+                continue
+
+            cmd = (
+                "LC_ALL=C apt policy %s 2>&1 | grep http -B1 | tr -d '*' | grep '+deb' | grep -v 'gbp' | head -n 1 | awk '{print $1}'"
+                % package
+            )
+            version_to_downgrade_to = check_output(cmd)
+            yield (package, version_to_downgrade_to)
+
+    def backports_in_sources_list(self):
+
+        cmd = "grep -q -nr '^ *deb .*-backports' /etc/apt/sources.list*"
+        return os.system(cmd) == 0
+
+    def number_of_recent_auth_failure(self):
+
+        # Those syslog facilities correspond to auth and authpriv
+        # c.f. https://unix.stackexchange.com/a/401398
+        # and https://wiki.archlinux.org/title/Systemd/Journal#Facility
+        cmd = "journalctl -q SYSLOG_FACILITY=10 SYSLOG_FACILITY=4 --since '1day ago' | grep 'authentication failure' | wc -l"
+
+        n_failures = check_output(cmd)
+        try:
+            return int(n_failures)
+        except Exception:
+            self.logger_warning(
+                "Failed to parse number of recent auth failures, expected an int, got '%s'"
+                % n_failures
+            )
+            return -1
+
+    def is_vulnerable_to_meltdown(self):
+        # meltdown CVE: https://security-tracker.debian.org/tracker/CVE-2017-5754
+
+        # We use a cache file to avoid re-running the script so many times,
+        # which can be expensive (up to around 5 seconds on ARM)
+        # and make the admin appear to be slow (c.f. the calls to diagnosis
+        # from the webadmin)
+        #
+        # The cache is in /tmp and shall disappear upon reboot
+        # *or* we compare it to dpkg.log modification time
+        # such that it's re-ran if there was package upgrades
+        # (e.g. from yunohost)
+        cache_file = "/tmp/yunohost-meltdown-diagnosis"
+        dpkg_log = "/var/log/dpkg.log"
+        if os.path.exists(cache_file):
+            if not os.path.exists(dpkg_log) or os.path.getmtime(
+                cache_file
+            ) > os.path.getmtime(dpkg_log):
+                self.logger_debug(
+                    "Using cached results for meltdown checker, from %s" % cache_file
+                )
+                return read_json(cache_file)[0]["VULNERABLE"]
+
+        # script taken from https://github.com/speed47/spectre-meltdown-checker
+        # script commit id is store directly in the script
+        SCRIPT_PATH = "/usr/lib/moulinette/yunohost/vendor/spectre-meltdown-checker/spectre-meltdown-checker.sh"
+
+        # '--variant 3' corresponds to Meltdown
+        # example output from the script:
+        # [{"NAME":"MELTDOWN","CVE":"CVE-2017-5754","VULNERABLE":false,"INFOS":"PTI mitigates the vulnerability"}]
+        try:
+            self.logger_debug("Running meltdown vulnerability checker")
+            call = subprocess.Popen(
+                "bash %s --batch json --variant 3" % SCRIPT_PATH,
+                shell=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+
+            # TODO / FIXME : here we are ignoring error messages ...
+            # in particular on RPi2 and other hardware, the script complains about
+            # "missing some kernel info (see -v), accuracy might be reduced"
+            # Dunno what to do about that but we probably don't want to harass
+            # users with this warning ...
+            output, _ = call.communicate()
+            output = output.decode()
+            assert call.returncode in (0, 2, 3), "Return code: %s" % call.returncode
+
+            # If there are multiple lines, sounds like there was some messages
+            # in stdout that are not json >.> ... Try to get the actual json
+            # stuff which should be the last line
+            output = output.strip()
+            if "\n" in output:
+                self.logger_debug("Original meltdown checker output : %s" % output)
+                output = output.split("\n")[-1]
+
+            CVEs = json.loads(output)
+            assert len(CVEs) == 1
+            assert CVEs[0]["NAME"] == "MELTDOWN"
+        except Exception as e:
+            import traceback
+
+            traceback.print_exc()
+            self.logger_warning(
+                "Something wrong happened when trying to diagnose Meltdown vunerability, exception: %s"
+                % e
+            )
+            raise Exception("Command output for failed meltdown check: '%s'" % output)
+
+        self.logger_debug(
+            "Writing results from meltdown checker to cache file, %s" % cache_file
+        )
+        write_to_json(cache_file, CVEs)
+        return CVEs[0]["VULNERABLE"]
+
+
+def main(args, env, loggers):
+    return BaseSystemDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/10-ip.py

@@ -0,0 +1,230 @@
+#!/usr/bin/env python
+
+import re
+import os
+import random
+
+from moulinette.utils.network import download_text
+from moulinette.utils.process import check_output
+from moulinette.utils.filesystem import read_file
+
+from yunohost.diagnosis import Diagnoser
+from yunohost.utils.network import get_network_interfaces
+
+
+class IPDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 600
+    dependencies = []
+
+    def run(self):
+
+        # ############################################################ #
+        # PING : Check that we can ping outside at least in ipv4 or v6 #
+        # ############################################################ #
+
+        can_ping_ipv4 = self.can_ping_outside(4)
+        can_ping_ipv6 = self.can_ping_outside(6)
+
+        if not can_ping_ipv4 and not can_ping_ipv6:
+            yield dict(
+                meta={"test": "ping"},
+                status="ERROR",
+                summary="diagnosis_ip_not_connected_at_all",
+            )
+            # Not much else we can do if there's no internet at all
+            return
+
+        # ###################################################### #
+        # DNS RESOLUTION : Check that we can resolve domain name #
+        # (later needed to talk to ip. and ip6.yunohost.org)     #
+        # ###################################################### #
+
+        can_resolve_dns = self.can_resolve_dns()
+
+        # In every case, we can check that resolvconf seems to be okay
+        # (symlink managed by resolvconf service + pointing to dnsmasq)
+        good_resolvconf = self.good_resolvconf()
+
+        # If we can't resolve domain names at all, that's a pretty big issue ...
+        # If it turns out that at the same time, resolvconf is bad, that's probably
+        # the cause of this, so we use a different message in that case
+        if not can_resolve_dns:
+            yield dict(
+                meta={"test": "dnsresolv"},
+                status="ERROR",
+                summary="diagnosis_ip_broken_dnsresolution"
+                if good_resolvconf
+                else "diagnosis_ip_broken_resolvconf",
+            )
+            return
+        # Otherwise, if the resolv conf is bad but we were able to resolve domain name,
+        # still warn that we're using a weird resolv conf ...
+        elif not good_resolvconf:
+            yield dict(
+                meta={"test": "dnsresolv"},
+                status="WARNING",
+                summary="diagnosis_ip_weird_resolvconf",
+                details=["diagnosis_ip_weird_resolvconf_details"],
+            )
+        else:
+            yield dict(
+                meta={"test": "dnsresolv"},
+                status="SUCCESS",
+                summary="diagnosis_ip_dnsresolution_working",
+            )
+
+        # ##################################################### #
+        # IP DIAGNOSIS : Check that we're actually able to talk #
+        # to a web server to fetch current IPv4 and v6          #
+        # ##################################################### #
+
+        ipv4 = self.get_public_ip(4) if can_ping_ipv4 else None
+        ipv6 = self.get_public_ip(6) if can_ping_ipv6 else None
+
+        network_interfaces = get_network_interfaces()
+
+        def get_local_ip(version):
+            local_ip = {
+                iface: addr[version].split("/")[0]
+                for iface, addr in network_interfaces.items()
+                if version in addr
+            }
+            if not local_ip:
+                return None
+            elif len(local_ip):
+                return next(iter(local_ip.values()))
+            else:
+                return local_ip
+
+        yield dict(
+            meta={"test": "ipv4"},
+            data={"global": ipv4, "local": get_local_ip("ipv4")},
+            status="SUCCESS" if ipv4 else "ERROR",
+            summary="diagnosis_ip_connected_ipv4" if ipv4 else "diagnosis_ip_no_ipv4",
+            details=["diagnosis_ip_global", "diagnosis_ip_local"] if ipv4 else None,
+        )
+
+        yield dict(
+            meta={"test": "ipv6"},
+            data={"global": ipv6, "local": get_local_ip("ipv6")},
+            status="SUCCESS" if ipv6 else "WARNING",
+            summary="diagnosis_ip_connected_ipv6" if ipv6 else "diagnosis_ip_no_ipv6",
+            details=["diagnosis_ip_global", "diagnosis_ip_local"]
+            if ipv6
+            else ["diagnosis_ip_no_ipv6_tip"],
+        )
+
+        # TODO / FIXME : add some attempt to detect ISP (using whois ?) ?
+
+    def can_ping_outside(self, protocol=4):
+
+        assert protocol in [
+            4,
+            6,
+        ], "Invalid protocol version, it should be either 4 or 6 and was '%s'" % repr(
+            protocol
+        )
+
+        # We can know that ipv6 is not available directly if this file does not exists
+        if protocol == 6 and not os.path.exists("/proc/net/if_inet6"):
+            return False
+
+        # If we are indeed connected in ipv4 or ipv6, we should find a default route
+        routes = check_output("ip -%s route show table all" % protocol).split("\n")
+
+        def is_default_route(r):
+            # Typically the default route starts with "default"
+            # But of course IPv6 is more complex ... e.g. on internet cube there's
+            # no default route but a /3 which acts as a default-like route...
+            # e.g. 2000:/3 dev tun0 ...
+            return r.startswith("default") or (
+                ":" in r and re.match(r".*/[0-3]$", r.split()[0])
+            )
+
+        if not any(is_default_route(r) for r in routes):
+            self.logger_debug(
+                "No default route for IPv%s, so assuming there's no IP address for that version"
+                % protocol
+            )
+            return None
+
+        # We use the resolver file as a list of well-known, trustable (ie not google ;)) IPs that we can ping
+        resolver_file = (
+            "/usr/share/yunohost/templates/dnsmasq/plain/resolv.dnsmasq.conf"
+        )
+        resolvers = [
+            r.split(" ")[1]
+            for r in read_file(resolver_file).split("\n")
+            if r.startswith("nameserver")
+        ]
+
+        if protocol == 4:
+            resolvers = [r for r in resolvers if ":" not in r]
+        if protocol == 6:
+            resolvers = [r for r in resolvers if ":" in r]
+
+        assert (
+            resolvers != []
+        ), "Uhoh, need at least one IPv%s DNS resolver in %s ..." % (
+            protocol,
+            resolver_file,
+        )
+
+        # So let's try to ping the first 4~5 resolvers (shuffled)
+        # If we succesfully ping any of them, we conclude that we are indeed connected
+        def ping(protocol, target):
+            return (
+                os.system(
+                    "ping%s -c1 -W 3 %s >/dev/null 2>/dev/null"
+                    % ("" if protocol == 4 else "6", target)
+                )
+                == 0
+            )
+
+        random.shuffle(resolvers)
+        return any(ping(protocol, resolver) for resolver in resolvers[:5])
+
+    def can_resolve_dns(self):
+        return os.system("dig +short ip.yunohost.org >/dev/null 2>/dev/null") == 0
+
+    def good_resolvconf(self):
+        content = read_file("/etc/resolv.conf").strip().split("\n")
+        # Ignore comments and empty lines
+        content = [
+            line.strip()
+            for line in content
+            if line.strip()
+            and not line.strip().startswith("#")
+            and not line.strip().startswith("search")
+        ]
+        # We should only find a "nameserver 127.0.0.1"
+        return len(content) == 1 and content[0].split() == ["nameserver", "127.0.0.1"]
+
+    def get_public_ip(self, protocol=4):
+
+        # FIXME - TODO : here we assume that DNS resolution for ip.yunohost.org is working
+        # but if we want to be able to diagnose DNS resolution issues independently from
+        # internet connectivity, we gotta rely on fixed IPs first....
+
+        assert protocol in [
+            4,
+            6,
+        ], "Invalid protocol version, it should be either 4 or 6 and was '%s'" % repr(
+            protocol
+        )
+
+        url = "https://ip%s.yunohost.org" % ("6" if protocol == 6 else "")
+
+        try:
+            return download_text(url, timeout=30).strip()
+        except Exception as e:
+            self.logger_debug(
+                "Could not get public IPv%s : %s" % (str(protocol), str(e))
+            )
+            return None
+
+
+def main(args, env, loggers):
+    return IPDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/12-dnsrecords.py

@@ -0,0 +1,299 @@
+#!/usr/bin/env python
+
+import os
+import re
+
+from datetime import datetime, timedelta
+from publicsuffix import PublicSuffixList
+
+from moulinette.utils.process import check_output
+
+from yunohost.utils.dns import dig, YNH_DYNDNS_DOMAINS
+from yunohost.diagnosis import Diagnoser
+from yunohost.domain import domain_list, _get_maindomain
+from yunohost.dns import _build_dns_conf, _get_dns_zone_for_domain
+
+SPECIAL_USE_TLDS = ["local", "localhost", "onion", "test"]
+
+
+class DNSRecordsDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 600
+    dependencies = ["ip"]
+
+    def run(self):
+
+        main_domain = _get_maindomain()
+
+        all_domains = domain_list(exclude_subdomains=True)["domains"]
+        for domain in all_domains:
+            self.logger_debug("Diagnosing DNS conf for %s" % domain)
+            is_specialusedomain = any(
+                domain.endswith("." + tld) for tld in SPECIAL_USE_TLDS
+            )
+            for report in self.check_domain(
+                domain,
+                domain == main_domain,
+                is_specialusedomain=is_specialusedomain,
+            ):
+                yield report
+
+        # Check if a domain buy by the user will expire soon
+        psl = PublicSuffixList()
+        domains_from_registrar = [
+            psl.get_public_suffix(domain) for domain in all_domains
+        ]
+        domains_from_registrar = [
+            domain for domain in domains_from_registrar if "." in domain
+        ]
+        domains_from_registrar = set(domains_from_registrar) - set(
+            YNH_DYNDNS_DOMAINS + ["netlib.re"]
+        )
+        for report in self.check_expiration_date(domains_from_registrar):
+            yield report
+
+    def check_domain(self, domain, is_main_domain, is_specialusedomain):
+
+        base_dns_zone = _get_dns_zone_for_domain(domain)
+        basename = domain.replace(base_dns_zone, "").rstrip(".") or "@"
+
+        expected_configuration = _build_dns_conf(
+            domain, include_empty_AAAA_if_no_ipv6=True
+        )
+
+        categories = ["basic", "mail", "xmpp", "extra"]
+
+        if is_specialusedomain:
+            categories = []
+            yield dict(
+                meta={"domain": domain},
+                data={},
+                status="INFO",
+                summary="diagnosis_dns_specialusedomain",
+            )
+
+        for category in categories:
+
+            records = expected_configuration[category]
+            discrepancies = []
+            results = {}
+
+            for r in records:
+
+                id_ = r["type"] + ":" + r["name"]
+                fqdn = r["name"] + "." + base_dns_zone if r["name"] != "@" else domain
+
+                # Ugly hack to not check mail records for subdomains stuff, otherwise will end up in a shitstorm of errors for people with many subdomains...
+                # Should find a cleaner solution in the suggested conf...
+                if r["type"] in ["MX", "TXT"] and fqdn not in [
+                    domain,
+                    f"mail._domainkey.{domain}",
+                    f"_dmarc.{domain}",
+                ]:
+                    continue
+
+                r["current"] = self.get_current_record(fqdn, r["type"])
+                if r["value"] == "@":
+                    r["value"] = domain + "."
+
+                if self.current_record_match_expected(r):
+                    results[id_] = "OK"
+                else:
+                    if r["current"] is None:
+                        results[id_] = "MISSING"
+                        discrepancies.append(("diagnosis_dns_missing_record", r))
+                    else:
+                        results[id_] = "WRONG"
+                        discrepancies.append(("diagnosis_dns_discrepancy", r))
+
+            def its_important():
+                # Every mail DNS records are important for main domain
+                # For other domain, we only report it as a warning for now...
+                if is_main_domain and category == "mail":
+                    return True
+                elif category == "basic":
+                    # A bad or missing A record is critical ...
+                    # And so is a wrong AAAA record
+                    # (However, a missing AAAA record is acceptable)
+                    if (
+                        results[f"A:{basename}"] != "OK"
+                        or results[f"AAAA:{basename}"] == "WRONG"
+                    ):
+                        return True
+
+                return False
+
+            if discrepancies:
+                status = "ERROR" if its_important() else "WARNING"
+                summary = "diagnosis_dns_bad_conf"
+            else:
+                status = "SUCCESS"
+                summary = "diagnosis_dns_good_conf"
+
+            output = dict(
+                meta={"domain": domain, "category": category},
+                data=results,
+                status=status,
+                summary=summary,
+            )
+
+            if discrepancies:
+                # For ynh-managed domains (nohost.me etc...), tell people to try to "yunohost dyndns update --force"
+                if any(
+                    domain.endswith(ynh_dyndns_domain)
+                    for ynh_dyndns_domain in YNH_DYNDNS_DOMAINS
+                ):
+                    output["details"] = ["diagnosis_dns_try_dyndns_update_force"]
+                # Otherwise point to the documentation
+                else:
+                    output["details"] = ["diagnosis_dns_point_to_doc"]
+                output["details"] += discrepancies
+
+            yield output
+
+    def get_current_record(self, fqdn, type_):
+
+        success, answers = dig(fqdn, type_, resolvers="force_external")
+
+        if success != "ok":
+            return None
+        else:
+            return answers[0] if len(answers) == 1 else answers
+
+    def current_record_match_expected(self, r):
+        if r["value"] is not None and r["current"] is None:
+            return False
+        if r["value"] is None and r["current"] is not None:
+            return False
+        elif isinstance(r["current"], list):
+            return False
+
+        if r["type"] == "TXT":
+            # Split expected/current
+            #  from  "v=DKIM1; k=rsa; p=hugekey;"
+            #  to a set like {'v=DKIM1', 'k=rsa', 'p=...'}
+            # Additionally, for DKIM, because the key is pretty long,
+            # some DNS registrar sometime split it into several pieces like this:
+            # "p=foo" "bar" (with a space and quotes in the middle)...
+            expected = set(r["value"].strip(';" ').replace(";", " ").split())
+            current = set(
+                r["current"].replace('" "', "").strip(';" ').replace(";", " ").split()
+            )
+
+            # For SPF, ignore parts starting by ip4: or ip6:
+            if "v=spf1" in r["value"]:
+                current = {
+                    part
+                    for part in current
+                    if not part.startswith("ip4:") and not part.startswith("ip6:")
+                }
+            return expected == current
+        elif r["type"] == "MX":
+            # For MX, we want to ignore the priority
+            expected = r["value"].split()[-1]
+            current = r["current"].split()[-1]
+            return expected == current
+        else:
+            return r["current"] == r["value"]
+
+    def check_expiration_date(self, domains):
+        """
+        Alert if expiration date of a domain is soon
+        """
+        details = {"not_found": [], "error": [], "warning": [], "success": []}
+
+        for domain in domains:
+            expire_date = self.get_domain_expiration(domain)
+
+            if isinstance(expire_date, str):
+                status_ns, _ = dig(domain, "NS", resolvers="force_external")
+                status_a, _ = dig(domain, "A", resolvers="force_external")
+                if "ok" not in [status_ns, status_a]:
+                    # i18n: diagnosis_domain_not_found_details
+                    details["not_found"].append(
+                        (
+                            "diagnosis_domain_%s_details" % (expire_date),
+                            {"domain": domain},
+                        )
+                    )
+                else:
+                    self.logger_debug("Dyndns domain: %s" % (domain))
+                continue
+
+            expire_in = expire_date - datetime.now()
+
+            alert_type = "success"
+            if expire_in <= timedelta(15):
+                alert_type = "error"
+            elif expire_in <= timedelta(45):
+                alert_type = "warning"
+
+            args = {
+                "domain": domain,
+                "days": expire_in.days - 1,
+                "expire_date": str(expire_date),
+            }
+            details[alert_type].append(("diagnosis_domain_expires_in", args))
+
+        for alert_type in ["success", "error", "warning", "not_found"]:
+            if details[alert_type]:
+                if alert_type == "not_found":
+                    meta = {"test": "domain_not_found"}
+                else:
+                    meta = {"test": "domain_expiration"}
+                # Allow to ignore specifically a single domain
+                if len(details[alert_type]) == 1:
+                    meta["domain"] = details[alert_type][0][1]["domain"]
+
+                # i18n: diagnosis_domain_expiration_not_found
+                # i18n: diagnosis_domain_expiration_error
+                # i18n: diagnosis_domain_expiration_warning
+                # i18n: diagnosis_domain_expiration_success
+                # i18n: diagnosis_domain_expiration_not_found_details
+                yield dict(
+                    meta=meta,
+                    data={},
+                    status=alert_type.upper()
+                    if alert_type != "not_found"
+                    else "WARNING",
+                    summary="diagnosis_domain_expiration_" + alert_type,
+                    details=details[alert_type],
+                )
+
+    def get_domain_expiration(self, domain):
+        """
+        Return the expiration datetime of a domain or None
+        """
+        command = "whois -H %s || echo failed" % (domain)
+        out = check_output(command).split("\n")
+
+        # Reduce output to determine if whois answer is equivalent to NOT FOUND
+        filtered_out = [
+            line
+            for line in out
+            if re.search(r"^[a-zA-Z0-9 ]{4,25}:", line, re.IGNORECASE)
+            and not re.match(r">>> Last update of whois", line, re.IGNORECASE)
+            and not re.match(r"^NOTICE:", line, re.IGNORECASE)
+            and not re.match(r"^%%", line, re.IGNORECASE)
+            and not re.match(r'"https?:"', line, re.IGNORECASE)
+        ]
+
+        # If there is less than 7 lines, it's NOT FOUND response
+        if len(filtered_out) <= 6:
+            return "not_found"
+
+        for line in out:
+            match = re.search(r"Expir.+(\d{4}-\d{2}-\d{2})", line, re.IGNORECASE)
+            if match is not None:
+                return datetime.strptime(match.group(1), "%Y-%m-%d")
+
+            match = re.search(r"Expir.+(\d{2}-\w{3}-\d{4})", line, re.IGNORECASE)
+            if match is not None:
+                return datetime.strptime(match.group(1), "%d-%b-%Y")
+
+        return "expiration_not_found"
+
+
+def main(args, env, loggers):
+    return DNSRecordsDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/14-ports.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python
+
+import os
+
+from yunohost.diagnosis import Diagnoser
+from yunohost.service import _get_services
+
+
+class PortsDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 600
+    dependencies = ["ip"]
+
+    def run(self):
+
+        # TODO: report a warning if port 53 or 5353 is exposed to the outside world...
+
+        # This dict is something like :
+        #   {   80: "nginx",
+        #       25: "postfix",
+        #       443: "nginx"
+        #       ... }
+        ports = {}
+        services = _get_services()
+        for service, infos in services.items():
+            for port in infos.get("needs_exposed_ports", []):
+                ports[port] = service
+
+        ipversions = []
+        ipv4 = Diagnoser.get_cached_report("ip", item={"test": "ipv4"}) or {}
+        if ipv4.get("status") == "SUCCESS":
+            ipversions.append(4)
+
+        # To be discussed: we could also make this check dependent on the
+        # existence of an AAAA record...
+        ipv6 = Diagnoser.get_cached_report("ip", item={"test": "ipv6"}) or {}
+        if ipv6.get("status") == "SUCCESS":
+            ipversions.append(6)
+
+        # Fetch test result for each relevant IP version
+        results = {}
+        for ipversion in ipversions:
+            try:
+                r = Diagnoser.remote_diagnosis(
+                    "check-ports", data={"ports": list(ports)}, ipversion=ipversion
+                )
+                results[ipversion] = r["ports"]
+            except Exception as e:
+                yield dict(
+                    meta={"reason": "remote_diagnosis_failed", "ipversion": ipversion},
+                    data={"error": str(e)},
+                    status="WARNING",
+                    summary="diagnosis_ports_could_not_diagnose",
+                    details=["diagnosis_ports_could_not_diagnose_details"],
+                )
+                continue
+
+        ipversions = results.keys()
+        if not ipversions:
+            return
+
+        for port, service in sorted(ports.items()):
+            port = str(port)
+            category = services[service].get("category", "[?]")
+
+            # If both IPv4 and IPv6 (if applicable) are good
+            if all(results[ipversion].get(port) is True for ipversion in ipversions):
+                yield dict(
+                    meta={"port": port},
+                    data={"service": service, "category": category},
+                    status="SUCCESS",
+                    summary="diagnosis_ports_ok",
+                    details=["diagnosis_ports_needed_by"],
+                )
+            # If both IPv4 and IPv6 (if applicable) are failed
+            elif all(
+                results[ipversion].get(port) is not True for ipversion in ipversions
+            ):
+                yield dict(
+                    meta={"port": port},
+                    data={"service": service, "category": category},
+                    status="ERROR",
+                    summary="diagnosis_ports_unreachable",
+                    details=[
+                        "diagnosis_ports_needed_by",
+                        "diagnosis_ports_forwarding_tip",
+                    ],
+                )
+            # If only IPv4 is failed or only IPv6 is failed (if applicable)
+            else:
+                passed, failed = (4, 6) if results[4].get(port) is True else (6, 4)
+
+                # Failing in ipv4 is critical.
+                # If we failed in IPv6 but there's in fact no AAAA record
+                # It's an acceptable situation and we shall not report an
+                # error
+                # If any AAAA record is set, IPv6 is important...
+                def ipv6_is_important():
+                    dnsrecords = Diagnoser.get_cached_report("dnsrecords") or {}
+                    return any(
+                        record["data"].get("AAAA:@") in ["OK", "WRONG"]
+                        for record in dnsrecords.get("items", [])
+                    )
+
+                if failed == 4 or ipv6_is_important():
+                    yield dict(
+                        meta={"port": port},
+                        data={
+                            "service": service,
+                            "category": category,
+                            "passed": passed,
+                            "failed": failed,
+                        },
+                        status="ERROR",
+                        summary="diagnosis_ports_partially_unreachable",
+                        details=[
+                            "diagnosis_ports_needed_by",
+                            "diagnosis_ports_forwarding_tip",
+                        ],
+                    )
+                # So otherwise we report a success
+                # And in addition we report an info about the failure in IPv6
+                # *with a different meta* (important to avoid conflicts when
+                # fetching the other info...)
+                else:
+                    yield dict(
+                        meta={"port": port},
+                        data={"service": service, "category": category},
+                        status="SUCCESS",
+                        summary="diagnosis_ports_ok",
+                        details=["diagnosis_ports_needed_by"],
+                    )
+                    yield dict(
+                        meta={"test": "ipv6", "port": port},
+                        data={
+                            "service": service,
+                            "category": category,
+                            "passed": passed,
+                            "failed": failed,
+                        },
+                        status="INFO",
+                        summary="diagnosis_ports_partially_unreachable",
+                        details=[
+                            "diagnosis_ports_needed_by",
+                            "diagnosis_ports_forwarding_tip",
+                        ],
+                    )
+
+
+def main(args, env, loggers):
+    return PortsDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/21-web.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python
+
+import os
+import random
+import requests
+
+from moulinette.utils.filesystem import read_file
+
+from yunohost.diagnosis import Diagnoser
+from yunohost.domain import domain_list
+
+DIAGNOSIS_SERVER = "diagnosis.yunohost.org"
+
+
+class WebDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 600
+    dependencies = ["ip"]
+
+    def run(self):
+
+        all_domains = domain_list()["domains"]
+        domains_to_check = []
+        for domain in all_domains:
+
+            # If the diagnosis location ain't defined, can't do diagnosis,
+            # probably because nginx conf manually modified...
+            nginx_conf = "/etc/nginx/conf.d/%s.conf" % domain
+            if ".well-known/ynh-diagnosis/" not in read_file(nginx_conf):
+                yield dict(
+                    meta={"domain": domain},
+                    status="WARNING",
+                    summary="diagnosis_http_nginx_conf_not_up_to_date",
+                    details=["diagnosis_http_nginx_conf_not_up_to_date_details"],
+                )
+            elif domain.endswith(".local"):
+                yield dict(
+                    meta={"domain": domain},
+                    status="INFO",
+                    summary="diagnosis_http_localdomain",
+                )
+            else:
+                domains_to_check.append(domain)
+
+        self.nonce = "".join(random.choice("0123456789abcedf") for i in range(16))
+        os.system("rm -rf /tmp/.well-known/ynh-diagnosis/")
+        os.system("mkdir -p /tmp/.well-known/ynh-diagnosis/")
+        os.system("touch /tmp/.well-known/ynh-diagnosis/%s" % self.nonce)
+
+        if not domains_to_check:
+            return
+
+        # To perform hairpinning test, we gotta make sure that port forwarding
+        # is working and therefore we'll do it only if at least one ipv4 domain
+        # works.
+        self.do_hairpinning_test = False
+
+        ipversions = []
+        ipv4 = Diagnoser.get_cached_report("ip", item={"test": "ipv4"}) or {}
+        if ipv4.get("status") == "SUCCESS":
+            ipversions.append(4)
+
+        # To be discussed: we could also make this check dependent on the
+        # existence of an AAAA record...
+        ipv6 = Diagnoser.get_cached_report("ip", item={"test": "ipv6"}) or {}
+        if ipv6.get("status") == "SUCCESS":
+            ipversions.append(6)
+
+        for item in self.test_http(domains_to_check, ipversions):
+            yield item
+
+        # If at least one domain is correctly exposed to the outside,
+        # attempt to diagnose hairpinning situations. On network with
+        # hairpinning issues, the server may be correctly exposed on the
+        # outside, but from the outside, it will be as if the port forwarding
+        # was not configured... Hence, calling for example
+        # "curl --head the.global.ip" will simply timeout...
+        if self.do_hairpinning_test:
+            global_ipv4 = ipv4.get("data", {}).get("global", None)
+            if global_ipv4:
+                try:
+                    requests.head("http://" + global_ipv4, timeout=5)
+                except requests.exceptions.Timeout:
+                    yield dict(
+                        meta={"test": "hairpinning"},
+                        status="WARNING",
+                        summary="diagnosis_http_hairpinning_issue",
+                        details=["diagnosis_http_hairpinning_issue_details"],
+                    )
+                except Exception:
+                    # Well I dunno what to do if that's another exception
+                    # type... That'll most probably *not* be an hairpinning
+                    # issue but something else super weird ...
+                    pass
+
+    def test_http(self, domains, ipversions):
+
+        results = {}
+        for ipversion in ipversions:
+            try:
+                r = Diagnoser.remote_diagnosis(
+                    "check-http",
+                    data={"domains": domains, "nonce": self.nonce},
+                    ipversion=ipversion,
+                )
+                results[ipversion] = r["http"]
+            except Exception as e:
+                yield dict(
+                    meta={"reason": "remote_diagnosis_failed", "ipversion": ipversion},
+                    data={"error": str(e)},
+                    status="WARNING",
+                    summary="diagnosis_http_could_not_diagnose",
+                    details=["diagnosis_http_could_not_diagnose_details"],
+                )
+                continue
+
+        ipversions = results.keys()
+        if not ipversions:
+            return
+
+        for domain in domains:
+
+            # i18n: diagnosis_http_bad_status_code
+            # i18n: diagnosis_http_connection_error
+            # i18n: diagnosis_http_timeout
+
+            # If both IPv4 and IPv6 (if applicable) are good
+            if all(
+                results[ipversion][domain]["status"] == "ok" for ipversion in ipversions
+            ):
+                if 4 in ipversions:
+                    self.do_hairpinning_test = True
+                yield dict(
+                    meta={"domain": domain},
+                    status="SUCCESS",
+                    summary="diagnosis_http_ok",
+                )
+            # If both IPv4 and IPv6 (if applicable) are failed
+            elif all(
+                results[ipversion][domain]["status"] != "ok" for ipversion in ipversions
+            ):
+                detail = results[4 if 4 in ipversions else 6][domain]["status"]
+                yield dict(
+                    meta={"domain": domain},
+                    status="ERROR",
+                    summary="diagnosis_http_unreachable",
+                    details=[detail.replace("error_http_check", "diagnosis_http")],
+                )
+            # If only IPv4 is failed or only IPv6 is failed (if applicable)
+            else:
+                passed, failed = (
+                    (4, 6) if results[4][domain]["status"] == "ok" else (6, 4)
+                )
+                detail = results[failed][domain]["status"]
+
+                # Failing in ipv4 is critical.
+                # If we failed in IPv6 but there's in fact no AAAA record
+                # It's an acceptable situation and we shall not report an
+                # error
+                def ipv6_is_important_for_this_domain():
+                    dnsrecords = (
+                        Diagnoser.get_cached_report(
+                            "dnsrecords", item={"domain": domain, "category": "basic"}
+                        )
+                        or {}
+                    )
+                    AAAA_status = dnsrecords.get("data", {}).get("AAAA:@")
+
+                    return AAAA_status in ["OK", "WRONG"]
+
+                if failed == 4 or ipv6_is_important_for_this_domain():
+                    yield dict(
+                        meta={"domain": domain},
+                        data={"passed": passed, "failed": failed},
+                        status="ERROR",
+                        summary="diagnosis_http_partially_unreachable",
+                        details=[detail.replace("error_http_check", "diagnosis_http")],
+                    )
+                # So otherwise we report a success (note that this info is
+                # later used to know that ACME challenge is doable)
+                #
+                # And in addition we report an info about the failure in IPv6
+                # *with a different meta* (important to avoid conflicts when
+                # fetching the other info...)
+                else:
+                    self.do_hairpinning_test = True
+                    yield dict(
+                        meta={"domain": domain},
+                        status="SUCCESS",
+                        summary="diagnosis_http_ok",
+                    )
+                    yield dict(
+                        meta={"test": "ipv6", "domain": domain},
+                        data={"passed": passed, "failed": failed},
+                        status="INFO",
+                        summary="diagnosis_http_partially_unreachable",
+                        details=[detail.replace("error_http_check", "diagnosis_http")],
+                    )
+
+
+def main(args, env, loggers):
+    return WebDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/24-mail.py

@@ -0,0 +1,298 @@
+#!/usr/bin/env python
+
+import os
+import dns.resolver
+import re
+
+from subprocess import CalledProcessError
+
+from moulinette.utils.process import check_output
+from moulinette.utils.filesystem import read_yaml
+
+from yunohost.diagnosis import Diagnoser
+from yunohost.domain import _get_maindomain, domain_list
+from yunohost.settings import settings_get
+from yunohost.utils.dns import dig
+
+DEFAULT_DNS_BLACKLIST = "/usr/share/yunohost/other/dnsbl_list.yml"
+
+
+class MailDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 600
+    dependencies = ["ip"]
+
+    def run(self):
+
+        self.ehlo_domain = _get_maindomain()
+        self.mail_domains = domain_list()["domains"]
+        self.ipversions, self.ips = self.get_ips_checked()
+
+        # TODO Is a A/AAAA and MX Record ?
+        # TODO Are outgoing public IPs authorized to send mail by SPF ?
+        # TODO Validate DKIM and dmarc ?
+        # TODO check that the recent mail logs are not filled with thousand of email sending (unusual number of mail sent)
+        # TODO check for unusual failed sending attempt being refused in the logs ?
+        checks = [
+            "check_outgoing_port_25",  # i18n: diagnosis_mail_outgoing_port_25_ok
+            "check_ehlo",  # i18n: diagnosis_mail_ehlo_ok
+            "check_fcrdns",  # i18n: diagnosis_mail_fcrdns_ok
+            "check_blacklist",  # i18n: diagnosis_mail_blacklist_ok
+            "check_queue",  # i18n: diagnosis_mail_queue_ok
+        ]
+        for check in checks:
+            self.logger_debug("Running " + check)
+            reports = list(getattr(self, check)())
+            for report in reports:
+                yield report
+            if not reports:
+                name = check[6:]
+                yield dict(
+                    meta={"test": "mail_" + name},
+                    status="SUCCESS",
+                    summary="diagnosis_mail_" + name + "_ok",
+                )
+
+    def check_outgoing_port_25(self):
+        """
+        Check outgoing port 25 is open and not blocked by router
+        This check is ran on IPs we could used to send mail.
+        """
+
+        for ipversion in self.ipversions:
+            cmd = "/bin/nc -{ipversion} -z -w2 yunohost.org 25".format(
+                ipversion=ipversion
+            )
+            if os.system(cmd) != 0:
+                yield dict(
+                    meta={"test": "outgoing_port_25", "ipversion": ipversion},
+                    data={},
+                    status="ERROR",
+                    summary="diagnosis_mail_outgoing_port_25_blocked",
+                    details=[
+                        "diagnosis_mail_outgoing_port_25_blocked_details",
+                        "diagnosis_mail_outgoing_port_25_blocked_relay_vpn",
+                    ],
+                )
+
+    def check_ehlo(self):
+        """
+        Check the server is reachable from outside and it's the good one
+        This check is ran on IPs we could used to send mail.
+        """
+
+        for ipversion in self.ipversions:
+            try:
+                r = Diagnoser.remote_diagnosis(
+                    "check-smtp", data={}, ipversion=ipversion
+                )
+            except Exception as e:
+                yield dict(
+                    meta={
+                        "test": "mail_ehlo",
+                        "reason": "remote_server_failed",
+                        "ipversion": ipversion,
+                    },
+                    data={"error": str(e)},
+                    status="WARNING",
+                    summary="diagnosis_mail_ehlo_could_not_diagnose",
+                    details=["diagnosis_mail_ehlo_could_not_diagnose_details"],
+                )
+                continue
+
+            if r["status"] != "ok":
+                # i18n: diagnosis_mail_ehlo_bad_answer
+                # i18n: diagnosis_mail_ehlo_bad_answer_details
+                # i18n: diagnosis_mail_ehlo_unreachable
+                # i18n: diagnosis_mail_ehlo_unreachable_details
+                summary = r["status"].replace("error_smtp_", "diagnosis_mail_ehlo_")
+                yield dict(
+                    meta={"test": "mail_ehlo", "ipversion": ipversion},
+                    data={},
+                    status="ERROR",
+                    summary=summary,
+                    details=[summary + "_details"],
+                )
+            elif r["helo"] != self.ehlo_domain:
+                yield dict(
+                    meta={"test": "mail_ehlo", "ipversion": ipversion},
+                    data={"wrong_ehlo": r["helo"], "right_ehlo": self.ehlo_domain},
+                    status="ERROR",
+                    summary="diagnosis_mail_ehlo_wrong",
+                    details=["diagnosis_mail_ehlo_wrong_details"],
+                )
+
+    def check_fcrdns(self):
+        """
+        Check the reverse DNS is well defined by doing a Forward-confirmed
+        reverse DNS check
+        This check is ran on IPs we could used to send mail.
+        """
+
+        for ip in self.ips:
+            if ":" in ip:
+                ipversion = 6
+                details = [
+                    "diagnosis_mail_fcrdns_nok_details",
+                    "diagnosis_mail_fcrdns_nok_alternatives_6",
+                ]
+            else:
+                ipversion = 4
+                details = [
+                    "diagnosis_mail_fcrdns_nok_details",
+                    "diagnosis_mail_fcrdns_nok_alternatives_4",
+                ]
+
+            rev = dns.reversename.from_address(ip)
+            subdomain = str(rev.split(3)[0])
+            query = subdomain
+            if ipversion == 4:
+                query += ".in-addr.arpa"
+            else:
+                query += ".ip6.arpa"
+
+            # Do the DNS Query
+            status, value = dig(query, "PTR", resolvers="force_external")
+            if status == "nok":
+                yield dict(
+                    meta={"test": "mail_fcrdns", "ipversion": ipversion},
+                    data={"ip": ip, "ehlo_domain": self.ehlo_domain},
+                    status="ERROR",
+                    summary="diagnosis_mail_fcrdns_dns_missing",
+                    details=details,
+                )
+                continue
+
+            rdns_domain = ""
+            if len(value) > 0:
+                rdns_domain = value[0][:-1] if value[0].endswith(".") else value[0]
+            if rdns_domain != self.ehlo_domain:
+                details = [
+                    "diagnosis_mail_fcrdns_different_from_ehlo_domain_details"
+                ] + details
+                yield dict(
+                    meta={"test": "mail_fcrdns", "ipversion": ipversion},
+                    data={
+                        "ip": ip,
+                        "ehlo_domain": self.ehlo_domain,
+                        "rdns_domain": rdns_domain,
+                    },
+                    status="ERROR",
+                    summary="diagnosis_mail_fcrdns_different_from_ehlo_domain",
+                    details=details,
+                )
+
+    def check_blacklist(self):
+        """
+        Check with dig onto blacklist DNS server
+        This check is ran on IPs and domains we could used to send mail.
+        """
+
+        dns_blacklists = read_yaml(DEFAULT_DNS_BLACKLIST)
+        for item in self.ips + self.mail_domains:
+            for blacklist in dns_blacklists:
+                item_type = "domain"
+                if ":" in item:
+                    item_type = "ipv6"
+                elif re.match(r"^\d+\.\d+\.\d+\.\d+$", item):
+                    item_type = "ipv4"
+
+                if not blacklist[item_type]:
+                    continue
+
+                # Build the query for DNSBL
+                subdomain = item
+                if item_type != "domain":
+                    rev = dns.reversename.from_address(item)
+                    subdomain = str(rev.split(3)[0])
+                query = subdomain + "." + blacklist["dns_server"]
+
+                # Do the DNS Query
+                status, _ = dig(query, "A")
+                if status != "ok":
+                    continue
+
+                # Try to get the reason
+                details = []
+                status, answers = dig(query, "TXT")
+                reason = "-"
+                if status == "ok":
+                    reason = ", ".join(answers)
+                    details.append("diagnosis_mail_blacklist_reason")
+
+                details.append("diagnosis_mail_blacklist_website")
+
+                yield dict(
+                    meta={
+                        "test": "mail_blacklist",
+                        "item": item,
+                        "blacklist": blacklist["dns_server"],
+                    },
+                    data={
+                        "blacklist_name": blacklist["name"],
+                        "blacklist_website": blacklist["website"],
+                        "reason": reason,
+                    },
+                    status="ERROR",
+                    summary="diagnosis_mail_blacklist_listed_by",
+                    details=details,
+                )
+
+    def check_queue(self):
+        """
+        Check mail queue is not filled with hundreds of email pending
+        """
+
+        command = (
+            'postqueue -p | grep -v "Mail queue is empty" | grep -c "^[A-Z0-9]" || true'
+        )
+        try:
+            output = check_output(command)
+            pending_emails = int(output)
+        except (ValueError, CalledProcessError) as e:
+            yield dict(
+                meta={"test": "mail_queue"},
+                data={"error": str(e)},
+                status="ERROR",
+                summary="diagnosis_mail_queue_unavailable",
+                details="diagnosis_mail_queue_unavailable_details",
+            )
+        else:
+            if pending_emails > 100:
+                yield dict(
+                    meta={"test": "mail_queue"},
+                    data={"nb_pending": pending_emails},
+                    status="WARNING",
+                    summary="diagnosis_mail_queue_too_big",
+                )
+            else:
+                yield dict(
+                    meta={"test": "mail_queue"},
+                    data={"nb_pending": pending_emails},
+                    status="SUCCESS",
+                    summary="diagnosis_mail_queue_ok",
+                )
+
+    def get_ips_checked(self):
+        outgoing_ipversions = []
+        outgoing_ips = []
+        ipv4 = Diagnoser.get_cached_report("ip", {"test": "ipv4"}) or {}
+        if ipv4.get("status") == "SUCCESS":
+            outgoing_ipversions.append(4)
+            global_ipv4 = ipv4.get("data", {}).get("global", {})
+            if global_ipv4:
+                outgoing_ips.append(global_ipv4)
+
+        if settings_get("smtp.allow_ipv6"):
+            ipv6 = Diagnoser.get_cached_report("ip", {"test": "ipv6"}) or {}
+            if ipv6.get("status") == "SUCCESS":
+                outgoing_ipversions.append(6)
+                global_ipv6 = ipv6.get("data", {}).get("global", {})
+                if global_ipv6:
+                    outgoing_ips.append(global_ipv6)
+        return (outgoing_ipversions, outgoing_ips)
+
+
+def main(args, env, loggers):
+    return MailDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/30-services.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+import os
+
+from yunohost.diagnosis import Diagnoser
+from yunohost.service import service_status
+
+
+class ServicesDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 300
+    dependencies = []
+
+    def run(self):
+
+        all_result = service_status()
+
+        for service, result in sorted(all_result.items()):
+
+            item = dict(
+                meta={"service": service},
+                data={
+                    "status": result["status"],
+                    "configuration": result["configuration"],
+                },
+            )
+
+            if result["status"] != "running":
+                item["status"] = "ERROR" if result["status"] != "unknown" else "WARNING"
+                item["summary"] = "diagnosis_services_bad_status"
+                item["details"] = ["diagnosis_services_bad_status_tip"]
+
+            elif result["configuration"] == "broken":
+                item["status"] = "WARNING"
+                item["summary"] = "diagnosis_services_conf_broken"
+                item["details"] = result["configuration-details"]
+
+            else:
+                item["status"] = "SUCCESS"
+                item["summary"] = "diagnosis_services_running"
+
+            yield item
+
+
+def main(args, env, loggers):
+    return ServicesDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/50-systemresources.py

@@ -0,0 +1,220 @@
+#!/usr/bin/env python
+import os
+import psutil
+import datetime
+import re
+
+from moulinette.utils.process import check_output
+
+from yunohost.diagnosis import Diagnoser
+
+
+class SystemResourcesDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 300
+    dependencies = []
+
+    def run(self):
+
+        MB = 1024 ** 2
+        GB = MB * 1024
+
+        #
+        # RAM
+        #
+
+        ram = psutil.virtual_memory()
+        ram_available_percent = 100 * ram.available / ram.total
+        item = dict(
+            meta={"test": "ram"},
+            data={
+                "total": human_size(ram.total),
+                "available": human_size(ram.available),
+                "available_percent": round_(ram_available_percent),
+            },
+        )
+
+        if ram.available < 100 * MB or ram_available_percent < 5:
+            item["status"] = "ERROR"
+            item["summary"] = "diagnosis_ram_verylow"
+        elif ram.available < 200 * MB or ram_available_percent < 10:
+            item["status"] = "WARNING"
+            item["summary"] = "diagnosis_ram_low"
+        else:
+            item["status"] = "SUCCESS"
+            item["summary"] = "diagnosis_ram_ok"
+        yield item
+
+        #
+        # Swap
+        #
+
+        swap = psutil.swap_memory()
+        item = dict(
+            meta={"test": "swap"},
+            data={"total": human_size(swap.total), "recommended": "512 MiB"},
+        )
+        if swap.total <= 1 * MB:
+            item["status"] = "INFO"
+            item["summary"] = "diagnosis_swap_none"
+        elif swap.total < 450 * MB:
+            item["status"] = "INFO"
+            item["summary"] = "diagnosis_swap_notsomuch"
+        else:
+            item["status"] = "SUCCESS"
+            item["summary"] = "diagnosis_swap_ok"
+        item["details"] = ["diagnosis_swap_tip"]
+        yield item
+
+        # FIXME : add a check that swapiness is low if swap is on a sdcard...
+
+        #
+        # Disks usage
+        #
+
+        disk_partitions = sorted(psutil.disk_partitions(), key=lambda k: k.mountpoint)
+
+        # Ignore /dev/loop stuff which are ~virtual partitions ? (e.g. mounted to /snap/)
+        disk_partitions = [
+            d
+            for d in disk_partitions
+            if d.mountpoint in ["/", "/var"] or not d.device.startswith("/dev/loop")
+        ]
+
+        for disk_partition in disk_partitions:
+            device = disk_partition.device
+            mountpoint = disk_partition.mountpoint
+
+            usage = psutil.disk_usage(mountpoint)
+            free_percent = 100 - round_(usage.percent)
+
+            item = dict(
+                meta={"test": "diskusage", "mountpoint": mountpoint},
+                data={
+                    "device": device,
+                    # N.B.: we do not use usage.total because we want
+                    # to take into account the 5% security margin
+                    # correctly (c.f. the doc of psutil ...)
+                    "total": human_size(usage.used + usage.free),
+                    "free": human_size(usage.free),
+                    "free_percent": free_percent,
+                },
+            )
+
+            # We have an additional absolute constrain on / and /var because
+            # system partitions are critical, having them full may prevent
+            # upgrades etc...
+            if free_percent < 2.5 or (
+                mountpoint in ["/", "/var"] and usage.free < 1 * GB
+            ):
+                item["status"] = "ERROR"
+                item["summary"] = "diagnosis_diskusage_verylow"
+            elif free_percent < 5 or (
+                mountpoint in ["/", "/var"] and usage.free < 2 * GB
+            ):
+                item["status"] = "WARNING"
+                item["summary"] = "diagnosis_diskusage_low"
+            else:
+                item["status"] = "SUCCESS"
+                item["summary"] = "diagnosis_diskusage_ok"
+
+            yield item
+
+        #
+        # Check for minimal space on / + /var
+        # because some stupid VPS provider only configure a stupidly
+        # low amount of disk space for the root partition
+        # which later causes issue when it gets full...
+        #
+
+        main_disk_partitions = [
+            d for d in disk_partitions if d.mountpoint in ["/", "/var"]
+        ]
+        main_space = sum(
+            [psutil.disk_usage(d.mountpoint).total for d in main_disk_partitions]
+        )
+        if main_space < 10 * GB:
+            yield dict(
+                meta={"test": "rootfstotalspace"},
+                data={"space": human_size(main_space)},
+                status="ERROR",
+                summary="diagnosis_rootfstotalspace_critical",
+            )
+        elif main_space < 14 * GB:
+            yield dict(
+                meta={"test": "rootfstotalspace"},
+                data={"space": human_size(main_space)},
+                status="WARNING",
+                summary="diagnosis_rootfstotalspace_warning",
+            )
+
+        #
+        # Recent kills by oom_reaper
+        #
+
+        kills_count = self.recent_kills_by_oom_reaper()
+        if kills_count:
+            kills_summary = "\n".join(
+                ["%s (x%s)" % (proc, count) for proc, count in kills_count]
+            )
+
+            yield dict(
+                meta={"test": "oom_reaper"},
+                status="WARNING",
+                summary="diagnosis_processes_killed_by_oom_reaper",
+                data={"kills_summary": kills_summary},
+            )
+
+    def recent_kills_by_oom_reaper(self):
+        if not os.path.exists("/var/log/kern.log"):
+            return []
+
+        def analyzed_kern_log():
+
+            cmd = 'tail -n 10000 /var/log/kern.log | grep "oom_reaper: reaped process" || true'
+            out = check_output(cmd)
+            lines = out.split("\n") if out else []
+
+            now = datetime.datetime.now()
+
+            for line in reversed(lines):
+                # Lines look like :
+                # Aug 25 18:48:21 yolo kernel: [ 9623.613667] oom_reaper: reaped process 11509 (uwsgi), now anon-rss:0kB, file-rss:0kB, shmem-rss:328kB
+                date_str = str(now.year) + " " + " ".join(line.split()[:3])
+                date = datetime.datetime.strptime(date_str, "%Y %b %d %H:%M:%S")
+                diff = now - date
+                if diff.days >= 1:
+                    break
+                process_killed = re.search(r"\(.*\)", line).group().strip("()")
+                yield process_killed
+
+        processes = list(analyzed_kern_log())
+        kills_count = [
+            (p, len([p_ for p_ in processes if p_ == p])) for p in set(processes)
+        ]
+        kills_count = sorted(kills_count, key=lambda p: p[1], reverse=True)
+
+        return kills_count
+
+
+def human_size(bytes_):
+    # Adapted from https://stackoverflow.com/a/1094933
+    for unit in ["", "ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
+        if abs(bytes_) < 1024.0:
+            return "%s %sB" % (round_(bytes_), unit)
+        bytes_ /= 1024.0
+    return "%s %sB" % (round_(bytes_), "Yi")
+
+
+def round_(n):
+    # round_(22.124) -> 22
+    # round_(9.45) -> 9.4
+    n = round(n, 1)
+    if n > 10:
+        n = int(round(n))
+    return n
+
+
+def main(args, env, loggers):
+    return SystemResourcesDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/70-regenconf.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+
+import os
+import re
+
+from yunohost.settings import settings_get
+from yunohost.diagnosis import Diagnoser
+from yunohost.regenconf import _get_regenconf_infos, _calculate_hash
+from moulinette.utils.filesystem import read_file
+
+
+class RegenconfDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 300
+    dependencies = []
+
+    def run(self):
+
+        regenconf_modified_files = list(self.manually_modified_files())
+
+        if not regenconf_modified_files:
+            yield dict(
+                meta={"test": "regenconf"},
+                status="SUCCESS",
+                summary="diagnosis_regenconf_allgood",
+            )
+        else:
+            for f in regenconf_modified_files:
+                yield dict(
+                    meta={
+                        "test": "regenconf",
+                        "category": f["category"],
+                        "file": f["path"],
+                    },
+                    status="WARNING",
+                    summary="diagnosis_regenconf_manually_modified",
+                    details=["diagnosis_regenconf_manually_modified_details"],
+                )
+
+        if (
+            any(f["path"] == "/etc/ssh/sshd_config" for f in regenconf_modified_files)
+            and os.system(
+                "grep -q '^ *AllowGroups\\|^ *AllowUsers' /etc/ssh/sshd_config"
+            )
+            != 0
+        ):
+            yield dict(
+                meta={"test": "sshd_config_insecure"},
+                status="ERROR",
+                summary="diagnosis_sshd_config_insecure",
+            )
+
+        # Check consistency between actual ssh port in sshd_config vs. setting
+        ssh_port_setting = settings_get("security.ssh.port")
+        ssh_port_line = re.findall(
+            r"\bPort *([0-9]{2,5})\b", read_file("/etc/ssh/sshd_config")
+        )
+        if len(ssh_port_line) == 1 and int(ssh_port_line[0]) != ssh_port_setting:
+            yield dict(
+                meta={"test": "sshd_config_port_inconsistency"},
+                status="WARNING",
+                summary="diagnosis_sshd_config_inconsistent",
+                details=["diagnosis_sshd_config_inconsistent_details"],
+            )
+
+    def manually_modified_files(self):
+
+        for category, infos in _get_regenconf_infos().items():
+            for path, hash_ in infos["conffiles"].items():
+                if hash_ != _calculate_hash(path):
+                    yield {"path": path, "category": category}
+
+
+def main(args, env, loggers):
+    return RegenconfDiagnoser(args, env, loggers).diagnose()

data/hooks/diagnosis/80-apps.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+
+import os
+
+from yunohost.app import app_list
+
+from yunohost.diagnosis import Diagnoser
+
+
+class AppDiagnoser(Diagnoser):
+
+    id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1]
+    cache_duration = 300
+    dependencies = []
+
+    def run(self):
+
+        apps = app_list(full=True)["apps"]
+        for app in apps:
+            app["issues"] = list(self.issues(app))
+
+        if not any(app["issues"] for app in apps):
+            yield dict(
+                meta={"test": "apps"},
+                status="SUCCESS",
+                summary="diagnosis_apps_allgood",
+            )
+        else:
+            for app in apps:
+
+                if not app["issues"]:
+                    continue
+
+                level = (
+                    "ERROR"
+                    if any(issue[0] == "error" for issue in app["issues"])
+                    else "WARNING"
+                )
+
+                yield dict(
+                    meta={"test": "apps", "app": app["name"]},
+                    status=level,
+                    summary="diagnosis_apps_issue",
+                    details=[issue[1] for issue in app["issues"]],
+                )
+
+    def issues(self, app):
+
+        # Check quality level in catalog
+
+        if not app.get("from_catalog") or app["from_catalog"].get("state") != "working":
+            yield ("error", "diagnosis_apps_not_in_app_catalog")
+        elif (
+            not isinstance(app["from_catalog"].get("level"), int)
+            or app["from_catalog"]["level"] == 0
+        ):
+            yield ("error", "diagnosis_apps_broken")
+        elif app["from_catalog"]["level"] <= 4:
+            yield ("warning", "diagnosis_apps_bad_quality")
+
+        # Check for super old, deprecated practices
+
+        yunohost_version_req = (
+            app["manifest"].get("requirements", {}).get("yunohost", "").strip(">= ")
+        )
+        if yunohost_version_req.startswith("2."):
+            yield ("error", "diagnosis_apps_outdated_ynh_requirement")
+
+        deprecated_helpers = [
+            "yunohost app setting",
+            "yunohost app checkurl",
+            "yunohost app checkport",
+            "yunohost app initdb",
+            "yunohost tools port-available",
+        ]
+        for deprecated_helper in deprecated_helpers:
+            if (
+                os.system(
+                    f"grep -hr '{deprecated_helper}' {app['setting_path']}/scripts/ | grep -v -q '^\s*#'"
+                )
+                == 0
+            ):
+                yield ("error", "diagnosis_apps_deprecated_practices")
+
+        old_arg_regex = r"^domain=\${?[0-9]"
+        if (
+            os.system(
+                f"grep -q '{old_arg_regex}' {app['setting_path']}/scripts/install"
+            )
+            == 0
+        ):
+            yield ("error", "diagnosis_apps_deprecated_practices")
+
+
+def main(args, env, loggers):
+    return AppDiagnoser(args, env, loggers).diagnose()

data/hooks/post_user_create/ynh_multimedia

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+user=$1
+ 
+readonly MEDIA_GROUP=multimedia
+readonly MEDIA_DIRECTORY=/home/yunohost.multimedia
+
+# We only do this if multimedia directory is enabled (= the folder exists)
+[ -e "$MEDIA_DIRECTORY" ] || exit 0
+
+mkdir -p "$MEDIA_DIRECTORY/$user"
+mkdir -p "$MEDIA_DIRECTORY/$user/Music"
+mkdir -p "$MEDIA_DIRECTORY/$user/Picture"
+mkdir -p "$MEDIA_DIRECTORY/$user/Video"
+mkdir -p "$MEDIA_DIRECTORY/$user/eBook"
+ln -sfn "$MEDIA_DIRECTORY/share" "$MEDIA_DIRECTORY/$user/Share"
+# Création du lien symbolique dans le home de l'utilisateur.
+#link will only be created if the home directory of the user exists and if it's located in '/home' folder
+user_home="$(getent passwd $user | cut -d: -f6 | grep '^/home/')"
+if [[ -d "$user_home" ]]; then
+    ln -sfn "$MEDIA_DIRECTORY/$user" "$user_home/Multimedia"
+fi
+# Propriétaires des dossiers utilisateurs.
+chown -R $user "$MEDIA_DIRECTORY/$user"
+
+## Application des droits étendus sur le dossier multimedia.
+# Droit d'écriture pour le groupe et le groupe multimedia en acl et droit de lecture pour other:
+setfacl -RnL -m g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$MEDIA_DIRECTORY/$user"
+# Application de la même règle que précédemment, mais par défaut pour les nouveaux fichiers.
+setfacl -RnL -m d:g:$MEDIA_GROUP:rwX,g::rwX,o:r-X "$MEDIA_DIRECTORY/$user"
+# Réglage du masque par défaut. Qui garantie (en principe...) un droit maximal à rwx. Donc pas de restriction de droits par l'acl.
+setfacl -RL -m m::rwx "$MEDIA_DIRECTORY/$user"

data/hooks/post_user_delete/ynh_multimedia

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+user=$1
+MEDIA_DIRECTORY=/home/yunohost.multimedia
+
+if [ -n "$user" ] && [ -e "$MEDIA_DIRECTORY/$user" ]; then
+    sudo rm -r "$MEDIA_DIRECTORY/$user"
+fi

data/hooks/restore/05-conf_ldap

@@ -1,58 +1,53 @@
+#!/bin/bash
+
 backup_dir="${1}/conf/ldap"
 
-if [[ $EUID -ne 0 ]]; then
+systemctl stop slapd
 
-    # We need to execute this script as root, since the ldap
-    # service will be shut down during the operation (and sudo
-    # won't be available)
-    sudo /bin/bash $(readlink -f $0) $1
+# Create a directory for backup
+TMPDIR="/tmp/$(date +%s)"
+mkdir -p "$TMPDIR"
 
-else
+die() {
+    state=$1
+    error=$2
 
-    service slapd stop || true
+    # Restore saved configuration and database
+    [[ $state -ge 1 ]] \
+      && (rm -rf /etc/ldap/slapd.d &&
+          mv "${TMPDIR}/slapd.d" /etc/ldap/slapd.d)
+    [[ $state -ge 2 ]] \
+      && (rm -rf /var/lib/ldap &&
+          mv "${TMPDIR}/ldap" /var/lib/ldap)
+    chown -R openldap: /etc/ldap/slapd.d /var/lib/ldap
 
-    # Create a directory for backup
-    TMPDIR="/tmp/$(date +%s)"
-    mkdir -p "$TMPDIR"
-
-    die() {
-        state=$1
-        error=$2
-
-        # Restore saved configuration and database
-        [[ $state -ge 1 ]] \
-          && (rm -rf /etc/ldap/slapd.d &&
-              mv "${TMPDIR}/slapd.d" /etc/ldap/slapd.d)
-        [[ $state -ge 2 ]] \
-          && (rm -rf /var/lib/ldap &&
-              mv "${TMPDIR}/ldap" /var/lib/ldap)
-        chown -R openldap: /etc/ldap/slapd.d /var/lib/ldap
-
-        service slapd start
-        rm -rf "$TMPDIR"
-
-        # Print an error message and exit
-        printf "%s" "$error" 1>&2
-        exit 1
-    }
-
-    # Restore the configuration
-    mv /etc/ldap/slapd.d "$TMPDIR"
-    mkdir -p /etc/ldap/slapd.d
-    cp -a "${backup_dir}/slapd.conf" /etc/ldap/slapd.conf
-    slapadd -F /etc/ldap/slapd.d -b cn=config \
-        -l "${backup_dir}/cn=config.master.ldif" \
-      || die 1 "Unable to restore LDAP configuration"
-    chown -R openldap: /etc/ldap/slapd.d
-
-    # Restore the database
-    mv /var/lib/ldap "$TMPDIR"
-    mkdir -p /var/lib/ldap
-    slapadd -F /etc/ldap/slapd.d -b dc=yunohost,dc=org \
-        -l "${backup_dir}/dc=yunohost-dc=org.ldif" \
-      || die 2 "Unable to restore LDAP database"
-    chown -R openldap: /var/lib/ldap
-
-    service slapd start
+    systemctl start slapd
     rm -rf "$TMPDIR"
-fi
+
+    # Print an error message and exit
+    printf "%s" "$error" 1>&2
+    exit 1
+}
+
+# Restore the configuration
+mv /etc/ldap/slapd.d "$TMPDIR"
+mkdir -p /etc/ldap/slapd.d
+cp -a "${backup_dir}/ldap.conf" /etc/ldap/ldap.conf
+# Legacy thing but we need it to force the regen-conf in case of it exist
+[ ! -e "${backup_dir}/slapd.conf" ] \
+    || cp -a "${backup_dir}/slapd.conf" /etc/ldap/slapd.conf
+slapadd -F /etc/ldap/slapd.d -b cn=config \
+    -l "${backup_dir}/cn=config.master.ldif" \
+  || die 1 "Unable to restore LDAP configuration"
+chown -R openldap: /etc/ldap/slapd.d
+
+# Restore the database
+mv /var/lib/ldap "$TMPDIR"
+mkdir -p /var/lib/ldap
+slapadd -F /etc/ldap/slapd.d -b dc=yunohost,dc=org \
+    -l "${backup_dir}/dc=yunohost-dc=org.ldif" \
+  || die 2 "Unable to restore LDAP database"
+chown -R openldap: /var/lib/ldap
+
+systemctl start slapd
+rm -rf "$TMPDIR"

data/hooks/restore/08-conf_ssh

@@ -1,9 +0,0 @@
-backup_dir="$1/conf/ssh"
-
-if [ -d /etc/ssh/ ]; then
-    sudo cp -a $backup_dir/. /etc/ssh
-    sudo service ssh restart
-else
-    echo "SSH is not installed"
-fi
-

data/hooks/restore/11-conf_ynh_mysql

@@ -1,42 +0,0 @@
-backup_dir="$1/conf/ynh/mysql"
-MYSQL_PKG="$(dpkg --list | sed -ne 's/^ii  \(mariadb-server-[[:digit:].]\+\) .*$/\1/p')"
-
-. /usr/share/yunohost/helpers
-
-# ensure that mysql is running
-service mysql status >/dev/null 2>&1 \
-  || service mysql start
-
-# retrieve current and new password
-[ -f /etc/yunohost/mysql ] \
-  && curr_pwd=$(sudo cat /etc/yunohost/mysql)
-new_pwd=$(sudo cat "${backup_dir}/root_pwd" || sudo cat "${backup_dir}/mysql")
-[ -z "$curr_pwd" ] && curr_pwd="yunohost"
-[ -z "$new_pwd" ] && {
-    new_pwd=$(ynh_string_random 10)
-}
-
-# attempt to change it
-sudo mysqladmin -s -u root -p"$curr_pwd" password "$new_pwd" || {
-
-  echo "It seems that you have already configured MySQL." \
-    "YunoHost needs to have a root access to MySQL to runs its" \
-    "applications, and is going to reset the MySQL root password." \
-    "You can find this new password in /etc/yunohost/mysql." >&2
-
-  # set new password with debconf
-  sudo debconf-set-selections << EOF
-$MYSQL_PKG mysql-server/root_password password $new_pwd
-$MYSQL_PKG mysql-server/root_password_again password $new_pwd
-EOF
-
-  # reconfigure Debian package
-  sudo dpkg-reconfigure -freadline -u "$MYSQL_PKG" 2>&1
-}
-
-# store new root password
-echo "$new_pwd" | sudo tee /etc/yunohost/mysql
-sudo chmod 400 /etc/yunohost/mysql
-
-# reload the grant tables
-sudo mysqladmin -s -u root -p"$new_pwd" reload

data/hooks/restore/14-conf_ssowat

@@ -1,3 +0,0 @@
-backup_dir="$1/conf/ssowat"
-
-sudo cp -a $backup_dir/. /etc/ssowat

data/hooks/restore/17-data_home

@@ -1,3 +1,3 @@
 backup_dir="$1/data/home"
 
-sudo cp -a $backup_dir/. /home
+cp -a $backup_dir/. /home

data/hooks/restore/18-data_multimedia

@@ -6,8 +6,4 @@ set -eu
 # Source YNH helpers
 source /usr/share/yunohost/helpers
 
-# Backup destination
-backup_dir="${1}/conf/ssowat"
-
-# Backup the configuration
-ynh_backup "/etc/ssowat" "$backup_dir"
+ynh_restore_file --origin_path="/home/yunohost.multimedia" --not_mandatory

data/hooks/restore/20-conf_ynh_firewall

@@ -1,4 +0,0 @@
-backup_dir="$1/conf/ynh/firewall"
-
-sudo cp -a $backup_dir/. /etc/yunohost
-sudo yunohost firewall reload

data/hooks/restore/20-conf_ynh_settings

@@ -0,0 +1,8 @@
+backup_dir="$1/conf/ynh"
+
+cp -a "${backup_dir}/current_host" /etc/yunohost/current_host
+cp -a "${backup_dir}/firewall.yml" /etc/yunohost/firewall.yml
+cp -a "${backup_dir}/domains" /etc/yunohost/domains
+[ ! -e "${backup_dir}/settings.json" ] || cp -a "${backup_dir}/settings.json" "/etc/yunohost/settings.json"
+[ ! -d "${backup_dir}/dyndns" ] || cp -raT "${backup_dir}/dyndns" "/etc/yunohost/dyndns"
+[ ! -d "${backup_dir}/dkim" ] || cp -raT "${backup_dir}/dkim" "/etc/dkim"

data/hooks/restore/21-conf_ynh_certs

@@ -1,8 +1,5 @@
 backup_dir="$1/conf/ynh/certs"
 
-sudo mkdir -p /etc/yunohost/certs/
+mkdir -p /etc/yunohost/certs/
 
-sudo cp -a $backup_dir/. /etc/yunohost/certs/
-sudo yunohost app ssowatconf
-sudo service nginx reload
-sudo service metronome reload
+cp -a $backup_dir/. /etc/yunohost/certs/

data/hooks/restore/23-data_mail

@@ -1,8 +1,4 @@
 backup_dir="$1/data/mail"
 
-sudo cp -a $backup_dir/. /var/mail/ || echo 'No mail found'
-sudo chown -R vmail:mail /var/mail/
-
-# Restart services to use migrated certs
-sudo service postfix restart
-sudo service dovecot restart
+cp -a $backup_dir/. /var/mail/ || echo 'No mail found'
+chown -R vmail:mail /var/mail/

data/hooks/restore/26-conf_xmpp

@@ -1,7 +0,0 @@
-backup_dir="$1/conf/xmpp"
-
-sudo cp -a $backup_dir/etc/. /etc/metronome
-sudo cp -a $backup_dir/var/. /var/lib/metronome
-
-# Restart to apply new conf and certs
-sudo service metronome restart

data/hooks/restore/27-data_xmpp

@@ -0,0 +1,4 @@
+backup_dir="$1/data/xmpp"
+
+cp -a $backup_dir/var_lib_metronome/. /var/lib/metronome
+cp -a $backup_dir/var_xmpp-upload/. /var/xmpp-upload