diff --git a/bin/yunohost b/bin/yunohost index 10a21a9da..b640c8c52 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -179,6 +179,10 @@ def _retrieve_namespaces(): ret.append(n) return ret +# Stupid PATH management because sometimes (e.g. some cron job) PATH is only /usr/bin:/bin ... +default_path = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +if os.environ["PATH"] != default_path: + os.environ["PATH"] = default_path + ":" + os.environ["PATH"] # Main action ---------------------------------------------------------- diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 3a4c9db97..44419a342 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -296,6 +296,13 @@ user: help: Display all info known about each permission, including the full user list of each group it is granted to. action: store_true + ### user_permission_info() + info: + action_help: Get information about a specific permission + api: GET /users/permissions/ + arguments: + permission: + help: Name of the permission to fetch info about ### user_permission_update() update: @@ -1452,6 +1459,11 @@ tools: help: List pending configuration files and exit action: store_true + ### tools_versions() + versions: + action_help: Display YunoHost's packages versions + api: GET /versions + subcategories: migrations: diff --git a/data/actionsmap/yunohost_completion.py b/data/actionsmap/yunohost_completion.py index a4c17c4d6..45d15f16c 100644 --- a/data/actionsmap/yunohost_completion.py +++ b/data/actionsmap/yunohost_completion.py @@ -3,7 +3,7 @@ Simple automated generation of a bash_completion file for yunohost command from the actionsmap. Generates a bash completion file assuming the structure -`yunohost domain action` +`yunohost category action` adds `--help` at the end if one presses [tab] again. author: Christophe Vuillot @@ -15,18 +15,39 @@ THIS_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) ACTIONSMAP_FILE = THIS_SCRIPT_DIR + '/yunohost.yml' BASH_COMPLETION_FILE = THIS_SCRIPT_DIR + '/../bash-completion.d/yunohost' +def get_dict_actions(OPTION_SUBTREE, category): + ACTIONS = [action for action in OPTION_SUBTREE[category]["actions"].keys() + if not action.startswith('_')] + ACTIONS_STR = '{}'.format(' '.join(ACTIONS)) + + DICT = { "actions_str": ACTIONS_STR } + + return DICT + with open(ACTIONSMAP_FILE, 'r') as stream: - # Getting the dictionary containning what actions are possible per domain + # Getting the dictionary containning what actions are possible per category OPTION_TREE = yaml.load(stream) - DOMAINS = [str for str in OPTION_TREE.keys() if not str.startswith('_')] - DOMAINS_STR = '"{}"'.format(' '.join(DOMAINS)) + + CATEGORY = [category for category in OPTION_TREE.keys() if not category.startswith('_')] + + CATEGORY_STR = '{}'.format(' '.join(CATEGORY)) ACTIONS_DICT = {} - for domain in DOMAINS: - ACTIONS = [str for str in OPTION_TREE[domain]['actions'].keys() - if not str.startswith('_')] - ACTIONS_STR = '"{}"'.format(' '.join(ACTIONS)) - ACTIONS_DICT[domain] = ACTIONS_STR + for category in CATEGORY: + ACTIONS_DICT[category] = get_dict_actions(OPTION_TREE, category) + + ACTIONS_DICT[category]["subcategories"] = {} + ACTIONS_DICT[category]["subcategories_str"] = "" + + if "subcategories" in OPTION_TREE[category].keys(): + SUBCATEGORIES = [ subcategory for subcategory in OPTION_TREE[category]["subcategories"].keys() ] + + SUBCATEGORIES_STR = '{}'.format(' '.join(SUBCATEGORIES)) + + ACTIONS_DICT[category]["subcategories_str"] = SUBCATEGORIES_STR + + for subcategory in SUBCATEGORIES: + ACTIONS_DICT[category]["subcategories"][subcategory] = get_dict_actions(OPTION_TREE[category]["subcategories"], subcategory) with open(BASH_COMPLETION_FILE, 'w') as generated_file: @@ -47,31 +68,49 @@ with open(ACTIONSMAP_FILE, 'r') as stream: generated_file.write('\tnarg=${#COMP_WORDS[@]}\n\n') generated_file.write('\t# the current word being typed\n') generated_file.write('\tcur="${COMP_WORDS[COMP_CWORD]}"\n\n') - generated_file.write('\t# the last typed word\n') - generated_file.write('\tprev="${COMP_WORDS[COMP_CWORD-1]}"\n\n') - # If one is currently typing a domain then match with the domain list - generated_file.write('\t# If one is currently typing a domain,\n') - generated_file.write('\t# match with domains\n') + # If one is currently typing a category then match with the category list + generated_file.write('\t# If one is currently typing a category,\n') + generated_file.write('\t# match with categorys\n') generated_file.write('\tif [[ $narg == 2 ]]; then\n') - generated_file.write('\t\topts={}\n'.format(DOMAINS_STR)) + generated_file.write('\t\topts="{}"\n'.format(CATEGORY_STR)) generated_file.write('\tfi\n\n') # If one is currently typing an action then match with the action list - # of the previously typed domain - generated_file.write('\t# If one already typed a domain,\n') - generated_file.write('\t# match the actions of that domain\n') + # of the previously typed category + generated_file.write('\t# If one already typed a category,\n') + generated_file.write('\t# match the actions or the subcategories of that category\n') generated_file.write('\tif [[ $narg == 3 ]]; then\n') - for domain in DOMAINS: - generated_file.write('\t\tif [[ $prev == "{}" ]]; then\n'.format(domain)) - generated_file.write('\t\t\topts={}\n'.format(ACTIONS_DICT[domain])) + generated_file.write('\t\t# the category typed\n') + generated_file.write('\t\tcategory="${COMP_WORDS[1]}"\n\n') + for category in CATEGORY: + generated_file.write('\t\tif [[ $category == "{}" ]]; then\n'.format(category)) + generated_file.write('\t\t\topts="{} {}"\n'.format(ACTIONS_DICT[category]["actions_str"], ACTIONS_DICT[category]["subcategories_str"])) generated_file.write('\t\tfi\n') generated_file.write('\tfi\n\n') - # If both domain and action have been typed or the domain + generated_file.write('\t# If one already typed an action or a subcategory,\n') + generated_file.write('\t# match the actions of that subcategory\n') + generated_file.write('\tif [[ $narg == 4 ]]; then\n') + generated_file.write('\t\t# the category typed\n') + generated_file.write('\t\tcategory="${COMP_WORDS[1]}"\n\n') + generated_file.write('\t\t# the action or the subcategory typed\n') + generated_file.write('\t\taction_or_subcategory="${COMP_WORDS[2]}"\n\n') + for category in CATEGORY: + if len(ACTIONS_DICT[category]["subcategories"]): + generated_file.write('\t\tif [[ $category == "{}" ]]; then\n'.format(category)) + for subcategory in ACTIONS_DICT[category]["subcategories"]: + generated_file.write('\t\t\tif [[ $action_or_subcategory == "{}" ]]; then\n'.format(subcategory)) + generated_file.write('\t\t\t\topts="{}"\n'.format(ACTIONS_DICT[category]["subcategories"][subcategory]["actions_str"])) + generated_file.write('\t\t\tfi\n') + generated_file.write('\t\tfi\n') + generated_file.write('\tfi\n\n') + + # If both category and action have been typed or the category # was not recognized propose --help (only once) generated_file.write('\t# If no options were found propose --help\n') generated_file.write('\tif [ -z "$opts" ]; then\n') + generated_file.write('\t\tprev="${COMP_WORDS[COMP_CWORD-1]}"\n\n') generated_file.write('\t\tif [[ $prev != "--help" ]]; then\n') generated_file.write('\t\t\topts=( --help )\n') generated_file.write('\t\tfi\n') diff --git a/data/helpers.d/apt b/data/helpers.d/apt index 7859d44c5..bcce02dcb 100644 --- a/data/helpers.d/apt +++ b/data/helpers.d/apt @@ -189,7 +189,16 @@ ynh_package_install_from_equivs () { # If install fails we use "apt-get check" to try to debug and diagnose possible unmet dependencies # Note the use of { } which allows to group commands without starting a subshell (otherwise the ynh_die wouldn't exit the current shell). # Be careful with the syntax : the semicolon + space at the end is important! - ynh_package_install -f || { apt-get check 2>&1; ynh_die --message="Unable to install dependencies"; } + + ynh_package_install -f || \ + { # If the installation failed + # Get the list of dependencies from the deb + local dependencies="$(dpkg --info "$TMPDIR/${pkgname}_${pkgversion}_all.deb" | grep Depends | \ + sed 's/^ Depends: //' | sed 's/,//g')" + # Fake an install of those dependencies to see the errors + # The sed command here is, Print only from '--fix-broken' to the end. + ynh_package_install $dependencies --dry-run | sed -n '/--fix-broken/,$p' >&2 + ynh_die --message="Unable to install dependencies"; } [[ -n "$TMPDIR" ]] && rm -rf $TMPDIR # Remove the temp dir. # check if the package is actually installed @@ -208,7 +217,8 @@ ynh_package_install_from_equivs () { # Requires YunoHost version 2.6.4 or higher. ynh_install_app_dependencies () { local dependencies=$@ - local dependencies=${dependencies// /, } + # Add a comma for each space between packages. But not add a comma if the space separate a version specification. (See below) + dependencies="$(echo "$dependencies" | sed 's/\([^\<=\>]\)\ \([^(]\)/\1, \2/g')" local dependencies=${dependencies//|/ | } local manifest_path="../manifest.json" if [ ! -e "$manifest_path" ]; then @@ -221,6 +231,20 @@ ynh_install_app_dependencies () { fi local dep_app=${app//_/-} # Replace all '_' by '-' + # Handle specific versions + if [[ "$dependencies" =~ [\<=\>] ]] + then + # Replace version specifications by relationships syntax + # https://www.debian.org/doc/debian-policy/ch-relationships.html + # Sed clarification + # [^(\<=\>] ignore if it begins by ( or < = >. To not apply twice. + # [\<=\>] matches < = or > + # \+ matches one or more occurence of the previous characters, for >= or >>. + # [^,]\+ matches all characters except ',' + # Ex: 'package>=1.0' will be replaced by 'package (>= 1.0)' + dependencies="$(echo "$dependencies" | sed 's/\([^(\<=\>]\)\([\<=\>]\+\)\([^,]\+\)/\1 (\2 \3)/g')" + fi + # # Epic ugly hack to fix the goddamn dependency nightmare of sury # Sponsored by the "Djeezusse Fokin Kraiste Why Do Adminsys Has To Be So Fucking Complicated I Should Go Grow Potatoes Instead Of This Shit" collective @@ -236,8 +260,11 @@ ynh_install_app_dependencies () { if ! grep -nrq "sury" /etc/apt/sources.list* then # Re-add sury - echo "deb https://packages.sury.org/php/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/sury.list - wget -O /etc/apt/trusted.gpg.d/sury.gpg https://packages.sury.org/php/apt.gpg + ynh_install_extra_repo --repo="https://packages.sury.org/php/ $(lsb_release -sc) main" --key="https://packages.sury.org/php/apt.gpg" --name=extra_php_version + + # Pin this sury repository to prevent sury of doing shit + ynh_pin_repo --package="*" --pin="origin \"packages.sury.org\"" --priority=200 --name=extra_php_version + ynh_pin_repo --package="php${$YNH_DEFAULT_PHP_VERSION}*" --pin="origin \"packages.sury.org\"" --priority=600 --name=extra_php_version --append fi fi fi @@ -258,6 +285,38 @@ EOF ynh_app_setting_set --app=$app --key=apt_dependencies --value="$dependencies" } +# Add dependencies to install with ynh_install_app_dependencies +# +# [internal] +# +# usage: ynh_add_app_dependencies --package=phpversion [--replace] +# | arg: -p, --package - Packages to add as dependencies for the app. +# | arg: -r, --replace - Replace dependencies instead of adding to existing ones. +ynh_add_app_dependencies () { + # Declare an array to define the options of this helper. + local legacy_args=pr + declare -Ar args_array=( [p]=package= [r]=replace) + local package + local replace + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + replace=${replace:-0} + + local current_dependencies="" + if [ $replace -eq 0 ] + then + local dep_app=${app//_/-} # Replace all '_' by '-' + if ynh_package_is_installed --package="${dep_app}-ynh-deps" + then + current_dependencies="$(dpkg-query --show --showformat='${Depends}' ${dep_app}-ynh-deps) " + fi + + current_dependencies=${current_dependencies// | /|} + fi + + ynh_install_app_dependencies "${current_dependencies}${package}" +} + # Remove fake package and its dependencies # # Dependencies will removed only if no other package need them. @@ -269,3 +328,223 @@ ynh_remove_app_dependencies () { local dep_app=${app//_/-} # Replace all '_' by '-' ynh_package_autopurge ${dep_app}-ynh-deps # Remove the fake package and its dependencies if they not still used. } + +#================================================= + +# Install packages from an extra repository properly. +# +# usage: ynh_install_extra_app_dependencies --repo="repo" --package="dep1 dep2" [--key=key_url] [--name=name] +# | arg: -r, --repo - Complete url of the extra repository. +# | arg: -p, --package - The packages to install from this extra repository +# | arg: -k, --key - url to get the public key. +# | arg: -n, --name - Name for the files for this repo, $app as default value. +ynh_install_extra_app_dependencies () { + # Declare an array to define the options of this helper. + local legacy_args=rpkn + declare -Ar args_array=( [r]=repo= [p]=package= [k]=key= [n]=name= ) + local repo + local package + local key + local name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + name="${name:-$app}" + key=${key:-} + + # Set a key only if asked + if [ -n "$key" ] + then + key="--key=$key" + fi + # Add an extra repository for those packages + ynh_install_extra_repo --repo="$repo" $key --priority=995 --name=$name + + # Install requested dependencies from this extra repository. + ynh_add_app_dependencies --package="$package" + + # Remove this extra repository after packages are installed + ynh_remove_extra_repo --name=$app +} + +# Add an extra repository correctly, pin it and get the key. +# +# [internal] +# +# usage: ynh_install_extra_repo --repo="repo" [--key=key_url] [--priority=priority_value] [--name=name] [--append] +# | arg: -r, --repo - Complete url of the extra repository. +# | arg: -k, --key - url to get the public key. +# | arg: -p, --priority - Priority for the pin +# | arg: -n, --name - Name for the files for this repo, $app as default value. +# | arg: -a, --append - Do not overwrite existing files. +ynh_install_extra_repo () { + # Declare an array to define the options of this helper. + local legacy_args=rkpna + declare -Ar args_array=( [r]=repo= [k]=key= [p]=priority= [n]=name= [a]=append ) + local repo + local key + local priority + local name + local append + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + name="${name:-$app}" + append=${append:-0} + key=${key:-} + priority=${priority:-} + + if [ $append -eq 1 ] + then + append="--append" + wget_append="tee -a" + else + append="" + wget_append="tee" + fi + + # Split the repository into uri, suite and components. + # Remove "deb " at the beginning of the repo. + repo="${repo#deb }" + + # Get the uri + local uri="$(echo "$repo" | awk '{ print $1 }')" + + # Get the suite + local suite="$(echo "$repo" | awk '{ print $2 }')" + + # Get the components + local component="${repo##$uri $suite }" + + # Add the repository into sources.list.d + ynh_add_repo --uri="$uri" --suite="$suite" --component="$component" --name="$name" $append + + # Pin the new repo with the default priority, so it won't be used for upgrades. + # Build $pin from the uri without http and any sub path + local pin="${uri#*://}" + pin="${pin%%/*}" + # Set a priority only if asked + if [ -n "$priority" ] + then + priority="--priority=$priority" + fi + ynh_pin_repo --package="*" --pin="origin \"$pin\"" $priority --name="$name" $append + + # Get the public key for the repo + if [ -n "$key" ] + then + mkdir -p "/etc/apt/trusted.gpg.d" + wget -q "$key" -O - | gpg --dearmor | $wget_append /etc/apt/trusted.gpg.d/$name.gpg > /dev/null + fi + + # Update the list of package with the new repo + ynh_package_update +} + +# Remove an extra repository and the assiociated configuration. +# +# [internal] +# +# usage: ynh_remove_extra_repo [--name=name] +# | arg: -n, --name - Name for the files for this repo, $app as default value. +ynh_remove_extra_repo () { + # Declare an array to define the options of this helper. + local legacy_args=n + declare -Ar args_array=( [n]=name= ) + local name + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + name="${name:-$app}" + + ynh_secure_remove "/etc/apt/sources.list.d/$name.list" + ynh_secure_remove "/etc/apt/preferences.d/$name" + ynh_secure_remove "/etc/apt/trusted.gpg.d/$name.gpg" + ynh_secure_remove "/etc/apt/trusted.gpg.d/$name.asc" + + # Update the list of package to exclude the old repo + ynh_package_update +} + +# Add a repository. +# +# [internal] +# +# usage: ynh_add_repo --uri=uri --suite=suite --component=component [--name=name] [--append] +# | arg: -u, --uri - Uri of the repository. +# | arg: -s, --suite - Suite of the repository. +# | arg: -c, --component - Component of the repository. +# | arg: -n, --name - Name for the files for this repo, $app as default value. +# | arg: -a, --append - Do not overwrite existing files. +# +# Example for a repo like deb http://forge.yunohost.org/debian/ stretch stable +# uri suite component +# ynh_add_repo --uri=http://forge.yunohost.org/debian/ --suite=stretch --component=stable +# +ynh_add_repo () { + # Declare an array to define the options of this helper. + local legacy_args=uscna + declare -Ar args_array=( [u]=uri= [s]=suite= [c]=component= [n]=name= [a]=append ) + local uri + local suite + local component + local name + local append + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + name="${name:-$app}" + append=${append:-0} + + if [ $append -eq 1 ] + then + append="tee -a" + else + append="tee" + fi + + mkdir -p "/etc/apt/sources.list.d" + # Add the new repo in sources.list.d + echo "deb $uri $suite $component" \ + | $append "/etc/apt/sources.list.d/$name.list" +} + +# Pin a repository. +# +# [internal] +# +# usage: ynh_pin_repo --package=packages --pin=pin_filter [--priority=priority_value] [--name=name] [--append] +# | arg: -p, --package - Packages concerned by the pin. Or all, *. +# | arg: -i, --pin - Filter for the pin. +# | arg: -p, --priority - Priority for the pin +# | arg: -n, --name - Name for the files for this repo, $app as default value. +# | arg: -a, --append - Do not overwrite existing files. +# +# See https://manpages.debian.org/stretch/apt/apt_preferences.5.en.html#How_APT_Interprets_Priorities for information about pinning. +# +ynh_pin_repo () { + # Declare an array to define the options of this helper. + local legacy_args=pirna + declare -Ar args_array=( [p]=package= [i]=pin= [r]=priority= [n]=name= [a]=append ) + local package + local pin + local priority + local name + local append + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + package="${package:-*}" + priority=${priority:-50} + name="${name:-$app}" + append=${append:-0} + + if [ $append -eq 1 ] + then + append="tee -a" + else + append="tee" + fi + + mkdir -p "/etc/apt/preferences.d" + echo "Package: $package +Pin: $pin +Pin-Priority: $priority +" \ + | $append "/etc/apt/preferences.d/$name" +} diff --git a/data/helpers.d/hardware b/data/helpers.d/hardware new file mode 100644 index 000000000..46e27caf4 --- /dev/null +++ b/data/helpers.d/hardware @@ -0,0 +1,102 @@ +#!/bin/bash + +# Get the total or free amount of RAM+swap on the system +# +# usage: ynh_get_ram [--free|--total] [--ignore_swap|--only_swap] +# | arg: -f, --free - Count free RAM+swap +# | arg: -t, --total - Count total RAM+swap +# | arg: -s, --ignore_swap - Ignore swap, consider only real RAM +# | arg: -o, --only_swap - Ignore real RAM, consider only swap +ynh_get_ram () { + # Declare an array to define the options of this helper. + local legacy_args=ftso + declare -Ar args_array=( [f]=free [t]=total [s]=ignore_swap [o]=only_swap ) + local free + local total + local ignore_swap + local only_swap + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + ignore_swap=${ignore_swap:-0} + only_swap=${only_swap:-0} + free=${free:-0} + total=${total:-0} + + local total_ram=$(vmstat --stats --unit M | grep "total memory" | awk '{print $1}') + local total_swap=$(vmstat --stats --unit M | grep "total swap" | awk '{print $1}') + local total_ram_swap=$(( total_ram + total_swap )) + + local free_ram=$(vmstat --stats --unit M | grep "free memory" | awk '{print $1}') + local free_swap=$(vmstat --stats --unit M | grep "free swap" | awk '{print $1}') + local free_ram_swap=$(( free_ram + free_swap )) + + # Use the total amount of ram + if [ $free -eq 1 ] + then + # Use the total amount of free ram + local ram=$free_ram_swap + if [ $ignore_swap -eq 1 ] + then + # Use only the amount of free ram + ram=$free_ram + elif [ $only_swap -eq 1 ] + then + # Use only the amount of free swap + ram=$free_swap + fi + elif [ $total -eq 1 ] + then + local ram=$total_ram_swap + if [ $ignore_swap -eq 1 ] + then + # Use only the amount of free ram + ram=$total_ram + elif [ $only_swap -eq 1 ] + then + # Use only the amount of free swap + ram=$total_swap + fi + else + ynh_print_warn --message="You have to choose --free or --total when using ynh_get_ram" + ram=0 + fi + + echo $ram +} + +# Return 0 or 1 depending if the system has a given amount of RAM+swap free or total +# +# usage: ynh_require_ram --required=RAM required in Mb [--free|--total] [--ignore_swap|--only_swap] +# | arg: -r, --required - The amount to require, in Mb +# | arg: -f, --free - Count free RAM+swap +# | arg: -t, --total - Count total RAM+swap +# | arg: -s, --ignore_swap - Ignore swap, consider only real RAM +# | arg: -o, --only_swap - Ignore real RAM, consider only swap +ynh_require_ram () { + # Declare an array to define the options of this helper. + local legacy_args=rftso + declare -Ar args_array=( [r]=required= [f]=free [t]=total [s]=ignore_swap [o]=only_swap ) + local required + local free + local total + local ignore_swap + local only_swap + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + # Dunno if that's the right way to do, but that's some black magic to be able to + # forward the bool args to ynh_get_ram easily? + # If the variable $free is not empty, set it to '--free' + free=${free:+--free} + total=${total:+--total} + ignore_swap=${ignore_swap:+--ignore_swap} + only_swap=${only_swap:+--only_swap} + + local ram=$(ynh_get_ram $free $total $ignore_swap $only_swap) + + if [ $ram -lt $required ] + then + return 1 + else + return 0 + fi +} diff --git a/data/helpers.d/nginx b/data/helpers.d/nginx index e3e45d2d4..b34ebb4e1 100644 --- a/data/helpers.d/nginx +++ b/data/helpers.d/nginx @@ -12,6 +12,7 @@ # __PORT__ by $port # __NAME__ by $app # __FINALPATH__ by $final_path +# __PHPVERSION__ by $YNH_PHP_VERSION ($YNH_PHP_VERSION is either the default php version or the version defined for the app) # # And dynamic variables (from the last example) : # __PATH_2__ by $path_2 @@ -44,6 +45,7 @@ ynh_add_nginx_config () { if test -n "${final_path:-}"; then ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalnginxconf" fi + ynh_replace_string --match_string="__PHPVERSION__" --replace_string="$YNH_PHP_VERSION" --target_file="$finalnginxconf" # Replace all other variable given as arguments for var_to_replace in $others_var diff --git a/data/helpers.d/php b/data/helpers.d/php index 7aefc697e..b479747c6 100644 --- a/data/helpers.d/php +++ b/data/helpers.d/php @@ -1,23 +1,102 @@ #!/bin/bash +YNH_DEFAULT_PHP_VERSION=7.0 +# Declare the actual php version to use. +# A packager willing to use another version of php can override the variable into its _common.sh. +YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} + # Create a dedicated php-fpm config # -# usage: ynh_add_fpm_config [--phpversion=7.X] [--dedicated_service] +# usage 1: ynh_add_fpm_config [--phpversion=7.X] [--use_template] [--package=packages] [--dedicated_service] # | arg: -v, --phpversion - Version of php to use. +# | arg: -t, --use_template - Use this helper in template mode. +# | arg: -p, --package - Additionnal php packages to install # | arg: -d, --dedicated_service - Use a dedicated php-fpm service instead of the common one. # +# ----------------------------------------------------------------------------- +# +# usage 2: ynh_add_fpm_config [--phpversion=7.X] --usage=usage --footprint=footprint [--package=packages] [--dedicated_service] +# | arg: -v, --phpversion - Version of php to use. +# | arg: -f, --footprint - Memory footprint of the service (low/medium/high). +# low - Less than 20Mb of ram by pool. +# medium - Between 20Mb and 40Mb of ram by pool. +# high - More than 40Mb of ram by pool. +# Or specify exactly the footprint, the load of the service as Mb by pool instead of having a standard value. +# To have this value, use the following command and stress the service. +# watch -n0.5 ps -o user,cmd,%cpu,rss -u APP +# +# | arg: -u, --usage - Expected usage of the service (low/medium/high). +# low - Personal usage, behind the sso. +# medium - Low usage, few people or/and publicly accessible. +# high - High usage, frequently visited website. +# +# | arg: -p, --package - Additionnal php packages to install for a specific version of php +# | arg: -d, --dedicated_service - Use a dedicated php-fpm service instead of the common one. +# +# +# The footprint of the service will be used to defined the maximum footprint we can allow, which is half the maximum RAM. +# So it will be used to defined 'pm.max_children' +# A lower value for the footprint will allow more children for 'pm.max_children'. And so for +# 'pm.start_servers', 'pm.min_spare_servers' and 'pm.max_spare_servers' which are defined from the +# value of 'pm.max_children' +# NOTE: 'pm.max_children' can't exceed 4 times the number of processor's cores. +# +# The usage value will defined the way php will handle the children for the pool. +# A value set as 'low' will set the process manager to 'ondemand'. Children will start only if the +# service is used, otherwise no child will stay alive. This config gives the lower footprint when the +# service is idle. But will use more proc since it has to start a child as soon it's used. +# Set as 'medium', the process manager will be at dynamic. If the service is idle, a number of children +# equal to pm.min_spare_servers will stay alive. So the service can be quick to answer to any request. +# The number of children can grow if needed. The footprint can stay low if the service is idle, but +# not null. The impact on the proc is a little bit less than 'ondemand' as there's always a few +# children already available. +# Set as 'high', the process manager will be set at 'static'. There will be always as many children as +# 'pm.max_children', the footprint is important (but will be set as maximum a quarter of the maximum +# RAM) but the impact on the proc is lower. The service will be quick to answer as there's always many +# children ready to answer. +# # Requires YunoHost version 2.7.2 or higher. ynh_add_fpm_config () { # Declare an array to define the options of this helper. - local legacy_args=vd - declare -Ar args_array=( [v]=phpversion= [d]=dedicated_service ) + local legacy_args=vtufpd + declare -Ar args_array=( [v]=phpversion= [t]=use_template [u]=usage= [f]=footprint= [p]=package= [d]=dedicated_service ) local phpversion + local use_template + local usage + local footprint + local package local dedicated_service # Manage arguments with getopts ynh_handle_getopts_args "$@" + package=${package:-} - # Configure PHP-FPM 7.0 by default - phpversion="${phpversion:-7.0}" + # The default behaviour is to use the template. + use_template="${use_template:-1}" + usage="${usage:-}" + footprint="${footprint:-}" + if [ -n "$usage" ] || [ -n "$footprint" ]; then + use_template=0 + fi + + # Set the default PHP-FPM version by default + phpversion="${phpversion:-$YNH_PHP_VERSION}" + + # If the requested php version is not the default version for YunoHost + if [ "$phpversion" != "$YNH_DEFAULT_PHP_VERSION" ] + then + # If the argument --package is used, add the packages to ynh_install_php to install them from sury + if [ -n "$package" ]; then + local additionnal_packages="--package=$package" + else + local additionnal_packages="" + fi + # Install this specific version of php. + ynh_install_php --phpversion=$phpversion "$additionnal_packages" + elif [ -n "$package" ] + then + # Install the additionnal packages from the default repository + ynh_add_app_dependencies --package="$package" + fi # Do not use a dedicated service by default dedicated_service=${dedicated_service:-0} @@ -43,6 +122,7 @@ ynh_add_fpm_config () { ynh_app_setting_set --app=$app --key=fpm_config_dir --value="$fpm_config_dir" ynh_app_setting_set --app=$app --key=fpm_service --value="$fpm_service" ynh_app_setting_set --app=$app --key=fpm_dedicated_service --value="$dedicated_service" + ynh_app_setting_set --app=$app --key=phpversion --value=$phpversion finalphpconf="$fpm_config_dir/pool.d/$app.conf" # Migrate from mutual php service to dedicated one. @@ -63,17 +143,74 @@ ynh_add_fpm_config () { ynh_backup_if_checksum_is_different --file="$finalphpconf" - cp ../conf/php-fpm.conf "$finalphpconf" - ynh_replace_string --match_string="__NAMETOCHANGE__" --replace_string="$app" --target_file="$finalphpconf" - ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalphpconf" - ynh_replace_string --match_string="__USER__" --replace_string="$app" --target_file="$finalphpconf" - ynh_replace_string --match_string="__PHPVERSION__" --replace_string="$phpversion" --target_file="$finalphpconf" + if [ $use_template -eq 1 ] + then + # Usage 1, use the template in ../conf/php-fpm.conf + cp ../conf/php-fpm.conf "$finalphpconf" + ynh_replace_string --match_string="__NAMETOCHANGE__" --replace_string="$app" --target_file="$finalphpconf" + ynh_replace_string --match_string="__FINALPATH__" --replace_string="$final_path" --target_file="$finalphpconf" + ynh_replace_string --match_string="__USER__" --replace_string="$app" --target_file="$finalphpconf" + ynh_replace_string --match_string="__PHPVERSION__" --replace_string="$phpversion" --target_file="$finalphpconf" + + else + # Usage 2, generate a php-fpm config file with ynh_get_scalable_phpfpm + + # Store settings + ynh_app_setting_set --app=$app --key=fpm_footprint --value=$footprint + ynh_app_setting_set --app=$app --key=fpm_usage --value=$usage + + # Define the values to use for the configuration of php. + ynh_get_scalable_phpfpm --usage=$usage --footprint=$footprint + + # Copy the default file + cp "$fpm_config_dir/pool.d/www.conf" "$finalphpconf" + + # Replace standard variables into the default file + ynh_replace_string --match_string="^\[www\]" --replace_string="[$app]" --target_file="$finalphpconf" + ynh_replace_string --match_string=".*listen = .*" --replace_string="listen = /var/run/php/php$phpversion-fpm-$app.sock" --target_file="$finalphpconf" + ynh_replace_string --match_string="^user = .*" --replace_string="user = $app" --target_file="$finalphpconf" + ynh_replace_string --match_string="^group = .*" --replace_string="group = $app" --target_file="$finalphpconf" + ynh_replace_string --match_string=".*chdir = .*" --replace_string="chdir = $final_path" --target_file="$finalphpconf" + + # Configure fpm children + ynh_replace_string --match_string=".*pm = .*" --replace_string="pm = $php_pm" --target_file="$finalphpconf" + ynh_replace_string --match_string=".*pm.max_children = .*" --replace_string="pm.max_children = $php_max_children" --target_file="$finalphpconf" + ynh_replace_string --match_string=".*pm.max_requests = .*" --replace_string="pm.max_requests = 500" --target_file="$finalphpconf" + ynh_replace_string --match_string=".*request_terminate_timeout = .*" --replace_string="request_terminate_timeout = 1d" --target_file="$finalphpconf" + if [ "$php_pm" = "dynamic" ] + then + ynh_replace_string --match_string=".*pm.start_servers = .*" --replace_string="pm.start_servers = $php_start_servers" --target_file="$finalphpconf" + ynh_replace_string --match_string=".*pm.min_spare_servers = .*" --replace_string="pm.min_spare_servers = $php_min_spare_servers" --target_file="$finalphpconf" + ynh_replace_string --match_string=".*pm.max_spare_servers = .*" --replace_string="pm.max_spare_servers = $php_max_spare_servers" --target_file="$finalphpconf" + elif [ "$php_pm" = "ondemand" ] + then + ynh_replace_string --match_string=".*pm.process_idle_timeout = .*" --replace_string="pm.process_idle_timeout = 10s" --target_file="$finalphpconf" + fi + + # Comment unused parameters + if [ "$php_pm" != "dynamic" ] + then + ynh_replace_string --match_string=".*\(pm.start_servers = .*\)" --replace_string=";\1" --target_file="$finalphpconf" + ynh_replace_string --match_string=".*\(pm.min_spare_servers = .*\)" --replace_string=";\1" --target_file="$finalphpconf" + ynh_replace_string --match_string=".*\(pm.max_spare_servers = .*\)" --replace_string=";\1" --target_file="$finalphpconf" + fi + if [ "$php_pm" != "ondemand" ] + then + ynh_replace_string --match_string=".*\(pm.process_idle_timeout = .*\)" --replace_string=";\1" --target_file="$finalphpconf" + fi + + # Concatene the extra config. + if [ -e ../conf/extra_php-fpm.conf ]; then + cat ../conf/extra_php-fpm.conf >> "$finalphpconf" + fi + fi + chown root: "$finalphpconf" ynh_store_file_checksum --file="$finalphpconf" if [ -e "../conf/php-fpm.ini" ] then - echo "Packagers ! Please do not use a separate php ini file, merge your directives in the pool file instead." >&2 + ynh_print_warn -message="Packagers ! Please do not use a separate php ini file, merge your directives in the pool file instead." finalphpini="$fpm_config_dir/conf.d/20-$app.ini" ynh_backup_if_checksum_is_different "$finalphpini" cp ../conf/php-fpm.ini "$finalphpini" @@ -131,10 +268,16 @@ ynh_remove_fpm_config () { local fpm_service=$(ynh_app_setting_get --app=$app --key=fpm_service) local dedicated_service=$(ynh_app_setting_get --app=$app --key=fpm_dedicated_service) dedicated_service=${dedicated_service:-0} - # Assume php version 7.0 if not set + # Get the version of php used by this app + local phpversion=$(ynh_app_setting_get $app phpversion) + + # Assume default PHP-FPM version by default + phpversion="${phpversion:-$YNH_DEFAULT_PHP_VERSION}" + + # Assume default php files if not set if [ -z "$fpm_config_dir" ]; then - fpm_config_dir="/etc/php/7.0/fpm" - fpm_service="php7.0-fpm" + fpm_config_dir="/etc/php/$YNH_DEFAULT_PHP_VERSION/fpm" + fpm_service="php$YNH_DEFAULT_PHP_VERSION-fpm" fi if [ $dedicated_service -eq 1 ] @@ -145,8 +288,240 @@ ynh_remove_fpm_config () { ynh_secure_remove --file="$fpm_config_dir/php-fpm-$app.conf" # Remove the service from the list of services known by Yunohost yunohost service remove $fpm_service + elif ynh_package_is_installed --package="php${phpversion}-fpm"; then + ynh_systemd_action --service_name=$fpm_service --action=reload fi ynh_secure_remove --file="$fpm_config_dir/pool.d/$app.conf" ynh_secure_remove --file="$fpm_config_dir/conf.d/20-$app.ini" 2>&1 + + # If the php version used is not the default version for YunoHost + if [ "$phpversion" != "$YNH_DEFAULT_PHP_VERSION" ] + then + # Remove this specific version of php + ynh_remove_php + fi +} + +# Install another version of php. +# +# [internal] +# +# usage: ynh_install_php --phpversion=phpversion [--package=packages] +# | arg: -v, --phpversion - Version of php to install. +# | arg: -p, --package - Additionnal php packages to install +ynh_install_php () { + # Declare an array to define the options of this helper. + local legacy_args=vp + declare -Ar args_array=( [v]=phpversion= [p]=package= ) + local phpversion + local package + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + package=${package:-} + + # Store phpversion into the config of this app + ynh_app_setting_set $app phpversion $phpversion + + if [ "$phpversion" == "$YNH_DEFAULT_PHP_VERSION" ] + then + ynh_die "Do not use ynh_install_php to install php$YNH_DEFAULT_PHP_VERSION" + fi + + # Create the file if doesn't exist already + touch /etc/php/ynh_app_version + + # Do not add twice the same line + if ! grep --quiet "$YNH_APP_INSTANCE_NAME:" "/etc/php/ynh_app_version" + then + # Store the ID of this app and the version of php requested for it + echo "$YNH_APP_INSTANCE_NAME:$phpversion" | tee --append "/etc/php/ynh_app_version" + fi + + # Add an extra repository for those packages + ynh_install_extra_repo --repo="https://packages.sury.org/php/ $(lsb_release -sc) main" --key="https://packages.sury.org/php/apt.gpg" --priority=995 --name=extra_php_version + + # Install requested dependencies from this extra repository. + # Install php-fpm first, otherwise php will install apache as a dependency. + ynh_add_app_dependencies --package="php${phpversion}-fpm" + ynh_add_app_dependencies --package="php$phpversion php${phpversion}-common $package" + + # Set the default php version back as the default version for php-cli. + update-alternatives --set php /usr/bin/php$YNH_DEFAULT_PHP_VERSION + + # Pin this extra repository after packages are installed to prevent sury of doing shit + ynh_pin_repo --package="*" --pin="origin \"packages.sury.org\"" --priority=200 --name=extra_php_version + ynh_pin_repo --package="php${YNH_DEFAULT_PHP_VERSION}*" --pin="origin \"packages.sury.org\"" --priority=600 --name=extra_php_version --append + + # Advertise service in admin panel + yunohost service add php${phpversion}-fpm --log "/var/log/php${phpversion}-fpm.log" +} + +# Remove the specific version of php used by the app. +# +# [internal] +# +# usage: ynh_install_php +ynh_remove_php () { + # Get the version of php used by this app + local phpversion=$(ynh_app_setting_get $app phpversion) + + if [ "$phpversion" == "$YNH_DEFAULT_PHP_VERSION" ] || [ -z "$phpversion" ] + then + if [ "$phpversion" == "$YNH_DEFAULT_PHP_VERSION" ] + then + ynh_print_err "Do not use ynh_remove_php to remove php$YNH_DEFAULT_PHP_VERSION !" + fi + return 0 + fi + + # Create the file if doesn't exist already + touch /etc/php/ynh_app_version + + # Remove the line for this app + sed --in-place "/$YNH_APP_INSTANCE_NAME:$phpversion/d" "/etc/php/ynh_app_version" + + # If no other app uses this version of php, remove it. + if ! grep --quiet "$phpversion" "/etc/php/ynh_app_version" + then + # Remove the service from the admin panel + if ynh_package_is_installed --package="php${phpversion}-fpm"; then + yunohost service remove php${phpversion}-fpm + fi + + # Purge php dependencies for this version. + ynh_package_autopurge "php$phpversion php${phpversion}-fpm php${phpversion}-common" + fi +} + +# Define the values to configure php-fpm +# +# [internal] +# +# usage: ynh_get_scalable_phpfpm --usage=usage --footprint=footprint [--print] +# | arg: -f, --footprint - Memory footprint of the service (low/medium/high). +# low - Less than 20Mb of ram by pool. +# medium - Between 20Mb and 40Mb of ram by pool. +# high - More than 40Mb of ram by pool. +# Or specify exactly the footprint, the load of the service as Mb by pool instead of having a standard value. +# To have this value, use the following command and stress the service. +# watch -n0.5 ps -o user,cmd,%cpu,rss -u APP +# +# | arg: -u, --usage - Expected usage of the service (low/medium/high). +# low - Personal usage, behind the sso. +# medium - Low usage, few people or/and publicly accessible. +# high - High usage, frequently visited website. +# +# | arg: -p, --print - Print the result (intended for debug purpose only when packaging the app) +ynh_get_scalable_phpfpm () { + local legacy_args=ufp + # Declare an array to define the options of this helper. + declare -Ar args_array=( [u]=usage= [f]=footprint= [p]=print ) + local usage + local footprint + local print + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + # Set all characters as lowercase + footprint=${footprint,,} + usage=${usage,,} + print=${print:-0} + + if [ "$footprint" = "low" ] + then + footprint=20 + elif [ "$footprint" = "medium" ] + then + footprint=35 + elif [ "$footprint" = "high" ] + then + footprint=50 + fi + + # Define the way the process manager handle child processes. + if [ "$usage" = "low" ] + then + php_pm=ondemand + elif [ "$usage" = "medium" ] + then + php_pm=dynamic + elif [ "$usage" = "high" ] + then + php_pm=static + else + ynh_die --message="Does not recognize '$usage' as an usage value." + fi + + # Get the total of RAM available, except swap. + local max_ram=$(ynh_get_ram --total --ignore_swap) + + at_least_one() { + # Do not allow value below 1 + if [ $1 -le 0 ] + then + echo 1 + else + echo $1 + fi + } + + # Define pm.max_children + # The value of pm.max_children is the total amount of ram divide by 2 and divide again by the footprint of a pool for this app. + # So if php-fpm start the maximum of children, it won't exceed half of the ram. + php_max_children=$(( $max_ram / 2 / $footprint )) + # If process manager is set as static, use half less children. + # Used as static, there's always as many children as the value of pm.max_children + if [ "$php_pm" = "static" ] + then + php_max_children=$(( $php_max_children / 2 )) + fi + php_max_children=$(at_least_one $php_max_children) + + # To not overload the proc, limit the number of children to 4 times the number of cores. + local core_number=$(nproc) + local max_proc=$(( $core_number * 4 )) + if [ $php_max_children -gt $max_proc ] + then + php_max_children=$max_proc + fi + + if [ "$php_pm" = "dynamic" ] + then + # Define pm.start_servers, pm.min_spare_servers and pm.max_spare_servers for a dynamic process manager + php_min_spare_servers=$(( $php_max_children / 8 )) + php_min_spare_servers=$(at_least_one $php_min_spare_servers) + + php_max_spare_servers=$(( $php_max_children / 2 )) + php_max_spare_servers=$(at_least_one $php_max_spare_servers) + + php_start_servers=$(( $php_min_spare_servers + ( $php_max_spare_servers - $php_min_spare_servers ) /2 )) + php_start_servers=$(at_least_one $php_start_servers) + else + php_min_spare_servers=0 + php_max_spare_servers=0 + php_start_servers=0 + fi + + if [ $print -eq 1 ] + then + ynh_debug --message="Footprint=${footprint}Mb by pool." + ynh_debug --message="Process manager=$php_pm" + ynh_debug --message="Max RAM=${max_ram}Mb" + if [ "$php_pm" != "static" ]; then + ynh_debug --message="\nMax estimated footprint=$(( $php_max_children * $footprint ))" + ynh_debug --message="Min estimated footprint=$(( $php_min_spare_servers * $footprint ))" + fi + if [ "$php_pm" = "dynamic" ]; then + ynh_debug --message="Estimated average footprint=$(( $php_max_spare_servers * $footprint ))" + elif [ "$php_pm" = "static" ]; then + ynh_debug --message="Estimated footprint=$(( $php_max_children * $footprint ))" + fi + ynh_debug --message="\nRaw php-fpm values:" + ynh_debug --message="pm.max_children = $php_max_children" + if [ "$php_pm" = "dynamic" ]; then + ynh_debug --message="pm.start_servers = $php_start_servers" + ynh_debug --message="pm.min_spare_servers = $php_min_spare_servers" + ynh_debug --message="pm.max_spare_servers = $php_max_spare_servers" + fi + fi } diff --git a/data/helpers.d/setting b/data/helpers.d/setting index f3692cf96..350ed3ea0 100644 --- a/data/helpers.d/setting +++ b/data/helpers.d/setting @@ -105,7 +105,7 @@ EOF if [[ "$1" == "set" ]] && [[ "${4:-}" == "/" ]] then ynh_permission_update --permission "main" --add "visitors" - elif [[ "$1" == "delete" ]] && [[ "${current_value:-}" == "/" ]] + elif [[ "$1" == "delete" ]] && [[ "${current_value:-}" == "/" ]] && [[ -n "$(ynh_app_setting_get --app=$2 --key='is_public' )" ]] then ynh_permission_update --permission "main" --remove "visitors" fi @@ -178,6 +178,8 @@ ynh_webpath_register () { # # Requires YunoHost version 3.7.0 or higher. ynh_permission_create() { + # Declare an array to define the options of this helper. + local legacy_args=pua declare -Ar args_array=( [p]=permission= [u]=url= [a]=allowed= ) local permission local url @@ -206,6 +208,8 @@ ynh_permission_create() { # # Requires YunoHost version 3.7.0 or higher. ynh_permission_delete() { + # Declare an array to define the options of this helper. + local legacy_args=p declare -Ar args_array=( [p]=permission= ) local permission ynh_handle_getopts_args "$@" @@ -220,6 +224,8 @@ ynh_permission_delete() { # # Requires YunoHost version 3.7.0 or higher. ynh_permission_exists() { + # Declare an array to define the options of this helper. + local legacy_args=p declare -Ar args_array=( [p]=permission= ) local permission ynh_handle_getopts_args "$@" @@ -235,6 +241,8 @@ ynh_permission_exists() { # # Requires YunoHost version 3.7.0 or higher. ynh_permission_url() { + # Declare an array to define the options of this helper. + local legacy_args=pu declare -Ar args_array=([p]=permission= [u]=url=) local permission local url @@ -260,6 +268,8 @@ ynh_permission_url() { # example: ynh_permission_update --permission admin --add samdoe --remove all_users # Requires YunoHost version 3.7.0 or higher. ynh_permission_update() { + # Declare an array to define the options of this helper. + local legacy_args=par declare -Ar args_array=( [p]=permission= [a]=add= [r]=remove= ) local permission local add @@ -275,3 +285,29 @@ ynh_permission_update() { yunohost user permission update "$app.$permission" ${add:-} ${remove:-} } + +# Check if a permission exists +# +# usage: ynh_permission_has_user --permission=permission --user=user +# | arg: -p, --permission - the permission to check +# | arg: -u, --user - the user seek in the permission +# +# example: ynh_permission_has_user --permission=main --user=visitors +# +# Requires YunoHost version 3.7.1 or higher. +ynh_permission_has_user() { + local legacy_args=pu + # Declare an array to define the options of this helper. + declare -Ar args_array=( [p]=permission= [u]=user= ) + local permission + local user + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + if ! ynh_permission_exists --permission=$permission + then + return 1 + fi + + yunohost user permission info "$app.$permission" | grep -w -q "$user" +} diff --git a/data/helpers.d/utils b/data/helpers.d/utils index 50671dba0..133a47247 100644 --- a/data/helpers.d/utils +++ b/data/helpers.d/utils @@ -237,9 +237,14 @@ ynh_local_curl () { # Wait untils nginx has fully reloaded (avoid curl fail with http2) sleep 2 + + local cookiefile=/tmp/ynh-$app-cookie.txt + touch $cookiefile + chown root $cookiefile + chmod 700 $cookiefile # Curl the URL - curl --silent --show-error -kL -H "Host: $domain" --resolve $domain:443:127.0.0.1 $POST_data "$full_page_url" --cookie-jar /tmp/ynh-$app-cookie.txt --cookie /tmp/ynh-$app-cookie.txt + curl --silent --show-error -kL -H "Host: $domain" --resolve $domain:443:127.0.0.1 $POST_data "$full_page_url" --cookie-jar $cookiefile --cookie $cookiefile } # Render templates with Jinja2 diff --git a/data/hooks/conf_regen/12-metronome b/data/hooks/conf_regen/12-metronome index 5c9c67f11..5a50b2b6e 100755 --- a/data/hooks/conf_regen/12-metronome +++ b/data/hooks/conf_regen/12-metronome @@ -54,7 +54,6 @@ do_post_regen() { chmod g+s "/var/xmpp-upload/${main_domain}/upload" chown -R metronome:www-data "/var/xmpp-upload/${main_domain}" - # fix some permissions chown -R metronome: /var/lib/metronome/ chown -R metronome: /etc/metronome/conf.d/ diff --git a/data/hooks/conf_regen/15-nginx b/data/hooks/conf_regen/15-nginx index 11e5f596c..f8b7d8062 100755 --- a/data/hooks/conf_regen/15-nginx +++ b/data/hooks/conf_regen/15-nginx @@ -23,6 +23,7 @@ do_init_regen() { rm -f "${nginx_dir}/sites-enabled/default" export compatibility="intermediate" + ynh_render_template "security.conf.inc" "${nginx_conf_dir}/security.conf.inc" ynh_render_template "yunohost_admin.conf" "${nginx_conf_dir}/yunohost_admin.conf" # Restart nginx if conf looks good, otherwise display error and exit unhappy @@ -110,6 +111,21 @@ do_post_regen() { mkdir -p "/etc/nginx/conf.d/${domain}.d" done + # Get rid of legacy lets encrypt snippets + for domain in $domain_list; do + # If the legacy letsencrypt / acme-challenge domain-specific snippet is still there + if [ -e /etc/nginx/conf.d/${domain}.d/000-acmechallenge.conf ] + then + # And if we're effectively including the new domain-independant snippet now + if grep -q "include /etc/nginx/conf.d/acme-challenge.conf.inc;" /etc/nginx/conf.d/${domain}.conf + then + # Delete the old domain-specific snippet + rm /etc/nginx/conf.d/${domain}.d/000-acmechallenge.conf + fi + fi + done + + # Reload nginx configuration pgrep nginx && service nginx reload } diff --git a/data/hooks/diagnosis/12-dnsrecords.py b/data/hooks/diagnosis/12-dnsrecords.py index 96ac31d55..a889201b9 100644 --- a/data/hooks/diagnosis/12-dnsrecords.py +++ b/data/hooks/diagnosis/12-dnsrecords.py @@ -28,21 +28,24 @@ class DNSRecordsDiagnoser(Diagnoser): all_domains = domain_list()["domains"] for domain in all_domains: self.logger_debug("Diagnosing DNS conf for %s" % domain) - for report in self.check_domain(domain, domain == main_domain): + is_subdomain = domain.split(".",1)[1] in all_domains + for report in self.check_domain(domain, domain == main_domain, is_subdomain=is_subdomain): yield report # FIXME : somewhere, should implement a check for reverse DNS ... # FIXME / TODO : somewhere, could also implement a check for domain expiring soon - def check_domain(self, domain, is_main_domain): + def check_domain(self, domain, is_main_domain, is_subdomain): expected_configuration = _build_dns_conf(domain) - # Here if there are no AAAA record, we should add something to expect "no" AAAA record + # FIXME: Here if there are no AAAA record, we should add something to expect "no" AAAA record # to properly diagnose situations where people have a AAAA record but no IPv6 - categories = ["basic", "mail", "xmpp", "extra"] + if is_subdomain: + categories = ["basic"] + for category in categories: records = expected_configuration[category] diff --git a/data/templates/dovecot/dovecot.conf b/data/templates/dovecot/dovecot.conf index 477ccbfb1..8fc0e75ae 100644 --- a/data/templates/dovecot/dovecot.conf +++ b/data/templates/dovecot/dovecot.conf @@ -12,10 +12,25 @@ protocols = imap sieve {% if pop3_enabled == "True" %}pop3{% endif %} mail_plugins = $mail_plugins quota -ssl = yes +############################################################################### + +# generated 2020-04-03, Mozilla Guideline v5.4, Dovecot 2.2.27, OpenSSL 1.1.0l, intermediate configuration +# https://ssl-config.mozilla.org/#server=dovecot&version=2.2.27&config=intermediate&openssl=1.1.0l&guideline=5.4 + +ssl = required + ssl_cert = openssl dhparam -out /etc/ssl/private/dh2048.pem -outform PEM -2 2048 #ssl_dhparam /etc/ssl/private/dh2048.pem; -{% endif %} +# Follows the Web Security Directives from the Mozilla Dev Lab and the Mozilla Obervatory + Partners +# https://wiki.mozilla.org/Security/Guidelines/Web_Security +# https://observatory.mozilla.org/ more_set_headers "Content-Security-Policy : upgrade-insecure-requests"; more_set_headers "Content-Security-Policy-Report-Only : default-src https: data: 'unsafe-inline' 'unsafe-eval'"; more_set_headers "X-Content-Type-Options : nosniff"; diff --git a/data/templates/nginx/server.tpl.conf b/data/templates/nginx/server.tpl.conf index 6316960c4..f2e9de2de 100644 --- a/data/templates/nginx/server.tpl.conf +++ b/data/templates/nginx/server.tpl.conf @@ -10,17 +10,19 @@ server { access_by_lua_file /usr/share/ssowat/access.lua; + include /etc/nginx/conf.d/acme-challenge.conf.inc; + include /etc/nginx/conf.d/{{ domain }}.d/*.conf; location /yunohost/admin { return 301 https://$http_host$request_uri; } - location /.well-known/ynh-diagnosis/ { + location ^~ '/.well-known/ynh-diagnosis/' { alias /tmp/.well-known/ynh-diagnosis/; } - location /.well-known/autoconfig/mail/ { + location ^~ '/.well-known/autoconfig/mail/' { alias /var/www/.well-known/{{ domain }}/autoconfig/mail/; } @@ -33,12 +35,10 @@ server { listen [::]:443 ssl http2; server_name {{ domain }}; + include /etc/nginx/conf.d/security.conf.inc; + ssl_certificate /etc/yunohost/certs/{{ domain }}/crt.pem; ssl_certificate_key /etc/yunohost/certs/{{ domain }}/key.pem; - ssl_session_timeout 5m; - ssl_session_cache shared:SSL:50m; - - include /etc/nginx/conf.d/security.conf.inc; {% if domain_cert_ca != "Self-signed" %} more_set_headers "Strict-Transport-Security : max-age=63072000; includeSubDomains; preload"; @@ -52,6 +52,10 @@ server { resolver_timeout 5s; {% endif %} + location ^~ '/.well-known/autoconfig/mail/' { + alias /var/www/.well-known/{{ domain }}/autoconfig/mail/; + } + access_by_lua_file /usr/share/ssowat/access.lua; include /etc/nginx/conf.d/{{ domain }}.d/*.conf; @@ -85,12 +89,10 @@ server { client_max_body_size 105M; # Choose a value a bit higher than the max upload configured in XMPP server } + include /etc/nginx/conf.d/security.conf.inc; + ssl_certificate /etc/yunohost/certs/{{ domain }}/crt.pem; ssl_certificate_key /etc/yunohost/certs/{{ domain }}/key.pem; - ssl_session_timeout 5m; - ssl_session_cache shared:SSL:50m; - - include /etc/nginx/conf.d/security.conf.inc; {% if domain_cert_ca != "Self-signed" %} more_set_headers "Strict-Transport-Security : max-age=63072000; includeSubDomains; preload"; diff --git a/data/templates/nginx/yunohost_admin.conf b/data/templates/nginx/yunohost_admin.conf index 63d466ecd..3df838c4a 100644 --- a/data/templates/nginx/yunohost_admin.conf +++ b/data/templates/nginx/yunohost_admin.conf @@ -20,6 +20,10 @@ server { ssl_certificate /etc/yunohost/certs/yunohost.org/crt.pem; ssl_certificate_key /etc/yunohost/certs/yunohost.org/key.pem; + more_set_headers "Strict-Transport-Security : max-age=63072000; includeSubDomains; preload"; + more_set_headers "Referrer-Policy : 'same-origin'"; + more_set_headers "Content-Security-Policy : upgrade-insecure-requests; object-src 'none'; script-src https: 'unsafe-eval'"; + location / { return 302 https://$http_host/yunohost/admin; } diff --git a/data/templates/postfix/main.cf b/data/templates/postfix/main.cf index 045b8edd0..2642fd8f0 100644 --- a/data/templates/postfix/main.cf +++ b/data/templates/postfix/main.cf @@ -18,35 +18,39 @@ append_dot_mydomain = no readme_directory = no # -- TLS for incoming connections -# By default, TLS is disabled in the Postfix SMTP server, so no difference to -# plain Postfix is visible. Explicitly switch it on with "smtpd_tls_security_level = may". -smtpd_tls_security_level=may +############################################################################### +# generated 2020-04-03, Mozilla Guideline v5.4, Postfix 3.1.14, OpenSSL 1.1.0l, intermediate configuration +# https://ssl-config.mozilla.org/#server=postfix&version=3.1.14&config=intermediate&openssl=1.1.0l&guideline=5.4 -# Sending AUTH data over an unencrypted channel poses a security risk. -# When TLS layer encryption is optional ("smtpd_tls_security_level = may"), it -# may however still be useful to only offer AUTH when TLS is active. To maintain -# compatibility with non-TLS clients, the default is to accept AUTH without -# encryption. In order to change this behavior, we set "smtpd_tls_auth_only = yes". -smtpd_tls_auth_only=yes +# (No modern conf support until we're on buster...) +# {% if compatibility == "intermediate" %} {% else %} {% endif %} + +smtpd_use_tls = yes + +smtpd_tls_security_level = may +smtpd_tls_auth_only = yes smtpd_tls_cert_file = /etc/yunohost/certs/{{ main_domain }}/crt.pem smtpd_tls_key_file = /etc/yunohost/certs/{{ main_domain }}/key.pem -smtpd_tls_exclude_ciphers = aNULL, MD5, DES, ADH, RC4, 3DES +smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1 +smtpd_tls_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1 +smtpd_tls_mandatory_ciphers = medium + +# curl https://ssl-config.mozilla.org/ffdhe2048.txt > /path/to/dhparam.pem +# not actually 1024 bits, this applies to all DHE >= 1024 bits +# smtpd_tls_dh1024_param_file = /path/to/dhparam.pem + +tls_medium_cipherlist = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 +tls_preempt_cipherlist = no +############################################################################### smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache smtpd_tls_loglevel=1 -{% if compatibility == "intermediate" %} -smtpd_tls_mandatory_protocols=!SSLv2,!SSLv3 -{% else %} -smtpd_tls_mandatory_protocols=!SSLv2,!SSLv3,!TLSv1,!TLSv1.1 -{% endif %} -smtpd_tls_mandatory_ciphers=high -smtpd_tls_eecdh_grade = ultra # -- TLS for outgoing connections # Use TLS if this is supported by the remote SMTP server, otherwise use plaintext. smtp_tls_security_level=may smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache -smtp_tls_exclude_ciphers = $smtpd_tls_exclude_ciphers -smtp_tls_mandatory_ciphers= $smtpd_tls_mandatory_ciphers +smtp_tls_exclude_ciphers = aNULL, MD5, DES, ADH, RC4, 3DES +smtp_tls_mandatory_ciphers= high smtp_tls_loglevel=1 # Configure Root CA certificates @@ -167,4 +171,4 @@ default_destination_rate_delay = 5s # By default it's possible to detect if the email adress exist # So it's easly possible to scan a server to know which email adress is valid # and after to send spam -disable_vrfy_command = yes \ No newline at end of file +disable_vrfy_command = yes diff --git a/debian/changelog b/debian/changelog index faa7db414..83c310d67 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,9 +1,86 @@ -yunohost (3.8.0~alpha) testing; urgency=low +yunohost (3.8.0) testing; urgency=low - Placeholder for upcoming 3.8 to avoid funky stuff with version numbers in - builds etc. + # Major stuff - -- Alexandre Aubin Mon, 16 Mar 2020 01:00:00 +0000 + - [enh] New diagnosis system (#534, #872, #919, a416044, a354425, 4ab3653, decb372, e686dc6, b5d18d6, 69bc124, 937d339, cc2288c, aaa9805, 526a3a2) + - [enh] App categories (#778, #853) + - [enh] Support XMPP http upload (#831) + - [enh] Many small improvements in the way we manage services (#838, fa5c0e9, dd92a34, c97a839) + - [enh] Add subcategories management in bash completion (#839) + - [mod] Add conflict with apache2 and bind9, other minor changes in Depends (#909, 3bd6a7a, 0a482fd) + - [enh] Setting to enable POP3 in email stack (#791) + - [enh] Better UX for CLI/API to change maindomain (#796) + + # Misc technical + + - Update ciphers for nginx, postfix and dovecot according to new Mozilla recommendation (#913, #914) + - Get rid of domain-specific acme-challenge snippet, use a single snippet included in every conf (#917) + - [enh] Persist cookies between multiple ynh_local_curl calls for the same app (#884, #903) + - [fix] ynh_find_port didn't detect port already used on UDP (#827, #907) + - [fix] prevent firefox to mix CA and server certificate (#857) + - [enh] add operation logger for config panel (#869) + - [fix] psql helpers: Revoke sessions before dropping tables (#895) + - [fix] moulinette logs were never displayed #lol (#758) + + # Tests, cleaning, refactoring + + - Add core CI, improve/fix tests (#856, #863, 6eb8efb, c4590ab, 711cc35, 6c24755) + - Refactoring (#805, 101d3be, #784) + - Drop some very-old deprecated app helpers (though still somewhat supporting them through hacky patching) (#780) + - Drop glances and the old monitoring system (#821) + - Drop app_debug (#824) + - Drop app's status.json (#834) + - Drop ynh_add_skipped/(un)protected_uris helpers (#910) + - Use a common security.conf.inc instead of having cipher setting in each nginx's domain file (1285776, 4d99cbe, be8427d, 22b9565) + - Don't add weird tmp redirected_urls after postinstall (#902) + - Don't do weird stuff with yunohost-firewall during debian's postinst (978d9d5) + + # i18n, messaging + + - Unit tests / lint / cleaning for translation files (#901) + - Improve message wording, spelling (8b0c9e5, 9fe43b1, f69ab4c, 0decb64, 986f38f, 8d40c73, 8fe343a, 1d84f17) + - Improve translations for French, Catalan, Bengali (Bangladesh), Italian, Dutch, Norwegian Bokmål, Chinese, Occitan, Spanish, Esperanto, German, Nepali, Portuguese, Arabic, Russian, Hungarian, Hindi, Polish, Greek + + Thanks to all contributors <3 ! (Aeris One, Aleks, Allan N., Alvaro, Armando F., Arthur L., Augustin T., Bram, ButterflyOfFire, Damien P., Gustavo M., Jeroen F., Jimmy M., Josué, Kay0u, Maniack Crudelis, Mario, Matthew D., Mélanie C., Patrick B., Quentí, Yasss Gurl, amirale qt, Elie G., ljf, pitchum, Romain R., tituspijean, xaloc33, yalh76) + + -- Kay0u Thu, 09 Apr 2020 19:59:18 +0000 + +yunohost (3.7.1.1) stable; urgency=low + + - [fix] lxc uid number is limited to 65536 by default (0c9a4509) + - [fix] also invalidate group cache when creating users (aaabf8c7) + - [fix] Make sure to have a path that include sbin for stupid cron jobs (f03bb82a) + + -- Alexandre Aubin Sun, 12 Apr 2020 23:15:00 +0000 + +yunohost (3.7.1) stable; urgency=low + + - [enh] Add ynh_permission_has_user helper (#905) + - [mod] Change behavior of ynh_setting_delete to try to make migrating away from legacy permissions easier (#906) + - [fix] app_config_apply should also return 'app' info (#918) + - [fix] uid/gid conflicts in user_create because of inconsistent comparison (#924) + - [fix] Ensure metronome owns its directories (1f623830, 031f8a6e) + - [mod] Remove useless sudos in helpers (be88a283) + - [enh] Improve message wording for services (3c844292) + - [enh] Attempt to anonymize data pasted to paste.yunohost.org (f56f4724) + - [enh] Lazy load yunohost.certificate to possibly improve perfs (af8981e4) + - [fix] Improve logging / debugging (1eef9b67, 7d323814, d17fcaf9, 210d5f3f) + + Thanks to all contributors <3 ! (Bram, Kay0u, Maniack, Matthew D.) + + -- Alexandre Aubin Thu, 9 Apr 2020 14:52:00 +0000 + +yunohost (3.7.0.12) stable; urgency=low + + - Fix previous buggy hotfix about deleting existing primary groups ... + + -- Alexandre Aubin Sat, 28 Mar 2020 14:52:00 +0000 + +yunohost (3.7.0.11) stable; urgency=low + + - [fix] Mess due to automatic translation tools ~_~ + + -- Kay0u Fri, 27 Mar 2020 23:49:45 +0000 yunohost (3.7.0.10) stable; urgency=low diff --git a/debian/control b/debian/control index 42bafc16c..5bcd78491 100644 --- a/debian/control +++ b/debian/control @@ -15,22 +15,23 @@ Depends: ${python:Depends}, ${misc:Depends} , python-psutil, python-requests, python-dnspython, python-openssl , python-apt, python-miniupnpc, python-dbus, python-jinja2 , python-toml - , apt-transport-https - , dnsutils, bind9utils, unzip, git, curl, cron, wget, jq - , ca-certificates, netcat-openbsd, iproute2 + , apt, apt-transport-https + , nginx, nginx-extras (>=1.6.2) + , php-fpm, php-ldap, php-intl , mariadb-server, php-mysql | php-mysqlnd + , openssh-server, iptables, fail2ban, dnsutils, bind9utils + , openssl, ca-certificates, netcat-openbsd, iproute2 , slapd, ldap-utils, sudo-ldap, libnss-ldapd, unscd, libpam-ldapd - , postfix-ldap, postfix-policyd-spf-perl, postfix-pcre, procmail, mailutils, postsrsd - , dovecot-ldap, dovecot-lmtpd, dovecot-managesieved - , dovecot-antispam, fail2ban, iptables - , nginx-extras (>=1.6.2), php-fpm, php-ldap, php-intl - , dnsmasq, openssl, avahi-daemon, libnss-mdns, resolvconf, libnss-myhostname + , dnsmasq, avahi-daemon, libnss-mdns, resolvconf, libnss-myhostname + , postfix, postfix-ldap, postfix-policyd-spf-perl, postfix-pcre + , dovecot-core, dovecot-ldap, dovecot-lmtpd, dovecot-managesieved, dovecot-antispam + , rspamd (>= 1.6.0), opendkim-tools, postsrsd, procmail, mailutils + , redis-server , metronome - , rspamd (>= 1.6.0), redis-server, opendkim-tools - , haveged, fake-hwclock - , equivs, lsof + , git, curl, wget, cron, unzip, jq + , lsb-release, haveged, fake-hwclock, equivs, lsof Recommends: yunohost-admin - , openssh-server, ntp, inetutils-ping | iputils-ping + , ntp, inetutils-ping | iputils-ping , bash-completion, rsyslog , php-gd, php-curl, php-gettext, php-mcrypt , python-pip @@ -43,6 +44,7 @@ Conflicts: iptables-persistent , yunohost-config-dovecot, yunohost-config-slapd , yunohost-config-nginx, yunohost-config-amavis , yunohost-config-mysql, yunohost-predepends + , apache2, bind9 Replaces: moulinette-yunohost, yunohost-config , yunohost-config-others, yunohost-config-postfix , yunohost-config-dovecot, yunohost-config-slapd diff --git a/locales/ar.json b/locales/ar.json index 502cc2cf6..a1349fde7 100644 --- a/locales/ar.json +++ b/locales/ar.json @@ -173,4 +173,4 @@ "app_remove_after_failed_install": "جارٍ حذف التطبيق بعدما فشل تنصيبها…", "apps_catalog_updating": "جارٍ تحديث فهرس التطبيقات…", "apps_catalog_update_success": "تم تحديث فهرس التطبيقات!" -} \ No newline at end of file +} diff --git a/locales/de.json b/locales/de.json index c53cb60d2..d250a22fd 100644 --- a/locales/de.json +++ b/locales/de.json @@ -315,4 +315,4 @@ "apps_catalog_update_success": "Der Apps-Katalog wurde aktualisiert!", "password_too_simple_1": "Das Passwort muss mindestens 8 Zeichen lang sein", "diagnosis_display_tip_cli": "Sie können 'yunohost diagnosis show --issues' ausführen, um die gefundenen Probleme anzuzeigen." -} \ No newline at end of file +} diff --git a/locales/en.json b/locales/en.json index 567b6a460..f6aa35f67 100644 --- a/locales/en.json +++ b/locales/en.json @@ -120,7 +120,6 @@ "certmanager_cert_renew_success": "Let's Encrypt certificate renewed for the domain '{domain:s}'", "certmanager_cert_signing_failed": "Could not sign the new certificate", "certmanager_certificate_fetching_or_enabling_failed": "Trying to use the new certificate for {domain:s} did not work…", - "certmanager_conflicting_nginx_file": "Could not prepare domain for ACME challenge: the NGINX configuration file {filepath:s} is conflicting and should be removed first", "certmanager_couldnt_fetch_intermediate_cert": "Timed out when trying to fetch intermediate certificate from Let's Encrypt. Certificate installation/renewal aborted—please try again later.", "certmanager_domain_cert_not_selfsigned": "The certificate for domain {domain:s} is not self-signed. Are you sure you want to replace it? (Use '--force' to do so.)", "certmanager_domain_dns_ip_differs_from_public_ip": "The DNS 'A' record for the domain '{domain:s}' is different from this server's IP. If you recently modified your A record, please wait for it to propagate (some DNS propagation checkers are available online). (If you know what you are doing, use '--no-checks' to turn off those checks.)", diff --git a/locales/eo.json b/locales/eo.json index d825c84c7..7142d9f72 100644 --- a/locales/eo.json +++ b/locales/eo.json @@ -591,4 +591,4 @@ "log_app_action_run": "Funkciigu agon de la apliko '{}'", "log_app_config_show_panel": "Montri la agordan panelon de la apliko '{}'", "log_app_config_apply": "Apliki agordon al la apliko '{}'" -} \ No newline at end of file +} diff --git a/locales/oc.json b/locales/oc.json index 49e3ab02e..5472c97e8 100644 --- a/locales/oc.json +++ b/locales/oc.json @@ -571,4 +571,4 @@ "diagnosis_diskusage_ok": "Lo lòc d’emmagazinatge {mountpoint} (sul periferic {device}) a encara {free_abs_GB} Go ({free_percent}%) de liure !", "diagnosis_swap_none": "Lo sistèma a pas cap de memòria d’escambi. Auriatz de considerar d’ajustar almens 256 Mo d’escambi per evitar las situacions ont lo sistèma manca de memòria.", "diagnosis_swap_notsomuch": "Lo sistèma a solament {total_MB} de memòria d’escambi. Auriatz de considerar d’ajustar almens 256 Mo d’escambi per evitar las situacions ont lo sistèma manca de memòria." -} \ No newline at end of file +} diff --git a/src/yunohost/app.py b/src/yunohost/app.py index de2a74c9c..39793ec1a 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1574,6 +1574,7 @@ def app_config_apply(operation_logger, app, args): logger.success("Config updated as expected") return { + "app": app, "logs": operation_logger.success(), } diff --git a/src/yunohost/certificate.py b/src/yunohost/certificate.py index 5fae59060..fd792ccae 100644 --- a/src/yunohost/certificate.py +++ b/src/yunohost/certificate.py @@ -285,7 +285,6 @@ def _certificate_install_letsencrypt(domain_list, force=False, no_checks=False, operation_logger.start() - _configure_for_acme_challenge(domain) _fetch_and_enable_new_certificate(domain, staging, no_checks=no_checks) _install_cron(no_checks=no_checks) @@ -468,52 +467,6 @@ Subject: %s smtp.quit() -def _configure_for_acme_challenge(domain): - - nginx_conf_folder = "/etc/nginx/conf.d/%s.d" % domain - nginx_conf_file = "%s/000-acmechallenge.conf" % nginx_conf_folder - - nginx_configuration = ''' -location ^~ '/.well-known/acme-challenge/' -{ - default_type "text/plain"; - alias %s; -} - ''' % WEBROOT_FOLDER - - # Check there isn't a conflicting file for the acme-challenge well-known - # uri - for path in glob.glob('%s/*.conf' % nginx_conf_folder): - - if path == nginx_conf_file: - continue - - with open(path) as f: - contents = f.read() - - if '/.well-known/acme-challenge' in contents: - raise YunohostError('certmanager_conflicting_nginx_file', filepath=path) - - # Write the conf - if os.path.exists(nginx_conf_file): - logger.debug( - "Nginx configuration file for ACME challenge already exists for domain, skipping.") - return - - logger.debug( - "Adding Nginx configuration file for Acme challenge for domain %s.", domain) - - with open(nginx_conf_file, "w") as f: - f.write(nginx_configuration) - - # Assume nginx conf is okay, and reload it - # (FIXME : maybe add a check that it is, using nginx -t, haven't found - # any clean function already implemented in yunohost to do this though) - _run_service_command("reload", "nginx") - - app_ssowatconf() - - def _check_acme_challenge_configuration(domain): # Check nginx conf file exists nginx_conf_folder = "/etc/nginx/conf.d/%s.d" % domain diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 456dfa4bf..23b5a4179 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -236,8 +236,7 @@ def domain_dns_conf(domain, ttl=None): for record in record_list: result += "\n{name} {ttl} IN {type} {value}".format(**record) - is_cli = True if msettings.get('interface') == 'cli' else False - if is_cli: + if msettings.get('interface') == 'cli': logger.info(m18n.n("domain_dns_conf_is_just_a_recommendation")) return result @@ -406,10 +405,8 @@ def _build_dns_conf(domain, ttl=3600): "basic": [ # if ipv4 available {"type": "A", "name": "@", "value": "123.123.123.123", "ttl": 3600}, - {"type": "A", "name": "*", "value": "123.123.123.123", "ttl": 3600}, # if ipv6 available {"type": "AAAA", "name": "@", "value": "valid-ipv6", "ttl": 3600}, - {"type": "AAAA", "name": "*", "value": "valid-ipv6", "ttl": 3600}, ], "xmpp": [ {"type": "SRV", "name": "_xmpp-client._tcp", "value": "0 5 5222 domain.tld.", "ttl": 3600}, @@ -426,6 +423,10 @@ def _build_dns_conf(domain, ttl=3600): {"type": "TXT", "name": "_dmarc", "value": "\"v=DMARC1; p=none\"", "ttl": 3600} ], "extra": [ + # if ipv4 available + {"type": "A", "name": "*", "value": "123.123.123.123", "ttl": 3600}, + # if ipv6 available + {"type": "AAAA", "name": "*", "value": "valid-ipv6", "ttl": 3600}, {"type": "CAA", "name": "@", "value": "128 issue \"letsencrypt.org\"", "ttl": 3600}, ], "example_of_a_custom_rule": [ @@ -437,32 +438,21 @@ def _build_dns_conf(domain, ttl=3600): ipv4 = get_public_ip() ipv6 = get_public_ip(6) - basic = [] + ########################### + # Basic ipv4/ipv6 records # + ########################### - # Basic ipv4/ipv6 records + basic = [] if ipv4: - basic += [ - ["@", ttl, "A", ipv4], - ["*", ttl, "A", ipv4], - ] + basic.append(["@", ttl, "A", ipv4]) if ipv6: - basic += [ - ["@", ttl, "AAAA", ipv6], - ["*", ttl, "AAAA", ipv6], - ] + basic.append(["@", ttl, "AAAA", ipv6]) - # XMPP - xmpp = [ - ["_xmpp-client._tcp", ttl, "SRV", "0 5 5222 %s." % domain], - ["_xmpp-server._tcp", ttl, "SRV", "0 5 5269 %s." % domain], - ["muc", ttl, "CNAME", "@"], - ["pubsub", ttl, "CNAME", "@"], - ["vjud", ttl, "CNAME", "@"], - ["xmpp-upload", ttl, "CNAME", "@"], - ] + ######### + # Email # + ######### - # SPF record spf_record = '"v=spf1 a mx' if ipv4: spf_record += ' ip4:{ip4}'.format(ip4=ipv4) @@ -470,7 +460,6 @@ def _build_dns_conf(domain, ttl=3600): spf_record += ' ip6:{ip6}'.format(ip6=ipv6) spf_record += ' -all"' - # Email mail = [ ["@", ttl, "MX", "10 %s." % domain], ["@", ttl, "TXT", spf_record], @@ -485,12 +474,36 @@ def _build_dns_conf(domain, ttl=3600): ["_dmarc", ttl, "TXT", '"v=DMARC1; p=none"'], ] - # Extra - extra = [ - ["@", ttl, "CAA", '128 issue "letsencrypt.org"'] + ######## + # XMPP # + ######## + + xmpp = [ + ["_xmpp-client._tcp", ttl, "SRV", "0 5 5222 %s." % domain], + ["_xmpp-server._tcp", ttl, "SRV", "0 5 5269 %s." % domain], + ["muc", ttl, "CNAME", "@"], + ["pubsub", ttl, "CNAME", "@"], + ["vjud", ttl, "CNAME", "@"], + ["xmpp-upload", ttl, "CNAME", "@"], ] - # Official record + ######### + # Extra # + ######### + + extra = [] + + if ipv4: + extra.append(["*", ttl, "A", ipv4]) + if ipv6: + extra.append(["*", ttl, "AAAA", ipv6]) + + extra.append(["@", ttl, "CAA", '128 issue "letsencrypt.org"']) + + #################### + # Standard records # + #################### + records = { "basic": [{"name": name, "ttl": ttl, "type": type_, "value": value} for name, ttl, type_, value in basic], "xmpp": [{"name": name, "ttl": ttl, "type": type_, "value": value} for name, ttl, type_, value in xmpp], @@ -498,7 +511,12 @@ def _build_dns_conf(domain, ttl=3600): "extra": [{"name": name, "ttl": ttl, "type": type_, "value": value} for name, ttl, type_, value in extra], } - # Custom records + ################## + # Custom records # + ################## + + # Defined by custom hooks ships in apps for example ... + hook_results = hook_callback('custom_dns_rules', args=[domain]) for hook_name, results in hook_results.items(): # diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index 70817b3fe..6e597fbbf 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -258,7 +258,17 @@ def dyndns_update(operation_logger, dyn_host="dyndns.yunohost.org", domain=None, logger.info("Updated needed, going on...") dns_conf = _build_dns_conf(domain) - del dns_conf["extra"] # Ignore records from the 'extra' category + + for i, record in enumerate(dns_conf["extra"]): + # Ignore CAA record ... not sure why, we could probably enforce it... + if record[3] == "CAA": + del dns_conf["extra"][i] + + # Delete custom DNS records, we don't support them (have to explicitly + # authorize them on dynette) + for category in dns_conf.keys(): + if category not in ["basic", "mail", "xmpp", "extra"]: + del dns_conf[category] # Delete the old records for all domain/subdomains diff --git a/src/yunohost/permission.py b/src/yunohost/permission.py index 2aea6f4c4..67a0f57b0 100644 --- a/src/yunohost/permission.py +++ b/src/yunohost/permission.py @@ -196,6 +196,28 @@ def user_permission_reset(operation_logger, permission, sync_perm=True): return new_permission + +def user_permission_info(permission): + """ + Return informations about a specific permission + + Keyword argument: + permission -- Name of the permission (e.g. mail or nextcloud or wordpress.editors) + """ + + # By default, manipulate main permission + if "." not in permission: + permission = permission + ".main" + + # Fetch existing permission + + existing_permission = user_permission_list(full=True)["permissions"].get(permission, None) + if existing_permission is None: + raise YunohostError('permission_not_found', permission=permission) + + return existing_permission + + # # # The followings methods are *not* directly exposed. @@ -440,7 +462,7 @@ def _update_ldap_group_permission(permission, allowed, sync_perm=True): return existing_permission allowed = [allowed] if not isinstance(allowed, list) else allowed - + # Guarantee uniqueness of values in allowed, which would otherwise make ldap.update angry. allowed = set(allowed) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 77a120d46..3208bda60 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -43,7 +43,7 @@ from yunohost.dyndns import _dyndns_available, _dyndns_provides from yunohost.firewall import firewall_upnp from yunohost.service import service_start, service_enable from yunohost.regenconf import regen_conf -from yunohost.utils.packages import _dump_sources_list, _list_upgradable_apt_packages +from yunohost.utils.packages import _dump_sources_list, _list_upgradable_apt_packages, ynh_packages_version from yunohost.utils.error import YunohostError from yunohost.log import is_unit_operation, OperationLogger @@ -53,6 +53,8 @@ MIGRATIONS_STATE_PATH = "/etc/yunohost/migrations.yaml" logger = getActionLogger('yunohost.tools') +def tools_versions(): + return ynh_packages_version() def tools_ldapinit(): """ @@ -316,7 +318,7 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False, 'touch %s/index.txt' % ssl_dir, 'cp %s/openssl.cnf %s/openssl.ca.cnf' % (ssl_dir, ssl_dir), 'sed -i s/yunohost.org/%s/g %s/openssl.ca.cnf ' % (domain, ssl_dir), - 'openssl req -x509 -new -config %s/openssl.ca.cnf -days 3650 -out %s/ca/cacert.pem -keyout %s/ca/cakey.pem -nodes -batch' % (ssl_dir, ssl_dir, ssl_dir), + 'openssl req -x509 -new -config %s/openssl.ca.cnf -days 3650 -out %s/ca/cacert.pem -keyout %s/ca/cakey.pem -nodes -batch -subj /CN=%s/O=%s' % (ssl_dir, ssl_dir, ssl_dir, domain, os.path.splitext(domain)[0]), 'cp %s/ca/cacert.pem /etc/ssl/certs/ca-yunohost_crt.pem' % ssl_dir, 'update-ca-certificates' ] diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 39a2d8f15..282ec8407 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -165,12 +165,13 @@ def user_create(operation_logger, username, firstname, lastname, mail, password, operation_logger.start() # Get random UID/GID - all_uid = {x.pw_uid for x in pwd.getpwall()} - all_gid = {x.gr_gid for x in grp.getgrall()} + all_uid = {str(x.pw_uid) for x in pwd.getpwall()} + all_gid = {str(x.gr_gid) for x in grp.getgrall()} uid_guid_found = False while not uid_guid_found: - uid = str(random.randint(200, 99999)) + # LXC uid number is limited to 65536 by default + uid = str(random.randint(200, 65000)) uid_guid_found = uid not in all_uid and uid not in all_gid # Adapt values for LDAP @@ -201,8 +202,9 @@ def user_create(operation_logger, username, firstname, lastname, mail, password, except Exception as e: raise YunohostError('user_creation_failed', user=username, error=e) - # Invalidate passwd to take user creation into account + # Invalidate passwd and group to take user and group creation into account subprocess.call(['nscd', '-i', 'passwd']) + subprocess.call(['nscd', '-i', 'group']) try: # Attempt to create user home folder @@ -780,6 +782,11 @@ def user_permission_reset(permission, sync_perm=True): sync_perm=sync_perm) +def user_permission_info(permission): + import yunohost.permission + return yunohost.permission.user_permission_info(permission) + + # # SSH subcategory #