From ea26182ad15e94cb32a4fc068a691aa347d8f9cc Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 3 Jun 2020 20:17:49 +0200 Subject: [PATCH 01/19] Fix regen-conf mechanism for SSH --- locales/en.json | 1 + src/yunohost/regenconf.py | 22 +++++++++++++-- src/yunohost/tests/test_regenconf.py | 40 +++++++++++++++++++++++++++- src/yunohost/tools.py | 5 +--- 4 files changed, 61 insertions(+), 7 deletions(-) diff --git a/locales/en.json b/locales/en.json index 05f097345..428cbd2ac 100644 --- a/locales/en.json +++ b/locales/en.json @@ -535,6 +535,7 @@ "regenconf_dry_pending_applying": "Checking pending configuration which would have been applied for category '{category}'…", "regenconf_failed": "Could not regenerate the configuration for category(s): {categories}", "regenconf_pending_applying": "Applying pending configuration for category '{category}'…", + "regenconf_need_to_explicitly_specify_ssh": "The ssh configuration has been manually modified, but you need to explicitly specify category 'ssh' with --force to actually apply the changes.", "restore_already_installed_app": "An app with the ID '{app:s}' is already installed", "restore_already_installed_apps": "The following apps can't be restored because they are already installed: {apps}", "restore_app_failed": "Could not restore the app '{app:s}'", diff --git a/src/yunohost/regenconf.py b/src/yunohost/regenconf.py index 53deb7073..5bfc5df77 100644 --- a/src/yunohost/regenconf.py +++ b/src/yunohost/regenconf.py @@ -125,11 +125,12 @@ def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run # return the arguments to pass to the script return pre_args + [category_pending_path, ] - # Don't regen SSH if not specifically specified + ssh_explicitly_specified = isinstance(names, list) and "ssh" in names + + # By default, we regen everything if not names: names = hook_list('conf_regen', list_by='name', show_info=False)['hooks'] - names.remove('ssh') # Dirty hack for legacy code : avoid attempting to regen the conf for # glances because it got removed ... This is only needed *once* @@ -185,6 +186,20 @@ def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run succeed_regen = {} failed_regen = {} + # Here we are doing some weird legacy shit + # The thing is, on some very old or specific setup, the sshd_config file + # was absolutely not managed by the regenconf ... + # But we now want to make sure that this file is managed. + # However, we don't want to overwrite a specific custom sshd_config + # which may make the admin unhappy ... + # So : if the hash for this file does not exists, we set the hash as the + # hash of the pending configuration ... + # That way, the file will later appear as manually modified. + sshd_config = "/etc/ssh/sshd_config" + if category == "ssh" and sshd_config not in conf_hashes and sshd_config in conf_files: + conf_hashes[sshd_config] = _calculate_hash(conf_files[sshd_config]) + _update_conf_hashes(category, conf_hashes) + for system_path, pending_path in conf_files.items(): logger.debug("processing pending conf '%s' to system conf '%s'", pending_path, system_path) @@ -267,6 +282,9 @@ def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run logger.debug("> new conf is as current system conf") conf_status = 'managed' regenerated = True + elif force and system_path == sshd_config and not ssh_explicitly_specified: + logger.warning(m18n.n('regenconf_need_to_explicitly_specify_ssh')) + conf_status = 'modified' elif force: regenerated = _regen(system_path, pending_path) conf_status = 'force-updated' diff --git a/src/yunohost/tests/test_regenconf.py b/src/yunohost/tests/test_regenconf.py index 357f96c88..f618278f3 100644 --- a/src/yunohost/tests/test_regenconf.py +++ b/src/yunohost/tests/test_regenconf.py @@ -11,10 +11,11 @@ from moulinette.utils.filesystem import mkdir from yunohost.domain import _get_maindomain, domain_add, domain_remove, domain_list from yunohost.utils.error import YunohostError -from yunohost.regenconf import manually_modified_files, _get_conf_hashes, _force_clear_hashes +from yunohost.regenconf import manually_modified_files, regen_conf, _get_conf_hashes, _force_clear_hashes TEST_DOMAIN = "secondarydomain.test" TEST_DOMAIN_NGINX_CONFIG = "/etc/nginx/conf.d/secondarydomain.test.conf" +SSHD_CONFIG = "/etc/ssh/sshd_config" def setup_function(function): @@ -43,6 +44,8 @@ def clean(): assert TEST_DOMAIN_NGINX_CONFIG not in _get_conf_hashes("nginx") assert TEST_DOMAIN_NGINX_CONFIG not in manually_modified_files() + regen_conf(['ssh'], force=True) + def test_add_domain(): @@ -78,3 +81,38 @@ def test_add_domain_conf_already_exists(): assert os.path.exists(TEST_DOMAIN_NGINX_CONFIG) assert TEST_DOMAIN_NGINX_CONFIG in _get_conf_hashes("nginx") assert TEST_DOMAIN_NGINX_CONFIG not in manually_modified_files() + + +def test_ssh_conf_unmanaged(): + + _force_clear_hashes([SSHD_CONFIG]) + + assert SSHD_CONFIG not in _get_conf_hashes("ssh") + + regen_conf() + + assert SSHD_CONFIG in _get_conf_hashes("ssh") + + +def test_ssh_conf_unmanaged_and_manually_modified(mocker): + + _force_clear_hashes([SSHD_CONFIG]) + os.system("echo ' ' >> %s" % SSHD_CONFIG) + + assert SSHD_CONFIG not in _get_conf_hashes("ssh") + + regen_conf() + + assert SSHD_CONFIG in _get_conf_hashes("ssh") + assert SSHD_CONFIG in manually_modified_files() + + with message(mocker, "regenconf_need_to_explicitly_specify_ssh"): + regen_conf(force=True) + + assert SSHD_CONFIG in _get_conf_hashes("ssh") + assert SSHD_CONFIG in manually_modified_files() + + regen_conf(['ssh'], force=True) + + assert SSHD_CONFIG in _get_conf_hashes("ssh") + assert SSHD_CONFIG not in manually_modified_files() diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 3b683e967..9221e6c7d 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -369,6 +369,7 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False, service_start("yunohost-firewall") regen_conf(force=True) + regen_conf(names=["ssh"], force=True) # Restore original ssh conf, as chosen by the # admin during the initial install @@ -382,10 +383,6 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False, original_sshd_conf = '/etc/ssh/sshd_config.before_yunohost' if os.path.exists(original_sshd_conf): os.rename(original_sshd_conf, '/etc/ssh/sshd_config') - else: - # We need to explicitly ask the regen conf to regen ssh - # (by default, i.e. first argument = None, it won't because it's too touchy) - regen_conf(names=["ssh"], force=True) logger.success(m18n.n('yunohost_configured')) From 6ffd0d8f5a0014e6a4f1fd408924c698d91f174f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Tille?= Date: Mon, 25 May 2020 10:45:40 +0200 Subject: [PATCH 02/19] Don't add unprotected_urls if it's already in skipped_urls --- src/yunohost/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 0cb420d16..6af892b03 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1358,7 +1358,7 @@ def app_ssowatconf(): url = _sanitized_absolute_url(perm_info["url"]) perm_info["url"] = url if "visitors" in perm_info["allowed"]: - if url not in unprotected_urls: + if url not in unprotected_urls and url not in skipped_urls: unprotected_urls.append(url) # Legacy stuff : we remove now protected-urls that might have been declared as unprotected earlier... From d00261d7a528b65670baad9c2ed8ec5785eb12e7 Mon Sep 17 00:00:00 2001 From: SohKa Date: Tue, 26 May 2020 19:17:03 +0200 Subject: [PATCH 03/19] Add pre-defined DHE group and set up Nginx to use it --- data/other/dh2048.pem | 8 ++++++++ data/templates/nginx/security.conf.inc | 5 ++--- debian/install | 1 + 3 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 data/other/dh2048.pem diff --git a/data/other/dh2048.pem b/data/other/dh2048.pem new file mode 100644 index 000000000..9b182b720 --- /dev/null +++ b/data/other/dh2048.pem @@ -0,0 +1,8 @@ +-----BEGIN DH PARAMETERS----- +MIIBCAKCAQEA//////////+t+FRYortKmq/cViAnPTzx2LnFg84tNpWp4TZBFGQz ++8yTnc4kmz75fS/jY2MMddj2gbICrsRhetPfHtXV/WVhJDP1H18GbtCFY2VVPe0a +87VXE15/V8k1mE8McODmi3fipona8+/och3xWKE2rec1MKzKT0g6eXq8CrGCsyT7 +YdEIqUuyyOP7uWrat2DX9GgdT0Kj3jlN9K5W7edjcrsZCwenyO4KbXCeAvzhzffi +7MA0BM0oNC9hkXL+nOmFg/+OTxIy7vKBg8P+OxtMb61zO7X8vC7CIAXFjvGDfRaD +ssbzSibBsu/6iGtCOGEoXJf//////////wIBAg== +-----END DH PARAMETERS----- diff --git a/data/templates/nginx/security.conf.inc b/data/templates/nginx/security.conf.inc index dea0f49db..e221dc0ff 100644 --- a/data/templates/nginx/security.conf.inc +++ b/data/templates/nginx/security.conf.inc @@ -14,9 +14,8 @@ ssl_protocols TLSv1.2; ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; ssl_prefer_server_ciphers off; -# Uncomment the following directive after DH generation -# > openssl dhparam -out /etc/ssl/private/dh2048.pem -outform PEM -2 2048 -#ssl_dhparam /etc/ssl/private/dh2048.pem; +# Pre-defined FFDHE group (RFC 7919) +ssl_dhparam /etc/ssl/dh2048.pem; # Follows the Web Security Directives from the Mozilla Dev Lab and the Mozilla Obervatory + Partners # https://wiki.mozilla.org/Security/Guidelines/Web_Security diff --git a/debian/install b/debian/install index a814d1617..e59214678 100644 --- a/debian/install +++ b/debian/install @@ -8,6 +8,7 @@ data/other/yunoprompt.service /etc/systemd/system/ data/other/password/* /usr/share/yunohost/other/password/ data/other/dpkg-origins/yunohost /etc/dpkg/origins data/other/dnsbl_list.yml /usr/share/yunohost/other/ +data/other/dh2048.pem /etc/ssl/ data/other/* /usr/share/yunohost/yunohost-config/moulinette/ data/templates/* /usr/share/yunohost/templates/ data/helpers /usr/share/yunohost/ From ea43f68552489779177dc78445abab1688544008 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 Jun 2020 18:41:05 +0200 Subject: [PATCH 04/19] Add comments about where the DH param comes from --- data/templates/nginx/security.conf.inc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data/templates/nginx/security.conf.inc b/data/templates/nginx/security.conf.inc index e221dc0ff..9de40f9cf 100644 --- a/data/templates/nginx/security.conf.inc +++ b/data/templates/nginx/security.conf.inc @@ -15,6 +15,8 @@ ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDS ssl_prefer_server_ciphers off; # Pre-defined FFDHE group (RFC 7919) +# From https://ssl-config.mozilla.org/ffdhe2048.txt +# https://security.stackexchange.com/a/149818 ssl_dhparam /etc/ssl/dh2048.pem; # Follows the Web Security Directives from the Mozilla Dev Lab and the Mozilla Obervatory + Partners From daed21c975579c5cc777c2cdfa37b7fa0f0c9a7d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 Jun 2020 18:42:40 +0200 Subject: [PATCH 05/19] Rename dh2048.pem to ffdhe2048.pem --- data/other/{dh2048.pem => ffdhe2048.pem} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename data/other/{dh2048.pem => ffdhe2048.pem} (100%) diff --git a/data/other/dh2048.pem b/data/other/ffdhe2048.pem similarity index 100% rename from data/other/dh2048.pem rename to data/other/ffdhe2048.pem From 5a74155dd4b9e33525a740b680ea6e8b0fe2868c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 Jun 2020 18:43:24 +0200 Subject: [PATCH 06/19] Put ffdhe2048.pem in /usr/share/yunohost/other/ instead of /etc/ssl --- debian/install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/install b/debian/install index e59214678..777d7973e 100644 --- a/debian/install +++ b/debian/install @@ -8,7 +8,7 @@ data/other/yunoprompt.service /etc/systemd/system/ data/other/password/* /usr/share/yunohost/other/password/ data/other/dpkg-origins/yunohost /etc/dpkg/origins data/other/dnsbl_list.yml /usr/share/yunohost/other/ -data/other/dh2048.pem /etc/ssl/ +data/other/ffdhe2048.pem /usr/share/yunohost/other/ data/other/* /usr/share/yunohost/yunohost-config/moulinette/ data/templates/* /usr/share/yunohost/templates/ data/helpers /usr/share/yunohost/ From 35dc0e797a3f69873b7c0a199dce99737645cbb8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 Jun 2020 18:44:02 +0200 Subject: [PATCH 07/19] Propagate location change for ffdhe2048.pem --- data/templates/nginx/security.conf.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/templates/nginx/security.conf.inc b/data/templates/nginx/security.conf.inc index 9de40f9cf..4c6461506 100644 --- a/data/templates/nginx/security.conf.inc +++ b/data/templates/nginx/security.conf.inc @@ -17,7 +17,7 @@ ssl_prefer_server_ciphers off; # Pre-defined FFDHE group (RFC 7919) # From https://ssl-config.mozilla.org/ffdhe2048.txt # https://security.stackexchange.com/a/149818 -ssl_dhparam /etc/ssl/dh2048.pem; +ssl_dhparam /usr/share/yunohost/other/ffdhe2048.pem; # Follows the Web Security Directives from the Mozilla Dev Lab and the Mozilla Obervatory + Partners # https://wiki.mozilla.org/Security/Guidelines/Web_Security From 33fc9d78d28498b6ce5b213b7adab739a2668bf1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 Jun 2020 19:24:50 +0200 Subject: [PATCH 08/19] Make sure to propagate change in slapd systemd conf during initial install --- data/hooks/conf_regen/06-slapd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data/hooks/conf_regen/06-slapd b/data/hooks/conf_regen/06-slapd index 5fd727a2d..c8fba52fc 100755 --- a/data/hooks/conf_regen/06-slapd +++ b/data/hooks/conf_regen/06-slapd @@ -12,6 +12,8 @@ do_init_regen() { do_pre_regen "" + systemctl daemon-reload + # fix some permissions chown root:openldap /etc/ldap/slapd.conf chown -R openldap:openldap /etc/ldap/schema/ From 5dfa45518d990a396a3d0250cea5e01625d63b38 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 5 Jun 2020 16:58:46 +0200 Subject: [PATCH 09/19] More accurate grep to avoid mistakenly grepping commented lines... --- data/helpers.d/apt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/apt b/data/helpers.d/apt index c48cc0add..9381c0f92 100644 --- a/data/helpers.d/apt +++ b/data/helpers.d/apt @@ -260,8 +260,8 @@ ynh_install_app_dependencies () { # And we have packages from sury installed (7.0.33-10+weirdshiftafter instead of 7.0.33-0 on debian) if dpkg --list | grep "php7.0" | grep --quiet --invert-match "7.0.33-0+deb9" then - # And sury ain't already installed - if ! grep --line-number --recursive --quiet "sury" /etc/apt/sources.list* + # And sury ain't already in sources.lists + if ! grep --recursive --quiet "^ *deb.*sury" /etc/apt/sources.list* then # Re-add sury ynh_install_extra_repo --repo="https://packages.sury.org/php/ $(ynh_get_debian_release) main" --key="https://packages.sury.org/php/apt.gpg" --name=extra_php_version From d30ec3953ad463891810921270c9eb59b54b888e Mon Sep 17 00:00:00 2001 From: ericgaspar Date: Sat, 30 May 2020 17:30:38 +0200 Subject: [PATCH 10/19] Update n to upstream version (6.5.1) --- data/helpers.d/nodejs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/data/helpers.d/nodejs b/data/helpers.d/nodejs index 3ad0c400f..61e02c8ad 100644 --- a/data/helpers.d/nodejs +++ b/data/helpers.d/nodejs @@ -16,8 +16,8 @@ ynh_install_n () { ynh_print_info --message="Installation of N - Node.js version management" # Build an app.src for n mkdir --parents "../conf" - echo "SOURCE_URL=https://github.com/tj/n/archive/v4.1.0.tar.gz -SOURCE_SUM=3983fa3f00d4bf85ba8e21f1a590f6e28938093abe0bb950aeea52b1717471fc" > "../conf/n.src" + echo "SOURCE_URL=https://github.com/tj/n/archive/v6.5.1.tar.gz +SOURCE_SUM=5833f15893b9951a9ed59487e87b6c181d96b83a525846255872c4f92f0d25dd" > "../conf/n.src" # Download and extract n ynh_setup_source --dest_dir="$n_install_dir/git" --source_id=n # Install n @@ -125,7 +125,10 @@ ynh_install_nodejs () { test -x /usr/bin/npm && mv /usr/bin/npm /usr/bin/npm_n # If n is not previously setup, install it - if ! test $(n --version > /dev/null 2>&1) + if ! type n > /dev/null 2>&1 + then + ynh_install_n + elif dpkg --compare-version "$($n_install_dir/bin/n --version | cut -d" " -f2)" lt "6.5.1" then ynh_install_n fi From 9810ccbf99a60cae3528dbc5316ebb520b47d92b Mon Sep 17 00:00:00 2001 From: ericgaspar Date: Sun, 31 May 2020 18:02:29 +0200 Subject: [PATCH 11/19] Update nodejs --- data/helpers.d/nodejs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/nodejs b/data/helpers.d/nodejs index 61e02c8ad..5cb5c4fae 100644 --- a/data/helpers.d/nodejs +++ b/data/helpers.d/nodejs @@ -125,7 +125,7 @@ ynh_install_nodejs () { test -x /usr/bin/npm && mv /usr/bin/npm /usr/bin/npm_n # If n is not previously setup, install it - if ! type n > /dev/null 2>&1 + if ! type $(n --version > /dev/null 2>&1) then ynh_install_n elif dpkg --compare-version "$($n_install_dir/bin/n --version | cut -d" " -f2)" lt "6.5.1" From 9c115709266831bed6d3815fd9dc5262256a44b5 Mon Sep 17 00:00:00 2001 From: ericgaspar Date: Wed, 3 Jun 2020 19:11:56 +0200 Subject: [PATCH 12/19] fix stupid mistake --- data/helpers.d/nodejs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/nodejs b/data/helpers.d/nodejs index 5cb5c4fae..ba9225293 100644 --- a/data/helpers.d/nodejs +++ b/data/helpers.d/nodejs @@ -125,7 +125,7 @@ ynh_install_nodejs () { test -x /usr/bin/npm && mv /usr/bin/npm /usr/bin/npm_n # If n is not previously setup, install it - if ! type $(n --version > /dev/null 2>&1) + if ! test $(n --version > /dev/null 2>&1) then ynh_install_n elif dpkg --compare-version "$($n_install_dir/bin/n --version | cut -d" " -f2)" lt "6.5.1" From 6e2e9b830d5cc81350038b1ac0af455adccf0f90 Mon Sep 17 00:00:00 2001 From: ericgaspar Date: Thu, 4 Jun 2020 11:23:51 +0200 Subject: [PATCH 13/19] add n_version variable --- data/helpers.d/nodejs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/nodejs b/data/helpers.d/nodejs index ba9225293..f1a4b3788 100644 --- a/data/helpers.d/nodejs +++ b/data/helpers.d/nodejs @@ -1,5 +1,6 @@ #!/bin/bash +n_version=6.5.1 n_install_dir="/opt/node_n" node_version_path="$n_install_dir/n/versions/node" # N_PREFIX is the directory of n, it needs to be loaded as a environment variable. @@ -16,7 +17,7 @@ ynh_install_n () { ynh_print_info --message="Installation of N - Node.js version management" # Build an app.src for n mkdir --parents "../conf" - echo "SOURCE_URL=https://github.com/tj/n/archive/v6.5.1.tar.gz + echo "SOURCE_URL=https://github.com/tj/n/archive/v${n_version}.tar.gz SOURCE_SUM=5833f15893b9951a9ed59487e87b6c181d96b83a525846255872c4f92f0d25dd" > "../conf/n.src" # Download and extract n ynh_setup_source --dest_dir="$n_install_dir/git" --source_id=n @@ -128,7 +129,7 @@ ynh_install_nodejs () { if ! test $(n --version > /dev/null 2>&1) then ynh_install_n - elif dpkg --compare-version "$($n_install_dir/bin/n --version | cut -d" " -f2)" lt "6.5.1" + elif dpkg --compare-version "$($n_install_dir/bin/n --version | cut -d" " -f2)" lt $n_version then ynh_install_n fi From f0c9f612af0ef4757ef4c4283c5de45d7644d3e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Gaspar?= <46165813+ericgaspar@users.noreply.github.com> Date: Thu, 4 Jun 2020 15:04:02 +0200 Subject: [PATCH 14/19] Update data/helpers.d/nodejs Co-authored-by: Kayou --- data/helpers.d/nodejs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/nodejs b/data/helpers.d/nodejs index f1a4b3788..7ba692f96 100644 --- a/data/helpers.d/nodejs +++ b/data/helpers.d/nodejs @@ -129,7 +129,7 @@ ynh_install_nodejs () { if ! test $(n --version > /dev/null 2>&1) then ynh_install_n - elif dpkg --compare-version "$($n_install_dir/bin/n --version | cut -d" " -f2)" lt $n_version + elif dpkg --compare-versions "$($n_install_dir/bin/n --version)" lt $n_version then ynh_install_n fi From 3b1a67d96395711add290b6b98d5783539d55a1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Gaspar?= <46165813+ericgaspar@users.noreply.github.com> Date: Fri, 5 Jun 2020 11:27:43 +0200 Subject: [PATCH 15/19] Update data/helpers.d/nodejs Co-authored-by: Kayou --- data/helpers.d/nodejs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/helpers.d/nodejs b/data/helpers.d/nodejs index 7ba692f96..75472c278 100644 --- a/data/helpers.d/nodejs +++ b/data/helpers.d/nodejs @@ -126,7 +126,7 @@ ynh_install_nodejs () { test -x /usr/bin/npm && mv /usr/bin/npm /usr/bin/npm_n # If n is not previously setup, install it - if ! test $(n --version > /dev/null 2>&1) + if ! $n_install_dir/bin/n --version > /dev/null 2>&1 then ynh_install_n elif dpkg --compare-versions "$($n_install_dir/bin/n --version)" lt $n_version From 5e877449ed818ae1223035e5789cd5c181edb1ed Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 6 Jun 2020 01:16:54 +0200 Subject: [PATCH 16/19] Set sury default pinning to 600 --- data/helpers.d/apt | 2 +- data/helpers.d/php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/helpers.d/apt b/data/helpers.d/apt index 9381c0f92..5589830c3 100644 --- a/data/helpers.d/apt +++ b/data/helpers.d/apt @@ -264,7 +264,7 @@ ynh_install_app_dependencies () { if ! grep --recursive --quiet "^ *deb.*sury" /etc/apt/sources.list* then # Re-add sury - ynh_install_extra_repo --repo="https://packages.sury.org/php/ $(ynh_get_debian_release) main" --key="https://packages.sury.org/php/apt.gpg" --name=extra_php_version + ynh_install_extra_repo --repo="https://packages.sury.org/php/ $(ynh_get_debian_release) main" --key="https://packages.sury.org/php/apt.gpg" --name=extra_php_version --priority=600 # Pin this sury repository to prevent sury of doing shit for package_to_not_upgrade in "php" "php-fpm" "php-mysql" "php-xml" "php-zip" "php-mbstring" "php-ldap" "php-gd" "php-curl" "php-bz2" "php-json" "php-sqlite3" "php-intl" "openssl" "libssl1.1" "libssl-dev" diff --git a/data/helpers.d/php b/data/helpers.d/php index b434c7002..4902e3292 100644 --- a/data/helpers.d/php +++ b/data/helpers.d/php @@ -354,7 +354,7 @@ ynh_install_php () { fi # Add an extra repository for those packages - ynh_install_extra_repo --repo="https://packages.sury.org/php/ $(ynh_get_debian_release) main" --key="https://packages.sury.org/php/apt.gpg" --priority=995 --name=extra_php_version + ynh_install_extra_repo --repo="https://packages.sury.org/php/ $(ynh_get_debian_release) main" --key="https://packages.sury.org/php/apt.gpg" --priority=995 --name=extra_php_version --priority=600 # Install requested dependencies from this extra repository. # Install php-fpm first, otherwise php will install apache as a dependency. From 28a922de51caec3c6ea44848eb928f404d4abd02 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 28 May 2020 21:37:15 +0200 Subject: [PATCH 17/19] Get rid of old hashes of file that 'are expected to be removed' and are indeed 'already removed' --- src/yunohost/regenconf.py | 6 +++ src/yunohost/tests/test_regenconf.py | 55 +++++++++++++++++++++++++--- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/yunohost/regenconf.py b/src/yunohost/regenconf.py index 5bfc5df77..ddaf7fe30 100644 --- a/src/yunohost/regenconf.py +++ b/src/yunohost/regenconf.py @@ -503,6 +503,12 @@ def _update_conf_hashes(category, hashes): if category_conf is None: category_conf = {} + # If a file shall be removed and is indeed removed, forget entirely about + # that path. + # It avoid keeping weird old entries like + # /etc/nginx/conf.d/some.domain.that.got.removed.conf + hashes = {path: hash_ for path, hash_ in hashes.items() if hash_ is not None or os.path.exists(path)} + category_conf['conffiles'] = hashes categories[category] = category_conf _save_regenconf_infos(categories) diff --git a/src/yunohost/tests/test_regenconf.py b/src/yunohost/tests/test_regenconf.py index f618278f3..0e2d7a418 100644 --- a/src/yunohost/tests/test_regenconf.py +++ b/src/yunohost/tests/test_regenconf.py @@ -1,8 +1,4 @@ -import glob import os -import pytest -import shutil -import requests from conftest import message, raiseYunohostError @@ -14,7 +10,8 @@ from yunohost.utils.error import YunohostError from yunohost.regenconf import manually_modified_files, regen_conf, _get_conf_hashes, _force_clear_hashes TEST_DOMAIN = "secondarydomain.test" -TEST_DOMAIN_NGINX_CONFIG = "/etc/nginx/conf.d/secondarydomain.test.conf" +TEST_DOMAIN_NGINX_CONFIG = "/etc/nginx/conf.d/%s.conf" % TEST_DOMAIN +TEST_DOMAIN_DNSMASQ_CONFIG = "/etc/dnsmasq.d/%s" % TEST_DOMAIN SSHD_CONFIG = "/etc/ssh/sshd_config" def setup_function(function): @@ -22,11 +19,13 @@ def setup_function(function): _force_clear_hashes([TEST_DOMAIN_NGINX_CONFIG]) clean() + def teardown_function(function): clean() _force_clear_hashes([TEST_DOMAIN_NGINX_CONFIG]) + def clean(): assert os.system("pgrep slapd >/dev/null") == 0 @@ -116,3 +115,49 @@ def test_ssh_conf_unmanaged_and_manually_modified(mocker): assert SSHD_CONFIG in _get_conf_hashes("ssh") assert SSHD_CONFIG not in manually_modified_files() + + +def test_stale_hashes_get_removed_if_empty(): + """ + This is intended to test that if a file gets removed and is indeed removed, + we don't keep a useless empty hash corresponding to an old file. + In this case, we test this using the dnsmasq conf file (we don't do this + using the nginx conf file because it's already force-removed during + domain_remove()) + """ + + domain_add(TEST_DOMAIN) + + assert os.path.exists(TEST_DOMAIN_DNSMASQ_CONFIG) + assert TEST_DOMAIN_DNSMASQ_CONFIG in _get_conf_hashes("dnsmasq") + + domain_remove(TEST_DOMAIN) + + assert not os.path.exists(TEST_DOMAIN_DNSMASQ_CONFIG) + assert TEST_DOMAIN_DNSMASQ_CONFIG not in _get_conf_hashes("dnsmasq") + + +def test_stale_hashes_if_file_manually_deleted(): + """ + Same as other test, but manually delete the file in between and check + behavior + """ + + domain_add(TEST_DOMAIN) + + assert os.path.exists(TEST_DOMAIN_DNSMASQ_CONFIG) + assert TEST_DOMAIN_DNSMASQ_CONFIG in _get_conf_hashes("dnsmasq") + + os.remove(TEST_DOMAIN_DNSMASQ_CONFIG) + + assert not os.path.exists(TEST_DOMAIN_DNSMASQ_CONFIG) + + regen_conf(names=["dnsmasq"]) + + assert not os.path.exists(TEST_DOMAIN_DNSMASQ_CONFIG) + assert TEST_DOMAIN_DNSMASQ_CONFIG in _get_conf_hashes("dnsmasq") + + domain_remove(TEST_DOMAIN) + + assert not os.path.exists(TEST_DOMAIN_DNSMASQ_CONFIG) + assert TEST_DOMAIN_DNSMASQ_CONFIG not in _get_conf_hashes("dnsmasq") From f27355d4b16560d7d9d694e3e9e3742a68dd63a5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 28 May 2020 21:43:32 +0200 Subject: [PATCH 18/19] Properly handle stale hashes of files even when file got manually removed/modified --- src/yunohost/regenconf.py | 73 +++++++++++++++++++++++++--- src/yunohost/tests/test_regenconf.py | 45 +++++++++++++++++ 2 files changed, 112 insertions(+), 6 deletions(-) diff --git a/src/yunohost/regenconf.py b/src/yunohost/regenconf.py index ddaf7fe30..4aad5f921 100644 --- a/src/yunohost/regenconf.py +++ b/src/yunohost/regenconf.py @@ -200,6 +200,42 @@ def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run conf_hashes[sshd_config] = _calculate_hash(conf_files[sshd_config]) _update_conf_hashes(category, conf_hashes) + # Consider the following scenario: + # - you add a domain foo.bar + # - the regen-conf creates file /etc/dnsmasq.d/foo.bar + # - the admin manually *deletes* /etc/dnsmasq.d/foo.bar + # - the file is now understood as manually deleted because there's the old file hash in regenconf.yml + # + # ... so far so good, that's the expected behavior. + # + # But then: + # - the admin remove domain foo.bar entirely + # - but now the hash for /etc/dnsmasq.d/foo.bar is *still* in + # regenconf.yml and and the file is still flagged as manually + # modified/deleted... And the user cannot even do anything about it + # except removing the hash in regenconf.yml... + # + # Expected behavior: it should forget about that + # hash because dnsmasq's regen-conf doesn't say anything about what's + # the state of that file so it should assume that it should be deleted. + # + # - then the admin tries to *re-add* foo.bar ! + # - ... but because the file is still flagged as manually modified + # the regen-conf refuses to re-create the file. + # + # Excepted behavior : the regen-conf should have forgot about the hash + # from earlier and this wouldnt happen. + # ------ + # conf_files contain files explicitly set by the current regen conf run + # conf_hashes contain all files known from the past runs + # we compare these to get the list of stale hashes and flag the file as + # "should be removed" + stale_files = set(conf_hashes.keys()) - set(conf_files.keys()) + stale_files_with_non_empty_hash = [f for f in stale_files if conf_hashes.get(f)] + for f in stale_files_with_non_empty_hash: + conf_files[f] = None + # End discussion about stale file hashes + for system_path, pending_path in conf_files.items(): logger.debug("processing pending conf '%s' to system conf '%s'", pending_path, system_path) @@ -211,20 +247,45 @@ def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run system_path, pending_path, True) if with_diff else None # Check if the conf must be removed - to_remove = True if os.path.getsize(pending_path) == 0 else False + to_remove = True if pending_path and os.path.getsize(pending_path) == 0 else False # Retrieve and calculate hashes system_hash = _calculate_hash(system_path) saved_hash = conf_hashes.get(system_path, None) new_hash = None if to_remove else _calculate_hash(pending_path) + # -> configuration was previously managed by yunohost but should now + # be removed / unmanaged + if system_path in stale_files_with_non_empty_hash: + # File is already deleted, so let's just silently forget about this hash entirely + if not system_hash: + logger.debug("> forgetting about stale file/hash") + conf_hashes[system_path] = None + conf_status = 'forget-about-it' + regenerated = True + # Otherwise there's still a file on the system but it's not managed by + # Yunohost anymore... But if user requested --force we shall + # force-erase it + elif force: + logger.debug("> force-remove stale file") + regenerated = _regen(system_path) + conf_status = 'force-removed' + # Otherwise, flag the file as manually modified + else: + logger.warning(m18n.n( + 'regenconf_file_manually_modified', + conf=system_path)) + conf_status = 'modified' + # -> system conf does not exists - if not system_hash: + elif not system_hash: if to_remove: logger.debug("> system conf is already removed") os.remove(pending_path) - continue - if not saved_hash or force: + conf_hashes[system_path] = None + conf_status = 'forget-about-it' + regenerated = True + elif not saved_hash or force: if force: logger.debug("> system conf has been manually removed") conf_status = 'force-created' @@ -301,7 +362,7 @@ def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run if regenerated: succeed_regen[system_path] = conf_result conf_hashes[system_path] = new_hash - if os.path.isfile(pending_path): + if pending_path and os.path.isfile(pending_path): os.remove(pending_path) else: failed_regen[system_path] = conf_result @@ -417,7 +478,7 @@ def _get_files_diff(orig_file, new_file, as_string=False, skip_header=True): def _calculate_hash(path): """Calculate the MD5 hash of a file""" - if not os.path.exists(path): + if not path or not os.path.exists(path): return None hasher = hashlib.md5() diff --git a/src/yunohost/tests/test_regenconf.py b/src/yunohost/tests/test_regenconf.py index 0e2d7a418..f50332b9a 100644 --- a/src/yunohost/tests/test_regenconf.py +++ b/src/yunohost/tests/test_regenconf.py @@ -161,3 +161,48 @@ def test_stale_hashes_if_file_manually_deleted(): assert not os.path.exists(TEST_DOMAIN_DNSMASQ_CONFIG) assert TEST_DOMAIN_DNSMASQ_CONFIG not in _get_conf_hashes("dnsmasq") + +# This test only works if you comment the part at the end of the regen-conf in +# dnsmasq that auto-flag /etc/dnsmasq.d/foo.bar as "to be removed" (using touch) +# ... But we want to keep it because they also possibly flag files that were +# never known by the regen-conf (e.g. if somebody adds a +# /etc/dnsmasq.d/my.custom.extension) +# Ideally we could use a system that's able to properly state 'no file in this +# folder should exist except the ones excplicitly defined by regen-conf' but +# that's too much work for the scope of this commit. +# +# ... Anyway, the proper way to write these tests would be to use a dummy +# regen-conf hook just for tests but meh I'm lazy +# +#def test_stale_hashes_if_file_manually_modified(): +# """ +# Same as other test, but manually delete the file in between and check +# behavior +# """ +# +# domain_add(TEST_DOMAIN) +# +# assert os.path.exists(TEST_DOMAIN_DNSMASQ_CONFIG) +# assert TEST_DOMAIN_DNSMASQ_CONFIG in _get_conf_hashes("dnsmasq") +# +# os.system("echo '#pwet' > %s" % TEST_DOMAIN_DNSMASQ_CONFIG) +# +# assert os.path.exists(TEST_DOMAIN_DNSMASQ_CONFIG) +# assert open(TEST_DOMAIN_DNSMASQ_CONFIG).read().strip() == "#pwet" +# +# regen_conf(names=["dnsmasq"]) +# +# assert os.path.exists(TEST_DOMAIN_DNSMASQ_CONFIG) +# assert open(TEST_DOMAIN_DNSMASQ_CONFIG).read().strip() == "#pwet" +# assert TEST_DOMAIN_DNSMASQ_CONFIG in _get_conf_hashes("dnsmasq") +# +# domain_remove(TEST_DOMAIN) +# +# assert os.path.exists(TEST_DOMAIN_DNSMASQ_CONFIG) +# assert open(TEST_DOMAIN_DNSMASQ_CONFIG).read().strip() == "#pwet" +# assert TEST_DOMAIN_DNSMASQ_CONFIG in _get_conf_hashes("dnsmasq") +# +# regen_conf(names=["dnsmasq"], force=True) +# +# assert not os.path.exists(TEST_DOMAIN_DNSMASQ_CONFIG) +# assert TEST_DOMAIN_DNSMASQ_CONFIG not in _get_conf_hashes("dnsmasq") From 051d14a945ef00317cf79061389fb15b0fee951b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 6 Jun 2020 02:08:05 +0200 Subject: [PATCH 19/19] Update changelog for 3.8.4.8 --- debian/changelog | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/debian/changelog b/debian/changelog index 5abadb3f4..9f07e62d1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,18 @@ +yunohost (3.8.4.8) stable; urgency=low + + - [fix] Don't add unprotected_urls if it's already in skipped_urls (#1005) + - [enh] Add pre-defined DHE group and set up Nginx to use it (#1007) + - [fix] Make sure to propagate change in slapd systemd conf during initial install (2d42480f) + - [fix] More accurate grep to avoid mistakenly grepping commented lines... (2408a620) + - [enh] Update n to 6.5.1 (#1012) + - [fix] Set sury default pinning to 600 (653c5fde) + - [enh] Clean stale file/hashes in regen-conf (#1009) + - [fix] Weirdness in regen-conf mechanism for SSH conf (#1014) + + Thanks to all contributors <3 ! (É. Gaspar, Josué, SohKa) + + -- Alexandre Aubin Sat, 06 Jun 2020 01:59:08 +0200 + yunohost (3.8.4.7) stable; urgency=low - [fix] Remove some remains of glances (17eec25e)