From 913b02e4fcc32622a3a3763680333980c4c04028 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 24 Apr 2021 21:34:57 +0200 Subject: [PATCH 1/6] Add diagnosis section to check that app are in catalog with good quality + check for deprecated practices --- data/hooks/diagnosis/80-apps.py | 82 +++++++++++++++++++++++++++++++++ locales/en.json | 8 ++++ src/yunohost/app.py | 10 ++-- 3 files changed, 96 insertions(+), 4 deletions(-) create mode 100644 data/hooks/diagnosis/80-apps.py diff --git a/data/hooks/diagnosis/80-apps.py b/data/hooks/diagnosis/80-apps.py new file mode 100644 index 000000000..ce54faef1 --- /dev/null +++ b/data/hooks/diagnosis/80-apps.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python + +import os +import re + +from yunohost.app import app_info, app_list +from moulinette.utils.filesystem import read_file + +from yunohost.settings import settings_get +from yunohost.diagnosis import Diagnoser +from yunohost.regenconf import _get_regenconf_infos, _calculate_hash +from moulinette.utils.filesystem import read_file + + +class AppDiagnoser(Diagnoser): + + id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] + cache_duration = 300 + dependencies = [] + + def run(self): + + apps = app_list(full=True)["apps"] + for app in apps: + app["issues"] = list(self.issues(app)) + + if not any(app["issues"] for app in apps): + yield dict( + meta={"test": "apps"}, + status="SUCCESS", + summary="diagnosis_apps_allgood", + ) + else: + for app in apps: + + if not app["issues"]: + continue + + level = "ERROR" if any(issue[0] == "error" for issue in app["issues"]) else "WARNING" + + yield dict( + meta={"test": "apps", "app": app["name"]}, + status=level, + summary="diagnosis_apps_issue", + details=[issue[1] for issue in app["issues"]] + ) + + def issues(self, app): + + # Check quality level in catalog + + if not app.get("from_catalog") or app["from_catalog"].get("state") != "working": + yield ("error", "diagnosis_apps_not_in_app_catalog") + elif not isinstance(app["from_catalog"].get("level"), int) or app["from_catalog"]["level"] == 0: + yield ("error", "diagnosis_apps_broken") + elif app["from_catalog"]["level"] <= 4: + yield ("warning", "diagnosis_apps_bad_quality") + + # Check for super old, deprecated practices + + yunohost_version_req = app["manifest"].get("requirements", {}).get("yunohost", "").strip(">= ") + if yunohost_version_req.startswith("2."): + yield ("error", "diagnosis_apps_outdated_ynh_requirement") + + deprecated_helpers = [ + "yunohost app setting", + "yunohost app checkurl", + "yunohost app checkport", + "yunohost app initdb", + "yunohost tools port-available", + ] + for deprecated_helper in deprecated_helpers: + if os.system(f"grep -nr -q '{deprecated_helper}' {app['setting_path']}/scripts/") == 0: + yield ("error", "diagnosis_apps_deprecated_practices") + + old_arg_regex = r'^domain=\${?[0-9]' + if os.system(f"grep -q '{old_arg_regex}' {app['setting_path']}/scripts/install") == 0: + yield ("error", "diagnosis_apps_deprecated_practices") + + +def main(args, env, loggers): + return AppDiagnoser(args, env, loggers).diagnose() diff --git a/locales/en.json b/locales/en.json index 938a38e20..8852f5587 100644 --- a/locales/en.json +++ b/locales/en.json @@ -248,6 +248,14 @@ "diagnosis_description_web": "Web", "diagnosis_description_mail": "Email", "diagnosis_description_regenconf": "System configurations", + "diagnosis_description_apps": "Applications", + "diagnosis_apps_allgood": "All installed apps respect basic packaging practices", + "diagnosis_apps_issue": "An issue was found for app {app}", + "diagnosis_apps_not_in_app_catalog": "This application is not in YunoHost's application catalog. If it was in the past and got removed, you should consider uninstalling this app as it won't receive upgrade, and may compromise the integrity and security of your system.", + "diagnosis_apps_broken": "This application is currently flagged as broken on YunoHost's application catalog. This may be a temporary issue while the maintainers attempt to fix the issue. In the meantime, upgrading this app is disabled.", + "diagnosis_apps_bad_quality": "This application is currently flagged as broken on YunoHost's application catalog. This may be a temporary issue while the maintainers attempt to fix the issue. In the meantime, upgrading this app is disabled.", + "diagnosis_apps_outdated_ynh_requirement": "This app's installed version only requires yunohost >= 2.x, which tends to indicate that it's not up to date with recommended packaging practices and helpers. You should really consider upgrading it.", + "diagnosis_apps_deprecated_practices": "This app's installed version still uses some super-old deprecated packaging practices. You should really consider upgrading it.", "diagnosis_ports_could_not_diagnose": "Could not diagnose if ports are reachable from outside in IPv{ipversion}.", "diagnosis_ports_could_not_diagnose_details": "Error: {error}", "diagnosis_ports_unreachable": "Port {port} is not reachable from outside.", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index c048ca5ea..646535fab 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -193,7 +193,8 @@ def app_info(app, full=False): "app_not_installed", app=app, all_apps=_get_all_installed_apps_id() ) - local_manifest = _get_manifest_of_app(os.path.join(APPS_SETTING_PATH, app)) + setting_path = os.path.join(APPS_SETTING_PATH, app) + local_manifest = _get_manifest_of_app(setting_path) permissions = user_permission_list(full=True, absolute_urls=True, apps=[app])[ "permissions" ] @@ -212,6 +213,7 @@ def app_info(app, full=False): if not full: return ret + ret["setting_path"] = setting_path ret["manifest"] = local_manifest ret["manifest"]["arguments"] = _set_default_ask_questions( ret["manifest"].get("arguments", {}) @@ -222,11 +224,11 @@ def app_info(app, full=False): ret["from_catalog"] = _load_apps_catalog()["apps"].get(absolute_app_name, {}) ret["upgradable"] = _app_upgradable(ret) ret["supports_change_url"] = os.path.exists( - os.path.join(APPS_SETTING_PATH, app, "scripts", "change_url") + os.path.join(setting_path, "scripts", "change_url") ) ret["supports_backup_restore"] = os.path.exists( - os.path.join(APPS_SETTING_PATH, app, "scripts", "backup") - ) and os.path.exists(os.path.join(APPS_SETTING_PATH, app, "scripts", "restore")) + os.path.join(setting_path, "scripts", "backup") + ) and os.path.exists(os.path.join(setting_path, "scripts", "restore")) ret["supports_multi_instance"] = is_true( local_manifest.get("multi_instance", False) ) From 569dc24267a499d9c231154ed64feab3a3c38fa9 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Thu, 26 Aug 2021 23:01:14 +0200 Subject: [PATCH 2/6] remove unused imports --- data/hooks/diagnosis/80-apps.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/data/hooks/diagnosis/80-apps.py b/data/hooks/diagnosis/80-apps.py index ce54faef1..4ab5a6c0d 100644 --- a/data/hooks/diagnosis/80-apps.py +++ b/data/hooks/diagnosis/80-apps.py @@ -1,16 +1,10 @@ #!/usr/bin/env python import os -import re -from yunohost.app import app_info, app_list -from moulinette.utils.filesystem import read_file +from yunohost.app import app_list -from yunohost.settings import settings_get from yunohost.diagnosis import Diagnoser -from yunohost.regenconf import _get_regenconf_infos, _calculate_hash -from moulinette.utils.filesystem import read_file - class AppDiagnoser(Diagnoser): From c3fe22f3f0b334375020ef93bdf66f3909c95d35 Mon Sep 17 00:00:00 2001 From: yunohost-bot Date: Fri, 27 Aug 2021 17:19:45 +0000 Subject: [PATCH 3/6] [CI] Remove stale translated strings --- locales/fr.json | 2 +- locales/gl.json | 2 +- locales/it.json | 2 +- locales/pt.json | 2 +- locales/uk.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/locales/fr.json b/locales/fr.json index 7719e1390..1cac6bbba 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -638,4 +638,4 @@ "global_settings_setting_security_webadmin_allowlist_enabled": "Autorisez seulement certaines IP à accéder à la page web du portail d'administration (webadmin).", "diagnosis_http_localdomain": "Le domaine {domain}, avec un TLD .local, ne devrait pas être atteint depuis l'extérieur du réseau local.", "diagnosis_dns_specialusedomain": "Le domaine {domain} est basé sur un domaine de premier niveau (TLD) à usage spécial et ne devrait donc pas avoir d'enregistrements DNS réels." -} +} \ No newline at end of file diff --git a/locales/gl.json b/locales/gl.json index 5c0bec604..c38d13ad1 100644 --- a/locales/gl.json +++ b/locales/gl.json @@ -523,4 +523,4 @@ "permission_deleted": "O permiso '{permission}' foi eliminado", "permission_cant_add_to_all_users": "O permiso {permission} non pode ser concecido a tódalas usuarias.", "permission_currently_allowed_for_all_users": "Este permiso está concedido actualmente a tódalas usuarias ademáis de a outros grupos. Probablemente queiras ben eliminar o permiso 'all_users' ou ben eliminar os outros grupos que teñen permiso." -} +} \ No newline at end of file diff --git a/locales/it.json b/locales/it.json index a9fb6f9c8..c38bae59d 100644 --- a/locales/it.json +++ b/locales/it.json @@ -635,4 +635,4 @@ "global_settings_setting_security_webadmin_allowlist_enabled": "Permetti solo ad alcuni IP di accedere al webadmin.", "disk_space_not_sufficient_update": "Non c'è abbastanza spazio libero per aggiornare questa applicazione", "disk_space_not_sufficient_install": "Non c'è abbastanza spazio libero per installare questa applicazione" -} +} \ No newline at end of file diff --git a/locales/pt.json b/locales/pt.json index ef0c41349..cc47da946 100644 --- a/locales/pt.json +++ b/locales/pt.json @@ -141,4 +141,4 @@ "app_install_failed": "Não foi possível instalar {app}: {error}", "app_full_domain_unavailable": "Desculpe, esse app deve ser instalado num domínio próprio mas já há outros apps instalados no domínio '{domain}'. Você pode usar um subdomínio dedicado a esse aplicativo.", "app_change_url_success": "A URL agora é {domain}{path}" -} +} \ No newline at end of file diff --git a/locales/uk.json b/locales/uk.json index c85b4fd0a..d51ec706c 100644 --- a/locales/uk.json +++ b/locales/uk.json @@ -33,4 +33,4 @@ "action_invalid": "Неприпустима дія '{action}'", "aborting": "Переривання.", "diagnosis_description_web": "Мережа" -} +} \ No newline at end of file From 712e2bb1c933550cb7f97d0afc79d779852094ab Mon Sep 17 00:00:00 2001 From: yunohost-bot Date: Fri, 27 Aug 2021 18:07:37 +0000 Subject: [PATCH 4/6] [CI] Format code --- src/yunohost/authenticators/ldap_admin.py | 6 ++- src/yunohost/tests/conftest.py | 3 +- .../tests/test_apps_arguments_parsing.py | 44 ++++++++++++++----- src/yunohost/tests/test_ldapauth.py | 1 + src/yunohost/utils/ldap.py | 5 +-- 5 files changed, 43 insertions(+), 16 deletions(-) diff --git a/src/yunohost/authenticators/ldap_admin.py b/src/yunohost/authenticators/ldap_admin.py index dd6eec03e..94d68a8db 100644 --- a/src/yunohost/authenticators/ldap_admin.py +++ b/src/yunohost/authenticators/ldap_admin.py @@ -12,6 +12,7 @@ from yunohost.utils.error import YunohostError logger = logging.getLogger("yunohost.authenticators.ldap_admin") + class Authenticator(BaseAuthenticator): name = "ldap_admin" @@ -57,7 +58,10 @@ class Authenticator(BaseAuthenticator): raise else: if who != self.admindn: - raise YunohostError(f"Not logged with the appropriate identity ? Found {who}, expected {self.admindn} !?", raw_msg=True) + raise YunohostError( + f"Not logged with the appropriate identity ? Found {who}, expected {self.admindn} !?", + raw_msg=True, + ) finally: # Free the connection, we don't really need it to keep it open as the point is only to check authentication... if con: diff --git a/src/yunohost/tests/conftest.py b/src/yunohost/tests/conftest.py index 9dfe2b39c..6b4e2c3fd 100644 --- a/src/yunohost/tests/conftest.py +++ b/src/yunohost/tests/conftest.py @@ -81,7 +81,8 @@ def pytest_cmdline_main(config): import yunohost yunohost.init(debug=config.option.yunodebug) - class DummyInterface(): + + class DummyInterface: type = "test" diff --git a/src/yunohost/tests/test_apps_arguments_parsing.py b/src/yunohost/tests/test_apps_arguments_parsing.py index 573c18cb2..fe5c5f8cd 100644 --- a/src/yunohost/tests/test_apps_arguments_parsing.py +++ b/src/yunohost/tests/test_apps_arguments_parsing.py @@ -180,7 +180,9 @@ def test_parse_args_in_yunohost_format_string_input_test_ask(): ] answers = {} - with patch.object(Moulinette.interface, "prompt", return_value="some_value") as prompt: + with patch.object( + Moulinette.interface, "prompt", return_value="some_value" + ) as prompt: _parse_args_in_yunohost_format(answers, questions) prompt.assert_called_with(ask_text, False) @@ -197,7 +199,9 @@ def test_parse_args_in_yunohost_format_string_input_test_ask_with_default(): ] answers = {} - with patch.object(Moulinette.interface, "prompt", return_value="some_value") as prompt: + with patch.object( + Moulinette.interface, "prompt", return_value="some_value" + ) as prompt: _parse_args_in_yunohost_format(answers, questions) prompt.assert_called_with("%s (default: %s)" % (ask_text, default_text), False) @@ -215,7 +219,9 @@ def test_parse_args_in_yunohost_format_string_input_test_ask_with_example(): ] answers = {} - with patch.object(Moulinette.interface, "prompt", return_value="some_value") as prompt: + with patch.object( + Moulinette.interface, "prompt", return_value="some_value" + ) as prompt: _parse_args_in_yunohost_format(answers, questions) assert ask_text in prompt.call_args[0][0] assert example_text in prompt.call_args[0][0] @@ -234,7 +240,9 @@ def test_parse_args_in_yunohost_format_string_input_test_ask_with_help(): ] answers = {} - with patch.object(Moulinette.interface, "prompt", return_value="some_value") as prompt: + with patch.object( + Moulinette.interface, "prompt", return_value="some_value" + ) as prompt: _parse_args_in_yunohost_format(answers, questions) assert ask_text in prompt.call_args[0][0] assert help_text in prompt.call_args[0][0] @@ -462,7 +470,9 @@ def test_parse_args_in_yunohost_format_password_input_test_ask(): ] answers = {} - with patch.object(Moulinette.interface, "prompt", return_value="some_value") as prompt: + with patch.object( + Moulinette.interface, "prompt", return_value="some_value" + ) as prompt: _parse_args_in_yunohost_format(answers, questions) prompt.assert_called_with(ask_text, True) @@ -481,7 +491,9 @@ def test_parse_args_in_yunohost_format_password_input_test_ask_with_example(): ] answers = {} - with patch.object(Moulinette.interface, "prompt", return_value="some_value") as prompt: + with patch.object( + Moulinette.interface, "prompt", return_value="some_value" + ) as prompt: _parse_args_in_yunohost_format(answers, questions) assert ask_text in prompt.call_args[0][0] assert example_text in prompt.call_args[0][0] @@ -501,7 +513,9 @@ def test_parse_args_in_yunohost_format_password_input_test_ask_with_help(): ] answers = {} - with patch.object(Moulinette.interface, "prompt", return_value="some_value") as prompt: + with patch.object( + Moulinette.interface, "prompt", return_value="some_value" + ) as prompt: _parse_args_in_yunohost_format(answers, questions) assert ask_text in prompt.call_args[0][0] assert help_text in prompt.call_args[0][0] @@ -697,7 +711,9 @@ def test_parse_args_in_yunohost_format_path_input_test_ask(): ] answers = {} - with patch.object(Moulinette.interface, "prompt", return_value="some_value") as prompt: + with patch.object( + Moulinette.interface, "prompt", return_value="some_value" + ) as prompt: _parse_args_in_yunohost_format(answers, questions) prompt.assert_called_with(ask_text, False) @@ -715,7 +731,9 @@ def test_parse_args_in_yunohost_format_path_input_test_ask_with_default(): ] answers = {} - with patch.object(Moulinette.interface, "prompt", return_value="some_value") as prompt: + with patch.object( + Moulinette.interface, "prompt", return_value="some_value" + ) as prompt: _parse_args_in_yunohost_format(answers, questions) prompt.assert_called_with("%s (default: %s)" % (ask_text, default_text), False) @@ -734,7 +752,9 @@ def test_parse_args_in_yunohost_format_path_input_test_ask_with_example(): ] answers = {} - with patch.object(Moulinette.interface, "prompt", return_value="some_value") as prompt: + with patch.object( + Moulinette.interface, "prompt", return_value="some_value" + ) as prompt: _parse_args_in_yunohost_format(answers, questions) assert ask_text in prompt.call_args[0][0] assert example_text in prompt.call_args[0][0] @@ -754,7 +774,9 @@ def test_parse_args_in_yunohost_format_path_input_test_ask_with_help(): ] answers = {} - with patch.object(Moulinette.interface, "prompt", return_value="some_value") as prompt: + with patch.object( + Moulinette.interface, "prompt", return_value="some_value" + ) as prompt: _parse_args_in_yunohost_format(answers, questions) assert ask_text in prompt.call_args[0][0] assert help_text in prompt.call_args[0][0] diff --git a/src/yunohost/tests/test_ldapauth.py b/src/yunohost/tests/test_ldapauth.py index 7560608f5..3a4e0dbb6 100644 --- a/src/yunohost/tests/test_ldapauth.py +++ b/src/yunohost/tests/test_ldapauth.py @@ -7,6 +7,7 @@ from yunohost.tools import tools_adminpw from moulinette import m18n from moulinette.core import MoulinetteError + def setup_function(function): if os.system("systemctl is-active slapd") != 0: diff --git a/src/yunohost/utils/ldap.py b/src/yunohost/utils/ldap.py index 1298eff69..4f571ce6f 100644 --- a/src/yunohost/utils/ldap.py +++ b/src/yunohost/utils/ldap.py @@ -56,7 +56,7 @@ def _get_ldap_interface(): def _ldap_path_extract(path, info): for element in path.split(","): if element.startswith(info + "="): - return element[len(info + "="):] + return element[len(info + "=") :] # Add this to properly close / delete the ldap interface / authenticator @@ -72,8 +72,7 @@ def _destroy_ldap_interface(): atexit.register(_destroy_ldap_interface) -class LDAPInterface(): - +class LDAPInterface: def __init__(self): logger.debug("initializing ldap interface") From 383aab55a66b3e94963f775e4e29134d22bbe25c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 28 Aug 2021 02:03:33 +0200 Subject: [PATCH 5/6] Fix tests --- locales/gl.json | 4 ++-- src/yunohost/tests/test_ldapauth.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/locales/gl.json b/locales/gl.json index c38d13ad1..5ccdad0c2 100644 --- a/locales/gl.json +++ b/locales/gl.json @@ -449,7 +449,7 @@ "migration_0019_slapd_config_will_be_overwritten": "Semella que editaches manualmente a configuración slapd. Para esta migración crítica YunoHost precisa forzar a actualización da configuración slapd. Os ficheiros orixinais van ser copiados en {conf_backup_folder}.", "migration_0019_add_new_attributes_in_ldap": "Engadir novos atributos para os permisos na base de datos LDAP", "migration_0018_failed_to_reset_legacy_rules": "Fallou o restablecemento das regras antigas de iptables: {error}", - "migration_0018_failed_to_migrate_iptables_rules": "Fallou a migración das regras antigas de iptables a nftables: {erro}", + "migration_0018_failed_to_migrate_iptables_rules": "Fallou a migración das regras antigas de iptables a nftables: {error}", "migration_0017_not_enough_space": "Crea espazo suficiente en {path} para executar a migración.", "migration_0017_postgresql_11_not_installed": "PostgreSQL 9.6 está instado, pero non postgresql 11? Algo raro debeu acontecer no teu sistema :(...", "migration_0017_postgresql_96_not_installed": "PostgreSQL non está instalado no teu sistema. Nada que facer.", @@ -523,4 +523,4 @@ "permission_deleted": "O permiso '{permission}' foi eliminado", "permission_cant_add_to_all_users": "O permiso {permission} non pode ser concecido a tódalas usuarias.", "permission_currently_allowed_for_all_users": "Este permiso está concedido actualmente a tódalas usuarias ademáis de a outros grupos. Probablemente queiras ben eliminar o permiso 'all_users' ou ben eliminar os outros grupos que teñen permiso." -} \ No newline at end of file +} diff --git a/src/yunohost/tests/test_ldapauth.py b/src/yunohost/tests/test_ldapauth.py index 3a4e0dbb6..a95dea443 100644 --- a/src/yunohost/tests/test_ldapauth.py +++ b/src/yunohost/tests/test_ldapauth.py @@ -24,7 +24,7 @@ def test_authenticate_with_wrong_password(): with pytest.raises(MoulinetteError) as exception: LDAPAuth().authenticate_credentials(credentials="bad_password_lul") - translation = m18n.g("invalid_password") + translation = m18n.n("invalid_password") expected_msg = translation.format() assert expected_msg in str(exception) @@ -52,7 +52,7 @@ def test_authenticate_change_password(): with pytest.raises(MoulinetteError) as exception: LDAPAuth().authenticate_credentials(credentials="yunohost") - translation = m18n.g("invalid_password") + translation = m18n.n("invalid_password") expected_msg = translation.format() assert expected_msg in str(exception) From ea76895fdfaa0006c5c6f6d8f67ce462cf5be368 Mon Sep 17 00:00:00 2001 From: Gregor Lenz Date: Sat, 28 Aug 2021 03:09:20 +0300 Subject: [PATCH 6/6] remove offensive language in package ban config (#1226) * remove offensive language in package ban config I understand that some users do silly things with their system configuration and this ban is supposed to prevent that. I just improved the wording for any user who like me stumbles across this file on their system... * [enh] Imrpove warnings * fix spelling mistakes Co-authored-by: ljf (zamentur) --- data/hooks/conf_regen/10-apt | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/data/hooks/conf_regen/10-apt b/data/hooks/conf_regen/10-apt index bb5caf67f..d2977ee92 100755 --- a/data/hooks/conf_regen/10-apt +++ b/data/hooks/conf_regen/10-apt @@ -17,19 +17,16 @@ Pin-Priority: -1" >> "${pending_dir}/etc/apt/preferences.d/extra_php_version" done echo " -# Yes ! -# This is what's preventing you from installing apache2 ! -# -# Maybe take two fucking minutes to realize that if you try to install -# apache2, this will break nginx and break the entire YunoHost ecosystem. -# on your server. -# -# So, *NO* -# DO NOT do this. -# DO NOT remove these lines. -# -# I warned you. I WARNED YOU! But did you listen to me? -# Oooooh, noooo. You knew it all, didn't you? + +# PLEASE READ THIS WARNING AND DON'T EDIT THIS FILE + +# You are probably reading this file because you tried to install apache2 or +# bind9. These 2 packages conflict with YunoHost. + +# Installing apache2 will break nginx and break the entire YunoHost ecosystem +# on your server, therefore don't remove those lines! + +# You have been warned. Package: apache2 Pin: release * @@ -39,9 +36,9 @@ Package: apache2-bin Pin: release * Pin-Priority: -1 -# Also yes, bind9 will conflict with dnsmasq. -# Same story than for apache2. -# Don't fucking install it. +# Also bind9 will conflict with dnsmasq. +# Same story as for apache2. +# Don't install it, don't remove those lines. Package: bind9 Pin: release *