diff --git a/data/hooks/diagnosis/50-systemresources.py b/data/hooks/diagnosis/50-systemresources.py index a8f3cb6df..a662e392e 100644 --- a/data/hooks/diagnosis/50-systemresources.py +++ b/data/hooks/diagnosis/50-systemresources.py @@ -77,7 +77,9 @@ class SystemResourcesDiagnoser(Diagnoser): # Ignore /dev/loop stuff which are ~virtual partitions ? (e.g. mounted to /snap/) disk_partitions = [ - d for d in disk_partitions if d.mountpoint in ["/", "/var"] or not d.device.startswith("/dev/loop") + d + for d in disk_partitions + if d.mountpoint in ["/", "/var"] or not d.device.startswith("/dev/loop") ] for disk_partition in disk_partitions: diff --git a/data/hooks/diagnosis/70-regenconf.py b/data/hooks/diagnosis/70-regenconf.py index 4e40c71fb..8ccbeed58 100644 --- a/data/hooks/diagnosis/70-regenconf.py +++ b/data/hooks/diagnosis/70-regenconf.py @@ -8,6 +8,7 @@ from yunohost.diagnosis import Diagnoser from yunohost.regenconf import _get_regenconf_infos, _calculate_hash from moulinette.utils.filesystem import read_file + class RegenconfDiagnoser(Diagnoser): id_ = os.path.splitext(os.path.basename(__file__))[0].split("-")[1] @@ -37,29 +38,30 @@ class RegenconfDiagnoser(Diagnoser): details=["diagnosis_regenconf_manually_modified_details"], ) - if any(f["path"] == '/etc/ssh/sshd_config' for f in regenconf_modified_files) \ - and os.system("grep -q '^ *AllowGroups\\|^ *AllowUsers' /etc/ssh/sshd_config") != 0: - yield dict( - meta={ - "test": "sshd_config_insecure" - }, - status="ERROR", - summary="diagnosis_sshd_config_insecure", - ) + if ( + any(f["path"] == "/etc/ssh/sshd_config" for f in regenconf_modified_files) + and os.system( + "grep -q '^ *AllowGroups\\|^ *AllowUsers' /etc/ssh/sshd_config" + ) + != 0 + ): + yield dict( + meta={"test": "sshd_config_insecure"}, + status="ERROR", + summary="diagnosis_sshd_config_insecure", + ) # Check consistency between actual ssh port in sshd_config vs. setting - ssh_port_setting = settings_get('security.ssh.port') + ssh_port_setting = settings_get("security.ssh.port") ssh_port_line = re.findall( r"\bPort *([0-9]{2,5})\b", read_file("/etc/ssh/sshd_config") ) if len(ssh_port_line) == 1 and int(ssh_port_line[0]) != ssh_port_setting: yield dict( - meta={ - "test": "sshd_config_port_inconsistency" - }, - status="WARNING", - summary="diagnosis_sshd_config_inconsistent", - details=["diagnosis_sshd_config_inconsistent_details"], + meta={"test": "sshd_config_port_inconsistency"}, + status="WARNING", + summary="diagnosis_sshd_config_inconsistent", + details=["diagnosis_sshd_config_inconsistent_details"], ) def manually_modified_files(self): diff --git a/src/yunohost/__init__.py b/src/yunohost/__init__.py index 04caefc7a..dad73e2a4 100644 --- a/src/yunohost/__init__.py +++ b/src/yunohost/__init__.py @@ -90,56 +90,54 @@ def init_logging(interface="cli", debug=False, quiet=False, logdir="/var/log/yun os.makedirs(logdir, 0o750) logging_configuration = { - 'version': 1, - 'disable_existing_loggers': True, - 'formatters': { - 'console': { - 'format': '%(relativeCreated)-5d %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s' + "version": 1, + "disable_existing_loggers": True, + "formatters": { + "console": { + "format": "%(relativeCreated)-5d %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s" }, - 'tty-debug': { - 'format': '%(relativeCreated)-4d %(fmessage)s' - }, - 'precise': { - 'format': '%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s' + "tty-debug": {"format": "%(relativeCreated)-4d %(fmessage)s"}, + "precise": { + "format": "%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s" }, }, - 'filters': { - 'action': { - '()': 'moulinette.utils.log.ActionFilter', + "filters": { + "action": { + "()": "moulinette.utils.log.ActionFilter", }, }, - 'handlers': { - 'cli': { - 'level': 'DEBUG' if debug else 'INFO', - 'class': 'moulinette.interfaces.cli.TTYHandler', - 'formatter': 'tty-debug' if debug else '', + "handlers": { + "cli": { + "level": "DEBUG" if debug else "INFO", + "class": "moulinette.interfaces.cli.TTYHandler", + "formatter": "tty-debug" if debug else "", }, - 'api': { - 'level': 'DEBUG' if debug else 'INFO', - 'class': 'moulinette.interfaces.api.APIQueueHandler', + "api": { + "level": "DEBUG" if debug else "INFO", + "class": "moulinette.interfaces.api.APIQueueHandler", }, - 'file': { - 'class': 'logging.FileHandler', - 'formatter': 'precise', - 'filename': logfile, - 'filters': ['action'], + "file": { + "class": "logging.FileHandler", + "formatter": "precise", + "filename": logfile, + "filters": ["action"], }, }, - 'loggers': { - 'yunohost': { - 'level': 'DEBUG', - 'handlers': ['file', interface] if not quiet else ['file'], - 'propagate': False, + "loggers": { + "yunohost": { + "level": "DEBUG", + "handlers": ["file", interface] if not quiet else ["file"], + "propagate": False, }, - 'moulinette': { - 'level': 'DEBUG', - 'handlers': ['file', interface] if not quiet else ['file'], - 'propagate': False, + "moulinette": { + "level": "DEBUG", + "handlers": ["file", interface] if not quiet else ["file"], + "propagate": False, }, }, - 'root': { - 'level': 'DEBUG', - 'handlers': ['file', interface] if debug else ['file'], + "root": { + "level": "DEBUG", + "handlers": ["file", interface] if debug else ["file"], }, } @@ -150,7 +148,9 @@ def init_logging(interface="cli", debug=False, quiet=False, logdir="/var/log/yun # Logging configuration for API # else: # We use a WatchedFileHandler instead of regular FileHandler to possibly support log rotation etc - logging_configuration["handlers"]["file"]["class"] = 'logging.handlers.WatchedFileHandler' + logging_configuration["handlers"]["file"][ + "class" + ] = "logging.handlers.WatchedFileHandler" # This is for when launching yunohost-api in debug mode, we want to display stuff in the console if debug: diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 301d37398..fc9f62373 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -193,7 +193,9 @@ def app_info(app, full=False): ) local_manifest = _get_manifest_of_app(os.path.join(APPS_SETTING_PATH, app)) - permissions = user_permission_list(full=True, absolute_urls=True, apps=[app])["permissions"] + permissions = user_permission_list(full=True, absolute_urls=True, apps=[app])[ + "permissions" + ] settings = _get_app_settings(app) @@ -325,7 +327,9 @@ def app_map(app=None, raw=False, user=None): else: apps = _installed_apps() - permissions = user_permission_list(full=True, absolute_urls=True, apps=apps)["permissions"] + permissions = user_permission_list(full=True, absolute_urls=True, apps=apps)[ + "permissions" + ] for app_id in apps: app_settings = _get_app_settings(app_id) if not app_settings: @@ -782,6 +786,7 @@ def app_manifest(app): return manifest + @is_unit_operation() def app_install( operation_logger, @@ -1106,10 +1111,7 @@ def app_install( permission_sync_to_user() - raise YunohostError( - failure_message_with_debug_instructions, - raw_msg=True - ) + raise YunohostError(failure_message_with_debug_instructions, raw_msg=True) # Clean hooks and add new ones hook_remove(app_instance_name) @@ -2232,9 +2234,13 @@ def _extract_app_from_file(path): extracted_app_folder = _make_tmp_workdir_for_app() if ".zip" in path: - extract_result = os.system(f"unzip '{path}' -d {extracted_app_folder} > /dev/null 2>&1") + extract_result = os.system( + f"unzip '{path}' -d {extracted_app_folder} > /dev/null 2>&1" + ) elif ".tar" in path: - extract_result = os.system(f"tar -xf '{path}' -C {extracted_app_folder} > /dev/null 2>&1") + extract_result = os.system( + f"tar -xf '{path}' -C {extracted_app_folder} > /dev/null 2>&1" + ) elif os.path.isdir(path): shutil.rmtree(extracted_app_folder) if path[-1] != "/": @@ -2746,7 +2752,9 @@ class YunoHostArgumentFormatParser(object): # we don't have an answer, check optional and default_value if question.value is None or question.value == "": if not question.optional and question.default is None: - raise YunohostValidationError("app_argument_required", name=question.name) + raise YunohostValidationError( + "app_argument_required", name=question.name + ) else: question.value = ( getattr(self, "default_value", None) @@ -2804,7 +2812,9 @@ class PasswordArgumentParser(YunoHostArgumentFormatParser): ) if question.default is not None: - raise YunohostValidationError("app_argument_password_no_default", name=question.name) + raise YunohostValidationError( + "app_argument_password_no_default", name=question.name + ) return question @@ -3114,7 +3124,9 @@ def _assert_no_conflicting_apps(domain, path, ignore_app=None, full_domain=False if full_domain: raise YunohostValidationError("app_full_domain_unavailable", domain=domain) else: - raise YunohostValidationError("app_location_unavailable", apps="\n".join(apps)) + raise YunohostValidationError( + "app_location_unavailable", apps="\n".join(apps) + ) def _make_environment_for_app_script(app, args={}, args_prefix="APP_ARG_"): @@ -3340,6 +3352,7 @@ def _load_apps_catalog(): # ############################### # # + def _make_tmp_workdir_for_app(app=None): # Create parent dir if it doesn't exists yet @@ -3425,15 +3438,27 @@ def _assert_system_is_sane_for_app(manifest, when): if not any(s for s in services if service_status(s)["status"] == "reloading"): break time.sleep(0.5) - test_nb+=1 + test_nb += 1 # List services currently down and raise an exception if any are found - services_status = {s:service_status(s) for s in services} - faulty_services = [f"{s} ({status['status']})" for s, status in services_status.items() if status['status'] != "running"] + services_status = {s: service_status(s) for s in services} + faulty_services = [ + f"{s} ({status['status']})" + for s, status in services_status.items() + if status["status"] != "running" + ] # Stupid tmp fix to try to track why the tests are failing - if "php7.3-fpm" in [s for s, status in services_status.items() if status['status'] != "running"]: - logger.info([status for s, status in services_status.items() if status['status'] != "running"]) + if "php7.3-fpm" in [ + s for s, status in services_status.items() if status["status"] != "running" + ]: + logger.info( + [ + status + for s, status in services_status.items() + if status["status"] != "running" + ] + ) os.system("journalctl -u php7.3-fpm -n 300 --no-hostname --no-pager") if faulty_services: @@ -3609,7 +3634,7 @@ def _patch_legacy_helpers(app_folder): content = read_file(filename) except MoulinetteError: continue - + replaced_stuff = False show_warning = False diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index c2d3085a3..c696e99da 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -53,7 +53,7 @@ from yunohost.app import ( _patch_legacy_php_versions, _patch_legacy_php_versions_in_settings, LEGACY_PHP_VERSION_REPLACEMENTS, - _make_tmp_workdir_for_app + _make_tmp_workdir_for_app, ) from yunohost.hook import ( hook_list, @@ -62,7 +62,11 @@ from yunohost.hook import ( hook_exec, CUSTOM_HOOK_FOLDER, ) -from yunohost.tools import tools_postinstall, _tools_migrations_run_after_system_restore, _tools_migrations_run_before_app_restore +from yunohost.tools import ( + tools_postinstall, + _tools_migrations_run_after_system_restore, + _tools_migrations_run_before_app_restore, +) from yunohost.regenconf import regen_conf from yunohost.log import OperationLogger, is_unit_operation from yunohost.utils.error import YunohostError, YunohostValidationError @@ -716,7 +720,9 @@ class BackupManager: hook_exec( f"{tmp_workdir_for_app}/scripts/backup", - raise_on_error=True, chdir=tmp_app_bkp_dir, env=env_dict + raise_on_error=True, + chdir=tmp_app_bkp_dir, + env=env_dict, )[0] self._import_to_list_to_backup(env_dict["YNH_BACKUP_CSV"]) @@ -724,10 +730,7 @@ class BackupManager: # backup permissions logger.debug(m18n.n("backup_permission", app=app)) permissions = user_permission_list(full=True, apps=[app])["permissions"] - this_app_permissions = { - name: infos - for name, infos in permissions.items() - } + this_app_permissions = {name: infos for name, infos in permissions.items()} write_to_yaml("%s/permissions.yml" % settings_dir, this_app_permissions) except Exception: @@ -857,7 +860,9 @@ class RestoreManager: # FIXME this way to get the info is not compatible with copy or custom # backup methods self.info = backup_info(name, with_details=True) - if not self.info["from_yunohost_version"] or version.parse(self.info["from_yunohost_version"]) < version.parse("3.8.0"): + if not self.info["from_yunohost_version"] or version.parse( + self.info["from_yunohost_version"] + ) < version.parse("3.8.0"): raise YunohostValidationError("restore_backup_too_old") self.archive_path = self.info["path"] @@ -1279,7 +1284,9 @@ class RestoreManager: regen_conf() - _tools_migrations_run_after_system_restore(backup_version=self.info["from_yunohost_version"]) + _tools_migrations_run_after_system_restore( + backup_version=self.info["from_yunohost_version"] + ) # Remove all permission for all app still in the LDAP for permission_name in user_permission_list(ignore_system_perms=True)[ @@ -1409,7 +1416,9 @@ class RestoreManager: # Restore permissions if not os.path.isfile("%s/permissions.yml" % app_settings_new_path): - raise YunohostError("Didnt find a permssions.yml for the app !?", raw_msg=True) + raise YunohostError( + "Didnt find a permssions.yml for the app !?", raw_msg=True + ) permissions = read_yaml("%s/permissions.yml" % app_settings_new_path) existing_groups = user_group_list()["groups"] @@ -1424,9 +1433,7 @@ class RestoreManager: should_be_allowed = ["all_users"] else: should_be_allowed = [ - g - for g in permission_infos["allowed"] - if g in existing_groups + g for g in permission_infos["allowed"] if g in existing_groups ] perm_name = permission_name.split(".")[1] @@ -1448,9 +1455,13 @@ class RestoreManager: os.remove("%s/permissions.yml" % app_settings_new_path) - _tools_migrations_run_before_app_restore(backup_version=self.info["from_yunohost_version"], app_id=app_instance_name) + _tools_migrations_run_before_app_restore( + backup_version=self.info["from_yunohost_version"], + app_id=app_instance_name, + ) except Exception: import traceback + error = m18n.n("unexpected_error", error="\n" + traceback.format_exc()) msg = m18n.n("app_restore_failed", app=app_instance_name, error=error) logger.error(msg) @@ -1493,20 +1504,27 @@ class RestoreManager: restore_failed = True if restore_retcode != 0 else False if restore_failed: error = m18n.n("app_restore_script_failed") - logger.error(m18n.n("app_restore_failed", app=app_instance_name, error=error)) + logger.error( + m18n.n("app_restore_failed", app=app_instance_name, error=error) + ) failure_message_with_debug_instructions = operation_logger.error(error) if msettings.get("interface") != "api": dump_app_log_extract_for_debugging(operation_logger) # Script got manually interrupted ... N.B. : KeyboardInterrupt does not inherit from Exception except (KeyboardInterrupt, EOFError): error = m18n.n("operation_interrupted") - logger.error(m18n.n("app_restore_failed", app=app_instance_name, error=error)) + logger.error( + m18n.n("app_restore_failed", app=app_instance_name, error=error) + ) failure_message_with_debug_instructions = operation_logger.error(error) # Something wrong happened in Yunohost's code (most probably hook_exec) except Exception: import traceback + error = m18n.n("unexpected_error", error="\n" + traceback.format_exc()) - logger.error(m18n.n("app_restore_failed", app=app_instance_name, error=error)) + logger.error( + m18n.n("app_restore_failed", app=app_instance_name, error=error) + ) failure_message_with_debug_instructions = operation_logger.error(error) finally: # Cleaning temporary scripts directory @@ -1551,6 +1569,7 @@ class RestoreManager: logger.error(failure_message_with_debug_instructions) + # # Backup methods # # @@ -2163,7 +2182,13 @@ class CustomBackupMethod(BackupMethod): @is_unit_operation() def backup_create( operation_logger, - name=None, description=None, methods=[], output_directory=None, system=[], apps=[], dry_run=False + name=None, + description=None, + methods=[], + output_directory=None, + system=[], + apps=[], + dry_run=False, ): """ Create a backup local archive @@ -2248,12 +2273,17 @@ def backup_create( if dry_run: return { "size": backup_manager.size, - "size_details": backup_manager.size_details + "size_details": backup_manager.size_details, } # Apply backup methods on prepared files logger.info(m18n.n("backup_actually_backuping")) - logger.info(m18n.n("backup_create_size_estimation", size=binary_to_human(backup_manager.size) + "B")) + logger.info( + m18n.n( + "backup_create_size_estimation", + size=binary_to_human(backup_manager.size) + "B", + ) + ) backup_manager.backup() logger.success(m18n.n("backup_created")) @@ -2291,9 +2321,9 @@ def backup_restore(name, system=[], apps=[], force=False): # if name.endswith(".tar.gz"): - name = name[:-len(".tar.gz")] + name = name[: -len(".tar.gz")] elif name.endswith(".tar"): - name = name[:-len(".tar")] + name = name[: -len(".tar")] restore_manager = RestoreManager(name) @@ -2407,7 +2437,9 @@ def backup_download(name): # Raise exception if link is broken (e.g. on unmounted external storage) if not os.path.exists(archive_file): - raise YunohostValidationError("backup_archive_broken_link", path=archive_file) + raise YunohostValidationError( + "backup_archive_broken_link", path=archive_file + ) # We return a raw bottle HTTPresponse (instead of serializable data like # list/dict, ...), which is gonna be picked and used directly by moulinette @@ -2429,9 +2461,9 @@ def backup_info(name, with_details=False, human_readable=False): """ if name.endswith(".tar.gz"): - name = name[:-len(".tar.gz")] + name = name[: -len(".tar.gz")] elif name.endswith(".tar"): - name = name[:-len(".tar")] + name = name[: -len(".tar")] archive_file = "%s/%s.tar" % (ARCHIVES_PATH, name) @@ -2447,7 +2479,9 @@ def backup_info(name, with_details=False, human_readable=False): # Raise exception if link is broken (e.g. on unmounted external storage) if not os.path.exists(archive_file): - raise YunohostValidationError("backup_archive_broken_link", path=archive_file) + raise YunohostValidationError( + "backup_archive_broken_link", path=archive_file + ) info_file = "%s/%s.info.json" % (ARCHIVES_PATH, name) diff --git a/src/yunohost/certificate.py b/src/yunohost/certificate.py index 1c2249ba7..7e6726f31 100644 --- a/src/yunohost/certificate.py +++ b/src/yunohost/certificate.py @@ -455,6 +455,7 @@ def certificate_renew( # Back-end stuff # # + def _email_renewing_failed(domain, exception_message, stack=""): from_ = "certmanager@%s (Certificate Manager)" % domain to_ = "root" @@ -872,7 +873,9 @@ def _check_domain_is_ready_for_ACME(domain): ) if not dnsrecords or not httpreachable: - raise YunohostValidationError("certmanager_domain_not_diagnosed_yet", domain=domain) + raise YunohostValidationError( + "certmanager_domain_not_diagnosed_yet", domain=domain + ) # Check if IP from DNS matches public IP if not dnsrecords.get("status") in [ @@ -885,7 +888,9 @@ def _check_domain_is_ready_for_ACME(domain): # Check if domain seems to be accessible through HTTP? if not httpreachable.get("status") == "SUCCESS": - raise YunohostValidationError("certmanager_domain_http_not_working", domain=domain) + raise YunohostValidationError( + "certmanager_domain_http_not_working", domain=domain + ) # FIXME / TODO : ideally this should not be needed. There should be a proper diff --git a/src/yunohost/data_migrations/0020_ssh_sftp_permissions.py b/src/yunohost/data_migrations/0020_ssh_sftp_permissions.py index c3b7a91ec..3fb36a6f3 100644 --- a/src/yunohost/data_migrations/0020_ssh_sftp_permissions.py +++ b/src/yunohost/data_migrations/0020_ssh_sftp_permissions.py @@ -9,7 +9,7 @@ from yunohost.tools import Migration from yunohost.permission import user_permission_update, permission_sync_to_user from yunohost.regenconf import manually_modified_files -logger = getActionLogger('yunohost.migration') +logger = getActionLogger("yunohost.migration") ################################################### # Tools used also for restoration @@ -18,7 +18,7 @@ logger = getActionLogger('yunohost.migration') class MyMigration(Migration): """ - Add new permissions around SSH/SFTP features + Add new permissions around SSH/SFTP features """ introduced_in_version = "4.2.2" @@ -28,38 +28,60 @@ class MyMigration(Migration): def run(self, *args): from yunohost.utils.ldap import _get_ldap_interface + ldap = _get_ldap_interface() - existing_perms_raw = ldap.search("ou=permission,dc=yunohost,dc=org", "(objectclass=permissionYnh)", ["cn"]) - existing_perms = [perm['cn'][0] for perm in existing_perms_raw] + existing_perms_raw = ldap.search( + "ou=permission,dc=yunohost,dc=org", "(objectclass=permissionYnh)", ["cn"] + ) + existing_perms = [perm["cn"][0] for perm in existing_perms_raw] # Add SSH and SFTP permissions - ldap_map = read_yaml('/usr/share/yunohost/yunohost-config/moulinette/ldap_scheme.yml') + ldap_map = read_yaml( + "/usr/share/yunohost/yunohost-config/moulinette/ldap_scheme.yml" + ) if "sftp.main" not in existing_perms: - ldap.add("cn=sftp.main,ou=permission", ldap_map['depends_children']["cn=sftp.main,ou=permission"]) + ldap.add( + "cn=sftp.main,ou=permission", + ldap_map["depends_children"]["cn=sftp.main,ou=permission"], + ) if "ssh.main" not in existing_perms: - ldap.add("cn=ssh.main,ou=permission", ldap_map['depends_children']["cn=ssh.main,ou=permission"]) + ldap.add( + "cn=ssh.main,ou=permission", + ldap_map["depends_children"]["cn=ssh.main,ou=permission"], + ) # Add a bash terminal to each users - users = ldap.search('ou=users,dc=yunohost,dc=org', filter="(loginShell=*)", attrs=["dn", "uid", "loginShell"]) + users = ldap.search( + "ou=users,dc=yunohost,dc=org", + filter="(loginShell=*)", + attrs=["dn", "uid", "loginShell"], + ) for user in users: - if user['loginShell'][0] == '/bin/false': - dn = user['dn'][0].replace(',dc=yunohost,dc=org', '') - ldap.update(dn, {'loginShell': ['/bin/bash']}) + if user["loginShell"][0] == "/bin/false": + dn = user["dn"][0].replace(",dc=yunohost,dc=org", "") + ldap.update(dn, {"loginShell": ["/bin/bash"]}) else: - user_permission_update("ssh.main", add=user["uid"][0], sync_perm=False) + user_permission_update( + "ssh.main", add=user["uid"][0], sync_perm=False + ) permission_sync_to_user() # Somehow this is needed otherwise the PAM thing doesn't forget about the # old loginShell value ? - subprocess.call(['nscd', '-i', 'passwd']) + subprocess.call(["nscd", "-i", "passwd"]) - if '/etc/ssh/sshd_config' in manually_modified_files() \ - and os.system("grep -q '^ *AllowGroups\\|^ *AllowUsers' /etc/ssh/sshd_config") != 0: - logger.error(m18n.n('diagnosis_sshd_config_insecure')) + if ( + "/etc/ssh/sshd_config" in manually_modified_files() + and os.system( + "grep -q '^ *AllowGroups\\|^ *AllowUsers' /etc/ssh/sshd_config" + ) + != 0 + ): + logger.error(m18n.n("diagnosis_sshd_config_insecure")) def run_after_system_restore(self): self.run() diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index b8a4a1f8a..29ffe686b 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -59,7 +59,9 @@ def diagnosis_get(category, item): all_categories_names = [c for c, _ in all_categories] if category not in all_categories_names: - raise YunohostValidationError("diagnosis_unknown_categories", categories=category) + raise YunohostValidationError( + "diagnosis_unknown_categories", categories=category + ) if isinstance(item, list): if any("=" not in criteria for criteria in item): diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 4543bbcd9..aaac3a995 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -122,7 +122,7 @@ def domain_add(operation_logger, domain, dyndns=False): # Do not allow to subscribe to multiple dyndns domains... if _guess_current_dyndns_domain("dyndns.yunohost.org") != (None, None): - raise YunohostValidationError('domain_dyndns_already_subscribed') + raise YunohostValidationError("domain_dyndns_already_subscribed") # Check that this domain can effectively be provided by # dyndns.yunohost.org. (i.e. is it a nohost.me / noho.st) @@ -133,6 +133,7 @@ def domain_add(operation_logger, domain, dyndns=False): if dyndns: from yunohost.dyndns import dyndns_subscribe + # Actually subscribe dyndns_subscribe(domain=domain) @@ -197,8 +198,8 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False): # the 'force' here is related to the exception happening in domain_add ... # we don't want to check the domain exists because the ldap add may have # failed - if not force and domain not in domain_list()['domains']: - raise YunohostValidationError('domain_name_unknown', domain=domain) + if not force and domain not in domain_list()["domains"]: + raise YunohostValidationError("domain_name_unknown", domain=domain) # Check domain is not the main domain if domain == _get_maindomain(): @@ -212,7 +213,9 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False): other_domains="\n * " + ("\n * ".join(other_domains)), ) else: - raise YunohostValidationError("domain_cannot_remove_main_add_new_one", domain=domain) + raise YunohostValidationError( + "domain_cannot_remove_main_add_new_one", domain=domain + ) # Check if apps are installed on the domain apps_on_that_domain = [] @@ -221,21 +224,37 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False): settings = _get_app_settings(app) label = app_info(app)["name"] if settings.get("domain") == domain: - apps_on_that_domain.append((app, " - %s \"%s\" on https://%s%s" % (app, label, domain, settings["path"]) if "path" in settings else app)) + apps_on_that_domain.append( + ( + app, + ' - %s "%s" on https://%s%s' + % (app, label, domain, settings["path"]) + if "path" in settings + else app, + ) + ) if apps_on_that_domain: if remove_apps: - if msettings.get('interface') == "cli" and not force: - answer = msignals.prompt(m18n.n('domain_remove_confirm_apps_removal', - apps="\n".join([x[1] for x in apps_on_that_domain]), - answers='y/N'), color="yellow") + if msettings.get("interface") == "cli" and not force: + answer = msignals.prompt( + m18n.n( + "domain_remove_confirm_apps_removal", + apps="\n".join([x[1] for x in apps_on_that_domain]), + answers="y/N", + ), + color="yellow", + ) if answer.upper() != "Y": raise YunohostError("aborting") for app, _ in apps_on_that_domain: app_remove(app) else: - raise YunohostValidationError('domain_uninstall_app_first', apps="\n".join([x[1] for x in apps_on_that_domain])) + raise YunohostValidationError( + "domain_uninstall_app_first", + apps="\n".join([x[1] for x in apps_on_that_domain]), + ) operation_logger.start() @@ -248,7 +267,7 @@ def domain_remove(operation_logger, domain, remove_apps=False, force=False): os.system("rm -rf /etc/yunohost/certs/%s" % domain) # Delete dyndns keys for this domain (if any) - os.system('rm -rf /etc/yunohost/dyndns/K%s.+*' % domain) + os.system("rm -rf /etc/yunohost/dyndns/K%s.+*" % domain) # Sometime we have weird issues with the regenconf where some files # appears as manually modified even though they weren't touched ... diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index c7a501b9c..c8249e439 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -124,7 +124,7 @@ def dyndns_subscribe( """ if _guess_current_dyndns_domain(subscribe_host) != (None, None): - raise YunohostValidationError('domain_dyndns_already_subscribed') + raise YunohostValidationError("domain_dyndns_already_subscribed") if domain is None: domain = _get_maindomain() @@ -192,12 +192,14 @@ def dyndns_subscribe( # Add some dyndns update in 2 and 4 minutes from now such that user should # not have to wait 10ish minutes for the conf to propagate - cmd = "at -M now + {t} >/dev/null 2>&1 <<< \"/bin/bash -c 'yunohost dyndns update'\"" + cmd = ( + "at -M now + {t} >/dev/null 2>&1 <<< \"/bin/bash -c 'yunohost dyndns update'\"" + ) # For some reason subprocess doesn't like the redirections so we have to use bash -c explicity... subprocess.check_call(["bash", "-c", cmd.format(t="2 min")]) subprocess.check_call(["bash", "-c", cmd.format(t="4 min")]) - logger.success(m18n.n('dyndns_registered')) + logger.success(m18n.n("dyndns_registered")) @is_unit_operation() @@ -231,7 +233,7 @@ def dyndns_update( (domain, key) = _guess_current_dyndns_domain(dyn_host) if domain is None: - raise YunohostValidationError('dyndns_no_domain_registered') + raise YunohostValidationError("dyndns_no_domain_registered") # If key is not given, pick the first file we find with the domain given else: @@ -374,11 +376,15 @@ def dyndns_update( def dyndns_installcron(): - logger.warning("This command is deprecated. The dyndns cron job should automatically be added/removed by the regenconf depending if there's a private key in /etc/yunohost/dyndns. You can run the regenconf yourself with 'yunohost tools regen-conf yunohost'.") + logger.warning( + "This command is deprecated. The dyndns cron job should automatically be added/removed by the regenconf depending if there's a private key in /etc/yunohost/dyndns. You can run the regenconf yourself with 'yunohost tools regen-conf yunohost'." + ) def dyndns_removecron(): - logger.warning("This command is deprecated. The dyndns cron job should automatically be added/removed by the regenconf depending if there's a private key in /etc/yunohost/dyndns. You can run the regenconf yourself with 'yunohost tools regen-conf yunohost'.") + logger.warning( + "This command is deprecated. The dyndns cron job should automatically be added/removed by the regenconf depending if there's a private key in /etc/yunohost/dyndns. You can run the regenconf yourself with 'yunohost tools regen-conf yunohost'." + ) def _guess_current_dyndns_domain(dyn_host): diff --git a/src/yunohost/firewall.py b/src/yunohost/firewall.py index af1cea2e3..b800cd42c 100644 --- a/src/yunohost/firewall.py +++ b/src/yunohost/firewall.py @@ -188,11 +188,17 @@ def firewall_list(raw=False, by_ip_version=False, list_forwarded=False): for i in ["ipv4", "ipv6"]: f = firewall[i] # Combine TCP and UDP ports - ports[i] = sorted(set(f["TCP"]) | set(f["UDP"]), key=lambda p: int(p.split(':')[0]) if isinstance(p, str) else p) + ports[i] = sorted( + set(f["TCP"]) | set(f["UDP"]), + key=lambda p: int(p.split(":")[0]) if isinstance(p, str) else p, + ) if not by_ip_version: # Combine IPv4 and IPv6 ports - ports = sorted(set(ports["ipv4"]) | set(ports["ipv6"]), key=lambda p: int(p.split(':')[0]) if isinstance(p, str) else p) + ports = sorted( + set(ports["ipv4"]) | set(ports["ipv6"]), + key=lambda p: int(p.split(":")[0]) if isinstance(p, str) else p, + ) # Format returned dict ret = {"opened_ports": ports} @@ -200,7 +206,7 @@ def firewall_list(raw=False, by_ip_version=False, list_forwarded=False): # Combine TCP and UDP forwarded ports ret["forwarded_ports"] = sorted( set(firewall["uPnP"]["TCP"]) | set(firewall["uPnP"]["UDP"]), - key=lambda p: int(p.split(':')[0]) if isinstance(p, str) else p + key=lambda p: int(p.split(":")[0]) if isinstance(p, str) else p, ) return ret diff --git a/src/yunohost/log.py b/src/yunohost/log.py index f8215955f..f8da40002 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -414,8 +414,15 @@ class RedactingFormatter(Formatter): # This matches stuff like db_pwd=the_secret or admin_password=other_secret # (the secret part being at least 3 chars to avoid catching some lines like just "db_pwd=") # Some names like "key" or "manifest_key" are ignored, used in helpers like ynh_app_setting_set or ynh_read_manifest - match = re.search(r'(pwd|pass|password|passphrase|secret\w*|\w+key|token)=(\S{3,})$', record.strip()) - if match and match.group(2) not in self.data_to_redact and match.group(1) not in ["key", "manifest_key"]: + match = re.search( + r"(pwd|pass|password|passphrase|secret\w*|\w+key|token)=(\S{3,})$", + record.strip(), + ) + if ( + match + and match.group(2) not in self.data_to_redact + and match.group(1) not in ["key", "manifest_key"] + ): self.data_to_redact.append(match.group(2)) except Exception as e: logger.warning( @@ -636,7 +643,11 @@ class OperationLogger(object): # we want to inject the log ref in the exception, such that it may be # transmitted to the webadmin which can then redirect to the appropriate # log page - if self.started_at and isinstance(error, Exception) and not isinstance(error, YunohostValidationError): + if ( + self.started_at + and isinstance(error, Exception) + and not isinstance(error, YunohostValidationError) + ): error.log_ref = self.name if self.ended_at is not None or self.started_at is None: diff --git a/src/yunohost/permission.py b/src/yunohost/permission.py index cee22a741..01330ad7f 100644 --- a/src/yunohost/permission.py +++ b/src/yunohost/permission.py @@ -80,7 +80,9 @@ def user_permission_list( apps_base_path = { app: app_setting(app, "domain") + app_setting(app, "path") for app in apps - if app in installed_apps and app_setting(app, "domain") and app_setting(app, "path") + if app in installed_apps + and app_setting(app, "domain") + and app_setting(app, "path") } permissions = {} @@ -179,7 +181,9 @@ def user_permission_update( # Refuse to add "visitors" to mail, xmpp ... they require an account to make sense. if add and "visitors" in add and permission.split(".")[0] in SYSTEM_PERMS: - raise YunohostValidationError("permission_require_account", permission=permission) + raise YunohostValidationError( + "permission_require_account", permission=permission + ) # Refuse to add "visitors" to protected permission if ( @@ -189,8 +193,14 @@ def user_permission_update( raise YunohostValidationError("permission_protected", permission=permission) # Refuse to add "all_users" to ssh/sftp permissions - if permission.split(".")[0] in ["ssh", "sftp"] and (add and "all_users" in add) and not force: - raise YunohostValidationError("permission_cant_add_to_all_users", permission=permission) + if ( + permission.split(".")[0] in ["ssh", "sftp"] + and (add and "all_users" in add) + and not force + ): + raise YunohostValidationError( + "permission_cant_add_to_all_users", permission=permission + ) # Fetch currently allowed groups for this permission diff --git a/src/yunohost/settings.py b/src/yunohost/settings.py index 7e9eb76d9..0466d8126 100644 --- a/src/yunohost/settings.py +++ b/src/yunohost/settings.py @@ -103,6 +103,7 @@ DEFAULTS = OrderedDict( ] ) + def settings_get(key, full=False): """ Get an entry value in the settings @@ -114,7 +115,9 @@ def settings_get(key, full=False): settings = _get_settings() if key not in settings: - raise YunohostValidationError("global_settings_key_doesnt_exists", settings_key=key) + raise YunohostValidationError( + "global_settings_key_doesnt_exists", settings_key=key + ) if full: return settings[key] @@ -142,7 +145,9 @@ def settings_set(key, value): settings = _get_settings() if key not in settings: - raise YunohostValidationError("global_settings_key_doesnt_exists", settings_key=key) + raise YunohostValidationError( + "global_settings_key_doesnt_exists", settings_key=key + ) key_type = settings[key]["type"] @@ -219,7 +224,9 @@ def settings_reset(key): settings = _get_settings() if key not in settings: - raise YunohostValidationError("global_settings_key_doesnt_exists", settings_key=key) + raise YunohostValidationError( + "global_settings_key_doesnt_exists", settings_key=key + ) settings[key]["value"] = settings[key]["default"] _save_settings(settings) @@ -381,6 +388,7 @@ def trigger_post_change_hook(setting_name, old_value, new_value): # # =========================================== + @post_change_hook("ssowat.panel_overlay.enabled") @post_change_hook("security.nginx.compatibility") def reconfigure_nginx(setting_name, old_value, new_value): diff --git a/src/yunohost/ssh.py b/src/yunohost/ssh.py index e2ecaeef3..caac00050 100644 --- a/src/yunohost/ssh.py +++ b/src/yunohost/ssh.py @@ -99,18 +99,16 @@ def user_ssh_remove_key(username, key): if not os.path.exists(authorized_keys_file): raise YunohostValidationError( "this key doesn't exists ({} dosesn't exists)".format(authorized_keys_file), - raw_msg=True + raw_msg=True, ) authorized_keys_content = read_file(authorized_keys_file) if key not in authorized_keys_content: raise YunohostValidationError( - "Key '{}' is not present in authorized_keys".format(key), - raw_msg=True + "Key '{}' is not present in authorized_keys".format(key), raw_msg=True ) - # don't delete the previous comment because we can't verify if it's legit # this regex approach failed for some reasons and I don't know why :( diff --git a/src/yunohost/tests/test_apps.py b/src/yunohost/tests/test_apps.py index b9e9e7530..eba5a5916 100644 --- a/src/yunohost/tests/test_apps.py +++ b/src/yunohost/tests/test_apps.py @@ -56,7 +56,9 @@ def clean(): shutil.rmtree(folderpath, ignore_errors=True) os.system("bash -c \"mysql -B 2>/dev/null <<< 'DROP DATABASE %s' \"" % test_app) - os.system("bash -c \"mysql -B 2>/dev/null <<< 'DROP USER %s@localhost'\"" % test_app) + os.system( + "bash -c \"mysql -B 2>/dev/null <<< 'DROP USER %s@localhost'\"" % test_app + ) # Reset failed quota for service to avoid running into start-limit rate ? os.system("systemctl reset-failed nginx") diff --git a/src/yunohost/tests/test_settings.py b/src/yunohost/tests/test_settings.py index a393e83c6..47f8efdf4 100644 --- a/src/yunohost/tests/test_settings.py +++ b/src/yunohost/tests/test_settings.py @@ -30,7 +30,7 @@ def setup_function(function): def teardown_function(function): os.system("mv /etc/yunohost/settings.json.saved /etc/yunohost/settings.json") for filename in glob.glob("/etc/yunohost/settings-*.json"): - os.remove(filename) + os.remove(filename) def test_settings_get_bool(): diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 8fb65bac8..ada43edaa 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -413,7 +413,9 @@ def tools_update(target=None, apps=False, system=False): # Legacy options (--system, --apps) if apps or system: - logger.warning("Using 'yunohost tools update' with --apps / --system is deprecated, just write 'yunohost tools update apps system' (no -- prefix anymore)") + logger.warning( + "Using 'yunohost tools update' with --apps / --system is deprecated, just write 'yunohost tools update apps system' (no -- prefix anymore)" + ) if apps and system: target = "all" elif apps: @@ -425,7 +427,10 @@ def tools_update(target=None, apps=False, system=False): target = "all" if target not in ["system", "apps", "all"]: - raise YunohostError("Unknown target %s, should be 'system', 'apps' or 'all'" % target, raw_msg=True) + raise YunohostError( + "Unknown target %s, should be 'system', 'apps' or 'all'" % target, + raw_msg=True, + ) upgradable_system_packages = [] if target in ["system", "all"]: @@ -548,7 +553,9 @@ def tools_upgrade( # Legacy options management (--system, --apps) if target is None: - logger.warning("Using 'yunohost tools upgrade' with --apps / --system is deprecated, just write 'yunohost tools upgrade apps' or 'system' (no -- prefix anymore)") + logger.warning( + "Using 'yunohost tools upgrade' with --apps / --system is deprecated, just write 'yunohost tools upgrade apps' or 'system' (no -- prefix anymore)" + ) if (system, apps) == (True, True): raise YunohostValidationError("tools_upgrade_cant_both") @@ -559,7 +566,9 @@ def tools_upgrade( target = "apps" if apps else "system" if target not in ["apps", "system"]: - raise Exception("Uhoh ?! tools_upgrade should have 'apps' or 'system' value for argument target") + raise Exception( + "Uhoh ?! tools_upgrade should have 'apps' or 'system' value for argument target" + ) # # Apps @@ -914,9 +923,13 @@ def tools_migrations_run( pending = [t.id for t in targets if t.state == "pending"] if skip and done: - raise YunohostValidationError("migrations_not_pending_cant_skip", ids=", ".join(done)) + raise YunohostValidationError( + "migrations_not_pending_cant_skip", ids=", ".join(done) + ) if force_rerun and pending: - raise YunohostValidationError("migrations_pending_cant_rerun", ids=", ".join(pending)) + raise YunohostValidationError( + "migrations_pending_cant_rerun", ids=", ".join(pending) + ) if not (skip or force_rerun) and done: raise YunohostValidationError("migrations_already_ran", ids=", ".join(done)) @@ -1129,9 +1142,11 @@ def _tools_migrations_run_after_system_restore(backup_version): return for migration in all_migrations: - if hasattr(migration, "introduced_in_version") \ - and version.parse(migration.introduced_in_version) > backup_version \ - and hasattr(migration, "run_after_system_restore"): + if ( + hasattr(migration, "introduced_in_version") + and version.parse(migration.introduced_in_version) > backup_version + and hasattr(migration, "run_after_system_restore") + ): try: logger.info(m18n.n("migrations_running_forward", id=migration.id)) migration.run_after_system_restore() @@ -1154,9 +1169,11 @@ def _tools_migrations_run_before_app_restore(backup_version, app_id): return for migration in all_migrations: - if hasattr(migration, "introduced_in_version") \ - and version.parse(migration.introduced_in_version) > backup_version \ - and hasattr(migration, "run_before_app_restore"): + if ( + hasattr(migration, "introduced_in_version") + and version.parse(migration.introduced_in_version) > backup_version + and hasattr(migration, "run_before_app_restore") + ): try: logger.info(m18n.n("migrations_running_forward", id=migration.id)) migration.run_before_app_restore(app_id) @@ -1167,6 +1184,7 @@ def _tools_migrations_run_before_app_restore(backup_version, app_id): logger.error(msg, exc_info=1) raise + class Migration(object): # Those are to be implemented by daughter classes @@ -1193,7 +1211,6 @@ class Migration(object): return m18n.n("migration_description_%s" % self.id) def ldap_migration(run): - def func(self): # Backup LDAP before the migration @@ -1206,7 +1223,9 @@ class Migration(object): os.system("systemctl stop slapd") os.system(f"cp -r --preserve /etc/ldap {backup_folder}/ldap_config") os.system(f"cp -r --preserve /var/lib/ldap {backup_folder}/ldap_db") - os.system(f"cp -r --preserve /etc/yunohost/apps {backup_folder}/apps_settings") + os.system( + f"cp -r --preserve /etc/yunohost/apps {backup_folder}/apps_settings" + ) except Exception as e: raise YunohostError( "migration_ldap_can_not_backup_before_migration", error=str(e) @@ -1217,13 +1236,17 @@ class Migration(object): try: run(self, backup_folder) except Exception: - logger.warning(m18n.n("migration_ldap_migration_failed_trying_to_rollback")) + logger.warning( + m18n.n("migration_ldap_migration_failed_trying_to_rollback") + ) os.system("systemctl stop slapd") # To be sure that we don't keep some part of the old config os.system("rm -r /etc/ldap/slapd.d") os.system(f"cp -r --preserve {backup_folder}/ldap_config/. /etc/ldap/") os.system(f"cp -r --preserve {backup_folder}/ldap_db/. /var/lib/ldap/") - os.system(f"cp -r --preserve {backup_folder}/apps_settings/. /etc/yunohost/apps/") + os.system( + f"cp -r --preserve {backup_folder}/apps_settings/. /etc/yunohost/apps/" + ) os.system("systemctl start slapd") os.system(f"rm -r {backup_folder}") logger.info(m18n.n("migration_ldap_rollback_success")) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 5143f610f..266c2774c 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -118,7 +118,9 @@ def user_create( # Validate domain used for email address/xmpp account if domain is None: if msettings.get("interface") == "api": - raise YunohostValidationError("Invalid usage, you should specify a domain argument") + raise YunohostValidationError( + "Invalid usage, you should specify a domain argument" + ) else: # On affiche les differents domaines possibles msignals.display(m18n.n("domains_available")) @@ -223,7 +225,9 @@ def user_create( logger.warning(m18n.n("user_home_creation_failed"), exc_info=1) try: - subprocess.check_call(["setfacl", "-m", "g:all_users:---", "/home/%s" % username]) + subprocess.check_call( + ["setfacl", "-m", "g:all_users:---", "/home/%s" % username] + ) except subprocess.CalledProcessError: logger.warning("Failed to protect /home/%s" % username, exc_info=1) @@ -412,7 +416,9 @@ def user_update( try: ldap.validate_uniqueness({"mail": mail}) except Exception as e: - raise YunohostValidationError("user_update_failed", user=username, error=e) + raise YunohostValidationError( + "user_update_failed", user=username, error=e + ) if mail[mail.find("@") + 1 :] not in domains: raise YunohostValidationError( "mail_domain_unknown", domain=mail[mail.find("@") + 1 :] @@ -649,7 +655,9 @@ def user_group_create( "sed --in-place '/^%s:/d' /etc/group" % groupname, shell=True ) else: - raise YunohostValidationError("group_already_exist_on_system", group=groupname) + raise YunohostValidationError( + "group_already_exist_on_system", group=groupname + ) if not gid: # Get random GID @@ -758,7 +766,9 @@ def user_group_update( elif groupname == "visitors": raise YunohostValidationError("group_cannot_edit_visitors") elif groupname in existing_users: - raise YunohostValidationError("group_cannot_edit_primary_group", group=groupname) + raise YunohostValidationError( + "group_cannot_edit_primary_group", group=groupname + ) # We extract the uid for each member of the group to keep a simple flat list of members current_group = user_group_info(groupname)["members"] @@ -864,9 +874,7 @@ def user_group_add(groupname, usernames, force=False, sync_perm=True): usernames -- User(s) to add in the group """ - return user_group_update( - groupname, add=usernames, force=force, sync_perm=sync_perm - ) + return user_group_update(groupname, add=usernames, force=force, sync_perm=sync_perm) def user_group_remove(groupname, usernames, force=False, sync_perm=True): @@ -891,7 +899,9 @@ def user_group_remove(groupname, usernames, force=False, sync_perm=True): def user_permission_list(short=False, full=False, apps=[]): import yunohost.permission - return yunohost.permission.user_permission_list(short, full, absolute_urls=True, apps=apps) + return yunohost.permission.user_permission_list( + short, full, absolute_urls=True, apps=apps + ) def user_permission_update(permission, label=None, show_tile=None, sync_perm=True): @@ -902,9 +912,7 @@ def user_permission_update(permission, label=None, show_tile=None, sync_perm=Tru ) -def user_permission_add( - permission, names, protected=None, force=False, sync_perm=True -): +def user_permission_add(permission, names, protected=None, force=False, sync_perm=True): import yunohost.permission return yunohost.permission.user_permission_update(