diff --git a/.gitlab/ci/lint.gitlab-ci.yml b/.gitlab/ci/lint.gitlab-ci.yml index 8db1ee756..dabe33d62 100644 --- a/.gitlab/ci/lint.gitlab-ci.yml +++ b/.gitlab/ci/lint.gitlab-ci.yml @@ -3,14 +3,6 @@ ######################################## # later we must fix lint and format-check jobs and remove "allow_failure" -lint27: - stage: lint - image: "before-install" - needs: [] - allow_failure: true - script: - - tox -e py27-lint - lint37: stage: lint image: "before-install" @@ -19,17 +11,9 @@ lint37: script: - tox -e py37-lint -invalidcode27: - stage: lint - image: "before-install" - needs: [] - script: - - tox -e py27-invalidcode - invalidcode37: stage: lint image: "before-install" - allow_failure: true needs: [] script: - tox -e py37-invalidcode diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index ef21731f3..6cbb89d0c 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -36,7 +36,7 @@ full-tests: - *install_debs - yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns script: - - python -m pytest --cov=yunohost tests/ src/yunohost/tests/ --junitxml=report.xml + - python3 -m pytest --cov=yunohost tests/ src/yunohost/tests/ --junitxml=report.xml needs: - job: build-yunohost artifacts: true @@ -51,70 +51,70 @@ full-tests: root-tests: extends: .test-stage script: - - python -m pytest tests + - python3 -m pytest tests test-apps: extends: .test-stage script: - cd src/yunohost - - python -m pytest tests/test_apps.py + - python3 -m pytest tests/test_apps.py test-appscatalog: extends: .test-stage script: - cd src/yunohost - - python -m pytest tests/test_appscatalog.py + - python3 -m pytest tests/test_appscatalog.py test-appurl: extends: .test-stage script: - cd src/yunohost - - python -m pytest tests/test_appurl.py + - python3 -m pytest tests/test_appurl.py test-apps-arguments-parsing: extends: .test-stage script: - cd src/yunohost - - python -m pytest tests/test_apps_arguments_parsing.py + - python3 -m pytest tests/test_apps_arguments_parsing.py test-backuprestore: extends: .test-stage script: - cd src/yunohost - - python -m pytest tests/test_backuprestore.py + - python3 -m pytest tests/test_backuprestore.py test-changeurl: extends: .test-stage script: - cd src/yunohost - - python -m pytest tests/test_changeurl.py + - python3 -m pytest tests/test_changeurl.py test-permission: extends: .test-stage script: - cd src/yunohost - - python -m pytest tests/test_permission.py + - python3 -m pytest tests/test_permission.py test-settings: extends: .test-stage script: - cd src/yunohost - - python -m pytest tests/test_settings.py + - python3 -m pytest tests/test_settings.py test-user-group: extends: .test-stage script: - cd src/yunohost - - python -m pytest tests/test_user-group.py + - python3 -m pytest tests/test_user-group.py test-regenconf: extends: .test-stage script: - cd src/yunohost - - python -m pytest tests/test_regenconf.py + - python3 -m pytest tests/test_regenconf.py test-service: extends: .test-stage script: - cd src/yunohost - - python -m pytest tests/test_service.py + - python3 -m pytest tests/test_service.py diff --git a/bin/yunohost b/bin/yunohost index 546d2d913..0220c5f09 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/python3 # -*- coding: utf-8 -*- import os diff --git a/bin/yunohost-api b/bin/yunohost-api index cc849590a..b3ed3a817 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/python3 # -*- coding: utf-8 -*- import sys diff --git a/data/helpers.d/logging b/data/helpers.d/logging index 45b5b7e67..dc32ecba9 100644 --- a/data/helpers.d/logging +++ b/data/helpers.d/logging @@ -35,7 +35,7 @@ ynh_print_info() { # Manage arguments with getopts ynh_handle_getopts_args "$@" - echo "$message" >> "$YNH_STDINFO" + echo "$message" >&$YNH_STDINFO } # Ignore the yunohost-cli log to prevent errors with conditional commands diff --git a/data/helpers.d/setting b/data/helpers.d/setting index e3c9c2f34..a66e0d1ea 100644 --- a/data/helpers.d/setting +++ b/data/helpers.d/setting @@ -77,9 +77,9 @@ ynh_app_setting_delete() { # [internal] # ynh_app_setting() -{ +{ set +o xtrace # set +x - ACTION="$1" APP="$2" KEY="$3" VALUE="${4:-}" python2.7 - < $output_path } diff --git a/data/hooks/conf_regen/01-yunohost b/data/hooks/conf_regen/01-yunohost index c4120d487..6ac61d07a 100755 --- a/data/hooks/conf_regen/01-yunohost +++ b/data/hooks/conf_regen/01-yunohost @@ -115,7 +115,7 @@ do_post_regen() { } _update_services() { - python2 - << EOF + python3 - << EOF import yaml diff --git a/data/hooks/diagnosis/00-basesystem.py b/data/hooks/diagnosis/00-basesystem.py index e92f9684b..a582805e9 100644 --- a/data/hooks/diagnosis/00-basesystem.py +++ b/data/hooks/diagnosis/00-basesystem.py @@ -159,7 +159,8 @@ class BaseSystemDiagnoser(Diagnoser): # "missing some kernel info (see -v), accuracy might be reduced" # Dunno what to do about that but we probably don't want to harass # users with this warning ... - output, err = call.communicate() + output, _ = call.communicate() + output = output.decode() assert call.returncode in (0, 2, 3), "Return code: %s" % call.returncode # If there are multiple lines, sounds like there was some messages diff --git a/data/hooks/diagnosis/50-systemresources.py b/data/hooks/diagnosis/50-systemresources.py index f0fac4974..64517f764 100644 --- a/data/hooks/diagnosis/50-systemresources.py +++ b/data/hooks/diagnosis/50-systemresources.py @@ -1,10 +1,11 @@ #!/usr/bin/env python import os import psutil -import subprocess import datetime import re +from moulinette.utils.process import check_output + from yunohost.diagnosis import Diagnoser @@ -119,7 +120,7 @@ class SystemResourcesDiagnoser(Diagnoser): def analyzed_kern_log(): cmd = 'tail -n 10000 /var/log/kern.log | grep "oom_reaper: reaped process" || true' - out = subprocess.check_output(cmd, shell=True).strip() + out = check_output(cmd) lines = out.split("\n") if out else [] now = datetime.datetime.now() diff --git a/debian/control b/debian/control index bfea80ccd..4e57c7a84 100644 --- a/debian/control +++ b/debian/control @@ -2,19 +2,18 @@ Source: yunohost Section: utils Priority: extra Maintainer: YunoHost Contributors -Build-Depends: debhelper (>=9), dh-systemd, dh-python, python-all (>= 2.7), python-yaml, python-jinja2 +Build-Depends: debhelper (>=9), dh-systemd, dh-python, python3-all (>= 3.7), python3-yaml, python3-jinja2 Standards-Version: 3.9.6 -X-Python-Version: >= 2.7 Homepage: https://yunohost.org/ Package: yunohost Essential: yes Architecture: all -Depends: ${python:Depends}, ${misc:Depends} +Depends: ${python3:Depends}, ${misc:Depends} , moulinette (>= 4.1.0.1), ssowat (>= 4.0) - , python-psutil, python-requests, python-dnspython, python-openssl - , python-miniupnpc, python-dbus, python-jinja2 - , python-toml, python-packaging, python-publicsuffix + , python3-psutil, python3-requests, python3-dnspython, python3-openssl + , python3-miniupnpc, python3-dbus, python3-jinja2 + , python3-toml, python3-packaging, python3-publicsuffix , apt, apt-transport-https, apt-utils, dirmngr , php7.3-common, php7.3-fpm, php7.3-ldap, php7.3-intl , mariadb-server, php7.3-mysql @@ -33,7 +32,7 @@ Recommends: yunohost-admin , ntp, inetutils-ping | iputils-ping , bash-completion, rsyslog , php7.3-gd, php7.3-curl, php-gettext - , python-pip + , python3-pip , unattended-upgrades , libdbd-ldap-perl, libnet-dns-perl Suggests: htop, vim, rsync, acpi-support-base, udisks2 diff --git a/debian/rules b/debian/rules index 8afe372b5..54ebc5781 100755 --- a/debian/rules +++ b/debian/rules @@ -5,7 +5,7 @@ #export DH_VERBOSE=1 %: - dh ${@} --with=python2,systemd + dh ${@} --with=python3,systemd override_dh_auto_build: # Generate bash completion file diff --git a/doc/generate_helper_doc.py b/doc/generate_helper_doc.py index bc9611c8f..908d6300e 100644 --- a/doc/generate_helper_doc.py +++ b/doc/generate_helper_doc.py @@ -1,4 +1,4 @@ -#!/usr/env/python2.7 +#!/usr/env/python3 import os import glob diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 0349f92e7..7a6521386 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -30,16 +30,16 @@ import shutil import yaml import time import re -import urlparse +import urllib.parse import subprocess import glob -import urllib +import urllib.request, urllib.parse, urllib.error from collections import OrderedDict from moulinette import msignals, m18n, msettings from moulinette.utils.log import getActionLogger from moulinette.utils.network import download_json -from moulinette.utils.process import run_commands +from moulinette.utils.process import run_commands, check_output from moulinette.utils.filesystem import read_file, read_json, read_toml, read_yaml, write_to_file, write_to_json, write_to_yaml, chmod, chown, mkdir from yunohost.service import service_status, _run_service_command @@ -426,10 +426,7 @@ def app_change_url(operation_logger, app, domain, path): # grab nginx errors # the "exit 0" is here to avoid check_output to fail because 'nginx -t' # will return != 0 since we are in a failed state - nginx_errors = subprocess.check_output("nginx -t; exit 0", - stderr=subprocess.STDOUT, - shell=True).rstrip() - + nginx_errors = check_output("nginx -t; exit 0") raise YunohostError("app_change_url_failed_nginx_reload", nginx_errors=nginx_errors) logger.success(m18n.n("app_change_url_success", @@ -751,7 +748,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu # Retrieve arguments list for install script args_dict = {} if not args else \ - dict(urlparse.parse_qsl(args, keep_blank_values=True)) + dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) args_odict = _parse_args_from_manifest(manifest, 'install', args=args_dict) # Validate domain / path availability for webapps @@ -774,7 +771,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu # Also redact the % escaped version of the password that might appear in # the 'args' section of metadata (relevant for password with non-alphanumeric char) data_to_redact = [value[0] for value in args_odict.values() if value[1] == "password"] - data_to_redact += [urllib.quote(data) for data in data_to_redact if urllib.quote(data) != data] + data_to_redact += [urllib.parse.quote(data) for data in data_to_redact if urllib.parse.quote(data) != data] operation_logger.data_to_redact.extend(data_to_redact) operation_logger.related_to = [s for s in operation_logger.related_to if s[0] != "app"] @@ -849,7 +846,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu logger.error(m18n.n("app_install_failed", app=app_id, error=error)) failure_message_with_debug_instructions = operation_logger.error(error) # Something wrong happened in Yunohost's code (most probably hook_exec) - except Exception as e: + except Exception: import traceback error = m18n.n('unexpected_error', error="\n" + traceback.format_exc()) logger.error(m18n.n("app_install_failed", app=app_id, error=error)) @@ -1421,7 +1418,7 @@ def app_ssowatconf(): write_to_json('/etc/ssowat/conf.json', conf_dict, sort_keys=True, indent=4) - from utils.legacy import translate_legacy_rules_in_ssowant_conf_json_persistent + from .utils.legacy import translate_legacy_rules_in_ssowant_conf_json_persistent translate_legacy_rules_in_ssowant_conf_json_persistent() logger.debug(m18n.n('ssowat_conf_generated')) @@ -1471,7 +1468,7 @@ def app_action_run(operation_logger, app, action, args=None): action_declaration = actions[action] # Retrieve arguments list for install script - args_dict = dict(urlparse.parse_qsl(args, keep_blank_values=True)) if args else {} + args_dict = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {} args_odict = _parse_args_for_action(actions[action], args=args_dict) args_list = [value[0] for value in args_odict.values()] @@ -1613,7 +1610,7 @@ def app_config_apply(operation_logger, app, args): "YNH_APP_INSTANCE_NAME": app, "YNH_APP_INSTANCE_NUMBER": str(app_instance_nb), } - args = dict(urlparse.parse_qsl(args, keep_blank_values=True)) if args else {} + args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {} for tab in config_panel.get("panel", []): tab_id = tab["id"] # this makes things easier to debug on crash @@ -1832,8 +1829,7 @@ def _get_app_config_panel(app_id): "panel": [], } - panels = filter(lambda key_value: key_value[0] not in ("name", "version") and isinstance(key_value[1], OrderedDict), - toml_config_panel.items()) + panels = [key_value for key_value in toml_config_panel.items() if key_value[0] not in ("name", "version") and isinstance(key_value[1], OrderedDict)] for key, value in panels: panel = { @@ -1842,8 +1838,7 @@ def _get_app_config_panel(app_id): "sections": [], } - sections = filter(lambda k_v1: k_v1[0] not in ("name",) and isinstance(k_v1[1], OrderedDict), - value.items()) + sections = [k_v1 for k_v1 in value.items() if k_v1[0] not in ("name",) and isinstance(k_v1[1], OrderedDict)] for section_key, section_value in sections: section = { @@ -1852,8 +1847,7 @@ def _get_app_config_panel(app_id): "options": [], } - options = filter(lambda k_v: k_v[0] not in ("name",) and isinstance(k_v[1], OrderedDict), - section_value.items()) + options = [k_v for k_v in section_value.items() if k_v[0] not in ("name",) and isinstance(k_v[1], OrderedDict)] for option_key, option_value in options: option = dict(option_value) @@ -1891,7 +1885,7 @@ def _get_app_settings(app_id): settings = yaml.load(f) # If label contains unicode char, this may later trigger issues when building strings... # FIXME: this should be propagated to read_yaml so that this fix applies everywhere I think... - settings = {k: _encode_string(v) for k, v in settings.items()} + settings = {k: v for k, v in settings.items()} if app_id == settings['id']: return settings except (IOError, TypeError, KeyError): @@ -2157,10 +2151,9 @@ def _get_git_last_commit_hash(repository, reference='HEAD'): """ try: - commit = subprocess.check_output( - "git ls-remote --exit-code {0} {1} | awk '{{print $1}}'".format( - repository, reference), - shell=True) + cmd = "git ls-remote --exit-code {0} {1} | awk '{{print $1}}'"\ + .format(repository, reference) + commit = check_output(cmd) except subprocess.CalledProcessError: logger.exception("unable to get last commit from %s", repository) raise ValueError("Unable to get last commit with git") @@ -2318,21 +2311,12 @@ def _value_for_locale(values): for lang in [m18n.locale, m18n.default_locale]: try: - return _encode_string(values[lang]) + return values[lang] except KeyError: continue # Fallback to first value - return _encode_string(values.values()[0]) - - -def _encode_string(value): - """ - Return the string encoded in utf-8 if needed - """ - if isinstance(value, unicode): - return value.encode('utf8') - return value + return list(values.values())[0] def _check_manifest_requirements(manifest, app_instance_name): @@ -2945,7 +2929,7 @@ def _load_apps_catalog(): try: apps_catalog_content = read_json(cache_file) if os.path.exists(cache_file) else None except Exception as e: - raise ("Unable to read cache for apps_catalog %s : %s" % (apps_catalog_id, str(e))) + raise YunohostError("Unable to read cache for apps_catalog %s : %s" % (cache_file, e), raw_msg=True) # Check that the version of the data matches version .... # ... otherwise it means we updated yunohost in the meantime @@ -2995,7 +2979,7 @@ def is_true(arg): """ if isinstance(arg, bool): return arg - elif isinstance(arg, basestring): + elif isinstance(arg, str): return arg.lower() in ['yes', 'true', 'on'] else: logger.debug('arg should be a boolean or a string, got %r', arg) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 242cd0bfd..7b37d1bf2 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -41,6 +41,7 @@ from moulinette import msignals, m18n, msettings from moulinette.utils import filesystem from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_file, mkdir, write_to_yaml, read_yaml +from moulinette.utils.process import check_output from yunohost.app import ( app_info, _is_installed, @@ -176,11 +177,11 @@ class BackupRestoreTargetsManager(object): or (exclude and isinstance(exclude, list) and not include) if include: - return [target.encode("Utf-8") for target in self.targets[category] + return [target for target in self.targets[category] if self.results[category][target] in include] if exclude: - return [target.encode("Utf-8") for target in self.targets[category] + return [target for target in self.targets[category] if self.results[category][target] not in exclude] @@ -599,7 +600,7 @@ class BackupManager(): for hook, infos in ret.items() if any(result["state"] == "failed" for result in infos.values())} - if ret_succeed.keys() != []: + if list(ret_succeed.keys()) != []: self.system_return = ret_succeed # Add files from targets (which they put in the CSV) to the list of @@ -882,7 +883,7 @@ class RestoreManager(): End a restore operations by cleaning the working directory and regenerate ssowat conf (if some apps were restored) """ - from permission import permission_sync_to_user + from .permission import permission_sync_to_user permission_sync_to_user() @@ -1643,7 +1644,7 @@ class BackupMethod(object): try: subprocess.check_call(["mount", "--rbind", src, dest]) subprocess.check_call(["mount", "-o", "remount,ro,bind", dest]) - except Exception as e: + except Exception: logger.warning(m18n.n("backup_couldnt_bind", src=src, dest=dest)) # To check if dest is mounted, use /proc/mounts that # escape spaces as \040 @@ -2165,7 +2166,7 @@ def backup_list(with_info=False, human_readable=False): d[archive] = backup_info(archive, human_readable=human_readable) except YunohostError as e: logger.warning(str(e)) - except Exception as e: + except Exception: import traceback logger.warning("Could not check infos for archive %s: %s" % (archive, '\n' + traceback.format_exc())) @@ -2386,7 +2387,7 @@ def _recursive_umount(directory): Args: directory -- a directory path """ - mount_lines = subprocess.check_output("mount").split("\n") + mount_lines = check_output("mount").split("\n") points_to_umount = [line.split(" ")[2] for line in mount_lines @@ -2412,8 +2413,8 @@ def disk_usage(path): # We don't do this in python with os.stat because we don't want # to follow symlinks - du_output = subprocess.check_output(['du', '-sb', path]) - return int(du_output.split()[0].decode('utf-8')) + du_output = check_output(['du', '-sb', path], shell=False) + return int(du_output.split()[0]) def binary_to_human(n, customary=False): diff --git a/src/yunohost/certificate.py b/src/yunohost/certificate.py index 2e4f1623d..7c633de4f 100644 --- a/src/yunohost/certificate.py +++ b/src/yunohost/certificate.py @@ -385,7 +385,7 @@ def certificate_renew(domain_list, force=False, no_checks=False, email=False, st _fetch_and_enable_new_certificate(domain, staging, no_checks=no_checks) except Exception as e: import traceback - from StringIO import StringIO + from io import StringIO stack = StringIO() traceback.print_exc(file=stack) msg = "Certificate renewing for %s failed !" % (domain) @@ -638,7 +638,7 @@ def _get_status(domain): cert_subject = cert.get_subject().CN cert_issuer = cert.get_issuer().CN organization_name = cert.get_issuer().O - valid_up_to = datetime.strptime(cert.get_notAfter(), "%Y%m%d%H%M%SZ") + valid_up_to = datetime.strptime(cert.get_notAfter().decode('utf-8'), "%Y%m%d%H%M%SZ") days_remaining = (valid_up_to - datetime.utcnow()).days if cert_issuer == _name_self_CA(): diff --git a/src/yunohost/data_migrations/0019_extend_permissions_features.py b/src/yunohost/data_migrations/0019_extend_permissions_features.py index b20b92e62..6c292e014 100644 --- a/src/yunohost/data_migrations/0019_extend_permissions_features.py +++ b/src/yunohost/data_migrations/0019_extend_permissions_features.py @@ -98,7 +98,7 @@ class MyMigration(Migration): # Migrate old settings migrate_legacy_permission_settings() - except Exception as e: + except Exception: logger.warn(m18n.n("migration_0019_migration_failed_trying_to_rollback")) os.system("systemctl stop slapd") os.system("rm -r /etc/ldap/slapd.d") # To be sure that we don't keep some part of the old config diff --git a/src/yunohost/diagnosis.py b/src/yunohost/diagnosis.py index e4dd84f6d..93ece21fc 100644 --- a/src/yunohost/diagnosis.py +++ b/src/yunohost/diagnosis.py @@ -451,7 +451,7 @@ class Diagnoser(): key = "diagnosis_description_" + id_ descr = m18n.n(key) # If no description available, fallback to id - return descr if descr.decode('utf-8') != key else id_ + return descr if descr != key else id_ @staticmethod def i18n(report, force_remove_html_tags=False): diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index d581b8426..af45d8757 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -62,18 +62,15 @@ def domain_list(exclude_subdomains=False): result_list.append(domain) - def cmp_domain(domain1, domain2): + def cmp_domain(domain): # Keep the main part of the domain and the extension together # eg: this.is.an.example.com -> ['example.com', 'an', 'is', 'this'] - domain1 = domain1.split('.') - domain2 = domain2.split('.') - domain1[-1] = domain1[-2] + domain1.pop() - domain2[-1] = domain2[-2] + domain2.pop() - domain1 = list(reversed(domain1)) - domain2 = list(reversed(domain2)) - return cmp(domain1, domain2) + domain = domain.split('.') + domain[-1] = domain[-2] + domain.pop() + domain = list(reversed(domain)) + return domain - result_list = sorted(result_list, cmp_domain) + result_list = sorted(result_list, key=cmp_domain) return { 'domains': result_list, diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index fe2a1bc9b..e43c180ad 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -105,7 +105,7 @@ def _dyndns_available(provider, domain): raise YunohostError('dyndns_could_not_check_available', domain=domain, provider=provider) - return r == u"Domain %s is available" % domain + return r == "Domain %s is available" % domain @is_unit_operation() diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index eafcaf825..067a5d7a7 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -182,7 +182,8 @@ def hook_list(action, list_by='name', show_info=False): def _append_folder(d, folder): # Iterate over and add hook from a folder for f in os.listdir(folder + action): - if f[0] == '.' or f[-1] == '~' or f.endswith(".pyc"): + if f[0] == '.' or f[-1] == '~' or f.endswith(".pyc") \ + or (f.startswith("__") and f.endswith("__")): continue path = '%s%s/%s' % (folder, action, f) priority, name = _extract_filename_parts(f) @@ -387,9 +388,6 @@ def _hook_exec_bash(path, args, no_trace, chdir, env, user, return_format, logge env['YNH_INTERFACE'] = msettings.get('interface') - stdinfo = os.path.join(tempfile.mkdtemp(), "stdinfo") - env['YNH_STDINFO'] = stdinfo - stdreturn = os.path.join(tempfile.mkdtemp(), "stdreturn") with open(stdreturn, 'w') as f: f.write('') @@ -415,10 +413,7 @@ def _hook_exec_bash(path, args, no_trace, chdir, env, user, return_format, logge logger.debug("Executing command '%s'" % ' '.join(command)) - returncode = call_async_output( - command, loggers, shell=False, cwd=chdir, - stdinfo=stdinfo - ) + returncode = call_async_output(command, loggers, shell=False, cwd=chdir) raw_content = None try: diff --git a/src/yunohost/log.py b/src/yunohost/log.py index 7e9ae18e6..850680237 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -65,8 +65,7 @@ def log_list(limit=None, with_details=False, with_suboperations=False): operations = {} - logs = filter(lambda x: x.endswith(METADATA_FILE_EXT), - os.listdir(OPERATIONS_PATH)) + logs = [x for x in os.listdir(OPERATIONS_PATH) if x.endswith(METADATA_FILE_EXT)] logs = list(reversed(sorted(logs))) if limit is not None: @@ -337,7 +336,7 @@ def is_unit_operation(entities=['app', 'domain', 'group', 'service', 'user'], entity_type = entity if entity in kwargs and kwargs[entity] is not None: - if isinstance(kwargs[entity], basestring): + if isinstance(kwargs[entity], str): related_to.append((entity_type, kwargs[entity])) else: for x in kwargs[entity]: @@ -596,7 +595,7 @@ class OperationLogger(object): """ if self.ended_at is not None or self.started_at is None: return - if error is not None and not isinstance(error, basestring): + if error is not None and not isinstance(error, str): error = str(error) self.ended_at = datetime.utcnow() self._error = error diff --git a/src/yunohost/permission.py b/src/yunohost/permission.py index d213ac61c..4db06e291 100644 --- a/src/yunohost/permission.py +++ b/src/yunohost/permission.py @@ -106,7 +106,7 @@ def user_permission_list(short=False, full=False, ignore_system_perms=False, abs infos["label"] = "%s (%s)" % (main_perm_label, infos["label"]) if short: - permissions = permissions.keys() + permissions = list(permissions.keys()) return {'permissions': permissions} @@ -668,7 +668,7 @@ def _validate_and_sanitize_permission_url(url, app_base_path, app): For example: re:/api/[A-Z]*$ -> domain.tld/app/api/[A-Z]*$ re:domain.tld/app/api/[A-Z]*$ -> domain.tld/app/api/[A-Z]*$ - + We can also have less-trivial regexes like: re:^\/api\/.*|\/scripts\/api.js$ """ diff --git a/src/yunohost/regenconf.py b/src/yunohost/regenconf.py index 6b369fc8c..9423b1b36 100644 --- a/src/yunohost/regenconf.py +++ b/src/yunohost/regenconf.py @@ -21,7 +21,6 @@ import os import yaml -import subprocess import shutil import hashlib @@ -30,6 +29,7 @@ from datetime import datetime from moulinette import m18n from moulinette.utils import log, filesystem +from moulinette.utils.process import check_output from yunohost.utils.error import YunohostError from yunohost.log import is_unit_operation @@ -654,10 +654,10 @@ def manually_modified_files(): def manually_modified_files_compared_to_debian_default(ignore_handled_by_regenconf=False): # from https://serverfault.com/a/90401 - files = subprocess.check_output("dpkg-query -W -f='${Conffiles}\n' '*' \ - | awk 'OFS=\" \"{print $2,$1}' \ - | md5sum -c 2>/dev/null \ - | awk -F': ' '$2 !~ /OK/{print $1}'", shell=True) + files = check_output("dpkg-query -W -f='${Conffiles}\n' '*' \ + | awk 'OFS=\" \"{print $2,$1}' \ + | md5sum -c 2>/dev/null \ + | awk -F': ' '$2 !~ /OK/{print $1}'") files = files.strip().split("\n") if ignore_handled_by_regenconf: diff --git a/src/yunohost/service.py b/src/yunohost/service.py index 1b1fec81b..347932add 100644 --- a/src/yunohost/service.py +++ b/src/yunohost/service.py @@ -35,6 +35,7 @@ from datetime import datetime from moulinette import m18n from yunohost.utils.error import YunohostError +from moulinette.utils.process import check_output from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_file, append_to_file, write_to_file @@ -358,7 +359,7 @@ def _get_and_format_service_status(service, infos): # that mean that we don't have a translation for this string # that's the only way to test for that for now # if we don't have it, uses the one provided by systemd - if description.decode('utf-8') == translation_key: + if description == translation_key: description = str(raw_status.get("Description", "")) output = { @@ -489,7 +490,7 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False, raise YunohostError('service_unknown', service=name) if names is []: - names = services.keys() + names = list(services.keys()) logger.warning(m18n.n("service_regen_conf_is_deprecated")) @@ -563,8 +564,7 @@ def _give_lock(action, service, p): while son_PID == 0 and p.poll() is None: # Call systemctl to get the PID # Output of the command is e.g. ControlPID=1234 - son_PID = subprocess.check_output(cmd_get_son_PID.split()) \ - .strip().split("=")[1] + son_PID = check_output(cmd_get_son_PID).split("=")[1] son_PID = int(son_PID) time.sleep(1) @@ -599,7 +599,7 @@ def _get_services(): # some services are marked as None to remove them from YunoHost # filter this - for key, value in services.items(): + for key, value in list(services.items()): if value is None: del services[key] @@ -720,7 +720,7 @@ def _get_journalctl_logs(service, number="all"): services = _get_services() systemd_service = services.get(service, {}).get("actual_systemd_service", service) try: - return subprocess.check_output("journalctl --no-hostname --no-pager -u {0} -n{1}".format(systemd_service, number), shell=True) + return check_output("journalctl --no-hostname --no-pager -u {0} -n{1}".format(systemd_service, number)) except: import traceback return "error while get services logs from journalctl:\n%s" % traceback.format_exc() diff --git a/src/yunohost/settings.py b/src/yunohost/settings.py index 3c79d7945..060aca6e4 100644 --- a/src/yunohost/settings.py +++ b/src/yunohost/settings.py @@ -29,7 +29,7 @@ def is_boolean(value): """ if isinstance(value, bool): return True, value - elif isinstance(value, basestring): + elif isinstance(value, str): if str(value).lower() in ['true', 'on', 'yes', 'false', 'off', 'no']: return True, str(value).lower() in ['true', 'on', 'yes'] else: @@ -141,7 +141,7 @@ def settings_set(key, value): raise YunohostError('global_settings_bad_type_for_setting', setting=key, received_type=type(value).__name__, expected_type=key_type) elif key_type == "string": - if not isinstance(value, basestring): + if not isinstance(value, str): raise YunohostError('global_settings_bad_type_for_setting', setting=key, received_type=type(value).__name__, expected_type=key_type) elif key_type == "enum": diff --git a/src/yunohost/tests/test_apps.py b/src/yunohost/tests/test_apps.py index 0f4c3749a..baa3d5181 100644 --- a/src/yunohost/tests/test_apps.py +++ b/src/yunohost/tests/test_apps.py @@ -4,7 +4,7 @@ import pytest import shutil import requests -from conftest import message, raiseYunohostError, get_test_apps_dir +from .conftest import message, raiseYunohostError, get_test_apps_dir from moulinette.utils.filesystem import mkdir diff --git a/src/yunohost/tests/test_apps_arguments_parsing.py b/src/yunohost/tests/test_apps_arguments_parsing.py index ca08f58ff..03a2fa0e6 100644 --- a/src/yunohost/tests/test_apps_arguments_parsing.py +++ b/src/yunohost/tests/test_apps_arguments_parsing.py @@ -2,7 +2,7 @@ import sys import pytest from mock import patch -from StringIO import StringIO +from io import StringIO from collections import OrderedDict from moulinette import msignals diff --git a/src/yunohost/tests/test_appurl.py b/src/yunohost/tests/test_appurl.py index 11ee7b4f5..3a2313b0e 100644 --- a/src/yunohost/tests/test_appurl.py +++ b/src/yunohost/tests/test_appurl.py @@ -1,7 +1,7 @@ import pytest import os -from conftest import get_test_apps_dir +from .conftest import get_test_apps_dir from yunohost.utils.error import YunohostError from yunohost.app import app_install, app_remove, _normalize_domain_path diff --git a/src/yunohost/tests/test_backuprestore.py b/src/yunohost/tests/test_backuprestore.py index 9c9448879..c1f211d1b 100644 --- a/src/yunohost/tests/test_backuprestore.py +++ b/src/yunohost/tests/test_backuprestore.py @@ -3,7 +3,7 @@ import os import shutil import subprocess -from conftest import message, raiseYunohostError, get_test_apps_dir +from .conftest import message, raiseYunohostError, get_test_apps_dir from yunohost.app import app_install, app_remove, app_ssowatconf from yunohost.app import _is_installed @@ -23,8 +23,6 @@ def setup_function(function): global maindomain maindomain = _get_maindomain() - print "" - assert backup_test_dependencies_are_met() clean_tmp_backup_directory() @@ -150,7 +148,7 @@ def clean_tmp_backup_directory(): if tmp_backup_directory_is_empty(): return - mount_lines = subprocess.check_output("mount").split("\n") + mount_lines = subprocess.check_output("mount").decode().split("\n") points_to_umount = [line.split(" ")[2] for line in mount_lines @@ -638,6 +636,7 @@ def test_backup_binds_are_readonly(mocker, monkeypatch): confssh = os.path.join(self.work_dir, "conf/ssh") output = subprocess.check_output("touch %s/test 2>&1 || true" % confssh, shell=True, env={'LANG': 'en_US.UTF-8'}) + output = output.decode() assert "Read-only file system" in output diff --git a/src/yunohost/tests/test_changeurl.py b/src/yunohost/tests/test_changeurl.py index ecd926a2e..ef076e67b 100644 --- a/src/yunohost/tests/test_changeurl.py +++ b/src/yunohost/tests/test_changeurl.py @@ -3,7 +3,7 @@ import time import requests import os -from conftest import get_test_apps_dir +from .conftest import get_test_apps_dir from yunohost.app import app_install, app_change_url, app_remove, app_map from yunohost.domain import _get_maindomain diff --git a/src/yunohost/tests/test_permission.py b/src/yunohost/tests/test_permission.py index 1d3961585..a1f411b80 100644 --- a/src/yunohost/tests/test_permission.py +++ b/src/yunohost/tests/test_permission.py @@ -6,7 +6,7 @@ import os import json import shutil -from conftest import message, raiseYunohostError, get_test_apps_dir +from .conftest import message, raiseYunohostError, get_test_apps_dir from yunohost.app import app_install, app_upgrade, app_remove, app_change_url, app_map, _installed_apps, APPS_SETTING_PATH, _set_app_settings, _get_app_settings from yunohost.user import user_list, user_create, user_delete, \ diff --git a/src/yunohost/tests/test_regenconf.py b/src/yunohost/tests/test_regenconf.py index 4e1ae679b..c9a3e68f5 100644 --- a/src/yunohost/tests/test_regenconf.py +++ b/src/yunohost/tests/test_regenconf.py @@ -1,6 +1,6 @@ import os -from conftest import message +from .conftest import message from yunohost.domain import domain_add, domain_remove, domain_list from yunohost.regenconf import regen_conf, manually_modified_files, _get_conf_hashes, _force_clear_hashes diff --git a/src/yunohost/tests/test_service.py b/src/yunohost/tests/test_service.py index d0a3d4fc2..0c65a864a 100644 --- a/src/yunohost/tests/test_service.py +++ b/src/yunohost/tests/test_service.py @@ -1,6 +1,6 @@ import os -from conftest import raiseYunohostError +from .conftest import raiseYunohostError from yunohost.service import _get_services, _save_services, service_status, service_add, service_remove, service_log diff --git a/src/yunohost/tests/test_user-group.py b/src/yunohost/tests/test_user-group.py index c0f51e35a..31131e939 100644 --- a/src/yunohost/tests/test_user-group.py +++ b/src/yunohost/tests/test_user-group.py @@ -1,6 +1,6 @@ import pytest -from conftest import message, raiseYunohostError +from .conftest import message, raiseYunohostError from yunohost.user import user_list, user_info, user_create, user_delete, user_update, \ user_group_list, user_group_create, user_group_delete, user_group_update diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 96bd01ed6..f01f6adb8 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -303,7 +303,7 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False, '/home/yunohost.app' ] - for folder in filter(lambda x: not os.path.exists(x), folders_to_create): + for folder in [x for x in folders_to_create if not os.path.exists(x)]: os.makedirs(folder) # Change folders permissions @@ -953,7 +953,7 @@ def _get_migrations_list(): # (in particular, pending migrations / not already ran are not listed states = tools_migrations_state()["migrations"] - for migration_file in filter(lambda x: re.match(r"^\d+_[a-zA-Z0-9_]+\.py$", x), os.listdir(migrations_path)): + for migration_file in [x for x in os.listdir(migrations_path) if re.match(r"^\d+_[a-zA-Z0-9_]+\.py$", x)]: m = _load_migration(migration_file) m.state = states.get(m.id, "pending") migrations.append(m) @@ -972,7 +972,7 @@ def _get_migration_by_name(migration_name): raise AssertionError("Unable to find migration with name %s" % migration_name) migrations_path = data_migrations.__path__[0] - migrations_found = filter(lambda x: re.match(r"^\d+_%s\.py$" % migration_name, x), os.listdir(migrations_path)) + migrations_found = [x for x in os.listdir(migrations_path) if re.match(r"^\d+_%s\.py$" % migration_name, x)] assert len(migrations_found) == 1, "Unable to find migration with name %s" % migration_name diff --git a/src/yunohost/user.py b/src/yunohost/user.py index edb1c7c8f..9798d7519 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -35,6 +35,7 @@ import copy from moulinette import msignals, msettings, m18n from moulinette.utils.log import getActionLogger +from moulinette.utils.process import check_output from yunohost.utils.error import YunohostError from yunohost.service import service_status @@ -473,8 +474,7 @@ def user_info(username): else: try: cmd = 'doveadm -f flow quota get -u %s' % user['uid'][0] - cmd_result = subprocess.check_output(cmd, stderr=subprocess.STDOUT, - shell=True) + cmd_result = check_output(cmd) except Exception as e: cmd_result = "" logger.warning("Failed to fetch quota info ... : %s " % str(e)) @@ -546,7 +546,7 @@ def user_group_list(short=False, full=False, include_primary_groups=True): groups[name]["permissions"] = [_ldap_path_extract(p, "cn") for p in infos.get("permission", [])] if short: - groups = groups.keys() + groups = list(groups.keys()) return {'groups': groups} @@ -631,7 +631,7 @@ def user_group_delete(operation_logger, groupname, force=False, sync_perm=True): from yunohost.permission import permission_sync_to_user from yunohost.utils.ldap import _get_ldap_interface - existing_groups = user_group_list()['groups'].keys() + existing_groups = list(user_group_list()['groups'].keys()) if groupname not in existing_groups: raise YunohostError('group_unknown', group=groupname) @@ -639,7 +639,7 @@ def user_group_delete(operation_logger, groupname, force=False, sync_perm=True): # without the force option... # # We also can't delete "all_users" because that's a special group... - existing_users = user_list()['users'].keys() + existing_users = list(user_list()['users'].keys()) undeletable_groups = existing_users + ["all_users", "visitors"] if groupname in undeletable_groups and not force: raise YunohostError('group_cannot_be_deleted', group=groupname) @@ -675,7 +675,7 @@ def user_group_update(operation_logger, groupname, add=None, remove=None, force= from yunohost.permission import permission_sync_to_user from yunohost.utils.ldap import _get_ldap_interface - existing_users = user_list()['users'].keys() + existing_users = list(user_list()['users'].keys()) # Refuse to edit a primary group of a user (e.g. group 'sam' related to user 'sam') # Those kind of group should only ever contain the user (e.g. sam) and only this one. diff --git a/src/yunohost/utils/legacy.py b/src/yunohost/utils/legacy.py index f3269cce1..73125dcb8 100644 --- a/src/yunohost/utils/legacy.py +++ b/src/yunohost/utils/legacy.py @@ -22,7 +22,7 @@ class SetupGroupPermissions(): try: objects = ldap.search(target + ",dc=yunohost,dc=org") # ldap search will raise an exception if no corresponding object is found >.> ... - except Exception as e: + except Exception: logger.debug("%s does not exist, no need to delete it" % target) return @@ -100,7 +100,7 @@ class SetupGroupPermissions(): url = "/" if domain and path else None if permission: - known_users = user_list()["users"].keys() + known_users = list(user_list()["users"].keys()) allowed = [user for user in permission.split(',') if user in known_users] else: allowed = ["all_users"] @@ -237,7 +237,7 @@ def translate_legacy_rules_in_ssowant_conf_json_persistent(): protected_urls = persistent.get("protected_urls", []) + ["re:" + r for r in persistent.get("protected_regex", [])] unprotected_urls = persistent.get("unprotected_urls", []) + ["re:" + r for r in persistent.get("unprotected_regex", [])] - known_users = user_list()["users"].keys() + known_users = list(user_list()["users"].keys()) for legacy_rule in legacy_rules: if legacy_rule in persistent: diff --git a/src/yunohost/utils/packages.py b/src/yunohost/utils/packages.py index f9ad385f8..5672a7233 100644 --- a/src/yunohost/utils/packages.py +++ b/src/yunohost/utils/packages.py @@ -70,7 +70,11 @@ def meets_version_specifier(pkg_name, specifier): op, req_version = re.search(r'(<<|<=|=|>=|>>) *([\d\.]+)', specifier).groups() req_version = version.parse(req_version) - # cmp is a python builtin that returns (-1, 0, 1) depending on comparison + # Python2 had a builtin that returns (-1, 0, 1) depending on comparison + # c.f. https://stackoverflow.com/a/22490617 + def cmp(a, b): + return (a > b) - (a < b) + deb_operators = { "<<": lambda v1, v2: cmp(v1, v2) in [-1], "<=": lambda v1, v2: cmp(v1, v2) in [-1, 0], diff --git a/src/yunohost/utils/password.py b/src/yunohost/utils/password.py index 14f42ca31..10f2cd9ad 100644 --- a/src/yunohost/utils/password.py +++ b/src/yunohost/utils/password.py @@ -173,7 +173,7 @@ class PasswordValidator(object): # stdin to avoid it being shown in ps -ef --forest... command = "grep -q -F -f - %s" % MOST_USED_PASSWORDS p = subprocess.Popen(command.split(), stdin=subprocess.PIPE) - p.communicate(input=password) + p.communicate(input=password.encode('utf-8')) return not bool(p.returncode) diff --git a/tox.ini b/tox.ini index 6c1139d52..36134e85a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,12 +1,12 @@ [tox] -envlist = py{27,37}-{lint,invalidcode},py37-black +envlist = py37-{lint,invalidcode},py37-black [testenv] skip_install=True deps = - py{27,37}-{lint,invalidcode}: flake8 + py37-{lint,invalidcode}: flake8 py37-black: black commands = - py{27,37}-lint: flake8 src doc data tests --ignore E402,E501 --exclude src/yunohost/vendor - py{27,37}-invalidcode: flake8 src data --exclude src/yunohost/tests,src/yunohost/vendor --select F + py37-lint: flake8 src doc data tests --ignore E402,E501 --exclude src/yunohost/vendor + py37-invalidcode: flake8 src data --exclude src/yunohost/tests,src/yunohost/vendor --select F py37-black: black --check --diff src doc data tests