YunoHost/yunohost
1
0
Fork 0
mirror of https://github.com/YunoHost/yunohost.git synced 2024-09-03 20:06:10 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #1116 from YunoHost/enh-python3

Browse source 
Python2 -> Python3
This commit is contained in:
Alexandre Aubin 2021-01-19 23:06:57 +01:00 • committed by GitHub
commit b98dd8585b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
41 changed files with 129 additions and 165 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
.gitlab/ci/lint.gitlab-ci.yml
View file

@ -3,14 +3,6 @@
########################################
# later we must fix lint and format-check jobs and remove "allow_failure"
lint27:
stage: lint
image: "before-install"
needs: []
allow_failure: true
script:
- tox -e py27-lint
lint37:
stage: lint
image: "before-install"
@ -19,17 +11,9 @@ lint37:
script:
- tox -e py37-lint
invalidcode27:
stage: lint
image: "before-install"
needs: []
script:
- tox -e py27-invalidcode
invalidcode37:
stage: lint
image: "before-install"
allow_failure: true
needs: []
script:
- tox -e py37-invalidcode

26
.gitlab/ci/test.gitlab-ci.yml
View file

@ -36,7 +36,7 @@ full-tests:
- *install_debs
- yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns
script:
- python -m pytest --cov=yunohost tests/ src/yunohost/tests/ --junitxml=report.xml
- python3 -m pytest --cov=yunohost tests/ src/yunohost/tests/ --junitxml=report.xml
needs:
- job: build-yunohost
artifacts: true
@ -51,70 +51,70 @@ full-tests:
root-tests:
extends: .test-stage
script:
- python -m pytest tests
- python3 -m pytest tests
test-apps:
extends: .test-stage
script:
- cd src/yunohost
- python -m pytest tests/test_apps.py
- python3 -m pytest tests/test_apps.py
test-appscatalog:
extends: .test-stage
script:
- cd src/yunohost
- python -m pytest tests/test_appscatalog.py
- python3 -m pytest tests/test_appscatalog.py
test-appurl:
extends: .test-stage
script:
- cd src/yunohost
- python -m pytest tests/test_appurl.py
- python3 -m pytest tests/test_appurl.py
test-apps-arguments-parsing:
extends: .test-stage
script:
- cd src/yunohost
- python -m pytest tests/test_apps_arguments_parsing.py
- python3 -m pytest tests/test_apps_arguments_parsing.py
test-backuprestore:
extends: .test-stage
script:
- cd src/yunohost
- python -m pytest tests/test_backuprestore.py
- python3 -m pytest tests/test_backuprestore.py
test-changeurl:
extends: .test-stage
script:
- cd src/yunohost
- python -m pytest tests/test_changeurl.py
- python3 -m pytest tests/test_changeurl.py
test-permission:
extends: .test-stage
script:
- cd src/yunohost
- python -m pytest tests/test_permission.py
- python3 -m pytest tests/test_permission.py
test-settings:
extends: .test-stage
script:
- cd src/yunohost
- python -m pytest tests/test_settings.py
- python3 -m pytest tests/test_settings.py
test-user-group:
extends: .test-stage
script:
- cd src/yunohost
- python -m pytest tests/test_user-group.py
- python3 -m pytest tests/test_user-group.py
test-regenconf:
extends: .test-stage
script:
- cd src/yunohost
- python -m pytest tests/test_regenconf.py
- python3 -m pytest tests/test_regenconf.py
test-service:
extends: .test-stage
script:
- cd src/yunohost
- python -m pytest tests/test_service.py
- python3 -m pytest tests/test_service.py

2
bin/yunohost
View file

@ -1,4 +1,4 @@
#! /usr/bin/python
#! /usr/bin/python3
# -*- coding: utf-8 -*-
import os

2
bin/yunohost-api
View file

@ -1,4 +1,4 @@
#! /usr/bin/python
#! /usr/bin/python3
# -*- coding: utf-8 -*-
import sys

2
data/helpers.d/logging
View file

@ -35,7 +35,7 @@ ynh_print_info() {
# Manage arguments with getopts
ynh_handle_getopts_args "$@"
echo "$message" >> "$YNH_STDINFO"
echo "$message" >&$YNH_STDINFO
}
# Ignore the yunohost-cli log to prevent errors with conditional commands

4
data/helpers.d/setting
View file

@ -77,9 +77,9 @@ ynh_app_setting_delete() {
# [internal]
#
ynh_app_setting()
{
{
set +o xtrace # set +x
ACTION="$1" APP="$2" KEY="$3" VALUE="${4:-}" python2.7 - <<EOF
ACTION="$1" APP="$2" KEY="$3" VALUE="${4:-}" python3 - <<EOF
import os, yaml, sys
app, action = os.environ['APP'], os.environ['ACTION'].lower()
key, value = os.environ['KEY'], os.environ.get('VALUE', None)

2
data/helpers.d/utils
View file

@ -422,7 +422,7 @@ ynh_render_template() {
local output_path=$2
mkdir -p "$(dirname $output_path)"
# Taken from https://stackoverflow.com/a/35009576
python2.7 -c 'import os, sys, jinja2; sys.stdout.write(
python3 -c 'import os, sys, jinja2; sys.stdout.write(
jinja2.Template(sys.stdin.read()
).render(os.environ));' < $template_path > $output_path
}

2
data/hooks/conf_regen/01-yunohost
View file

@ -115,7 +115,7 @@ do_post_regen() {
}
_update_services() {
python2 - << EOF
python3 - << EOF
import yaml

3
data/hooks/diagnosis/00-basesystem.py
View file

@ -159,7 +159,8 @@ class BaseSystemDiagnoser(Diagnoser):
# "missing some kernel info (see -v), accuracy might be reduced"
# Dunno what to do about that but we probably don't want to harass
# users with this warning ...
output, err = call.communicate()
output, _ = call.communicate()
output = output.decode()
assert call.returncode in (0, 2, 3), "Return code: %s" % call.returncode
# If there are multiple lines, sounds like there was some messages

5
data/hooks/diagnosis/50-systemresources.py
View file

@ -1,10 +1,11 @@
#!/usr/bin/env python
import os
import psutil
import subprocess
import datetime
import re
from moulinette.utils.process import check_output
from yunohost.diagnosis import Diagnoser
@ -119,7 +120,7 @@ class SystemResourcesDiagnoser(Diagnoser):
def analyzed_kern_log():
cmd = 'tail -n 10000 /var/log/kern.log | grep "oom_reaper: reaped process" || true'
out = subprocess.check_output(cmd, shell=True).strip()
out = check_output(cmd)
lines = out.split("\n") if out else []
now = datetime.datetime.now()

13
debian/control vendored
View file

@ -2,19 +2,18 @@ Source: yunohost
Section: utils
Priority: extra
Maintainer: YunoHost Contributors <contrib@yunohost.org>
Build-Depends: debhelper (>=9), dh-systemd, dh-python, python-all (>= 2.7), python-yaml, python-jinja2
Build-Depends: debhelper (>=9), dh-systemd, dh-python, python3-all (>= 3.7), python3-yaml, python3-jinja2
Standards-Version: 3.9.6
X-Python-Version: >= 2.7
Homepage: https://yunohost.org/
Package: yunohost
Essential: yes
Architecture: all
Depends: ${python:Depends}, ${misc:Depends}
Depends: ${python3:Depends}, ${misc:Depends}
, moulinette (>= 4.1.0.1), ssowat (>= 4.0)
, python-psutil, python-requests, python-dnspython, python-openssl
, python-miniupnpc, python-dbus, python-jinja2
, python-toml, python-packaging, python-publicsuffix
, python3-psutil, python3-requests, python3-dnspython, python3-openssl
, python3-miniupnpc, python3-dbus, python3-jinja2
, python3-toml, python3-packaging, python3-publicsuffix
, apt, apt-transport-https, apt-utils, dirmngr
, php7.3-common, php7.3-fpm, php7.3-ldap, php7.3-intl
, mariadb-server, php7.3-mysql
@ -33,7 +32,7 @@ Recommends: yunohost-admin
, ntp, inetutils-ping | iputils-ping
, bash-completion, rsyslog
, php7.3-gd, php7.3-curl, php-gettext
, python-pip
, python3-pip
, unattended-upgrades
, libdbd-ldap-perl, libnet-dns-perl
Suggests: htop, vim, rsync, acpi-support-base, udisks2

2
debian/rules vendored
View file

@ -5,7 +5,7 @@
#export DH_VERBOSE=1
%:
dh ${@} --with=python2,systemd
dh ${@} --with=python3,systemd
override_dh_auto_build:
# Generate bash completion file

2
doc/generate_helper_doc.py
View file

@ -1,4 +1,4 @@
#!/usr/env/python2.7
#!/usr/env/python3
import os
import glob

58
src/yunohost/app.py
View file

@ -30,16 +30,16 @@ import shutil
import yaml
import time
import re
import urlparse
import urllib.parse
import subprocess
import glob
import urllib
import urllib.request, urllib.parse, urllib.error
from collections import OrderedDict
from moulinette import msignals, m18n, msettings
from moulinette.utils.log import getActionLogger
from moulinette.utils.network import download_json
from moulinette.utils.process import run_commands
from moulinette.utils.process import run_commands, check_output
from moulinette.utils.filesystem import read_file, read_json, read_toml, read_yaml, write_to_file, write_to_json, write_to_yaml, chmod, chown, mkdir
from yunohost.service import service_status, _run_service_command
@ -426,10 +426,7 @@ def app_change_url(operation_logger, app, domain, path):
# grab nginx errors
# the "exit 0" is here to avoid check_output to fail because 'nginx -t'
# will return != 0 since we are in a failed state
nginx_errors = subprocess.check_output("nginx -t; exit 0",
stderr=subprocess.STDOUT,
shell=True).rstrip()
nginx_errors = check_output("nginx -t; exit 0")
raise YunohostError("app_change_url_failed_nginx_reload", nginx_errors=nginx_errors)
logger.success(m18n.n("app_change_url_success",
@ -751,7 +748,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu
# Retrieve arguments list for install script
args_dict = {} if not args else \
dict(urlparse.parse_qsl(args, keep_blank_values=True))
dict(urllib.parse.parse_qsl(args, keep_blank_values=True))
args_odict = _parse_args_from_manifest(manifest, 'install', args=args_dict)
# Validate domain / path availability for webapps
@ -774,7 +771,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu
# Also redact the % escaped version of the password that might appear in
# the 'args' section of metadata (relevant for password with non-alphanumeric char)
data_to_redact = [value[0] for value in args_odict.values() if value[1] == "password"]
data_to_redact += [urllib.quote(data) for data in data_to_redact if urllib.quote(data) != data]
data_to_redact += [urllib.parse.quote(data) for data in data_to_redact if urllib.parse.quote(data) != data]
operation_logger.data_to_redact.extend(data_to_redact)
operation_logger.related_to = [s for s in operation_logger.related_to if s[0] != "app"]
@ -849,7 +846,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu
logger.error(m18n.n("app_install_failed", app=app_id, error=error))
failure_message_with_debug_instructions = operation_logger.error(error)
# Something wrong happened in Yunohost's code (most probably hook_exec)
except Exception as e:
except Exception:
import traceback
error = m18n.n('unexpected_error', error="\n" + traceback.format_exc())
logger.error(m18n.n("app_install_failed", app=app_id, error=error))
@ -1421,7 +1418,7 @@ def app_ssowatconf():
write_to_json('/etc/ssowat/conf.json', conf_dict, sort_keys=True, indent=4)
from utils.legacy import translate_legacy_rules_in_ssowant_conf_json_persistent
from .utils.legacy import translate_legacy_rules_in_ssowant_conf_json_persistent
translate_legacy_rules_in_ssowant_conf_json_persistent()
logger.debug(m18n.n('ssowat_conf_generated'))
@ -1471,7 +1468,7 @@ def app_action_run(operation_logger, app, action, args=None):
action_declaration = actions[action]
# Retrieve arguments list for install script
args_dict = dict(urlparse.parse_qsl(args, keep_blank_values=True)) if args else {}
args_dict = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {}
args_odict = _parse_args_for_action(actions[action], args=args_dict)
args_list = [value[0] for value in args_odict.values()]
@ -1613,7 +1610,7 @@ def app_config_apply(operation_logger, app, args):
"YNH_APP_INSTANCE_NAME": app,
"YNH_APP_INSTANCE_NUMBER": str(app_instance_nb),
}
args = dict(urlparse.parse_qsl(args, keep_blank_values=True)) if args else {}
args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {}
for tab in config_panel.get("panel", []):
tab_id = tab["id"] # this makes things easier to debug on crash
@ -1832,8 +1829,7 @@ def _get_app_config_panel(app_id):
"panel": [],
}
panels = filter(lambda key_value: key_value[0] not in ("name", "version") and isinstance(key_value[1], OrderedDict),
toml_config_panel.items())
panels = [key_value for key_value in toml_config_panel.items() if key_value[0] not in ("name", "version") and isinstance(key_value[1], OrderedDict)]
for key, value in panels:
panel = {
@ -1842,8 +1838,7 @@ def _get_app_config_panel(app_id):
"sections": [],
}
sections = filter(lambda k_v1: k_v1[0] not in ("name",) and isinstance(k_v1[1], OrderedDict),
value.items())
sections = [k_v1 for k_v1 in value.items() if k_v1[0] not in ("name",) and isinstance(k_v1[1], OrderedDict)]
for section_key, section_value in sections:
section = {
@ -1852,8 +1847,7 @@ def _get_app_config_panel(app_id):
"options": [],
}
options = filter(lambda k_v: k_v[0] not in ("name",) and isinstance(k_v[1], OrderedDict),
section_value.items())
options = [k_v for k_v in section_value.items() if k_v[0] not in ("name",) and isinstance(k_v[1], OrderedDict)]
for option_key, option_value in options:
option = dict(option_value)
@ -1891,7 +1885,7 @@ def _get_app_settings(app_id):
settings = yaml.load(f)
# If label contains unicode char, this may later trigger issues when building strings...
# FIXME: this should be propagated to read_yaml so that this fix applies everywhere I think...
settings = {k: _encode_string(v) for k, v in settings.items()}
settings = {k: v for k, v in settings.items()}
if app_id == settings['id']:
return settings
except (IOError, TypeError, KeyError):
@ -2157,10 +2151,9 @@ def _get_git_last_commit_hash(repository, reference='HEAD'):
"""
try:
commit = subprocess.check_output(
"git ls-remote --exit-code {0} {1} | awk '{{print $1}}'".format(
repository, reference),
shell=True)
cmd = "git ls-remote --exit-code {0} {1} | awk '{{print $1}}'"\
.format(repository, reference)
commit = check_output(cmd)
except subprocess.CalledProcessError:
logger.exception("unable to get last commit from %s", repository)
raise ValueError("Unable to get last commit with git")
@ -2318,21 +2311,12 @@ def _value_for_locale(values):
for lang in [m18n.locale, m18n.default_locale]:
try:
return _encode_string(values[lang])
return values[lang]
except KeyError:
continue
# Fallback to first value
return _encode_string(values.values()[0])
def _encode_string(value):
"""
Return the string encoded in utf-8 if needed
"""
if isinstance(value, unicode):
return value.encode('utf8')
return value
return list(values.values())[0]
def _check_manifest_requirements(manifest, app_instance_name):
@ -2945,7 +2929,7 @@ def _load_apps_catalog():
try:
apps_catalog_content = read_json(cache_file) if os.path.exists(cache_file) else None
except Exception as e:
raise ("Unable to read cache for apps_catalog %s : %s" % (apps_catalog_id, str(e)))
raise YunohostError("Unable to read cache for apps_catalog %s : %s" % (cache_file, e), raw_msg=True)
# Check that the version of the data matches version ....
# ... otherwise it means we updated yunohost in the meantime
@ -2995,7 +2979,7 @@ def is_true(arg):
"""
if isinstance(arg, bool):
return arg
elif isinstance(arg, basestring):
elif isinstance(arg, str):
return arg.lower() in ['yes', 'true', 'on']
else:
logger.debug('arg should be a boolean or a string, got %r', arg)

19
src/yunohost/backup.py
View file

@ -41,6 +41,7 @@ from moulinette import msignals, m18n, msettings
from moulinette.utils import filesystem
from moulinette.utils.log import getActionLogger
from moulinette.utils.filesystem import read_file, mkdir, write_to_yaml, read_yaml
from moulinette.utils.process import check_output
from yunohost.app import (
app_info, _is_installed,
@ -176,11 +177,11 @@ class BackupRestoreTargetsManager(object):
or (exclude and isinstance(exclude, list) and not include)
if include:
return [target.encode("Utf-8") for target in self.targets[category]
return [target for target in self.targets[category]
if self.results[category][target] in include]
if exclude:
return [target.encode("Utf-8") for target in self.targets[category]
return [target for target in self.targets[category]
if self.results[category][target] not in exclude]
@ -599,7 +600,7 @@ class BackupManager():
for hook, infos in ret.items()
if any(result["state"] == "failed" for result in infos.values())}
if ret_succeed.keys() != []:
if list(ret_succeed.keys()) != []:
self.system_return = ret_succeed
# Add files from targets (which they put in the CSV) to the list of
@ -882,7 +883,7 @@ class RestoreManager():
End a restore operations by cleaning the working directory and
regenerate ssowat conf (if some apps were restored)
"""
from permission import permission_sync_to_user
from .permission import permission_sync_to_user
permission_sync_to_user()
@ -1643,7 +1644,7 @@ class BackupMethod(object):
try:
subprocess.check_call(["mount", "--rbind", src, dest])
subprocess.check_call(["mount", "-o", "remount,ro,bind", dest])
except Exception as e:
except Exception:
logger.warning(m18n.n("backup_couldnt_bind", src=src, dest=dest))
# To check if dest is mounted, use /proc/mounts that
# escape spaces as \040
@ -2165,7 +2166,7 @@ def backup_list(with_info=False, human_readable=False):
d[archive] = backup_info(archive, human_readable=human_readable)
except YunohostError as e:
logger.warning(str(e))
except Exception as e:
except Exception:
import traceback
logger.warning("Could not check infos for archive %s: %s" % (archive, '\n' + traceback.format_exc()))
@ -2386,7 +2387,7 @@ def _recursive_umount(directory):
Args:
directory -- a directory path
"""
mount_lines = subprocess.check_output("mount").split("\n")
mount_lines = check_output("mount").split("\n")
points_to_umount = [line.split(" ")[2]
for line in mount_lines
@ -2412,8 +2413,8 @@ def disk_usage(path):
# We don't do this in python with os.stat because we don't want
# to follow symlinks
du_output = subprocess.check_output(['du', '-sb', path])
return int(du_output.split()[0].decode('utf-8'))
du_output = check_output(['du', '-sb', path], shell=False)
return int(du_output.split()[0])
def binary_to_human(n, customary=False):

4
src/yunohost/certificate.py
View file

@ -385,7 +385,7 @@ def certificate_renew(domain_list, force=False, no_checks=False, email=False, st
_fetch_and_enable_new_certificate(domain, staging, no_checks=no_checks)
except Exception as e:
import traceback
from StringIO import StringIO
from io import StringIO
stack = StringIO()
traceback.print_exc(file=stack)
msg = "Certificate renewing for %s failed !" % (domain)
@ -638,7 +638,7 @@ def _get_status(domain):
cert_subject = cert.get_subject().CN
cert_issuer = cert.get_issuer().CN
organization_name = cert.get_issuer().O
valid_up_to = datetime.strptime(cert.get_notAfter(), "%Y%m%d%H%M%SZ")
valid_up_to = datetime.strptime(cert.get_notAfter().decode('utf-8'), "%Y%m%d%H%M%SZ")
days_remaining = (valid_up_to - datetime.utcnow()).days
if cert_issuer == _name_self_CA():

2
src/yunohost/data_migrations/0019_extend_permissions_features.py
View file

@ -98,7 +98,7 @@ class MyMigration(Migration):
# Migrate old settings
migrate_legacy_permission_settings()
except Exception as e:
except Exception:
logger.warn(m18n.n("migration_0019_migration_failed_trying_to_rollback"))
os.system("systemctl stop slapd")
os.system("rm -r /etc/ldap/slapd.d") # To be sure that we don't keep some part of the old config

2
src/yunohost/diagnosis.py
View file

@ -451,7 +451,7 @@ class Diagnoser():
key = "diagnosis_description_" + id_
descr = m18n.n(key)
# If no description available, fallback to id
return descr if descr.decode('utf-8') != key else id_
return descr if descr != key else id_
@staticmethod
def i18n(report, force_remove_html_tags=False):

15
src/yunohost/domain.py
View file

@ -62,18 +62,15 @@ def domain_list(exclude_subdomains=False):
result_list.append(domain)
def cmp_domain(domain1, domain2):
def cmp_domain(domain):
# Keep the main part of the domain and the extension together
# eg: this.is.an.example.com -> ['example.com', 'an', 'is', 'this']
domain1 = domain1.split('.')
domain2 = domain2.split('.')
domain1[-1] = domain1[-2] + domain1.pop()
domain2[-1] = domain2[-2] + domain2.pop()
domain1 = list(reversed(domain1))
domain2 = list(reversed(domain2))
return cmp(domain1, domain2)
domain = domain.split('.')
domain[-1] = domain[-2] + domain.pop()
domain = list(reversed(domain))
return domain
result_list = sorted(result_list, cmp_domain)
result_list = sorted(result_list, key=cmp_domain)
return {
'domains': result_list,

2
src/yunohost/dyndns.py
View file

@ -105,7 +105,7 @@ def _dyndns_available(provider, domain):
raise YunohostError('dyndns_could_not_check_available',
domain=domain, provider=provider)
return r == u"Domain %s is available" % domain
return r == "Domain %s is available" % domain
@is_unit_operation()

11
src/yunohost/hook.py
View file

@ -182,7 +182,8 @@ def hook_list(action, list_by='name', show_info=False):
def _append_folder(d, folder):
# Iterate over and add hook from a folder
for f in os.listdir(folder + action):
if f[0] == '.' or f[-1] == '~' or f.endswith(".pyc"):
if f[0] == '.' or f[-1] == '~' or f.endswith(".pyc") \
or (f.startswith("__") and f.endswith("__")):
continue
path = '%s%s/%s' % (folder, action, f)
priority, name = _extract_filename_parts(f)
@ -387,9 +388,6 @@ def _hook_exec_bash(path, args, no_trace, chdir, env, user, return_format, logge
env['YNH_INTERFACE'] = msettings.get('interface')
stdinfo = os.path.join(tempfile.mkdtemp(), "stdinfo")
env['YNH_STDINFO'] = stdinfo
stdreturn = os.path.join(tempfile.mkdtemp(), "stdreturn")
with open(stdreturn, 'w') as f:
f.write('')
@ -415,10 +413,7 @@ def _hook_exec_bash(path, args, no_trace, chdir, env, user, return_format, logge
logger.debug("Executing command '%s'" % ' '.join(command))
returncode = call_async_output(
command, loggers, shell=False, cwd=chdir,
stdinfo=stdinfo
)
returncode = call_async_output(command, loggers, shell=False, cwd=chdir)
raw_content = None
try:

7
src/yunohost/log.py
View file

@ -65,8 +65,7 @@ def log_list(limit=None, with_details=False, with_suboperations=False):
operations = {}
logs = filter(lambda x: x.endswith(METADATA_FILE_EXT),
os.listdir(OPERATIONS_PATH))
logs = [x for x in os.listdir(OPERATIONS_PATH) if x.endswith(METADATA_FILE_EXT)]
logs = list(reversed(sorted(logs)))
if limit is not None:
@ -337,7 +336,7 @@ def is_unit_operation(entities=['app', 'domain', 'group', 'service', 'user'],
entity_type = entity
if entity in kwargs and kwargs[entity] is not None:
if isinstance(kwargs[entity], basestring):
if isinstance(kwargs[entity], str):
related_to.append((entity_type, kwargs[entity]))
else:
for x in kwargs[entity]:
@ -596,7 +595,7 @@ class OperationLogger(object):
"""
if self.ended_at is not None or self.started_at is None:
return
if error is not None and not isinstance(error, basestring):
if error is not None and not isinstance(error, str):
error = str(error)
self.ended_at = datetime.utcnow()
self._error = error

4
src/yunohost/permission.py
View file

@ -106,7 +106,7 @@ def user_permission_list(short=False, full=False, ignore_system_perms=False, abs
infos["label"] = "%s (%s)" % (main_perm_label, infos["label"])
if short:
permissions = permissions.keys()
permissions = list(permissions.keys())
return {'permissions': permissions}
@ -668,7 +668,7 @@ def _validate_and_sanitize_permission_url(url, app_base_path, app):
For example:
re:/api/[A-Z]*$ -> domain.tld/app/api/[A-Z]*$
re:domain.tld/app/api/[A-Z]*$ -> domain.tld/app/api/[A-Z]*$
We can also have less-trivial regexes like:
re:^\/api\/.*|\/scripts\/api.js$
"""

10
src/yunohost/regenconf.py
View file

@ -21,7 +21,6 @@
import os
import yaml
import subprocess
import shutil
import hashlib
@ -30,6 +29,7 @@ from datetime import datetime
from moulinette import m18n
from moulinette.utils import log, filesystem
from moulinette.utils.process import check_output
from yunohost.utils.error import YunohostError
from yunohost.log import is_unit_operation
@ -654,10 +654,10 @@ def manually_modified_files():
def manually_modified_files_compared_to_debian_default(ignore_handled_by_regenconf=False):
# from https://serverfault.com/a/90401
files = subprocess.check_output("dpkg-query -W -f='${Conffiles}\n' '*' \
| awk 'OFS=\" \"{print $2,$1}' \
| md5sum -c 2>/dev/null \
| awk -F': ' '$2 !~ /OK/{print $1}'", shell=True)
files = check_output("dpkg-query -W -f='${Conffiles}\n' '*' \
| awk 'OFS=\" \"{print $2,$1}' \
| md5sum -c 2>/dev/null \
| awk -F': ' '$2 !~ /OK/{print $1}'")
files = files.strip().split("\n")
if ignore_handled_by_regenconf:

12
src/yunohost/service.py
View file

@ -35,6 +35,7 @@ from datetime import datetime
from moulinette import m18n
from yunohost.utils.error import YunohostError
from moulinette.utils.process import check_output
from moulinette.utils.log import getActionLogger
from moulinette.utils.filesystem import read_file, append_to_file, write_to_file
@ -358,7 +359,7 @@ def _get_and_format_service_status(service, infos):
# that mean that we don't have a translation for this string
# that's the only way to test for that for now
# if we don't have it, uses the one provided by systemd
if description.decode('utf-8') == translation_key:
if description == translation_key:
description = str(raw_status.get("Description", ""))
output = {
@ -489,7 +490,7 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
raise YunohostError('service_unknown', service=name)
if names is []:
names = services.keys()
names = list(services.keys())
logger.warning(m18n.n("service_regen_conf_is_deprecated"))
@ -563,8 +564,7 @@ def _give_lock(action, service, p):
while son_PID == 0 and p.poll() is None:
# Call systemctl to get the PID
# Output of the command is e.g. ControlPID=1234
son_PID = subprocess.check_output(cmd_get_son_PID.split()) \
.strip().split("=")[1]
son_PID = check_output(cmd_get_son_PID).split("=")[1]
son_PID = int(son_PID)
time.sleep(1)
@ -599,7 +599,7 @@ def _get_services():
# some services are marked as None to remove them from YunoHost
# filter this
for key, value in services.items():
for key, value in list(services.items()):
if value is None:
del services[key]
@ -720,7 +720,7 @@ def _get_journalctl_logs(service, number="all"):
services = _get_services()
systemd_service = services.get(service, {}).get("actual_systemd_service", service)
try:
return subprocess.check_output("journalctl --no-hostname --no-pager -u {0} -n{1}".format(systemd_service, number), shell=True)
return check_output("journalctl --no-hostname --no-pager -u {0} -n{1}".format(systemd_service, number))
except:
import traceback
return "error while get services logs from journalctl:\n%s" % traceback.format_exc()

4
src/yunohost/settings.py
View file

@ -29,7 +29,7 @@ def is_boolean(value):
"""
if isinstance(value, bool):
return True, value
elif isinstance(value, basestring):
elif isinstance(value, str):
if str(value).lower() in ['true', 'on', 'yes', 'false', 'off', 'no']:
return True, str(value).lower() in ['true', 'on', 'yes']
else:
@ -141,7 +141,7 @@ def settings_set(key, value):
raise YunohostError('global_settings_bad_type_for_setting', setting=key,
received_type=type(value).__name__, expected_type=key_type)
elif key_type == "string":
if not isinstance(value, basestring):
if not isinstance(value, str):
raise YunohostError('global_settings_bad_type_for_setting', setting=key,
received_type=type(value).__name__, expected_type=key_type)
elif key_type == "enum":

2
src/yunohost/tests/test_apps.py
View file

@ -4,7 +4,7 @@ import pytest
import shutil
import requests
from conftest import message, raiseYunohostError, get_test_apps_dir
from .conftest import message, raiseYunohostError, get_test_apps_dir
from moulinette.utils.filesystem import mkdir

2
src/yunohost/tests/test_apps_arguments_parsing.py
View file

@ -2,7 +2,7 @@ import sys
import pytest
from mock import patch
from StringIO import StringIO
from io import StringIO
from collections import OrderedDict
from moulinette import msignals

2
src/yunohost/tests/test_appurl.py
View file

@ -1,7 +1,7 @@
import pytest
import os
from conftest import get_test_apps_dir
from .conftest import get_test_apps_dir
from yunohost.utils.error import YunohostError
from yunohost.app import app_install, app_remove, _normalize_domain_path

7
src/yunohost/tests/test_backuprestore.py
View file

@ -3,7 +3,7 @@ import os
import shutil
import subprocess
from conftest import message, raiseYunohostError, get_test_apps_dir
from .conftest import message, raiseYunohostError, get_test_apps_dir
from yunohost.app import app_install, app_remove, app_ssowatconf
from yunohost.app import _is_installed
@ -23,8 +23,6 @@ def setup_function(function):
global maindomain
maindomain = _get_maindomain()
print ""
assert backup_test_dependencies_are_met()
clean_tmp_backup_directory()
@ -150,7 +148,7 @@ def clean_tmp_backup_directory():
if tmp_backup_directory_is_empty():
return
mount_lines = subprocess.check_output("mount").split("\n")
mount_lines = subprocess.check_output("mount").decode().split("\n")
points_to_umount = [line.split(" ")[2]
for line in mount_lines
@ -638,6 +636,7 @@ def test_backup_binds_are_readonly(mocker, monkeypatch):
confssh = os.path.join(self.work_dir, "conf/ssh")
output = subprocess.check_output("touch %s/test 2>&1 || true" % confssh,
shell=True, env={'LANG': 'en_US.UTF-8'})
output = output.decode()
assert "Read-only file system" in output

2
src/yunohost/tests/test_changeurl.py
View file

@ -3,7 +3,7 @@ import time
import requests
import os
from conftest import get_test_apps_dir
from .conftest import get_test_apps_dir
from yunohost.app import app_install, app_change_url, app_remove, app_map
from yunohost.domain import _get_maindomain

2
src/yunohost/tests/test_permission.py
View file

@ -6,7 +6,7 @@ import os
import json
import shutil
from conftest import message, raiseYunohostError, get_test_apps_dir
from .conftest import message, raiseYunohostError, get_test_apps_dir
from yunohost.app import app_install, app_upgrade, app_remove, app_change_url, app_map, _installed_apps, APPS_SETTING_PATH, _set_app_settings, _get_app_settings
from yunohost.user import user_list, user_create, user_delete, \

2
src/yunohost/tests/test_regenconf.py
View file

@ -1,6 +1,6 @@
import os
from conftest import message
from .conftest import message
from yunohost.domain import domain_add, domain_remove, domain_list
from yunohost.regenconf import regen_conf, manually_modified_files, _get_conf_hashes, _force_clear_hashes

2
src/yunohost/tests/test_service.py
View file

@ -1,6 +1,6 @@
import os
from conftest import raiseYunohostError
from .conftest import raiseYunohostError
from yunohost.service import _get_services, _save_services, service_status, service_add, service_remove, service_log

2
src/yunohost/tests/test_user-group.py
View file

@ -1,6 +1,6 @@
import pytest
from conftest import message, raiseYunohostError
from .conftest import message, raiseYunohostError
from yunohost.user import user_list, user_info, user_create, user_delete, user_update, \
user_group_list, user_group_create, user_group_delete, user_group_update

6
src/yunohost/tools.py
View file

@ -303,7 +303,7 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False,
'/home/yunohost.app'
]
for folder in filter(lambda x: not os.path.exists(x), folders_to_create):
for folder in [x for x in folders_to_create if not os.path.exists(x)]:
os.makedirs(folder)
# Change folders permissions
@ -953,7 +953,7 @@ def _get_migrations_list():
# (in particular, pending migrations / not already ran are not listed
states = tools_migrations_state()["migrations"]
for migration_file in filter(lambda x: re.match(r"^\d+_[a-zA-Z0-9_]+\.py$", x), os.listdir(migrations_path)):
for migration_file in [x for x in os.listdir(migrations_path) if re.match(r"^\d+_[a-zA-Z0-9_]+\.py$", x)]:
m = _load_migration(migration_file)
m.state = states.get(m.id, "pending")
migrations.append(m)
@ -972,7 +972,7 @@ def _get_migration_by_name(migration_name):
raise AssertionError("Unable to find migration with name %s" % migration_name)
migrations_path = data_migrations.__path__[0]
migrations_found = filter(lambda x: re.match(r"^\d+_%s\.py$" % migration_name, x), os.listdir(migrations_path))
migrations_found = [x for x in os.listdir(migrations_path) if re.match(r"^\d+_%s\.py$" % migration_name, x)]
assert len(migrations_found) == 1, "Unable to find migration with name %s" % migration_name

12
src/yunohost/user.py
View file

@ -35,6 +35,7 @@ import copy
from moulinette import msignals, msettings, m18n
from moulinette.utils.log import getActionLogger
from moulinette.utils.process import check_output
from yunohost.utils.error import YunohostError
from yunohost.service import service_status
@ -473,8 +474,7 @@ def user_info(username):
else:
try:
cmd = 'doveadm -f flow quota get -u %s' % user['uid'][0]
cmd_result = subprocess.check_output(cmd, stderr=subprocess.STDOUT,
shell=True)
cmd_result = check_output(cmd)
except Exception as e:
cmd_result = ""
logger.warning("Failed to fetch quota info ... : %s " % str(e))
@ -546,7 +546,7 @@ def user_group_list(short=False, full=False, include_primary_groups=True):
groups[name]["permissions"] = [_ldap_path_extract(p, "cn") for p in infos.get("permission", [])]
if short:
groups = groups.keys()
groups = list(groups.keys())
return {'groups': groups}
@ -631,7 +631,7 @@ def user_group_delete(operation_logger, groupname, force=False, sync_perm=True):
from yunohost.permission import permission_sync_to_user
from yunohost.utils.ldap import _get_ldap_interface
existing_groups = user_group_list()['groups'].keys()
existing_groups = list(user_group_list()['groups'].keys())
if groupname not in existing_groups:
raise YunohostError('group_unknown', group=groupname)
@ -639,7 +639,7 @@ def user_group_delete(operation_logger, groupname, force=False, sync_perm=True):
# without the force option...
#
# We also can't delete "all_users" because that's a special group...
existing_users = user_list()['users'].keys()
existing_users = list(user_list()['users'].keys())
undeletable_groups = existing_users + ["all_users", "visitors"]
if groupname in undeletable_groups and not force:
raise YunohostError('group_cannot_be_deleted', group=groupname)
@ -675,7 +675,7 @@ def user_group_update(operation_logger, groupname, add=None, remove=None, force=
from yunohost.permission import permission_sync_to_user
from yunohost.utils.ldap import _get_ldap_interface
existing_users = user_list()['users'].keys()
existing_users = list(user_list()['users'].keys())
# Refuse to edit a primary group of a user (e.g. group 'sam' related to user 'sam')
# Those kind of group should only ever contain the user (e.g. sam) and only this one.

6
src/yunohost/utils/legacy.py
View file

@ -22,7 +22,7 @@ class SetupGroupPermissions():
try:
objects = ldap.search(target + ",dc=yunohost,dc=org")
# ldap search will raise an exception if no corresponding object is found >.> ...
except Exception as e:
except Exception:
logger.debug("%s does not exist, no need to delete it" % target)
return
@ -100,7 +100,7 @@ class SetupGroupPermissions():
url = "/" if domain and path else None
if permission:
known_users = user_list()["users"].keys()
known_users = list(user_list()["users"].keys())
allowed = [user for user in permission.split(',') if user in known_users]
else:
allowed = ["all_users"]
@ -237,7 +237,7 @@ def translate_legacy_rules_in_ssowant_conf_json_persistent():
protected_urls = persistent.get("protected_urls", []) + ["re:" + r for r in persistent.get("protected_regex", [])]
unprotected_urls = persistent.get("unprotected_urls", []) + ["re:" + r for r in persistent.get("unprotected_regex", [])]
known_users = user_list()["users"].keys()
known_users = list(user_list()["users"].keys())
for legacy_rule in legacy_rules:
if legacy_rule in persistent:

6
src/yunohost/utils/packages.py
View file

@ -70,7 +70,11 @@ def meets_version_specifier(pkg_name, specifier):
op, req_version = re.search(r'(<<|<=|=|>=|>>) *([\d\.]+)', specifier).groups()
req_version = version.parse(req_version)
# cmp is a python builtin that returns (-1, 0, 1) depending on comparison
# Python2 had a builtin that returns (-1, 0, 1) depending on comparison
# c.f. https://stackoverflow.com/a/22490617
def cmp(a, b):
return (a > b) - (a < b)
deb_operators = {
"<<": lambda v1, v2: cmp(v1, v2) in [-1],
"<=": lambda v1, v2: cmp(v1, v2) in [-1, 0],

2
src/yunohost/utils/password.py
View file

@ -173,7 +173,7 @@ class PasswordValidator(object):
# stdin to avoid it being shown in ps -ef --forest...
command = "grep -q -F -f - %s" % MOST_USED_PASSWORDS
p = subprocess.Popen(command.split(), stdin=subprocess.PIPE)
p.communicate(input=password)
p.communicate(input=password.encode('utf-8'))
return not bool(p.returncode)

8
tox.ini
View file

@ -1,12 +1,12 @@
[tox]
envlist = py{27,37}-{lint,invalidcode},py37-black
envlist = py37-{lint,invalidcode},py37-black
[testenv]
skip_install=True
deps =
py{27,37}-{lint,invalidcode}: flake8
py37-{lint,invalidcode}: flake8
py37-black: black
commands =
py{27,37}-lint: flake8 src doc data tests --ignore E402,E501 --exclude src/yunohost/vendor
py{27,37}-invalidcode: flake8 src data --exclude src/yunohost/tests,src/yunohost/vendor --select F
py37-lint: flake8 src doc data tests --ignore E402,E501 --exclude src/yunohost/vendor
py37-invalidcode: flake8 src data --exclude src/yunohost/tests,src/yunohost/vendor --select F
py37-black: black --check --diff src doc data tests