Merge pull request #1116 from YunoHost/enh-python3

Python2 -> Python3

.gitlab/ci/lint.gitlab-ci.yml

@@ -3,14 +3,6 @@
 ########################################
 # later we must fix lint and format-check jobs and remove "allow_failure"
 
-lint27:
-  stage: lint
-  image: "before-install"
-  needs: []
-  allow_failure: true
-  script:
-    - tox -e py27-lint
-
 lint37:
   stage: lint
   image: "before-install"
@@ -19,17 +11,9 @@ lint37:
   script:
     - tox -e py37-lint
 
-invalidcode27:
-  stage: lint
-  image: "before-install"
-  needs: []
-  script:
-    - tox -e py27-invalidcode
-
 invalidcode37:
   stage: lint
   image: "before-install"
-  allow_failure: true
   needs: []
   script:
     - tox -e py37-invalidcode

.gitlab/ci/test.gitlab-ci.yml

@@ -36,7 +36,7 @@ full-tests:
     - *install_debs
     - yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns
   script:
-    - python -m pytest --cov=yunohost tests/ src/yunohost/tests/ --junitxml=report.xml
+    - python3 -m pytest --cov=yunohost tests/ src/yunohost/tests/ --junitxml=report.xml
   needs:
       - job: build-yunohost
         artifacts: true
@@ -51,70 +51,70 @@ full-tests:
 root-tests:
   extends: .test-stage
   script:
-    - python -m pytest tests
+    - python3 -m pytest tests
 
 test-apps:
   extends: .test-stage
   script:
     - cd src/yunohost
-    - python -m pytest tests/test_apps.py
+    - python3 -m pytest tests/test_apps.py
 
 test-appscatalog:
   extends: .test-stage
   script:
     - cd src/yunohost
-    - python -m pytest tests/test_appscatalog.py
+    - python3 -m pytest tests/test_appscatalog.py
 
 test-appurl:
   extends: .test-stage
   script:
     - cd src/yunohost
-    - python -m pytest tests/test_appurl.py
+    - python3 -m pytest tests/test_appurl.py
 
 test-apps-arguments-parsing:
   extends: .test-stage
   script:
     - cd src/yunohost
-    - python -m pytest tests/test_apps_arguments_parsing.py
+    - python3 -m pytest tests/test_apps_arguments_parsing.py
 
 test-backuprestore:
   extends: .test-stage
   script:
     - cd src/yunohost
-    - python -m pytest tests/test_backuprestore.py
+    - python3 -m pytest tests/test_backuprestore.py
 
 test-changeurl:
   extends: .test-stage
   script:
     - cd src/yunohost
-    - python -m pytest tests/test_changeurl.py
+    - python3 -m pytest tests/test_changeurl.py
 
 test-permission:
   extends: .test-stage
   script:
     - cd src/yunohost
-    - python -m pytest tests/test_permission.py
+    - python3 -m pytest tests/test_permission.py
 
 test-settings:
   extends: .test-stage
   script:
     - cd src/yunohost
-    - python -m pytest tests/test_settings.py
+    - python3 -m pytest tests/test_settings.py
 
 test-user-group:
   extends: .test-stage
   script:
     - cd src/yunohost
-    - python -m pytest tests/test_user-group.py
+    - python3 -m pytest tests/test_user-group.py
     
 test-regenconf:
   extends: .test-stage
   script:
     - cd src/yunohost
-    - python -m pytest tests/test_regenconf.py
+    - python3 -m pytest tests/test_regenconf.py
 
 test-service:
   extends: .test-stage
   script:
     - cd src/yunohost
-    - python -m pytest tests/test_service.py
+    - python3 -m pytest tests/test_service.py

bin/yunohost

@@ -1,4 +1,4 @@
-#! /usr/bin/python
+#! /usr/bin/python3
 # -*- coding: utf-8 -*-
 
 import os

bin/yunohost-api

@@ -1,4 +1,4 @@
-#! /usr/bin/python
+#! /usr/bin/python3
 # -*- coding: utf-8 -*-
 
 import sys

data/helpers.d/logging

@@ -35,7 +35,7 @@ ynh_print_info() {
     # Manage arguments with getopts
     ynh_handle_getopts_args "$@"
 
-    echo "$message" >> "$YNH_STDINFO"
+    echo "$message" >&$YNH_STDINFO
 }
 
 # Ignore the yunohost-cli log to prevent errors with conditional commands

data/helpers.d/setting

@@ -79,7 +79,7 @@ ynh_app_setting_delete() {
 ynh_app_setting()
 {   
     set +o xtrace # set +x
-    ACTION="$1" APP="$2" KEY="$3" VALUE="${4:-}" python2.7 - <<EOF
+    ACTION="$1" APP="$2" KEY="$3" VALUE="${4:-}" python3 - <<EOF
 import os, yaml, sys
 app, action = os.environ['APP'], os.environ['ACTION'].lower()
 key, value = os.environ['KEY'], os.environ.get('VALUE', None)

data/helpers.d/utils

@@ -422,7 +422,7 @@ ynh_render_template() {
     local output_path=$2
     mkdir -p "$(dirname $output_path)"
     # Taken from https://stackoverflow.com/a/35009576
-    python2.7 -c 'import os, sys, jinja2; sys.stdout.write(
+    python3 -c 'import os, sys, jinja2; sys.stdout.write(
                     jinja2.Template(sys.stdin.read()
                     ).render(os.environ));' < $template_path > $output_path
 }

data/hooks/conf_regen/01-yunohost

@@ -115,7 +115,7 @@ do_post_regen() {
 }
 
 _update_services() {
-  python2 - << EOF
+  python3 - << EOF
 import yaml
 
 

data/hooks/diagnosis/00-basesystem.py

@@ -159,7 +159,8 @@ class BaseSystemDiagnoser(Diagnoser):
             # "missing some kernel info (see -v), accuracy might be reduced"
             # Dunno what to do about that but we probably don't want to harass
             # users with this warning ...
-            output, err = call.communicate()
+            output, _ = call.communicate()
+            output = output.decode()
             assert call.returncode in (0, 2, 3), "Return code: %s" % call.returncode
 
             # If there are multiple lines, sounds like there was some messages

data/hooks/diagnosis/50-systemresources.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python
 import os
 import psutil
-import subprocess
 import datetime
 import re
 
+from moulinette.utils.process import check_output
+
 from yunohost.diagnosis import Diagnoser
 
 
@@ -119,7 +120,7 @@ class SystemResourcesDiagnoser(Diagnoser):
         def analyzed_kern_log():
 
             cmd = 'tail -n 10000 /var/log/kern.log | grep "oom_reaper: reaped process" || true'
-            out = subprocess.check_output(cmd, shell=True).strip()
+            out = check_output(cmd)
             lines = out.split("\n") if out else []
 
             now = datetime.datetime.now()

debian/control

@@ -2,19 +2,18 @@ Source: yunohost
 Section: utils
 Priority: extra
 Maintainer: YunoHost Contributors <contrib@yunohost.org>
-Build-Depends: debhelper (>=9), dh-systemd, dh-python, python-all (>= 2.7), python-yaml, python-jinja2
+Build-Depends: debhelper (>=9), dh-systemd, dh-python, python3-all (>= 3.7), python3-yaml, python3-jinja2
 Standards-Version: 3.9.6
-X-Python-Version: >= 2.7
 Homepage: https://yunohost.org/
 
 Package: yunohost
 Essential: yes
 Architecture: all
-Depends: ${python:Depends}, ${misc:Depends}
+Depends: ${python3:Depends}, ${misc:Depends}
  , moulinette (>= 4.1.0.1), ssowat (>= 4.0)
- , python-psutil, python-requests, python-dnspython, python-openssl
+ , python3-psutil, python3-requests, python3-dnspython, python3-openssl
- , python-miniupnpc, python-dbus, python-jinja2
+ , python3-miniupnpc, python3-dbus, python3-jinja2
- , python-toml, python-packaging, python-publicsuffix
+ , python3-toml, python3-packaging, python3-publicsuffix
  , apt, apt-transport-https, apt-utils, dirmngr
  , php7.3-common, php7.3-fpm, php7.3-ldap, php7.3-intl
  , mariadb-server, php7.3-mysql
@@ -33,7 +32,7 @@ Recommends: yunohost-admin
  , ntp, inetutils-ping | iputils-ping
  , bash-completion, rsyslog
  , php7.3-gd, php7.3-curl, php-gettext
- , python-pip
+ , python3-pip
  , unattended-upgrades
  , libdbd-ldap-perl, libnet-dns-perl
 Suggests: htop, vim, rsync, acpi-support-base, udisks2

debian/rules

@@ -5,7 +5,7 @@
 #export DH_VERBOSE=1
 
 %:
-	dh ${@} --with=python2,systemd
+	dh ${@} --with=python3,systemd
 
 override_dh_auto_build:
 	# Generate bash completion file

doc/generate_helper_doc.py

@@ -1,4 +1,4 @@
-#!/usr/env/python2.7
+#!/usr/env/python3
 
 import os
 import glob

src/yunohost/app.py

@@ -30,16 +30,16 @@ import shutil
 import yaml
 import time
 import re
-import urlparse
+import urllib.parse
 import subprocess
 import glob
-import urllib
+import urllib.request, urllib.parse, urllib.error
 from collections import OrderedDict
 
 from moulinette import msignals, m18n, msettings
 from moulinette.utils.log import getActionLogger
 from moulinette.utils.network import download_json
-from moulinette.utils.process import run_commands
+from moulinette.utils.process import run_commands, check_output
 from moulinette.utils.filesystem import read_file, read_json, read_toml, read_yaml, write_to_file, write_to_json, write_to_yaml, chmod, chown, mkdir
 
 from yunohost.service import service_status, _run_service_command
@@ -426,10 +426,7 @@ def app_change_url(operation_logger, app, domain, path):
         # grab nginx errors
         # the "exit 0" is here to avoid check_output to fail because 'nginx -t'
         # will return != 0 since we are in a failed state
-        nginx_errors = subprocess.check_output("nginx -t; exit 0",
+        nginx_errors = check_output("nginx -t; exit 0")
-                                               stderr=subprocess.STDOUT,
-                                               shell=True).rstrip()
-
         raise YunohostError("app_change_url_failed_nginx_reload", nginx_errors=nginx_errors)
 
     logger.success(m18n.n("app_change_url_success",
@@ -751,7 +748,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu
 
     # Retrieve arguments list for install script
     args_dict = {} if not args else \
-        dict(urlparse.parse_qsl(args, keep_blank_values=True))
+        dict(urllib.parse.parse_qsl(args, keep_blank_values=True))
     args_odict = _parse_args_from_manifest(manifest, 'install', args=args_dict)
 
     # Validate domain / path availability for webapps
@@ -774,7 +771,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu
     # Also redact the % escaped version of the password that might appear in
     # the 'args' section of metadata (relevant for password with non-alphanumeric char)
     data_to_redact = [value[0] for value in args_odict.values() if value[1] == "password"]
-    data_to_redact += [urllib.quote(data) for data in data_to_redact if urllib.quote(data) != data]
+    data_to_redact += [urllib.parse.quote(data) for data in data_to_redact if urllib.parse.quote(data) != data]
     operation_logger.data_to_redact.extend(data_to_redact)
 
     operation_logger.related_to = [s for s in operation_logger.related_to if s[0] != "app"]
@@ -849,7 +846,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu
         logger.error(m18n.n("app_install_failed", app=app_id, error=error))
         failure_message_with_debug_instructions = operation_logger.error(error)
     # Something wrong happened in Yunohost's code (most probably hook_exec)
-    except Exception as e:
+    except Exception:
         import traceback
         error = m18n.n('unexpected_error', error="\n" + traceback.format_exc())
         logger.error(m18n.n("app_install_failed", app=app_id, error=error))
@@ -1421,7 +1418,7 @@ def app_ssowatconf():
 
     write_to_json('/etc/ssowat/conf.json', conf_dict, sort_keys=True, indent=4)
 
-    from utils.legacy import translate_legacy_rules_in_ssowant_conf_json_persistent
+    from .utils.legacy import translate_legacy_rules_in_ssowant_conf_json_persistent
     translate_legacy_rules_in_ssowant_conf_json_persistent()
 
     logger.debug(m18n.n('ssowat_conf_generated'))
@@ -1471,7 +1468,7 @@ def app_action_run(operation_logger, app, action, args=None):
     action_declaration = actions[action]
 
     # Retrieve arguments list for install script
-    args_dict = dict(urlparse.parse_qsl(args, keep_blank_values=True)) if args else {}
+    args_dict = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {}
     args_odict = _parse_args_for_action(actions[action], args=args_dict)
     args_list = [value[0] for value in args_odict.values()]
 
@@ -1613,7 +1610,7 @@ def app_config_apply(operation_logger, app, args):
         "YNH_APP_INSTANCE_NAME": app,
         "YNH_APP_INSTANCE_NUMBER": str(app_instance_nb),
     }
-    args = dict(urlparse.parse_qsl(args, keep_blank_values=True)) if args else {}
+    args = dict(urllib.parse.parse_qsl(args, keep_blank_values=True)) if args else {}
 
     for tab in config_panel.get("panel", []):
         tab_id = tab["id"]  # this makes things easier to debug on crash
@@ -1832,8 +1829,7 @@ def _get_app_config_panel(app_id):
             "panel": [],
         }
 
-        panels = filter(lambda key_value: key_value[0] not in ("name", "version") and isinstance(key_value[1], OrderedDict),
+        panels = [key_value for key_value in toml_config_panel.items() if key_value[0] not in ("name", "version") and isinstance(key_value[1], OrderedDict)]
-                        toml_config_panel.items())
 
         for key, value in panels:
             panel = {
@@ -1842,8 +1838,7 @@ def _get_app_config_panel(app_id):
                 "sections": [],
             }
 
-            sections = filter(lambda k_v1: k_v1[0] not in ("name",) and isinstance(k_v1[1], OrderedDict),
+            sections = [k_v1 for k_v1 in value.items() if k_v1[0] not in ("name",) and isinstance(k_v1[1], OrderedDict)]
-                              value.items())
 
             for section_key, section_value in sections:
                 section = {
@@ -1852,8 +1847,7 @@ def _get_app_config_panel(app_id):
                     "options": [],
                 }
 
-                options = filter(lambda k_v: k_v[0] not in ("name",) and isinstance(k_v[1], OrderedDict),
+                options = [k_v for k_v in section_value.items() if k_v[0] not in ("name",) and isinstance(k_v[1], OrderedDict)]
-                                 section_value.items())
 
                 for option_key, option_value in options:
                     option = dict(option_value)
@@ -1891,7 +1885,7 @@ def _get_app_settings(app_id):
             settings = yaml.load(f)
         # If label contains unicode char, this may later trigger issues when building strings...
         # FIXME: this should be propagated to read_yaml so that this fix applies everywhere I think...
-        settings = {k: _encode_string(v) for k, v in settings.items()}
+        settings = {k: v for k, v in settings.items()}
         if app_id == settings['id']:
             return settings
     except (IOError, TypeError, KeyError):
@@ -2157,10 +2151,9 @@ def _get_git_last_commit_hash(repository, reference='HEAD'):
 
     """
     try:
-        commit = subprocess.check_output(
+        cmd = "git ls-remote --exit-code {0} {1} | awk '{{print $1}}'"\
-            "git ls-remote --exit-code {0} {1} | awk '{{print $1}}'".format(
+            .format(repository, reference)
-                repository, reference),
+        commit = check_output(cmd)
-            shell=True)
     except subprocess.CalledProcessError:
         logger.exception("unable to get last commit from %s", repository)
         raise ValueError("Unable to get last commit with git")
@@ -2318,21 +2311,12 @@ def _value_for_locale(values):
 
     for lang in [m18n.locale, m18n.default_locale]:
         try:
-            return _encode_string(values[lang])
+            return values[lang]
         except KeyError:
             continue
 
     # Fallback to first value
-    return _encode_string(values.values()[0])
+    return list(values.values())[0]
-
-
-def _encode_string(value):
-    """
-    Return the string encoded in utf-8 if needed
-    """
-    if isinstance(value, unicode):
-        return value.encode('utf8')
-    return value
 
 
 def _check_manifest_requirements(manifest, app_instance_name):
@@ -2945,7 +2929,7 @@ def _load_apps_catalog():
         try:
             apps_catalog_content = read_json(cache_file) if os.path.exists(cache_file) else None
         except Exception as e:
-            raise ("Unable to read cache for apps_catalog %s : %s" % (apps_catalog_id, str(e)))
+            raise YunohostError("Unable to read cache for apps_catalog %s : %s" % (cache_file, e), raw_msg=True)
 
         # Check that the version of the data matches version ....
         # ... otherwise it means we updated yunohost in the meantime
@@ -2995,7 +2979,7 @@ def is_true(arg):
     """
     if isinstance(arg, bool):
         return arg
-    elif isinstance(arg, basestring):
+    elif isinstance(arg, str):
         return arg.lower() in ['yes', 'true', 'on']
     else:
         logger.debug('arg should be a boolean or a string, got %r', arg)

src/yunohost/backup.py

@@ -41,6 +41,7 @@ from moulinette import msignals, m18n, msettings
 from moulinette.utils import filesystem
 from moulinette.utils.log import getActionLogger
 from moulinette.utils.filesystem import read_file, mkdir, write_to_yaml, read_yaml
+from moulinette.utils.process import check_output
 
 from yunohost.app import (
     app_info, _is_installed,
@@ -176,11 +177,11 @@ class BackupRestoreTargetsManager(object):
             or (exclude and isinstance(exclude, list) and not include)
 
         if include:
-            return [target.encode("Utf-8") for target in self.targets[category]
+            return [target for target in self.targets[category]
                     if self.results[category][target] in include]
 
         if exclude:
-            return [target.encode("Utf-8") for target in self.targets[category]
+            return [target for target in self.targets[category]
                     if self.results[category][target] not in exclude]
 
 
@@ -599,7 +600,7 @@ class BackupManager():
                       for hook, infos in ret.items()
                       if any(result["state"] == "failed" for result in infos.values())}
 
-        if ret_succeed.keys() != []:
+        if list(ret_succeed.keys()) != []:
             self.system_return = ret_succeed
 
         # Add files from targets (which they put in the CSV) to the list of
@@ -882,7 +883,7 @@ class RestoreManager():
         End a restore operations by cleaning the working directory and
         regenerate ssowat conf (if some apps were restored)
         """
-        from permission import permission_sync_to_user
+        from .permission import permission_sync_to_user
 
         permission_sync_to_user()
 
@@ -1643,7 +1644,7 @@ class BackupMethod(object):
                 try:
                     subprocess.check_call(["mount", "--rbind", src, dest])
                     subprocess.check_call(["mount", "-o", "remount,ro,bind", dest])
-                except Exception as e:
+                except Exception:
                     logger.warning(m18n.n("backup_couldnt_bind", src=src, dest=dest))
                     # To check if dest is mounted, use /proc/mounts that
                     # escape spaces as \040
@@ -2165,7 +2166,7 @@ def backup_list(with_info=False, human_readable=False):
                 d[archive] = backup_info(archive, human_readable=human_readable)
             except YunohostError as e:
                 logger.warning(str(e))
-            except Exception as e:
+            except Exception:
                 import traceback
                 logger.warning("Could not check infos for archive %s: %s" % (archive, '\n' + traceback.format_exc()))
 
@@ -2386,7 +2387,7 @@ def _recursive_umount(directory):
     Args:
         directory -- a directory path
     """
-    mount_lines = subprocess.check_output("mount").split("\n")
+    mount_lines = check_output("mount").split("\n")
 
     points_to_umount = [line.split(" ")[2]
                         for line in mount_lines
@@ -2412,8 +2413,8 @@ def disk_usage(path):
     # We don't do this in python with os.stat because we don't want
     # to follow symlinks
 
-    du_output = subprocess.check_output(['du', '-sb', path])
+    du_output = check_output(['du', '-sb', path], shell=False)
-    return int(du_output.split()[0].decode('utf-8'))
+    return int(du_output.split()[0])
 
 
 def binary_to_human(n, customary=False):

src/yunohost/certificate.py

@@ -385,7 +385,7 @@ def certificate_renew(domain_list, force=False, no_checks=False, email=False, st
             _fetch_and_enable_new_certificate(domain, staging, no_checks=no_checks)
         except Exception as e:
             import traceback
-            from StringIO import StringIO
+            from io import StringIO
             stack = StringIO()
             traceback.print_exc(file=stack)
             msg = "Certificate renewing for %s failed !" % (domain)
@@ -638,7 +638,7 @@ def _get_status(domain):
     cert_subject = cert.get_subject().CN
     cert_issuer = cert.get_issuer().CN
     organization_name = cert.get_issuer().O
-    valid_up_to = datetime.strptime(cert.get_notAfter(), "%Y%m%d%H%M%SZ")
+    valid_up_to = datetime.strptime(cert.get_notAfter().decode('utf-8'), "%Y%m%d%H%M%SZ")
     days_remaining = (valid_up_to - datetime.utcnow()).days
 
     if cert_issuer == _name_self_CA():

src/yunohost/data_migrations/0019_extend_permissions_features.py

@@ -98,7 +98,7 @@ class MyMigration(Migration):
             # Migrate old settings
             migrate_legacy_permission_settings()
 
-        except Exception as e:
+        except Exception:
             logger.warn(m18n.n("migration_0019_migration_failed_trying_to_rollback"))
             os.system("systemctl stop slapd")
             os.system("rm -r /etc/ldap/slapd.d")  # To be sure that we don't keep some part of the old config

src/yunohost/diagnosis.py

@@ -451,7 +451,7 @@ class Diagnoser():
         key = "diagnosis_description_" + id_
         descr = m18n.n(key)
         # If no description available, fallback to id
-        return descr if descr.decode('utf-8') != key else id_
+        return descr if descr != key else id_
 
     @staticmethod
     def i18n(report, force_remove_html_tags=False):

src/yunohost/domain.py

@@ -62,18 +62,15 @@ def domain_list(exclude_subdomains=False):
 
         result_list.append(domain)
 
-    def cmp_domain(domain1, domain2):
+    def cmp_domain(domain):
         # Keep the main part of the domain and the extension together
         # eg: this.is.an.example.com -> ['example.com', 'an', 'is', 'this']
-        domain1 = domain1.split('.')
+        domain = domain.split('.')
-        domain2 = domain2.split('.')
+        domain[-1] = domain[-2] + domain.pop()
-        domain1[-1] = domain1[-2] + domain1.pop()
+        domain = list(reversed(domain))
-        domain2[-1] = domain2[-2] + domain2.pop()
+        return domain
-        domain1 = list(reversed(domain1))
-        domain2 = list(reversed(domain2))
-        return cmp(domain1, domain2)
 
-    result_list = sorted(result_list, cmp_domain)
+    result_list = sorted(result_list, key=cmp_domain)
 
     return {
         'domains': result_list,

src/yunohost/dyndns.py

@@ -105,7 +105,7 @@ def _dyndns_available(provider, domain):
         raise YunohostError('dyndns_could_not_check_available',
                             domain=domain, provider=provider)
 
-    return r == u"Domain %s is available" % domain
+    return r == "Domain %s is available" % domain
 
 
 @is_unit_operation()

src/yunohost/hook.py

@@ -182,7 +182,8 @@ def hook_list(action, list_by='name', show_info=False):
     def _append_folder(d, folder):
         # Iterate over and add hook from a folder
         for f in os.listdir(folder + action):
-            if f[0] == '.' or f[-1] == '~' or f.endswith(".pyc"):
+            if f[0] == '.' or f[-1] == '~' or f.endswith(".pyc") \
+               or (f.startswith("__") and f.endswith("__")):
                 continue
             path = '%s%s/%s' % (folder, action, f)
             priority, name = _extract_filename_parts(f)
@@ -387,9 +388,6 @@ def _hook_exec_bash(path, args, no_trace, chdir, env, user, return_format, logge
 
     env['YNH_INTERFACE'] = msettings.get('interface')
 
-    stdinfo = os.path.join(tempfile.mkdtemp(), "stdinfo")
-    env['YNH_STDINFO'] = stdinfo
-
     stdreturn = os.path.join(tempfile.mkdtemp(), "stdreturn")
     with open(stdreturn, 'w') as f:
         f.write('')
@@ -415,10 +413,7 @@ def _hook_exec_bash(path, args, no_trace, chdir, env, user, return_format, logge
 
     logger.debug("Executing command '%s'" % ' '.join(command))
 
-    returncode = call_async_output(
+    returncode = call_async_output(command, loggers, shell=False, cwd=chdir)
-        command, loggers, shell=False, cwd=chdir,
-        stdinfo=stdinfo
-    )
 
     raw_content = None
     try:

src/yunohost/log.py

@@ -65,8 +65,7 @@ def log_list(limit=None, with_details=False, with_suboperations=False):
 
     operations = {}
 
-    logs = filter(lambda x: x.endswith(METADATA_FILE_EXT),
+    logs = [x for x in os.listdir(OPERATIONS_PATH) if x.endswith(METADATA_FILE_EXT)]
-                  os.listdir(OPERATIONS_PATH))
     logs = list(reversed(sorted(logs)))
 
     if limit is not None:
@@ -337,7 +336,7 @@ def is_unit_operation(entities=['app', 'domain', 'group', 'service', 'user'],
                     entity_type = entity
 
                 if entity in kwargs and kwargs[entity] is not None:
-                    if isinstance(kwargs[entity], basestring):
+                    if isinstance(kwargs[entity], str):
                         related_to.append((entity_type, kwargs[entity]))
                     else:
                         for x in kwargs[entity]:
@@ -596,7 +595,7 @@ class OperationLogger(object):
         """
         if self.ended_at is not None or self.started_at is None:
             return
-        if error is not None and not isinstance(error, basestring):
+        if error is not None and not isinstance(error, str):
             error = str(error)
         self.ended_at = datetime.utcnow()
         self._error = error

src/yunohost/permission.py

@@ -106,7 +106,7 @@ def user_permission_list(short=False, full=False, ignore_system_perms=False, abs
             infos["label"] = "%s (%s)" % (main_perm_label, infos["label"])
 
     if short:
-        permissions = permissions.keys()
+        permissions = list(permissions.keys())
 
     return {'permissions': permissions}
 

src/yunohost/regenconf.py

@@ -21,7 +21,6 @@
 
 import os
 import yaml
-import subprocess
 import shutil
 import hashlib
 
@@ -30,6 +29,7 @@ from datetime import datetime
 
 from moulinette import m18n
 from moulinette.utils import log, filesystem
+from moulinette.utils.process import check_output
 
 from yunohost.utils.error import YunohostError
 from yunohost.log import is_unit_operation
@@ -654,10 +654,10 @@ def manually_modified_files():
 def manually_modified_files_compared_to_debian_default(ignore_handled_by_regenconf=False):
 
     # from https://serverfault.com/a/90401
-    files = subprocess.check_output("dpkg-query -W -f='${Conffiles}\n' '*' \
+    files = check_output("dpkg-query -W -f='${Conffiles}\n' '*' \
-                                   | awk 'OFS=\"  \"{print $2,$1}' \
+                        | awk 'OFS=\"  \"{print $2,$1}' \
-                                   | md5sum -c 2>/dev/null \
+                        | md5sum -c 2>/dev/null \
-                                   | awk -F': ' '$2 !~ /OK/{print $1}'", shell=True)
+                        | awk -F': ' '$2 !~ /OK/{print $1}'")
     files = files.strip().split("\n")
 
     if ignore_handled_by_regenconf:

src/yunohost/service.py

@@ -35,6 +35,7 @@ from datetime import datetime
 
 from moulinette import m18n
 from yunohost.utils.error import YunohostError
+from moulinette.utils.process import check_output
 from moulinette.utils.log import getActionLogger
 from moulinette.utils.filesystem import read_file, append_to_file, write_to_file
 
@@ -358,7 +359,7 @@ def _get_and_format_service_status(service, infos):
         # that mean that we don't have a translation for this string
         # that's the only way to test for that for now
         # if we don't have it, uses the one provided by systemd
-        if description.decode('utf-8') == translation_key:
+        if description == translation_key:
             description = str(raw_status.get("Description", ""))
 
     output = {
@@ -489,7 +490,7 @@ def service_regen_conf(names=[], with_diff=False, force=False, dry_run=False,
             raise YunohostError('service_unknown', service=name)
 
     if names is []:
-        names = services.keys()
+        names = list(services.keys())
 
     logger.warning(m18n.n("service_regen_conf_is_deprecated"))
 
@@ -563,8 +564,7 @@ def _give_lock(action, service, p):
     while son_PID == 0 and p.poll() is None:
         # Call systemctl to get the PID
         # Output of the command is e.g. ControlPID=1234
-        son_PID = subprocess.check_output(cmd_get_son_PID.split()) \
+        son_PID = check_output(cmd_get_son_PID).split("=")[1]
-                            .strip().split("=")[1]
         son_PID = int(son_PID)
         time.sleep(1)
 
@@ -599,7 +599,7 @@ def _get_services():
 
     # some services are marked as None to remove them from YunoHost
     # filter this
-    for key, value in services.items():
+    for key, value in list(services.items()):
         if value is None:
             del services[key]
 
@@ -720,7 +720,7 @@ def _get_journalctl_logs(service, number="all"):
     services = _get_services()
     systemd_service = services.get(service, {}).get("actual_systemd_service", service)
     try:
-        return subprocess.check_output("journalctl --no-hostname --no-pager -u {0} -n{1}".format(systemd_service, number), shell=True)
+        return check_output("journalctl --no-hostname --no-pager -u {0} -n{1}".format(systemd_service, number))
     except:
         import traceback
         return "error while get services logs from journalctl:\n%s" % traceback.format_exc()

src/yunohost/settings.py

@@ -29,7 +29,7 @@ def is_boolean(value):
     """
     if isinstance(value, bool):
         return True, value
-    elif isinstance(value, basestring):
+    elif isinstance(value, str):
         if str(value).lower() in ['true', 'on', 'yes', 'false', 'off', 'no']:
             return True, str(value).lower() in ['true', 'on', 'yes']
         else:
@@ -141,7 +141,7 @@ def settings_set(key, value):
                 raise YunohostError('global_settings_bad_type_for_setting', setting=key,
                                     received_type=type(value).__name__, expected_type=key_type)
     elif key_type == "string":
-        if not isinstance(value, basestring):
+        if not isinstance(value, str):
             raise YunohostError('global_settings_bad_type_for_setting', setting=key,
                                 received_type=type(value).__name__, expected_type=key_type)
     elif key_type == "enum":

src/yunohost/tests/test_apps.py

@@ -4,7 +4,7 @@ import pytest
 import shutil
 import requests
 
-from conftest import message, raiseYunohostError, get_test_apps_dir
+from .conftest import message, raiseYunohostError, get_test_apps_dir
 
 from moulinette.utils.filesystem import mkdir
 

src/yunohost/tests/test_apps_arguments_parsing.py

@@ -2,7 +2,7 @@ import sys
 import pytest
 
 from mock import patch
-from StringIO import StringIO
+from io import StringIO
 from collections import OrderedDict
 
 from moulinette import msignals

src/yunohost/tests/test_appurl.py

@@ -1,7 +1,7 @@
 import pytest
 import os
 
-from conftest import get_test_apps_dir
+from .conftest import get_test_apps_dir
 
 from yunohost.utils.error import YunohostError
 from yunohost.app import app_install, app_remove, _normalize_domain_path

src/yunohost/tests/test_backuprestore.py

@@ -3,7 +3,7 @@ import os
 import shutil
 import subprocess
 
-from conftest import message, raiseYunohostError, get_test_apps_dir
+from .conftest import message, raiseYunohostError, get_test_apps_dir
 
 from yunohost.app import app_install, app_remove, app_ssowatconf
 from yunohost.app import _is_installed
@@ -23,8 +23,6 @@ def setup_function(function):
     global maindomain
     maindomain = _get_maindomain()
 
-    print ""
-
     assert backup_test_dependencies_are_met()
 
     clean_tmp_backup_directory()
@@ -150,7 +148,7 @@ def clean_tmp_backup_directory():
     if tmp_backup_directory_is_empty():
         return
 
-    mount_lines = subprocess.check_output("mount").split("\n")
+    mount_lines = subprocess.check_output("mount").decode().split("\n")
 
     points_to_umount = [line.split(" ")[2]
                         for line in mount_lines
@@ -638,6 +636,7 @@ def test_backup_binds_are_readonly(mocker, monkeypatch):
         confssh = os.path.join(self.work_dir, "conf/ssh")
         output = subprocess.check_output("touch %s/test 2>&1 || true" % confssh,
                                          shell=True, env={'LANG': 'en_US.UTF-8'})
+        output = output.decode()
 
         assert "Read-only file system" in output
 

src/yunohost/tests/test_changeurl.py

@@ -3,7 +3,7 @@ import time
 import requests
 import os
 
-from conftest import get_test_apps_dir
+from .conftest import get_test_apps_dir
 
 from yunohost.app import app_install, app_change_url, app_remove, app_map
 from yunohost.domain import _get_maindomain

src/yunohost/tests/test_permission.py

@@ -6,7 +6,7 @@ import os
 import json
 import shutil
 
-from conftest import message, raiseYunohostError, get_test_apps_dir
+from .conftest import message, raiseYunohostError, get_test_apps_dir
 
 from yunohost.app import app_install, app_upgrade, app_remove, app_change_url, app_map, _installed_apps, APPS_SETTING_PATH, _set_app_settings, _get_app_settings
 from yunohost.user import user_list, user_create, user_delete, \

src/yunohost/tests/test_regenconf.py

@@ -1,6 +1,6 @@
 import os
 
-from conftest import message
+from .conftest import message
 from yunohost.domain import domain_add, domain_remove, domain_list
 from yunohost.regenconf import regen_conf, manually_modified_files, _get_conf_hashes, _force_clear_hashes
 

src/yunohost/tests/test_service.py

@@ -1,6 +1,6 @@
 import os
 
-from conftest import raiseYunohostError
+from .conftest import raiseYunohostError
 
 from yunohost.service import _get_services, _save_services, service_status, service_add, service_remove, service_log
 

src/yunohost/tests/test_user-group.py

@@ -1,6 +1,6 @@
 import pytest
 
-from conftest import message, raiseYunohostError
+from .conftest import message, raiseYunohostError
 
 from yunohost.user import user_list, user_info, user_create, user_delete, user_update, \
     user_group_list, user_group_create, user_group_delete, user_group_update

src/yunohost/tools.py

@@ -303,7 +303,7 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False,
         '/home/yunohost.app'
     ]
 
-    for folder in filter(lambda x: not os.path.exists(x), folders_to_create):
+    for folder in [x for x in folders_to_create if not os.path.exists(x)]:
         os.makedirs(folder)
 
     # Change folders permissions
@@ -953,7 +953,7 @@ def _get_migrations_list():
     # (in particular, pending migrations / not already ran are not listed
     states = tools_migrations_state()["migrations"]
 
-    for migration_file in filter(lambda x: re.match(r"^\d+_[a-zA-Z0-9_]+\.py$", x), os.listdir(migrations_path)):
+    for migration_file in [x for x in os.listdir(migrations_path) if re.match(r"^\d+_[a-zA-Z0-9_]+\.py$", x)]:
         m = _load_migration(migration_file)
         m.state = states.get(m.id, "pending")
         migrations.append(m)
@@ -972,7 +972,7 @@ def _get_migration_by_name(migration_name):
         raise AssertionError("Unable to find migration with name %s" % migration_name)
 
     migrations_path = data_migrations.__path__[0]
-    migrations_found = filter(lambda x: re.match(r"^\d+_%s\.py$" % migration_name, x), os.listdir(migrations_path))
+    migrations_found = [x for x in os.listdir(migrations_path) if re.match(r"^\d+_%s\.py$" % migration_name, x)]
 
     assert len(migrations_found) == 1, "Unable to find migration with name %s" % migration_name
 

src/yunohost/user.py

@@ -35,6 +35,7 @@ import copy
 
 from moulinette import msignals, msettings, m18n
 from moulinette.utils.log import getActionLogger
+from moulinette.utils.process import check_output
 
 from yunohost.utils.error import YunohostError
 from yunohost.service import service_status
@@ -473,8 +474,7 @@ def user_info(username):
         else:
             try:
                 cmd = 'doveadm -f flow quota get -u %s' % user['uid'][0]
-                cmd_result = subprocess.check_output(cmd, stderr=subprocess.STDOUT,
+                cmd_result = check_output(cmd)
-                                                     shell=True)
             except Exception as e:
                 cmd_result = ""
                 logger.warning("Failed to fetch quota info ... : %s " % str(e))
@@ -546,7 +546,7 @@ def user_group_list(short=False, full=False, include_primary_groups=True):
             groups[name]["permissions"] = [_ldap_path_extract(p, "cn") for p in infos.get("permission", [])]
 
     if short:
-        groups = groups.keys()
+        groups = list(groups.keys())
 
     return {'groups': groups}
 
@@ -631,7 +631,7 @@ def user_group_delete(operation_logger, groupname, force=False, sync_perm=True):
     from yunohost.permission import permission_sync_to_user
     from yunohost.utils.ldap import _get_ldap_interface
 
-    existing_groups = user_group_list()['groups'].keys()
+    existing_groups = list(user_group_list()['groups'].keys())
     if groupname not in existing_groups:
         raise YunohostError('group_unknown', group=groupname)
 
@@ -639,7 +639,7 @@ def user_group_delete(operation_logger, groupname, force=False, sync_perm=True):
     # without the force option...
     #
     # We also can't delete "all_users" because that's a special group...
-    existing_users = user_list()['users'].keys()
+    existing_users = list(user_list()['users'].keys())
     undeletable_groups = existing_users + ["all_users", "visitors"]
     if groupname in undeletable_groups and not force:
         raise YunohostError('group_cannot_be_deleted', group=groupname)
@@ -675,7 +675,7 @@ def user_group_update(operation_logger, groupname, add=None, remove=None, force=
     from yunohost.permission import permission_sync_to_user
     from yunohost.utils.ldap import _get_ldap_interface
 
-    existing_users = user_list()['users'].keys()
+    existing_users = list(user_list()['users'].keys())
 
     # Refuse to edit a primary group of a user (e.g. group 'sam' related to user 'sam')
     # Those kind of group should only ever contain the user (e.g. sam) and only this one.

src/yunohost/utils/legacy.py

@@ -22,7 +22,7 @@ class SetupGroupPermissions():
         try:
             objects = ldap.search(target + ",dc=yunohost,dc=org")
         # ldap search will raise an exception if no corresponding object is found >.> ...
-        except Exception as e:
+        except Exception:
             logger.debug("%s does not exist, no need to delete it" % target)
             return
 
@@ -100,7 +100,7 @@ class SetupGroupPermissions():
 
             url = "/" if domain and path else None
             if permission:
-                known_users = user_list()["users"].keys()
+                known_users = list(user_list()["users"].keys())
                 allowed = [user for user in permission.split(',') if user in known_users]
             else:
                 allowed = ["all_users"]
@@ -237,7 +237,7 @@ def translate_legacy_rules_in_ssowant_conf_json_persistent():
     protected_urls = persistent.get("protected_urls", []) + ["re:" + r for r in persistent.get("protected_regex", [])]
     unprotected_urls = persistent.get("unprotected_urls", []) + ["re:" + r for r in persistent.get("unprotected_regex", [])]
 
-    known_users = user_list()["users"].keys()
+    known_users = list(user_list()["users"].keys())
 
     for legacy_rule in legacy_rules:
         if legacy_rule in persistent:

src/yunohost/utils/packages.py

@@ -70,7 +70,11 @@ def meets_version_specifier(pkg_name, specifier):
     op, req_version = re.search(r'(<<|<=|=|>=|>>) *([\d\.]+)', specifier).groups()
     req_version = version.parse(req_version)
 
-    # cmp is a python builtin that returns (-1, 0, 1) depending on comparison
+    # Python2 had a builtin that returns (-1, 0, 1) depending on comparison
+    # c.f. https://stackoverflow.com/a/22490617
+    def cmp(a, b):
+        return (a > b) - (a < b)
+
     deb_operators = {
         "<<": lambda v1, v2: cmp(v1, v2) in [-1],
         "<=": lambda v1, v2: cmp(v1, v2) in [-1, 0],

src/yunohost/utils/password.py

@@ -173,7 +173,7 @@ class PasswordValidator(object):
         # stdin to avoid it being shown in ps -ef --forest...
         command = "grep -q -F -f - %s" % MOST_USED_PASSWORDS
         p = subprocess.Popen(command.split(), stdin=subprocess.PIPE)
-        p.communicate(input=password)
+        p.communicate(input=password.encode('utf-8'))
         return not bool(p.returncode)
 
 

tox.ini

@@ -1,12 +1,12 @@
 [tox]
-envlist = py{27,37}-{lint,invalidcode},py37-black
+envlist = py37-{lint,invalidcode},py37-black
 
 [testenv]
 skip_install=True
 deps =
-  py{27,37}-{lint,invalidcode}: flake8
+  py37-{lint,invalidcode}: flake8
   py37-black: black
 commands =
-        py{27,37}-lint: flake8 src doc data tests --ignore E402,E501 --exclude src/yunohost/vendor
+        py37-lint: flake8 src doc data tests --ignore E402,E501 --exclude src/yunohost/vendor
-        py{27,37}-invalidcode: flake8 src data --exclude src/yunohost/tests,src/yunohost/vendor --select F
+        py37-invalidcode: flake8 src data --exclude src/yunohost/tests,src/yunohost/vendor --select F
         py37-black: black --check --diff src doc data tests