From 2f0dd973b8686de62f92b1aefa91a6d68ebb6100 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 22 Mar 2019 01:14:45 +0100 Subject: [PATCH 1/4] Rework system-part of tools_update... --- src/yunohost/tools.py | 51 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 42114c7e9..92c68eca9 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -41,7 +41,7 @@ from moulinette import msettings, msignals, m18n from moulinette.core import init_authenticator from yunohost.utils.error import YunohostError from moulinette.utils.log import getActionLogger -from moulinette.utils.process import check_output +from moulinette.utils.process import check_output, call_async_output from moulinette.utils.filesystem import read_json, write_to_json from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, app_list, _install_appslist_fetch_cron from yunohost.domain import domain_add, domain_list, _get_maindomain, _set_maindomain @@ -474,23 +474,54 @@ def tools_update(ignore_apps=False, ignore_packages=False): # "packages" will list upgradable packages packages = [] if not ignore_packages: - cache = apt.Cache() # Update APT cache + # LC_ALL=C is here to make sure the results are in english + command = "LC_ALL=C apt update" + # TODO : add @is_unit_operation to tools_update so that the + # debug output can be fetched when there's an issue... + callbacks = ( + # stdout goes to debug + lambda l: logger.debug(l.rstrip()), + # stderr goes to warning + # FIXME : filter the damn "CLI interface not stable" from apt >.> + lambda l: logger.warning(l.rstrip()), + ) + logger.info(m18n.n('updating_apt_cache')) - if not cache.update(): + + returncode = call_async_output(command, callbacks, shell=True) + + if returncode != 0: + + # TODO : here, we should run something like a + # `cat /etc/apt/sources.list /etc/apt/sources.list.d/*` + # and append it to the error message to improve debugging + raise YunohostError('update_cache_failed') - cache.open(None) - cache.upgrade(True) + # List upgradable packages + # LC_ALL=C is here to make sure the results are in english + upgradable_raw = check_output("LC_ALL=C apt list --upgradable") - # Add changelogs to the result - for pkg in cache.get_changes(): + # Dirty parsing of the output + upgradable_raw = [l.strip() for l in upgradable_raw.split("\n") if l.strip()] + for line in upgradable_raw: + # Remove stupid warning and verbose messages >.> + if "apt does not have a stable CLI interface" in line or "Listing..." in line: + continue + # line should look like : + # yunohost/stable 3.5.0.2+201903211853 all [upgradable from: 3.4.2.4+201903080053] + line = line.split() + if len(line) != 6: + logger.warning("Failed to parse this line : %s" % ' '.join(line)) + continue packages.append({ - 'name': pkg.name, - 'fullname': pkg.fullname, - 'changelog': pkg.get_changelog() + "name": line[0].split("/")[0], + "new_version": line[1], + "current_version": line[5].strip("]"), }) + logger.debug(m18n.n('done')) # "apps" will list upgradable packages From 2f034bb7c9b6d62943a9ea1897b3000b5c33f193 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 22 Mar 2019 04:01:08 +0100 Subject: [PATCH 2/4] Factorize function to list upgradable packages --- src/yunohost/tools.py | 50 ++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 92c68eca9..154ad086a 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -500,28 +500,7 @@ def tools_update(ignore_apps=False, ignore_packages=False): raise YunohostError('update_cache_failed') - # List upgradable packages - # LC_ALL=C is here to make sure the results are in english - upgradable_raw = check_output("LC_ALL=C apt list --upgradable") - - # Dirty parsing of the output - upgradable_raw = [l.strip() for l in upgradable_raw.split("\n") if l.strip()] - for line in upgradable_raw: - # Remove stupid warning and verbose messages >.> - if "apt does not have a stable CLI interface" in line or "Listing..." in line: - continue - # line should look like : - # yunohost/stable 3.5.0.2+201903211853 all [upgradable from: 3.4.2.4+201903080053] - line = line.split() - if len(line) != 6: - logger.warning("Failed to parse this line : %s" % ' '.join(line)) - continue - packages.append({ - "name": line[0].split("/")[0], - "new_version": line[1], - "current_version": line[5].strip("]"), - }) - + packages = list(_list_upgradable_apt_packages()) logger.debug(m18n.n('done')) # "apps" will list upgradable packages @@ -550,6 +529,33 @@ def tools_update(ignore_apps=False, ignore_packages=False): return {'packages': packages, 'apps': apps} +# TODO : move this to utils/packages.py ? +def _list_upgradable_apt_packages(): + + # List upgradable packages + # LC_ALL=C is here to make sure the results are in english + upgradable_raw = check_output("LC_ALL=C apt list --upgradable") + + # Dirty parsing of the output + upgradable_raw = [l.strip() for l in upgradable_raw.split("\n") if l.strip()] + for line in upgradable_raw: + # Remove stupid warning and verbose messages >.> + if "apt does not have a stable CLI interface" in line or "Listing..." in line: + continue + # line should look like : + # yunohost/stable 3.5.0.2+201903211853 all [upgradable from: 3.4.2.4+201903080053] + line = line.split() + if len(line) != 6: + logger.warning("Failed to parse this line : %s" % ' '.join(line)) + continue + + yield { + "name": line[0].split("/")[0], + "new_version": line[1], + "current_version": line[5].strip("]"), + } + + @is_unit_operation() def tools_upgrade(operation_logger, auth, ignore_apps=False, ignore_packages=False): """ From e298838949ce3b2ab924a303dff3cbeffc9b39f3 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 26 Mar 2019 19:45:16 +0100 Subject: [PATCH 3/4] Filter boring apt warnings + report an error if there was real warnings --- locales/en.json | 1 + src/yunohost/tools.py | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/locales/en.json b/locales/en.json index 694df0707..da27c7cb0 100644 --- a/locales/en.json +++ b/locales/en.json @@ -483,6 +483,7 @@ "unlimit": "No quota", "unrestore_app": "App '{app:s}' will not be restored", "update_cache_failed": "Unable to update APT cache", + "update_cache_warning": "Some errors happened while updating APT cache", "updating_apt_cache": "Fetching available upgrades for system packages…", "upgrade_complete": "Upgrade complete", "upgrading_packages": "Upgrading packages…", diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 154ad086a..f9d86ad09 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -480,12 +480,21 @@ def tools_update(ignore_apps=False, ignore_packages=False): command = "LC_ALL=C apt update" # TODO : add @is_unit_operation to tools_update so that the # debug output can be fetched when there's an issue... + + # Filter boring message about "apt not having a stable CLI interface" + # Also keep track of wether or not we encountered a warning... + warnings = [] + def is_legit_warning(m): + legit_warning = m.rstrip() and "apt does not have a stable CLI interface" not in m.rstrip() + if legit_warning: + warnings.append(m) + return legit_warning + callbacks = ( # stdout goes to debug lambda l: logger.debug(l.rstrip()), - # stderr goes to warning - # FIXME : filter the damn "CLI interface not stable" from apt >.> - lambda l: logger.warning(l.rstrip()), + # stderr goes to warning except for the boring apt messages + lambda l: logger.warning(l.rstrip()) if is_legit_warning(l) else logger.debug(l.rstrip()) ) logger.info(m18n.n('updating_apt_cache')) @@ -499,6 +508,8 @@ def tools_update(ignore_apps=False, ignore_packages=False): # and append it to the error message to improve debugging raise YunohostError('update_cache_failed') + elif warnings: + logger.error(m18n.n('update_cache_warning')) packages = list(_list_upgradable_apt_packages()) logger.debug(m18n.n('done')) From 162eeb7e06e04ec32ed66331a184c9527fa68e82 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 27 Mar 2019 18:08:46 +0100 Subject: [PATCH 4/4] Dump sources list in error message to help debugging --- locales/en.json | 4 ++-- src/yunohost/tools.py | 21 ++++++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/locales/en.json b/locales/en.json index da27c7cb0..64f2e6766 100644 --- a/locales/en.json +++ b/locales/en.json @@ -482,8 +482,8 @@ "unit_unknown": "Unknown unit '{unit:s}'", "unlimit": "No quota", "unrestore_app": "App '{app:s}' will not be restored", - "update_cache_failed": "Unable to update APT cache", - "update_cache_warning": "Some errors happened while updating APT cache", + "update_apt_cache_failed": "Unable to update the cache of APT (Debian's package manager). Here is a dump of the sources.list lines which might help to identify problematic lines : \n{sourceslist}", + "update_apt_cache_warning": "Some errors happened while updating the cache of APT (Debian's package manager). Here is a dump of the sources.list lines which might help to identify problematic lines : \n{sourceslist}", "updating_apt_cache": "Fetching available upgrades for system packages…", "upgrade_complete": "Upgrade complete", "upgrading_packages": "Upgrading packages…", diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index f9d86ad09..635399801 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -30,6 +30,7 @@ import json import subprocess import pwd import socket +from glob import glob from xmlrpclib import Fault from importlib import import_module from collections import OrderedDict @@ -502,14 +503,9 @@ def tools_update(ignore_apps=False, ignore_packages=False): returncode = call_async_output(command, callbacks, shell=True) if returncode != 0: - - # TODO : here, we should run something like a - # `cat /etc/apt/sources.list /etc/apt/sources.list.d/*` - # and append it to the error message to improve debugging - - raise YunohostError('update_cache_failed') + raise YunohostError('update_apt_cache_failed', sourceslist='\n'.join(_dump_sources_list())) elif warnings: - logger.error(m18n.n('update_cache_warning')) + logger.error(m18n.n('update_apt_cache_warning', sourceslist='\n'.join(_dump_sources_list()))) packages = list(_list_upgradable_apt_packages()) logger.debug(m18n.n('done')) @@ -567,6 +563,17 @@ def _list_upgradable_apt_packages(): } +def _dump_sources_list(): + + filenames = glob("/etc/apt/sources.list") + glob("/etc/apt/sources.list.d/*") + for filename in filenames: + with open(filename, "r") as f: + for line in f.readlines(): + if line.startswith("#") or not line.strip(): + continue + yield filename.replace("/etc/apt/", "") + ":" + line.strip() + + @is_unit_operation() def tools_upgrade(operation_logger, auth, ignore_apps=False, ignore_packages=False): """