Merge pull request #695 from YunoHost/rework_tools_update_only

[fix] Rework tools update (only)

locales/en.json

@@ -483,7 +483,8 @@
     "unit_unknown": "Unknown unit '{unit:s}'",
     "unlimit": "No quota",
     "unrestore_app": "App '{app:s}' will not be restored",
-    "update_cache_failed": "Unable to update APT cache",
+    "update_apt_cache_failed": "Unable to update the cache of APT (Debian's package manager). Here is a dump of the sources.list lines which might help to identify problematic lines : \n{sourceslist}",
+    "update_apt_cache_warning": "Some errors happened while updating the cache of APT (Debian's package manager). Here is a dump of the sources.list lines which might help to identify problematic lines : \n{sourceslist}",
     "updating_apt_cache": "Fetching available upgrades for system packages…",
     "upgrade_complete": "Upgrade complete",
     "upgrading_packages": "Upgrading packages…",

src/yunohost/tools.py

@@ -30,6 +30,7 @@ import json
 import subprocess
 import pwd
 import socket
+from glob import glob
 from xmlrpclib import Fault
 from importlib import import_module
 from collections import OrderedDict
@@ -41,7 +42,7 @@ from moulinette import msettings, msignals, m18n
 from moulinette.core import init_authenticator
 from yunohost.utils.error import YunohostError
 from moulinette.utils.log import getActionLogger
-from moulinette.utils.process import check_output
+from moulinette.utils.process import check_output, call_async_output
 from moulinette.utils.filesystem import read_json, write_to_json
 from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, app_list, _install_appslist_fetch_cron
 from yunohost.domain import domain_add, domain_list, _get_maindomain, _set_maindomain
@@ -474,23 +475,39 @@ def tools_update(ignore_apps=False, ignore_packages=False):
     # "packages" will list upgradable packages
     packages = []
     if not ignore_packages:
-        cache = apt.Cache()
 
         # Update APT cache
+        # LC_ALL=C is here to make sure the results are in english
+        command = "LC_ALL=C apt update"
+        # TODO : add @is_unit_operation to tools_update so that the
+        # debug output can be fetched when there's an issue...
+
+        # Filter boring message about "apt not having a stable CLI interface"
+        # Also keep track of wether or not we encountered a warning...
+        warnings = []
+        def is_legit_warning(m):
+            legit_warning = m.rstrip() and "apt does not have a stable CLI interface" not in m.rstrip()
+            if legit_warning:
+                warnings.append(m)
+            return legit_warning
+
+        callbacks = (
+            # stdout goes to debug
+            lambda l: logger.debug(l.rstrip()),
+            # stderr goes to warning except for the boring apt messages
+            lambda l: logger.warning(l.rstrip()) if is_legit_warning(l) else logger.debug(l.rstrip())
+        )
+
         logger.info(m18n.n('updating_apt_cache'))
-        if not cache.update():
-            raise YunohostError('update_cache_failed')
 
-        cache.open(None)
+        returncode = call_async_output(command, callbacks, shell=True)
-        cache.upgrade(True)
 
-        # Add changelogs to the result
+        if returncode != 0:
-        for pkg in cache.get_changes():
+            raise YunohostError('update_apt_cache_failed', sourceslist='\n'.join(_dump_sources_list()))
-            packages.append({
+        elif warnings:
-                'name': pkg.name,
+            logger.error(m18n.n('update_apt_cache_warning', sourceslist='\n'.join(_dump_sources_list())))
-                'fullname': pkg.fullname,
+
-                'changelog': pkg.get_changelog()
+        packages = list(_list_upgradable_apt_packages())
-            })
         logger.debug(m18n.n('done'))
 
     # "apps" will list upgradable packages
@@ -519,6 +536,44 @@ def tools_update(ignore_apps=False, ignore_packages=False):
     return {'packages': packages, 'apps': apps}
 
 
+# TODO : move this to utils/packages.py ?
+def _list_upgradable_apt_packages():
+
+    # List upgradable packages
+    # LC_ALL=C is here to make sure the results are in english
+    upgradable_raw = check_output("LC_ALL=C apt list --upgradable")
+
+    # Dirty parsing of the output
+    upgradable_raw = [l.strip() for l in upgradable_raw.split("\n") if l.strip()]
+    for line in upgradable_raw:
+        # Remove stupid warning and verbose messages >.>
+        if "apt does not have a stable CLI interface" in line or "Listing..." in line:
+            continue
+        # line should look like :
+        # yunohost/stable 3.5.0.2+201903211853 all [upgradable from: 3.4.2.4+201903080053]
+        line = line.split()
+        if len(line) != 6:
+            logger.warning("Failed to parse this line : %s" % ' '.join(line))
+            continue
+
+        yield {
+            "name": line[0].split("/")[0],
+            "new_version": line[1],
+            "current_version": line[5].strip("]"),
+        }
+
+
+def _dump_sources_list():
+
+    filenames = glob("/etc/apt/sources.list") + glob("/etc/apt/sources.list.d/*")
+    for filename in filenames:
+        with open(filename, "r") as f:
+            for line in f.readlines():
+                if line.startswith("#") or not line.strip():
+                    continue
+                yield filename.replace("/etc/apt/", "") + ":" + line.strip()
+
+
 @is_unit_operation()
 def tools_upgrade(operation_logger, auth, ignore_apps=False, ignore_packages=False):
     """