[enh] Refactor applist management (#160)

* [mod] directly use python to retreive json list

* [enh] app_fetchlist fetch all app_list by default

* [fix] name variable doesn't exists here

* [fix] re returns None when there is not matchs

* [enh] app_fetchlist fetch all app_list by default

* Some cleaning for better readability

* Simpler variable name

* Prepare a function that register lists to be fetched

* Skeletong for applist system migration

* Add implementation of migration system with tests

* Refactorize app_fetchlist

* Misc fixes + adding test for single app fetching

* Fixing a few issues + test removelist

* Adding fetchlist and cron install during postinstall

* Adding debug messages

* Adding particular exception for SSL connection error

* Update actionmap help

* We don't use urlretrieve

* Clean tests, some description were bad

* [mod] some cleaning

* Moving to a .json file to store lists url + adjusting tests

* Adding missing string in locale

* Moving exception to logger.error when fetching fails

* Adding name of applist in error messages

* Fixing cron job stuff + adding proper tests

* Using None instead of -1 for applist lastupdate

* Handling exceptions when writing applist files

* More exception handling...

* [mod] pep8

* Updating test for migration of conflicting lists

* More general error when return code is not 200

* [enh] Improve app_fetchlist help.

* [fix] Use appslist instead of applist.

* [fix] Consistent user string for translation.

data/actionsmap/yunohost.yml

@@ -391,28 +391,28 @@ app:
 
         ### app_fetchlist()
         fetchlist:
-            action_help: Fetch application list from app server
+            action_help: Fetch application lists from app servers, or register a new one.
             api: PUT /appslists
             arguments:
-                -u:
-                    full: --url
-                    help: URL of remote JSON list (default https://app.yunohost.org/official.json)
                 -n:
                     full: --name
-                    help: Name of the list (default yunohost)
+                    help: Name of the list to fetch (fetches all registered lists if empty)
                     extra:
                         pattern: &pattern_listname
                             - !!str ^[a-z0-9_]+$
                             - "pattern_listname"
+                -u:
+                    full: --url
+                    help: URL of a new application list to register. To be specified with -n.
 
         ### app_listlists()
         listlists:
-            action_help: List fetched lists
+            action_help: List registered application lists
             api: GET /appslists
 
         ### app_removelist()
         removelist:
-            action_help: Remove list from the repositories
+            action_help: Remove and forget about a given application list
             api: DELETE /appslists
             arguments:
                 name:

locales/en.json

@@ -28,11 +28,16 @@
     "app_unsupported_remote_type": "Unsupported remote type used for the app",
     "app_upgrade_failed": "Unable to upgrade {app:s}",
     "app_upgraded": "{app:s} has been upgraded",
-    "appslist_fetched": "The app list has been fetched",
+    "appslist_fetched": "The application list {appslist:s} has been fetched",
-    "appslist_removed": "The app list has been removed",
+    "appslist_removed": "The application list {appslist:s} has been removed",
-    "appslist_retrieve_error": "Unable to retrieve the remote app list: {error}",
+    "appslist_unknown": "Application list {appslist:s} unknown.",
-    "appslist_retrieve_bad_format": "Retrieved file is not a valid app list",
+    "appslist_retrieve_error": "Unable to retrieve the remote application list {appslist:s}: {error:s}",
-    "appslist_unknown": "Unknown app list",
+    "appslist_retrieve_bad_format": "Retrieved file for application list {appslist:s} is not a valid app list",
+    "appslist_name_already_tracked": "There is already a registered application list with name {name:s}.",
+    "appslist_url_already_tracked": "There is already a registered application list with url {url:s}.",
+    "appslist_migrating": "Migrating application list {appslist:s} ...",
+    "appslist_could_not_migrate": "Could not migrate app list {appslist:s} ! Unable to parse the url... The old cron job has been kept in {bkp_file:s}.",
+    "appslist_corrupted_json": "Could not load the application lists. It looks like {filename:s} is corrupted.",
     "ask_current_admin_password": "Current administration password",
     "ask_email": "Email address",
     "ask_firstname": "First name",

src/yunohost/app.py

@@ -34,6 +34,9 @@ import urlparse
 import errno
 import subprocess
 import requests
+import glob
+import pwd
+import grp
 from collections import OrderedDict
 
 from moulinette.core import MoulinetteError
@@ -46,9 +49,10 @@ logger = getActionLogger('yunohost.app')
 
 REPO_PATH         = '/var/cache/yunohost/repo'
 APPS_PATH         = '/usr/share/yunohost/apps'
-APPS_SETTING_PATH= '/etc/yunohost/apps/'
+APPS_SETTING_PATH = '/etc/yunohost/apps/'
 INSTALL_TMP       = '/var/cache/yunohost'
 APP_TMP_FOLDER    = INSTALL_TMP + '/from_file'
+APPSLISTS_JSON     = '/etc/yunohost/appslists.json'
 
 re_github_repo = re.compile(
     r'^(http[s]?://|git@)github.com[/:]'
@@ -65,66 +69,122 @@ def app_listlists():
     """
     List fetched lists
 
-
     """
-    list_list = []
-    try:
-        for filename in os.listdir(REPO_PATH):
-            if '.json' in filename:
-                list_list.append(filename[:len(filename)-5])
-    except OSError:
-        raise MoulinetteError(1, m18n.n('no_appslist_found'))
 
-    return { 'lists' : list_list }
+    # Migrate appslist system if needed
+    # XXX move to a migration when those are implemented
+    if _using_legacy_appslist_system():
+        _migrate_appslist_system()
+
+    # Get the list
+    appslist_list = _read_appslist_list()
+
+    return appslist_list
 
 
 def app_fetchlist(url=None, name=None):
     """
-    Fetch application list from app server
+    Fetch application list(s) from app server. By default, fetch all lists.
 
     Keyword argument:
-        name -- Name of the list (default yunohost)
+        name -- Name of the list
-        url -- URL of remote JSON list (default https://app.yunohost.org/official.json)
+        url -- URL of remote JSON list
-
     """
-    # Create app path if not exists
+    # If needed, create folder where actual appslists are stored
     if not os.path.exists(REPO_PATH):
         os.makedirs(REPO_PATH)
 
-    if url is None:
+    # Migrate appslist system if needed
-        url = 'https://app.yunohost.org/official.json'
+    # XXX move that to a migration once they are finished
-        name = 'yunohost'
+    if _using_legacy_appslist_system():
-    elif name is None:
+        _migrate_appslist_system()
+
+    # Read the list of appslist...
+    appslists = _read_appslist_list()
+
+    # Determine the list of appslist to be fetched
+    appslists_to_be_fetched = []
+
+    # If a url and and a name is given, try to register new list,
+    # the fetch only this list
+    if url is not None:
+        if name:
+            _register_new_appslist(url, name)
+            # Refresh the appslists dict
+            appslists = _read_appslist_list()
+            appslists_to_be_fetched = [name]
+        else:
             raise MoulinetteError(errno.EINVAL,
                                   m18n.n('custom_appslist_name_required'))
 
+    # If a name is given, look for an appslist with that name and fetch it
+    elif name is not None:
+        if name not in appslists.keys():
+            raise MoulinetteError(errno.EINVAL,
+                                  m18n.n('appslist_unknown', appslist=name))
+        else:
+            appslists_to_be_fetched = [name]
+
+    # Otherwise, fetch all lists
+    else:
+        appslists_to_be_fetched = appslists.keys()
+
+    # Fetch all appslists to be fetched
+    for name in appslists_to_be_fetched:
+
+        url = appslists[name]["url"]
+
+        logger.debug("Attempting to fetch list %s at %s" % (name, url))
+
         # Download file
         try:
-        applist_request = requests.get(url, timeout=30)
+            appslist_request = requests.get(url, timeout=30)
+        except requests.exceptions.SSLError:
+            logger.error(m18n.n('appslist_retrieve_error',
+                                appslist=name,
+                                error="SSL connection error"))
+            continue
         except Exception as e:
-        raise MoulinetteError(errno.EBADR, m18n.n('appslist_retrieve_error', error=str(e)))
+            logger.error(m18n.n('appslist_retrieve_error',
-
+                                appslist=name,
-    if (applist_request.status_code != 200):
+                                error=str(e)))
-        raise MoulinetteError(errno.EBADR, m18n.n('appslist_retrieve_error', error="404, not found"))
+            continue
+        if appslist_request.status_code != 200:
+            logger.error(m18n.n('appslist_retrieve_error',
+                                appslist=name,
+                                error="Server returned code %s " %
+                                str(appslist_request.status_code)))
+            continue
 
         # Validate app list format
-    # TODO / Possible improvement : better validation for app list (check that
+        # TODO / Possible improvement : better validation for app list (check
-    # json fields actually look like an app list and not any json file)
+        # that json fields actually look like an app list and not any json
-    applist = applist_request.text
+        # file)
+        appslist = appslist_request.text
         try:
-        json.loads(applist)
+            json.loads(appslist)
         except ValueError, e:
-        raise MoulinetteError(errno.EBADR, m18n.n('appslist_retrieve_bad_format'))
+            logger.error(m18n.n('appslist_retrieve_bad_format',
+                                appslist=name))
+            continue
 
         # Write app list to file
         list_file = '%s/%s.json' % (REPO_PATH, name)
+        try:
             with open(list_file, "w") as f:
-        f.write(applist)
+                f.write(appslist)
+        except Exception as e:
+            raise MoulinetteError(errno.EIO,
+                                  "Error while writing appslist %s: %s" %
+                                  (name, str(e)))
 
-    # Setup a cron job to re-fetch the list at midnight
+        now = int(time.time())
-    open("/etc/cron.d/yunohost-applist-%s" % name, "w").write('00 00 * * * root yunohost app fetchlist -u %s -n %s > /dev/null 2>&1\n' % (url, name))
+        appslists[name]["lastUpdate"] = now
 
-    logger.success(m18n.n('appslist_fetched'))
+        logger.success(m18n.n('appslist_fetched', appslist=name))
+
+    # Write updated list of appslist
+    _write_appslist_list(appslists)
 
 
 def app_removelist(name):
@@ -135,13 +195,22 @@ def app_removelist(name):
         name -- Name of the list to remove
 
     """
-    try:
+    appslists = _read_appslist_list()
-        os.remove('%s/%s.json' % (REPO_PATH, name))
-        os.remove("/etc/cron.d/yunohost-applist-%s" % name)
-    except OSError:
-        raise MoulinetteError(errno.ENOENT, m18n.n('appslist_unknown'))
 
-    logger.success(m18n.n('appslist_removed'))
+    # Make sure we know this appslist
+    if name not in appslists.keys():
+        raise MoulinetteError(errno.ENOENT, m18n.n('appslist_unknown', appslist=name))
+
+    # Remove json
+    json_path = '%s/%s.json' % (REPO_PATH, name)
+    if os.path.exists(json_path):
+        os.remove(json_path)
+
+    # Forget about this appslist
+    del appslists[name]
+    _write_appslist_list(appslists)
+
+    logger.success(m18n.n('appslist_removed', appslist=name))
 
 
 def app_list(filter=None, raw=False, installed=False, with_backup=False):
@@ -162,19 +231,18 @@ def app_list(filter=None, raw=False, installed=False, with_backup=False):
     app_dict = {}
     list_dict = {} if raw else []
 
-    try:
+    appslists = _read_appslist_list()
-        applists = app_listlists()['lists']
-        applists[0]
-    except (IOError, IndexError):
-        app_fetchlist()
-        applists = app_listlists()['lists']
 
-    # Construct a dictionnary of apps, based on known app lists
+    for appslist in appslists.keys():
-    for applist in applists:
+
-        with open(os.path.join(REPO_PATH, applist + '.json')) as json_list:
+        json_path = "%s/%s.json" % (REPO_PATH, appslist)
-            for app, info in json.load(json_list).items():
+        if not os.path.exists(json_path):
+            app_fetchlist(name=appslist)
+
+        with open(json_path) as json_list:
+            for app, info in json.loads(str(json_list.read())).items():
                 if app not in app_dict:
-                    info['repository'] = applist
+                    info['repository'] = appslist
                     app_dict[app] = info
 
     # Get app list from the app settings directory
@@ -1657,6 +1725,151 @@ def _parse_app_instance_name(app_instance_name):
     app_instance_nb = int(match.groupdict().get('appinstancenb')) if match.groupdict().get('appinstancenb') is not None else 1
     return (appid, app_instance_nb)
 
+
+def _using_legacy_appslist_system():
+    """
+    Return True if we're using the old fetchlist scheme.
+    This is determined by the presence of some cron job yunohost-appslist-foo
+    """
+
+    return glob.glob("/etc/cron.d/yunohost-appslist-*") != []
+
+
+def _migrate_appslist_system():
+    """
+    Migrate from the legacy fetchlist system to the new one
+    """
+    legacy_crons = glob.glob("/etc/cron.d/yunohost-appslist-*")
+
+    for cron_path in legacy_crons:
+        appslist_name = os.path.basename(cron_path).replace("yunohost-appslist-", "")
+        logger.info(m18n.n('appslist_migrating', appslist=appslist_name))
+
+        # Parse appslist url in cron
+        cron_file_content = open(cron_path).read().strip()
+        appslist_url_parse = re.search("-u (https?://[^ ]+)", cron_file_content)
+
+        # Abort if we did not find an url
+        if not appslist_url_parse or not appslist_url_parse.groups():
+            # Bkp the old cron job somewhere else
+            bkp_file = "/etc/yunohost/%s.oldlist.bkp" % appslist_name
+            os.rename(cron_path, bkp_file)
+            # Notice the user
+            logger.warning(m18n.n('appslist_could_not_migrate',
+                           appslist=appslist_name,
+                           bkp_file=bkp_file))
+        # Otherwise, register the list and remove the legacy cron
+        else:
+            appslist_url = appslist_url_parse.groups()[0]
+            try:
+                _register_new_appslist(appslist_url, appslist_name)
+            # Might get an exception if two legacy cron jobs conflict
+            # in terms of url...
+            except Exception as e:
+                logger.error(str(e))
+                # Bkp the old cron job somewhere else
+                bkp_file = "/etc/yunohost/%s.oldlist.bkp" % appslist_name
+                os.rename(cron_path, bkp_file)
+                # Notice the user
+                logger.warning(m18n.n('appslist_could_not_migrate',
+                               appslist=appslist_name,
+                               bkp_file=bkp_file))
+            else:
+                os.remove(cron_path)
+
+
+def _install_appslist_fetch_cron():
+
+    cron_job_file = "/etc/cron.daily/yunohost-fetch-appslists"
+
+    logger.debug("Installing appslist fetch cron job")
+
+    with open(cron_job_file, "w") as f:
+        f.write('#!/bin/bash\n\nyunohost app fetchlist > /dev/null 2>&1\n')
+
+    _set_permissions(cron_job_file, "root", "root", 0755)
+
+
+# FIXME - Duplicate from certificate.py, should be moved into a common helper
+# thing...
+def _set_permissions(path, user, group, permissions):
+    uid = pwd.getpwnam(user).pw_uid
+    gid = grp.getgrnam(group).gr_gid
+
+    os.chown(path, uid, gid)
+    os.chmod(path, permissions)
+
+
+def _read_appslist_list():
+    """
+    Read the json corresponding to the list of appslists
+    """
+
+    # If file does not exists yet, return empty dict
+    if not os.path.exists(APPSLISTS_JSON):
+        return {}
+
+    # Read file content
+    with open(APPSLISTS_JSON, "r") as f:
+        appslists_json = f.read()
+
+    # Parse json, throw exception if what we got from file is not a valid json
+    try:
+        appslists = json.loads(appslists_json)
+    except ValueError:
+        raise MoulinetteError(errno.EBADR,
+                              m18n.n('appslist_corrupted_json', filename=APPSLISTS_JSON))
+
+    return appslists
+
+
+def _write_appslist_list(appslist_lists):
+    """
+    Update the json containing list of appslists
+    """
+
+    # Write appslist list
+    try:
+        with open(APPSLISTS_JSON, "w") as f:
+            json.dump(appslist_lists, f)
+    except Exception as e:
+            raise MoulinetteError(errno.EIO,
+                                  "Error while writing list of appslist %s: %s" %
+                                  (APPSLISTS_JSON, str(e)))
+
+
+def _register_new_appslist(url, name):
+    """
+    Add a new appslist to be fetched regularly.
+    Raise an exception if url or name conflicts with an existing list.
+    """
+
+    appslist_list = _read_appslist_list()
+
+    # Check if name conflicts with an existing list
+    if name in appslist_list:
+        raise MoulinetteError(errno.EEXIST,
+                              m18n.n('appslist_name_already_tracked', name=name))
+
+    # Check if url conflicts with an existing list
+    known_appslist_urls = [appslist["url"] for _, appslist in appslist_list.items()]
+
+    if url in known_appslist_urls:
+        raise MoulinetteError(errno.EEXIST,
+                              m18n.n('appslist_url_already_tracked', url=url))
+
+    logger.debug("Registering new appslist %s at %s" % (name, url))
+
+    appslist_list[name] = {
+        "url": url,
+        "lastUpdate": None
+    }
+
+    _write_appslist_list(appslist_list)
+
+    _install_appslist_fetch_cron()
+
+
 def is_true(arg):
     """
     Convert a string into a boolean

src/yunohost/tests/test_appslist.py

@@ -0,0 +1,389 @@
+import os
+import pytest
+import requests
+import requests_mock
+import glob
+import time
+
+from moulinette.core import MoulinetteError
+
+from yunohost.app import app_fetchlist, app_removelist, app_listlists, _using_legacy_appslist_system, _migrate_appslist_system, _register_new_appslist
+
+URL_OFFICIAL_APP_LIST = "https://app.yunohost.org/official.json"
+REPO_PATH = '/var/cache/yunohost/repo'
+APPSLISTS_JSON = '/etc/yunohost/appslists.json'
+
+
+def setup_function(function):
+
+    # Clear all appslist
+    files = glob.glob(REPO_PATH+"/*")
+    for f in files:
+        os.remove(f)
+
+    # Clear appslist crons
+    files = glob.glob("/etc/cron.d/yunohost-appslist-*")
+    for f in files:
+        os.remove(f)
+
+    if os.path.exists("/etc/cron.daily/yunohost-fetch-appslists"):
+        os.remove("/etc/cron.daily/yunohost-fetch-appslists")
+
+    if os.path.exists(APPSLISTS_JSON):
+        os.remove(APPSLISTS_JSON)
+
+
+def teardown_function(function):
+    pass
+
+
+def cron_job_is_there():
+    r = os.system("run-parts -v --test /etc/cron.daily/ | grep yunohost-fetch-appslists")
+    return r == 0
+
+
+###############################################################################
+#   Test listing of appslists and registering of appslists                      #
+###############################################################################
+
+
+def test_appslist_list_empty():
+    """
+    Calling app_listlists() with no registered list should return empty dict
+    """
+
+    assert app_listlists() == {}
+
+
+def test_appslist_list_register():
+    """
+    Register a new list
+    """
+
+    # Assume we're starting with an empty app list
+    assert app_listlists() == {}
+
+    # Register a new dummy list
+    _register_new_appslist("https://lol.com/appslist.json", "dummy")
+
+    appslist_dict = app_listlists()
+    assert "dummy" in appslist_dict.keys()
+    assert appslist_dict["dummy"]["url"] == "https://lol.com/appslist.json"
+
+    assert cron_job_is_there()
+
+
+def test_appslist_list_register_conflict_name():
+    """
+    Attempt to register a new list with conflicting name
+    """
+
+    _register_new_appslist("https://lol.com/appslist.json", "dummy")
+    with pytest.raises(MoulinetteError):
+        _register_new_appslist("https://lol.com/appslist2.json", "dummy")
+
+    appslist_dict = app_listlists()
+
+    assert "dummy" in appslist_dict.keys()
+    assert "dummy2" not in appslist_dict.keys()
+
+
+def test_appslist_list_register_conflict_url():
+    """
+    Attempt to register a new list with conflicting url
+    """
+
+    _register_new_appslist("https://lol.com/appslist.json", "dummy")
+    with pytest.raises(MoulinetteError):
+        _register_new_appslist("https://lol.com/appslist.json", "plopette")
+
+    appslist_dict = app_listlists()
+
+    assert "dummy" in appslist_dict.keys()
+    assert "plopette" not in appslist_dict.keys()
+
+
+###############################################################################
+#   Test fetching of appslists                                                 #
+###############################################################################
+
+
+def test_appslist_fetch():
+    """
+    Do a fetchlist and test the .json got updated.
+    """
+    assert app_listlists() == {}
+
+    _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
+
+    with requests_mock.Mocker() as m:
+
+        # Mock the server response with a valid (well, empty, yep) json
+        m.register_uri("GET", URL_OFFICIAL_APP_LIST, text='{ }')
+
+        official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"]
+        app_fetchlist()
+        new_official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"]
+
+    assert new_official_lastUpdate > official_lastUpdate
+
+
+def test_appslist_fetch_single_appslist():
+    """
+    Register several lists but only fetch one. Check only one got updated.
+    """
+
+    assert app_listlists() == {}
+    _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
+    _register_new_appslist("https://lol.com/appslist.json", "dummy")
+
+    time.sleep(1)
+
+    with requests_mock.Mocker() as m:
+
+        # Mock the server response with a valid (well, empty, yep) json
+        m.register_uri("GET", URL_OFFICIAL_APP_LIST, text='{ }')
+
+        official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"]
+        dummy_lastUpdate = app_listlists()["dummy"]["lastUpdate"]
+        app_fetchlist(name="yunohost")
+        new_official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"]
+        new_dummy_lastUpdate = app_listlists()["dummy"]["lastUpdate"]
+
+    assert new_official_lastUpdate > official_lastUpdate
+    assert new_dummy_lastUpdate == dummy_lastUpdate
+
+
+def test_appslist_fetch_unknownlist():
+    """
+    Attempt to fetch an unknown list
+    """
+
+    assert app_listlists() == {}
+
+    with pytest.raises(MoulinetteError):
+        app_fetchlist(name="swag")
+
+
+def test_appslist_fetch_url_but_no_name():
+    """
+    Do a fetchlist with url given, but no name given
+    """
+
+    with pytest.raises(MoulinetteError):
+        app_fetchlist(url=URL_OFFICIAL_APP_LIST)
+
+
+def test_appslist_fetch_badurl():
+    """
+    Do a fetchlist with a bad url
+    """
+
+    app_fetchlist(url="https://not.a.valid.url/plop.json", name="plop")
+
+
+def test_appslist_fetch_badfile():
+    """
+    Do a fetchlist and mock a response with a bad json
+    """
+    assert app_listlists() == {}
+
+    _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
+
+    with requests_mock.Mocker() as m:
+
+        m.register_uri("GET", URL_OFFICIAL_APP_LIST, text='{ not json lol }')
+
+        app_fetchlist()
+
+
+def test_appslist_fetch_404():
+    """
+    Do a fetchlist and mock a 404 response
+    """
+    assert app_listlists() == {}
+
+    _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
+
+    with requests_mock.Mocker() as m:
+
+        m.register_uri("GET", URL_OFFICIAL_APP_LIST, status_code=404)
+
+        app_fetchlist()
+
+
+def test_appslist_fetch_sslerror():
+    """
+    Do a fetchlist and mock an SSL error
+    """
+    assert app_listlists() == {}
+
+    _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
+
+    with requests_mock.Mocker() as m:
+
+        m.register_uri("GET", URL_OFFICIAL_APP_LIST,
+                       exc=requests.exceptions.SSLError)
+
+        app_fetchlist()
+
+
+def test_appslist_fetch_timeout():
+    """
+    Do a fetchlist and mock a timeout
+    """
+    assert app_listlists() == {}
+
+    _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
+
+    with requests_mock.Mocker() as m:
+
+        m.register_uri("GET", URL_OFFICIAL_APP_LIST,
+                       exc=requests.exceptions.ConnectTimeout)
+
+        app_fetchlist()
+
+
+###############################################################################
+#   Test remove of appslist                                                    #
+###############################################################################
+
+
+def test_appslist_remove():
+    """
+    Register a new appslist, then remove it
+    """
+
+    # Assume we're starting with an empty app list
+    assert app_listlists() == {}
+
+    # Register a new dummy list
+    _register_new_appslist("https://lol.com/appslist.json", "dummy")
+    app_removelist("dummy")
+
+    # Should end up with no list registered
+    assert app_listlists() == {}
+
+
+def test_appslist_remove_unknown():
+    """
+    Attempt to remove an unknown list
+    """
+
+    with pytest.raises(MoulinetteError):
+        app_removelist("dummy")
+
+
+###############################################################################
+#   Test migration from legacy appslist system                                 #
+###############################################################################
+
+
+def add_legacy_cron(name, url):
+    with open("/etc/cron.d/yunohost-appslist-%s" % name, "w") as f:
+        f.write('00 00 * * * root yunohost app fetchlist -u %s -n %s > /dev/null 2>&1\n' % (url, name))
+
+
+def test_appslist_check_using_legacy_system_testFalse():
+    """
+    If no legacy cron job is there, the check should return False
+    """
+    assert glob.glob("/etc/cron.d/yunohost-appslist-*") == []
+    assert _using_legacy_appslist_system() is False
+
+
+def test_appslist_check_using_legacy_system_testTrue():
+    """
+    If there's a legacy cron job, the check should return True
+    """
+    assert glob.glob("/etc/cron.d/yunohost-appslist-*") == []
+    add_legacy_cron("yunohost", "https://app.yunohost.org/official.json")
+    assert _using_legacy_appslist_system() is True
+
+
+def test_appslist_system_migration():
+    """
+    Test that legacy cron jobs get migrated correctly when calling app_listlists
+    """
+
+    # Start with no legacy cron, no appslist registered
+    assert glob.glob("/etc/cron.d/yunohost-appslist-*") == []
+    assert app_listlists() == {}
+    assert not os.path.exists("/etc/cron.daily/yunohost-fetch-appslists")
+
+    # Add a few legacy crons
+    add_legacy_cron("yunohost", "https://app.yunohost.org/official.json")
+    add_legacy_cron("dummy", "https://swiggitty.swaggy.lol/yolo.json")
+
+    # Migrate
+    assert _using_legacy_appslist_system() is True
+    _migrate_appslist_system()
+    assert _using_legacy_appslist_system() is False
+
+    # No legacy cron job should remain
+    assert glob.glob("/etc/cron.d/yunohost-appslist-*") == []
+
+    # Check they are in app_listlists anyway
+    appslist_dict = app_listlists()
+    assert "yunohost" in appslist_dict.keys()
+    assert appslist_dict["yunohost"]["url"] == "https://app.yunohost.org/official.json"
+    assert "dummy" in appslist_dict.keys()
+    assert appslist_dict["dummy"]["url"] == "https://swiggitty.swaggy.lol/yolo.json"
+
+    assert cron_job_is_there()
+
+
+def test_appslist_system_migration_badcron():
+    """
+    Test the migration on a bad legacy cron (no url found inside cron job)
+    """
+
+    # Start with no legacy cron, no appslist registered
+    assert glob.glob("/etc/cron.d/yunohost-appslist-*") == []
+    assert app_listlists() == {}
+    assert not os.path.exists("/etc/cron.daily/yunohost-fetch-appslists")
+
+    # Add a "bad" legacy cron
+    add_legacy_cron("wtflist", "ftp://the.fuck.is.this")
+
+    # Migrate
+    assert _using_legacy_appslist_system() is True
+    _migrate_appslist_system()
+    assert _using_legacy_appslist_system() is False
+
+    # No legacy cron should remain, but it should be backuped in /etc/yunohost
+    assert glob.glob("/etc/cron.d/yunohost-appslist-*") == []
+    assert os.path.exists("/etc/yunohost/wtflist.oldlist.bkp")
+
+    # Appslist should still be empty
+    assert app_listlists() == {}
+
+
+def test_appslist_system_migration_conflict():
+    """
+    Test migration of conflicting cron job (in terms of url)
+    """
+
+    # Start with no legacy cron, no appslist registered
+    assert glob.glob("/etc/cron.d/yunohost-appslist-*") == []
+    assert app_listlists() == {}
+    assert not os.path.exists("/etc/cron.daily/yunohost-fetch-appslists")
+
+    # Add a few legacy crons
+    add_legacy_cron("yunohost", "https://app.yunohost.org/official.json")
+    add_legacy_cron("dummy", "https://app.yunohost.org/official.json")
+
+    # Migrate
+    assert _using_legacy_appslist_system() is True
+    _migrate_appslist_system()
+    assert _using_legacy_appslist_system() is False
+
+    # No legacy cron job should remain
+    assert glob.glob("/etc/cron.d/yunohost-appslist-*") == []
+
+    # Only one among "dummy" and "yunohost" should be listed
+    appslist_dict = app_listlists()
+    assert (len(appslist_dict.keys()) == 1)
+    assert ("dummy" in appslist_dict.keys()) or ("yunohost" in appslist_dict.keys())
+
+    assert cron_job_is_there()

src/yunohost/tools.py

@@ -38,7 +38,7 @@ import apt.progress
 
 from moulinette.core import MoulinetteError, init_authenticator
 from moulinette.utils.log import getActionLogger
-from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, app_list
+from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, app_list, _install_appslist_fetch_cron
 from yunohost.domain import domain_add, domain_list, get_public_ip, _get_maindomain, _set_maindomain
 from yunohost.dyndns import dyndns_subscribe
 from yunohost.firewall import firewall_upnp
@@ -327,6 +327,15 @@ def tools_postinstall(domain, password, ignore_dyndns=False):
     # Enable UPnP silently and reload firewall
     firewall_upnp('enable', no_refresh=True)
 
+    # Setup the default official app list with cron job
+    try:
+        app_fetchlist(name="yunohost",
+                      url="https://app.yunohost.org/official.json")
+    except Exception as e:
+        logger.warning(str(e))
+
+    _install_appslist_fetch_cron()
+
     os.system('touch /etc/yunohost/installed')
 
     # Enable and start YunoHost firewall at boot time