From a4c487a0aad96e1fe69edec6cc917f2e130f247f Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Thu, 6 Apr 2017 22:21:25 +0200 Subject: [PATCH] [enh] Refactor applist management (#160) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [mod] directly use python to retreive json list * [enh] app_fetchlist fetch all app_list by default * [fix] name variable doesn't exists here * [fix] re returns None when there is not matchs * [enh] app_fetchlist fetch all app_list by default * Some cleaning for better readability * Simpler variable name * Prepare a function that register lists to be fetched * Skeletong for applist system migration * Add implementation of migration system with tests * Refactorize app_fetchlist * Misc fixes + adding test for single app fetching * Fixing a few issues + test removelist * Adding fetchlist and cron install during postinstall * Adding debug messages * Adding particular exception for SSL connection error * Update actionmap help * We don't use urlretrieve * Clean tests, some description were bad * [mod] some cleaning * Moving to a .json file to store lists url + adjusting tests * Adding missing string in locale * Moving exception to logger.error when fetching fails * Adding name of applist in error messages * Fixing cron job stuff + adding proper tests * Using None instead of -1 for applist lastupdate * Handling exceptions when writing applist files * More exception handling... * [mod] pep8 * Updating test for migration of conflicting lists * More general error when return code is not 200 * [enh] Improve app_fetchlist help. * [fix] Use appslist instead of applist. * [fix] Consistent user string for translation. --- data/actionsmap/yunohost.yml | 14 +- locales/en.json | 15 +- src/yunohost/app.py | 341 +++++++++++++++++++----- src/yunohost/tests/test_appslist.py | 389 ++++++++++++++++++++++++++++ src/yunohost/tools.py | 11 +- 5 files changed, 693 insertions(+), 77 deletions(-) create mode 100644 src/yunohost/tests/test_appslist.py diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index c37a2c2f1..8d549ebb3 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -391,28 +391,28 @@ app: ### app_fetchlist() fetchlist: - action_help: Fetch application list from app server + action_help: Fetch application lists from app servers, or register a new one. api: PUT /appslists arguments: - -u: - full: --url - help: URL of remote JSON list (default https://app.yunohost.org/official.json) -n: full: --name - help: Name of the list (default yunohost) + help: Name of the list to fetch (fetches all registered lists if empty) extra: pattern: &pattern_listname - !!str ^[a-z0-9_]+$ - "pattern_listname" + -u: + full: --url + help: URL of a new application list to register. To be specified with -n. ### app_listlists() listlists: - action_help: List fetched lists + action_help: List registered application lists api: GET /appslists ### app_removelist() removelist: - action_help: Remove list from the repositories + action_help: Remove and forget about a given application list api: DELETE /appslists arguments: name: diff --git a/locales/en.json b/locales/en.json index 9c3d1ce76..b0dad75b4 100644 --- a/locales/en.json +++ b/locales/en.json @@ -28,11 +28,16 @@ "app_unsupported_remote_type": "Unsupported remote type used for the app", "app_upgrade_failed": "Unable to upgrade {app:s}", "app_upgraded": "{app:s} has been upgraded", - "appslist_fetched": "The app list has been fetched", - "appslist_removed": "The app list has been removed", - "appslist_retrieve_error": "Unable to retrieve the remote app list: {error}", - "appslist_retrieve_bad_format": "Retrieved file is not a valid app list", - "appslist_unknown": "Unknown app list", + "appslist_fetched": "The application list {appslist:s} has been fetched", + "appslist_removed": "The application list {appslist:s} has been removed", + "appslist_unknown": "Application list {appslist:s} unknown.", + "appslist_retrieve_error": "Unable to retrieve the remote application list {appslist:s}: {error:s}", + "appslist_retrieve_bad_format": "Retrieved file for application list {appslist:s} is not a valid app list", + "appslist_name_already_tracked": "There is already a registered application list with name {name:s}.", + "appslist_url_already_tracked": "There is already a registered application list with url {url:s}.", + "appslist_migrating": "Migrating application list {appslist:s} ...", + "appslist_could_not_migrate": "Could not migrate app list {appslist:s} ! Unable to parse the url... The old cron job has been kept in {bkp_file:s}.", + "appslist_corrupted_json": "Could not load the application lists. It looks like {filename:s} is corrupted.", "ask_current_admin_password": "Current administration password", "ask_email": "Email address", "ask_firstname": "First name", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 226fbdb93..66e5785da 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -34,6 +34,9 @@ import urlparse import errno import subprocess import requests +import glob +import pwd +import grp from collections import OrderedDict from moulinette.core import MoulinetteError @@ -44,11 +47,12 @@ from yunohost.utils import packages logger = getActionLogger('yunohost.app') -REPO_PATH = '/var/cache/yunohost/repo' -APPS_PATH = '/usr/share/yunohost/apps' -APPS_SETTING_PATH= '/etc/yunohost/apps/' -INSTALL_TMP = '/var/cache/yunohost' -APP_TMP_FOLDER = INSTALL_TMP + '/from_file' +REPO_PATH = '/var/cache/yunohost/repo' +APPS_PATH = '/usr/share/yunohost/apps' +APPS_SETTING_PATH = '/etc/yunohost/apps/' +INSTALL_TMP = '/var/cache/yunohost' +APP_TMP_FOLDER = INSTALL_TMP + '/from_file' +APPSLISTS_JSON = '/etc/yunohost/appslists.json' re_github_repo = re.compile( r'^(http[s]?://|git@)github.com[/:]' @@ -65,66 +69,122 @@ def app_listlists(): """ List fetched lists - """ - list_list = [] - try: - for filename in os.listdir(REPO_PATH): - if '.json' in filename: - list_list.append(filename[:len(filename)-5]) - except OSError: - raise MoulinetteError(1, m18n.n('no_appslist_found')) - return { 'lists' : list_list } + # Migrate appslist system if needed + # XXX move to a migration when those are implemented + if _using_legacy_appslist_system(): + _migrate_appslist_system() + + # Get the list + appslist_list = _read_appslist_list() + + return appslist_list def app_fetchlist(url=None, name=None): """ - Fetch application list from app server + Fetch application list(s) from app server. By default, fetch all lists. Keyword argument: - name -- Name of the list (default yunohost) - url -- URL of remote JSON list (default https://app.yunohost.org/official.json) - + name -- Name of the list + url -- URL of remote JSON list """ - # Create app path if not exists + # If needed, create folder where actual appslists are stored if not os.path.exists(REPO_PATH): os.makedirs(REPO_PATH) - if url is None: - url = 'https://app.yunohost.org/official.json' - name = 'yunohost' - elif name is None: - raise MoulinetteError(errno.EINVAL, - m18n.n('custom_appslist_name_required')) + # Migrate appslist system if needed + # XXX move that to a migration once they are finished + if _using_legacy_appslist_system(): + _migrate_appslist_system() - # Download file - try: - applist_request = requests.get(url, timeout=30) - except Exception as e: - raise MoulinetteError(errno.EBADR, m18n.n('appslist_retrieve_error', error=str(e))) + # Read the list of appslist... + appslists = _read_appslist_list() - if (applist_request.status_code != 200): - raise MoulinetteError(errno.EBADR, m18n.n('appslist_retrieve_error', error="404, not found")) + # Determine the list of appslist to be fetched + appslists_to_be_fetched = [] - # Validate app list format - # TODO / Possible improvement : better validation for app list (check that - # json fields actually look like an app list and not any json file) - applist = applist_request.text - try: - json.loads(applist) - except ValueError, e: - raise MoulinetteError(errno.EBADR, m18n.n('appslist_retrieve_bad_format')) + # If a url and and a name is given, try to register new list, + # the fetch only this list + if url is not None: + if name: + _register_new_appslist(url, name) + # Refresh the appslists dict + appslists = _read_appslist_list() + appslists_to_be_fetched = [name] + else: + raise MoulinetteError(errno.EINVAL, + m18n.n('custom_appslist_name_required')) - # Write app list to file - list_file = '%s/%s.json' % (REPO_PATH, name) - with open(list_file, "w") as f: - f.write(applist) + # If a name is given, look for an appslist with that name and fetch it + elif name is not None: + if name not in appslists.keys(): + raise MoulinetteError(errno.EINVAL, + m18n.n('appslist_unknown', appslist=name)) + else: + appslists_to_be_fetched = [name] - # Setup a cron job to re-fetch the list at midnight - open("/etc/cron.d/yunohost-applist-%s" % name, "w").write('00 00 * * * root yunohost app fetchlist -u %s -n %s > /dev/null 2>&1\n' % (url, name)) + # Otherwise, fetch all lists + else: + appslists_to_be_fetched = appslists.keys() - logger.success(m18n.n('appslist_fetched')) + # Fetch all appslists to be fetched + for name in appslists_to_be_fetched: + + url = appslists[name]["url"] + + logger.debug("Attempting to fetch list %s at %s" % (name, url)) + + # Download file + try: + appslist_request = requests.get(url, timeout=30) + except requests.exceptions.SSLError: + logger.error(m18n.n('appslist_retrieve_error', + appslist=name, + error="SSL connection error")) + continue + except Exception as e: + logger.error(m18n.n('appslist_retrieve_error', + appslist=name, + error=str(e))) + continue + if appslist_request.status_code != 200: + logger.error(m18n.n('appslist_retrieve_error', + appslist=name, + error="Server returned code %s " % + str(appslist_request.status_code))) + continue + + # Validate app list format + # TODO / Possible improvement : better validation for app list (check + # that json fields actually look like an app list and not any json + # file) + appslist = appslist_request.text + try: + json.loads(appslist) + except ValueError, e: + logger.error(m18n.n('appslist_retrieve_bad_format', + appslist=name)) + continue + + # Write app list to file + list_file = '%s/%s.json' % (REPO_PATH, name) + try: + with open(list_file, "w") as f: + f.write(appslist) + except Exception as e: + raise MoulinetteError(errno.EIO, + "Error while writing appslist %s: %s" % + (name, str(e))) + + now = int(time.time()) + appslists[name]["lastUpdate"] = now + + logger.success(m18n.n('appslist_fetched', appslist=name)) + + # Write updated list of appslist + _write_appslist_list(appslists) def app_removelist(name): @@ -135,13 +195,22 @@ def app_removelist(name): name -- Name of the list to remove """ - try: - os.remove('%s/%s.json' % (REPO_PATH, name)) - os.remove("/etc/cron.d/yunohost-applist-%s" % name) - except OSError: - raise MoulinetteError(errno.ENOENT, m18n.n('appslist_unknown')) + appslists = _read_appslist_list() - logger.success(m18n.n('appslist_removed')) + # Make sure we know this appslist + if name not in appslists.keys(): + raise MoulinetteError(errno.ENOENT, m18n.n('appslist_unknown', appslist=name)) + + # Remove json + json_path = '%s/%s.json' % (REPO_PATH, name) + if os.path.exists(json_path): + os.remove(json_path) + + # Forget about this appslist + del appslists[name] + _write_appslist_list(appslists) + + logger.success(m18n.n('appslist_removed', appslist=name)) def app_list(filter=None, raw=False, installed=False, with_backup=False): @@ -162,19 +231,18 @@ def app_list(filter=None, raw=False, installed=False, with_backup=False): app_dict = {} list_dict = {} if raw else [] - try: - applists = app_listlists()['lists'] - applists[0] - except (IOError, IndexError): - app_fetchlist() - applists = app_listlists()['lists'] + appslists = _read_appslist_list() - # Construct a dictionnary of apps, based on known app lists - for applist in applists: - with open(os.path.join(REPO_PATH, applist + '.json')) as json_list: - for app, info in json.load(json_list).items(): + for appslist in appslists.keys(): + + json_path = "%s/%s.json" % (REPO_PATH, appslist) + if not os.path.exists(json_path): + app_fetchlist(name=appslist) + + with open(json_path) as json_list: + for app, info in json.loads(str(json_list.read())).items(): if app not in app_dict: - info['repository'] = applist + info['repository'] = appslist app_dict[app] = info # Get app list from the app settings directory @@ -1657,6 +1725,151 @@ def _parse_app_instance_name(app_instance_name): app_instance_nb = int(match.groupdict().get('appinstancenb')) if match.groupdict().get('appinstancenb') is not None else 1 return (appid, app_instance_nb) + +def _using_legacy_appslist_system(): + """ + Return True if we're using the old fetchlist scheme. + This is determined by the presence of some cron job yunohost-appslist-foo + """ + + return glob.glob("/etc/cron.d/yunohost-appslist-*") != [] + + +def _migrate_appslist_system(): + """ + Migrate from the legacy fetchlist system to the new one + """ + legacy_crons = glob.glob("/etc/cron.d/yunohost-appslist-*") + + for cron_path in legacy_crons: + appslist_name = os.path.basename(cron_path).replace("yunohost-appslist-", "") + logger.info(m18n.n('appslist_migrating', appslist=appslist_name)) + + # Parse appslist url in cron + cron_file_content = open(cron_path).read().strip() + appslist_url_parse = re.search("-u (https?://[^ ]+)", cron_file_content) + + # Abort if we did not find an url + if not appslist_url_parse or not appslist_url_parse.groups(): + # Bkp the old cron job somewhere else + bkp_file = "/etc/yunohost/%s.oldlist.bkp" % appslist_name + os.rename(cron_path, bkp_file) + # Notice the user + logger.warning(m18n.n('appslist_could_not_migrate', + appslist=appslist_name, + bkp_file=bkp_file)) + # Otherwise, register the list and remove the legacy cron + else: + appslist_url = appslist_url_parse.groups()[0] + try: + _register_new_appslist(appslist_url, appslist_name) + # Might get an exception if two legacy cron jobs conflict + # in terms of url... + except Exception as e: + logger.error(str(e)) + # Bkp the old cron job somewhere else + bkp_file = "/etc/yunohost/%s.oldlist.bkp" % appslist_name + os.rename(cron_path, bkp_file) + # Notice the user + logger.warning(m18n.n('appslist_could_not_migrate', + appslist=appslist_name, + bkp_file=bkp_file)) + else: + os.remove(cron_path) + + +def _install_appslist_fetch_cron(): + + cron_job_file = "/etc/cron.daily/yunohost-fetch-appslists" + + logger.debug("Installing appslist fetch cron job") + + with open(cron_job_file, "w") as f: + f.write('#!/bin/bash\n\nyunohost app fetchlist > /dev/null 2>&1\n') + + _set_permissions(cron_job_file, "root", "root", 0755) + + +# FIXME - Duplicate from certificate.py, should be moved into a common helper +# thing... +def _set_permissions(path, user, group, permissions): + uid = pwd.getpwnam(user).pw_uid + gid = grp.getgrnam(group).gr_gid + + os.chown(path, uid, gid) + os.chmod(path, permissions) + + +def _read_appslist_list(): + """ + Read the json corresponding to the list of appslists + """ + + # If file does not exists yet, return empty dict + if not os.path.exists(APPSLISTS_JSON): + return {} + + # Read file content + with open(APPSLISTS_JSON, "r") as f: + appslists_json = f.read() + + # Parse json, throw exception if what we got from file is not a valid json + try: + appslists = json.loads(appslists_json) + except ValueError: + raise MoulinetteError(errno.EBADR, + m18n.n('appslist_corrupted_json', filename=APPSLISTS_JSON)) + + return appslists + + +def _write_appslist_list(appslist_lists): + """ + Update the json containing list of appslists + """ + + # Write appslist list + try: + with open(APPSLISTS_JSON, "w") as f: + json.dump(appslist_lists, f) + except Exception as e: + raise MoulinetteError(errno.EIO, + "Error while writing list of appslist %s: %s" % + (APPSLISTS_JSON, str(e))) + + +def _register_new_appslist(url, name): + """ + Add a new appslist to be fetched regularly. + Raise an exception if url or name conflicts with an existing list. + """ + + appslist_list = _read_appslist_list() + + # Check if name conflicts with an existing list + if name in appslist_list: + raise MoulinetteError(errno.EEXIST, + m18n.n('appslist_name_already_tracked', name=name)) + + # Check if url conflicts with an existing list + known_appslist_urls = [appslist["url"] for _, appslist in appslist_list.items()] + + if url in known_appslist_urls: + raise MoulinetteError(errno.EEXIST, + m18n.n('appslist_url_already_tracked', url=url)) + + logger.debug("Registering new appslist %s at %s" % (name, url)) + + appslist_list[name] = { + "url": url, + "lastUpdate": None + } + + _write_appslist_list(appslist_list) + + _install_appslist_fetch_cron() + + def is_true(arg): """ Convert a string into a boolean diff --git a/src/yunohost/tests/test_appslist.py b/src/yunohost/tests/test_appslist.py new file mode 100644 index 000000000..4750e32a8 --- /dev/null +++ b/src/yunohost/tests/test_appslist.py @@ -0,0 +1,389 @@ +import os +import pytest +import requests +import requests_mock +import glob +import time + +from moulinette.core import MoulinetteError + +from yunohost.app import app_fetchlist, app_removelist, app_listlists, _using_legacy_appslist_system, _migrate_appslist_system, _register_new_appslist + +URL_OFFICIAL_APP_LIST = "https://app.yunohost.org/official.json" +REPO_PATH = '/var/cache/yunohost/repo' +APPSLISTS_JSON = '/etc/yunohost/appslists.json' + + +def setup_function(function): + + # Clear all appslist + files = glob.glob(REPO_PATH+"/*") + for f in files: + os.remove(f) + + # Clear appslist crons + files = glob.glob("/etc/cron.d/yunohost-appslist-*") + for f in files: + os.remove(f) + + if os.path.exists("/etc/cron.daily/yunohost-fetch-appslists"): + os.remove("/etc/cron.daily/yunohost-fetch-appslists") + + if os.path.exists(APPSLISTS_JSON): + os.remove(APPSLISTS_JSON) + + +def teardown_function(function): + pass + + +def cron_job_is_there(): + r = os.system("run-parts -v --test /etc/cron.daily/ | grep yunohost-fetch-appslists") + return r == 0 + + +############################################################################### +# Test listing of appslists and registering of appslists # +############################################################################### + + +def test_appslist_list_empty(): + """ + Calling app_listlists() with no registered list should return empty dict + """ + + assert app_listlists() == {} + + +def test_appslist_list_register(): + """ + Register a new list + """ + + # Assume we're starting with an empty app list + assert app_listlists() == {} + + # Register a new dummy list + _register_new_appslist("https://lol.com/appslist.json", "dummy") + + appslist_dict = app_listlists() + assert "dummy" in appslist_dict.keys() + assert appslist_dict["dummy"]["url"] == "https://lol.com/appslist.json" + + assert cron_job_is_there() + + +def test_appslist_list_register_conflict_name(): + """ + Attempt to register a new list with conflicting name + """ + + _register_new_appslist("https://lol.com/appslist.json", "dummy") + with pytest.raises(MoulinetteError): + _register_new_appslist("https://lol.com/appslist2.json", "dummy") + + appslist_dict = app_listlists() + + assert "dummy" in appslist_dict.keys() + assert "dummy2" not in appslist_dict.keys() + + +def test_appslist_list_register_conflict_url(): + """ + Attempt to register a new list with conflicting url + """ + + _register_new_appslist("https://lol.com/appslist.json", "dummy") + with pytest.raises(MoulinetteError): + _register_new_appslist("https://lol.com/appslist.json", "plopette") + + appslist_dict = app_listlists() + + assert "dummy" in appslist_dict.keys() + assert "plopette" not in appslist_dict.keys() + + +############################################################################### +# Test fetching of appslists # +############################################################################### + + +def test_appslist_fetch(): + """ + Do a fetchlist and test the .json got updated. + """ + assert app_listlists() == {} + + _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost") + + with requests_mock.Mocker() as m: + + # Mock the server response with a valid (well, empty, yep) json + m.register_uri("GET", URL_OFFICIAL_APP_LIST, text='{ }') + + official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"] + app_fetchlist() + new_official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"] + + assert new_official_lastUpdate > official_lastUpdate + + +def test_appslist_fetch_single_appslist(): + """ + Register several lists but only fetch one. Check only one got updated. + """ + + assert app_listlists() == {} + _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost") + _register_new_appslist("https://lol.com/appslist.json", "dummy") + + time.sleep(1) + + with requests_mock.Mocker() as m: + + # Mock the server response with a valid (well, empty, yep) json + m.register_uri("GET", URL_OFFICIAL_APP_LIST, text='{ }') + + official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"] + dummy_lastUpdate = app_listlists()["dummy"]["lastUpdate"] + app_fetchlist(name="yunohost") + new_official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"] + new_dummy_lastUpdate = app_listlists()["dummy"]["lastUpdate"] + + assert new_official_lastUpdate > official_lastUpdate + assert new_dummy_lastUpdate == dummy_lastUpdate + + +def test_appslist_fetch_unknownlist(): + """ + Attempt to fetch an unknown list + """ + + assert app_listlists() == {} + + with pytest.raises(MoulinetteError): + app_fetchlist(name="swag") + + +def test_appslist_fetch_url_but_no_name(): + """ + Do a fetchlist with url given, but no name given + """ + + with pytest.raises(MoulinetteError): + app_fetchlist(url=URL_OFFICIAL_APP_LIST) + + +def test_appslist_fetch_badurl(): + """ + Do a fetchlist with a bad url + """ + + app_fetchlist(url="https://not.a.valid.url/plop.json", name="plop") + + +def test_appslist_fetch_badfile(): + """ + Do a fetchlist and mock a response with a bad json + """ + assert app_listlists() == {} + + _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost") + + with requests_mock.Mocker() as m: + + m.register_uri("GET", URL_OFFICIAL_APP_LIST, text='{ not json lol }') + + app_fetchlist() + + +def test_appslist_fetch_404(): + """ + Do a fetchlist and mock a 404 response + """ + assert app_listlists() == {} + + _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost") + + with requests_mock.Mocker() as m: + + m.register_uri("GET", URL_OFFICIAL_APP_LIST, status_code=404) + + app_fetchlist() + + +def test_appslist_fetch_sslerror(): + """ + Do a fetchlist and mock an SSL error + """ + assert app_listlists() == {} + + _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost") + + with requests_mock.Mocker() as m: + + m.register_uri("GET", URL_OFFICIAL_APP_LIST, + exc=requests.exceptions.SSLError) + + app_fetchlist() + + +def test_appslist_fetch_timeout(): + """ + Do a fetchlist and mock a timeout + """ + assert app_listlists() == {} + + _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost") + + with requests_mock.Mocker() as m: + + m.register_uri("GET", URL_OFFICIAL_APP_LIST, + exc=requests.exceptions.ConnectTimeout) + + app_fetchlist() + + +############################################################################### +# Test remove of appslist # +############################################################################### + + +def test_appslist_remove(): + """ + Register a new appslist, then remove it + """ + + # Assume we're starting with an empty app list + assert app_listlists() == {} + + # Register a new dummy list + _register_new_appslist("https://lol.com/appslist.json", "dummy") + app_removelist("dummy") + + # Should end up with no list registered + assert app_listlists() == {} + + +def test_appslist_remove_unknown(): + """ + Attempt to remove an unknown list + """ + + with pytest.raises(MoulinetteError): + app_removelist("dummy") + + +############################################################################### +# Test migration from legacy appslist system # +############################################################################### + + +def add_legacy_cron(name, url): + with open("/etc/cron.d/yunohost-appslist-%s" % name, "w") as f: + f.write('00 00 * * * root yunohost app fetchlist -u %s -n %s > /dev/null 2>&1\n' % (url, name)) + + +def test_appslist_check_using_legacy_system_testFalse(): + """ + If no legacy cron job is there, the check should return False + """ + assert glob.glob("/etc/cron.d/yunohost-appslist-*") == [] + assert _using_legacy_appslist_system() is False + + +def test_appslist_check_using_legacy_system_testTrue(): + """ + If there's a legacy cron job, the check should return True + """ + assert glob.glob("/etc/cron.d/yunohost-appslist-*") == [] + add_legacy_cron("yunohost", "https://app.yunohost.org/official.json") + assert _using_legacy_appslist_system() is True + + +def test_appslist_system_migration(): + """ + Test that legacy cron jobs get migrated correctly when calling app_listlists + """ + + # Start with no legacy cron, no appslist registered + assert glob.glob("/etc/cron.d/yunohost-appslist-*") == [] + assert app_listlists() == {} + assert not os.path.exists("/etc/cron.daily/yunohost-fetch-appslists") + + # Add a few legacy crons + add_legacy_cron("yunohost", "https://app.yunohost.org/official.json") + add_legacy_cron("dummy", "https://swiggitty.swaggy.lol/yolo.json") + + # Migrate + assert _using_legacy_appslist_system() is True + _migrate_appslist_system() + assert _using_legacy_appslist_system() is False + + # No legacy cron job should remain + assert glob.glob("/etc/cron.d/yunohost-appslist-*") == [] + + # Check they are in app_listlists anyway + appslist_dict = app_listlists() + assert "yunohost" in appslist_dict.keys() + assert appslist_dict["yunohost"]["url"] == "https://app.yunohost.org/official.json" + assert "dummy" in appslist_dict.keys() + assert appslist_dict["dummy"]["url"] == "https://swiggitty.swaggy.lol/yolo.json" + + assert cron_job_is_there() + + +def test_appslist_system_migration_badcron(): + """ + Test the migration on a bad legacy cron (no url found inside cron job) + """ + + # Start with no legacy cron, no appslist registered + assert glob.glob("/etc/cron.d/yunohost-appslist-*") == [] + assert app_listlists() == {} + assert not os.path.exists("/etc/cron.daily/yunohost-fetch-appslists") + + # Add a "bad" legacy cron + add_legacy_cron("wtflist", "ftp://the.fuck.is.this") + + # Migrate + assert _using_legacy_appslist_system() is True + _migrate_appslist_system() + assert _using_legacy_appslist_system() is False + + # No legacy cron should remain, but it should be backuped in /etc/yunohost + assert glob.glob("/etc/cron.d/yunohost-appslist-*") == [] + assert os.path.exists("/etc/yunohost/wtflist.oldlist.bkp") + + # Appslist should still be empty + assert app_listlists() == {} + + +def test_appslist_system_migration_conflict(): + """ + Test migration of conflicting cron job (in terms of url) + """ + + # Start with no legacy cron, no appslist registered + assert glob.glob("/etc/cron.d/yunohost-appslist-*") == [] + assert app_listlists() == {} + assert not os.path.exists("/etc/cron.daily/yunohost-fetch-appslists") + + # Add a few legacy crons + add_legacy_cron("yunohost", "https://app.yunohost.org/official.json") + add_legacy_cron("dummy", "https://app.yunohost.org/official.json") + + # Migrate + assert _using_legacy_appslist_system() is True + _migrate_appslist_system() + assert _using_legacy_appslist_system() is False + + # No legacy cron job should remain + assert glob.glob("/etc/cron.d/yunohost-appslist-*") == [] + + # Only one among "dummy" and "yunohost" should be listed + appslist_dict = app_listlists() + assert (len(appslist_dict.keys()) == 1) + assert ("dummy" in appslist_dict.keys()) or ("yunohost" in appslist_dict.keys()) + + assert cron_job_is_there() diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 5f1e0bb0f..8a9cb0664 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -38,7 +38,7 @@ import apt.progress from moulinette.core import MoulinetteError, init_authenticator from moulinette.utils.log import getActionLogger -from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, app_list +from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, app_list, _install_appslist_fetch_cron from yunohost.domain import domain_add, domain_list, get_public_ip, _get_maindomain, _set_maindomain from yunohost.dyndns import dyndns_subscribe from yunohost.firewall import firewall_upnp @@ -327,6 +327,15 @@ def tools_postinstall(domain, password, ignore_dyndns=False): # Enable UPnP silently and reload firewall firewall_upnp('enable', no_refresh=True) + # Setup the default official app list with cron job + try: + app_fetchlist(name="yunohost", + url="https://app.yunohost.org/official.json") + except Exception as e: + logger.warning(str(e)) + + _install_appslist_fetch_cron() + os.system('touch /etc/yunohost/installed') # Enable and start YunoHost firewall at boot time