From a6607eaf305e82269464eb6292d8c59cfda19130 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 16 Aug 2019 16:26:01 +0200 Subject: [PATCH 1/9] Buuuuurn the appslist system --- data/actionsmap/yunohost.yml | 32 ---- locales/en.json | 16 -- src/yunohost/app.py | 309 ----------------------------------- 3 files changed, 357 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 2b87b6daa..cc1cf738c 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -568,38 +568,6 @@ app: category_help: Manage apps actions: - ### app_fetchlist() - fetchlist: - action_help: Fetch application lists from app servers, or register a new one. - api: PUT /appslists - arguments: - -n: - full: --name - help: Name of the list to fetch (fetches all registered lists if empty) - extra: - pattern: &pattern_listname - - !!str ^[a-z0-9_]+$ - - "pattern_listname" - -u: - full: --url - help: URL of a new application list to register. To be specified with -n. - - ### app_listlists() - listlists: - action_help: List registered application lists - api: GET /appslists - - ### app_removelist() - removelist: - action_help: Remove and forget about a given application list - api: DELETE /appslists - arguments: - name: - help: Name of the list to remove - extra: - ask: ask_list_to_remove - pattern: *pattern_listname - ### app_list() list: action_help: List apps diff --git a/locales/en.json b/locales/en.json index b45739149..c018289b1 100644 --- a/locales/en.json +++ b/locales/en.json @@ -51,21 +51,10 @@ "app_upgraded": "{app:s} has been upgraded", "apps_permission_not_found": "No permission found for the installed apps", "apps_permission_restoration_failed": "Permission '{permission:s}' for app {app:s} restoration has failed", - "appslist_corrupted_json": "Could not load the application lists. It looks like {filename:s} is corrupted.", - "appslist_could_not_migrate": "Could not migrate app list {appslist:s}! Unable to parse the url… The old cron job has been kept in {bkp_file:s}.", - "appslist_fetched": "The application list {appslist:s} has been fetched", - "appslist_migrating": "Migrating application list {appslist:s}…", - "appslist_name_already_tracked": "There is already a registered application list with name {name:s}.", - "appslist_removed": "The application list {appslist:s} has been removed", - "appslist_retrieve_bad_format": "Retrieved file for application list {appslist:s} is not valid", - "appslist_retrieve_error": "Unable to retrieve the remote application list {appslist:s}: {error:s}", - "appslist_unknown": "Application list {appslist:s} unknown.", - "appslist_url_already_tracked": "There is already a registered application list with url {url:s}.", "ask_current_admin_password": "Current administration password", "ask_email": "Email address", "ask_firstname": "First name", "ask_lastname": "Last name", - "ask_list_to_remove": "List to remove", "ask_main_domain": "Main domain", "ask_new_admin_password": "New administration password", "ask_new_domain": "New domain", @@ -152,7 +141,6 @@ "confirm_app_install_danger": "WARNING! This application is still experimental (if not explicitly not working) and it is likely to break your system! You should probably NOT install it unless you know what you are doing. Are you willing to take that risk? [{answers:s}] ", "confirm_app_install_thirdparty": "WARNING! Installing 3rd party applications may compromise the integrity and security of your system. You should probably NOT install it unless you know what you are doing. Are you willing to take that risk? [{answers:s}] ", "custom_app_url_required": "You must provide a URL to upgrade your custom app {app:s}", - "custom_appslist_name_required": "You must provide a name for your custom app list", "diagnosis_debian_version_error": "Can't retrieve the Debian version: {error}", "diagnosis_kernel_version_error": "Can't retrieve kernel version: {error}", "diagnosis_monitor_disk_error": "Can't monitor disks: {error}", @@ -263,8 +251,6 @@ "log_app_addaccess": "Add access to '{}'", "log_app_removeaccess": "Remove access to '{}'", "log_app_clearaccess": "Remove all access to '{}'", - "log_app_fetchlist": "Add an application list", - "log_app_removelist": "Remove an application list", "log_app_change_url": "Change the url of '{}' application", "log_app_install": "Install '{}' application", "log_app_remove": "Remove '{}' application", @@ -401,7 +387,6 @@ "network_check_smtp_ko": "Outbound mail (SMTP port 25) seems to be blocked by your network", "network_check_smtp_ok": "Outbound mail (SMTP port 25) is not blocked", "new_domain_required": "You must provide the new main domain", - "no_appslist_found": "No app list found", "no_internet_connection": "Server is not connected to the Internet", "no_ipv6_connectivity": "IPv6 connectivity is not available", "no_restore_script": "No restore script found for the app '{app:s}'", @@ -538,7 +523,6 @@ "system_upgraded": "The system has been upgraded", "system_username_exists": "Username already exists in the system users", "this_action_broke_dpkg": "This action broke dpkg/apt (the system package managers)... You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.", - "tools_update_failed_to_app_fetchlist": "Failed to update YunoHost's applists because: {error}", "tools_upgrade_at_least_one": "Please specify --apps OR --system", "tools_upgrade_cant_both": "Cannot upgrade both system and apps at the same time", "tools_upgrade_cant_hold_critical_packages": "Unable to hold critical packages ...", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 4831f050c..d80144ff1 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -33,8 +33,6 @@ import re import urlparse import subprocess import glob -import pwd -import grp import urllib from collections import OrderedDict from datetime import datetime @@ -68,166 +66,6 @@ re_app_instance_name = re.compile( ) -def app_listlists(): - """ - List fetched lists - - """ - - # Migrate appslist system if needed - # XXX move to a migration when those are implemented - if _using_legacy_appslist_system(): - _migrate_appslist_system() - - # Get the list - appslist_list = _read_appslist_list() - - # Convert 'lastUpdate' timestamp to datetime - for name, infos in appslist_list.items(): - if infos["lastUpdate"] is None: - infos["lastUpdate"] = 0 - infos["lastUpdate"] = datetime.utcfromtimestamp(infos["lastUpdate"]) - - return appslist_list - - -def app_fetchlist(url=None, name=None): - """ - Fetch application list(s) from app server. By default, fetch all lists. - - Keyword argument: - name -- Name of the list - url -- URL of remote JSON list - """ - if url and not url.endswith(".json"): - raise YunohostError("This is not a valid application list url. It should end with .json.") - - # If needed, create folder where actual appslists are stored - if not os.path.exists(REPO_PATH): - os.makedirs(REPO_PATH) - - # Migrate appslist system if needed - # XXX move that to a migration once they are finished - if _using_legacy_appslist_system(): - _migrate_appslist_system() - - # Read the list of appslist... - appslists = _read_appslist_list() - - # Determine the list of appslist to be fetched - appslists_to_be_fetched = [] - - # If a url and and a name is given, try to register new list, - # the fetch only this list - if url is not None: - if name: - operation_logger = OperationLogger('app_fetchlist') - operation_logger.start() - _register_new_appslist(url, name) - # Refresh the appslists dict - appslists = _read_appslist_list() - appslists_to_be_fetched = [name] - operation_logger.success() - else: - raise YunohostError('custom_appslist_name_required') - - # If a name is given, look for an appslist with that name and fetch it - elif name is not None: - if name not in appslists.keys(): - raise YunohostError('appslist_unknown', appslist=name) - else: - appslists_to_be_fetched = [name] - - # Otherwise, fetch all lists - else: - appslists_to_be_fetched = appslists.keys() - - import requests # lazy loading this module for performance reasons - # Fetch all appslists to be fetched - for name in appslists_to_be_fetched: - - url = appslists[name]["url"] - - logger.debug("Attempting to fetch list %s at %s" % (name, url)) - - # Download file - try: - appslist_request = requests.get(url, timeout=30) - except requests.exceptions.SSLError: - logger.error(m18n.n('appslist_retrieve_error', - appslist=name, - error="SSL connection error")) - continue - except Exception as e: - logger.error(m18n.n('appslist_retrieve_error', - appslist=name, - error=str(e))) - continue - if appslist_request.status_code != 200: - logger.error(m18n.n('appslist_retrieve_error', - appslist=name, - error="Server returned code %s " % - str(appslist_request.status_code))) - continue - - # Validate app list format - # TODO / Possible improvement : better validation for app list (check - # that json fields actually look like an app list and not any json - # file) - appslist = appslist_request.text - try: - json.loads(appslist) - except ValueError as e: - logger.error(m18n.n('appslist_retrieve_bad_format', - appslist=name)) - continue - - # Write app list to file - list_file = '%s/%s.json' % (REPO_PATH, name) - try: - with open(list_file, "w") as f: - f.write(appslist) - except Exception as e: - raise YunohostError("Error while writing appslist %s: %s" % (name, str(e)), raw_msg=True) - - now = int(time.time()) - appslists[name]["lastUpdate"] = now - - logger.success(m18n.n('appslist_fetched', appslist=name)) - - # Write updated list of appslist - _write_appslist_list(appslists) - - -@is_unit_operation() -def app_removelist(operation_logger, name): - """ - Remove list from the repositories - - Keyword argument: - name -- Name of the list to remove - - """ - appslists = _read_appslist_list() - - # Make sure we know this appslist - if name not in appslists.keys(): - raise YunohostError('appslist_unknown', appslist=name) - - operation_logger.start() - - # Remove json - json_path = '%s/%s.json' % (REPO_PATH, name) - if os.path.exists(json_path): - os.remove(json_path) - - # Forget about this appslist - del appslists[name] - _write_appslist_list(appslists) - - logger.success(m18n.n('appslist_removed', appslist=name)) - - def app_list(filter=None, raw=False, installed=False, with_backup=False): """ List apps @@ -2704,153 +2542,6 @@ def _parse_app_instance_name(app_instance_name): return (appid, app_instance_nb) -def _using_legacy_appslist_system(): - """ - Return True if we're using the old fetchlist scheme. - This is determined by the presence of some cron job yunohost-applist-foo - """ - - return glob.glob("/etc/cron.d/yunohost-applist-*") != [] - - -def _migrate_appslist_system(): - """ - Migrate from the legacy fetchlist system to the new one - """ - legacy_crons = glob.glob("/etc/cron.d/yunohost-applist-*") - - for cron_path in legacy_crons: - appslist_name = os.path.basename(cron_path).replace("yunohost-applist-", "") - logger.debug(m18n.n('appslist_migrating', appslist=appslist_name)) - - # Parse appslist url in cron - cron_file_content = open(cron_path).read().strip() - appslist_url_parse = re.search("-u (https?://[^ ]+)", cron_file_content) - - # Abort if we did not find an url - if not appslist_url_parse or not appslist_url_parse.groups(): - # Bkp the old cron job somewhere else - bkp_file = "/etc/yunohost/%s.oldlist.bkp" % appslist_name - os.rename(cron_path, bkp_file) - # Notice the user - logger.warning(m18n.n('appslist_could_not_migrate', - appslist=appslist_name, - bkp_file=bkp_file)) - # Otherwise, register the list and remove the legacy cron - else: - appslist_url = appslist_url_parse.groups()[0] - try: - _register_new_appslist(appslist_url, appslist_name) - # Might get an exception if two legacy cron jobs conflict - # in terms of url... - except Exception as e: - logger.error(str(e)) - # Bkp the old cron job somewhere else - bkp_file = "/etc/yunohost/%s.oldlist.bkp" % appslist_name - os.rename(cron_path, bkp_file) - # Notice the user - logger.warning(m18n.n('appslist_could_not_migrate', - appslist=appslist_name, - bkp_file=bkp_file)) - else: - os.remove(cron_path) - - -def _install_appslist_fetch_cron(): - - cron_job_file = "/etc/cron.daily/yunohost-fetch-appslists" - - logger.debug("Installing appslist fetch cron job") - - cron_job = [] - cron_job.append("#!/bin/bash") - # We add a random delay between 0 and 60 min to avoid every instance fetching - # the appslist at the same time every night - cron_job.append("(sleep $((RANDOM%3600));") - cron_job.append("yunohost app fetchlist > /dev/null 2>&1) &") - - with open(cron_job_file, "w") as f: - f.write('\n'.join(cron_job)) - - _set_permissions(cron_job_file, "root", "root", 0o755) - - -# FIXME - Duplicate from certificate.py, should be moved into a common helper -# thing... -def _set_permissions(path, user, group, permissions): - uid = pwd.getpwnam(user).pw_uid - gid = grp.getgrnam(group).gr_gid - - os.chown(path, uid, gid) - os.chmod(path, permissions) - - -def _read_appslist_list(): - """ - Read the json corresponding to the list of appslists - """ - - # If file does not exists yet, return empty dict - if not os.path.exists(APPSLISTS_JSON): - return {} - - # Read file content - with open(APPSLISTS_JSON, "r") as f: - appslists_json = f.read() - - # Parse json, throw exception if what we got from file is not a valid json - try: - appslists = json.loads(appslists_json) - except ValueError: - raise YunohostError('appslist_corrupted_json', filename=APPSLISTS_JSON) - - return appslists - - -def _write_appslist_list(appslist_lists): - """ - Update the json containing list of appslists - """ - - # Write appslist list - try: - with open(APPSLISTS_JSON, "w") as f: - json.dump(appslist_lists, f) - except Exception as e: - raise YunohostError("Error while writing list of appslist %s: %s" % - (APPSLISTS_JSON, str(e)), raw_msg=True) - - -def _register_new_appslist(url, name): - """ - Add a new appslist to be fetched regularly. - Raise an exception if url or name conflicts with an existing list. - """ - - appslist_list = _read_appslist_list() - - # Check if name conflicts with an existing list - if name in appslist_list: - raise YunohostError('appslist_name_already_tracked', name=name) - - # Check if url conflicts with an existing list - known_appslist_urls = [appslist["url"] for _, appslist in appslist_list.items()] - - if url in known_appslist_urls: - raise YunohostError('appslist_url_already_tracked', url=url) - - logger.debug("Registering new appslist %s at %s" % (name, url)) - - appslist_list[name] = { - "url": url, - "lastUpdate": None - } - - _write_appslist_list(appslist_list) - - _install_appslist_fetch_cron() - - def is_true(arg): """ Convert a string into a boolean From 65b81e8677ec64ed6aac440650e852093dc02ddd Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 17 Aug 2019 16:51:47 +0200 Subject: [PATCH 2/9] Rewrite appslist system --- locales/en.json | 5 + src/yunohost/app.py | 160 +++++++- src/yunohost/tests/test_appslist.py | 547 ++++++++++++---------------- 3 files changed, 396 insertions(+), 316 deletions(-) diff --git a/locales/en.json b/locales/en.json index c018289b1..fdabad9d0 100644 --- a/locales/en.json +++ b/locales/en.json @@ -51,6 +51,11 @@ "app_upgraded": "{app:s} has been upgraded", "apps_permission_not_found": "No permission found for the installed apps", "apps_permission_restoration_failed": "Permission '{permission:s}' for app {app:s} restoration has failed", + "appslist_init_success": "Appslist system initialized!", + "appslist_updating": "Updating application list...", + "appslist_failed_to_download": "Unable to download the {applist} appslist : {error}", + "appslist_obsolete_cache": "The applist cache is empty or obsolete.", + "appslist_update_success": "The application list has been updated!", "ask_current_admin_password": "Current administration password", "ask_email": "Email address", "ask_firstname": "First name", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index d80144ff1..b2e276fe3 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -39,7 +39,8 @@ from datetime import datetime from moulinette import msignals, m18n, msettings from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import read_json, read_toml +from moulinette.utils.network import download_json +from moulinette.utils.filesystem import read_json, read_toml, read_yaml, write_to_file, write_to_json, write_to_yaml, chmod, chown, mkdir from yunohost.service import service_log, service_status, _run_service_command from yunohost.utils import packages @@ -48,12 +49,16 @@ from yunohost.log import is_unit_operation, OperationLogger logger = getActionLogger('yunohost.app') -REPO_PATH = '/var/cache/yunohost/repo' APPS_PATH = '/usr/share/yunohost/apps' APPS_SETTING_PATH = '/etc/yunohost/apps/' INSTALL_TMP = '/var/cache/yunohost' APP_TMP_FOLDER = INSTALL_TMP + '/from_file' -APPSLISTS_JSON = '/etc/yunohost/appslists.json' + +APPSLISTS_CACHE = '/var/cache/yunohost/repo' +APPSLISTS_CONF = '/etc/yunohost/appslists.yml' +APPSLISTS_CRON_PATH = "/etc/cron.daily/yunohost-fetch-appslists" +APPSLISTS_API_VERSION = 1 +APPSLISTS_DEFAULT_URL = "https://app.yunohost.org/default" re_github_repo = re.compile( r'^(http[s]?://|git@)github.com[/:]' @@ -2542,6 +2547,155 @@ def _parse_app_instance_name(app_instance_name): return (appid, app_instance_nb) +# +# ############################### # +# Applications list management # +# ############################### # +# + + +def _initialize_appslists_system(): + """ + This function is meant to intialize the appslist system with YunoHost's default applist. + + It also creates the cron job that will update the list every day + """ + + default_appslist_list = [{"id": "default", "url": APPSLISTS_DEFAULT_URL}] + + cron_job = [] + cron_job.append("#!/bin/bash") + # We add a random delay between 0 and 60 min to avoid every instance fetching + # the appslist at the same time every night + cron_job.append("(sleep $((RANDOM%3600));") + cron_job.append("yunohost tools update --apps > /dev/null) &") + try: + logger.debug("Initializing appslist system with YunoHost's default app list") + write_to_yaml(APPSLISTS_CONF, default_appslist_list) + + logger.debug("Installing appslist fetch daily cron job") + write_to_file(APPSLISTS_CRON_PATH, '\n'.join(cron_job)) + chown(APPSLISTS_CRON_PATH, uid="root", gid="root") + chmod(APPSLISTS_CRON_PATH, 0o755) + except Exception as e: + raise YunohostError("Could not initialize the appslist system... : %s" % str(e)) + + logger.success(m18n.n("appslist_init_success")) + + +def _read_appslist_list(): + """ + Read the json corresponding to the list of appslists + """ + + try: + list_ = read_yaml(APPSLISTS_CONF) + # Support the case where file exists but is empty + # by returning [] if list_ is None + return list_ if list_ else [] + except Exception as e: + raise YunohostError("Could not read the appslist list ... : %s" % str(e)) + + +def _actual_appslist_api_url(base_url): + + return "{base_url}/v{version}/apps.json".format(base_url=base_url, version=APPSLISTS_API_VERSION) + + +def _update_appslist(): + """ + Fetches the json for each appslist and update the cache + + appslist_list is for example : + [ {"id": "default", "url": "https://app.yunohost.org/default/"} ] + + Then for each appslist, the actual json URL to be fetched is like : + https://app.yunohost.org/default/vX/apps.json + + And store it in : + /var/cache/yunohost/repo/default.json + """ + + appslist_list = _read_appslist_list() + + logger.info(m18n.n("appslist_updating")) + + # Create cache folder if needed + if not os.path.exists(APPSLISTS_CACHE): + logger.debug("Initialize folder for appslist cache") + mkdir(APPSLISTS_CACHE, mode=0o750, parents=True, uid='root') + + for appslist in appslist_list: + applist_id = appslist["id"] + actual_api_url = _actual_appslist_api_url(appslist["url"]) + + # Fetch the json + try: + appslist_content = download_json(actual_api_url) + except Exception as e: + raise YunohostError("appslist_failed_to_download", applist=applist_id, error=str(e)) + + # Remember the appslist api version for later + appslist_content["from_api_version"] = APPSLISTS_API_VERSION + + # Save the appslist data in the cache + cache_file = "{cache_folder}/{list}.json".format(cache_folder=APPSLISTS_CACHE, list=applist_id) + try: + write_to_json(cache_file, appslist_content) + except Exception as e: + raise YunohostError("Unable to write cache data for %s appslist : %s" % (applist_id, str(e))) + + logger.success(m18n.n("appslist_update_success")) + + +def _load_appslist(): + """ + Read all the appslist cache file and build a single dict (app_dict) + corresponding to all known apps in all indexes + """ + + app_dict = {} + + for appslist_id in [L["id"] for L in _read_appslist_list()]: + + # Let's load the json from cache for this appslist + cache_file = "{cache_folder}/{list}.json".format(cache_folder=APPSLISTS_CACHE, list=appslist_id) + + try: + appslist_content = read_json(cache_file) if os.path.exists(cache_file) else None + except Exception as e: + raise ("Unable to read cache for appslist %s : %s" % (appslist_id, str(e))) + + # Check that the version of the data matches version .... + # ... otherwise it means we updated yunohost in the meantime + # and need to update the cache for everything to be consistent + if not appslist_content or appslist_content.get("from_api_version") != APPSLISTS_API_VERSION: + logger.info(m18n.n("appslist_obsolete_cache")) + _update_appslist() + appslist_content = read_json(cache_file) + + del appslist_content["from_api_version"] + + # Add apps from this applist to the output + for app, info in appslist_content.items(): + + # (N.B. : there's a small edge case where multiple appslist could be listing the same apps ... + # in which case we keep only the first one found) + if app in app_dict: + logger.warning("Duplicate app %s found between appslist %s and %s" % (app, appslist_id, app_dict[app]['repository'])) + continue + + info['repository'] = appslist_id + app_dict[app] = info + + return app_dict + +# +# ############################### # +# Small utilities # +# ############################### # +# + def is_true(arg): """ Convert a string into a boolean diff --git a/src/yunohost/tests/test_appslist.py b/src/yunohost/tests/test_appslist.py index 817807ed9..057a5f3fe 100644 --- a/src/yunohost/tests/test_appslist.py +++ b/src/yunohost/tests/test_appslist.py @@ -3,34 +3,49 @@ import pytest import requests import requests_mock import glob -import time +import shutil + +from moulinette import m18n +from moulinette.utils.filesystem import read_json, write_to_json, write_to_yaml from yunohost.utils.error import YunohostError +from yunohost.app import (_initialize_appslists_system, + _read_appslist_list, + _update_appslist, + _actual_appslist_api_url, + _load_appslist, + logger, + APPSLISTS_CACHE, + APPSLISTS_CONF, + APPSLISTS_CRON_PATH, + APPSLISTS_API_VERSION, + APPSLISTS_DEFAULT_URL) -from yunohost.app import app_fetchlist, app_removelist, app_listlists, _using_legacy_appslist_system, _migrate_appslist_system, _register_new_appslist +APPSLISTS_DEFAULT_URL_FULL = _actual_appslist_api_url(APPSLISTS_DEFAULT_URL) +CRON_FOLDER, CRON_NAME = APPSLISTS_CRON_PATH.rsplit("/", 1) -URL_OFFICIAL_APP_LIST = "https://app.yunohost.org/official.json" -REPO_PATH = '/var/cache/yunohost/repo' -APPSLISTS_JSON = '/etc/yunohost/appslists.json' +DUMMY_APPLIST = """{ + "foo": {"id": "foo", "level": 4}, + "bar": {"id": "bar", "level": 7} +} +""" +class AnyStringWith(str): + def __eq__(self, other): + return self in other def setup_function(function): - # Clear all appslist - files = glob.glob(REPO_PATH + "/*") - for f in files: - os.remove(f) + # Clear applist cache + shutil.rmtree(APPSLISTS_CACHE, ignore_errors=True) - # Clear appslist crons - files = glob.glob("/etc/cron.d/yunohost-applist-*") - for f in files: - os.remove(f) + # Clear appslist cron + if os.path.exists(APPSLISTS_CRON_PATH): + os.remove(APPSLISTS_CRON_PATH) - if os.path.exists("/etc/cron.daily/yunohost-fetch-appslists"): - os.remove("/etc/cron.daily/yunohost-fetch-appslists") - - if os.path.exists(APPSLISTS_JSON): - os.remove(APPSLISTS_JSON) + # Clear appslist conf + if os.path.exists(APPSLISTS_CONF): + os.remove(APPSLISTS_CONF) def teardown_function(function): @@ -38,352 +53,258 @@ def teardown_function(function): def cron_job_is_there(): - r = os.system("run-parts -v --test /etc/cron.daily/ | grep yunohost-fetch-appslists") + r = os.system("run-parts -v --test %s | grep %s" % (CRON_FOLDER, CRON_NAME)) return r == 0 - # -# Test listing of appslists and registering of appslists # +# ################################################ # -def test_appslist_list_empty(): - """ - Calling app_listlists() with no registered list should return empty dict - """ +def test_appslist_init(mocker): - assert app_listlists() == {} + # Cache is empty + assert not glob.glob(APPSLISTS_CACHE + "/*") + # Conf doesn't exist yet + assert not os.path.exists(APPSLISTS_CONF) + # Conf doesn't exist yet + assert not os.path.exists(APPSLISTS_CRON_PATH) + # Initialize ... + mocker.spy(m18n, "n") + _initialize_appslists_system() + m18n.n.assert_any_call('appslist_init_success') -def test_appslist_list_register(): - """ - Register a new list - """ - - # Assume we're starting with an empty app list - assert app_listlists() == {} - - # Register a new dummy list - _register_new_appslist("https://lol.com/appslist.json", "dummy") - - appslist_dict = app_listlists() - assert "dummy" in appslist_dict.keys() - assert appslist_dict["dummy"]["url"] == "https://lol.com/appslist.json" - + # Then there's a cron enabled assert cron_job_is_there() + # And a conf with at least one list + assert os.path.exists(APPSLISTS_CONF) + appslist_list = _read_appslist_list() + assert len(appslist_list) -def test_appslist_list_register_conflict_name(): - """ - Attempt to register a new list with conflicting name - """ - - _register_new_appslist("https://lol.com/appslist.json", "dummy") - with pytest.raises(YunohostError): - _register_new_appslist("https://lol.com/appslist2.json", "dummy") - - appslist_dict = app_listlists() - - assert "dummy" in appslist_dict.keys() - assert "dummy2" not in appslist_dict.keys() + # Cache is expected to still be empty though + # (if we did update the appslist during init, + # we couldn't differentiate easily exceptions + # related to lack of network connectivity) + assert not glob.glob(APPSLISTS_CACHE + "/*") -def test_appslist_list_register_conflict_url(): - """ - Attempt to register a new list with conflicting url - """ +def test_appslist_emptylist(): - _register_new_appslist("https://lol.com/appslist.json", "dummy") - with pytest.raises(YunohostError): - _register_new_appslist("https://lol.com/appslist.json", "plopette") + # Initialize ... + _initialize_appslists_system() - appslist_dict = app_listlists() + # Let's imagine somebody removed the default applist because uh idk they dont want to use our default applist + os.system("rm %s" % APPSLISTS_CONF) + os.system("touch %s" % APPSLISTS_CONF) - assert "dummy" in appslist_dict.keys() - assert "plopette" not in appslist_dict.keys() + appslist_list = _read_appslist_list() + assert not len(appslist_list) -# -# Test fetching of appslists # -# +def test_appslist_update_success(mocker): + + # Initialize ... + _initialize_appslists_system() + + # Cache is empty + assert not glob.glob(APPSLISTS_CACHE + "/*") + + # Update + with requests_mock.Mocker() as m: + + _actual_appslist_api_url, + # Mock the server response with a dummy applist + m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, text=DUMMY_APPLIST) + + mocker.spy(m18n, "n") + _update_appslist() + m18n.n.assert_any_call("appslist_updating") + m18n.n.assert_any_call("appslist_update_success") + + # Cache shouldn't be empty anymore empty + assert glob.glob(APPSLISTS_CACHE + "/*") + + app_dict = _load_appslist() + assert "foo" in app_dict.keys() + assert "bar" in app_dict.keys() -def test_appslist_fetch(): - """ - Do a fetchlist and test the .json got updated. - """ - assert app_listlists() == {} +def test_appslist_update_404(mocker): - _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost") + # Initialize ... + _initialize_appslists_system() with requests_mock.Mocker() as m: - # Mock the server response with a valid (well, empty, yep) json - m.register_uri("GET", URL_OFFICIAL_APP_LIST, text='{ }') + # 404 error + m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, + status_code=404) - official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"] - app_fetchlist() - new_official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"] + with pytest.raises(YunohostError): + mocker.spy(m18n, "n") + _update_appslist() + m18n.n.assert_any_call("appslist_failed_to_download") - assert new_official_lastUpdate > official_lastUpdate +def test_appslist_update_timeout(mocker): - -def test_appslist_fetch_single_appslist(): - """ - Register several lists but only fetch one. Check only one got updated. - """ - - assert app_listlists() == {} - _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost") - _register_new_appslist("https://lol.com/appslist.json", "dummy") - - time.sleep(1) + # Initialize ... + _initialize_appslists_system() with requests_mock.Mocker() as m: - # Mock the server response with a valid (well, empty, yep) json - m.register_uri("GET", URL_OFFICIAL_APP_LIST, text='{ }') - - official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"] - dummy_lastUpdate = app_listlists()["dummy"]["lastUpdate"] - app_fetchlist(name="yunohost") - new_official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"] - new_dummy_lastUpdate = app_listlists()["dummy"]["lastUpdate"] - - assert new_official_lastUpdate > official_lastUpdate - assert new_dummy_lastUpdate == dummy_lastUpdate - - -def test_appslist_fetch_unknownlist(): - """ - Attempt to fetch an unknown list - """ - - assert app_listlists() == {} - - with pytest.raises(YunohostError): - app_fetchlist(name="swag") - - -def test_appslist_fetch_url_but_no_name(): - """ - Do a fetchlist with url given, but no name given - """ - - with pytest.raises(YunohostError): - app_fetchlist(url=URL_OFFICIAL_APP_LIST) - - -def test_appslist_fetch_badurl(): - """ - Do a fetchlist with a bad url - """ - - app_fetchlist(url="https://not.a.valid.url/plop.json", name="plop") - - -def test_appslist_fetch_badfile(): - """ - Do a fetchlist and mock a response with a bad json - """ - assert app_listlists() == {} - - _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost") - - with requests_mock.Mocker() as m: - - m.register_uri("GET", URL_OFFICIAL_APP_LIST, text='{ not json lol }') - - app_fetchlist() - - -def test_appslist_fetch_404(): - """ - Do a fetchlist and mock a 404 response - """ - assert app_listlists() == {} - - _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost") - - with requests_mock.Mocker() as m: - - m.register_uri("GET", URL_OFFICIAL_APP_LIST, status_code=404) - - app_fetchlist() - - -def test_appslist_fetch_sslerror(): - """ - Do a fetchlist and mock an SSL error - """ - assert app_listlists() == {} - - _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost") - - with requests_mock.Mocker() as m: - - m.register_uri("GET", URL_OFFICIAL_APP_LIST, - exc=requests.exceptions.SSLError) - - app_fetchlist() - - -def test_appslist_fetch_timeout(): - """ - Do a fetchlist and mock a timeout - """ - assert app_listlists() == {} - - _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost") - - with requests_mock.Mocker() as m: - - m.register_uri("GET", URL_OFFICIAL_APP_LIST, + # Timeout + m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, exc=requests.exceptions.ConnectTimeout) - app_fetchlist() + with pytest.raises(YunohostError): + mocker.spy(m18n, "n") + _update_appslist() + m18n.n.assert_any_call("appslist_failed_to_download") -# -# Test remove of appslist # -# +def test_appslist_update_sslerror(mocker): + + # Initialize ... + _initialize_appslists_system() + + with requests_mock.Mocker() as m: + + # SSL error + m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, + exc=requests.exceptions.SSLError) + + with pytest.raises(YunohostError): + mocker.spy(m18n, "n") + _update_appslist() + m18n.n.assert_any_call("appslist_failed_to_download") -def test_appslist_remove(): - """ - Register a new appslist, then remove it - """ +def test_appslist_update_corrupted(mocker): - # Assume we're starting with an empty app list - assert app_listlists() == {} + # Initialize ... + _initialize_appslists_system() - # Register a new dummy list - _register_new_appslist("https://lol.com/appslist.json", "dummy") - app_removelist("dummy") + with requests_mock.Mocker() as m: - # Should end up with no list registered - assert app_listlists() == {} + # Corrupted json + m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, + text=DUMMY_APPLIST[:-2]) + + with pytest.raises(YunohostError): + mocker.spy(m18n, "n") + _update_appslist() + m18n.n.assert_any_call("appslist_failed_to_download") -def test_appslist_remove_unknown(): - """ - Attempt to remove an unknown list - """ +def test_appslist_load_with_empty_cache(mocker): - with pytest.raises(YunohostError): - app_removelist("dummy") + # Initialize ... + _initialize_appslists_system() + + # Cache is empty + assert not glob.glob(APPSLISTS_CACHE + "/*") + + # Update + with requests_mock.Mocker() as m: + + # Mock the server response with a dummy applist + m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, text=DUMMY_APPLIST) + + # Try to load the applist + # This should implicitly trigger an update in the background + mocker.spy(m18n, "n") + app_dict = _load_appslist() + m18n.n.assert_any_call("appslist_obsolete_cache") + m18n.n.assert_any_call("appslist_update_success") -# -# Test migration from legacy appslist system # -# + # Cache shouldn't be empty anymore empty + assert glob.glob(APPSLISTS_CACHE + "/*") + + assert "foo" in app_dict.keys() + assert "bar" in app_dict.keys() -def add_legacy_cron(name, url): - with open("/etc/cron.d/yunohost-applist-%s" % name, "w") as f: - f.write('00 00 * * * root yunohost app fetchlist -u %s -n %s > /dev/null 2>&1\n' % (url, name)) +def test_appslist_load_with_conflicts_between_lists(mocker): + + # Initialize ... + _initialize_appslists_system() + + conf = [{"id": "default", "url": APPSLISTS_DEFAULT_URL}, + {"id": "default2", "url": APPSLISTS_DEFAULT_URL.replace("yunohost.org", "yolohost.org")}] + + write_to_yaml(APPSLISTS_CONF, conf) + + # Update + with requests_mock.Mocker() as m: + + # Mock the server response with a dummy applist + # + the same applist for the second list + m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, text=DUMMY_APPLIST) + m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL.replace("yunohost.org", "yolohost.org"), text=DUMMY_APPLIST) + + # Try to load the applist + # This should implicitly trigger an update in the background + mocker.spy(logger, "warning") + app_dict = _load_appslist() + logger.warning.assert_any_call(AnyStringWith("Duplicate")) + + # Cache shouldn't be empty anymore empty + assert glob.glob(APPSLISTS_CACHE + "/*") + + assert "foo" in app_dict.keys() + assert "bar" in app_dict.keys() -def test_appslist_check_using_legacy_system_testFalse(): - """ - If no legacy cron job is there, the check should return False - """ - assert glob.glob("/etc/cron.d/yunohost-applist-*") == [] - assert _using_legacy_appslist_system() is False +def test_appslist_load_with_oudated_api_version(mocker): + + # Initialize ... + _initialize_appslists_system() + + # Update + with requests_mock.Mocker() as m: + + mocker.spy(m18n, "n") + m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, text=DUMMY_APPLIST) + _update_appslist() + + # Cache shouldn't be empty anymore empty + assert glob.glob(APPSLISTS_CACHE + "/*") + + # Tweak the cache to replace the from_api_version with a different one + for cache_file in glob.glob(APPSLISTS_CACHE + "/*"): + cache_json = read_json(cache_file) + assert cache_json["from_api_version"] == APPSLISTS_API_VERSION + cache_json["from_api_version"] = 0 + write_to_json(cache_file, cache_json) + + # Update + with requests_mock.Mocker() as m: + + # Mock the server response with a dummy applist + m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, text=DUMMY_APPLIST) + + mocker.spy(m18n, "n") + app_dict = _load_appslist() + m18n.n.assert_any_call("appslist_update_success") + + assert "foo" in app_dict.keys() + assert "bar" in app_dict.keys() + + # Check that we indeed have the new api number in cache + for cache_file in glob.glob(APPSLISTS_CACHE + "/*"): + cache_json = read_json(cache_file) + assert cache_json["from_api_version"] == APPSLISTS_API_VERSION -def test_appslist_check_using_legacy_system_testTrue(): - """ - If there's a legacy cron job, the check should return True - """ - assert glob.glob("/etc/cron.d/yunohost-applist-*") == [] - add_legacy_cron("yunohost", "https://app.yunohost.org/official.json") - assert _using_legacy_appslist_system() is True + +def test_appslist_migrate_legacy_explicitly(): + + raise NotImplementedError -def test_appslist_system_migration(): - """ - Test that legacy cron jobs get migrated correctly when calling app_listlists - """ +def test_appslist_migrate_legacy_implicitly(): - # Start with no legacy cron, no appslist registered - assert glob.glob("/etc/cron.d/yunohost-applist-*") == [] - assert app_listlists() == {} - assert not os.path.exists("/etc/cron.daily/yunohost-fetch-appslists") - - # Add a few legacy crons - add_legacy_cron("yunohost", "https://app.yunohost.org/official.json") - add_legacy_cron("dummy", "https://swiggitty.swaggy.lol/yolo.json") - - # Migrate - assert _using_legacy_appslist_system() is True - _migrate_appslist_system() - assert _using_legacy_appslist_system() is False - - # No legacy cron job should remain - assert glob.glob("/etc/cron.d/yunohost-applist-*") == [] - - # Check they are in app_listlists anyway - appslist_dict = app_listlists() - assert "yunohost" in appslist_dict.keys() - assert appslist_dict["yunohost"]["url"] == "https://app.yunohost.org/official.json" - assert "dummy" in appslist_dict.keys() - assert appslist_dict["dummy"]["url"] == "https://swiggitty.swaggy.lol/yolo.json" - - assert cron_job_is_there() - - -def test_appslist_system_migration_badcron(): - """ - Test the migration on a bad legacy cron (no url found inside cron job) - """ - - # Start with no legacy cron, no appslist registered - assert glob.glob("/etc/cron.d/yunohost-applist-*") == [] - assert app_listlists() == {} - assert not os.path.exists("/etc/cron.daily/yunohost-fetch-appslists") - - # Add a "bad" legacy cron - add_legacy_cron("wtflist", "ftp://the.fuck.is.this") - - # Migrate - assert _using_legacy_appslist_system() is True - _migrate_appslist_system() - assert _using_legacy_appslist_system() is False - - # No legacy cron should remain, but it should be backuped in /etc/yunohost - assert glob.glob("/etc/cron.d/yunohost-applist-*") == [] - assert os.path.exists("/etc/yunohost/wtflist.oldlist.bkp") - - # Appslist should still be empty - assert app_listlists() == {} - - -def test_appslist_system_migration_conflict(): - """ - Test migration of conflicting cron job (in terms of url) - """ - - # Start with no legacy cron, no appslist registered - assert glob.glob("/etc/cron.d/yunohost-applist-*") == [] - assert app_listlists() == {} - assert not os.path.exists("/etc/cron.daily/yunohost-fetch-appslists") - - # Add a few legacy crons - add_legacy_cron("yunohost", "https://app.yunohost.org/official.json") - add_legacy_cron("dummy", "https://app.yunohost.org/official.json") - - # Migrate - assert _using_legacy_appslist_system() is True - _migrate_appslist_system() - assert _using_legacy_appslist_system() is False - - # No legacy cron job should remain - assert glob.glob("/etc/cron.d/yunohost-applist-*") == [] - - # Only one among "dummy" and "yunohost" should be listed - appslist_dict = app_listlists() - assert (len(appslist_dict.keys()) == 1) - assert ("dummy" in appslist_dict.keys()) or ("yunohost" in appslist_dict.keys()) - - assert cron_job_is_there() + raise NotImplementedError From 050750b7c986f064e4a8c1f54a9c23911f3dc9f7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 17 Aug 2019 17:43:38 +0200 Subject: [PATCH 3/9] Propagate the changes on other parts of the code relying on the appslist system --- src/yunohost/app.py | 22 ++-------------------- src/yunohost/tools.py | 20 +++++++++++--------- 2 files changed, 13 insertions(+), 29 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index b2e276fe3..aa2163cdf 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -86,28 +86,10 @@ def app_list(filter=None, raw=False, installed=False, with_backup=False): """ installed = with_backup or installed - app_dict = {} list_dict = {} if raw else [] - appslists = _read_appslist_list() - - for appslist in appslists.keys(): - - json_path = "%s/%s.json" % (REPO_PATH, appslist) - - # If we don't have the json yet, try to fetch it - if not os.path.exists(json_path): - app_fetchlist(name=appslist) - - # If it now exist - if os.path.exists(json_path): - appslist_content = read_json(json_path) - for app, info in appslist_content.items(): - if app not in app_dict: - info['repository'] = appslist - app_dict[app] = info - else: - logger.warning("Uh there's no data for applist '%s' ... (That should be just a temporary issue?)" % appslist) + # Get app list from applist cache + app_dict = _load_appslist() # Get app list from the app settings directory for app in os.listdir(APPS_SETTING_PATH): diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index c63f1ed33..d9b50280c 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -38,7 +38,7 @@ from moulinette import msignals, m18n from moulinette.utils.log import getActionLogger from moulinette.utils.process import check_output, call_async_output from moulinette.utils.filesystem import read_json, write_to_json -from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, app_list, _install_appslist_fetch_cron +from yunohost.app import _update_appslist, app_info, app_upgrade, app_ssowatconf, app_list from yunohost.domain import domain_add, domain_list, _get_maindomain, _set_maindomain from yunohost.dyndns import _dyndns_available, _dyndns_provides from yunohost.firewall import firewall_upnp @@ -411,15 +411,17 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False, # Enable UPnP silently and reload firewall firewall_upnp('enable', no_refresh=True) - # Setup the default apps list with cron job + # Initialize the appslist system + _initialize_appslist_system() + + # Try to update the appslist ... + # we don't fail miserably if this fails, + # because that could be for example an offline installation... try: - app_fetchlist(name="yunohost", - url="https://app.yunohost.org/apps.json") + _update_appslist() except Exception as e: logger.warning(str(e)) - _install_appslist_fetch_cron() - # Init migrations (skip them, no need to run them on a fresh system) _skip_all_migrations() @@ -465,6 +467,7 @@ def tools_update(apps=False, system=False): Keyword arguments: system -- Fetch available system packages upgrades (equivalent to apt update) apps -- Fetch the application list to check which apps can be upgraded + appslist -- Just update the application list cache """ # If neither --apps nor --system specified, do both @@ -510,11 +513,10 @@ def tools_update(apps=False, system=False): upgradable_apps = [] if apps: - logger.info(m18n.n('updating_app_lists')) try: - app_fetchlist() + _update_appslist() except YunohostError as e: - logger.error(m18n.n('tools_update_failed_to_app_fetchlist'), error=e) + logger.error(str(e)) upgradable_apps = list(_list_upgradable_apps()) From 8dd590e6fcd05f4a340511a4c0d7f7ea14088413 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 17 Aug 2019 18:43:34 +0200 Subject: [PATCH 4/9] Implement explicit and implicit migrations --- locales/en.json | 3 +- src/yunohost/app.py | 6 ++ .../0010_migrate_to_apps_json.py | 34 ++--------- src/yunohost/tests/test_appslist.py | 56 +++++++++++++++++-- 4 files changed, 64 insertions(+), 35 deletions(-) diff --git a/locales/en.json b/locales/en.json index fdabad9d0..0e3667749 100644 --- a/locales/en.json +++ b/locales/en.json @@ -320,8 +320,9 @@ "migration_description_0007_ssh_conf_managed_by_yunohost_step1": "Let the SSH configuration be managed by YunoHost (step 1, automatic)", "migration_description_0008_ssh_conf_managed_by_yunohost_step2": "Let the SSH configuration be managed by YunoHost (step 2, manual)", "migration_description_0009_decouple_regenconf_from_services": "Decouple the regen-conf mechanism from services", - "migration_description_0010_migrate_to_apps_json": "Remove deprecated appslists and use the new unified 'apps.json' list instead", + "migration_description_0010_migrate_to_apps_json": "Remove deprecated appslists and use the new unified 'apps.json' list instead (outdated, replaced by migration 12)", "migration_description_0011_setup_group_permission": "Setup user group and setup permission for apps and services", + "migration_description_0012_futureproof_appslist_system": "Migrate to the new future-proof appslist system", "migration_0003_backward_impossible": "The stretch migration cannot be reverted.", "migration_0003_start": "Starting migration to Stretch. The logs will be available in {logfile}.", "migration_0003_patching_sources_list": "Patching the sources.lists…", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index aa2163cdf..3a6d2216a 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -2570,6 +2570,12 @@ def _read_appslist_list(): Read the json corresponding to the list of appslists """ + # Legacy code - can be removed after moving to buster (if the migration got merged before buster) + if os.path.exists('/etc/yunohost/appslists.json'): + from yunohost.tools import _get_migration_by_name + migration = _get_migration_by_name("futureproof_appslist_system") + migration.migrate() + try: list_ = read_yaml(APPSLISTS_CONF) # Support the case where file exists but is empty diff --git a/src/yunohost/data_migrations/0010_migrate_to_apps_json.py b/src/yunohost/data_migrations/0010_migrate_to_apps_json.py index 43ae9a86f..1333e54e8 100644 --- a/src/yunohost/data_migrations/0010_migrate_to_apps_json.py +++ b/src/yunohost/data_migrations/0010_migrate_to_apps_json.py @@ -1,44 +1,18 @@ -import os - from moulinette.utils.log import getActionLogger -from yunohost.app import app_fetchlist, app_removelist, _read_appslist_list, APPSLISTS_JSON from yunohost.tools import Migration logger = getActionLogger('yunohost.migration') -BASE_CONF_PATH = '/home/yunohost.conf' -BACKUP_CONF_DIR = os.path.join(BASE_CONF_PATH, 'backup') -APPSLISTS_BACKUP = os.path.join(BACKUP_CONF_DIR, "appslist_before_migration_to_unified_list.json") - class MyMigration(Migration): - "Migrate from official.json to apps.json" + "Migrate from official.json to apps.json (outdated, replaced by migration 12)" def migrate(self): - # Backup current app list json - os.system("cp %s %s" % (APPSLISTS_JSON, APPSLISTS_BACKUP)) - - # Remove all the deprecated lists - lists_to_remove = [ - "app.yunohost.org/list.json", # Old list on old installs, alias to official.json - "app.yunohost.org/official.json", - "app.yunohost.org/community.json", - "labriqueinter.net/apps/labriqueinternet.json", - "labriqueinter.net/internetcube.json" - ] - - appslists = _read_appslist_list() - for appslist, infos in appslists.items(): - if infos["url"].split("//")[-1] in lists_to_remove: - app_removelist(name=appslist) - - # Replace by apps.json list - app_fetchlist(name="yunohost", - url="https://app.yunohost.org/apps.json") + logger.info("This is migration is oudated and doesn't do anything anymore. The migration 12 will handle this instead.") + pass def backward(self): - if os.path.exists(APPSLISTS_BACKUP): - os.system("cp %s %s" % (APPSLISTS_BACKUP, APPSLISTS_JSON)) + pass diff --git a/src/yunohost/tests/test_appslist.py b/src/yunohost/tests/test_appslist.py index 057a5f3fe..d7b8e429b 100644 --- a/src/yunohost/tests/test_appslist.py +++ b/src/yunohost/tests/test_appslist.py @@ -6,7 +6,7 @@ import glob import shutil from moulinette import m18n -from moulinette.utils.filesystem import read_json, write_to_json, write_to_yaml +from moulinette.utils.filesystem import read_json, write_to_json, write_to_yaml, mkdir from yunohost.utils.error import YunohostError from yunohost.app import (_initialize_appslists_system, @@ -49,7 +49,11 @@ def setup_function(function): def teardown_function(function): - pass + + # Clear applist cache + # Otherwise when using apps stuff after running the test, + # we'll still have the dummy unusable list + shutil.rmtree(APPSLISTS_CACHE, ignore_errors=True) def cron_job_is_there(): @@ -302,9 +306,53 @@ def test_appslist_load_with_oudated_api_version(mocker): def test_appslist_migrate_legacy_explicitly(): - raise NotImplementedError + open("/etc/yunohost/appslists.json", "w").write('{"yunohost": {"yolo":"swag"}}') + mkdir(APPSLISTS_CACHE, 0o750, parents=True) + open(APPSLISTS_CACHE+"/yunohost_old.json", "w").write('{"foo":{}, "bar": {}}') + open(APPSLISTS_CRON_PATH, "w").write("# Some old cron") + + from yunohost.tools import _get_migration_by_name + migration = _get_migration_by_name("futureproof_appslist_system") + + with requests_mock.Mocker() as m: + + # Mock the server response with a dummy applist + m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, text=DUMMY_APPLIST) + migration.migrate() + + # Old conf shouldnt be there anymore (got renamed to .old) + assert not os.path.exists("/etc/yunohost/appslists.json") + # Old cache should have been removed + assert not os.path.exists(APPSLISTS_CACHE+"/yunohost_old.json") + # Cron should have been changed + assert "/bin/bash" in open(APPSLISTS_CRON_PATH, "r").read() + assert cron_job_is_there() + + # Reading the appslist should work + app_dict = _load_appslist() + assert "foo" in app_dict.keys() + assert "bar" in app_dict.keys() def test_appslist_migrate_legacy_implicitly(): - raise NotImplementedError + open("/etc/yunohost/appslists.json", "w").write('{"yunohost": {"yolo":"swag"}}') + mkdir(APPSLISTS_CACHE, 0o750, parents=True) + open(APPSLISTS_CACHE+"/yunohost_old.json", "w").write('{"old_foo":{}, "old_bar": {}}') + open(APPSLISTS_CRON_PATH, "w").write("# Some old cron") + + with requests_mock.Mocker() as m: + m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, text=DUMMY_APPLIST) + app_dict = _load_appslist() + + assert "foo" in app_dict.keys() + assert "bar" in app_dict.keys() + + # Old conf shouldnt be there anymore (got renamed to .old) + assert not os.path.exists("/etc/yunohost/appslists.json") + # Old cache should have been removed + assert not os.path.exists(APPSLISTS_CACHE+"/yunohost_old.json") + # Cron should have been changed + assert "/bin/bash" in open(APPSLISTS_CRON_PATH, "r").read() + assert cron_job_is_there() + From 005ccea610a6fa263f182069e3238976078d18a3 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 17 Aug 2019 19:02:38 +0200 Subject: [PATCH 5/9] Make the PEP8 happy even though that's unrelated code to the current PR but meh ... --- src/yunohost/app.py | 59 +++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 31 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 3a6d2216a..f4f645f2c 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -230,7 +230,8 @@ def app_map(app=None, raw=False, user=None): app -- Specific app to map """ - from yunohost.permission import user_permission_list + # TODO / FIXME : Aleks to Josue : not sure why this was included but not used ... + # from yunohost.permission import user_permission_list from yunohost.utils.ldap import _get_ldap_interface apps = [] @@ -327,7 +328,7 @@ def app_change_url(operation_logger, app, domain, path): # Retrieve arguments list for change_url script # TODO: Allow to specify arguments args_odict = _parse_args_from_manifest(manifest, 'change_url') - args_list = [ value[0] for value in args_odict.values() ] + args_list = [value[0] for value in args_odict.values()] args_list.append(app) # Prepare env. var. to pass to script @@ -380,7 +381,7 @@ def app_change_url(operation_logger, app, domain, path): app_setting(app, 'domain', value=domain) app_setting(app, 'path', value=path) - permission_update(app, permission="main", add_url=[domain+path], remove_url=[old_domain+old_path], sync_perm=True) + permission_update(app, permission="main", add_url=[domain + path], remove_url=[old_domain + old_path], sync_perm=True) # avoid common mistakes if _run_service_command("reload", "nginx") is False: @@ -415,9 +416,6 @@ def app_upgrade(app=[], url=None, file=None): from yunohost.hook import hook_add, hook_remove, hook_exec, hook_callback from yunohost.permission import permission_sync_to_user - # Retrieve interface - is_api = msettings.get('interface') == 'api' - try: app_list() except YunohostError: @@ -430,15 +428,15 @@ def app_upgrade(app=[], url=None, file=None): if not apps: # FIXME : not sure what's supposed to happen if there is a url and a file but no apps... if not url and not file: - apps = [app["id"] for app in app_list(installed=True)["apps"]] + apps = [app_["id"] for app_ in app_list(installed=True)["apps"]] elif not isinstance(app, list): apps = [app] # Remove possible duplicates - apps = [app for i,app in enumerate(apps) if apps not in apps[:i]] + apps = [app_ for i, app_ in enumerate(apps) if app_ not in apps[:i]] # Abort if any of those app is in fact not installed.. - for app in [app for app in apps if not _is_installed(app)]: + for app in [app_ for app_ in apps if not _is_installed(app_)]: raise YunohostError('app_not_installed', app=app, all_apps=_get_all_installed_apps_id()) if len(apps) == 0: @@ -477,7 +475,7 @@ def app_upgrade(app=[], url=None, file=None): # Retrieve arguments list for upgrade script # TODO: Allow to specify arguments args_odict = _parse_args_from_manifest(manifest, 'upgrade') - args_list = [ value[0] for value in args_odict.values() ] + args_list = [value[0] for value in args_odict.values()] args_list.append(app_instance_name) # Prepare env. var. to pass to script @@ -585,7 +583,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu return answer = msignals.prompt(m18n.n('confirm_app_install_' + confirm, - answers='Y/N')) + answers='Y/N')) if answer.upper() != "Y": raise YunohostError("aborting") @@ -639,7 +637,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu args_dict = {} if not args else \ dict(urlparse.parse_qsl(args, keep_blank_values=True)) args_odict = _parse_args_from_manifest(manifest, 'install', args=args_dict) - args_list = [ value[0] for value in args_odict.values() ] + args_list = [value[0] for value in args_odict.values()] args_list.append(app_instance_name) # Prepare env. var. to pass to script @@ -654,8 +652,8 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu # Tell the operation_logger to redact all password-type args # Also redact the % escaped version of the password that might appear in # the 'args' section of metadata (relevant for password with non-alphanumeric char) - data_to_redact = [ value[0] for value in args_odict.values() if value[1] == "password" ] - data_to_redact += [ urllib.quote(data) for data in data_to_redact if urllib.quote(data) != data ] + data_to_redact = [value[0] for value in args_odict.values() if value[1] == "password"] + data_to_redact += [urllib.quote(data) for data in data_to_redact if urllib.quote(data) != data] operation_logger.data_to_redact.extend(data_to_redact) operation_logger.related_to = [s for s in operation_logger.related_to if s[0] != "app"] @@ -736,7 +734,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu )[0] # Remove all permission in LDAP result = ldap.search(base='ou=permission,dc=yunohost,dc=org', - filter='(&(objectclass=permissionYnh)(cn=*.%s))' % app_instance_name, attrs=['cn']) + filter='(&(objectclass=permissionYnh)(cn=*.%s))' % app_instance_name, attrs=['cn']) permission_list = [p['cn'][0] for p in result] for l in permission_list: permission_remove(app_instance_name, l.split('.')[0], force=True) @@ -785,7 +783,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu domain = app_settings.get('domain', None) path = app_settings.get('path', None) if domain and path: - permission_update(app_instance_name, permission="main", add_url=[domain+path], sync_perm=False) + permission_update(app_instance_name, permission="main", add_url=[domain + path], sync_perm=False) permission_sync_to_user() @@ -818,7 +816,7 @@ def app_remove(operation_logger, app): # TODO: display fail messages from script try: shutil.rmtree('/tmp/yunohost_remove') - except: + except Exception: pass # Apply dirty patch to make php5 apps compatible with php7 (e.g. the remove @@ -864,7 +862,7 @@ def app_remove(operation_logger, app): raise YunohostError("this_action_broke_dpkg") -@is_unit_operation(['permission','app']) +@is_unit_operation(['permission', 'app']) def app_addaccess(operation_logger, apps, users=[]): """ Grant access right to users (everyone by default) @@ -878,12 +876,12 @@ def app_addaccess(operation_logger, apps, users=[]): permission = user_permission_update(operation_logger, app=apps, permission="main", add_username=users) - result = {p : v['main']['allowed_users'] for p, v in permission['permissions'].items()} + result = {p: v['main']['allowed_users'] for p, v in permission['permissions'].items()} return {'allowed_users': result} -@is_unit_operation(['permission','app']) +@is_unit_operation(['permission', 'app']) def app_removeaccess(operation_logger, apps, users=[]): """ Revoke access right to users (everyone by default) @@ -897,12 +895,12 @@ def app_removeaccess(operation_logger, apps, users=[]): permission = user_permission_update(operation_logger, app=apps, permission="main", del_username=users) - result = {p : v['main']['allowed_users'] for p, v in permission['permissions'].items()} + result = {p: v['main']['allowed_users'] for p, v in permission['permissions'].items()} return {'allowed_users': result} -@is_unit_operation(['permission','app']) +@is_unit_operation(['permission', 'app']) def app_clearaccess(operation_logger, apps): """ Reset access rights for the app @@ -915,10 +913,11 @@ def app_clearaccess(operation_logger, apps): permission = user_permission_clear(operation_logger, app=apps, permission="main") - result = {p : v['main']['allowed_users'] for p, v in permission['permissions'].items()} + result = {p: v['main']['allowed_users'] for p, v in permission['permissions'].items()} return {'allowed_users': result} + def app_debug(app): """ Display debug informations for an app @@ -1686,8 +1685,7 @@ def _get_app_config_panel(app_id): "panel": [], } - panels = filter(lambda (key, value): key not in ("name", "version") - and isinstance(value, OrderedDict), + panels = filter(lambda (key, value): key not in ("name", "version") and isinstance(value, OrderedDict), toml_config_panel.items()) for key, value in panels: @@ -1697,8 +1695,7 @@ def _get_app_config_panel(app_id): "sections": [], } - sections = filter(lambda (k, v): k not in ("name",) - and isinstance(v, OrderedDict), + sections = filter(lambda (k, v): k not in ("name",) and isinstance(v, OrderedDict), value.items()) for section_key, section_value in sections: @@ -1708,8 +1705,7 @@ def _get_app_config_panel(app_id): "options": [], } - options = filter(lambda (k, v): k not in ("name",) - and isinstance(v, OrderedDict), + options = filter(lambda (k, v): k not in ("name",) and isinstance(v, OrderedDict), section_value.items()) for option_key, option_value in options: @@ -2455,8 +2451,8 @@ def _parse_args_in_yunohost_format(args, action_args): # If there's only one "domain" and "path", validate that domain/path # is an available url and normalize the path. - domain_args = [ (name, value[0]) for name, value in args_dict.items() if value[1] == "domain" ] - path_args = [ (name, value[0]) for name, value in args_dict.items() if value[1] == "path" ] + domain_args = [(name, value[0]) for name, value in args_dict.items() if value[1] == "domain"] + path_args = [(name, value[0]) for name, value in args_dict.items() if value[1] == "path"] if len(domain_args) == 1 and len(path_args) == 1: @@ -2684,6 +2680,7 @@ def _load_appslist(): # ############################### # # + def is_true(arg): """ Convert a string into a boolean From b22ad09e63c5b0096aaac4ff1ca3b0401b3eaade Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 24 Aug 2019 12:52:09 +0200 Subject: [PATCH 6/9] Update src/yunohost/data_migrations/0010_migrate_to_apps_json.py Co-Authored-By: decentral1se --- src/yunohost/data_migrations/0010_migrate_to_apps_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/data_migrations/0010_migrate_to_apps_json.py b/src/yunohost/data_migrations/0010_migrate_to_apps_json.py index 1333e54e8..4358a7476 100644 --- a/src/yunohost/data_migrations/0010_migrate_to_apps_json.py +++ b/src/yunohost/data_migrations/0010_migrate_to_apps_json.py @@ -10,7 +10,7 @@ class MyMigration(Migration): def migrate(self): - logger.info("This is migration is oudated and doesn't do anything anymore. The migration 12 will handle this instead.") + logger.info("This migration is oudated and doesn't do anything anymore. The migration 12 will handle this instead.") pass def backward(self): From d764b02701f22e6e007e4fb2de6b57c37bcc5306 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 24 Aug 2019 17:53:32 +0200 Subject: [PATCH 7/9] Annnnd I forgot to add the migration --- .../0012_futureproof_appslist_system.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 src/yunohost/data_migrations/0012_futureproof_appslist_system.py diff --git a/src/yunohost/data_migrations/0012_futureproof_appslist_system.py b/src/yunohost/data_migrations/0012_futureproof_appslist_system.py new file mode 100644 index 000000000..e0bf70d04 --- /dev/null +++ b/src/yunohost/data_migrations/0012_futureproof_appslist_system.py @@ -0,0 +1,46 @@ + +import os +import shutil + +from moulinette.utils.log import getActionLogger +from moulinette.utils.filesystem import read_json + +from yunohost.tools import Migration +from yunohost.app import (_initialize_appslists_system, + _update_appslist, + APPSLISTS_CACHE, + APPSLISTS_CONF) + +logger = getActionLogger('yunohost.migration') + +LEGACY_APPSLISTS_CONF = '/etc/yunohost/appslists.json' +LEGACY_APPSLISTS_CONF_BACKUP = LEGACY_APPSLISTS_CONF + ".old" + + +class MyMigration(Migration): + + "Migrate to the new future-proof appslist system" + + def migrate(self): + + if not os.path.exists(LEGACY_APPSLISTS_CONF): + logger.info("No need to do anything") + + # Destroy old lecacy cache + if os.path.exists(APPSLISTS_CACHE): + shutil.rmtree(APPSLISTS_CACHE) + + # Backup the legacy file + try: + legacy_list = read_json(LEGACY_APPSLISTS_CONF) + # If there's only one list, we assume it's just the old official list + # Otherwise, warn the (power-?)users that they should migrate their old list manually + if len(legacy_list) > 1: + logger.warning("It looks like you had additional appslist in the configuration file %s! YunoHost now uses %s instead, but it won't migrate your custom appslist. You should do this manually. The old file has been backuped in %s." % (LEGACY_APPSLISTS_CONF, APPSLISTS_CONF, LEGACY_APPSLISTS_CONF_BACKUP)) + except Exception as e: + logger.warning("Unable to parse the legacy conf %s (error : %s) ... migrating anyway" % (LEGACY_APPSLISTS_CONF, str(e))) + + os.rename(LEGACY_APPSLISTS_CONF, LEGACY_APPSLISTS_CONF_BACKUP) + + _initialize_appslists_system() + _update_appslist() From 66c5ad18c9f66ba0fcea99683d69de98a31a86b0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 8 Nov 2019 23:17:32 +0100 Subject: [PATCH 8/9] Migration 12 -> 13 --- locales/en.json | 2 +- src/yunohost/data_migrations/0010_migrate_to_apps_json.py | 6 +++--- ...pslist_system.py => 0013_futureproof_appslist_system.py} | 0 3 files changed, 4 insertions(+), 4 deletions(-) rename src/yunohost/data_migrations/{0012_futureproof_appslist_system.py => 0013_futureproof_appslist_system.py} (100%) diff --git a/locales/en.json b/locales/en.json index 6f4df6cd8..4ad404da7 100644 --- a/locales/en.json +++ b/locales/en.json @@ -313,7 +313,7 @@ "migration_description_0010_migrate_to_apps_json": "Remove deprecated applists and use the new unified 'apps.json' list instead (outdated, replaced by migration 12)", "migration_description_0011_setup_group_permission": "Set up user group and set up permission for apps and services", "migration_description_0012_postgresql_password_to_md5_authentication": "Force PostgreSQL authentication to use MD5 for local connections", - "migration_description_0012_futureproof_appslist_system": "Migrate to the new future-proof appslist system", + "migration_description_0013_futureproof_appslist_system": "Migrate to the new future-proof appslist system", "migration_0003_start": "Starting migration to Stretch. The logs will be available in {logfile}.", "migration_0003_patching_sources_list": "Patching the sources.lists…", "migration_0003_main_upgrade": "Starting main upgrade…", diff --git a/src/yunohost/data_migrations/0010_migrate_to_apps_json.py b/src/yunohost/data_migrations/0010_migrate_to_apps_json.py index c83408ce8..e5ce65608 100644 --- a/src/yunohost/data_migrations/0010_migrate_to_apps_json.py +++ b/src/yunohost/data_migrations/0010_migrate_to_apps_json.py @@ -6,8 +6,8 @@ logger = getActionLogger('yunohost.migration') class MyMigration(Migration): - "Migrate from official.json to apps.json (outdated, replaced by migration 12)" + "Migrate from official.json to apps.json (outdated, replaced by migration 13)" def run(self): - logger.info("This migration is oudated and doesn't do anything anymore. The migration 12 will handle this instead.") - pass \ No newline at end of file + logger.info("This migration is oudated and doesn't do anything anymore. The migration 13 will handle this instead.") + pass diff --git a/src/yunohost/data_migrations/0012_futureproof_appslist_system.py b/src/yunohost/data_migrations/0013_futureproof_appslist_system.py similarity index 100% rename from src/yunohost/data_migrations/0012_futureproof_appslist_system.py rename to src/yunohost/data_migrations/0013_futureproof_appslist_system.py From 3b7899db8cd0482ac606837468ff8cc35b285451 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 9 Nov 2019 00:06:50 +0100 Subject: [PATCH 9/9] appslists -> apps catalog --- locales/en.json | 17 +- src/yunohost/app.py | 128 +++---- .../0013_futureproof_apps_catalog_system.py | 46 +++ .../0013_futureproof_appslist_system.py | 46 --- src/yunohost/tests/test_appscatalog.py | 358 ++++++++++++++++++ src/yunohost/tests/test_appslist.py | 358 ------------------ src/yunohost/tools.py | 13 +- 7 files changed, 482 insertions(+), 484 deletions(-) create mode 100644 src/yunohost/data_migrations/0013_futureproof_apps_catalog_system.py delete mode 100644 src/yunohost/data_migrations/0013_futureproof_appslist_system.py create mode 100644 src/yunohost/tests/test_appscatalog.py delete mode 100644 src/yunohost/tests/test_appslist.py diff --git a/locales/en.json b/locales/en.json index c3d636c2f..027d9fc0b 100644 --- a/locales/en.json +++ b/locales/en.json @@ -55,11 +55,11 @@ "apps_already_up_to_date": "All apps are already up-to-date", "apps_permission_not_found": "No permission found for the installed apps", "apps_permission_restoration_failed": "Permission '{permission:s}' for app {app:s} restoration has failed", - "appslist_init_success": "Appslist system initialized!", - "appslist_updating": "Updating application list...", - "appslist_failed_to_download": "Unable to download the {applist} appslist : {error}", - "appslist_obsolete_cache": "The applist cache is empty or obsolete.", - "appslist_update_success": "The application list has been updated!", + "apps_catalog_init_success": "Apps catalog system initialized!", + "apps_catalog_updating": "Updating applications catalog...", + "apps_catalog_failed_to_download": "Unable to download the {apps_catalog} apps catalog: {error}", + "apps_catalog_obsolete_cache": "The apps catalog cache is empty or obsolete.", + "apps_catalog_update_success": "The application catalog has been updated!", "ask_current_admin_password": "Current administration password", "ask_email": "E-mail address", "ask_firstname": "First name", @@ -144,7 +144,6 @@ "confirm_app_install_danger": "DANGER! This app is known to be still experimental (if not explicitly not working)! You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or break your system… If you are willing to take that risk anyway, type '{answers:s}'", "confirm_app_install_thirdparty": "DANGER! This app is not part of Yunohost's app catalog. Installing third-party apps may compromise the integrity and security of your system. You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or break your system… If you are willing to take that risk anyway, type '{answers:s}'", "custom_app_url_required": "You must provide a URL to upgrade your custom app {app:s}", - "custom_appslist_name_required": "You must provide a name for your custom app list", "diagnosis_basesystem_host": "Server is running Debian {debian_version}.", "diagnosis_basesystem_kernel": "Server is running Linux kernel {kernel_version}", "diagnosis_basesystem_ynh_single_version": "{0} version: {1} ({2})", @@ -374,10 +373,10 @@ "migration_description_0007_ssh_conf_managed_by_yunohost_step1": "Let the SSH configuration be managed by YunoHost (step 1, automatic)", "migration_description_0008_ssh_conf_managed_by_yunohost_step2": "Let the SSH configuration be managed by YunoHost (step 2, manual)", "migration_description_0009_decouple_regenconf_from_services": "Decouple the regen-conf mechanism from services", - "migration_description_0010_migrate_to_apps_json": "Remove deprecated applists and use the new unified 'apps.json' list instead (outdated, replaced by migration 12)", + "migration_description_0010_migrate_to_apps_json": "Remove deprecated apps catalogs and use the new unified 'apps.json' list instead (outdated, replaced by migration 13)", "migration_description_0011_setup_group_permission": "Set up user group and set up permission for apps and services", "migration_description_0012_postgresql_password_to_md5_authentication": "Force PostgreSQL authentication to use MD5 for local connections", - "migration_description_0013_futureproof_appslist_system": "Migrate to the new future-proof appslist system", + "migration_description_0013_futureproof_apps_catalog_system": "Migrate to the new future-proof apps catalog system", "migration_0003_start": "Starting migration to Stretch. The logs will be available in {logfile}.", "migration_0003_patching_sources_list": "Patching the sources.lists…", "migration_0003_main_upgrade": "Starting main upgrade…", @@ -388,7 +387,7 @@ "migration_0003_system_not_fully_up_to_date": "Your system is not fully up-to-date. Please perform a regular upgrade before running the migration to Stretch.", "migration_0003_still_on_jessie_after_main_upgrade": "Something went wrong during the main upgrade: Is the system still on Jessie‽ To investigate the issue, please look at {log}:s…", "migration_0003_general_warning": "Please note that this migration is a delicate operation. The YunoHost team did its best to review and test it, but the migration might still break parts of the system or its apps.\n\nTherefore, it is recommended to:\n - Perform a backup of any critical data or app. More info on https://yunohost.org/backup;\n - Be patient after launching the migration: Depending on your Internet connection and hardware, it might take up to a few hours for everything to upgrade.\n\nAdditionally, the port for SMTP, used by external e-mail clients (like Thunderbird or K9-Mail) was changed from 465 (SSL/TLS) to 587 (STARTTLS). The old port (465) will automatically be closed, and the new port (587) will be opened in the firewall. You and your users *will* have to adapt the configuration of your e-mail clients accordingly.", - "migration_0003_problematic_apps_warning": "Please note that the following possibly problematic installed apps were detected. It looks like those were not installed from an applist, or are not flagged as 'working'. Consequently, it cannot be guaranteed that they will still work after the upgrade: {problematic_apps}", + "migration_0003_problematic_apps_warning": "Please note that the following possibly problematic installed apps were detected. It looks like those were not installed from an apps_catalog, or are not flagged as 'working'. Consequently, it cannot be guaranteed that they will still work after the upgrade: {problematic_apps}", "migration_0003_modified_files": "Please note that the following files were found to be manually modified and might be overwritten following the upgrade: {manually_modified_files}", "migration_0005_postgresql_94_not_installed": "PostgreSQL was not installed on your system. Nothing to do.", "migration_0005_postgresql_96_not_installed": "PostgreSQL 9.4 is installed, but not postgresql 9.6‽ Something weird might have happened on your system:(…", diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 73f96ba0b..d17ea8424 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -54,11 +54,11 @@ APPS_SETTING_PATH = '/etc/yunohost/apps/' INSTALL_TMP = '/var/cache/yunohost' APP_TMP_FOLDER = INSTALL_TMP + '/from_file' -APPSLISTS_CACHE = '/var/cache/yunohost/repo' -APPSLISTS_CONF = '/etc/yunohost/appslists.yml' -APPSLISTS_CRON_PATH = "/etc/cron.daily/yunohost-fetch-appslists" -APPSLISTS_API_VERSION = 1 -APPSLISTS_DEFAULT_URL = "https://app.yunohost.org/default" +APPS_CATALOG_CACHE = '/var/cache/yunohost/repo' +APPS_CATALOG_CONF = '/etc/yunohost/apps_catalog.yml' +APPS_CATALOG_CRON_PATH = "/etc/cron.daily/yunohost-fetch-apps-catalog" +APPS_CATALOG_API_VERSION = 1 +APPS_CATALOG_DEFAULT_URL = "https://app.yunohost.org/default" re_github_repo = re.compile( r'^(http[s]?://|git@)github.com[/:]' @@ -88,8 +88,8 @@ def app_list(filter=None, raw=False, installed=False, with_backup=False): list_dict = {} if raw else [] - # Get app list from applist cache - app_dict = _load_appslist() + # Get app list from catalog cache + app_dict = _load_apps_catalog() # Get app list from the app settings directory for app in os.listdir(APPS_SETTING_PATH): @@ -2740,144 +2740,144 @@ def _parse_app_instance_name(app_instance_name): # -def _initialize_appslists_system(): +def _initialize_apps_catalog_system(): """ - This function is meant to intialize the appslist system with YunoHost's default applist. + This function is meant to intialize the apps_catalog system with YunoHost's default app catalog. It also creates the cron job that will update the list every day """ - default_appslist_list = [{"id": "default", "url": APPSLISTS_DEFAULT_URL}] + default_apps_catalog_list = [{"id": "default", "url": APPS_CATALOG_DEFAULT_URL}] cron_job = [] cron_job.append("#!/bin/bash") # We add a random delay between 0 and 60 min to avoid every instance fetching - # the appslist at the same time every night + # the apps catalog at the same time every night cron_job.append("(sleep $((RANDOM%3600));") cron_job.append("yunohost tools update --apps > /dev/null) &") try: - logger.debug("Initializing appslist system with YunoHost's default app list") - write_to_yaml(APPSLISTS_CONF, default_appslist_list) + logger.debug("Initializing apps catalog system with YunoHost's default app list") + write_to_yaml(APPS_CATALOG_CONF, default_apps_catalog_list) - logger.debug("Installing appslist fetch daily cron job") - write_to_file(APPSLISTS_CRON_PATH, '\n'.join(cron_job)) - chown(APPSLISTS_CRON_PATH, uid="root", gid="root") - chmod(APPSLISTS_CRON_PATH, 0o755) + logger.debug("Installing apps catalog fetch daily cron job") + write_to_file(APPS_CATALOG_CRON_PATH, '\n'.join(cron_job)) + chown(APPS_CATALOG_CRON_PATH, uid="root", gid="root") + chmod(APPS_CATALOG_CRON_PATH, 0o755) except Exception as e: - raise YunohostError("Could not initialize the appslist system... : %s" % str(e)) + raise YunohostError("Could not initialize the apps catalog system... : %s" % str(e)) - logger.success(m18n.n("appslist_init_success")) + logger.success(m18n.n("apps_catalog_init_success")) -def _read_appslist_list(): +def _read_apps_catalog_list(): """ - Read the json corresponding to the list of appslists + Read the json corresponding to the list of apps catalogs """ # Legacy code - can be removed after moving to buster (if the migration got merged before buster) if os.path.exists('/etc/yunohost/appslists.json'): from yunohost.tools import _get_migration_by_name - migration = _get_migration_by_name("futureproof_appslist_system") + migration = _get_migration_by_name("futureproof_apps_catalog_system") migration.migrate() try: - list_ = read_yaml(APPSLISTS_CONF) + list_ = read_yaml(APPS_CATALOG_CONF) # Support the case where file exists but is empty # by returning [] if list_ is None return list_ if list_ else [] except Exception as e: - raise YunohostError("Could not read the appslist list ... : %s" % str(e)) + raise YunohostError("Could not read the apps_catalog list ... : %s" % str(e)) -def _actual_appslist_api_url(base_url): +def _actual_apps_catalog_api_url(base_url): - return "{base_url}/v{version}/apps.json".format(base_url=base_url, version=APPSLISTS_API_VERSION) + return "{base_url}/v{version}/apps.json".format(base_url=base_url, version=APPS_CATALOG_API_VERSION) -def _update_appslist(): +def _update_apps_catalog(): """ - Fetches the json for each appslist and update the cache + Fetches the json for each apps_catalog and update the cache - appslist_list is for example : + apps_catalog_list is for example : [ {"id": "default", "url": "https://app.yunohost.org/default/"} ] - Then for each appslist, the actual json URL to be fetched is like : + Then for each apps_catalog, the actual json URL to be fetched is like : https://app.yunohost.org/default/vX/apps.json And store it in : /var/cache/yunohost/repo/default.json """ - appslist_list = _read_appslist_list() + apps_catalog_list = _read_apps_catalog_list() - logger.info(m18n.n("appslist_updating")) + logger.info(m18n.n("apps_catalog_updating")) # Create cache folder if needed - if not os.path.exists(APPSLISTS_CACHE): - logger.debug("Initialize folder for appslist cache") - mkdir(APPSLISTS_CACHE, mode=0o750, parents=True, uid='root') + if not os.path.exists(APPS_CATALOG_CACHE): + logger.debug("Initialize folder for apps catalog cache") + mkdir(APPS_CATALOG_CACHE, mode=0o750, parents=True, uid='root') - for appslist in appslist_list: - applist_id = appslist["id"] - actual_api_url = _actual_appslist_api_url(appslist["url"]) + for apps_catalog in apps_catalog_list: + apps_catalog_id = apps_catalog["id"] + actual_api_url = _actual_apps_catalog_api_url(apps_catalog["url"]) # Fetch the json try: - appslist_content = download_json(actual_api_url) + apps_catalog_content = download_json(actual_api_url) except Exception as e: - raise YunohostError("appslist_failed_to_download", applist=applist_id, error=str(e)) + raise YunohostError("apps_catalog_failed_to_download", apps_catalog=apps_catalog_id, error=str(e)) - # Remember the appslist api version for later - appslist_content["from_api_version"] = APPSLISTS_API_VERSION + # Remember the apps_catalog api version for later + apps_catalog_content["from_api_version"] = APPS_CATALOG_API_VERSION - # Save the appslist data in the cache - cache_file = "{cache_folder}/{list}.json".format(cache_folder=APPSLISTS_CACHE, list=applist_id) + # Save the apps_catalog data in the cache + cache_file = "{cache_folder}/{list}.json".format(cache_folder=APPS_CATALOG_CACHE, list=apps_catalog_id) try: - write_to_json(cache_file, appslist_content) + write_to_json(cache_file, apps_catalog_content) except Exception as e: - raise YunohostError("Unable to write cache data for %s appslist : %s" % (applist_id, str(e))) + raise YunohostError("Unable to write cache data for %s apps_catalog : %s" % (apps_catalog_id, str(e))) - logger.success(m18n.n("appslist_update_success")) + logger.success(m18n.n("apps_catalog_update_success")) -def _load_appslist(): +def _load_apps_catalog(): """ - Read all the appslist cache file and build a single dict (app_dict) + Read all the apps catalog cache files and build a single dict (app_dict) corresponding to all known apps in all indexes """ app_dict = {} - for appslist_id in [L["id"] for L in _read_appslist_list()]: + for apps_catalog_id in [L["id"] for L in _read_apps_catalog_list()]: - # Let's load the json from cache for this appslist - cache_file = "{cache_folder}/{list}.json".format(cache_folder=APPSLISTS_CACHE, list=appslist_id) + # Let's load the json from cache for this catalog + cache_file = "{cache_folder}/{list}.json".format(cache_folder=APPS_CATALOG_CACHE, list=apps_catalog_id) try: - appslist_content = read_json(cache_file) if os.path.exists(cache_file) else None + apps_catalog_content = read_json(cache_file) if os.path.exists(cache_file) else None except Exception as e: - raise ("Unable to read cache for appslist %s : %s" % (appslist_id, str(e))) + raise ("Unable to read cache for apps_catalog %s : %s" % (apps_catalog_id, str(e))) # Check that the version of the data matches version .... # ... otherwise it means we updated yunohost in the meantime # and need to update the cache for everything to be consistent - if not appslist_content or appslist_content.get("from_api_version") != APPSLISTS_API_VERSION: - logger.info(m18n.n("appslist_obsolete_cache")) - _update_appslist() - appslist_content = read_json(cache_file) + if not apps_catalog_content or apps_catalog_content.get("from_api_version") != APPS_CATALOG_API_VERSION: + logger.info(m18n.n("apps_catalog_obsolete_cache")) + _update_apps_catalog() + apps_catalog_content = read_json(cache_file) - del appslist_content["from_api_version"] + del apps_catalog_content["from_api_version"] - # Add apps from this applist to the output - for app, info in appslist_content.items(): + # Add apps from this catalog to the output + for app, info in apps_catalog_content.items(): - # (N.B. : there's a small edge case where multiple appslist could be listing the same apps ... + # (N.B. : there's a small edge case where multiple apps catalog could be listing the same apps ... # in which case we keep only the first one found) if app in app_dict: - logger.warning("Duplicate app %s found between appslist %s and %s" % (app, appslist_id, app_dict[app]['repository'])) + logger.warning("Duplicate app %s found between apps catalog %s and %s" % (app, apps_catalog_id, app_dict[app]['repository'])) continue - info['repository'] = appslist_id + info['repository'] = apps_catalog_id app_dict[app] = info return app_dict diff --git a/src/yunohost/data_migrations/0013_futureproof_apps_catalog_system.py b/src/yunohost/data_migrations/0013_futureproof_apps_catalog_system.py new file mode 100644 index 000000000..2215d4681 --- /dev/null +++ b/src/yunohost/data_migrations/0013_futureproof_apps_catalog_system.py @@ -0,0 +1,46 @@ + +import os +import shutil + +from moulinette.utils.log import getActionLogger +from moulinette.utils.filesystem import read_json + +from yunohost.tools import Migration +from yunohost.app import (_initialize_apps_catalog_system, + _update_apps_catalog, + APPS_CATALOG_CACHE, + APPS_CATALOG_CONF) + +logger = getActionLogger('yunohost.migration') + +LEGACY_APPS_CATALOG_CONF = '/etc/yunohost/appslists.json' +LEGACY_APPS_CATALOG_CONF_BACKUP = LEGACY_APPS_CATALOG_CONF + ".old" + + +class MyMigration(Migration): + + "Migrate to the new future-proof apps catalog system" + + def migrate(self): + + if not os.path.exists(LEGACY_APPS_CATALOG_CONF): + logger.info("No need to do anything") + + # Destroy old lecacy cache + if os.path.exists(APPS_CATALOG_CACHE): + shutil.rmtree(APPS_CATALOG_CACHE) + + # Backup the legacy file + try: + legacy_catalogs = read_json(LEGACY_APPS_CATALOG_CONF) + # If there's only one catalog, we assume it's just the old official catalog + # Otherwise, warn the (power-?)users that they should migrate their old catalogs manually + if len(legacy_catalogs) > 1: + logger.warning("It looks like you had additional apps_catalog in the configuration file %s! YunoHost now uses %s instead, but it won't migrate your custom apps_catalog. You should do this manually. The old file has been backuped in %s." % (LEGACY_APPS_CATALOG_CONF, APPS_CATALOG_CONF, LEGACY_APPS_CATALOG_CONF_BACKUP)) + except Exception as e: + logger.warning("Unable to parse the legacy conf %s (error : %s) ... migrating anyway" % (LEGACY_APPS_CATALOG_CONF, str(e))) + + os.rename(LEGACY_APPS_CATALOG_CONF, LEGACY_APPS_CATALOG_CONF_BACKUP) + + _initialize_apps_catalog_system() + _update_apps_catalog() diff --git a/src/yunohost/data_migrations/0013_futureproof_appslist_system.py b/src/yunohost/data_migrations/0013_futureproof_appslist_system.py deleted file mode 100644 index e0bf70d04..000000000 --- a/src/yunohost/data_migrations/0013_futureproof_appslist_system.py +++ /dev/null @@ -1,46 +0,0 @@ - -import os -import shutil - -from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import read_json - -from yunohost.tools import Migration -from yunohost.app import (_initialize_appslists_system, - _update_appslist, - APPSLISTS_CACHE, - APPSLISTS_CONF) - -logger = getActionLogger('yunohost.migration') - -LEGACY_APPSLISTS_CONF = '/etc/yunohost/appslists.json' -LEGACY_APPSLISTS_CONF_BACKUP = LEGACY_APPSLISTS_CONF + ".old" - - -class MyMigration(Migration): - - "Migrate to the new future-proof appslist system" - - def migrate(self): - - if not os.path.exists(LEGACY_APPSLISTS_CONF): - logger.info("No need to do anything") - - # Destroy old lecacy cache - if os.path.exists(APPSLISTS_CACHE): - shutil.rmtree(APPSLISTS_CACHE) - - # Backup the legacy file - try: - legacy_list = read_json(LEGACY_APPSLISTS_CONF) - # If there's only one list, we assume it's just the old official list - # Otherwise, warn the (power-?)users that they should migrate their old list manually - if len(legacy_list) > 1: - logger.warning("It looks like you had additional appslist in the configuration file %s! YunoHost now uses %s instead, but it won't migrate your custom appslist. You should do this manually. The old file has been backuped in %s." % (LEGACY_APPSLISTS_CONF, APPSLISTS_CONF, LEGACY_APPSLISTS_CONF_BACKUP)) - except Exception as e: - logger.warning("Unable to parse the legacy conf %s (error : %s) ... migrating anyway" % (LEGACY_APPSLISTS_CONF, str(e))) - - os.rename(LEGACY_APPSLISTS_CONF, LEGACY_APPSLISTS_CONF_BACKUP) - - _initialize_appslists_system() - _update_appslist() diff --git a/src/yunohost/tests/test_appscatalog.py b/src/yunohost/tests/test_appscatalog.py new file mode 100644 index 000000000..450c2846e --- /dev/null +++ b/src/yunohost/tests/test_appscatalog.py @@ -0,0 +1,358 @@ +import os +import pytest +import requests +import requests_mock +import glob +import shutil + +from moulinette import m18n +from moulinette.utils.filesystem import read_json, write_to_json, write_to_yaml, mkdir + +from yunohost.utils.error import YunohostError +from yunohost.app import (_initialize_apps_catalog_system, + _read_apps_catalog_list, + _update_apps_catalog, + _actual_apps_catalog_api_url, + _load_apps_catalog, + logger, + APPS_CATALOG_CACHE, + APPS_CATALOG_CONF, + APPS_CATALOG_CRON_PATH, + APPS_CATALOG_API_VERSION, + APPS_CATALOG_DEFAULT_URL) + +APPS_CATALOG_DEFAULT_URL_FULL = _actual_apps_catalog_api_url(APPS_CATALOG_DEFAULT_URL) +CRON_FOLDER, CRON_NAME = APPS_CATALOG_CRON_PATH.rsplit("/", 1) + +DUMMY_APP_CATALOG = """{ + "foo": {"id": "foo", "level": 4}, + "bar": {"id": "bar", "level": 7} +} +""" + +class AnyStringWith(str): + def __eq__(self, other): + return self in other + +def setup_function(function): + + # Clear apps catalog cache + shutil.rmtree(APPS_CATALOG_CACHE, ignore_errors=True) + + # Clear apps_catalog cron + if os.path.exists(APPS_CATALOG_CRON_PATH): + os.remove(APPS_CATALOG_CRON_PATH) + + # Clear apps_catalog conf + if os.path.exists(APPS_CATALOG_CONF): + os.remove(APPS_CATALOG_CONF) + + +def teardown_function(function): + + # Clear apps catalog cache + # Otherwise when using apps stuff after running the test, + # we'll still have the dummy unusable list + shutil.rmtree(APPS_CATALOG_CACHE, ignore_errors=True) + + +def cron_job_is_there(): + r = os.system("run-parts -v --test %s | grep %s" % (CRON_FOLDER, CRON_NAME)) + return r == 0 + +# +# ################################################ +# + + +def test_apps_catalog_init(mocker): + + # Cache is empty + assert not glob.glob(APPS_CATALOG_CACHE + "/*") + # Conf doesn't exist yet + assert not os.path.exists(APPS_CATALOG_CONF) + # Conf doesn't exist yet + assert not os.path.exists(APPS_CATALOG_CRON_PATH) + + # Initialize ... + mocker.spy(m18n, "n") + _initialize_apps_catalog_system() + m18n.n.assert_any_call('apps_catalog_init_success') + + # Then there's a cron enabled + assert cron_job_is_there() + + # And a conf with at least one list + assert os.path.exists(APPS_CATALOG_CONF) + apps_catalog_list = _read_apps_catalog_list() + assert len(apps_catalog_list) + + # Cache is expected to still be empty though + # (if we did update the apps_catalog during init, + # we couldn't differentiate easily exceptions + # related to lack of network connectivity) + assert not glob.glob(APPS_CATALOG_CACHE + "/*") + + +def test_apps_catalog_emptylist(): + + # Initialize ... + _initialize_apps_catalog_system() + + # Let's imagine somebody removed the default apps catalog because uh idk they dont want to use our default apps catalog + os.system("rm %s" % APPS_CATALOG_CONF) + os.system("touch %s" % APPS_CATALOG_CONF) + + apps_catalog_list = _read_apps_catalog_list() + assert not len(apps_catalog_list) + + +def test_apps_catalog_update_success(mocker): + + # Initialize ... + _initialize_apps_catalog_system() + + # Cache is empty + assert not glob.glob(APPS_CATALOG_CACHE + "/*") + + # Update + with requests_mock.Mocker() as m: + + _actual_apps_catalog_api_url, + # Mock the server response with a dummy apps catalog + m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG) + + mocker.spy(m18n, "n") + _update_apps_catalog() + m18n.n.assert_any_call("apps_catalog_updating") + m18n.n.assert_any_call("apps_catalog_update_success") + + # Cache shouldn't be empty anymore empty + assert glob.glob(APPS_CATALOG_CACHE + "/*") + + app_dict = _load_apps_catalog() + assert "foo" in app_dict.keys() + assert "bar" in app_dict.keys() + + +def test_apps_catalog_update_404(mocker): + + # Initialize ... + _initialize_apps_catalog_system() + + with requests_mock.Mocker() as m: + + # 404 error + m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, + status_code=404) + + with pytest.raises(YunohostError): + mocker.spy(m18n, "n") + _update_apps_catalog() + m18n.n.assert_any_call("apps_catalog_failed_to_download") + +def test_apps_catalog_update_timeout(mocker): + + # Initialize ... + _initialize_apps_catalog_system() + + with requests_mock.Mocker() as m: + + # Timeout + m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, + exc=requests.exceptions.ConnectTimeout) + + with pytest.raises(YunohostError): + mocker.spy(m18n, "n") + _update_apps_catalog() + m18n.n.assert_any_call("apps_catalog_failed_to_download") + + +def test_apps_catalog_update_sslerror(mocker): + + # Initialize ... + _initialize_apps_catalog_system() + + with requests_mock.Mocker() as m: + + # SSL error + m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, + exc=requests.exceptions.SSLError) + + with pytest.raises(YunohostError): + mocker.spy(m18n, "n") + _update_apps_catalog() + m18n.n.assert_any_call("apps_catalog_failed_to_download") + + +def test_apps_catalog_update_corrupted(mocker): + + # Initialize ... + _initialize_apps_catalog_system() + + with requests_mock.Mocker() as m: + + # Corrupted json + m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, + text=DUMMY_APP_CATALOG[:-2]) + + with pytest.raises(YunohostError): + mocker.spy(m18n, "n") + _update_apps_catalog() + m18n.n.assert_any_call("apps_catalog_failed_to_download") + + +def test_apps_catalog_load_with_empty_cache(mocker): + + # Initialize ... + _initialize_apps_catalog_system() + + # Cache is empty + assert not glob.glob(APPS_CATALOG_CACHE + "/*") + + # Update + with requests_mock.Mocker() as m: + + # Mock the server response with a dummy apps catalog + m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG) + + # Try to load the apps catalog + # This should implicitly trigger an update in the background + mocker.spy(m18n, "n") + app_dict = _load_apps_catalog() + m18n.n.assert_any_call("apps_catalog_obsolete_cache") + m18n.n.assert_any_call("apps_catalog_update_success") + + + # Cache shouldn't be empty anymore empty + assert glob.glob(APPS_CATALOG_CACHE + "/*") + + assert "foo" in app_dict.keys() + assert "bar" in app_dict.keys() + + +def test_apps_catalog_load_with_conflicts_between_lists(mocker): + + # Initialize ... + _initialize_apps_catalog_system() + + conf = [{"id": "default", "url": APPS_CATALOG_DEFAULT_URL}, + {"id": "default2", "url": APPS_CATALOG_DEFAULT_URL.replace("yunohost.org", "yolohost.org")}] + + write_to_yaml(APPS_CATALOG_CONF, conf) + + # Update + with requests_mock.Mocker() as m: + + # Mock the server response with a dummy apps catalog + # + the same apps catalog for the second list + m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG) + m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL.replace("yunohost.org", "yolohost.org"), text=DUMMY_APP_CATALOG) + + # Try to load the apps catalog + # This should implicitly trigger an update in the background + mocker.spy(logger, "warning") + app_dict = _load_apps_catalog() + logger.warning.assert_any_call(AnyStringWith("Duplicate")) + + # Cache shouldn't be empty anymore empty + assert glob.glob(APPS_CATALOG_CACHE + "/*") + + assert "foo" in app_dict.keys() + assert "bar" in app_dict.keys() + + +def test_apps_catalog_load_with_oudated_api_version(mocker): + + # Initialize ... + _initialize_apps_catalog_system() + + # Update + with requests_mock.Mocker() as m: + + mocker.spy(m18n, "n") + m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG) + _update_apps_catalog() + + # Cache shouldn't be empty anymore empty + assert glob.glob(APPS_CATALOG_CACHE + "/*") + + # Tweak the cache to replace the from_api_version with a different one + for cache_file in glob.glob(APPS_CATALOG_CACHE + "/*"): + cache_json = read_json(cache_file) + assert cache_json["from_api_version"] == APPS_CATALOG_API_VERSION + cache_json["from_api_version"] = 0 + write_to_json(cache_file, cache_json) + + # Update + with requests_mock.Mocker() as m: + + # Mock the server response with a dummy apps catalog + m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG) + + mocker.spy(m18n, "n") + app_dict = _load_apps_catalog() + m18n.n.assert_any_call("apps_catalog_update_success") + + assert "foo" in app_dict.keys() + assert "bar" in app_dict.keys() + + # Check that we indeed have the new api number in cache + for cache_file in glob.glob(APPS_CATALOG_CACHE + "/*"): + cache_json = read_json(cache_file) + assert cache_json["from_api_version"] == APPS_CATALOG_API_VERSION + + + +def test_apps_catalog_migrate_legacy_explicitly(): + + open("/etc/yunohost/appslists.json", "w").write('{"yunohost": {"yolo":"swag"}}') + mkdir(APPS_CATALOG_CACHE, 0o750, parents=True) + open(APPS_CATALOG_CACHE+"/yunohost_old.json", "w").write('{"foo":{}, "bar": {}}') + open(APPS_CATALOG_CRON_PATH, "w").write("# Some old cron") + + from yunohost.tools import _get_migration_by_name + migration = _get_migration_by_name("futureproof_apps_catalog_system") + + with requests_mock.Mocker() as m: + + # Mock the server response with a dummy apps catalog + m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG) + migration.migrate() + + # Old conf shouldnt be there anymore (got renamed to .old) + assert not os.path.exists("/etc/yunohost/appslists.json") + # Old cache should have been removed + assert not os.path.exists(APPS_CATALOG_CACHE+"/yunohost_old.json") + # Cron should have been changed + assert "/bin/bash" in open(APPS_CATALOG_CRON_PATH, "r").read() + assert cron_job_is_there() + + # Reading the apps_catalog should work + app_dict = _load_apps_catalog() + assert "foo" in app_dict.keys() + assert "bar" in app_dict.keys() + + +def test_apps_catalog_migrate_legacy_implicitly(): + + open("/etc/yunohost/appslists.json", "w").write('{"yunohost": {"yolo":"swag"}}') + mkdir(APPS_CATALOG_CACHE, 0o750, parents=True) + open(APPS_CATALOG_CACHE+"/yunohost_old.json", "w").write('{"old_foo":{}, "old_bar": {}}') + open(APPS_CATALOG_CRON_PATH, "w").write("# Some old cron") + + with requests_mock.Mocker() as m: + m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG) + app_dict = _load_apps_catalog() + + assert "foo" in app_dict.keys() + assert "bar" in app_dict.keys() + + # Old conf shouldnt be there anymore (got renamed to .old) + assert not os.path.exists("/etc/yunohost/appslists.json") + # Old cache should have been removed + assert not os.path.exists(APPS_CATALOG_CACHE+"/yunohost_old.json") + # Cron should have been changed + assert "/bin/bash" in open(APPS_CATALOG_CRON_PATH, "r").read() + assert cron_job_is_there() + diff --git a/src/yunohost/tests/test_appslist.py b/src/yunohost/tests/test_appslist.py deleted file mode 100644 index d7b8e429b..000000000 --- a/src/yunohost/tests/test_appslist.py +++ /dev/null @@ -1,358 +0,0 @@ -import os -import pytest -import requests -import requests_mock -import glob -import shutil - -from moulinette import m18n -from moulinette.utils.filesystem import read_json, write_to_json, write_to_yaml, mkdir - -from yunohost.utils.error import YunohostError -from yunohost.app import (_initialize_appslists_system, - _read_appslist_list, - _update_appslist, - _actual_appslist_api_url, - _load_appslist, - logger, - APPSLISTS_CACHE, - APPSLISTS_CONF, - APPSLISTS_CRON_PATH, - APPSLISTS_API_VERSION, - APPSLISTS_DEFAULT_URL) - -APPSLISTS_DEFAULT_URL_FULL = _actual_appslist_api_url(APPSLISTS_DEFAULT_URL) -CRON_FOLDER, CRON_NAME = APPSLISTS_CRON_PATH.rsplit("/", 1) - -DUMMY_APPLIST = """{ - "foo": {"id": "foo", "level": 4}, - "bar": {"id": "bar", "level": 7} -} -""" - -class AnyStringWith(str): - def __eq__(self, other): - return self in other - -def setup_function(function): - - # Clear applist cache - shutil.rmtree(APPSLISTS_CACHE, ignore_errors=True) - - # Clear appslist cron - if os.path.exists(APPSLISTS_CRON_PATH): - os.remove(APPSLISTS_CRON_PATH) - - # Clear appslist conf - if os.path.exists(APPSLISTS_CONF): - os.remove(APPSLISTS_CONF) - - -def teardown_function(function): - - # Clear applist cache - # Otherwise when using apps stuff after running the test, - # we'll still have the dummy unusable list - shutil.rmtree(APPSLISTS_CACHE, ignore_errors=True) - - -def cron_job_is_there(): - r = os.system("run-parts -v --test %s | grep %s" % (CRON_FOLDER, CRON_NAME)) - return r == 0 - -# -# ################################################ -# - - -def test_appslist_init(mocker): - - # Cache is empty - assert not glob.glob(APPSLISTS_CACHE + "/*") - # Conf doesn't exist yet - assert not os.path.exists(APPSLISTS_CONF) - # Conf doesn't exist yet - assert not os.path.exists(APPSLISTS_CRON_PATH) - - # Initialize ... - mocker.spy(m18n, "n") - _initialize_appslists_system() - m18n.n.assert_any_call('appslist_init_success') - - # Then there's a cron enabled - assert cron_job_is_there() - - # And a conf with at least one list - assert os.path.exists(APPSLISTS_CONF) - appslist_list = _read_appslist_list() - assert len(appslist_list) - - # Cache is expected to still be empty though - # (if we did update the appslist during init, - # we couldn't differentiate easily exceptions - # related to lack of network connectivity) - assert not glob.glob(APPSLISTS_CACHE + "/*") - - -def test_appslist_emptylist(): - - # Initialize ... - _initialize_appslists_system() - - # Let's imagine somebody removed the default applist because uh idk they dont want to use our default applist - os.system("rm %s" % APPSLISTS_CONF) - os.system("touch %s" % APPSLISTS_CONF) - - appslist_list = _read_appslist_list() - assert not len(appslist_list) - - -def test_appslist_update_success(mocker): - - # Initialize ... - _initialize_appslists_system() - - # Cache is empty - assert not glob.glob(APPSLISTS_CACHE + "/*") - - # Update - with requests_mock.Mocker() as m: - - _actual_appslist_api_url, - # Mock the server response with a dummy applist - m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, text=DUMMY_APPLIST) - - mocker.spy(m18n, "n") - _update_appslist() - m18n.n.assert_any_call("appslist_updating") - m18n.n.assert_any_call("appslist_update_success") - - # Cache shouldn't be empty anymore empty - assert glob.glob(APPSLISTS_CACHE + "/*") - - app_dict = _load_appslist() - assert "foo" in app_dict.keys() - assert "bar" in app_dict.keys() - - -def test_appslist_update_404(mocker): - - # Initialize ... - _initialize_appslists_system() - - with requests_mock.Mocker() as m: - - # 404 error - m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, - status_code=404) - - with pytest.raises(YunohostError): - mocker.spy(m18n, "n") - _update_appslist() - m18n.n.assert_any_call("appslist_failed_to_download") - -def test_appslist_update_timeout(mocker): - - # Initialize ... - _initialize_appslists_system() - - with requests_mock.Mocker() as m: - - # Timeout - m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, - exc=requests.exceptions.ConnectTimeout) - - with pytest.raises(YunohostError): - mocker.spy(m18n, "n") - _update_appslist() - m18n.n.assert_any_call("appslist_failed_to_download") - - -def test_appslist_update_sslerror(mocker): - - # Initialize ... - _initialize_appslists_system() - - with requests_mock.Mocker() as m: - - # SSL error - m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, - exc=requests.exceptions.SSLError) - - with pytest.raises(YunohostError): - mocker.spy(m18n, "n") - _update_appslist() - m18n.n.assert_any_call("appslist_failed_to_download") - - -def test_appslist_update_corrupted(mocker): - - # Initialize ... - _initialize_appslists_system() - - with requests_mock.Mocker() as m: - - # Corrupted json - m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, - text=DUMMY_APPLIST[:-2]) - - with pytest.raises(YunohostError): - mocker.spy(m18n, "n") - _update_appslist() - m18n.n.assert_any_call("appslist_failed_to_download") - - -def test_appslist_load_with_empty_cache(mocker): - - # Initialize ... - _initialize_appslists_system() - - # Cache is empty - assert not glob.glob(APPSLISTS_CACHE + "/*") - - # Update - with requests_mock.Mocker() as m: - - # Mock the server response with a dummy applist - m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, text=DUMMY_APPLIST) - - # Try to load the applist - # This should implicitly trigger an update in the background - mocker.spy(m18n, "n") - app_dict = _load_appslist() - m18n.n.assert_any_call("appslist_obsolete_cache") - m18n.n.assert_any_call("appslist_update_success") - - - # Cache shouldn't be empty anymore empty - assert glob.glob(APPSLISTS_CACHE + "/*") - - assert "foo" in app_dict.keys() - assert "bar" in app_dict.keys() - - -def test_appslist_load_with_conflicts_between_lists(mocker): - - # Initialize ... - _initialize_appslists_system() - - conf = [{"id": "default", "url": APPSLISTS_DEFAULT_URL}, - {"id": "default2", "url": APPSLISTS_DEFAULT_URL.replace("yunohost.org", "yolohost.org")}] - - write_to_yaml(APPSLISTS_CONF, conf) - - # Update - with requests_mock.Mocker() as m: - - # Mock the server response with a dummy applist - # + the same applist for the second list - m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, text=DUMMY_APPLIST) - m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL.replace("yunohost.org", "yolohost.org"), text=DUMMY_APPLIST) - - # Try to load the applist - # This should implicitly trigger an update in the background - mocker.spy(logger, "warning") - app_dict = _load_appslist() - logger.warning.assert_any_call(AnyStringWith("Duplicate")) - - # Cache shouldn't be empty anymore empty - assert glob.glob(APPSLISTS_CACHE + "/*") - - assert "foo" in app_dict.keys() - assert "bar" in app_dict.keys() - - -def test_appslist_load_with_oudated_api_version(mocker): - - # Initialize ... - _initialize_appslists_system() - - # Update - with requests_mock.Mocker() as m: - - mocker.spy(m18n, "n") - m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, text=DUMMY_APPLIST) - _update_appslist() - - # Cache shouldn't be empty anymore empty - assert glob.glob(APPSLISTS_CACHE + "/*") - - # Tweak the cache to replace the from_api_version with a different one - for cache_file in glob.glob(APPSLISTS_CACHE + "/*"): - cache_json = read_json(cache_file) - assert cache_json["from_api_version"] == APPSLISTS_API_VERSION - cache_json["from_api_version"] = 0 - write_to_json(cache_file, cache_json) - - # Update - with requests_mock.Mocker() as m: - - # Mock the server response with a dummy applist - m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, text=DUMMY_APPLIST) - - mocker.spy(m18n, "n") - app_dict = _load_appslist() - m18n.n.assert_any_call("appslist_update_success") - - assert "foo" in app_dict.keys() - assert "bar" in app_dict.keys() - - # Check that we indeed have the new api number in cache - for cache_file in glob.glob(APPSLISTS_CACHE + "/*"): - cache_json = read_json(cache_file) - assert cache_json["from_api_version"] == APPSLISTS_API_VERSION - - - -def test_appslist_migrate_legacy_explicitly(): - - open("/etc/yunohost/appslists.json", "w").write('{"yunohost": {"yolo":"swag"}}') - mkdir(APPSLISTS_CACHE, 0o750, parents=True) - open(APPSLISTS_CACHE+"/yunohost_old.json", "w").write('{"foo":{}, "bar": {}}') - open(APPSLISTS_CRON_PATH, "w").write("# Some old cron") - - from yunohost.tools import _get_migration_by_name - migration = _get_migration_by_name("futureproof_appslist_system") - - with requests_mock.Mocker() as m: - - # Mock the server response with a dummy applist - m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, text=DUMMY_APPLIST) - migration.migrate() - - # Old conf shouldnt be there anymore (got renamed to .old) - assert not os.path.exists("/etc/yunohost/appslists.json") - # Old cache should have been removed - assert not os.path.exists(APPSLISTS_CACHE+"/yunohost_old.json") - # Cron should have been changed - assert "/bin/bash" in open(APPSLISTS_CRON_PATH, "r").read() - assert cron_job_is_there() - - # Reading the appslist should work - app_dict = _load_appslist() - assert "foo" in app_dict.keys() - assert "bar" in app_dict.keys() - - -def test_appslist_migrate_legacy_implicitly(): - - open("/etc/yunohost/appslists.json", "w").write('{"yunohost": {"yolo":"swag"}}') - mkdir(APPSLISTS_CACHE, 0o750, parents=True) - open(APPSLISTS_CACHE+"/yunohost_old.json", "w").write('{"old_foo":{}, "old_bar": {}}') - open(APPSLISTS_CRON_PATH, "w").write("# Some old cron") - - with requests_mock.Mocker() as m: - m.register_uri("GET", APPSLISTS_DEFAULT_URL_FULL, text=DUMMY_APPLIST) - app_dict = _load_appslist() - - assert "foo" in app_dict.keys() - assert "bar" in app_dict.keys() - - # Old conf shouldnt be there anymore (got renamed to .old) - assert not os.path.exists("/etc/yunohost/appslists.json") - # Old cache should have been removed - assert not os.path.exists(APPSLISTS_CACHE+"/yunohost_old.json") - # Cron should have been changed - assert "/bin/bash" in open(APPSLISTS_CRON_PATH, "r").read() - assert cron_job_is_there() - diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index 6aab198e2..bcd39b0e2 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -37,7 +37,7 @@ from moulinette.utils.log import getActionLogger from moulinette.utils.process import check_output, call_async_output from moulinette.utils.filesystem import read_json, write_to_json, read_yaml, write_to_yaml -from yunohost.app import _update_appslist, app_info, app_upgrade, app_ssowatconf, app_list +from yunohost.app import _update_apps_catalog, app_info, app_upgrade, app_ssowatconf, app_list from yunohost.domain import domain_add, domain_list from yunohost.dyndns import _dyndns_available, _dyndns_provides from yunohost.firewall import firewall_upnp @@ -351,14 +351,14 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False, # Enable UPnP silently and reload firewall firewall_upnp('enable', no_refresh=True) - # Initialize the appslist system - _initialize_appslist_system() + # Initialize the apps catalog system + _initialize_apps_catalog_system() - # Try to update the appslist ... + # Try to update the apps catalog ... # we don't fail miserably if this fails, # because that could be for example an offline installation... try: - _update_appslist() + _update_apps_catalog() except Exception as e: logger.warning(str(e)) @@ -407,7 +407,6 @@ def tools_update(apps=False, system=False): Keyword arguments: system -- Fetch available system packages upgrades (equivalent to apt update) apps -- Fetch the application list to check which apps can be upgraded - appslist -- Just update the application list cache """ # If neither --apps nor --system specified, do both @@ -454,7 +453,7 @@ def tools_update(apps=False, system=False): upgradable_apps = [] if apps: try: - _update_appslist() + _update_apps_catalog() except YunohostError as e: logger.error(str(e))