diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 4cb108e3a..3a4c9db97 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -543,42 +543,44 @@ app: category_help: Manage apps actions: + catalog: + action_help: Show the catalog of installable application + api: GET /appscatalog + arguments: + -f: + full: --full + help: Display all details, including the app manifest and various other infos + action: store_true + -c: + full: --with-categories + help: Also return a list of app categories + action: store_true + ### app_list() list: - action_help: List apps + action_help: List installed apps api: GET /apps arguments: -f: - full: --filter - help: Name filter of app_id or app_name - -r: - full: --raw - help: Return the full app_dict - action: store_true - -i: - full: --installed - help: Return only installed apps - action: store_true - -b: - full: --with-backup - help: Return only apps with backup feature (force --installed filter) + full: --full + help: Display all details, including the app manifest and various other infos action: store_true ### app_info() info: - action_help: Get information about an installed app + action_help: Show infos about a specific installed app api: GET /apps/ arguments: app: help: Specific app ID - -r: - full: --raw - help: Return the full app_dict + -f: + full: --full + help: Display all details, including the app manifest and various other infos action: store_true ### app_map() map: - action_help: List apps by domain + action_help: Show the mapping between urls and apps api: GET /appsmap arguments: -a: diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 39eb78365..30d3ab31b 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -57,7 +57,7 @@ APP_TMP_FOLDER = INSTALL_TMP + '/from_file' APPS_CATALOG_CACHE = '/var/cache/yunohost/repo' APPS_CATALOG_CONF = '/etc/yunohost/apps_catalog.yml' APPS_CATALOG_CRON_PATH = "/etc/cron.daily/yunohost-fetch-apps-catalog" -APPS_CATALOG_API_VERSION = 1 +APPS_CATALOG_API_VERSION = 2 APPS_CATALOG_DEFAULT_URL = "https://app.yunohost.org/default" re_github_repo = re.compile( @@ -71,135 +71,106 @@ re_app_instance_name = re.compile( ) -def app_list(filter=None, raw=False, installed=False, with_backup=False): +def app_catalog(full=False, with_categories=False): """ - List apps - - Keyword argument: - filter -- Name filter of app_id or app_name - raw -- Return the full app_dict - installed -- Return only installed apps - with_backup -- Return only apps with backup feature (force --installed filter) - + Return a dict of apps available to installation from Yunohost's app catalog """ - installed = with_backup or installed - - list_dict = {} if raw else [] # Get app list from catalog cache - app_dict = _load_apps_catalog() + catalog = _load_apps_catalog() + installed_apps = set(_installed_apps()) - # Get app list from the app settings directory - for app in os.listdir(APPS_SETTING_PATH): - if app not in app_dict: - # Handle multi-instance case like wordpress__2 - if '__' in app: - original_app = app[:app.index('__')] - if original_app in app_dict: - app_dict[app] = app_dict[original_app] - continue - # FIXME : What if it's not !?!? + # Trim info for apps if not using --full + for app, infos in catalog["apps"].items(): + infos["installed"] = app in installed_apps - manifest = _get_manifest_of_app(os.path.join(APPS_SETTING_PATH, app)) - app_dict[app] = {"manifest": manifest} + infos["manifest"]["description"] = _value_for_locale(infos['manifest']['description']) - app_dict[app]['repository'] = None + if not full: + catalog["apps"][app] = { + "description": infos['manifest']['description'], + "level": infos["level"], + } - # Sort app list - sorted_app_list = sorted(app_dict.keys()) + # Trim info for categories if not using --full + for category in catalog["categories"]: + category["title"] = _value_for_locale(category["title"]) + category["description"] = _value_for_locale(category["description"]) + for subtags in category.get("subtags", []): + subtags["title"] = _value_for_locale(subtags["title"]) - for app_id in sorted_app_list: + if not full: + catalog["categories"] = [{"id": c["id"], + "description": c["description"]} + for c in catalog["categories"]] - app_info_dict = app_dict[app_id] - - # Apply filter if there's one - if (filter and - (filter not in app_id) and - (filter not in app_info_dict['manifest']['name'])): - continue - - # Ignore non-installed app if user wants only installed apps - app_installed = _is_installed(app_id) - if installed and not app_installed: - continue - - # Ignore apps which don't have backup/restore script if user wants - # only apps with backup features - if with_backup and ( - not os.path.isfile(APPS_SETTING_PATH + app_id + '/scripts/backup') or - not os.path.isfile(APPS_SETTING_PATH + app_id + '/scripts/restore') - ): - continue - - if raw: - app_info_dict['installed'] = app_installed - - # dirty: we used to have manifest containing multi_instance value in form of a string - # but we've switched to bool, this line ensure retrocompatibility - - app_info_dict["manifest"]["multi_instance"] = is_true(app_info_dict["manifest"].get("multi_instance", False)) - - list_dict[app_id] = app_info_dict - - else: - list_dict.append({ - 'id': app_id, - 'name': app_info_dict['manifest']['name'], - 'label': _get_app_settings(app_id).get("label", "?") if app_installed else None, - 'description': _value_for_locale(app_info_dict['manifest']['description']), - # FIXME: Temporarly allow undefined license - 'license': app_info_dict['manifest'].get('license', m18n.n('license_undefined')), - 'installed': app_installed - }) - - return {'apps': list_dict} if not raw else list_dict + if not with_categories: + return {"apps": catalog["apps"]} + else: + return {"apps": catalog["apps"], "categories": catalog["categories"]} -def app_info(app, raw=False): +def app_list(full=False): """ - Get app info + List installed apps + """ + out = [] + for app_id in sorted(_installed_apps()): + app_info_dict = app_info(app_id, full=full) + app_info_dict["id"] = app_id + out.append(app_info_dict) - Keyword argument: - app -- Specific app ID - raw -- Return the full app_dict + return {'apps': out} + +def app_info(app, full=False): + """ + Get info for a specific app """ if not _is_installed(app): raise YunohostError('app_not_installed', app=app, all_apps=_get_all_installed_apps_id()) - app_setting_path = APPS_SETTING_PATH + app + local_manifest = _get_manifest_of_app(os.path.join(APPS_SETTING_PATH, app)) + settings = _get_app_settings(app) - # Retrieve manifest and status - manifest = _get_manifest_of_app(app_setting_path) - - if raw: - ret = app_list(filter=app, raw=True)[app] - ret['settings'] = _get_app_settings(app) - - # Determine upgradability - # In case there is neither update_time nor install_time, we assume the app can/has to be upgraded - local_update_time = ret['settings'].get('update_time', ret['settings'].get('install_time', 0)) - - if 'lastUpdate' not in ret or 'git' not in ret: - upgradable = "url_required" - elif ret['lastUpdate'] > local_update_time: - upgradable = "yes" - else: - upgradable = "no" - - ret['upgradable'] = upgradable - ret['change_url'] = os.path.exists(os.path.join(app_setting_path, "scripts", "change_url")) - ret['version'] = manifest.get('version', '-') + ret = { + 'description': _value_for_locale(local_manifest['description']), + 'name': local_manifest['name'], + 'version': local_manifest.get('version', '-'), + } + if not full: return ret - info = { - 'name': manifest['name'], - 'description': _value_for_locale(manifest['description']), - 'license': manifest.get('license', m18n.n('license_undefined')), - 'version': manifest.get('version', '-'), - } - return info + ret["manifest"] = local_manifest + ret['settings'] = settings + + absolute_app_name = app if "__" not in app else app[:app.index('__')] # idk this is the name of the app even for multiinstance apps (so wordpress__2 -> wordpress) + ret["from_catalog"] = _load_apps_catalog()["apps"].get(absolute_app_name, {}) + ret['upgradable'] = _app_upgradable(ret) + ret['supports_change_url'] = os.path.exists(os.path.join(APPS_SETTING_PATH, app, "scripts", "change_url")) + ret['supports_backup_restore'] = (os.path.exists(os.path.join(APPS_SETTING_PATH, app, "scripts", "backup")) and + os.path.exists(os.path.join(APPS_SETTING_PATH, app, "scripts", "restore"))) + ret['supports_multi_instance'] = is_true(local_manifest.get("multi_instance", False)) + return ret + + +def _app_upgradable(app_infos): + + # Determine upgradability + # In case there is neither update_time nor install_time, we assume the app can/has to be upgraded + + if not app_infos.get("from_catalog", None): + return "url_required" + if not app_infos["from_catalog"].get("lastUpdate") or not app_infos["from_catalog"].get("git"): + return "url_required" + + settings = app_infos["settings"] + local_update_time = settings.get('update_time', settings.get('install_time', 0)) + if app_infos["from_catalog"]['lastUpdate'] > local_update_time: + return "yes" + else: + return "no" def app_map(app=None, raw=False, user=None): @@ -452,19 +423,12 @@ def app_upgrade(app=[], url=None, file=None): from yunohost.hook import hook_add, hook_remove, hook_exec, hook_callback from yunohost.permission import permission_sync_to_user - try: - app_list() - except YunohostError: - raise YunohostError('apps_already_up_to_date') - - not_upgraded_apps = [] - apps = app # If no app is specified, upgrade all apps if not apps: # FIXME : not sure what's supposed to happen if there is a url and a file but no apps... if not url and not file: - apps = [app_["id"] for app_ in app_list(installed=True)["apps"]] + apps = _installed_apps() elif not isinstance(app, list): apps = [app] @@ -483,7 +447,7 @@ def app_upgrade(app=[], url=None, file=None): for number, app_instance_name in enumerate(apps): logger.info(m18n.n('app_upgrade_app_name', app=app_instance_name)) - app_dict = app_info(app_instance_name, raw=True) + app_dict = app_info(app_instance_name, full=True) if file and isinstance(file, dict): # We use this dirty hack to test chained upgrades in unit/functional tests @@ -658,7 +622,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu if answer.upper() != "Y": raise YunohostError("aborting") - raw_app_list = app_list(raw=True) + raw_app_list = _load_apps_catalog()["apps"] if app in raw_app_list or ('@' in app) or ('http://' in app) or ('https://' in app): @@ -668,7 +632,10 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu # If we got an url like "https://github.com/foo/bar_ynh, we want to # extract "bar" and test if we know this app elif ('http://' in app) or ('https://' in app): - app_name_to_test = app.strip("/").split("/")[-1].replace("_ynh","") + app_name_to_test = app.strip("/").split("/")[-1].replace("_ynh", "") + else: + # FIXME : watdo if '@' in app ? + app_name_to_test = None if app_name_to_test in raw_app_list: @@ -1216,8 +1183,7 @@ def app_register_url(app, domain, path): # We cannot change the url of an app already installed simply by changing # the settings... - installed = app in app_list(installed=True, raw=True).keys() - if installed: + if _is_installed(app): settings = _get_app_settings(app) if "path" in settings.keys() and "domain" in settings.keys(): raise YunohostError('app_already_installed_cant_change_url') @@ -1263,19 +1229,13 @@ def app_ssowatconf(): redirected_regex = {main_domain + '/yunohost[\/]?$': 'https://' + main_domain + '/yunohost/sso/'} redirected_urls = {} - try: - apps_list = app_list(installed=True)['apps'] - except Exception as e: - logger.debug("cannot get installed app list because %s", e) - apps_list = [] - def _get_setting(settings, name): s = settings.get(name, None) return s.split(',') if s else [] - for app in apps_list: + for app in _installed_apps(): - app_settings = read_yaml(APPS_SETTING_PATH + app['id'] + '/settings.yml') + app_settings = read_yaml(APPS_SETTING_PATH + app + '/settings.yml') if 'domain' not in app_settings: continue @@ -1318,7 +1278,7 @@ def app_ssowatconf(): protected_regex += _get_setting(app_settings, 'protected_regex') # New permission system - this_app_perms = {name: info for name, info in all_permissions.items() if name.startswith(app['id'] + ".")} + this_app_perms = {name: info for name, info in all_permissions.items() if name.startswith(app + ".")} for perm_name, perm_info in this_app_perms.items(): # Ignore permissions for which there's no url defined @@ -1624,8 +1584,7 @@ def _get_all_installed_apps_id(): * ...' """ - all_apps_ids = [x["id"] for x in app_list(installed=True)["apps"]] - all_apps_ids = sorted(all_apps_ids) + all_apps_ids = sorted(_installed_apps()) all_apps_ids_formatted = "\n * ".join(all_apps_ids) all_apps_ids_formatted = "\n * " + all_apps_ids_formatted @@ -2164,17 +2123,16 @@ def _fetch_app_from_git(app): else: manifest['remote']['revision'] = revision else: - app_dict = app_list(raw=True) + app_dict = _load_apps_catalog()["apps"] - if app in app_dict: - app_info = app_dict[app] - app_info['manifest']['lastUpdate'] = app_info['lastUpdate'] - manifest = app_info['manifest'] - else: + if app not in app_dict: raise YunohostError('app_unknown') - - if 'git' not in app_info: + elif 'git' not in app_dict[app]: raise YunohostError('app_unsupported_remote_type') + + app_info = app_dict[app] + app_info['manifest']['lastUpdate'] = app_info['lastUpdate'] + manifest = app_info['manifest'] url = app_info['git']['url'] if 'github.com' in url: @@ -2272,6 +2230,10 @@ def _is_installed(app): return os.path.isdir(APPS_SETTING_PATH + app) +def _installed_apps(): + return os.listdir(APPS_SETTING_PATH) + + def _value_for_locale(values): """ Return proper value for current locale @@ -2698,11 +2660,14 @@ def _update_apps_catalog(): def _load_apps_catalog(): """ - Read all the apps catalog cache files and build a single dict (app_dict) - corresponding to all known apps in all indexes + Read all the apps catalog cache files and build a single dict (merged_catalog) + corresponding to all known apps and categories """ - app_dict = {} + merged_catalog = { + "apps": {}, + "categories": [] + } for apps_catalog_id in [L["id"] for L in _read_apps_catalog_list()]: @@ -2725,18 +2690,22 @@ def _load_apps_catalog(): del apps_catalog_content["from_api_version"] # Add apps from this catalog to the output - for app, info in apps_catalog_content.items(): + for app, info in apps_catalog_content["apps"].items(): # (N.B. : there's a small edge case where multiple apps catalog could be listing the same apps ... # in which case we keep only the first one found) - if app in app_dict: - logger.warning("Duplicate app %s found between apps catalog %s and %s" % (app, apps_catalog_id, app_dict[app]['repository'])) + if app in merged_catalog["apps"]: + logger.warning("Duplicate app %s found between apps catalog %s and %s" + % (app, apps_catalog_id, merged_catalog["apps"][app]['repository'])) continue info['repository'] = apps_catalog_id - app_dict[app] = info + merged_catalog["apps"][app] = info - return app_dict + # Annnnd categories + merged_catalog["categories"] += apps_catalog_content["categories"] + + return merged_catalog # # ############################### # @@ -2782,16 +2751,12 @@ def random_password(length=8): def unstable_apps(): - raw_app_installed = app_list(installed=True, raw=True) output = [] - for app, infos in raw_app_installed.items(): + for infos in app_list(full=True): - repo = infos.get("repository", None) - state = infos.get("state", None) - - if repo is None or state in ["inprogress", "notworking"]: - output.append(app) + if not infos.get("from_catalog") or infos.get("from_catalog").get("state") in ["inprogress", "notworking"]: + output.append(infos["id"]) return output diff --git a/src/yunohost/data_migrations/0011_setup_group_permission.py b/src/yunohost/data_migrations/0011_setup_group_permission.py index b07e5d21b..c55e33cab 100644 --- a/src/yunohost/data_migrations/0011_setup_group_permission.py +++ b/src/yunohost/data_migrations/0011_setup_group_permission.py @@ -8,7 +8,7 @@ from moulinette.utils.filesystem import read_yaml from yunohost.tools import Migration from yunohost.user import user_list, user_group_create, user_group_update -from yunohost.app import app_setting, app_list +from yunohost.app import app_setting, _installed_apps from yunohost.regenconf import regen_conf, BACKUP_CONF_DIR from yunohost.permission import permission_create, user_permission_update, permission_sync_to_user @@ -96,13 +96,16 @@ class MyMigration(Migration): def migrate_app_permission(self, app=None): logger.info(m18n.n("migration_0011_migrate_permission")) - if app: - apps = app_list(installed=True, filter=app)['apps'] - else: - apps = app_list(installed=True)['apps'] + apps = _installed_apps() - for app_info in apps: - app = app_info['id'] + if app: + if app not in apps: + logger.error("Can't migrate permission for app %s because it ain't installed..." % app) + apps = [] + else: + apps = [app] + + for app in apps: permission = app_setting(app, 'allowed_users') path = app_setting(app, 'path') domain = app_setting(app, 'domain') diff --git a/src/yunohost/tests/test_appscatalog.py b/src/yunohost/tests/test_appscatalog.py index 613b59012..39a0be206 100644 --- a/src/yunohost/tests/test_appscatalog.py +++ b/src/yunohost/tests/test_appscatalog.py @@ -14,6 +14,7 @@ from yunohost.app import (_initialize_apps_catalog_system, _update_apps_catalog, _actual_apps_catalog_api_url, _load_apps_catalog, + app_catalog, logger, APPS_CATALOG_CACHE, APPS_CATALOG_CONF, @@ -25,8 +26,14 @@ APPS_CATALOG_DEFAULT_URL_FULL = _actual_apps_catalog_api_url(APPS_CATALOG_DEFAUL CRON_FOLDER, CRON_NAME = APPS_CATALOG_CRON_PATH.rsplit("/", 1) DUMMY_APP_CATALOG = """{ - "foo": {"id": "foo", "level": 4}, - "bar": {"id": "bar", "level": 7} + "apps": { + "foo": {"id": "foo", "level": 4, "category": "yolo", "manifest":{"description": "Foo"}}, + "bar": {"id": "bar", "level": 7, "category": "swag", "manifest":{"description": "Bar"}} + }, + "categories": [ + {"id": "yolo", "description": "YoLo"}, + {"id": "swag", "description": "sWaG"} + ] } """ @@ -107,7 +114,7 @@ def test_apps_catalog_emptylist(): assert not len(apps_catalog_list) -def test_apps_catalog_update_success(mocker): +def test_apps_catalog_update_nominal(mocker): # Initialize ... _initialize_apps_catalog_system() @@ -130,9 +137,16 @@ def test_apps_catalog_update_success(mocker): # Cache shouldn't be empty anymore empty assert glob.glob(APPS_CATALOG_CACHE + "/*") - app_dict = _load_apps_catalog() - assert "foo" in app_dict.keys() - assert "bar" in app_dict.keys() + # And if we load the catalog, we sould find + # - foo and bar as apps (unordered), + # - yolo and swag as categories (ordered) + catalog = app_catalog(with_categories=True) + + assert "apps" in catalog + assert set(catalog["apps"].keys()) == set(["foo", "bar"]) + + assert "categories" in catalog + assert [c["id"] for c in catalog["categories"]] == ["yolo", "swag"] def test_apps_catalog_update_404(mocker): @@ -219,7 +233,7 @@ def test_apps_catalog_load_with_empty_cache(mocker): # Try to load the apps catalog # This should implicitly trigger an update in the background mocker.spy(m18n, "n") - app_dict = _load_apps_catalog() + app_dict = _load_apps_catalog()["apps"] m18n.n.assert_any_call("apps_catalog_obsolete_cache") m18n.n.assert_any_call("apps_catalog_update_success") @@ -252,7 +266,7 @@ def test_apps_catalog_load_with_conflicts_between_lists(mocker): # Try to load the apps catalog # This should implicitly trigger an update in the background mocker.spy(logger, "warning") - app_dict = _load_apps_catalog() + app_dict = _load_apps_catalog()["apps"] logger.warning.assert_any_call(AnyStringWith("Duplicate")) # Cache shouldn't be empty anymore empty @@ -291,7 +305,7 @@ def test_apps_catalog_load_with_oudated_api_version(mocker): m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG) mocker.spy(m18n, "n") - app_dict = _load_apps_catalog() + app_dict = _load_apps_catalog()["apps"] m18n.n.assert_any_call("apps_catalog_update_success") assert "foo" in app_dict.keys() @@ -329,7 +343,7 @@ def test_apps_catalog_migrate_legacy_explicitly(): assert cron_job_is_there() # Reading the apps_catalog should work - app_dict = _load_apps_catalog() + app_dict = _load_apps_catalog()["apps"] assert "foo" in app_dict.keys() assert "bar" in app_dict.keys() @@ -343,7 +357,7 @@ def test_apps_catalog_migrate_legacy_implicitly(): with requests_mock.Mocker() as m: m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG) - app_dict = _load_apps_catalog() + app_dict = _load_apps_catalog()["apps"] assert "foo" in app_dict.keys() assert "bar" in app_dict.keys() diff --git a/src/yunohost/tests/test_permission.py b/src/yunohost/tests/test_permission.py index 2e53f13a7..636d9b4b1 100644 --- a/src/yunohost/tests/test_permission.py +++ b/src/yunohost/tests/test_permission.py @@ -3,7 +3,7 @@ import pytest from conftest import message, raiseYunohostError -from yunohost.app import app_install, app_remove, app_change_url, app_list, app_map +from yunohost.app import app_install, app_remove, app_change_url, app_list, app_map, _installed_apps from yunohost.user import user_list, user_create, user_delete, \ user_group_list, user_group_delete from yunohost.permission import user_permission_update, user_permission_list, user_permission_reset, \ @@ -163,9 +163,7 @@ def check_permission_for_apps(): app_perms_prefix = set(p.split(".")[0] for p in app_perms) - installed_apps = {app['id'] for app in app_list(installed=True)['apps']} - - assert installed_apps == app_perms_prefix + assert set(_installed_apps()) == app_perms_prefix def can_access_webpage(webpath, logged_as=None): diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index e081f1b3a..ea9521020 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -470,17 +470,17 @@ def _list_upgradable_apps(): app_list_installed = os.listdir(APPS_SETTING_PATH) for app_id in app_list_installed: - app_dict = app_info(app_id, raw=True) + app_dict = app_info(app_id, full=True) if app_dict["upgradable"] == "yes": # FIXME : would make more sense for these infos to be computed # directly in app_info and used to check the upgradability of # the app... - current_version = app_dict.get("version", "?") + current_version = app_dict.get("manifest", {}).get("version", "?") current_commit = app_dict.get("settings", {}).get("current_revision", "?")[:7] - new_version = app_dict.get("manifest",{}).get("version","?") - new_commit = app_dict.get("git", {}).get("revision", "?")[:7] + new_version = app_dict.get("from_catalog", {}).get("manifest", {}).get("version", "?") + new_commit = app_dict.get("from_catalog", {}).get("git", {}).get("revision", "?")[:7] if current_version == new_version: current_version += " (" + current_commit + ")"