From a4acd4c4e50dda6072242c01ebbf3ba764822d94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Thu, 1 Oct 2015 22:07:43 +0200 Subject: [PATCH 1/4] [fix] Update backup_restore api route and configuration --- data/actionsmap/yunohost.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index ca539090b..b338616f4 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -600,9 +600,7 @@ backup: ### backup_restore() restore: action_help: Restore from a local backup archive - api: POST /restore - configuration: - authenticate: false + api: POST /backup/restore/ arguments: name: help: Name of the local backup archive From 843f68e8173cf2737e704b1f387bdbdd4bcdea90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Thu, 1 Oct 2015 22:14:40 +0200 Subject: [PATCH 2/4] [enh] Introduce app status file to store install app info --- data/actionsmap/yunohost.yml | 2 +- lib/yunohost/app.py | 114 +++++++++++++++++++++++++++++++---- 2 files changed, 103 insertions(+), 13 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index b338616f4..3c3db49f5 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -358,7 +358,7 @@ app: ### app_info() info: - action_help: Get app info + action_help: Get information about an installed app api: GET /apps/ arguments: app: diff --git a/lib/yunohost/app.py b/lib/yunohost/app.py index 01c93960d..08c3a73d3 100644 --- a/lib/yunohost/app.py +++ b/lib/yunohost/app.py @@ -34,8 +34,12 @@ import re import socket import urlparse import errno +import subprocess from moulinette.core import MoulinetteError +from moulinette.utils.log import getActionLogger + +logger = getActionLogger('yunohost.app') repo_path = '/var/cache/yunohost/repo' apps_path = '/usr/share/yunohost/apps' @@ -274,7 +278,7 @@ def app_upgrade(auth, app=[], url=None, file=None): url -- Git url to fetch for upgrade """ - from yunohost.hook import hook_add, hook_exec + from yunohost.hook import hook_add, hook_remove, hook_exec try: app_list() @@ -330,6 +334,19 @@ def app_upgrade(auth, app=[], url=None, file=None): app_setting_path = apps_setting_path +'/'+ app_id + # Retrieve current app status + status = {} + try: + with open(app_setting_path + '/status.json') as f: + status = json.loads(str(f.read())) + except IOError: + logger.exception("status file not found for '%s'", app_id) + status = { + 'installed_at': app_setting(app_id, 'install_time'), + } + finally: + status['remote'] = manifest.get('remote', None) + if original_app_id != app_id: # Replace original_app_id with the forked one in scripts for script in os.listdir(app_tmp_folder +'/scripts'): @@ -362,7 +379,14 @@ def app_upgrade(auth, app=[], url=None, file=None): if hook_exec(app_tmp_folder +'/scripts/upgrade') != 0: raise MoulinetteError(errno.EIO, m18n.n('installation_failed')) else: - app_setting(app_id, 'update_time', int(time.time())) + now = int(time.time()) + # TODO: Move install_time away from app_setting + app_setting(app_id, 'update_time', now) + status['upgraded_at'] = now + + # Store app status + with open(app_setting_path + '/status.json', 'w+') as f: + json.dump(status, f) # Replace scripts and manifest os.system('rm -rf "%s/scripts" "%s/manifest.json"' % (app_setting_path, app_setting_path)) @@ -394,12 +418,21 @@ def app_install(auth, app, label=None, args=None): try: os.listdir(install_tmp) except OSError: os.makedirs(install_tmp) + status = { + 'installed_at': None, + 'upgraded_at': None, + 'remote': { + 'type': None, + }, + } + if app in app_list(raw=True) or ('@' in app) or ('http://' in app) or ('https://' in app): manifest = _fetch_app_from_git(app) elif os.path.exists(app): manifest = _extract_app_from_file(app) else: raise MoulinetteError(errno.EINVAL, m18n.n('app_unknown')) + status['remote'] = manifest.get('remote', {}) # Check ID if 'id' not in manifest or '__' in manifest['id']: @@ -458,8 +491,11 @@ def app_install(auth, app, label=None, args=None): for file in os.listdir(app_tmp_folder +'/hooks'): hook_add(app_id, app_tmp_folder +'/hooks/'+ file) + now = int(time.time()) app_setting(app_id, 'id', app_id) - app_setting(app_id, 'install_time', int(time.time())) + # TODO: Move install_time away from app_setting + app_setting(app_id, 'install_time', now) + status['installed_at'] = now if label: app_setting(app_id, 'label', label) @@ -482,6 +518,11 @@ def app_install(auth, app, label=None, args=None): os.system('cp -R %s/scripts %s' % (app_tmp_folder, app_setting_path)) try: if hook_exec(app_tmp_folder + '/scripts/install', args_dict) == 0: + # Store app status + with open(app_setting_path + '/status.json', 'w+') as f: + json.dump(status, f) + + # Clean and set permissions shutil.rmtree(app_tmp_folder) os.system('chmod -R 400 %s' % app_setting_path) os.system('chown -R root: %s' % app_setting_path) @@ -1009,9 +1050,30 @@ def _extract_app_from_file(path, remove=False): msignals.display(m18n.n('done')) + manifest['remote'] = {'type': 'file', 'path': path} return manifest +def _get_git_last_commit_hash(repository): + """ + Attempt to retrieve the last commit hash of a git repository + + Keyword arguments: + repository -- The URL or path of the repository + + """ + try: + commit = subprocess.check_output( + "git ls-remote --exit-code {:s} HEAD | awk '{{print $1}}'".format( + repository), + shell=True) + except subprocess.CalledProcessError: + logger.exception("unable to get last commit from %s", repository) + raise ValueError("Unable to get last commit with git") + else: + return commit.strip() + + def _fetch_app_from_git(app): """ Unzip or untar application tarball in app_tmp_folder @@ -1027,6 +1089,8 @@ def _fetch_app_from_git(app): msignals.display(m18n.n('downloading')) + git_result_2 = 1 + if ('@' in app) or ('http://' in app) or ('https://' in app): if "github.com" in app: url = app.replace("git@github.com:", "https://github.com/") @@ -1034,16 +1098,28 @@ def _fetch_app_from_git(app): if "/" in url [-1:]: url = url[:-1] url = url + "/archive/master.zip" if os.system('wget "%s" -O "%s.zip" > /dev/null 2>&1' % (url, app_tmp_folder)) == 0: - return _extract_app_from_file(app_tmp_folder +'.zip', remove=True) + manifest = _extract_app_from_file(app_tmp_folder +'.zip', remove=True) + del manifest['remote'] + else: + git_result = os.system('git clone --depth=1 %s %s' % (app, app_tmp_folder)) + git_result_2 = 0 + try: + with open(app_tmp_folder + '/manifest.json') as json_manifest: + manifest = json.loads(str(json_manifest.read())) + manifest['lastUpdate'] = int(time.time()) + except IOError: + raise MoulinetteError(errno.EIO, m18n.n('app_manifest_invalid')) - git_result = os.system('git clone --depth=1 %s %s' % (app, app_tmp_folder)) - git_result_2 = 0 + # Store remote repository info into the returned manifest + manifest['remote'] = {'type': 'git', 'url': app, 'branch': 'master'} try: - with open(app_tmp_folder + '/manifest.json') as json_manifest: - manifest = json.loads(str(json_manifest.read())) - manifest['lastUpdate'] = int(time.time()) - except IOError: - raise MoulinetteError(errno.EIO, m18n.n('app_manifest_invalid')) + revision = _get_git_last_commit_hash(app) + except: pass + else: + manifest['remote']['revision'] = revision + + if git_result_2 == 1: + return manifest else: app_dict = app_list(raw=True) @@ -1056,12 +1132,26 @@ def _fetch_app_from_git(app): raise MoulinetteError(errno.EINVAL, m18n.n('app_unknown')) if "github.com" in app_info['git']['url']: + # FIXME: Retrieve branch defined in app_info url = app_info['git']['url'].replace("git@github.com:", "https://github.com/") if ".git" in url[-4:]: url = url[:-4] if "/" in url [-1:]: url = url[:-1] url = url + "/archive/"+ str(app_info['git']['revision']) + ".zip" if os.system('wget "%s" -O "%s.zip" > /dev/null 2>&1' % (url, app_tmp_folder)) == 0: - return _extract_app_from_file(app_tmp_folder +'.zip', remove=True) + manifest = _extract_app_from_file(app_tmp_folder +'.zip', remove=True) + else: + git_result_2 = 0 + + # Store remote repository info into the returned manifest + manifest['remote'] = { + 'type': 'git', + 'url': app_info['git']['url'], + 'branch': app_info['git']['branch'], + 'revision': app_info['git']['revision'], + } + + if git_result_2 == 1: + return manifest app_tmp_folder = install_tmp +'/'+ app if os.path.exists(app_tmp_folder): shutil.rmtree(app_tmp_folder) From 7048e98a0a4f0d1a8822d00ce74ff3e694be1a7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Fri, 2 Oct 2015 00:01:00 +0200 Subject: [PATCH 3/4] [enh] Update app_info to show app installation status --- data/actionsmap/yunohost.yml | 4 ++ lib/yunohost/app.py | 109 ++++++++++++++++++++++++----------- 2 files changed, 79 insertions(+), 34 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 3c3db49f5..2b0a76050 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -363,6 +363,10 @@ app: arguments: app: help: Specific app ID + -s: + full: --show-status + help: Show app installation status + action: store_true -r: full: --raw help: Return the full app_dict diff --git a/lib/yunohost/app.py b/lib/yunohost/app.py index 08c3a73d3..a38b87e89 100644 --- a/lib/yunohost/app.py +++ b/lib/yunohost/app.py @@ -173,6 +173,8 @@ def app_list(offset=None, limit=None, filter=None, raw=False): if raw: app_info['installed'] = installed + if installed: + app_info['status'] = _get_app_status(app_id) list_dict[app_id] = app_info else: list_dict.append({ @@ -193,37 +195,44 @@ def app_list(offset=None, limit=None, filter=None, raw=False): return list_dict -def app_info(app, raw=False): +def app_info(app, show_status=False, raw=False): """ Get app info Keyword argument: app -- Specific app ID + show_status -- Show app installation status raw -- Return the full app_dict """ - try: - app_info = app_list(filter=app, raw=True)[app] - except: - app_info = {} - - if _is_installed(app): + if not _is_installed(app): + raise MoulinetteError(errno.EINVAL, + m18n.n('app_not_installed', app)) + if raw: + ret = app_list(filter=app, raw=True)[app] with open(apps_setting_path + app +'/settings.yml') as f: - app_info['settings'] = yaml.load(f) + ret['settings'] = yaml.load(f) + return ret - if raw: - return app_info - else: - return { - 'name': app_info['manifest']['name'], - 'description': _value_for_locale(app_info['manifest']['description']), - # FIXME: Temporarly allow undefined license - 'license': app_info['manifest'].get('license', - m18n.n('license_undefined')), - # FIXME: Temporarly allow undefined version - 'version' : app_info['manifest'].get('version', '-'), - #TODO: Add more info - } + app_setting_path = apps_setting_path + app + + # Retrieve manifest and status + with open(app_setting_path + '/manifest.json') as f: + manifest = json.loads(str(f.read())) + status = _get_app_status(app, format_date=True) + + info = { + 'name': manifest['name'], + 'description': _value_for_locale(manifest['description']), + # FIXME: Temporarly allow undefined license + 'license': manifest.get('license', m18n.n('license_undefined')), + # FIXME: Temporarly allow undefined version + 'version': manifest.get('version', '-'), + #TODO: Add more info + } + if show_status: + info['status'] = status + return info def app_map(app=None, raw=False, user=None): @@ -335,17 +344,8 @@ def app_upgrade(auth, app=[], url=None, file=None): app_setting_path = apps_setting_path +'/'+ app_id # Retrieve current app status - status = {} - try: - with open(app_setting_path + '/status.json') as f: - status = json.loads(str(f.read())) - except IOError: - logger.exception("status file not found for '%s'", app_id) - status = { - 'installed_at': app_setting(app_id, 'install_time'), - } - finally: - status['remote'] = manifest.get('remote', None) + status = _get_app_status(app_id) + status['remote'] = manifest.get('remote', None) if original_app_id != app_id: # Replace original_app_id with the forked one in scripts @@ -791,9 +791,11 @@ def app_setting(app, key, value=None, delete=False): app_settings = {} if value is None and not delete: - # Get the value - if app_settings is not None and key in app_settings: + try: return app_settings[key] + except: + logger.exception("cannot get app setting '%s' for '%s'", key, app) + return None else: yaml_settings=['redirected_urls','redirected_regex'] # Set the value @@ -1000,6 +1002,45 @@ def app_ssowatconf(auth): msignals.display(m18n.n('ssowat_conf_generated'), 'success') +def _get_app_status(app_id, format_date=False): + """ + Get app status or create it if needed + + Keyword arguments: + app_id -- The app id + format_date -- Format date fields + + """ + app_setting_path = apps_setting_path + app_id + if not os.path.isdir(app_setting_path): + raise MoulinetteError(errno.EINVAL, m18n.n('app_unknown')) + status = {} + + try: + with open(app_setting_path + '/status.json') as f: + status = json.loads(str(f.read())) + except IOError: + logger.exception("status file not found for '%s'", app_id) + # Create app status + status = { + 'installed_at': app_setting(app_id, 'install_time'), + 'upgraded_at': app_setting(app_id, 'update_time'), + 'remote': { 'type': None }, + } + with open(app_setting_path + '/status.json', 'w+') as f: + json.dump(status, f) + + if format_date: + for f in ['installed_at', 'upgraded_at']: + v = status.get(f, None) + if not v: + status[f] = '-' + else: + status[f] = time.strftime(m18n.n('format_datetime_short'), + time.gmtime(v)) + return status + + def _extract_app_from_file(path, remove=False): """ Unzip or untar application tarball in app_tmp_folder, or copy it from a directory From 820fcd5b60261a07970b792a9f8baa7682b31a6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Fri, 2 Oct 2015 01:17:05 +0200 Subject: [PATCH 4/4] [enh] Add python-miniupnpc alternative dependency to pyminiupnpc --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 4c96298da..89149ef8a 100644 --- a/debian/control +++ b/debian/control @@ -13,7 +13,7 @@ Depends: moulinette (>= 2.2.1), python-requests, glances, python-pip, - pyminiupnpc, + python-miniupnpc | pyminiupnpc, dnsutils, bind9utils, python-apt,