From d10e318dd19b9c4a833caf4de82a51b2dae5fe4b Mon Sep 17 00:00:00 2001 From: Julien Malik Date: Mon, 14 Mar 2016 09:27:05 +0100 Subject: [PATCH 01/15] [enh] Use kwargs consistently --- src/yunohost/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index c5ea1bdec..982e5cd46 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -462,7 +462,7 @@ def app_install(auth, app, label=None, args=None): # Retrieve arguments list for install script args_dict = {} if not args else \ dict(urlparse.parse_qsl(args, keep_blank_values=True)) - args_list = _parse_args_from_manifest(manifest, 'install', args_dict, auth) + args_list = _parse_args_from_manifest(manifest, 'install', args=args_dict, auth=auth) args_list.append(app_id) # Create app directory From 91daa7fc36c2ea3143fc0794b9b5905abd616b41 Mon Sep 17 00:00:00 2001 From: Julien Malik Date: Mon, 14 Mar 2016 09:48:49 +0100 Subject: [PATCH 02/15] [enh] return manifest arguments as an Orderedict --- src/yunohost/app.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 982e5cd46..4f3e9d283 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -375,7 +375,8 @@ def app_upgrade(auth, app=[], url=None, file=None): # Retrieve arguments list for upgrade script # TODO: Allow to specify arguments - args_list = _parse_args_from_manifest(manifest, 'upgrade', auth=auth) + args_odict = _parse_args_from_manifest(manifest, 'upgrade', auth=auth) + args_list = args_odict.values() args_list.append(app_id) # Execute App upgrade script @@ -462,7 +463,8 @@ def app_install(auth, app, label=None, args=None): # Retrieve arguments list for install script args_dict = {} if not args else \ dict(urlparse.parse_qsl(args, keep_blank_values=True)) - args_list = _parse_args_from_manifest(manifest, 'install', args=args_dict, auth=auth) + args_odict = _parse_args_from_manifest(manifest, 'install', args=args_dict, auth=auth) + args_list = args_odict.values() args_list.append(app_id) # Create app directory @@ -1397,8 +1399,7 @@ def _parse_args_from_manifest(manifest, action, args={}, auth=None): Retrieve specified arguments for the action from the manifest, and parse given args according to that. If some required arguments are not provided, its values will be asked if interaction is possible. - Parsed arguments will be returned as a list of strings to pass directly - to the proper script. + Parsed arguments will be returned as an OrderedDict Keyword arguments: manifest -- The app manifest to use @@ -1409,7 +1410,7 @@ def _parse_args_from_manifest(manifest, action, args={}, auth=None): from yunohost.domain import domain_list from yunohost.user import user_info - args_list = [] + args_list = OrderedDict() try: action_args = manifest['arguments'][action] except KeyError: @@ -1497,7 +1498,7 @@ def _parse_args_from_manifest(manifest, action, args={}, auth=None): raise MoulinetteError(errno.EINVAL, m18n.n('app_argument_choice_invalid', name=arg_name, choices='0, 1')) - args_list.append(arg_value) + args_list[arg_name] = arg_value return args_list From d9a33072b78ac70346c5f97e352c8b61a3e4c38c Mon Sep 17 00:00:00 2001 From: Julien Malik Date: Mon, 14 Mar 2016 11:04:14 +0100 Subject: [PATCH 03/15] [enh] Support passing env.var. to hook_exec --- src/yunohost/hook.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index 2b0902b6f..90a02f9f8 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -275,7 +275,7 @@ def hook_callback(action, hooks=[], args=None): return result -def hook_exec(path, args=None, raise_on_error=False, no_trace=False): +def hook_exec(path, args=None, raise_on_error=False, no_trace=False, env=None): """ Execute hook from a file with arguments @@ -284,6 +284,7 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False): args -- A list of arguments to pass to the script raise_on_error -- Raise if the script returns a non-zero exit code no_trace -- Do not print each command that will be executed + env -- Dictionnary of environment variables to export """ from moulinette.utils.process import call_async_output @@ -305,14 +306,18 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False): # bash related issue if an argument is empty and is not the last cmd_args = '"{:s}"'.format('" "'.join(str(s) for s in args)) + envcli = '' + if env is not None and isinstance(env, dict): + envcli = ' '.join([ '{key}="{val}"'.format(key=key, val=val) for key,val in env.items()]) + # Construct command to execute command = ['sudo', '-u', 'admin', '-H', 'sh', '-c'] if no_trace: - cmd = 'cd "{0:s}" && /bin/bash "{1:s}" {2:s}' + cmd = 'cd "{0:s}" && {1:s} /bin/bash "{2:s}" {3:s}' else: # use xtrace on fd 7 which is redirected to stdout - cmd = 'cd "{0:s}" && BASH_XTRACEFD=7 /bin/bash -x "{1:s}" {2:s} 7>&1' - command.append(cmd.format(cmd_fdir, cmd_fname, cmd_args)) + cmd = 'cd "{0:s}" && {1:s} BASH_XTRACEFD=7 /bin/bash -x "{2:s}" {3:s} 7>&1' + command.append(cmd.format(cmd_fdir, envcli, cmd_fname, cmd_args)) if logger.isEnabledFor(log.DEBUG): logger.info(m18n.n('executing_command', command=' '.join(command))) From eaf0fabc7f80e942008b097d2324776f550dbd0f Mon Sep 17 00:00:00 2001 From: Julien Malik Date: Mon, 14 Mar 2016 15:04:39 +0100 Subject: [PATCH 04/15] [fix] missing collections import --- src/yunohost/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 4f3e9d283..5c610deeb 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -35,6 +35,7 @@ import socket import urlparse import errno import subprocess +from collections import OrderedDict from moulinette.core import MoulinetteError from moulinette.utils.log import getActionLogger From e20ec39cb218009ad047a485fe6049ce2ed4f709 Mon Sep 17 00:00:00 2001 From: Julien Malik Date: Mon, 14 Mar 2016 15:05:33 +0100 Subject: [PATCH 05/15] [enh] Set env. var YNH_APP_ARG_xxx for each manifest argument during install --- src/yunohost/app.py | 7 ++++++- src/yunohost/hook.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 5c610deeb..476f03a87 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -468,6 +468,11 @@ def app_install(auth, app, label=None, args=None): args_list = args_odict.values() args_list.append(app_id) + # Prepare env. var. to pass to script + env_dict = {} + for arg_name, arg_value in args_odict.items(): + env_dict[ "YNH_APP_ARG_%s" % arg_name ] = arg_value + # Create app directory app_setting_path = os.path.join(apps_setting_path, app_id) if os.path.exists(app_setting_path): @@ -497,7 +502,7 @@ def app_install(auth, app, label=None, args=None): os.system('cp %s/manifest.json %s' % (app_tmp_folder, app_setting_path)) os.system('cp -R %s/scripts %s' % (app_tmp_folder, app_setting_path)) try: - if hook_exec(app_tmp_folder + '/scripts/install', args_list) == 0: + if hook_exec(app_tmp_folder + '/scripts/install', args=args_list, env=env_dict) == 0: # Store app status with open(app_setting_path + '/status.json', 'w+') as f: json.dump(status, f) diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index 90a02f9f8..a61e3eb1c 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -307,7 +307,7 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False, env=None): cmd_args = '"{:s}"'.format('" "'.join(str(s) for s in args)) envcli = '' - if env is not None and isinstance(env, dict): + if env is not None: envcli = ' '.join([ '{key}="{val}"'.format(key=key, val=val) for key,val in env.items()]) # Construct command to execute From c655d10c2f782507e45e7681120efa86205acef1 Mon Sep 17 00:00:00 2001 From: Julien Malik Date: Mon, 14 Mar 2016 17:38:48 +0100 Subject: [PATCH 06/15] [enh] Factor code for setting env. var. to scripts, also support upgrade script --- src/yunohost/app.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 476f03a87..070e0742e 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -380,6 +380,11 @@ def app_upgrade(auth, app=[], url=None, file=None): args_list = args_odict.values() args_list.append(app_id) + # Prepare env. var. to pass to script + env_dict = _make_environment_dict(args_odict) + env_dict["YNH_APP_INSTANCE_NUMBER"] = app_id.split('__')[-1] + env_dict["YNH_APP_ID"] = app_id + # Execute App upgrade script os.system('chown -hR admin: %s' % install_tmp) if hook_exec(app_tmp_folder +'/scripts/upgrade', args_list) != 0: @@ -469,9 +474,9 @@ def app_install(auth, app, label=None, args=None): args_list.append(app_id) # Prepare env. var. to pass to script - env_dict = {} - for arg_name, arg_value in args_odict.items(): - env_dict[ "YNH_APP_ARG_%s" % arg_name ] = arg_value + env_dict = _make_environment_dict(args_odict) + env_dict["YNH_APP_INSTANCE_NUMBER"] = str(instance_number) + env_dict["YNH_APP_ID"] = app_id # Create app directory app_setting_path = os.path.join(apps_setting_path, app_id) @@ -1507,6 +1512,19 @@ def _parse_args_from_manifest(manifest, action, args={}, auth=None): args_list[arg_name] = arg_value return args_list +def _make_environment_dict(args_dict): + """ + Convert a dictionnary containing manifest arguments + to a dictionnary of env. var. to be passed to scripts + + Keyword arguments: + arg -- A key/value dictionnary of manifest arguments + + """ + env_dict = {} + for arg_name, arg_value in args_dict.items(): + env_dict[ "YNH_APP_ARG_%s" % arg_name ] = arg_value + return env_dict def is_true(arg): """ From 03779fa7c39cf89cd8d5aca95c1b1aea2c8a7ef7 Mon Sep 17 00:00:00 2001 From: Julien Malik Date: Tue, 15 Mar 2016 15:28:36 +0100 Subject: [PATCH 07/15] [fix] Forward env. var to scripts --- src/yunohost/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 070e0742e..44e563e23 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -387,7 +387,7 @@ def app_upgrade(auth, app=[], url=None, file=None): # Execute App upgrade script os.system('chown -hR admin: %s' % install_tmp) - if hook_exec(app_tmp_folder +'/scripts/upgrade', args_list) != 0: + if hook_exec(app_tmp_folder +'/scripts/upgrade', args=args_list, env=env_dict) != 0: logger.error(m18n.n('app_upgrade_failed', app=app_id)) else: now = int(time.time()) From 02358b9b1eeb03a01dfa7f37a689f569a38a461a Mon Sep 17 00:00:00 2001 From: Julien Malik Date: Tue, 15 Mar 2016 16:15:15 +0100 Subject: [PATCH 08/15] [enh] Use uppercase argument for building the env. var. name --- src/yunohost/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 44e563e23..4c1d0112b 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1523,7 +1523,7 @@ def _make_environment_dict(args_dict): """ env_dict = {} for arg_name, arg_value in args_dict.items(): - env_dict[ "YNH_APP_ARG_%s" % arg_name ] = arg_value + env_dict[ "YNH_APP_ARG_%s" % arg_name.upper() ] = arg_value return env_dict def is_true(arg): From 34154b01574109a127b31292c8dbad758512ead3 Mon Sep 17 00:00:00 2001 From: Julien Malik Date: Wed, 16 Mar 2016 11:00:23 +0100 Subject: [PATCH 09/15] [fix] Extracting app instance number from appid in all cases --- src/yunohost/app.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 4c1d0112b..7cbdabfb1 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -57,6 +57,10 @@ re_github_repo = re.compile( '(/tree/(?P.+))?' ) +re_appid = re.compile( + r'^(?P[\w]+?)(__(?P[1-9][0-9]*))?$' +) + def app_listlists(): """ @@ -382,7 +386,8 @@ def app_upgrade(auth, app=[], url=None, file=None): # Prepare env. var. to pass to script env_dict = _make_environment_dict(args_odict) - env_dict["YNH_APP_INSTANCE_NUMBER"] = app_id.split('__')[-1] + app_id_withoutinstance, appinstance = _parse_appid(app_id) + env_dict["YNH_APP_INSTANCE_NUMBER"] = str(appinstance) env_dict["YNH_APP_ID"] = app_id # Execute App upgrade script @@ -1526,6 +1531,33 @@ def _make_environment_dict(args_dict): env_dict[ "YNH_APP_ARG_%s" % arg_name.upper() ] = arg_value return env_dict +def _parse_appid(appid): + """ + Parse a Yunohost app identifier and extracts the original appid + and the application instance number + + >>> _parse_appid('yolo') == ('yolo', 1) + True + >>> _parse_appid('yolo1') == ('yolo1', 1) + True + >>> _parse_appid('yolo__0') == ('yolo__0', 1) + True + >>> _parse_appid('yolo__1') == ('yolo', 1) + True + >>> _parse_appid('yolo__23') == ('yolo', 23) + True + >>> _parse_appid('yolo__42__72') == ('yolo__42', 72) + True + >>> _parse_appid('yolo__23qdqsd') == ('yolo__23qdqsd', 1) + True + >>> _parse_appid('yolo__23qdqsd56') == ('yolo__23qdqsd56', 1) + True + """ + match = re_appid.match(appid) + appid = m.groupdict().get('appid') + appinstance = int(m.groupdict().get('appinstance')) if m.groupdict().get('appinstance') is not None else 1 + return (appid, appinstance) + def is_true(arg): """ Convert a string into a boolean From 3641dead2edba23677d3a612c1126b85d20a2d10 Mon Sep 17 00:00:00 2001 From: Julien Malik Date: Wed, 16 Mar 2016 11:02:17 +0100 Subject: [PATCH 10/15] [fix] Wrong copy-paste of code --- src/yunohost/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 7cbdabfb1..9b2790bba 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1554,8 +1554,8 @@ def _parse_appid(appid): True """ match = re_appid.match(appid) - appid = m.groupdict().get('appid') - appinstance = int(m.groupdict().get('appinstance')) if m.groupdict().get('appinstance') is not None else 1 + appid = match.groupdict().get('appid') + appinstance = int(match.groupdict().get('appinstance')) if match.groupdict().get('appinstance') is not None else 1 return (appid, appinstance) def is_true(arg): From ace470e04a76c9861514fd2f3f9dfa76ef8696dc Mon Sep 17 00:00:00 2001 From: Julien Malik Date: Fri, 18 Mar 2016 16:49:15 +0100 Subject: [PATCH 11/15] [enh] rename app_id to app_instance_name when relevant --- src/yunohost/app.py | 88 +++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 43 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 9b2790bba..83febbb98 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -57,8 +57,8 @@ re_github_repo = re.compile( '(/tree/(?P.+))?' ) -re_appid = re.compile( - r'^(?P[\w]+?)(__(?P[1-9][0-9]*))?$' +re_app_instance_name = re.compile( + r'^(?P[\w]+?)(__(?P[1-9][0-9]*))?$' ) @@ -335,69 +335,69 @@ def app_upgrade(auth, app=[], url=None, file=None): elif not isinstance(app, list): app = [ app ] - for app_id in app: - installed = _is_installed(app_id) + for app_instance_name in app: + installed = _is_installed(app_instance_name) if not installed: raise MoulinetteError(errno.ENOPKG, - m18n.n('app_not_installed', app=app_id)) + m18n.n('app_not_installed', app=app_instance_name)) - if app_id in upgraded_apps: + if app_instance_name in upgraded_apps: continue - current_app_dict = app_info(app_id, raw=True) - new_app_dict = app_info(app_id, raw=True) + current_app_dict = app_info(app_instance_name, raw=True) + new_app_dict = app_info(app_instance_name, raw=True) if file: manifest = _extract_app_from_file(file) elif url: manifest = _fetch_app_from_git(url) elif new_app_dict is None or 'lastUpdate' not in new_app_dict or 'git' not in new_app_dict: - logger.warning(m18n.n('custom_app_url_required', app=app_id)) + logger.warning(m18n.n('custom_app_url_required', app=app_instance_name)) continue elif (new_app_dict['lastUpdate'] > current_app_dict['lastUpdate']) \ or ('update_time' not in current_app_dict['settings'] \ and (new_app_dict['lastUpdate'] > current_app_dict['settings']['install_time'])) \ or ('update_time' in current_app_dict['settings'] \ and (new_app_dict['lastUpdate'] > current_app_dict['settings']['update_time'])): - manifest = _fetch_app_from_git(app_id) + manifest = _fetch_app_from_git(app_instance_name) else: continue # Check requirements _check_manifest_requirements(manifest) - app_setting_path = apps_setting_path +'/'+ app_id + app_setting_path = apps_setting_path +'/'+ app_instance_name # Retrieve current app status - status = _get_app_status(app_id) + status = _get_app_status(app_instance_name) status['remote'] = manifest.get('remote', None) # Clean hooks and add new ones - hook_remove(app_id) + hook_remove(app_instance_name) if 'hooks' in os.listdir(app_tmp_folder): for hook in os.listdir(app_tmp_folder +'/hooks'): - hook_add(app_id, app_tmp_folder +'/hooks/'+ hook) + hook_add(app_instance_name, app_tmp_folder +'/hooks/'+ hook) # Retrieve arguments list for upgrade script # TODO: Allow to specify arguments args_odict = _parse_args_from_manifest(manifest, 'upgrade', auth=auth) args_list = args_odict.values() - args_list.append(app_id) + args_list.append(app_instance_name) # Prepare env. var. to pass to script env_dict = _make_environment_dict(args_odict) - app_id_withoutinstance, appinstance = _parse_appid(app_id) - env_dict["YNH_APP_INSTANCE_NUMBER"] = str(appinstance) + app_id, app_instance_nb = _parse_app_instance_name(app_instance_name) + env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb) env_dict["YNH_APP_ID"] = app_id # Execute App upgrade script os.system('chown -hR admin: %s' % install_tmp) if hook_exec(app_tmp_folder +'/scripts/upgrade', args=args_list, env=env_dict) != 0: - logger.error(m18n.n('app_upgrade_failed', app=app_id)) + logger.error(m18n.n('app_upgrade_failed', app=app_instance_name)) else: now = int(time.time()) # TODO: Move install_time away from app_setting - app_setting(app_id, 'update_time', now) + app_setting(app_instance_name, 'update_time', now) status['upgraded_at'] = now # Store app status @@ -409,8 +409,8 @@ def app_upgrade(auth, app=[], url=None, file=None): os.system('mv "%s/manifest.json" "%s/scripts" %s' % (app_tmp_folder, app_tmp_folder, app_setting_path)) # So much win - upgraded_apps.append(app_id) - logger.success(m18n.n('app_upgraded', app=app_id)) + upgraded_apps.append(app_instance_name) + logger.success(m18n.n('app_upgraded', app=app_instance_name)) if not upgraded_apps: raise MoulinetteError(errno.ENODATA, m18n.n('app_no_upgrade')) @@ -469,40 +469,42 @@ def app_install(auth, app, label=None, args=None): m18n.n('app_already_installed', app=app_id)) # Change app_id to the forked app id - app_id = app_id + '__' + str(instance_number) + app_instance_name = app_id + '__' + str(instance_number) + else: + app_instance_name = app_id # Retrieve arguments list for install script args_dict = {} if not args else \ dict(urlparse.parse_qsl(args, keep_blank_values=True)) args_odict = _parse_args_from_manifest(manifest, 'install', args=args_dict, auth=auth) args_list = args_odict.values() - args_list.append(app_id) + args_list.append(app_instance_name) # Prepare env. var. to pass to script env_dict = _make_environment_dict(args_odict) env_dict["YNH_APP_INSTANCE_NUMBER"] = str(instance_number) - env_dict["YNH_APP_ID"] = app_id + env_dict["YNH_APP_ID"] = app_instance_name # Create app directory - app_setting_path = os.path.join(apps_setting_path, app_id) + app_setting_path = os.path.join(apps_setting_path, app_instance_name) if os.path.exists(app_setting_path): shutil.rmtree(app_setting_path) os.makedirs(app_setting_path) # Clean hooks and add new ones - hook_remove(app_id) + hook_remove(app_instance_name) if 'hooks' in os.listdir(app_tmp_folder): for file in os.listdir(app_tmp_folder +'/hooks'): - hook_add(app_id, app_tmp_folder +'/hooks/'+ file) + hook_add(app_instance_name, app_tmp_folder +'/hooks/'+ file) # Set initial app settings app_settings = { - 'id': app_id, + 'id': app_instance_name, 'label': label if label else manifest['name'], } # TODO: Move install_time away from app settings app_settings['install_time'] = status['installed_at'] - _set_app_settings(app_id, app_settings) + _set_app_settings(app_instance_name, app_settings) os.system('chown -R admin: '+ app_tmp_folder) @@ -528,7 +530,7 @@ def app_install(auth, app, label=None, args=None): raise MoulinetteError(errno.EIO, m18n.n('installation_failed')) except: # Execute remove script and clean folders - hook_remove(app_id) + hook_remove(app_instance_name) shutil.rmtree(app_setting_path) shutil.rmtree(app_tmp_folder) @@ -1531,32 +1533,32 @@ def _make_environment_dict(args_dict): env_dict[ "YNH_APP_ARG_%s" % arg_name.upper() ] = arg_value return env_dict -def _parse_appid(appid): +def _parse_app_instance_name(app_instance_name): """ - Parse a Yunohost app identifier and extracts the original appid + Parse a Yunohost app instance name and extracts the original appid and the application instance number - >>> _parse_appid('yolo') == ('yolo', 1) + >>> _parse_app_instance_name('yolo') == ('yolo', 1) True - >>> _parse_appid('yolo1') == ('yolo1', 1) + >>> _parse_app_instance_name('yolo1') == ('yolo1', 1) True - >>> _parse_appid('yolo__0') == ('yolo__0', 1) + >>> _parse_app_instance_name('yolo__0') == ('yolo__0', 1) True - >>> _parse_appid('yolo__1') == ('yolo', 1) + >>> _parse_app_instance_name('yolo__1') == ('yolo', 1) True - >>> _parse_appid('yolo__23') == ('yolo', 23) + >>> _parse_app_instance_name('yolo__23') == ('yolo', 23) True - >>> _parse_appid('yolo__42__72') == ('yolo__42', 72) + >>> _parse_app_instance_name('yolo__42__72') == ('yolo__42', 72) True - >>> _parse_appid('yolo__23qdqsd') == ('yolo__23qdqsd', 1) + >>> _parse_app_instance_name('yolo__23qdqsd') == ('yolo__23qdqsd', 1) True - >>> _parse_appid('yolo__23qdqsd56') == ('yolo__23qdqsd56', 1) + >>> _parse_app_instance_name('yolo__23qdqsd56') == ('yolo__23qdqsd56', 1) True """ - match = re_appid.match(appid) + match = re_app_instance_name.match(app_instance_name) appid = match.groupdict().get('appid') - appinstance = int(match.groupdict().get('appinstance')) if match.groupdict().get('appinstance') is not None else 1 - return (appid, appinstance) + app_instance_nb = int(match.groupdict().get('appinstancenb')) if match.groupdict().get('appinstancenb') is not None else 1 + return (appid, app_instance_nb) def is_true(arg): """ From 1c520c59b99e1191793a55e12d923140a4a47d89 Mon Sep 17 00:00:00 2001 From: Julien Malik Date: Fri, 18 Mar 2016 16:55:04 +0100 Subject: [PATCH 12/15] [enh] Add YNH_APP_INSTANCE_NAME for app instance, use YNH_APP_ID for original manifest id --- src/yunohost/app.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 83febbb98..716dcf4cd 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -387,8 +387,9 @@ def app_upgrade(auth, app=[], url=None, file=None): # Prepare env. var. to pass to script env_dict = _make_environment_dict(args_odict) app_id, app_instance_nb = _parse_app_instance_name(app_instance_name) - env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb) env_dict["YNH_APP_ID"] = app_id + env_dict["YNH_APP_INSTANCE_NAME"] = app_instance_name + env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb) # Execute App upgrade script os.system('chown -hR admin: %s' % install_tmp) @@ -482,8 +483,9 @@ def app_install(auth, app, label=None, args=None): # Prepare env. var. to pass to script env_dict = _make_environment_dict(args_odict) + env_dict["YNH_APP_ID"] = app_id + env_dict["YNH_APP_INSTANCE_NAME"] = app_instance_name env_dict["YNH_APP_INSTANCE_NUMBER"] = str(instance_number) - env_dict["YNH_APP_ID"] = app_instance_name # Create app directory app_setting_path = os.path.join(apps_setting_path, app_instance_name) From 7d1d461ef2d7e6007698b5f39ed90172f2896d8f Mon Sep 17 00:00:00 2001 From: Julien Malik Date: Fri, 18 Mar 2016 18:03:19 +0100 Subject: [PATCH 13/15] [enh] also pass YNH_APP_ID YNH_APP_INSTANCE_NAME & YNH_APP_INSTANCE_NUMBER to backup/restore scripts --- src/yunohost/backup.py | 64 +++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 478dfe648..76b4d9508 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -168,6 +168,7 @@ def backup_create(name=None, description=None, output_directory=None, # Backup apps if not ignore_apps: from yunohost.app import app_info + from yunohost.app import _parse_app_instance_name # Filter applications to backup apps_list = set(os.listdir('/etc/yunohost/apps')) @@ -183,21 +184,21 @@ def backup_create(name=None, description=None, output_directory=None, # Run apps backup scripts tmp_script = '/tmp/backup_' + str(timestamp) - for app_id in apps_filtered: - app_setting_path = '/etc/yunohost/apps/' + app_id + for app_instance_name in apps_filtered: + app_setting_path = '/etc/yunohost/apps/' + app_instance_name # Check if the app has a backup and restore script app_script = app_setting_path + '/scripts/backup' app_restore_script = app_setting_path + '/scripts/restore' if not os.path.isfile(app_script): - logger.warning(m18n.n('unbackup_app', app=app_id)) + logger.warning(m18n.n('unbackup_app', app=app_instance_name)) continue elif not os.path.isfile(app_restore_script): - logger.warning(m18n.n('unrestore_app', app=app_id)) + logger.warning(m18n.n('unrestore_app', app=app_instance_name)) - tmp_app_dir = '{:s}/apps/{:s}'.format(tmp_dir, app_id) + tmp_app_dir = '{:s}/apps/{:s}'.format(tmp_dir, app_instance_name) tmp_app_bkp_dir = tmp_app_dir + '/backup' - logger.info(m18n.n('backup_running_app_script', app=app_id)) + logger.info(m18n.n('backup_running_app_script', app=app_instance_name)) try: # Prepare backup directory for the app filesystem.mkdir(tmp_app_bkp_dir, 0750, True, uid='admin') @@ -205,16 +206,24 @@ def backup_create(name=None, description=None, output_directory=None, # Copy app backup script in a temporary folder and execute it subprocess.call(['install', '-Dm555', app_script, tmp_script]) - hook_exec(tmp_script, args=[tmp_app_bkp_dir, app_id], - raise_on_error=True) + + # Prepare env. var. to pass to script + env_dict = {} + app_id, app_instance_nb = _parse_app_instance_name(app_instance_name) + env_dict["YNH_APP_ID"] = app_id + env_dict["YNH_APP_INSTANCE_NAME"] = app_instance_name + env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb) + + hook_exec(tmp_script, args=[tmp_app_bkp_dir, app_instance_name], + raise_on_error=True, env=env_dict) except: - logger.exception(m18n.n('backup_app_failed', app=app_id)) + logger.exception(m18n.n('backup_app_failed', app=app_instance_name)) # Cleaning app backup directory shutil.rmtree(tmp_app_dir, ignore_errors=True) else: # Add app info - i = app_info(app_id) - info['apps'][app_id] = { + i = app_info(app_instance_name) + info['apps'][app_instance_name] = { 'version': i['version'], 'name': i['name'], 'description': i['description'], @@ -429,6 +438,7 @@ def backup_restore(name, hooks=[], apps=[], ignore_apps=False, ignore_hooks=Fals # Add apps restore hook if not ignore_apps: from yunohost.app import _is_installed + from yunohost.app import _parse_app_instance_name # Filter applications to restore apps_list = set(info['apps'].keys()) @@ -442,24 +452,24 @@ def backup_restore(name, hooks=[], apps=[], ignore_apps=False, ignore_hooks=Fals else: apps_filtered = apps_list - for app_id in apps_filtered: - tmp_app_dir = '{:s}/apps/{:s}'.format(tmp_dir, app_id) + for app_instance_name in apps_filtered: + tmp_app_dir = '{:s}/apps/{:s}'.format(tmp_dir, app_instance_name) # Check if the app is not already installed - if _is_installed(app_id): + if _is_installed(app_instance_name): logger.error(m18n.n('restore_already_installed_app', - app=app_id)) + app=app_instance_name)) continue # Check if the app has a restore script app_script = tmp_app_dir + '/settings/scripts/restore' if not os.path.isfile(app_script): - logger.warning(m18n.n('unrestore_app', app=app_id)) + logger.warning(m18n.n('unrestore_app', app=app_instance_name)) continue - tmp_script = '/tmp/restore_' + app_id - app_setting_path = '/etc/yunohost/apps/' + app_id - logger.info(m18n.n('restore_running_app_script', app=app_id)) + tmp_script = '/tmp/restore_' + app_instance_name + app_setting_path = '/etc/yunohost/apps/' + app_instance_name + logger.info(m18n.n('restore_running_app_script', app=app_instance_name)) try: # Copy app settings and set permissions shutil.copytree(tmp_app_dir + '/settings', app_setting_path) @@ -468,14 +478,22 @@ def backup_restore(name, hooks=[], apps=[], ignore_apps=False, ignore_hooks=Fals # Execute app restore script subprocess.call(['install', '-Dm555', app_script, tmp_script]) - hook_exec(tmp_script, args=[tmp_app_dir + '/backup', app_id], - raise_on_error=True) + + # Prepare env. var. to pass to script + env_dict = {} + app_id, app_instance_nb = _parse_app_instance_name(app_instance_name) + env_dict["YNH_APP_ID"] = app_id + env_dict["YNH_APP_INSTANCE_NAME"] = app_instance_name + env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb) + + hook_exec(tmp_script, args=[tmp_app_dir + '/backup', app_instance_name], + raise_on_error=True, env=env_dict) except: - logger.exception(m18n.n('restore_app_failed', app=app_id)) + logger.exception(m18n.n('restore_app_failed', app=app_instance_name)) # Cleaning app directory shutil.rmtree(app_setting_path, ignore_errors=True) else: - result['apps'].append(app_id) + result['apps'].append(app_instance_name) finally: filesystem.rm(tmp_script, force=True) From 205d6db7bdc3996d5160e9c96595d7f9a19026fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Mon, 25 Apr 2016 18:10:25 +0200 Subject: [PATCH 14/15] [enh] Small changes to env prepending in hook_exec --- src/yunohost/backup.py | 2 +- src/yunohost/hook.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index e9d945a28..d1abc22ca 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -486,7 +486,7 @@ def backup_restore(auth, name, hooks=[], ignore_hooks=False, env_dict["YNH_APP_INSTANCE_NAME"] = app_instance_name env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb) - hook_exec(tmp_script, args=[tmp_app_dir + '/backup', app_instance_name], + hook_exec(tmp_script, args=[tmp_app_bkp_dir, app_instance_name], raise_on_error=True, chdir=tmp_app_bkp_dir, env=env_dict) except: logger.exception(m18n.n('restore_app_failed', app=app_instance_name)) diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index 08000d793..da85aaad3 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -329,18 +329,18 @@ def hook_exec(path, args=None, raise_on_error=False, no_trace=False, else: cmd_script = path - envcli = '' - if env is not None: - envcli = ' '.join([ '{key}="{val}"'.format(key=key, val=val) for key,val in env.items()]) - # Construct command to execute command = ['sudo', '-n', '-u', 'admin', '-H', 'sh', '-c'] if no_trace: - cmd = '{envcli} /bin/bash "{script}" {args}' + cmd = '/bin/bash "{script}" {args}' else: # use xtrace on fd 7 which is redirected to stdout - cmd = '{envcli} BASH_XTRACEFD=7 /bin/bash -x "{script}" {args} 7>&1' - command.append(cmd.format(envcli=envcli, script=cmd_script, args=cmd_args)) + cmd = 'BASH_XTRACEFD=7 /bin/bash -x "{script}" {args} 7>&1' + if env: + # prepend environment variables + cmd = '{0} {1}'.format( + ' '.join(['{0}="{1}"'.format(k, v) for k, v in env.items()]), cmd) + command.append(cmd.format(script=cmd_script, args=cmd_args)) if logger.isEnabledFor(log.DEBUG): logger.info(m18n.n('executing_command', command=' '.join(command))) From fd6bd9a348b3aa95c3cb3564f69694cf91249dda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lebleu?= Date: Tue, 26 Apr 2016 16:46:22 +0200 Subject: [PATCH 15/15] [enh] Set YNH_APP_BACKUP_DIR env var for backup/restore scripts --- src/yunohost/backup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index d1abc22ca..ccf75881f 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -217,6 +217,7 @@ def backup_create(name=None, description=None, output_directory=None, env_dict["YNH_APP_ID"] = app_id env_dict["YNH_APP_INSTANCE_NAME"] = app_instance_name env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb) + env_dict["YNH_APP_BACKUP_DIR"] = tmp_app_bkp_dir hook_exec(tmp_script, args=[tmp_app_bkp_dir, app_instance_name], raise_on_error=True, chdir=tmp_app_bkp_dir, env=env_dict) @@ -485,6 +486,7 @@ def backup_restore(auth, name, hooks=[], ignore_hooks=False, env_dict["YNH_APP_ID"] = app_id env_dict["YNH_APP_INSTANCE_NAME"] = app_instance_name env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb) + env_dict["YNH_APP_BACKUP_DIR"] = tmp_app_bkp_dir hook_exec(tmp_script, args=[tmp_app_bkp_dir, app_instance_name], raise_on_error=True, chdir=tmp_app_bkp_dir, env=env_dict)