# -*- coding: utf-8 -*- import os import re import errno import logging import yaml import cPickle as pickle from time import time from collections import OrderedDict from moulinette.core import (MoulinetteError, MoulinetteLock) from moulinette.interfaces import ( BaseActionsMapParser, GLOBAL_SECTION, TO_RETURN_PROP ) from moulinette.utils.log import start_action_logging logger = logging.getLogger('moulinette.actionsmap') # Extra parameters ---------------------------------------------------- # Extra parameters definition class _ExtraParameter(object): """ Argument parser for an extra parameter. It is a pure virtual class that each extra parameter classes must implement. """ def __init__(self, iface): # TODO: Add conn argument which contains authentification object self.iface = iface # Required variables # Each extra parameters classes must overwrite these variables. """The extra parameter name""" name = None # Optional variables # Each extra parameters classes can overwrite these variables. """A list of interface for which the parameter doesn't apply""" skipped_iface = [] # Virtual methods # Each extra parameters classes can implement these methods. def __call__(self, parameter, arg_name, arg_value): """ Parse the argument Keyword arguments: - parameter -- The value of this parameter for the action - arg_name -- The argument name - arg_value -- The argument value Returns: The new argument value """ return arg_value @staticmethod def validate(value, arg_name): """ Validate the parameter value for an argument Keyword arguments: - value -- The parameter value - arg_name -- The argument name Returns: The validated parameter value """ return value class AskParameter(_ExtraParameter): """ Ask for the argument value if possible and needed. The value of this parameter corresponds to the message to display when asking the argument value. """ name = 'ask' skipped_iface = ['api'] def __call__(self, message, arg_name, arg_value): if arg_value: return arg_value try: # Ask for the argument value return msignals.prompt(m18n.n(message)) except NotImplementedError: return arg_value @classmethod def validate(klass, value, arg_name): # Deprecated boolean or empty string if isinstance(value, bool) or (isinstance(value, str) and not value): logger.warning("expecting a string for extra parameter '%s' of " "argument '%s'", klass.name, arg_name) value = arg_name elif not isinstance(value, str): raise TypeError("parameter value must be a string, got %r" % value) return value class PasswordParameter(AskParameter): """ Ask for the password argument value if possible and needed. The value of this parameter corresponds to the message to display when asking the password. """ name = 'password' def __call__(self, message, arg_name, arg_value): if arg_value: return arg_value try: # Ask for the password return msignals.prompt(m18n.n(message), True, True) except NotImplementedError: return arg_value class PatternParameter(_ExtraParameter): """ Check if the argument value match a pattern. The value of this parameter corresponds to a list of the pattern and the message to display if it doesn't match. """ name = 'pattern' def __call__(self, arguments, arg_name, arg_value): pattern, message = (arguments[0], arguments[1]) # Use temporarly utf-8 encoded value try: v = unicode(arg_value, 'utf-8') except: v = arg_value if v and not re.match(pattern, v or '', re.UNICODE): logger.debug("argument value '%s' for '%s' doesn't match pattern '%s'", v, arg_name, pattern) # Attempt to retrieve message translation msg = m18n.n(message) if msg == message: msg = m18n.g(message) raise MoulinetteError(errno.EINVAL, m18n.g('invalid_argument', argument=arg_name, error=msg)) return arg_value @staticmethod def validate(value, arg_name): # Deprecated string type if isinstance(value, str): logger.warning("expecting a list for extra parameter 'pattern' of " "argument '%s'", arg_name) value = [value, 'pattern_not_match'] elif not isinstance(value, list) or len(value) != 2: raise TypeError("parameter value must be a list, got %r" % value) return value class RequiredParameter(_ExtraParameter): """ Check if a required argument is defined or not. The value of this parameter must be a boolean which is set to False by default. """ name = 'required' def __call__(self, required, arg_name, arg_value): if required and (arg_value is None or arg_value == ''): logger.debug("argument '%s' is required", arg_name) raise MoulinetteError(errno.EINVAL, m18n.g('argument_required', argument=arg_name)) return arg_value @staticmethod def validate(value, arg_name): if not isinstance(value, bool): raise TypeError("parameter value must be a list, got %r" % value) return value """ The list of available extra parameters classes. It will keep to this list order on argument parsing. """ extraparameters_list = [AskParameter, PasswordParameter, RequiredParameter, PatternParameter] # Extra parameters argument Parser class ExtraArgumentParser(object): """ Argument validator and parser for the extra parameters. Keyword arguments: - iface -- The running interface """ def __init__(self, iface): self.iface = iface self.extra = OrderedDict() self._extra_params = {GLOBAL_SECTION: {}} # Append available extra parameters for the current interface for klass in extraparameters_list: if iface in klass.skipped_iface: continue self.extra[klass.name] = klass logger.debug('extra parameter classes loaded: %s', self.extra.keys()) def validate(self, arg_name, parameters): """ Validate values of extra parameters for an argument Keyword arguments: - arg_name -- The argument name - parameters -- A dict of extra parameters with their values """ # Iterate over parameters to validate for p, v in parameters.items(): klass = self.extra.get(p, None) if not klass: # Remove unknown parameters del parameters[p] else: try: # Validate parameter value parameters[p] = klass.validate(v, arg_name) except Exception as e: logger.error("unable to validate extra parameter '%s' " "for argument '%s': %s", p, arg_name, e) raise MoulinetteError(errno.EINVAL, m18n.g('error_see_log')) return parameters def add_argument(self, tid, arg_name, parameters, validate=True): """ Add extra parameters to apply on an action argument Keyword arguments: - tid -- The tuple identifier of the action or GLOBAL_SECTION for global extra parameters - arg_name -- The argument name - parameters -- A dict of extra parameters with their values - validate -- False to not validate extra parameters values """ if validate: parameters = self.validate(arg_name, parameters) try: self._extra_params[tid][arg_name] = parameters except KeyError: self._extra_params[tid] = OrderedDict({arg_name: parameters}) def parse_args(self, tid, args): """ Parse arguments for an action with extra parameters Keyword arguments: - tid -- The tuple identifier of the action - args -- A dict of argument name associated to their value """ extra_args = OrderedDict(self._extra_params.get(GLOBAL_SECTION, {})) extra_args.update(self._extra_params.get(tid, {})) # Iterate over action arguments with extra parameters for arg_name, extra_params in extra_args.items(): # Iterate over available extra parameters for p, cls in self.extra.items(): try: extra_value = extra_params[p] except KeyError: continue arg_value = args.get(arg_name, None) # Initialize the extra parser parser = cls(self.iface) # Parse the argument if isinstance(arg_value, list): for v in arg_value: r = parser(extra_value, arg_name, v) if r not in arg_value: arg_value.append(r) else: arg_value = parser(extra_value, arg_name, arg_value) # Update argument value if arg_value is not None: args[arg_name] = arg_value return args # Main class ---------------------------------------------------------- def ordered_yaml_load(stream): class OrderedLoader(yaml.Loader): pass OrderedLoader.add_constructor( yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, lambda loader, node: OrderedDict(loader.construct_pairs(node))) return yaml.load(stream, OrderedLoader) class ActionsMap(object): """Validate and process actions defined into an actions map The actions map defines the features - and their usage - of an application which will be available through the moulinette. It is composed by categories which contain one or more action(s). Moreover, the action can have specific argument(s). This class allows to manipulate one or several actions maps associated to a namespace. If no namespace is given, it will load all available namespaces. Keyword arguments: - parser_class -- The BaseActionsMapParser derived class to use for parsing the actions map - namespaces -- The list of namespaces to use - use_cache -- False if it should parse the actions map file instead of using the cached one - parser_kwargs -- A dict of arguments to pass to the parser class at construction """ def __init__(self, parser_class, namespaces=[], use_cache=True, parser_kwargs={}): if not issubclass(parser_class, BaseActionsMapParser): raise ValueError("Invalid parser class '%s'" % parser_class.__name__) self.parser_class = parser_class self.use_cache = use_cache if len(namespaces) == 0: namespaces = self.get_namespaces() actionsmaps = OrderedDict() # Iterate over actions map namespaces for n in namespaces: logger.debug("loading actions map namespace '%s'", n) actionsmap_yml = '%s/actionsmap/%s.yml' % (pkg.datadir, n) actionsmap_yml_stat = os.stat(actionsmap_yml) actionsmap_pkl = '%s/actionsmap/%s-%d-%d.pkl' % ( pkg.cachedir, n, actionsmap_yml_stat.st_size, actionsmap_yml_stat.st_mtime ) if use_cache and os.path.exists(actionsmap_pkl): try: # Attempt to load cache with open(actionsmap_pkl) as f: actionsmaps[n] = pickle.load(f) # TODO: Switch to python3 and catch proper exception except IOError: self.use_cache = False actionsmaps = self.generate_cache(namespaces) elif use_cache: # cached file doesn't exists self.use_cache = False actionsmaps = self.generate_cache(namespaces) elif n not in actionsmaps: with open(actionsmap_yml) as f: actionsmaps[n] = ordered_yaml_load(f) # Load translations m18n.load_namespace(n) # Generate parsers self.extraparser = ExtraArgumentParser(parser_class.interface) self._parser = self._construct_parser(actionsmaps, **parser_kwargs) @property def parser(self): """Return the instance of the interface's actions map parser""" return self._parser def get_authenticator(self, profile='default'): """Get an authenticator instance Retrieve the authenticator for the given profile and return a new instance. Keyword arguments: - profile -- An authenticator profile name Returns: A new _BaseAuthenticator derived instance """ try: auth = self.parser.get_global_conf('authenticator', profile)[1] except KeyError: raise ValueError("Unknown authenticator profile '%s'" % profile) else: return auth() def process(self, args, timeout=None, **kwargs): """ Parse arguments and process the proper action Keyword arguments: - args -- The arguments to parse - timeout -- The time period before failing if the lock cannot be acquired for the action - **kwargs -- Additional interface arguments """ # Parse arguments arguments = vars(self.parser.parse_args(args, **kwargs)) # Retrieve tid and parse arguments with extra parameters tid = arguments.pop('_tid') arguments = self.extraparser.parse_args(tid, arguments) # Return immediately if a value is defined if TO_RETURN_PROP in arguments: return arguments.get(TO_RETURN_PROP) # Retrieve action information namespace, category, action = tid func_name = '%s_%s' % (category, action.replace('-', '_')) # Lock the moulinette for the namespace with MoulinetteLock(namespace, timeout): try: mod = __import__('%s.%s' % (namespace, category), globals=globals(), level=0, fromlist=[func_name]) func = getattr(mod, func_name) except (AttributeError, ImportError): logger.exception("unable to load function %s.%s.%s", namespace, category, func_name) raise MoulinetteError(errno.EIO, m18n.g('error_see_log')) else: log_id = start_action_logging() if logger.isEnabledFor(logging.DEBUG): # Log arguments in debug mode only for safety reasons logger.info('processing action [%s]: %s.%s.%s with args=%s', log_id, namespace, category, action, arguments) else: logger.info('processing action [%s]: %s.%s.%s', log_id, namespace, category, action) # Load translation and process the action m18n.load_namespace(namespace) start = time() try: return func(**arguments) finally: stop = time() logger.debug('action [%s] ended after %.3fs', log_id, stop - start) @staticmethod def get_namespaces(): """ Retrieve available actions map namespaces Returns: A list of available namespaces """ namespaces = [] for f in os.listdir('%s/actionsmap' % pkg.datadir): if f.endswith('.yml'): namespaces.append(f[:-4]) return namespaces @classmethod def generate_cache(klass, namespaces=None): """ Generate cache for the actions map's file(s) Keyword arguments: - namespaces -- A list of namespaces to generate cache for Returns: A dict of actions map for each namespaces """ actionsmaps = {} if not namespaces: namespaces = klass.get_namespaces() # Iterate over actions map namespaces for n in namespaces: logger.debug("generating cache for actions map namespace '%s'", n) # Read actions map from yaml file am_file = '%s/actionsmap/%s.yml' % (pkg.datadir, n) with open(am_file, 'r') as f: actionsmaps[n] = ordered_yaml_load(f) # at installation, cachedir might not exists if os.path.exists('%s/actionsmap/' % pkg.cachedir): # clean old cached files for i in os.listdir('%s/actionsmap/' % pkg.cachedir): if i.endswith(".pkl"): os.remove('%s/actionsmap/%s' % (pkg.cachedir, i)) # Cache actions map into pickle file am_file_stat = os.stat(am_file) pkl = '%s-%d-%d.pkl' % (n, am_file_stat.st_size, am_file_stat.st_mtime) with pkg.open_cachefile(pkl, 'w', subdir='actionsmap') as f: pickle.dump(actionsmaps[n], f) return actionsmaps # Private methods def _construct_parser(self, actionsmaps, **kwargs): """ Construct the parser with the actions map Keyword arguments: - actionsmaps -- A dict of multi-level dictionnary of categories/actions/arguments list for each namespaces - **kwargs -- Additionnal arguments to pass at the parser class instantiation Returns: An interface relevant's parser object """ # Get extra parameters if not self.use_cache: validate_extra = True else: validate_extra = False # Add arguments to the parser def _add_arguments(tid, parser, arguments): for argn, argp in arguments.items(): names = top_parser.format_arg_names(str(argn), argp.pop('full', None)) try: argp['type'] = eval(argp['type']) except: pass try: extra = argp.pop('extra') arg_dest = (parser.add_argument(*names, **argp)).dest self.extraparser.add_argument(tid, arg_dest, extra, validate_extra) except KeyError: # No extra parameters parser.add_argument(*names, **argp) # Instantiate parser # # this either returns: # * moulinette.interfaces.cli.ActionsMapParser # * moulinette.interfaces.api.ActionsMapParser top_parser = self.parser_class(**kwargs) # namespace, actionsmap is a tuple where: # # * namespace define the top "name", for us it will always be # "yunohost" and there well be only this one # * actionsmap is the actual actionsmap that we care about for namespace, actionsmap in actionsmaps.items(): # Retrieve global parameters _global = actionsmap.pop('_global', {}) top_parser.set_global_conf(_global['configuration']) _add_arguments(GLOBAL_SECTION, top_parser.add_global_parser(), _global['arguments']) # category_name is stuff like "user", "domain", "hooks"... # category_values is the values of this category (like actions) for category_name, category_values in actionsmap.items(): if "actions" not in category_values: # Invalid category without actions logger.warning("no actions found in category '%s' in " "namespace '%s'", category_name, namespace) continue actions = category_values.pop('actions') # Get category parser category_parser = top_parser.add_category_parser(category_name, **category_values) # action_name is like "list" of "domain list" # action_options are the values for action_name, action_options in actions.items(): arguments = action_options.pop('arguments', {}) tid = (namespace, category_name, action_name) if 'configuration' in action_options: configuration = action_options.pop('configuration') _set_conf = lambda p: p.set_conf(tid, configuration) else: # No action configuration _set_conf = lambda p: False try: # Get action parser action_parser = category_parser.add_action_parser(action_name, tid, **action_options) except AttributeError: # No parser for the action continue except ValueError as e: logger.warning("cannot add action (%s, %s, %s): %s", namespace, category_name, action_name, e) continue # Store action identifier and add arguments action_parser.set_defaults(_tid=tid) _add_arguments(tid, action_parser, arguments) _set_conf(category_parser) return top_parser