[enh] Add a new callback action and start to normalize parsing

locales/en.json

@@ -15,6 +15,7 @@
     "ldap_operation_error" : "An error occured during LDAP operation",
     "ldap_attribute_already_exists" : "Attribute already exists: '{:s}={:s}'",
 
+    "invalid_usage" : "Invalid usage, pass --help to see help",
     "argument_required" : "Argument {:s} is required",
     "invalid_argument": "Invalid argument '{:s}': {:s}",
     "pattern_not_match": "Does not match pattern",

moulinette/actionsmap.py

@@ -10,11 +10,11 @@ from time import time
 from collections import OrderedDict
 
 from moulinette.core import (MoulinetteError, MoulinetteLock)
-from moulinette.interfaces import BaseActionsMapParser
+from moulinette.interfaces import (
+    BaseActionsMapParser, GLOBAL_SECTION, TO_RETURN_PROP
+)
 from moulinette.utils.log import start_action_logging
 
-GLOBAL_ARGUMENT = '_global'
-
 logger = logging.getLogger('moulinette.actionsmap')
 
 
@@ -224,7 +224,7 @@ class ExtraArgumentParser(object):
     def __init__(self, iface):
         self.iface = iface
         self.extra = OrderedDict()
-        self._extra_params = { GLOBAL_ARGUMENT: {} }
+        self._extra_params = {GLOBAL_SECTION: {}}
 
         # Append available extra parameters for the current interface
         for klass in extraparameters_list:
@@ -264,7 +264,7 @@ class ExtraArgumentParser(object):
         Add extra parameters to apply on an action argument
 
         Keyword arguments:
-            - tid -- The tuple identifier of the action or GLOBAL_ARGUMENT
+            - tid -- The tuple identifier of the action or GLOBAL_SECTION
                 for global extra parameters
             - arg_name -- The argument name
             - parameters -- A dict of extra parameters with their values
@@ -276,7 +276,7 @@ class ExtraArgumentParser(object):
         try:
             self._extra_params[tid][arg_name] = parameters
         except KeyError:
-            self._extra_params[tid] = OrderedDict({ arg_name: parameters })
+            self._extra_params[tid] = OrderedDict({arg_name: parameters})
 
     def parse_args(self, tid, args):
         """
@@ -287,7 +287,7 @@ class ExtraArgumentParser(object):
             - args -- A dict of argument name associated to their value
 
         """
-        extra_args = OrderedDict(self._extra_params.get(GLOBAL_ARGUMENT, {}))
+        extra_args = OrderedDict(self._extra_params.get(GLOBAL_SECTION, {}))
         extra_args.update(self._extra_params.get(tid, {}))
 
         # Iterate over action arguments with extra parameters
@@ -426,6 +426,10 @@ class ActionsMap(object):
         tid = arguments.pop('_tid')
         arguments = self.extraparser.parse_args(tid, arguments)
 
+        # Return immediately if a value is defined
+        if TO_RETURN_PROP in arguments:
+            return arguments.get(TO_RETURN_PROP)
+
         # Retrieve action information
         namespace, category, action = tid
         func_name = '%s_%s' % (category, action.replace('-', '_'))
@@ -569,7 +573,7 @@ class ActionsMap(object):
                     pass
                 else:
                     # Add arguments
-                    _add_arguments(GLOBAL_ARGUMENT, parser,
+                    _add_arguments(GLOBAL_SECTION, parser,
                                    _global['arguments'])
 
             # -- Parse categories
@@ -598,7 +602,7 @@ class ActionsMap(object):
 
                     try:
                         # Get action parser
-                        parser = cat_parser.add_action_parser(an, tid, **ap)
+                        a_parser = cat_parser.add_action_parser(an, tid, **ap)
                     except AttributeError:
                         # No parser for the action
                         continue
@@ -608,8 +612,8 @@ class ActionsMap(object):
                         continue
                     else:
                         # Store action identifier and add arguments
-                        parser.set_defaults(_tid=tid)
+                        a_parser.set_defaults(_tid=tid)
-                        _add_arguments(tid, parser, args)
+                        _add_arguments(tid, a_parser, args)
                         _set_conf(cat_parser)
 
         return top_parser

moulinette/interfaces/__init__.py

@@ -1,12 +1,19 @@
 # -*- coding: utf-8 -*-
 
+import sys
 import errno
 import logging
+import argparse
+from collections import deque
 
 from moulinette.core import (init_authenticator, MoulinetteError)
 
 logger = logging.getLogger('moulinette.interface')
 
+GLOBAL_SECTION = '_global'
+TO_RETURN_PROP = '_to_return'
+CALLBACKS_PROP = '_callbacks'
+
 
 # Base Class -----------------------------------------------------------
 
@@ -125,6 +132,42 @@ class BaseActionsMapParser(object):
                                       self.__class__.__name__)
 
 
+    ## Arguments helpers
+
+    def prepare_action_namespace(self, tid, namespace=None):
+        """Prepare the namespace for a given action"""
+        # Validate tid and namespace
+        if not isinstance(tid, tuple) and \
+                (namespace is None or not hasattr(namespace, TO_RETURN_PROP)):
+            raise MoulinetteError(errno.EINVAL, m18n.g('invalid_usage'))
+        elif not tid:
+            tid = GLOBAL_SECTION
+
+        # Prepare namespace
+        if namespace is None:
+            namespace = argparse.Namespace()
+        namespace._tid = tid
+
+        # Check lock
+        if not self.get_conf(tid, 'lock'):
+            os.environ['BYPASS_LOCK'] = 'yes'
+
+        # Perform authentication if needed
+        if self.get_conf(tid, 'authenticate'):
+            auth_conf, cls = self.get_conf(tid, 'authenticator')
+
+            # TODO: Catch errors
+            auth = msignals.authenticate(cls(), **auth_conf)
+            if not auth.is_authenticated:
+                raise MoulinetteError(errno.EACCES,
+                                      m18n.g('authentication_required_long'))
+            if self.get_conf(tid, 'argument_auth') and \
+                    self.get_conf(tid, 'authenticate') == 'all':
+                namespace.auth = auth
+
+        return namespace
+
+
     ## Configuration access
 
     @property
@@ -328,3 +371,125 @@ class BaseInterface(object):
     def __init__(self, actionsmap):
         raise NotImplementedError("derived class '%s' must override this method" % \
                                       self.__class__.__name__)
+
+
+# Argument parser ------------------------------------------------------
+
+class _CallbackAction(argparse.Action):
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 nargs=0,
+                 callback={},
+                 default=argparse.SUPPRESS,
+                 help=None):
+        if not callback or 'method' not in callback:
+            raise ValueError('callback must be provided with at least '
+                             'a method key')
+        super(_CallbackAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=nargs,
+            default=default,
+            help=help)
+        self.callback_method = callback.get('method')
+        self.callback_kwargs = callback.get('kwargs', {})
+        self.callback_return = callback.get('return', False)
+        logger.debug("registering new callback action '{0}' to {1}".format(
+            self.callback_method, option_strings))
+
+    @property
+    def callback(self):
+        if not hasattr(self, '_callback'):
+            self._retrieve_callback()
+        return self._callback
+
+    def _retrieve_callback(self):
+        # Attempt to retrieve callback method
+        mod_name, func_name = (self.callback_method).rsplit('.', 1)
+        try:
+            mod = __import__(mod_name, globals=globals(), level=0,
+                             fromlist=[func_name])
+            func = getattr(mod, func_name)
+        except (AttributeError, ImportError):
+            raise ValueError('unable to import method {0}'.format(
+                self.callback_method))
+        self._callback = func
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        parser.enqueue_callback(namespace, self, values)
+        if self.callback_return:
+            setattr(namespace, TO_RETURN_PROP, {})
+
+    def execute(self, namespace, values):
+        try:
+            # Execute callback and get returned value
+            value = self.callback(namespace, values, **self.callback_kwargs)
+        except:
+            logger.exception("cannot get value from callback method " \
+                "'{0}'".format(self.callback_method))
+            raise MoulinetteError(errno.EINVAL, m18n.g('error_see_log'))
+        else:
+            if value:
+                if self.callback_return:
+                    setattr(namespace, TO_RETURN_PROP, value)
+                else:
+                    setattr(namespace, self.dest, value)
+
+class _OptionalSubParsersAction(argparse._SubParsersAction):
+    def __init__(self, *args, **kwargs):
+        required = kwargs.pop('required', False)
+        super(_OptionalSubParsersAction, self).__init__(*args, **kwargs)
+        self.required = required
+
+
+class ExtendedArgumentParser(argparse.ArgumentParser):
+    def __init__(self, *args, **kwargs):
+        super(ExtendedArgumentParser, self).__init__(*args, **kwargs)
+
+        # Register additional actions
+        self.register('action', 'callback', _CallbackAction)
+        self.register('action', 'parsers', _OptionalSubParsersAction)
+
+    def enqueue_callback(self, namespace, callback, values):
+        queue = self._get_callbacks_queue(namespace)
+        queue.append((callback, values))
+
+    def dequeue_callbacks(self, namespace):
+        queue = self._get_callbacks_queue(namespace, False)
+        for _i in xrange(len(queue)):
+            c, v = queue.popleft()
+            # FIXME: break dequeue if callback returns
+            c.execute(namespace, v)
+        try: delattr(namespace, CALLBACKS_PROP)
+        except: pass
+
+    def _get_callbacks_queue(self, namespace, create=True):
+        try:
+            queue = getattr(namespace, CALLBACKS_PROP)
+        except AttributeError:
+            if create:
+                queue = deque()
+                setattr(namespace, CALLBACKS_PROP, queue)
+            else:
+                queue = list()
+        return queue
+
+    def _get_nargs_pattern(self, action):
+        if action.nargs == argparse.PARSER and not action.required:
+            return '([-AO]*)'
+        else:
+            return super(ExtendedArgumentParser, self)._get_nargs_pattern(
+                    action)
+
+    def _get_values(self, action, arg_strings):
+        if action.nargs == argparse.PARSER and not action.required:
+            value = [self._get_value(action, v) for v in arg_strings]
+            if value:
+                self._check_value(action, value[0])
+            else:
+                value = argparse.SUPPRESS
+        else:
+            value = super(ExtendedArgumentParser, self)._get_values(
+                    action, arg_strings)
+        return value

moulinette/interfaces/api.py

@@ -15,7 +15,9 @@ from geventwebsocket import WebSocketError
 from bottle import run, request, response, Bottle, HTTPResponse
 
 from moulinette.core import MoulinetteError, clean_session
-from moulinette.interfaces import (BaseActionsMapParser, BaseInterface)
+from moulinette.interfaces import (
+    BaseActionsMapParser, BaseInterface, ExtendedArgumentParser,
+)
 from moulinette.utils.serialize import JSONExtendedEncoder
 
 logger = logging.getLogger('moulinette.interface.api')
@@ -31,12 +33,12 @@ class _HTTPArgumentParser(object):
     """Argument parser for HTTP requests
 
     Object for parsing HTTP requests into Python objects. It is based
-    on argparse.ArgumentParser class and implements some of its methods.
+    on ExtendedArgumentParser class and implements some of its methods.
 
     """
     def __init__(self):
         # Initialize the ArgumentParser object
-        self._parser = argparse.ArgumentParser(usage='',
+        self._parser = ExtendedArgumentParser(usage='',
                                               prefix_chars='@',
                                               add_help=False)
         self._parser.error = self._error
@@ -106,6 +108,9 @@ class _HTTPArgumentParser(object):
 
         return self._parser.parse_args(arg_strings, namespace)
 
+    def dequeue_callbacks(self, *args, **kwargs):
+        return self._parser.dequeue_callbacks(*args, **kwargs)
+
     def _error(self, message):
         # TODO: Raise a proper exception
         raise MoulinetteError(1, message)
@@ -472,7 +477,7 @@ class ActionsMapParser(BaseActionsMapParser):
     """Actions map's Parser for the API
 
     Provide actions map parsing methods for a CLI usage. The parser for
-    the arguments is represented by a argparse.ArgumentParser object.
+    the arguments is represented by a ExtendedArgumentParser object.
 
     """
     def __init__(self, parent=None):
@@ -580,7 +585,10 @@ class ActionsMapParser(BaseActionsMapParser):
                self.get_conf(tid, 'authenticate') == 'all':
                 ret.auth = auth
 
-        return parser.parse_args(args, ret)
+        # TODO: Catch errors?
+        ret = parser.parse_args(args, ret)
+        parser.dequeue_callbacks(ret)
+        return ret
 
 
     ## Private methods

moulinette/interfaces/cli.py

@@ -3,11 +3,17 @@
 import os
 import errno
 import getpass
-import argparse
 import locale
+import logging
 
 from moulinette.core import MoulinetteError
-from moulinette.interfaces import (BaseActionsMapParser, BaseInterface)
+from moulinette.interfaces import (
+    BaseActionsMapParser, BaseInterface, ExtendedArgumentParser,
+)
+
+
+logger = logging.getLogger('moulinette.cli')
+
 
 # CLI helpers ----------------------------------------------------------
 
@@ -46,7 +52,7 @@ def pretty_print_dict(d, depth=0):
         - depth -- The recursive depth of the dictionary
 
     """
-    for k,v in sorted(d.items(), key=lambda x: x[0]):
+    for k,v in d.items():
         k = colorize(str(k), 'purple')
         if isinstance(v, (tuple, set)):
             v = list(v)
@@ -78,23 +84,28 @@ def get_locale():
         return ''
     return lang[:2]
 
+
 # CLI Classes Implementation -------------------------------------------
 
 class ActionsMapParser(BaseActionsMapParser):
     """Actions map's Parser for the CLI
 
     Provide actions map parsing methods for a CLI usage. The parser for
-    the arguments is represented by a argparse.ArgumentParser object.
+    the arguments is represented by a ExtendedArgumentParser object.
 
     Keyword arguments:
-        - parser -- The argparse.ArgumentParser object to use
+        - parser -- The ExtendedArgumentParser object to use
+        - subparser_kwargs -- Arguments to pass to the sub-parser group
 
     """
-    def __init__(self, parent=None, parser=None):
+    def __init__(self, parent=None, parser=None, subparser_kwargs=None):
         super(ActionsMapParser, self).__init__(parent)
 
-        self._parser = parser or argparse.ArgumentParser()
+        if subparser_kwargs is None:
-        self._subparsers = self._parser.add_subparsers()
+            subparser_kwargs = {'title': "categories", 'required': False}
+
+        self._parser = parser or ExtendedArgumentParser()
+        self._subparsers = self._parser.add_subparsers(**subparser_kwargs)
 
 
     ## Implement virtual properties
@@ -111,7 +122,7 @@ class ActionsMapParser(BaseActionsMapParser):
         return [name]
 
     def add_global_parser(self, **kwargs):
-        return self._parser
+        return self._parser.add_mutually_exclusive_group()
 
     def add_category_parser(self, name, category_help=None, **kwargs):
         """Add a parser for a category
@@ -123,8 +134,10 @@ class ActionsMapParser(BaseActionsMapParser):
             A new ActionsMapParser object for the category
 
         """
-        parser = self._subparsers.add_parser(name, help=category_help)
+        parser = self._subparsers.add_parser(name, help=category_help, **kwargs)
-        return self.__class__(self, parser)
+        return self.__class__(self, parser, {
+                'title': "actions", 'required': True
+        })
 
     def add_action_parser(self, name, tid, action_help=None, **kwargs):
         """Add a parser for an action
@@ -133,30 +146,22 @@ class ActionsMapParser(BaseActionsMapParser):
             - action_help -- A brief description for the action
 
         Returns:
-            A new argparse.ArgumentParser object for the action
+            A new ExtendedArgumentParser object for the action
 
         """
         return self._subparsers.add_parser(name, help=action_help)
 
     def parse_args(self, args, **kwargs):
+        try:
             ret = self._parser.parse_args(args)
-
+        except SystemExit:
-        if not self.get_conf(ret._tid, 'lock'):
+            raise
-            os.environ['BYPASS_LOCK'] = 'yes'
+        except:
-
+            logger.exception("unable to parse arguments '%s'", ' '.join(args))
-        # Perform authentication if needed
+            raise MoulinetteError(errno.EINVAL, m18n.g('error_see_log'))
-        if self.get_conf(ret._tid, 'authenticate'):
+        else:
-            auth_conf, klass = self.get_conf(ret._tid, 'authenticator')
+            self.prepare_action_namespace(getattr(ret, '_tid', None), ret)
-
+            self._parser.dequeue_callbacks(ret)
-            # TODO: Catch errors
-            auth = msignals.authenticate(klass(), **auth_conf)
-            if not auth.is_authenticated:
-                raise MoulinetteError(errno.EACCES,
-                                      m18n.g('authentication_required_long'))
-            if self.get_conf(ret._tid, 'argument_auth') and \
-               self.get_conf(ret._tid, 'authenticate') == 'all':
-                ret.auth = auth
-
             return ret