From a802bcfd6fc1f914044c2c719898b250cebe9f4d Mon Sep 17 00:00:00 2001 From: Jerome Lebleu Date: Wed, 5 Feb 2014 02:01:03 +0100 Subject: [PATCH 01/18] Initial refactoring --- bin/yunohost | 42 ++ bin/yunohost-api | 43 ++ action_map.yml => etc/actionsmap/yunohost.yml | 16 +- firewall.yml => etc/firewall.yml | 4 +- ldap_scheme.yml => etc/ldap_scheme.yml | 0 bash/yunohost_cli => etc/moulinette_cli | 8 +- services.yml => etc/services.yml | 0 src/moulinette/__init__.py | 89 ++++ src/moulinette/config.py | 16 + src/moulinette/core/__init__.py | 0 src/moulinette/core/actionsmap.py | 490 ++++++++++++++++++ src/moulinette/core/api.py | 211 ++++++++ yunohost.py => src/moulinette/core/helpers.py | 42 -- src/moulinette/yunohost/__init__.py | 0 .../moulinette/yunohost/app.py | 9 +- .../moulinette/yunohost/backup.py | 3 +- .../moulinette/yunohost/domain.py | 5 +- .../moulinette/yunohost/dyndns.py | 3 +- .../moulinette/yunohost/firewall.py | 3 +- .../moulinette/yunohost/hook.py | 3 +- .../moulinette/yunohost/monitor.py | 7 +- .../moulinette/yunohost/service.py | 7 +- .../moulinette/yunohost/tools.py | 11 +- .../moulinette/yunohost/user.py | 11 +- txrestapi/.gitignore | 3 - txrestapi/README.rst | 146 ------ txrestapi/__init__.py | 1 - txrestapi/methods.py | 29 -- txrestapi/resource.py | 65 --- txrestapi/service.py | 7 - txrestapi/tests.py | 194 ------- 31 files changed, 941 insertions(+), 527 deletions(-) create mode 100755 bin/yunohost create mode 100755 bin/yunohost-api rename action_map.yml => etc/actionsmap/yunohost.yml (98%) rename firewall.yml => etc/firewall.yml (93%) rename ldap_scheme.yml => etc/ldap_scheme.yml (100%) rename bash/yunohost_cli => etc/moulinette_cli (87%) rename services.yml => etc/services.yml (100%) create mode 100755 src/moulinette/__init__.py create mode 100644 src/moulinette/config.py create mode 100755 src/moulinette/core/__init__.py create mode 100644 src/moulinette/core/actionsmap.py create mode 100644 src/moulinette/core/api.py rename yunohost.py => src/moulinette/core/helpers.py (91%) create mode 100755 src/moulinette/yunohost/__init__.py rename yunohost_app.py => src/moulinette/yunohost/app.py (99%) rename yunohost_backup.py => src/moulinette/yunohost/backup.py (94%) rename yunohost_domain.py => src/moulinette/yunohost/domain.py (98%) rename yunohost_dyndns.py => src/moulinette/yunohost/dyndns.py (98%) rename yunohost_firewall.py => src/moulinette/yunohost/firewall.py (99%) rename yunohost_hook.py => src/moulinette/yunohost/hook.py (98%) rename yunohost_monitor.py => src/moulinette/yunohost/monitor.py (99%) rename yunohost_service.py => src/moulinette/yunohost/service.py (99%) rename yunohost_tools.py => src/moulinette/yunohost/tools.py (96%) rename yunohost_user.py => src/moulinette/yunohost/user.py (98%) delete mode 100644 txrestapi/.gitignore delete mode 100644 txrestapi/README.rst delete mode 100644 txrestapi/__init__.py delete mode 100644 txrestapi/methods.py delete mode 100644 txrestapi/resource.py delete mode 100644 txrestapi/service.py delete mode 100644 txrestapi/tests.py diff --git a/bin/yunohost b/bin/yunohost new file mode 100755 index 00000000..d4483df5 --- /dev/null +++ b/bin/yunohost @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os.path +import gettext + +# Debug option +if '--debug' in sys.argv: + sys.path.append(os.path.abspath(os.path.dirname(__file__) +'/../src')) +from moulinette import cli +from moulinette.core.helpers import YunoHostError, colorize + +gettext.install('YunoHost') + + +## Main action + +if __name__ == '__main__': + # Additional arguments + use_cache = True + if '--no-cache' in sys.argv: + use_cache = False + sys.argv.remove('--no-cache') + if '--debug' in sys.argv: + sys.argv.remove('--debug') + + try: + args = list(sys.argv) + args.pop(0) + + # Check that YunoHost is installed + if not os.path.isfile('/etc/yunohost/installed') \ + and (len(args) < 2 or args[1] != 'tools' or args[2] != 'postinstall'): + raise YunoHostError(17, _("YunoHost is not correctly installed, please execute 'yunohost tools postinstall'")) + + # Execute the action + cli(args, use_cache) + except YunoHostError as e: + print(colorize(_("Error: "), 'red') + e.message) + sys.exit(e.code) + sys.exit(0) diff --git a/bin/yunohost-api b/bin/yunohost-api new file mode 100755 index 00000000..625d2b7e --- /dev/null +++ b/bin/yunohost-api @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os.path +import gettext + +# Debug option +if '--debug' in sys.argv: + sys.path.append(os.path.abspath(os.path.dirname(__file__) +'/../src')) +from moulinette import api + +gettext.install('YunoHost') + + +## Callbacks for additional routes + +def is_installed(): + """ + Check whether YunoHost is installed or not + + """ + installed = False + if os.path.isfile('/etc/yunohost/installed'): + installed = True + return { 'installed': installed } + + +## Main action + +if __name__ == '__main__': + # Additional arguments + use_cache = True + if '--no-cache' in sys.argv: + use_cache = False + sys.argv.remove('--no-cache') + if '--debug' in sys.argv: + sys.argv.remove('--debug') + # TODO: Add log argument + + # Rune the server + api(6787, {('GET', '/installed'): is_installed}, use_cache) + sys.exit(0) diff --git a/action_map.yml b/etc/actionsmap/yunohost.yml similarity index 98% rename from action_map.yml rename to etc/actionsmap/yunohost.yml index 15b41c16..cf2df613 100644 --- a/action_map.yml +++ b/etc/actionsmap/yunohost.yml @@ -95,7 +95,7 @@ user: ### user_delete() delete: action_help: Delete user - api: 'DELETE /users/{users}' + api: 'DELETE /users/' arguments: -u: full: --users @@ -109,7 +109,7 @@ user: ### user_update() update: action_help: Update user informations - api: 'PUT /users/{username}' + api: 'PUT /users/' arguments: username: help: Username of user to update @@ -143,7 +143,7 @@ user: ### user_info() info: action_help: Get user informations - api: 'GET /users/{username}' + api: 'GET /users/' arguments: username: help: Username or mail to get informations @@ -202,7 +202,7 @@ domain: ### domain_info() info: action_help: Get domain informations - api: 'GET /domains/{domain}' + api: 'GET /domains/' arguments: domain: help: "" @@ -266,7 +266,7 @@ app: ### app_info() info: action_help: Get app info - api: GET /app/{app} + api: GET /app/ arguments: app: help: Specific app ID @@ -310,7 +310,7 @@ app: ### app_remove() TODO: Write help remove: action_help: Remove app - api: DELETE /app/{app} + api: DELETE /app/ arguments: app: help: App(s) to delete @@ -333,7 +333,7 @@ app: ### app_setting() setting: action_help: Set ou get an app setting value - api: GET /app/{app}/setting + api: GET /app//setting arguments: app: help: App ID @@ -350,7 +350,7 @@ app: ### app_service() service: action_help: Add or remove a YunoHost monitored service - api: POST /app/service/{service} + api: POST /app/service/ arguments: service: help: Service to add/remove diff --git a/firewall.yml b/etc/firewall.yml similarity index 93% rename from firewall.yml rename to etc/firewall.yml index 075116cf..5a7383d1 100644 --- a/firewall.yml +++ b/etc/firewall.yml @@ -1,6 +1,6 @@ -UPNP: +UPNP: cron: false - ports: + ports: TCP: [22, 25, 53, 80, 443, 465, 993, 5222, 5269, 5280, 6767, 7676] UDP: [53, 137, 138] ipv4: diff --git a/ldap_scheme.yml b/etc/ldap_scheme.yml similarity index 100% rename from ldap_scheme.yml rename to etc/ldap_scheme.yml diff --git a/bash/yunohost_cli b/etc/moulinette_cli similarity index 87% rename from bash/yunohost_cli rename to etc/moulinette_cli index 20ed5855..15ac21f2 100644 --- a/bash/yunohost_cli +++ b/etc/moulinette_cli @@ -7,14 +7,14 @@ COMPREPLY=() argc=${COMP_CWORD} cur="${COMP_WORDS[argc]}" -prev="${COMP_WORDS[argc-1]}" +prev="${COMP_WORDS[argc-1]}" opts=$(yunohost -h | sed -n "/usage/,/}/p" | awk -F"{" '{print $2}' | awk -F"}" '{print $1}' | tr ',' ' ') if [[ $argc = 1 ]]; then COMPREPLY=( $(compgen -W "$opts --help" -- $cur ) ) fi - + if [[ "$prev" != "--help" ]]; then if [[ $argc = 2 ]]; @@ -23,9 +23,9 @@ then COMPREPLY=( $(compgen -W "$opts2 --help" -- $cur ) ) elif [[ $argc = 3 ]]; then - COMPREPLY=( $(compgen -W "--help" $cur ) ) + COMPREPLY=( $(compgen -W "--help" $cur ) ) fi -else +else COMPREPLY=() fi diff --git a/services.yml b/etc/services.yml similarity index 100% rename from services.yml rename to etc/services.yml diff --git a/src/moulinette/__init__.py b/src/moulinette/__init__.py new file mode 100755 index 00000000..784ed407 --- /dev/null +++ b/src/moulinette/__init__.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- + +__title__ = 'moulinette' +__version__ = '695' +__author__ = ['Kload', + 'jlebleu', + 'titoko', + 'beudbeud', + 'npze'] +__license__ = 'AGPL 3.0' +__credits__ = """ + Copyright (C) 2014 YUNOHOST.ORG + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program; if not, see http://www.gnu.org/licenses + """ + + +## Fast access functions + +def api(port, routes={}, use_cache=True): + """ + Run a HTTP server with the moulinette for an API usage. + + Keyword arguments: + + - port -- Port to run on + + - routes -- A dict of additional routes to add in the form of + {(method, uri): callback} + + - use_cache -- False if it should parse the actions map file + instead of using the cached one + + """ + from bottle import run + from core.actionsmap import ActionsMap + from core.api import MoulinetteAPI + + amap = ActionsMap(ActionsMap.IFACE_API, use_cache=use_cache) + moulinette = MoulinetteAPI(amap, routes) + + run(moulinette.app, port=port) + +def cli(args, use_cache=True): + """ + Execute an action with the moulinette from the CLI and print its + result in a readable format. + + Keyword arguments: + + - args -- A list of argument strings + + - use_cache -- False if it should parse the actions map file + instead of using the cached one + + """ + import os + from core.actionsmap import ActionsMap + from core.helpers import YunoHostError, pretty_print_dict + + lock_file = '/var/run/moulinette.lock' + + # Check the lock + if os.path.isfile(lock_file): + raise YunoHostError(1, _("The moulinette is already running")) + + # Create a lock + with open(lock_file, 'w') as f: pass + os.system('chmod 400 '+ lock_file) + + try: + amap = ActionsMap(ActionsMap.IFACE_CLI, use_cache=use_cache) + pretty_print_dict(amap.process(args)) + except KeyboardInterrupt, EOFError: + raise YunoHostError(125, _("Interrupted")) + finally: + # Remove the lock + os.remove(lock_file) diff --git a/src/moulinette/config.py b/src/moulinette/config.py new file mode 100644 index 00000000..b1bcdfb9 --- /dev/null +++ b/src/moulinette/config.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +# TODO: Remove permanent debug values +import os + +# Path for the the web sessions +session_path = '/var/cache/yunohost/session' + +# Path of the actions map definition(s) +actionsmap_path = os.path.dirname(__file__) +'/../../etc/actionsmap' + +# Path for the actions map cache +actionsmap_cache_path = '/var/cache/yunohost/actionsmap' + +# Path of the doc in json format +doc_json_path = os.path.dirname(__file__) +'/../../doc' diff --git a/src/moulinette/core/__init__.py b/src/moulinette/core/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/src/moulinette/core/actionsmap.py b/src/moulinette/core/actionsmap.py new file mode 100644 index 00000000..022dc06c --- /dev/null +++ b/src/moulinette/core/actionsmap.py @@ -0,0 +1,490 @@ +# -*- coding: utf-8 -*- + +import argparse +import getpass +import marshal +import pickle +import yaml +import re +import os + +from .. import __version__ +from ..config import actionsmap_path, actionsmap_cache_path +from helpers import YunoHostError, colorize + +class _HTTPArgumentParser(object): + + def __init__(self, method, uri): + # Initialize the ArgumentParser object + self._parser = argparse.ArgumentParser(usage='', + prefix_chars='@', + add_help=False) + self._parser.error = self._error + + self.method = method + self.uri = uri + + self._positional = [] # list(arg_name) + self._optional = {} # dict({arg_name: option_strings}) + + def set_defaults(self, **kwargs): + return self._parser.set_defaults(**kwargs) + + def get_default(self, dest): + return self._parser.get_default(dest) + + def add_argument(self, *args, **kwargs): + action = self._parser.add_argument(*args, **kwargs) + + # Append newly created action + if len(action.option_strings) == 0: + self._positional.append(action.dest) + else: + self._optional[action.dest] = action.option_strings + + return action + + def parse_args(self, args): + arg_strings = [] + + ## Append an argument to the current one + def append(arg_strings, value, option_string=None): + # TODO: Process list arguments + if isinstance(value, bool): + # Append the option string only + if option_string is not None: + arg_strings.append(option_string) + elif isinstance(value, str): + if option_string is not None: + arg_strings.append(option_string) + arg_strings.append(value) + else: + arg_strings.append(value) + + return arg_strings + + # Iterate over positional arguments + for dest in self._positional: + if dest in args: + arg_strings = append(arg_strings, args[dest]) + + # Iterate over optional arguments + for dest, opt in self._optional.items(): + if dest in args: + arg_strings = append(arg_strings, args[dest], opt[0]) + return self._parser.parse_args(arg_strings) + + def _error(self, message): + # TODO: Raise a proper exception + raise Exception(message) + +class HTTPParser(object): + """ + Object for parsing HTTP requests into Python objects. + + """ + + def __init__(self): + self._parsers = {} # dict({(method, uri): _HTTPArgumentParser}) + + @property + def routes(self): + """Get current routes""" + return self._parsers.keys() + + def add_parser(self, method, uri): + """ + Add a parser for a given route + + Keyword arguments: + - method -- The route's HTTP method (GET, POST, PUT, DELETE) + - uri -- The route's URI + + Returns: + A new _HTTPArgumentParser object for the route + + """ + # Check if a parser already exists for the route + key = (method, uri) + if key in self.routes: + raise ValueError("A parser for '%s' already exists" % key) + + # Create and append parser + parser = _HTTPArgumentParser(method, uri) + self._parsers[key] = parser + + # Return the created parser + return parser + + def parse_args(self, method, uri, args={}): + """ + Convert argument variables to objects and assign them as + attributes of the namespace for a given route + + Keyword arguments: + - method -- The route's HTTP method (GET, POST, PUT, DELETE) + - uri -- The route's URI + - args -- Argument variables for the route + + Returns: + The populated namespace + + """ + # Retrieve the parser for the route + key = (method, uri) + if key not in self.routes: + raise ValueError("No parser for '%s %s' found" % key) + + return self._parsers[key].parse_args(args) + + +class _ExtraParameters(object): + + CLI_PARAMETERS = ['ask', 'password', 'pattern'] + API_PARAMETERS = ['pattern'] + AVAILABLE_PARAMETERS = CLI_PARAMETERS + + def __init__(self, **kwargs): + self._params = {} + + for k, v in kwargs.items(): + if k in self.AVAILABLE_PARAMETERS: + self._params[k] = v + + def validate(self, p_name, p_value): + ret = type(p_value)() if p_value is not None else None + + for p, v in self._params.items(): + func = getattr(self, 'process_' + p) + + if isinstance(ret, list): + for p_v in p_value: + r = func(v, p_name, p_v) + if r is not None: + ret.append(r) + else: + r = func(v, p_name, p_value) + if r is not None: + ret = r + + return ret + + + ## Parameters validating's method + # TODO: Add doc + + def process_ask(self, message, p_name, p_value): + # TODO: Fix asked arguments ordering + if not self._can_prompt(p_value): + return p_value + + # Skip password asking + if 'password' in self._params.keys(): + return None + + ret = raw_input(colorize(message + ': ', 'cyan')) + return ret + + def process_password(self, is_password, p_name, p_value): + if not self._can_prompt(p_value): + return p_value + + message = self._params['ask'] + pwd1 = getpass.getpass(colorize(message + ': ', 'cyan')) + pwd2 = getpass.getpass(colorize('Retype ' + message + ': ', 'cyan')) + if pwd1 != pwd2: + raise YunoHostError(22, _("Passwords don't match")) + return pwd1 + + def process_pattern(self, pattern, p_name, p_value): + # TODO: Add a pattern_help parameter + # TODO: Fix missing pattern matching on asking + if p_value is not None and not re.match(pattern, p_value): + raise YunoHostError(22, _("'%s' argument not match pattern" % p_name)) + return p_value + + + ## Private method + + def _can_prompt(self, p_value): + if os.isatty(1) and (p_value is None or p_value == ''): + return True + return False + + +class ActionsMap(object): + """ + Validate and process action defined into the actions map. + + The actions map defines features and their usage of the main + application. It is composed by categories which contain one or more + action(s). Moreover, the action can have specific argument(s). + + Keyword arguments: + + - interface -- Interface type that requires the actions map. + Possible value is one of: + - 'cli' for the command line interface + - 'api' for an API usage (HTTP requests) + + - use_cache -- False if it should parse the actions map file + instead of using the cached one. + + """ + IFACE_CLI = 'cli' + IFACE_API = 'api' + + def __init__(self, interface, use_cache=True): + if interface not in [self.IFACE_CLI,self.IFACE_API]: + raise ValueError(_("Invalid interface '%s'" % interface)) + self.interface = interface + + # Iterate over actions map namespaces + actionsmap = {} + for n in self.get_actionsmap_namespaces(): + if use_cache: + cache_file = '%s/%s.pkl' % (actionsmap_cache_path, n) + if os.path.isfile(cache_file): + with open(cache_file, 'r') as f: + actionsmap[n] = pickle.load(f) + else: + actionsmap = self.generate_cache() + else: + am_file = '%s/%s.yml' % (actionsmap_path, n) + with open(am_file, 'r') as f: + actionsmap[n] = yaml.load(f) + + self.parser = self._construct_parser(actionsmap) + + def process(self, args, route=None): + """ + Parse arguments and process the proper action + + Keyword arguments: + - args -- The arguments to parse + - route -- A tupple (method, uri) of the requested route (api only) + + """ + arguments = None + + # Parse arguments + if self.interface ==self.IFACE_CLI: + arguments = self.parser.parse_args(args) + elif self.interface ==self.IFACE_API: + if route is None: + # TODO: Raise a proper exception + raise Exception(_("Missing route argument")) + method, uri = route + arguments = self.parser.parse_args(method, uri, args) + arguments = vars(arguments) + + # Parse extra parameters + arguments = self._parse_extra_parameters(arguments) + + # Retrieve action information + namespace = arguments['_info']['namespace'] + category = arguments['_info']['category'] + action = arguments['_info']['action'] + del arguments['_info'] + + module = '%s.%s' % (namespace, category) + function = '%s_%s' % (category, action) + + try: + mod = __import__(module, globals=globals(), fromlist=[function], level=2) + func = getattr(mod, function) + except (AttributeError, ImportError): + raise YunoHostError(168, _('Function is not defined')) + else: + # Process the action + return func(**arguments) + + @staticmethod + def get_actionsmap_namespaces(path=actionsmap_path): + """ + Retrieve actions map namespaces in a given path + + """ + namespaces = [] + + for f in os.listdir(path): + if f.endswith('.yml'): + namespaces.append(f[:-4]) + return namespaces + + @classmethod + def generate_cache(cls): + """ + Generate cache for the actions map's file(s) + + """ + actionsmap = {} + + if not os.path.isdir(actionsmap_cache_path): + os.makedirs(actionsmap_cache_path) + + for n in cls.get_actionsmap_namespaces(): + am_file = '%s/%s.yml' % (actionsmap_path, n) + with open(am_file, 'r') as f: + actionsmap[n] = yaml.load(f) + cache_file = '%s/%s.pkl' % (actionsmap_cache_path, n) + with open(cache_file, 'w') as f: + pickle.dump(actionsmap[n], f) + + return actionsmap + + + ## Private class and methods + + def _store_extra_parameters(self, parser, arg_name, arg_params): + """ + Store extra parameters for a given parser's argument name + + Keyword arguments: + - parser -- Parser object of the argument + - arg_name -- Argument name + - arg_params -- Argument parameters + + """ + params = {} + keys = [] + + # Get available parameters for the current interface + if self.interface ==self.IFACE_CLI: + keys = _ExtraParameters.CLI_PARAMETERS + elif self.interface ==self.IFACE_API: + keys = _ExtraParameters.API_PARAMETERS + + for k in keys: + if k in arg_params: + params[k] = arg_params[k] + + if len(params) > 0: + # Retrieve all extra parameters from the parser + extra = parser.get_default('_extra') + if not extra or not isinstance(extra, dict): + extra = {} + + # Add completed extra parameters to the parser + extra[arg_name] = _ExtraParameters(**params) + parser.set_defaults(_extra=extra) + + return parser + + def _parse_extra_parameters(self, args): + # Retrieve extra parameters from the arguments + if '_extra' not in args: + return args + extra = args['_extra'] + del args['_extra'] + + # Validate extra parameters for each arguments + for n, e in extra.items(): + args[n] = e.validate(n, args[n]) + + return args + + def _construct_parser(self, actionsmap): + """ + Construct the parser with the actions map + + Keyword arguments: + - actionsmap -- Multi-level dictionnary of + categories/actions/arguments list + + Returns: + Interface relevant's parser object + + """ + top_parser = None + iface = self.interface + + # Create parser object + if iface ==self.IFACE_CLI: + # TODO: Add descritpion (from __description__) + top_parser = argparse.ArgumentParser() + top_subparsers = top_parser.add_subparsers() + elif iface ==self.IFACE_API: + top_parser = HTTPParser() + + ## Extract option strings from parameters + def _option_strings(arg_name, arg_params): + if iface ==self.IFACE_CLI: + if arg_name[0] == '-' and 'full' in arg_params: + return [arg_name, arg_params['full']] + return [arg_name] + elif iface ==self.IFACE_API: + if arg_name[0] != '-': + return [arg_name] + if 'full' in arg_params: + return [arg_params['full'].replace('--', '@', 1)] + if arg_name.startswith('--'): + return [arg_name.replace('--', '@', 1)] + return [arg_name.replace('-', '@', 1)] + + ## Extract a key from parameters + def _key(arg_params, key, default=str()): + if key in arg_params: + return arg_params[key] + return default + + ## Remove extra parameters + def _clean_params(arg_params): + keys = list(_ExtraParameters.AVAILABLE_PARAMETERS) + keys.append('full') + + for k in keys: + if k in arg_params: + del arg_params[k] + return arg_params + + # Iterate over actions map namespaces + for n in self.get_actionsmap_namespaces(): + # Parse general arguments for the cli only + if iface ==self.IFACE_CLI: + for an, ap in actionsmap[n]['general_arguments'].items(): + if 'version' in ap: + ap['version'] = ap['version'].replace('%version%', __version__) + top_parser.add_argument(*_option_strings(an, ap), **_clean_params(ap)) + del actionsmap[n]['general_arguments'] + + # Parse categories + for cn, cp in actionsmap[n].items(): + if 'actions' not in cp: + continue + + # Add category subparsers for the cli only + if iface ==self.IFACE_CLI: + c_help = _key(cp, 'category_help') + subparsers = top_subparsers.add_parser(cn, help=c_help).add_subparsers() + + # Parse actions + for an, ap in cp['actions'].items(): + parser = None + + # Add parser for the current action + if iface ==self.IFACE_CLI: + a_help = _key(ap, 'action_help') + parser = subparsers.add_parser(an, help=a_help) + elif iface ==self.IFACE_API and 'api' in ap: + # Extract method and uri + m = re.match('(GET|POST|PUT|DELETE) (/\S+)', ap['api']) + if m: + parser = top_parser.add_parser(m.group(1), m.group(2)) + if not parser: + continue + + # Store action information + parser.set_defaults(_info={'namespace': n, + 'category': cn, + 'action': an}) + + # Add arguments + if not 'arguments' in ap: + continue + for argn, argp in ap['arguments'].items(): + arg = parser.add_argument(*_option_strings(argn, argp), + **_clean_params(argp.copy())) + parser = self._store_extra_parameters(parser, arg.dest, argp) + + return top_parser diff --git a/src/moulinette/core/api.py b/src/moulinette/core/api.py new file mode 100644 index 00000000..ea00cc9e --- /dev/null +++ b/src/moulinette/core/api.py @@ -0,0 +1,211 @@ +# -*- coding: utf-8 -*- + +import os.path +from bottle import Bottle, request, response, HTTPResponse +from beaker.middleware import SessionMiddleware + +from ..config import session_path, doc_json_path +from helpers import YunoHostError, YunoHostLDAP + +class APIAuthPlugin(object): + """ + This Bottle plugin manages the authentication for the API access. + + """ + name = 'apiauth' + api = 2 + + def __init__(self): + # TODO: Add options (e.g. session type, content type, ...) + if not os.path.isdir(session_path): + os.makedirs(session_path) + + @property + def app(self): + """Get Bottle application with session integration""" + if hasattr(self, '_app'): + return self._app + raise Exception(_("The APIAuth Plugin is not installed yet.")) + + def setup(self, app): + """ + Setup the plugin and install the session into the app + + Keyword argument: + app -- The associated application object + + """ + app.route('/login', name='login', method='POST', callback=self.login) + app.route('/logout', name='logout', method='GET', callback=self.logout) + + session_opts = { + 'session.type': 'file', + 'session.cookie_expires': True, + 'session.data_dir': session_path, + 'session.secure': True + } + self._app = SessionMiddleware(app, session_opts) + + def apply(self, callback, context): + """ + Check authentication before executing the route callback + + Keyword argument: + callback -- The route callback + context -- An instance of Route + + """ + # Check the authentication + if self._is_authenticated: + if context.name == 'login': + self.logout() + else: + # TODO: Fix this tweak to retrieve uri from the callback + def wrapper(*args, **kwargs): + if hasattr(context.config, '_uri'): + kwargs['_uri'] = context.config._uri + return callback(*args, **kwargs) + return wrapper + + # Process login route + if context.name == 'login': + password = request.POST.get('password', None) + if password is not None and self.login(password): + raise HTTPResponse(status=200) + else: + raise HTTPResponse(_("Wrong password"), 401) + + # Deny access to the requested route + raise HTTPResponse(_("Unauthorized"), 401) + + def login(self, password): + """ + Attempt to log in with the given password + + Keyword argument: + password -- Cleartext password + + """ + try: YunoHostLDAP(password=password) + except YunoHostError: + return False + else: + session = self._beaker_session + session['authenticated'] = True + session.save() + return True + return False + + def logout(self): + """ + Log out and delete the session + + """ + # TODO: Delete the cached session file + session = self._beaker_session + session.delete() + + + ## Private methods + + @property + def _beaker_session(self): + """Get Beaker session""" + return request.environ.get('beaker.session') + + @property + def _is_authenticated(self): + """Check authentication""" + # TODO: Clear the session path on password changing to avoid invalid access + if 'authenticated' in self._beaker_session: + return True + return False + + +class MoulinetteAPI(object): + """ + Initialize a HTTP server which serves the API to access to the + moulinette actions. + + Keyword arguments: + + - actionsmap -- The relevant ActionsMap instance + + - routes -- A dict of additional routes to add in the form of + {(method, uri): callback} + + """ + + def __init__(self, actionsmap, routes={}): + self.actionsmap = actionsmap + + # Initialize app and default routes + # TODO: Return OK to 'OPTIONS' xhr requests (l173) + app = Bottle() + app.route(['/api', '/api/'], method='GET', callback=self.doc, skip=['apiauth']) + + # Append routes from the actions map + for (m, u) in actionsmap.parser.routes: + app.route(u, method=m, callback=self._route_wrapper, _uri=u) + + # Append additional routes + for (m, u), c in routes.items(): + app.route(u, method=m, callback=c) + + # Define and install a plugin which sets proper header + def apiheader(callback): + def wrapper(*args, **kwargs): + response.content_type = 'application/json' + response.set_header('Access-Control-Allow-Origin', '*') + return callback(*args, **kwargs) + return wrapper + app.install(apiheader) + + # Install authentication plugin + apiauth = APIAuthPlugin() + app.install(apiauth) + + self._app = apiauth.app + + @property + def app(self): + """Get Bottle application""" + return self._app + + def doc(self, category=None): + """ + Get API documentation for a category (all by default) + + Keyword argument: + category -- Name of the category + + """ + if category is None: + with open(doc_json_path +'/resources.json') as f: + return f.read() + + try: + with open(doc_json_path +'/'+ category +'.json') as f: + return f.read() + except IOError: + return 'unknown' + + def _route_wrapper(self, *args, **kwargs): + """Process the relevant action for the request""" + # Retrieve uri + if '_uri' in kwargs: + uri = kwargs['_uri'] + del kwargs['_uri'] + else: + uri = request.path + + # Bring arguments together + params = kwargs + for a in args: + params[a] = True + for k, v in request.params.items(): + params[k] = v + + # Process the action + # TODO: Catch errors + return self.actionsmap.process(params, (request.method, uri)) diff --git a/yunohost.py b/src/moulinette/core/helpers.py similarity index 91% rename from yunohost.py rename to src/moulinette/core/helpers.py index f75ee7ac..cf4e383c 100644 --- a/yunohost.py +++ b/src/moulinette/core/helpers.py @@ -1,47 +1,5 @@ # -*- coding: utf-8 -*- -""" License - - Copyright (C) 2013 YunoHost - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program; if not, see http://www.gnu.org/licenses - -""" - -""" - YunoHost core classes & functions -""" - -__credits__ = """ - Copyright (C) 2012 YunoHost - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program; if not, see http://www.gnu.org/licenses - """ -__author__ = 'Kload ' -__version__ = '695' - import os import sys try: diff --git a/src/moulinette/yunohost/__init__.py b/src/moulinette/yunohost/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/yunohost_app.py b/src/moulinette/yunohost/app.py similarity index 99% rename from yunohost_app.py rename to src/moulinette/yunohost/app.py index 24a7299d..70d8f978 100644 --- a/yunohost_app.py +++ b/src/moulinette/yunohost/app.py @@ -33,10 +33,11 @@ import time import re import socket import urlparse -from yunohost import YunoHostError, YunoHostLDAP, win_msg, random_password, is_true, validate -from yunohost_domain import domain_list, domain_add -from yunohost_user import user_info, user_list -from yunohost_hook import hook_exec, hook_add, hook_remove +from domain import domain_list, domain_add +from user import user_info, user_list +from hook import hook_exec, hook_add, hook_remove + +from ..core.helpers import YunoHostError, YunoHostLDAP, win_msg, random_password, is_true, validate repo_path = '/var/cache/yunohost/repo' apps_path = '/usr/share/yunohost/apps' diff --git a/yunohost_backup.py b/src/moulinette/yunohost/backup.py similarity index 94% rename from yunohost_backup.py rename to src/moulinette/yunohost/backup.py index 2edea6bb..18363bbb 100644 --- a/yunohost_backup.py +++ b/src/moulinette/yunohost/backup.py @@ -28,7 +28,8 @@ import sys import json import yaml import glob -from yunohost import YunoHostError, YunoHostLDAP, validate, colorize, win_msg + +from ..core.helpers import YunoHostError, YunoHostLDAP, validate, colorize, win_msg def backup_init(helper=False): """ diff --git a/yunohost_domain.py b/src/moulinette/yunohost/domain.py similarity index 98% rename from yunohost_domain.py rename to src/moulinette/yunohost/domain.py index 0bf39c2f..6c6f1127 100644 --- a/yunohost_domain.py +++ b/src/moulinette/yunohost/domain.py @@ -32,8 +32,9 @@ import json import yaml import requests from urllib import urlopen -from yunohost import YunoHostError, YunoHostLDAP, win_msg, colorize, validate, get_required_args -from yunohost_dyndns import dyndns_subscribe +from dyndns import dyndns_subscribe + +from ..core.helpers import YunoHostError, YunoHostLDAP, win_msg, colorize, validate, get_required_args def domain_list(filter=None, limit=None, offset=None): diff --git a/yunohost_dyndns.py b/src/moulinette/yunohost/dyndns.py similarity index 98% rename from yunohost_dyndns.py rename to src/moulinette/yunohost/dyndns.py index 2d4355ce..13d5259c 100644 --- a/yunohost_dyndns.py +++ b/src/moulinette/yunohost/dyndns.py @@ -29,7 +29,8 @@ import requests import json import glob import base64 -from yunohost import YunoHostError, YunoHostLDAP, validate, colorize, win_msg + +from ..core.helpers import YunoHostError, YunoHostLDAP, validate, colorize, win_msg def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None): """ diff --git a/yunohost_firewall.py b/src/moulinette/yunohost/firewall.py similarity index 99% rename from yunohost_firewall.py rename to src/moulinette/yunohost/firewall.py index 1e9844e0..ae83a7cc 100644 --- a/yunohost_firewall.py +++ b/src/moulinette/yunohost/firewall.py @@ -36,7 +36,8 @@ except ImportError: sys.stderr.write('Error: Yunohost CLI Require yaml lib\n') sys.stderr.write('apt-get install python-yaml\n') sys.exit(1) -from yunohost import YunoHostError, win_msg + +from ..core.helpers import YunoHostError, win_msg def firewall_allow(protocol=None, port=None, ipv6=None, upnp=False): diff --git a/yunohost_hook.py b/src/moulinette/yunohost/hook.py similarity index 98% rename from yunohost_hook.py rename to src/moulinette/yunohost/hook.py index 14b719ee..2aea8706 100644 --- a/yunohost_hook.py +++ b/src/moulinette/yunohost/hook.py @@ -27,7 +27,8 @@ import os import sys import re import json -from yunohost import YunoHostError, YunoHostLDAP, win_msg, colorize + +from ..core.helpers import YunoHostError, YunoHostLDAP, win_msg, colorize hook_folder = '/usr/share/yunohost/hooks/' diff --git a/yunohost_monitor.py b/src/moulinette/yunohost/monitor.py similarity index 99% rename from yunohost_monitor.py rename to src/moulinette/yunohost/monitor.py index ed4f7db1..069fa39f 100644 --- a/yunohost_monitor.py +++ b/src/moulinette/yunohost/monitor.py @@ -34,10 +34,11 @@ import os.path import cPickle as pickle from urllib import urlopen from datetime import datetime, timedelta -from yunohost import YunoHostError, win_msg -from yunohost_service import (service_enable, service_disable, +from service import (service_enable, service_disable, service_start, service_stop, service_status) +from ..core.helpers import YunoHostError, win_msg + glances_uri = 'http://127.0.0.1:61209' stats_path = '/var/lib/yunohost/stats' crontab_path = '/etc/cron.d/yunohost-monitor' @@ -359,7 +360,7 @@ def monitor_enable(no_stats=False): # Install crontab if not no_stats: - cmd = 'yunohost monitor update-stats' + cmd = 'cd /home/admin/dev/moulinette && ./yunohost monitor update-stats' # day: every 5 min # week: every 1 h # month: every 4 h # rules = ('*/5 * * * * root %(cmd)s day --no-ldap >> /dev/null\n' + \ '3 * * * * root %(cmd)s week --no-ldap >> /dev/null\n' + \ diff --git a/yunohost_service.py b/src/moulinette/yunohost/service.py similarity index 99% rename from yunohost_service.py rename to src/moulinette/yunohost/service.py index cc9467bf..cfbfa875 100644 --- a/yunohost_service.py +++ b/src/moulinette/yunohost/service.py @@ -27,7 +27,8 @@ import yaml import glob import subprocess import os.path -from yunohost import YunoHostError, win_msg + +from ..core.helpers import YunoHostError, win_msg def service_start(names): @@ -169,7 +170,7 @@ def service_log(name, number=50): if name not in services.keys(): raise YunoHostError(1, _("Unknown service '%s'") % service) - + if 'log' in services[name]: log_list = services[name]['log'] result = {} @@ -253,7 +254,7 @@ def _tail(file, n, offset=None): pos = f.tell() lines = f.read().splitlines() if len(lines) >= to_read or pos == 0: - return lines[-to_read:offset and -offset or None] + return lines[-to_read:offset and -offset or None] avg_line_length *= 1.3 except IOError: return [] diff --git a/yunohost_tools.py b/src/moulinette/yunohost/tools.py similarity index 96% rename from yunohost_tools.py rename to src/moulinette/yunohost/tools.py index 3cac46a6..1300063c 100644 --- a/yunohost_tools.py +++ b/src/moulinette/yunohost/tools.py @@ -31,11 +31,12 @@ import getpass import subprocess import requests import json -from yunohost import YunoHostError, YunoHostLDAP, validate, colorize, get_required_args, win_msg -from yunohost_domain import domain_add, domain_list -from yunohost_dyndns import dyndns_subscribe -from yunohost_backup import backup_init -from yunohost_app import app_ssowatconf +from domain import domain_add, domain_list +from dyndns import dyndns_subscribe +from backup import backup_init +from app import app_ssowatconf + +from ..core.helpers import YunoHostError, YunoHostLDAP, validate, colorize, get_required_args, win_msg def tools_ldapinit(password=None): diff --git a/yunohost_user.py b/src/moulinette/yunohost/user.py similarity index 98% rename from yunohost_user.py rename to src/moulinette/yunohost/user.py index 5d74f323..16ac8cd4 100644 --- a/yunohost_user.py +++ b/src/moulinette/yunohost/user.py @@ -30,9 +30,10 @@ import crypt import random import string import getpass -from yunohost import YunoHostError, YunoHostLDAP, win_msg, colorize, validate, get_required_args -from yunohost_domain import domain_list -from yunohost_hook import hook_callback +from domain import domain_list +from hook import hook_callback + +from ..core.helpers import YunoHostError, YunoHostLDAP, win_msg, colorize, validate, get_required_args def user_list(fields=None, filter=None, limit=None, offset=None): """ @@ -117,7 +118,7 @@ def user_create(username, firstname, lastname, mail, password): uid = str(random.randint(200, 99999)) uid_check = os.system("getent passwd " + uid) gid_check = os.system("getent group " + uid) - + # Adapt values for LDAP fullname = firstname + ' ' + lastname rdn = 'uid=' + username + ',ou=users' @@ -139,7 +140,7 @@ def user_create(username, firstname, lastname, mail, password): 'uidNumber' : uid, 'homeDirectory' : '/home/' + username, 'loginShell' : '/bin/false' - + } if yldap.add(rdn, attr_dict): diff --git a/txrestapi/.gitignore b/txrestapi/.gitignore deleted file mode 100644 index 2193e643..00000000 --- a/txrestapi/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -_trial_temp -txrestapi.egg-info -txrestapi/_trial_temp diff --git a/txrestapi/README.rst b/txrestapi/README.rst deleted file mode 100644 index caf2cf32..00000000 --- a/txrestapi/README.rst +++ /dev/null @@ -1,146 +0,0 @@ -============ -Introduction -============ - -``txrestapi`` makes it easier to create Twisted REST API services. Normally, one -would create ``Resource`` subclasses defining each segment of a path; this is -cubersome to implement and results in output that isn't very readable. -``txrestapi`` provides an ``APIResource`` class allowing complex mapping of path to -callback (a la Django) with a readable decorator. - -=============================== -Basic URL callback registration -=============================== - -First, let's create a bare API service:: - - >>> from txrestapi.resource import APIResource - >>> api = APIResource() - -and a web server to serve it:: - - >>> from twisted.web.server import Site - >>> from twisted.internet import reactor - >>> site = Site(api, timeout=None) - -and a function to make it easy for us to make requests (only for doctest -purposes; normally you would of course use ``reactor.listenTCP(8080, site)``):: - - >>> from twisted.web.server import Request - >>> class FakeChannel(object): - ... transport = None - >>> def makeRequest(method, path): - ... req = Request(FakeChannel(), None) - ... req.prepath = req.postpath = None - ... req.method = method; req.path = path - ... resource = site.getChildWithDefault(path, req) - ... return resource.render(req) - -We can now register callbacks for paths we care about. We can provide different -callbacks for different methods; they must accept ``request`` as the first -argument:: - - >>> def get_callback(request): return 'GET callback' - >>> api.register('GET', '^/path/to/method', get_callback) - >>> def post_callback(request): return 'POST callback' - >>> api.register('POST', '^/path/to/method', post_callback) - -Then, when we make a call, the request is routed to the proper callback:: - - >>> print makeRequest('GET', '/path/to/method') - GET callback - >>> print makeRequest('POST', '/path/to/method') - POST callback - -We can register multiple callbacks for different requests; the first one that -matches wins:: - - >>> def default_callback(request): - ... return 'Default callback' - >>> api.register('GET', '^/.*$', default_callback) # Matches everything - >>> print makeRequest('GET', '/path/to/method') - GET callback - >>> print makeRequest('GET', '/path/to/different/method') - Default callback - -Our default callback, however, will only match GET requests. For a true default -callback, we can either register callbacks for each method individually, or we -can use ALL:: - - >>> api.register('ALL', '^/.*$', default_callback) - >>> print makeRequest('PUT', '/path/to/method') - Default callback - >>> print makeRequest('DELETE', '/path/to/method') - Default callback - >>> print makeRequest('GET', '/path/to/method') - GET callback - -Let's unregister all references to the default callback so it doesn't interfere -with later tests (default callbacks should, of course, always be registered -last, so they don't get called before other callbacks):: - - >>> api.unregister(callback=default_callback) - -============= -URL Arguments -============= - -Since callbacks accept ``request``, they have access to POST data or query -arguments, but we can also pull arguments out of the URL by using named groups -in the regular expression (similar to Django). These will be passed into the -callback as keyword arguments:: - - >>> def get_info(request, id): - ... return 'Information for id %s' % id - >>> api.register('GET', '/(?P[^/]+)/info$', get_info) - >>> print makeRequest('GET', '/someid/info') - Information for id someid - -Bear in mind all arguments will come in as strings, so code should be -accordingly defensive. - -================ -Decorator syntax -================ - -Registration via the ``register()`` method is somewhat awkward, so decorators -are provided making it much more straightforward. :: - - >>> from txrestapi.methods import GET, POST, PUT, ALL - >>> class MyResource(APIResource): - ... - ... @GET('^/(?P[^/]+)/info') - ... def get_info(self, request, id): - ... return 'Info for id %s' % id - ... - ... @PUT('^/(?P[^/]+)/update') - ... @POST('^/(?P[^/]+)/update') - ... def set_info(self, request, id): - ... return "Setting info for id %s" % id - ... - ... @ALL('^/') - ... def default_view(self, request): - ... return "I match any URL" - -Again, registrations occur top to bottom, so methods should be written from -most specific to least. Also notice that one can use the decorator syntax as -one would expect to register a method as the target for two URLs :: - - >>> site = Site(MyResource(), timeout=None) - >>> print makeRequest('GET', '/anid/info') - Info for id anid - >>> print makeRequest('PUT', '/anid/update') - Setting info for id anid - >>> print makeRequest('POST', '/anid/update') - Setting info for id anid - >>> print makeRequest('DELETE', '/anid/delete') - I match any URL - -====================== -Callback return values -====================== - -You can return Resource objects from a callback if you wish, allowing you to -have APIs that send you to other kinds of resources, or even other APIs. -Normally, however, you'll most likely want to return strings, which will be -wrapped in a Resource object for convenience. diff --git a/txrestapi/__init__.py b/txrestapi/__init__.py deleted file mode 100644 index 792d6005..00000000 --- a/txrestapi/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# diff --git a/txrestapi/methods.py b/txrestapi/methods.py deleted file mode 100644 index 8d5a89d9..00000000 --- a/txrestapi/methods.py +++ /dev/null @@ -1,29 +0,0 @@ -from zope.interface.advice import addClassAdvisor - -def method_factory_factory(method): - def factory(regex): - _f = {} - def decorator(f): - _f[f.__name__] = f - return f - def advisor(cls): - def wrapped(f): - def __init__(self, *args, **kwargs): - f(self, *args, **kwargs) - for func_name in _f: - orig = _f[func_name] - func = getattr(self, func_name) - if func.im_func==orig: - self.register(method, regex, func) - return __init__ - cls.__init__ = wrapped(cls.__init__) - return cls - addClassAdvisor(advisor) - return decorator - return factory - -ALL = method_factory_factory('ALL') -GET = method_factory_factory('GET') -POST = method_factory_factory('POST') -PUT = method_factory_factory('PUT') -DELETE = method_factory_factory('DELETE') diff --git a/txrestapi/resource.py b/txrestapi/resource.py deleted file mode 100644 index 322acd5f..00000000 --- a/txrestapi/resource.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -from itertools import ifilter -from functools import wraps -from twisted.web.resource import Resource, NoResource - -class _FakeResource(Resource): - _result = '' - isLeaf = True - def __init__(self, result): - Resource.__init__(self) - self._result = result - def render(self, request): - return self._result - - -def maybeResource(f): - @wraps(f) - def inner(*args, **kwargs): - result = f(*args, **kwargs) - if not isinstance(result, Resource): - result = _FakeResource(result) - return result - return inner - - -class APIResource(Resource): - - _registry = None - - def __init__(self, *args, **kwargs): - Resource.__init__(self, *args, **kwargs) - self._registry = [] - - def _get_callback(self, request): - filterf = lambda t:t[0] in (request.method, 'ALL') - path_to_check = getattr(request, '_remaining_path', request.path) - for m, r, cb in ifilter(filterf, self._registry): - result = r.search(path_to_check) - if result: - request._remaining_path = path_to_check[result.span()[1]:] - return cb, result.groupdict() - return None, None - - def register(self, method, regex, callback): - self._registry.append((method, re.compile(regex), callback)) - - def unregister(self, method=None, regex=None, callback=None): - if regex is not None: regex = re.compile(regex) - for m, r, cb in self._registry[:]: - if not method or (method and m==method): - if not regex or (regex and r==regex): - if not callback or (callback and cb==callback): - self._registry.remove((m, r, cb)) - - def getChild(self, name, request): - r = self.children.get(name, None) - if r is None: - # Go into the thing - callback, args = self._get_callback(request) - if callback is None: - return NoResource() - else: - return maybeResource(callback)(request, **args) - else: - return r diff --git a/txrestapi/service.py b/txrestapi/service.py deleted file mode 100644 index 78d031a8..00000000 --- a/txrestapi/service.py +++ /dev/null @@ -1,7 +0,0 @@ -from twisted.web.server import Site -from .resource import APIResource - - -class RESTfulService(Site): - def __init__(self, port=8080): - self.root = APIResource() diff --git a/txrestapi/tests.py b/txrestapi/tests.py deleted file mode 100644 index 3ee1b53c..00000000 --- a/txrestapi/tests.py +++ /dev/null @@ -1,194 +0,0 @@ -import txrestapi -__package__="txrestapi" -import re -import os.path -from twisted.internet import reactor -from twisted.internet.defer import inlineCallbacks -from twisted.web.resource import Resource, NoResource -from twisted.web.server import Request, Site -from twisted.web.client import getPage -from twisted.trial import unittest -from .resource import APIResource -from .methods import GET, PUT - -class FakeChannel(object): - transport = None - -def getRequest(method, url): - req = Request(FakeChannel(), None) - req.method = method - req.path = url - return req - -class APIResourceTest(unittest.TestCase): - - def test_returns_normal_resources(self): - r = APIResource() - a = Resource() - r.putChild('a', a) - req = Request(FakeChannel(), None) - a_ = r.getChild('a', req) - self.assertEqual(a, a_) - - def test_registry(self): - compiled = re.compile('regex') - r = APIResource() - r.register('GET', 'regex', None) - self.assertEqual([x[0] for x in r._registry], ['GET']) - self.assertEqual(r._registry[0], ('GET', compiled, None)) - - def test_method_matching(self): - r = APIResource() - r.register('GET', 'regex', 1) - r.register('PUT', 'regex', 2) - r.register('GET', 'another', 3) - - req = getRequest('GET', 'regex') - result = r._get_callback(req) - self.assert_(result) - self.assertEqual(result[0], 1) - - req = getRequest('PUT', 'regex') - result = r._get_callback(req) - self.assert_(result) - self.assertEqual(result[0], 2) - - req = getRequest('GET', 'another') - result = r._get_callback(req) - self.assert_(result) - self.assertEqual(result[0], 3) - - req = getRequest('PUT', 'another') - result = r._get_callback(req) - self.assertEqual(result, (None, None)) - - def test_callback(self): - marker = object() - def cb(request): - return marker - r = APIResource() - r.register('GET', 'regex', cb) - req = getRequest('GET', 'regex') - result = r.getChild('regex', req) - self.assertEqual(result.render(req), marker) - - def test_longerpath(self): - marker = object() - r = APIResource() - def cb(request): - return marker - r.register('GET', '/regex/a/b/c', cb) - req = getRequest('GET', '/regex/a/b/c') - result = r.getChild('regex', req) - self.assertEqual(result.render(req), marker) - - def test_args(self): - r = APIResource() - def cb(request, **kwargs): - return kwargs - r.register('GET', '/(?P[^/]*)/a/(?P[^/]*)/c', cb) - req = getRequest('GET', '/regex/a/b/c') - result = r.getChild('regex', req) - self.assertEqual(sorted(result.render(req).keys()), ['a', 'b']) - - def test_order(self): - r = APIResource() - def cb1(request, **kwargs): - kwargs.update({'cb1':True}) - return kwargs - def cb(request, **kwargs): - return kwargs - # Register two regexes that will match - r.register('GET', '/(?P[^/]*)/a/(?P[^/]*)/c', cb1) - r.register('GET', '/(?P[^/]*)/a/(?P[^/]*)', cb) - req = getRequest('GET', '/regex/a/b/c') - result = r.getChild('regex', req) - # Make sure the first one got it - self.assert_('cb1' in result.render(req)) - - def test_no_resource(self): - r = APIResource() - r.register('GET', '^/(?P[^/]*)/a/(?P[^/]*)$', None) - req = getRequest('GET', '/definitely/not/a/match') - result = r.getChild('regex', req) - self.assert_(isinstance(result, NoResource)) - - def test_all(self): - r = APIResource() - def get_cb(r): return 'GET' - def put_cb(r): return 'PUT' - def all_cb(r): return 'ALL' - r.register('GET', '^path', get_cb) - r.register('ALL', '^path', all_cb) - r.register('PUT', '^path', put_cb) - # Test that the ALL registration picks it up before the PUT one - for method in ('GET', 'PUT', 'ALL'): - req = getRequest(method, 'path') - result = r.getChild('path', req) - self.assertEqual(result.render(req), 'ALL' if method=='PUT' else method) - - -class TestResource(Resource): - isLeaf = True - def render(self, request): - return 'aresource' - - -class TestAPI(APIResource): - - @GET('^/(?Ptest[^/]*)/?') - def _on_test_get(self, request, a): - return 'GET %s' % a - - @PUT('^/(?Ptest[^/]*)/?') - def _on_test_put(self, request, a): - return 'PUT %s' % a - - @GET('^/gettest') - def _on_gettest(self, request): - return TestResource() - - -class DecoratorsTest(unittest.TestCase): - def _listen(self, site): - return reactor.listenTCP(0, site, interface="127.0.0.1") - - def setUp(self): - r = TestAPI() - site = Site(r, timeout=None) - self.port = self._listen(site) - self.portno = self.port.getHost().port - - def tearDown(self): - return self.port.stopListening() - - def getURL(self, path): - return "http://127.0.0.1:%d/%s" % (self.portno, path) - - @inlineCallbacks - def test_get(self): - url = self.getURL('test_thing/') - result = yield getPage(url, method='GET') - self.assertEqual(result, 'GET test_thing') - - @inlineCallbacks - def test_put(self): - url = self.getURL('test_thing/') - result = yield getPage(url, method='PUT') - self.assertEqual(result, 'PUT test_thing') - - @inlineCallbacks - def test_resource_wrapper(self): - url = self.getURL('gettest') - result = yield getPage(url, method='GET') - self.assertEqual(result, 'aresource') - - -def test_suite(): - import unittest as ut - suite = unittest.TestSuite() - suite.addTest(ut.makeSuite(DecoratorsTest)) - suite.addTest(ut.makeSuite(APIResourceTest)) - suite.addTest(unittest.doctest.DocFileSuite(os.path.join('..', 'README.rst'))) - return suite - From a2be4d6e12cfe9ac328ccf470c52165d37c6ade4 Mon Sep 17 00:00:00 2001 From: Jerome Lebleu Date: Fri, 14 Feb 2014 23:51:17 +0100 Subject: [PATCH 02/18] Implement an actions map plugin for the api --- src/moulinette/core/api.py | 83 +++++++++++++++++++----------- src/moulinette/yunohost/monitor.py | 2 +- 2 files changed, 54 insertions(+), 31 deletions(-) diff --git a/src/moulinette/core/api.py b/src/moulinette/core/api.py index ea00cc9e..19e0af18 100644 --- a/src/moulinette/core/api.py +++ b/src/moulinette/core/api.py @@ -7,9 +7,12 @@ from beaker.middleware import SessionMiddleware from ..config import session_path, doc_json_path from helpers import YunoHostError, YunoHostLDAP + +## Bottle Plugins + class APIAuthPlugin(object): """ - This Bottle plugin manages the authentication for the API access. + Manage the authentication for the API access. """ name = 'apiauth' @@ -60,12 +63,7 @@ class APIAuthPlugin(object): if context.name == 'login': self.logout() else: - # TODO: Fix this tweak to retrieve uri from the callback - def wrapper(*args, **kwargs): - if hasattr(context.config, '_uri'): - kwargs['_uri'] = context.config._uri - return callback(*args, **kwargs) - return wrapper + return callback # Process login route if context.name == 'login': @@ -121,6 +119,46 @@ class APIAuthPlugin(object): return True return False +class ActionsMapPlugin(object): + """ + Process action for the request using the actions map. + + """ + name = 'actionsmap' + api = 2 + + def __init__(self, actionsmap): + self.actionsmap = actionsmap + + def setup(self, app): + pass + + def apply(self, callback, context): + """ + Process the relevant action for the request + + Keyword argument: + callback -- The route callback + context -- An instance of Route + + """ + method = request.method + uri = context.rule + + def wrapper(*args, **kwargs): + # Bring arguments together + params = kwargs + for a in args: + params[a] = True + for k, v in request.params.items(): + params[k] = v + + # Process the action + return self.actionsmap.process(params, route=(method, uri)) + return wrapper + + +## Main class class MoulinetteAPI(object): """ @@ -137,18 +175,19 @@ class MoulinetteAPI(object): """ def __init__(self, actionsmap, routes={}): - self.actionsmap = actionsmap - # Initialize app and default routes # TODO: Return OK to 'OPTIONS' xhr requests (l173) app = Bottle() - app.route(['/api', '/api/'], method='GET', callback=self.doc, skip=['apiauth']) + app.route(['/api', '/api/'], method='GET', + callback=self.doc, skip=['apiauth']) # Append routes from the actions map + amap = ActionsMapPlugin(actionsmap) for (m, u) in actionsmap.parser.routes: - app.route(u, method=m, callback=self._route_wrapper, _uri=u) + app.route(u, method=m, callback=self._error, apply=amap) # Append additional routes + # TODO: Add an option to skip auth for the route for (m, u), c in routes.items(): app.route(u, method=m, callback=c) @@ -190,22 +229,6 @@ class MoulinetteAPI(object): except IOError: return 'unknown' - def _route_wrapper(self, *args, **kwargs): - """Process the relevant action for the request""" - # Retrieve uri - if '_uri' in kwargs: - uri = kwargs['_uri'] - del kwargs['_uri'] - else: - uri = request.path - - # Bring arguments together - params = kwargs - for a in args: - params[a] = True - for k, v in request.params.items(): - params[k] = v - - # Process the action - # TODO: Catch errors - return self.actionsmap.process(params, (request.method, uri)) + def _error(self, *args, **kwargs): + # TODO: Raise or return an error + print('error') diff --git a/src/moulinette/yunohost/monitor.py b/src/moulinette/yunohost/monitor.py index 069fa39f..8791c807 100644 --- a/src/moulinette/yunohost/monitor.py +++ b/src/moulinette/yunohost/monitor.py @@ -360,7 +360,7 @@ def monitor_enable(no_stats=False): # Install crontab if not no_stats: - cmd = 'cd /home/admin/dev/moulinette && ./yunohost monitor update-stats' + cmd = 'yunohost monitor update-stats' # day: every 5 min # week: every 1 h # month: every 4 h # rules = ('*/5 * * * * root %(cmd)s day --no-ldap >> /dev/null\n' + \ '3 * * * * root %(cmd)s week --no-ldap >> /dev/null\n' + \ From 0bfc63e4b88330f79f846db81d914ed6c0593c14 Mon Sep 17 00:00:00 2001 From: Jerome Lebleu Date: Mon, 24 Feb 2014 18:28:11 +0100 Subject: [PATCH 03/18] Standardize extra parameters and make them modular --- etc/actionsmap/yunohost.yml | 78 ++++--- src/moulinette/__init__.py | 8 +- src/moulinette/core/actionsmap.py | 283 +++++++++++++------------ src/moulinette/core/extraparameters.py | 158 ++++++++++++++ src/moulinette/core/helpers.py | 15 ++ 5 files changed, 374 insertions(+), 168 deletions(-) create mode 100644 src/moulinette/core/extraparameters.py diff --git a/etc/actionsmap/yunohost.yml b/etc/actionsmap/yunohost.yml index cf2df613..fe5ab237 100644 --- a/etc/actionsmap/yunohost.yml +++ b/etc/actionsmap/yunohost.yml @@ -74,23 +74,29 @@ user: -u: full: --username help: Must be unique - ask: "Username" - pattern: '^[a-z0-9_]+$' + extra: + ask: "Username" + pattern: '^[a-z0-9_]+$' -f: full: --firstname - ask: "Firstname" + extra: + ask: "Firstname" -l: full: --lastname - ask: "Lastname" + extra: + ask: "Lastname" -m: full: --mail help: Main mail address must be unique - ask: "Mail address" - pattern: '^[\w.-]+@[\w.-]+\.[a-zA-Z]{2,6}$' + extra: + ask: "Mail address" + pattern: + - '^[\w.-]+@[\w.-]+\.[a-zA-Z]{2,6}$' + - "Must be a valid email address (e.g. someone@domain.org)" -p: full: --password - ask: "User password" - password: yes + extra: + password: "User password" ### user_delete() delete: @@ -100,9 +106,10 @@ user: -u: full: --users help: Username of users to delete - ask: "Users to delete" - pattern: '^[a-z0-9_]+$' nargs: "*" + extra: + ask: "Users to delete" + pattern: '^[a-z0-9_]+$' --purge: action: store_true @@ -179,7 +186,8 @@ domain: domains: help: Domain name to add nargs: '+' - pattern: '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + extra: + pattern: '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' -m: full: --main help: Is the main domain @@ -197,7 +205,8 @@ domain: domains: help: Domain(s) to delete nargs: "+" - pattern: '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + extra: + pattern: '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' ### domain_info() info: @@ -206,7 +215,8 @@ domain: arguments: domain: help: "" - pattern: '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + extra: + pattern: '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' ############################# @@ -241,8 +251,9 @@ app: -n: full: --name help: Name of the list to remove - ask: "List to remove" - pattern: '^[a-z0-9_]+$' + extra: + ask: "List to remove" + pattern: '^[a-z0-9_]+$' ### app_list() list: @@ -290,7 +301,8 @@ app: -u: full: --user help: Allowed app map for a user - pattern: '^[a-z0-9_]+$' + extra: + pattern: '^[a-z0-9_]+$' ### app_install() TODO: Write help @@ -375,7 +387,8 @@ app: arguments: port: help: Port to check - pattern: '^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$' + extra: + pattern: '^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$' ### app_checkurl() checkurl: @@ -644,8 +657,9 @@ service: -n: full: --number help: Number of lines to display - pattern: '^[0-9]+$' default: "50" + extra: + pattern: '^[0-9]+$' ############################# @@ -676,7 +690,8 @@ firewall: arguments: port: help: Port to open - pattern: '^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$' + extra: + pattern: '^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$' protocol: help: Protocol associated with port choices: @@ -810,12 +825,12 @@ tools: arguments: -o: full: --old-password - ask: "Current admin password" - password: yes + extra: + password: "Current admin password" -n: full: --new-password - ask: "New admin password" - password: yes + extra: + password: "New admin password" ### tools_maindomain() maindomain: @@ -824,11 +839,13 @@ tools: arguments: -o: full: --old-domain - pattern: '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + extra: + pattern: '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' -n: full: --new-domain - ask: "New main domain" - pattern: '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + extra: + ask: "New main domain" + pattern: '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' ### tools_postinstall() postinstall: @@ -838,13 +855,14 @@ tools: -d: full: --domain help: YunoHost main domain - ask: "Main domain" - pattern: '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + extra: + ask: "Main domain" + pattern: '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' -p: full: --password help: YunoHost admin password - ask: "New admin password" - password: yes + extra: + password: "New admin password" --dyndns: help: Subscribe domain to a DynDNS service action: store_true diff --git a/src/moulinette/__init__.py b/src/moulinette/__init__.py index 784ed407..f96eae48 100755 --- a/src/moulinette/__init__.py +++ b/src/moulinette/__init__.py @@ -46,8 +46,9 @@ def api(port, routes={}, use_cache=True): from bottle import run from core.actionsmap import ActionsMap from core.api import MoulinetteAPI + from core.helpers import Interface - amap = ActionsMap(ActionsMap.IFACE_API, use_cache=use_cache) + amap = ActionsMap(Interface.api, use_cache=use_cache) moulinette = MoulinetteAPI(amap, routes) run(moulinette.app, port=port) @@ -67,10 +68,11 @@ def cli(args, use_cache=True): """ import os from core.actionsmap import ActionsMap - from core.helpers import YunoHostError, pretty_print_dict + from core.helpers import Interface, YunoHostError, pretty_print_dict lock_file = '/var/run/moulinette.lock' + # TODO: Move the lock checking into the ActionsMap class # Check the lock if os.path.isfile(lock_file): raise YunoHostError(1, _("The moulinette is already running")) @@ -80,7 +82,7 @@ def cli(args, use_cache=True): os.system('chmod 400 '+ lock_file) try: - amap = ActionsMap(ActionsMap.IFACE_CLI, use_cache=use_cache) + amap = ActionsMap(Interface.cli, use_cache=use_cache) pretty_print_dict(amap.process(args)) except KeyboardInterrupt, EOFError: raise YunoHostError(125, _("Interrupted")) diff --git a/src/moulinette/core/actionsmap.py b/src/moulinette/core/actionsmap.py index 022dc06c..90ff9f43 100644 --- a/src/moulinette/core/actionsmap.py +++ b/src/moulinette/core/actionsmap.py @@ -1,16 +1,21 @@ # -*- coding: utf-8 -*- import argparse -import getpass -import marshal import pickle import yaml import re import os +from collections import OrderedDict + +import logging from .. import __version__ from ..config import actionsmap_path, actionsmap_cache_path -from helpers import YunoHostError, colorize + +from extraparameters import extraparameters_list +from helpers import Interface, YunoHostError + +## Additional parsers class _HTTPArgumentParser(object): @@ -137,80 +142,75 @@ class HTTPParser(object): return self._parsers[key].parse_args(args) +class ExtraParser(object): + """ + Global parser for the extra parameters. -class _ExtraParameters(object): + """ + def __init__(self, iface): + self.iface = iface + self.extra = OrderedDict() - CLI_PARAMETERS = ['ask', 'password', 'pattern'] - API_PARAMETERS = ['pattern'] - AVAILABLE_PARAMETERS = CLI_PARAMETERS + # Append available extra parameters for the current interface + for klass in extraparameters_list: + if iface in klass.skipped_iface: + continue + if klass.name in self.extra: + logging.warning("extra parameter named '%s' was already added" % klass.name) + continue + self.extra[klass.name] = klass - def __init__(self, **kwargs): - self._params = {} + def validate(self, arg_name, parameters): + """ + Validate values of extra parameters for an argument - for k, v in kwargs.items(): - if k in self.AVAILABLE_PARAMETERS: - self._params[k] = v + Keyword arguments: + - arg_name -- The argument name + - parameters -- A dict of extra parameters with their values - def validate(self, p_name, p_value): - ret = type(p_value)() if p_value is not None else None + """ + # Iterate over parameters to validate + for p, v in parameters.items(): + # Remove unknow parameters + if p not in self.extra.keys(): + del parameters[p] - for p, v in self._params.items(): - func = getattr(self, 'process_' + p) + # Validate parameter value + parameters[p] = self.extra[p].validate(v, arg_name) - if isinstance(ret, list): - for p_v in p_value: - r = func(v, p_name, p_v) - if r is not None: - ret.append(r) + return parameters + + def parse(self, arg_name, arg_value, parameters): + """ + Parse argument with extra parameters + + Keyword arguments: + - arg_name -- The argument name + - arg_value -- The argument value + - parameters -- A dict of extra parameters with their values + + """ + # Iterate over available parameters + for p, klass in self.extra.items(): + if p not in parameters.keys(): + continue + + # Initialize the extra parser + parser = klass(self.iface) + + # Parse the argument + if isinstance(arg_value, list): + for v in arg_value: + r = parser(parameters[p], arg_name, v) + if r not in arg_value: + arg_value.append(r) else: - r = func(v, p_name, p_value) - if r is not None: - ret = r + arg_value = parser(parameters[p], arg_name, arg_value) - return ret + return arg_value - ## Parameters validating's method - # TODO: Add doc - - def process_ask(self, message, p_name, p_value): - # TODO: Fix asked arguments ordering - if not self._can_prompt(p_value): - return p_value - - # Skip password asking - if 'password' in self._params.keys(): - return None - - ret = raw_input(colorize(message + ': ', 'cyan')) - return ret - - def process_password(self, is_password, p_name, p_value): - if not self._can_prompt(p_value): - return p_value - - message = self._params['ask'] - pwd1 = getpass.getpass(colorize(message + ': ', 'cyan')) - pwd2 = getpass.getpass(colorize('Retype ' + message + ': ', 'cyan')) - if pwd1 != pwd2: - raise YunoHostError(22, _("Passwords don't match")) - return pwd1 - - def process_pattern(self, pattern, p_name, p_value): - # TODO: Add a pattern_help parameter - # TODO: Fix missing pattern matching on asking - if p_value is not None and not re.match(pattern, p_value): - raise YunoHostError(22, _("'%s' argument not match pattern" % p_name)) - return p_value - - - ## Private method - - def _can_prompt(self, p_value): - if os.isatty(1) and (p_value is None or p_value == ''): - return True - return False - +## Main class class ActionsMap(object): """ @@ -231,30 +231,37 @@ class ActionsMap(object): instead of using the cached one. """ - IFACE_CLI = 'cli' - IFACE_API = 'api' - def __init__(self, interface, use_cache=True): - if interface not in [self.IFACE_CLI,self.IFACE_API]: + if interface not in Interface.all(): raise ValueError(_("Invalid interface '%s'" % interface)) self.interface = interface + self.use_cache = use_cache + + logging.debug("initializing ActionsMap for the '%s' interface" % interface) # Iterate over actions map namespaces - actionsmap = {} + actionsmaps = {} for n in self.get_actionsmap_namespaces(): + logging.debug("loading '%s' actions map namespace" % n) + if use_cache: + # Attempt to load cache if it exists cache_file = '%s/%s.pkl' % (actionsmap_cache_path, n) if os.path.isfile(cache_file): with open(cache_file, 'r') as f: - actionsmap[n] = pickle.load(f) + actionsmaps[n] = pickle.load(f) else: - actionsmap = self.generate_cache() + self.use_cache = False + actionsmaps = self.generate_cache() + break else: am_file = '%s/%s.yml' % (actionsmap_path, n) with open(am_file, 'r') as f: - actionsmap[n] = yaml.load(f) + actionsmaps[n] = yaml.load(f) - self.parser = self._construct_parser(actionsmap) + # Generate parsers + self.extraparser = ExtraParser(interface) + self.parser = self._construct_parser(actionsmaps) def process(self, args, route=None): """ @@ -268,9 +275,9 @@ class ActionsMap(object): arguments = None # Parse arguments - if self.interface ==self.IFACE_CLI: + if self.interface == Interface.cli: arguments = self.parser.parse_args(args) - elif self.interface ==self.IFACE_API: + elif self.interface == Interface.api: if route is None: # TODO: Raise a proper exception raise Exception(_("Missing route argument")) @@ -302,7 +309,10 @@ class ActionsMap(object): @staticmethod def get_actionsmap_namespaces(path=actionsmap_path): """ - Retrieve actions map namespaces in a given path + Retrieve actions map namespaces from a given path + + Returns: + A list of available namespaces """ namespaces = [] @@ -313,65 +323,77 @@ class ActionsMap(object): return namespaces @classmethod - def generate_cache(cls): + def generate_cache(klass): """ Generate cache for the actions map's file(s) + Returns: + A dict of actions map for each namespaces + """ - actionsmap = {} + actionsmaps = {} if not os.path.isdir(actionsmap_cache_path): os.makedirs(actionsmap_cache_path) - for n in cls.get_actionsmap_namespaces(): + # Iterate over actions map namespaces + for n in klass.get_actionsmap_namespaces(): + logging.debug("generating cache for '%s' actions map namespace" % n) + + # Read actions map from yaml file am_file = '%s/%s.yml' % (actionsmap_path, n) with open(am_file, 'r') as f: - actionsmap[n] = yaml.load(f) + actionsmaps[n] = yaml.load(f) + + # Cache actions map into pickle file cache_file = '%s/%s.pkl' % (actionsmap_cache_path, n) with open(cache_file, 'w') as f: - pickle.dump(actionsmap[n], f) + pickle.dump(actionsmaps[n], f) - return actionsmap + return actionsmaps ## Private class and methods def _store_extra_parameters(self, parser, arg_name, arg_params): """ - Store extra parameters for a given parser's argument name + Store extra parameters for a given argument Keyword arguments: - - parser -- Parser object of the argument + - parser -- Parser object for the arguments - arg_name -- Argument name - arg_params -- Argument parameters + Returns: + The parser object + """ - params = {} - keys = [] - - # Get available parameters for the current interface - if self.interface ==self.IFACE_CLI: - keys = _ExtraParameters.CLI_PARAMETERS - elif self.interface ==self.IFACE_API: - keys = _ExtraParameters.API_PARAMETERS - - for k in keys: - if k in arg_params: - params[k] = arg_params[k] - - if len(params) > 0: - # Retrieve all extra parameters from the parser + if 'extra' in arg_params: + # Retrieve current extra parameters dict extra = parser.get_default('_extra') if not extra or not isinstance(extra, dict): extra = {} - # Add completed extra parameters to the parser - extra[arg_name] = _ExtraParameters(**params) + if not self.use_cache: + # Validate extra parameters for the argument + extra[arg_name] = self.extraparser.validate(arg_name, arg_params['extra']) + else: + extra[arg_name] = arg_params['extra'] parser.set_defaults(_extra=extra) return parser def _parse_extra_parameters(self, args): + """ + Parse arguments with their extra parameters + + Keyword arguments: + - args -- A dict of all arguments + + Return: + The parsed arguments dict + + """ # Retrieve extra parameters from the arguments if '_extra' not in args: return args @@ -379,41 +401,41 @@ class ActionsMap(object): del args['_extra'] # Validate extra parameters for each arguments - for n, e in extra.items(): - args[n] = e.validate(n, args[n]) + for an, parameters in extra.items(): + args[an] = self.extraparser.parse(an, args[an], parameters) return args - def _construct_parser(self, actionsmap): + def _construct_parser(self, actionsmaps): """ Construct the parser with the actions map Keyword arguments: - - actionsmap -- Multi-level dictionnary of - categories/actions/arguments list + - actionsmaps -- A dict of multi-level dictionnary of + categories/actions/arguments list for each namespaces Returns: - Interface relevant's parser object + An interface relevant's parser object """ top_parser = None iface = self.interface # Create parser object - if iface ==self.IFACE_CLI: - # TODO: Add descritpion (from __description__) + if iface == Interface.cli: + # TODO: Add descritpion (from __description__?) top_parser = argparse.ArgumentParser() top_subparsers = top_parser.add_subparsers() - elif iface ==self.IFACE_API: + elif iface == Interface.api: top_parser = HTTPParser() - ## Extract option strings from parameters + ## Format option strings from argument parameters def _option_strings(arg_name, arg_params): - if iface ==self.IFACE_CLI: + if iface == Interface.cli: if arg_name[0] == '-' and 'full' in arg_params: return [arg_name, arg_params['full']] return [arg_name] - elif iface ==self.IFACE_API: + elif iface == Interface.api: if arg_name[0] != '-': return [arg_name] if 'full' in arg_params: @@ -422,40 +444,31 @@ class ActionsMap(object): return [arg_name.replace('--', '@', 1)] return [arg_name.replace('-', '@', 1)] - ## Extract a key from parameters - def _key(arg_params, key, default=str()): - if key in arg_params: - return arg_params[key] - return default - ## Remove extra parameters def _clean_params(arg_params): - keys = list(_ExtraParameters.AVAILABLE_PARAMETERS) - keys.append('full') - - for k in keys: + for k in {'full', 'extra'}: if k in arg_params: del arg_params[k] return arg_params # Iterate over actions map namespaces - for n in self.get_actionsmap_namespaces(): + for n, actionsmap in actionsmaps.items(): # Parse general arguments for the cli only - if iface ==self.IFACE_CLI: - for an, ap in actionsmap[n]['general_arguments'].items(): + if iface == Interface.cli: + for an, ap in actionsmap['general_arguments'].items(): if 'version' in ap: ap['version'] = ap['version'].replace('%version%', __version__) top_parser.add_argument(*_option_strings(an, ap), **_clean_params(ap)) - del actionsmap[n]['general_arguments'] + del actionsmap['general_arguments'] # Parse categories - for cn, cp in actionsmap[n].items(): + for cn, cp in actionsmap.items(): if 'actions' not in cp: continue # Add category subparsers for the cli only - if iface ==self.IFACE_CLI: - c_help = _key(cp, 'category_help') + if iface == Interface.cli: + c_help = cp.get('category_help') subparsers = top_subparsers.add_parser(cn, help=c_help).add_subparsers() # Parse actions @@ -463,10 +476,10 @@ class ActionsMap(object): parser = None # Add parser for the current action - if iface ==self.IFACE_CLI: - a_help = _key(ap, 'action_help') + if iface == Interface.cli: + a_help = ap.get('action_help') parser = subparsers.add_parser(an, help=a_help) - elif iface ==self.IFACE_API and 'api' in ap: + elif iface == Interface.api and 'api' in ap: # Extract method and uri m = re.match('(GET|POST|PUT|DELETE) (/\S+)', ap['api']) if m: diff --git a/src/moulinette/core/extraparameters.py b/src/moulinette/core/extraparameters.py new file mode 100644 index 00000000..3f4248c0 --- /dev/null +++ b/src/moulinette/core/extraparameters.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- + +import getpass +import re +import logging + +from helpers import Interface, colorize, YunoHostError + +class _ExtraParameter(object): + """ + Argument parser for an extra parameter. + + It is a pure virtual class that each extra parameter classes must + implement. + + """ + def __init__(self, iface): + # TODO: Add conn argument which contains authentification object + self.iface = iface + + + ## Required variables + # Each extra parameters classes must overwrite these variables. + + """The extra parameter name""" + name = None + + + ## Optional variables + # Each extra parameters classes can overwrite these variables. + + """A list of interface for which the parameter doesn't apply""" + skipped_iface = {} + + + ## Virtual methods + # Each extra parameters classes can implement these methods. + + def __call__(self, parameter, arg_name, arg_value): + """ + Parse the argument + + Keyword arguments: + - parameter -- The value of this parameter for the action + - arg_name -- The argument name + - arg_value -- The argument value + + Returns: + The new argument value + + """ + return arg_value + + @staticmethod + def validate(value, arg_name): + """ + Validate the parameter value for an argument + + Keyword arguments: + - value -- The parameter value + - arg_name -- The argument name + + Returns: + The validated parameter value + + """ + return value + + +## Extra parameters definitions + +class AskParameter(_ExtraParameter): + """ + Ask for the argument value if possible and needed. + + The value of this parameter corresponds to the message to display + when asking the argument value. + + """ + name = 'ask' + skipped_iface = {Interface.api} + + def __call__(self, message, arg_name, arg_value): + # TODO: Fix asked arguments ordering + if arg_value: + return arg_value + + # Ask for the argument value + ret = raw_input(colorize(message + ': ', 'cyan')) + return ret + + @classmethod + def validate(klass, value, arg_name): + # Allow boolean or empty string + if isinstance(value, bool) or (isinstance(value, str) and not value): + logging.debug("value of '%s' extra parameter for '%s' argument should be a string" \ + % (klass.name, arg_name)) + value = arg_name + elif not isinstance(value, str): + raise TypeError("Invalid type of '%s' extra parameter for '%s' argument" \ + % (klass.name, arg_name)) + return value + +class PasswordParameter(AskParameter): + """ + Ask for the password argument value if possible and needed. + + The value of this parameter corresponds to the message to display + when asking the password. + + """ + name = 'password' + + def __call__(self, message, arg_name, arg_value): + if arg_value: + return arg_value + + # Ask for the password + pwd1 = getpass.getpass(colorize(message + ': ', 'cyan')) + pwd2 = getpass.getpass(colorize('Retype ' + message + ': ', 'cyan')) + if pwd1 != pwd2: + raise YunoHostError(22, _("Passwords don't match")) + return pwd1 + +class PatternParameter(_ExtraParameter): + """ + Check if the argument value match a pattern. + + The value of this parameter corresponds to a list of the pattern and + the message to display if it doesn't match. + + """ + name = 'pattern' + + def __call__(self, arguments, arg_name, arg_value): + pattern = arguments[0] + message = arguments[1] + + if arg_value is not None and not re.match(pattern, arg_value): + raise YunoHostError(22, message) + return arg_value + + @staticmethod + def validate(value, arg_name): + # Tolerate string type + if isinstance(value, str): + logging.warning("value of 'pattern' extra parameter for '%s' argument should be a list" % arg_name) + value = [value, _("'%s' argument is not matching the pattern") % arg_name] + elif not isinstance(value, list) or len(value) != 2: + raise TypeError("Invalid type of 'pattern' extra parameter for '%s' argument" % arg_name) + return value + +""" +The list of available extra parameters classes. It will keep to this list +order on argument parsing. + +""" +extraparameters_list = {AskParameter, PasswordParameter, PatternParameter} diff --git a/src/moulinette/core/helpers.py b/src/moulinette/core/helpers.py index cf4e383c..3d0517aa 100644 --- a/src/moulinette/core/helpers.py +++ b/src/moulinette/core/helpers.py @@ -21,6 +21,21 @@ import getpass if not __debug__: import traceback + +class Interface(): + """ + Contain available interfaces to use with the moulinette. + + """ + api = 'api' + cli = 'cli' + + @classmethod + def all(klass): + """Get a list of all interfaces""" + ifaces = set(i for i in dir(klass) if not i.startswith('_')) + return ifaces + win = [] def random_password(length=8): From 2b370be233c4f0fa78e7aa69640c444f3802be52 Mon Sep 17 00:00:00 2001 From: Jerome Lebleu Date: Mon, 24 Feb 2014 19:11:09 +0100 Subject: [PATCH 04/18] Rename 'etc' folder and move yunohost into a separate package --- {etc => data}/actionsmap/yunohost.yml | 0 {etc => data}/firewall.yml | 0 {etc => data}/ldap_scheme.yml | 0 {etc => data}/moulinette_cli | 0 {etc => data}/services.yml | 0 src/moulinette/config.py | 2 +- src/moulinette/core/actionsmap.py | 2 +- src/{moulinette => }/yunohost/__init__.py | 0 src/{moulinette => }/yunohost/app.py | 2 +- src/{moulinette => }/yunohost/backup.py | 2 +- src/{moulinette => }/yunohost/domain.py | 2 +- src/{moulinette => }/yunohost/dyndns.py | 2 +- src/{moulinette => }/yunohost/firewall.py | 2 +- src/{moulinette => }/yunohost/hook.py | 2 +- src/{moulinette => }/yunohost/monitor.py | 2 +- src/{moulinette => }/yunohost/service.py | 2 +- src/{moulinette => }/yunohost/tools.py | 2 +- src/{moulinette => }/yunohost/user.py | 2 +- 18 files changed, 12 insertions(+), 12 deletions(-) rename {etc => data}/actionsmap/yunohost.yml (100%) rename {etc => data}/firewall.yml (100%) rename {etc => data}/ldap_scheme.yml (100%) rename {etc => data}/moulinette_cli (100%) rename {etc => data}/services.yml (100%) rename src/{moulinette => }/yunohost/__init__.py (100%) rename src/{moulinette => }/yunohost/app.py (99%) rename src/{moulinette => }/yunohost/backup.py (94%) rename src/{moulinette => }/yunohost/domain.py (99%) rename src/{moulinette => }/yunohost/dyndns.py (98%) rename src/{moulinette => }/yunohost/firewall.py (99%) rename src/{moulinette => }/yunohost/hook.py (98%) rename src/{moulinette => }/yunohost/monitor.py (99%) rename src/{moulinette => }/yunohost/service.py (99%) rename src/{moulinette => }/yunohost/tools.py (98%) rename src/{moulinette => }/yunohost/user.py (99%) diff --git a/etc/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml similarity index 100% rename from etc/actionsmap/yunohost.yml rename to data/actionsmap/yunohost.yml diff --git a/etc/firewall.yml b/data/firewall.yml similarity index 100% rename from etc/firewall.yml rename to data/firewall.yml diff --git a/etc/ldap_scheme.yml b/data/ldap_scheme.yml similarity index 100% rename from etc/ldap_scheme.yml rename to data/ldap_scheme.yml diff --git a/etc/moulinette_cli b/data/moulinette_cli similarity index 100% rename from etc/moulinette_cli rename to data/moulinette_cli diff --git a/etc/services.yml b/data/services.yml similarity index 100% rename from etc/services.yml rename to data/services.yml diff --git a/src/moulinette/config.py b/src/moulinette/config.py index b1bcdfb9..578c8ef1 100644 --- a/src/moulinette/config.py +++ b/src/moulinette/config.py @@ -7,7 +7,7 @@ import os session_path = '/var/cache/yunohost/session' # Path of the actions map definition(s) -actionsmap_path = os.path.dirname(__file__) +'/../../etc/actionsmap' +actionsmap_path = os.path.dirname(__file__) +'/../../data/actionsmap' # Path for the actions map cache actionsmap_cache_path = '/var/cache/yunohost/actionsmap' diff --git a/src/moulinette/core/actionsmap.py b/src/moulinette/core/actionsmap.py index 90ff9f43..8c90d973 100644 --- a/src/moulinette/core/actionsmap.py +++ b/src/moulinette/core/actionsmap.py @@ -298,7 +298,7 @@ class ActionsMap(object): function = '%s_%s' % (category, action) try: - mod = __import__(module, globals=globals(), fromlist=[function], level=2) + mod = __import__(module, globals=globals(), fromlist=[function], level=0) func = getattr(mod, function) except (AttributeError, ImportError): raise YunoHostError(168, _('Function is not defined')) diff --git a/src/moulinette/yunohost/__init__.py b/src/yunohost/__init__.py similarity index 100% rename from src/moulinette/yunohost/__init__.py rename to src/yunohost/__init__.py diff --git a/src/moulinette/yunohost/app.py b/src/yunohost/app.py similarity index 99% rename from src/moulinette/yunohost/app.py rename to src/yunohost/app.py index d1b84c38..03a6cd77 100644 --- a/src/moulinette/yunohost/app.py +++ b/src/yunohost/app.py @@ -37,7 +37,7 @@ from domain import domain_list, domain_add from user import user_info, user_list from hook import hook_exec, hook_add, hook_remove -from ..core.helpers import YunoHostError, YunoHostLDAP, win_msg, random_password, is_true, validate +from .moulinette.core.helpers import YunoHostError, YunoHostLDAP, win_msg, random_password, is_true, validate repo_path = '/var/cache/yunohost/repo' apps_path = '/usr/share/yunohost/apps' diff --git a/src/moulinette/yunohost/backup.py b/src/yunohost/backup.py similarity index 94% rename from src/moulinette/yunohost/backup.py rename to src/yunohost/backup.py index 18363bbb..0e245a62 100644 --- a/src/moulinette/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -29,7 +29,7 @@ import json import yaml import glob -from ..core.helpers import YunoHostError, YunoHostLDAP, validate, colorize, win_msg +from moulinette.core.helpers import YunoHostError, YunoHostLDAP, validate, colorize, win_msg def backup_init(helper=False): """ diff --git a/src/moulinette/yunohost/domain.py b/src/yunohost/domain.py similarity index 99% rename from src/moulinette/yunohost/domain.py rename to src/yunohost/domain.py index 6c6f1127..8545dacb 100644 --- a/src/moulinette/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -34,7 +34,7 @@ import requests from urllib import urlopen from dyndns import dyndns_subscribe -from ..core.helpers import YunoHostError, YunoHostLDAP, win_msg, colorize, validate, get_required_args +from moulinette.core.helpers import YunoHostError, YunoHostLDAP, win_msg, colorize, validate, get_required_args def domain_list(filter=None, limit=None, offset=None): diff --git a/src/moulinette/yunohost/dyndns.py b/src/yunohost/dyndns.py similarity index 98% rename from src/moulinette/yunohost/dyndns.py rename to src/yunohost/dyndns.py index 13d5259c..200af870 100644 --- a/src/moulinette/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -30,7 +30,7 @@ import json import glob import base64 -from ..core.helpers import YunoHostError, YunoHostLDAP, validate, colorize, win_msg +from moulinette.core.helpers import YunoHostError, YunoHostLDAP, validate, colorize, win_msg def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None): """ diff --git a/src/moulinette/yunohost/firewall.py b/src/yunohost/firewall.py similarity index 99% rename from src/moulinette/yunohost/firewall.py rename to src/yunohost/firewall.py index 1c4529c5..f55d4f59 100644 --- a/src/moulinette/yunohost/firewall.py +++ b/src/yunohost/firewall.py @@ -37,7 +37,7 @@ except ImportError: sys.stderr.write('apt-get install python-yaml\n') sys.exit(1) -from ..core.helpers import YunoHostError, win_msg +from moulinette.core.helpers import YunoHostError, win_msg def firewall_allow(protocol=None, port=None, ipv6=None, upnp=False): diff --git a/src/moulinette/yunohost/hook.py b/src/yunohost/hook.py similarity index 98% rename from src/moulinette/yunohost/hook.py rename to src/yunohost/hook.py index 4d7c7838..7acdc2a8 100644 --- a/src/moulinette/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -28,7 +28,7 @@ import sys import re import json -from ..core.helpers import YunoHostError, YunoHostLDAP, win_msg, colorize +from moulinette.core.helpers import YunoHostError, YunoHostLDAP, win_msg, colorize hook_folder = '/usr/share/yunohost/hooks/' diff --git a/src/moulinette/yunohost/monitor.py b/src/yunohost/monitor.py similarity index 99% rename from src/moulinette/yunohost/monitor.py rename to src/yunohost/monitor.py index 626b2437..36b81684 100644 --- a/src/moulinette/yunohost/monitor.py +++ b/src/yunohost/monitor.py @@ -37,7 +37,7 @@ from datetime import datetime, timedelta from service import (service_enable, service_disable, service_start, service_stop, service_status) -from ..core.helpers import YunoHostError, win_msg +from moulinette.core.helpers import YunoHostError, win_msg glances_uri = 'http://127.0.0.1:61209' stats_path = '/var/lib/yunohost/stats' diff --git a/src/moulinette/yunohost/service.py b/src/yunohost/service.py similarity index 99% rename from src/moulinette/yunohost/service.py rename to src/yunohost/service.py index cfbfa875..3186608d 100644 --- a/src/moulinette/yunohost/service.py +++ b/src/yunohost/service.py @@ -28,7 +28,7 @@ import glob import subprocess import os.path -from ..core.helpers import YunoHostError, win_msg +from moulinette.core.helpers import YunoHostError, win_msg def service_start(names): diff --git a/src/moulinette/yunohost/tools.py b/src/yunohost/tools.py similarity index 98% rename from src/moulinette/yunohost/tools.py rename to src/yunohost/tools.py index 651e2afb..cf0408bd 100644 --- a/src/moulinette/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -36,7 +36,7 @@ from dyndns import dyndns_subscribe from backup import backup_init from app import app_ssowatconf -from ..core.helpers import YunoHostError, YunoHostLDAP, validate, colorize, get_required_args, win_msg +from moulinette.core.helpers import YunoHostError, YunoHostLDAP, validate, colorize, get_required_args, win_msg def tools_ldapinit(password=None): diff --git a/src/moulinette/yunohost/user.py b/src/yunohost/user.py similarity index 99% rename from src/moulinette/yunohost/user.py rename to src/yunohost/user.py index 16ac8cd4..86c79577 100644 --- a/src/moulinette/yunohost/user.py +++ b/src/yunohost/user.py @@ -33,7 +33,7 @@ import getpass from domain import domain_list from hook import hook_callback -from ..core.helpers import YunoHostError, YunoHostLDAP, win_msg, colorize, validate, get_required_args +from moulinette.core.helpers import YunoHostError, YunoHostLDAP, win_msg, colorize, validate, get_required_args def user_list(fields=None, filter=None, limit=None, offset=None): """ From 9104024fa166b9adc6b500d787b0c5f0a5916c2f Mon Sep 17 00:00:00 2001 From: Jerome Lebleu Date: Tue, 4 Mar 2014 12:31:04 +0100 Subject: [PATCH 05/18] Refactore interfaces and reorganize files a bit * Remove 'core' folder and reorganize root package structure * Introduce interface's base class and implement 'api' and 'cli' * Add a Package class and a moulinette initialization method * Start to replace YunoHostError by MoulinetteError * Fix actionsmap/yunohost.yml to follow extra parameters rules --- bin/yunohost | 23 +- bin/yunohost-api | 16 +- data/actionsmap/yunohost.yml | 52 +- src/moulinette/__init__.py | 64 ++- src/moulinette/actionsmap.py | 338 ++++++++++++ src/moulinette/config.py | 16 - src/moulinette/core.py | 155 ++++++ src/moulinette/core/actionsmap.py | 503 ------------------ src/moulinette/{core => extra}/__init__.py | 0 .../parameters.py} | 9 +- src/moulinette/{core => }/helpers.py | 154 ------ src/moulinette/interface/__init__.py | 114 ++++ src/moulinette/{core => interface}/api.py | 183 ++++++- src/moulinette/interface/cli.py | 35 ++ src/yunohost/app.py | 2 +- src/yunohost/backup.py | 2 +- src/yunohost/domain.py | 2 +- src/yunohost/dyndns.py | 2 +- src/yunohost/firewall.py | 2 +- src/yunohost/hook.py | 2 +- src/yunohost/monitor.py | 2 +- src/yunohost/service.py | 2 +- src/yunohost/tools.py | 2 +- src/yunohost/user.py | 2 +- 24 files changed, 931 insertions(+), 751 deletions(-) create mode 100644 src/moulinette/actionsmap.py delete mode 100644 src/moulinette/config.py create mode 100644 src/moulinette/core.py delete mode 100644 src/moulinette/core/actionsmap.py rename src/moulinette/{core => extra}/__init__.py (100%) mode change 100755 => 100644 rename src/moulinette/{core/extraparameters.py => extra/parameters.py} (95%) rename src/moulinette/{core => }/helpers.py (63%) create mode 100755 src/moulinette/interface/__init__.py rename src/moulinette/{core => interface}/api.py (53%) create mode 100644 src/moulinette/interface/cli.py diff --git a/bin/yunohost b/bin/yunohost index d4483df5..fb03b34d 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -5,25 +5,29 @@ import sys import os.path import gettext -# Debug option -if '--debug' in sys.argv: - sys.path.append(os.path.abspath(os.path.dirname(__file__) +'/../src')) -from moulinette import cli -from moulinette.core.helpers import YunoHostError, colorize +# Run from source +basedir = os.path.abspath(os.path.dirname(__file__) +'/../') +if os.path.isdir(basedir +'/src'): + sys.path.append(basedir +'/src') -gettext.install('YunoHost') +from moulinette import init, cli, MoulinetteError +from moulinette.helpers import YunoHostError, colorize + +gettext.install('yunohost') ## Main action if __name__ == '__main__': + # Run from source (prefix and libdir set to None) + init('yunohost', prefix=None, libdir=None, + cachedir=os.path.join(basedir, 'cache')) + # Additional arguments use_cache = True if '--no-cache' in sys.argv: use_cache = False sys.argv.remove('--no-cache') - if '--debug' in sys.argv: - sys.argv.remove('--debug') try: args = list(sys.argv) @@ -36,6 +40,9 @@ if __name__ == '__main__': # Execute the action cli(args, use_cache) + except MoulinetteError as e: + print(e.colorize()) + sys.exit(e.code) except YunoHostError as e: print(colorize(_("Error: "), 'red') + e.message) sys.exit(e.code) diff --git a/bin/yunohost-api b/bin/yunohost-api index 625d2b7e..18d10d09 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -5,12 +5,14 @@ import sys import os.path import gettext -# Debug option -if '--debug' in sys.argv: - sys.path.append(os.path.abspath(os.path.dirname(__file__) +'/../src')) -from moulinette import api +# Run from source +basedir = os.path.abspath(os.path.dirname(__file__) +'/../') +if os.path.isdir(basedir +'/src'): + sys.path.append(basedir +'/src') -gettext.install('YunoHost') +from moulinette import init, api + +gettext.install('yunohost') ## Callbacks for additional routes @@ -29,6 +31,10 @@ def is_installed(): ## Main action if __name__ == '__main__': + # Run from source (prefix and libdir set to None) + init('yunohost', prefix=None, libdir=None, + cachedir=os.path.join(basedir, 'cache')) + # Additional arguments use_cache = True if '--no-cache' in sys.argv: diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index fe5ab237..52a62654 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -76,7 +76,9 @@ user: help: Must be unique extra: ask: "Username" - pattern: '^[a-z0-9_]+$' + pattern: + - '^[a-z0-9_]+$' + - "Must be alphanumeric and underscore characters only" -f: full: --firstname extra: @@ -109,7 +111,9 @@ user: nargs: "*" extra: ask: "Users to delete" - pattern: '^[a-z0-9_]+$' + pattern: + - '^[a-z0-9_]+$' + - "Must be alphanumeric and underscore characters only" --purge: action: store_true @@ -187,7 +191,9 @@ domain: help: Domain name to add nargs: '+' extra: - pattern: '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + pattern: + - '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + - "Must be a valid domain name (e.g. my-domain.org)" -m: full: --main help: Is the main domain @@ -206,7 +212,9 @@ domain: help: Domain(s) to delete nargs: "+" extra: - pattern: '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + pattern: + - '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + - "Must be a valid domain name (e.g. my-domain.org)" ### domain_info() info: @@ -216,7 +224,9 @@ domain: domain: help: "" extra: - pattern: '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + pattern: + - '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + - "Must be a valid domain name (e.g. my-domain.org)" ############################# @@ -253,7 +263,9 @@ app: help: Name of the list to remove extra: ask: "List to remove" - pattern: '^[a-z0-9_]+$' + pattern: + - '^[a-z0-9_]+$' + - "Must be alphanumeric and underscore characters only" ### app_list() list: @@ -302,7 +314,9 @@ app: full: --user help: Allowed app map for a user extra: - pattern: '^[a-z0-9_]+$' + pattern: + - '^[a-z0-9_]+$' + - "Must be alphanumeric and underscore characters only" ### app_install() TODO: Write help @@ -388,7 +402,9 @@ app: port: help: Port to check extra: - pattern: '^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$' + pattern: + - '^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$' + - "Must be a valid port number (i.e. 0-65535)" ### app_checkurl() checkurl: @@ -659,7 +675,9 @@ service: help: Number of lines to display default: "50" extra: - pattern: '^[0-9]+$' + pattern: + - '^[0-9]+$' + - "Must be a valid number" ############################# @@ -691,7 +709,9 @@ firewall: port: help: Port to open extra: - pattern: '^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$' + pattern: + - '^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$' + - "Must be a valid port number (i.e. 0-65535)" protocol: help: Protocol associated with port choices: @@ -840,12 +860,16 @@ tools: -o: full: --old-domain extra: - pattern: '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + pattern: + - '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + - "Must be a valid domain name (e.g. my-domain.org)" -n: full: --new-domain extra: ask: "New main domain" - pattern: '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + pattern: + - '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + - "Must be a valid domain name (e.g. my-domain.org)" ### tools_postinstall() postinstall: @@ -857,7 +881,9 @@ tools: help: YunoHost main domain extra: ask: "Main domain" - pattern: '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + pattern: + - '^([a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)(\.[a-zA-Z0-9]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)*(\.[a-zA-Z]{1}([a-zA-Z0-9\-]*[a-zA-Z0-9])*)$' + - "Must be a valid domain name (e.g. my-domain.org)" -p: full: --password help: YunoHost admin password diff --git a/src/moulinette/__init__.py b/src/moulinette/__init__.py index f96eae48..aa8c6753 100755 --- a/src/moulinette/__init__.py +++ b/src/moulinette/__init__.py @@ -24,68 +24,98 @@ __credits__ = """ You should have received a copy of the GNU Affero General Public License along with this program; if not, see http://www.gnu.org/licenses """ +__all__ = [ + 'init', 'api', 'cli', + 'MoulinetteError', +] + +from .core import MoulinetteError + +curr_namespace = None -## Fast access functions +## Package functions + +def init(namespace=None, **kwargs): + """Package initialization + + Initialize directories and global variables. It must be called + before any of package method is used - even the easy access + functions. + + Keyword arguments: + - namespace -- The namespace to initialize and use + - **kwargs -- See helpers.Package + + At the end, the global variable 'pkg' will contain a Package + instance. See helpers.Package for available methods and variables. + + """ + import __builtin__ + from .core import Package + + global curr_namespace + curr_namespace = namespace + + __builtin__.__dict__['pkg'] = Package(**kwargs) + + +## Easy access to interfaces def api(port, routes={}, use_cache=True): - """ + """Web server (API) interface + Run a HTTP server with the moulinette for an API usage. Keyword arguments: - - port -- Port to run on - - routes -- A dict of additional routes to add in the form of {(method, uri): callback} - - use_cache -- False if it should parse the actions map file instead of using the cached one """ from bottle import run - from core.actionsmap import ActionsMap - from core.api import MoulinetteAPI - from core.helpers import Interface + from .actionsmap import ActionsMap + from .interface.api import MoulinetteAPI - amap = ActionsMap(Interface.api, use_cache=use_cache) + amap = ActionsMap('api', use_cache=use_cache) moulinette = MoulinetteAPI(amap, routes) run(moulinette.app, port=port) def cli(args, use_cache=True): - """ + """Command line interface + Execute an action with the moulinette from the CLI and print its result in a readable format. Keyword arguments: - - args -- A list of argument strings - - use_cache -- False if it should parse the actions map file instead of using the cached one """ import os - from core.actionsmap import ActionsMap - from core.helpers import Interface, YunoHostError, pretty_print_dict + from .actionsmap import ActionsMap + from .helpers import YunoHostError, pretty_print_dict lock_file = '/var/run/moulinette.lock' # TODO: Move the lock checking into the ActionsMap class # Check the lock if os.path.isfile(lock_file): - raise YunoHostError(1, _("The moulinette is already running")) + raise MoulinetteError(1, _("The moulinette is already running")) # Create a lock with open(lock_file, 'w') as f: pass os.system('chmod 400 '+ lock_file) try: - amap = ActionsMap(Interface.cli, use_cache=use_cache) + amap = ActionsMap('cli', use_cache=use_cache) pretty_print_dict(amap.process(args)) except KeyboardInterrupt, EOFError: - raise YunoHostError(125, _("Interrupted")) + raise MoulinetteError(125, _("Interrupted")) finally: # Remove the lock os.remove(lock_file) diff --git a/src/moulinette/actionsmap.py b/src/moulinette/actionsmap.py new file mode 100644 index 00000000..f2eb5464 --- /dev/null +++ b/src/moulinette/actionsmap.py @@ -0,0 +1,338 @@ +# -*- coding: utf-8 -*- + +import pickle +import yaml +import re +import os +from collections import OrderedDict + +import logging + +from . import __version__, curr_namespace, MoulinetteError +from .extra.parameters import extraparameters_list + +## Extra parameters Parser + +class ExtraParser(object): + """ + Global parser for the extra parameters. + + """ + def __init__(self, iface): + self.iface = iface + self.extra = OrderedDict() + + # Append available extra parameters for the current interface + for klass in extraparameters_list: + if iface in klass.skipped_iface: + continue + self.extra[klass.name] = klass + + def validate(self, arg_name, parameters): + """ + Validate values of extra parameters for an argument + + Keyword arguments: + - arg_name -- The argument name + - parameters -- A dict of extra parameters with their values + + """ + # Iterate over parameters to validate + for p, v in parameters.items(): + # Remove unknow parameters + if p not in self.extra.keys(): + del parameters[p] + + # Validate parameter value + parameters[p] = self.extra[p].validate(v, arg_name) + + return parameters + + def parse(self, arg_name, arg_value, parameters): + """ + Parse argument with extra parameters + + Keyword arguments: + - arg_name -- The argument name + - arg_value -- The argument value + - parameters -- A dict of extra parameters with their values + + """ + # Iterate over available parameters + for p, klass in self.extra.items(): + if p not in parameters.keys(): + continue + + # Initialize the extra parser + parser = klass(self.iface) + + # Parse the argument + if isinstance(arg_value, list): + for v in arg_value: + r = parser(parameters[p], arg_name, v) + if r not in arg_value: + arg_value.append(r) + else: + arg_value = parser(parameters[p], arg_name, arg_value) + + return arg_value + + +## Main class + +class ActionsMap(object): + """ + Validate and process action defined into the actions map. + + The actions map defines features and their usage of the main + application. It is composed by categories which contain one or more + action(s). Moreover, the action can have specific argument(s). + + Keyword arguments: + + - interface -- Interface type that requires the actions map. + Possible value is one of: + - 'cli' for the command line interface + - 'api' for an API usage (HTTP requests) + + - use_cache -- False if it should parse the actions map file + instead of using the cached one. + + """ + def __init__(self, interface, use_cache=True): + self.use_cache = use_cache + + try: + # Retrieve the interface parser + mod = __import__('interface.%s' % interface, + globals=globals(), level=1, + fromlist=['actionsmap_parser']) + parser = getattr(mod, 'actionsmap_parser') + except (AttributeError, ImportError): + raise MoulinetteError(22, _("Invalid interface '%s'" % interface)) + else: + self._parser_class = parser + + logging.debug("initializing ActionsMap for the '%s' interface" % interface) + + actionsmaps = {} + namespaces = self.get_actionsmap_namespaces() + if curr_namespace and curr_namespace in namespaces: + namespaces = [curr_namespace] + + # Iterate over actions map namespaces + for n in namespaces: + logging.debug("loading '%s' actions map namespace" % n) + + if use_cache: + # Attempt to load cache if it exists + cache_file = '%s/%s.pkl' % (pkg.cachedir('actionsmap'), n) + if os.path.isfile(cache_file): + with open(cache_file, 'r') as f: + actionsmaps[n] = pickle.load(f) + else: + self.use_cache = False + actionsmaps = self.generate_cache(namespaces) + break + else: + am_file = '%s/%s.yml' % (pkg.datadir('actionsmap'), n) + with open(am_file, 'r') as f: + actionsmaps[n] = yaml.load(f) + + # Generate parsers + self.extraparser = ExtraParser(interface) + self.parser = self._construct_parser(actionsmaps) + + def process(self, args, **kwargs): + """ + Parse arguments and process the proper action + + Keyword arguments: + - args -- The arguments to parse + - **kwargs -- Additional interface arguments + + """ + # Parse arguments + arguments = vars(self.parser.parse_args(args, **kwargs)) + arguments = self._parse_extra_parameters(arguments) + + # Retrieve action information + namespace, category, action = arguments.pop('_info') + func_name = '%s_%s' % (category, action) + + try: + mod = __import__('%s.%s' % (namespace, category), + globals=globals(), level=0, + fromlist=[func_name]) + func = getattr(mod, func_name) + except (AttributeError, ImportError): + raise MoulinetteError(168, _('Function is not defined')) + else: + # Process the action + return func(**arguments) + + @staticmethod + def get_actionsmap_namespaces(): + """ + Retrieve actions map namespaces from a given path + + Returns: + A list of available namespaces + + """ + namespaces = [] + + for f in os.listdir(pkg.datadir('actionsmap')): + if f.endswith('.yml'): + namespaces.append(f[:-4]) + return namespaces + + @classmethod + def generate_cache(klass, namespaces=None): + """ + Generate cache for the actions map's file(s) + + Keyword arguments: + - namespaces -- A list of namespaces to generate cache for + + Returns: + A dict of actions map for each namespaces + + """ + actionsmaps = {} + if not namespaces: + namespaces = klass.get_actionsmap_namespaces() + + # Iterate over actions map namespaces + for n in namespaces: + logging.debug("generating cache for '%s' actions map namespace" % n) + + # Read actions map from yaml file + am_file = pkg.datafile('actionsmap/%s.yml' % n) + with open(am_file, 'r') as f: + actionsmaps[n] = yaml.load(f) + + # Cache actions map into pickle file + cache_file = pkg.cachefile('actionsmap/%s.pkl' % n, make_dir=True) + with open(cache_file, 'w') as f: + pickle.dump(actionsmaps[n], f) + + return actionsmaps + + + ## Private class and methods + + def _store_extra_parameters(self, parser, arg_name, arg_extra): + """ + Store extra parameters for a given argument + + Keyword arguments: + - parser -- Parser object for the arguments + - arg_name -- Argument name + - arg_extra -- Argument extra parameters + + Returns: + The parser object + + """ + if arg_extra: + # Retrieve current extra parameters dict + extra = parser.get_default('_extra') + if not extra or not isinstance(extra, dict): + extra = {} + + if not self.use_cache: + # Validate extra parameters for the argument + extra[arg_name] = self.extraparser.validate(arg_name, arg_extra) + else: + extra[arg_name] = arg_extra + parser.set_defaults(_extra=extra) + + return parser + + def _parse_extra_parameters(self, args): + """ + Parse arguments with their extra parameters + + Keyword arguments: + - args -- A dict of all arguments + + Return: + The parsed arguments dict + + """ + # Retrieve extra parameters for the arguments + extra = args.pop('_extra', None) + if not extra: + return args + + # Validate extra parameters for each arguments + for an, parameters in extra.items(): + args[an] = self.extraparser.parse(an, args[an], parameters) + + return args + + def _construct_parser(self, actionsmaps): + """ + Construct the parser with the actions map + + Keyword arguments: + - actionsmaps -- A dict of multi-level dictionnary of + categories/actions/arguments list for each namespaces + + Returns: + An interface relevant's parser object + + """ + # Instantiate parser + top_parser = self._parser_class() + + # Iterate over actions map namespaces + for n, actionsmap in actionsmaps.items(): + if 'general_arguments' in actionsmap: + # Parse general arguments + if top_parser.parse_general: + parser = top_parser.add_general_parser() + for an, ap in actionsmap['general_arguments'].items(): + if 'version' in ap: + ap['version'] = ap['version'].replace('%version%', + __version__) + argname = top_parser.format_arg_name(an, ap.pop('full', None)) + parser.add_argument(*argname, **ap) + del actionsmap['general_arguments'] + + # Parse categories + for cn, cp in actionsmap.items(): + if 'actions' not in cp: + continue + actions = cp.pop('actions') + + # Add category parser + if top_parser.parse_category: + cat_parser = top_parser.add_category_parser(cn, **cp) + else: + cat_parser = top_parser + + # Parse actions + if not top_parser.parse_action: + continue + for an, ap in actions.items(): + arguments = ap.pop('arguments', {}) + + # Add action parser + parser = cat_parser.add_action_parser(an, **ap) + if not parser: + continue + + # Store action information + parser.set_defaults(_info=(n, cn, an)) + + # Add action arguments + for argn, argp in arguments.items(): + name = top_parser.format_arg_name(argn, argp.pop('full', None)) + extra = argp.pop('extra', None) + + arg = parser.add_argument(*name, **argp) + parser = self._store_extra_parameters(parser, arg.dest, extra) + + return top_parser diff --git a/src/moulinette/config.py b/src/moulinette/config.py deleted file mode 100644 index 578c8ef1..00000000 --- a/src/moulinette/config.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- - -# TODO: Remove permanent debug values -import os - -# Path for the the web sessions -session_path = '/var/cache/yunohost/session' - -# Path of the actions map definition(s) -actionsmap_path = os.path.dirname(__file__) +'/../../data/actionsmap' - -# Path for the actions map cache -actionsmap_cache_path = '/var/cache/yunohost/actionsmap' - -# Path of the doc in json format -doc_json_path = os.path.dirname(__file__) +'/../../doc' diff --git a/src/moulinette/core.py b/src/moulinette/core.py new file mode 100644 index 00000000..62fe34fe --- /dev/null +++ b/src/moulinette/core.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- + +import os +import sys +import gettext +from .helpers import colorize + +class Package(object): + """Package representation and easy access + + Initialize directories and variables for the package and give them + easy access. + + Keyword arguments: + - prefix -- The installation prefix + - libdir -- The library directory; usually, this would be + prefix + '/lib' (or '/lib64') when installed + - cachedir -- The cache directory; usually, this would be + '/var/cache' when installed + - destdir -- The destination prefix only if it's an installation + + 'prefix' and 'libdir' arguments should be empty in order to run + package from source. + + """ + def __init__(self, prefix, libdir, cachedir, destdir=None): + if not prefix and not libdir: + # Running from source directory + basedir = os.path.abspath(os.path.dirname(sys.argv[0]) +'/../') + self._datadir = os.path.join(basedir, 'data') + self._libdir = os.path.join(basedir, 'src') + self._cachedir = cachedir + else: + self._datadir = os.path.join(prefix, 'share/moulinette') + self._libdir = os.path.join(libdir, 'moulinette') + self._cachedir = os.path.join(cachedir, 'moulinette') + + # Append library path to python's path + sys.path.append(self._libdir) + self._destdir = destdir or None + + + ## Easy access to directories and files + + def datadir(self, subdir=None, **kwargs): + """Return the path to a data directory""" + return self.get_dir(self._datadir, subdir, **kwargs) + + def datafile(self, filename, **kwargs): + """Return the path to a data file""" + return self.get_file(self._datadir, filename, **kwargs) + + def libdir(self, subdir=None, **kwargs): + """Return the path to a lib directory""" + return self.get_dir(self._libdir, subdir, **kwargs) + + def libfile(self, filename, **kwargs): + """Return the path to a lib file""" + return self.get_file(self._libdir, filename, **kwargs) + + def cachedir(self, subdir=None, **kwargs): + """Return the path to a cache directory""" + return self.get_dir(self._cachedir, subdir, **kwargs) + + def cachefile(self, filename, **kwargs): + """Return the path to a cache file""" + return self.get_file(self._cachedir, filename, **kwargs) + + + ## Standard methods + + def get_dir(self, basedir, subdir=None, make_dir=False): + """Get a directory path + + Return a path composed by a base directory and an optional + subdirectory. The path will be created if needed. + + Keyword arguments: + - basedir -- The base directory + - subdir -- An optional subdirectory + - make_dir -- True if it should create needed directory + + """ + # Retrieve path + path = basedir + if self._destdir: + path = os.path.join(self._destdir, path) + if subdir: + path = os.path.join(path, subdir) + + # Create directory + if make_dir and not os.path.isdir(path): + os.makedirs(path) + return path + + def get_file(self, basedir, filename, **kwargs): + """Get a file path + + Return the path of the filename in the specified directory. This + directory will be created if needed. + + Keyword arguments: + - basedir -- The base directory of the file + - filename -- The filename or a path relative to basedir + - **kwargs -- Additional arguments for Package.get_dir + + """ + # Check for a directory in filename + subdir = os.path.dirname(filename) or None + if subdir: + filename = os.path.basename(filename) + + # Get directory path + dirpath = self.get_dir(basedir, subdir, **kwargs) + return os.path.join(dirpath, filename) + + +class MoulinetteError(Exception): + """Moulinette base exception + + Keyword arguments: + - code -- Integer error code + - message -- Error message to display + + """ + def __init__(self, code, message): + self.code = code + self.message = message + + errorcode_desc = { + 1 : _('Fail'), + 13 : _('Permission denied'), + 17 : _('Already exists'), + 22 : _('Invalid arguments'), + 87 : _('Too many users'), + 111 : _('Connection refused'), + 122 : _('Quota exceeded'), + 125 : _('Operation canceled'), + 167 : _('Not found'), + 168 : _('Undefined'), + 169 : _('LDAP operation error') + } + if code in errorcode_desc: + self.desc = errorcode_desc[code] + else: + self.desc = _('Error %s' % code) + + def __str__(self, colorized=False): + desc = self.desc + if colorized: + desc = colorize(self.desc, 'red') + return _('%s: %s' % (desc, self.message)) + + def colorize(self): + return self.__str__(colorized=True) diff --git a/src/moulinette/core/actionsmap.py b/src/moulinette/core/actionsmap.py deleted file mode 100644 index 8c90d973..00000000 --- a/src/moulinette/core/actionsmap.py +++ /dev/null @@ -1,503 +0,0 @@ -# -*- coding: utf-8 -*- - -import argparse -import pickle -import yaml -import re -import os -from collections import OrderedDict - -import logging - -from .. import __version__ -from ..config import actionsmap_path, actionsmap_cache_path - -from extraparameters import extraparameters_list -from helpers import Interface, YunoHostError - -## Additional parsers - -class _HTTPArgumentParser(object): - - def __init__(self, method, uri): - # Initialize the ArgumentParser object - self._parser = argparse.ArgumentParser(usage='', - prefix_chars='@', - add_help=False) - self._parser.error = self._error - - self.method = method - self.uri = uri - - self._positional = [] # list(arg_name) - self._optional = {} # dict({arg_name: option_strings}) - - def set_defaults(self, **kwargs): - return self._parser.set_defaults(**kwargs) - - def get_default(self, dest): - return self._parser.get_default(dest) - - def add_argument(self, *args, **kwargs): - action = self._parser.add_argument(*args, **kwargs) - - # Append newly created action - if len(action.option_strings) == 0: - self._positional.append(action.dest) - else: - self._optional[action.dest] = action.option_strings - - return action - - def parse_args(self, args): - arg_strings = [] - - ## Append an argument to the current one - def append(arg_strings, value, option_string=None): - # TODO: Process list arguments - if isinstance(value, bool): - # Append the option string only - if option_string is not None: - arg_strings.append(option_string) - elif isinstance(value, str): - if option_string is not None: - arg_strings.append(option_string) - arg_strings.append(value) - else: - arg_strings.append(value) - - return arg_strings - - # Iterate over positional arguments - for dest in self._positional: - if dest in args: - arg_strings = append(arg_strings, args[dest]) - - # Iterate over optional arguments - for dest, opt in self._optional.items(): - if dest in args: - arg_strings = append(arg_strings, args[dest], opt[0]) - return self._parser.parse_args(arg_strings) - - def _error(self, message): - # TODO: Raise a proper exception - raise Exception(message) - -class HTTPParser(object): - """ - Object for parsing HTTP requests into Python objects. - - """ - - def __init__(self): - self._parsers = {} # dict({(method, uri): _HTTPArgumentParser}) - - @property - def routes(self): - """Get current routes""" - return self._parsers.keys() - - def add_parser(self, method, uri): - """ - Add a parser for a given route - - Keyword arguments: - - method -- The route's HTTP method (GET, POST, PUT, DELETE) - - uri -- The route's URI - - Returns: - A new _HTTPArgumentParser object for the route - - """ - # Check if a parser already exists for the route - key = (method, uri) - if key in self.routes: - raise ValueError("A parser for '%s' already exists" % key) - - # Create and append parser - parser = _HTTPArgumentParser(method, uri) - self._parsers[key] = parser - - # Return the created parser - return parser - - def parse_args(self, method, uri, args={}): - """ - Convert argument variables to objects and assign them as - attributes of the namespace for a given route - - Keyword arguments: - - method -- The route's HTTP method (GET, POST, PUT, DELETE) - - uri -- The route's URI - - args -- Argument variables for the route - - Returns: - The populated namespace - - """ - # Retrieve the parser for the route - key = (method, uri) - if key not in self.routes: - raise ValueError("No parser for '%s %s' found" % key) - - return self._parsers[key].parse_args(args) - -class ExtraParser(object): - """ - Global parser for the extra parameters. - - """ - def __init__(self, iface): - self.iface = iface - self.extra = OrderedDict() - - # Append available extra parameters for the current interface - for klass in extraparameters_list: - if iface in klass.skipped_iface: - continue - if klass.name in self.extra: - logging.warning("extra parameter named '%s' was already added" % klass.name) - continue - self.extra[klass.name] = klass - - def validate(self, arg_name, parameters): - """ - Validate values of extra parameters for an argument - - Keyword arguments: - - arg_name -- The argument name - - parameters -- A dict of extra parameters with their values - - """ - # Iterate over parameters to validate - for p, v in parameters.items(): - # Remove unknow parameters - if p not in self.extra.keys(): - del parameters[p] - - # Validate parameter value - parameters[p] = self.extra[p].validate(v, arg_name) - - return parameters - - def parse(self, arg_name, arg_value, parameters): - """ - Parse argument with extra parameters - - Keyword arguments: - - arg_name -- The argument name - - arg_value -- The argument value - - parameters -- A dict of extra parameters with their values - - """ - # Iterate over available parameters - for p, klass in self.extra.items(): - if p not in parameters.keys(): - continue - - # Initialize the extra parser - parser = klass(self.iface) - - # Parse the argument - if isinstance(arg_value, list): - for v in arg_value: - r = parser(parameters[p], arg_name, v) - if r not in arg_value: - arg_value.append(r) - else: - arg_value = parser(parameters[p], arg_name, arg_value) - - return arg_value - - -## Main class - -class ActionsMap(object): - """ - Validate and process action defined into the actions map. - - The actions map defines features and their usage of the main - application. It is composed by categories which contain one or more - action(s). Moreover, the action can have specific argument(s). - - Keyword arguments: - - - interface -- Interface type that requires the actions map. - Possible value is one of: - - 'cli' for the command line interface - - 'api' for an API usage (HTTP requests) - - - use_cache -- False if it should parse the actions map file - instead of using the cached one. - - """ - def __init__(self, interface, use_cache=True): - if interface not in Interface.all(): - raise ValueError(_("Invalid interface '%s'" % interface)) - self.interface = interface - self.use_cache = use_cache - - logging.debug("initializing ActionsMap for the '%s' interface" % interface) - - # Iterate over actions map namespaces - actionsmaps = {} - for n in self.get_actionsmap_namespaces(): - logging.debug("loading '%s' actions map namespace" % n) - - if use_cache: - # Attempt to load cache if it exists - cache_file = '%s/%s.pkl' % (actionsmap_cache_path, n) - if os.path.isfile(cache_file): - with open(cache_file, 'r') as f: - actionsmaps[n] = pickle.load(f) - else: - self.use_cache = False - actionsmaps = self.generate_cache() - break - else: - am_file = '%s/%s.yml' % (actionsmap_path, n) - with open(am_file, 'r') as f: - actionsmaps[n] = yaml.load(f) - - # Generate parsers - self.extraparser = ExtraParser(interface) - self.parser = self._construct_parser(actionsmaps) - - def process(self, args, route=None): - """ - Parse arguments and process the proper action - - Keyword arguments: - - args -- The arguments to parse - - route -- A tupple (method, uri) of the requested route (api only) - - """ - arguments = None - - # Parse arguments - if self.interface == Interface.cli: - arguments = self.parser.parse_args(args) - elif self.interface == Interface.api: - if route is None: - # TODO: Raise a proper exception - raise Exception(_("Missing route argument")) - method, uri = route - arguments = self.parser.parse_args(method, uri, args) - arguments = vars(arguments) - - # Parse extra parameters - arguments = self._parse_extra_parameters(arguments) - - # Retrieve action information - namespace = arguments['_info']['namespace'] - category = arguments['_info']['category'] - action = arguments['_info']['action'] - del arguments['_info'] - - module = '%s.%s' % (namespace, category) - function = '%s_%s' % (category, action) - - try: - mod = __import__(module, globals=globals(), fromlist=[function], level=0) - func = getattr(mod, function) - except (AttributeError, ImportError): - raise YunoHostError(168, _('Function is not defined')) - else: - # Process the action - return func(**arguments) - - @staticmethod - def get_actionsmap_namespaces(path=actionsmap_path): - """ - Retrieve actions map namespaces from a given path - - Returns: - A list of available namespaces - - """ - namespaces = [] - - for f in os.listdir(path): - if f.endswith('.yml'): - namespaces.append(f[:-4]) - return namespaces - - @classmethod - def generate_cache(klass): - """ - Generate cache for the actions map's file(s) - - Returns: - A dict of actions map for each namespaces - - """ - actionsmaps = {} - - if not os.path.isdir(actionsmap_cache_path): - os.makedirs(actionsmap_cache_path) - - # Iterate over actions map namespaces - for n in klass.get_actionsmap_namespaces(): - logging.debug("generating cache for '%s' actions map namespace" % n) - - # Read actions map from yaml file - am_file = '%s/%s.yml' % (actionsmap_path, n) - with open(am_file, 'r') as f: - actionsmaps[n] = yaml.load(f) - - # Cache actions map into pickle file - cache_file = '%s/%s.pkl' % (actionsmap_cache_path, n) - with open(cache_file, 'w') as f: - pickle.dump(actionsmaps[n], f) - - return actionsmaps - - - ## Private class and methods - - def _store_extra_parameters(self, parser, arg_name, arg_params): - """ - Store extra parameters for a given argument - - Keyword arguments: - - parser -- Parser object for the arguments - - arg_name -- Argument name - - arg_params -- Argument parameters - - Returns: - The parser object - - """ - if 'extra' in arg_params: - # Retrieve current extra parameters dict - extra = parser.get_default('_extra') - if not extra or not isinstance(extra, dict): - extra = {} - - if not self.use_cache: - # Validate extra parameters for the argument - extra[arg_name] = self.extraparser.validate(arg_name, arg_params['extra']) - else: - extra[arg_name] = arg_params['extra'] - parser.set_defaults(_extra=extra) - - return parser - - def _parse_extra_parameters(self, args): - """ - Parse arguments with their extra parameters - - Keyword arguments: - - args -- A dict of all arguments - - Return: - The parsed arguments dict - - """ - # Retrieve extra parameters from the arguments - if '_extra' not in args: - return args - extra = args['_extra'] - del args['_extra'] - - # Validate extra parameters for each arguments - for an, parameters in extra.items(): - args[an] = self.extraparser.parse(an, args[an], parameters) - - return args - - def _construct_parser(self, actionsmaps): - """ - Construct the parser with the actions map - - Keyword arguments: - - actionsmaps -- A dict of multi-level dictionnary of - categories/actions/arguments list for each namespaces - - Returns: - An interface relevant's parser object - - """ - top_parser = None - iface = self.interface - - # Create parser object - if iface == Interface.cli: - # TODO: Add descritpion (from __description__?) - top_parser = argparse.ArgumentParser() - top_subparsers = top_parser.add_subparsers() - elif iface == Interface.api: - top_parser = HTTPParser() - - ## Format option strings from argument parameters - def _option_strings(arg_name, arg_params): - if iface == Interface.cli: - if arg_name[0] == '-' and 'full' in arg_params: - return [arg_name, arg_params['full']] - return [arg_name] - elif iface == Interface.api: - if arg_name[0] != '-': - return [arg_name] - if 'full' in arg_params: - return [arg_params['full'].replace('--', '@', 1)] - if arg_name.startswith('--'): - return [arg_name.replace('--', '@', 1)] - return [arg_name.replace('-', '@', 1)] - - ## Remove extra parameters - def _clean_params(arg_params): - for k in {'full', 'extra'}: - if k in arg_params: - del arg_params[k] - return arg_params - - # Iterate over actions map namespaces - for n, actionsmap in actionsmaps.items(): - # Parse general arguments for the cli only - if iface == Interface.cli: - for an, ap in actionsmap['general_arguments'].items(): - if 'version' in ap: - ap['version'] = ap['version'].replace('%version%', __version__) - top_parser.add_argument(*_option_strings(an, ap), **_clean_params(ap)) - del actionsmap['general_arguments'] - - # Parse categories - for cn, cp in actionsmap.items(): - if 'actions' not in cp: - continue - - # Add category subparsers for the cli only - if iface == Interface.cli: - c_help = cp.get('category_help') - subparsers = top_subparsers.add_parser(cn, help=c_help).add_subparsers() - - # Parse actions - for an, ap in cp['actions'].items(): - parser = None - - # Add parser for the current action - if iface == Interface.cli: - a_help = ap.get('action_help') - parser = subparsers.add_parser(an, help=a_help) - elif iface == Interface.api and 'api' in ap: - # Extract method and uri - m = re.match('(GET|POST|PUT|DELETE) (/\S+)', ap['api']) - if m: - parser = top_parser.add_parser(m.group(1), m.group(2)) - if not parser: - continue - - # Store action information - parser.set_defaults(_info={'namespace': n, - 'category': cn, - 'action': an}) - - # Add arguments - if not 'arguments' in ap: - continue - for argn, argp in ap['arguments'].items(): - arg = parser.add_argument(*_option_strings(argn, argp), - **_clean_params(argp.copy())) - parser = self._store_extra_parameters(parser, arg.dest, argp) - - return top_parser diff --git a/src/moulinette/core/__init__.py b/src/moulinette/extra/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from src/moulinette/core/__init__.py rename to src/moulinette/extra/__init__.py diff --git a/src/moulinette/core/extraparameters.py b/src/moulinette/extra/parameters.py similarity index 95% rename from src/moulinette/core/extraparameters.py rename to src/moulinette/extra/parameters.py index 3f4248c0..f1bf0ba9 100644 --- a/src/moulinette/core/extraparameters.py +++ b/src/moulinette/extra/parameters.py @@ -4,7 +4,8 @@ import getpass import re import logging -from helpers import Interface, colorize, YunoHostError +from .. import MoulinetteError +from ..helpers import colorize class _ExtraParameter(object): """ @@ -78,7 +79,7 @@ class AskParameter(_ExtraParameter): """ name = 'ask' - skipped_iface = {Interface.api} + skipped_iface = { 'api' } def __call__(self, message, arg_name, arg_value): # TODO: Fix asked arguments ordering @@ -119,7 +120,7 @@ class PasswordParameter(AskParameter): pwd1 = getpass.getpass(colorize(message + ': ', 'cyan')) pwd2 = getpass.getpass(colorize('Retype ' + message + ': ', 'cyan')) if pwd1 != pwd2: - raise YunoHostError(22, _("Passwords don't match")) + raise MoulinetteError(22, _("Passwords don't match")) return pwd1 class PatternParameter(_ExtraParameter): @@ -137,7 +138,7 @@ class PatternParameter(_ExtraParameter): message = arguments[1] if arg_value is not None and not re.match(pattern, arg_value): - raise YunoHostError(22, message) + raise MoulinetteError(22, message) return arg_value @staticmethod diff --git a/src/moulinette/core/helpers.py b/src/moulinette/helpers.py similarity index 63% rename from src/moulinette/core/helpers.py rename to src/moulinette/helpers.py index 3d0517aa..69efaf1f 100644 --- a/src/moulinette/core/helpers.py +++ b/src/moulinette/helpers.py @@ -21,21 +21,6 @@ import getpass if not __debug__: import traceback - -class Interface(): - """ - Contain available interfaces to use with the moulinette. - - """ - api = 'api' - cli = 'cli' - - @classmethod - def all(klass): - """Get a list of all interfaces""" - ifaces = set(i for i in dir(klass) if not i.startswith('_')) - return ifaces - win = [] def random_password(length=8): @@ -105,33 +90,6 @@ def win_msg(astr): win.append(astr) -def str_to_func(astr): - """ - Call a function from a string name - - Keyword arguments: - astr -- Name of function to call - - Returns: - Function - - """ - try: - module, _, function = astr.rpartition('.') - if module: - __import__(module) - mod = sys.modules[module] - else: - mod = sys.modules['__main__'] # default module - - func = getattr(mod, function) - except (AttributeError, ImportError): - #raise YunoHostError(168, _('Function is not defined')) - return None - else: - return func - - def validate(pattern, array): """ Validate attributes with a pattern @@ -441,115 +399,3 @@ class YunoHostLDAP(Singleton): else: raise YunoHostError(17, _('Attribute already exists') + ' "' + attr + '=' + value + '"') return True - - -def parse_dict(action_map): - """ - Turn action dictionnary to parser, subparsers and arguments - - Keyword arguments: - action_map -- Multi-level dictionnary of categories/actions/arguments list - - Returns: - Namespace of args - - """ - # Intialize parsers - parsers = subparsers_category = subparsers_action = {} - parsers['general'] = argparse.ArgumentParser() - subparsers = parsers['general'].add_subparsers() - new_args = [] - patterns = {} - - # Add general arguments - for arg_name, arg_params in action_map['general_arguments'].items(): - if 'version' in arg_params: - v = arg_params['version'] - arg_params['version'] = v.replace('%version%', __version__) - if 'full' in arg_params: - arg_names = [arg_name, arg_params['full']] - arg_fullname = arg_params['full'] - del arg_params['full'] - else: arg_names = [arg_name] - parsers['general'].add_argument(*arg_names, **arg_params) - - del action_map['general_arguments'] - - # Split categories into subparsers - for category, category_params in action_map.items(): - if 'category_help' not in category_params: category_params['category_help'] = '' - subparsers_category[category] = subparsers.add_parser(category, help=category_params['category_help']) - subparsers_action[category] = subparsers_category[category].add_subparsers() - # Split actions - if 'actions' in category_params: - for action, action_params in category_params['actions'].items(): - if 'action_help' not in action_params: action_params['action_help'] = '' - parsers[category + '_' + action] = subparsers_action[category].add_parser(action, help=action_params['action_help']) - # Set the action s related function - parsers[category + '_' + action].set_defaults( - func=str_to_func('yunohost_' + category + '.' - + category + '_' + action.replace('-', '_'))) - # Add arguments - if 'arguments' in action_params: - for arg_name, arg_params in action_params['arguments'].items(): - arg_fullname = False - - if 'password' in arg_params: - if arg_params['password']: is_password = True - del arg_params['password'] - else: is_password = False - - if 'full' in arg_params: - arg_names = [arg_name, arg_params['full']] - arg_fullname = arg_params['full'] - del arg_params['full'] - else: arg_names = [arg_name] - - if 'ask' in arg_params: - require_input = True - if '-h' in sys.argv or '--help' in sys.argv: - require_input = False - if (category != sys.argv[1]) or (action != sys.argv[2]): - require_input = False - for name in arg_names: - if name in sys.argv[2:]: require_input = False - - if require_input: - if is_password: - if os.isatty(1): - pwd1 = getpass.getpass(colorize(arg_params['ask'] + ': ', 'cyan')) - pwd2 = getpass.getpass(colorize('Retype ' + arg_params['ask'][0].lower() + arg_params['ask'][1:] + ': ', 'cyan')) - if pwd1 != pwd2: - raise YunoHostError(22, _("Passwords don't match")) - sys.exit(1) - else: - raise YunoHostError(22, _("Missing arguments") + ': ' + arg_name) - if arg_name[0] == '-': arg_extend = [arg_name, pwd1] - else: arg_extend = [pwd1] - else: - if os.isatty(1): - arg_value = raw_input(colorize(arg_params['ask'] + ': ', 'cyan')) - else: - raise YunoHostError(22, _("Missing arguments") + ': ' + arg_name) - if arg_name[0] == '-': arg_extend = [arg_name, arg_value] - else: arg_extend = [arg_value] - new_args.extend(arg_extend) - del arg_params['ask'] - - if 'pattern' in arg_params: - if (category == sys.argv[1]) and (action == sys.argv[2]): - if 'dest' in arg_params: name = arg_params['dest'] - elif arg_fullname: name = arg_fullname[2:] - else: name = arg_name - name = name.replace('-', '_') - patterns[name] = arg_params['pattern'] - del arg_params['pattern'] - - parsers[category + '_' + action].add_argument(*arg_names, **arg_params) - - args = parsers['general'].parse_args(sys.argv.extend(new_args)) - args_dict = vars(args) - for key, value in patterns.items(): - validate(value, args_dict[key]) - - return args diff --git a/src/moulinette/interface/__init__.py b/src/moulinette/interface/__init__.py new file mode 100755 index 00000000..3a99535f --- /dev/null +++ b/src/moulinette/interface/__init__.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- + +class BaseParser(object): + """Actions map's base Parser + + Each interfaces must implement a parser class derived from this + class. It is used to parse the main parts of the actions map (i.e. + general arguments, categories and actions). + + """ + + ## Optional variables + # Each parser classes can overwrite these variables. + + """Either it will parse general arguments, or not""" + parse_general = True + + """Either it will parse categories, or not""" + parse_category = True + + """Either it will parse actions, or not""" + parse_action = True + + + ## Virtual methods + # Each parser classes can implement these methods. + + @staticmethod + def format_arg_name(name, full): + """Format argument name + + Format agument name depending on its 'full' parameters and return + a list to use it as option string for the argument parser. + + Keyword arguments: + - name -- The argument name + - full -- The argument's 'full' parameter + + Returns: + A list of option strings + + """ + raise NotImplementedError("derived class '%s' must override this method" % \ + self.__class__.__name__) + + def add_general_parser(self, **kwargs): + """Add a parser for general arguments + + Create and return an argument parser for general arguments. + + Returns: + An ArgumentParser based object + + """ + if not self.parse_general: + msg = "doesn't parse general arguments" + else: + msg = "must override this method" + raise NotImplementedError("derived class '%s' %s" % \ + (self.__class__.__name__, msg)) + + def add_category_parser(self, name, **kwargs): + """Add a parser for a category + + Create a new category and return a parser for it. + + Keyword arguments: + - name -- The category name + + Returns: + A BaseParser based object + + """ + if not self.parse_categories: + msg = "doesn't parse categories" + else: + msg = "must override this method" + raise NotImplementedError("derived class '%s' %s" % \ + (self.__class__.__name__, msg)) + + def add_action_parser(self, name, **kwargs): + """Add a parser for an action + + Create a new action and return an argument parser for it. + + Keyword arguments: + - name -- The action name + + Returns: + An ArgumentParser based object + + """ + if not self.parse_general: + msg = "doesn't parse actions" + else: + msg = "must override this method" + raise NotImplementedError("derived class '%s' %s" % \ + (self.__class__.__name__, msg)) + + def parse_args(self, args, **kwargs): + """Parse arguments + + Convert argument variables to objects and assign them as + attributes of the namespace. + + Keyword arguments: + - args -- Arguments string or dict (TODO) + + Returns: + The populated namespace + + """ + raise NotImplementedError("derived class '%s' must override this method" % \ + self.__class__.__name__) diff --git a/src/moulinette/core/api.py b/src/moulinette/interface/api.py similarity index 53% rename from src/moulinette/core/api.py rename to src/moulinette/interface/api.py index 19e0af18..6c959c9f 100644 --- a/src/moulinette/core/api.py +++ b/src/moulinette/interface/api.py @@ -1,16 +1,162 @@ # -*- coding: utf-8 -*- +import re +import argparse import os.path from bottle import Bottle, request, response, HTTPResponse from beaker.middleware import SessionMiddleware -from ..config import session_path, doc_json_path -from helpers import YunoHostError, YunoHostLDAP +from . import BaseParser +from .. import MoulinetteError +from ..helpers import YunoHostError, YunoHostLDAP + +## API arguments Parser + +class _HTTPArgumentParser(object): + """Argument parser for HTTP requests + + Object for parsing HTTP requests into Python objects. It is based + on argparse.ArgumentParser class and implements some of its methods. + + """ + def __init__(self): + # Initialize the ArgumentParser object + self._parser = argparse.ArgumentParser(usage='', + prefix_chars='@', + add_help=False) + self._parser.error = self._error + + self._positional = [] # list(arg_name) + self._optional = {} # dict({arg_name: option_strings}) + + def set_defaults(self, **kwargs): + return self._parser.set_defaults(**kwargs) + + def get_default(self, dest): + return self._parser.get_default(dest) + + def add_argument(self, *args, **kwargs): + action = self._parser.add_argument(*args, **kwargs) + + # Append newly created action + if len(action.option_strings) == 0: + self._positional.append(action.dest) + else: + self._optional[action.dest] = action.option_strings + + return action + + def parse_args(self, args): + arg_strings = [] + + ## Append an argument to the current one + def append(arg_strings, value, option_string=None): + # TODO: Process list arguments + if isinstance(value, bool): + # Append the option string only + if option_string is not None: + arg_strings.append(option_string) + elif isinstance(value, str): + if option_string is not None: + arg_strings.append(option_string) + arg_strings.append(value) + else: + arg_strings.append(value) + + return arg_strings + + # Iterate over positional arguments + for dest in self._positional: + if dest in args: + arg_strings = append(arg_strings, args[dest]) + + # Iterate over optional arguments + for dest, opt in self._optional.items(): + if dest in args: + arg_strings = append(arg_strings, args[dest], opt[0]) + return self._parser.parse_args(arg_strings) + + def _error(self, message): + # TODO: Raise a proper exception + raise MoulinetteError(1, message) + +class APIParser(BaseParser): + """Actions map's API Parser + + """ + parse_category = False + parse_general = False + + def __init__(self): + self._parsers = {} # dict({(method, path): _HTTPArgumentParser}) + + @property + def routes(self): + """Get current routes""" + return self._parsers.keys() + + + ## Implement virtual methods + + @staticmethod + def format_arg_name(name, full): + if name[0] != '-': + return [name] + if full: + return [full.replace('--', '@', 1)] + if name.startswith('--'): + return [name.replace('--', '@', 1)] + return [name.replace('-', '@', 1)] + + def add_action_parser(self, name, api=None, **kwargs): + """Add a parser for an action + + Keyword arguments: + - api -- The action route (e.g. 'GET /' ) + + Returns: + A new _HTTPArgumentParser object for the route + + """ + if not api: + return None + + # Validate action route + m = re.match('(GET|POST|PUT|DELETE) (/\S+)', api) + if not m: + return None + + # Check if a parser already exists for the route + key = (m.group(1), m.group(2)) + if key in self.routes: + raise ValueError("A parser for '%s' already exists" % key) + + # Create and append parser + parser = _HTTPArgumentParser() + self._parsers[key] = parser + + # Return the created parser + return parser + + def parse_args(self, args, route, **kwargs): + """Parse arguments + + Keyword arguments: + - route -- The action route (e.g. 'GET /' ) + + """ + # Retrieve the parser for the route + if route not in self.routes: + raise MoulinetteError(22, "No parser for '%s %s' found" % key) + + return self._parsers[route].parse_args(args) + +actionsmap_parser = APIParser -## Bottle Plugins +## API moulinette interface -class APIAuthPlugin(object): +class _APIAuthPlugin(object): """ Manage the authentication for the API access. @@ -20,8 +166,7 @@ class APIAuthPlugin(object): def __init__(self): # TODO: Add options (e.g. session type, content type, ...) - if not os.path.isdir(session_path): - os.makedirs(session_path) + pass @property def app(self): @@ -44,7 +189,7 @@ class APIAuthPlugin(object): session_opts = { 'session.type': 'file', 'session.cookie_expires': True, - 'session.data_dir': session_path, + 'session.data_dir': pkg.cachedir('session', make_dir=True), 'session.secure': True } self._app = SessionMiddleware(app, session_opts) @@ -119,7 +264,7 @@ class APIAuthPlugin(object): return True return False -class ActionsMapPlugin(object): +class _ActionsMapPlugin(object): """ Process action for the request using the actions map. @@ -158,19 +303,15 @@ class ActionsMapPlugin(object): return wrapper -## Main class - class MoulinetteAPI(object): """ Initialize a HTTP server which serves the API to access to the moulinette actions. Keyword arguments: - - actionsmap -- The relevant ActionsMap instance - - routes -- A dict of additional routes to add in the form of - {(method, uri): callback} + {(method, path): callback} """ @@ -182,14 +323,14 @@ class MoulinetteAPI(object): callback=self.doc, skip=['apiauth']) # Append routes from the actions map - amap = ActionsMapPlugin(actionsmap) - for (m, u) in actionsmap.parser.routes: - app.route(u, method=m, callback=self._error, apply=amap) + amap = _ActionsMapPlugin(actionsmap) + for (m, p) in actionsmap.parser.routes: + app.route(p, method=m, callback=self._error, apply=amap) # Append additional routes # TODO: Add an option to skip auth for the route - for (m, u), c in routes.items(): - app.route(u, method=m, callback=c) + for (m, p), c in routes.items(): + app.route(p, method=m, callback=c) # Define and install a plugin which sets proper header def apiheader(callback): @@ -201,7 +342,7 @@ class MoulinetteAPI(object): app.install(apiheader) # Install authentication plugin - apiauth = APIAuthPlugin() + apiauth = _APIAuthPlugin() app.install(apiauth) self._app = apiauth.app @@ -220,11 +361,11 @@ class MoulinetteAPI(object): """ if category is None: - with open(doc_json_path +'/resources.json') as f: + with open(pkg.datafile('doc/resources.json')) as f: return f.read() try: - with open(doc_json_path +'/'+ category +'.json') as f: + with open(pkg.datafile('doc/%s.json' % category)) as f: return f.read() except IOError: return 'unknown' diff --git a/src/moulinette/interface/cli.py b/src/moulinette/interface/cli.py new file mode 100644 index 00000000..c4be4c57 --- /dev/null +++ b/src/moulinette/interface/cli.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +import argparse +from . import BaseParser + +## CLI arguments Parser + +class CLIParser(BaseParser): + """Actions map's CLI Parser + + """ + def __init__(self, parser=None): + self._parser = parser or argparse.ArgumentParser() + self._subparsers = self._parser.add_subparsers() + + @staticmethod + def format_arg_name(name, full): + if name[0] == '-' and full: + return [name, full] + return [name] + + def add_general_parser(self, **kwargs): + return self._parser + + def add_category_parser(self, name, category_help=None, **kwargs): + parser = self._subparsers.add_parser(name, help=category_help) + return CLIParser(parser) + + def add_action_parser(self, name, action_help, **kwargs): + return self._subparsers.add_parser(name, help=action_help) + + def parse_args(self, args, **kwargs): + return self._parser.parse_args(args) + +actionsmap_parser = CLIParser diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 03a6cd77..eeb9808b 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -37,7 +37,7 @@ from domain import domain_list, domain_add from user import user_info, user_list from hook import hook_exec, hook_add, hook_remove -from .moulinette.core.helpers import YunoHostError, YunoHostLDAP, win_msg, random_password, is_true, validate +from moulinette.helpers import YunoHostError, YunoHostLDAP, win_msg, random_password, is_true, validate repo_path = '/var/cache/yunohost/repo' apps_path = '/usr/share/yunohost/apps' diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 0e245a62..d4723d9d 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -29,7 +29,7 @@ import json import yaml import glob -from moulinette.core.helpers import YunoHostError, YunoHostLDAP, validate, colorize, win_msg +from moulinette.helpers import YunoHostError, YunoHostLDAP, validate, colorize, win_msg def backup_init(helper=False): """ diff --git a/src/yunohost/domain.py b/src/yunohost/domain.py index 8545dacb..b10a92a7 100644 --- a/src/yunohost/domain.py +++ b/src/yunohost/domain.py @@ -34,7 +34,7 @@ import requests from urllib import urlopen from dyndns import dyndns_subscribe -from moulinette.core.helpers import YunoHostError, YunoHostLDAP, win_msg, colorize, validate, get_required_args +from moulinette.helpers import YunoHostError, YunoHostLDAP, win_msg, colorize, validate, get_required_args def domain_list(filter=None, limit=None, offset=None): diff --git a/src/yunohost/dyndns.py b/src/yunohost/dyndns.py index 200af870..1bd6b6ba 100644 --- a/src/yunohost/dyndns.py +++ b/src/yunohost/dyndns.py @@ -30,7 +30,7 @@ import json import glob import base64 -from moulinette.core.helpers import YunoHostError, YunoHostLDAP, validate, colorize, win_msg +from moulinette.helpers import YunoHostError, YunoHostLDAP, validate, colorize, win_msg def dyndns_subscribe(subscribe_host="dyndns.yunohost.org", domain=None, key=None): """ diff --git a/src/yunohost/firewall.py b/src/yunohost/firewall.py index f55d4f59..3fb68dea 100644 --- a/src/yunohost/firewall.py +++ b/src/yunohost/firewall.py @@ -37,7 +37,7 @@ except ImportError: sys.stderr.write('apt-get install python-yaml\n') sys.exit(1) -from moulinette.core.helpers import YunoHostError, win_msg +from moulinette.helpers import YunoHostError, win_msg def firewall_allow(protocol=None, port=None, ipv6=None, upnp=False): diff --git a/src/yunohost/hook.py b/src/yunohost/hook.py index 7acdc2a8..9136acfb 100644 --- a/src/yunohost/hook.py +++ b/src/yunohost/hook.py @@ -28,7 +28,7 @@ import sys import re import json -from moulinette.core.helpers import YunoHostError, YunoHostLDAP, win_msg, colorize +from moulinette.helpers import YunoHostError, YunoHostLDAP, win_msg, colorize hook_folder = '/usr/share/yunohost/hooks/' diff --git a/src/yunohost/monitor.py b/src/yunohost/monitor.py index 36b81684..cd0a22ae 100644 --- a/src/yunohost/monitor.py +++ b/src/yunohost/monitor.py @@ -37,7 +37,7 @@ from datetime import datetime, timedelta from service import (service_enable, service_disable, service_start, service_stop, service_status) -from moulinette.core.helpers import YunoHostError, win_msg +from moulinette.helpers import YunoHostError, win_msg glances_uri = 'http://127.0.0.1:61209' stats_path = '/var/lib/yunohost/stats' diff --git a/src/yunohost/service.py b/src/yunohost/service.py index 3186608d..139e1ca2 100644 --- a/src/yunohost/service.py +++ b/src/yunohost/service.py @@ -28,7 +28,7 @@ import glob import subprocess import os.path -from moulinette.core.helpers import YunoHostError, win_msg +from moulinette.helpers import YunoHostError, win_msg def service_start(names): diff --git a/src/yunohost/tools.py b/src/yunohost/tools.py index cf0408bd..dd57d47f 100644 --- a/src/yunohost/tools.py +++ b/src/yunohost/tools.py @@ -36,7 +36,7 @@ from dyndns import dyndns_subscribe from backup import backup_init from app import app_ssowatconf -from moulinette.core.helpers import YunoHostError, YunoHostLDAP, validate, colorize, get_required_args, win_msg +from moulinette.helpers import YunoHostError, YunoHostLDAP, validate, colorize, get_required_args, win_msg def tools_ldapinit(password=None): diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 86c79577..b619a227 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -33,7 +33,7 @@ import getpass from domain import domain_list from hook import hook_callback -from moulinette.core.helpers import YunoHostError, YunoHostLDAP, win_msg, colorize, validate, get_required_args +from moulinette.helpers import YunoHostError, YunoHostLDAP, win_msg, colorize, validate, get_required_args def user_list(fields=None, filter=None, limit=None, offset=None): """ From 9c9ccc1271209dbb4fc988e92e3e89739482a87a Mon Sep 17 00:00:00 2001 From: Jerome Lebleu Date: Tue, 11 Mar 2014 00:57:28 +0100 Subject: [PATCH 06/18] Try to improve time execution and continue refactoring * Revert to classes centralization into actionsmap.py * Try to optimize conditions and loops * Revisit Package class and get directories from a file generated at build * Early refactoring of i18n * Move yunohost library into /lib --- bin/yunohost | 16 +- bin/yunohost-api | 10 +- .../extra => lib/yunohost}/__init__.py | 0 {src => lib}/yunohost/app.py | 0 {src => lib}/yunohost/backup.py | 0 {data => lib/yunohost/data}/firewall.yml | 0 {data => lib/yunohost/data}/ldap_scheme.yml | 0 {data => lib/yunohost/data}/services.yml | 0 {src => lib}/yunohost/domain.py | 0 {src => lib}/yunohost/dyndns.py | 0 {src => lib}/yunohost/firewall.py | 0 {src => lib}/yunohost/hook.py | 0 {src => lib}/yunohost/monitor.py | 0 {src => lib}/yunohost/service.py | 0 {src => lib}/yunohost/tools.py | 0 {src => lib}/yunohost/user.py | 0 src/moulinette/__init__.py | 36 +- src/moulinette/actionsmap.py | 566 ++++++++++++++++-- src/moulinette/core.py | 155 ++--- src/moulinette/extra/parameters.py | 159 ----- src/moulinette/helpers.py | 12 +- src/moulinette/interface/__init__.py | 114 ---- src/moulinette/interface/api.py | 158 +---- src/moulinette/interface/cli.py | 36 +- src/moulinette/package.py | 15 + src/moulinette/package.py.in | 15 + src/yunohost/__init__.py | 0 27 files changed, 660 insertions(+), 632 deletions(-) rename {src/moulinette/extra => lib/yunohost}/__init__.py (100%) mode change 100644 => 100755 rename {src => lib}/yunohost/app.py (100%) rename {src => lib}/yunohost/backup.py (100%) rename {data => lib/yunohost/data}/firewall.yml (100%) rename {data => lib/yunohost/data}/ldap_scheme.yml (100%) rename {data => lib/yunohost/data}/services.yml (100%) rename {src => lib}/yunohost/domain.py (100%) rename {src => lib}/yunohost/dyndns.py (100%) rename {src => lib}/yunohost/firewall.py (100%) rename {src => lib}/yunohost/hook.py (100%) rename {src => lib}/yunohost/monitor.py (100%) rename {src => lib}/yunohost/service.py (100%) rename {src => lib}/yunohost/tools.py (100%) rename {src => lib}/yunohost/user.py (100%) delete mode 100644 src/moulinette/extra/parameters.py mode change 100755 => 100644 src/moulinette/interface/__init__.py create mode 100644 src/moulinette/package.py create mode 100644 src/moulinette/package.py.in delete mode 100755 src/yunohost/__init__.py diff --git a/bin/yunohost b/bin/yunohost index fb03b34d..78a354bf 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -3,25 +3,21 @@ import sys import os.path -import gettext # Run from source -basedir = os.path.abspath(os.path.dirname(__file__) +'/../') -if os.path.isdir(basedir +'/src'): - sys.path.append(basedir +'/src') +basedir = os.path.abspath('%s/../' % os.path.dirname(__file__)) +if os.path.isdir('%s/src' % basedir): + sys.path.append('%s/src' % basedir) from moulinette import init, cli, MoulinetteError from moulinette.helpers import YunoHostError, colorize -gettext.install('yunohost') - ## Main action if __name__ == '__main__': - # Run from source (prefix and libdir set to None) - init('yunohost', prefix=None, libdir=None, - cachedir=os.path.join(basedir, 'cache')) + # Run from source + init(_from_source=True) # Additional arguments use_cache = True @@ -39,7 +35,7 @@ if __name__ == '__main__': raise YunoHostError(17, _("YunoHost is not correctly installed, please execute 'yunohost tools postinstall'")) # Execute the action - cli(args, use_cache) + cli(['yunohost'], args, use_cache) except MoulinetteError as e: print(e.colorize()) sys.exit(e.code) diff --git a/bin/yunohost-api b/bin/yunohost-api index 18d10d09..11e889c5 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -3,7 +3,6 @@ import sys import os.path -import gettext # Run from source basedir = os.path.abspath(os.path.dirname(__file__) +'/../') @@ -12,8 +11,6 @@ if os.path.isdir(basedir +'/src'): from moulinette import init, api -gettext.install('yunohost') - ## Callbacks for additional routes @@ -31,9 +28,8 @@ def is_installed(): ## Main action if __name__ == '__main__': - # Run from source (prefix and libdir set to None) - init('yunohost', prefix=None, libdir=None, - cachedir=os.path.join(basedir, 'cache')) + # Run from source + init(_from_source=True) # Additional arguments use_cache = True @@ -45,5 +41,5 @@ if __name__ == '__main__': # TODO: Add log argument # Rune the server - api(6787, {('GET', '/installed'): is_installed}, use_cache) + api(['yunohost'], 6787, {('GET', '/installed'): is_installed}, use_cache) sys.exit(0) diff --git a/src/moulinette/extra/__init__.py b/lib/yunohost/__init__.py old mode 100644 new mode 100755 similarity index 100% rename from src/moulinette/extra/__init__.py rename to lib/yunohost/__init__.py diff --git a/src/yunohost/app.py b/lib/yunohost/app.py similarity index 100% rename from src/yunohost/app.py rename to lib/yunohost/app.py diff --git a/src/yunohost/backup.py b/lib/yunohost/backup.py similarity index 100% rename from src/yunohost/backup.py rename to lib/yunohost/backup.py diff --git a/data/firewall.yml b/lib/yunohost/data/firewall.yml similarity index 100% rename from data/firewall.yml rename to lib/yunohost/data/firewall.yml diff --git a/data/ldap_scheme.yml b/lib/yunohost/data/ldap_scheme.yml similarity index 100% rename from data/ldap_scheme.yml rename to lib/yunohost/data/ldap_scheme.yml diff --git a/data/services.yml b/lib/yunohost/data/services.yml similarity index 100% rename from data/services.yml rename to lib/yunohost/data/services.yml diff --git a/src/yunohost/domain.py b/lib/yunohost/domain.py similarity index 100% rename from src/yunohost/domain.py rename to lib/yunohost/domain.py diff --git a/src/yunohost/dyndns.py b/lib/yunohost/dyndns.py similarity index 100% rename from src/yunohost/dyndns.py rename to lib/yunohost/dyndns.py diff --git a/src/yunohost/firewall.py b/lib/yunohost/firewall.py similarity index 100% rename from src/yunohost/firewall.py rename to lib/yunohost/firewall.py diff --git a/src/yunohost/hook.py b/lib/yunohost/hook.py similarity index 100% rename from src/yunohost/hook.py rename to lib/yunohost/hook.py diff --git a/src/yunohost/monitor.py b/lib/yunohost/monitor.py similarity index 100% rename from src/yunohost/monitor.py rename to lib/yunohost/monitor.py diff --git a/src/yunohost/service.py b/lib/yunohost/service.py similarity index 100% rename from src/yunohost/service.py rename to lib/yunohost/service.py diff --git a/src/yunohost/tools.py b/lib/yunohost/tools.py similarity index 100% rename from src/yunohost/tools.py rename to lib/yunohost/tools.py diff --git a/src/yunohost/user.py b/lib/yunohost/user.py similarity index 100% rename from src/yunohost/user.py rename to lib/yunohost/user.py diff --git a/src/moulinette/__init__.py b/src/moulinette/__init__.py index aa8c6753..0bc0ff72 100755 --- a/src/moulinette/__init__.py +++ b/src/moulinette/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- __title__ = 'moulinette' -__version__ = '695' +__version__ = '0.1' __author__ = ['Kload', 'jlebleu', 'titoko', @@ -31,12 +31,10 @@ __all__ = [ from .core import MoulinetteError -curr_namespace = None - ## Package functions -def init(namespace=None, **kwargs): +def init(**kwargs): """Package initialization Initialize directories and global variables. It must be called @@ -44,30 +42,33 @@ def init(namespace=None, **kwargs): functions. Keyword arguments: - - namespace -- The namespace to initialize and use - - **kwargs -- See helpers.Package + - **kwargs -- See core.Package At the end, the global variable 'pkg' will contain a Package - instance. See helpers.Package for available methods and variables. + instance. See core.Package for available methods and variables. """ + import sys import __builtin__ - from .core import Package - - global curr_namespace - curr_namespace = namespace - + from .core import Package, install_i18n __builtin__.__dict__['pkg'] = Package(**kwargs) + # Initialize internationalization + install_i18n() + + # Add library directory to python path + sys.path.append(pkg.libdir) + ## Easy access to interfaces -def api(port, routes={}, use_cache=True): +def api(namespaces, port, routes={}, use_cache=True): """Web server (API) interface Run a HTTP server with the moulinette for an API usage. Keyword arguments: + - namespaces -- The list of namespaces to use - port -- Port to run on - routes -- A dict of additional routes to add in the form of {(method, uri): callback} @@ -79,18 +80,19 @@ def api(port, routes={}, use_cache=True): from .actionsmap import ActionsMap from .interface.api import MoulinetteAPI - amap = ActionsMap('api', use_cache=use_cache) + amap = ActionsMap('api', namespaces, use_cache) moulinette = MoulinetteAPI(amap, routes) run(moulinette.app, port=port) -def cli(args, use_cache=True): +def cli(namespaces, args, use_cache=True): """Command line interface Execute an action with the moulinette from the CLI and print its result in a readable format. Keyword arguments: + - namespaces -- The list of namespaces to use - args -- A list of argument strings - use_cache -- False if it should parse the actions map file instead of using the cached one @@ -98,7 +100,7 @@ def cli(args, use_cache=True): """ import os from .actionsmap import ActionsMap - from .helpers import YunoHostError, pretty_print_dict + from .helpers import pretty_print_dict lock_file = '/var/run/moulinette.lock' @@ -112,7 +114,7 @@ def cli(args, use_cache=True): os.system('chmod 400 '+ lock_file) try: - amap = ActionsMap('cli', use_cache=use_cache) + amap = ActionsMap('cli', namespaces, use_cache) pretty_print_dict(amap.process(args)) except KeyboardInterrupt, EOFError: raise MoulinetteError(125, _("Interrupted")) diff --git a/src/moulinette/actionsmap.py b/src/moulinette/actionsmap.py index f2eb5464..db07318b 100644 --- a/src/moulinette/actionsmap.py +++ b/src/moulinette/actionsmap.py @@ -1,21 +1,478 @@ # -*- coding: utf-8 -*- -import pickle +import argparse import yaml import re import os +import cPickle as pickle from collections import OrderedDict import logging -from . import __version__, curr_namespace, MoulinetteError -from .extra.parameters import extraparameters_list +from . import __version__ +from .core import MoulinetteError -## Extra parameters Parser +## Interfaces' Actions map Parser -------------------------------------- + +class _AMapParser(object): + """Actions map's base Parser + + Each interfaces must implement a parser class derived from this + class. It is used to parse the main parts of the actions map (i.e. + general arguments, categories and actions). -class ExtraParser(object): """ - Global parser for the extra parameters. + + ## Optional variables + # Each parser classes can overwrite these variables. + + """Either it will parse general arguments, or not""" + parse_general_arguments = True + + + ## Virtual methods + # Each parser classes can implement these methods. + + @staticmethod + def format_arg_name(name, full): + """Format argument name + + Format agument name depending on its 'full' parameters and return + a list to use it as option string for the argument parser. + + Keyword arguments: + - name -- The argument name + - full -- The argument's 'full' parameter + + Returns: + A list of option strings + + """ + raise NotImplementedError("derived class '%s' must override this method" % \ + self.__class__.__name__) + + def add_general_parser(self, **kwargs): + """Add a parser for general arguments + + Create and return an argument parser for general arguments. + + Returns: + An ArgumentParser based object + + """ + if not self.parse_general_arguments: + msg = "doesn't parse general arguments" + else: + msg = "must override this method" + raise NotImplementedError("derived class '%s' %s" % \ + (self.__class__.__name__, msg)) + + def add_category_parser(self, name, **kwargs): + """Add a parser for a category + + Create a new category and return a parser for it. + + Keyword arguments: + - name -- The category name + + Returns: + A BaseParser based object + + """ + raise NotImplementedError("derived class '%s' must override this method" % \ + self.__class__.__name__) + + def add_action_parser(self, name, **kwargs): + """Add a parser for an action + + Create a new action and return an argument parser for it. + + Keyword arguments: + - name -- The action name + + Returns: + An ArgumentParser based object + + """ + raise NotImplementedError("derived class '%s' must override this method" % \ + self.__class__.__name__) + + def parse_args(self, args, **kwargs): + """Parse arguments + + Convert argument variables to objects and assign them as + attributes of the namespace. + + Keyword arguments: + - args -- Arguments string or dict (TODO) + + Returns: + The populated namespace + + """ + raise NotImplementedError("derived class '%s' must override this method" % \ + self.__class__.__name__) + +# CLI Actions map Parser + +class CLIAMapParser(_AMapParser): + """Actions map's CLI Parser + + """ + def __init__(self, parser=None): + self._parser = parser or argparse.ArgumentParser() + self._subparsers = self._parser.add_subparsers() + + @staticmethod + def format_arg_name(name, full): + if name[0] == '-' and full: + return [name, full] + return [name] + + def add_general_parser(self, **kwargs): + return self._parser + + def add_category_parser(self, name, category_help=None, **kwargs): + """Add a parser for a category + + Keyword arguments: + - category_help -- A brief description for the category + + Returns: + A new CLIParser object for the category + + """ + parser = self._subparsers.add_parser(name, help=category_help) + return self.__class__(parser) + + def add_action_parser(self, name, action_help, **kwargs): + """Add a parser for an action + + Keyword arguments: + - action_help -- A brief description for the action + + Returns: + A new argparse.ArgumentParser object for the action + + """ + return self._subparsers.add_parser(name, help=action_help) + + def parse_args(self, args, **kwargs): + return self._parser.parse_args(args) + +# API Actions map Parser + +class _HTTPArgumentParser(object): + """Argument parser for HTTP requests + + Object for parsing HTTP requests into Python objects. It is based + on argparse.ArgumentParser class and implements some of its methods. + + """ + def __init__(self): + # Initialize the ArgumentParser object + self._parser = argparse.ArgumentParser(usage='', + prefix_chars='@', + add_help=False) + self._parser.error = self._error + + self._positional = [] # list(arg_name) + self._optional = {} # dict({arg_name: option_strings}) + + def set_defaults(self, **kwargs): + return self._parser.set_defaults(**kwargs) + + def get_default(self, dest): + return self._parser.get_default(dest) + + def add_argument(self, *args, **kwargs): + action = self._parser.add_argument(*args, **kwargs) + + # Append newly created action + if len(action.option_strings) == 0: + self._positional.append(action.dest) + else: + self._optional[action.dest] = action.option_strings + + return action + + def parse_args(self, args): + arg_strings = [] + + ## Append an argument to the current one + def append(arg_strings, value, option_string=None): + # TODO: Process list arguments + if isinstance(value, bool): + # Append the option string only + if option_string is not None: + arg_strings.append(option_string) + elif isinstance(value, str): + if option_string is not None: + arg_strings.append(option_string) + arg_strings.append(value) + else: + arg_strings.append(value) + + return arg_strings + + # Iterate over positional arguments + for dest in self._positional: + if dest in args: + arg_strings = append(arg_strings, args[dest]) + + # Iterate over optional arguments + for dest, opt in self._optional.items(): + if dest in args: + arg_strings = append(arg_strings, args[dest], opt[0]) + return self._parser.parse_args(arg_strings) + + def _error(self, message): + # TODO: Raise a proper exception + raise MoulinetteError(1, message) + +class APIAMapParser(_AMapParser): + """Actions map's API Parser + + """ + parse_general_arguments = False + + def __init__(self): + self._parsers = {} # dict({(method, path): _HTTPArgumentParser}) + + @property + def routes(self): + """Get current routes""" + return self._parsers.keys() + + + ## Implement virtual methods + + @staticmethod + def format_arg_name(name, full): + if name[0] != '-': + return [name] + if full: + return [full.replace('--', '@', 1)] + if name.startswith('--'): + return [name.replace('--', '@', 1)] + return [name.replace('-', '@', 1)] + + def add_category_parser(self, name, **kwargs): + return self + + def add_action_parser(self, name, api=None, **kwargs): + """Add a parser for an action + + Keyword arguments: + - api -- The action route (e.g. 'GET /' ) + + Returns: + A new _HTTPArgumentParser object for the route + + """ + if not api: + return None + + # Validate action route + m = re.match('(GET|POST|PUT|DELETE) (/\S+)', api) + if not m: + return None + + # Check if a parser already exists for the route + key = (m.group(1), m.group(2)) + if key in self.routes: + raise ValueError("A parser for '%s' already exists" % key) + + # Create and append parser + parser = _HTTPArgumentParser() + self._parsers[key] = parser + + # Return the created parser + return parser + + def parse_args(self, args, route, **kwargs): + """Parse arguments + + Keyword arguments: + - route -- The action route (e.g. 'GET /' ) + + """ + # Retrieve the parser for the route + if route not in self.routes: + raise MoulinetteError(22, "No parser for '%s %s' found" % key) + + return self._parsers[route].parse_args(args) + +""" +The dict of interfaces names and their associated parser class. + +""" +actionsmap_parsers = { + 'api': APIAMapParser, + 'cli': CLIAMapParser +} + + +## Extra parameters ---------------------------------------------------- + +# Extra parameters definition + +class _ExtraParameter(object): + """ + Argument parser for an extra parameter. + + It is a pure virtual class that each extra parameter classes must + implement. + + """ + def __init__(self, iface): + # TODO: Add conn argument which contains authentification object + self.iface = iface + + + ## Required variables + # Each extra parameters classes must overwrite these variables. + + """The extra parameter name""" + name = None + + + ## Optional variables + # Each extra parameters classes can overwrite these variables. + + """A list of interface for which the parameter doesn't apply""" + skipped_iface = {} + + + ## Virtual methods + # Each extra parameters classes can implement these methods. + + def __call__(self, parameter, arg_name, arg_value): + """ + Parse the argument + + Keyword arguments: + - parameter -- The value of this parameter for the action + - arg_name -- The argument name + - arg_value -- The argument value + + Returns: + The new argument value + + """ + return arg_value + + @staticmethod + def validate(value, arg_name): + """ + Validate the parameter value for an argument + + Keyword arguments: + - value -- The parameter value + - arg_name -- The argument name + + Returns: + The validated parameter value + + """ + return value + +class AskParameter(_ExtraParameter): + """ + Ask for the argument value if possible and needed. + + The value of this parameter corresponds to the message to display + when asking the argument value. + + """ + name = 'ask' + skipped_iface = { 'api' } + + def __call__(self, message, arg_name, arg_value): + # TODO: Fix asked arguments ordering + if arg_value: + return arg_value + + # Ask for the argument value + ret = raw_input(colorize(message + ': ', 'cyan')) + return ret + + @classmethod + def validate(klass, value, arg_name): + # Allow boolean or empty string + if isinstance(value, bool) or (isinstance(value, str) and not value): + logging.debug("value of '%s' extra parameter for '%s' argument should be a string" \ + % (klass.name, arg_name)) + value = arg_name + elif not isinstance(value, str): + raise TypeError("Invalid type of '%s' extra parameter for '%s' argument" \ + % (klass.name, arg_name)) + return value + +class PasswordParameter(AskParameter): + """ + Ask for the password argument value if possible and needed. + + The value of this parameter corresponds to the message to display + when asking the password. + + """ + name = 'password' + + def __call__(self, message, arg_name, arg_value): + if arg_value: + return arg_value + + # Ask for the password + pwd1 = getpass.getpass(colorize(message + ': ', 'cyan')) + pwd2 = getpass.getpass(colorize('Retype ' + message + ': ', 'cyan')) + if pwd1 != pwd2: + raise MoulinetteError(22, _("Passwords don't match")) + return pwd1 + +class PatternParameter(_ExtraParameter): + """ + Check if the argument value match a pattern. + + The value of this parameter corresponds to a list of the pattern and + the message to display if it doesn't match. + + """ + name = 'pattern' + + def __call__(self, arguments, arg_name, arg_value): + pattern = arguments[0] + message = arguments[1] + + if arg_value is not None and not re.match(pattern, arg_value): + raise MoulinetteError(22, message) + return arg_value + + @staticmethod + def validate(value, arg_name): + # Tolerate string type + if isinstance(value, str): + logging.warning("value of 'pattern' extra parameter for '%s' argument should be a list" % arg_name) + value = [value, _("'%s' argument is not matching the pattern") % arg_name] + elif not isinstance(value, list) or len(value) != 2: + raise TypeError("Invalid type of 'pattern' extra parameter for '%s' argument" % arg_name) + return value + +""" +The list of available extra parameters classes. It will keep to this list +order on argument parsing. + +""" +extraparameters_list = {AskParameter, PasswordParameter, PatternParameter} + +# Extra parameters argument Parser + +class ExtraArgumentParser(object): + """ + Argument validator and parser for the extra parameters. + + Keyword arguments: + - iface -- The running interface """ def __init__(self, iface): @@ -78,69 +535,64 @@ class ExtraParser(object): return arg_value -## Main class +## Main class ---------------------------------------------------------- class ActionsMap(object): - """ - Validate and process action defined into the actions map. + """Validate and process actions defined into an actions map - The actions map defines features and their usage of the main + The actions map defines the features and their usage of the main application. It is composed by categories which contain one or more action(s). Moreover, the action can have specific argument(s). - Keyword arguments: + This class allows to manipulate one or several actions maps + associated to a namespace. If no namespace is given, it will load + all available namespaces. - - interface -- Interface type that requires the actions map. - Possible value is one of: + Keyword arguments: + - interface -- The type of interface which needs the actions map. + Possible values are: - 'cli' for the command line interface - 'api' for an API usage (HTTP requests) - + - namespaces -- The list of namespaces to use - use_cache -- False if it should parse the actions map file instead of using the cached one. """ - def __init__(self, interface, use_cache=True): + def __init__(self, interface, namespaces=[], use_cache=True): self.use_cache = use_cache try: # Retrieve the interface parser - mod = __import__('interface.%s' % interface, - globals=globals(), level=1, - fromlist=['actionsmap_parser']) - parser = getattr(mod, 'actionsmap_parser') - except (AttributeError, ImportError): + self._parser_class = actionsmap_parsers[interface] + except KeyError: raise MoulinetteError(22, _("Invalid interface '%s'" % interface)) - else: - self._parser_class = parser logging.debug("initializing ActionsMap for the '%s' interface" % interface) + if len(namespaces) == 0: + namespaces = self.get_namespaces() actionsmaps = {} - namespaces = self.get_actionsmap_namespaces() - if curr_namespace and curr_namespace in namespaces: - namespaces = [curr_namespace] # Iterate over actions map namespaces for n in namespaces: logging.debug("loading '%s' actions map namespace" % n) if use_cache: - # Attempt to load cache if it exists - cache_file = '%s/%s.pkl' % (pkg.cachedir('actionsmap'), n) - if os.path.isfile(cache_file): - with open(cache_file, 'r') as f: + try: + # Attempt to load cache + with open('%s/actionsmap/%s.pkl' % (pkg.cachedir, n)) as f: actionsmaps[n] = pickle.load(f) - else: + # TODO: Switch to python3 and catch proper exception + except IOError: self.use_cache = False actionsmaps = self.generate_cache(namespaces) break else: - am_file = '%s/%s.yml' % (pkg.datadir('actionsmap'), n) - with open(am_file, 'r') as f: + with open('%s/actionsmap/%s.yml' % (pkg.datadir, n)) as f: actionsmaps[n] = yaml.load(f) # Generate parsers - self.extraparser = ExtraParser(interface) + self.extraparser = ExtraArgumentParser(interface) self.parser = self._construct_parser(actionsmaps) def process(self, args, **kwargs): @@ -152,6 +604,8 @@ class ActionsMap(object): - **kwargs -- Additional interface arguments """ + # Check moulinette status + # Parse arguments arguments = vars(self.parser.parse_args(args, **kwargs)) arguments = self._parse_extra_parameters(arguments) @@ -170,11 +624,12 @@ class ActionsMap(object): else: # Process the action return func(**arguments) + return {} @staticmethod - def get_actionsmap_namespaces(): + def get_namespaces(): """ - Retrieve actions map namespaces from a given path + Retrieve available actions map namespaces Returns: A list of available namespaces @@ -182,7 +637,7 @@ class ActionsMap(object): """ namespaces = [] - for f in os.listdir(pkg.datadir('actionsmap')): + for f in os.listdir('%s/actionsmap' % pkg.datadir): if f.endswith('.yml'): namespaces.append(f[:-4]) return namespaces @@ -201,20 +656,19 @@ class ActionsMap(object): """ actionsmaps = {} if not namespaces: - namespaces = klass.get_actionsmap_namespaces() + namespaces = klass.get_namespaces() # Iterate over actions map namespaces for n in namespaces: logging.debug("generating cache for '%s' actions map namespace" % n) # Read actions map from yaml file - am_file = pkg.datafile('actionsmap/%s.yml' % n) + am_file = '%s/actionsmap/%s.yml' % (pkg.datadir, n) with open(am_file, 'r') as f: actionsmaps[n] = yaml.load(f) # Cache actions map into pickle file - cache_file = pkg.cachefile('actionsmap/%s.pkl' % n, make_dir=True) - with open(cache_file, 'w') as f: + with pkg.open_cache('%s.pkl' % n, subdir='actionsmap') as f: pickle.dump(actionsmaps[n], f) return actionsmaps @@ -291,41 +745,41 @@ class ActionsMap(object): for n, actionsmap in actionsmaps.items(): if 'general_arguments' in actionsmap: # Parse general arguments - if top_parser.parse_general: + if top_parser.parse_general_arguments: parser = top_parser.add_general_parser() for an, ap in actionsmap['general_arguments'].items(): - if 'version' in ap: - ap['version'] = ap['version'].replace('%version%', - __version__) + # Replace version number + version = ap.get('version', None) + if version: + ap['version'] = version.replace('%version%', + __version__) argname = top_parser.format_arg_name(an, ap.pop('full', None)) parser.add_argument(*argname, **ap) del actionsmap['general_arguments'] # Parse categories for cn, cp in actionsmap.items(): - if 'actions' not in cp: + try: + actions = cp.pop('actions') + except KeyError: continue - actions = cp.pop('actions') # Add category parser - if top_parser.parse_category: - cat_parser = top_parser.add_category_parser(cn, **cp) - else: - cat_parser = top_parser + cat_parser = top_parser.add_category_parser(cn, **cp) # Parse actions - if not top_parser.parse_action: - continue for an, ap in actions.items(): arguments = ap.pop('arguments', {}) # Add action parser parser = cat_parser.add_action_parser(an, **ap) - if not parser: - continue - # Store action information - parser.set_defaults(_info=(n, cn, an)) + try: + # Store action information + parser.set_defaults(_info=(n, cn, an)) + except AttributeError: + # No parser for the action + break # Add action arguments for argn, argp in arguments.items(): diff --git a/src/moulinette/core.py b/src/moulinette/core.py index 62fe34fe..5dbbe557 100644 --- a/src/moulinette/core.py +++ b/src/moulinette/core.py @@ -5,115 +5,126 @@ import sys import gettext from .helpers import colorize +# Package manipulation ------------------------------------------------- + +def install_i18n(namespace=None): + """Install internationalization + + Install translation based on the package's default gettext domain or + on 'namespace' if provided. + + Keyword arguments: + - namespace -- The namespace to initialize i18n for + + """ + if namespace: + try: + t = gettext.translation(namespace, pkg.localedir) + except IOError: + # TODO: Log error + return + else: + t.install() + else: + gettext.install('moulinette', pkg.localedir) + class Package(object): - """Package representation and easy access + """Package representation and easy access methods Initialize directories and variables for the package and give them easy access. Keyword arguments: - - prefix -- The installation prefix - - libdir -- The library directory; usually, this would be - prefix + '/lib' (or '/lib64') when installed - - cachedir -- The cache directory; usually, this would be - '/var/cache' when installed - - destdir -- The destination prefix only if it's an installation - - 'prefix' and 'libdir' arguments should be empty in order to run - package from source. + - _from_source -- Either the package is running from source or + not (only for debugging) """ - def __init__(self, prefix, libdir, cachedir, destdir=None): - if not prefix and not libdir: - # Running from source directory + def __init__(self, _from_source=False): + if _from_source: + import sys basedir = os.path.abspath(os.path.dirname(sys.argv[0]) +'/../') - self._datadir = os.path.join(basedir, 'data') - self._libdir = os.path.join(basedir, 'src') - self._cachedir = cachedir + + # Set local directories + self._datadir = '%s/data' % basedir + self._libdir = '%s/lib' % basedir + self._localedir = '%s/po' % basedir + self._cachedir = '%s/cache' % basedir else: - self._datadir = os.path.join(prefix, 'share/moulinette') - self._libdir = os.path.join(libdir, 'moulinette') - self._cachedir = os.path.join(cachedir, 'moulinette') + import package - # Append library path to python's path - sys.path.append(self._libdir) - self._destdir = destdir or None + # Set system directories + self._datadir = package.datadir + self._libdir = package.libdir + self._localedir = package.localedir + self._cachedir = package.cachedir + + def __setattr__(self, name, value): + if name[0] == '_' and self.__dict__.has_key(name): + # Deny reassignation of package directories + raise TypeError("cannot reassign constant '%s'") + self.__dict__[name] = value - ## Easy access to directories and files + ## Easy access to package directories - def datadir(self, subdir=None, **kwargs): - """Return the path to a data directory""" - return self.get_dir(self._datadir, subdir, **kwargs) + @property + def datadir(self): + """Return the data directory of the package""" + return self._datadir - def datafile(self, filename, **kwargs): - """Return the path to a data file""" - return self.get_file(self._datadir, filename, **kwargs) + @property + def libdir(self): + """Return the lib directory of the package""" + return self._libdir - def libdir(self, subdir=None, **kwargs): - """Return the path to a lib directory""" - return self.get_dir(self._libdir, subdir, **kwargs) + @property + def localedir(self): + """Return the locale directory of the package""" + return self._localedir - def libfile(self, filename, **kwargs): - """Return the path to a lib file""" - return self.get_file(self._libdir, filename, **kwargs) - - def cachedir(self, subdir=None, **kwargs): - """Return the path to a cache directory""" - return self.get_dir(self._cachedir, subdir, **kwargs) - - def cachefile(self, filename, **kwargs): - """Return the path to a cache file""" - return self.get_file(self._cachedir, filename, **kwargs) + @property + def cachedir(self): + """Return the cache directory of the package""" + return self._cachedir - ## Standard methods + ## Additional methods - def get_dir(self, basedir, subdir=None, make_dir=False): - """Get a directory path + def get_cachedir(self, subdir='', make_dir=True): + """Get the path to a cache directory - Return a path composed by a base directory and an optional - subdirectory. The path will be created if needed. + Return the path to the cache directory from an optional + subdirectory and create it if needed. Keyword arguments: - - basedir -- The base directory - - subdir -- An optional subdirectory - - make_dir -- True if it should create needed directory + - subdir -- A cache subdirectory + - make_dir -- False to not make directory if it not exists """ - # Retrieve path - path = basedir - if self._destdir: - path = os.path.join(self._destdir, path) - if subdir: - path = os.path.join(path, subdir) + path = os.path.join(self.cachedir, subdir) - # Create directory if make_dir and not os.path.isdir(path): os.makedirs(path) return path - def get_file(self, basedir, filename, **kwargs): - """Get a file path + def open_cache(self, filename, subdir='', mode='w'): + """Open a cache file and return a stream - Return the path of the filename in the specified directory. This - directory will be created if needed. + Attempt to open in 'mode' the cache file 'filename' from the + default cache directory and in the subdirectory 'subdir' if + given. Directories are created if needed and a stream is + returned if the file can be written. Keyword arguments: - - basedir -- The base directory of the file - - filename -- The filename or a path relative to basedir - - **kwargs -- Additional arguments for Package.get_dir + - filename -- The cache filename + - subdir -- A subdirectory which contains the file + - mode -- The mode in which the file is opened """ - # Check for a directory in filename - subdir = os.path.dirname(filename) or None - if subdir: - filename = os.path.basename(filename) + return open('%s/%s' % (self.get_cachedir(subdir), filename), mode) - # Get directory path - dirpath = self.get_dir(basedir, subdir, **kwargs) - return os.path.join(dirpath, filename) +# Moulinette core classes ---------------------------------------------- class MoulinetteError(Exception): """Moulinette base exception diff --git a/src/moulinette/extra/parameters.py b/src/moulinette/extra/parameters.py deleted file mode 100644 index f1bf0ba9..00000000 --- a/src/moulinette/extra/parameters.py +++ /dev/null @@ -1,159 +0,0 @@ -# -*- coding: utf-8 -*- - -import getpass -import re -import logging - -from .. import MoulinetteError -from ..helpers import colorize - -class _ExtraParameter(object): - """ - Argument parser for an extra parameter. - - It is a pure virtual class that each extra parameter classes must - implement. - - """ - def __init__(self, iface): - # TODO: Add conn argument which contains authentification object - self.iface = iface - - - ## Required variables - # Each extra parameters classes must overwrite these variables. - - """The extra parameter name""" - name = None - - - ## Optional variables - # Each extra parameters classes can overwrite these variables. - - """A list of interface for which the parameter doesn't apply""" - skipped_iface = {} - - - ## Virtual methods - # Each extra parameters classes can implement these methods. - - def __call__(self, parameter, arg_name, arg_value): - """ - Parse the argument - - Keyword arguments: - - parameter -- The value of this parameter for the action - - arg_name -- The argument name - - arg_value -- The argument value - - Returns: - The new argument value - - """ - return arg_value - - @staticmethod - def validate(value, arg_name): - """ - Validate the parameter value for an argument - - Keyword arguments: - - value -- The parameter value - - arg_name -- The argument name - - Returns: - The validated parameter value - - """ - return value - - -## Extra parameters definitions - -class AskParameter(_ExtraParameter): - """ - Ask for the argument value if possible and needed. - - The value of this parameter corresponds to the message to display - when asking the argument value. - - """ - name = 'ask' - skipped_iface = { 'api' } - - def __call__(self, message, arg_name, arg_value): - # TODO: Fix asked arguments ordering - if arg_value: - return arg_value - - # Ask for the argument value - ret = raw_input(colorize(message + ': ', 'cyan')) - return ret - - @classmethod - def validate(klass, value, arg_name): - # Allow boolean or empty string - if isinstance(value, bool) or (isinstance(value, str) and not value): - logging.debug("value of '%s' extra parameter for '%s' argument should be a string" \ - % (klass.name, arg_name)) - value = arg_name - elif not isinstance(value, str): - raise TypeError("Invalid type of '%s' extra parameter for '%s' argument" \ - % (klass.name, arg_name)) - return value - -class PasswordParameter(AskParameter): - """ - Ask for the password argument value if possible and needed. - - The value of this parameter corresponds to the message to display - when asking the password. - - """ - name = 'password' - - def __call__(self, message, arg_name, arg_value): - if arg_value: - return arg_value - - # Ask for the password - pwd1 = getpass.getpass(colorize(message + ': ', 'cyan')) - pwd2 = getpass.getpass(colorize('Retype ' + message + ': ', 'cyan')) - if pwd1 != pwd2: - raise MoulinetteError(22, _("Passwords don't match")) - return pwd1 - -class PatternParameter(_ExtraParameter): - """ - Check if the argument value match a pattern. - - The value of this parameter corresponds to a list of the pattern and - the message to display if it doesn't match. - - """ - name = 'pattern' - - def __call__(self, arguments, arg_name, arg_value): - pattern = arguments[0] - message = arguments[1] - - if arg_value is not None and not re.match(pattern, arg_value): - raise MoulinetteError(22, message) - return arg_value - - @staticmethod - def validate(value, arg_name): - # Tolerate string type - if isinstance(value, str): - logging.warning("value of 'pattern' extra parameter for '%s' argument should be a list" % arg_name) - value = [value, _("'%s' argument is not matching the pattern") % arg_name] - elif not isinstance(value, list) or len(value) != 2: - raise TypeError("Invalid type of 'pattern' extra parameter for '%s' argument" % arg_name) - return value - -""" -The list of available extra parameters classes. It will keep to this list -order on argument parsing. - -""" -extraparameters_list = {AskParameter, PasswordParameter, PatternParameter} diff --git a/src/moulinette/helpers.py b/src/moulinette/helpers.py index 69efaf1f..d632d03d 100644 --- a/src/moulinette/helpers.py +++ b/src/moulinette/helpers.py @@ -1,25 +1,15 @@ # -*- coding: utf-8 -*- import os -import sys -try: - import ldap -except ImportError: - sys.stderr.write('Error: Yunohost CLI Require LDAP lib\n') - sys.stderr.write('apt-get install python-ldap\n') - sys.exit(1) +import ldap import ldap.modlist as modlist -import yaml import json import re import getpass import random import string -import argparse import gettext import getpass -if not __debug__: - import traceback win = [] diff --git a/src/moulinette/interface/__init__.py b/src/moulinette/interface/__init__.py old mode 100755 new mode 100644 index 3a99535f..e69de29b --- a/src/moulinette/interface/__init__.py +++ b/src/moulinette/interface/__init__.py @@ -1,114 +0,0 @@ -# -*- coding: utf-8 -*- - -class BaseParser(object): - """Actions map's base Parser - - Each interfaces must implement a parser class derived from this - class. It is used to parse the main parts of the actions map (i.e. - general arguments, categories and actions). - - """ - - ## Optional variables - # Each parser classes can overwrite these variables. - - """Either it will parse general arguments, or not""" - parse_general = True - - """Either it will parse categories, or not""" - parse_category = True - - """Either it will parse actions, or not""" - parse_action = True - - - ## Virtual methods - # Each parser classes can implement these methods. - - @staticmethod - def format_arg_name(name, full): - """Format argument name - - Format agument name depending on its 'full' parameters and return - a list to use it as option string for the argument parser. - - Keyword arguments: - - name -- The argument name - - full -- The argument's 'full' parameter - - Returns: - A list of option strings - - """ - raise NotImplementedError("derived class '%s' must override this method" % \ - self.__class__.__name__) - - def add_general_parser(self, **kwargs): - """Add a parser for general arguments - - Create and return an argument parser for general arguments. - - Returns: - An ArgumentParser based object - - """ - if not self.parse_general: - msg = "doesn't parse general arguments" - else: - msg = "must override this method" - raise NotImplementedError("derived class '%s' %s" % \ - (self.__class__.__name__, msg)) - - def add_category_parser(self, name, **kwargs): - """Add a parser for a category - - Create a new category and return a parser for it. - - Keyword arguments: - - name -- The category name - - Returns: - A BaseParser based object - - """ - if not self.parse_categories: - msg = "doesn't parse categories" - else: - msg = "must override this method" - raise NotImplementedError("derived class '%s' %s" % \ - (self.__class__.__name__, msg)) - - def add_action_parser(self, name, **kwargs): - """Add a parser for an action - - Create a new action and return an argument parser for it. - - Keyword arguments: - - name -- The action name - - Returns: - An ArgumentParser based object - - """ - if not self.parse_general: - msg = "doesn't parse actions" - else: - msg = "must override this method" - raise NotImplementedError("derived class '%s' %s" % \ - (self.__class__.__name__, msg)) - - def parse_args(self, args, **kwargs): - """Parse arguments - - Convert argument variables to objects and assign them as - attributes of the namespace. - - Keyword arguments: - - args -- Arguments string or dict (TODO) - - Returns: - The populated namespace - - """ - raise NotImplementedError("derived class '%s' must override this method" % \ - self.__class__.__name__) diff --git a/src/moulinette/interface/api.py b/src/moulinette/interface/api.py index 6c959c9f..60e72228 100644 --- a/src/moulinette/interface/api.py +++ b/src/moulinette/interface/api.py @@ -1,160 +1,12 @@ # -*- coding: utf-8 -*- -import re -import argparse -import os.path from bottle import Bottle, request, response, HTTPResponse from beaker.middleware import SessionMiddleware -from . import BaseParser -from .. import MoulinetteError +from ..core import MoulinetteError from ..helpers import YunoHostError, YunoHostLDAP -## API arguments Parser - -class _HTTPArgumentParser(object): - """Argument parser for HTTP requests - - Object for parsing HTTP requests into Python objects. It is based - on argparse.ArgumentParser class and implements some of its methods. - - """ - def __init__(self): - # Initialize the ArgumentParser object - self._parser = argparse.ArgumentParser(usage='', - prefix_chars='@', - add_help=False) - self._parser.error = self._error - - self._positional = [] # list(arg_name) - self._optional = {} # dict({arg_name: option_strings}) - - def set_defaults(self, **kwargs): - return self._parser.set_defaults(**kwargs) - - def get_default(self, dest): - return self._parser.get_default(dest) - - def add_argument(self, *args, **kwargs): - action = self._parser.add_argument(*args, **kwargs) - - # Append newly created action - if len(action.option_strings) == 0: - self._positional.append(action.dest) - else: - self._optional[action.dest] = action.option_strings - - return action - - def parse_args(self, args): - arg_strings = [] - - ## Append an argument to the current one - def append(arg_strings, value, option_string=None): - # TODO: Process list arguments - if isinstance(value, bool): - # Append the option string only - if option_string is not None: - arg_strings.append(option_string) - elif isinstance(value, str): - if option_string is not None: - arg_strings.append(option_string) - arg_strings.append(value) - else: - arg_strings.append(value) - - return arg_strings - - # Iterate over positional arguments - for dest in self._positional: - if dest in args: - arg_strings = append(arg_strings, args[dest]) - - # Iterate over optional arguments - for dest, opt in self._optional.items(): - if dest in args: - arg_strings = append(arg_strings, args[dest], opt[0]) - return self._parser.parse_args(arg_strings) - - def _error(self, message): - # TODO: Raise a proper exception - raise MoulinetteError(1, message) - -class APIParser(BaseParser): - """Actions map's API Parser - - """ - parse_category = False - parse_general = False - - def __init__(self): - self._parsers = {} # dict({(method, path): _HTTPArgumentParser}) - - @property - def routes(self): - """Get current routes""" - return self._parsers.keys() - - - ## Implement virtual methods - - @staticmethod - def format_arg_name(name, full): - if name[0] != '-': - return [name] - if full: - return [full.replace('--', '@', 1)] - if name.startswith('--'): - return [name.replace('--', '@', 1)] - return [name.replace('-', '@', 1)] - - def add_action_parser(self, name, api=None, **kwargs): - """Add a parser for an action - - Keyword arguments: - - api -- The action route (e.g. 'GET /' ) - - Returns: - A new _HTTPArgumentParser object for the route - - """ - if not api: - return None - - # Validate action route - m = re.match('(GET|POST|PUT|DELETE) (/\S+)', api) - if not m: - return None - - # Check if a parser already exists for the route - key = (m.group(1), m.group(2)) - if key in self.routes: - raise ValueError("A parser for '%s' already exists" % key) - - # Create and append parser - parser = _HTTPArgumentParser() - self._parsers[key] = parser - - # Return the created parser - return parser - - def parse_args(self, args, route, **kwargs): - """Parse arguments - - Keyword arguments: - - route -- The action route (e.g. 'GET /' ) - - """ - # Retrieve the parser for the route - if route not in self.routes: - raise MoulinetteError(22, "No parser for '%s %s' found" % key) - - return self._parsers[route].parse_args(args) - -actionsmap_parser = APIParser - - -## API moulinette interface +# API moulinette interface --------------------------------------------- class _APIAuthPlugin(object): """ @@ -189,7 +41,7 @@ class _APIAuthPlugin(object): session_opts = { 'session.type': 'file', 'session.cookie_expires': True, - 'session.data_dir': pkg.cachedir('session', make_dir=True), + 'session.data_dir': pkg.get_cachedir('session'), 'session.secure': True } self._app = SessionMiddleware(app, session_opts) @@ -361,11 +213,11 @@ class MoulinetteAPI(object): """ if category is None: - with open(pkg.datafile('doc/resources.json')) as f: + with open('%s/doc/resources.json' % pkg.datadir) as f: return f.read() try: - with open(pkg.datafile('doc/%s.json' % category)) as f: + with open('%s/doc/%s.json' % (pkg.datadir, category)) as f: return f.read() except IOError: return 'unknown' diff --git a/src/moulinette/interface/cli.py b/src/moulinette/interface/cli.py index c4be4c57..e10b0a25 100644 --- a/src/moulinette/interface/cli.py +++ b/src/moulinette/interface/cli.py @@ -1,35 +1,5 @@ # -*- coding: utf-8 -*- -import argparse -from . import BaseParser - -## CLI arguments Parser - -class CLIParser(BaseParser): - """Actions map's CLI Parser - - """ - def __init__(self, parser=None): - self._parser = parser or argparse.ArgumentParser() - self._subparsers = self._parser.add_subparsers() - - @staticmethod - def format_arg_name(name, full): - if name[0] == '-' and full: - return [name, full] - return [name] - - def add_general_parser(self, **kwargs): - return self._parser - - def add_category_parser(self, name, category_help=None, **kwargs): - parser = self._subparsers.add_parser(name, help=category_help) - return CLIParser(parser) - - def add_action_parser(self, name, action_help, **kwargs): - return self._subparsers.add_parser(name, help=action_help) - - def parse_args(self, args, **kwargs): - return self._parser.parse_args(args) - -actionsmap_parser = CLIParser +class MoulinetteCLI(object): + # TODO: Implement this class + pass diff --git a/src/moulinette/package.py b/src/moulinette/package.py new file mode 100644 index 00000000..e4326da3 --- /dev/null +++ b/src/moulinette/package.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +# Public constants defined during build + +"""Package's data directory (e.g. /usr/share/moulinette)""" +datadir = '/usr/share/moulinette' + +"""Package's library directory (e.g. /usr/lib/moulinette)""" +libdir = '/usr/lib/moulinette' + +"""Locale directory for the package (e.g. /usr/lib/moulinette/locale)""" +localedir = '/usr/lib/moulinette/locale' + +"""Cache directory for the package (e.g. /var/cache/moulinette)""" +cachedir = '/var/cache/moulinette' diff --git a/src/moulinette/package.py.in b/src/moulinette/package.py.in new file mode 100644 index 00000000..a72b5486 --- /dev/null +++ b/src/moulinette/package.py.in @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +# Public constants defined during build + +"""Package's data directory (e.g. /usr/share/moulinette)""" +datadir = %PKGDATADIR% + +"""Package's library directory (e.g. /usr/lib/moulinette)""" +libdir = %PKGLIBDIR% + +"""Locale directory for the package (e.g. /usr/lib/moulinette/locale)""" +localedir = %PKGLOCALEDIR% + +"""Cache directory for the package (e.g. /var/cache/moulinette)""" +cachedir = %PKGCACHEDIR% diff --git a/src/yunohost/__init__.py b/src/yunohost/__init__.py deleted file mode 100755 index e69de29b..00000000 From 66f60381e4782b4bcf7f8a732f9df7d031bb4907 Mon Sep 17 00:00:00 2001 From: Jerome Lebleu Date: Tue, 11 Mar 2014 15:03:33 +0100 Subject: [PATCH 07/18] Implement a moulinette lock and revisit extra params a bit --- src/moulinette/__init__.py | 15 ----- src/moulinette/actionsmap.py | 116 ++++++++++++----------------------- src/moulinette/core.py | 69 +++++++++++++++++++++ 3 files changed, 109 insertions(+), 91 deletions(-) diff --git a/src/moulinette/__init__.py b/src/moulinette/__init__.py index 0bc0ff72..56e9c582 100755 --- a/src/moulinette/__init__.py +++ b/src/moulinette/__init__.py @@ -98,26 +98,11 @@ def cli(namespaces, args, use_cache=True): instead of using the cached one """ - import os from .actionsmap import ActionsMap from .helpers import pretty_print_dict - lock_file = '/var/run/moulinette.lock' - - # TODO: Move the lock checking into the ActionsMap class - # Check the lock - if os.path.isfile(lock_file): - raise MoulinetteError(1, _("The moulinette is already running")) - - # Create a lock - with open(lock_file, 'w') as f: pass - os.system('chmod 400 '+ lock_file) - try: amap = ActionsMap('cli', namespaces, use_cache) pretty_print_dict(amap.process(args)) except KeyboardInterrupt, EOFError: raise MoulinetteError(125, _("Interrupted")) - finally: - # Remove the lock - os.remove(lock_file) diff --git a/src/moulinette/actionsmap.py b/src/moulinette/actionsmap.py index db07318b..5fef75b7 100644 --- a/src/moulinette/actionsmap.py +++ b/src/moulinette/actionsmap.py @@ -10,7 +10,7 @@ from collections import OrderedDict import logging from . import __version__ -from .core import MoulinetteError +from .core import MoulinetteError, MoulinetteLock ## Interfaces' Actions map Parser -------------------------------------- @@ -441,10 +441,9 @@ class PatternParameter(_ExtraParameter): name = 'pattern' def __call__(self, arguments, arg_name, arg_value): - pattern = arguments[0] - message = arguments[1] + pattern, message = (arguments[0], arguments[1]) - if arg_value is not None and not re.match(pattern, arg_value): + if not re.match(pattern, arg_value or ''): raise MoulinetteError(22, message) return arg_value @@ -496,12 +495,13 @@ class ExtraArgumentParser(object): """ # Iterate over parameters to validate for p, v in parameters.items(): - # Remove unknow parameters - if p not in self.extra.keys(): + klass = self.extra.get(p, None) + if not klass: + # Remove unknown parameters del parameters[p] - - # Validate parameter value - parameters[p] = self.extra[p].validate(v, arg_name) + else: + # Validate parameter value + parameters[p] = klass.validate(v, arg_name) return parameters @@ -595,36 +595,38 @@ class ActionsMap(object): self.extraparser = ExtraArgumentParser(interface) self.parser = self._construct_parser(actionsmaps) - def process(self, args, **kwargs): + def process(self, args, timeout=0, **kwargs): """ Parse arguments and process the proper action Keyword arguments: - args -- The arguments to parse + - timeout -- The time period before failing if the lock + cannot be acquired for the action - **kwargs -- Additional interface arguments """ - # Check moulinette status - # Parse arguments arguments = vars(self.parser.parse_args(args, **kwargs)) - arguments = self._parse_extra_parameters(arguments) + for an, parameters in (arguments.pop('_extra', {})).items(): + arguments[an] = self.extraparser.parse(an, arguments[an], parameters) # Retrieve action information namespace, category, action = arguments.pop('_info') func_name = '%s_%s' % (category, action) - try: - mod = __import__('%s.%s' % (namespace, category), - globals=globals(), level=0, - fromlist=[func_name]) - func = getattr(mod, func_name) - except (AttributeError, ImportError): - raise MoulinetteError(168, _('Function is not defined')) - else: - # Process the action - return func(**arguments) - return {} + # Lock the moulinette for the namespace + with MoulinetteLock(namespace, timeout): + try: + mod = __import__('%s.%s' % (namespace, category), + globals=globals(), level=0, + fromlist=[func_name]) + func = getattr(mod, func_name) + except (AttributeError, ImportError): + raise MoulinetteError(168, _('Function is not defined')) + else: + # Process the action + return func(**arguments) @staticmethod def get_namespaces(): @@ -674,57 +676,7 @@ class ActionsMap(object): return actionsmaps - ## Private class and methods - - def _store_extra_parameters(self, parser, arg_name, arg_extra): - """ - Store extra parameters for a given argument - - Keyword arguments: - - parser -- Parser object for the arguments - - arg_name -- Argument name - - arg_extra -- Argument extra parameters - - Returns: - The parser object - - """ - if arg_extra: - # Retrieve current extra parameters dict - extra = parser.get_default('_extra') - if not extra or not isinstance(extra, dict): - extra = {} - - if not self.use_cache: - # Validate extra parameters for the argument - extra[arg_name] = self.extraparser.validate(arg_name, arg_extra) - else: - extra[arg_name] = arg_extra - parser.set_defaults(_extra=extra) - - return parser - - def _parse_extra_parameters(self, args): - """ - Parse arguments with their extra parameters - - Keyword arguments: - - args -- A dict of all arguments - - Return: - The parsed arguments dict - - """ - # Retrieve extra parameters for the arguments - extra = args.pop('_extra', None) - if not extra: - return args - - # Validate extra parameters for each arguments - for an, parameters in extra.items(): - args[an] = self.extraparser.parse(an, args[an], parameters) - - return args + ## Private methods def _construct_parser(self, actionsmaps): """ @@ -738,6 +690,12 @@ class ActionsMap(object): An interface relevant's parser object """ + # Define setter for extra parameters + if not self.use_cache: + _set_extra = lambda an, e: self.extraparser.validate(an, e) + else: + _set_extra = lambda an, e: e + # Instantiate parser top_parser = self._parser_class() @@ -787,6 +745,12 @@ class ActionsMap(object): extra = argp.pop('extra', None) arg = parser.add_argument(*name, **argp) - parser = self._store_extra_parameters(parser, arg.dest, extra) + if not extra: + continue + + # Store extra parameters + extras = parser.get_default('_extra') or {} + extras[arg.dest] = _set_extra(arg.dest, extra) + parser.set_defaults(_extra=extras) return top_parser diff --git a/src/moulinette/core.py b/src/moulinette/core.py index 5dbbe557..41e53ed8 100644 --- a/src/moulinette/core.py +++ b/src/moulinette/core.py @@ -2,7 +2,9 @@ import os import sys +import time import gettext + from .helpers import colorize # Package manipulation ------------------------------------------------- @@ -164,3 +166,70 @@ class MoulinetteError(Exception): def colorize(self): return self.__str__(colorized=True) + + +class MoulinetteLock(object): + """Locker for a moulinette instance + + It provides a lock mechanism for a given moulinette instance. It can + be used in a with statement as it has a context-manager support. + + Keyword arguments: + - namespace -- The namespace to lock + - timeout -- The time period before failing if the lock cannot + be acquired + - interval -- The time period before trying again to acquire the + lock + + """ + def __init__(self, namespace, timeout=0, interval=.5): + self.namespace = namespace + self.timeout = timeout + self.interval = interval + + self._lockfile = '/var/run/moulinette_%s.lock' % namespace + self._locked = False + + def acquire(self): + """Attempt to acquire the lock for the moulinette instance + + It will try to write to the lock file only if it doesn't exist. + Otherwise, it will wait and try again until the timeout expires + or the lock file doesn't exist. + + """ + start_time = time.time() + + while True: + if not os.path.isfile(self._lockfile): + # Create the lock file + (open(self._lockfile, 'w')).close() + break + + if (time.time() - start_time) > self.timeout: + raise MoulinetteError(1, _("An instance is already running for '%s'") \ + % self.namespace) + # Wait before checking again + time.sleep(self.interval) + self._locked = True + + def release(self): + """Release the lock of the moulinette instance + + It will delete the lock file if the lock has been acquired. + + """ + if self._locked: + os.unlink(self._lockfile) + self._locked = False + + def __enter__(self): + if not self._locked: + self.acquire() + return self + + def __exit__(self, *args): + self.release() + + def __del__(self): + self.release() From cdcfa2418029fd966009e0a0f291d0c1e7aa8a8b Mon Sep 17 00:00:00 2001 From: Jerome Lebleu Date: Wed, 12 Mar 2014 00:50:09 +0100 Subject: [PATCH 08/18] Standardize arguments in the actions map * Move general_arguments to _global.arguments in the actions map * Introduce global configuration in the actions map (need implementation!) * Standardize arguments addition during the parser construction * Fix action name with '-' --- data/actionsmap/yunohost.yml | 10 ++- src/moulinette/actionsmap.py | 138 +++++++++++++++++------------------ 2 files changed, 75 insertions(+), 73 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 52a62654..351eae68 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -29,12 +29,14 @@ # ########################################################################## -# TODO: Add patern for all this - ############################# -# General args # +# Global parameters # ############################# -general_arguments: +_global: + configuration: + auth: + - api + arguments: -v: full: --version help: Display moulinette version diff --git a/src/moulinette/actionsmap.py b/src/moulinette/actionsmap.py index 5fef75b7..21b73ab6 100644 --- a/src/moulinette/actionsmap.py +++ b/src/moulinette/actionsmap.py @@ -19,26 +19,20 @@ class _AMapParser(object): Each interfaces must implement a parser class derived from this class. It is used to parse the main parts of the actions map (i.e. - general arguments, categories and actions). + global arguments, categories and actions). """ - ## Optional variables - # Each parser classes can overwrite these variables. - - """Either it will parse general arguments, or not""" - parse_general_arguments = True - - ## Virtual methods # Each parser classes can implement these methods. @staticmethod - def format_arg_name(name, full): + def format_arg_names(name, full): """Format argument name - Format agument name depending on its 'full' parameters and return - a list to use it as option string for the argument parser. + Format agument name depending on its 'full' parameter and return + a list of strings which will be used as name or option strings + for the argument parser. Keyword arguments: - name -- The argument name @@ -51,21 +45,17 @@ class _AMapParser(object): raise NotImplementedError("derived class '%s' must override this method" % \ self.__class__.__name__) - def add_general_parser(self, **kwargs): - """Add a parser for general arguments + def add_global_parser(self, **kwargs): + """Add a parser for global arguments - Create and return an argument parser for general arguments. + Create and return an argument parser for global arguments. Returns: An ArgumentParser based object """ - if not self.parse_general_arguments: - msg = "doesn't parse general arguments" - else: - msg = "must override this method" - raise NotImplementedError("derived class '%s' %s" % \ - (self.__class__.__name__, msg)) + raise NotImplementedError("derived class '%s' must override this method" % \ + self.__class__.__name__) def add_category_parser(self, name, **kwargs): """Add a parser for a category @@ -124,12 +114,12 @@ class CLIAMapParser(_AMapParser): self._subparsers = self._parser.add_subparsers() @staticmethod - def format_arg_name(name, full): + def format_arg_names(name, full): if name[0] == '-' and full: return [name, full] return [name] - def add_general_parser(self, **kwargs): + def add_global_parser(self, **kwargs): return self._parser def add_category_parser(self, name, category_help=None, **kwargs): @@ -145,7 +135,7 @@ class CLIAMapParser(_AMapParser): parser = self._subparsers.add_parser(name, help=category_help) return self.__class__(parser) - def add_action_parser(self, name, action_help, **kwargs): + def add_action_parser(self, name, action_help=None, **kwargs): """Add a parser for an action Keyword arguments: @@ -234,7 +224,6 @@ class APIAMapParser(_AMapParser): """Actions map's API Parser """ - parse_general_arguments = False def __init__(self): self._parsers = {} # dict({(method, path): _HTTPArgumentParser}) @@ -248,7 +237,7 @@ class APIAMapParser(_AMapParser): ## Implement virtual methods @staticmethod - def format_arg_name(name, full): + def format_arg_names(name, full): if name[0] != '-': return [name] if full: @@ -257,6 +246,9 @@ class APIAMapParser(_AMapParser): return [name.replace('--', '@', 1)] return [name.replace('-', '@', 1)] + def add_global_parser(self, **kwargs): + raise AttributeError("global arguments are not managed") + def add_category_parser(self, name, **kwargs): return self @@ -271,17 +263,17 @@ class APIAMapParser(_AMapParser): """ if not api: - return None + raise AttributeError("the action '%s' doesn't provide api access" % name) # Validate action route m = re.match('(GET|POST|PUT|DELETE) (/\S+)', api) if not m: - return None + raise ValueError("the action '%s' doesn't provide api access" % name) # Check if a parser already exists for the route key = (m.group(1), m.group(2)) if key in self.routes: - raise ValueError("A parser for '%s' already exists" % key) + raise AttributeError("a parser for '%s' already exists" % key) # Create and append parser parser = _HTTPArgumentParser() @@ -612,8 +604,8 @@ class ActionsMap(object): arguments[an] = self.extraparser.parse(an, arguments[an], parameters) # Retrieve action information - namespace, category, action = arguments.pop('_info') - func_name = '%s_%s' % (category, action) + namespace, category, action = arguments.pop('_id') + func_name = '%s_%s' % (category, action.replace('-', '_')) # Lock the moulinette for the namespace with MoulinetteLock(namespace, timeout): @@ -690,67 +682,75 @@ class ActionsMap(object): An interface relevant's parser object """ - # Define setter for extra parameters + ## Get extra parameters if not self.use_cache: - _set_extra = lambda an, e: self.extraparser.validate(an, e) + _get_extra = lambda an, e: self.extraparser.validate(an, e) else: - _set_extra = lambda an, e: e + _get_extra = lambda an, e: e + + ## Add arguments to the parser + def _add_arguments(parser, arguments): + extras = {} + for argn, argp in arguments.items(): + names = top_parser.format_arg_names(argn, + argp.pop('full', None)) + extra = argp.pop('extra', None) + + arg = parser.add_argument(*names, **argp) + if extra: + extras[arg.dest] = _get_extra(arg.dest, extra) + parser.set_defaults(_extra=extras) # Instantiate parser top_parser = self._parser_class() # Iterate over actions map namespaces for n, actionsmap in actionsmaps.items(): - if 'general_arguments' in actionsmap: - # Parse general arguments - if top_parser.parse_general_arguments: - parser = top_parser.add_general_parser() - for an, ap in actionsmap['general_arguments'].items(): - # Replace version number - version = ap.get('version', None) - if version: - ap['version'] = version.replace('%version%', - __version__) - argname = top_parser.format_arg_name(an, ap.pop('full', None)) - parser.add_argument(*argname, **ap) - del actionsmap['general_arguments'] + # Retrieve global parameters + _global = actionsmap.pop('_global', {}) - # Parse categories + # -- Parse global configuration + # TODO + + # -- Parse global arguments + if 'arguments' in _global: + try: + # Get global arguments parser + parser = top_parser.add_global_parser() + except AttributeError: + # No parser for global arguments + pass + else: + # Add arguments + _add_arguments(parser, _global['arguments']) + + # -- Parse categories for cn, cp in actionsmap.items(): try: actions = cp.pop('actions') except KeyError: + # Invalid category without actions continue - # Add category parser + # Get category parser cat_parser = top_parser.add_category_parser(cn, **cp) - # Parse actions + # -- Parse actions for an, ap in actions.items(): arguments = ap.pop('arguments', {}) - # Add action parser - parser = cat_parser.add_action_parser(an, **ap) - try: - # Store action information - parser.set_defaults(_info=(n, cn, an)) + # Get action parser + parser = cat_parser.add_action_parser(an, **ap) except AttributeError: # No parser for the action - break - - # Add action arguments - for argn, argp in arguments.items(): - name = top_parser.format_arg_name(argn, argp.pop('full', None)) - extra = argp.pop('extra', None) - - arg = parser.add_argument(*name, **argp) - if not extra: - continue - - # Store extra parameters - extras = parser.get_default('_extra') or {} - extras[arg.dest] = _set_extra(arg.dest, extra) - parser.set_defaults(_extra=extras) + continue + except ValueError: + # TODO: Log error + continue + else: + # Store action identification and add arguments + parser.set_defaults(_id=(n, cn, an)) + _add_arguments(parser, arguments) return top_parser From 33752ce01b6b7791dd9544f9f879b7abe437658a Mon Sep 17 00:00:00 2001 From: Jerome Lebleu Date: Mon, 17 Mar 2014 00:47:33 +0100 Subject: [PATCH 09/18] Implement global/actions configuration and MoulinetteCLI * Modify global configuration in the actions map * Implement getter/setter for global and action configuration * Implement quickly authenticators classes and add a todo LDAP authenticator * Implement an actions map signals system and add some signals * Add a Moulinette Interface for the cli and make it support signals * Add a test namespace which implements configuration and authentication --- bin/yunohost | 4 +- data/actionsmap/test.yml | 48 ++++ data/actionsmap/yunohost.yml | 16 +- lib/test/__init__.py | 0 lib/test/test.py | 12 + src/moulinette/__init__.py | 13 +- src/moulinette/actionsmap.py | 404 +++++++++++++++++++++++++++++--- src/moulinette/core.py | 74 ++++++ src/moulinette/interface/api.py | 4 +- src/moulinette/interface/cli.py | 128 +++++++++- 10 files changed, 655 insertions(+), 48 deletions(-) create mode 100644 data/actionsmap/test.yml create mode 100755 lib/test/__init__.py create mode 100644 lib/test/test.py diff --git a/bin/yunohost b/bin/yunohost index 78a354bf..39dfaf97 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -7,7 +7,7 @@ import os.path # Run from source basedir = os.path.abspath('%s/../' % os.path.dirname(__file__)) if os.path.isdir('%s/src' % basedir): - sys.path.append('%s/src' % basedir) + sys.path.insert(0, '%s/src' % basedir) from moulinette import init, cli, MoulinetteError from moulinette.helpers import YunoHostError, colorize @@ -35,7 +35,7 @@ if __name__ == '__main__': raise YunoHostError(17, _("YunoHost is not correctly installed, please execute 'yunohost tools postinstall'")) # Execute the action - cli(['yunohost'], args, use_cache) + cli(['yunohost', 'test'], args, use_cache) except MoulinetteError as e: print(e.colorize()) sys.exit(e.code) diff --git a/data/actionsmap/test.yml b/data/actionsmap/test.yml new file mode 100644 index 00000000..d3947cd1 --- /dev/null +++ b/data/actionsmap/test.yml @@ -0,0 +1,48 @@ + +############################# +# Global parameters # +############################# +_global: + configuration: + authenticate: + - api + authenticator: + default: + type: ldap + help: Admin Password + parameters: + uri: ldap://localhost:389 + base: dc=yunohost,dc=org + anonymous: false + ldap-anonymous: + type: ldap + parameters: + uri: ldap://localhost:389 + base: dc=yunohost,dc=org + anonymous: true + argument_auth: true + +############################# +# Test Actions # +############################# +test: + actions: + non-auth: + api: GET /test/non-auth + configuration: + authenticate: false + auth: + api: GET /test/auth + configuration: + authenticate: all + auth-cli: + api: GET /test/auth-cli + configuration: + authenticate: + - cli + anonymous: + api: GET /test/anon + configuration: + authenticate: all + authenticator: ldap-anonymous + argument_auth: false diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 351eae68..cbb06872 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -34,8 +34,22 @@ ############################# _global: configuration: - auth: + authenticate: - api + authenticator: + default: + type: ldap + help: Admin Password + parameters: + uri: ldap://localhost:389 + base: dc=yunohost,dc=org + anonymous: false + ldap-anonymous: + type: ldap + parameters: + uri: ldap://localhost:389 + base: dc=yunohost,dc=org + anonymous: true arguments: -v: full: --version diff --git a/lib/test/__init__.py b/lib/test/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/lib/test/test.py b/lib/test/test.py new file mode 100644 index 00000000..c22c3e9c --- /dev/null +++ b/lib/test/test.py @@ -0,0 +1,12 @@ + +def test_non_auth(): + print('non-auth') + +def test_auth(auth): + print('[default] / all / auth: %r' % auth) + +def test_auth_cli(): + print('[default] / cli') + +def test_anonymous(): + print('[ldap-anonymous] / all') diff --git a/src/moulinette/__init__.py b/src/moulinette/__init__.py index 56e9c582..4e73f6db 100755 --- a/src/moulinette/__init__.py +++ b/src/moulinette/__init__.py @@ -57,7 +57,7 @@ def init(**kwargs): install_i18n() # Add library directory to python path - sys.path.append(pkg.libdir) + sys.path.insert(0, pkg.libdir) ## Easy access to interfaces @@ -99,10 +99,9 @@ def cli(namespaces, args, use_cache=True): """ from .actionsmap import ActionsMap - from .helpers import pretty_print_dict + from .interface.cli import MoulinetteCLI - try: - amap = ActionsMap('cli', namespaces, use_cache) - pretty_print_dict(amap.process(args)) - except KeyboardInterrupt, EOFError: - raise MoulinetteError(125, _("Interrupted")) + amap = ActionsMap('cli', namespaces, use_cache) + moulinette = MoulinetteCLI(amap) + + moulinette.run(args) diff --git a/src/moulinette/actionsmap.py b/src/moulinette/actionsmap.py index 21b73ab6..21e704a7 100644 --- a/src/moulinette/actionsmap.py +++ b/src/moulinette/actionsmap.py @@ -10,7 +10,91 @@ from collections import OrderedDict import logging from . import __version__ -from .core import MoulinetteError, MoulinetteLock +from .core import MoulinetteError, MoulinetteLock, init_authenticator + +## Actions map Signals ------------------------------------------------- + +class _AMapSignals(object): + """Actions map's Signals interface + + Allow to easily connect signals of the actions map to handlers. They + can be given as arguments in the form of { signal: handler }. + + """ + def __init__(self, **kwargs): + # Initialize handlers + for s in self.signals: + self.clear_handler(s) + + # Iterate over signals to connect + for s, h in kwargs.items(): + self.set_handler(s, h) + + def set_handler(self, signal, handler): + """Set the handler for a signal""" + if signal not in self.signals: + raise ValueError("unknown signal '%s'" % signal) + setattr(self, '_%s' % signal, handler) + + def clear_handler(self, signal): + """Clear the handler of a signal""" + if signal not in self.signals: + raise ValueError("unknown signal '%s'" % signal) + setattr(self, '_%s' % signal, self._notimplemented) + + + ## Signals definitions + + """The list of available signals""" + signals = { 'authenticate', 'prompt' } + + def authenticate(self, authenticator, name, help): + """Process the authentication + + Attempt to authenticate to the given authenticator and return + it. + It is called when authentication is needed (e.g. to process an + action). + + Keyword arguments: + - authenticator -- The authenticator to use + - name -- The authenticator name in the actions map + - help -- A help message for the authenticator + + Returns: + The authenticator object + + """ + if authenticator.is_authenticated: + return authenticator + return self._authenticate(authenticator, name, help) + + def prompt(self, message, is_password=False, confirm=False): + """Prompt for a value + + Prompt the interface for a parameter value which is a password + if 'is_password' and must be confirmed if 'confirm'. + Is is called when a parameter value is needed and when the + current interface should allow user interaction (e.g. to parse + extra parameter 'ask' in the cli). + + Keyword arguments: + - message -- The message to display + - is_password -- True if the parameter is a password + - confirm -- True if the value must be confirmed + + Returns: + The collected value + + """ + return self._prompt(message, is_password, confirm) + + @staticmethod + def _notimplemented(**kwargs): + raise NotImplementedError("this signal is not handled") + +shandler = _AMapSignals() + ## Interfaces' Actions map Parser -------------------------------------- @@ -22,9 +106,24 @@ class _AMapParser(object): global arguments, categories and actions). """ + def __init__(self, parent=None): + if parent: + self._o = parent + else: + self._o = self + self._global_conf = {} + self._conf = {} + + + ## Virtual properties + # Each parser classes must implement these properties. + + """The name of the interface for which it is the parser""" + name = None + ## Virtual methods - # Each parser classes can implement these methods. + # Each parser classes must implement these methods. @staticmethod def format_arg_names(name, full): @@ -72,13 +171,18 @@ class _AMapParser(object): raise NotImplementedError("derived class '%s' must override this method" % \ self.__class__.__name__) - def add_action_parser(self, name, **kwargs): + def add_action_parser(self, name, tid, conf=None, **kwargs): """Add a parser for an action - Create a new action and return an argument parser for it. + Create a new action and return an argument parser for it. It + should set the configuration 'conf' for the action which can be + identified by the tuple identifier 'tid' - it is usually in the + form of (namespace, category, action). Keyword arguments: - name -- The action name + - tid -- The tuple identifier of the action + - conf -- A dict of configuration for the action Returns: An ArgumentParser based object @@ -103,16 +207,194 @@ class _AMapParser(object): raise NotImplementedError("derived class '%s' must override this method" % \ self.__class__.__name__) + + ## Configuration access + + @property + def global_conf(self): + """Return the global configuration of the parser""" + return self._o._global_conf + + def get_global_conf(self, name, profile='default'): + """Get the global value of a configuration + + Return the formated global value of the configuration 'name' for + the given profile. If the configuration doesn't provide profile, + the formated default value is returned. + + Keyword arguments: + - name -- The configuration name + - profile -- The profile of the configuration + + """ + try: + if name == 'authenticator': + value = self.global_conf[name][profile] + else: + value = self.global_conf[name] + except KeyError: + return None + else: + return self._format_conf(name, value) + + def set_global_conf(self, configuration): + """Set global configuration + + Set the global configuration to use for the parser. + + Keyword arguments: + - configuration -- The global configuration + + """ + self._o._global_conf.update(self._validate_conf(configuration, True)) + + def get_conf(self, action, name): + """Get the value of an action configuration + + Return the formated value of configuration 'name' for the action + identified by 'action'. If the configuration for the action is + not set, the default one is returned. + + Keyword arguments: + - action -- An action identifier + - name -- The configuration name + + """ + try: + value = self._o._conf[action][name] + except KeyError: + return self.get_global_conf(name) + else: + return self._format_conf(name, value) + + def set_conf(self, action, configuration): + """Set configuration for an action + + Set the configuration to use for a given action identified by + 'action' which is specific to the parser. + + Keyword arguments: + - action -- The action identifier + - configuration -- The configuration for the action + + """ + self._o._conf[action] = self._validate_conf(configuration) + + + def _validate_conf(self, configuration, is_global=False): + """Validate configuration for the parser + + Return the validated configuration for the interface's actions + map parser. + + Keyword arguments: + - configuration -- The configuration to pre-format + + """ + conf = {} + + # -- 'authenficate' + try: + ifaces = configuration['authenticate'] + except KeyError: + pass + else: + if ifaces == 'all': + conf['authenticate'] = ifaces + elif ifaces == False: + conf['authenticate'] = False + elif isinstance(ifaces, list): + # Store only if authentication is needed + conf['authenticate'] = True if self.name in ifaces else False + else: + # TODO: Log error instead and tell valid values + raise MoulinetteError(22, "Invalid value '%r' for configuration 'authenticate'" % ifaces) + + # -- 'authenticator' + try: + auth = configuration['authenticator'] + except KeyError: + pass + else: + if not is_global and isinstance(auth, str): + try: + # Store parameters of the required authenticator + conf['authenticator'] = self.global_conf['authenticator'][auth] + except KeyError: + raise MoulinetteError(22, "Authenticator '%s' is not defined in global configuration" % auth) + elif is_global and isinstance(auth, dict): + if len(auth) == 0: + logging.warning('no authenticator defined in global configuration') + else: + auths = {} + for auth_name, auth_conf in auth.items(): + # Add authenticator name + auths[auth_name] = ({ 'name': auth_name, + 'type': auth_conf.get('type'), + 'help': auth_conf.get('help', None) + }, + auth_conf.get('parameters', {})) + conf['authenticator'] = auths + else: + # TODO: Log error instead and tell valid values + raise MoulinetteError(22, "Invalid value '%r' for configuration 'authenticator'" % auth) + + # -- 'argument_auth' + try: + arg_auth = configuration['argument_auth'] + except KeyError: + pass + else: + if isinstance(arg_auth, bool): + conf['argument_auth'] = arg_auth + else: + # TODO: Log error instead and tell valid values + raise MoulinetteError(22, "Invalid value '%r' for configuration 'argument_auth'" % arg_auth) + + return conf + + def _format_conf(self, name, value): + """Format a configuration value + + Return the formated value of the configuration 'name' from its + given value. + + Keyword arguments: + - name -- The name of the configuration + - value -- The value to format + + """ + if name == 'authenticator' and value: + auth_conf, auth_params = value + auth_type = auth_conf.pop('type') + + # Return authenticator configuration and an instanciator for + # it as a 2-tuple + return (auth_conf, + lambda: init_authenticator(auth_type, **auth_params)) + + return value + # CLI Actions map Parser class CLIAMapParser(_AMapParser): """Actions map's CLI Parser """ - def __init__(self, parser=None): + def __init__(self, parent=None, parser=None): + super(CLIAMapParser, self).__init__(parent) + self._parser = parser or argparse.ArgumentParser() self._subparsers = self._parser.add_subparsers() + + ## Implement virtual properties + + name = 'cli' + + + ## Implement virtual methods + @staticmethod def format_arg_names(name, full): if name[0] == '-' and full: @@ -133,9 +415,9 @@ class CLIAMapParser(_AMapParser): """ parser = self._subparsers.add_parser(name, help=category_help) - return self.__class__(parser) + return self.__class__(self, parser) - def add_action_parser(self, name, action_help=None, **kwargs): + def add_action_parser(self, name, tid, conf=None, action_help=None, **kwargs): """Add a parser for an action Keyword arguments: @@ -145,10 +427,27 @@ class CLIAMapParser(_AMapParser): A new argparse.ArgumentParser object for the action """ + if conf: + self.set_conf(tid, conf) return self._subparsers.add_parser(name, help=action_help) def parse_args(self, args, **kwargs): - return self._parser.parse_args(args) + ret = self._parser.parse_args(args) + + # Perform authentication if needed + if self.get_conf(ret._tid, 'authenticate'): + auth_conf, klass = self.get_conf(ret._tid, 'authenticator') + + # TODO: Catch errors + auth = shandler.authenticate(klass(), **auth_conf) + if not auth.is_authenticated: + # TODO: Set proper error code + raise MoulinetteError(1, _("This action need authentication")) + if self.get_conf(ret._tid, 'argument_auth') and \ + self.get_conf(ret._tid, 'authenticate') == 'all': + ret.auth = auth + + return ret # API Actions map Parser @@ -226,13 +525,20 @@ class APIAMapParser(_AMapParser): """ def __init__(self): - self._parsers = {} # dict({(method, path): _HTTPArgumentParser}) + super(APIAMapParser, self).__init__() + + self._parsers = {} # dict({(method, path): _HTTPArgumentParser}) @property def routes(self): """Get current routes""" return self._parsers.keys() + + ## Implement virtual properties + + name = 'api' + ## Implement virtual methods @@ -252,7 +558,7 @@ class APIAMapParser(_AMapParser): def add_category_parser(self, name, **kwargs): return self - def add_action_parser(self, name, api=None, **kwargs): + def add_action_parser(self, name, tid, conf=None, api=None, **kwargs): """Add a parser for an action Keyword arguments: @@ -262,12 +568,13 @@ class APIAMapParser(_AMapParser): A new _HTTPArgumentParser object for the route """ - if not api: + try: + # Validate action route + m = re.match('(GET|POST|PUT|DELETE) (/\S+)', api) + except TypeError: raise AttributeError("the action '%s' doesn't provide api access" % name) - - # Validate action route - m = re.match('(GET|POST|PUT|DELETE) (/\S+)', api) if not m: + # TODO: Log error raise ValueError("the action '%s' doesn't provide api access" % name) # Check if a parser already exists for the route @@ -278,6 +585,8 @@ class APIAMapParser(_AMapParser): # Create and append parser parser = _HTTPArgumentParser() self._parsers[key] = parser + if conf: + self.set_conf(key, conf) # Return the created parser return parser @@ -293,6 +602,8 @@ class APIAMapParser(_AMapParser): if route not in self.routes: raise MoulinetteError(22, "No parser for '%s %s' found" % key) + # TODO: Implement authentication + return self._parsers[route].parse_args(args) """ @@ -385,9 +696,11 @@ class AskParameter(_ExtraParameter): if arg_value: return arg_value - # Ask for the argument value - ret = raw_input(colorize(message + ': ', 'cyan')) - return ret + try: + # Ask for the argument value + return shandler.prompt(message) + except NotImplementedError: + return arg_value @classmethod def validate(klass, value, arg_name): @@ -415,12 +728,11 @@ class PasswordParameter(AskParameter): if arg_value: return arg_value - # Ask for the password - pwd1 = getpass.getpass(colorize(message + ': ', 'cyan')) - pwd2 = getpass.getpass(colorize('Retype ' + message + ': ', 'cyan')) - if pwd1 != pwd2: - raise MoulinetteError(22, _("Passwords don't match")) - return pwd1 + try: + # Ask for the password + return shandler.prompt(message, True, True) + except NotImplementedError: + return arg_value class PatternParameter(_ExtraParameter): """ @@ -552,6 +864,7 @@ class ActionsMap(object): """ def __init__(self, interface, namespaces=[], use_cache=True): self.use_cache = use_cache + self.interface = interface try: # Retrieve the interface parser @@ -563,7 +876,7 @@ class ActionsMap(object): if len(namespaces) == 0: namespaces = self.get_namespaces() - actionsmaps = {} + actionsmaps = OrderedDict() # Iterate over actions map namespaces for n in namespaces: @@ -585,7 +898,26 @@ class ActionsMap(object): # Generate parsers self.extraparser = ExtraArgumentParser(interface) - self.parser = self._construct_parser(actionsmaps) + self._parser = self._construct_parser(actionsmaps) + + @property + def parser(self): + """Return the instance of the interface's actions map parser""" + return self._parser + + def connect(self, signal, handler): + """Connect a signal to a handler + + Connect a signal emitted by actions map while processing to a + handler. Note that some signals need a return value. + + Keyword arguments: + - signal -- The name of the signal + - handler -- The method to handle the signal + + """ + global shandler + shandler.set_handler(signal, handler) def process(self, args, timeout=0, **kwargs): """ @@ -604,7 +936,7 @@ class ActionsMap(object): arguments[an] = self.extraparser.parse(an, arguments[an], parameters) # Retrieve action information - namespace, category, action = arguments.pop('_id') + namespace, category, action = arguments.pop('_tid') func_name = '%s_%s' % (category, action.replace('-', '_')) # Lock the moulinette for the namespace @@ -710,7 +1042,9 @@ class ActionsMap(object): _global = actionsmap.pop('_global', {}) # -- Parse global configuration - # TODO + if 'configuration' in _global: + # Set global configuration + top_parser.set_global_conf(_global['configuration']) # -- Parse global arguments if 'arguments' in _global: @@ -737,20 +1071,22 @@ class ActionsMap(object): # -- Parse actions for an, ap in actions.items(): - arguments = ap.pop('arguments', {}) + conf = ap.pop('configuration', None) + args = ap.pop('arguments', {}) + tid = (n, cn, an) try: # Get action parser - parser = cat_parser.add_action_parser(an, **ap) + parser = cat_parser.add_action_parser(an, tid, conf, **ap) except AttributeError: # No parser for the action continue - except ValueError: - # TODO: Log error + except ValueError as e: + logging.warning("cannot add action (%s, %s, %s): %s" % (n, cn, an, e)) continue else: - # Store action identification and add arguments - parser.set_defaults(_id=(n, cn, an)) - _add_arguments(parser, arguments) + # Store action identifier and add arguments + parser.set_defaults(_tid=tid) + _add_arguments(parser, args) return top_parser diff --git a/src/moulinette/core.py b/src/moulinette/core.py index 41e53ed8..35d52432 100644 --- a/src/moulinette/core.py +++ b/src/moulinette/core.py @@ -126,6 +126,80 @@ class Package(object): return open('%s/%s' % (self.get_cachedir(subdir), filename), mode) +# Authenticators ------------------------------------------------------- + +class _BaseAuthenticator(object): + + ## Virtual properties + # Each authenticator classes must implement these properties. + + """The name of the authenticator""" + name = None + + @property + def is_authenticated(self): + """Either the instance is authenticated or not""" + raise NotImplementedError("derived class '%s' must override this property" % \ + self.__class__.__name__) + + + ## Virtual methods + # Each authenticator classes must implement these methods. + + def authenticate(password=None, token=None): + """Attempt to authenticate + + Attempt to authenticate with given password or session token. + + Keyword arguments: + - password -- A clear text password + - token -- A session token + + Returns: + An optional session token + + """ + raise NotImplementedError("derived class '%s' must override this method" % \ + self.__class__.__name__) + + +class LDAPAuthenticator(object): + + def __init__(self, uri, base, anonymous=False): + # TODO: Initialize LDAP connection + + if anonymous: + self._authenticated = True + else: + self._authenticated = False + + + ## Implement virtual properties + + name = 'ldap' + + @property + def is_authenticated(self): + return self._authenticated + + + ## Implement virtual methods + + def authenticate(self, password=None, token=None): + # TODO: Perform LDAP authentication + if password == 'test': + self._authenticated = True + else: + raise MoulinetteError(13, _("Invalid password")) + + return self + + +def init_authenticator(_name, **kwargs): + if _name == 'ldap': + return LDAPAuthenticator(**kwargs) + + # Moulinette core classes ---------------------------------------------- class MoulinetteError(Exception): diff --git a/src/moulinette/interface/api.py b/src/moulinette/interface/api.py index 60e72228..a8136933 100644 --- a/src/moulinette/interface/api.py +++ b/src/moulinette/interface/api.py @@ -213,11 +213,11 @@ class MoulinetteAPI(object): """ if category is None: - with open('%s/doc/resources.json' % pkg.datadir) as f: + with open('%s/../doc/resources.json' % pkg.datadir) as f: return f.read() try: - with open('%s/doc/%s.json' % (pkg.datadir, category)) as f: + with open('%s/../doc/%s.json' % (pkg.datadir, category)) as f: return f.read() except IOError: return 'unknown' diff --git a/src/moulinette/interface/cli.py b/src/moulinette/interface/cli.py index e10b0a25..481413c0 100644 --- a/src/moulinette/interface/cli.py +++ b/src/moulinette/interface/cli.py @@ -1,5 +1,129 @@ # -*- coding: utf-8 -*- +import getpass + +from ..core import MoulinetteError + +# CLI helpers ---------------------------------------------------------- + +colors_codes = { + 'red' : 31, + 'green' : 32, + 'yellow': 33, + 'cyan' : 34, + 'purple': 35 +} + +def colorize(astr, color): + """Colorize a string + + Return a colorized string for printing in shell with style ;) + + Keyword arguments: + - astr -- String to colorize + - color -- Name of the color + + """ + return '\033[{:d}m\033[1m{:s}\033[m'.format(colors_codes[color], astr) + +def pretty_print_dict(d, depth=0): + """Print a dictionary recursively + + Print a dictionary recursively with colors to the standard output. + + Keyword arguments: + - d -- The dictionary to print + - depth -- The recursive depth of the dictionary + + """ + for k,v in sorted(d.items(), key=lambda x: x[0]): + k = colorize(str(k), 'purple') + if isinstance(v, list) and len(v) == 1: + v = v[0] + if isinstance(v, dict): + print((" ") * depth + ("%s: " % str(k))) + pretty_print_dict(v, depth+1) + elif isinstance(v, list): + print((" ") * depth + ("%s: " % str(k))) + for key, value in enumerate(v): + if isinstance(value, tuple): + pretty_print_dict({value[0]: value[1]}, depth+1) + elif isinstance(value, dict): + pretty_print_dict({key: value}, depth+1) + else: + print((" ") * (depth+1) + "- " +str(value)) + else: + if not isinstance(v, basestring): + v = str(v) + print((" ") * depth + "%s: %s" % (str(k), v)) + + +# Moulinette Interface ------------------------------------------------- + class MoulinetteCLI(object): - # TODO: Implement this class - pass + """Moulinette command-line Interface + + Initialize an interface connected to the standard input and output + stream which allows to process moulinette action. + + Keyword arguments: + - actionsmap -- The interface relevant ActionsMap instance + + """ + def __init__(self, actionsmap): + # Connect signals to handlers + actionsmap.connect('authenticate', self._do_authenticate) + actionsmap.connect('prompt', self._do_prompt) + + self.actionsmap = actionsmap + + def run(self, args): + """Run the moulinette + + Process the action corresponding to the given arguments 'args' + and print the result. + + Keyword arguments: + - args -- A list of argument strings + + """ + try: + ret = self.actionsmap.process(args, timeout=5) + except KeyboardInterrupt, EOFError: + raise MoulinetteError(125, _("Interrupted")) + + if isinstance(ret, dict): + pretty_print_dict(ret) + elif ret: + print(ret) + + + ## Signals handlers + + def _do_authenticate(self, authenticator, name, help): + """Process the authentication + + Handle the actionsmap._AMapSignals.authenticate signal. + + """ + # TODO: Allow token authentication? + msg = help or _("Password") + return authenticator.authenticate(password=self._do_prompt(msg, True, False)) + + def _do_prompt(self, message, is_password, confirm): + """Prompt for a value + + Handle the actionsmap._AMapSignals.prompt signal. + + """ + if is_password: + prompt = lambda m: getpass.getpass(colorize(_('%s: ') % m, 'cyan')) + else: + prompt = lambda m: raw_input(colorize(_('%s: ') % m, 'cyan')) + value = prompt(message) + + if confirm: + if prompt(_('Retype %s: ') % message) != value: + raise MoulinetteError(22, _("Values don't match")) + + return value From 5da9f6add820df0b01958b66a56b4dfb3e2f5820 Mon Sep 17 00:00:00 2001 From: Jerome Lebleu Date: Thu, 20 Mar 2014 21:19:30 +0100 Subject: [PATCH 10/18] Work on Authenticators (support for session token and LDAP) --- data/actionsmap/test.yml | 11 +- data/actionsmap/yunohost.yml | 11 +- src/moulinette/actionsmap.py | 9 +- src/moulinette/core.py | 189 ++++++++++++++++++++++++++------ src/moulinette/interface/cli.py | 2 +- 5 files changed, 169 insertions(+), 53 deletions(-) diff --git a/data/actionsmap/test.yml b/data/actionsmap/test.yml index d3947cd1..723ed49c 100644 --- a/data/actionsmap/test.yml +++ b/data/actionsmap/test.yml @@ -8,18 +8,17 @@ _global: - api authenticator: default: - type: ldap + vendor: ldap help: Admin Password parameters: uri: ldap://localhost:389 - base: dc=yunohost,dc=org - anonymous: false + base_dn: dc=yunohost,dc=org + user_rdn: cn=admin ldap-anonymous: - type: ldap + vendor: ldap parameters: uri: ldap://localhost:389 - base: dc=yunohost,dc=org - anonymous: true + base_dn: dc=yunohost,dc=org argument_auth: true ############################# diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index cbb06872..39d39aeb 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -38,18 +38,17 @@ _global: - api authenticator: default: - type: ldap + vendor: ldap help: Admin Password parameters: uri: ldap://localhost:389 - base: dc=yunohost,dc=org - anonymous: false + base_dn: dc=yunohost,dc=org + user_rdn: cn=admin ldap-anonymous: - type: ldap + vendor: ldap parameters: uri: ldap://localhost:389 - base: dc=yunohost,dc=org - anonymous: true + base_dn: dc=yunohost,dc=org arguments: -v: full: --version diff --git a/src/moulinette/actionsmap.py b/src/moulinette/actionsmap.py index 21e704a7..28c4306b 100644 --- a/src/moulinette/actionsmap.py +++ b/src/moulinette/actionsmap.py @@ -330,7 +330,7 @@ class _AMapParser(object): for auth_name, auth_conf in auth.items(): # Add authenticator name auths[auth_name] = ({ 'name': auth_name, - 'type': auth_conf.get('type'), + 'vendor': auth_conf.get('vendor'), 'help': auth_conf.get('help', None) }, auth_conf.get('parameters', {})) @@ -366,12 +366,13 @@ class _AMapParser(object): """ if name == 'authenticator' and value: auth_conf, auth_params = value - auth_type = auth_conf.pop('type') + auth_vendor = auth_conf.pop('vendor') # Return authenticator configuration and an instanciator for # it as a 2-tuple return (auth_conf, - lambda: init_authenticator(auth_type, **auth_params)) + lambda: init_authenticator(auth_conf['name'], + auth_vendor, **auth_params)) return value @@ -994,7 +995,7 @@ class ActionsMap(object): actionsmaps[n] = yaml.load(f) # Cache actions map into pickle file - with pkg.open_cache('%s.pkl' % n, subdir='actionsmap') as f: + with pkg.open_cachefile('%s.pkl' % n, 'w', subdir='actionsmap') as f: pickle.dump(actionsmaps[n], f) return actionsmaps diff --git a/src/moulinette/core.py b/src/moulinette/core.py index 35d52432..4c4d6932 100644 --- a/src/moulinette/core.py +++ b/src/moulinette/core.py @@ -4,6 +4,7 @@ import os import sys import time import gettext +import logging from .helpers import colorize @@ -109,7 +110,7 @@ class Package(object): os.makedirs(path) return path - def open_cache(self, filename, subdir='', mode='w'): + def open_cachefile(self, filename, mode='r', **kwargs): """Open a cache file and return a stream Attempt to open in 'mode' the cache file 'filename' from the @@ -119,85 +120,201 @@ class Package(object): Keyword arguments: - filename -- The cache filename - - subdir -- A subdirectory which contains the file - mode -- The mode in which the file is opened + - **kwargs -- Optional arguments for get_cachedir """ - return open('%s/%s' % (self.get_cachedir(subdir), filename), mode) + # Set make_dir if not given + kwargs['make_dir'] = kwargs.get('make_dir', + True if mode[0] == 'w' else False) + return open('%s/%s' % (self.get_cachedir(**kwargs), filename), mode) # Authenticators ------------------------------------------------------- +import ldap +import gnupg + +class AuthenticationError(Exception): + pass + class _BaseAuthenticator(object): + def __init__(self, name): + self._name = name + + @property + def name(self): + """Return the name of the authenticator instance""" + return self._name + + ## Virtual properties # Each authenticator classes must implement these properties. - """The name of the authenticator""" - name = None + """The vendor of the authenticator""" + vendor = None @property def is_authenticated(self): """Either the instance is authenticated or not""" raise NotImplementedError("derived class '%s' must override this property" % \ - self.__class__.__name__) + self.__class__.__name__) ## Virtual methods # Each authenticator classes must implement these methods. - def authenticate(password=None, token=None): + def authenticate(password=None): """Attempt to authenticate - Attempt to authenticate with given password or session token. + Attempt to authenticate with given password. It should raise an + AuthenticationError exception if authentication fails. Keyword arguments: - password -- A clear text password - - token -- A session token - - Returns: - An optional session token """ raise NotImplementedError("derived class '%s' must override this method" % \ - self.__class__.__name__) + self.__class__.__name__) -class LDAPAuthenticator(object): + ## Authentication methods - def __init__(self, uri, base, anonymous=False): - # TODO: Initialize LDAP connection + def __call__(self, password=None, token=None): + """Attempt to authenticate - if anonymous: - self._authenticated = True - else: - self._authenticated = False + Attempt to authenticate either with password or with session + token if 'password' is None. If the authentication succeed, the + instance is returned and the session is registered for the token + if 'token' and 'password' are given. + The token is composed by the session identifier and a session + hash - to use for encryption - as a 2-tuple. + Keyword arguments: + - password -- A clear text password + - token -- The session token in the form of (id, hash) - ## Implement virtual properties + Returns: + The authenticated instance - name = 'ldap' + """ + if self.is_authenticated: + return self + store_session = True if password and token else False - @property - def is_authenticated(self): - return self._authenticated + if token: + try: + # Extract id and hash from token + s_id, s_hash = token + except TypeError: + if not password: + raise MoulinetteError(22, _("Invalid format for token")) + else: + # TODO: Log error + store_session = False + else: + if password is None: + # Retrieve session + password = self._retrieve_session(s_id, s_hash) + try: + # Attempt to authenticate + self.authenticate(password) + except AuthenticationError as e: + raise MoulinetteError(13, str(e)) + except Exception as e: + logging.error("authentication (name: '%s', type: '%s') fails: %s" % \ + (self.name, self.vendor, e)) + raise MoulinetteError(13, _("Unable to authenticate")) - ## Implement virtual methods - - def authenticate(self, password=None, token=None): - # TODO: Perform LDAP authentication - if password == 'test': - self._authenticated = True - else: - raise MoulinetteError(13, _("Invalid password")) + # Store session + if store_session: + self._store_session(s_id, s_hash, password) return self -def init_authenticator(_name, **kwargs): - if _name == 'ldap': - return LDAPAuthenticator(**kwargs) + ## Private methods + + def _open_sessionfile(self, session_id, mode='r'): + """Open a session file for this instance in given mode""" + return pkg.open_cachefile('%s.asc' % session_id, mode, + subdir='session/%s' % self.name) + + def _store_session(self, session_id, session_hash, password): + """Store a session and its associated password""" + gpg = gnupg.GPG() + gpg.encoding = 'utf-8' + with self._open_sessionfile(session_id, 'w') as f: + f.write(str(gpg.encrypt(password, None, symmetric=True, + passphrase=session_hash))) + + def _retrieve_session(self, session_id, session_hash): + """Retrieve a session and return its associated password""" + try: + with self._open_sessionfile(session_id, 'r') as f: + enc_pwd = f.read() + except IOError: + # TODO: Set proper error code + raise MoulinetteError(167, _("Unable to retrieve session")) + else: + gpg = gnupg.GPG() + gpg.encoding = 'utf-8' + return str(gpg.decrypt(enc_pwd, passphrase=session_hash)) + + +class LDAPAuthenticator(_BaseAuthenticator): + + def __init__(self, uri, base_dn, user_rdn=None, **kwargs): + super(LDAPAuthenticator, self).__init__(**kwargs) + + self.uri = uri + self.basedn = base_dn + if user_rdn: + self.userdn = '%s,%s' % (user_rdn, base_dn) + self.con = None + else: + # Initialize anonymous usage + self.userdn = '' + self.authenticate(None) + + + ## Implement virtual properties + + vendor = 'ldap' + + @property + def is_authenticated(self): + try: + # Retrieve identity + who = self.con.whoami_s() + except: + return False + else: + if who[3:] == self.userdn: + return True + return False + + + ## Implement virtual methods + + def authenticate(self, password): + try: + con = ldap.initialize(self.uri) + if self.userdn: + con.simple_bind_s(self.userdn, password) + else: + con.simple_bind_s() + except ldap.INVALID_CREDENTIALS: + raise AuthenticationError(_("Invalid password")) + else: + self.con = con + + +def init_authenticator(_name, _vendor, **kwargs): + if _vendor == 'ldap': + return LDAPAuthenticator(name=_name, **kwargs) # Moulinette core classes ---------------------------------------------- diff --git a/src/moulinette/interface/cli.py b/src/moulinette/interface/cli.py index 481413c0..1262b336 100644 --- a/src/moulinette/interface/cli.py +++ b/src/moulinette/interface/cli.py @@ -108,7 +108,7 @@ class MoulinetteCLI(object): """ # TODO: Allow token authentication? msg = help or _("Password") - return authenticator.authenticate(password=self._do_prompt(msg, True, False)) + return authenticator(password=self._do_prompt(msg, True, False)) def _do_prompt(self, message, is_password, confirm): """Prompt for a value From 3fae8bf1ff44bef946f1c9bb427ff1e1cc070a9e Mon Sep 17 00:00:00 2001 From: Jerome Lebleu Date: Sat, 22 Mar 2014 17:01:30 +0100 Subject: [PATCH 11/18] Add support for Authenticators in the API * Update the API to support authenticators and session token * Add a function to clean sessions * Make the API fit in better with actions map * Minor changes on test namespace --- bin/yunohost-api | 2 +- data/actionsmap/test.yml | 12 + lib/test/test.py | 15 +- src/moulinette/__init__.py | 5 +- src/moulinette/actionsmap.py | 61 +++-- src/moulinette/core.py | 11 + src/moulinette/interface/api.py | 392 +++++++++++++++++++------------- 7 files changed, 320 insertions(+), 178 deletions(-) diff --git a/bin/yunohost-api b/bin/yunohost-api index 11e889c5..74778536 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -41,5 +41,5 @@ if __name__ == '__main__': # TODO: Add log argument # Rune the server - api(['yunohost'], 6787, {('GET', '/installed'): is_installed}, use_cache) + api(['yunohost', 'test'], 6787, {('GET', '/installed'): is_installed}, use_cache) sys.exit(0) diff --git a/data/actionsmap/test.yml b/data/actionsmap/test.yml index 723ed49c..40f632d7 100644 --- a/data/actionsmap/test.yml +++ b/data/actionsmap/test.yml @@ -19,6 +19,13 @@ _global: parameters: uri: ldap://localhost:389 base_dn: dc=yunohost,dc=org + test-profile: + vendor: ldap + help: Admin Password (profile) + parameters: + uri: ldap://localhost:389 + base_dn: dc=yunohost,dc=org + user_rdn: cn=admin argument_auth: true ############################# @@ -34,6 +41,11 @@ test: api: GET /test/auth configuration: authenticate: all + auth-profile: + api: GET /test/auth-profile + configuration: + authenticate: all + authenticator: test-profile auth-cli: api: GET /test/auth-cli configuration: diff --git a/lib/test/test.py b/lib/test/test.py index c22c3e9c..8a9e6e6c 100644 --- a/lib/test/test.py +++ b/lib/test/test.py @@ -1,12 +1,19 @@ def test_non_auth(): - print('non-auth') + return {'action': 'non-auth'} def test_auth(auth): - print('[default] / all / auth: %r' % auth) + return {'action': 'auth', + 'authenticator': 'default', 'authenticate': 'all'} + +def test_auth_profile(auth): + return {'action': 'auth-profile', + 'authenticator': 'test-profile', 'authenticate': 'all'} def test_auth_cli(): - print('[default] / cli') + return {'action': 'auth-cli', + 'authenticator': 'default', 'authenticate': ['cli']} def test_anonymous(): - print('[ldap-anonymous] / all') + return {'action': 'anonymous', + 'authenticator': 'ldap-anonymous', 'authenticate': 'all'} diff --git a/src/moulinette/__init__.py b/src/moulinette/__init__.py index 4e73f6db..b503a8e7 100755 --- a/src/moulinette/__init__.py +++ b/src/moulinette/__init__.py @@ -69,21 +69,20 @@ def api(namespaces, port, routes={}, use_cache=True): Keyword arguments: - namespaces -- The list of namespaces to use - - port -- Port to run on + - port -- Port number to run on - routes -- A dict of additional routes to add in the form of {(method, uri): callback} - use_cache -- False if it should parse the actions map file instead of using the cached one """ - from bottle import run from .actionsmap import ActionsMap from .interface.api import MoulinetteAPI amap = ActionsMap('api', namespaces, use_cache) moulinette = MoulinetteAPI(amap, routes) - run(moulinette.app, port=port) + moulinette.run(port) def cli(namespaces, args, use_cache=True): """Command line interface diff --git a/src/moulinette/actionsmap.py b/src/moulinette/actionsmap.py index 28c4306b..fb54e3e3 100644 --- a/src/moulinette/actionsmap.py +++ b/src/moulinette/actionsmap.py @@ -48,7 +48,7 @@ class _AMapSignals(object): """The list of available signals""" signals = { 'authenticate', 'prompt' } - def authenticate(self, authenticator, name, help): + def authenticate(self, authenticator, name, help, vendor=None): """Process the authentication Attempt to authenticate to the given authenticator and return @@ -60,6 +60,7 @@ class _AMapSignals(object): - authenticator -- The authenticator to use - name -- The authenticator name in the actions map - help -- A help message for the authenticator + - vendor -- Not expected (TODO: Remove it) Returns: The authenticator object @@ -227,15 +228,11 @@ class _AMapParser(object): - profile -- The profile of the configuration """ - try: - if name == 'authenticator': - value = self.global_conf[name][profile] - else: - value = self.global_conf[name] - except KeyError: - return None + if name == 'authenticator': + value = self.global_conf[name][profile] else: - return self._format_conf(name, value) + value = self.global_conf[name] + return self._format_conf(name, value) def set_global_conf(self, configuration): """Set global configuration @@ -366,13 +363,13 @@ class _AMapParser(object): """ if name == 'authenticator' and value: auth_conf, auth_params = value - auth_vendor = auth_conf.pop('vendor') # Return authenticator configuration and an instanciator for # it as a 2-tuple return (auth_conf, lambda: init_authenticator(auth_conf['name'], - auth_vendor, **auth_params)) + auth_conf['vendor'], + **auth_params)) return value @@ -486,7 +483,7 @@ class _HTTPArgumentParser(object): return action - def parse_args(self, args): + def parse_args(self, args={}, namespace=None): arg_strings = [] ## Append an argument to the current one @@ -514,7 +511,7 @@ class _HTTPArgumentParser(object): for dest, opt in self._optional.items(): if dest in args: arg_strings = append(arg_strings, args[dest], opt[0]) - return self._parser.parse_args(arg_strings) + return self._parser.parse_args(arg_strings, namespace) def _error(self, message): # TODO: Raise a proper exception @@ -596,16 +593,28 @@ class APIAMapParser(_AMapParser): """Parse arguments Keyword arguments: - - route -- The action route (e.g. 'GET /' ) + - route -- The action route as a 2-tuple (method, path) """ # Retrieve the parser for the route if route not in self.routes: raise MoulinetteError(22, "No parser for '%s %s' found" % key) + ret = argparse.Namespace() - # TODO: Implement authentication + # Perform authentication if needed + if self.get_conf(route, 'authenticate'): + auth_conf, klass = self.get_conf(route, 'authenticator') - return self._parsers[route].parse_args(args) + # TODO: Catch errors + auth = shandler.authenticate(klass(), **auth_conf) + if not auth.is_authenticated: + # TODO: Set proper error code + raise MoulinetteError(1, _("This action need authentication")) + if self.get_conf(route, 'argument_auth') and \ + self.get_conf(route, 'authenticate') == 'all': + ret.auth = auth + + return self._parsers[route].parse_args(args, ret) """ The dict of interfaces names and their associated parser class. @@ -906,6 +915,26 @@ class ActionsMap(object): """Return the instance of the interface's actions map parser""" return self._parser + def get_authenticator(self, profile='default'): + """Get an authenticator instance + + Retrieve the authenticator for the given profile and return a + new instance. + + Keyword arguments: + - profile -- An authenticator profile name + + Returns: + A new _BaseAuthenticator derived instance + + """ + try: + auth = self.parser.get_global_conf('authenticator', profile)[1] + except KeyError: + raise MoulinetteError(167, _("Unknown authenticator profile '%s'") % profile) + else: + return auth() + def connect(self, signal, handler): """Connect a signal to a handler diff --git a/src/moulinette/core.py b/src/moulinette/core.py index 4c4d6932..1b2805e9 100644 --- a/src/moulinette/core.py +++ b/src/moulinette/core.py @@ -316,6 +316,17 @@ def init_authenticator(_name, _vendor, **kwargs): if _vendor == 'ldap': return LDAPAuthenticator(name=_name, **kwargs) +def clean_session(session_id, profiles=[]): + sessiondir = pkg.get_cachedir('session') + if len(profiles) == 0: + profiles = os.listdir(sessiondir) + + for p in profiles: + try: + os.unlink(os.path.join(sessiondir, p, '%s.asc' % session_id)) + except OSError: + pass + # Moulinette core classes ---------------------------------------------- diff --git a/src/moulinette/interface/api.py b/src/moulinette/interface/api.py index a8136933..a47d000e 100644 --- a/src/moulinette/interface/api.py +++ b/src/moulinette/interface/api.py @@ -1,147 +1,118 @@ # -*- coding: utf-8 -*- -from bottle import Bottle, request, response, HTTPResponse -from beaker.middleware import SessionMiddleware +from bottle import run, request, response, Bottle, HTTPResponse +from json import dumps as json_encode -from ..core import MoulinetteError +from ..core import MoulinetteError, clean_session from ..helpers import YunoHostError, YunoHostLDAP +# API helpers ---------------------------------------------------------- + +import os +import binascii + +def random20(): + return binascii.hexlify(os.urandom(20)).decode('ascii') + + +# HTTP Responses ------------------------------------------------------- + +class HTTPOKResponse(HTTPResponse): + def __init__(self, output=''): + super(HTTPOKResponse, self).__init__(output, 200) + +class HTTPBadRequestResponse(HTTPResponse): + def __init__(self, output=''): + super(HTTPBadRequestResponse, self).__init__(output, 400) + +class HTTPUnauthorizedResponse(HTTPResponse): + def __init__(self, output=''): + super(HTTPUnauthorizedResponse, self).__init__(output, 401) + +class HTTPErrorResponse(HTTPResponse): + def __init__(self, output=''): + super(HTTPErrorResponse, self).__init__(output, 500) + + # API moulinette interface --------------------------------------------- -class _APIAuthPlugin(object): - """ - Manage the authentication for the API access. - - """ - name = 'apiauth' - api = 2 - - def __init__(self): - # TODO: Add options (e.g. session type, content type, ...) - pass - - @property - def app(self): - """Get Bottle application with session integration""" - if hasattr(self, '_app'): - return self._app - raise Exception(_("The APIAuth Plugin is not installed yet.")) - - def setup(self, app): - """ - Setup the plugin and install the session into the app - - Keyword argument: - app -- The associated application object - - """ - app.route('/login', name='login', method='POST', callback=self.login) - app.route('/logout', name='logout', method='GET', callback=self.logout) - - session_opts = { - 'session.type': 'file', - 'session.cookie_expires': True, - 'session.data_dir': pkg.get_cachedir('session'), - 'session.secure': True - } - self._app = SessionMiddleware(app, session_opts) - - def apply(self, callback, context): - """ - Check authentication before executing the route callback - - Keyword argument: - callback -- The route callback - context -- An instance of Route - - """ - # Check the authentication - if self._is_authenticated: - if context.name == 'login': - self.logout() - else: - return callback - - # Process login route - if context.name == 'login': - password = request.POST.get('password', None) - if password is not None and self.login(password): - raise HTTPResponse(status=200) - else: - raise HTTPResponse(_("Wrong password"), 401) - - # Deny access to the requested route - raise HTTPResponse(_("Unauthorized"), 401) - - def login(self, password): - """ - Attempt to log in with the given password - - Keyword argument: - password -- Cleartext password - - """ - try: YunoHostLDAP(password=password) - except YunoHostError: - return False - else: - session = self._beaker_session - session['authenticated'] = True - session.save() - return True - return False - - def logout(self): - """ - Log out and delete the session - - """ - # TODO: Delete the cached session file - session = self._beaker_session - session.delete() - - - ## Private methods - - @property - def _beaker_session(self): - """Get Beaker session""" - return request.environ.get('beaker.session') - - @property - def _is_authenticated(self): - """Check authentication""" - # TODO: Clear the session path on password changing to avoid invalid access - if 'authenticated' in self._beaker_session: - return True - return False - class _ActionsMapPlugin(object): - """ - Process action for the request using the actions map. + """Actions map Bottle Plugin + + Process relevant action for the request using the actions map and + manage authentication. + + Keyword arguments: + - actionsmap -- An ActionsMap instance """ name = 'actionsmap' api = 2 def __init__(self, actionsmap): + # Connect signals to handlers + actionsmap.connect('authenticate', self._do_authenticate) + self.actionsmap = actionsmap + # TODO: Save and load secrets? + self.secrets = {} def setup(self, app): - pass + """Setup plugin on the application + + Add routes according to the actions map to the application. + + Keyword arguments: + - app -- The application instance + + """ + ## Login wrapper + def _login(callback): + def wrapper(): + kwargs = {} + try: + kwargs['password'] = request.POST['password'] + except KeyError: + raise HTTPBadRequestResponse(_("Missing password parameter")) + try: + kwargs['profile'] = request.POST['profile'] + except KeyError: + pass + return callback(**kwargs) + return wrapper + + ## Logout wrapper + def _logout(callback): + def wrapper(): + kwargs = {} + try: + kwargs['profile'] = request.POST.get('profile') + except KeyError: + pass + return callback(**kwargs) + return wrapper + + # Append authentication routes + app.route('/login', name='login', method='POST', + callback=self.login, skip=['actionsmap'], apply=_login) + app.route('/logout', name='logout', method='GET', + callback=self.logout, skip=['actionsmap'], apply=_logout) + + # Append routes from the actions map + for (m, p) in self.actionsmap.parser.routes: + app.route(p, method=m, callback=self.process) def apply(self, callback, context): - """ - Process the relevant action for the request + """Apply plugin to the route callback - Keyword argument: + Install a wrapper which replace callback and process the + relevant action for the route. + + Keyword arguments: callback -- The route callback context -- An instance of Route """ - method = request.method - uri = context.rule - def wrapper(*args, **kwargs): # Bring arguments together params = kwargs @@ -151,14 +122,127 @@ class _ActionsMapPlugin(object): params[k] = v # Process the action - return self.actionsmap.process(params, route=(method, uri)) + return callback((request.method, context.rule), params) return wrapper + ## Routes callbacks + + def login(self, password, profile='default'): + """Log in to an authenticator profile + + Attempt to authenticate to a given authenticator profile and + register it with the current session - a new one will be created + if needed. + + Keyword arguments: + - password -- A clear text password + - profile -- The authenticator profile name to log in + + """ + # Retrieve session values + s_id = request.get_cookie('session.id') or random20() + try: + s_secret = self.secrets[s_id] + except KeyError: + s_hashes = {} + else: + s_hashes = request.get_cookie('session.hashes', + secret=s_secret) or {} + s_hash = random20() + + try: + # Attempt to authenticate + auth = self.actionsmap.get_authenticator(profile) + auth(password, token=(s_id, s_hash)) + except MoulinetteError as e: + if len(s_hashes) > 0: + try: self.logout(profile) + except: pass + # TODO: Replace by proper exception + if e.code == 13: + raise HTTPUnauthorizedResponse(e.message) + raise HTTPErrorResponse(e.message) + else: + # Update dicts with new values + s_hashes[profile] = s_hash + self.secrets[s_id] = s_secret = random20() + + response.set_cookie('session.id', s_id, secure=True) + response.set_cookie('session.hashes', s_hashes, secure=True, + secret=s_secret) + raise HTTPOKResponse() + + def logout(self, profile=None): + """Log out from an authenticator profile + + Attempt to unregister a given profile - or all by default - from + the current session. + + Keyword arguments: + - profile -- The authenticator profile name to log out + + """ + s_id = request.get_cookie('session.id') + try: + del self.secrets[s_id] + except KeyError: + raise HTTPUnauthorizedResponse(_("You are not logged in")) + else: + # TODO: Clean the session for profile only + # Delete cookie and clean the session + response.set_cookie('session.hashes', '', max_age=-1) + clean_session(s_id) + raise HTTPOKResponse() + + def process(self, _route, arguments={}): + """Process the relevant action for the route + + Call the actions map in order to process the relevant action for + the route with the given arguments and process the returned + value. + + Keyword arguments: + - _route -- The action route as a 2-tuple (method, path) + - arguments -- A dict of arguments for the route + + """ + try: + ret = self.actionsmap.process(arguments, route=_route) + except MoulinetteError as e: + raise HTTPErrorResponse(e.message) + else: + return ret + + + ## Signals handlers + + def _do_authenticate(self, authenticator, name, help): + """Process the authentication + + Handle the actionsmap._AMapSignals.authenticate signal. + + """ + s_id = request.get_cookie('session.id') + try: + s_secret = self.secrets[s_id] + s_hash = request.get_cookie('session.hashes', + secret=s_secret)[name] + except KeyError: + if name == 'default': + msg = _("Needing authentication") + else: + msg = _("Needing authentication to profile '%s'") % name + raise HTTPUnauthorizedResponse(msg) + else: + return authenticator(token=(s_id, s_hash)) + + class MoulinetteAPI(object): - """ - Initialize a HTTP server which serves the API to access to the - moulinette actions. + """Moulinette Application Programming Interface + + Initialize a HTTP server which serves the API to process moulinette + actions. Keyword arguments: - actionsmap -- The relevant ActionsMap instance @@ -166,43 +250,47 @@ class MoulinetteAPI(object): {(method, path): callback} """ - def __init__(self, actionsmap, routes={}): - # Initialize app and default routes # TODO: Return OK to 'OPTIONS' xhr requests (l173) - app = Bottle() - app.route(['/api', '/api/'], method='GET', - callback=self.doc, skip=['apiauth']) + app = Bottle(autojson=False) - # Append routes from the actions map - amap = _ActionsMapPlugin(actionsmap) - for (m, p) in actionsmap.parser.routes: - app.route(p, method=m, callback=self._error, apply=amap) - - # Append additional routes - # TODO: Add an option to skip auth for the route - for (m, p), c in routes.items(): - app.route(p, method=m, callback=c) - - # Define and install a plugin which sets proper header + ## Wrapper which sets proper header def apiheader(callback): def wrapper(*args, **kwargs): response.content_type = 'application/json' response.set_header('Access-Control-Allow-Origin', '*') - return callback(*args, **kwargs) + return json_encode(callback(*args, **kwargs)) return wrapper + + # Install plugins app.install(apiheader) + app.install(_ActionsMapPlugin(actionsmap)) - # Install authentication plugin - apiauth = _APIAuthPlugin() - app.install(apiauth) + # Append default routes + app.route(['/api', '/api/'], method='GET', + callback=self.doc, skip=['actionsmap']) - self._app = apiauth.app + # Append additional routes + # TODO: Add optional authentication to those routes? + for (m, p), c in routes.items(): + app.route(p, method=m, callback=c, skip=['actionsmap']) - @property - def app(self): - """Get Bottle application""" - return self._app + self._app = app + + def run(self, _port): + """Run the moulinette + + Start a server instance on the given port to serve moulinette + actions. + + Keyword arguments: + - _port -- Port number to run on + + """ + run(self._app, port=_port) + + + ## Routes handlers def doc(self, category=None): """ @@ -220,8 +308,4 @@ class MoulinetteAPI(object): with open('%s/../doc/%s.json' % (pkg.datadir, category)) as f: return f.read() except IOError: - return 'unknown' - - def _error(self, *args, **kwargs): - # TODO: Raise or return an error - print('error') + return None From c95469073a81d52c37357f8ae42231e2c1a87770 Mon Sep 17 00:00:00 2001 From: Jerome Lebleu Date: Mon, 24 Mar 2014 15:52:26 +0100 Subject: [PATCH 12/18] Fix extra parameters absence --- src/moulinette/actionsmap.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/moulinette/actionsmap.py b/src/moulinette/actionsmap.py index fb54e3e3..1935a34d 100644 --- a/src/moulinette/actionsmap.py +++ b/src/moulinette/actionsmap.py @@ -1061,7 +1061,8 @@ class ActionsMap(object): arg = parser.add_argument(*names, **argp) if extra: extras[arg.dest] = _get_extra(arg.dest, extra) - parser.set_defaults(_extra=extras) + if extras: + parser.set_defaults(_extra=extras) # Instantiate parser top_parser = self._parser_class() From ecd88ce8532c91fe3b3acc360d33d14f8fcf90fc Mon Sep 17 00:00:00 2001 From: Jerome Lebleu Date: Tue, 25 Mar 2014 00:51:39 +0100 Subject: [PATCH 13/18] Some refactoring (again) in authenticators and exceptions * Move authenticators classes into distinct modules * Standardize MoulinetteError which is now a child class of IOError * Add methods from helpers.py to LDAP authenticator class * Review authenticator and action configuration storage * Small changes in error printing on MoulinetteError raising --- bin/yunohost | 7 +- bin/yunohost-api | 12 +- src/moulinette/__init__.py | 23 +- src/moulinette/actionsmap.py | 131 ++++++------ src/moulinette/authenticators/__init__.py | 146 +++++++++++++ src/moulinette/authenticators/ldap.py | 192 +++++++++++++++++ src/moulinette/core.py | 244 ++-------------------- src/moulinette/interface/api.py | 29 +-- src/moulinette/interface/cli.py | 11 +- 9 files changed, 477 insertions(+), 318 deletions(-) create mode 100644 src/moulinette/authenticators/__init__.py create mode 100644 src/moulinette/authenticators/ldap.py diff --git a/bin/yunohost b/bin/yunohost index 39dfaf97..001148a5 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -35,11 +35,8 @@ if __name__ == '__main__': raise YunoHostError(17, _("YunoHost is not correctly installed, please execute 'yunohost tools postinstall'")) # Execute the action - cli(['yunohost', 'test'], args, use_cache) - except MoulinetteError as e: - print(e.colorize()) - sys.exit(e.code) + ret = cli(['yunohost', 'test'], args, use_cache) except YunoHostError as e: print(colorize(_("Error: "), 'red') + e.message) sys.exit(e.code) - sys.exit(0) + sys.exit(ret) diff --git a/bin/yunohost-api b/bin/yunohost-api index 74778536..8bd869d0 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -9,7 +9,7 @@ basedir = os.path.abspath(os.path.dirname(__file__) +'/../') if os.path.isdir(basedir +'/src'): sys.path.append(basedir +'/src') -from moulinette import init, api +from moulinette import init, api, MoulinetteError ## Callbacks for additional routes @@ -40,6 +40,12 @@ if __name__ == '__main__': sys.argv.remove('--debug') # TODO: Add log argument - # Rune the server - api(['yunohost', 'test'], 6787, {('GET', '/installed'): is_installed}, use_cache) + try: + # Run the server + api(['yunohost', 'test'], 6787, + {('GET', '/installed'): is_installed}, use_cache) + except MoulinetteError as e: + from moulinette.interface.cli import colorize + print(_('%s: %s' % (colorize(_('Error'), 'red'), e.strerror))) + sys.exit(e.code) sys.exit(0) diff --git a/src/moulinette/__init__.py b/src/moulinette/__init__.py index b503a8e7..f589a881 100755 --- a/src/moulinette/__init__.py +++ b/src/moulinette/__init__.py @@ -29,7 +29,7 @@ __all__ = [ 'MoulinetteError', ] -from .core import MoulinetteError +from moulinette.core import MoulinetteError ## Package functions @@ -50,7 +50,7 @@ def init(**kwargs): """ import sys import __builtin__ - from .core import Package, install_i18n + from moulinette.core import Package, install_i18n __builtin__.__dict__['pkg'] = Package(**kwargs) # Initialize internationalization @@ -76,8 +76,8 @@ def api(namespaces, port, routes={}, use_cache=True): instead of using the cached one """ - from .actionsmap import ActionsMap - from .interface.api import MoulinetteAPI + from moulinette.actionsmap import ActionsMap + from moulinette.interface.api import MoulinetteAPI amap = ActionsMap('api', namespaces, use_cache) moulinette = MoulinetteAPI(amap, routes) @@ -97,10 +97,15 @@ def cli(namespaces, args, use_cache=True): instead of using the cached one """ - from .actionsmap import ActionsMap - from .interface.cli import MoulinetteCLI + from moulinette.actionsmap import ActionsMap + from moulinette.interface.cli import MoulinetteCLI, colorize - amap = ActionsMap('cli', namespaces, use_cache) - moulinette = MoulinetteCLI(amap) + try: + amap = ActionsMap('cli', namespaces, use_cache) + moulinette = MoulinetteCLI(amap) - moulinette.run(args) + moulinette.run(args) + except MoulinetteError as e: + print(_('%s: %s' % (colorize(_('Error'), 'red'), e.strerror))) + return e.errno + return 0 diff --git a/src/moulinette/actionsmap.py b/src/moulinette/actionsmap.py index 1935a34d..ec28da5c 100644 --- a/src/moulinette/actionsmap.py +++ b/src/moulinette/actionsmap.py @@ -1,16 +1,17 @@ # -*- coding: utf-8 -*- -import argparse -import yaml -import re import os +import re +import errno +import yaml +import argparse import cPickle as pickle from collections import OrderedDict import logging -from . import __version__ -from .core import MoulinetteError, MoulinetteLock, init_authenticator +from moulinette.core import (MoulinetteError, MoulinetteLock, + init_authenticator) ## Actions map Signals ------------------------------------------------- @@ -48,7 +49,7 @@ class _AMapSignals(object): """The list of available signals""" signals = { 'authenticate', 'prompt' } - def authenticate(self, authenticator, name, help, vendor=None): + def authenticate(self, authenticator, help): """Process the authentication Attempt to authenticate to the given authenticator and return @@ -57,10 +58,8 @@ class _AMapSignals(object): action). Keyword arguments: - - authenticator -- The authenticator to use - - name -- The authenticator name in the actions map + - authenticator -- The authenticator object to use - help -- A help message for the authenticator - - vendor -- Not expected (TODO: Remove it) Returns: The authenticator object @@ -68,7 +67,7 @@ class _AMapSignals(object): """ if authenticator.is_authenticated: return authenticator - return self._authenticate(authenticator, name, help) + return self._authenticate(authenticator, help) def prompt(self, message, is_password=False, confirm=False): """Prompt for a value @@ -172,18 +171,14 @@ class _AMapParser(object): raise NotImplementedError("derived class '%s' must override this method" % \ self.__class__.__name__) - def add_action_parser(self, name, tid, conf=None, **kwargs): + def add_action_parser(self, name, tid, **kwargs): """Add a parser for an action - Create a new action and return an argument parser for it. It - should set the configuration 'conf' for the action which can be - identified by the tuple identifier 'tid' - it is usually in the - form of (namespace, category, action). + Create a new action and return an argument parser for it. Keyword arguments: - name -- The action name - tid -- The tuple identifier of the action - - conf -- A dict of configuration for the action Returns: An ArgumentParser based object @@ -288,6 +283,7 @@ class _AMapParser(object): - configuration -- The configuration to pre-format """ + # TODO: Create a class with a validator method for each configuration conf = {} # -- 'authenficate' @@ -305,7 +301,7 @@ class _AMapParser(object): conf['authenticate'] = True if self.name in ifaces else False else: # TODO: Log error instead and tell valid values - raise MoulinetteError(22, "Invalid value '%r' for configuration 'authenticate'" % ifaces) + raise MoulinetteError(errno.EINVAL, "Invalid value '%r' for configuration 'authenticate'" % ifaces) # -- 'authenticator' try: @@ -315,26 +311,31 @@ class _AMapParser(object): else: if not is_global and isinstance(auth, str): try: - # Store parameters of the required authenticator + # Store needed authenticator profile conf['authenticator'] = self.global_conf['authenticator'][auth] except KeyError: - raise MoulinetteError(22, "Authenticator '%s' is not defined in global configuration" % auth) + raise MoulinetteError(errno.EINVAL, "Undefined authenticator '%s' in global configuration" % auth) elif is_global and isinstance(auth, dict): if len(auth) == 0: logging.warning('no authenticator defined in global configuration') else: auths = {} for auth_name, auth_conf in auth.items(): - # Add authenticator name - auths[auth_name] = ({ 'name': auth_name, - 'vendor': auth_conf.get('vendor'), - 'help': auth_conf.get('help', None) - }, + # Add authenticator profile as a 3-tuple + # (identifier, configuration, parameters) with + # - identifier: the authenticator vendor and its + # profile name as a 2-tuple + # - configuration: a dict of additional global + # configuration (i.e. 'help') + # - parameters: a dict of arguments for the + # authenticator profile + auths[auth_name] = ((auth_conf.get('vendor'), auth_name), + { 'help': auth_conf.get('help', None) }, auth_conf.get('parameters', {})) conf['authenticator'] = auths else: # TODO: Log error instead and tell valid values - raise MoulinetteError(22, "Invalid value '%r' for configuration 'authenticator'" % auth) + raise MoulinetteError(errno.EINVAL, "Invalid value '%r' for configuration 'authenticator'" % auth) # -- 'argument_auth' try: @@ -346,7 +347,7 @@ class _AMapParser(object): conf['argument_auth'] = arg_auth else: # TODO: Log error instead and tell valid values - raise MoulinetteError(22, "Invalid value '%r' for configuration 'argument_auth'" % arg_auth) + raise MoulinetteError(errno.EINVAL, "Invalid value '%r' for configuration 'argument_auth'" % arg_auth) return conf @@ -362,14 +363,12 @@ class _AMapParser(object): """ if name == 'authenticator' and value: - auth_conf, auth_params = value + (identifier, configuration, parameters) = value - # Return authenticator configuration and an instanciator for - # it as a 2-tuple - return (auth_conf, - lambda: init_authenticator(auth_conf['name'], - auth_conf['vendor'], - **auth_params)) + # Return global configuration and an authenticator + # instanciator as a 2-tuple + return (configuration, + lambda: init_authenticator(identifier, parameters)) return value @@ -415,7 +414,7 @@ class CLIAMapParser(_AMapParser): parser = self._subparsers.add_parser(name, help=category_help) return self.__class__(self, parser) - def add_action_parser(self, name, tid, conf=None, action_help=None, **kwargs): + def add_action_parser(self, name, tid, action_help=None, **kwargs): """Add a parser for an action Keyword arguments: @@ -425,8 +424,6 @@ class CLIAMapParser(_AMapParser): A new argparse.ArgumentParser object for the action """ - if conf: - self.set_conf(tid, conf) return self._subparsers.add_parser(name, help=action_help) def parse_args(self, args, **kwargs): @@ -440,7 +437,7 @@ class CLIAMapParser(_AMapParser): auth = shandler.authenticate(klass(), **auth_conf) if not auth.is_authenticated: # TODO: Set proper error code - raise MoulinetteError(1, _("This action need authentication")) + raise MoulinetteError(errno.EACCES, _("This action need authentication")) if self.get_conf(ret._tid, 'argument_auth') and \ self.get_conf(ret._tid, 'authenticate') == 'all': ret.auth = auth @@ -521,7 +518,6 @@ class APIAMapParser(_AMapParser): """Actions map's API Parser """ - def __init__(self): super(APIAMapParser, self).__init__() @@ -556,7 +552,7 @@ class APIAMapParser(_AMapParser): def add_category_parser(self, name, **kwargs): return self - def add_action_parser(self, name, tid, conf=None, api=None, **kwargs): + def add_action_parser(self, name, tid, api=None, **kwargs): """Add a parser for an action Keyword arguments: @@ -582,9 +578,7 @@ class APIAMapParser(_AMapParser): # Create and append parser parser = _HTTPArgumentParser() - self._parsers[key] = parser - if conf: - self.set_conf(key, conf) + self._parsers[key] = (tid, parser) # Return the created parser return parser @@ -596,25 +590,27 @@ class APIAMapParser(_AMapParser): - route -- The action route as a 2-tuple (method, path) """ - # Retrieve the parser for the route - if route not in self.routes: - raise MoulinetteError(22, "No parser for '%s %s' found" % key) + try: + # Retrieve the tid and the parser for the route + tid, parser = self._parsers[route] + except KeyError: + raise MoulinetteError(errno.EINVAL, "No parser found for route '%s'" % route) ret = argparse.Namespace() # Perform authentication if needed - if self.get_conf(route, 'authenticate'): - auth_conf, klass = self.get_conf(route, 'authenticator') + if self.get_conf(tid, 'authenticate'): + auth_conf, klass = self.get_conf(tid, 'authenticator') # TODO: Catch errors auth = shandler.authenticate(klass(), **auth_conf) if not auth.is_authenticated: # TODO: Set proper error code - raise MoulinetteError(1, _("This action need authentication")) - if self.get_conf(route, 'argument_auth') and \ - self.get_conf(route, 'authenticate') == 'all': + raise MoulinetteError(errno.EACCES, _("This action need authentication")) + if self.get_conf(tid, 'argument_auth') and \ + self.get_conf(tid, 'authenticate') == 'all': ret.auth = auth - return self._parsers[route].parse_args(args, ret) + return parser.parse_args(args, ret) """ The dict of interfaces names and their associated parser class. @@ -758,7 +754,7 @@ class PatternParameter(_ExtraParameter): pattern, message = (arguments[0], arguments[1]) if not re.match(pattern, arg_value or ''): - raise MoulinetteError(22, message) + raise MoulinetteError(errno.EINVAL, message) return arg_value @staticmethod @@ -776,7 +772,7 @@ The list of available extra parameters classes. It will keep to this list order on argument parsing. """ -extraparameters_list = {AskParameter, PasswordParameter, PatternParameter} +extraparameters_list = [AskParameter, PasswordParameter, PatternParameter] # Extra parameters argument Parser @@ -880,7 +876,7 @@ class ActionsMap(object): # Retrieve the interface parser self._parser_class = actionsmap_parsers[interface] except KeyError: - raise MoulinetteError(22, _("Invalid interface '%s'" % interface)) + raise MoulinetteError(errno.EINVAL, _("Unknown interface '%s'" % interface)) logging.debug("initializing ActionsMap for the '%s' interface" % interface) @@ -931,7 +927,7 @@ class ActionsMap(object): try: auth = self.parser.get_global_conf('authenticator', profile)[1] except KeyError: - raise MoulinetteError(167, _("Unknown authenticator profile '%s'") % profile) + raise MoulinetteError(errno.EINVAL, _("Unknown authenticator profile '%s'") % profile) else: return auth() @@ -977,7 +973,7 @@ class ActionsMap(object): fromlist=[func_name]) func = getattr(mod, func_name) except (AttributeError, ImportError): - raise MoulinetteError(168, _('Function is not defined')) + raise MoulinetteError(errno.ENOSYS, _('Function is not defined')) else: # Process the action return func(**arguments) @@ -1056,11 +1052,13 @@ class ActionsMap(object): for argn, argp in arguments.items(): names = top_parser.format_arg_names(argn, argp.pop('full', None)) - extra = argp.pop('extra', None) - - arg = parser.add_argument(*names, **argp) - if extra: - extras[arg.dest] = _get_extra(arg.dest, extra) + try: + extra = argp.pop('extra') + arg_dest = (parser.add_argument(*names, **argp)).dest + extras[arg_dest] = _get_extra(arg_dest, extra) + except KeyError: + # No extra parameters + parser.add_argument(*names, **argp) if extras: parser.set_defaults(_extra=extras) @@ -1095,6 +1093,7 @@ class ActionsMap(object): actions = cp.pop('actions') except KeyError: # Invalid category without actions + logging.warning("no actions found in category '%s'" % cn) continue # Get category parser @@ -1102,13 +1101,18 @@ class ActionsMap(object): # -- Parse actions for an, ap in actions.items(): - conf = ap.pop('configuration', None) args = ap.pop('arguments', {}) tid = (n, cn, an) + try: + conf = ap.pop('configuration') + _set_conf = lambda p: p.set_conf(tid, conf) + except KeyError: + # No action configuration + _set_conf = lambda p: False try: # Get action parser - parser = cat_parser.add_action_parser(an, tid, conf, **ap) + parser = cat_parser.add_action_parser(an, tid, **ap) except AttributeError: # No parser for the action continue @@ -1119,5 +1123,6 @@ class ActionsMap(object): # Store action identifier and add arguments parser.set_defaults(_tid=tid) _add_arguments(parser, args) + _set_conf(cat_parser) return top_parser diff --git a/src/moulinette/authenticators/__init__.py b/src/moulinette/authenticators/__init__.py new file mode 100644 index 00000000..4d98d7ac --- /dev/null +++ b/src/moulinette/authenticators/__init__.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- + +import errno +import gnupg +import logging + +from moulinette.core import MoulinetteError + +# Base Class ----------------------------------------------------------- + +class BaseAuthenticator(object): + """Authenticator base representation + + Each authenticators must implement an Authenticator class derived + from this class. It implements base methods to authenticate with a + password or a session token. + Authenticators configurations are identified by a profile name which + must be given on instantiation - with the corresponding vendor + configuration of the authenticator. + + Keyword arguments: + - name -- The authenticator profile name + + """ + def __init__(self, name): + self._name = name + + @property + def name(self): + """Return the name of the authenticator instance""" + return self._name + + + ## Virtual properties + # Each authenticator classes must implement these properties. + + """The vendor name of the authenticator""" + vendor = None + + @property + def is_authenticated(self): + """Either the instance is authenticated or not""" + raise NotImplementedError("derived class '%s' must override this property" % \ + self.__class__.__name__) + + + ## Virtual methods + # Each authenticator classes must implement these methods. + + def authenticate(password=None): + """Attempt to authenticate + + Attempt to authenticate with given password. It should raise an + AuthenticationError exception if authentication fails. + + Keyword arguments: + - password -- A clear text password + + """ + raise NotImplementedError("derived class '%s' must override this method" % \ + self.__class__.__name__) + + + ## Authentication methods + + def __call__(self, password=None, token=None): + """Attempt to authenticate + + Attempt to authenticate either with password or with session + token if 'password' is None. If the authentication succeed, the + instance is returned and the session is registered for the token + if 'token' and 'password' are given. + The token is composed by the session identifier and a session + hash - to use for encryption - as a 2-tuple. + + Keyword arguments: + - password -- A clear text password + - token -- The session token in the form of (id, hash) + + Returns: + The authenticated instance + + """ + if self.is_authenticated: + return self + store_session = True if password and token else False + + if token: + try: + # Extract id and hash from token + s_id, s_hash = token + except TypeError: + if not password: + raise MoulinetteError(errno.EINVAL, _("Invalid format for token")) + else: + # TODO: Log error + store_session = False + else: + if password is None: + # Retrieve session + password = self._retrieve_session(s_id, s_hash) + + try: + # Attempt to authenticate + self.authenticate(password) + except MoulinetteError: + raise + except Exception as e: + logging.error("authentication (name: '%s', type: '%s') fails: %s" % \ + (self.name, self.vendor, e)) + raise MoulinetteError(errno.EACCES, _("Unable to authenticate")) + + # Store session + if store_session: + self._store_session(s_id, s_hash, password) + + return self + + + ## Private methods + + def _open_sessionfile(self, session_id, mode='r'): + """Open a session file for this instance in given mode""" + return pkg.open_cachefile('%s.asc' % session_id, mode, + subdir='session/%s' % self.name) + + def _store_session(self, session_id, session_hash, password): + """Store a session and its associated password""" + gpg = gnupg.GPG() + gpg.encoding = 'utf-8' + with self._open_sessionfile(session_id, 'w') as f: + f.write(str(gpg.encrypt(password, None, symmetric=True, + passphrase=session_hash))) + + def _retrieve_session(self, session_id, session_hash): + """Retrieve a session and return its associated password""" + try: + with self._open_sessionfile(session_id, 'r') as f: + enc_pwd = f.read() + except IOError: + # TODO: Set proper error code + raise MoulinetteError(167, _("Unable to retrieve session")) + else: + gpg = gnupg.GPG() + gpg.encoding = 'utf-8' + return str(gpg.decrypt(enc_pwd, passphrase=session_hash)) diff --git a/src/moulinette/authenticators/ldap.py b/src/moulinette/authenticators/ldap.py new file mode 100644 index 00000000..4a5ce193 --- /dev/null +++ b/src/moulinette/authenticators/ldap.py @@ -0,0 +1,192 @@ +# -*- coding: utf-8 -*- + +# TODO: Use Python3 to remove this fix! +from __future__ import absolute_import +import ldap +import ldap.modlist as modlist + +from moulinette.core import MoulinetteError +from moulinette.authenticators import BaseAuthenticator + +# LDAP Class Implementation -------------------------------------------- + +class Authenticator(BaseAuthenticator): + """LDAP Authenticator + + Initialize a LDAP connexion for the given arguments. It attempts to + authenticate a user if 'user_rdn' is given - by associating user_rdn + and base_dn - and provides extra methods to manage opened connexion. + + Keyword arguments: + - uri -- The LDAP server URI + - base_dn -- The base dn + - user_rdn -- The user rdn to authenticate + + """ + def __init__(self, name, uri, base_dn, user_rdn=None): + super(Authenticator, self).__init__(name) + + self.uri = uri + self.basedn = base_dn + if user_rdn: + self.userdn = '%s,%s' % (user_rdn, base_dn) + self.con = None + else: + # Initialize anonymous usage + self.userdn = '' + self.authenticate(None) + + + ## Implement virtual properties + + vendor = 'ldap' + + @property + def is_authenticated(self): + try: + # Retrieve identity + who = self.con.whoami_s() + except: + return False + else: + if who[3:] == self.userdn: + return True + return False + + + ## Implement virtual methods + + def authenticate(self, password): + try: + con = ldap.initialize(self.uri) + if self.userdn: + con.simple_bind_s(self.userdn, password) + else: + con.simple_bind_s() + except ldap.INVALID_CREDENTIALS: + raise MoulinetteError(errno.EACCES, _("Invalid password")) + else: + self.con = con + + + ## Additional LDAP methods + # TODO: Review these methods + + def search(self, base=None, filter='(objectClass=*)', attrs=['dn']): + """ + Search in LDAP base + + Keyword arguments: + base -- Base to search into + filter -- LDAP filter + attrs -- Array of attributes to fetch + + Returns: + Boolean | Dict + + """ + if not base: + base = self.basedn + + try: + result = self.con.search_s(base, ldap.SCOPE_SUBTREE, filter, attrs) + except: + raise MoulinetteError(169, _('An error occured during LDAP search')) + + if result: + result_list = [] + for dn, entry in result: + if attrs != None: + if 'dn' in attrs: + entry['dn'] = [dn] + result_list.append(entry) + return result_list + else: + return False + + def add(self, rdn, attr_dict): + """ + Add LDAP entry + + Keyword arguments: + rdn -- DN without domain + attr_dict -- Dictionnary of attributes/values to add + + Returns: + Boolean | MoulinetteError + + """ + dn = rdn + ',' + self.basedn + ldif = modlist.addModlist(attr_dict) + + try: + self.con.add_s(dn, ldif) + except: + raise MoulinetteError(169, _('An error occured during LDAP entry creation')) + else: + return True + + def remove(self, rdn): + """ + Remove LDAP entry + + Keyword arguments: + rdn -- DN without domain + + Returns: + Boolean | MoulinetteError + + """ + dn = rdn + ',' + self.basedn + try: + self.con.delete_s(dn) + except: + raise MoulinetteError(169, _('An error occured during LDAP entry deletion')) + else: + return True + + def update(self, rdn, attr_dict, new_rdn=False): + """ + Modify LDAP entry + + Keyword arguments: + rdn -- DN without domain + attr_dict -- Dictionnary of attributes/values to add + new_rdn -- New RDN for modification + + Returns: + Boolean | MoulinetteError + + """ + dn = rdn + ',' + self.basedn + actual_entry = self.search(base=dn, attrs=None) + ldif = modlist.modifyModlist(actual_entry[0], attr_dict, ignore_oldexistent=1) + + try: + if new_rdn: + self.con.rename_s(dn, new_rdn) + dn = new_rdn + ',' + self.basedn + + self.con.modify_ext_s(dn, ldif) + except: + raise MoulinetteError(169, _('An error occured during LDAP entry update')) + else: + return True + + def validate_uniqueness(self, value_dict): + """ + Check uniqueness of values + + Keyword arguments: + value_dict -- Dictionnary of attributes/values to check + + Returns: + Boolean | MoulinetteError + + """ + for attr, value in value_dict.items(): + if not self.search(filter=attr + '=' + value): + continue + else: + raise MoulinetteError(17, _('Attribute already exists') + ' "' + attr + '=' + value + '"') + return True diff --git a/src/moulinette/core.py b/src/moulinette/core.py index 1b2805e9..ee4d6a1b 100644 --- a/src/moulinette/core.py +++ b/src/moulinette/core.py @@ -3,10 +3,11 @@ import os import sys import time +import errno import gettext import logging -from .helpers import colorize +from importlib import import_module # Package manipulation ------------------------------------------------- @@ -132,189 +133,25 @@ class Package(object): # Authenticators ------------------------------------------------------- -import ldap -import gnupg +def init_authenticator((vendor, name), kwargs={}): + """Return a new authenticator instance -class AuthenticationError(Exception): - pass + Retrieve the given authenticator vendor and return a new instance of + its Authenticator class for the given profile. -class _BaseAuthenticator(object): + Keyword arguments: + - vendor -- The authenticator vendor name + - name -- The authenticator profile name + - kwargs -- A dict of arguments for the authenticator profile - def __init__(self, name): - self._name = name - - @property - def name(self): - """Return the name of the authenticator instance""" - return self._name - - - ## Virtual properties - # Each authenticator classes must implement these properties. - - """The vendor of the authenticator""" - vendor = None - - @property - def is_authenticated(self): - """Either the instance is authenticated or not""" - raise NotImplementedError("derived class '%s' must override this property" % \ - self.__class__.__name__) - - - ## Virtual methods - # Each authenticator classes must implement these methods. - - def authenticate(password=None): - """Attempt to authenticate - - Attempt to authenticate with given password. It should raise an - AuthenticationError exception if authentication fails. - - Keyword arguments: - - password -- A clear text password - - """ - raise NotImplementedError("derived class '%s' must override this method" % \ - self.__class__.__name__) - - - ## Authentication methods - - def __call__(self, password=None, token=None): - """Attempt to authenticate - - Attempt to authenticate either with password or with session - token if 'password' is None. If the authentication succeed, the - instance is returned and the session is registered for the token - if 'token' and 'password' are given. - The token is composed by the session identifier and a session - hash - to use for encryption - as a 2-tuple. - - Keyword arguments: - - password -- A clear text password - - token -- The session token in the form of (id, hash) - - Returns: - The authenticated instance - - """ - if self.is_authenticated: - return self - store_session = True if password and token else False - - if token: - try: - # Extract id and hash from token - s_id, s_hash = token - except TypeError: - if not password: - raise MoulinetteError(22, _("Invalid format for token")) - else: - # TODO: Log error - store_session = False - else: - if password is None: - # Retrieve session - password = self._retrieve_session(s_id, s_hash) - - try: - # Attempt to authenticate - self.authenticate(password) - except AuthenticationError as e: - raise MoulinetteError(13, str(e)) - except Exception as e: - logging.error("authentication (name: '%s', type: '%s') fails: %s" % \ - (self.name, self.vendor, e)) - raise MoulinetteError(13, _("Unable to authenticate")) - - # Store session - if store_session: - self._store_session(s_id, s_hash, password) - - return self - - - ## Private methods - - def _open_sessionfile(self, session_id, mode='r'): - """Open a session file for this instance in given mode""" - return pkg.open_cachefile('%s.asc' % session_id, mode, - subdir='session/%s' % self.name) - - def _store_session(self, session_id, session_hash, password): - """Store a session and its associated password""" - gpg = gnupg.GPG() - gpg.encoding = 'utf-8' - with self._open_sessionfile(session_id, 'w') as f: - f.write(str(gpg.encrypt(password, None, symmetric=True, - passphrase=session_hash))) - - def _retrieve_session(self, session_id, session_hash): - """Retrieve a session and return its associated password""" - try: - with self._open_sessionfile(session_id, 'r') as f: - enc_pwd = f.read() - except IOError: - # TODO: Set proper error code - raise MoulinetteError(167, _("Unable to retrieve session")) - else: - gpg = gnupg.GPG() - gpg.encoding = 'utf-8' - return str(gpg.decrypt(enc_pwd, passphrase=session_hash)) - - -class LDAPAuthenticator(_BaseAuthenticator): - - def __init__(self, uri, base_dn, user_rdn=None, **kwargs): - super(LDAPAuthenticator, self).__init__(**kwargs) - - self.uri = uri - self.basedn = base_dn - if user_rdn: - self.userdn = '%s,%s' % (user_rdn, base_dn) - self.con = None - else: - # Initialize anonymous usage - self.userdn = '' - self.authenticate(None) - - - ## Implement virtual properties - - vendor = 'ldap' - - @property - def is_authenticated(self): - try: - # Retrieve identity - who = self.con.whoami_s() - except: - return False - else: - if who[3:] == self.userdn: - return True - return False - - - ## Implement virtual methods - - def authenticate(self, password): - try: - con = ldap.initialize(self.uri) - if self.userdn: - con.simple_bind_s(self.userdn, password) - else: - con.simple_bind_s() - except ldap.INVALID_CREDENTIALS: - raise AuthenticationError(_("Invalid password")) - else: - self.con = con - - -def init_authenticator(_name, _vendor, **kwargs): - if _vendor == 'ldap': - return LDAPAuthenticator(name=_name, **kwargs) + """ + try: + mod = import_module('moulinette.authenticators.%s' % vendor) + except ImportError: + # TODO: List available authenticator vendors + raise MoulinetteError(errno.EINVAL, _("Unknown authenticator vendor '%s'" % vendor)) + else: + return mod.Authenticator(name, **kwargs) def clean_session(session_id, profiles=[]): sessiondir = pkg.get_cachedir('session') @@ -330,44 +167,9 @@ def clean_session(session_id, profiles=[]): # Moulinette core classes ---------------------------------------------- -class MoulinetteError(Exception): - """Moulinette base exception - - Keyword arguments: - - code -- Integer error code - - message -- Error message to display - - """ - def __init__(self, code, message): - self.code = code - self.message = message - - errorcode_desc = { - 1 : _('Fail'), - 13 : _('Permission denied'), - 17 : _('Already exists'), - 22 : _('Invalid arguments'), - 87 : _('Too many users'), - 111 : _('Connection refused'), - 122 : _('Quota exceeded'), - 125 : _('Operation canceled'), - 167 : _('Not found'), - 168 : _('Undefined'), - 169 : _('LDAP operation error') - } - if code in errorcode_desc: - self.desc = errorcode_desc[code] - else: - self.desc = _('Error %s' % code) - - def __str__(self, colorized=False): - desc = self.desc - if colorized: - desc = colorize(self.desc, 'red') - return _('%s: %s' % (desc, self.message)) - - def colorize(self): - return self.__str__(colorized=True) +class MoulinetteError(OSError): + """Moulinette base exception""" + pass class MoulinetteLock(object): @@ -409,8 +211,8 @@ class MoulinetteLock(object): break if (time.time() - start_time) > self.timeout: - raise MoulinetteError(1, _("An instance is already running for '%s'") \ - % self.namespace) + raise MoulinetteError(errno.EBUSY, _("An instance is already running for '%s'") \ + % self.namespace) # Wait before checking again time.sleep(self.interval) self._locked = True diff --git a/src/moulinette/interface/api.py b/src/moulinette/interface/api.py index a47d000e..9ff67309 100644 --- a/src/moulinette/interface/api.py +++ b/src/moulinette/interface/api.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- +import errno from bottle import run, request, response, Bottle, HTTPResponse from json import dumps as json_encode -from ..core import MoulinetteError, clean_session -from ..helpers import YunoHostError, YunoHostLDAP +from moulinette.core import MoulinetteError, clean_session +from moulinette.helpers import YunoHostError, YunoHostLDAP # API helpers ---------------------------------------------------------- @@ -159,10 +160,9 @@ class _ActionsMapPlugin(object): if len(s_hashes) > 0: try: self.logout(profile) except: pass - # TODO: Replace by proper exception - if e.code == 13: - raise HTTPUnauthorizedResponse(e.message) - raise HTTPErrorResponse(e.message) + if e.errno == errno.EACCES: + raise HTTPUnauthorizedResponse(e.strerror) + raise HTTPErrorResponse(e.strerror) else: # Update dicts with new values s_hashes[profile] = s_hash @@ -210,14 +210,14 @@ class _ActionsMapPlugin(object): try: ret = self.actionsmap.process(arguments, route=_route) except MoulinetteError as e: - raise HTTPErrorResponse(e.message) + raise HTTPErrorResponse(e.strerror) else: return ret ## Signals handlers - def _do_authenticate(self, authenticator, name, help): + def _do_authenticate(self, authenticator, help): """Process the authentication Handle the actionsmap._AMapSignals.authenticate signal. @@ -227,12 +227,12 @@ class _ActionsMapPlugin(object): try: s_secret = self.secrets[s_id] s_hash = request.get_cookie('session.hashes', - secret=s_secret)[name] + secret=s_secret)[authenticator.name] except KeyError: - if name == 'default': + if authenticator.name == 'default': msg = _("Needing authentication") else: - msg = _("Needing authentication to profile '%s'") % name + msg = _("Needing authentication to profile '%s'") % authenticator.name raise HTTPUnauthorizedResponse(msg) else: return authenticator(token=(s_id, s_hash)) @@ -287,7 +287,12 @@ class MoulinetteAPI(object): - _port -- Port number to run on """ - run(self._app, port=_port) + try: + run(self._app, port=_port) + except IOError as e: + if e.args[0] == errno.EADDRINUSE: + raise MoulinetteError(errno.EADDRINUSE, _("A server is already running")) + raise ## Routes handlers diff --git a/src/moulinette/interface/cli.py b/src/moulinette/interface/cli.py index 1262b336..576ed622 100644 --- a/src/moulinette/interface/cli.py +++ b/src/moulinette/interface/cli.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- +import errno import getpass -from ..core import MoulinetteError +from moulinette.core import MoulinetteError # CLI helpers ---------------------------------------------------------- @@ -64,7 +65,7 @@ class MoulinetteCLI(object): """Moulinette command-line Interface Initialize an interface connected to the standard input and output - stream which allows to process moulinette action. + stream which allows to process moulinette actions. Keyword arguments: - actionsmap -- The interface relevant ActionsMap instance @@ -90,7 +91,7 @@ class MoulinetteCLI(object): try: ret = self.actionsmap.process(args, timeout=5) except KeyboardInterrupt, EOFError: - raise MoulinetteError(125, _("Interrupted")) + raise MoulinetteError(errno.EINTR, _("Interrupted")) if isinstance(ret, dict): pretty_print_dict(ret) @@ -100,7 +101,7 @@ class MoulinetteCLI(object): ## Signals handlers - def _do_authenticate(self, authenticator, name, help): + def _do_authenticate(self, authenticator, help): """Process the authentication Handle the actionsmap._AMapSignals.authenticate signal. @@ -124,6 +125,6 @@ class MoulinetteCLI(object): if confirm: if prompt(_('Retype %s: ') % message) != value: - raise MoulinetteError(22, _("Values don't match")) + raise MoulinetteError(errno.EINVAL, _("Values don't match")) return value From b3af4ddaea25f8e572a5fd79ca13d04d9cf1abf9 Mon Sep 17 00:00:00 2001 From: Jerome Lebleu Date: Tue, 25 Mar 2014 18:13:44 +0100 Subject: [PATCH 14/18] One more refactoring in interfaces * Move actions map parsers classes into their respective interface modules * Introduce an Interface base class for a futur usage * Each interfaces must now implement ActionsMapParser and Interface classes * Standardize interface instantiation --- bin/yunohost-api | 4 +- src/moulinette/__init__.py | 23 +- src/moulinette/actionsmap.py | 569 +----------------- src/moulinette/authenticators/__init__.py | 6 +- src/moulinette/core.py | 57 +- src/moulinette/interface/__init__.py | 0 src/moulinette/interfaces/__init__.py | 307 ++++++++++ .../{interface => interfaces}/api.py | 227 ++++++- .../{interface => interfaces}/cli.py | 91 ++- 9 files changed, 679 insertions(+), 605 deletions(-) delete mode 100644 src/moulinette/interface/__init__.py create mode 100644 src/moulinette/interfaces/__init__.py rename src/moulinette/{interface => interfaces}/api.py (60%) rename src/moulinette/{interface => interfaces}/cli.py (55%) diff --git a/bin/yunohost-api b/bin/yunohost-api index 8bd869d0..2baf31b4 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -45,7 +45,7 @@ if __name__ == '__main__': api(['yunohost', 'test'], 6787, {('GET', '/installed'): is_installed}, use_cache) except MoulinetteError as e: - from moulinette.interface.cli import colorize + from moulinette.interfaces.cli import colorize print(_('%s: %s' % (colorize(_('Error'), 'red'), e.strerror))) - sys.exit(e.code) + sys.exit(e.errno) sys.exit(0) diff --git a/src/moulinette/__init__.py b/src/moulinette/__init__.py index f589a881..04272431 100755 --- a/src/moulinette/__init__.py +++ b/src/moulinette/__init__.py @@ -26,10 +26,10 @@ __credits__ = """ """ __all__ = [ 'init', 'api', 'cli', - 'MoulinetteError', + 'init_interface', 'MoulinetteError', ] -from moulinette.core import MoulinetteError +from moulinette.core import init_interface, MoulinetteError ## Package functions @@ -76,12 +76,10 @@ def api(namespaces, port, routes={}, use_cache=True): instead of using the cached one """ - from moulinette.actionsmap import ActionsMap - from moulinette.interface.api import MoulinetteAPI - - amap = ActionsMap('api', namespaces, use_cache) - moulinette = MoulinetteAPI(amap, routes) - + moulinette = init_interface('api', + kwargs={'routes': routes}, + actionsmap={'namespaces': namespaces, + 'use_cache': use_cache}) moulinette.run(port) def cli(namespaces, args, use_cache=True): @@ -97,13 +95,12 @@ def cli(namespaces, args, use_cache=True): instead of using the cached one """ - from moulinette.actionsmap import ActionsMap - from moulinette.interface.cli import MoulinetteCLI, colorize + from moulinette.interfaces.cli import colorize try: - amap = ActionsMap('cli', namespaces, use_cache) - moulinette = MoulinetteCLI(amap) - + moulinette = init_interface('cli', + actionsmap={'namespaces': namespaces, + 'use_cache': use_cache}) moulinette.run(args) except MoulinetteError as e: print(_('%s: %s' % (colorize(_('Error'), 'red'), e.strerror))) diff --git a/src/moulinette/actionsmap.py b/src/moulinette/actionsmap.py index ec28da5c..a60e3aac 100644 --- a/src/moulinette/actionsmap.py +++ b/src/moulinette/actionsmap.py @@ -3,19 +3,17 @@ import os import re import errno +import logging import yaml -import argparse import cPickle as pickle from collections import OrderedDict -import logging - -from moulinette.core import (MoulinetteError, MoulinetteLock, - init_authenticator) +from moulinette.core import (MoulinetteError, MoulinetteLock) +from moulinette.interfaces import BaseActionsMapParser ## Actions map Signals ------------------------------------------------- -class _AMapSignals(object): +class ActionsMapSignals(object): """Actions map's Signals interface Allow to easily connect signals of the actions map to handlers. They @@ -93,533 +91,7 @@ class _AMapSignals(object): def _notimplemented(**kwargs): raise NotImplementedError("this signal is not handled") -shandler = _AMapSignals() - - -## Interfaces' Actions map Parser -------------------------------------- - -class _AMapParser(object): - """Actions map's base Parser - - Each interfaces must implement a parser class derived from this - class. It is used to parse the main parts of the actions map (i.e. - global arguments, categories and actions). - - """ - def __init__(self, parent=None): - if parent: - self._o = parent - else: - self._o = self - self._global_conf = {} - self._conf = {} - - - ## Virtual properties - # Each parser classes must implement these properties. - - """The name of the interface for which it is the parser""" - name = None - - - ## Virtual methods - # Each parser classes must implement these methods. - - @staticmethod - def format_arg_names(name, full): - """Format argument name - - Format agument name depending on its 'full' parameter and return - a list of strings which will be used as name or option strings - for the argument parser. - - Keyword arguments: - - name -- The argument name - - full -- The argument's 'full' parameter - - Returns: - A list of option strings - - """ - raise NotImplementedError("derived class '%s' must override this method" % \ - self.__class__.__name__) - - def add_global_parser(self, **kwargs): - """Add a parser for global arguments - - Create and return an argument parser for global arguments. - - Returns: - An ArgumentParser based object - - """ - raise NotImplementedError("derived class '%s' must override this method" % \ - self.__class__.__name__) - - def add_category_parser(self, name, **kwargs): - """Add a parser for a category - - Create a new category and return a parser for it. - - Keyword arguments: - - name -- The category name - - Returns: - A BaseParser based object - - """ - raise NotImplementedError("derived class '%s' must override this method" % \ - self.__class__.__name__) - - def add_action_parser(self, name, tid, **kwargs): - """Add a parser for an action - - Create a new action and return an argument parser for it. - - Keyword arguments: - - name -- The action name - - tid -- The tuple identifier of the action - - Returns: - An ArgumentParser based object - - """ - raise NotImplementedError("derived class '%s' must override this method" % \ - self.__class__.__name__) - - def parse_args(self, args, **kwargs): - """Parse arguments - - Convert argument variables to objects and assign them as - attributes of the namespace. - - Keyword arguments: - - args -- Arguments string or dict (TODO) - - Returns: - The populated namespace - - """ - raise NotImplementedError("derived class '%s' must override this method" % \ - self.__class__.__name__) - - - ## Configuration access - - @property - def global_conf(self): - """Return the global configuration of the parser""" - return self._o._global_conf - - def get_global_conf(self, name, profile='default'): - """Get the global value of a configuration - - Return the formated global value of the configuration 'name' for - the given profile. If the configuration doesn't provide profile, - the formated default value is returned. - - Keyword arguments: - - name -- The configuration name - - profile -- The profile of the configuration - - """ - if name == 'authenticator': - value = self.global_conf[name][profile] - else: - value = self.global_conf[name] - return self._format_conf(name, value) - - def set_global_conf(self, configuration): - """Set global configuration - - Set the global configuration to use for the parser. - - Keyword arguments: - - configuration -- The global configuration - - """ - self._o._global_conf.update(self._validate_conf(configuration, True)) - - def get_conf(self, action, name): - """Get the value of an action configuration - - Return the formated value of configuration 'name' for the action - identified by 'action'. If the configuration for the action is - not set, the default one is returned. - - Keyword arguments: - - action -- An action identifier - - name -- The configuration name - - """ - try: - value = self._o._conf[action][name] - except KeyError: - return self.get_global_conf(name) - else: - return self._format_conf(name, value) - - def set_conf(self, action, configuration): - """Set configuration for an action - - Set the configuration to use for a given action identified by - 'action' which is specific to the parser. - - Keyword arguments: - - action -- The action identifier - - configuration -- The configuration for the action - - """ - self._o._conf[action] = self._validate_conf(configuration) - - - def _validate_conf(self, configuration, is_global=False): - """Validate configuration for the parser - - Return the validated configuration for the interface's actions - map parser. - - Keyword arguments: - - configuration -- The configuration to pre-format - - """ - # TODO: Create a class with a validator method for each configuration - conf = {} - - # -- 'authenficate' - try: - ifaces = configuration['authenticate'] - except KeyError: - pass - else: - if ifaces == 'all': - conf['authenticate'] = ifaces - elif ifaces == False: - conf['authenticate'] = False - elif isinstance(ifaces, list): - # Store only if authentication is needed - conf['authenticate'] = True if self.name in ifaces else False - else: - # TODO: Log error instead and tell valid values - raise MoulinetteError(errno.EINVAL, "Invalid value '%r' for configuration 'authenticate'" % ifaces) - - # -- 'authenticator' - try: - auth = configuration['authenticator'] - except KeyError: - pass - else: - if not is_global and isinstance(auth, str): - try: - # Store needed authenticator profile - conf['authenticator'] = self.global_conf['authenticator'][auth] - except KeyError: - raise MoulinetteError(errno.EINVAL, "Undefined authenticator '%s' in global configuration" % auth) - elif is_global and isinstance(auth, dict): - if len(auth) == 0: - logging.warning('no authenticator defined in global configuration') - else: - auths = {} - for auth_name, auth_conf in auth.items(): - # Add authenticator profile as a 3-tuple - # (identifier, configuration, parameters) with - # - identifier: the authenticator vendor and its - # profile name as a 2-tuple - # - configuration: a dict of additional global - # configuration (i.e. 'help') - # - parameters: a dict of arguments for the - # authenticator profile - auths[auth_name] = ((auth_conf.get('vendor'), auth_name), - { 'help': auth_conf.get('help', None) }, - auth_conf.get('parameters', {})) - conf['authenticator'] = auths - else: - # TODO: Log error instead and tell valid values - raise MoulinetteError(errno.EINVAL, "Invalid value '%r' for configuration 'authenticator'" % auth) - - # -- 'argument_auth' - try: - arg_auth = configuration['argument_auth'] - except KeyError: - pass - else: - if isinstance(arg_auth, bool): - conf['argument_auth'] = arg_auth - else: - # TODO: Log error instead and tell valid values - raise MoulinetteError(errno.EINVAL, "Invalid value '%r' for configuration 'argument_auth'" % arg_auth) - - return conf - - def _format_conf(self, name, value): - """Format a configuration value - - Return the formated value of the configuration 'name' from its - given value. - - Keyword arguments: - - name -- The name of the configuration - - value -- The value to format - - """ - if name == 'authenticator' and value: - (identifier, configuration, parameters) = value - - # Return global configuration and an authenticator - # instanciator as a 2-tuple - return (configuration, - lambda: init_authenticator(identifier, parameters)) - - return value - -# CLI Actions map Parser - -class CLIAMapParser(_AMapParser): - """Actions map's CLI Parser - - """ - def __init__(self, parent=None, parser=None): - super(CLIAMapParser, self).__init__(parent) - - self._parser = parser or argparse.ArgumentParser() - self._subparsers = self._parser.add_subparsers() - - - ## Implement virtual properties - - name = 'cli' - - - ## Implement virtual methods - - @staticmethod - def format_arg_names(name, full): - if name[0] == '-' and full: - return [name, full] - return [name] - - def add_global_parser(self, **kwargs): - return self._parser - - def add_category_parser(self, name, category_help=None, **kwargs): - """Add a parser for a category - - Keyword arguments: - - category_help -- A brief description for the category - - Returns: - A new CLIParser object for the category - - """ - parser = self._subparsers.add_parser(name, help=category_help) - return self.__class__(self, parser) - - def add_action_parser(self, name, tid, action_help=None, **kwargs): - """Add a parser for an action - - Keyword arguments: - - action_help -- A brief description for the action - - Returns: - A new argparse.ArgumentParser object for the action - - """ - return self._subparsers.add_parser(name, help=action_help) - - def parse_args(self, args, **kwargs): - ret = self._parser.parse_args(args) - - # Perform authentication if needed - if self.get_conf(ret._tid, 'authenticate'): - auth_conf, klass = self.get_conf(ret._tid, 'authenticator') - - # TODO: Catch errors - auth = shandler.authenticate(klass(), **auth_conf) - if not auth.is_authenticated: - # TODO: Set proper error code - raise MoulinetteError(errno.EACCES, _("This action need authentication")) - if self.get_conf(ret._tid, 'argument_auth') and \ - self.get_conf(ret._tid, 'authenticate') == 'all': - ret.auth = auth - - return ret - -# API Actions map Parser - -class _HTTPArgumentParser(object): - """Argument parser for HTTP requests - - Object for parsing HTTP requests into Python objects. It is based - on argparse.ArgumentParser class and implements some of its methods. - - """ - def __init__(self): - # Initialize the ArgumentParser object - self._parser = argparse.ArgumentParser(usage='', - prefix_chars='@', - add_help=False) - self._parser.error = self._error - - self._positional = [] # list(arg_name) - self._optional = {} # dict({arg_name: option_strings}) - - def set_defaults(self, **kwargs): - return self._parser.set_defaults(**kwargs) - - def get_default(self, dest): - return self._parser.get_default(dest) - - def add_argument(self, *args, **kwargs): - action = self._parser.add_argument(*args, **kwargs) - - # Append newly created action - if len(action.option_strings) == 0: - self._positional.append(action.dest) - else: - self._optional[action.dest] = action.option_strings - - return action - - def parse_args(self, args={}, namespace=None): - arg_strings = [] - - ## Append an argument to the current one - def append(arg_strings, value, option_string=None): - # TODO: Process list arguments - if isinstance(value, bool): - # Append the option string only - if option_string is not None: - arg_strings.append(option_string) - elif isinstance(value, str): - if option_string is not None: - arg_strings.append(option_string) - arg_strings.append(value) - else: - arg_strings.append(value) - - return arg_strings - - # Iterate over positional arguments - for dest in self._positional: - if dest in args: - arg_strings = append(arg_strings, args[dest]) - - # Iterate over optional arguments - for dest, opt in self._optional.items(): - if dest in args: - arg_strings = append(arg_strings, args[dest], opt[0]) - return self._parser.parse_args(arg_strings, namespace) - - def _error(self, message): - # TODO: Raise a proper exception - raise MoulinetteError(1, message) - -class APIAMapParser(_AMapParser): - """Actions map's API Parser - - """ - def __init__(self): - super(APIAMapParser, self).__init__() - - self._parsers = {} # dict({(method, path): _HTTPArgumentParser}) - - @property - def routes(self): - """Get current routes""" - return self._parsers.keys() - - - ## Implement virtual properties - - name = 'api' - - - ## Implement virtual methods - - @staticmethod - def format_arg_names(name, full): - if name[0] != '-': - return [name] - if full: - return [full.replace('--', '@', 1)] - if name.startswith('--'): - return [name.replace('--', '@', 1)] - return [name.replace('-', '@', 1)] - - def add_global_parser(self, **kwargs): - raise AttributeError("global arguments are not managed") - - def add_category_parser(self, name, **kwargs): - return self - - def add_action_parser(self, name, tid, api=None, **kwargs): - """Add a parser for an action - - Keyword arguments: - - api -- The action route (e.g. 'GET /' ) - - Returns: - A new _HTTPArgumentParser object for the route - - """ - try: - # Validate action route - m = re.match('(GET|POST|PUT|DELETE) (/\S+)', api) - except TypeError: - raise AttributeError("the action '%s' doesn't provide api access" % name) - if not m: - # TODO: Log error - raise ValueError("the action '%s' doesn't provide api access" % name) - - # Check if a parser already exists for the route - key = (m.group(1), m.group(2)) - if key in self.routes: - raise AttributeError("a parser for '%s' already exists" % key) - - # Create and append parser - parser = _HTTPArgumentParser() - self._parsers[key] = (tid, parser) - - # Return the created parser - return parser - - def parse_args(self, args, route, **kwargs): - """Parse arguments - - Keyword arguments: - - route -- The action route as a 2-tuple (method, path) - - """ - try: - # Retrieve the tid and the parser for the route - tid, parser = self._parsers[route] - except KeyError: - raise MoulinetteError(errno.EINVAL, "No parser found for route '%s'" % route) - ret = argparse.Namespace() - - # Perform authentication if needed - if self.get_conf(tid, 'authenticate'): - auth_conf, klass = self.get_conf(tid, 'authenticator') - - # TODO: Catch errors - auth = shandler.authenticate(klass(), **auth_conf) - if not auth.is_authenticated: - # TODO: Set proper error code - raise MoulinetteError(errno.EACCES, _("This action need authentication")) - if self.get_conf(tid, 'argument_auth') and \ - self.get_conf(tid, 'authenticate') == 'all': - ret.auth = auth - - return parser.parse_args(args, ret) - -""" -The dict of interfaces names and their associated parser class. - -""" -actionsmap_parsers = { - 'api': APIAMapParser, - 'cli': CLIAMapParser -} +shandler = ActionsMapSignals() ## Extra parameters ---------------------------------------------------- @@ -850,35 +322,30 @@ class ExtraArgumentParser(object): class ActionsMap(object): """Validate and process actions defined into an actions map - The actions map defines the features and their usage of the main - application. It is composed by categories which contain one or more - action(s). Moreover, the action can have specific argument(s). + The actions map defines the features - and their usage - of an + application which will be available through the moulinette. + It is composed by categories which contain one or more action(s). + Moreover, the action can have specific argument(s). This class allows to manipulate one or several actions maps associated to a namespace. If no namespace is given, it will load all available namespaces. Keyword arguments: - - interface -- The type of interface which needs the actions map. - Possible values are: - - 'cli' for the command line interface - - 'api' for an API usage (HTTP requests) + - parser -- The BaseActionsMapParser derived class to use for + parsing the actions map - namespaces -- The list of namespaces to use - use_cache -- False if it should parse the actions map file instead of using the cached one. """ - def __init__(self, interface, namespaces=[], use_cache=True): + def __init__(self, parser, namespaces=[], use_cache=True): self.use_cache = use_cache - self.interface = interface + if not issubclass(parser, BaseActionsMapParser): + raise MoulinetteError(errno.EINVAL, _("Invalid parser class '%s'" % parser.__name__)) + self._parser_class = parser - try: - # Retrieve the interface parser - self._parser_class = actionsmap_parsers[interface] - except KeyError: - raise MoulinetteError(errno.EINVAL, _("Unknown interface '%s'" % interface)) - - logging.debug("initializing ActionsMap for the '%s' interface" % interface) + logging.debug("initializing ActionsMap for the interface '%s'" % parser.interface) if len(namespaces) == 0: namespaces = self.get_namespaces() @@ -903,7 +370,7 @@ class ActionsMap(object): actionsmaps[n] = yaml.load(f) # Generate parsers - self.extraparser = ExtraArgumentParser(interface) + self.extraparser = ExtraArgumentParser(parser.interface) self._parser = self._construct_parser(actionsmaps) @property @@ -1063,7 +530,7 @@ class ActionsMap(object): parser.set_defaults(_extra=extras) # Instantiate parser - top_parser = self._parser_class() + top_parser = self._parser_class(shandler) # Iterate over actions map namespaces for n, actionsmap in actionsmaps.items(): diff --git a/src/moulinette/authenticators/__init__.py b/src/moulinette/authenticators/__init__.py index 4d98d7ac..601f7d06 100644 --- a/src/moulinette/authenticators/__init__.py +++ b/src/moulinette/authenticators/__init__.py @@ -12,8 +12,10 @@ class BaseAuthenticator(object): """Authenticator base representation Each authenticators must implement an Authenticator class derived - from this class. It implements base methods to authenticate with a - password or a session token. + from this class which must overrides virtual properties and methods. + It is used to authenticate and manage session. It implements base + methods to authenticate with a password or a session token. + Authenticators configurations are identified by a profile name which must be given on instantiation - with the corresponding vendor configuration of the authenticator. diff --git a/src/moulinette/core.py b/src/moulinette/core.py index ee4d6a1b..72ae8ea4 100644 --- a/src/moulinette/core.py +++ b/src/moulinette/core.py @@ -131,7 +131,48 @@ class Package(object): return open('%s/%s' % (self.get_cachedir(**kwargs), filename), mode) -# Authenticators ------------------------------------------------------- +# Interfaces & Authenticators management ------------------------------- + +def init_interface(name, kwargs={}, actionsmap={}): + """Return a new interface instance + + Retrieve the given interface module and return a new instance of its + Interface class. It is initialized with arguments 'kwargs' and + connected to 'actionsmap' if it's an ActionsMap object, otherwise + a new ActionsMap instance will be initialized with arguments + 'actionsmap'. + + Keyword arguments: + - name -- The interface name + - kwargs -- A dict of arguments to pass to Interface + - actionsmap -- Either an ActionsMap instance or a dict of + arguments to pass to ActionsMap + + """ + from moulinette.actionsmap import ActionsMap + + try: + mod = import_module('moulinette.interfaces.%s' % name) + except ImportError: + # TODO: List available interfaces + raise MoulinetteError(errno.EINVAL, _("Unknown interface '%s'" % name)) + else: + try: + # Retrieve interface classes + parser = mod.ActionsMapParser + interface = mod.Interface + except AttributeError as e: + raise MoulinetteError(errno.EFAULT, _("Invalid interface '%s': %s") % (name, e)) + + # Instantiate or retrieve ActionsMap + if isinstance(actionsmap, dict): + amap = ActionsMap(actionsmap.pop('parser', parser), **actionsmap) + elif isinstance(actionsmap, ActionsMap): + amap = actionsmap + else: + raise MoulinetteError(errno.EINVAL, _("Invalid actions map '%r'" % actionsmap)) + + return interface(amap, **kwargs) def init_authenticator((vendor, name), kwargs={}): """Return a new authenticator instance @@ -148,14 +189,24 @@ def init_authenticator((vendor, name), kwargs={}): try: mod = import_module('moulinette.authenticators.%s' % vendor) except ImportError: - # TODO: List available authenticator vendors + # TODO: List available authenticators vendors raise MoulinetteError(errno.EINVAL, _("Unknown authenticator vendor '%s'" % vendor)) else: return mod.Authenticator(name, **kwargs) def clean_session(session_id, profiles=[]): + """Clean a session cache + + Remove cache for the session 'session_id' and for profiles in + 'profiles' or for all of them if the list is empty. + + Keyword arguments: + - session_id -- The session id to clean + - profiles -- A list of profiles to clean + + """ sessiondir = pkg.get_cachedir('session') - if len(profiles) == 0: + if not profiles: profiles = os.listdir(sessiondir) for p in profiles: diff --git a/src/moulinette/interface/__init__.py b/src/moulinette/interface/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/moulinette/interfaces/__init__.py b/src/moulinette/interfaces/__init__.py new file mode 100644 index 00000000..778d7dc9 --- /dev/null +++ b/src/moulinette/interfaces/__init__.py @@ -0,0 +1,307 @@ +# -*- coding: utf-8 -*- + +import errno +import logging + +from moulinette.core import (init_authenticator, MoulinetteError) + +# Base Class ----------------------------------------------------------- + +class BaseActionsMapParser(object): + """Actions map's base Parser + + Each interfaces must implement an ActionsMapParser class derived + from this class which must overrides virtual properties and methods. + It is used to parse the main parts of the actions map (i.e. global + arguments, categories and actions). It implements methods to set/get + the global and actions configuration. + + Keyword arguments: + - shandler -- A actionsmap.ActionsMapSignals instance + - parent -- A parent BaseActionsMapParser derived object + + """ + def __init__(self, shandler, parent=None): + if parent: + self.shandler = parent.shandler + self._o = parent + else: + self.shandler = shandler + self._o = self + self._global_conf = {} + self._conf = {} + + + ## Virtual properties + # Each parser classes must implement these properties. + + """The name of the interface for which it is the parser""" + interface = None + + + ## Virtual methods + # Each parser classes must implement these methods. + + @staticmethod + def format_arg_names(name, full): + """Format argument name + + Format agument name depending on its 'full' parameter and return + a list of strings which will be used as name or option strings + for the argument parser. + + Keyword arguments: + - name -- The argument name + - full -- The argument's 'full' parameter + + Returns: + A list of option strings + + """ + raise NotImplementedError("derived class '%s' must override this method" % \ + self.__class__.__name__) + + def add_global_parser(self, **kwargs): + """Add a parser for global arguments + + Create and return an argument parser for global arguments. + + Returns: + An ArgumentParser based object + + """ + raise NotImplementedError("derived class '%s' must override this method" % \ + self.__class__.__name__) + + def add_category_parser(self, name, **kwargs): + """Add a parser for a category + + Create a new category and return a parser for it. + + Keyword arguments: + - name -- The category name + + Returns: + A BaseParser based object + + """ + raise NotImplementedError("derived class '%s' must override this method" % \ + self.__class__.__name__) + + def add_action_parser(self, name, tid, **kwargs): + """Add a parser for an action + + Create a new action and return an argument parser for it. + + Keyword arguments: + - name -- The action name + - tid -- The tuple identifier of the action + + Returns: + An ArgumentParser based object + + """ + raise NotImplementedError("derived class '%s' must override this method" % \ + self.__class__.__name__) + + def parse_args(self, args, **kwargs): + """Parse arguments + + Convert argument variables to objects and assign them as + attributes of the namespace. + + Keyword arguments: + - args -- Arguments string or dict (TODO) + + Returns: + The populated namespace + + """ + raise NotImplementedError("derived class '%s' must override this method" % \ + self.__class__.__name__) + + + ## Configuration access + + @property + def global_conf(self): + """Return the global configuration of the parser""" + return self._o._global_conf + + def get_global_conf(self, name, profile='default'): + """Get the global value of a configuration + + Return the formated global value of the configuration 'name' for + the given profile. If the configuration doesn't provide profile, + the formated default value is returned. + + Keyword arguments: + - name -- The configuration name + - profile -- The profile of the configuration + + """ + if name == 'authenticator': + value = self.global_conf[name][profile] + else: + value = self.global_conf[name] + return self._format_conf(name, value) + + def set_global_conf(self, configuration): + """Set global configuration + + Set the global configuration to use for the parser. + + Keyword arguments: + - configuration -- The global configuration + + """ + self._o._global_conf.update(self._validate_conf(configuration, True)) + + def get_conf(self, action, name): + """Get the value of an action configuration + + Return the formated value of configuration 'name' for the action + identified by 'action'. If the configuration for the action is + not set, the default one is returned. + + Keyword arguments: + - action -- An action identifier + - name -- The configuration name + + """ + try: + value = self._o._conf[action][name] + except KeyError: + return self.get_global_conf(name) + else: + return self._format_conf(name, value) + + def set_conf(self, action, configuration): + """Set configuration for an action + + Set the configuration to use for a given action identified by + 'action' which is specific to the parser. + + Keyword arguments: + - action -- The action identifier + - configuration -- The configuration for the action + + """ + self._o._conf[action] = self._validate_conf(configuration) + + + def _validate_conf(self, configuration, is_global=False): + """Validate configuration for the parser + + Return the validated configuration for the interface's actions + map parser. + + Keyword arguments: + - configuration -- The configuration to pre-format + + """ + # TODO: Create a class with a validator method for each configuration + conf = {} + + # -- 'authenficate' + try: + ifaces = configuration['authenticate'] + except KeyError: + pass + else: + if ifaces == 'all': + conf['authenticate'] = ifaces + elif ifaces == False: + conf['authenticate'] = False + elif isinstance(ifaces, list): + # Store only if authentication is needed + conf['authenticate'] = True if self.interface in ifaces else False + else: + # TODO: Log error instead and tell valid values + raise MoulinetteError(errno.EINVAL, "Invalid value '%r' for configuration 'authenticate'" % ifaces) + + # -- 'authenticator' + try: + auth = configuration['authenticator'] + except KeyError: + pass + else: + if not is_global and isinstance(auth, str): + try: + # Store needed authenticator profile + conf['authenticator'] = self.global_conf['authenticator'][auth] + except KeyError: + raise MoulinetteError(errno.EINVAL, "Undefined authenticator '%s' in global configuration" % auth) + elif is_global and isinstance(auth, dict): + if len(auth) == 0: + logging.warning('no authenticator defined in global configuration') + else: + auths = {} + for auth_name, auth_conf in auth.items(): + # Add authenticator profile as a 3-tuple + # (identifier, configuration, parameters) with + # - identifier: the authenticator vendor and its + # profile name as a 2-tuple + # - configuration: a dict of additional global + # configuration (i.e. 'help') + # - parameters: a dict of arguments for the + # authenticator profile + auths[auth_name] = ((auth_conf.get('vendor'), auth_name), + { 'help': auth_conf.get('help', None) }, + auth_conf.get('parameters', {})) + conf['authenticator'] = auths + else: + # TODO: Log error instead and tell valid values + raise MoulinetteError(errno.EINVAL, "Invalid value '%r' for configuration 'authenticator'" % auth) + + # -- 'argument_auth' + try: + arg_auth = configuration['argument_auth'] + except KeyError: + pass + else: + if isinstance(arg_auth, bool): + conf['argument_auth'] = arg_auth + else: + # TODO: Log error instead and tell valid values + raise MoulinetteError(errno.EINVAL, "Invalid value '%r' for configuration 'argument_auth'" % arg_auth) + + return conf + + def _format_conf(self, name, value): + """Format a configuration value + + Return the formated value of the configuration 'name' from its + given value. + + Keyword arguments: + - name -- The name of the configuration + - value -- The value to format + + """ + if name == 'authenticator' and value: + (identifier, configuration, parameters) = value + + # Return global configuration and an authenticator + # instanciator as a 2-tuple + return (configuration, + lambda: init_authenticator(identifier, parameters)) + + return value + + +class BaseInterface(object): + """Moulinette's base Interface + + Each interfaces must implement an Interface class derived from this + class which must overrides virtual properties and methods. + It is used to provide a user interface for an actions map. + + Keyword arguments: + - actionsmap -- The ActionsMap instance to connect to + + """ + # TODO: Add common interface methods and try to standardize default ones + def __init__(self, actionsmap): + raise NotImplementedError("derived class '%s' must override this method" % \ + self.__class__.__name__) diff --git a/src/moulinette/interface/api.py b/src/moulinette/interfaces/api.py similarity index 60% rename from src/moulinette/interface/api.py rename to src/moulinette/interfaces/api.py index 9ff67309..9e1b975e 100644 --- a/src/moulinette/interface/api.py +++ b/src/moulinette/interfaces/api.py @@ -1,41 +1,89 @@ # -*- coding: utf-8 -*- +import os +import re import errno -from bottle import run, request, response, Bottle, HTTPResponse +import binascii +import argparse from json import dumps as json_encode +from bottle import run, request, response, Bottle, HTTPResponse from moulinette.core import MoulinetteError, clean_session -from moulinette.helpers import YunoHostError, YunoHostLDAP +from moulinette.interfaces import (BaseActionsMapParser, BaseInterface) # API helpers ---------------------------------------------------------- -import os -import binascii +def random_ascii(length=20): + """Return a random ascii string""" + return binascii.hexlify(os.urandom(length)).decode('ascii') -def random20(): - return binascii.hexlify(os.urandom(20)).decode('ascii') +class _HTTPArgumentParser(object): + """Argument parser for HTTP requests + Object for parsing HTTP requests into Python objects. It is based + on argparse.ArgumentParser class and implements some of its methods. -# HTTP Responses ------------------------------------------------------- + """ + def __init__(self): + # Initialize the ArgumentParser object + self._parser = argparse.ArgumentParser(usage='', + prefix_chars='@', + add_help=False) + self._parser.error = self._error -class HTTPOKResponse(HTTPResponse): - def __init__(self, output=''): - super(HTTPOKResponse, self).__init__(output, 200) + self._positional = [] # list(arg_name) + self._optional = {} # dict({arg_name: option_strings}) -class HTTPBadRequestResponse(HTTPResponse): - def __init__(self, output=''): - super(HTTPBadRequestResponse, self).__init__(output, 400) + def set_defaults(self, **kwargs): + return self._parser.set_defaults(**kwargs) -class HTTPUnauthorizedResponse(HTTPResponse): - def __init__(self, output=''): - super(HTTPUnauthorizedResponse, self).__init__(output, 401) + def get_default(self, dest): + return self._parser.get_default(dest) -class HTTPErrorResponse(HTTPResponse): - def __init__(self, output=''): - super(HTTPErrorResponse, self).__init__(output, 500) + def add_argument(self, *args, **kwargs): + action = self._parser.add_argument(*args, **kwargs) + # Append newly created action + if len(action.option_strings) == 0: + self._positional.append(action.dest) + else: + self._optional[action.dest] = action.option_strings -# API moulinette interface --------------------------------------------- + return action + + def parse_args(self, args={}, namespace=None): + arg_strings = [] + + ## Append an argument to the current one + def append(arg_strings, value, option_string=None): + # TODO: Process list arguments + if isinstance(value, bool): + # Append the option string only + if option_string is not None: + arg_strings.append(option_string) + elif isinstance(value, str): + if option_string is not None: + arg_strings.append(option_string) + arg_strings.append(value) + else: + arg_strings.append(value) + + return arg_strings + + # Iterate over positional arguments + for dest in self._positional: + if dest in args: + arg_strings = append(arg_strings, args[dest]) + + # Iterate over optional arguments + for dest, opt in self._optional.items(): + if dest in args: + arg_strings = append(arg_strings, args[dest], opt[0]) + return self._parser.parse_args(arg_strings, namespace) + + def _error(self, message): + # TODO: Raise a proper exception + raise MoulinetteError(1, message) class _ActionsMapPlugin(object): """Actions map Bottle Plugin @@ -142,7 +190,7 @@ class _ActionsMapPlugin(object): """ # Retrieve session values - s_id = request.get_cookie('session.id') or random20() + s_id = request.get_cookie('session.id') or random_ascii() try: s_secret = self.secrets[s_id] except KeyError: @@ -150,7 +198,7 @@ class _ActionsMapPlugin(object): else: s_hashes = request.get_cookie('session.hashes', secret=s_secret) or {} - s_hash = random20() + s_hash = random_ascii() try: # Attempt to authenticate @@ -166,7 +214,7 @@ class _ActionsMapPlugin(object): else: # Update dicts with new values s_hashes[profile] = s_hash - self.secrets[s_id] = s_secret = random20() + self.secrets[s_id] = s_secret = random_ascii() response.set_cookie('session.id', s_id, secure=True) response.set_cookie('session.hashes', s_hashes, secure=True, @@ -238,14 +286,137 @@ class _ActionsMapPlugin(object): return authenticator(token=(s_id, s_hash)) -class MoulinetteAPI(object): - """Moulinette Application Programming Interface +# HTTP Responses ------------------------------------------------------- - Initialize a HTTP server which serves the API to process moulinette - actions. +class HTTPOKResponse(HTTPResponse): + def __init__(self, output=''): + super(HTTPOKResponse, self).__init__(output, 200) + +class HTTPBadRequestResponse(HTTPResponse): + def __init__(self, output=''): + super(HTTPBadRequestResponse, self).__init__(output, 400) + +class HTTPUnauthorizedResponse(HTTPResponse): + def __init__(self, output=''): + super(HTTPUnauthorizedResponse, self).__init__(output, 401) + +class HTTPErrorResponse(HTTPResponse): + def __init__(self, output=''): + super(HTTPErrorResponse, self).__init__(output, 500) + + +# API Classes Implementation ------------------------------------------- + +class ActionsMapParser(BaseActionsMapParser): + """Actions map's Parser for the API + + Provide actions map parsing methods for a CLI usage. The parser for + the arguments is represented by a argparse.ArgumentParser object. + + """ + def __init__(self, shandler, parent=None): + super(ActionsMapParser, self).__init__(shandler, parent) + + self._parsers = {} # dict({(method, path): _HTTPArgumentParser}) + + @property + def routes(self): + """Get current routes""" + return self._parsers.keys() + + + ## Implement virtual properties + + name = 'api' + + + ## Implement virtual methods + + @staticmethod + def format_arg_names(name, full): + if name[0] != '-': + return [name] + if full: + return [full.replace('--', '@', 1)] + if name.startswith('--'): + return [name.replace('--', '@', 1)] + return [name.replace('-', '@', 1)] + + def add_global_parser(self, **kwargs): + raise AttributeError("global arguments are not managed") + + def add_category_parser(self, name, **kwargs): + return self + + def add_action_parser(self, name, tid, api=None, **kwargs): + """Add a parser for an action + + Keyword arguments: + - api -- The action route (e.g. 'GET /' ) + + Returns: + A new _HTTPArgumentParser object for the route + + """ + try: + # Validate action route + m = re.match('(GET|POST|PUT|DELETE) (/\S+)', api) + except TypeError: + raise AttributeError("the action '%s' doesn't provide api access" % name) + if not m: + # TODO: Log error + raise ValueError("the action '%s' doesn't provide api access" % name) + + # Check if a parser already exists for the route + key = (m.group(1), m.group(2)) + if key in self.routes: + raise AttributeError("a parser for '%s' already exists" % key) + + # Create and append parser + parser = _HTTPArgumentParser() + self._parsers[key] = (tid, parser) + + # Return the created parser + return parser + + def parse_args(self, args, route, **kwargs): + """Parse arguments + + Keyword arguments: + - route -- The action route as a 2-tuple (method, path) + + """ + try: + # Retrieve the tid and the parser for the route + tid, parser = self._parsers[route] + except KeyError: + raise MoulinetteError(errno.EINVAL, "No parser found for route '%s'" % route) + ret = argparse.Namespace() + + # Perform authentication if needed + if self.get_conf(tid, 'authenticate'): + auth_conf, klass = self.get_conf(tid, 'authenticator') + + # TODO: Catch errors + auth = self.shandler.authenticate(klass(), **auth_conf) + if not auth.is_authenticated: + # TODO: Set proper error code + raise MoulinetteError(errno.EACCES, _("This action need authentication")) + if self.get_conf(tid, 'argument_auth') and \ + self.get_conf(tid, 'authenticate') == 'all': + ret.auth = auth + + return parser.parse_args(args, ret) + + +class Interface(BaseInterface): + """Application Programming Interface for the moulinette + + Initialize a HTTP server which serves the API connected to a given + actions map. Keyword arguments: - - actionsmap -- The relevant ActionsMap instance + - actionsmap -- The ActionsMap instance to connect to - routes -- A dict of additional routes to add in the form of {(method, path): callback} diff --git a/src/moulinette/interface/cli.py b/src/moulinette/interfaces/cli.py similarity index 55% rename from src/moulinette/interface/cli.py rename to src/moulinette/interfaces/cli.py index 576ed622..597b1ff7 100644 --- a/src/moulinette/interface/cli.py +++ b/src/moulinette/interfaces/cli.py @@ -2,8 +2,10 @@ import errno import getpass +import argparse from moulinette.core import MoulinetteError +from moulinette.interfaces import (BaseActionsMapParser, BaseInterface) # CLI helpers ---------------------------------------------------------- @@ -59,16 +61,93 @@ def pretty_print_dict(d, depth=0): print((" ") * depth + "%s: %s" % (str(k), v)) -# Moulinette Interface ------------------------------------------------- +# CLI Classes Implementation ------------------------------------------- -class MoulinetteCLI(object): - """Moulinette command-line Interface +class ActionsMapParser(BaseActionsMapParser): + """Actions map's Parser for the CLI - Initialize an interface connected to the standard input and output - stream which allows to process moulinette actions. + Provide actions map parsing methods for a CLI usage. The parser for + the arguments is represented by a argparse.ArgumentParser object. Keyword arguments: - - actionsmap -- The interface relevant ActionsMap instance + - parser -- The argparse.ArgumentParser object to use + + """ + def __init__(self, shandler, parent=None, parser=None): + super(ActionsMapParser, self).__init__(shandler, parent) + + self._parser = parser or argparse.ArgumentParser() + self._subparsers = self._parser.add_subparsers() + + + ## Implement virtual properties + + interface = 'cli' + + + ## Implement virtual methods + + @staticmethod + def format_arg_names(name, full): + if name[0] == '-' and full: + return [name, full] + return [name] + + def add_global_parser(self, **kwargs): + return self._parser + + def add_category_parser(self, name, category_help=None, **kwargs): + """Add a parser for a category + + Keyword arguments: + - category_help -- A brief description for the category + + Returns: + A new ActionsMapParser object for the category + + """ + parser = self._subparsers.add_parser(name, help=category_help) + return self.__class__(None, self, parser) + + def add_action_parser(self, name, tid, action_help=None, **kwargs): + """Add a parser for an action + + Keyword arguments: + - action_help -- A brief description for the action + + Returns: + A new argparse.ArgumentParser object for the action + + """ + return self._subparsers.add_parser(name, help=action_help) + + def parse_args(self, args, **kwargs): + ret = self._parser.parse_args(args) + + # Perform authentication if needed + if self.get_conf(ret._tid, 'authenticate'): + auth_conf, klass = self.get_conf(ret._tid, 'authenticator') + + # TODO: Catch errors + auth = self.shandler.authenticate(klass(), **auth_conf) + if not auth.is_authenticated: + # TODO: Set proper error code + raise MoulinetteError(errno.EACCES, _("This action need authentication")) + if self.get_conf(ret._tid, 'argument_auth') and \ + self.get_conf(ret._tid, 'authenticate') == 'all': + ret.auth = auth + + return ret + + +class Interface(BaseInterface): + """Command-line Interface for the moulinette + + Initialize an interface connected to the standard input/output + stream and to a given actions map. + + Keyword arguments: + - actionsmap -- The ActionsMap instance to connect to """ def __init__(self, actionsmap): From fa0a7affddad6c4c1d12f0fd6dbf5b32dd978a7c Mon Sep 17 00:00:00 2001 From: Jerome Lebleu Date: Tue, 25 Mar 2014 18:56:51 +0100 Subject: [PATCH 15/18] Warn about non-reviewed yunohost package and clean up --- .gitignore | 7 +- bin/yunohost | 3 +- bin/yunohost-api | 2 +- lib/yunohost/app.py | 3 + lib/yunohost/backup.py | 3 + lib/yunohost/domain.py | 3 + lib/yunohost/dyndns.py | 3 + lib/yunohost/firewall.py | 3 + lib/yunohost/hook.py | 3 + lib/yunohost/monitor.py | 3 + lib/yunohost/service.py | 3 + lib/yunohost/tools.py | 3 + lib/yunohost/user.py | 3 + yunohost | 134 ------------------- yunohost.tac | 273 --------------------------------------- 15 files changed, 38 insertions(+), 411 deletions(-) delete mode 100755 yunohost delete mode 100755 yunohost.tac diff --git a/.gitignore b/.gitignore index 19bf7c38..4492aeb8 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ build eggs parts bin +cache var sdist develop-eggs @@ -22,10 +23,12 @@ pip-log.txt .coverage .tox -#Translations +# Translations *.mo -#Mr Developer +# Mr Developer .mr.developer.cfg +# Moulinette doc/*.json +src/moulinette/package.py diff --git a/bin/yunohost b/bin/yunohost index 001148a5..4c0919f7 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -35,8 +35,9 @@ if __name__ == '__main__': raise YunoHostError(17, _("YunoHost is not correctly installed, please execute 'yunohost tools postinstall'")) # Execute the action - ret = cli(['yunohost', 'test'], args, use_cache) + ret = cli(['yunohost'], args, use_cache) except YunoHostError as e: + # TODO: Remove this and associated import when yunohost package has been revisited print(colorize(_("Error: "), 'red') + e.message) sys.exit(e.code) sys.exit(ret) diff --git a/bin/yunohost-api b/bin/yunohost-api index 2baf31b4..1c1538a8 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -42,7 +42,7 @@ if __name__ == '__main__': try: # Run the server - api(['yunohost', 'test'], 6787, + api(['yunohost'], 6787, {('GET', '/installed'): is_installed}, use_cache) except MoulinetteError as e: from moulinette.interfaces.cli import colorize diff --git a/lib/yunohost/app.py b/lib/yunohost/app.py index eeb9808b..f75cae7a 100644 --- a/lib/yunohost/app.py +++ b/lib/yunohost/app.py @@ -23,6 +23,9 @@ Manage apps """ +import logging +logging.warning('the module yunohost.app has not been revisited and updated yet') + import os import sys import json diff --git a/lib/yunohost/backup.py b/lib/yunohost/backup.py index d4723d9d..390c8637 100644 --- a/lib/yunohost/backup.py +++ b/lib/yunohost/backup.py @@ -23,6 +23,9 @@ Manage backups """ +import logging +logging.warning('the module yunohost.backup has not been revisited and updated yet') + import os import sys import json diff --git a/lib/yunohost/domain.py b/lib/yunohost/domain.py index b10a92a7..ff0baaed 100644 --- a/lib/yunohost/domain.py +++ b/lib/yunohost/domain.py @@ -23,6 +23,9 @@ Manage domains """ +import logging +logging.warning('the module yunohost.backup has not been revisited and updated yet') + import os import sys import datetime diff --git a/lib/yunohost/dyndns.py b/lib/yunohost/dyndns.py index 1bd6b6ba..83a4db36 100644 --- a/lib/yunohost/dyndns.py +++ b/lib/yunohost/dyndns.py @@ -23,6 +23,9 @@ Subscribe and Update DynDNS Hosts """ +import logging +logging.warning('the module yunohost.dyndns has not been revisited and updated yet') + import os import sys import requests diff --git a/lib/yunohost/firewall.py b/lib/yunohost/firewall.py index 3fb68dea..ce011841 100644 --- a/lib/yunohost/firewall.py +++ b/lib/yunohost/firewall.py @@ -23,6 +23,9 @@ Manage firewall rules """ +import logging +logging.warning('the module yunohost.firewall has not been revisited and updated yet') + import os import sys try: diff --git a/lib/yunohost/hook.py b/lib/yunohost/hook.py index 9136acfb..7d57f733 100644 --- a/lib/yunohost/hook.py +++ b/lib/yunohost/hook.py @@ -23,6 +23,9 @@ Manage hooks """ +import logging +logging.warning('the module yunohost.hook has not been revisited and updated yet') + import os import sys import re diff --git a/lib/yunohost/monitor.py b/lib/yunohost/monitor.py index cd0a22ae..be01eadb 100644 --- a/lib/yunohost/monitor.py +++ b/lib/yunohost/monitor.py @@ -23,6 +23,9 @@ Monitoring functions """ +import logging +logging.warning('the module yunohost.monitor has not been revisited and updated yet') + import re import json import time diff --git a/lib/yunohost/service.py b/lib/yunohost/service.py index 139e1ca2..c3ce1fa9 100644 --- a/lib/yunohost/service.py +++ b/lib/yunohost/service.py @@ -23,6 +23,9 @@ Manage services """ +import logging +logging.warning('the module yunohost.service has not been revisited and updated yet') + import yaml import glob import subprocess diff --git a/lib/yunohost/tools.py b/lib/yunohost/tools.py index dd57d47f..e69dfac2 100644 --- a/lib/yunohost/tools.py +++ b/lib/yunohost/tools.py @@ -23,6 +23,9 @@ Specific tools """ +import logging +logging.warning('the module yunohost.tools has not been revisited and updated yet') + import os import sys import yaml diff --git a/lib/yunohost/user.py b/lib/yunohost/user.py index b619a227..06a09c7b 100644 --- a/lib/yunohost/user.py +++ b/lib/yunohost/user.py @@ -23,6 +23,9 @@ Manage users """ +import logging +logging.warning('the module yunohost.user has not been revisited and updated yet') + import os import sys import ldap diff --git a/yunohost b/yunohost deleted file mode 100755 index 5eebbe9b..00000000 --- a/yunohost +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import os -import sys -import argparse -import gettext -import getpass -try: - import yaml -except ImportError: - sys.stderr.write('Error: Yunohost CLI Require yaml lib\n') - sys.stderr.write('apt-get install python-yaml\n') - sys.exit(1) -import json -if not __debug__: - import traceback - -gettext.install('YunoHost') - -try: - from yunohost import YunoHostError, YunoHostLDAP, str_to_func, colorize, pretty_print_dict, display_error, validate, win, parse_dict -except ImportError: - sys.stderr.write('Error: Yunohost CLI Require YunoHost lib\n') - sys.exit(1) - - -def main(): - """ - Main instructions - - Parse the action_dict and execute the action-specific function, - then print json or pretty result if executed in a tty :) - - Returns: - int -- 0 or error code - - """ - - if len(sys.argv) < 2: - sys.argv.append('-h') - - with open('action_map.yml') as f: - action_map = yaml.load(f) - - admin_password_provided = False - json_print = False - write_ldap = True - postinstall = False - - for key, arg in enumerate(sys.argv): - if arg == '--admin-password': - admin_password_provided = True - admin_password = sys.argv[key+1] - sys.argv.pop(key) - sys.argv.pop(key) - if arg == '--no-ldap': - write_ldap = False - sys.argv.pop(key) - if arg == '--json': - json_print = True - sys.argv.pop(key) - - try: - try: - with open('/etc/yunohost/installed') as f: pass - except IOError: - postinstall = True - if len(sys.argv) < 3 or sys.argv[1] != 'tools' or sys.argv[2] != 'postinstall': - raise YunoHostError(17, _("YunoHost is not correctly installed, please execute 'yunohost tools postinstall'")) - - args = parse_dict(action_map) - args_dict = vars(args).copy() - for key in args_dict.keys(): - sanitized_key = key.replace('-', '_') - if sanitized_key is not key: - args_dict[sanitized_key] = args_dict[key] - del args_dict[key] - del args_dict['func'] - try: - with open('/etc/yunohost/passwd') as f: - admin_password = f.read() - admin_password_provided = True - except IOError: pass - if postinstall: - result = args.func(**args_dict) - elif admin_password_provided: - with YunoHostLDAP(password=admin_password): - result = args.func(**args_dict) - elif os.isatty(1) and write_ldap: - admin_password = getpass.getpass(colorize(_('Admin Password: '), 'yellow')) - with YunoHostLDAP(password=admin_password): - try: - with open('/var/run/yunohost.pid', 'r'): - raise YunoHostError(1, _("A YunoHost command is already running")) - except IOError: - with open('/var/run/yunohost.pid', 'w') as f: - f.write('ldap') - os.system('chmod 400 /var/run/yunohost.pid') - with open('/etc/yunohost/passwd', 'w') as f: - f.write(admin_password) - os.system('chmod 400 /etc/yunohost/passwd') - try: - result = args.func(**args_dict) - except KeyboardInterrupt, EOFError: - raise YunoHostError(125, _("Interrupted")) - finally: - os.remove('/etc/yunohost/passwd') - os.remove('/var/run/yunohost.pid') - else: - with YunoHostLDAP(anonymous=True): - result = args.func(**args_dict) - #except TypeError, error: - #if not __debug__ : - #traceback.print_exc() - #print(_("Not (yet) implemented function")) - #return 1 - except YunoHostError, error: - display_error(error, json_print) - return error.code - else: - if json_print or not os.isatty(1) and result is not None: - if len(win) > 0: - result['success'] = win - print(json.dumps(result)) - elif result is not None: - pretty_print_dict(result) - else: - pass - - return 0 - -if __name__ == '__main__': - sys.exit(main()) diff --git a/yunohost.tac b/yunohost.tac deleted file mode 100755 index b3d1ac61..00000000 --- a/yunohost.tac +++ /dev/null @@ -1,273 +0,0 @@ -# -*- mode: python -*- -import os -import sys -import gettext -import ldap -import yaml -import json - -sys.path.append('/usr/share/pyshared') - -from twisted.python.log import ILogObserver, FileLogObserver, startLogging, msg -from twisted.python.logfile import DailyLogFile -from twisted.web.server import Site, http -from twisted.internet import reactor -from twisted.application import internet,service -from txrestapi.resource import APIResource -from yunohost import YunoHostError, YunoHostLDAP, str_to_func, colorize, pretty_print_dict, display_error, validate, win, parse_dict -import yunohost - -if not __debug__: - import traceback - -gettext.install('YunoHost') - -dev = False -installed = True -action_dict = {} -api = APIResource() - -def http_exec(request, **kwargs): - global installed - - request.setHeader('Access-Control-Allow-Origin', '*') # Allow cross-domain requests - request.setHeader('Content-Type', 'application/json') # Return JSON anyway - - # Return OK to 'OPTIONS' xhr requests - if request.method == 'OPTIONS': - request.setResponseCode(200, 'OK') - request.setHeader('Access-Control-Allow-Headers', 'Authorization, Content-Type') - request.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS') - return '' - - # Simple HTTP auth - elif installed: - authorized = False - pwd = request.getPassword() - if request.getUser() == 'admin' and pwd != '': - authorized = True - if dev and 'api_key' in request.args: - pwd = request.args['api_key'][0] - authorized = True - if authorized: - try: YunoHostLDAP(password=pwd) - except YunoHostError: authorized = False - if not authorized: - request.setResponseCode(401, 'Unauthorized') - request.setHeader('Access-Control-Allow-Origin', '*') - request.setHeader('www-authenticate', 'Basic realm="Restricted Area"') - return 'Unauthorized' - - path = request.path - if request.method == 'PUT': - given_args = http.parse_qs(request.content.read(), 1) - else: - given_args = request.args - if kwargs: - for k, v in kwargs.iteritems(): - dynamic_key = path.split('/')[-1] - path = path.replace(dynamic_key, '{'+ k +'}') - given_args[k] = [v] - - #msg(given_args) - # Sanitize arguments - dict = action_dict[request.method +' '+ path] - if 'arguments' in dict: possible_args = dict['arguments'] - else: possible_args = {} - for arg, params in possible_args.items(): - sanitized_key = arg.replace('-', '_') - if sanitized_key is not arg: - possible_args[sanitized_key] = possible_args[arg] - del possible_args[arg] - arg = sanitized_key - if arg[0] == '_': - if 'nargs' not in params: - possible_args[arg]['nargs'] = '*' - if 'full' in params: - new_key = params['full'][2:] - else: - new_key = arg[2:] - new_key = new_key.replace('-', '_') - possible_args[new_key] = possible_args[arg] - del possible_args[arg] - - try: - - # Validate arguments - validated_args = {} - for key, value in given_args.items(): - if key in possible_args: - # Validate args - if 'pattern' in possible_args[key]: - validate(possible_args[key]['pattern'], value) - if 'nargs' not in possible_args[key] or ('nargs' != '*' and 'nargs' != '+'): - value = value[0] - if 'choices' in possible_args[key] and value not in possible_args[key]['choices']: - raise YunoHostError(22, _('Invalid argument') + ' ' + value) - if 'action' in possible_args[key] and possible_args[key]['action'] == 'store_true': - yes = ['true', 'True', 'yes', 'Yes'] - value = value in yes - validated_args[key] = value - - func = str_to_func(dict['function']) - if func is None: - raise YunoHostError(168, _('Function not yet implemented : ') + dict['function'].split('.')[1]) - - # Execute requested function - try: - with open('/var/run/yunohost.pid', 'r'): - raise YunoHostError(1, _("A YunoHost command is already running")) - except IOError: - if dict['function'].split('.')[1] != 'tools_postinstall': - try: - with open('/etc/yunohost/installed'): pass - except IOError: - raise YunoHostError(1, _("You must run postinstall before any other actions")) - with open('/var/run/yunohost.pid', 'w') as f: - f.write('ldap') - os.system('chmod 400 /var/run/yunohost.pid') - with open('/etc/yunohost/passwd', 'w') as f: - f.write(request.getPassword()) - os.system('chmod 400 /etc/yunohost/passwd') - try: - result = func(**validated_args) - except KeyboardInterrupt, EOFError: - raise YunoHostError(125, _("Interrupted")) - finally: - try: - os.remove('/etc/yunohost/passwd') - os.remove('/var/run/yunohost.pid') - except: pass - if result is None: - result = {} - if len(yunohost.win) > 0: - result['win'] = yunohost.win - yunohost.win = [] - - # Build response - if request.method == 'POST': - request.setResponseCode(201, 'Created') - if not installed: - installed = True - elif request.method == 'DELETE': - request.setResponseCode(204, 'No Content') - else: - request.setResponseCode(200, 'OK') - - except YunoHostError, error: - - # Set response code with function's raised code - server_errors = [1, 111, 168, 169] - client_errors = [13, 17, 22, 87, 122, 125, 167] - if error.code in client_errors: - request.setResponseCode(400, 'Bad Request') - else: - request.setResponseCode(500, 'Internal Server Error') - - result = { 'error' : error.message } - - return json.dumps(result) - -def api_doc(request): - request.setHeader('Access-Control-Allow-Origin', '*') # Allow cross-domain requests - request.setHeader('Content-Type', 'application/json') # Return JSON anyway - - # Return OK to 'OPTIONS' xhr requests - if request.method == 'OPTIONS': - request.setResponseCode(200, 'OK') - request.setHeader('Access-Control-Allow-Headers', 'Authorization') - return '' - - if request.path == '/api': - with open('doc/resources.json') as f: - return f.read() - - category = request.path.split('/')[2] - try: - with open('doc/'+ category +'.json') as f: - return f.read() - except IOError: - return '' - -def favicon(request): - request.setHeader('Access-Control-Allow-Origin', '*') # Allow cross-domain requests - request.setResponseCode(404, 'Not Found') - return '' - -def is_installed(request): - global installed - - try: - with open('/etc/yunohost/installed'): - installed = True - except IOError: - installed = False - request.setHeader('Access-Control-Allow-Origin', '*') # Allow cross-domain requests - request.setResponseCode(200, 'OK') - return json.dumps({ 'installed': installed }) - -def main(): - global action_dict - global api - global installed - - # Generate API doc - os.system('python ./generate_api_doc.py') - - # Register API doc service - api.register('ALL', '/api', api_doc) - - # favicon.ico error - api.register('ALL', '/favicon.ico', favicon) - - # Load & parse yaml file - with open('action_map.yml') as f: - action_map = yaml.load(f) - - # Register only postinstall action if YunoHost isn't completely set up - try: - with open('/etc/yunohost/installed'): - installed = True - except IOError: - installed = False - - del action_map['general_arguments'] - for category, category_params in action_map.items(): - api.register('ALL', '/api/'+ category, api_doc) - for action, action_params in category_params['actions'].items(): - if 'action_help' not in action_params: - action_params['action_help'] = '' - if 'api' not in action_params: - action_params['api'] = 'GET /'+ category +'/'+ action - method, path = action_params['api'].split(' ') - # Register route - if '{' in path: - path = path.replace('{', '(?P<').replace('}', '>[^/]+)') - api.register(method, path, http_exec) - api.register('OPTIONS', path, http_exec) - action_dict[action_params['api']] = { - 'function': 'yunohost_'+ category +'.'+ category +'_'+ action.replace('-', '_'), - 'help' : action_params['action_help'] - } - if 'arguments' in action_params: - action_dict[action_params['api']]['arguments'] = action_params['arguments'] - - api.register('ALL', '/installed', is_installed) - - - -if __name__ == '__main__': - if '--dev' in sys.argv: - dev = True - startLogging(sys.stdout) - else: - startLogging(open('/var/log/yunohost.log', 'a+')) # Log actions to file - main() - reactor.listenTCP(6787, Site(api, timeout=None)) - reactor.run() -else: - application = service.Application("YunoHost API") - logfile = DailyLogFile("yunohost.log", "/var/log") - application.setComponent(ILogObserver, FileLogObserver(logfile).emit) - main() - internet.TCPServer(6787, Site(api, timeout=None)).setServiceParent(application) From f4d2f2b062b49d24df24bbaefc2ba8d6eaa02aec Mon Sep 17 00:00:00 2001 From: Jerome Lebleu Date: Wed, 26 Mar 2014 01:44:58 +0100 Subject: [PATCH 16/18] Update README.md --- README.md | 203 +++++++++++------------------------------------------- 1 file changed, 39 insertions(+), 164 deletions(-) diff --git a/README.md b/README.md index a65e36c4..bdbd720c 100644 --- a/README.md +++ b/README.md @@ -1,172 +1,47 @@ -YunoHost CLI -============ +Moulinette +========== + +The *moulinette* is a Python package that allows to quickly and easily +deploy an application for different interfaces. -Specifications --------------- +Overview +-------- + +Initially, the moulinette was an application made for the +[YunoHost](https://yunohost.org/) project in order to regroup all its +related operations into a single program called *moulinette*. Those +operations were available from a command-line interface and a Web server +providing an API. Moreover, the usage of these operations (e.g. +required/optional arguments) was defined into a simple yaml file - +called *actionsmap*. This file was parsed in order to construct an +*ArgumentParser* object and to parse the command arguments to process +the proper operation. + +During a long refactoring with the goal of unify both interfaces, the +idea to separate the core of the YunoHost operations has emerged. +The core kept the same name *moulinette* and try to follow the same +initial principle. An [Actions Map](#actions-map) - which defines +available operations and their usage - is parsed and it's used to +process an operation from several unified [Interfaces](#interfaces). It +also supports a configuration mechanism - which allows to restrict an +operation on an interface for example (see +[Authenticators](#authenticators)). -### User +### Actions Map +... - yunohost user list [-h] [--fields FIELDS [FIELDS ...]] [-o OFFSET] - [-f FILTER] [-l LIMIT] - yunohost user create [-h] [-u USERNAME] [-l LASTNAME] [-f FIRSTNAME] - [-p PASSWORD] [-m MAIL] - yunohost user delete [-h] [--purge] users [users ...] - yunohost user update [-h] [-f FIRSTNAME] - [--add-mailalias MAIL [MAIL ...]] [-m MAIL] - [-l LASTNAME] - [--remove-mailforward MAIL [MAIL ...]] - [--remove-mailalias MAIL [MAIL ...]] - [--add-mailforward MAIL [MAIL ...]] - [-p PASSWORD] - user - yunohost user info [-h] [-u USER] [-m MAIL] - - -### Domain - - yunohost domain list [-h] [-l LIMIT] [-o OFFSET] [-f FILTER] - yunohost domain add [-h] [domain [domain ...]] - yunohost domain remove [-h] [domain [domain ...]] - yunohost domain info [-h] domain - yunohost domain renewcert [-h] domain - - -### App - - yunohost app updatelist [-h] [-u URL] - yunohost app list [-h] [--fields FIELDS [FIELDS ...]] [-o OFFSET] - [-f FILTER] [-l LIMIT] - yunohost app install [-h] [-d DOMAIN] [--public] [-l LABEL] [-p PATH] - [--protected] - app - yunohost app remove [-h] app [app ...] - yunohost app upgrade [-h] [app [app ...]] - yunohost app info [-h] app - yunohost app addaccess [-h] [-u USER [USER ...]] app [app ...] - yunohost app removeaccess [-h] [-u USER [USER ...]] app [app ...] - - -### Firewall - - yunohost firewall list [-h] - yunohost firewall allow [-h] {UDP,TCP,Both} port name - yunohost firewall disallow [-h] name - - -### Monitoring - - yunohost monitor disk [-h] [-m MOUNTPOINT] [-t] [-f] [-H] - yunohost monitor network [-h] [-u] [-i] [-H] - yunohost monitor system [-h] [-m] [-u] [-i] [-p] [-c] [-H] - - -### Services - - yunohost service status [-h] [NAME [NAME ...]] - yunohost service start [-h] NAME [NAME ...] - yunohost service stop [-h] NAME [NAME ...] - yunohost service enable [-h] NAME [NAME ...] - yunohost service disable [-h] NAME [NAME ...] - - -### Tools - - yunohost tools postinstall [-h] [-d DOMAIN] [-p PASSWORD] - yunohost tools maindomain [-h] [-o OLD_DOMAIN] [-n NEW_DOMAIN] - yunohost tools adminpw [-h] [-o OLD_PASSWORD] [-n NEW_PASSWORD] - yunohost tools ldapinit [-h] [-d DOMAIN] - - -How to use "as is" ? --------------------- - -The executable file is yunohost, for example: - - ./yunohost user create - - -Contribute / FAQ ----------------- - -*What a lovely idea !* :) - -### Dafuq is dat moulinette ? -We decided to regroup all YunoHost related operations into a single program called "moulinette". This will allow us to entirely manipulate our YunoHost instances through a wonderful CLI. Additionally the web interface will just have to call the same "moulinette" functions. Magic power inside :p - -### Important files -* `` yunohost `` File executed on function calling - i.e `` ./yunohost user create ``. -* `` action_map.yml `` Defines all CLI actions and links arguments. -* `` yunohost.py `` Contains all YunoHost functions likely to be shared between moulinette files. Also contains service connections classes (erk). -* `` yunohost_*.py `` Files containing action functions. `` * `` is the category: user, domain, firewall, etc. - -### How to add a function ? -1. Check if the action is already in the `` action_map.yml `` file. If not, follow the file documentation to add it. -2. Also check if the file `` yunohost_category.py `` is created in the working tree. If not, just create it (you may take example of `` yunohost_user.py `` file). -3. Add your function `` category_action() `` in this file - i.e `` user_create() `` - -**Note:** `` category_action() `` takes one parameter,`` args `` which contains the arguments passed to the command. Refers to `` action_map.yml `` documentation for more informations. - -### Error handling -Moulinette has a unified way to handle errors. First, you need to import the ``YunoHostError`` exception: -`` from yunohost import YunoHostError `` - -Then you need to raise errors like this: -`` raise YunoHostError(, ) `` - -For example: -`` raise YunoHostError(125, _("Interrupted, user not created")) `` - -**Note:** Standard error codes can be found in the ``YunoHostError`` class in `` yunohost.py `` file. - -### Print results -Moulinette also have a unified way to print results. In fact we don't only print result for the CLI, but we also have to export the result in a JSON way. -Results are automatically printed OR exported, you don't have to print it yourself in the action's functions. Your function just need is to return results as a dictionary, for example: -`` return { 'Fullname' : 'Homer Simpson', 'Mail' : 'homer@simpson.org', 'Username' : 'hsimpson' } `` - -### i18n -We will have to translate YunoHost, and we have already initialized i18n module in the moulinette. As a result, do not forget to put potentially translated strings into `` _() `` function. For example: -`` raise YunoHostError(125, _("Interrupted, user not created")) `` - -### Git is pissing me off ! -OK, this is the workflow ! - -**For gitlab:** -Development is handle with git branches and you have your own (i.e dev_beudbeud). -``` -git clone git@dev.yunohost.org:moulinette.git -git checkout -b dev_beudbeud `` -git rebase origin/dev -``` - - -Do your modifications, then : -``` -git commit -am 'My commit message' -git pull origin dev (merge manually if conflicts) -git push origin dev_beudbeud -``` - -Then you could ask for a 'merge request' in gitlab. - -**For github (here):** -Development is handle with forked repos and you have your own (i.e beudbeud/moulinette). -``` -git clone https://github.com/beudbeud/moulinette.git `` -git checkout -b dev -git rebase origin/dev -``` - -Do your modifications, then: -``` -git commit -am 'My commit message' -git remote add vanilla https://github.com/YunoHost/moulinette.git -git pull vanilla dev (merge manually if conflicts) -git push origin dev -``` - -Then you could ask for a 'pull request' in github. +### Interfaces +... + +### Authenticators +... +Requirements +------------ +* Python 2.7 +* python-gnupg (>= 0.3) +* python-ldap (>= 2.4) From 7258e21adbebe498da67b26099900c2bb3a61dc4 Mon Sep 17 00:00:00 2001 From: Jerome Lebleu Date: Wed, 26 Mar 2014 01:50:14 +0100 Subject: [PATCH 17/18] Move src/moulinette to the root --- bin/yunohost | 9 ++++++--- bin/yunohost-api | 13 +++++++------ {src/moulinette => moulinette}/__init__.py | 0 {src/moulinette => moulinette}/actionsmap.py | 0 .../authenticators/__init__.py | 0 .../authenticators/ldap.py | 0 {src/moulinette => moulinette}/core.py | 0 {src/moulinette => moulinette}/helpers.py | 0 .../interfaces/__init__.py | 0 {src/moulinette => moulinette}/interfaces/api.py | 0 {src/moulinette => moulinette}/interfaces/cli.py | 0 {src/moulinette => moulinette}/package.py | 0 {src/moulinette => moulinette}/package.py.in | 0 13 files changed, 13 insertions(+), 9 deletions(-) rename {src/moulinette => moulinette}/__init__.py (100%) rename {src/moulinette => moulinette}/actionsmap.py (100%) rename {src/moulinette => moulinette}/authenticators/__init__.py (100%) rename {src/moulinette => moulinette}/authenticators/ldap.py (100%) rename {src/moulinette => moulinette}/core.py (100%) rename {src/moulinette => moulinette}/helpers.py (100%) rename {src/moulinette => moulinette}/interfaces/__init__.py (100%) rename {src/moulinette => moulinette}/interfaces/api.py (100%) rename {src/moulinette => moulinette}/interfaces/cli.py (100%) rename {src/moulinette => moulinette}/package.py (100%) rename {src/moulinette => moulinette}/package.py.in (100%) diff --git a/bin/yunohost b/bin/yunohost index 4c0919f7..6b253334 100755 --- a/bin/yunohost +++ b/bin/yunohost @@ -4,10 +4,13 @@ import sys import os.path +from_source = False + # Run from source basedir = os.path.abspath('%s/../' % os.path.dirname(__file__)) -if os.path.isdir('%s/src' % basedir): - sys.path.insert(0, '%s/src' % basedir) +if os.path.isdir('%s/moulinette' % basedir): + sys.path.insert(0, basedir) + from_source = True from moulinette import init, cli, MoulinetteError from moulinette.helpers import YunoHostError, colorize @@ -17,7 +20,7 @@ from moulinette.helpers import YunoHostError, colorize if __name__ == '__main__': # Run from source - init(_from_source=True) + init(_from_source=from_source) # Additional arguments use_cache = True diff --git a/bin/yunohost-api b/bin/yunohost-api index 1c1538a8..744e16b7 100755 --- a/bin/yunohost-api +++ b/bin/yunohost-api @@ -4,10 +4,13 @@ import sys import os.path +from_source = False + # Run from source -basedir = os.path.abspath(os.path.dirname(__file__) +'/../') -if os.path.isdir(basedir +'/src'): - sys.path.append(basedir +'/src') +basedir = os.path.abspath('%s/../' % os.path.dirname(__file__)) +if os.path.isdir('%s/moulinette' % basedir): + sys.path.insert(0, basedir) + from_source = True from moulinette import init, api, MoulinetteError @@ -29,15 +32,13 @@ def is_installed(): if __name__ == '__main__': # Run from source - init(_from_source=True) + init(_from_source=from_source) # Additional arguments use_cache = True if '--no-cache' in sys.argv: use_cache = False sys.argv.remove('--no-cache') - if '--debug' in sys.argv: - sys.argv.remove('--debug') # TODO: Add log argument try: diff --git a/src/moulinette/__init__.py b/moulinette/__init__.py similarity index 100% rename from src/moulinette/__init__.py rename to moulinette/__init__.py diff --git a/src/moulinette/actionsmap.py b/moulinette/actionsmap.py similarity index 100% rename from src/moulinette/actionsmap.py rename to moulinette/actionsmap.py diff --git a/src/moulinette/authenticators/__init__.py b/moulinette/authenticators/__init__.py similarity index 100% rename from src/moulinette/authenticators/__init__.py rename to moulinette/authenticators/__init__.py diff --git a/src/moulinette/authenticators/ldap.py b/moulinette/authenticators/ldap.py similarity index 100% rename from src/moulinette/authenticators/ldap.py rename to moulinette/authenticators/ldap.py diff --git a/src/moulinette/core.py b/moulinette/core.py similarity index 100% rename from src/moulinette/core.py rename to moulinette/core.py diff --git a/src/moulinette/helpers.py b/moulinette/helpers.py similarity index 100% rename from src/moulinette/helpers.py rename to moulinette/helpers.py diff --git a/src/moulinette/interfaces/__init__.py b/moulinette/interfaces/__init__.py similarity index 100% rename from src/moulinette/interfaces/__init__.py rename to moulinette/interfaces/__init__.py diff --git a/src/moulinette/interfaces/api.py b/moulinette/interfaces/api.py similarity index 100% rename from src/moulinette/interfaces/api.py rename to moulinette/interfaces/api.py diff --git a/src/moulinette/interfaces/cli.py b/moulinette/interfaces/cli.py similarity index 100% rename from src/moulinette/interfaces/cli.py rename to moulinette/interfaces/cli.py diff --git a/src/moulinette/package.py b/moulinette/package.py similarity index 100% rename from src/moulinette/package.py rename to moulinette/package.py diff --git a/src/moulinette/package.py.in b/moulinette/package.py.in similarity index 100% rename from src/moulinette/package.py.in rename to moulinette/package.py.in From a855326fadc68fe0761dc07d18754906565650d2 Mon Sep 17 00:00:00 2001 From: Jerome Lebleu Date: Wed, 26 Mar 2014 01:54:44 +0100 Subject: [PATCH 18/18] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index bdbd720c..3ec05a9a 100644 --- a/README.md +++ b/README.md @@ -43,5 +43,6 @@ Requirements ------------ * Python 2.7 +* python-bottle (>= 0.10) * python-gnupg (>= 0.3) * python-ldap (>= 2.4)