mirror of
https://github.com/YunoHost/moulinette.git
synced 2024-09-03 20:06:31 +02:00
Merge pull request #245 from YunoHost/simplify-interface-init
Simplify interface initialization + ugly optimization hack to speed up parser building
This commit is contained in:
commit
4635c555d8
10 changed files with 142 additions and 254 deletions
|
@ -1,7 +1,6 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
from moulinette.core import (
|
from moulinette.core import (
|
||||||
init_interface,
|
|
||||||
MoulinetteError,
|
MoulinetteError,
|
||||||
MoulinetteSignals,
|
MoulinetteSignals,
|
||||||
Moulinette18n,
|
Moulinette18n,
|
||||||
|
@ -72,85 +71,58 @@ def init(logging_config=None, **kwargs):
|
||||||
|
|
||||||
|
|
||||||
# Easy access to interfaces
|
# Easy access to interfaces
|
||||||
|
def api(host="localhost", port=80, routes={}):
|
||||||
|
|
||||||
def api(
|
|
||||||
namespaces, host="localhost", port=80, routes={}, use_websocket=True, use_cache=True
|
|
||||||
):
|
|
||||||
"""Web server (API) interface
|
"""Web server (API) interface
|
||||||
|
|
||||||
Run a HTTP server with the moulinette for an API usage.
|
Run a HTTP server with the moulinette for an API usage.
|
||||||
|
|
||||||
Keyword arguments:
|
Keyword arguments:
|
||||||
- namespaces -- The list of namespaces to use
|
|
||||||
- host -- Server address to bind to
|
- host -- Server address to bind to
|
||||||
- port -- Server port to bind to
|
- port -- Server port to bind to
|
||||||
- routes -- A dict of additional routes to add in the form of
|
- routes -- A dict of additional routes to add in the form of
|
||||||
{(method, uri): callback}
|
{(method, uri): callback}
|
||||||
- use_websocket -- Serve via WSGI to handle asynchronous responses
|
|
||||||
- use_cache -- False if it should parse the actions map file
|
|
||||||
instead of using the cached one
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
from moulinette.interfaces.api import Interface as Api
|
||||||
|
|
||||||
try:
|
try:
|
||||||
moulinette = init_interface(
|
Api(routes=routes).run(host, port)
|
||||||
"api",
|
|
||||||
kwargs={"routes": routes, "use_websocket": use_websocket},
|
|
||||||
actionsmap={"namespaces": namespaces, "use_cache": use_cache},
|
|
||||||
)
|
|
||||||
moulinette.run(host, port)
|
|
||||||
except MoulinetteError as e:
|
except MoulinetteError as e:
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
logging.getLogger(namespaces[0]).error(e.strerror)
|
logging.getLogger().error(e.strerror)
|
||||||
return e.errno if hasattr(e, "errno") else 1
|
return 1
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
logging.getLogger(namespaces[0]).info(m18n.g("operation_interrupted"))
|
logging.getLogger().info(m18n.g("operation_interrupted"))
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def cli(
|
def cli(args, top_parser, output_as=None, timeout=None):
|
||||||
namespaces,
|
|
||||||
args,
|
|
||||||
use_cache=True,
|
|
||||||
output_as=None,
|
|
||||||
password=None,
|
|
||||||
timeout=None,
|
|
||||||
parser_kwargs={},
|
|
||||||
):
|
|
||||||
"""Command line interface
|
"""Command line interface
|
||||||
|
|
||||||
Execute an action with the moulinette from the CLI and print its
|
Execute an action with the moulinette from the CLI and print its
|
||||||
result in a readable format.
|
result in a readable format.
|
||||||
|
|
||||||
Keyword arguments:
|
Keyword arguments:
|
||||||
- namespaces -- The list of namespaces to use
|
|
||||||
- args -- A list of argument strings
|
- args -- A list of argument strings
|
||||||
- use_cache -- False if it should parse the actions map file
|
|
||||||
instead of using the cached one
|
|
||||||
- output_as -- Output result in another format, see
|
- output_as -- Output result in another format, see
|
||||||
moulinette.interfaces.cli.Interface for possible values
|
moulinette.interfaces.cli.Interface for possible values
|
||||||
- password -- The password to use in case of authentication
|
- top_parser -- The top parser used to build the ActionsMapParser
|
||||||
- parser_kwargs -- A dict of arguments to pass to the parser
|
|
||||||
class at construction
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
from moulinette.interfaces.cli import Interface as Cli
|
||||||
|
|
||||||
try:
|
try:
|
||||||
moulinette = init_interface(
|
load_only_category = args[0] if args and not args[0].startswith("-") else None
|
||||||
"cli",
|
Cli(top_parser=top_parser, load_only_category=load_only_category).run(
|
||||||
actionsmap={
|
args, output_as=output_as, timeout=timeout
|
||||||
"namespaces": namespaces,
|
|
||||||
"use_cache": use_cache,
|
|
||||||
"parser_kwargs": parser_kwargs,
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
moulinette.run(args, output_as=output_as, password=password, timeout=timeout)
|
|
||||||
except MoulinetteError as e:
|
except MoulinetteError as e:
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
logging.getLogger(namespaces[0]).error(e.strerror)
|
logging.getLogger().error(e.strerror)
|
||||||
return 1
|
return 1
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
|
@ -4,6 +4,7 @@ import os
|
||||||
import re
|
import re
|
||||||
import logging
|
import logging
|
||||||
import yaml
|
import yaml
|
||||||
|
import glob
|
||||||
import cPickle as pickle
|
import cPickle as pickle
|
||||||
from time import time
|
from time import time
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
|
@ -282,7 +283,6 @@ class ExtraArgumentParser(object):
|
||||||
if iface in klass.skipped_iface:
|
if iface in klass.skipped_iface:
|
||||||
continue
|
continue
|
||||||
self.extra[klass.name] = klass
|
self.extra[klass.name] = klass
|
||||||
logger.debug("extra parameter classes loaded: %s", self.extra.keys())
|
|
||||||
|
|
||||||
def validate(self, arg_name, parameters):
|
def validate(self, arg_name, parameters):
|
||||||
"""
|
"""
|
||||||
|
@ -398,36 +398,32 @@ class ActionsMap(object):
|
||||||
Moreover, the action can have specific argument(s).
|
Moreover, the action can have specific argument(s).
|
||||||
|
|
||||||
This class allows to manipulate one or several actions maps
|
This class allows to manipulate one or several actions maps
|
||||||
associated to a namespace. If no namespace is given, it will load
|
associated to a namespace.
|
||||||
all available namespaces.
|
|
||||||
|
|
||||||
Keyword arguments:
|
Keyword arguments:
|
||||||
- parser_class -- The BaseActionsMapParser derived class to use
|
- top_parser -- A BaseActionsMapParser-derived instance to use for
|
||||||
for parsing the actions map
|
parsing the actions map
|
||||||
- namespaces -- The list of namespaces to use
|
- load_only_category -- A name of a category that should only be the
|
||||||
- use_cache -- False if it should parse the actions map file
|
one loaded because it's been already determined
|
||||||
instead of using the cached one
|
that's the only one relevant ... used for optimization
|
||||||
- parser_kwargs -- A dict of arguments to pass to the parser
|
purposes...
|
||||||
class at construction
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, parser_class, namespaces=[], use_cache=True, parser_kwargs={}):
|
def __init__(self, top_parser, load_only_category=None):
|
||||||
if not issubclass(parser_class, BaseActionsMapParser):
|
|
||||||
raise ValueError("Invalid parser class '%s'" % parser_class.__name__)
|
assert isinstance(top_parser, BaseActionsMapParser), (
|
||||||
self.parser_class = parser_class
|
"Invalid parser class '%s'" % top_parser.__class__.__name__
|
||||||
self.use_cache = use_cache
|
)
|
||||||
|
|
||||||
moulinette_env = init_moulinette_env()
|
moulinette_env = init_moulinette_env()
|
||||||
DATA_DIR = moulinette_env["DATA_DIR"]
|
DATA_DIR = moulinette_env["DATA_DIR"]
|
||||||
CACHE_DIR = moulinette_env["CACHE_DIR"]
|
CACHE_DIR = moulinette_env["CACHE_DIR"]
|
||||||
|
|
||||||
if len(namespaces) == 0:
|
|
||||||
namespaces = self.get_namespaces()
|
|
||||||
actionsmaps = OrderedDict()
|
actionsmaps = OrderedDict()
|
||||||
|
|
||||||
|
self.from_cache = False
|
||||||
# Iterate over actions map namespaces
|
# Iterate over actions map namespaces
|
||||||
for n in namespaces:
|
for n in self.get_namespaces():
|
||||||
logger.debug("loading actions map namespace '%s'", n)
|
logger.debug("loading actions map namespace '%s'", n)
|
||||||
|
|
||||||
actionsmap_yml = "%s/actionsmap/%s.yml" % (DATA_DIR, n)
|
actionsmap_yml = "%s/actionsmap/%s.yml" % (DATA_DIR, n)
|
||||||
|
@ -439,33 +435,36 @@ class ActionsMap(object):
|
||||||
actionsmap_yml_stat.st_mtime,
|
actionsmap_yml_stat.st_mtime,
|
||||||
)
|
)
|
||||||
|
|
||||||
if use_cache and os.path.exists(actionsmap_pkl):
|
if os.path.exists(actionsmap_pkl):
|
||||||
try:
|
try:
|
||||||
# Attempt to load cache
|
# Attempt to load cache
|
||||||
with open(actionsmap_pkl) as f:
|
with open(actionsmap_pkl) as f:
|
||||||
actionsmaps[n] = pickle.load(f)
|
actionsmaps[n] = pickle.load(f)
|
||||||
|
|
||||||
|
self.from_cache = True
|
||||||
# TODO: Switch to python3 and catch proper exception
|
# TODO: Switch to python3 and catch proper exception
|
||||||
except (IOError, EOFError):
|
except (IOError, EOFError):
|
||||||
self.use_cache = False
|
actionsmaps[n] = self.generate_cache(n)
|
||||||
actionsmaps = self.generate_cache(namespaces)
|
else: # cache file doesn't exists
|
||||||
elif use_cache: # cached file doesn't exists
|
actionsmaps[n] = self.generate_cache(n)
|
||||||
self.use_cache = False
|
|
||||||
actionsmaps = self.generate_cache(namespaces)
|
# If load_only_category is set, and *if* the target category
|
||||||
elif n not in actionsmaps:
|
# is in the actionsmap, we'll load only that one.
|
||||||
with open(actionsmap_yml) as f:
|
# If we filter it even if it doesn't exist, we'll end up with a
|
||||||
actionsmaps[n] = ordered_yaml_load(f)
|
# weird help message when we do a typo in the category name..
|
||||||
|
if load_only_category and load_only_category in actionsmaps[n]:
|
||||||
|
actionsmaps[n] = {
|
||||||
|
k: v
|
||||||
|
for k, v in actionsmaps[n].items()
|
||||||
|
if k in [load_only_category, "_global"]
|
||||||
|
}
|
||||||
|
|
||||||
# Load translations
|
# Load translations
|
||||||
m18n.load_namespace(n)
|
m18n.load_namespace(n)
|
||||||
|
|
||||||
# Generate parsers
|
# Generate parsers
|
||||||
self.extraparser = ExtraArgumentParser(parser_class.interface)
|
self.extraparser = ExtraArgumentParser(top_parser.interface)
|
||||||
self._parser = self._construct_parser(actionsmaps, **parser_kwargs)
|
self.parser = self._construct_parser(actionsmaps, top_parser)
|
||||||
|
|
||||||
@property
|
|
||||||
def parser(self):
|
|
||||||
"""Return the instance of the interface's actions map parser"""
|
|
||||||
return self._parser
|
|
||||||
|
|
||||||
def get_authenticator_for_profile(self, auth_profile):
|
def get_authenticator_for_profile(self, auth_profile):
|
||||||
|
|
||||||
|
@ -604,85 +603,81 @@ class ActionsMap(object):
|
||||||
moulinette_env = init_moulinette_env()
|
moulinette_env = init_moulinette_env()
|
||||||
DATA_DIR = moulinette_env["DATA_DIR"]
|
DATA_DIR = moulinette_env["DATA_DIR"]
|
||||||
|
|
||||||
for f in os.listdir("%s/actionsmap" % DATA_DIR):
|
# This var is ['*'] by default but could be set for example to
|
||||||
if f.endswith(".yml"):
|
# ['yunohost', 'yml_*']
|
||||||
namespaces.append(f[:-4])
|
NAMESPACE_PATTERNS = moulinette_env["NAMESPACES"]
|
||||||
|
|
||||||
|
# Look for all files that match the given patterns in the actionsmap dir
|
||||||
|
for namespace_pattern in NAMESPACE_PATTERNS:
|
||||||
|
namespaces.extend(
|
||||||
|
glob.glob("%s/actionsmap/%s.yml" % (DATA_DIR, namespace_pattern))
|
||||||
|
)
|
||||||
|
|
||||||
|
# Keep only the filenames with extension
|
||||||
|
namespaces = [os.path.basename(n)[:-4] for n in namespaces]
|
||||||
|
|
||||||
return namespaces
|
return namespaces
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def generate_cache(klass, namespaces=None):
|
def generate_cache(klass, namespace):
|
||||||
"""
|
"""
|
||||||
Generate cache for the actions map's file(s)
|
Generate cache for the actions map's file(s)
|
||||||
|
|
||||||
Keyword arguments:
|
Keyword arguments:
|
||||||
- namespaces -- A list of namespaces to generate cache for
|
- namespace -- The namespace to generate cache for
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
A dict of actions map for each namespaces
|
The action map for the namespace
|
||||||
|
|
||||||
"""
|
"""
|
||||||
moulinette_env = init_moulinette_env()
|
moulinette_env = init_moulinette_env()
|
||||||
CACHE_DIR = moulinette_env["CACHE_DIR"]
|
CACHE_DIR = moulinette_env["CACHE_DIR"]
|
||||||
DATA_DIR = moulinette_env["DATA_DIR"]
|
DATA_DIR = moulinette_env["DATA_DIR"]
|
||||||
|
|
||||||
actionsmaps = {}
|
|
||||||
if not namespaces:
|
|
||||||
namespaces = klass.get_namespaces()
|
|
||||||
|
|
||||||
# Iterate over actions map namespaces
|
# Iterate over actions map namespaces
|
||||||
for n in namespaces:
|
logger.debug("generating cache for actions map namespace '%s'", namespace)
|
||||||
logger.debug("generating cache for actions map namespace '%s'", n)
|
|
||||||
|
|
||||||
# Read actions map from yaml file
|
# Read actions map from yaml file
|
||||||
am_file = "%s/actionsmap/%s.yml" % (DATA_DIR, n)
|
am_file = "%s/actionsmap/%s.yml" % (DATA_DIR, namespace)
|
||||||
with open(am_file, "r") as f:
|
with open(am_file, "r") as f:
|
||||||
actionsmaps[n] = ordered_yaml_load(f)
|
actionsmap = ordered_yaml_load(f)
|
||||||
|
|
||||||
# at installation, cachedir might not exists
|
# at installation, cachedir might not exists
|
||||||
if os.path.exists("%s/actionsmap/" % CACHE_DIR):
|
for old_cache in glob.glob("%s/actionsmap/%s-*.pkl" % (CACHE_DIR, namespace)):
|
||||||
# clean old cached files
|
os.remove(old_cache)
|
||||||
for i in os.listdir("%s/actionsmap/" % CACHE_DIR):
|
|
||||||
if i.endswith(".pkl"):
|
|
||||||
os.remove("%s/actionsmap/%s" % (CACHE_DIR, i))
|
|
||||||
|
|
||||||
# Cache actions map into pickle file
|
# Cache actions map into pickle file
|
||||||
am_file_stat = os.stat(am_file)
|
am_file_stat = os.stat(am_file)
|
||||||
|
|
||||||
pkl = "%s-%d-%d.pkl" % (n, am_file_stat.st_size, am_file_stat.st_mtime)
|
pkl = "%s-%d-%d.pkl" % (namespace, am_file_stat.st_size, am_file_stat.st_mtime)
|
||||||
|
|
||||||
with open_cachefile(pkl, "w", subdir="actionsmap") as f:
|
with open_cachefile(pkl, "w", subdir="actionsmap") as f:
|
||||||
pickle.dump(actionsmaps[n], f)
|
pickle.dump(actionsmap, f)
|
||||||
|
|
||||||
return actionsmaps
|
return actionsmap
|
||||||
|
|
||||||
# Private methods
|
# Private methods
|
||||||
|
|
||||||
def _construct_parser(self, actionsmaps, **kwargs):
|
def _construct_parser(self, actionsmaps, top_parser):
|
||||||
"""
|
"""
|
||||||
Construct the parser with the actions map
|
Construct the parser with the actions map
|
||||||
|
|
||||||
Keyword arguments:
|
Keyword arguments:
|
||||||
- actionsmaps -- A dict of multi-level dictionnary of
|
- actionsmaps -- A dict of multi-level dictionnary of
|
||||||
categories/actions/arguments list for each namespaces
|
categories/actions/arguments list for each namespaces
|
||||||
- **kwargs -- Additionnal arguments to pass at the parser
|
- top_parser -- A BaseActionsMapParser-derived instance to use for
|
||||||
class instantiation
|
parsing the actions map
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
An interface relevant's parser object
|
An interface relevant's parser object
|
||||||
|
|
||||||
"""
|
"""
|
||||||
# Get extra parameters
|
|
||||||
if self.use_cache:
|
|
||||||
validate_extra = False
|
|
||||||
else:
|
|
||||||
validate_extra = True
|
|
||||||
|
|
||||||
# Instantiate parser
|
logger.debug("building parser...")
|
||||||
#
|
start = time()
|
||||||
# this either returns:
|
|
||||||
# * moulinette.interfaces.cli.ActionsMapParser
|
# If loading from cache, extra were already checked when cache was
|
||||||
# * moulinette.interfaces.api.ActionsMapParser
|
# loaded ? Not sure about this ... old code is a bit mysterious...
|
||||||
top_parser = self.parser_class(**kwargs)
|
validate_extra = not self.from_cache
|
||||||
|
|
||||||
# namespace, actionsmap is a tuple where:
|
# namespace, actionsmap is a tuple where:
|
||||||
#
|
#
|
||||||
|
@ -784,4 +779,5 @@ class ActionsMap(object):
|
||||||
tid, action_options["configuration"]
|
tid, action_options["configuration"]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
logger.debug("building parser took %.3fs", time() - start)
|
||||||
return top_parser
|
return top_parser
|
||||||
|
|
|
@ -5,8 +5,6 @@ import time
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from importlib import import_module
|
|
||||||
|
|
||||||
import moulinette
|
import moulinette
|
||||||
from moulinette.globals import init_moulinette_env
|
from moulinette.globals import init_moulinette_env
|
||||||
|
|
||||||
|
@ -375,53 +373,6 @@ class MoulinetteSignals(object):
|
||||||
raise NotImplementedError("this signal is not handled")
|
raise NotImplementedError("this signal is not handled")
|
||||||
|
|
||||||
|
|
||||||
# Interfaces & Authenticators management -------------------------------
|
|
||||||
|
|
||||||
|
|
||||||
def init_interface(name, kwargs={}, actionsmap={}):
|
|
||||||
"""Return a new interface instance
|
|
||||||
|
|
||||||
Retrieve the given interface module and return a new instance of its
|
|
||||||
Interface class. It is initialized with arguments 'kwargs' and
|
|
||||||
connected to 'actionsmap' if it's an ActionsMap object, otherwise
|
|
||||||
a new ActionsMap instance will be initialized with arguments
|
|
||||||
'actionsmap'.
|
|
||||||
|
|
||||||
Keyword arguments:
|
|
||||||
- name -- The interface name
|
|
||||||
- kwargs -- A dict of arguments to pass to Interface
|
|
||||||
- actionsmap -- Either an ActionsMap instance or a dict of
|
|
||||||
arguments to pass to ActionsMap
|
|
||||||
|
|
||||||
"""
|
|
||||||
from moulinette.actionsmap import ActionsMap
|
|
||||||
|
|
||||||
try:
|
|
||||||
mod = import_module("moulinette.interfaces.%s" % name)
|
|
||||||
except ImportError as e:
|
|
||||||
logger.exception("unable to load interface '%s' : %s", name, e)
|
|
||||||
raise MoulinetteError("error_see_log")
|
|
||||||
else:
|
|
||||||
try:
|
|
||||||
# Retrieve interface classes
|
|
||||||
parser = mod.ActionsMapParser
|
|
||||||
interface = mod.Interface
|
|
||||||
except AttributeError:
|
|
||||||
logger.exception("unable to retrieve classes of interface '%s'", name)
|
|
||||||
raise MoulinetteError("error_see_log")
|
|
||||||
|
|
||||||
# Instantiate or retrieve ActionsMap
|
|
||||||
if isinstance(actionsmap, dict):
|
|
||||||
amap = ActionsMap(actionsmap.pop("parser", parser), **actionsmap)
|
|
||||||
elif isinstance(actionsmap, ActionsMap):
|
|
||||||
amap = actionsmap
|
|
||||||
else:
|
|
||||||
logger.error("invalid actionsmap value %r", actionsmap)
|
|
||||||
raise MoulinetteError("error_see_log")
|
|
||||||
|
|
||||||
return interface(amap, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
# Moulinette core classes ----------------------------------------------
|
# Moulinette core classes ----------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -11,4 +11,7 @@ def init_moulinette_env():
|
||||||
"MOULINETTE_LOCALES_DIR", "/usr/share/moulinette/locale"
|
"MOULINETTE_LOCALES_DIR", "/usr/share/moulinette/locale"
|
||||||
),
|
),
|
||||||
"CACHE_DIR": environ.get("MOULINETTE_CACHE_DIR", "/var/cache/moulinette"),
|
"CACHE_DIR": environ.get("MOULINETTE_CACHE_DIR", "/var/cache/moulinette"),
|
||||||
|
"NAMESPACES": environ.get(
|
||||||
|
"MOULINETTE_NAMESPACES", "*"
|
||||||
|
).split(), # By default we'll load every namespace we find
|
||||||
}
|
}
|
||||||
|
|
|
@ -342,11 +342,6 @@ class _CallbackAction(argparse.Action):
|
||||||
self.callback_method = callback.get("method")
|
self.callback_method = callback.get("method")
|
||||||
self.callback_kwargs = callback.get("kwargs", {})
|
self.callback_kwargs = callback.get("kwargs", {})
|
||||||
self.callback_return = callback.get("return", False)
|
self.callback_return = callback.get("return", False)
|
||||||
logger.debug(
|
|
||||||
"registering new callback action '{0}' to {1}".format(
|
|
||||||
self.callback_method, option_strings
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def callback(self):
|
def callback(self):
|
||||||
|
|
|
@ -10,10 +10,11 @@ from gevent import sleep
|
||||||
from gevent.queue import Queue
|
from gevent.queue import Queue
|
||||||
from geventwebsocket import WebSocketError
|
from geventwebsocket import WebSocketError
|
||||||
|
|
||||||
from bottle import run, request, response, Bottle, HTTPResponse
|
from bottle import request, response, Bottle, HTTPResponse
|
||||||
from bottle import abort
|
from bottle import abort
|
||||||
|
|
||||||
from moulinette import msignals, m18n, env
|
from moulinette import msignals, m18n, env
|
||||||
|
from moulinette.actionsmap import ActionsMap
|
||||||
from moulinette.core import MoulinetteError
|
from moulinette.core import MoulinetteError
|
||||||
from moulinette.interfaces import (
|
from moulinette.interfaces import (
|
||||||
BaseActionsMapParser,
|
BaseActionsMapParser,
|
||||||
|
@ -219,22 +220,18 @@ class _ActionsMapPlugin(object):
|
||||||
|
|
||||||
Keyword arguments:
|
Keyword arguments:
|
||||||
- actionsmap -- An ActionsMap instance
|
- actionsmap -- An ActionsMap instance
|
||||||
- use_websocket -- If true, install a WebSocket on /messages in order
|
|
||||||
to serve messages coming from the 'display' signal
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
name = "actionsmap"
|
name = "actionsmap"
|
||||||
api = 2
|
api = 2
|
||||||
|
|
||||||
def __init__(self, actionsmap, use_websocket, log_queues={}):
|
def __init__(self, actionsmap, log_queues={}):
|
||||||
# Connect signals to handlers
|
# Connect signals to handlers
|
||||||
msignals.set_handler("authenticate", self._do_authenticate)
|
msignals.set_handler("authenticate", self._do_authenticate)
|
||||||
if use_websocket:
|
msignals.set_handler("display", self._do_display)
|
||||||
msignals.set_handler("display", self._do_display)
|
|
||||||
|
|
||||||
self.actionsmap = actionsmap
|
self.actionsmap = actionsmap
|
||||||
self.use_websocket = use_websocket
|
|
||||||
self.log_queues = log_queues
|
self.log_queues = log_queues
|
||||||
# TODO: Save and load secrets?
|
# TODO: Save and load secrets?
|
||||||
self.secrets = {}
|
self.secrets = {}
|
||||||
|
@ -290,13 +287,9 @@ class _ActionsMapPlugin(object):
|
||||||
)
|
)
|
||||||
|
|
||||||
# Append messages route
|
# Append messages route
|
||||||
if self.use_websocket:
|
app.route(
|
||||||
app.route(
|
"/messages", name="messages", callback=self.messages, skip=["actionsmap"],
|
||||||
"/messages",
|
)
|
||||||
name="messages",
|
|
||||||
callback=self.messages,
|
|
||||||
skip=["actionsmap"],
|
|
||||||
)
|
|
||||||
|
|
||||||
# Append routes from the actions map
|
# Append routes from the actions map
|
||||||
for (m, p) in self.actionsmap.parser.routes:
|
for (m, p) in self.actionsmap.parser.routes:
|
||||||
|
@ -745,17 +738,16 @@ class Interface(BaseInterface):
|
||||||
actions map.
|
actions map.
|
||||||
|
|
||||||
Keyword arguments:
|
Keyword arguments:
|
||||||
- actionsmap -- The ActionsMap instance to connect to
|
|
||||||
- routes -- A dict of additional routes to add in the form of
|
- routes -- A dict of additional routes to add in the form of
|
||||||
{(method, path): callback}
|
{(method, path): callback}
|
||||||
- use_websocket -- Serve via WSGI to handle asynchronous responses
|
|
||||||
- log_queues -- A LogQueues object or None to retrieve it from
|
- log_queues -- A LogQueues object or None to retrieve it from
|
||||||
registered logging handlers
|
registered logging handlers
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, actionsmap, routes={}, use_websocket=True, log_queues=None):
|
def __init__(self, routes={}, log_queues=None):
|
||||||
self.use_websocket = use_websocket
|
|
||||||
|
actionsmap = ActionsMap(ActionsMapParser())
|
||||||
|
|
||||||
# Attempt to retrieve log queues from an APIQueueHandler
|
# Attempt to retrieve log queues from an APIQueueHandler
|
||||||
if log_queues is None:
|
if log_queues is None:
|
||||||
|
@ -787,7 +779,7 @@ class Interface(BaseInterface):
|
||||||
app.install(filter_csrf)
|
app.install(filter_csrf)
|
||||||
app.install(apiheader)
|
app.install(apiheader)
|
||||||
app.install(api18n)
|
app.install(api18n)
|
||||||
app.install(_ActionsMapPlugin(actionsmap, use_websocket, log_queues))
|
app.install(_ActionsMapPlugin(actionsmap, log_queues))
|
||||||
|
|
||||||
# Append default routes
|
# Append default routes
|
||||||
# app.route(['/api', '/api/<category:re:[a-z]+>'], method='GET',
|
# app.route(['/api', '/api/<category:re:[a-z]+>'], method='GET',
|
||||||
|
@ -812,23 +804,15 @@ class Interface(BaseInterface):
|
||||||
|
|
||||||
"""
|
"""
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"starting the server instance in %s:%d with websocket=%s",
|
"starting the server instance in %s:%d", host, port,
|
||||||
host,
|
|
||||||
port,
|
|
||||||
self.use_websocket,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if self.use_websocket:
|
from gevent.pywsgi import WSGIServer
|
||||||
from gevent.pywsgi import WSGIServer
|
from geventwebsocket.handler import WebSocketHandler
|
||||||
from geventwebsocket.handler import WebSocketHandler
|
|
||||||
|
|
||||||
server = WSGIServer(
|
server = WSGIServer((host, port), self._app, handler_class=WebSocketHandler)
|
||||||
(host, port), self._app, handler_class=WebSocketHandler
|
server.serve_forever()
|
||||||
)
|
|
||||||
server.serve_forever()
|
|
||||||
else:
|
|
||||||
run(self._app, host=host, port=port)
|
|
||||||
except IOError as e:
|
except IOError as e:
|
||||||
logger.exception("unable to start the server instance on %s:%d", host, port)
|
logger.exception("unable to start the server instance on %s:%d", host, port)
|
||||||
if e.args[0] == errno.EADDRINUSE:
|
if e.args[0] == errno.EADDRINUSE:
|
||||||
|
|
|
@ -12,6 +12,7 @@ from datetime import date, datetime
|
||||||
import argcomplete
|
import argcomplete
|
||||||
|
|
||||||
from moulinette import msignals, m18n
|
from moulinette import msignals, m18n
|
||||||
|
from moulinette.actionsmap import ActionsMap
|
||||||
from moulinette.core import MoulinetteError
|
from moulinette.core import MoulinetteError
|
||||||
from moulinette.interfaces import (
|
from moulinette.interfaces import (
|
||||||
BaseActionsMapParser,
|
BaseActionsMapParser,
|
||||||
|
@ -424,7 +425,8 @@ class Interface(BaseInterface):
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, actionsmap):
|
def __init__(self, top_parser=None, load_only_category=None):
|
||||||
|
|
||||||
# Set user locale
|
# Set user locale
|
||||||
m18n.set_locale(get_locale())
|
m18n.set_locale(get_locale())
|
||||||
|
|
||||||
|
@ -434,9 +436,12 @@ class Interface(BaseInterface):
|
||||||
msignals.set_handler("authenticate", self._do_authenticate)
|
msignals.set_handler("authenticate", self._do_authenticate)
|
||||||
msignals.set_handler("prompt", self._do_prompt)
|
msignals.set_handler("prompt", self._do_prompt)
|
||||||
|
|
||||||
self.actionsmap = actionsmap
|
self.actionsmap = ActionsMap(
|
||||||
|
ActionsMapParser(top_parser=top_parser),
|
||||||
|
load_only_category=load_only_category,
|
||||||
|
)
|
||||||
|
|
||||||
def run(self, args, output_as=None, password=None, timeout=None):
|
def run(self, args, output_as=None, timeout=None):
|
||||||
"""Run the moulinette
|
"""Run the moulinette
|
||||||
|
|
||||||
Process the action corresponding to the given arguments 'args'
|
Process the action corresponding to the given arguments 'args'
|
||||||
|
@ -448,7 +453,6 @@ class Interface(BaseInterface):
|
||||||
- json: return a JSON encoded string
|
- json: return a JSON encoded string
|
||||||
- plain: return a script-readable output
|
- plain: return a script-readable output
|
||||||
- none: do not output the result
|
- none: do not output the result
|
||||||
- password -- The password to use in case of authentication
|
|
||||||
- timeout -- Number of seconds before this command will timeout because it can't acquire the lock (meaning that another command is currently running), by default there is no timeout and the command will wait until it can get the lock
|
- timeout -- Number of seconds before this command will timeout because it can't acquire the lock (meaning that another command is currently running), by default there is no timeout and the command will wait until it can get the lock
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
@ -459,11 +463,7 @@ class Interface(BaseInterface):
|
||||||
argcomplete.autocomplete(self.actionsmap.parser._parser)
|
argcomplete.autocomplete(self.actionsmap.parser._parser)
|
||||||
|
|
||||||
# Set handler for authentication
|
# Set handler for authentication
|
||||||
if password:
|
msignals.set_handler("authenticate", self._do_authenticate)
|
||||||
msignals.set_handler("authenticate", lambda a: a(password=password))
|
|
||||||
else:
|
|
||||||
if os.isatty(1):
|
|
||||||
msignals.set_handler("authenticate", self._do_authenticate)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
ret = self.actionsmap.process(args, timeout=timeout)
|
ret = self.actionsmap.process(args, timeout=timeout)
|
||||||
|
@ -495,7 +495,11 @@ class Interface(BaseInterface):
|
||||||
Handle the core.MoulinetteSignals.authenticate signal.
|
Handle the core.MoulinetteSignals.authenticate signal.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
# TODO: Allow token authentication?
|
# Hmpf we have no-use case in yunohost anymore where we need to auth
|
||||||
|
# because everything is run as root ...
|
||||||
|
# I guess we could imagine some yunohost-independant use-case where
|
||||||
|
# moulinette is used to create a CLI for non-root user that needs to
|
||||||
|
# auth somehow but hmpf -.-
|
||||||
help = authenticator.extra.get("help")
|
help = authenticator.extra.get("help")
|
||||||
msg = m18n.n(help) if help else m18n.g("password")
|
msg = m18n.n(help) if help else m18n.g("password")
|
||||||
return authenticator(password=self._do_prompt(msg, True, False, color="yellow"))
|
return authenticator(password=self._do_prompt(msg, True, False, color="yellow"))
|
||||||
|
|
|
@ -125,13 +125,9 @@ def moulinette_webapi(moulinette):
|
||||||
|
|
||||||
CookiePolicy.return_ok_secure = return_true
|
CookiePolicy.return_ok_secure = return_true
|
||||||
|
|
||||||
moulinette_webapi = moulinette.core.init_interface(
|
from moulinette.interfaces.api import Interface as Api
|
||||||
"api",
|
|
||||||
kwargs={"routes": {}, "use_websocket": False},
|
|
||||||
actionsmap={"namespaces": ["moulitest"], "use_cache": True},
|
|
||||||
)
|
|
||||||
|
|
||||||
return TestApp(moulinette_webapi._app)
|
return TestApp(Api(routes={})._app)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
|
@ -148,17 +144,12 @@ def moulinette_cli(moulinette, mocker):
|
||||||
help="Log and print debug messages",
|
help="Log and print debug messages",
|
||||||
)
|
)
|
||||||
mocker.patch("os.isatty", return_value=True)
|
mocker.patch("os.isatty", return_value=True)
|
||||||
moulinette_cli = moulinette.core.init_interface(
|
from moulinette.interfaces.cli import Interface as Cli
|
||||||
"cli",
|
|
||||||
actionsmap={
|
cli = Cli(top_parser=parser)
|
||||||
"namespaces": ["moulitest"],
|
|
||||||
"use_cache": False,
|
|
||||||
"parser_kwargs": {"top_parser": parser},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
mocker.stopall()
|
mocker.stopall()
|
||||||
|
|
||||||
return moulinette_cli
|
return cli
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
|
|
|
@ -158,10 +158,10 @@ def test_required_paremeter_missing_value(iface, caplog):
|
||||||
|
|
||||||
def test_actions_map_unknown_authenticator(monkeypatch, tmp_path):
|
def test_actions_map_unknown_authenticator(monkeypatch, tmp_path):
|
||||||
monkeypatch.setenv("MOULINETTE_DATA_DIR", str(tmp_path))
|
monkeypatch.setenv("MOULINETTE_DATA_DIR", str(tmp_path))
|
||||||
actionsmap_dir = actionsmap_dir = tmp_path / "actionsmap"
|
actionsmap_dir = tmp_path / "actionsmap"
|
||||||
actionsmap_dir.mkdir()
|
actionsmap_dir.mkdir()
|
||||||
|
|
||||||
amap = ActionsMap(BaseActionsMapParser)
|
amap = ActionsMap(BaseActionsMapParser())
|
||||||
with pytest.raises(ValueError) as exception:
|
with pytest.raises(ValueError) as exception:
|
||||||
amap.get_authenticator_for_profile("unknown")
|
amap.get_authenticator_for_profile("unknown")
|
||||||
assert "Unknown authenticator" in str(exception)
|
assert "Unknown authenticator" in str(exception)
|
||||||
|
@ -225,7 +225,7 @@ def test_extra_argument_parser_parse_args(iface, mocker):
|
||||||
def test_actions_map_api():
|
def test_actions_map_api():
|
||||||
from moulinette.interfaces.api import ActionsMapParser
|
from moulinette.interfaces.api import ActionsMapParser
|
||||||
|
|
||||||
amap = ActionsMap(ActionsMapParser, use_cache=False)
|
amap = ActionsMap(ActionsMapParser())
|
||||||
|
|
||||||
assert amap.parser.global_conf["authenticate"] == "all"
|
assert amap.parser.global_conf["authenticate"] == "all"
|
||||||
assert "default" in amap.parser.global_conf["authenticator"]
|
assert "default" in amap.parser.global_conf["authenticator"]
|
||||||
|
@ -233,9 +233,9 @@ def test_actions_map_api():
|
||||||
assert ("GET", "/test-auth/default") in amap.parser.routes
|
assert ("GET", "/test-auth/default") in amap.parser.routes
|
||||||
assert ("POST", "/test-auth/subcat/post") in amap.parser.routes
|
assert ("POST", "/test-auth/subcat/post") in amap.parser.routes
|
||||||
|
|
||||||
amap.generate_cache()
|
amap.generate_cache("moulitest")
|
||||||
|
|
||||||
amap = ActionsMap(ActionsMapParser, use_cache=True)
|
amap = ActionsMap(ActionsMapParser())
|
||||||
|
|
||||||
assert amap.parser.global_conf["authenticate"] == "all"
|
assert amap.parser.global_conf["authenticate"] == "all"
|
||||||
assert "default" in amap.parser.global_conf["authenticator"]
|
assert "default" in amap.parser.global_conf["authenticator"]
|
||||||
|
@ -247,7 +247,7 @@ def test_actions_map_api():
|
||||||
def test_actions_map_import_error(mocker):
|
def test_actions_map_import_error(mocker):
|
||||||
from moulinette.interfaces.api import ActionsMapParser
|
from moulinette.interfaces.api import ActionsMapParser
|
||||||
|
|
||||||
amap = ActionsMap(ActionsMapParser)
|
amap = ActionsMap(ActionsMapParser())
|
||||||
|
|
||||||
from moulinette.core import MoulinetteLock
|
from moulinette.core import MoulinetteLock
|
||||||
|
|
||||||
|
@ -281,9 +281,7 @@ def test_actions_map_cli():
|
||||||
default=False,
|
default=False,
|
||||||
help="Log and print debug messages",
|
help="Log and print debug messages",
|
||||||
)
|
)
|
||||||
amap = ActionsMap(
|
amap = ActionsMap(ActionsMapParser(top_parser=parser))
|
||||||
ActionsMapParser, use_cache=False, parser_kwargs={"top_parser": parser}
|
|
||||||
)
|
|
||||||
|
|
||||||
assert amap.parser.global_conf["authenticate"] == "all"
|
assert amap.parser.global_conf["authenticate"] == "all"
|
||||||
assert "default" in amap.parser.global_conf["authenticator"]
|
assert "default" in amap.parser.global_conf["authenticator"]
|
||||||
|
@ -300,11 +298,9 @@ def test_actions_map_cli():
|
||||||
.choices
|
.choices
|
||||||
)
|
)
|
||||||
|
|
||||||
amap.generate_cache()
|
amap.generate_cache("moulitest")
|
||||||
|
|
||||||
amap = ActionsMap(
|
amap = ActionsMap(ActionsMapParser(top_parser=parser))
|
||||||
ActionsMapParser, use_cache=True, parser_kwargs={"top_parser": parser}
|
|
||||||
)
|
|
||||||
|
|
||||||
assert amap.parser.global_conf["authenticate"] == "all"
|
assert amap.parser.global_conf["authenticate"] == "all"
|
||||||
assert "default" in amap.parser.global_conf["authenticator"]
|
assert "default" in amap.parser.global_conf["authenticator"]
|
||||||
|
|
|
@ -216,18 +216,15 @@ class TestAuthCLI:
|
||||||
|
|
||||||
assert "some_data_from_default" in message.out
|
assert "some_data_from_default" in message.out
|
||||||
|
|
||||||
moulinette_cli.run(
|
moulinette_cli.run(["testauth", "default"], output_as="plain")
|
||||||
["testauth", "default"], output_as="plain", password="default"
|
|
||||||
)
|
|
||||||
message = capsys.readouterr()
|
message = capsys.readouterr()
|
||||||
|
|
||||||
assert "some_data_from_default" in message.out
|
assert "some_data_from_default" in message.out
|
||||||
|
|
||||||
def test_login_bad_password(self, moulinette_cli, capsys, mocker):
|
def test_login_bad_password(self, moulinette_cli, capsys, mocker):
|
||||||
|
mocker.patch("getpass.getpass", return_value="Bad Password")
|
||||||
with pytest.raises(MoulinetteError):
|
with pytest.raises(MoulinetteError):
|
||||||
moulinette_cli.run(
|
moulinette_cli.run(["testauth", "default"], output_as="plain")
|
||||||
["testauth", "default"], output_as="plain", password="Bad Password"
|
|
||||||
)
|
|
||||||
|
|
||||||
mocker.patch("getpass.getpass", return_value="Bad Password")
|
mocker.patch("getpass.getpass", return_value="Bad Password")
|
||||||
with pytest.raises(MoulinetteError):
|
with pytest.raises(MoulinetteError):
|
||||||
|
@ -242,10 +239,9 @@ class TestAuthCLI:
|
||||||
expected_msg = translation.format()
|
expected_msg = translation.format()
|
||||||
assert expected_msg in str(exception)
|
assert expected_msg in str(exception)
|
||||||
|
|
||||||
|
mocker.patch("getpass.getpass", return_value="yoloswag")
|
||||||
with pytest.raises(MoulinetteError) as exception:
|
with pytest.raises(MoulinetteError) as exception:
|
||||||
moulinette_cli.run(
|
moulinette_cli.run(["testauth", "default"], output_as="none")
|
||||||
["testauth", "default"], output_as="none", password="yoloswag"
|
|
||||||
)
|
|
||||||
|
|
||||||
expected_msg = translation.format()
|
expected_msg = translation.format()
|
||||||
assert expected_msg in str(exception)
|
assert expected_msg in str(exception)
|
||||||
|
|
Loading…
Add table
Reference in a new issue