Merge pull request #245 from YunoHost/simplify-interface-init

Simplify interface initialization + ugly optimization hack to speed up parser building
This commit is contained in:
Alexandre Aubin 2020-08-14 15:39:31 +02:00 committed by GitHub
commit 4635c555d8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 142 additions and 254 deletions

View file

@ -1,7 +1,6 @@
# -*- coding: utf-8 -*-
from moulinette.core import (
init_interface,
MoulinetteError,
MoulinetteSignals,
Moulinette18n,
@ -72,85 +71,58 @@ def init(logging_config=None, **kwargs):
# Easy access to interfaces
def api(
namespaces, host="localhost", port=80, routes={}, use_websocket=True, use_cache=True
):
def api(host="localhost", port=80, routes={}):
"""Web server (API) interface
Run a HTTP server with the moulinette for an API usage.
Keyword arguments:
- namespaces -- The list of namespaces to use
- host -- Server address to bind to
- port -- Server port to bind to
- routes -- A dict of additional routes to add in the form of
{(method, uri): callback}
- use_websocket -- Serve via WSGI to handle asynchronous responses
- use_cache -- False if it should parse the actions map file
instead of using the cached one
"""
from moulinette.interfaces.api import Interface as Api
try:
moulinette = init_interface(
"api",
kwargs={"routes": routes, "use_websocket": use_websocket},
actionsmap={"namespaces": namespaces, "use_cache": use_cache},
)
moulinette.run(host, port)
Api(routes=routes).run(host, port)
except MoulinetteError as e:
import logging
logging.getLogger(namespaces[0]).error(e.strerror)
return e.errno if hasattr(e, "errno") else 1
logging.getLogger().error(e.strerror)
return 1
except KeyboardInterrupt:
import logging
logging.getLogger(namespaces[0]).info(m18n.g("operation_interrupted"))
logging.getLogger().info(m18n.g("operation_interrupted"))
return 0
def cli(
namespaces,
args,
use_cache=True,
output_as=None,
password=None,
timeout=None,
parser_kwargs={},
):
def cli(args, top_parser, output_as=None, timeout=None):
"""Command line interface
Execute an action with the moulinette from the CLI and print its
result in a readable format.
Keyword arguments:
- namespaces -- The list of namespaces to use
- args -- A list of argument strings
- use_cache -- False if it should parse the actions map file
instead of using the cached one
- output_as -- Output result in another format, see
moulinette.interfaces.cli.Interface for possible values
- password -- The password to use in case of authentication
- parser_kwargs -- A dict of arguments to pass to the parser
class at construction
- top_parser -- The top parser used to build the ActionsMapParser
"""
from moulinette.interfaces.cli import Interface as Cli
try:
moulinette = init_interface(
"cli",
actionsmap={
"namespaces": namespaces,
"use_cache": use_cache,
"parser_kwargs": parser_kwargs,
},
load_only_category = args[0] if args and not args[0].startswith("-") else None
Cli(top_parser=top_parser, load_only_category=load_only_category).run(
args, output_as=output_as, timeout=timeout
)
moulinette.run(args, output_as=output_as, password=password, timeout=timeout)
except MoulinetteError as e:
import logging
logging.getLogger(namespaces[0]).error(e.strerror)
logging.getLogger().error(e.strerror)
return 1
return 0

View file

@ -4,6 +4,7 @@ import os
import re
import logging
import yaml
import glob
import cPickle as pickle
from time import time
from collections import OrderedDict
@ -282,7 +283,6 @@ class ExtraArgumentParser(object):
if iface in klass.skipped_iface:
continue
self.extra[klass.name] = klass
logger.debug("extra parameter classes loaded: %s", self.extra.keys())
def validate(self, arg_name, parameters):
"""
@ -398,36 +398,32 @@ class ActionsMap(object):
Moreover, the action can have specific argument(s).
This class allows to manipulate one or several actions maps
associated to a namespace. If no namespace is given, it will load
all available namespaces.
associated to a namespace.
Keyword arguments:
- parser_class -- The BaseActionsMapParser derived class to use
for parsing the actions map
- namespaces -- The list of namespaces to use
- use_cache -- False if it should parse the actions map file
instead of using the cached one
- parser_kwargs -- A dict of arguments to pass to the parser
class at construction
- top_parser -- A BaseActionsMapParser-derived instance to use for
parsing the actions map
- load_only_category -- A name of a category that should only be the
one loaded because it's been already determined
that's the only one relevant ... used for optimization
purposes...
"""
def __init__(self, parser_class, namespaces=[], use_cache=True, parser_kwargs={}):
if not issubclass(parser_class, BaseActionsMapParser):
raise ValueError("Invalid parser class '%s'" % parser_class.__name__)
self.parser_class = parser_class
self.use_cache = use_cache
def __init__(self, top_parser, load_only_category=None):
assert isinstance(top_parser, BaseActionsMapParser), (
"Invalid parser class '%s'" % top_parser.__class__.__name__
)
moulinette_env = init_moulinette_env()
DATA_DIR = moulinette_env["DATA_DIR"]
CACHE_DIR = moulinette_env["CACHE_DIR"]
if len(namespaces) == 0:
namespaces = self.get_namespaces()
actionsmaps = OrderedDict()
self.from_cache = False
# Iterate over actions map namespaces
for n in namespaces:
for n in self.get_namespaces():
logger.debug("loading actions map namespace '%s'", n)
actionsmap_yml = "%s/actionsmap/%s.yml" % (DATA_DIR, n)
@ -439,33 +435,36 @@ class ActionsMap(object):
actionsmap_yml_stat.st_mtime,
)
if use_cache and os.path.exists(actionsmap_pkl):
if os.path.exists(actionsmap_pkl):
try:
# Attempt to load cache
with open(actionsmap_pkl) as f:
actionsmaps[n] = pickle.load(f)
self.from_cache = True
# TODO: Switch to python3 and catch proper exception
except (IOError, EOFError):
self.use_cache = False
actionsmaps = self.generate_cache(namespaces)
elif use_cache: # cached file doesn't exists
self.use_cache = False
actionsmaps = self.generate_cache(namespaces)
elif n not in actionsmaps:
with open(actionsmap_yml) as f:
actionsmaps[n] = ordered_yaml_load(f)
actionsmaps[n] = self.generate_cache(n)
else: # cache file doesn't exists
actionsmaps[n] = self.generate_cache(n)
# If load_only_category is set, and *if* the target category
# is in the actionsmap, we'll load only that one.
# If we filter it even if it doesn't exist, we'll end up with a
# weird help message when we do a typo in the category name..
if load_only_category and load_only_category in actionsmaps[n]:
actionsmaps[n] = {
k: v
for k, v in actionsmaps[n].items()
if k in [load_only_category, "_global"]
}
# Load translations
m18n.load_namespace(n)
# Generate parsers
self.extraparser = ExtraArgumentParser(parser_class.interface)
self._parser = self._construct_parser(actionsmaps, **parser_kwargs)
@property
def parser(self):
"""Return the instance of the interface's actions map parser"""
return self._parser
self.extraparser = ExtraArgumentParser(top_parser.interface)
self.parser = self._construct_parser(actionsmaps, top_parser)
def get_authenticator_for_profile(self, auth_profile):
@ -604,85 +603,81 @@ class ActionsMap(object):
moulinette_env = init_moulinette_env()
DATA_DIR = moulinette_env["DATA_DIR"]
for f in os.listdir("%s/actionsmap" % DATA_DIR):
if f.endswith(".yml"):
namespaces.append(f[:-4])
# This var is ['*'] by default but could be set for example to
# ['yunohost', 'yml_*']
NAMESPACE_PATTERNS = moulinette_env["NAMESPACES"]
# Look for all files that match the given patterns in the actionsmap dir
for namespace_pattern in NAMESPACE_PATTERNS:
namespaces.extend(
glob.glob("%s/actionsmap/%s.yml" % (DATA_DIR, namespace_pattern))
)
# Keep only the filenames with extension
namespaces = [os.path.basename(n)[:-4] for n in namespaces]
return namespaces
@classmethod
def generate_cache(klass, namespaces=None):
def generate_cache(klass, namespace):
"""
Generate cache for the actions map's file(s)
Keyword arguments:
- namespaces -- A list of namespaces to generate cache for
- namespace -- The namespace to generate cache for
Returns:
A dict of actions map for each namespaces
The action map for the namespace
"""
moulinette_env = init_moulinette_env()
CACHE_DIR = moulinette_env["CACHE_DIR"]
DATA_DIR = moulinette_env["DATA_DIR"]
actionsmaps = {}
if not namespaces:
namespaces = klass.get_namespaces()
# Iterate over actions map namespaces
for n in namespaces:
logger.debug("generating cache for actions map namespace '%s'", n)
logger.debug("generating cache for actions map namespace '%s'", namespace)
# Read actions map from yaml file
am_file = "%s/actionsmap/%s.yml" % (DATA_DIR, n)
with open(am_file, "r") as f:
actionsmaps[n] = ordered_yaml_load(f)
# Read actions map from yaml file
am_file = "%s/actionsmap/%s.yml" % (DATA_DIR, namespace)
with open(am_file, "r") as f:
actionsmap = ordered_yaml_load(f)
# at installation, cachedir might not exists
if os.path.exists("%s/actionsmap/" % CACHE_DIR):
# clean old cached files
for i in os.listdir("%s/actionsmap/" % CACHE_DIR):
if i.endswith(".pkl"):
os.remove("%s/actionsmap/%s" % (CACHE_DIR, i))
# at installation, cachedir might not exists
for old_cache in glob.glob("%s/actionsmap/%s-*.pkl" % (CACHE_DIR, namespace)):
os.remove(old_cache)
# Cache actions map into pickle file
am_file_stat = os.stat(am_file)
# Cache actions map into pickle file
am_file_stat = os.stat(am_file)
pkl = "%s-%d-%d.pkl" % (n, am_file_stat.st_size, am_file_stat.st_mtime)
pkl = "%s-%d-%d.pkl" % (namespace, am_file_stat.st_size, am_file_stat.st_mtime)
with open_cachefile(pkl, "w", subdir="actionsmap") as f:
pickle.dump(actionsmaps[n], f)
with open_cachefile(pkl, "w", subdir="actionsmap") as f:
pickle.dump(actionsmap, f)
return actionsmaps
return actionsmap
# Private methods
def _construct_parser(self, actionsmaps, **kwargs):
def _construct_parser(self, actionsmaps, top_parser):
"""
Construct the parser with the actions map
Keyword arguments:
- actionsmaps -- A dict of multi-level dictionnary of
categories/actions/arguments list for each namespaces
- **kwargs -- Additionnal arguments to pass at the parser
class instantiation
- top_parser -- A BaseActionsMapParser-derived instance to use for
parsing the actions map
Returns:
An interface relevant's parser object
"""
# Get extra parameters
if self.use_cache:
validate_extra = False
else:
validate_extra = True
# Instantiate parser
#
# this either returns:
# * moulinette.interfaces.cli.ActionsMapParser
# * moulinette.interfaces.api.ActionsMapParser
top_parser = self.parser_class(**kwargs)
logger.debug("building parser...")
start = time()
# If loading from cache, extra were already checked when cache was
# loaded ? Not sure about this ... old code is a bit mysterious...
validate_extra = not self.from_cache
# namespace, actionsmap is a tuple where:
#
@ -784,4 +779,5 @@ class ActionsMap(object):
tid, action_options["configuration"]
)
logger.debug("building parser took %.3fs", time() - start)
return top_parser

View file

@ -5,8 +5,6 @@ import time
import json
import logging
from importlib import import_module
import moulinette
from moulinette.globals import init_moulinette_env
@ -375,53 +373,6 @@ class MoulinetteSignals(object):
raise NotImplementedError("this signal is not handled")
# Interfaces & Authenticators management -------------------------------
def init_interface(name, kwargs={}, actionsmap={}):
"""Return a new interface instance
Retrieve the given interface module and return a new instance of its
Interface class. It is initialized with arguments 'kwargs' and
connected to 'actionsmap' if it's an ActionsMap object, otherwise
a new ActionsMap instance will be initialized with arguments
'actionsmap'.
Keyword arguments:
- name -- The interface name
- kwargs -- A dict of arguments to pass to Interface
- actionsmap -- Either an ActionsMap instance or a dict of
arguments to pass to ActionsMap
"""
from moulinette.actionsmap import ActionsMap
try:
mod = import_module("moulinette.interfaces.%s" % name)
except ImportError as e:
logger.exception("unable to load interface '%s' : %s", name, e)
raise MoulinetteError("error_see_log")
else:
try:
# Retrieve interface classes
parser = mod.ActionsMapParser
interface = mod.Interface
except AttributeError:
logger.exception("unable to retrieve classes of interface '%s'", name)
raise MoulinetteError("error_see_log")
# Instantiate or retrieve ActionsMap
if isinstance(actionsmap, dict):
amap = ActionsMap(actionsmap.pop("parser", parser), **actionsmap)
elif isinstance(actionsmap, ActionsMap):
amap = actionsmap
else:
logger.error("invalid actionsmap value %r", actionsmap)
raise MoulinetteError("error_see_log")
return interface(amap, **kwargs)
# Moulinette core classes ----------------------------------------------

View file

@ -11,4 +11,7 @@ def init_moulinette_env():
"MOULINETTE_LOCALES_DIR", "/usr/share/moulinette/locale"
),
"CACHE_DIR": environ.get("MOULINETTE_CACHE_DIR", "/var/cache/moulinette"),
"NAMESPACES": environ.get(
"MOULINETTE_NAMESPACES", "*"
).split(), # By default we'll load every namespace we find
}

View file

@ -342,11 +342,6 @@ class _CallbackAction(argparse.Action):
self.callback_method = callback.get("method")
self.callback_kwargs = callback.get("kwargs", {})
self.callback_return = callback.get("return", False)
logger.debug(
"registering new callback action '{0}' to {1}".format(
self.callback_method, option_strings
)
)
@property
def callback(self):

View file

@ -10,10 +10,11 @@ from gevent import sleep
from gevent.queue import Queue
from geventwebsocket import WebSocketError
from bottle import run, request, response, Bottle, HTTPResponse
from bottle import request, response, Bottle, HTTPResponse
from bottle import abort
from moulinette import msignals, m18n, env
from moulinette.actionsmap import ActionsMap
from moulinette.core import MoulinetteError
from moulinette.interfaces import (
BaseActionsMapParser,
@ -219,22 +220,18 @@ class _ActionsMapPlugin(object):
Keyword arguments:
- actionsmap -- An ActionsMap instance
- use_websocket -- If true, install a WebSocket on /messages in order
to serve messages coming from the 'display' signal
"""
name = "actionsmap"
api = 2
def __init__(self, actionsmap, use_websocket, log_queues={}):
def __init__(self, actionsmap, log_queues={}):
# Connect signals to handlers
msignals.set_handler("authenticate", self._do_authenticate)
if use_websocket:
msignals.set_handler("display", self._do_display)
msignals.set_handler("display", self._do_display)
self.actionsmap = actionsmap
self.use_websocket = use_websocket
self.log_queues = log_queues
# TODO: Save and load secrets?
self.secrets = {}
@ -290,13 +287,9 @@ class _ActionsMapPlugin(object):
)
# Append messages route
if self.use_websocket:
app.route(
"/messages",
name="messages",
callback=self.messages,
skip=["actionsmap"],
)
app.route(
"/messages", name="messages", callback=self.messages, skip=["actionsmap"],
)
# Append routes from the actions map
for (m, p) in self.actionsmap.parser.routes:
@ -745,17 +738,16 @@ class Interface(BaseInterface):
actions map.
Keyword arguments:
- actionsmap -- The ActionsMap instance to connect to
- routes -- A dict of additional routes to add in the form of
{(method, path): callback}
- use_websocket -- Serve via WSGI to handle asynchronous responses
- log_queues -- A LogQueues object or None to retrieve it from
registered logging handlers
"""
def __init__(self, actionsmap, routes={}, use_websocket=True, log_queues=None):
self.use_websocket = use_websocket
def __init__(self, routes={}, log_queues=None):
actionsmap = ActionsMap(ActionsMapParser())
# Attempt to retrieve log queues from an APIQueueHandler
if log_queues is None:
@ -787,7 +779,7 @@ class Interface(BaseInterface):
app.install(filter_csrf)
app.install(apiheader)
app.install(api18n)
app.install(_ActionsMapPlugin(actionsmap, use_websocket, log_queues))
app.install(_ActionsMapPlugin(actionsmap, log_queues))
# Append default routes
# app.route(['/api', '/api/<category:re:[a-z]+>'], method='GET',
@ -812,23 +804,15 @@ class Interface(BaseInterface):
"""
logger.debug(
"starting the server instance in %s:%d with websocket=%s",
host,
port,
self.use_websocket,
"starting the server instance in %s:%d", host, port,
)
try:
if self.use_websocket:
from gevent.pywsgi import WSGIServer
from geventwebsocket.handler import WebSocketHandler
from gevent.pywsgi import WSGIServer
from geventwebsocket.handler import WebSocketHandler
server = WSGIServer(
(host, port), self._app, handler_class=WebSocketHandler
)
server.serve_forever()
else:
run(self._app, host=host, port=port)
server = WSGIServer((host, port), self._app, handler_class=WebSocketHandler)
server.serve_forever()
except IOError as e:
logger.exception("unable to start the server instance on %s:%d", host, port)
if e.args[0] == errno.EADDRINUSE:

View file

@ -12,6 +12,7 @@ from datetime import date, datetime
import argcomplete
from moulinette import msignals, m18n
from moulinette.actionsmap import ActionsMap
from moulinette.core import MoulinetteError
from moulinette.interfaces import (
BaseActionsMapParser,
@ -424,7 +425,8 @@ class Interface(BaseInterface):
"""
def __init__(self, actionsmap):
def __init__(self, top_parser=None, load_only_category=None):
# Set user locale
m18n.set_locale(get_locale())
@ -434,9 +436,12 @@ class Interface(BaseInterface):
msignals.set_handler("authenticate", self._do_authenticate)
msignals.set_handler("prompt", self._do_prompt)
self.actionsmap = actionsmap
self.actionsmap = ActionsMap(
ActionsMapParser(top_parser=top_parser),
load_only_category=load_only_category,
)
def run(self, args, output_as=None, password=None, timeout=None):
def run(self, args, output_as=None, timeout=None):
"""Run the moulinette
Process the action corresponding to the given arguments 'args'
@ -448,7 +453,6 @@ class Interface(BaseInterface):
- json: return a JSON encoded string
- plain: return a script-readable output
- none: do not output the result
- password -- The password to use in case of authentication
- timeout -- Number of seconds before this command will timeout because it can't acquire the lock (meaning that another command is currently running), by default there is no timeout and the command will wait until it can get the lock
"""
@ -459,11 +463,7 @@ class Interface(BaseInterface):
argcomplete.autocomplete(self.actionsmap.parser._parser)
# Set handler for authentication
if password:
msignals.set_handler("authenticate", lambda a: a(password=password))
else:
if os.isatty(1):
msignals.set_handler("authenticate", self._do_authenticate)
msignals.set_handler("authenticate", self._do_authenticate)
try:
ret = self.actionsmap.process(args, timeout=timeout)
@ -495,7 +495,11 @@ class Interface(BaseInterface):
Handle the core.MoulinetteSignals.authenticate signal.
"""
# TODO: Allow token authentication?
# Hmpf we have no-use case in yunohost anymore where we need to auth
# because everything is run as root ...
# I guess we could imagine some yunohost-independant use-case where
# moulinette is used to create a CLI for non-root user that needs to
# auth somehow but hmpf -.-
help = authenticator.extra.get("help")
msg = m18n.n(help) if help else m18n.g("password")
return authenticator(password=self._do_prompt(msg, True, False, color="yellow"))

View file

@ -125,13 +125,9 @@ def moulinette_webapi(moulinette):
CookiePolicy.return_ok_secure = return_true
moulinette_webapi = moulinette.core.init_interface(
"api",
kwargs={"routes": {}, "use_websocket": False},
actionsmap={"namespaces": ["moulitest"], "use_cache": True},
)
from moulinette.interfaces.api import Interface as Api
return TestApp(moulinette_webapi._app)
return TestApp(Api(routes={})._app)
@pytest.fixture
@ -148,17 +144,12 @@ def moulinette_cli(moulinette, mocker):
help="Log and print debug messages",
)
mocker.patch("os.isatty", return_value=True)
moulinette_cli = moulinette.core.init_interface(
"cli",
actionsmap={
"namespaces": ["moulitest"],
"use_cache": False,
"parser_kwargs": {"top_parser": parser},
},
)
from moulinette.interfaces.cli import Interface as Cli
cli = Cli(top_parser=parser)
mocker.stopall()
return moulinette_cli
return cli
@pytest.fixture

View file

@ -158,10 +158,10 @@ def test_required_paremeter_missing_value(iface, caplog):
def test_actions_map_unknown_authenticator(monkeypatch, tmp_path):
monkeypatch.setenv("MOULINETTE_DATA_DIR", str(tmp_path))
actionsmap_dir = actionsmap_dir = tmp_path / "actionsmap"
actionsmap_dir = tmp_path / "actionsmap"
actionsmap_dir.mkdir()
amap = ActionsMap(BaseActionsMapParser)
amap = ActionsMap(BaseActionsMapParser())
with pytest.raises(ValueError) as exception:
amap.get_authenticator_for_profile("unknown")
assert "Unknown authenticator" in str(exception)
@ -225,7 +225,7 @@ def test_extra_argument_parser_parse_args(iface, mocker):
def test_actions_map_api():
from moulinette.interfaces.api import ActionsMapParser
amap = ActionsMap(ActionsMapParser, use_cache=False)
amap = ActionsMap(ActionsMapParser())
assert amap.parser.global_conf["authenticate"] == "all"
assert "default" in amap.parser.global_conf["authenticator"]
@ -233,9 +233,9 @@ def test_actions_map_api():
assert ("GET", "/test-auth/default") in amap.parser.routes
assert ("POST", "/test-auth/subcat/post") in amap.parser.routes
amap.generate_cache()
amap.generate_cache("moulitest")
amap = ActionsMap(ActionsMapParser, use_cache=True)
amap = ActionsMap(ActionsMapParser())
assert amap.parser.global_conf["authenticate"] == "all"
assert "default" in amap.parser.global_conf["authenticator"]
@ -247,7 +247,7 @@ def test_actions_map_api():
def test_actions_map_import_error(mocker):
from moulinette.interfaces.api import ActionsMapParser
amap = ActionsMap(ActionsMapParser)
amap = ActionsMap(ActionsMapParser())
from moulinette.core import MoulinetteLock
@ -281,9 +281,7 @@ def test_actions_map_cli():
default=False,
help="Log and print debug messages",
)
amap = ActionsMap(
ActionsMapParser, use_cache=False, parser_kwargs={"top_parser": parser}
)
amap = ActionsMap(ActionsMapParser(top_parser=parser))
assert amap.parser.global_conf["authenticate"] == "all"
assert "default" in amap.parser.global_conf["authenticator"]
@ -300,11 +298,9 @@ def test_actions_map_cli():
.choices
)
amap.generate_cache()
amap.generate_cache("moulitest")
amap = ActionsMap(
ActionsMapParser, use_cache=True, parser_kwargs={"top_parser": parser}
)
amap = ActionsMap(ActionsMapParser(top_parser=parser))
assert amap.parser.global_conf["authenticate"] == "all"
assert "default" in amap.parser.global_conf["authenticator"]

View file

@ -216,18 +216,15 @@ class TestAuthCLI:
assert "some_data_from_default" in message.out
moulinette_cli.run(
["testauth", "default"], output_as="plain", password="default"
)
moulinette_cli.run(["testauth", "default"], output_as="plain")
message = capsys.readouterr()
assert "some_data_from_default" in message.out
def test_login_bad_password(self, moulinette_cli, capsys, mocker):
mocker.patch("getpass.getpass", return_value="Bad Password")
with pytest.raises(MoulinetteError):
moulinette_cli.run(
["testauth", "default"], output_as="plain", password="Bad Password"
)
moulinette_cli.run(["testauth", "default"], output_as="plain")
mocker.patch("getpass.getpass", return_value="Bad Password")
with pytest.raises(MoulinetteError):
@ -242,10 +239,9 @@ class TestAuthCLI:
expected_msg = translation.format()
assert expected_msg in str(exception)
mocker.patch("getpass.getpass", return_value="yoloswag")
with pytest.raises(MoulinetteError) as exception:
moulinette_cli.run(
["testauth", "default"], output_as="none", password="yoloswag"
)
moulinette_cli.run(["testauth", "default"], output_as="none")
expected_msg = translation.format()
assert expected_msg in str(exception)