From e8f10ce54e5fb7cb8d466d688404561ab0cf8d80 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 May 2023 20:31:44 +0200 Subject: [PATCH 01/22] Update changelog for 12.0.0 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index a1d37106..8bdabb46 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +moulinette (12.0.0) unstable; urgency=low + + - Tmp changelog to prepare Bookworm + + -- Alexandre Aubin Thu, 04 May 2023 20:30:19 +0200 + moulinette (11.1.4) stable; urgency=low - Releasing as stable From 34a2a660272c4501d96276acc2b19e4251b3f87f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 14 Jul 2023 19:09:00 +0200 Subject: [PATCH 02/22] Fix boring login API expecting a weird form/multiparam thing instead of classic JSON for credentials ... --- moulinette/interfaces/api.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/moulinette/interfaces/api.py b/moulinette/interfaces/api.py index 34b51143..bf3b0c7f 100644 --- a/moulinette/interfaces/api.py +++ b/moulinette/interfaces/api.py @@ -353,11 +353,18 @@ class _ActionsMapPlugin: """ - if "credentials" not in request.params: - raise HTTPResponse("Missing credentials parameter", 400) - credentials = request.params["credentials"] + if request.get_header("Content-Type") == "application/json": + if "credentials" not in request.json: + raise HTTPResponse("Missing credentials parameter", 400) + credentials = request.json["credentials"] + profile = request.json.get("profile", self.actionsmap.default_authentication) + else: + if "credentials" not in request.params: + raise HTTPResponse("Missing credentials parameter", 400) + credentials = request.params["credentials"] + + profile = request.params.get("profile", self.actionsmap.default_authentication) - profile = request.params.get("profile", self.actionsmap.default_authentication) authenticator = self.actionsmap.get_authenticator(profile) try: @@ -732,7 +739,7 @@ class Interface: def wrapper(*args, **kwargs): try: locale = request.params.pop("locale") - except KeyError: + except (KeyError, ValueError): locale = m18n.default_locale m18n.set_locale(locale) return callback(*args, **kwargs) From 0f056d66d7572eb070a0ebf5be591503b9dbacd5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 18 Jul 2023 00:20:24 +0200 Subject: [PATCH 03/22] Moulinette logging is an unecessarily complex mess, episode 57682 --- moulinette/actionsmap.py | 19 +------- moulinette/interfaces/__init__.py | 3 +- moulinette/interfaces/cli.py | 22 ++++----- moulinette/utils/log.py | 81 ++----------------------------- 4 files changed, 16 insertions(+), 109 deletions(-) diff --git a/moulinette/actionsmap.py b/moulinette/actionsmap.py index e5e9547b..c213ab2e 100644 --- a/moulinette/actionsmap.py +++ b/moulinette/actionsmap.py @@ -19,7 +19,6 @@ from moulinette.core import ( MoulinetteValidationError, ) from moulinette.interfaces import BaseActionsMapParser -from moulinette.utils.log import start_action_logging from moulinette.utils.filesystem import read_yaml logger = logging.getLogger("moulinette.actionsmap") @@ -398,8 +397,6 @@ class ActionsMap: self.from_cache = False - logger.debug("loading actions map") - actionsmap_yml_dir = os.path.dirname(actionsmap_yml) actionsmap_yml_file = os.path.basename(actionsmap_yml) actionsmap_yml_stat = os.stat(actionsmap_yml) @@ -562,17 +559,7 @@ class ActionsMap: logger.exception(error_message) raise MoulinetteError(error_message, raw_msg=True) else: - log_id = start_action_logging() - if logger.isEnabledFor(logging.DEBUG): - # Log arguments in debug mode only for safety reasons - logger.debug( - "processing action [%s]: %s with args=%s", - log_id, - full_action_name, - arguments, - ) - else: - logger.debug("processing action [%s]: %s", log_id, full_action_name) + logger.debug("processing action '%s'", full_action_name) # Load translation and process the action start = time() @@ -580,7 +567,7 @@ class ActionsMap: return func(**arguments) finally: stop = time() - logger.debug("action [%s] executed in %.3fs", log_id, stop - start) + logger.debug("action executed in %.3fs", stop - start) # Private methods @@ -598,7 +585,6 @@ class ActionsMap: """ - logger.debug("building parser...") start = time() interface_type = top_parser.interface @@ -717,5 +703,4 @@ class ActionsMap: else: action_parser.want_to_take_lock = True - logger.debug("building parser took %.3fs", time() - start) return top_parser diff --git a/moulinette/interfaces/__init__.py b/moulinette/interfaces/__init__.py index 54776552..73d75f1e 100644 --- a/moulinette/interfaces/__init__.py +++ b/moulinette/interfaces/__init__.py @@ -33,8 +33,7 @@ class BaseActionsMapParser: """ def __init__(self, parent=None, **kwargs): - if not parent: - logger.debug("initializing base actions map parser for %s", self.interface) + pass # Virtual properties # Each parser classes must implement these properties. diff --git a/moulinette/interfaces/cli.py b/moulinette/interfaces/cli.py index 84e72185..f1005558 100644 --- a/moulinette/interfaces/cli.py +++ b/moulinette/interfaces/cli.py @@ -235,28 +235,26 @@ class TTYHandler(logging.StreamHandler): log.CRITICAL: "red", } - def __init__(self, message_key="fmessage"): + def __init__(self, message_key="message_with_color"): logging.StreamHandler.__init__(self) self.message_key = message_key def format(self, record): """Enhance message with level and colors if supported.""" msg = record.getMessage() + level = record.levelname + level_with_color = level if self.supports_color(): - level = "" - if self.level <= log.DEBUG: - # add level name before message - level = "%s " % record.levelname - elif record.levelname in ["SUCCESS", "WARNING", "ERROR", "INFO"]: - # add translated level name before message - level = "%s " % m18n.g(record.levelname.lower()) + if self.level > log.DEBUG and record.levelname in ["SUCCESS", "WARNING", "ERROR", "INFO"]: + level = m18n.g(record.levelname.lower()) color = self.LEVELS_COLOR.get(record.levelno, "white") - msg = "{}{}{}{}".format(colors_codes[color], level, END_CLI_COLOR, msg) + level_with_color = f"{colors_codes[color]}{level}{END_CLI_COLOR}" + if self.level == log.DEBUG: + level_with_color = level_with_color + " " * max(0, 7 - len(level)) if self.formatter: - # use user-defined formatter - record.__dict__[self.message_key] = msg + record.__dict__["level_with_color"] = level_with_color return self.formatter.format(record) - return msg + return level_with_color + " " + msg def emit(self, record): # set proper stream first diff --git a/moulinette/utils/log.py b/moulinette/utils/log.py index 02f8a30a..3d010726 100644 --- a/moulinette/utils/log.py +++ b/moulinette/utils/log.py @@ -110,89 +110,14 @@ class MoulinetteLogger(Logger): f = currentframe() if f is not None: f = f.f_back - rv = "(unknown file)", 0, "(unknown function)" + rv = "(unknown file)", 0, "(unknown function)", None while hasattr(f, "f_code"): co = f.f_code filename = os.path.normcase(co.co_filename) if filename == _srcfile or filename == __file__: f = f.f_back continue - rv = (co.co_filename, f.f_lineno, co.co_name) + rv = (co.co_filename, f.f_lineno, co.co_name, None) break + return rv - - def _log(self, *args, **kwargs): - """Append action_id if available to the extra.""" - if self.action_id is not None: - extra = kwargs.get("extra", {}) - if "action_id" not in extra: - # FIXME: Get real action_id instead of logger/current one - extra["action_id"] = _get_action_id() - kwargs["extra"] = extra - return super()._log(*args, **kwargs) - - -# Action logging ------------------------------------------------------- - -pid = os.getpid() -action_id = 0 - - -def _get_action_id(): - return "%d.%d" % (pid, action_id) - - -def start_action_logging(): - """Configure logging for a new action - - Returns: - The new action id - - """ - global action_id - action_id += 1 - - return _get_action_id() - - -def getActionLogger(name=None, logger=None, action_id=None): - """Get the logger adapter for an action - - Return a logger for the specified name - or use given logger - and - optionally for a given action id, retrieving it if necessary. - - Either a name or a logger must be specified. - - """ - if not name and not logger: - raise ValueError("Either a name or a logger must be specified") - - logger = logger or getLogger(name) - logger.action_id = action_id if action_id else _get_action_id() - return logger - - -class ActionFilter: - - """Extend log record for an optionnal action - - Filter a given record and look for an `action_id` key. If it is not found - and `strict` is True, the record will not be logged. Otherwise, the key - specified by `message_key` will be added to the record, containing the - message formatted for the action or just the original one. - - """ - - def __init__(self, message_key="fmessage", strict=False): - self.message_key = message_key - self.strict = strict - - def filter(self, record): - msg = record.getMessage() - action_id = record.__dict__.get("action_id", None) - if action_id is not None: - msg = "[{:s}] {:s}".format(action_id, msg) - elif self.strict: - return False - record.__dict__[self.message_key] = msg - return True From 4104704a87b97daa96985c1c48c3f6d6f29131c9 Mon Sep 17 00:00:00 2001 From: axolotle Date: Fri, 28 Jul 2023 17:17:51 +0200 Subject: [PATCH 04/22] allow json requests --- moulinette/interfaces/api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/moulinette/interfaces/api.py b/moulinette/interfaces/api.py index bf3b0c7f..c0e18be8 100644 --- a/moulinette/interfaces/api.py +++ b/moulinette/interfaces/api.py @@ -312,6 +312,9 @@ class _ActionsMapPlugin: return value def wrapper(*args, **kwargs): + if request.get_header("Content-Type") == "application/json": + return callback((request.method, context.rule), request.json) + params = kwargs # Format boolean params for a in args: From 328107c94699074cd00692451fd55589c96f16f2 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 29 Jul 2023 19:09:52 +0200 Subject: [PATCH 05/22] api: Add a proper mechanism to allow specific, configurable CORS origins --- moulinette/__init__.py | 3 ++- moulinette/interfaces/api.py | 27 +++++++++++++++++++++------ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/moulinette/__init__.py b/moulinette/__init__.py index 6df5abef..92b7e691 100755 --- a/moulinette/__init__.py +++ b/moulinette/__init__.py @@ -53,7 +53,7 @@ class Moulinette: # Easy access to interfaces -def api(host="localhost", port=80, routes={}, actionsmap=None, locales_dir=None): +def api(host="localhost", port=80, routes={}, actionsmap=None, locales_dir=None, allowed_cors_origins=[]): """Web server (API) interface Run a HTTP server with the moulinette for an API usage. @@ -73,6 +73,7 @@ def api(host="localhost", port=80, routes={}, actionsmap=None, locales_dir=None) Api( routes=routes, actionsmap=actionsmap, + allowed_cors_origins=allowed_cors_origins, ).run(host, port) except MoulinetteError as e: import logging diff --git a/moulinette/interfaces/api.py b/moulinette/interfaces/api.py index c0e18be8..32383977 100644 --- a/moulinette/interfaces/api.py +++ b/moulinette/interfaces/api.py @@ -717,9 +717,11 @@ class Interface: type = "api" - def __init__(self, routes={}, actionsmap=None): + def __init__(self, routes={}, actionsmap=None, allowed_cors_origins=[]): actionsmap = ActionsMap(actionsmap, ActionsMapParser()) + self.allowed_cors_origins = allowed_cors_origins + # Attempt to retrieve log queues from an APIQueueHandler handler = log.getHandlersByClass(APIQueueHandler, limit=1) if handler: @@ -729,11 +731,18 @@ class Interface: # TODO: Return OK to 'OPTIONS' xhr requests (l173) app = Bottle(autojson=True) - # Wrapper which sets proper header - def apiheader(callback): + def cors(callback): def wrapper(*args, **kwargs): - response.set_header("Access-Control-Allow-Origin", "*") - return callback(*args, **kwargs) + r = callback(*args, **kwargs) + origin = request.headers.environ.get("HTTP_ORIGIN", "") + if origin and origin in self.allowed_cors_origins: + resp = r if isinstance(r, HTTPResponse) else response + resp.headers['Access-Control-Allow-Origin'] = origin + resp.headers['Access-Control-Allow-Methods'] = 'GET, HEAD, POST, PUT, OPTIONS, DELETE' + resp.headers['Access-Control-Allow-Headers'] = 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' + resp.headers['Access-Control-Allow-Credentials'] = 'true' + + return r return wrapper @@ -751,7 +760,7 @@ class Interface: # Install plugins app.install(filter_csrf) - app.install(apiheader) + app.install(cors) app.install(api18n) actionsmapplugin = _ActionsMapPlugin(actionsmap, log_queues) app.install(actionsmapplugin) @@ -760,6 +769,12 @@ class Interface: self.display = actionsmapplugin.display self.prompt = actionsmapplugin.prompt + def handle_options(): + return HTTPResponse("", 204) + + app.route('/<:re:.*>', method="OPTIONS", + callback=handle_options, skip=["actionsmap"]) + # Append additional routes # TODO: Add optional authentication to those routes? for (m, p), c in routes.items(): From a6c7e55d1dcfc26c94cc97713b270037d236b30a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 29 Jul 2023 19:10:47 +0200 Subject: [PATCH 06/22] api: fix authentication failure not deleting expired cookies --- moulinette/interfaces/api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moulinette/interfaces/api.py b/moulinette/interfaces/api.py index 32383977..ac632ec0 100644 --- a/moulinette/interfaces/api.py +++ b/moulinette/interfaces/api.py @@ -387,6 +387,7 @@ class _ActionsMapPlugin: try: session_infos = authenticator.get_session_cookie() except Exception: + authenticator.delete_session_cookie() msg = m18n.g("authentication_required") raise HTTPResponse(msg, 401) From e24d56d5f18be388499d451a128a18d01e833b37 Mon Sep 17 00:00:00 2001 From: selfhoster1312 Date: Mon, 14 Aug 2023 22:00:28 +0200 Subject: [PATCH 07/22] /yunohost/sso/log{in,out} 303 to referer when GET/POST param referer_redirect is set --- moulinette/interfaces/api.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/moulinette/interfaces/api.py b/moulinette/interfaces/api.py index ac632ec0..6864c5cf 100644 --- a/moulinette/interfaces/api.py +++ b/moulinette/interfaces/api.py @@ -13,7 +13,7 @@ from gevent import sleep from gevent.queue import Queue from geventwebsocket import WebSocketError -from bottle import request, response, Bottle, HTTPResponse, FileUpload +from bottle import redirect, request, response, Bottle, HTTPResponse, FileUpload from bottle import abort from moulinette import m18n, Moulinette @@ -380,7 +380,11 @@ class _ActionsMapPlugin: raise HTTPResponse(e.strerror, 401) else: authenticator.set_session_cookie(auth_infos) - return m18n.g("logged_in") + referer = request.get_header("Referer") + if "referer_redirect" in request.params and referer: + redirect(referer) + else: + return m18n.g("logged_in") # This is called before each time a route is going to be processed def authenticate(self, authenticator): @@ -404,7 +408,11 @@ class _ActionsMapPlugin: else: # Delete cookie and clean the session authenticator.delete_session_cookie() - return m18n.g("logged_out") + referer = request.get_header("Referer") + if "referer_redirect" in request.params and referer: + redirect(referer) + else: + return m18n.g("logged_in") def messages(self): """Listen to the messages WebSocket stream From 7daa50459a5c1b8ab63f3b51c51b1fb46eb1fd5c Mon Sep 17 00:00:00 2001 From: selfhoster1312 Date: Mon, 14 Aug 2023 15:44:28 +0200 Subject: [PATCH 08/22] Bypass CSRF protection for the /yunohost/portalapi/login route Allowing login from simple HTML form Also allow to pass username/password as two params instead of a combined "credentials" --- moulinette/interfaces/api.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/moulinette/interfaces/api.py b/moulinette/interfaces/api.py index ac632ec0..f1bf9883 100644 --- a/moulinette/interfaces/api.py +++ b/moulinette/interfaces/api.py @@ -272,13 +272,14 @@ class _ActionsMapPlugin: name="login", method="POST", callback=self.login, - skip=["actionsmap"], + skip=[filter_csrf, "actionsmap"], ) app.route( "/logout", name="logout", method="GET", callback=self.logout, + # No need to bypass CSRF here because filter allows GET requests skip=["actionsmap"], ) @@ -362,9 +363,12 @@ class _ActionsMapPlugin: credentials = request.json["credentials"] profile = request.json.get("profile", self.actionsmap.default_authentication) else: - if "credentials" not in request.params: + if "credentials" in request.params: + credentials = request.params["credentials"] + elif "username" in request.params and "password" in request.params: + credentials = request.params["username"] + ":" + request.params["password"] + else: raise HTTPResponse("Missing credentials parameter", 400) - credentials = request.params["credentials"] profile = request.params.get("profile", self.actionsmap.default_authentication) From 2696e811cef26f2249d33f8b7fc864a1a579ef92 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 27 Sep 2023 20:02:41 +0200 Subject: [PATCH 09/22] quality: make linter gods happy --- moulinette/actionsmap.py | 2 -- moulinette/utils/log.py | 1 - 2 files changed, 3 deletions(-) diff --git a/moulinette/actionsmap.py b/moulinette/actionsmap.py index c213ab2e..a0f60a13 100644 --- a/moulinette/actionsmap.py +++ b/moulinette/actionsmap.py @@ -585,8 +585,6 @@ class ActionsMap: """ - start = time() - interface_type = top_parser.interface # If loading from cache, extra were already checked when cache was diff --git a/moulinette/utils/log.py b/moulinette/utils/log.py index 3d010726..be7eb6f3 100644 --- a/moulinette/utils/log.py +++ b/moulinette/utils/log.py @@ -6,7 +6,6 @@ from logging import ( addLevelName, setLoggerClass, Logger, - getLogger, NOTSET, # noqa DEBUG, INFO, From 7210b66fbad4ed47b7e61380d87fbc3d3c8682e6 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 27 Sep 2023 20:04:15 +0200 Subject: [PATCH 10/22] quality: update tox.ini, bookworm has python 3.11 --- tox.ini | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tox.ini b/tox.ini index ff656a7b..837dd19e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{37,39}-{pytest,lint,invalidcode,mypy} + py311-{pytest,lint,invalidcode,mypy} format format-check docs @@ -11,20 +11,19 @@ usedevelop = True passenv = * extras = tests deps = - py{37,39}-pytest: .[tests] - py{37,39}-lint: flake8 - py{37,39}-invalidcode: flake8 - py{37,39}-mypy: mypy >= 0.761 + py311-pytest: .[tests] + py311-lint: flake8 + py311-invalidcode: flake8 + py311-mypy: mypy >= 0.761 commands = - py{37,39}-pytest: pytest {posargs} -c pytest.ini - py{37,39}-lint: flake8 moulinette test - py{37,39}-invalidcode: flake8 moulinette test --select F - py{37,39}-mypy: mypy --ignore-missing-imports --install-types --non-interactive moulinette/ + py311-pytest: pytest {posargs} -c pytest.ini + py311-lint: flake8 moulinette test + py311-invalidcode: flake8 moulinette test --select F + py311-mypy: mypy --ignore-missing-imports --install-types --non-interactive moulinette/ [gh-actions] python = - 3.7: py37 - 3.9: py39 + 3.11: py311 [testenv:format] basepython = python3 From 37331cb1d6ed83463d66f5bb43e92a6420c961b5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 27 Sep 2023 20:06:34 +0200 Subject: [PATCH 11/22] quality: fix test/conftest.py, there's no ActionFilter anymore --- test/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/conftest.py b/test/conftest.py index 50b73580..c211e81b 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -46,7 +46,7 @@ def logging_configuration(moulinette): "format": "%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s" # noqa }, }, - "filters": {"action": {"()": "moulinette.utils.log.ActionFilter"}}, + "filters": {}, "handlers": { "api": { "level": level, From fc1eef2d928861823578e1ad1533a8434318b592 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 27 Sep 2023 20:08:38 +0200 Subject: [PATCH 12/22] quality: we're in python 3.11 bruh --- .github/workflows/tox.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 2c4cf7f5..1f53d1eb 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.9] + python-version: [3.11] steps: - uses: actions/checkout@v1 - name: Set up Python ${{ matrix.python-version }} @@ -26,7 +26,7 @@ jobs: python -m pip install --upgrade pip pip install tox tox-gh-actions - name: Test with tox - run: tox -e py39-pytest + run: tox -e py311-pytest invalidcode: runs-on: ubuntu-latest @@ -44,6 +44,6 @@ jobs: python -m pip install --upgrade pip pip install tox tox-gh-actions - name: Linter - run: tox -e py39-invalidcode + run: tox -e py311-invalidcode - name: Mypy - run: tox -e py39-mypy + run: tox -e py311-mypy From 8c28a573e2e86185aa694504e9d7feedeb8298fa Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 27 Sep 2023 20:09:48 +0200 Subject: [PATCH 13/22] =?UTF-8?q?quality:=20we're=20in=20python=203.11=20b?= =?UTF-8?q?ruh=C2=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/tox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 1f53d1eb..d06d7c48 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.9] + python-version: [3.11] steps: - uses: actions/checkout@v1 - name: Set up Python ${{ matrix.python-version }} From bd9736efc1f59dc04f10614bfd972eb8d0801a6f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 27 Sep 2023 20:11:16 +0200 Subject: [PATCH 14/22] =?UTF-8?q?quality:=20we're=20in=20python=203.11=20b?= =?UTF-8?q?ruh=C2=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 435d43f6..9cb279ca 100755 --- a/setup.py +++ b/setup.py @@ -60,7 +60,7 @@ setup( license="AGPL", packages=find_packages(exclude=["test"]), data_files=[("/usr/share/moulinette/locales", locale_files)], - python_requires=">=3.7.0,<3.10", + python_requires=">=3.11.0,<3.12", install_requires=install_deps, tests_require=test_deps, extras_require=extras, From f562a9b333799c2889d316522a86b2cd661f02f8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 27 Sep 2023 20:15:57 +0200 Subject: [PATCH 15/22] fix old logger mechanism remants --- moulinette/interfaces/api.py | 2 +- moulinette/interfaces/cli.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moulinette/interfaces/api.py b/moulinette/interfaces/api.py index ac632ec0..9c17ca17 100644 --- a/moulinette/interfaces/api.py +++ b/moulinette/interfaces/api.py @@ -30,7 +30,7 @@ from moulinette.interfaces import ( ) from moulinette.utils import log -logger = log.getLogger("moulinette.interface.api") +logger = logging.getLogger("moulinette.interface.api") # API helpers ---------------------------------------------------------- diff --git a/moulinette/interfaces/cli.py b/moulinette/interfaces/cli.py index f1005558..010f3346 100644 --- a/moulinette/interfaces/cli.py +++ b/moulinette/interfaces/cli.py @@ -50,7 +50,7 @@ def monkey_get_action_name(argument): argparse._get_action_name = monkey_get_action_name -logger = log.getLogger("moulinette.cli") +logger = logging.getLogger("moulinette.cli") # CLI helpers ---------------------------------------------------------- From 20d3b82340ee4f1705f87a2cd5a68a6b38b26add Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 27 Sep 2023 20:23:11 +0200 Subject: [PATCH 16/22] fix test ... apparently the API now returns 405 when no action is specified, I guess that's okay --- test/test_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_auth.py b/test/test_auth.py index 1f557329..033901a2 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -180,7 +180,7 @@ class TestAuthAPI: def test_request_arg_without_action(self, moulinette_webapi, caplog, mocker): self.login(moulinette_webapi) - moulinette_webapi.get("/test-auth", status=404) + moulinette_webapi.get("/test-auth", status=405) class TestAuthCLI: From 924fd7825e3e43d53daeb7c33e7ff1d7816bc223 Mon Sep 17 00:00:00 2001 From: axolotle Date: Wed, 8 Nov 2023 19:11:18 +0100 Subject: [PATCH 17/22] cors: fix some http response error not being catched by cors decorator --- moulinette/interfaces/api.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/moulinette/interfaces/api.py b/moulinette/interfaces/api.py index 9c17ca17..20c9ec11 100644 --- a/moulinette/interfaces/api.py +++ b/moulinette/interfaces/api.py @@ -734,7 +734,11 @@ class Interface: def cors(callback): def wrapper(*args, **kwargs): - r = callback(*args, **kwargs) + try: + r = callback(*args, **kwargs) + except HTTPResponse as e: + r = e + origin = request.headers.environ.get("HTTP_ORIGIN", "") if origin and origin in self.allowed_cors_origins: resp = r if isinstance(r, HTTPResponse) else response From d53dfc29970b4fc0728984911a9395b20f2fccb5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 13 Nov 2023 15:35:49 +0100 Subject: [PATCH 18/22] debug: print stacktrace to stderr upon 500 errors, because otherwise APIs are hell to debug ~_~ --- moulinette/interfaces/api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moulinette/interfaces/api.py b/moulinette/interfaces/api.py index 20c9ec11..3a3450be 100644 --- a/moulinette/interfaces/api.py +++ b/moulinette/interfaces/api.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +import sys import re import errno import logging @@ -471,6 +471,7 @@ class _ActionsMapPlugin: tb = traceback.format_exc() logs = {"route": _route, "arguments": arguments, "traceback": tb} + print(tb, file=sys.stderr) return HTTPResponse(json_encode(logs), 500) else: return format_for_response(ret) From 976aac0d052796b2bbb0ab213b204d6bf96ed08f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 Nov 2023 15:55:45 +0100 Subject: [PATCH 19/22] Do not log about loading auth module, it creates tricky issue when manually launching yunohost APIs to debug them --- moulinette/actionsmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moulinette/actionsmap.py b/moulinette/actionsmap.py index a0f60a13..fc93ed9f 100644 --- a/moulinette/actionsmap.py +++ b/moulinette/actionsmap.py @@ -464,7 +464,7 @@ class ActionsMap: # Load and initialize the authenticator module auth_module = f"{self.namespace}.authenticators.{auth_method}" - logger.debug(f"Loading auth module {auth_module}") + #logger.debug(f"Loading auth module {auth_module}") try: mod = import_module(auth_module) except ImportError as e: From cfb840c5ccb8417494fb4d9556b59068abf4150e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 Nov 2023 15:56:52 +0100 Subject: [PATCH 20/22] perf: in call_async_output: only wait for 0.1 sec, should speed up things significantly for stuff that calls a lot of hooks... --- moulinette/utils/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moulinette/utils/process.py b/moulinette/utils/process.py index 99945428..d015f8c9 100644 --- a/moulinette/utils/process.py +++ b/moulinette/utils/process.py @@ -82,7 +82,7 @@ def call_async_output(args, callback, **kwargs): while p.poll() is None: while True: try: - callback, message = log_queue.get(True, 1) + callback, message = log_queue.get(True, 0.1) except queue.Empty: break From 9cc786e83c4038b80d5954c61bbcc5b16a3f1567 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 26 Jul 2024 21:57:48 +0200 Subject: [PATCH 21/22] Update changelog for 12.0.1 testing --- debian/changelog | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index f148a393..c4b270dc 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,8 @@ -moulinette (12.0.0) unstable; urgency=low +moulinette (12.0.1) testing; urgency=low - - Tmp changelog to prepare Bookworm + - Tweaks and fixes for new portal API / ssowat refactoring ([#341](https://github.com/YunoHost/moulinette/pull/341)) + - Simplify logging : unecessary messages + obscure concept of "action id" ([#337](https://github.com/YunoHost/moulinette/pull/337)) + - Misc tweaks to adapt code and tests to Python 3.11 -- Alexandre Aubin Thu, 04 May 2023 20:30:19 +0200 From 709585be83304cbbe2eb8dedb75005fafdf3291f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 31 Aug 2024 20:13:48 +0200 Subject: [PATCH 22/22] Update changelog for 12.0.2 --- debian/changelog | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/debian/changelog b/debian/changelog index c4b270dc..2e49d2c8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +moulinette (12.0.2) testing; urgency=low + + - portal-api: Bypass CSRF protection for login route ([#340](http://github.com/YunoHost/moulinette/pull/340)) + - portal-api: login/logout redirect to referer when param referer_redirect is set ([#339](http://github.com/YunoHost/moulinette/pull/339)) + + Thanks to all contributors <3 ! (selfhoster1312) + + -- Alexandre Aubin Sat, 31 Aug 2024 20:12:43 +0200 + moulinette (12.0.1) testing; urgency=low - Tweaks and fixes for new portal API / ssowat refactoring ([#341](https://github.com/YunoHost/moulinette/pull/341))