Compare commits

..

5 commits

Author SHA1 Message Date
Alexandre Aubin
8de98670e9
Merge pull request #358 from yunohost-bot/weblate-yunohost-moulinette
Translations update from Weblate
2024-08-20 16:28:26 +02:00
xabirequejo
068d6d369a Translated using Weblate (Basque)
Currently translated at 100.0% (45 of 45 strings)

Translation: YunoHost/moulinette
Translate-URL: https://translate.yunohost.org/projects/yunohost/moulinette/eu/
2024-08-16 17:54:48 +02:00
Ivan Davydov
3c7f55c610 Translated using Weblate (Russian)
Currently translated at 100.0% (45 of 45 strings)

Translation: YunoHost/moulinette
Translate-URL: https://translate.yunohost.org/projects/yunohost/moulinette/ru/
2024-08-05 12:13:55 +02:00
cjdw
531c972bed Translated using Weblate (Indonesian)
Currently translated at 100.0% (45 of 45 strings)

Translation: YunoHost/moulinette
Translate-URL: https://translate.yunohost.org/projects/yunohost/moulinette/id/
2024-07-16 11:03:01 +02:00
Ivan Davydov
7c378210fa Translated using Weblate (Russian)
Currently translated at 100.0% (45 of 45 strings)

Translation: YunoHost/moulinette
Translate-URL: https://translate.yunohost.org/projects/yunohost/moulinette/ru/
2024-07-04 20:54:46 +02:00
16 changed files with 163 additions and 127 deletions

View file

@ -12,7 +12,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [3.11]
python-version: [3.9]
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
@ -26,13 +26,13 @@ jobs:
python -m pip install --upgrade pip
pip install tox tox-gh-actions
- name: Test with tox
run: tox -e py311-pytest
run: tox -e py39-pytest
invalidcode:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [3.11]
python-version: [3.9]
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
@ -44,6 +44,6 @@ jobs:
python -m pip install --upgrade pip
pip install tox tox-gh-actions
- name: Linter
run: tox -e py311-invalidcode
run: tox -e py39-invalidcode
- name: Mypy
run: tox -e py311-mypy
run: tox -e py39-mypy

17
debian/changelog vendored
View file

@ -1,20 +1,3 @@
moulinette (12.0.2) testing; urgency=low
- portal-api: Bypass CSRF protection for login route ([#340](http://github.com/YunoHost/moulinette/pull/340))
- portal-api: login/logout redirect to referer when param referer_redirect is set ([#339](http://github.com/YunoHost/moulinette/pull/339))
Thanks to all contributors <3 ! (selfhoster1312)
-- Alexandre Aubin <alex.aubin@mailoo.org> Sat, 31 Aug 2024 20:12:43 +0200
moulinette (12.0.1) testing; urgency=low
- Tweaks and fixes for new portal API / ssowat refactoring ([#341](https://github.com/YunoHost/moulinette/pull/341))
- Simplify logging : unecessary messages + obscure concept of "action id" ([#337](https://github.com/YunoHost/moulinette/pull/337))
- Misc tweaks to adapt code and tests to Python 3.11
-- Alexandre Aubin <alex.aubin@mailoo.org> Thu, 04 May 2023 20:30:19 +0200
moulinette (11.2.1) stable; urgency=low
- repo chores: various black enhancements

View file

@ -29,7 +29,7 @@
"pattern_not_match": "Ez dator ereduarekin bat",
"root_required": "Ezinbestekoa da 'root' izatea eragiketa hau exekutatzeko",
"server_already_running": "Zerbitzari bat martxan dago dagoeneko ataka horretan",
"unable_authenticate": "Ezinezkoa izan da autentifikatzea",
"unable_authenticate": "Ezin da autentifikatu",
"values_mismatch": "Balioak ez datoz bat",
"warning": "Adi:",
"cannot_open_file": "Ezinezkoa izan da {file} fitxategia irekitzea (zergatia: {error})",

View file

@ -13,16 +13,16 @@
"error_removing": "Terjadi galat ketika menghapus {path}: {error}",
"success": "Berhasil!",
"warn_the_user_about_waiting_lock": "Perintah YunoHost lain sedang berjalan saat ini, kami sedang menunggu itu selesai sebelum menjalankan yang ini",
"warn_the_user_about_waiting_lock_again": "Masih menunggu",
"warn_the_user_about_waiting_lock_again": "Masih menunggu...",
"unable_authenticate": "Tidak dapat mengautentikasi",
"warn_the_user_that_lock_is_acquired": "Perintah yang tadi baru saja selesai, akan memulai perintah ini",
"server_already_running": "Sebuah peladen telah berjalan di porta tersebut",
"unknown_group": "Kelompok '{group}' tidak diketahui",
"unknown_user": "Pengguna '{user}' tidak diketahui",
"values_mismatch": "Tidak sama",
"values_mismatch": "Nnilai berbeda",
"cannot_write_file": "Tidak dapat menyimpan berkas {file} (alasan: {error})",
"unknown_error_reading_file": "Galat yang tidak diketahui ketika membaca berkas {file} (alasan: {error})",
"invalid_url": "Gagal terhubung dengan {url} … mungkin layanan tersebut sedang tidak berjalan atau Anda tidak terhubung ke internet di IPv4/IPv6.",
"invalid_url": "Gagal terhubung dengan {url}... mungkin layanan tersebut sedang turun, atau Anda tidak terhubung dengan benar ke internet di IPv4/IPv6.",
"download_timeout": "{url} memakan waktu yang lama untuk menjawab, menyerah.",
"download_unknown_error": "Galat ketika mengunduh data dari {url}: {error}",
"download_bad_status_code": "{url} menjawab dengan kode status {code}",
@ -42,5 +42,6 @@
"root_required": "Anda harus berada di root untuk melakukan tindakan ini",
"corrupted_yaml": "Pembacaan rusak untuk YAML {ressource} (alasan: {error})",
"corrupted_toml": "Pembacaan rusak untuk TOML {ressource} (alasan: {error})",
"corrupted_json": "Pembacaan rusak untuk JSON {ressource} (alasan: {error})"
"corrupted_json": "Pembacaan rusak untuk JSON {ressource} (alasan: {error})",
"websocket_request_expected": "Mengharapkan permintaan WebSocket"
}

View file

@ -1,5 +1,5 @@
{
"argument_required": "Требуется'{argument}' аргумент",
"argument_required": "Требуется аргумент «{argument}»",
"authentication_required": "Требуется аутентификация",
"confirm": "Подтвердить {prompt}",
"deprecated_command": "'{prog} {command}' устарела и будет удалена",
@ -10,7 +10,7 @@
"invalid_argument": "Неправильный аргумент '{argument}': {error}",
"logged_in": "Вы вошли",
"logged_out": "Вы вышли из системы",
"not_logged_in": "Вы не залогинились",
"not_logged_in": "Вы не вошли в систему",
"operation_interrupted": "Действие прервано",
"password": "Пароль",
"pattern_not_match": "Не соответствует образцу",
@ -28,7 +28,7 @@
"corrupted_yaml": "Повреждённой YAML получен от {ressource} (причина: {error})",
"error_writing_file": "Ошибка при записи файла {file}: {error}",
"error_removing": "Ошибка при удалении {path}: {error}",
"invalid_url": "Не удалось подключиться к {url} возможно этот сервис недоступен или вы не подключены к Интернету через IPv4/IPv6.",
"invalid_url": "Не удалось подключиться к {url}... возможно этот сервис недоступен или вы не подключены к Интернету через IPv4/IPv6.",
"download_ssl_error": "Ошибка SSL при соединении с {url}",
"download_timeout": "Превышено время ожидания ответа от {url}.",
"download_unknown_error": "Ошибка при загрузке данных с {url} : {error}",
@ -36,7 +36,7 @@
"root_required": "Чтобы выполнить это действие, вы должны иметь права root",
"corrupted_json": "Повреждённый json получен от {ressource} (причина: {error})",
"warn_the_user_that_lock_is_acquired": "Другая команда только что завершилась, теперь запускается эта команда",
"warn_the_user_about_waiting_lock_again": "Все еще жду",
"warn_the_user_about_waiting_lock_again": "Все еще жду...",
"warn_the_user_about_waiting_lock": "Сейчас запускается еще одна команда YunoHost, мы ждем ее завершения, прежде чем запустить эту",
"download_bad_status_code": "{url} вернул код состояния {code}",
"error_changing_file_permissions": "Ошибка при изменении разрешений для {path}: {error}",

View file

@ -53,7 +53,7 @@ class Moulinette:
# Easy access to interfaces
def api(host="localhost", port=80, routes={}, actionsmap=None, locales_dir=None, allowed_cors_origins=[]):
def api(host="localhost", port=80, routes={}, actionsmap=None, locales_dir=None):
"""Web server (API) interface
Run a HTTP server with the moulinette for an API usage.
@ -73,7 +73,6 @@ def api(host="localhost", port=80, routes={}, actionsmap=None, locales_dir=None,
Api(
routes=routes,
actionsmap=actionsmap,
allowed_cors_origins=allowed_cors_origins,
).run(host, port)
except MoulinetteError as e:
import logging

View file

@ -19,6 +19,7 @@ from moulinette.core import (
MoulinetteValidationError,
)
from moulinette.interfaces import BaseActionsMapParser
from moulinette.utils.log import start_action_logging
from moulinette.utils.filesystem import read_yaml
logger = logging.getLogger("moulinette.actionsmap")
@ -391,6 +392,8 @@ class ActionsMap:
self.from_cache = False
logger.debug("loading actions map")
actionsmap_yml_dir = os.path.dirname(actionsmap_yml)
actionsmap_yml_file = os.path.basename(actionsmap_yml)
actionsmap_yml_stat = os.stat(actionsmap_yml)
@ -458,7 +461,7 @@ class ActionsMap:
# Load and initialize the authenticator module
auth_module = f"{self.namespace}.authenticators.{auth_method}"
#logger.debug(f"Loading auth module {auth_module}")
logger.debug(f"Loading auth module {auth_module}")
try:
mod = import_module(auth_module)
except ImportError as e:
@ -553,7 +556,17 @@ class ActionsMap:
logger.exception(error_message)
raise MoulinetteError(error_message, raw_msg=True)
else:
logger.debug("processing action '%s'", full_action_name)
log_id = start_action_logging()
if logger.isEnabledFor(logging.DEBUG):
# Log arguments in debug mode only for safety reasons
logger.debug(
"processing action [%s]: %s with args=%s",
log_id,
full_action_name,
arguments,
)
else:
logger.debug("processing action [%s]: %s", log_id, full_action_name)
# Load translation and process the action
start = time()
@ -561,7 +574,7 @@ class ActionsMap:
return func(**arguments)
finally:
stop = time()
logger.debug("action executed in %.3fs", stop - start)
logger.debug("action [%s] executed in %.3fs", log_id, stop - start)
# Private methods
@ -579,6 +592,9 @@ class ActionsMap:
"""
logger.debug("building parser...")
start = time()
interface_type = top_parser.interface
# If loading from cache, extra were already checked when cache was
@ -695,4 +711,5 @@ class ActionsMap:
else:
action_parser.want_to_take_lock = True
logger.debug("building parser took %.3fs", time() - start)
return top_parser

View file

@ -32,7 +32,8 @@ class BaseActionsMapParser:
"""
def __init__(self, parent=None, **kwargs):
pass
if not parent:
logger.debug("initializing base actions map parser for %s", self.interface)
# Virtual properties
# Each parser classes must implement these properties.

View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
import sys
import re
import errno
import logging
@ -13,7 +13,7 @@ from gevent import sleep
from gevent.queue import Queue
from geventwebsocket import WebSocketError
from bottle import redirect, request, response, Bottle, HTTPResponse, FileUpload
from bottle import request, response, Bottle, HTTPResponse, FileUpload
from bottle import abort
from moulinette import m18n, Moulinette
@ -30,7 +30,7 @@ from moulinette.interfaces import (
)
from moulinette.utils import log
logger = logging.getLogger("moulinette.interface.api")
logger = log.getLogger("moulinette.interface.api")
# API helpers ----------------------------------------------------------
@ -269,14 +269,13 @@ class _ActionsMapPlugin:
name="login",
method="POST",
callback=self.login,
skip=[filter_csrf, "actionsmap"],
skip=["actionsmap"],
)
app.route(
"/logout",
name="logout",
method="GET",
callback=self.logout,
# No need to bypass CSRF here because filter allows GET requests
skip=["actionsmap"],
)
@ -310,9 +309,6 @@ class _ActionsMapPlugin:
return value
def wrapper(*args, **kwargs):
if request.get_header("Content-Type") == "application/json":
return callback((request.method, context.rule), request.json)
params = kwargs
# Format boolean params
for a in args:
@ -354,21 +350,11 @@ class _ActionsMapPlugin:
"""
if request.get_header("Content-Type") == "application/json":
if "credentials" not in request.json:
raise HTTPResponse("Missing credentials parameter", 400)
credentials = request.json["credentials"]
profile = request.json.get("profile", self.actionsmap.default_authentication)
else:
if "credentials" in request.params:
credentials = request.params["credentials"]
elif "username" in request.params and "password" in request.params:
credentials = request.params["username"] + ":" + request.params["password"]
else:
raise HTTPResponse("Missing credentials parameter", 400)
profile = request.params.get("profile", self.actionsmap.default_authentication)
if "credentials" not in request.params:
raise HTTPResponse("Missing credentials parameter", 400)
credentials = request.params["credentials"]
profile = request.params.get("profile", self.actionsmap.default_authentication)
authenticator = self.actionsmap.get_authenticator(profile)
try:
@ -381,18 +367,13 @@ class _ActionsMapPlugin:
raise HTTPResponse(e.strerror, 401)
else:
authenticator.set_session_cookie(auth_infos)
referer = request.get_header("Referer")
if "referer_redirect" in request.params and referer:
redirect(referer)
else:
return m18n.g("logged_in")
return m18n.g("logged_in")
# This is called before each time a route is going to be processed
def authenticate(self, authenticator):
try:
session_infos = authenticator.get_session_cookie()
except Exception:
authenticator.delete_session_cookie()
msg = m18n.g("authentication_required")
raise HTTPResponse(msg, 401)
@ -409,11 +390,7 @@ class _ActionsMapPlugin:
else:
# Delete cookie and clean the session
authenticator.delete_session_cookie()
referer = request.get_header("Referer")
if "referer_redirect" in request.params and referer:
redirect(referer)
else:
return m18n.g("logged_in")
return m18n.g("logged_out")
def messages(self):
"""Listen to the messages WebSocket stream
@ -480,7 +457,6 @@ class _ActionsMapPlugin:
tb = traceback.format_exc()
logs = {"route": _route, "arguments": arguments, "traceback": tb}
print(tb, file=sys.stderr)
return HTTPResponse(json_encode(logs), 500)
else:
return format_for_response(ret)
@ -726,11 +702,9 @@ class Interface:
type = "api"
def __init__(self, routes={}, actionsmap=None, allowed_cors_origins=[]):
def __init__(self, routes={}, actionsmap=None):
actionsmap = ActionsMap(actionsmap, ActionsMapParser())
self.allowed_cors_origins = allowed_cors_origins
# Attempt to retrieve log queues from an APIQueueHandler
handler = log.getHandlersByClass(APIQueueHandler, limit=1)
if handler:
@ -740,22 +714,11 @@ class Interface:
# TODO: Return OK to 'OPTIONS' xhr requests (l173)
app = Bottle(autojson=True)
def cors(callback):
# Wrapper which sets proper header
def apiheader(callback):
def wrapper(*args, **kwargs):
try:
r = callback(*args, **kwargs)
except HTTPResponse as e:
r = e
origin = request.headers.environ.get("HTTP_ORIGIN", "")
if origin and origin in self.allowed_cors_origins:
resp = r if isinstance(r, HTTPResponse) else response
resp.headers['Access-Control-Allow-Origin'] = origin
resp.headers['Access-Control-Allow-Methods'] = 'GET, HEAD, POST, PUT, OPTIONS, DELETE'
resp.headers['Access-Control-Allow-Headers'] = 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token'
resp.headers['Access-Control-Allow-Credentials'] = 'true'
return r
response.set_header("Access-Control-Allow-Origin", "*")
return callback(*args, **kwargs)
return wrapper
@ -764,7 +727,7 @@ class Interface:
def wrapper(*args, **kwargs):
try:
locale = request.params.pop("locale")
except (KeyError, ValueError):
except KeyError:
locale = m18n.default_locale
m18n.set_locale(locale)
return callback(*args, **kwargs)
@ -773,7 +736,7 @@ class Interface:
# Install plugins
app.install(filter_csrf)
app.install(cors)
app.install(apiheader)
app.install(api18n)
actionsmapplugin = _ActionsMapPlugin(actionsmap, log_queues)
app.install(actionsmapplugin)
@ -782,12 +745,6 @@ class Interface:
self.display = actionsmapplugin.display
self.prompt = actionsmapplugin.prompt
def handle_options():
return HTTPResponse("", 204)
app.route('/<:re:.*>', method="OPTIONS",
callback=handle_options, skip=["actionsmap"])
# Append additional routes
# TODO: Add optional authentication to those routes?
for (m, p), c in routes.items():

View file

@ -50,7 +50,7 @@ def monkey_get_action_name(argument):
argparse._get_action_name = monkey_get_action_name
logger = logging.getLogger("moulinette.cli")
logger = log.getLogger("moulinette.cli")
# CLI helpers ----------------------------------------------------------
@ -234,26 +234,28 @@ class TTYHandler(logging.StreamHandler):
log.CRITICAL: "red",
}
def __init__(self, message_key="message_with_color"):
def __init__(self, message_key="fmessage"):
logging.StreamHandler.__init__(self)
self.message_key = message_key
def format(self, record):
"""Enhance message with level and colors if supported."""
msg = record.getMessage()
level = record.levelname
level_with_color = level
if self.supports_color():
if self.level > log.DEBUG and record.levelname in ["SUCCESS", "WARNING", "ERROR", "INFO"]:
level = m18n.g(record.levelname.lower())
level = ""
if self.level <= log.DEBUG:
# add level name before message
level = "%s " % record.levelname
elif record.levelname in ["SUCCESS", "WARNING", "ERROR", "INFO"]:
# add translated level name before message
level = "%s " % m18n.g(record.levelname.lower())
color = self.LEVELS_COLOR.get(record.levelno, "white")
level_with_color = f"{colors_codes[color]}{level}{END_CLI_COLOR}"
if self.level == log.DEBUG:
level_with_color = level_with_color + " " * max(0, 7 - len(level))
msg = "{}{}{}{}".format(colors_codes[color], level, END_CLI_COLOR, msg)
if self.formatter:
record.__dict__["level_with_color"] = level_with_color
# use user-defined formatter
record.__dict__[self.message_key] = msg
return self.formatter.format(record)
return level_with_color + " " + msg
return msg
def emit(self, record):
# set proper stream first

View file

@ -6,6 +6,7 @@ from logging import (
addLevelName,
setLoggerClass,
Logger,
getLogger,
NOTSET, # noqa
DEBUG,
INFO,
@ -108,14 +109,88 @@ class MoulinetteLogger(Logger):
f = currentframe()
if f is not None:
f = f.f_back
rv = "(unknown file)", 0, "(unknown function)", None
rv = "(unknown file)", 0, "(unknown function)"
while hasattr(f, "f_code"):
co = f.f_code
filename = os.path.normcase(co.co_filename)
if filename == _srcfile or filename == __file__:
f = f.f_back
continue
rv = (co.co_filename, f.f_lineno, co.co_name, None)
rv = (co.co_filename, f.f_lineno, co.co_name)
break
return rv
def _log(self, *args, **kwargs):
"""Append action_id if available to the extra."""
if self.action_id is not None:
extra = kwargs.get("extra", {})
if "action_id" not in extra:
# FIXME: Get real action_id instead of logger/current one
extra["action_id"] = _get_action_id()
kwargs["extra"] = extra
return super()._log(*args, **kwargs)
# Action logging -------------------------------------------------------
pid = os.getpid()
action_id = 0
def _get_action_id():
return "%d.%d" % (pid, action_id)
def start_action_logging():
"""Configure logging for a new action
Returns:
The new action id
"""
global action_id
action_id += 1
return _get_action_id()
def getActionLogger(name=None, logger=None, action_id=None):
"""Get the logger adapter for an action
Return a logger for the specified name - or use given logger - and
optionally for a given action id, retrieving it if necessary.
Either a name or a logger must be specified.
"""
if not name and not logger:
raise ValueError("Either a name or a logger must be specified")
logger = logger or getLogger(name)
logger.action_id = action_id if action_id else _get_action_id()
return logger
class ActionFilter:
"""Extend log record for an optionnal action
Filter a given record and look for an `action_id` key. If it is not found
and `strict` is True, the record will not be logged. Otherwise, the key
specified by `message_key` will be added to the record, containing the
message formatted for the action or just the original one.
"""
def __init__(self, message_key="fmessage", strict=False):
self.message_key = message_key
self.strict = strict
def filter(self, record):
msg = record.getMessage()
action_id = record.__dict__.get("action_id", None)
if action_id is not None:
msg = "[{:s}] {:s}".format(action_id, msg)
elif self.strict:
return False
record.__dict__[self.message_key] = msg
return True

View file

@ -82,7 +82,7 @@ def call_async_output(args, callback, **kwargs):
while p.poll() is None:
while True:
try:
callback, message = log_queue.get(True, 0.1)
callback, message = log_queue.get(True, 1)
except queue.Empty:
break

View file

@ -60,7 +60,7 @@ setup(
license="AGPL",
packages=find_packages(exclude=["test"]),
data_files=[("/usr/share/moulinette/locales", locale_files)],
python_requires=">=3.11.0,<3.12",
python_requires=">=3.7.0,<3.10",
install_requires=install_deps,
tests_require=test_deps,
extras_require=extras,

View file

@ -46,7 +46,7 @@ def logging_configuration(moulinette):
"format": "%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s" # noqa
},
},
"filters": {},
"filters": {"action": {"()": "moulinette.utils.log.ActionFilter"}},
"handlers": {
"api": {
"level": level,

View file

@ -180,7 +180,7 @@ class TestAuthAPI:
def test_request_arg_without_action(self, moulinette_webapi, caplog, mocker):
self.login(moulinette_webapi)
moulinette_webapi.get("/test-auth", status=405)
moulinette_webapi.get("/test-auth", status=404)
class TestAuthCLI:

21
tox.ini
View file

@ -1,6 +1,6 @@
[tox]
envlist =
py311-{pytest,lint,invalidcode,mypy}
py{37,39}-{pytest,lint,invalidcode,mypy}
format
format-check
docs
@ -11,19 +11,20 @@ usedevelop = True
passenv = *
extras = tests
deps =
py311-pytest: .[tests]
py311-lint: flake8
py311-invalidcode: flake8
py311-mypy: mypy >= 0.761
py{37,39}-pytest: .[tests]
py{37,39}-lint: flake8
py{37,39}-invalidcode: flake8
py{37,39}-mypy: mypy >= 0.761
commands =
py311-pytest: pytest {posargs} -c pytest.ini
py311-lint: flake8 moulinette test
py311-invalidcode: flake8 moulinette test --select F
py311-mypy: mypy --ignore-missing-imports --install-types --non-interactive moulinette/
py{37,39}-pytest: pytest {posargs} -c pytest.ini
py{37,39}-lint: flake8 moulinette test
py{37,39}-invalidcode: flake8 moulinette test --select F
py{37,39}-mypy: mypy --ignore-missing-imports --install-types --non-interactive moulinette/
[gh-actions]
python =
3.11: py311
3.7: py37
3.9: py39
[testenv:format]
basepython = python3