Merge branch 'dev' into rework-authenticator-system

.github/workflows/autoblack.yml

@@ -0,0 +1,28 @@
+name: Check / auto apply Black
+on:
+  push:
+      branches:
+          - dev
+jobs:
+  black:
+    name: Check / auto apply black
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Check files using the black formatter
+        uses: rickstaa/action-black@v1
+        id: action_black
+        with:
+          black_args: "."
+        continue-on-error: true
+      - name: Create Pull Request
+        if: steps.action_black.outputs.is_formatted == 'true'
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          title: "Format Python code with Black"
+          commit-message: ":art: Format Python code with Black"
+          body: |
+            This pull request uses the [psf/black](https://github.com/psf/black) formatter.
+          base: ${{ github.head_ref }} # Creates pull request onto pull request or commit branch
+          branch: actions/black

doc/conf.py

@@ -18,18 +18,21 @@
 
 import os
 import sys
-sys.path.insert(0, os.path.abspath('..'))
+
+sys.path.insert(0, os.path.abspath(".."))
 
 
 import sys
 from mock import Mock as MagicMock
 
+
 class Mock(MagicMock):
     @classmethod
     def __getattr__(cls, name):
-            return MagicMock()
+        return MagicMock()
 
-MOCK_MODULES = ['ldap', 'ldap.modlist', 'ldap.sasl']
+
+MOCK_MODULES = ["ldap", "ldap.modlist", "ldap.sasl"]
 sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
 
 
@@ -42,36 +45,38 @@ sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['sphinx.ext.autodoc',
+extensions = [
-    'sphinx.ext.intersphinx',
+    "sphinx.ext.autodoc",
-    'sphinx.ext.todo',
+    "sphinx.ext.intersphinx",
-    'sphinx.ext.viewcode']
+    "sphinx.ext.todo",
+    "sphinx.ext.viewcode",
+]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 #
 # source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = u'Moulinette'
+project = u"Moulinette"
-copyright = u'2017, YunoHost Collective'
+copyright = u"2017, YunoHost Collective"
-author = u'YunoHost Collective'
+author = u"YunoHost Collective"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = u'2.6.1'
+version = u"2.6.1"
 # The full version, including alpha/beta/rc tags.
-release = u'2.6.1'
+release = u"2.6.1"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -83,10 +88,10 @@ language = None
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This patterns also effect to html_static_path and html_extra_path
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = True
@@ -97,7 +102,7 @@ todo_include_todos = True
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'classic'
+html_theme = "classic"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -108,7 +113,7 @@ html_theme = 'classic'
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 # Custom sidebar templates, must be a dictionary that maps document names
 # to template names.
@@ -116,11 +121,11 @@ html_static_path = ['_static']
 # This is required for the alabaster theme
 # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
 html_sidebars = {
-    '**': [
+    "**": [
         # 'about.html',
         # 'navigation.html',
         # 'relations.html',  # needs 'show_related': True theme option to display
-        'searchbox.html',
+        "searchbox.html",
         # 'donate.html',
     ]
 }
@@ -129,7 +134,7 @@ html_sidebars = {
 # -- Options for HTMLHelp output ------------------------------------------
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'Moulinettedoc'
+htmlhelp_basename = "Moulinettedoc"
 
 
 # -- Options for LaTeX output ---------------------------------------------
@@ -138,15 +143,12 @@ latex_elements = {
     # The paper size ('letterpaper' or 'a4paper').
     #
     # 'papersize': 'letterpaper',
-
     # The font size ('10pt', '11pt' or '12pt').
     #
     # 'pointsize': '10pt',
-
     # Additional stuff for the LaTeX preamble.
     #
     # 'preamble': '',
-
     # Latex figure (float) alignment
     #
     # 'figure_align': 'htbp',
@@ -156,8 +158,13 @@ latex_elements = {
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'Moulinette.tex', u'Moulinette Documentation',
+    (
-     u'YunoHost Collective', 'manual'),
+        master_doc,
+        "Moulinette.tex",
+        u"Moulinette Documentation",
+        u"YunoHost Collective",
+        "manual",
+    ),
 ]
 
 
@@ -165,10 +172,7 @@ latex_documents = [
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
+man_pages = [(master_doc, "moulinette", u"Moulinette Documentation", [author], 1)]
-    (master_doc, 'moulinette', u'Moulinette Documentation',
-     [author], 1)
-]
 
 
 # -- Options for Texinfo output -------------------------------------------
@@ -177,13 +181,17 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'Moulinette', u'Moulinette Documentation',
+    (
-     author, 'Moulinette', 'One line description of project.',
+        master_doc,
-     'Miscellaneous'),
+        "Moulinette",
+        u"Moulinette Documentation",
+        author,
+        "Moulinette",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
 ]
 
 
-
-
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'https://docs.python.org/': None}
+intersphinx_mapping = {"https://docs.python.org/": None}

moulinette/interfaces/api.py

@@ -5,12 +5,14 @@ import errno
 import logging
 import argparse
 from json import dumps as json_encode
+from tempfile import mkdtemp
+from shutil import rmtree
 
 from gevent import sleep
 from gevent.queue import Queue
 from geventwebsocket import WebSocketError
 
-from bottle import request, response, Bottle, HTTPResponse
+from bottle import request, response, Bottle, HTTPResponse, FileUpload
 from bottle import abort
 
 from moulinette import m18n, Moulinette
@@ -28,6 +30,8 @@ logger = log.getLogger("moulinette.interface.api")
 
 
 # API helpers ----------------------------------------------------------
+# We define a global variable to manage in a dirty way the upload...
+UPLOAD_DIR = None
 
 CSRF_TYPES = set(
     ["text/plain", "application/x-www-form-urlencoded", "multipart/form-data"]
@@ -111,6 +115,7 @@ class _HTTPArgumentParser(object):
 
         self._positional = []  # list(arg_name)
         self._optional = {}  # dict({arg_name: option_strings})
+        self._upload_dir = None
 
     def set_defaults(self, **kwargs):
         return self._parser.set_defaults(**kwargs)
@@ -145,9 +150,9 @@ class _HTTPArgumentParser(object):
 
         # Append newly created action
         if len(action.option_strings) == 0:
-            self._positional.append(action.dest)
+            self._positional.append(action)
         else:
-            self._optional[action.dest] = action.option_strings
+            self._optional[action.dest] = action
 
         return action
 
@@ -155,11 +160,26 @@ class _HTTPArgumentParser(object):
         arg_strings = []
 
         # Append an argument to the current one
-        def append(arg_strings, value, option_string=None):
+        def append(arg_strings, value, action):
-            if isinstance(value, bool):
+            option_string = None
+            if len(action.option_strings) > 0:
+                option_string = action.option_strings[0]
+
+            if isinstance(value, bool) or isinstance(action.const, bool):
                 # Append the option string only
+                if option_string is not None and value != 0:
+                    arg_strings.append(option_string)
+            elif isinstance(value, FileUpload) and (
+                isinstance(action.type, argparse.FileType) or action.type == open
+            ):
+                # Upload the file in a temp directory
+                global UPLOAD_DIR
+                if UPLOAD_DIR is None:
+                    UPLOAD_DIR = mkdtemp(prefix="moulinette_upload_")
+                value.save(UPLOAD_DIR)
                 if option_string is not None:
                     arg_strings.append(option_string)
+                arg_strings.append(UPLOAD_DIR + "/" + value.filename)
             elif isinstance(value, str):
                 if option_string is not None:
                     arg_strings.append(option_string)
@@ -192,14 +212,14 @@ class _HTTPArgumentParser(object):
             return arg_strings
 
         # Iterate over positional arguments
-        for dest in self._positional:
+        for action in self._positional:
-            if dest in args:
+            if action.dest in args:
-                arg_strings = append(arg_strings, args[dest])
+                arg_strings = append(arg_strings, args[action.dest], action)
 
         # Iterate over optional arguments
-        for dest, opt in self._optional.items():
+        for dest, action in self._optional.items():
             if dest in args:
-                arg_strings = append(arg_strings, args[dest], opt[0])
+                arg_strings = append(arg_strings, args[dest], action)
 
         return self._parser.parse_args(arg_strings, namespace)
 
@@ -319,8 +339,12 @@ class _ActionsMapPlugin(object):
             # Format boolean params
             for a in args:
                 params[a] = True
+
             # Append other request params
-            for k, v in request.params.decode().dict.items():
+            req_params = list(request.params.decode().dict.items())
+            # TODO test special chars in filename
+            req_params += list(request.files.dict.items())
+            for k, v in req_params:
                 v = _format(v)
                 if k not in params.keys():
                     params[k] = v
@@ -464,6 +488,14 @@ class _ActionsMapPlugin(object):
         else:
             return format_for_response(ret)
         finally:
+
+            # Clean upload directory
+            # FIXME do that in a better way
+            global UPLOAD_DIR
+            if UPLOAD_DIR is not None:
+                rmtree(UPLOAD_DIR, True)
+                UPLOAD_DIR = None
+
             # Close opened WebSocket by putting StopIteration in the queue
             try:
                 s_id = Session.get_infos()["id"]

setup.py

@@ -6,53 +6,54 @@ from setuptools import setup, find_packages
 from moulinette import init_moulinette_env
 
 
-LOCALES_DIR = init_moulinette_env()['LOCALES_DIR']
+LOCALES_DIR = init_moulinette_env()["LOCALES_DIR"]
 
 # Extend installation
 locale_files = []
 
 if "install" in sys.argv:
     # Evaluate locale files
-    for f in os.listdir('locales'):
+    for f in os.listdir("locales"):
-        if f.endswith('.json'):
+        if f.endswith(".json"):
-            locale_files.append('locales/%s' % f)
+            locale_files.append("locales/%s" % f)
 
 install_deps = [
-    'argcomplete',
+    "argcomplete",
-    'psutil',
+    "psutil",
-    'pytz',
+    "pytz",
-    'pyyaml',
+    "pyyaml",
-    'toml',
+    "toml",
-    'gevent-websocket',
+    "gevent-websocket",
-    'bottle',
+    "bottle",
 ]
 
 test_deps = [
-    'pytest',
+    "pytest",
-    'pytest-cov',
+    "pytest-cov",
-    'pytest-env',
+    "pytest-env",
-    'pytest-mock',
+    "pytest-mock",
-    'requests',
+    "requests",
-    'requests-mock',
+    "requests-mock",
-    'webtest'
+    "webtest",
 ]
 extras = {
-    'install': install_deps,
+    "install": install_deps,
-    'tests': test_deps,
+    "tests": test_deps,
 }
 
 
-setup(name='Moulinette',
+setup(
-      version='2.0.0',
+    name="Moulinette",
-      description='Prototype interfaces quickly and easily',
+    version="2.0.0",
-      author='Yunohost Team',
+    description="Prototype interfaces quickly and easily",
-      author_email='yunohost@yunohost.org',
+    author="Yunohost Team",
-      url='http://yunohost.org',
+    author_email="yunohost@yunohost.org",
-      license='AGPL',
+    url="http://yunohost.org",
-      packages=find_packages(exclude=['test']),
+    license="AGPL",
-      data_files=[(LOCALES_DIR, locale_files)],
+    packages=find_packages(exclude=["test"]),
-      python_requires='>=3.7.*,  <3.8',
+    data_files=[(LOCALES_DIR, locale_files)],
-      install_requires=install_deps,
+    python_requires=">=3.7.*,  <3.8",
-      tests_require=test_deps,
+    install_requires=install_deps,
-      extras_require=extras,
+    tests_require=test_deps,
-      )
+    extras_require=extras,
+)

test/test_i18n_keys.py

@@ -38,6 +38,7 @@ def find_expected_string_keys():
                 continue
             yield m
 
+
 ###############################################################################
 #   Load en locale json keys                                                  #
 ###############################################################################