diff --git a/.github/workflows/autoblack.yml b/.github/workflows/autoblack.yml new file mode 100644 index 00000000..35380607 --- /dev/null +++ b/.github/workflows/autoblack.yml @@ -0,0 +1,28 @@ +name: Check / auto apply Black +on: + push: + branches: + - dev +jobs: + black: + name: Check / auto apply black + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Check files using the black formatter + uses: rickstaa/action-black@v1 + id: action_black + with: + black_args: "." + continue-on-error: true + - name: Create Pull Request + if: steps.action_black.outputs.is_formatted == 'true' + uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + title: "Format Python code with Black" + commit-message: ":art: Format Python code with Black" + body: | + This pull request uses the [psf/black](https://github.com/psf/black) formatter. + base: ${{ github.head_ref }} # Creates pull request onto pull request or commit branch + branch: actions/black diff --git a/doc/conf.py b/doc/conf.py index 3eafc4ea..b68fec4b 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -18,18 +18,21 @@ import os import sys -sys.path.insert(0, os.path.abspath('..')) + +sys.path.insert(0, os.path.abspath("..")) import sys from mock import Mock as MagicMock + class Mock(MagicMock): @classmethod def __getattr__(cls, name): - return MagicMock() + return MagicMock() -MOCK_MODULES = ['ldap', 'ldap.modlist', 'ldap.sasl'] + +MOCK_MODULES = ["ldap", "ldap.modlist", "ldap.sasl"] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) @@ -42,36 +45,38 @@ sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autodoc', - 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', - 'sphinx.ext.viewcode'] +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "sphinx.ext.viewcode", +] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = u'Moulinette' -copyright = u'2017, YunoHost Collective' -author = u'YunoHost Collective' +project = u"Moulinette" +copyright = u"2017, YunoHost Collective" +author = u"YunoHost Collective" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = u'2.6.1' +version = u"2.6.1" # The full version, including alpha/beta/rc tags. -release = u'2.6.1' +release = u"2.6.1" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -83,10 +88,10 @@ language = None # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = True @@ -97,7 +102,7 @@ todo_include_todos = True # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'classic' +html_theme = "classic" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -108,7 +113,7 @@ html_theme = 'classic' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -116,11 +121,11 @@ html_static_path = ['_static'] # This is required for the alabaster theme # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars html_sidebars = { - '**': [ + "**": [ # 'about.html', # 'navigation.html', # 'relations.html', # needs 'show_related': True theme option to display - 'searchbox.html', + "searchbox.html", # 'donate.html', ] } @@ -129,7 +134,7 @@ html_sidebars = { # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. -htmlhelp_basename = 'Moulinettedoc' +htmlhelp_basename = "Moulinettedoc" # -- Options for LaTeX output --------------------------------------------- @@ -138,15 +143,12 @@ latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -156,8 +158,13 @@ latex_elements = { # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'Moulinette.tex', u'Moulinette Documentation', - u'YunoHost Collective', 'manual'), + ( + master_doc, + "Moulinette.tex", + u"Moulinette Documentation", + u"YunoHost Collective", + "manual", + ), ] @@ -165,10 +172,7 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'moulinette', u'Moulinette Documentation', - [author], 1) -] +man_pages = [(master_doc, "moulinette", u"Moulinette Documentation", [author], 1)] # -- Options for Texinfo output ------------------------------------------- @@ -177,13 +181,17 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'Moulinette', u'Moulinette Documentation', - author, 'Moulinette', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "Moulinette", + u"Moulinette Documentation", + author, + "Moulinette", + "One line description of project.", + "Miscellaneous", + ), ] - - # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/': None} +intersphinx_mapping = {"https://docs.python.org/": None} diff --git a/moulinette/interfaces/api.py b/moulinette/interfaces/api.py index c155d7a3..db942649 100644 --- a/moulinette/interfaces/api.py +++ b/moulinette/interfaces/api.py @@ -5,12 +5,14 @@ import errno import logging import argparse from json import dumps as json_encode +from tempfile import mkdtemp +from shutil import rmtree from gevent import sleep from gevent.queue import Queue from geventwebsocket import WebSocketError -from bottle import request, response, Bottle, HTTPResponse +from bottle import request, response, Bottle, HTTPResponse, FileUpload from bottle import abort from moulinette import m18n, Moulinette @@ -28,6 +30,8 @@ logger = log.getLogger("moulinette.interface.api") # API helpers ---------------------------------------------------------- +# We define a global variable to manage in a dirty way the upload... +UPLOAD_DIR = None CSRF_TYPES = set( ["text/plain", "application/x-www-form-urlencoded", "multipart/form-data"] @@ -111,6 +115,7 @@ class _HTTPArgumentParser(object): self._positional = [] # list(arg_name) self._optional = {} # dict({arg_name: option_strings}) + self._upload_dir = None def set_defaults(self, **kwargs): return self._parser.set_defaults(**kwargs) @@ -145,9 +150,9 @@ class _HTTPArgumentParser(object): # Append newly created action if len(action.option_strings) == 0: - self._positional.append(action.dest) + self._positional.append(action) else: - self._optional[action.dest] = action.option_strings + self._optional[action.dest] = action return action @@ -155,11 +160,26 @@ class _HTTPArgumentParser(object): arg_strings = [] # Append an argument to the current one - def append(arg_strings, value, option_string=None): - if isinstance(value, bool): + def append(arg_strings, value, action): + option_string = None + if len(action.option_strings) > 0: + option_string = action.option_strings[0] + + if isinstance(value, bool) or isinstance(action.const, bool): # Append the option string only + if option_string is not None and value != 0: + arg_strings.append(option_string) + elif isinstance(value, FileUpload) and ( + isinstance(action.type, argparse.FileType) or action.type == open + ): + # Upload the file in a temp directory + global UPLOAD_DIR + if UPLOAD_DIR is None: + UPLOAD_DIR = mkdtemp(prefix="moulinette_upload_") + value.save(UPLOAD_DIR) if option_string is not None: arg_strings.append(option_string) + arg_strings.append(UPLOAD_DIR + "/" + value.filename) elif isinstance(value, str): if option_string is not None: arg_strings.append(option_string) @@ -192,14 +212,14 @@ class _HTTPArgumentParser(object): return arg_strings # Iterate over positional arguments - for dest in self._positional: - if dest in args: - arg_strings = append(arg_strings, args[dest]) + for action in self._positional: + if action.dest in args: + arg_strings = append(arg_strings, args[action.dest], action) # Iterate over optional arguments - for dest, opt in self._optional.items(): + for dest, action in self._optional.items(): if dest in args: - arg_strings = append(arg_strings, args[dest], opt[0]) + arg_strings = append(arg_strings, args[dest], action) return self._parser.parse_args(arg_strings, namespace) @@ -319,8 +339,12 @@ class _ActionsMapPlugin(object): # Format boolean params for a in args: params[a] = True + # Append other request params - for k, v in request.params.decode().dict.items(): + req_params = list(request.params.decode().dict.items()) + # TODO test special chars in filename + req_params += list(request.files.dict.items()) + for k, v in req_params: v = _format(v) if k not in params.keys(): params[k] = v @@ -464,6 +488,14 @@ class _ActionsMapPlugin(object): else: return format_for_response(ret) finally: + + # Clean upload directory + # FIXME do that in a better way + global UPLOAD_DIR + if UPLOAD_DIR is not None: + rmtree(UPLOAD_DIR, True) + UPLOAD_DIR = None + # Close opened WebSocket by putting StopIteration in the queue try: s_id = Session.get_infos()["id"] diff --git a/setup.py b/setup.py index e315070c..a7f7c1b6 100755 --- a/setup.py +++ b/setup.py @@ -6,53 +6,54 @@ from setuptools import setup, find_packages from moulinette import init_moulinette_env -LOCALES_DIR = init_moulinette_env()['LOCALES_DIR'] +LOCALES_DIR = init_moulinette_env()["LOCALES_DIR"] # Extend installation locale_files = [] if "install" in sys.argv: # Evaluate locale files - for f in os.listdir('locales'): - if f.endswith('.json'): - locale_files.append('locales/%s' % f) + for f in os.listdir("locales"): + if f.endswith(".json"): + locale_files.append("locales/%s" % f) install_deps = [ - 'argcomplete', - 'psutil', - 'pytz', - 'pyyaml', - 'toml', - 'gevent-websocket', - 'bottle', + "argcomplete", + "psutil", + "pytz", + "pyyaml", + "toml", + "gevent-websocket", + "bottle", ] test_deps = [ - 'pytest', - 'pytest-cov', - 'pytest-env', - 'pytest-mock', - 'requests', - 'requests-mock', - 'webtest' + "pytest", + "pytest-cov", + "pytest-env", + "pytest-mock", + "requests", + "requests-mock", + "webtest", ] extras = { - 'install': install_deps, - 'tests': test_deps, + "install": install_deps, + "tests": test_deps, } -setup(name='Moulinette', - version='2.0.0', - description='Prototype interfaces quickly and easily', - author='Yunohost Team', - author_email='yunohost@yunohost.org', - url='http://yunohost.org', - license='AGPL', - packages=find_packages(exclude=['test']), - data_files=[(LOCALES_DIR, locale_files)], - python_requires='>=3.7.*, <3.8', - install_requires=install_deps, - tests_require=test_deps, - extras_require=extras, - ) +setup( + name="Moulinette", + version="2.0.0", + description="Prototype interfaces quickly and easily", + author="Yunohost Team", + author_email="yunohost@yunohost.org", + url="http://yunohost.org", + license="AGPL", + packages=find_packages(exclude=["test"]), + data_files=[(LOCALES_DIR, locale_files)], + python_requires=">=3.7.*, <3.8", + install_requires=install_deps, + tests_require=test_deps, + extras_require=extras, +) diff --git a/test/test_i18n_keys.py b/test/test_i18n_keys.py index 815549b8..912cf448 100644 --- a/test/test_i18n_keys.py +++ b/test/test_i18n_keys.py @@ -38,6 +38,7 @@ def find_expected_string_keys(): continue yield m + ############################################################################### # Load en locale json keys # ###############################################################################