Apply manageproject updates

This commit is contained in:
Jens Diemer 2024-08-02 17:01:38 +02:00
parent ca6defd5f4
commit ceec1325fb
16 changed files with 1814 additions and 1494 deletions

View file

@ -16,5 +16,5 @@ max_line_length = 119
indent_style = tab
insert_final_newline = false
[*.yml]
[{*.yaml,*.yml}]
indent_size = 2

View file

@ -1,3 +1,5 @@
name: tests
on:
@ -14,10 +16,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ["3.12", "3.11", "3.10", "3.9"]
env:
PYTHONUNBUFFERED: 1
PYTHONWARNINGS: always
python-version: ["3.12", "3.11", "3.10"]
steps:
- name: Checkout
run: |
@ -35,7 +34,7 @@ jobs:
with:
python-version: '${{ matrix.python-version }}'
cache: 'pip' # caching pip dependencies
cache-dependency-path: '**/requirements.dev.txt'
cache-dependency-path: '**/requirements.*.txt'
- name: 'Bootstrap dev venv'
# The first CLI call will create the .venv
@ -46,11 +45,14 @@ jobs:
run: |
./dev-cli.py --help
- name: 'Safety'
- name: 'Run pip-audit'
run: |
./dev-cli.py safety
./dev-cli.py pip-audit
- name: 'Run tests with Python v${{ matrix.python-version }}'
env:
PYTHONUNBUFFERED: 1
PYTHONWARNINGS: always
run: |
.venv/bin/coverage erase
./dev-cli.py coverage
@ -61,3 +63,4 @@ jobs:
with:
fail_ci_if_error: false
verbose: true

1
.gitignore vendored
View file

@ -2,6 +2,7 @@
*.egg-info
__pycache__
/dist/
/build/
/coverage.*
/htmlcov/
*.orig

File diff suppressed because it is too large Load diff

View file

@ -59,7 +59,9 @@ if 'axes' not in INSTALLED_APPS:
INSTALLED_APPS.append('django_yunohost_integration.apps.YunohostIntegrationConfig')
SECRET_KEY = __get_or_create_secret(DATA_DIR_PATH / 'secret.txt') # /home/yunohost.app/$app/secret.txt
SECRET_KEY = __get_or_create_secret(
DATA_DIR_PATH / 'secret.txt'
) # /home/yunohost.app/$app/secret.txt
MIDDLEWARE.insert(

View file

@ -1,6 +1,7 @@
from django.conf import settings
from django.contrib import admin
from django.urls import path
from django.views.generic import RedirectView
from for_runners.views.media_files import UserMediaView
@ -9,6 +10,7 @@ if settings.PATH_URL:
# settings.PATH_URL is __PATH__
# Prefix all urls with "PATH_URL":
urlpatterns = [
path('', RedirectView.as_view(url=f'{settings.PATH_URL}/')),
path(f'{settings.PATH_URL}/media/<slug:user_name>/<path:path>', UserMediaView.as_view()),
# TODO: https://github.com/jedie/django-for-runners/issues/25
# path(settings.MEDIA_URL.lstrip('/'), include('django_tools.serve_media_app.urls')),

View file

@ -8,6 +8,7 @@
"""
import hashlib
import shlex
import subprocess
import sys
import venv
@ -32,7 +33,7 @@ else:
sys.exit(-1)
assert sys.version_info >= (3, 9), 'Python version is too old!'
assert sys.version_info >= (3, 10), f'Python version {sys.version_info} is too old!'
if sys.platform == 'win32': # wtf
@ -77,7 +78,7 @@ def venv_up2date():
def verbose_check_call(*popen_args):
print(f'\n+ {" ".join(str(arg) for arg in popen_args)}\n')
print(f'\n+ {shlex.join(str(arg) for arg in popen_args)}\n')
return subprocess.check_call(popen_args)
@ -86,17 +87,17 @@ def main(argv):
# Create virtual env in ".venv/":
if not PYTHON_PATH.is_file():
print('Create virtual env here:', VENV_PATH.absolute())
print(f'Create virtual env here: {VENV_PATH.absolute()}')
builder = venv.EnvBuilder(symlinks=True, upgrade=True, with_pip=True)
builder.create(env_dir=VENV_PATH)
if not PROJECT_SHELL_SCRIPT.is_file() or not venv_up2date():
# Update pip
verbose_check_call(PYTHON_PATH, '-m', 'pip', 'install', '-U', 'pip')
if not PIP_SYNC_PATH.is_file():
# Install pip-tools
verbose_check_call(PYTHON_PATH, '-m', 'pip', 'install', '-U', 'pip-tools')
if not PROJECT_SHELL_SCRIPT.is_file() or not venv_up2date():
# install requirements via "pip-sync"
verbose_check_call(PIP_SYNC_PATH, str(DEP_LOCK_PATH))

View file

@ -2,31 +2,34 @@
CLI for development
"""
import logging
import os
import shlex
import sys
from pathlib import Path
import django
import rich_click as click
from bx_py_utils.path import assert_is_file
from cli_base.cli_tools.subprocess_utils import verbose_check_call
from cli_base.cli_tools.version_info import print_version
from django.core.management.commands.test import Command as DjangoTestCommand
from django_yunohost_integration.local_test import CreateResults, create_local_test
from manageprojects.utilities import code_style
from manageprojects.utilities.publish import publish_package
from rich import print # noqa; noqa
from rich_click import RichGroup
import for_runners_ynh
import rich_click as click
from cli_base.cli_tools import code_style
from cli_base.cli_tools.dev_tools import run_tox
from cli_base.cli_tools.subprocess_utils import verbose_check_call
from cli_base.cli_tools.test_utils.snapshot import UpdateTestSnapshotFiles
from cli_base.cli_tools.verbosity import OPTION_KWARGS_VERBOSE
from cli_base.cli_tools.version_info import print_version
from cli_base.run_pip_audit import run_pip_audit
from django.core.management.commands.test import Command as DjangoTestCommand
from for_runners_ynh import constants
from for_runners_ynh.constants import PACKAGE_ROOT
from for_runners_ynh.tests import setup_ynh_tests
from django_yunohost_integration.local_test import create_local_test
from manageprojects.utilities.publish import publish_package
from rich import print
from rich.console import Console
from rich.traceback import install as rich_traceback_install
from rich_click import RichGroup
logger = logging.getLogger(__name__)
PACKAGE_ROOT = Path(for_runners_ynh.__file__).parent.parent
assert_is_file(PACKAGE_ROOT / 'pyproject.toml')
OPTION_ARGS_DEFAULT_TRUE = dict(is_flag=True, show_default=True, default=True)
OPTION_ARGS_DEFAULT_FALSE = dict(is_flag=True, show_default=True, default=False)
ARGUMENT_EXISTING_DIR = dict(
@ -45,7 +48,6 @@ ARGUMENT_NOT_EXISTING_DIR = dict(
ARGUMENT_EXISTING_FILE = dict(
type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True, path_type=Path)
)
CLI_EPILOG = 'Project Homepage: https://github.com/YunoHost-Apps/django-for-runners_ynh'
class ClickGroup(RichGroup): # FIXME: How to set the "info_name" easier?
@ -54,38 +56,22 @@ class ClickGroup(RichGroup): # FIXME: How to set the "info_name" easier?
return super().make_context(info_name, *args, **kwargs)
@click.group(cls=ClickGroup, epilog=CLI_EPILOG)
@click.group(
cls=ClickGroup,
epilog=constants.CLI_EPILOG,
)
def cli():
pass
@click.command()
@click.option('--verbose/--no-verbose', **OPTION_ARGS_DEFAULT_FALSE)
def mypy(verbose: bool = True):
@cli.command()
@click.option('-v', '--verbosity', **OPTION_KWARGS_VERBOSE)
def mypy(verbosity: int):
"""Run Mypy (configured in pyproject.toml)"""
verbose_check_call('mypy', '.', cwd=PACKAGE_ROOT, verbose=verbose, exit_on_error=True)
verbose_check_call('mypy', '.', cwd=PACKAGE_ROOT, verbose=verbosity > 0, exit_on_error=True)
cli.add_command(mypy)
@click.command()
@click.option('--verbose/--no-verbose', **OPTION_ARGS_DEFAULT_FALSE)
def coverage(verbose: bool = True):
"""
Run and show coverage.
"""
verbose_check_call('coverage', 'run', verbose=verbose, exit_on_error=True)
verbose_check_call('coverage', 'combine', '--append', verbose=verbose, exit_on_error=True)
verbose_check_call('coverage', 'report', '--fail-under=10', verbose=verbose, exit_on_error=True)
verbose_check_call('coverage', 'xml', verbose=verbose, exit_on_error=True)
verbose_check_call('coverage', 'json', verbose=verbose, exit_on_error=True)
cli.add_command(coverage)
@click.command()
@cli.command()
def install():
"""
Run pip-sync and install 'for_runners_ynh' via pip as editable.
@ -94,21 +80,16 @@ def install():
verbose_check_call('pip', 'install', '--no-deps', '-e', '.')
cli.add_command(install)
@click.command()
def safety():
@cli.command()
@click.option('-v', '--verbosity', **OPTION_KWARGS_VERBOSE)
def pip_audit(verbosity: int):
"""
Run safety check against current requirements files
Run pip-audit check against current requirements files
"""
verbose_check_call('safety', 'check', '-r', 'requirements.dev.txt')
run_pip_audit(base_path=PACKAGE_ROOT, verbosity=verbosity)
cli.add_command(safety)
@click.command()
@cli.command()
def update():
"""
Update "requirements*.txt" dependencies files
@ -150,22 +131,19 @@ def update():
extra_env=extra_env,
)
verbose_check_call(bin_path / 'safety', 'check', '-r', 'requirements.dev.txt')
run_pip_audit(base_path=PACKAGE_ROOT)
# Install new dependencies in current .venv:
verbose_check_call(bin_path / 'pip-sync', 'requirements.dev.txt')
cli.add_command(update)
@click.command()
@cli.command()
def publish():
"""
Build and upload this project to PyPi
"""
try:
_run_django_test_cli() # Don't publish a broken state
_run_django_test_cli(argv=sys.argv, exit_after_run=True) # Don't publish a broken state
except SystemExit as err:
assert err.code == 0, f'Exit code is not 0: {err.code}'
@ -176,133 +154,79 @@ def publish():
)
cli.add_command(publish)
@click.command()
@cli.command()
@click.option('--color/--no-color', **OPTION_ARGS_DEFAULT_TRUE)
@click.option('--verbose/--no-verbose', **OPTION_ARGS_DEFAULT_FALSE)
def fix_code_style(color: bool = True, verbose: bool = False):
@click.option('-v', '--verbosity', **OPTION_KWARGS_VERBOSE)
def fix_code_style(color: bool, verbosity: int):
"""
Fix code style of all for_runners_ynh source code files via darker
Fix code style of all your_cool_package source code files via darker
"""
code_style.fix(package_root=PACKAGE_ROOT, color=color, verbose=verbose)
code_style.fix(package_root=PACKAGE_ROOT, darker_color=color, darker_verbose=verbosity > 0)
cli.add_command(fix_code_style)
@click.command()
@cli.command()
@click.option('--color/--no-color', **OPTION_ARGS_DEFAULT_TRUE)
@click.option('--verbose/--no-verbose', **OPTION_ARGS_DEFAULT_FALSE)
def check_code_style(color: bool = True, verbose: bool = False):
@click.option('-v', '--verbosity', **OPTION_KWARGS_VERBOSE)
def check_code_style(color: bool, verbosity: int):
"""
Check code style by calling darker + flake8
"""
code_style.check(package_root=PACKAGE_ROOT, color=color, verbose=verbose)
code_style.check(package_root=PACKAGE_ROOT, darker_color=color, darker_verbose=verbosity > 0)
cli.add_command(check_code_style)
@click.command()
@cli.command()
def update_test_snapshot_files():
"""
Update all test snapshot files (by remove and recreate all snapshot files)
"""
def iter_snapshot_files():
yield from PACKAGE_ROOT.rglob('*.snapshot.*')
removed_file_count = 0
for item in iter_snapshot_files():
item.unlink()
removed_file_count += 1
print(f'{removed_file_count} test snapshot files removed... run tests...')
with UpdateTestSnapshotFiles(root_path=PACKAGE_ROOT, verbose=True):
# Just recreate them by running tests:
os.environ['RAISE_SNAPSHOT_ERRORS'] = '0' # Recreate snapshot files without error
try:
_run_django_test_cli()
finally:
new_files = len(list(iter_snapshot_files()))
print(f'{new_files} test snapshot files created, ok.\n')
_run_django_test_cli(argv=sys.argv, exit_after_run=False)
cli.add_command(update_test_snapshot_files)
def _run_django_test_cli():
def _run_django_test_cli(argv, exit_after_run=True):
"""
Call the origin Django test manage command CLI and pass all args to it.
"""
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
setup_ynh_tests()
print('Compile YunoHost files...')
result: CreateResults = create_local_test(
django_settings_path=PACKAGE_ROOT / 'conf' / 'settings.py',
destination=PACKAGE_ROOT / 'local_test',
runserver=False,
extra_replacements={
'__DEBUG_ENABLED__': '0', # "1" or "0" string
'__LOG_LEVEL__': 'INFO',
'__ADMIN_EMAIL__': 'foo-bar@test.tld',
'__DEFAULT_FROM_EMAIL__': 'django_app@test.tld',
},
)
print('Local test files created:')
print(result)
data_dir = str(result.data_dir_path)
if data_dir not in sys.path:
sys.path.insert(0, data_dir)
django.setup()
os.chdir(Path(for_runners_ynh.__file__).parent)
print('\nStart Django unittests with:')
for default_arg in ('shuffle', 'buffer'):
if default_arg not in argv and f'--no-{default_arg}' not in argv:
argv.append(f'--{default_arg}')
print(shlex.join(argv))
print()
test_command = DjangoTestCommand()
test_command.run_from_argv(sys.argv)
test_command.run_from_argv(argv)
if exit_after_run:
sys.exit(0)
@click.command() # Dummy command
@cli.command() # Dummy command
def test():
"""
Compile YunoHost files and run Django unittests
"""
_run_django_test_cli()
_run_django_test_cli(argv=sys.argv, exit_after_run=True)
cli.add_command(test)
def _run_tox():
verbose_check_call(sys.executable, '-m', 'tox', *sys.argv[2:])
sys.exit(0)
@click.command() # Dummy "tox" command
@cli.command() # Dummy "tox" command
def tox():
"""
Run tox
"""
_run_tox()
run_tox()
cli.add_command(tox)
@click.command()
@cli.command()
def version():
"""Print version and exit"""
# Pseudo command, because the version always printed on every CLI call ;)
sys.exit(0)
cli.add_command(version)
@click.command()
@cli.command()
def local_test():
"""
Build a "local_test" YunoHost installation and start the Django dev. server against it.
@ -317,10 +241,7 @@ def local_test():
)
cli.add_command(local_test)
@click.command()
@cli.command()
def diffsettings():
"""
Run "diffsettings" manage command against a "local_test" YunoHost installation.
@ -343,22 +264,26 @@ def diffsettings():
)
cli.add_command(diffsettings)
def main():
print_version(for_runners_ynh)
print(f'{sys.argv=}')
if len(sys.argv) >= 2:
# Check if we just pass a command call
# Check if we can just pass a command call to origin CLI:
command = sys.argv[1]
if command == 'test':
_run_django_test_cli()
sys.exit(0)
elif command == 'tox':
_run_tox()
sys.exit(0)
command_map = {
'test': _run_django_test_cli,
'tox': run_tox,
}
if real_func := command_map.get(command):
real_func(argv=sys.argv, exit_after_run=True)
console = Console()
rich_traceback_install(
width=console.size.width, # full terminal width
show_locals=True,
suppress=[click],
max_frames=2,
)
print('Execute Click CLI')
cli()

View file

@ -0,0 +1,11 @@
from pathlib import Path
import for_runners_ynh
from bx_py_utils.path import assert_is_file
PACKAGE_ROOT = Path(for_runners_ynh.__file__).parent.parent
assert_is_file(PACKAGE_ROOT / 'pyproject.toml')
CLI_EPILOG = 'Project Homepage: https://github.com/YunoHost-Apps/django-for-runners_ynh'

View file

@ -1,6 +1,63 @@
import os
import sys
import unittest.util
from pathlib import Path
import django
import for_runners_ynh
from bx_py_utils.test_utils.deny_requests import deny_any_real_request
from cli_base.cli_tools.verbosity import MAX_LOG_LEVEL, setup_logging
from for_runners_ynh.constants import PACKAGE_ROOT
from django_yunohost_integration.local_test import CreateResults, create_local_test
from rich import print # noqa
# Hacky way to expand the failed test output:
unittest.util._MAX_LENGTH = os.environ.get('UNITTEST_MAX_LENGTH', 300)
def pre_configure_tests() -> None:
print(f'Configure unittests via "load_tests Protocol" from {Path(__file__).relative_to(Path.cwd())}')
# Hacky way to display more "assert"-Context in failing tests:
_MIN_MAX_DIFF = unittest.util._MAX_LENGTH - unittest.util._MIN_DIFF_LEN
unittest.util._MAX_LENGTH = int(os.environ.get('UNITTEST_MAX_LENGTH', 300))
unittest.util._MIN_DIFF_LEN = unittest.util._MAX_LENGTH - _MIN_MAX_DIFF
# Deny any request via docket/urllib3 because tests they should mock all requests:
deny_any_real_request()
# Display DEBUG logs in tests:
setup_logging(verbosity=MAX_LOG_LEVEL)
def setup_ynh_tests() -> None:
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
print('Compile YunoHost files...')
result: CreateResults = create_local_test(
django_settings_path=PACKAGE_ROOT / 'conf' / 'settings.py',
destination=PACKAGE_ROOT / 'local_test',
runserver=False,
extra_replacements={
'__DEBUG_ENABLED__': '0', # "1" or "0" string
'__LOG_LEVEL__': 'INFO',
'__ADMIN_EMAIL__': 'foo-bar@test.tld',
'__DEFAULT_FROM_EMAIL__': 'django_app@test.tld',
},
)
print('Local test files created:')
print(result)
data_dir = str(result.data_dir_path)
if data_dir not in sys.path:
sys.path.insert(0, data_dir)
django.setup()
os.chdir(Path(for_runners_ynh.__file__).parent)
def load_tests(loader, tests, pattern):
"""
Use unittest "load_tests Protocol" as a hook to setup test environment before running tests.
https://docs.python.org/3/library/unittest.html#load-tests-protocol
"""
pre_configure_tests()
return loader.discover(start_dir=Path(__file__).parent, pattern=pattern)

View file

@ -1,4 +1,3 @@
import os
from for_runners_ynh.cli.dev import PACKAGE_ROOT
@ -12,9 +11,8 @@ from bx_django_utils.filename import clean_filename
from bx_py_utils.path import assert_is_dir, assert_is_file
from django.test.testcases import TestCase
from django_tools.unittest_utils.project_setup import check_editor_config
from django_yunohost_integration.test_utils import assert_project_version
from for_runners import __version__ as upstream_version
from django_example import __version__ as upstream_version
from for_runners_ynh import __version__ as ynh_pkg_version
@ -47,13 +45,6 @@ class ProjectSetupTestCase(TestCase):
manifest_version = ynh_pkg_version.replace('+', '~')
self.assertEqual(self.manifest_cfg['version'], manifest_version)
if 'GITHUB_ACTION' not in os.environ:
# Github has a rate-limiting... So don't fetch the API if we run as GitHub action
assert_project_version(
current_version=ynh_pkg_version,
github_project_url='https://github.com/jedie/django-for-runners',
)
def test_screenshot_filenames(self):
"""
https://forum.yunohost.org/t/yunohost-bot-cant-handle-spaces-in-screenshots/19483

View file

@ -1,11 +1,11 @@
from pathlib import Path
from bx_py_utils.auto_doc import assert_readme_block
from for_runners_ynh.cli.dev import PACKAGE_ROOT, cli
from for_runners_ynh.constants import CLI_EPILOG
from manageprojects.test_utils.click_cli_utils import invoke_click
from manageprojects.tests.base import BaseTestCase
from for_runners_ynh.cli.dev import CLI_EPILOG, PACKAGE_ROOT, cli
def assert_cli_help_in_readme(text_block: str, marker: str, readme_path: Path):
text_block = text_block.replace(CLI_EPILOG, '')

25
local_settings_source.py Normal file
View file

@ -0,0 +1,25 @@
# This file will be copied to the "local test" files, to overwrite Django settings
import os
print('Load local settings file:', __file__)
ENV_TYPE = os.environ.get('ENV_TYPE', None)
print(f'ENV_TYPE: {ENV_TYPE!r}')
if ENV_TYPE == 'local':
print(f'Activate settings overwrite by {__file__}')
SECURE_SSL_REDIRECT = False # Don't redirect http to https
SERVE_FILES = True # May used in urls.py
AUTH_PASSWORD_VALIDATORS = [] # accept all passwords
ALLOWED_HOSTS = ['127.0.0.1', 'localhost'] # For local dev. server
CACHES = { # Setup a working cache, without Redis ;)
'default': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'unique-snowflake',
},
}
elif ENV_TYPE == 'test':
SILENCED_SYSTEM_CHECKS = ['security.W018'] # tests runs with DEBUG=True
ALLOWED_HOSTS = [] # For unittests (Django's setup_test_environment() will add 'testserver')

View file

@ -19,7 +19,7 @@ code = "https://github.com/jedie/django-for-runners"
[integration]
# https://yunohost.org/en/packaging_manifest#integration-section
yunohost = ">=11.2.12"
yunohost = ">=11"
architectures = "all"
multi_instance = true
ldap = true

View file

@ -25,7 +25,7 @@ dependencies = [
dev = [
"bx_django_utils", # https://github.com/boxine/bx_django_utils
"beautifulsoup4", # https://pypi.org/project/beautifulsoup4/
"manageprojects>=0.15.0", # https://github.com/jedie/manageprojects
"manageprojects", # https://github.com/jedie/manageprojects
"pip-tools", # https://github.com/jazzband/pip-tools/
"tblib", # https://github.com/ionelmc/python-tblib
"tox", # https://github.com/tox-dev/tox
@ -36,7 +36,7 @@ dev = [
"pyflakes", # https://github.com/PyCQA/pyflakes
"codespell", # https://github.com/codespell-project/codespell
"EditorConfig", # https://github.com/editorconfig/editorconfig-core-py
"safety", # https://github.com/pyupio/safety
"pip-audit", # https://github.com/pypa/pip-audit
"mypy", # https://github.com/python/mypy
"twine", # https://github.com/pypa/twine
@ -61,6 +61,9 @@ Source = "https://github.com/YunoHost-Apps/django-for-runners_ynh"
for_runners_ynh_app = "for_runners_ynh.__main__:main"
for_runners_ynh_dev = "for_runners_ynh.cli.dev:main"
[ynh-integration]
local_settings_source= "local_settings_source.py"
[build-system]
requires = ["setuptools>=61.0", "setuptools_scm>=7.1"]
build-backend = "setuptools.build_meta"
@ -73,6 +76,14 @@ include = ["for_runners_ynh*"]
version = {attr = "for_runners_ynh.__version__"}
[tool.cli_base.pip_audit]
# https://github.com/jedie/cli-base-utilities/blob/main/docs/pip_audit.md
requirements=["requirements.dev.txt"]
strict=true
require_hashes=true
ignore-vuln=[]
[tool.darker]
src = ['.']
# YunoHost apps still use "master" istead of "main", isn't it?
@ -163,6 +174,7 @@ applied_migrations = [
"3383cb0", # 2023-11-09T20:14:05+01:00
"4abd4c0", # 2023-11-25T15:59:31+01:00
"2f9fd7b", # 2023-11-26T20:13:32+01:00
"52669d0", # 2024-08-02T15:47:04+02:00
]
[manageprojects.cookiecutter_context.cookiecutter]

File diff suppressed because it is too large Load diff