From 5db169b2496b5a305f62117807ed40aa47577182 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Sun, 18 Sep 2022 19:00:39 +0200 Subject: [PATCH] Update project setup from upstream --- conf/gunicorn.conf.py | 2 +- conf/manage.py | 2 +- conf/settings.py | 86 ++++++++++++---------------- conf/urls.py | 2 - config_panel.toml | 35 ++++++++++++ local_test.py | 3 + manifest.json | 15 ++++- pyproject.toml | 4 +- scripts/_common.sh | 45 +++++++-------- scripts/change_url | 42 +++++++------- scripts/install | 100 ++++++++++++++++----------------- scripts/restore | 26 ++++----- scripts/upgrade | 105 +++++++++++++++-------------------- tests/test_django_project.py | 41 +++++++++----- tests/test_project_setup.py | 21 +++++-- 15 files changed, 272 insertions(+), 257 deletions(-) create mode 100644 config_panel.toml diff --git a/conf/gunicorn.conf.py b/conf/gunicorn.conf.py index bcab967..f49a397 100644 --- a/conf/gunicorn.conf.py +++ b/conf/gunicorn.conf.py @@ -17,4 +17,4 @@ accesslog = '__LOG_FILE__' errorlog = '__LOG_FILE__' # https://docs.gunicorn.org/en/latest/settings.html#pidfile -pidfile = '__FINAL_HOME_PATH__/gunicorn.pid' +pidfile = '__FINALPATH__/gunicorn.pid' diff --git a/conf/manage.py b/conf/manage.py index 9962cbd..ec26808 100755 --- a/conf/manage.py +++ b/conf/manage.py @@ -1,4 +1,4 @@ -#!__FINAL_HOME_PATH__/venv/bin/python +#!__FINALPATH__/venv/bin/python import os import sys diff --git a/conf/settings.py b/conf/settings.py index 49bf1f6..8df3787 100644 --- a/conf/settings.py +++ b/conf/settings.py @@ -2,7 +2,7 @@ ################################################################################ # Please do not modify this file, it will be reset at the next update. -# You can edit the file __FINAL_HOME_PATH__/local_settings.py and add/modify the settings you need. +# You can edit the file __FINALPATH__/local_settings.py and add/modify the settings you need. # The parameters you add in local_settings.py will overwrite these, # but you can use the options and documentation in this file to find out what can be done. @@ -11,35 +11,42 @@ from pathlib import Path as __Path -from django_yunohost_integration.base_settings import * # noqa +from django_yunohost_integration.base_settings import * # noqa:F401,F403 from django_yunohost_integration.secret_key import get_or_create_secret as __get_or_create_secret -from for_runners_project.settings.base import * # noqa -DEBUG = False # Don't turn DEBUG on in production! +from for_runners_project.settings.base import * # noqa isort:skip +from django_yunohost_integration.base_settings import LOGGING # noqa:F401 isort:skip -# ----------------------------------------------------------------------------- +FINALPATH = __Path('__FINALPATH__') # /opt/yunohost/$app +assert FINALPATH.is_dir(), f'Directory not exists: {FINALPATH}' -FINAL_HOME_PATH = __Path('__FINAL_HOME_PATH__') # /opt/yunohost/$app -assert FINAL_HOME_PATH.is_dir(), f'Directory not exists: {FINAL_HOME_PATH}' +PUBLIC_PATH = __Path('__PUBLIC_PATH__') # /var/www/$app +assert PUBLIC_PATH.is_dir(), f'Directory not exists: {PUBLIC_PATH}' -FINAL_WWW_PATH = __Path('__FINAL_WWW_PATH__') # /var/www/$app -assert FINAL_WWW_PATH.is_dir(), f'Directory not exists: {FINAL_WWW_PATH}' - -LOG_FILE = __Path('__LOG_FILE__') # /var/log/$app/django-for-runners.log +LOG_FILE = __Path('__LOG_FILE__') # /var/log/$app/django_example_ynh.log assert LOG_FILE.is_file(), f'File not exists: {LOG_FILE}' PATH_URL = '__PATH_URL__' # $YNH_APP_ARG_PATH PATH_URL = PATH_URL.strip('/') # ----------------------------------------------------------------------------- +# config_panel.toml settings: -ROOT_URLCONF = 'urls' # /opt/yunohost/django-for-runners/ynh_urls.py +DEBUG_ENABLED = '__DEBUG_ENABLED__' +DEBUG = bool(int(DEBUG_ENABLED)) + +LOG_LEVEL = '__LOG_LEVEL__' +ADMIN_EMAIL = '__ADMIN_EMAIL__' +DEFAULT_FROM_EMAIL = '__DEFAULT_FROM_EMAIL__' + + +# ----------------------------------------------------------------------------- # Function that will be called to finalize a user profile: YNH_SETUP_USER = 'setup_user.setup_project_user' -SECRET_KEY = __get_or_create_secret(FINAL_HOME_PATH / 'secret.txt') # /opt/yunohost/$app/secret.txt +SECRET_KEY = __get_or_create_secret(FINALPATH / 'secret.txt') # /opt/yunohost/$app/secret.txt INSTALLED_APPS.append('django_yunohost_integration') @@ -65,10 +72,12 @@ LOGIN_URL = '/yunohost/sso/' LOGOUT_REDIRECT_URL = '/yunohost/sso/' # /yunohost/sso/?action=logout +ROOT_URLCONF = 'urls' # .../conf/urls.py + # ----------------------------------------------------------------------------- -ADMINS = (('__ADMIN__', '__ADMINMAIL__'),) +ADMINS = (('__ADMIN__', ADMIN_EMAIL),) MANAGERS = ADMINS @@ -95,11 +104,10 @@ EMAIL_SUBJECT_PREFIX = f'[{SITE_TITLE}] ' # E-mail address that error messages come from. -SERVER_EMAIL = 'noreply@__DOMAIN__' +SERVER_EMAIL = ADMIN_EMAIL # Default email address to use for various automated correspondence from # the site managers. Used for registration emails. -DEFAULT_FROM_EMAIL = '__ADMINMAIL__' # List of URLs your site is supposed to serve ALLOWED_HOSTS = ['__DOMAIN__'] @@ -131,47 +139,25 @@ else: STATIC_URL = '/static/' MEDIA_URL = '/media/' -STATIC_ROOT = str(FINAL_WWW_PATH / 'static') -MEDIA_ROOT = str(FINAL_WWW_PATH / 'media') +STATIC_ROOT = str(PUBLIC_PATH / 'static') +MEDIA_ROOT = str(PUBLIC_PATH / 'media') + # ----------------------------------------------------------------------------- -LOGGING = { - 'version': 1, - 'disable_existing_loggers': True, - 'formatters': { - 'verbose': { - 'format': '{asctime} {levelname} {name} {module}.{funcName} {message}', - 'style': '{', - }, - }, - 'handlers': { - 'mail_admins': { - 'level': 'ERROR', - 'formatter': 'verbose', - 'class': 'django.utils.log.AdminEmailHandler', - 'include_html': True, - }, - 'log_file': { - 'level': 'DEBUG', - 'class': 'logging.handlers.WatchedFileHandler', - 'formatter': 'verbose', - 'filename': str(LOG_FILE), - }, - }, - 'loggers': { - '': {'handlers': ['log_file', 'mail_admins'], 'level': 'INFO', 'propagate': False}, - 'django': {'handlers': ['log_file', 'mail_admins'], 'level': 'INFO', 'propagate': False}, - 'axes': {'handlers': ['log_file', 'mail_admins'], 'level': 'WARNING', 'propagate': False}, - 'django_tools': {'handlers': ['log_file', 'mail_admins'], 'level': 'INFO', 'propagate': False}, - 'django_ynh': {'handlers': ['log_file', 'mail_admins'], 'level': 'INFO', 'propagate': False}, - 'for_runners': {'handlers': ['log_file', 'mail_admins'], 'level': 'INFO', 'propagate': False}, - }, +# Set log file to e.g.: /var/log/$app/$app.log +LOGGING['handlers']['log_file']['filename'] = str(LOG_FILE) + +# Example how to add logging to own app: +LOGGING['loggers']['for_runners'] = { + 'handlers': ['syslog', 'log_file', 'mail_admins'], + 'level': 'INFO', + 'propagate': False, } # ----------------------------------------------------------------------------- try: - from local_settings import * # noqa + from local_settings import * # noqa:F401,F403 except ImportError: pass diff --git a/conf/urls.py b/conf/urls.py index f0fd05f..8512891 100644 --- a/conf/urls.py +++ b/conf/urls.py @@ -1,8 +1,6 @@ from django.conf import settings -from django.conf.urls import static from django.contrib import admin from django.urls import path - from for_runners.views.media_files import UserMediaView diff --git a/config_panel.toml b/config_panel.toml new file mode 100644 index 0000000..921a8cb --- /dev/null +++ b/config_panel.toml @@ -0,0 +1,35 @@ +version = "1.0" + +[main] +name = "django_example_ynh configuration" +services = ["__APP__"] + + [main.config] + name = "Configuration Options" + + [main.config.default_from_email] + ask = "from email" + type = "email" + help = "Default email address to use for various automated emails." + bind = "default_from_email:__FINALPATH__/settings.py" + + [main.config.admin_email] + ask = "ADMIN email" + type = "email" + help = "EMail address for error emails." + bind = "admin_email:__FINALPATH__/settings.py" + + [main.config.debug_enabled] + ask = "DEBUG mode" + type = "boolean" + yes = "1" + no = "0" + help = "Should be never enabled in production!" + bind = "debug_enabled:__FINALPATH__/settings.py" + + [main.config.log_level] + type = "string" + ask = "Log Level" + choices = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] + default = "WARNING" + bind = "log_level:__FINALPATH__/settings.py" diff --git a/local_test.py b/local_test.py index e2664e5..1f857e9 100644 --- a/local_test.py +++ b/local_test.py @@ -22,6 +22,9 @@ def main(): django_settings_path=BASE_PATH / 'conf' / 'settings.py', destination=BASE_PATH / 'local_test', runserver=True, + extra_replacements={ + '__DEBUG_ENABLED__': '1', + }, ) diff --git a/manifest.json b/manifest.json index 7334a6b..cccac44 100644 --- a/manifest.json +++ b/manifest.json @@ -19,11 +19,11 @@ }, "previous_maintainers": [], "requirements": { - "yunohost": ">= 4.0" + "yunohost": ">= 4.4" }, "multi_instance": true, "services": [ - "nginx" + "nginx", "postgresql", "redis" ], "arguments": { "install" : [ @@ -40,7 +40,16 @@ { "name": "admin", "type": "user" - } + }, + { + "name": "is_public", + "type": "boolean", + "help": { + "en": "Any YunoHost user and anonymous people from the web will be able to access the application", + "fr": "Tout utilisateur YunoHost et les personnes anonymes pourront accéder à l'application" + }, + "default": false + } ] } } diff --git a/pyproject.toml b/pyproject.toml index fb7094a..e94ce2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,8 +54,8 @@ line_length = 100 # https://pycqa.github.io/isort/docs/configuration/config_files/#pyprojecttoml-preferred-format atomic=true profile='black' +skip_glob=["*/htmlcov/*","*/migrations/*","*/local_test/*"] line_length=100 -skip_glob=["*/htmlcov/*","*/migrations/*"] known_first_party=["for_runners"] lines_after_imports=2 @@ -63,7 +63,7 @@ lines_after_imports=2 [tool.pytest.ini_options] # https://docs.pytest.org/en/latest/customize.html#pyproject-toml minversion = "6.0" -norecursedirs = ".* .git __pycache__ conf coverage* dist htmlcov" +norecursedirs = ".* .git __pycache__ conf local_test coverage* dist htmlcov" # sometimes helpfull "addopts" arguments: # -vv # --verbose diff --git a/scripts/_common.sh b/scripts/_common.sh index 2c3cd69..24083aa 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -11,6 +11,22 @@ admin=$YNH_APP_ARG_ADMIN is_public=$YNH_APP_ARG_IS_PUBLIC app=$YNH_APP_INSTANCE_NAME +#================================================= +# ARGUMENTS FROM CONFIG PANEL +#================================================= + +# 'debug_enabled' -> '__DEBUG_ENABLED__' -> settings.DEBUG +debug_enabled="0" + +# 'log_level' -> '__LOG_LEVEL__' -> settings.LOG_LEVEL +log_level="WARNING" + +# 'admin_email' -> '__ADMIN_EMAIL__' add in settings.ADMINS +admin_email="${admin}@${domain}" + +# 'default_from_email' -> '__DEFAULT_FROM_EMAIL__' -> settings.DEFAULT_FROM_EMAIL +default_from_email="${app}@${domain}" + #================================================= # SET CONSTANTS #================================================= @@ -18,23 +34,14 @@ app=$YNH_APP_INSTANCE_NAME public_path=/var/www/$app final_path=/opt/yunohost/$app log_path=/var/log/$app -log_file="${log_path}/django-for-runners.log" +log_file="${log_path}/${app}.log" #================================================= # COMMON VARIABLES #================================================= -# Needed base dependencies: -pkg_dependencies="build-essential python3-dev python3-pip python3-venv git" - -# For pillow: -pkg_dependencies="${pkg_dependencies} libjpeg-dev" - -# Postgres and Python's "psycopg2": -pkg_dependencies="${pkg_dependencies} libpq-dev postgresql postgresql-contrib" - -# Needed for lxml: https://lxml.de/installation.html#requirements -pkg_dependencies="${pkg_dependencies} libxml2-dev libxslt-dev" +# dependencies used by the app +pkg_dependencies="build-essential python3-dev python3-pip python3-venv git libpq-dev postgresql postgresql-contrib" #================================================= # Redis HELPERS @@ -78,17 +85,3 @@ ynh_redis_remove_db() { redis-cli -n "$db" flushall } -#================================================= - -# Execute a command as another user -# usage: ynh_exec_as USER COMMAND [ARG ...] -ynh_exec_as() { - local USER=$1 - shift 1 - - if [[ $USER = $(whoami) ]]; then - eval "$@" - else - sudo -u "$USER" "$@" - fi -} diff --git a/scripts/change_url b/scripts/change_url index 8a037d3..cb77f04 100644 --- a/scripts/change_url +++ b/scripts/change_url @@ -6,6 +6,9 @@ # IMPORT GENERIC HELPERS #================================================= +YNH_APP_ARG_DOMAIN=$YNH_APP_NEW_DOMAIN +YNH_APP_ARG_PATH=$YNH_APP_NEW_PATH + source _common.sh source /usr/share/yunohost/helpers @@ -25,16 +28,26 @@ new_path=$YNH_APP_NEW_PATH ynh_script_progression --message="Loading installation settings..." admin=$(ynh_app_setting_get --app="$app" --key=admin) -is_public=$(ynh_app_setting_get --app="$app" --key=is_public) public_path=$(ynh_app_setting_get --app="$app" --key=public_path) final_path=$(ynh_app_setting_get --app="$app" --key=final_path) log_path=$(ynh_app_setting_get --app="$app" --key=log_path) port=$(ynh_app_setting_get --app="$app" --key=port) + db_pwd=$(ynh_app_setting_get --app="$app" --key=psqlpwd) -admin_mail=$(ynh_user_get_info "$admin" mail) +db_name=$(ynh_sanitize_dbid --db_name="$app") +db_user=$db_name + redis_db=$(ynh_app_setting_get --app="$app" --key=redis_db) +#------------------------------------------------- +# config_panel.toml settings: + +debug_enabled=$(ynh_app_setting_get --app="$app" --key=debug_enabled) +log_level=$(ynh_app_setting_get --app="$app" --key=log_level) +admin_email=$(ynh_app_setting_get --app="$app" --key=admin_email) +default_from_email=$(ynh_app_setting_get --app="$app" --key=default_from_email) + #================================================= # BACKUP BEFORE UPGRADE THEN ACTIVE TRAP #================================================= @@ -113,29 +126,12 @@ fi #================================================= # MODIFY SETTINGS #================================================= -ynh_script_progression --message="Modify project config file..." +ynh_script_progression --message="Modify $app config file..." -# save old settings file -settings="$final_path/settings.py" -ynh_backup_if_checksum_is_different --file="$settings" +domain=$YNH_APP_NEW_DOMAIN +path_url=$YNH_APP_NEW_PATH -cp "../conf/settings.py" "$settings" - -ynh_replace_string --match_string="__APP__" --replace_string="$app" --target_file="$settings" -ynh_replace_string --match_string="__DB_PWD__" --replace_string="$db_pwd" --target_file="$settings" -ynh_replace_string --match_string="__ADMIN__" --replace_string="$admin" --target_file="$settings" -ynh_replace_string --match_string="__ADMINMAIL__" --replace_string="$admin_mail" --target_file="$settings" -ynh_replace_string --match_string="__FINAL_HOME_PATH__" --replace_string="$final_path" --target_file="$settings" -ynh_replace_string --match_string="__FINAL_WWW_PATH__" --replace_string="$public_path" --target_file="$settings" -ynh_replace_string --match_string="__LOG_FILE__" --replace_string="$log_file" --target_file="$settings" -ynh_replace_string --match_string="__REDIS_DB__" --replace_string="$redis_db" --target_file="$settings" - -# Difference to install/upgrade scripts: Use $new_domain and $new_path here: -ynh_replace_string --match_string="__DOMAIN__" --replace_string="$new_domain" --target_file="$settings" -ynh_replace_string --match_string="__PATH_URL__" --replace_string="$new_path" --target_file="$settings" - -# Recalculate and store the config file checksum into the app settings -ynh_store_file_checksum --file="$settings" +ynh_add_config --template="settings.py" --destination="$final_path/settings.py" #================================================= # GENERIC FINALISATION diff --git a/scripts/install b/scripts/install index 6f5c2d4..b56bb3b 100755 --- a/scripts/install +++ b/scripts/install @@ -40,7 +40,6 @@ touch "${log_file}" ynh_script_progression --message="Storing installation settings..." ynh_app_setting_set --app="$app" --key=admin --value="$admin" -ynh_app_setting_set --app="$app" --key=is_public --value="$is_public" ynh_app_setting_set --app="$app" --key=public_path --value="$public_path" ynh_app_setting_set --app="$app" --key=final_path --value="$final_path" ynh_app_setting_set --app="$app" --key=log_path --value="$log_file" @@ -51,19 +50,29 @@ ynh_app_setting_set --app="$app" --key=path --value="$path_url" # Find a free port port=$(ynh_find_port --port=8000) # Set port as application setting -# https://github.com/YunoHost/yunohost/blob/dev/data/helpers.d/setting +# https://yunohost.org/en/contribute/packaging_apps/helpers +# https://github.com/YunoHost/yunohost/blob/dev/helpers/setting ynh_app_setting_set --app="$app" --key=port --value="$port" db_pwd=$(ynh_app_setting_get --app="$app" --key=psqlpwd) -admin_mail=$(ynh_user_get_info --username="$admin" --key=mail) + redis_db=$(ynh_redis_get_free_db) +ynh_app_setting_set --app="$app" --key=redis_db --value="$redis_db" + +#------------------------------------------------- +# config_panel.toml settings: + +ynh_app_setting_set --app="$app" --key=debug_enabled --value="$debug_enabled" +ynh_app_setting_set --app="$app" --key=log_level --value="$log_level" +ynh_app_setting_set --app="$app" --key=admin_email --value="$admin_email" +ynh_app_setting_set --app="$app" --key=default_from_email --value="$default_from_email" #================================================= # STANDARD MODIFICATIONS #================================================= # INSTALL DEPENDENCIES #================================================= -ynh_script_progression --message="Installing dependencies..." --weight=20 +ynh_script_progression --message="Installing $app dependencies..." --weight=20 ynh_exec_warn_less ynh_install_app_dependencies "$pkg_dependencies" @@ -87,21 +96,22 @@ ynh_psql_setup_db --db_user="$db_user" --db_name="$db_name" ynh_script_progression --message="Configuring nginx web server..." # Create a dedicated nginx config -# https://github.com/YunoHost/yunohost/blob/dev/data/helpers.d/nginx +# https://yunohost.org/en/contribute/packaging_apps/helpers +# https://github.com/YunoHost/yunohost/blob/dev/helpers/nginx ynh_add_nginx_config "public_path" "port" #================================================= # CREATE DEDICATED USER #================================================= -ynh_script_progression --message="Configuring system user..." +ynh_script_progression --message="Configuring system user '$app'..." # A home directory for venv and settings etc. ynh_system_user_create --username="$app" --home_dir="$final_path" --use_shell #================================================= -# PIP INSTALLATION +# PYTHON VIRTUALENV #================================================= -ynh_script_progression --message="Install project via pip..." --weight=50 +ynh_script_progression --message="Create Python virtualenv..." --weight=5 # Always recreate everything fresh with current python version ynh_secure_remove "${final_path}/venv" @@ -112,64 +122,35 @@ python3 -m venv --without-pip "${final_path}/venv" cp ../conf/requirements.txt "$final_path/requirements.txt" chown -R "$app:" "$final_path" +#================================================= +# PIP INSTALLATION +#================================================= +ynh_script_progression --message="Install project via pip..." --weight=45 + #run source in a 'sub shell' ( set +o nounset source "${final_path}/venv/bin/activate" set -o nounset - ynh_exec_as $app $final_path/venv/bin/python3 -m ensurepip ynh_exec_as $app $final_path/venv/bin/pip3 install --upgrade wheel pip setuptools - - # for installing packages like "matplotlib" - # See: https://github.com/YunoHost-Apps/django-for-runners_ynh/issues/15 - ynh_exec_as $app $final_path/venv/bin/pip install --upgrade --prefer-binary Cython - ynh_exec_as $app $final_path/venv/bin/pip3 install --no-deps -r "$final_path/requirements.txt" ) #================================================= # copy config files # ================================================ -ynh_script_progression --message="Create project configuration files..." +ynh_script_progression --message="Create $app configuration files..." -gunicorn_conf="$final_path/gunicorn.conf.py" -cp "../conf/gunicorn.conf.py" "$gunicorn_conf" -ynh_replace_string --match_string="__FINAL_HOME_PATH__" --replace_string="$final_path" --target_file="$gunicorn_conf" -ynh_replace_string --match_string="__LOG_FILE__" --replace_string="$log_file" --target_file="$gunicorn_conf" -ynh_replace_string --match_string="__PORT__" --replace_string="$port" --target_file="$gunicorn_conf" -ynh_store_file_checksum --file="$gunicorn_conf" +ynh_add_config --template="gunicorn.conf.py" --destination="$final_path/gunicorn.conf.py" -cp ../conf/manage.py "$final_path/manage.py" -ynh_replace_string --match_string="__FINAL_HOME_PATH__" --replace_string="$final_path" --target_file="$final_path/manage.py" -ynh_store_file_checksum --file="$final_path/manage.py" +ynh_add_config --template="manage.py" --destination="$final_path/manage.py" chmod +x "$final_path/manage.py" -settings="$final_path/settings.py" -cp "../conf/settings.py" "$settings" - -ynh_replace_string --match_string="__APP__" --replace_string="$app" --target_file="$settings" -ynh_replace_string --match_string="__DB_NAME__" --replace_string="$db_name" --target_file="$settings" -ynh_replace_string --match_string="__DB_USER__" --replace_string="$db_user" --target_file="$settings" -ynh_replace_string --match_string="__DB_PWD__" --replace_string="$db_pwd" --target_file="$settings" -ynh_replace_string --match_string="__ADMIN__" --replace_string="$admin" --target_file="$settings" -ynh_replace_string --match_string="__ADMINMAIL__" --replace_string="$admin_mail" --target_file="$settings" -ynh_replace_string --match_string="__FINAL_HOME_PATH__" --replace_string="$final_path" --target_file="$settings" -ynh_replace_string --match_string="__FINAL_WWW_PATH__" --replace_string="$public_path" --target_file="$settings" -ynh_replace_string --match_string="__LOG_FILE__" --replace_string="$log_file" --target_file="$settings" -ynh_replace_string --match_string="__REDIS_DB__" --replace_string="$redis_db" --target_file="$settings" - -ynh_replace_string --match_string="__DOMAIN__" --replace_string="$domain" --target_file="$settings" -ynh_replace_string --match_string="__PATH_URL__" --replace_string="$path_url" --target_file="$settings" - -# Calculate and store the config file checksum into the app settings -ynh_store_file_checksum --file="$settings" - -ynh_app_setting_set --app="$app" --key=redis_db --value="$redis_db" - -cp ../conf/setup_user.py "$final_path/setup_user.py" -cp ../conf/urls.py "$final_path/urls.py" -cp ../conf/wsgi.py "$final_path/wsgi.py" +ynh_add_config --template="settings.py" --destination="$final_path/settings.py" +ynh_add_config --template="setup_user.py" --destination="$final_path/setup_user.py" +ynh_add_config --template="urls.py" --destination="$final_path/urls.py" +ynh_add_config --template="wsgi.py" --destination="$final_path/wsgi.py" touch "$final_path/local_settings.py" @@ -187,7 +168,7 @@ cd "$final_path" || exit ./manage.py collectstatic --no-input # Create/update Django superuser (set unusable password, because auth done via SSOwat): -./manage.py create_superuser --username="$admin" --email="$admin_mail" +./manage.py create_superuser --username="$admin" --email="$(ynh_user_get_info "$admin" mail)" # Check the configuration # This may fail in some cases with errors, etc., but the app works and the user can fix issues later. @@ -207,7 +188,7 @@ ynh_use_logrotate "$log_file" #================================================= ynh_script_progression --message="Integrating service in YunoHost..." -yunohost service add $app --description="Web based management to catalog things" --log="${log_file}" +yunohost service add $app --log="${log_file}" #================================================= # GENERIC FINALIZATION @@ -234,9 +215,22 @@ ynh_script_progression --message="Configuring systemd service '$app'..." --weigh ynh_add_systemd_config --service="$app" --template="systemd.service" #================================================= -# Start for_runners via systemd +# SETUP SSOWAT #================================================= -ynh_script_progression --message="Starting project services..." --weight=5 +ynh_script_progression --message="Configuring SSOwat..." + +# Make app public if necessary or protect it +if [ $is_public -eq 1 ] +then + # Everyone can access the app. + # The "main" permission is automatically created before the install script. + ynh_permission_update --permission "main" --add "visitors" +fi + +#================================================= +# Start the app server via systemd +#================================================= +ynh_script_progression --message="Starting systemd service '$app'..." --weight=5 ynh_systemd_action --service_name="$app" --action="start" diff --git a/scripts/restore b/scripts/restore index 5460ea0..a72837e 100755 --- a/scripts/restore +++ b/scripts/restore @@ -20,8 +20,8 @@ ynh_abort_if_errors #================================================= ynh_script_progression --message="Loading settings..." -public_path=$(ynh_app_setting_get --app="$app" --key=public_path) final_path=$(ynh_app_setting_get --app="$app" --key=final_path) +public_path=$(ynh_app_setting_get --app="$app" --key=public_path) db_name=$(ynh_app_setting_get --app="$app" --key=db_name) db_user=$db_name db_pwd=$(ynh_app_setting_get --app="$app" --key=psqlpwd) @@ -34,8 +34,6 @@ path_url=$(ynh_app_setting_get --app="$app" --key=path) #================================================= ynh_script_progression --message="Validating restoration parameters..." -ynh_webpath_available --domain=$domain --path_url=$path_url \ - || ynh_die --message="Path not available: ${domain}${path_url}" test ! -d $final_path \ || ynh_die --message="There is already a directory: $final_path " @@ -50,12 +48,10 @@ ynh_restore_file --origin_path="/etc/nginx/conf.d/$domain.d/$app.conf" #================================================= # RESTORE THE APP MAIN DIR #================================================= -ynh_script_progression --message="Restoring the app main directory..." +ynh_script_progression --message="Restoring $app main directory..." -ynh_restore_file --origin_path="$public_path" ynh_restore_file --origin_path="$final_path" - -touch "$final_path/local_settings.py" +ynh_restore_file --origin_path="$public_path" #================================================= # RECREATE THE DEDICATED USER @@ -83,10 +79,10 @@ ynh_script_progression --message="Reinstalling dependencies..." --weight=20 ynh_exec_warn_less ynh_install_app_dependencies "$pkg_dependencies" #================================================= -# REINSTALL PYTHON VIRTUALENV +# PYTHON VIRTUALENV # Maybe the backup contains a other Python version #================================================= -ynh_script_progression --message="Upgrade Python virtualenv..." --weight=50 +ynh_script_progression --message="Recreate Python virtualenv..." --weight=5 # Always recreate everything fresh with current python version ynh_secure_remove "${final_path}/venv" @@ -95,19 +91,17 @@ ynh_secure_remove "${final_path}/venv" python3 -m venv --without-pip "${final_path}/venv" chown -R "$app:" "$final_path" +#================================================= +# PIP INSTALLATION +#================================================= +ynh_script_progression --message="Install project via pip..." --weight=45 #run source in a 'sub shell' ( set +o nounset source "${final_path}/venv/bin/activate" set -o nounset - ynh_exec_as $app $final_path/venv/bin/python3 -m ensurepip ynh_exec_as $app $final_path/venv/bin/pip3 install --upgrade wheel pip setuptools - - # for installing packages like "matplotlib" - # See: https://github.com/YunoHost-Apps/django-for-runners_ynh/issues/15 - ynh_exec_as $app $final_path/venv/bin/pip install --upgrade --prefer-binary Cython - ynh_exec_as $app $final_path/venv/bin/pip3 install --no-deps -r "$final_path/requirements.txt" ) @@ -133,7 +127,7 @@ systemctl enable $app.service --quiet #================================================= ynh_script_progression --message="Integrating service in YunoHost..." -yunohost service add $app --description="Web based management to catalog things" --log="${log_file}" +yunohost service add $app --log="${log_file}" #================================================= # RESTORE THE LOGROTATE CONFIGURATION diff --git a/scripts/upgrade b/scripts/upgrade index 98f46e4..9bdc8f1 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -13,7 +13,6 @@ source /usr/share/yunohost/helpers ynh_script_progression --message="Loading installation settings..." admin=$(ynh_app_setting_get --app="$app" --key=admin) -is_public=$(ynh_app_setting_get --app="$app" --key=is_public) public_path=$(ynh_app_setting_get --app="$app" --key=public_path) final_path=$(ynh_app_setting_get --app="$app" --key=final_path) log_path=$(ynh_app_setting_get --app="$app" --key=log_path) @@ -27,13 +26,39 @@ db_pwd=$(ynh_app_setting_get --app="$app" --key=psqlpwd) db_name=$(ynh_sanitize_dbid --db_name="$app") db_user=$db_name -admin_mail=$(ynh_user_get_info "$admin" mail) redis_db=$(ynh_app_setting_get --app="$app" --key=redis_db) +#------------------------------------------------- +# config_panel.toml settings: + +debug_enabled=$(ynh_app_setting_get --app="$app" --key=debug_enabled) +if [ -z "$debug_enabled" ]; then + debug_enabled="0" + ynh_app_setting_set --app="$app" --key=debug_enabled --value="$debug_enabled" +fi + +log_level=$(ynh_app_setting_get --app="$app" --key=log_level) +if [ -z "$log_level" ]; then + log_level="WARNING" + ynh_app_setting_set --app="$app" --key=log_level --value="$log_level" +fi + +admin_email=$(ynh_app_setting_get --app="$app" --key=admin_email) +if [ -z "$admin_email" ]; then + admin_email="${admin}@${domain}" + ynh_app_setting_set --app="$app" --key=admin_email --value="$admin_email" +fi + +default_from_email=$(ynh_app_setting_get --app="$app" --key=default_from_email) +if [ -z "$default_from_email" ]; then + default_from_email="${app}@${domain}" + ynh_app_setting_set --app="$app" --key=default_from_email --value="$default_from_email" +fi + #================================================= # BACKUP BEFORE UPGRADE THEN ACTIVE TRAP #================================================= -ynh_script_progression --message="Backing up the app before upgrading (may take a while)..." --weight=40 +ynh_script_progression --message="Backing up $app before upgrading (may take a while)..." --weight=40 # Backup the current version of the app ynh_backup_before_upgrade @@ -59,7 +84,8 @@ ynh_systemd_action --service_name="$app" --action="stop" ynh_script_progression --message="Upgrading nginx web server configuration..." # Create a dedicated nginx config -# https://github.com/YunoHost/yunohost/blob/dev/data/helpers.d/nginx +# https://yunohost.org/en/contribute/packaging_apps/helpers +# https://github.com/YunoHost/yunohost/blob/dev/helpers/nginx ynh_add_nginx_config "public_path" "port" #================================================= @@ -82,14 +108,14 @@ ynh_system_user_create --username="$app" --home_dir="$final_path" --use_shell #================================================= # SETUP SYSTEMD #================================================= -ynh_script_progression --message="Configuring a systemd service..." +ynh_script_progression --message="Configuring systemd service '$app'..." --weight=5 -ynh_add_systemd_config --service="$app" --template="django-for-runners.service" +ynh_add_systemd_config --service="$app" --template="systemd.service" #================================================= -# UPGRADE VENV +# PYTHON VIRTUALENV #================================================= -ynh_script_progression --message="Upgrade project via pip..." --weight=50 +ynh_script_progression --message="Recreate Python virtualenv..." --weight=5 # Always recreate everything fresh with current python version ynh_secure_remove "${final_path}/venv" @@ -100,19 +126,17 @@ python3 -m venv --without-pip "${final_path}/venv" cp ../conf/requirements.txt "$final_path/requirements.txt" chown -R "$app:" "$final_path" +#================================================= +# PIP INSTALLATION +#================================================= +ynh_script_progression --message="Install project via pip..." --weight=45 #run source in a 'sub shell' ( set +o nounset source "${final_path}/venv/bin/activate" set -o nounset - ynh_exec_as $app $final_path/venv/bin/python3 -m ensurepip ynh_exec_as $app $final_path/venv/bin/pip3 install --upgrade wheel pip setuptools - - # for installing packages like "matplotlib" - # See: https://github.com/YunoHost-Apps/django-for-runners_ynh/issues/15 - ynh_exec_as $app $final_path/venv/bin/pip install --upgrade --prefer-binary Cython - ynh_exec_as $app $final_path/venv/bin/pip3 install --no-deps -r "$final_path/requirements.txt" ) @@ -121,52 +145,15 @@ chown -R "$app:" "$final_path" # ================================================ ynh_script_progression --message="Create project configuration files..." -gunicorn_conf="$final_path/gunicorn.conf.py" -ynh_backup_if_checksum_is_different --file="$gunicorn_conf" -cp "../conf/gunicorn.conf.py" "$gunicorn_conf" -ynh_replace_string --match_string="__FINAL_HOME_PATH__" --replace_string="$final_path" --target_file="$gunicorn_conf" -ynh_replace_string --match_string="__LOG_FILE__" --replace_string="$log_file" --target_file="$gunicorn_conf" -ynh_replace_string --match_string="__PORT__" --replace_string="$port" --target_file="$gunicorn_conf" -ynh_store_file_checksum --file="$gunicorn_conf" +ynh_add_config --template="gunicorn.conf.py" --destination="$final_path/gunicorn.conf.py" -cp ../conf/manage.py "$final_path/manage.py" -ynh_replace_string --match_string="__FINAL_HOME_PATH__" --replace_string="$final_path" --target_file="$final_path/manage.py" -ynh_store_file_checksum --file="$final_path/manage.py" +ynh_add_config --template="manage.py" --destination="$final_path/manage.py" chmod +x "$final_path/manage.py" -# save old settings file -settings="$final_path/settings.py" -ynh_backup_if_checksum_is_different --file="$settings" - -cp "../conf/settings.py" "$settings" - -ynh_replace_string --match_string="__APP__" --replace_string="$app" --target_file="$settings" -ynh_replace_string --match_string="__DB_NAME__" --replace_string="$db_name" --target_file="$settings" -ynh_replace_string --match_string="__DB_USER__" --replace_string="$db_user" --target_file="$settings" -ynh_replace_string --match_string="__DB_PWD__" --replace_string="$db_pwd" --target_file="$settings" -ynh_replace_string --match_string="__ADMIN__" --replace_string="$admin" --target_file="$settings" -ynh_replace_string --match_string="__ADMINMAIL__" --replace_string="$admin_mail" --target_file="$settings" -ynh_replace_string --match_string="__FINAL_HOME_PATH__" --replace_string="$final_path" --target_file="$settings" -ynh_replace_string --match_string="__FINAL_WWW_PATH__" --replace_string="$public_path" --target_file="$settings" -ynh_replace_string --match_string="__LOG_FILE__" --replace_string="$log_file" --target_file="$settings" -ynh_replace_string --match_string="__REDIS_DB__" --replace_string="$redis_db" --target_file="$settings" - -ynh_replace_string --match_string="__DOMAIN__" --replace_string="$domain" --target_file="$settings" -ynh_replace_string --match_string="__PATH_URL__" --replace_string="$path_url" --target_file="$settings" - -# Recalculate and store the config file checksum into the app settings -ynh_store_file_checksum --file="$settings" - -ynh_backup_if_checksum_is_different --file="$final_path/setup_user.py" -cp ../conf/setup_user.py "$final_path/setup_user.py" - -ynh_backup_if_checksum_is_different --file="$final_path/urls.py" -cp ../conf/urls.py "$final_path/urls.py" - -ynh_backup_if_checksum_is_different --file="$final_path/wsgi.py" -cp ../conf/wsgi.py "$final_path/wsgi.py" - -touch "$final_path/local_settings.py" +ynh_add_config --template="settings.py" --destination="$final_path/settings.py" +ynh_add_config --template="setup_user.py" --destination="$final_path/setup_user.py" +ynh_add_config --template="urls.py" --destination="$final_path/urls.py" +ynh_add_config --template="wsgi.py" --destination="$final_path/wsgi.py" #================================================= # MIGRATE PYINVENTORY @@ -182,7 +169,7 @@ cd "$final_path" || exit ./manage.py collectstatic --no-input # Create/update Django superuser (set unusable password, because auth done via SSOwat): -./manage.py create_superuser --username="$admin" --email="$admin_mail" +./manage.py create_superuser --username="$admin" --email="$(ynh_user_get_info "$admin" mail)" # Check the configuration # This may fail in some cases with errors, etc., but the app works and the user can fix issues later. @@ -202,7 +189,7 @@ ynh_use_logrotate --non-append #================================================= ynh_script_progression --message="Integrating service in YunoHost..." -yunohost service add $app --description="Web based management to catalog things" --log="${log_file}" +yunohost service add $app --log="${log_file}" #================================================= # GENERIC FINALIZATION diff --git a/tests/test_django_project.py b/tests/test_django_project.py index c48af1f..dad61a6 100644 --- a/tests/test_django_project.py +++ b/tests/test_django_project.py @@ -1,3 +1,4 @@ +import for_runners from axes.models import AccessLog from bx_django_utils.test_utils.html_assertion import HtmlAssertionMixin from django.conf import settings @@ -6,11 +7,10 @@ from django.test import override_settings from django.test.testcases import TestCase from django.urls import NoReverseMatch from django.urls.base import reverse + from django_yunohost_integration.test_utils import generate_basic_auth from django_yunohost_integration.views import request_media_debug_view -import for_runners - @override_settings(DEBUG=False) class DjangoYnhTestCase(HtmlAssertionMixin, TestCase): @@ -23,12 +23,19 @@ class DjangoYnhTestCase(HtmlAssertionMixin, TestCase): def test_settings(self): assert settings.PATH_URL == 'app_path' - assert str(settings.FINAL_HOME_PATH).endswith('/local_test/opt_yunohost') - assert str(settings.FINAL_WWW_PATH).endswith('/local_test/var_www') + assert str(settings.FINALPATH).endswith('/local_test/opt_yunohost') + assert str(settings.PUBLIC_PATH).endswith('/local_test/var_www') assert str(settings.LOG_FILE).endswith('/local_test/var_log_django-for-runners.log') assert settings.ROOT_URLCONF == 'urls' + def test_config_panel_settings(self): + # config_panel.toml settings, set via tests.conftest.pytest_configure(): + assert settings.DEBUG_ENABLED == '0' and settings.DEBUG is False + assert settings.LOG_LEVEL == 'INFO' + assert settings.ADMIN_EMAIL == 'foo-bar@test.tld' + assert settings.DEFAULT_FROM_EMAIL == 'django_app@test.tld' + def test_urls(self): assert reverse('admin:index') == '/app_path/' @@ -44,6 +51,10 @@ class DjangoYnhTestCase(HtmlAssertionMixin, TestCase): # ) == ('/app_path/media/token/foo/bar/') def test_auth(self): + assert settings.PATH_URL == 'app_path' + url = reverse('admin:index') + assert url == '/app_path/' + # SecurityMiddleware should redirects all non-HTTPS requests to HTTPS: assert settings.SECURE_SSL_REDIRECT is True response = self.client.get('/app_path/', secure=False) @@ -59,7 +70,6 @@ class DjangoYnhTestCase(HtmlAssertionMixin, TestCase): response, expected_url='/app_path/login/?next=/app_path/', fetch_redirect_response=False ) - @override_settings(SECURE_SSL_REDIRECT=False) def test_create_unknown_user(self): assert User.objects.count() == 0 @@ -70,16 +80,16 @@ class DjangoYnhTestCase(HtmlAssertionMixin, TestCase): HTTP_REMOTE_USER='test', HTTP_AUTH_USER='test', HTTP_AUTHORIZATION='basic dGVzdDp0ZXN0MTIz', + secure=True, ) assert User.objects.count() == 1 user = User.objects.first() assert user.username == 'test' assert user.is_active is True - assert user.is_staff is True # Set by: conf.django_ynh_demo_urls.setup_user_handler + assert user.is_staff is True # Set by: conf.setup_user.setup_project_user assert user.is_superuser is False - assert response.status_code == 200 self.assert_html_parts( response, parts=( @@ -88,7 +98,6 @@ class DjangoYnhTestCase(HtmlAssertionMixin, TestCase): ), ) - @override_settings(SECURE_SSL_REDIRECT=False) def test_wrong_auth_user(self): assert User.objects.count() == 0 assert AccessLog.objects.count() == 0 @@ -100,20 +109,20 @@ class DjangoYnhTestCase(HtmlAssertionMixin, TestCase): HTTP_REMOTE_USER='test', HTTP_AUTH_USER='foobar', # <<< wrong user name HTTP_AUTHORIZATION='basic dGVzdDp0ZXN0MTIz', + secure=True, ) assert User.objects.count() == 1 user = User.objects.first() assert user.username == 'test' assert user.is_active is True - assert user.is_staff is True # Set by: conf.django_ynh_demo_urls.setup_user_handler + assert user.is_staff is True # Set by: conf.setup_user.setup_project_user assert user.is_superuser is False assert AccessLog.objects.count() == 1 assert response.status_code == 403 # Forbidden - @override_settings(SECURE_SSL_REDIRECT=False) def test_wrong_cookie(self): assert User.objects.count() == 0 assert AccessLog.objects.count() == 0 @@ -125,20 +134,20 @@ class DjangoYnhTestCase(HtmlAssertionMixin, TestCase): HTTP_REMOTE_USER='test', HTTP_AUTH_USER='test', HTTP_AUTHORIZATION='basic dGVzdDp0ZXN0MTIz', + secure=True, ) assert User.objects.count() == 1 user = User.objects.first() assert user.username == 'test' assert user.is_active is True - assert user.is_staff is True # Set by: conf.django_ynh_demo_urls.setup_user_handler + assert user.is_staff is True # Set by: conf.setup_user.setup_project_user assert user.is_superuser is False assert AccessLog.objects.count() == 1 assert response.status_code == 403 # Forbidden - @override_settings(SECURE_SSL_REDIRECT=False) def test_wrong_authorization_user(self): assert User.objects.count() == 0 @@ -149,15 +158,17 @@ class DjangoYnhTestCase(HtmlAssertionMixin, TestCase): HTTP_REMOTE_USER='test', HTTP_AUTH_USER='test', HTTP_AUTHORIZATION=generate_basic_auth( - username='foobar', password='test123' - ), # <<< wrong user name + username='foobar', # <<< wrong user name + password='test123', + ), + secure=True, ) assert User.objects.count() == 1 user = User.objects.first() assert user.username == 'test' assert user.is_active is True - assert user.is_staff is True # Set by: conf.django_ynh_demo_urls.setup_user_handler + assert user.is_staff is True # Set by: conf.setup_user.setup_project_user assert user.is_superuser is False assert AccessLog.objects.count() == 1 diff --git a/tests/test_project_setup.py b/tests/test_project_setup.py index afb09c3..19b98ff 100644 --- a/tests/test_project_setup.py +++ b/tests/test_project_setup.py @@ -4,11 +4,11 @@ import shutil import subprocess from pathlib import Path +import for_runners +import tomli from bx_django_utils.filename import clean_filename from bx_py_utils.path import assert_is_dir, assert_is_file -import for_runners - PACKAGE_ROOT = Path(__file__).parent.parent @@ -22,13 +22,20 @@ def assert_file_contains_string(file_path, string): def test_version(): - version = for_runners.__version__ + pyproject_toml_path = Path(PACKAGE_ROOT, 'pyproject.toml') + pyproject_toml = tomli.loads(pyproject_toml_path.read_text(encoding='UTF-8')) + pyproject_version = pyproject_toml['tool']['poetry']['version'] + assert '~ynh' in pyproject_version + assert pyproject_version[0].isdigit() + assert pyproject_version.startswith(for_runners.__version__) assert_file_contains_string( - file_path=Path(PACKAGE_ROOT, 'pyproject.toml'), string=f'version = "{version}~ynh' + file_path=Path(PACKAGE_ROOT, 'manifest.json'), + string=f'"version": "{pyproject_version}"', ) assert_file_contains_string( - file_path=Path(PACKAGE_ROOT, 'manifest.json'), string=f'"version": "{version}~ynh' + file_path=pyproject_toml_path, + string=f'django-for-runners = ">={for_runners.__version__}"', ) @@ -52,7 +59,7 @@ def test_poetry_check(): def test_requirements_txt(): - requirements_txt = Path('conf', 'requirements.txt') + requirements_txt = PACKAGE_ROOT / 'conf' / 'requirements.txt' assert_is_file(requirements_txt) output = poetry_check_output('export', '-f', 'requirements.txt') @@ -81,6 +88,8 @@ def test_screenshot_filenames(): renamed = [] for file_path in screenshot_path.iterdir(): file_name = file_path.name + if file_name.startswith('.'): + continue cleaned_name = clean_filename(file_name) if cleaned_name != file_name: new_path = file_path.with_name(cleaned_name)