From 8a9abca957b1037b9c62fd8f16aba72e48043ca4 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Wed, 16 Dec 2020 17:34:33 +0100 Subject: [PATCH 01/12] Bugfix change url script --- scripts/change_url | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/scripts/change_url b/scripts/change_url index 90475fb..54a5a85 100644 --- a/scripts/change_url +++ b/scripts/change_url @@ -24,10 +24,18 @@ new_path=$YNH_APP_NEW_PATH #================================================= ynh_script_progression --message="Loading installation settings..." +domain=$(ynh_app_setting_get --app="$app" --key=domain) +path_url=$(ynh_app_setting_get --app="$app" --key=path) +admin=$(ynh_app_setting_get --app="$app" --key=admin) +is_public=$(ynh_app_setting_get --app="$app" --key=is_public) public_path=$(ynh_app_setting_get --app="$app" --key=public_path) final_path=$(ynh_app_setting_get --app="$app" --key=final_path) -is_public=$(ynh_app_setting_get --app="$app" --key=is_public) +log_path=$(ynh_app_setting_get --app="$app" --key=log_path) + port=$(ynh_app_setting_get --app="$app" --key=port) +db_pwd=$(ynh_app_setting_get --app="$app" --key=psqlpwd) +admin_mail=$(ynh_user_get_info "$admin" mail) +redis_db=$(ynh_app_setting_get --app="$app" --key=redis_db) #================================================= # BACKUP BEFORE UPGRADE THEN ACTIVE TRAP @@ -109,26 +117,24 @@ fi #================================================= ynh_script_progression --message="Modify Django-For-Runners's config file..." +# save old settings file settings="$final_path/ynh_for_runners_settings.py" ynh_backup_if_checksum_is_different --file="$settings" -# Change the path in the nginx config file -if [ $change_path -eq 1 ] -then - ynh_replace_string --match_string="URL_PREFIX = \"${old_path%/}\"" --replace_string="URL_PREFIX = \"${new_path%/}\"" --target_file="$settings" -fi +cp "../conf/ynh_for_runners_settings.py" "$settings" -# Change the domain for nginx -if [ $change_domain -eq 1 ] -then - # replace SERVER_EMAIL - ynh_replace_string --match_string="noreply@$old_domain" --replace_string="noreply@$new_domain" --target_file="$settings" - # replace SITE_DOMAIN - ynh_replace_string --match_string="SITE_DOMAIN = \"$old_domain\"" --replace_string="SITE_DOMAIN = \"$new_domain\"" --target_file="$settings" - # replace ALLOWED_HOSTS - ynh_replace_string --match_string="ALLOWED_HOSTS = \[\"$old_domain\"\]" --replace_string="ALLOWED_HOSTS = \[\"$new_domain\"\]" --target_file="$settings" -fi +ynh_replace_string --match_string="__APP__" --replace_string="$app" --target_file="$settings" +ynh_replace_string --match_string="__DB_PWD__" --replace_string="$db_pwd" --target_file="$settings" +ynh_replace_string --match_string="__ADMIN__" --replace_string="$admin" --target_file="$settings" +ynh_replace_string --match_string="__ADMINMAIL__" --replace_string="$admin_mail" --target_file="$settings" +ynh_replace_string --match_string="__DOMAIN__" --replace_string="$domain" --target_file="$settings" +ynh_replace_string --match_string="__FINAL_HOME_PATH__" --replace_string="$final_path" --target_file="$settings" +ynh_replace_string --match_string="__FINAL_WWW_PATH__" --replace_string="$public_path" --target_file="$settings" +ynh_replace_string --match_string="__PATH_URL__" --replace_string="$path_url" --target_file="$settings" +ynh_replace_string --match_string="__LOG_FILE__" --replace_string="$log_file" --target_file="$settings" +ynh_replace_string --match_string="__REDIS_DB__" --replace_string="$redis_db" --target_file="$settings" +# Recalculate and store the config file checksum into the app settings ynh_store_file_checksum --file="$settings" #================================================= From d4269c42afae928845bfaf6394875e3810f2fb5b Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Tue, 22 Dec 2020 18:47:11 +0100 Subject: [PATCH 02/12] add editorconfig --- .editorconfig | 20 ++++++++++++++++++++ .gitignore | 1 + 2 files changed, 21 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..7022146 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,20 @@ +# see http://editorconfig.org +root = true + +[*] +indent_style = space +indent_size = 4 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.py] +line_length = 119 + +[{Makefile,**.mk}] +indent_style = tab +insert_final_newline = false + +[*.yml] +indent_style = tab diff --git a/.gitignore b/.gitignore index 145be6a..a7c2f58 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .* !.github +!.editorconfig !.gitignore __pycache__ secret.txt From 32a6ab35fa6be8706d95564a7c692dc9dfe10e3c Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Tue, 22 Dec 2020 18:51:40 +0100 Subject: [PATCH 03/12] setup existing user --- conf/create_superuser.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/conf/create_superuser.py b/conf/create_superuser.py index b35ab77..b6312d8 100644 --- a/conf/create_superuser.py +++ b/conf/create_superuser.py @@ -24,12 +24,15 @@ def main(): from django.contrib.auth import get_user_model User = get_user_model() - super_user = User.objects.filter(username=username).first() - if super_user: - print('Update existing super user and set his password.', file=sys.stderr) - super_user.set_password(password) - super_user.email=email - super_user.save() + user = User.objects.filter(username=username).first() + if user: + print('Update existing user and set his password.', file=sys.stderr) + user.is_active = True + user.is_staff = True + user.is_superuser = True + user.set_password(password) + user.email = email + user.save() else: print('Create new super user', file=sys.stderr) User.objects.create_superuser( From 17b45f8310ead224fecfacddd5bac5cf6de97fdb Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Tue, 22 Dec 2020 18:52:00 +0100 Subject: [PATCH 04/12] code cleanup --- conf/nginx.conf | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/conf/nginx.conf b/conf/nginx.conf index af9d9e4..65adebf 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -1,8 +1,8 @@ location __PATH__/static/ { - # Django static files - alias __PUBLIC_PATH__/static/; - expires 30d; + # Django static files + alias __PUBLIC_PATH__/static/; + expires 30d; } # TODO: django-sendfile2: @@ -12,26 +12,23 @@ location __PATH__/static/ { # expires 30d; #} -location __PATH__/ { - # https://github.com/benoitc/gunicorn/blob/master/examples/nginx.conf +location / { + # https://github.com/benoitc/gunicorn/blob/master/examples/nginx.conf - # this is needed if you have file import via upload enabled - client_max_body_size 100M; + # this is needed if you have file import via upload enabled + client_max_body_size 100M; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Protocol $scheme; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Scheme $scheme; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Protocol $scheme; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Scheme $scheme; - proxy_read_timeout 30; - proxy_send_timeout 30; - proxy_connect_timeout 30; - proxy_redirect off; + proxy_read_timeout 30; + proxy_send_timeout 30; + proxy_connect_timeout 30; + proxy_redirect off; - proxy_pass http://127.0.0.1:__PORT__/; - - # Include SSOWAT user panel. - #include conf.d/yunohost_panel.conf.inc; + proxy_pass http://127.0.0.1:__PORT__/; } From a52bb7b8c4a575faf2b3eda12dcee6e5c44a9b35 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Tue, 22 Dec 2020 18:52:25 +0100 Subject: [PATCH 05/12] create local_settings.py in local test --- conf/ynh_for_runners_settings.py | 2 +- local_test.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/conf/ynh_for_runners_settings.py b/conf/ynh_for_runners_settings.py index e6330db..9eacb76 100644 --- a/conf/ynh_for_runners_settings.py +++ b/conf/ynh_for_runners_settings.py @@ -185,6 +185,6 @@ LOGGING = { # ----------------------------------------------------------------------------- try: - from .local_settings import * # noqa + from local_settings import * # noqa except ImportError: pass diff --git a/local_test.py b/local_test.py index 9a3afd2..063b42e 100755 --- a/local_test.py +++ b/local_test.py @@ -130,6 +130,10 @@ def main(): # conf/ynh_urls.py -> local_test/ynh_urls.py copy_patch(src_file=URLS_FILE, replaces=REPLACES) + with Path(FINAL_HOME_PATH / 'local_settings.py').open('w') as f: + f.write('# Only for local test run\n') + f.write('SERVE_FILES=True # used in src/for_runners_project/urls.py\n') + # call "local_test/manage.py" via subprocess: call_manage_py('check --deploy') call_manage_py('migrate --no-input') From cf58061b93e9e15c518150ab41edbf8f534171ca Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Tue, 22 Dec 2020 18:52:38 +0100 Subject: [PATCH 06/12] simplify urls --- conf/ynh_urls.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/conf/ynh_urls.py b/conf/ynh_urls.py index 8140ee5..dc21199 100644 --- a/conf/ynh_urls.py +++ b/conf/ynh_urls.py @@ -1,21 +1,23 @@ from django.conf import settings from django.contrib import admin from django.urls import path -from django.views.generic import RedirectView + +# def debug_view(request): +# """ debug request.META """ +# if not request.user.is_authenticated: +# from django.shortcuts import redirect +# return redirect('admin:index') +# +# import pprint +# meta = pprint.pformat(request.META) +# html = f'request.META:
{meta}
' +# from django.http import HttpResponse +# return HttpResponse(html) -# settings.PATH_URL is the $YNH_APP_ARG_PATH -if settings.PATH_URL: - admin.autodiscover() +admin.autodiscover() - urlpatterns = [ - # TODO: - # XXX: Hack - the MEDIA_URL contains the "PATH_URL" already: - # path(settings.MEDIA_URL.lstrip('/'), include('django_tools.serve_media_app.urls')), - - path(f'{settings.PATH_URL}/admin/', admin.site.urls), - path('', RedirectView.as_view(pattern_name='admin:index')), - ] -else: - # Installed to domain root, without a path prefix? - from inventory_project.urls import urlpatterns # noqa +urlpatterns = [ + # path(f'{settings.PATH_URL}/debug/', debug_view), + path(f'{settings.PATH_URL}/', admin.site.urls), +] From 88de499e6de66a88db9b2a137e85907113419f3b Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Tue, 22 Dec 2020 18:53:02 +0100 Subject: [PATCH 07/12] backup "$public_path", too --- scripts/backup | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/backup b/scripts/backup index 91b17e3..308ba1c 100755 --- a/scripts/backup +++ b/scripts/backup @@ -18,10 +18,12 @@ ynh_print_info --message="Loading installation settings..." app=$YNH_APP_INSTANCE_NAME +public_path=$(ynh_app_setting_get --app="$app" --key=public_path) final_path=$(ynh_app_setting_get --app="$app" --key=final_path) -domain=$(ynh_app_setting_get --app="$app" --key=domain) db_name=$(ynh_app_setting_get --app="$app" --key=db_name) +domain=$(ynh_app_setting_get --app="$app" --key=domain) + #================================================= # DECLARE DATA AND CONF FILES TO BACKUP #================================================= @@ -32,6 +34,7 @@ ynh_print_info --message="Declaring files to be backed up..." #================================================= ynh_backup --src_path="$final_path" +ynh_backup --src_path="$public_path" #================================================= # BACKUP THE NGINX CONFIGURATION From d7b014cbf609465e1b867d66539b7c76356eb8c8 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Tue, 22 Dec 2020 18:53:32 +0100 Subject: [PATCH 08/12] fix change url by using $new_domain and $new_path in settings --- scripts/change_url | 10 +++++----- scripts/install | 10 ++++++---- scripts/restore | 5 +++-- scripts/upgrade | 10 ++++++---- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/scripts/change_url b/scripts/change_url index 54a5a85..6e300ed 100644 --- a/scripts/change_url +++ b/scripts/change_url @@ -24,8 +24,6 @@ new_path=$YNH_APP_NEW_PATH #================================================= ynh_script_progression --message="Loading installation settings..." -domain=$(ynh_app_setting_get --app="$app" --key=domain) -path_url=$(ynh_app_setting_get --app="$app" --key=path) admin=$(ynh_app_setting_get --app="$app" --key=admin) is_public=$(ynh_app_setting_get --app="$app" --key=is_public) public_path=$(ynh_app_setting_get --app="$app" --key=public_path) @@ -97,7 +95,7 @@ then domain="$old_domain" path_url="$new_path" # Create a dedicated nginx config - ynh_add_nginx_config + ynh_add_nginx_config "public_path" "port" fi # Change the domain for nginx @@ -127,13 +125,15 @@ ynh_replace_string --match_string="__APP__" --replace_string="$app" --target_fil ynh_replace_string --match_string="__DB_PWD__" --replace_string="$db_pwd" --target_file="$settings" ynh_replace_string --match_string="__ADMIN__" --replace_string="$admin" --target_file="$settings" ynh_replace_string --match_string="__ADMINMAIL__" --replace_string="$admin_mail" --target_file="$settings" -ynh_replace_string --match_string="__DOMAIN__" --replace_string="$domain" --target_file="$settings" ynh_replace_string --match_string="__FINAL_HOME_PATH__" --replace_string="$final_path" --target_file="$settings" ynh_replace_string --match_string="__FINAL_WWW_PATH__" --replace_string="$public_path" --target_file="$settings" -ynh_replace_string --match_string="__PATH_URL__" --replace_string="$path_url" --target_file="$settings" ynh_replace_string --match_string="__LOG_FILE__" --replace_string="$log_file" --target_file="$settings" ynh_replace_string --match_string="__REDIS_DB__" --replace_string="$redis_db" --target_file="$settings" +# Difference to install/upgrade scripts: Use $new_domain and $new_path here: +ynh_replace_string --match_string="__DOMAIN__" --replace_string="$new_domain" --target_file="$settings" +ynh_replace_string --match_string="__PATH_URL__" --replace_string="$new_path" --target_file="$settings" + # Recalculate and store the config file checksum into the app settings ynh_store_file_checksum --file="$settings" diff --git a/scripts/install b/scripts/install index 7769f04..50b7f89 100755 --- a/scripts/install +++ b/scripts/install @@ -39,14 +39,15 @@ touch "${log_file}" #================================================= ynh_script_progression --message="Storing installation settings..." -ynh_app_setting_set --app="$app" --key=domain --value="$domain" -ynh_app_setting_set --app="$app" --key=path --value="$path_url" ynh_app_setting_set --app="$app" --key=admin --value="$admin" ynh_app_setting_set --app="$app" --key=is_public --value="$is_public" ynh_app_setting_set --app="$app" --key=public_path --value="$public_path" ynh_app_setting_set --app="$app" --key=final_path --value="$final_path" ynh_app_setting_set --app="$app" --key=log_path --value="$log_file" +ynh_app_setting_set --app="$app" --key=domain --value="$domain" +ynh_app_setting_set --app="$app" --key=path --value="$path_url" + # Find a free port port=$(ynh_find_port --port=8000) # Set port as application setting @@ -143,13 +144,14 @@ ynh_replace_string --match_string="__APP__" --replace_string="$app" --target_fil ynh_replace_string --match_string="__DB_PWD__" --replace_string="$db_pwd" --target_file="$settings" ynh_replace_string --match_string="__ADMIN__" --replace_string="$admin" --target_file="$settings" ynh_replace_string --match_string="__ADMINMAIL__" --replace_string="$admin_mail" --target_file="$settings" -ynh_replace_string --match_string="__DOMAIN__" --replace_string="$domain" --target_file="$settings" ynh_replace_string --match_string="__FINAL_HOME_PATH__" --replace_string="$final_path" --target_file="$settings" ynh_replace_string --match_string="__FINAL_WWW_PATH__" --replace_string="$public_path" --target_file="$settings" -ynh_replace_string --match_string="__PATH_URL__" --replace_string="$path_url" --target_file="$settings" ynh_replace_string --match_string="__LOG_FILE__" --replace_string="$log_file" --target_file="$settings" ynh_replace_string --match_string="__REDIS_DB__" --replace_string="$redis_db" --target_file="$settings" +ynh_replace_string --match_string="__DOMAIN__" --replace_string="$domain" --target_file="$settings" +ynh_replace_string --match_string="__PATH_URL__" --replace_string="$path_url" --target_file="$settings" + # Calculate and store the config file checksum into the app settings ynh_store_file_checksum --file="$settings" diff --git a/scripts/restore b/scripts/restore index 1bbf501..216ff7f 100755 --- a/scripts/restore +++ b/scripts/restore @@ -20,14 +20,15 @@ ynh_abort_if_errors #================================================= ynh_script_progression --message="Loading settings..." -domain=$(ynh_app_setting_get --app="$app" --key=domain) -path_url=$(ynh_app_setting_get --app="$app" --key=path) public_path=$(ynh_app_setting_get --app="$app" --key=public_path) final_path=$(ynh_app_setting_get --app="$app" --key=final_path) db_name=$(ynh_app_setting_get --app="$app" --key=db_name) db_user=$db_name db_pwd=$(ynh_app_setting_get --app="$app" --key=psqlpwd) +domain=$(ynh_app_setting_get --app="$app" --key=domain) +path_url=$(ynh_app_setting_get --app="$app" --key=path) + #================================================= # CHECK IF THE APP CAN BE RESTORED #================================================= diff --git a/scripts/upgrade b/scripts/upgrade index a5d053b..78cccce 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -12,14 +12,15 @@ source /usr/share/yunohost/helpers #================================================= ynh_script_progression --message="Loading installation settings..." -domain=$(ynh_app_setting_get --app="$app" --key=domain) -path_url=$(ynh_app_setting_get --app="$app" --key=path) admin=$(ynh_app_setting_get --app="$app" --key=admin) is_public=$(ynh_app_setting_get --app="$app" --key=is_public) public_path=$(ynh_app_setting_get --app="$app" --key=public_path) final_path=$(ynh_app_setting_get --app="$app" --key=final_path) log_path=$(ynh_app_setting_get --app="$app" --key=log_path) +domain=$(ynh_app_setting_get --app="$app" --key=domain) +path_url=$(ynh_app_setting_get --app="$app" --key=path) + port=$(ynh_app_setting_get --app="$app" --key=port) db_pwd=$(ynh_app_setting_get --app="$app" --key=psqlpwd) admin_mail=$(ynh_user_get_info "$admin" mail) @@ -135,13 +136,14 @@ ynh_replace_string --match_string="__APP__" --replace_string="$app" --target_fil ynh_replace_string --match_string="__DB_PWD__" --replace_string="$db_pwd" --target_file="$settings" ynh_replace_string --match_string="__ADMIN__" --replace_string="$admin" --target_file="$settings" ynh_replace_string --match_string="__ADMINMAIL__" --replace_string="$admin_mail" --target_file="$settings" -ynh_replace_string --match_string="__DOMAIN__" --replace_string="$domain" --target_file="$settings" ynh_replace_string --match_string="__FINAL_HOME_PATH__" --replace_string="$final_path" --target_file="$settings" ynh_replace_string --match_string="__FINAL_WWW_PATH__" --replace_string="$public_path" --target_file="$settings" -ynh_replace_string --match_string="__PATH_URL__" --replace_string="$path_url" --target_file="$settings" ynh_replace_string --match_string="__LOG_FILE__" --replace_string="$log_file" --target_file="$settings" ynh_replace_string --match_string="__REDIS_DB__" --replace_string="$redis_db" --target_file="$settings" +ynh_replace_string --match_string="__DOMAIN__" --replace_string="$domain" --target_file="$settings" +ynh_replace_string --match_string="__PATH_URL__" --replace_string="$path_url" --target_file="$settings" + # Recalculate and store the config file checksum into the app settings ynh_store_file_checksum --file="$settings" From a5cc74340c1fc30877d8c3570c9c977a4beafe74 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Tue, 22 Dec 2020 18:54:32 +0100 Subject: [PATCH 09/12] code style --- scripts/_common.sh | 1 + scripts/install | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/scripts/_common.sh b/scripts/_common.sh index 7a4d614..93e5c97 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -6,6 +6,7 @@ domain=$YNH_APP_ARG_DOMAIN path_url=$YNH_APP_ARG_PATH + admin=$YNH_APP_ARG_ADMIN is_public=$YNH_APP_ARG_IS_PUBLIC app=$YNH_APP_INSTANCE_NAME diff --git a/scripts/install b/scripts/install index 50b7f89..fbc7b25 100755 --- a/scripts/install +++ b/scripts/install @@ -167,21 +167,21 @@ cp "../conf/ynh_urls.py" "$final_path/ynh_urls.py" ynh_script_progression --message="migrate/collectstatic/createadmin..." --weight=10 ( - set +o nounset - source "${final_path}/venv/bin/activate" - set -o nounset - cd "${final_path}" + set +o nounset + source "${final_path}/venv/bin/activate" + set -o nounset + cd "${final_path}" - # Just for debugging: - ./manage.py diffsettings + # Just for debugging: + ./manage.py diffsettings - ./manage.py migrate --no-input - ./manage.py collectstatic --no-input - ./create_superuser.py --username="$admin" --email="$admin_mail" --password="django-for-runners" + ./manage.py migrate --no-input + ./manage.py collectstatic --no-input + ./create_superuser.py --username="$admin" --email="$admin_mail" --password="django-for-runners" - # Check the configuration - # This may fail in some cases with errors, etc., but the app works and the user can fix issues later. - ./manage.py check --deploy || true + # Check the configuration + # This may fail in some cases with errors, etc., but the app works and the user can fix issues later. + ./manage.py check --deploy || true ) #================================================= From c1205e7c47d008b85a7b2df24648439940146765 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Tue, 22 Dec 2020 18:55:11 +0100 Subject: [PATCH 10/12] local test via Makefile --- .gitignore | 1 + Makefile | 36 ++++++++++++++++++++++++++++++++++++ README.md | 22 +++++++++++----------- manifest.json | 2 +- pyproject.toml | 16 ++++++++++++++++ 5 files changed, 65 insertions(+), 12 deletions(-) create mode 100644 Makefile create mode 100644 pyproject.toml diff --git a/.gitignore b/.gitignore index a7c2f58..d18d5cc 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ __pycache__ secret.txt /local_test/ +/poetry.lock diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2edd58a --- /dev/null +++ b/Makefile @@ -0,0 +1,36 @@ +SHELL := /bin/bash + +all: help + +help: + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z0-9 -]+:.*?## / {printf "\033[36m%-22s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) + +check-poetry: + @if [[ "$(shell poetry --version 2>/dev/null)" == *"Poetry"* ]] ; \ + then \ + echo "Poetry found, ok." ; \ + else \ + echo 'Please install poetry first, with e.g.:' ; \ + echo 'make install-poetry' ; \ + exit 1 ; \ + fi + +install-poetry: ## install or update poetry + pip3 install -U poetry + +install: check-poetry ## install project via poetry + poetry install + +update: install-poetry ## update the sources and installation + poetry update + +local-test: check-poetry ## Run local_test.py to run the project locally + poetry run ./local_test.py + +local-diff-settings: ## Run "manage.py diffsettings" with local test + poetry run python3 local_test/opt_yunohost/manage.py diffsettings + + +############################################################################## + +.PHONY: help check-poetry install-poetry install update local-test \ No newline at end of file diff --git a/README.md b/README.md index 2d0b2d4..e7fac34 100644 --- a/README.md +++ b/README.md @@ -111,20 +111,20 @@ For quicker developing of Django-For-Runners in the context of YunoHost app, it's possible to run the Django developer server with the settings and urls made for YunoHost installation. -For this, just run `local_test.py` in a Django-For-Runners virtualenv. - e.g.: ```bash -~$ git clone https://github.com/jedie/Django-For-Runners.git ~$ git clone https://github.com/YunoHost-Apps/django-for-runners_ynh.git -~$ cd Django-For-Runners/ -~/Django-For-Runners$ make install -~/Django-For-Runners$ poetry shell -(django-for-runners-yd_5sxYx-py3.8) ~/Django-For-Runners$ cd ../django-for-runners_ynh/ -(django-for-runners-yd_5sxYx-py3.8) ~/django-for-runners_ynh$ ./local_test.py -... -Django version 2.2.17, using settings 'ynh_django-for-runners_settings' -Starting development server at http://127.0.0.1:8000/ +~$ cd django-for-runners_ynh/ +~/django-for-runners_ynh$ make +install-poetry install or update poetry +install install project via poetry +update update the sources and installation +local-test Run local_test.py to run the project locally +local-diff-settings Run "manage.py diffsettings" with local test + +~/django-for-runners_ynh$ make install-poetry +~/django-for-runners_ynh$ make install +~/django-for-runners_ynh$ make local-test ``` Notes: diff --git a/manifest.json b/manifest.json index 71f217f..c9bd2f3 100644 --- a/manifest.json +++ b/manifest.json @@ -5,7 +5,7 @@ "description": { "en": "Store your GPX tracks of your running (or other sports activity)" }, - "version": "0.12.0rc2~ynh1", + "version": "0.12.0rc2~ynh2", "url": "https://github.com/jedie/django-for-runners", "license": "GPL-3.0", "maintainer": { diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4d17b07 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,16 @@ +[tool.poetry] +name = "django-for-runners_ynh" +version = "0.12.0rc2~ynh2" +description = "Test django-for-runners_ynh via local_test.py" +authors = ["JensDiemer "] +license = "GPL" + +[tool.poetry.dependencies] +python = ">=3.7,<4.0.0" +django-for-runners = "*" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" From 85b573046b28b4276586bbbf87236e7631212481 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Tue, 22 Dec 2020 18:55:27 +0100 Subject: [PATCH 11/12] restore $public_path, too --- scripts/restore | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/restore b/scripts/restore index 216ff7f..3145133 100755 --- a/scripts/restore +++ b/scripts/restore @@ -52,6 +52,7 @@ ynh_restore_file --origin_path="/etc/nginx/conf.d/$domain.d/$app.conf" #================================================= ynh_script_progression --message="Restoring the app main directory..." +ynh_restore_file --origin_path="$public_path" ynh_restore_file --origin_path="$final_path" touch "$final_path/local_settings.py" From 7d225c56633866c8249dbb5c5f2c79b68126cdd5 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Tue, 22 Dec 2020 18:55:57 +0100 Subject: [PATCH 12/12] Autenticate completely with SSOwat --- README.md | 17 ++- conf/ynh_authenticate.py | 185 +++++++++++++++++++++++++++++++ conf/ynh_for_runners_settings.py | 67 +++++------ scripts/_common.sh | 6 +- scripts/install | 1 + scripts/upgrade | 1 + 6 files changed, 227 insertions(+), 50 deletions(-) create mode 100644 conf/ynh_authenticate.py diff --git a/README.md b/README.md index e7fac34..8a13efb 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # Django-For-Runners for YunoHost -[![Integration level](https://dash.yunohost.org/integration/django-for-runners.svg)](https://dash.yunohost.org/appci/app/django-for-runners) ![](https://ci-apps.yunohost.org/ci/badges/django-for-runners.status.svg) ![](https://ci-apps.yunohost.org/ci/badges/django-for-runners.maintain.svg) +[![Integration level](https://dash.yunohost.org/integration/django-for-runners.svg)](https://dash.yunohost.org/appci/app/django-for-runners) ![](https://ci-apps.yunohost.org/ci/badges/django-for-runners.status.svg) ![](https://ci-apps.yunohost.org/ci/badges/django-for-runners.maintain.svg) [![Install Django-For-Runners with YunoHost](https://install-app.yunohost.org/install-with-yunohost.png)](https://install-app.yunohost.org/?app=django-for-runners) -> *This package allows you to install Django-For-Runners quickly and simply on a YunoHost server. +> *This package allows you to install Django-For-Runners quickly and simply on a YunoHost server. If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/install) to learn how to install it.* Current status is pre-alpha: This app doesn't work, yet ;) @@ -31,9 +31,16 @@ You can edit the file `$final_path/local_settings.py` to enable or disable featu # Miscellaneous -## LDAP connection -Supported by https://github.com/django-auth-ldap/django-auth-ldap +## SSO authentication + +[SSOwat](https://github.com/YunoHost/SSOwat) is fully supported: + +* First user (`$YNH_APP_ARG_ADMIN`) will be created as Django's super user +* All new users will be created as normal users +* Login via SSO is fully supported +* User Email, First / Last name will be updated from SSO data + ## Links @@ -131,4 +138,4 @@ Notes: * SQlite database will be used * A super user with username `test` and password `test` is created -* The page is available under `http://127.0.0.1:8000/app_path/` \ No newline at end of file +* The page is available under `http://127.0.0.1:8000/app_path/` diff --git a/conf/ynh_authenticate.py b/conf/ynh_authenticate.py new file mode 100644 index 0000000..1f3d03c --- /dev/null +++ b/conf/ynh_authenticate.py @@ -0,0 +1,185 @@ +""" + * remote user authentication backend + * remote user middleware + + Note: SSOwat/nginx add authentication headers: + + 'HTTP_AUTHORIZATION': 'Basic XXXXXXXXXXXXXXXX=' + 'HTTP_AUTH_USER': 'username' + 'HTTP_REMOTE_USER': 'username' + + Basic auth contains "{username}:{plaintext-password}" + + and we get SSOwat cookies like: + + 'HTTP_COOKIE': 'SSOwAuthUser=username; ' + 'SSOwAuthHash=593876aa66...99e69f88af1e; ' + 'SSOwAuthExpire=1609227697.998; ' + + * Login a user via HTTP_REMOTE_USER header, but check also username in: + * SSOwAuthUser + * HTTP_AUTH_USER + * HTTP_AUTHORIZATION (Basic auth) + * Create new users + * Update Email, First / Last name for existing users +""" +import base64 +import logging + +from axes.exceptions import AxesBackendPermissionDenied +from django.contrib.auth.backends import RemoteUserBackend as OriginRemoteUserBackend +from django.contrib.auth.middleware import RemoteUserMiddleware as OriginRemoteUserMiddleware +from django.core.exceptions import ValidationError +from inventory.permissions import get_or_create_normal_user_group + +logger = logging.getLogger(__name__) + + +def update_user_profile(request): + """ + Update existing user information: + * Email + * First / Last name + """ + user = request.user + assert user.is_authenticated + + update_fields = [] + + if not user.password: + # Empty password is not valid, so we can't save the model, because of full_clean() call + logger.info('Set unusable password for user: %s', user) + user.set_unusable_password() + update_fields.append('password') + + email = request.META.get('HTTP_EMAIL') + if email and user.email != email: + logger.info('Update email: %r -> %r', user.email, email) + user.email = email + update_fields.append('email') + + raw_username = request.META.get('HTTP_NAME') + if raw_username: + if ' ' in raw_username: + first_name, last_name = raw_username.split(' ', 1) + else: + first_name = '' + last_name = raw_username + + if user.first_name != first_name: + logger.info('Update first name: %r -> %r', user.first_name, first_name) + user.first_name = first_name + update_fields.append('first_name') + + if user.last_name != last_name: + logger.info('Update last name: %r -> %r', user.last_name, last_name) + user.last_name = last_name + update_fields.append('last_name') + + if update_fields: + try: + user.full_clean() + except ValidationError: + logger.exception('Can not update user: %s', user) + else: + user.save(update_fields=update_fields) + + +class RemoteUserMiddleware(OriginRemoteUserMiddleware): + """ + Middleware to login a user HTTP_REMOTE_USER header. + Use Django Axes if something is wrong. + Update exising user informations. + """ + header = 'HTTP_REMOTE_USER' + force_logout_if_no_header = True + + def process_request(self, request): + # Keep the information if the user is already logged in + was_authenticated = request.user.is_authenticated + + super().process_request(request) # login remote user + + if not request.user.is_authenticated: + # Not logged in -> nothing to verify here + return + + # Check SSOwat cookie informations: + try: + username = request.COOKIES['SSOwAuthUser'] + except KeyError: + logger.error('SSOwAuthUser cookie missing!') + + # emits a signal indicating user login failed, which is processed by + # axes.signals.log_user_login_failed which logs and flags the failed request. + raise AxesBackendPermissionDenied('Cookie missing') + + logger.info('SSOwat username from cookies: %r', username) + if username != request.user.username: + raise AxesBackendPermissionDenied('Wrong username') + + # Compare with HTTP_AUTH_USER + try: + username = request.META['HTTP_AUTH_USER'] + except KeyError: + logger.error('HTTP_AUTH_USER missing!') + raise AxesBackendPermissionDenied('No HTTP_AUTH_USER') + + if username != request.user.username: + raise AxesBackendPermissionDenied('Wrong HTTP_AUTH_USER username') + + # Also check 'HTTP_AUTHORIZATION', but only the username ;) + try: + auth = request.META['HTTP_AUTHORIZATION'] + except KeyError: + logger.error('HTTP_AUTHORIZATION missing!') + raise AxesBackendPermissionDenied('No HTTP_AUTHORIZATION') + + scheme, creds = auth.split(' ', 1) + if scheme.lower() != 'basic': + logger.error('HTTP_AUTHORIZATION with %r not supported', scheme) + raise AxesBackendPermissionDenied('HTTP_AUTHORIZATION scheme not supported') + + creds = str(base64.b64decode(creds), encoding='utf-8') + username = creds.split(':', 1)[0] + if username != request.user.username: + raise AxesBackendPermissionDenied('Wrong HTTP_AUTHORIZATION username') + + if not was_authenticated: + # First request, after login -> update user informations + logger.info('Remote used was logged in') + update_user_profile(request) + + +class RemoteUserBackend(OriginRemoteUserBackend): + """ + Authentication backend via SSO/nginx header + """ + create_unknown_user = True + + def authenticate(self, request, remote_user): + logger.info('Remote user authenticate: %r', remote_user) + return super().authenticate(request, remote_user) + + def configure_user(self, request, user): + """ + Configure a user after creation and return the updated user. + Setup a normal, non-superuser + """ + logger.warning('Configure user %s', user) + + user.set_unusable_password() # Always login via SSO + user.is_staff = True + user.is_superuser = False + user.save() + + pyinventory_user_group = get_or_create_normal_user_group()[0] + user.groups.set([pyinventory_user_group]) + + update_user_profile(request) + + return user + + def user_can_authenticate(self, user): + logger.warning('Remote user login: %s', user) + return True diff --git a/conf/ynh_for_runners_settings.py b/conf/ynh_for_runners_settings.py index 9eacb76..55864e6 100644 --- a/conf/ynh_for_runners_settings.py +++ b/conf/ynh_for_runners_settings.py @@ -11,8 +11,6 @@ from pathlib import Path as __Path -import ldap -from django_auth_ldap.config import LDAPSearch from for_runners_project.settings.base import * # noqa DEBUG = False @@ -36,44 +34,30 @@ PATH_URL = PATH_URL.strip('/') ROOT_URLCONF = 'ynh_urls' # /opt/yunohost/django-for-runners/ynh_urls.py # ----------------------------------------------------------------------------- -# https://github.com/django-auth-ldap/django-auth-ldap - -LDAP_SERVER_URI = 'ldap://localhost:389' -LDAP_START_TLS = True - -# enable anonymous searches -# https://django-auth-ldap.readthedocs.io/en/latest/authentication.html?highlight=anonymous#search-bind -LDAP_BIND_DN = '' -LDAP_BIND_PASSWORD = '' - -LDAP_ROOT_DN = 'ou=users,dc=yunohost,dc=org' - -AUTH_LDAP_USER_SEARCH = LDAPSearch(LDAP_ROOT_DN, ldap.SCOPE_SUBTREE, '(uid=%(user)s)') - -# Populate the Django user from the LDAP directory. -AUTH_LDAP_USER_ATTR_MAP = { - 'username': 'uid', - 'first_name': 'givenName', - 'last_name': 'sn', - 'email': 'mail', -} - -AUTH_LDAP_ALWAYS_UPDATE_USER = True - -# Don't use LDAP group membership to calculate group permissions -AUTH_LDAP_FIND_GROUP_PERMS = False - -# TODO: -# AUTH_LDAP_GROUP_TYPE = 'normal user' # Same as: inventory.permissions.NORMAL_USER_GROUP_NAME - -# Cache distinguished names and group memberships for an hour to minimize LDAP traffic -AUTH_LDAP_CACHE_TIMEOUT = 3600 # Keep ModelBackend around for per-user permissions and superuser AUTHENTICATION_BACKENDS = ( - 'django_auth_ldap.backend.LDAPBackend', + 'axes.backends.AxesBackend', # AxesBackend should be the first backend! + + # Authenticate via SSO and nginx 'HTTP_REMOTE_USER' header: + 'ynh_authenticate.RemoteUserBackend', + + # Fallback to normal Django model backend: 'django.contrib.auth.backends.ModelBackend', ) +LOGIN_REDIRECT_URL = None +LOGIN_URL = '/yunohost/sso/' +LOGOUT_REDIRECT_URL = '/yunohost/sso/' +# /yunohost/sso/?action=logout + +# ----------------------------------------------------------------------------- +# https://docs.djangoproject.com/en/2.2/howto/auth-remote-user/ +# Add RemoteUserMiddleware after AuthenticationMiddleware + +MIDDLEWARE.insert( + MIDDLEWARE.index('django.contrib.auth.middleware.AuthenticationMiddleware') + 1, + 'ynh_authenticate.RemoteUserMiddleware', +) # ----------------------------------------------------------------------------- @@ -165,7 +149,7 @@ LOGGING = { 'class': 'django.utils.log.AdminEmailHandler', 'include_html': True, }, - 'syslog': { + 'log_file': { 'level': 'DEBUG', 'class': 'logging.handlers.WatchedFileHandler', 'formatter': 'verbose', @@ -173,12 +157,11 @@ LOGGING = { }, }, 'loggers': { - '': {'handlers': ['syslog', 'mail_admins'], 'level': 'INFO', 'propagate': False}, - 'django': {'handlers': ['syslog', 'mail_admins'], 'level': 'INFO', 'propagate': False}, - 'axes': {'handlers': ['syslog', 'mail_admins'], 'level': 'WARNING', 'propagate': False}, - 'django_tools': {'handlers': ['syslog', 'mail_admins'], 'level': 'INFO', 'propagate': False}, - 'django_auth_ldap': {'handlers': ['syslog', 'mail_admins'], 'level': 'DEBUG', 'propagate': False}, - 'inventory': {'handlers': ['syslog', 'mail_admins'], 'level': 'INFO', 'propagate': False}, + '': {'handlers': ['log_file', 'mail_admins'], 'level': 'INFO', 'propagate': False}, + 'django': {'handlers': ['log_file', 'mail_admins'], 'level': 'INFO', 'propagate': False}, + 'axes': {'handlers': ['log_file', 'mail_admins'], 'level': 'WARNING', 'propagate': False}, + 'django_tools': {'handlers': ['log_file', 'mail_admins'], 'level': 'INFO', 'propagate': False}, + 'for_runners': {'handlers': ['log_file', 'mail_admins'], 'level': 'INFO', 'propagate': False}, }, } diff --git a/scripts/_common.sh b/scripts/_common.sh index 93e5c97..013a0bc 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -25,14 +25,14 @@ log_file="${log_path}/django-for-runners.log" #================================================= # dependencies used by the app -pkg_dependencies="build-essential python3-dev python3-pip python3-virtualenv git \ - postgresql postgresql-contrib python3-ldap libldap2-dev libsasl2-dev" +pkg_dependencies="build-essential python3-dev python3-pip python3-venv git \ + postgresql postgresql-contrib" # Django-For-Runners's version for PIP and settings file for_runners_version="0.12.0rc2" # Extra python packages: -pypi_extras="django-redis django-auth-ldap" +pypi_extras="django-redis" #================================================= # Redis HELPERS diff --git a/scripts/install b/scripts/install index fbc7b25..5a7ff0b 100755 --- a/scripts/install +++ b/scripts/install @@ -159,6 +159,7 @@ ynh_app_setting_set --app="$app" --key=redis_db --value="$redis_db" touch "$final_path/local_settings.py" +cp "../conf/ynh_authenticate.py" "$final_path/ynh_authenticate.py" cp "../conf/ynh_urls.py" "$final_path/ynh_urls.py" #================================================= diff --git a/scripts/upgrade b/scripts/upgrade index 78cccce..b9e5262 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -149,6 +149,7 @@ ynh_store_file_checksum --file="$settings" touch "$final_path/local_settings.py" +cp "../conf/ynh_authenticate.py" "$final_path/ynh_authenticate.py" cp "../conf/ynh_urls.py" "$final_path/ynh_urls.py" #=================================================