From fd84fbcc64fa0f692110203cf37256e8bcd33642 Mon Sep 17 00:00:00 2001 From: Chris Vogel Date: Fri, 26 Apr 2024 21:42:05 +0200 Subject: [PATCH 1/6] test --- logger.json | 27 ++++++++++++++++++++++ run-uvicorn.example | 55 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 logger.json create mode 100644 run-uvicorn.example diff --git a/logger.json b/logger.json new file mode 100644 index 0000000..cc0676e --- /dev/null +++ b/logger.json @@ -0,0 +1,27 @@ +{ + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "json": { + "()": "pythonjsonlogger.jsonlogger.JsonFormatter", + "fmt": "%(asctime)s %(name)s %(levelname)s %(message)s" + } + }, + "handlers": { + "stderr": { + "formatter": "json", + "class": "logging.StreamHandler", + "stream": "ext://sys.stderr" + }, + "stdout": { + "formatter": "json", + "class": "logging.StreamHandler", + "stream": "ext://sys.stdout" + } + }, + "loggers": { + "uvicorn": {"handlers": ["stderr"], "level": "INFO", "propagate": false}, + "uvicorn.error": {"level": "INFO"}, + "uvicorn.access": {"handlers": ["stdout"], "level": "INFO", "propagate": false} + } +} diff --git a/run-uvicorn.example b/run-uvicorn.example new file mode 100644 index 0000000..a203da5 --- /dev/null +++ b/run-uvicorn.example @@ -0,0 +1,55 @@ +/opt/flohmarkt/venv/bin/uvicorn --host 127.0.0.1 --port 8000 --reload --log-config logger.json flohmarkt.web:start + +Fehler: + +Traceback (most recent call last): + File "/usr/lib/python3.9/logging/config.py", line 385, in resolve + found = self.importer(used) +ModuleNotFoundError: No module named 'pythonjsonlogger' + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/usr/lib/python3.9/logging/config.py", line 543, in configure + formatters[name] = self.configure_formatter( + File "/usr/lib/python3.9/logging/config.py", line 655, in configure_formatter + result = self.configure_custom(config) + File "/usr/lib/python3.9/logging/config.py", line 470, in configure_custom + c = self.resolve(c) + File "/usr/lib/python3.9/logging/config.py", line 398, in resolve + raise v + File "/usr/lib/python3.9/logging/config.py", line 385, in resolve + found = self.importer(used) +ValueError: Cannot resolve 'pythonjsonlogger.jsonlogger.JsonFormatter': No module named 'pythonjsonlogger' + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/opt/flohmarkt/venv/bin/uvicorn", line 8, in + sys.exit(main()) + File "/opt/flohmarkt/venv/lib/python3.9/site-packages/click/core.py", line 1157, in __call__ + return self.main(*args, **kwargs) + File "/opt/flohmarkt/venv/lib/python3.9/site-packages/click/core.py", line 1078, in main + rv = self.invoke(ctx) + File "/opt/flohmarkt/venv/lib/python3.9/site-packages/click/core.py", line 1434, in invoke + return ctx.invoke(self.callback, **ctx.params) + File "/opt/flohmarkt/venv/lib/python3.9/site-packages/click/core.py", line 783, in invoke + return __callback(*args, **kwargs) + File "/opt/flohmarkt/venv/lib/python3.9/site-packages/uvicorn/main.py", line 404, in main + run( + File "/opt/flohmarkt/venv/lib/python3.9/site-packages/uvicorn/main.py", line 506, in run + config = Config( + File "/opt/flohmarkt/venv/lib/python3.9/site-packages/uvicorn/config.py", line 299, in __init__ + self.configure_logging() + File "/opt/flohmarkt/venv/lib/python3.9/site-packages/uvicorn/config.py", line 411, in configure_logging + logging.config.dictConfig(loaded_config) + File "/usr/lib/python3.9/logging/config.py", line 809, in dictConfig + dictConfigClass(config).configure() + File "/usr/lib/python3.9/logging/config.py", line 546, in configure + raise ValueError('Unable to configure ' +ValueError: Unable to configure formatter 'json' + +(venv) root@yt:/opt/flohmarkt/flohmarkt# pip install python-json-logger +Requirement already satisfied: python-json-logger in /usr/lib/python3.9/site-packages (2.0.7) + +Ausgabe in json will ich eigentlich auch nicht falls das in der config steht. Der Kram ist aber so dokumentiert, dass ich ihn nicht verstehe... From 17424c781cc3f6db381bec07f331fa73ef5055a8 Mon Sep 17 00:00:00 2001 From: Chris Vogel Date: Sat, 27 Apr 2024 13:07:26 +0200 Subject: [PATCH 2/6] fixed typo --- doc/DESCRIPTION.md | 2 +- doc/DEVELOPMENT.md | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/DESCRIPTION.md b/doc/DESCRIPTION.md index e508bc8..55c7d51 100644 --- a/doc/DESCRIPTION.md +++ b/doc/DESCRIPTION.md @@ -10,7 +10,7 @@ Registered users can access a simple form to publish there small ads. The small ## Federation -To communicate with someone who published a small ad the server hints to an unregistred user _"To answer this offer please log in or create an account. OR use another fediverse-account". +To communicate with someone who published a small ad the server hints to an unregistred user _"To answer this offer please log in or create an account. OR use another fediverse-account"._ The small add visited turns out to be a _note_ in the fediverse. It's url can be opened with your favourite fediverse client at the server you're already registered to. You then can boost the small ad like any other note you read. diff --git a/doc/DEVELOPMENT.md b/doc/DEVELOPMENT.md index 01fa970..c9acee1 100644 --- a/doc/DEVELOPMENT.md +++ b/doc/DEVELOPMENT.md @@ -29,6 +29,12 @@ To do so we'll push the according changes to the [flohmarkt repository at github One **pitfall** doing so is that we can't rely on the yunohost CI for testing for our codeberg repository this way. If need'll be and developers would like to use yunohost as their base for active work on flohmarkt we might release another app **flohmarkt-devel_ynh** in future that closely follows the repository we use for development. +#### master and main + +The development startet on codeberg on a branch named **main**. That worked all the way until the repository got mirrored to github to be included into the yunohost app catalog. The last change that needed to be done for flohmarkt to appear on the catalog has been to change the name of the branch to **master** for the workflows on github to recognize it for catalog inclusion. + +Short: for **historic** reason we use the branch **main** on codeberg and publish the versions for the catalog as **master** on github. + ### pushing to github * make sure the local git and the codeberg git are in sync on their main branch @@ -37,17 +43,11 @@ One **pitfall** doing so is that we can't rely on the yunohost CI for testing fo → change `.-ynh` according to the new flohmarkt version * changes in integration: scripts, conf files or `doc/*` changed → increment `` to signal a new version of the yunohost integration -* **help needed** make sure the main branch contains the version to publish -* try to push to github and maybe fail: - * on github the README.md and README_LANG.md files are automatically generated and might have changed - * check the difference between github and the local git - * pull the newer versions from github and merge them -* push the update to github -* push the local git to codeberg +* try to push the local main branch to github which might fail + * there might for some reason exist an old main branch that had not been deleted after the PR to the github master branch - check carefully and delete the existing main branch +* on github open an PR from the main branch into the master branch + * the PR can be tested on the CI workflow if a comment containing `!testme` is added to the PR +* the PR will be included after + * it successfully ran through the CI workflow (results will show up inside the PR) + * it has been reviewed -#### help wanted - -At time of writing the author still is learning about git and didn't know a way to push a branch/tag from their local git repository onto a different branch on a remote repository which would help to -* tag a release version on codeberg -* checkout the version into a local git and -* push that version to github From 26d4aa7ca2de0308aea2cd2190c6df00e7e09b38 Mon Sep 17 00:00:00 2001 From: Chris Vogel Date: Fri, 26 Apr 2024 21:42:05 +0200 Subject: [PATCH 3/6] test --- logger.json | 27 ++++++++++++++++++++++ run-uvicorn.example | 55 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 logger.json create mode 100644 run-uvicorn.example diff --git a/logger.json b/logger.json new file mode 100644 index 0000000..cc0676e --- /dev/null +++ b/logger.json @@ -0,0 +1,27 @@ +{ + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "json": { + "()": "pythonjsonlogger.jsonlogger.JsonFormatter", + "fmt": "%(asctime)s %(name)s %(levelname)s %(message)s" + } + }, + "handlers": { + "stderr": { + "formatter": "json", + "class": "logging.StreamHandler", + "stream": "ext://sys.stderr" + }, + "stdout": { + "formatter": "json", + "class": "logging.StreamHandler", + "stream": "ext://sys.stdout" + } + }, + "loggers": { + "uvicorn": {"handlers": ["stderr"], "level": "INFO", "propagate": false}, + "uvicorn.error": {"level": "INFO"}, + "uvicorn.access": {"handlers": ["stdout"], "level": "INFO", "propagate": false} + } +} diff --git a/run-uvicorn.example b/run-uvicorn.example new file mode 100644 index 0000000..a203da5 --- /dev/null +++ b/run-uvicorn.example @@ -0,0 +1,55 @@ +/opt/flohmarkt/venv/bin/uvicorn --host 127.0.0.1 --port 8000 --reload --log-config logger.json flohmarkt.web:start + +Fehler: + +Traceback (most recent call last): + File "/usr/lib/python3.9/logging/config.py", line 385, in resolve + found = self.importer(used) +ModuleNotFoundError: No module named 'pythonjsonlogger' + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/usr/lib/python3.9/logging/config.py", line 543, in configure + formatters[name] = self.configure_formatter( + File "/usr/lib/python3.9/logging/config.py", line 655, in configure_formatter + result = self.configure_custom(config) + File "/usr/lib/python3.9/logging/config.py", line 470, in configure_custom + c = self.resolve(c) + File "/usr/lib/python3.9/logging/config.py", line 398, in resolve + raise v + File "/usr/lib/python3.9/logging/config.py", line 385, in resolve + found = self.importer(used) +ValueError: Cannot resolve 'pythonjsonlogger.jsonlogger.JsonFormatter': No module named 'pythonjsonlogger' + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/opt/flohmarkt/venv/bin/uvicorn", line 8, in + sys.exit(main()) + File "/opt/flohmarkt/venv/lib/python3.9/site-packages/click/core.py", line 1157, in __call__ + return self.main(*args, **kwargs) + File "/opt/flohmarkt/venv/lib/python3.9/site-packages/click/core.py", line 1078, in main + rv = self.invoke(ctx) + File "/opt/flohmarkt/venv/lib/python3.9/site-packages/click/core.py", line 1434, in invoke + return ctx.invoke(self.callback, **ctx.params) + File "/opt/flohmarkt/venv/lib/python3.9/site-packages/click/core.py", line 783, in invoke + return __callback(*args, **kwargs) + File "/opt/flohmarkt/venv/lib/python3.9/site-packages/uvicorn/main.py", line 404, in main + run( + File "/opt/flohmarkt/venv/lib/python3.9/site-packages/uvicorn/main.py", line 506, in run + config = Config( + File "/opt/flohmarkt/venv/lib/python3.9/site-packages/uvicorn/config.py", line 299, in __init__ + self.configure_logging() + File "/opt/flohmarkt/venv/lib/python3.9/site-packages/uvicorn/config.py", line 411, in configure_logging + logging.config.dictConfig(loaded_config) + File "/usr/lib/python3.9/logging/config.py", line 809, in dictConfig + dictConfigClass(config).configure() + File "/usr/lib/python3.9/logging/config.py", line 546, in configure + raise ValueError('Unable to configure ' +ValueError: Unable to configure formatter 'json' + +(venv) root@yt:/opt/flohmarkt/flohmarkt# pip install python-json-logger +Requirement already satisfied: python-json-logger in /usr/lib/python3.9/site-packages (2.0.7) + +Ausgabe in json will ich eigentlich auch nicht falls das in der config steht. Der Kram ist aber so dokumentiert, dass ich ihn nicht verstehe... From b271454e8d9c963dfbdd3268c793ef46428ae55c Mon Sep 17 00:00:00 2001 From: Chris Vogel Date: Sat, 27 Apr 2024 13:36:14 +0200 Subject: [PATCH 4/6] extend logging * make systemd.service log a timestamp that can be read by fail2ban * add dependency on moreutils to manifest.toml (needed in systemd.service) --- conf/systemd.service | 2 +- manifest.toml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/conf/systemd.service b/conf/systemd.service index c5318c2..96467e7 100644 --- a/conf/systemd.service +++ b/conf/systemd.service @@ -8,7 +8,7 @@ User=__APP__ Group=__APP__ WorkingDirectory=__INSTALL_DIR__/__APP__ Environment="VENV_DIR=__INSTALL_DIR__/venv/" -ExecStart=__INSTALL_DIR__/venv/bin/uvicorn --host 127.0.0.1 --port __PORT__ --reload flohmarkt.web:start +ExecStart=/bin/bash -c "/opt/flohmarkt/venv/bin/uvicorn --host 127.0.0.1 --port 8000 --reload flohmarkt.web:start 2>&1 | /usr/bin/ts '%%Y-%%m-%%d %%H:%%M:%%S'" StandardOutput=append:/var/log/__APP__/__APP__.log StandardError=inherit diff --git a/manifest.toml b/manifest.toml index 62f6ec1..592c9b2 100644 --- a/manifest.toml +++ b/manifest.toml @@ -156,7 +156,8 @@ ram.runtime = "100M" [resources.apt] # python dependencies shall be installed in a venv using pip. - packages = "python3-pip python3-full curl apt-transport-https gnupg" + # moreutils is needed for `ts` used in systemd.service + packages = "python3-pip python3-full curl apt-transport-https gnupg moreutils" # repo for couchdb - doesn't work, yet # extras.couchdb.repo = "deb https://apache.jfrog.io/artifactory/couchdb-deb/ __YNH_DEBIAN_VERSION__ main" # extras.couchdb.key = "https://couchdb.apache.org/repo/keys.asc" From 4ce465d31a40af1909402cce2ef7dc3628443223 Mon Sep 17 00:00:00 2001 From: Chris Vogel Date: Sat, 27 Apr 2024 14:04:21 +0200 Subject: [PATCH 5/6] replaced " by ' because content itself contained " --- scripts/install | 5 ++--- scripts/upgrade | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/scripts/install b/scripts/install index 94505c1..7e79ec3 100755 --- a/scripts/install +++ b/scripts/install @@ -122,8 +122,7 @@ chmod 640 "/var/log/$app/$app.log" ynh_systemd_action --service_name=$app --action="start" # SETUP FAIL2BAN -# @@ todo -# ynh_script_progression --message="Configuring Fail2Ban..." --weight=3 -# ynh_add_fail2ban_config --logpath="/var/log/$app/$app.log" --failregex=":" --max_retry=5 +ynh_script_progression --message="Configuring Fail2Ban..." --weight=3 +ynh_add_fail2ban_config --logpath="/var/log/$app/$app.log" --failregex='INFO: +:\d+ - "POST /token HTTP/\d+\.\d+" 403 Forbidden' --max_retry=5 ynh_script_progression --message="Installation of $app completed" --last diff --git a/scripts/upgrade b/scripts/upgrade index 33362ef..e36f6c7 100644 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -93,9 +93,8 @@ ynh_script_progression --message="Upgrading NGINX web server configuration..." - ynh_add_nginx_config # UPGRADE FAIL2BAN -# ynh_script_progression --message="Reconfiguring Fail2Ban..." --weight=3 -# Create a dedicated Fail2Ban config -# ynh_add_fail2ban_config --logpath="/var/log/couchdb/couchdb.log" --failregex="[warning] .*couch_httpd_auth: Authentication failed for user .+ from " --max_retry=5 +ynh_script_progression --message="Reconfiguring Fail2Ban..." --weight=3 +ynh_add_fail2ban_config --logpath="/var/log/$app/$app.log" --failregex='INFO: +:\d+ - "POST /token HTTP/\d+\.\d+" 403 Forbidden' --max_retry=5 # start flohmarkt ynh_script_progression --message="Starting flohmarkt..." --weight=3 From 955db35ef46c7d3a75a088c1b94ff4c50627af35 Mon Sep 17 00:00:00 2001 From: Chris Vogel Date: Sat, 27 Apr 2024 18:26:41 +0200 Subject: [PATCH 6/6] ynh4 fail2ban configured and tested --- manifest.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest.toml b/manifest.toml index 592c9b2..3e76fa7 100644 --- a/manifest.toml +++ b/manifest.toml @@ -6,7 +6,7 @@ id = "flohmarkt" name = "flohmarkt" description.en = "A decentral federated small ads platform" -version = "0.0~ynh3" +version = "0.0~ynh4" maintainers = ["Chris Vogel"]