Add promtail

.github/workflows/updater.sh

@@ -64,11 +64,17 @@ echo "Handling asset at $asset_url"
 # Here we base the source file name upon a unique keyword in the assets url (admin vs. update)
 # Leave $src empty to ignore the asset
 case $asset_url in
+  *"promtail-linux-amd64.zip")
+    src="promtail-amd64"
+    ;;
+  *"promtail-linux-arm64.zip")
+    src="promtail-arm64"
+    ;;
   *"loki-linux-amd64.zip")
-    src="amd64"
+    src="loki-amd64"
     ;;
   *"loki-linux-arm64.zip")
-    src="arm64"
+    src="loki-arm64"
     ;;
   *)
     src=""

conf/loki.service

@@ -7,8 +7,8 @@ Type=simple
 User=__APP__
 Group=__APP__
 WorkingDirectory=__FINALPATH__/
-ExecStart=__FINALPATH__/loki-linux-__ARCH__ --config.file __FINALPATH__/local-config.yaml
+ExecStart=/bin/bash -c '__FINALPATH__/loki-linux-__ARCH__ --config.file <(/bin/bash __FINALPATH__/merge_yaml.bash /etc/__APP__/loki-default.yaml /etc/__APP__/loki.d/*.y{a,}ml)'
-StandardOutput=append:/var/log/__APP__/__APP__.log
+StandardOutput=append:/var/log/__APP__/loki.log
 StandardError=inherit
 
 # Sandboxing options to harden security

conf/promtail-amd64.src

@@ -0,0 +1,6 @@
+SOURCE_URL=https://github.com/grafana/loki/releases/download/v2.7.1/promtail-linux-amd64.zip
+SOURCE_SUM=aeccf5c2ac067210e7737ab408f1e36cd55be4dfc521a56f0cc5ee01582e71d6
+SOURCE_SUM_PRG=sha256sum
+SOURCE_FORMAT=zip
+SOURCE_IN_SUBDIR=false
+SOURCE_FILENAME=

conf/promtail-arm64.src

@@ -0,0 +1,6 @@
+SOURCE_URL=https://github.com/grafana/loki/releases/download/v2.7.1/promtail-linux-arm64.zip
+SOURCE_SUM=770283cabc274cbbd969e12d0f58c75205394ee23fd99ec262c943532d64a12e
+SOURCE_SUM_PRG=sha256sum
+SOURCE_FORMAT=zip
+SOURCE_IN_SUBDIR=false
+SOURCE_FILENAME=

conf/promtail-default.yaml

@@ -0,0 +1,11 @@
+server:
+  http_listen_port: __PROMTAIL_PORT__
+
+clients:
+  - url: http://localhost:__HTTP_PORT__/loki/api/v1/push
+
+positions:
+  filename: /tmp/promtail_positions.yaml
+
+target_config:
+   sync_period: 10s

conf/promtail.service

@@ -0,0 +1,49 @@
+[Unit]
+Description=Promtail daemon
+After=network.target
+
+[Service]
+Type=simple
+User=__APP__
+Group=__APP__
+WorkingDirectory=__FINALPATH__/
+ExecStart=/bin/bash -c '__FINALPATH__/promtail-linux-__ARCH__ --config.file <(/bin/bash __FINALPATH__/merge_yaml.bash /etc/__APP__/promtail-default.yaml /etc/__APP__/promtail.d/*.y{a,}ml)'
+StandardOutput=append:/var/log/__APP__/promtail.log
+StandardError=inherit
+
+# Sandboxing options to harden security
+# Depending on specificities of your service/app, you may need to tweak these
+# .. but this should be a good baseline
+# Details for these options: https://www.freedesktop.org/software/systemd/man/systemd.exec.html
+NoNewPrivileges=yes
+PrivateTmp=yes
+PrivateDevices=yes
+RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
+RestrictNamespaces=yes
+RestrictRealtime=yes
+DevicePolicy=closed
+ProtectClock=yes
+ProtectHostname=yes
+ProtectProc=invisible
+ProtectSystem=full
+ProtectControlGroups=yes
+ProtectKernelModules=yes
+ProtectKernelTunables=yes
+LockPersonality=yes
+SystemCallArchitectures=native
+SystemCallFilter=~@clock @debug @module @mount @obsolete @reboot @setuid @swap @cpu-emulation @privileged
+
+# Denying access to capabilities that should not be relevant for webapps
+# Doc: https://man7.org/linux/man-pages/man7/capabilities.7.html
+CapabilityBoundingSet=~CAP_RAWIO CAP_MKNOD
+CapabilityBoundingSet=~CAP_AUDIT_CONTROL CAP_AUDIT_READ CAP_AUDIT_WRITE
+CapabilityBoundingSet=~CAP_SYS_BOOT CAP_SYS_TIME CAP_SYS_MODULE CAP_SYS_PACCT
+CapabilityBoundingSet=~CAP_LEASE CAP_LINUX_IMMUTABLE CAP_IPC_LOCK
+CapabilityBoundingSet=~CAP_BLOCK_SUSPEND CAP_WAKE_ALARM
+CapabilityBoundingSet=~CAP_SYS_TTY_CONFIG
+CapabilityBoundingSet=~CAP_MAC_ADMIN CAP_MAC_OVERRIDE
+CapabilityBoundingSet=~CAP_NET_ADMIN CAP_NET_BROADCAST CAP_NET_RAW
+CapabilityBoundingSet=~CAP_SYS_ADMIN CAP_SYS_PTRACE CAP_SYSLOG
+
+[Install]
+WantedBy=multi-user.target

manifest.json

@@ -1,5 +1,5 @@
 {
-    "name": "Loki",
+    "name": "Loki + Promtail",
     "id": "loki",
     "packaging_format": 1,
     "description": {

scripts/backup

@@ -43,6 +43,12 @@ ynh_backup --src_path="$final_path"
 
 #=================================================
 # SPECIFIC BACKUP
+#=================================================
+# BACKUP CONFIGURATION
+#=================================================
+
+ynh_backup --src_path="/etc/$app"
+
 #=================================================
 # BACKUP LOGROTATE
 #=================================================
@@ -54,6 +60,7 @@ ynh_backup --src_path="/etc/logrotate.d/$app"
 #=================================================
 
 ynh_backup --src_path="/etc/systemd/system/$app.service"
+ynh_backup --src_path="/etc/systemd/system/$app-promtail.service"
 
 #=================================================
 # END OF SCRIPT

scripts/install

@@ -44,6 +44,9 @@ ynh_app_setting_set --app=$app --key=http_port --value=$http_port
 grpc_port=$(ynh_find_port --port=9096)
 ynh_app_setting_set --app=$app --key=grpc_port --value=$grpc_port
 
+promtail_port=$(ynh_find_port --port=9080)
+ynh_app_setting_set --app=$app --key=promtail_port --value=$promtail_port
+
 #=================================================
 # INSTALL DEPENDENCIES
 #=================================================
@@ -76,21 +79,37 @@ chown -R $app:www-data "$final_path"
 #=================================================
 ynh_script_progression --message="Adding a configuration file..." --weight=1
 
-ynh_add_config --template="local-config.yaml" --destination="$final_path/local-config.yaml"
+ynh_add_config --template="loki-default.yaml" --destination="/etc/$app/loki-default.yaml"
 
 # FIXME: this should be handled by the core in the future
 # You may need to use chmod 600 instead of 400,
 # for example if the app is expected to be able to modify its own config
-chmod 400 "$final_path/local-config.yaml"
+chmod 400 "/etc/$app/loki-default.yaml"
-chown $app:www-data "$final_path/local-config.yaml"
+chown $app:www-data "/etc/$app/loki-default.yaml"
+
+ynh_add_config --template="promtail-default.yaml" --destination="/etc/$app/promtail-default.yaml"
+
+# FIXME: this should be handled by the core in the future
+# You may need to use chmod 600 instead of 400,
+# for example if the app is expected to be able to modify its own config
+chmod 400 "/etc/$app/promtail-default.yaml"
+chown $app:www-data "/etc/$app/promtail-default.yaml"
+
+mkdir -p "/etc/$app/loki.d"
+chmod 700 "/etc/$app/loki.d"
+chown $app:www-data "/etc/$app/loki.d"
+
+mkdir -p "/etc/$app/promtail.d"
+chmod 700 "/etc/$app/promtail.d"
+chown $app:www-data "/etc/$app/promtail.d"
 
 #=================================================
 # SETUP SYSTEMD
 #=================================================
 ynh_script_progression --message="Configuring a systemd service..." --weight=1
 
-# Faire 2 applications différentes
+ynh_add_systemd_config --template="loki.service"
-ynh_add_systemd_config
+ynh_add_systemd_config --template="promtail.service" --service="$app-promtail"
 
 #=================================================
 # GENERIC FINALIZATION
@@ -107,7 +126,8 @@ ynh_use_logrotate
 #=================================================
 ynh_script_progression --message="Integrating service in YunoHost..." --weight=1
 
-yunohost service add $app --description="Loki daemon" --log="/var/log/$app/$app.log"
+yunohost service add $app --description="Loki daemon" --log="/var/log/$app/loki.log"
+yunohost service add $app-promtail --description="Promtail daemon" --log="/var/log/$app/promtail.log"
 
 #=================================================
 # START SYSTEMD SERVICE
@@ -115,7 +135,8 @@ yunohost service add $app --description="Loki daemon" --log="/var/log/$app/$app.
 ynh_script_progression --message="Starting a systemd service..." --weight=1
 
 # Start a systemd service
-ynh_systemd_action --service_name=$app --action="start" --log_path="/var/log/$app/$app.log"
+ynh_systemd_action --service_name=$app --action="start" --log_path="/var/log/$app/loki.log"
+ynh_systemd_action --service_name=$app-promtail --action="start" --log_path="/var/log/$app/promtail.log"
 
 #=================================================
 # END OF SCRIPT

scripts/restore

@@ -70,6 +70,14 @@ ynh_script_progression --message="Reinstalling dependencies..." --weight=1
 # Define and install dependencies
 ynh_install_app_dependencies $pkg_dependencies
 
+#=================================================
+# RESTORE CONFIGURATION
+#=================================================
+
+ynh_script_progression --message="Restoring the configuration files..." --weight=1
+
+ynh_restore_file --origin_path="/etc/$app"
+
 #=================================================
 # RESTORE SYSTEMD
 #=================================================
@@ -78,6 +86,9 @@ ynh_script_progression --message="Restoring the systemd configuration..." --weig
 ynh_restore_file --origin_path="/etc/systemd/system/$app.service"
 systemctl enable $app.service --quiet
 
+ynh_restore_file --origin_path="/etc/systemd/system/$app-promtail.service"
+systemctl enable $app-promtail.service --quiet
+
 #=================================================
 # RESTORE THE LOGROTATE CONFIGURATION
 #=================================================
@@ -90,14 +101,16 @@ ynh_restore_file --origin_path="/etc/logrotate.d/$app"
 #=================================================
 ynh_script_progression --message="Integrating service in YunoHost..." --weight=1
 
-yunohost service add $app --description="Loki daemon" --log="/var/log/$app/$app.log"
+yunohost service add $app --description="Loki daemon" --log="/var/log/$app/loki.log"
+yunohost service add $app-promtail --description="Promtail daemon" --log="/var/log/$app/promtail.log"
 
 #=================================================
 # START SYSTEMD SERVICE
 #=================================================
 ynh_script_progression --message="Starting a systemd service..." --weight=1
 
-ynh_systemd_action --service_name=$app --action="start" --log_path="/var/log/$app/$app.log"
+ynh_systemd_action --service_name=$app --action="start" --log_path="/var/log/$app/loki.log"
+ynh_systemd_action --service_name=$app-promtail --action="start" --log_path="/var/log/$app/promtail.log"
 
 #=================================================
 # END OF SCRIPT

scripts/upgrade

@@ -92,13 +92,29 @@ ynh_install_app_dependencies $pkg_dependencies
 #=================================================
 ynh_script_progression --message="Updating a configuration file..." --weight=1
 
-ynh_add_config --template="local-config.yaml" --destination="$final_path/local-config.yaml"
+ynh_add_config --template="loki-default.yaml" --destination="/etc/$app/loki-default.yaml"
 
 # FIXME: this should be handled by the core in the future
 # You may need to use chmod 600 instead of 400,
 # for example if the app is expected to be able to modify its own config
-chmod 400 "$final_path/local-config.yaml"
+chmod 400 "/etc/$app/loki-default.yaml"
-chown $app:www-data "$final_path/local-config.yaml"
+chown $app:www-data "/etc/$app/loki-default.yaml"
+
+ynh_add_config --template="promtail-default.yaml" --destination="/etc/$app/promtail-default.yaml"
+
+# FIXME: this should be handled by the core in the future
+# You may need to use chmod 600 instead of 400,
+# for example if the app is expected to be able to modify its own config
+chmod 400 "/etc/$app/promtail-default.yaml"
+chown $app:www-data "/etc/$app/promtail-default.yaml"
+
+mkdir -p "/etc/$app/loki.d"
+chmod 700 "/etc/$app/loki.d"
+chown $app:www-data "/etc/$app/loki.d"
+
+mkdir -p "/etc/$app/promtail.d"
+chmod 700 "/etc/$app/promtail.d"
+chown $app:www-data "/etc/$app/promtail.d"
 
 #=================================================
 # SETUP SYSTEMD
@@ -106,7 +122,8 @@ chown $app:www-data "$final_path/local-config.yaml"
 ynh_script_progression --message="Upgrading systemd configuration..." --weight=1
 
 # Create a dedicated systemd config
-ynh_add_systemd_config
+ynh_add_systemd_config --template="loki.service"
+ynh_add_systemd_config --template="promtail.service" --service="$app-promtail"
 
 #=================================================
 # GENERIC FINALIZATION
@@ -123,14 +140,16 @@ ynh_use_logrotate --non-append
 #=================================================
 ynh_script_progression --message="Integrating service in YunoHost..." --weight=1
 
-yunohost service add $app --description="Loki daemon" --log="/var/log/$app/$app.log"
+yunohost service add $app --description="Loki daemon" --log="/var/log/$app/loki.log"
+yunohost service add $app --description="Promtail daemon" --log="/var/log/$app/promtail.log"
 
 #=================================================
 # START SYSTEMD SERVICE
 #=================================================
 ynh_script_progression --message="Starting a systemd service..." --weight=1
 
-ynh_systemd_action --service_name=$app --action="start" --log_path="/var/log/$app/$app.log"
+ynh_systemd_action --service_name=$app --action="start" --log_path="/var/log/$app/loki.log"
+ynh_systemd_action --service_name=$app-promtail --action="start" --log_path="/var/log/$app/promtail.log"
 
 #=================================================
 # END OF SCRIPT

sources/extra_files/app/merge_yaml.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+{
+	for file in $@
+	do
+		/usr/bin/python3 -c "import yaml, json; print(json.dumps(yaml.safe_load(open('$file'))))"
+	done
+} \
+	| /usr/bin/jq --slurp 'reduce .[] as $item ({}; . * $item)' \ # Credits: https://stackoverflow.com/a/36218044
+	| /usr/bin/python3 -c "import yaml, json; print(yaml.dump(json.load(open('/dev/stdin'))))"