From 1433997d5bb1e4d33f5c85ba1af82d5151e6c464 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ullrich=20Sch=C3=A4fer?= <hi+github@ullrich.is>
Date: Sun, 12 Apr 2026 15:00:00 +0200
Subject: [PATCH] Fix journal health check and improve log collection

Switch journal Docker health check from `node -e "fetch(...)"` to `curl -sf`,
matching the planner. The Node.js process spawn was heavy and prone to timeout
under memory pressure, causing false health check failures and 502s.

Upgrade Promtail from static file scraping to Docker service discovery so logs
get `service` and `container` labels. Parse Pino JSON logs to extract the
`level` label, enabling Loki queries filtered by log level.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 infrastructure/docker-compose.yml           |  4 +--
 infrastructure/promtail/promtail-config.yml | 33 ++++++++++++++++-----
 2 files changed, 27 insertions(+), 10 deletions(-)

diff --git a/infrastructure/docker-compose.yml b/infrastructure/docker-compose.yml
index 3f63dc5..b9e8055 100644
--- a/infrastructure/docker-compose.yml
+++ b/infrastructure/docker-compose.yml
@@ -35,7 +35,7 @@ services:
       WAHOO_CLIENT_SECRET: ${WAHOO_CLIENT_SECRET:-}
       WAHOO_WEBHOOK_TOKEN: ${WAHOO_WEBHOOK_TOKEN:-}
     healthcheck:
-      test: ["CMD-SHELL", "node -e \"fetch('http://localhost:3000/api/health').then(r=>{process.exit(r.ok?0:1)}).catch(()=>process.exit(1))\""]
+      test: ["CMD-SHELL", "curl -sf http://localhost:3000/api/health || exit 1"]
       interval: 15s
       timeout: 5s
       retries: 3
@@ -135,7 +135,7 @@ services:
     image: grafana/promtail:latest
     restart: unless-stopped
     volumes:
-      - /var/log:/var/log:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
       - /var/lib/docker/containers:/var/lib/docker/containers:ro
       - ./promtail/promtail-config.yml:/etc/promtail/config.yml:ro
     command: ["-config.file=/etc/promtail/config.yml"]
diff --git a/infrastructure/promtail/promtail-config.yml b/infrastructure/promtail/promtail-config.yml
index f7414fe..248f2a5 100644
--- a/infrastructure/promtail/promtail-config.yml
+++ b/infrastructure/promtail/promtail-config.yml
@@ -10,13 +10,30 @@ clients:
 
 scrape_configs:
   - job_name: docker
-    static_configs:
-      - targets:
-          - localhost
-        labels:
-          job: docker
-          __path__: /var/lib/docker/containers/*/*.log
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 10s
+    relabel_configs:
+      # Use the container name as a label
+      - source_labels: ["__meta_docker_container_name"]
+        regex: "/?(.*)"
+        target_label: container
+      # Use the compose service name if available
+      - source_labels: ["__meta_docker_container_label_com_docker_compose_service"]
+        target_label: service
+      # Drop containers we don't need logs from (exporters, promtail itself)
+      - source_labels: ["__meta_docker_container_label_com_docker_compose_service"]
+        regex: "promtail|node-exporter|cadvisor|postgres-exporter"
+        action: drop
     pipeline_stages:
       - docker: {}
-      - labeldrop:
-          - filename
+      # Parse Pino JSON logs (journal, planner) to extract level
+      - match:
+          selector: '{service=~"journal|planner"}'
+          stages:
+            - json:
+                expressions:
+                  level: level
+                  msg: msg
+            - labels:
+                level: