Fix journal health check and improve log collection

Switch journal Docker health check from `node -e "fetch(...)"` to `curl -sf`,
matching the planner. The Node.js process spawn was heavy and prone to timeout
under memory pressure, causing false health check failures and 502s.

Upgrade Promtail from static file scraping to Docker service discovery so logs
get `service` and `container` labels. Parse Pino JSON logs to extract the
`level` label, enabling Loki queries filtered by log level.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ullrich Schäfer 2026-04-12 15:00:00 +02:00
parent e69f93800b
commit 1433997d5b
2 changed files with 27 additions and 10 deletions

View file

@ -35,7 +35,7 @@ services:
WAHOO_CLIENT_SECRET: ${WAHOO_CLIENT_SECRET:-}
WAHOO_WEBHOOK_TOKEN: ${WAHOO_WEBHOOK_TOKEN:-}
healthcheck:
test: ["CMD-SHELL", "node -e \"fetch('http://localhost:3000/api/health').then(r=>{process.exit(r.ok?0:1)}).catch(()=>process.exit(1))\""]
test: ["CMD-SHELL", "curl -sf http://localhost:3000/api/health || exit 1"]
interval: 15s
timeout: 5s
retries: 3
@ -135,7 +135,7 @@ services:
image: grafana/promtail:latest
restart: unless-stopped
volumes:
- /var/log:/var/log:ro
- /var/run/docker.sock:/var/run/docker.sock:ro
- /var/lib/docker/containers:/var/lib/docker/containers:ro
- ./promtail/promtail-config.yml:/etc/promtail/config.yml:ro
command: ["-config.file=/etc/promtail/config.yml"]

View file

@ -10,13 +10,30 @@ clients:
scrape_configs:
- job_name: docker
static_configs:
- targets:
- localhost
labels:
job: docker
__path__: /var/lib/docker/containers/*/*.log
docker_sd_configs:
- host: unix:///var/run/docker.sock
refresh_interval: 10s
relabel_configs:
# Use the container name as a label
- source_labels: ["__meta_docker_container_name"]
regex: "/?(.*)"
target_label: container
# Use the compose service name if available
- source_labels: ["__meta_docker_container_label_com_docker_compose_service"]
target_label: service
# Drop containers we don't need logs from (exporters, promtail itself)
- source_labels: ["__meta_docker_container_label_com_docker_compose_service"]
regex: "promtail|node-exporter|cadvisor|postgres-exporter"
action: drop
pipeline_stages:
- docker: {}
- labeldrop:
- filename
# Parse Pino JSON logs (journal, planner) to extract level
- match:
selector: '{service=~"journal|planner"}'
stages:
- json:
expressions:
level: level
msg: msg
- labels:
level: