Fix journal health check and improve log collection
Switch journal Docker health check from `node -e "fetch(...)"` to `curl -sf`, matching the planner. The Node.js process spawn was heavy and prone to timeout under memory pressure, causing false health check failures and 502s. Upgrade Promtail from static file scraping to Docker service discovery so logs get `service` and `container` labels. Parse Pino JSON logs to extract the `level` label, enabling Loki queries filtered by log level. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
e69f93800b
commit
1433997d5b
2 changed files with 27 additions and 10 deletions
|
|
@ -35,7 +35,7 @@ services:
|
|||
WAHOO_CLIENT_SECRET: ${WAHOO_CLIENT_SECRET:-}
|
||||
WAHOO_WEBHOOK_TOKEN: ${WAHOO_WEBHOOK_TOKEN:-}
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "node -e \"fetch('http://localhost:3000/api/health').then(r=>{process.exit(r.ok?0:1)}).catch(()=>process.exit(1))\""]
|
||||
test: ["CMD-SHELL", "curl -sf http://localhost:3000/api/health || exit 1"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
|
@ -135,7 +135,7 @@ services:
|
|||
image: grafana/promtail:latest
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /var/log:/var/log:ro
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
- /var/lib/docker/containers:/var/lib/docker/containers:ro
|
||||
- ./promtail/promtail-config.yml:/etc/promtail/config.yml:ro
|
||||
command: ["-config.file=/etc/promtail/config.yml"]
|
||||
|
|
|
|||
|
|
@ -10,13 +10,30 @@ clients:
|
|||
|
||||
scrape_configs:
|
||||
- job_name: docker
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost
|
||||
labels:
|
||||
job: docker
|
||||
__path__: /var/lib/docker/containers/*/*.log
|
||||
docker_sd_configs:
|
||||
- host: unix:///var/run/docker.sock
|
||||
refresh_interval: 10s
|
||||
relabel_configs:
|
||||
# Use the container name as a label
|
||||
- source_labels: ["__meta_docker_container_name"]
|
||||
regex: "/?(.*)"
|
||||
target_label: container
|
||||
# Use the compose service name if available
|
||||
- source_labels: ["__meta_docker_container_label_com_docker_compose_service"]
|
||||
target_label: service
|
||||
# Drop containers we don't need logs from (exporters, promtail itself)
|
||||
- source_labels: ["__meta_docker_container_label_com_docker_compose_service"]
|
||||
regex: "promtail|node-exporter|cadvisor|postgres-exporter"
|
||||
action: drop
|
||||
pipeline_stages:
|
||||
- docker: {}
|
||||
- labeldrop:
|
||||
- filename
|
||||
# Parse Pino JSON logs (journal, planner) to extract level
|
||||
- match:
|
||||
selector: '{service=~"journal|planner"}'
|
||||
stages:
|
||||
- json:
|
||||
expressions:
|
||||
level: level
|
||||
msg: msg
|
||||
- labels:
|
||||
level:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue