49aadd04a9
Health endpoints: - /api/health (Journal) and /health (Planner) with DB connectivity check - Docker healthchecks updated to use app health endpoints Structured logging: - Pino with JSON output in production, pretty-print in dev - Request logging middleware in Planner (method, path, status, duration) - Replaced console.log/error with structured logger in email and auth flows Prometheus metrics: - prom-client with default Node.js metrics + custom histograms/gauges - /metrics endpoints on both apps - http_request_duration, planner_active_sessions, brouter_request_duration Monitoring stack: - Prometheus, Loki, Grafana containers in docker-compose - Grafana provisioned with datasources, dashboards, and alert rules - Caddy access logging (JSON to stdout for Loki) - grafana.trails.cool with basic auth via Caddy Dashboards and alerting: - Overview: request rate, error rate, latency p50/p95/p99 - Planner: active sessions, connected clients, BRouter latency - Infrastructure: memory, CPU, event loop lag - Alerts: disk >80%, app down 2min, error rate >5% Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
89 lines
2.6 KiB
YAML
89 lines
2.6 KiB
YAML
apiVersion: 1
|
|
|
|
groups:
|
|
- orgId: 1
|
|
name: trails.cool alerts
|
|
folder: trails.cool
|
|
interval: 1m
|
|
rules:
|
|
- uid: disk-usage-high
|
|
title: Disk usage > 80%
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 600, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
- refId: C
|
|
relativeTimeRange: { from: 600, to: 0 }
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
conditions:
|
|
- evaluator: { params: [80], type: gt }
|
|
reducer: { type: last }
|
|
for: 15m
|
|
annotations:
|
|
summary: "Disk usage is above 80%"
|
|
|
|
- uid: app-health-failing
|
|
title: App health check failing
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: up{job=~"journal|planner"} == 0
|
|
intervalMs: 15000
|
|
maxDataPoints: 43200
|
|
- refId: C
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
conditions:
|
|
- evaluator: { params: [0], type: gt }
|
|
reducer: { type: last }
|
|
for: 2m
|
|
annotations:
|
|
summary: "{{ $labels.job }} is down"
|
|
|
|
- uid: error-rate-high
|
|
title: Error rate > 5%
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 600, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: sum(rate(http_request_duration_seconds_count{status=~"5.."}[5m])) / sum(rate(http_request_duration_seconds_count[5m])) * 100
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
- refId: C
|
|
relativeTimeRange: { from: 600, to: 0 }
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
conditions:
|
|
- evaluator: { params: [5], type: gt }
|
|
reducer: { type: last }
|
|
for: 5m
|
|
annotations:
|
|
summary: "Error rate is above 5%"
|
|
|
|
contactPoints:
|
|
- orgId: 1
|
|
name: email
|
|
receivers:
|
|
- uid: email-default
|
|
type: email
|
|
settings:
|
|
addresses: ${ALERT_EMAIL:-admin@trails.cool}
|
|
|
|
policies:
|
|
- orgId: 1
|
|
receiver: email
|