trails/infrastructure/grafana/dashboards/overview.json
Ullrich Schäfer 37fb5fa1c9 Add Journal dashboard, fix Caddy metrics, and improve alerts
Fix blind spot where Caddy 502 errors were invisible: the dashboards and
alerts queried caddy_http_response_duration_seconds (upstream responses only),
missing 502s that Caddy generates itself. Switch to
caddy_http_request_duration_seconds (server-level, all responses).

Add Journal Grafana dashboard with: 502 rate, response codes, request rate
by route, latency percentiles, container restarts/memory/CPU, Node.js event
loop lag and heap, and Loki log panels for errors and Caddy 5xx entries.

Add color coding to Caddy status code panel (green=2xx, blue=3xx, yellow=4xx,
red=5xx). Add log-based error rate panel to the overview dashboard.

New alerts: container restart loop, PostgreSQL connections > 80, application
crash log detection (Loki), and Caddy 502 rate.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-12 14:00:00 +02:00

181 lines
4.1 KiB
JSON

{
"title": "trails.cool Overview",
"uid": "trails-overview",
"annotations": {
"list": [
{
"name": "Deploys",
"enable": true,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"iconColor": "rgba(0, 211, 255, 1)",
"target": {
"limit": 100,
"matchAny": false,
"tags": [
"deploy"
],
"type": "tags"
}
}
]
},
"timezone": "browser",
"refresh": "30s",
"panels": [
{
"title": "Request Rate",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"targets": [
{
"expr": "sum(rate(http_request_duration_seconds_count[5m])) by (job)",
"legendFormat": "{{job}}"
}
]
},
{
"title": "Error Rate",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"targets": [
{
"expr": "sum(rate(caddy_http_request_duration_seconds_count{code=~\"5..\"}[5m])) / sum(rate(caddy_http_request_duration_seconds_count[5m])) * 100",
"legendFormat": "Error %"
}
],
"fieldConfig": {
"defaults": {
"unit": "percent",
"thresholds": {
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 5
}
]
}
}
}
},
{
"title": "Latency p50 / p95 / p99",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"targets": [
{
"expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))",
"legendFormat": "p50"
},
{
"expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))",
"legendFormat": "p95"
},
{
"expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))",
"legendFormat": "p99"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
}
}
},
{
"title": "Health Status",
"type": "stat",
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 8
},
"targets": [
{
"expr": "up",
"legendFormat": "{{job}}"
}
],
"fieldConfig": {
"defaults": {
"mappings": [
{
"type": "value",
"options": {
"0": {
"text": "DOWN",
"color": "red"
},
"1": {
"text": "UP",
"color": "green"
}
}
}
]
}
}
}
{
"title": "App Error Rate (from logs)",
"description": "Error-level log entries per second from Pino (level 50=error, 60=fatal). Captures crashes and unhandled errors that metrics may miss.",
"type": "timeseries",
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 16
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum(rate({service=\"journal\"} |~ \"\\\"level\\\":(50|60)\" [5m])) or vector(0)",
"legendFormat": "journal"
},
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum(rate({service=\"planner\"} |~ \"\\\"level\\\":(50|60)\" [5m])) or vector(0)",
"legendFormat": "planner"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"fillOpacity": 15
},
"color": {
"mode": "palette-classic"
}
}
}
}
]
}