Fix blind spot where Caddy 502 errors were invisible: the dashboards and alerts queried caddy_http_response_duration_seconds (upstream responses only), missing 502s that Caddy generates itself. Switch to caddy_http_request_duration_seconds (server-level, all responses). Add Journal Grafana dashboard with: 502 rate, response codes, request rate by route, latency percentiles, container restarts/memory/CPU, Node.js event loop lag and heap, and Loki log panels for errors and Caddy 5xx entries. Add color coding to Caddy status code panel (green=2xx, blue=3xx, yellow=4xx, red=5xx). Add log-based error rate panel to the overview dashboard. New alerts: container restart loop, PostgreSQL connections > 80, application crash log detection (Loki), and Caddy 502 rate. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
181 lines
4.1 KiB
JSON
181 lines
4.1 KiB
JSON
{
|
|
"title": "trails.cool Overview",
|
|
"uid": "trails-overview",
|
|
"annotations": {
|
|
"list": [
|
|
{
|
|
"name": "Deploys",
|
|
"enable": true,
|
|
"datasource": {
|
|
"type": "grafana",
|
|
"uid": "-- Grafana --"
|
|
},
|
|
"iconColor": "rgba(0, 211, 255, 1)",
|
|
"target": {
|
|
"limit": 100,
|
|
"matchAny": false,
|
|
"tags": [
|
|
"deploy"
|
|
],
|
|
"type": "tags"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"timezone": "browser",
|
|
"refresh": "30s",
|
|
"panels": [
|
|
{
|
|
"title": "Request Rate",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(http_request_duration_seconds_count[5m])) by (job)",
|
|
"legendFormat": "{{job}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"title": "Error Rate",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(caddy_http_request_duration_seconds_count{code=~\"5..\"}[5m])) / sum(rate(caddy_http_request_duration_seconds_count[5m])) * 100",
|
|
"legendFormat": "Error %"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"thresholds": {
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 5
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"title": "Latency p50 / p95 / p99",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 8
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))",
|
|
"legendFormat": "p50"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))",
|
|
"legendFormat": "p95"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))",
|
|
"legendFormat": "p99"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"title": "Health Status",
|
|
"type": "stat",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 8
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "up",
|
|
"legendFormat": "{{job}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"mappings": [
|
|
{
|
|
"type": "value",
|
|
"options": {
|
|
"0": {
|
|
"text": "DOWN",
|
|
"color": "red"
|
|
},
|
|
"1": {
|
|
"text": "UP",
|
|
"color": "green"
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
{
|
|
"title": "App Error Rate (from logs)",
|
|
"description": "Error-level log entries per second from Pino (level 50=error, 60=fatal). Captures crashes and unhandled errors that metrics may miss.",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 16
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum(rate({service=\"journal\"} |~ \"\\\"level\\\":(50|60)\" [5m])) or vector(0)",
|
|
"legendFormat": "journal"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum(rate({service=\"planner\"} |~ \"\\\"level\\\":(50|60)\" [5m])) or vector(0)",
|
|
"legendFormat": "planner"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 15
|
|
},
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|