37fb5fa1c9
Fix blind spot where Caddy 502 errors were invisible: the dashboards and alerts queried caddy_http_response_duration_seconds (upstream responses only), missing 502s that Caddy generates itself. Switch to caddy_http_request_duration_seconds (server-level, all responses). Add Journal Grafana dashboard with: 502 rate, response codes, request rate by route, latency percentiles, container restarts/memory/CPU, Node.js event loop lag and heap, and Loki log panels for errors and Caddy 5xx entries. Add color coding to Caddy status code panel (green=2xx, blue=3xx, yellow=4xx, red=5xx). Add log-based error rate panel to the overview dashboard. New alerts: container restart loop, PostgreSQL connections > 80, application crash log detection (Loki), and Caddy 502 rate. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
312 lines
7.5 KiB
JSON
312 lines
7.5 KiB
JSON
{
|
|
"title": "Journal",
|
|
"uid": "trails-journal",
|
|
"annotations": {
|
|
"list": [
|
|
{
|
|
"name": "Deploys",
|
|
"enable": true,
|
|
"datasource": {
|
|
"type": "grafana",
|
|
"uid": "-- Grafana --"
|
|
},
|
|
"iconColor": "rgba(0, 211, 255, 1)",
|
|
"target": {
|
|
"limit": 100,
|
|
"matchAny": false,
|
|
"tags": [
|
|
"deploy"
|
|
],
|
|
"type": "tags"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"timezone": "browser",
|
|
"refresh": "30s",
|
|
"panels": [
|
|
{
|
|
"title": "Caddy 502 Rate (Journal)",
|
|
"description": "502 errors returned by Caddy for the journal upstream. These indicate Caddy could not reach the journal container.",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(caddy_http_request_duration_seconds_count{server=~\".*trails.cool.*\", code=\"502\"}[5m])) or vector(0)",
|
|
"legendFormat": "502/s"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 20,
|
|
"lineWidth": 2
|
|
},
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "red"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"title": "Caddy Response Codes (Journal)",
|
|
"description": "All HTTP response codes from Caddy for the journal host, including Caddy-generated errors like 502.",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(caddy_http_request_duration_seconds_count{server=~\".*trails.cool.*\", code!~\"(101|2..|3..)\"}[5m])) by (code)",
|
|
"legendFormat": "{{code}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"title": "Journal Request Rate by Status",
|
|
"description": "Request rate from the journal app's own metrics, grouped by HTTP status code.",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 8
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(http_request_duration_seconds_count{job=\"journal\"}[5m])) by (status)",
|
|
"legendFormat": "{{status}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"title": "Journal Latency p50 / p95 / p99",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 8
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))",
|
|
"legendFormat": "p50"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))",
|
|
"legendFormat": "p95"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))",
|
|
"legendFormat": "p99"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"title": "Request Rate by Route",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 16
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(http_request_duration_seconds_count{job=\"journal\"}[5m])) by (route)",
|
|
"legendFormat": "{{route}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"title": "Container Restarts",
|
|
"description": "Journal container restart count. Spikes here correlate with 502 errors — Caddy returns 502 while the container is restarting.",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 24
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "changes(container_start_time_seconds{name=\"trails-cool-journal-1\"}[5m])",
|
|
"legendFormat": "journal"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 30,
|
|
"drawStyle": "bars"
|
|
},
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "orange"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"title": "Container Memory",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 24
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "container_memory_usage_bytes{name=\"trails-cool-journal-1\"}",
|
|
"legendFormat": "used"
|
|
},
|
|
{
|
|
"expr": "container_memory_working_set_bytes{name=\"trails-cool-journal-1\"}",
|
|
"legendFormat": "working set"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"title": "Container CPU",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 24
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "rate(container_cpu_usage_seconds_total{name=\"trails-cool-journal-1\"}[5m]) * 100",
|
|
"legendFormat": "CPU %"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"title": "Node.js Event Loop Lag",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 32
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "nodejs_eventloop_lag_seconds{job=\"journal\"}",
|
|
"legendFormat": "lag"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"title": "Node.js Heap Used",
|
|
"type": "timeseries",
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 32
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "nodejs_heap_size_used_bytes{job=\"journal\"}",
|
|
"legendFormat": "used"
|
|
},
|
|
{
|
|
"expr": "nodejs_heap_size_total_bytes{job=\"journal\"}",
|
|
"legendFormat": "total"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"title": "Journal Logs (Errors & Warnings)",
|
|
"description": "Recent error and warning logs from the journal container. Requires Promtail docker_sd_configs with Pino JSON parsing.",
|
|
"type": "logs",
|
|
"gridPos": {
|
|
"h": 10,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 40
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{service=\"journal\"} |~ \"(error|warn|ERR|WARN|level.*(40|50|60))\"",
|
|
"legendFormat": ""
|
|
}
|
|
],
|
|
"options": {
|
|
"showTime": true,
|
|
"sortOrder": "Descending",
|
|
"enableLogDetails": true,
|
|
"dedupStrategy": "none"
|
|
}
|
|
},
|
|
{
|
|
"title": "Caddy Logs (5xx Responses)",
|
|
"description": "Caddy access log entries with 5xx status codes. Shows the raw request that triggered the error.",
|
|
"type": "logs",
|
|
"gridPos": {
|
|
"h": 10,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 50
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{service=\"caddy\"} |~ \"\\\"status\\\":\\s*5\\d\\d\"",
|
|
"legendFormat": ""
|
|
}
|
|
],
|
|
"options": {
|
|
"showTime": true,
|
|
"sortOrder": "Descending",
|
|
"enableLogDetails": true,
|
|
"dedupStrategy": "none"
|
|
}
|
|
}
|
|
]
|
|
}
|