Add Journal dashboard, fix Caddy metrics, and improve alerts
Fix blind spot where Caddy 502 errors were invisible: the dashboards and alerts queried caddy_http_response_duration_seconds (upstream responses only), missing 502s that Caddy generates itself. Switch to caddy_http_request_duration_seconds (server-level, all responses). Add Journal Grafana dashboard with: 502 rate, response codes, request rate by route, latency percentiles, container restarts/memory/CPU, Node.js event loop lag and heap, and Loki log panels for errors and Caddy 5xx entries. Add color coding to Caddy status code panel (green=2xx, blue=3xx, yellow=4xx, red=5xx). Add log-based error rate panel to the overview dashboard. New alerts: container restart loop, PostgreSQL connections > 80, application crash log detection (Loki), and Caddy 502 rate. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ullrich Schäfer
parent
1433997d5b
commit
37fb5fa1c9
5 changed files with 478 additions and 8 deletions
|
|
@ -68,12 +68,11 @@ export async function computeRoute(request: RouteRequest): Promise<EnrichedRoute
|
|||
}
|
||||
|
||||
export class BRouterError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly statusCode: number,
|
||||
) {
|
||||
readonly statusCode: number;
|
||||
constructor(message: string, statusCode: number) {
|
||||
super(message);
|
||||
this.name = "BRouterError";
|
||||
this.statusCode = statusCode;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
312
infrastructure/grafana/dashboards/journal.json
Normal file
312
infrastructure/grafana/dashboards/journal.json
Normal file
|
|
@ -0,0 +1,312 @@
|
|||
{
|
||||
"title": "Journal",
|
||||
"uid": "trails-journal",
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"name": "Deploys",
|
||||
"enable": true,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"target": {
|
||||
"limit": 100,
|
||||
"matchAny": false,
|
||||
"tags": [
|
||||
"deploy"
|
||||
],
|
||||
"type": "tags"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"timezone": "browser",
|
||||
"refresh": "30s",
|
||||
"panels": [
|
||||
{
|
||||
"title": "Caddy 502 Rate (Journal)",
|
||||
"description": "502 errors returned by Caddy for the journal upstream. These indicate Caddy could not reach the journal container.",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(caddy_http_request_duration_seconds_count{server=~\".*trails.cool.*\", code=\"502\"}[5m])) or vector(0)",
|
||||
"legendFormat": "502/s"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"fillOpacity": 20,
|
||||
"lineWidth": 2
|
||||
},
|
||||
"color": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "red"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "Caddy Response Codes (Journal)",
|
||||
"description": "All HTTP response codes from Caddy for the journal host, including Caddy-generated errors like 502.",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(caddy_http_request_duration_seconds_count{server=~\".*trails.cool.*\", code!~\"(101|2..|3..)\"}[5m])) by (code)",
|
||||
"legendFormat": "{{code}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Journal Request Rate by Status",
|
||||
"description": "Request rate from the journal app's own metrics, grouped by HTTP status code.",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(http_request_duration_seconds_count{job=\"journal\"}[5m])) by (status)",
|
||||
"legendFormat": "{{status}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Journal Latency p50 / p95 / p99",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job=\"journal\"}[5m])) by (le))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "Request Rate by Route",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(http_request_duration_seconds_count{job=\"journal\"}[5m])) by (route)",
|
||||
"legendFormat": "{{route}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Container Restarts",
|
||||
"description": "Journal container restart count. Spikes here correlate with 502 errors — Caddy returns 502 while the container is restarting.",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 24
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "changes(container_start_time_seconds{name=\"trails-cool-journal-1\"}[5m])",
|
||||
"legendFormat": "journal"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"fillOpacity": 30,
|
||||
"drawStyle": "bars"
|
||||
},
|
||||
"color": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "orange"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "Container Memory",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 24
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "container_memory_usage_bytes{name=\"trails-cool-journal-1\"}",
|
||||
"legendFormat": "used"
|
||||
},
|
||||
{
|
||||
"expr": "container_memory_working_set_bytes{name=\"trails-cool-journal-1\"}",
|
||||
"legendFormat": "working set"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "Container CPU",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 24
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(container_cpu_usage_seconds_total{name=\"trails-cool-journal-1\"}[5m]) * 100",
|
||||
"legendFormat": "CPU %"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "Node.js Event Loop Lag",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 32
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "nodejs_eventloop_lag_seconds{job=\"journal\"}",
|
||||
"legendFormat": "lag"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "Node.js Heap Used",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 32
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "nodejs_heap_size_used_bytes{job=\"journal\"}",
|
||||
"legendFormat": "used"
|
||||
},
|
||||
{
|
||||
"expr": "nodejs_heap_size_total_bytes{job=\"journal\"}",
|
||||
"legendFormat": "total"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "Journal Logs (Errors & Warnings)",
|
||||
"description": "Recent error and warning logs from the journal container. Requires Promtail docker_sd_configs with Pino JSON parsing.",
|
||||
"type": "logs",
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 40
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "{service=\"journal\"} |~ \"(error|warn|ERR|WARN|level.*(40|50|60))\"",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"sortOrder": "Descending",
|
||||
"enableLogDetails": true,
|
||||
"dedupStrategy": "none"
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "Caddy Logs (5xx Responses)",
|
||||
"description": "Caddy access log entries with 5xx status codes. Shows the raw request that triggered the error.",
|
||||
"type": "logs",
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 50
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "{service=\"caddy\"} |~ \"\\\"status\\\":\\s*5\\d\\d\"",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"sortOrder": "Descending",
|
||||
"enableLogDetails": true,
|
||||
"dedupStrategy": "none"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -52,7 +52,7 @@
|
|||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(caddy_http_response_duration_seconds_count{code=~\"5..\"}[5m])) / sum(rate(caddy_http_response_duration_seconds_count[5m])) * 100",
|
||||
"expr": "sum(rate(caddy_http_request_duration_seconds_count{code=~\"5..\"}[5m])) / sum(rate(caddy_http_request_duration_seconds_count[5m])) * 100",
|
||||
"legendFormat": "Error %"
|
||||
}
|
||||
],
|
||||
|
|
@ -138,5 +138,44 @@
|
|||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
"title": "App Error Rate (from logs)",
|
||||
"description": "Error-level log entries per second from Pino (level 50=error, 60=fatal). Captures crashes and unhandled errors that metrics may miss.",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "sum(rate({service=\"journal\"} |~ \"\\\"level\\\":(50|60)\" [5m])) or vector(0)",
|
||||
"legendFormat": "journal"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "sum(rate({service=\"planner\"} |~ \"\\\"level\\\":(50|60)\" [5m])) or vector(0)",
|
||||
"legendFormat": "planner"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"fillOpacity": 15
|
||||
},
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -418,10 +418,30 @@
|
|||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(caddy_http_response_duration_seconds_count[5m])) by (code)",
|
||||
"expr": "sum(rate(caddy_http_request_duration_seconds_count[5m])) by (code)",
|
||||
"legendFormat": "{{code}}"
|
||||
}
|
||||
]
|
||||
],
|
||||
"fieldConfig": {
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byRegexp", "options": "/^2\\d\\d$/" },
|
||||
"properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byRegexp", "options": "/^3\\d\\d$/" },
|
||||
"properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byRegexp", "options": "/^4\\d\\d$/" },
|
||||
"properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "yellow" } }]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byRegexp", "options": "/^5\\d\\d$/" },
|
||||
"properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "Caddy \u2014 Request Duration by Host",
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ groups:
|
|||
relativeTimeRange: { from: 600, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
expr: (sum(rate(http_request_duration_seconds_count{status=~"5.."}[5m])) or vector(0)) / clamp_min(sum(rate(http_request_duration_seconds_count[5m])), 0.001) * 100
|
||||
expr: (sum(rate(caddy_http_request_duration_seconds_count{code=~"5.."}[5m])) or vector(0)) / clamp_min(sum(rate(caddy_http_request_duration_seconds_count[5m])), 0.001) * 100
|
||||
instant: true
|
||||
- refId: B
|
||||
datasourceUid: __expr__
|
||||
|
|
@ -76,6 +76,106 @@ groups:
|
|||
annotations:
|
||||
summary: "Error rate is above 5%"
|
||||
|
||||
- uid: container-restart-loop
|
||||
title: Container restart loop
|
||||
condition: B
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 300, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
expr: changes(container_start_time_seconds{name=~"trails-cool.*"}[5m])
|
||||
instant: true
|
||||
- refId: B
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator: { params: [2], type: gt }
|
||||
operator: { type: and }
|
||||
reducer: { type: last }
|
||||
for: 1m
|
||||
annotations:
|
||||
summary: "{{ $labels.name }} has restarted more than 2 times in 5 minutes"
|
||||
|
||||
- uid: db-connections-high
|
||||
title: PostgreSQL connections > 80
|
||||
condition: B
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 300, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
expr: sum(pg_stat_activity_count{datname="trails"})
|
||||
instant: true
|
||||
- refId: B
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator: { params: [80], type: gt }
|
||||
operator: { type: and }
|
||||
reducer: { type: last }
|
||||
for: 5m
|
||||
annotations:
|
||||
summary: "PostgreSQL active connections above 80 — approaching default max_connections limit"
|
||||
|
||||
- uid: app-crash-log
|
||||
title: Application crash detected
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 300, to: 0 }
|
||||
datasourceUid: loki
|
||||
model:
|
||||
expr: sum(count_over_time({service=~"journal|planner"} |~ "uncaughtException|unhandledRejection|ERR_|FATAL|segfault|OOMKilled"[5m]))
|
||||
instant: true
|
||||
- refId: B
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
type: reduce
|
||||
expression: A
|
||||
reducer: last
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
type: threshold
|
||||
expression: B
|
||||
conditions:
|
||||
- evaluator: { params: [0], type: gt }
|
||||
operator: { type: and }
|
||||
reducer: { type: last }
|
||||
for: 0s
|
||||
noDataState: OK
|
||||
annotations:
|
||||
summary: "Crash signature detected in application logs"
|
||||
|
||||
- uid: caddy-502-rate
|
||||
title: Caddy 502 errors detected
|
||||
condition: B
|
||||
noDataState: OK
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 300, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
expr: sum(rate(caddy_http_request_duration_seconds_count{code="502"}[5m])) or vector(0)
|
||||
instant: true
|
||||
- refId: B
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
type: threshold
|
||||
expression: A
|
||||
conditions:
|
||||
- evaluator: { params: [0], type: gt }
|
||||
operator: { type: and }
|
||||
reducer: { type: last }
|
||||
for: 2m
|
||||
annotations:
|
||||
summary: "Caddy is returning 502 errors — journal or planner upstream unreachable"
|
||||
|
||||
contactPoints:
|
||||
- orgId: 1
|
||||
name: email
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue