From 3a43784f0713fa6ebc76a9f954aeba176f22705a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ullrich=20Sch=C3=A4fer?= Date: Tue, 9 Jun 2026 12:21:04 +0200 Subject: [PATCH] feat(infra): self-monitor the observability stack + Overpass alert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the visibility that would have caught the corrupt-WAL incident, plus a direct Overpass upstream alert. - prometheus.yml: self-scrape Prometheus (localhost:9090) and Loki (loki:3100). Prometheus scraped everything except itself, so TSDB health (compaction failures, WAL corruption, head series, retention deletions) was invisible. - monitoring-health.json: new "Monitoring Health" dashboard — TSDB compaction/WAL failures, retention deletions/hour, head series, samples/s, block bytes vs size-retention limit, retention depth (oldest-sample age), Loki ingestion rate + memory chunks. - alerts.yml: prometheus-compaction-failing (any compaction failure or WAL corruption in 1h) and overpass-upstream-unhealthy (>20% upstream failure over 10m — sustained public-Overpass degradation, distinct from the symptom-level Caddy-502 alert). Validated: promtool check config, YAML parse, JSON parse. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../grafana/dashboards/monitoring-health.json | 179 ++++++++++++++++++ .../grafana/provisioning/alerting/alerts.yml | 61 ++++++ infrastructure/prometheus/prometheus.yml | 14 ++ 3 files changed, 254 insertions(+) create mode 100644 infrastructure/grafana/dashboards/monitoring-health.json diff --git a/infrastructure/grafana/dashboards/monitoring-health.json b/infrastructure/grafana/dashboards/monitoring-health.json new file mode 100644 index 0000000..dbeb2a8 --- /dev/null +++ b/infrastructure/grafana/dashboards/monitoring-health.json @@ -0,0 +1,179 @@ +{ + "title": "Monitoring Health", + "uid": "monitoring-health", + "description": "Self-health of the observability stack (Prometheus TSDB + Loki ingestion). Added after a corrupt Prometheus WAL silently collapsed metric retention to a ~2h window — these panels make that condition visible.", + "timezone": "browser", + "refresh": "30s", + "panels": [ + { + "title": "Monitoring Components Up", + "type": "stat", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "up{job=~\"prometheus|loki|cadvisor|node|caddy|postgres\"}", + "legendFormat": "{{job}}" + } + ], + "fieldConfig": { + "defaults": { + "mappings": [ + { + "type": "value", + "options": { + "0": { "text": "DOWN", "color": "red" }, + "1": { "text": "UP", "color": "green" } + } + } + ] + } + } + }, + { + "title": "Prometheus — Compaction & WAL Failures (1h)", + "description": "Any non-zero value means the TSDB is failing to persist or truncate — the exact signature of the corrupt-WAL incident. The prometheus-compaction-failing alert watches this.", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "increase(prometheus_tsdb_compactions_failed_total[1h])", + "legendFormat": "compaction failures" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "increase(prometheus_tsdb_wal_corruptions_total[1h])", + "legendFormat": "WAL corruptions" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "increase(prometheus_tsdb_wal_truncations_failed_total[1h])", + "legendFormat": "WAL truncation failures" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + } + } + } + }, + { + "title": "Prometheus — Retention Deletions / hour", + "description": "Blocks deleted by size- vs time-based retention. Sustained size-retention deletions with low data age means the size cap is too small (counts the WAL) — blocks get deleted on creation.", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "increase(prometheus_tsdb_size_retentions_total[1h])", + "legendFormat": "size-based" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "increase(prometheus_tsdb_time_retentions_total[1h])", + "legendFormat": "time-based" + } + ] + }, + { + "title": "Prometheus — Head Series (active cardinality)", + "description": "Active in-memory series. Watch this after dropping cAdvisor families / adding scrape targets.", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "prometheus_tsdb_head_series", + "legendFormat": "head series" + } + ] + }, + { + "title": "Prometheus — Samples Appended / sec", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(prometheus_tsdb_head_samples_appended_total[5m])", + "legendFormat": "samples/s" + } + ] + }, + { + "title": "Prometheus — Block Storage vs Retention Limit", + "description": "On-disk block bytes against the configured size-retention limit. If block bytes track the limit while data age stays low, the cap is too tight.", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "prometheus_tsdb_storage_blocks_bytes", + "legendFormat": "block bytes" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "prometheus_tsdb_retention_limit_bytes", + "legendFormat": "size-retention limit" + } + ], + "fieldConfig": { "defaults": { "unit": "bytes" } } + }, + { + "title": "Prometheus — Retention Depth (oldest sample age)", + "description": "How far back queryable metrics actually go. Should track ~15d once healthy; it sat near ~2h during the incident.", + "type": "stat", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "time() - prometheus_tsdb_lowest_timestamp_seconds", + "legendFormat": "oldest sample age" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "steps": [ + { "color": "red", "value": null }, + { "color": "orange", "value": 86400 }, + { "color": "green", "value": 604800 } + ] + } + } + } + }, + { + "title": "Loki — Lines Received / sec", + "description": "Log ingestion rate. A drop to zero means Promtail stopped shipping.", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(loki_distributor_lines_received_total[5m]))", + "legendFormat": "lines/s" + } + ] + }, + { + "title": "Loki — In-Memory Chunks", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(loki_ingester_memory_chunks)", + "legendFormat": "memory chunks" + } + ] + } + ] +} diff --git a/infrastructure/grafana/provisioning/alerting/alerts.yml b/infrastructure/grafana/provisioning/alerting/alerts.yml index 1ce2546..cc9f2c0 100644 --- a/infrastructure/grafana/provisioning/alerting/alerts.yml +++ b/infrastructure/grafana/provisioning/alerting/alerts.yml @@ -236,6 +236,67 @@ groups: annotations: summary: "Caddy is returning 502 errors — journal or planner upstream unreachable" + # Prometheus self-health. A corrupt WAL segment once made every + # compaction fail for days, silently collapsing metric retention to + # the ~2h head block (size-retention kept deleting fresh blocks). + # This fires on any compaction failure OR WAL corruption in the last + # hour. Depends on the `prometheus` self-scrape job. + - uid: prometheus-compaction-failing + title: Prometheus TSDB compaction failing + condition: B + noDataState: OK + data: + - refId: A + relativeTimeRange: { from: 3600, to: 0 } + datasourceUid: prometheus + model: + expr: increase(prometheus_tsdb_compactions_failed_total[1h]) + increase(prometheus_tsdb_wal_corruptions_total[1h]) + instant: true + - refId: B + datasourceUid: __expr__ + model: + type: threshold + expression: A + conditions: + - evaluator: { params: [0], type: gt } + operator: { type: and } + reducer: { type: last } + for: 10m + annotations: + summary: "Prometheus TSDB compaction or WAL is failing — metric retention is at risk. Check the WAL for a corrupt segment (this silently capped retention to ~2h before)." + + # Overpass upstream health. The public Overpass servers the Planner + # proxies to (overpass-api.de / lz4) flake — rate-limiting (429) and + # stalls. This alerts on a *sustained* upstream failure rate (>20% + # over 10m), distinct from the Caddy-502 alert which only sees the + # symptom. Brief blips (e.g. the ~30s stall on 2026-06-08) won't + # page; sustained degradation will. client-abort is excluded (that's + # the caller giving up, not the upstream failing). Per-status detail + # lives on the Planner dashboard. + - uid: overpass-upstream-unhealthy + title: Overpass upstream failure rate high + condition: B + noDataState: OK + data: + - refId: A + relativeTimeRange: { from: 600, to: 0 } + datasourceUid: prometheus + model: + expr: (sum(rate(overpass_upstream_requests_total{status!~"2..|client-abort"}[10m])) or vector(0)) / clamp_min(sum(rate(overpass_upstream_requests_total[10m])), 0.001) * 100 + instant: true + - refId: B + datasourceUid: __expr__ + model: + type: threshold + expression: A + conditions: + - evaluator: { params: [20], type: gt } + operator: { type: and } + reducer: { type: last } + for: 10m + annotations: + summary: "Overpass upstream failure rate above 20% over 10m — public Overpass servers are degraded or rate-limiting. See the Planner dashboard; weigh self-hosting Overpass." + contactPoints: # Single "default" contact point with multiple integrations: every alert # fan-outs to both email and Pushover. Grafana resolves $VAR / ${VAR} diff --git a/infrastructure/prometheus/prometheus.yml b/infrastructure/prometheus/prometheus.yml index 6785c59..c64726b 100644 --- a/infrastructure/prometheus/prometheus.yml +++ b/infrastructure/prometheus/prometheus.yml @@ -43,6 +43,20 @@ scrape_configs: static_configs: - targets: ["caddy:2019"] + # Self-monitoring. Prometheus did not scrape its own /metrics, so TSDB + # health (compaction failures, WAL corruption, head series, retention + # deletions) was invisible — which is how a corrupt WAL silently + # collapsed retention to a ~2h window for days. Scraping itself + Loki + # surfaces that on the "Monitoring Health" dashboard and the + # prometheus-compaction-failing alert. + - job_name: "prometheus" + static_configs: + - targets: ["localhost:9090"] + + - job_name: "loki" + static_configs: + - targets: ["loki:3100"] + # BRouter runs on a separate Hetzner Robot host (ullrich.is), reached # over the vSwitch at 10.0.1.10. cAdvisor there is scoped to trails- # labeled containers only — we don't collect metrics for any of the