feat(infra): self-monitor the observability stack + Overpass alert

Adds the visibility that would have caught the corrupt-WAL incident,
plus a direct Overpass upstream alert.

- prometheus.yml: self-scrape Prometheus (localhost:9090) and Loki
  (loki:3100). Prometheus scraped everything except itself, so TSDB
  health (compaction failures, WAL corruption, head series, retention
  deletions) was invisible.
- monitoring-health.json: new "Monitoring Health" dashboard — TSDB
  compaction/WAL failures, retention deletions/hour, head series,
  samples/s, block bytes vs size-retention limit, retention depth
  (oldest-sample age), Loki ingestion rate + memory chunks.
- alerts.yml: prometheus-compaction-failing (any compaction failure or
  WAL corruption in 1h) and overpass-upstream-unhealthy (>20% upstream
  failure over 10m — sustained public-Overpass degradation, distinct
  from the symptom-level Caddy-502 alert).

Validated: promtool check config, YAML parse, JSON parse.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

infrastructure/grafana/dashboards/monitoring-health.json

@@ -0,0 +1,179 @@
+{
+  "title": "Monitoring Health",
+  "uid": "monitoring-health",
+  "description": "Self-health of the observability stack (Prometheus TSDB + Loki ingestion). Added after a corrupt Prometheus WAL silently collapsed metric retention to a ~2h window — these panels make that condition visible.",
+  "timezone": "browser",
+  "refresh": "30s",
+  "panels": [
+    {
+      "title": "Monitoring Components Up",
+      "type": "stat",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "up{job=~\"prometheus|loki|cadvisor|node|caddy|postgres\"}",
+          "legendFormat": "{{job}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": { "text": "DOWN", "color": "red" },
+                "1": { "text": "UP", "color": "green" }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "title": "Prometheus — Compaction & WAL Failures (1h)",
+      "description": "Any non-zero value means the TSDB is failing to persist or truncate — the exact signature of the corrupt-WAL incident. The prometheus-compaction-failing alert watches this.",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "increase(prometheus_tsdb_compactions_failed_total[1h])",
+          "legendFormat": "compaction failures"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "increase(prometheus_tsdb_wal_corruptions_total[1h])",
+          "legendFormat": "WAL corruptions"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "increase(prometheus_tsdb_wal_truncations_failed_total[1h])",
+          "legendFormat": "WAL truncation failures"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "red", "value": 1 }
+            ]
+          }
+        }
+      }
+    },
+    {
+      "title": "Prometheus — Retention Deletions / hour",
+      "description": "Blocks deleted by size- vs time-based retention. Sustained size-retention deletions with low data age means the size cap is too small (counts the WAL) — blocks get deleted on creation.",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "increase(prometheus_tsdb_size_retentions_total[1h])",
+          "legendFormat": "size-based"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "increase(prometheus_tsdb_time_retentions_total[1h])",
+          "legendFormat": "time-based"
+        }
+      ]
+    },
+    {
+      "title": "Prometheus — Head Series (active cardinality)",
+      "description": "Active in-memory series. Watch this after dropping cAdvisor families / adding scrape targets.",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "prometheus_tsdb_head_series",
+          "legendFormat": "head series"
+        }
+      ]
+    },
+    {
+      "title": "Prometheus — Samples Appended / sec",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "rate(prometheus_tsdb_head_samples_appended_total[5m])",
+          "legendFormat": "samples/s"
+        }
+      ]
+    },
+    {
+      "title": "Prometheus — Block Storage vs Retention Limit",
+      "description": "On-disk block bytes against the configured size-retention limit. If block bytes track the limit while data age stays low, the cap is too tight.",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "prometheus_tsdb_storage_blocks_bytes",
+          "legendFormat": "block bytes"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "prometheus_tsdb_retention_limit_bytes",
+          "legendFormat": "size-retention limit"
+        }
+      ],
+      "fieldConfig": { "defaults": { "unit": "bytes" } }
+    },
+    {
+      "title": "Prometheus — Retention Depth (oldest sample age)",
+      "description": "How far back queryable metrics actually go. Should track ~15d once healthy; it sat near ~2h during the incident.",
+      "type": "stat",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "time() - prometheus_tsdb_lowest_timestamp_seconds",
+          "legendFormat": "oldest sample age"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "thresholds": {
+            "steps": [
+              { "color": "red", "value": null },
+              { "color": "orange", "value": 86400 },
+              { "color": "green", "value": 604800 }
+            ]
+          }
+        }
+      }
+    },
+    {
+      "title": "Loki — Lines Received / sec",
+      "description": "Log ingestion rate. A drop to zero means Promtail stopped shipping.",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "sum(rate(loki_distributor_lines_received_total[5m]))",
+          "legendFormat": "lines/s"
+        }
+      ]
+    },
+    {
+      "title": "Loki — In-Memory Chunks",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 },
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "sum(loki_ingester_memory_chunks)",
+          "legendFormat": "memory chunks"
+        }
+      ]
+    }
+  ]
+}

infrastructure/grafana/provisioning/alerting/alerts.yml

@@ -236,6 +236,67 @@ groups:
         annotations:
           summary: "Caddy is returning 502 errors — journal or planner upstream unreachable"
 
+      # Prometheus self-health. A corrupt WAL segment once made every
+      # compaction fail for days, silently collapsing metric retention to
+      # the ~2h head block (size-retention kept deleting fresh blocks).
+      # This fires on any compaction failure OR WAL corruption in the last
+      # hour. Depends on the `prometheus` self-scrape job.
+      - uid: prometheus-compaction-failing
+        title: Prometheus TSDB compaction failing
+        condition: B
+        noDataState: OK
+        data:
+          - refId: A
+            relativeTimeRange: { from: 3600, to: 0 }
+            datasourceUid: prometheus
+            model:
+              expr: increase(prometheus_tsdb_compactions_failed_total[1h]) + increase(prometheus_tsdb_wal_corruptions_total[1h])
+              instant: true
+          - refId: B
+            datasourceUid: __expr__
+            model:
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { params: [0], type: gt }
+                  operator: { type: and }
+                  reducer: { type: last }
+        for: 10m
+        annotations:
+          summary: "Prometheus TSDB compaction or WAL is failing — metric retention is at risk. Check the WAL for a corrupt segment (this silently capped retention to ~2h before)."
+
+      # Overpass upstream health. The public Overpass servers the Planner
+      # proxies to (overpass-api.de / lz4) flake — rate-limiting (429) and
+      # stalls. This alerts on a *sustained* upstream failure rate (>20%
+      # over 10m), distinct from the Caddy-502 alert which only sees the
+      # symptom. Brief blips (e.g. the ~30s stall on 2026-06-08) won't
+      # page; sustained degradation will. client-abort is excluded (that's
+      # the caller giving up, not the upstream failing). Per-status detail
+      # lives on the Planner dashboard.
+      - uid: overpass-upstream-unhealthy
+        title: Overpass upstream failure rate high
+        condition: B
+        noDataState: OK
+        data:
+          - refId: A
+            relativeTimeRange: { from: 600, to: 0 }
+            datasourceUid: prometheus
+            model:
+              expr: (sum(rate(overpass_upstream_requests_total{status!~"2..|client-abort"}[10m])) or vector(0)) / clamp_min(sum(rate(overpass_upstream_requests_total[10m])), 0.001) * 100
+              instant: true
+          - refId: B
+            datasourceUid: __expr__
+            model:
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { params: [20], type: gt }
+                  operator: { type: and }
+                  reducer: { type: last }
+        for: 10m
+        annotations:
+          summary: "Overpass upstream failure rate above 20% over 10m — public Overpass servers are degraded or rate-limiting. See the Planner dashboard; weigh self-hosting Overpass."
+
 contactPoints:
   # Single "default" contact point with multiple integrations: every alert
   # fan-outs to both email and Pushover. Grafana resolves $VAR / ${VAR}

infrastructure/prometheus/prometheus.yml

@@ -43,6 +43,20 @@ scrape_configs:
     static_configs:
       - targets: ["caddy:2019"]
 
+  # Self-monitoring. Prometheus did not scrape its own /metrics, so TSDB
+  # health (compaction failures, WAL corruption, head series, retention
+  # deletions) was invisible — which is how a corrupt WAL silently
+  # collapsed retention to a ~2h window for days. Scraping itself + Loki
+  # surfaces that on the "Monitoring Health" dashboard and the
+  # prometheus-compaction-failing alert.
+  - job_name: "prometheus"
+    static_configs:
+      - targets: ["localhost:9090"]
+
+  - job_name: "loki"
+    static_configs:
+      - targets: ["loki:3100"]
+
   # BRouter runs on a separate Hetzner Robot host (ullrich.is), reached
   # over the vSwitch at 10.0.1.10. cAdvisor there is scoped to trails-
   # labeled containers only — we don't collect metrics for any of the