3a43784f07
Adds the visibility that would have caught the corrupt-WAL incident, plus a direct Overpass upstream alert. - prometheus.yml: self-scrape Prometheus (localhost:9090) and Loki (loki:3100). Prometheus scraped everything except itself, so TSDB health (compaction failures, WAL corruption, head series, retention deletions) was invisible. - monitoring-health.json: new "Monitoring Health" dashboard — TSDB compaction/WAL failures, retention deletions/hour, head series, samples/s, block bytes vs size-retention limit, retention depth (oldest-sample age), Loki ingestion rate + memory chunks. - alerts.yml: prometheus-compaction-failing (any compaction failure or WAL corruption in 1h) and overpass-upstream-unhealthy (>20% upstream failure over 10m — sustained public-Overpass degradation, distinct from the symptom-level Caddy-502 alert). Validated: promtool check config, YAML parse, JSON parse. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
325 lines
12 KiB
YAML
325 lines
12 KiB
YAML
apiVersion: 1
|
|
|
|
groups:
|
|
- orgId: 1
|
|
name: trails.cool alerts
|
|
folder: trails.cool
|
|
interval: 1m
|
|
rules:
|
|
- uid: disk-usage-high
|
|
title: Disk usage > 80%
|
|
condition: B
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 600, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [80], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 15m
|
|
annotations:
|
|
summary: "Disk usage is above 80%"
|
|
|
|
- uid: app-health-failing
|
|
title: App health check failing
|
|
condition: B
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: up{job=~"journal|planner"}
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [1], type: lt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 2m
|
|
annotations:
|
|
summary: "{{ $labels.job }} is down"
|
|
|
|
- uid: error-rate-high
|
|
title: Error rate > 5%
|
|
condition: B
|
|
noDataState: OK
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 600, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: (sum(rate(caddy_http_request_duration_seconds_count{code=~"5.."}[5m])) or vector(0)) / clamp_min(sum(rate(caddy_http_request_duration_seconds_count[5m])), 0.001) * 100
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [5], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 5m
|
|
annotations:
|
|
summary: "Error rate is above 5%"
|
|
|
|
- uid: container-restart-loop
|
|
title: Container restart loop
|
|
condition: B
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: changes(container_start_time_seconds{name=~"trails-cool.*"}[5m])
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [2], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 1m
|
|
annotations:
|
|
summary: "{{ $labels.name }} has restarted more than 2 times in 5 minutes"
|
|
|
|
- uid: db-connections-high
|
|
title: PostgreSQL connections > 80
|
|
condition: B
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: sum(pg_stat_activity_count{datname="trails"})
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [80], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 5m
|
|
annotations:
|
|
summary: "PostgreSQL active connections above 80 — approaching default max_connections limit"
|
|
|
|
- uid: app-crash-log
|
|
title: Application crash detected
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: loki
|
|
model:
|
|
expr: sum(count_over_time({service=~"journal|planner"} |~ "uncaughtException|unhandledRejection|ERR_|FATAL|segfault|OOMKilled"[5m]))
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: reduce
|
|
expression: A
|
|
reducer: last
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: B
|
|
conditions:
|
|
- evaluator: { params: [0], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 0s
|
|
noDataState: OK
|
|
annotations:
|
|
summary: "Crash signature detected in application logs"
|
|
|
|
- uid: background-job-failures
|
|
title: Background job failures
|
|
condition: C
|
|
noDataState: OK
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 3600, to: 0 }
|
|
datasourceUid: postgres
|
|
model:
|
|
rawSql: "SELECT count(*) AS failed FROM pgboss.job WHERE state = 'failed' AND completed_on > now() - interval '1 hour'"
|
|
format: table
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: reduce
|
|
expression: A
|
|
reducer: last
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: B
|
|
conditions:
|
|
- evaluator: { params: [0], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 0s
|
|
annotations:
|
|
summary: "Background jobs have failed in the last hour — check Grafana Service Health dashboard"
|
|
|
|
- uid: brouter-scrape-down
|
|
title: BRouter host unreachable
|
|
condition: B
|
|
noDataState: Alerting
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: up{job="brouter-cadvisor"}
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [1], type: lt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 2m
|
|
annotations:
|
|
summary: "BRouter host metrics scrape has been failing for 2+ minutes — the dedicated host, vSwitch, or cAdvisor may be down"
|
|
|
|
# The threshold here is intentionally `> 0` for 2m — *any*
|
|
# sustained 502 stream is real. Deploy-time restarts no longer
|
|
# produce 502s thanks to `lb_try_duration` on Caddy's reverse
|
|
# proxy (see `infrastructure/Caddyfile`); if 502s appear here
|
|
# it means the upstream has been unreachable for longer than
|
|
# Caddy's retry window, which is a genuine outage.
|
|
- uid: caddy-502-rate
|
|
title: Caddy 502 errors detected
|
|
condition: B
|
|
noDataState: OK
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: sum(rate(caddy_http_request_duration_seconds_count{code="502"}[5m])) or vector(0)
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [0], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 2m
|
|
annotations:
|
|
summary: "Caddy is returning 502 errors — journal or planner upstream unreachable"
|
|
|
|
# Prometheus self-health. A corrupt WAL segment once made every
|
|
# compaction fail for days, silently collapsing metric retention to
|
|
# the ~2h head block (size-retention kept deleting fresh blocks).
|
|
# This fires on any compaction failure OR WAL corruption in the last
|
|
# hour. Depends on the `prometheus` self-scrape job.
|
|
- uid: prometheus-compaction-failing
|
|
title: Prometheus TSDB compaction failing
|
|
condition: B
|
|
noDataState: OK
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 3600, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: increase(prometheus_tsdb_compactions_failed_total[1h]) + increase(prometheus_tsdb_wal_corruptions_total[1h])
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [0], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 10m
|
|
annotations:
|
|
summary: "Prometheus TSDB compaction or WAL is failing — metric retention is at risk. Check the WAL for a corrupt segment (this silently capped retention to ~2h before)."
|
|
|
|
# Overpass upstream health. The public Overpass servers the Planner
|
|
# proxies to (overpass-api.de / lz4) flake — rate-limiting (429) and
|
|
# stalls. This alerts on a *sustained* upstream failure rate (>20%
|
|
# over 10m), distinct from the Caddy-502 alert which only sees the
|
|
# symptom. Brief blips (e.g. the ~30s stall on 2026-06-08) won't
|
|
# page; sustained degradation will. client-abort is excluded (that's
|
|
# the caller giving up, not the upstream failing). Per-status detail
|
|
# lives on the Planner dashboard.
|
|
- uid: overpass-upstream-unhealthy
|
|
title: Overpass upstream failure rate high
|
|
condition: B
|
|
noDataState: OK
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 600, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: (sum(rate(overpass_upstream_requests_total{status!~"2..|client-abort"}[10m])) or vector(0)) / clamp_min(sum(rate(overpass_upstream_requests_total[10m])), 0.001) * 100
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [20], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 10m
|
|
annotations:
|
|
summary: "Overpass upstream failure rate above 20% over 10m — public Overpass servers are degraded or rate-limiting. See the Planner dashboard; weigh self-hosting Overpass."
|
|
|
|
contactPoints:
|
|
# Single "default" contact point with multiple integrations: every alert
|
|
# fan-outs to both email and Pushover. Grafana resolves $VAR / ${VAR}
|
|
# tokens from the container's env at startup, so the secrets never land
|
|
# in the provisioning YAML or the git history. See SOPS
|
|
# `secrets.infra.env` for PUSHOVER_*.
|
|
- orgId: 1
|
|
name: default
|
|
receivers:
|
|
- uid: email-default
|
|
type: email
|
|
settings:
|
|
addresses: admin@trails.cool
|
|
- uid: pushover-ullrich
|
|
type: pushover
|
|
settings:
|
|
apiToken: $PUSHOVER_API_TOKEN
|
|
userKey: $PUSHOVER_USER_KEY_ULLRICH
|
|
# High-priority push (bypass quiet hours) on alert; default
|
|
# priority on resolve. Tweak if this gets noisy.
|
|
priority: "1"
|
|
okPriority: "0"
|
|
|
|
policies:
|
|
- orgId: 1
|
|
receiver: default
|