5c4b6fd9af
The journal/planner deploy in cd-apps.yml does `docker compose up -d journal planner`, which stops the old container and starts the new one — Caddy keeps forwarding requests during the ~10–30s gap and returns 502s. The caddy-502-rate alert (threshold > 0 for 2m) correctly trips, every time. Two production changes plus a long-broken workflow detail: - infrastructure/Caddyfile — add `lb_try_duration 30s` / `lb_try_interval 250ms` to the journal and planner reverse_proxy blocks. Caddy now holds and retries the upstream for up to 30s during a restart instead of 502'ing immediately. Real outages (upstream unreachable longer than 30s) still 502 and the alert still fires for those. - infrastructure/grafana/provisioning/alerting/alerts.yml — add a comment documenting why caddy-502-rate stays at threshold > 0: with lb_try_duration in front of it, the alert no longer conflates "deploy in flight" with "real outage." - .github/workflows/cd-apps.yml — fix a long-silent bug: the Grafana deploy-annotation step was reading GRAFANA_SERVICE_TOKEN from `.env`, but the secrets file we scp to /opt/trails-cool is named `app.env`. The token check failed silently and the curl was being skipped on every deploy. Switching to `app.env` so deploys actually annotate Grafana. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
264 lines
8.7 KiB
YAML
264 lines
8.7 KiB
YAML
apiVersion: 1
|
|
|
|
groups:
|
|
- orgId: 1
|
|
name: trails.cool alerts
|
|
folder: trails.cool
|
|
interval: 1m
|
|
rules:
|
|
- uid: disk-usage-high
|
|
title: Disk usage > 80%
|
|
condition: B
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 600, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [80], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 15m
|
|
annotations:
|
|
summary: "Disk usage is above 80%"
|
|
|
|
- uid: app-health-failing
|
|
title: App health check failing
|
|
condition: B
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: up{job=~"journal|planner"}
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [1], type: lt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 2m
|
|
annotations:
|
|
summary: "{{ $labels.job }} is down"
|
|
|
|
- uid: error-rate-high
|
|
title: Error rate > 5%
|
|
condition: B
|
|
noDataState: OK
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 600, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: (sum(rate(caddy_http_request_duration_seconds_count{code=~"5.."}[5m])) or vector(0)) / clamp_min(sum(rate(caddy_http_request_duration_seconds_count[5m])), 0.001) * 100
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [5], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 5m
|
|
annotations:
|
|
summary: "Error rate is above 5%"
|
|
|
|
- uid: container-restart-loop
|
|
title: Container restart loop
|
|
condition: B
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: changes(container_start_time_seconds{name=~"trails-cool.*"}[5m])
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [2], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 1m
|
|
annotations:
|
|
summary: "{{ $labels.name }} has restarted more than 2 times in 5 minutes"
|
|
|
|
- uid: db-connections-high
|
|
title: PostgreSQL connections > 80
|
|
condition: B
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: sum(pg_stat_activity_count{datname="trails"})
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [80], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 5m
|
|
annotations:
|
|
summary: "PostgreSQL active connections above 80 — approaching default max_connections limit"
|
|
|
|
- uid: app-crash-log
|
|
title: Application crash detected
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: loki
|
|
model:
|
|
expr: sum(count_over_time({service=~"journal|planner"} |~ "uncaughtException|unhandledRejection|ERR_|FATAL|segfault|OOMKilled"[5m]))
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: reduce
|
|
expression: A
|
|
reducer: last
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: B
|
|
conditions:
|
|
- evaluator: { params: [0], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 0s
|
|
noDataState: OK
|
|
annotations:
|
|
summary: "Crash signature detected in application logs"
|
|
|
|
- uid: background-job-failures
|
|
title: Background job failures
|
|
condition: C
|
|
noDataState: OK
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 3600, to: 0 }
|
|
datasourceUid: postgres
|
|
model:
|
|
rawSql: "SELECT count(*) AS failed FROM pgboss.job WHERE state = 'failed' AND completed_on > now() - interval '1 hour'"
|
|
format: table
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: reduce
|
|
expression: A
|
|
reducer: last
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: B
|
|
conditions:
|
|
- evaluator: { params: [0], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 0s
|
|
annotations:
|
|
summary: "Background jobs have failed in the last hour — check Grafana Service Health dashboard"
|
|
|
|
- uid: brouter-scrape-down
|
|
title: BRouter host unreachable
|
|
condition: B
|
|
noDataState: Alerting
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: up{job="brouter-cadvisor"}
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [1], type: lt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 2m
|
|
annotations:
|
|
summary: "BRouter host metrics scrape has been failing for 2+ minutes — the dedicated host, vSwitch, or cAdvisor may be down"
|
|
|
|
# The threshold here is intentionally `> 0` for 2m — *any*
|
|
# sustained 502 stream is real. Deploy-time restarts no longer
|
|
# produce 502s thanks to `lb_try_duration` on Caddy's reverse
|
|
# proxy (see `infrastructure/Caddyfile`); if 502s appear here
|
|
# it means the upstream has been unreachable for longer than
|
|
# Caddy's retry window, which is a genuine outage.
|
|
- uid: caddy-502-rate
|
|
title: Caddy 502 errors detected
|
|
condition: B
|
|
noDataState: OK
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: sum(rate(caddy_http_request_duration_seconds_count{code="502"}[5m])) or vector(0)
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [0], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 2m
|
|
annotations:
|
|
summary: "Caddy is returning 502 errors — journal or planner upstream unreachable"
|
|
|
|
contactPoints:
|
|
# Single "default" contact point with multiple integrations: every alert
|
|
# fan-outs to both email and Pushover. Grafana resolves $VAR / ${VAR}
|
|
# tokens from the container's env at startup, so the secrets never land
|
|
# in the provisioning YAML or the git history. See SOPS
|
|
# `secrets.infra.env` for PUSHOVER_*.
|
|
- orgId: 1
|
|
name: default
|
|
receivers:
|
|
- uid: email-default
|
|
type: email
|
|
settings:
|
|
addresses: admin@trails.cool
|
|
- uid: pushover-ullrich
|
|
type: pushover
|
|
settings:
|
|
apiToken: $PUSHOVER_API_TOKEN
|
|
userKey: $PUSHOVER_USER_KEY_ULLRICH
|
|
# High-priority push (bypass quiet hours) on alert; default
|
|
# priority on resolve. Tweak if this gets noisy.
|
|
priority: "1"
|
|
okPriority: "0"
|
|
|
|
policies:
|
|
- orgId: 1
|
|
receiver: default
|