98341e3603
The notification policy and contact points live in `infrastructure/grafana/provisioning/alerting/alerts.yml`, so UI-created contact points can't be attached to provisioned alerts. Provision the Pushover integration instead. Rather than add a second contact point and do the fan-out in the notification tree (Grafana's tree only fires one receiver per match without duplication workarounds), put both email + pushover under a single `default` contact point. Every alert now fans out to both. Secrets go to SOPS `secrets.infra.env`. The Grafana container reads `PUSHOVER_API_TOKEN` and `PUSHOVER_USER_KEY_ULLRICH` from env; the provisioning YAML references them via `$VAR` and Grafana substitutes at startup, so no secret lands in git. Priority 1 on firing (bypass quiet hours), 0 on resolve. User can tweak in the YAML later. Post-deploy: delete the UI-created "Pushover Ullrich" contact point in Grafana to avoid a duplicate. It's unused (no policy references it) but shows up in the contact-point list. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
258 lines
8.3 KiB
YAML
258 lines
8.3 KiB
YAML
apiVersion: 1
|
|
|
|
groups:
|
|
- orgId: 1
|
|
name: trails.cool alerts
|
|
folder: trails.cool
|
|
interval: 1m
|
|
rules:
|
|
- uid: disk-usage-high
|
|
title: Disk usage > 80%
|
|
condition: B
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 600, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [80], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 15m
|
|
annotations:
|
|
summary: "Disk usage is above 80%"
|
|
|
|
- uid: app-health-failing
|
|
title: App health check failing
|
|
condition: B
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: up{job=~"journal|planner"}
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [1], type: lt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 2m
|
|
annotations:
|
|
summary: "{{ $labels.job }} is down"
|
|
|
|
- uid: error-rate-high
|
|
title: Error rate > 5%
|
|
condition: B
|
|
noDataState: OK
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 600, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: (sum(rate(caddy_http_request_duration_seconds_count{code=~"5.."}[5m])) or vector(0)) / clamp_min(sum(rate(caddy_http_request_duration_seconds_count[5m])), 0.001) * 100
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [5], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 5m
|
|
annotations:
|
|
summary: "Error rate is above 5%"
|
|
|
|
- uid: container-restart-loop
|
|
title: Container restart loop
|
|
condition: B
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: changes(container_start_time_seconds{name=~"trails-cool.*"}[5m])
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [2], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 1m
|
|
annotations:
|
|
summary: "{{ $labels.name }} has restarted more than 2 times in 5 minutes"
|
|
|
|
- uid: db-connections-high
|
|
title: PostgreSQL connections > 80
|
|
condition: B
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: sum(pg_stat_activity_count{datname="trails"})
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [80], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 5m
|
|
annotations:
|
|
summary: "PostgreSQL active connections above 80 — approaching default max_connections limit"
|
|
|
|
- uid: app-crash-log
|
|
title: Application crash detected
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: loki
|
|
model:
|
|
expr: sum(count_over_time({service=~"journal|planner"} |~ "uncaughtException|unhandledRejection|ERR_|FATAL|segfault|OOMKilled"[5m]))
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: reduce
|
|
expression: A
|
|
reducer: last
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: B
|
|
conditions:
|
|
- evaluator: { params: [0], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 0s
|
|
noDataState: OK
|
|
annotations:
|
|
summary: "Crash signature detected in application logs"
|
|
|
|
- uid: background-job-failures
|
|
title: Background job failures
|
|
condition: C
|
|
noDataState: OK
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 3600, to: 0 }
|
|
datasourceUid: postgres
|
|
model:
|
|
rawSql: "SELECT count(*) AS failed FROM pgboss.job WHERE state = 'failed' AND completed_on > now() - interval '1 hour'"
|
|
format: table
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: reduce
|
|
expression: A
|
|
reducer: last
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: B
|
|
conditions:
|
|
- evaluator: { params: [0], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 0s
|
|
annotations:
|
|
summary: "Background jobs have failed in the last hour — check Grafana Service Health dashboard"
|
|
|
|
- uid: brouter-scrape-down
|
|
title: BRouter host unreachable
|
|
condition: B
|
|
noDataState: Alerting
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: up{job="brouter-cadvisor"}
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [1], type: lt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 2m
|
|
annotations:
|
|
summary: "BRouter host metrics scrape has been failing for 2+ minutes — the dedicated host, vSwitch, or cAdvisor may be down"
|
|
|
|
- uid: caddy-502-rate
|
|
title: Caddy 502 errors detected
|
|
condition: B
|
|
noDataState: OK
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: sum(rate(caddy_http_request_duration_seconds_count{code="502"}[5m])) or vector(0)
|
|
instant: true
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
expression: A
|
|
conditions:
|
|
- evaluator: { params: [0], type: gt }
|
|
operator: { type: and }
|
|
reducer: { type: last }
|
|
for: 2m
|
|
annotations:
|
|
summary: "Caddy is returning 502 errors — journal or planner upstream unreachable"
|
|
|
|
contactPoints:
|
|
# Single "default" contact point with multiple integrations: every alert
|
|
# fan-outs to both email and Pushover. Grafana resolves $VAR / ${VAR}
|
|
# tokens from the container's env at startup, so the secrets never land
|
|
# in the provisioning YAML or the git history. See SOPS
|
|
# `secrets.infra.env` for PUSHOVER_*.
|
|
- orgId: 1
|
|
name: default
|
|
receivers:
|
|
- uid: email-default
|
|
type: email
|
|
settings:
|
|
addresses: admin@trails.cool
|
|
- uid: pushover-ullrich
|
|
type: pushover
|
|
settings:
|
|
apiToken: $PUSHOVER_API_TOKEN
|
|
userKey: $PUSHOVER_USER_KEY_ULLRICH
|
|
# High-priority push (bypass quiet hours) on alert; default
|
|
# priority on resolve. Tweak if this gets noisy.
|
|
priority: "1"
|
|
okPriority: "0"
|
|
|
|
policies:
|
|
- orgId: 1
|
|
receiver: default
|