c47426213b
Grafana provisioning YAML doesn't support env var substitution.
Replace ${ALERT_EMAIL} with hardcoded admin@trails.cool.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
89 lines
2.6 KiB
YAML
89 lines
2.6 KiB
YAML
apiVersion: 1
|
|
|
|
groups:
|
|
- orgId: 1
|
|
name: trails.cool alerts
|
|
folder: trails.cool
|
|
interval: 1m
|
|
rules:
|
|
- uid: disk-usage-high
|
|
title: Disk usage > 80%
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 600, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
- refId: C
|
|
relativeTimeRange: { from: 600, to: 0 }
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
conditions:
|
|
- evaluator: { params: [80], type: gt }
|
|
reducer: { type: last }
|
|
for: 15m
|
|
annotations:
|
|
summary: "Disk usage is above 80%"
|
|
|
|
- uid: app-health-failing
|
|
title: App health check failing
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: up{job=~"journal|planner"} == 0
|
|
intervalMs: 15000
|
|
maxDataPoints: 43200
|
|
- refId: C
|
|
relativeTimeRange: { from: 300, to: 0 }
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
conditions:
|
|
- evaluator: { params: [0], type: gt }
|
|
reducer: { type: last }
|
|
for: 2m
|
|
annotations:
|
|
summary: "{{ $labels.job }} is down"
|
|
|
|
- uid: error-rate-high
|
|
title: Error rate > 5%
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 600, to: 0 }
|
|
datasourceUid: prometheus
|
|
model:
|
|
expr: sum(rate(http_request_duration_seconds_count{status=~"5.."}[5m])) / sum(rate(http_request_duration_seconds_count[5m])) * 100
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
- refId: C
|
|
relativeTimeRange: { from: 600, to: 0 }
|
|
datasourceUid: __expr__
|
|
model:
|
|
type: threshold
|
|
conditions:
|
|
- evaluator: { params: [5], type: gt }
|
|
reducer: { type: last }
|
|
for: 5m
|
|
annotations:
|
|
summary: "Error rate is above 5%"
|
|
|
|
contactPoints:
|
|
- orgId: 1
|
|
name: email
|
|
receivers:
|
|
- uid: email-default
|
|
type: email
|
|
settings:
|
|
addresses: admin@trails.cool
|
|
|
|
policies:
|
|
- orgId: 1
|
|
receiver: email
|