From 98341e36037817afba02a2ec1119505fbdd557fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ullrich=20Sch=C3=A4fer?= Date: Fri, 24 Apr 2026 19:07:04 +0200 Subject: [PATCH] Provision Grafana Pushover contact point The notification policy and contact points live in `infrastructure/grafana/provisioning/alerting/alerts.yml`, so UI-created contact points can't be attached to provisioned alerts. Provision the Pushover integration instead. Rather than add a second contact point and do the fan-out in the notification tree (Grafana's tree only fires one receiver per match without duplication workarounds), put both email + pushover under a single `default` contact point. Every alert now fans out to both. Secrets go to SOPS `secrets.infra.env`. The Grafana container reads `PUSHOVER_API_TOKEN` and `PUSHOVER_USER_KEY_ULLRICH` from env; the provisioning YAML references them via `$VAR` and Grafana substitutes at startup, so no secret lands in git. Priority 1 on firing (bypass quiet hours), 0 on resolve. User can tweak in the YAML later. Post-deploy: delete the UI-created "Pushover Ullrich" contact point in Grafana to avoid a duplicate. It's unused (no policy references it) but shows up in the contact-point list. Co-Authored-By: Claude Opus 4.7 (1M context) --- infrastructure/docker-compose.yml | 5 +++++ .../grafana/provisioning/alerting/alerts.yml | 18 ++++++++++++++++-- infrastructure/secrets.infra.env | 8 +++++--- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/infrastructure/docker-compose.yml b/infrastructure/docker-compose.yml index 7937332..2d844e0 100644 --- a/infrastructure/docker-compose.yml +++ b/infrastructure/docker-compose.yml @@ -201,6 +201,11 @@ services: GF_SMTP_FROM_ADDRESS: noreply@trails.cool GF_SMTP_FROM_NAME: trails.cool Grafana GRAFANA_DB_PASSWORD: ${GRAFANA_DB_PASSWORD:-} + # Pushover app + user keys consumed by alerts.yml provisioning. + # App token identifies the trails.cool Pushover application; per-user + # keys receive the notifications. Keys (un)set in SOPS secrets.infra.env. + PUSHOVER_API_TOKEN: ${PUSHOVER_API_TOKEN:-} + PUSHOVER_USER_KEY_ULLRICH: ${PUSHOVER_USER_KEY_ULLRICH:-} volumes: - ./grafana/provisioning:/etc/grafana/provisioning:ro - ./grafana/dashboards:/var/lib/grafana/dashboards:ro diff --git a/infrastructure/grafana/provisioning/alerting/alerts.yml b/infrastructure/grafana/provisioning/alerting/alerts.yml index 0ec3fe1..aaae096 100644 --- a/infrastructure/grafana/provisioning/alerting/alerts.yml +++ b/infrastructure/grafana/provisioning/alerting/alerts.yml @@ -231,14 +231,28 @@ groups: summary: "Caddy is returning 502 errors — journal or planner upstream unreachable" contactPoints: + # Single "default" contact point with multiple integrations: every alert + # fan-outs to both email and Pushover. Grafana resolves $VAR / ${VAR} + # tokens from the container's env at startup, so the secrets never land + # in the provisioning YAML or the git history. See SOPS + # `secrets.infra.env` for PUSHOVER_*. - orgId: 1 - name: email + name: default receivers: - uid: email-default type: email settings: addresses: admin@trails.cool + - uid: pushover-ullrich + type: pushover + settings: + apiToken: $PUSHOVER_API_TOKEN + userKey: $PUSHOVER_USER_KEY_ULLRICH + # High-priority push (bypass quiet hours) on alert; default + # priority on resolve. Tweak if this gets noisy. + priority: "1" + okPriority: "0" policies: - orgId: 1 - receiver: email + receiver: default diff --git a/infrastructure/secrets.infra.env b/infrastructure/secrets.infra.env index 62916c8..d0ece01 100644 --- a/infrastructure/secrets.infra.env +++ b/infrastructure/secrets.infra.env @@ -7,9 +7,11 @@ GRAFANA_SERVICE_TOKEN=ENC[AES256_GCM,data:en8+/UsM/wy7Y3Ae9N0p7WgQsq+PlHdNdHeoya GF_SMTP_HOST=ENC[AES256_GCM,data:rfGovWBsyxZsQ6sSv20/Sui0Pcww,iv:t4dlQXafGVYcE62udWGSwVcl/puz4yr52xLhczytsGw=,tag:TZvQpsdxJI4F9YPbiwOAhA==,type:str] GF_SMTP_USER=ENC[AES256_GCM,data:R80opLVhdfTz9eK+jw5Enni5,iv:4lS+qDLIYBb3Pm6e70lOOLQQzWNGasI8xU8IMjSE8Tc=,tag:VJTSK+r7rMgFZvCtFLFruw==,type:str] GF_SMTP_PASSWORD=ENC[AES256_GCM,data:GUREzRWoUpRKicl0hyRsGQ==,iv:Yv64LhLcJ/aDW+0zMcrlgFDMTxuIfSS8BZmp3TYAcvU=,tag:OEReZeEgxqbblGgUp2E6iw==,type:str] +PUSHOVER_API_TOKEN=ENC[AES256_GCM,data:Q3WyEbN1lZpwt4gK92kyLlNaCYkMUe+Fx1OHxEvD,iv:wRtn2YCAhMdBI/m4cDd1AhRp0G3eN8i870msO2TJhLc=,tag:98RMeJM3myxDZvyLbP2YKg==,type:str] +PUSHOVER_USER_KEY_ULLRICH=ENC[AES256_GCM,data:Hkc9KpZVHyJyr1MfGaG3PUiJagpkkAWbHUjEopQ0,iv:iVhIb/kpa+eo0qOBqCtyli/lEnzPrXTN+rMHyhyCq2Q=,tag:zWX99z4jIc/eNCSCykGSwQ==,type:str] sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwOFVLYi9VYlRQdWwzK0g3\nNmRwemxjeU5zYWhPRzIxbzJWc0E2cXNYN2xRClVYUlkxWnIreUhtZ044dkhBeFBk\ncW9YZi9RN1E0Qjg4cVp4T2ZPaGR4WEUKLS0tIGFxQTV2blhzSmUvT2ljeFN2blVj\na3l1NGc5dEtZYkNJZG1GYlo2YXM3OVEKd1UVotkdkOqaA23UeEJckItDV7HU72uO\nH7Nza9clHgQwXqwYFFxBBAPRiVo/aX+J+jXELNp2fA/Wt/66SjrBig==\n-----END AGE ENCRYPTED FILE-----\n sops_age__list_0__map_recipient=age1vukt8py0s2mm47fx6qh6vykckfkdqslf9w68ekytfzvc8xp7e3rqn6p64n -sops_lastmodified=2026-03-29T17:47:20Z -sops_mac=ENC[AES256_GCM,data:GywX/4bsPvDrlOXAyv18xPEA/eEuF7rrk5UUphP3+7M2SWwNEMQSQhz0YsuIxlctdVVyMmCnK9SwJtru03FHUE0X3GY48IRvQcYwz9xSXkws/znKn90KkFUpLouIEfchNHL0xxY9xQSgGVZT1/IDKQMEc1+4hyEQqBrxgy8nDUk=,iv:IBEAYarsCAs10o8bipLvpeI9knHknytkd+5Kj24uFTY=,tag:Y2RAhx/ceGPnxKNsWjRDDw==,type:str] +sops_lastmodified=2026-04-24T17:04:54Z +sops_mac=ENC[AES256_GCM,data:zlgyPMPgTw8w6kWjWhbsSy9BmhydUDz9PPUO/X1hx/ABczG5gwcoMQ14Dv6z/FT+uFlNxauWlRkOIaOujlaDLd/zqOAodhG4gqELPsq0swVFxVZbdvY2R8EL4eOYPq7vKBUFrXm2M8NS9Su43tXkZcbm3g7bRNTz4Zm0NQjobCw=,iv:wSY8/Advd7gJ8QnXsyk1YWCjAtRQ7m1BRxGe0LDtAmM=,tag:yfFD0HO9iRU10TubqRFoVg==,type:str] sops_unencrypted_suffix=_unencrypted -sops_version=3.9.4 +sops_version=3.12.2