Provision Grafana Pushover contact point

The notification policy and contact points live in
`infrastructure/grafana/provisioning/alerting/alerts.yml`, so
UI-created contact points can't be attached to provisioned alerts.
Provision the Pushover integration instead.

Rather than add a second contact point and do the fan-out in the
notification tree (Grafana's tree only fires one receiver per match
without duplication workarounds), put both email + pushover under a
single `default` contact point. Every alert now fans out to both.

Secrets go to SOPS `secrets.infra.env`. The Grafana container reads
`PUSHOVER_API_TOKEN` and `PUSHOVER_USER_KEY_ULLRICH` from env; the
provisioning YAML references them via `$VAR` and Grafana substitutes
at startup, so no secret lands in git.

Priority 1 on firing (bypass quiet hours), 0 on resolve. User can
tweak in the YAML later.

Post-deploy: delete the UI-created "Pushover Ullrich" contact point
in Grafana to avoid a duplicate. It's unused (no policy references it)
but shows up in the contact-point list.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

infrastructure/docker-compose.yml

@@ -201,6 +201,11 @@ services:
       GF_SMTP_FROM_ADDRESS: noreply@trails.cool
       GF_SMTP_FROM_NAME: trails.cool Grafana
       GRAFANA_DB_PASSWORD: ${GRAFANA_DB_PASSWORD:-}
+      # Pushover app + user keys consumed by alerts.yml provisioning.
+      # App token identifies the trails.cool Pushover application; per-user
+      # keys receive the notifications. Keys (un)set in SOPS secrets.infra.env.
+      PUSHOVER_API_TOKEN: ${PUSHOVER_API_TOKEN:-}
+      PUSHOVER_USER_KEY_ULLRICH: ${PUSHOVER_USER_KEY_ULLRICH:-}
     volumes:
       - ./grafana/provisioning:/etc/grafana/provisioning:ro
       - ./grafana/dashboards:/var/lib/grafana/dashboards:ro

infrastructure/grafana/provisioning/alerting/alerts.yml

@@ -231,14 +231,28 @@ groups:
           summary: "Caddy is returning 502 errors — journal or planner upstream unreachable"
 
 contactPoints:
+  # Single "default" contact point with multiple integrations: every alert
+  # fan-outs to both email and Pushover. Grafana resolves $VAR / ${VAR}
+  # tokens from the container's env at startup, so the secrets never land
+  # in the provisioning YAML or the git history. See SOPS
+  # `secrets.infra.env` for PUSHOVER_*.
   - orgId: 1
-    name: email
+    name: default
     receivers:
       - uid: email-default
         type: email
         settings:
           addresses: admin@trails.cool
+      - uid: pushover-ullrich
+        type: pushover
+        settings:
+          apiToken: $PUSHOVER_API_TOKEN
+          userKey: $PUSHOVER_USER_KEY_ULLRICH
+          # High-priority push (bypass quiet hours) on alert; default
+          # priority on resolve. Tweak if this gets noisy.
+          priority: "1"
+          okPriority: "0"
 
 policies:
   - orgId: 1
-    receiver: email
+    receiver: default

infrastructure/secrets.infra.env

@@ -7,9 +7,11 @@ GRAFANA_SERVICE_TOKEN=ENC[AES256_GCM,data:en8+/UsM/wy7Y3Ae9N0p7WgQsq+PlHdNdHeoya
 GF_SMTP_HOST=ENC[AES256_GCM,data:rfGovWBsyxZsQ6sSv20/Sui0Pcww,iv:t4dlQXafGVYcE62udWGSwVcl/puz4yr52xLhczytsGw=,tag:TZvQpsdxJI4F9YPbiwOAhA==,type:str]
 GF_SMTP_USER=ENC[AES256_GCM,data:R80opLVhdfTz9eK+jw5Enni5,iv:4lS+qDLIYBb3Pm6e70lOOLQQzWNGasI8xU8IMjSE8Tc=,tag:VJTSK+r7rMgFZvCtFLFruw==,type:str]
 GF_SMTP_PASSWORD=ENC[AES256_GCM,data:GUREzRWoUpRKicl0hyRsGQ==,iv:Yv64LhLcJ/aDW+0zMcrlgFDMTxuIfSS8BZmp3TYAcvU=,tag:OEReZeEgxqbblGgUp2E6iw==,type:str]
+PUSHOVER_API_TOKEN=ENC[AES256_GCM,data:Q3WyEbN1lZpwt4gK92kyLlNaCYkMUe+Fx1OHxEvD,iv:wRtn2YCAhMdBI/m4cDd1AhRp0G3eN8i870msO2TJhLc=,tag:98RMeJM3myxDZvyLbP2YKg==,type:str]
+PUSHOVER_USER_KEY_ULLRICH=ENC[AES256_GCM,data:Hkc9KpZVHyJyr1MfGaG3PUiJagpkkAWbHUjEopQ0,iv:iVhIb/kpa+eo0qOBqCtyli/lEnzPrXTN+rMHyhyCq2Q=,tag:zWX99z4jIc/eNCSCykGSwQ==,type:str]
 sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwOFVLYi9VYlRQdWwzK0g3\nNmRwemxjeU5zYWhPRzIxbzJWc0E2cXNYN2xRClVYUlkxWnIreUhtZ044dkhBeFBk\ncW9YZi9RN1E0Qjg4cVp4T2ZPaGR4WEUKLS0tIGFxQTV2blhzSmUvT2ljeFN2blVj\na3l1NGc5dEtZYkNJZG1GYlo2YXM3OVEKd1UVotkdkOqaA23UeEJckItDV7HU72uO\nH7Nza9clHgQwXqwYFFxBBAPRiVo/aX+J+jXELNp2fA/Wt/66SjrBig==\n-----END AGE ENCRYPTED FILE-----\n
 sops_age__list_0__map_recipient=age1vukt8py0s2mm47fx6qh6vykckfkdqslf9w68ekytfzvc8xp7e3rqn6p64n
-sops_lastmodified=2026-03-29T17:47:20Z
-sops_mac=ENC[AES256_GCM,data:GywX/4bsPvDrlOXAyv18xPEA/eEuF7rrk5UUphP3+7M2SWwNEMQSQhz0YsuIxlctdVVyMmCnK9SwJtru03FHUE0X3GY48IRvQcYwz9xSXkws/znKn90KkFUpLouIEfchNHL0xxY9xQSgGVZT1/IDKQMEc1+4hyEQqBrxgy8nDUk=,iv:IBEAYarsCAs10o8bipLvpeI9knHknytkd+5Kj24uFTY=,tag:Y2RAhx/ceGPnxKNsWjRDDw==,type:str]
+sops_lastmodified=2026-04-24T17:04:54Z
+sops_mac=ENC[AES256_GCM,data:zlgyPMPgTw8w6kWjWhbsSy9BmhydUDz9PPUO/X1hx/ABczG5gwcoMQ14Dv6z/FT+uFlNxauWlRkOIaOujlaDLd/zqOAodhG4gqELPsq0swVFxVZbdvY2R8EL4eOYPq7vKBUFrXm2M8NS9Su43tXkZcbm3g7bRNTz4Zm0NQjobCw=,iv:wSY8/Advd7gJ8QnXsyk1YWCjAtRQ7m1BRxGe0LDtAmM=,tag:yfFD0HO9iRU10TubqRFoVg==,type:str]
 sops_unencrypted_suffix=_unencrypted
-sops_version=3.9.4
+sops_version=3.12.2