cd-infra: fix Prometheus 3.10 SIGHUP startup kill + add readiness gate

- Capture container ID before/after 'docker compose up -d' and only
  send SIGHUP when the same container persisted (config-only change).
  A recreated container already loaded the fresh config; sending HUP
  immediately after startup kills Prometheus 3.10 with exit code 2.
- Add Prometheus /-/ready gate alongside the postgres/journal health
  check to fail the deploy if monitoring stack never comes up.
- Observed 2026-06-09: infra deploy left trails-cool-prometheus-1
  Exited (2) for ~2h, causing a Grafana alert storm.

.github/workflows/cd-infra.yml

@@ -84,6 +84,12 @@ jobs:
               docker compose exec -T postgres psql -U trails -d trails -c "ALTER ROLE grafana_reader PASSWORD '$GRAFANA_DB_PW'" 2>/dev/null || true
             fi
 
+            # Capture whether prometheus was recreated. A fresh container already
+            # reads the new config on startup; sending it SIGHUP immediately can
+            # kill Prometheus 3.10 during early boot (exit 2 observed on
+            # 2026-06-09).
+            PROMETHEUS_BEFORE_ID=$(docker inspect -f '{{.Id}}' trails-cool-prometheus-1 2>/dev/null || true)
+
             # Full restart: gh workflow run cd-infra.yml -f restart_all=true
             if [ "${{ github.event.inputs.restart_all }}" = "true" ]; then
               docker compose --env-file .env up -d --remove-orphans
@@ -95,24 +101,31 @@ jobs:
               docker compose --env-file .env up -d --remove-orphans postgres prometheus loki promtail grafana postgres-exporter node-exporter cadvisor
             fi
 
+            PROMETHEUS_AFTER_ID=$(docker inspect -f '{{.Id}}' trails-cool-prometheus-1 2>/dev/null || true)
+
             # Apply config-only changes that `up -d` skips — it recreates a
             # container only when its compose *definition* changes, not when a
             # mounted config file's content changes. The configs are mounted as
             # directories (not single files), so a reload/restart re-reads the
             # freshly scp'd file; a single-file mount would have pinned the old
-            # inode. Prometheus hot-reloads on SIGHUP (zero downtime); Loki and
+            # inode. Prometheus hot-reloads on SIGHUP (zero downtime) only when
-            # Promtail reload their main config only on restart; Caddy reloads
+            # the same container stayed up; a recreated container already loaded
-            # gracefully (validates, swaps live, no downtime).
+            # the new config on startup. Loki and Promtail reload their main
-            docker compose --env-file .env kill -s SIGHUP prometheus
+            # config only on restart; Caddy reloads gracefully (validates, swaps
+            # live, no downtime).
+            if [ -n "$PROMETHEUS_BEFORE_ID" ] && [ "$PROMETHEUS_BEFORE_ID" = "$PROMETHEUS_AFTER_ID" ]; then
+              docker compose --env-file .env kill -s SIGHUP prometheus
+            fi
             docker compose --env-file .env restart loki promtail
             docker compose exec caddy caddy reload --config /etc/caddy/Caddyfile
 
             docker compose ps
 
-            # Gate on the stack actually being up: postgres healthy and —
+            # Gate on the stack actually being up: postgres healthy, journal
-            # since an infra restart bounces the app containers' database —
+            # back to healthy after the DB bounce, and Prometheus answering
-            # journal back to healthy too. A deploy that leaves either down
+            # /-/ready. A deploy that leaves any of them down must fail loudly
-            # must fail loudly (see the 2026-06-06/07 outage).
+            # (see the 2026-06-06/07 outage, plus the 2026-06-09 Prometheus
+            # startup/HUP regression).
             for ctr in trails-cool-postgres-1 trails-cool-journal-1; do
               for i in $(seq 1 36); do
                 status=$(docker inspect -f '{{.State.Health.Status}}' "$ctr" 2>/dev/null || echo missing)
@@ -125,6 +138,23 @@ jobs:
               fi
             done
 
+            PROMETHEUS_READY=
+            for i in $(seq 1 36); do
+              prom_status=$(docker inspect -f '{{.State.Status}}' trails-cool-prometheus-1 2>/dev/null || echo missing)
+              if [ "$prom_status" = "running" ]; then
+                prom_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' trails-cool-prometheus-1)
+                if curl -sf "http://$prom_ip:9090/-/ready" >/dev/null; then
+                  PROMETHEUS_READY=1
+                  break
+                fi
+              fi
+              sleep 5
+            done
+            if [ -z "$PROMETHEUS_READY" ]; then
+              echo "trails-cool-prometheus-1 did not become ready (last status: $prom_status)"
+              exit 1
+            fi
+
             # Annotate deploy in Grafana
             GRAFANA_TOKEN=$(grep GRAFANA_SERVICE_TOKEN .env | cut -d= -f2-)
             if [ -n "$GRAFANA_TOKEN" ]; then