diff --git a/.github/workflows/cd-infra.yml b/.github/workflows/cd-infra.yml index bdc3b31..333251e 100644 --- a/.github/workflows/cd-infra.yml +++ b/.github/workflows/cd-infra.yml @@ -84,6 +84,12 @@ jobs: docker compose exec -T postgres psql -U trails -d trails -c "ALTER ROLE grafana_reader PASSWORD '$GRAFANA_DB_PW'" 2>/dev/null || true fi + # Capture whether prometheus was recreated. A fresh container already + # reads the new config on startup; sending it SIGHUP immediately can + # kill Prometheus 3.10 during early boot (exit 2 observed on + # 2026-06-09). + PROMETHEUS_BEFORE_ID=$(docker inspect -f '{{.Id}}' trails-cool-prometheus-1 2>/dev/null || true) + # Full restart: gh workflow run cd-infra.yml -f restart_all=true if [ "${{ github.event.inputs.restart_all }}" = "true" ]; then docker compose --env-file .env up -d --remove-orphans @@ -95,24 +101,31 @@ jobs: docker compose --env-file .env up -d --remove-orphans postgres prometheus loki promtail grafana postgres-exporter node-exporter cadvisor fi + PROMETHEUS_AFTER_ID=$(docker inspect -f '{{.Id}}' trails-cool-prometheus-1 2>/dev/null || true) + # Apply config-only changes that `up -d` skips — it recreates a # container only when its compose *definition* changes, not when a # mounted config file's content changes. The configs are mounted as # directories (not single files), so a reload/restart re-reads the # freshly scp'd file; a single-file mount would have pinned the old - # inode. Prometheus hot-reloads on SIGHUP (zero downtime); Loki and - # Promtail reload their main config only on restart; Caddy reloads - # gracefully (validates, swaps live, no downtime). - docker compose --env-file .env kill -s SIGHUP prometheus + # inode. Prometheus hot-reloads on SIGHUP (zero downtime) only when + # the same container stayed up; a recreated container already loaded + # the new config on startup. Loki and Promtail reload their main + # config only on restart; Caddy reloads gracefully (validates, swaps + # live, no downtime). + if [ -n "$PROMETHEUS_BEFORE_ID" ] && [ "$PROMETHEUS_BEFORE_ID" = "$PROMETHEUS_AFTER_ID" ]; then + docker compose --env-file .env kill -s SIGHUP prometheus + fi docker compose --env-file .env restart loki promtail docker compose exec caddy caddy reload --config /etc/caddy/Caddyfile docker compose ps - # Gate on the stack actually being up: postgres healthy and — - # since an infra restart bounces the app containers' database — - # journal back to healthy too. A deploy that leaves either down - # must fail loudly (see the 2026-06-06/07 outage). + # Gate on the stack actually being up: postgres healthy, journal + # back to healthy after the DB bounce, and Prometheus answering + # /-/ready. A deploy that leaves any of them down must fail loudly + # (see the 2026-06-06/07 outage, plus the 2026-06-09 Prometheus + # startup/HUP regression). for ctr in trails-cool-postgres-1 trails-cool-journal-1; do for i in $(seq 1 36); do status=$(docker inspect -f '{{.State.Health.Status}}' "$ctr" 2>/dev/null || echo missing) @@ -125,6 +138,23 @@ jobs: fi done + PROMETHEUS_READY= + for i in $(seq 1 36); do + prom_status=$(docker inspect -f '{{.State.Status}}' trails-cool-prometheus-1 2>/dev/null || echo missing) + if [ "$prom_status" = "running" ]; then + prom_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' trails-cool-prometheus-1) + if curl -sf "http://$prom_ip:9090/-/ready" >/dev/null; then + PROMETHEUS_READY=1 + break + fi + fi + sleep 5 + done + if [ -z "$PROMETHEUS_READY" ]; then + echo "trails-cool-prometheus-1 did not become ready (last status: $prom_status)" + exit 1 + fi + # Annotate deploy in Grafana GRAFANA_TOKEN=$(grep GRAFANA_SERVICE_TOKEN .env | cut -d= -f2-) if [ -n "$GRAFANA_TOKEN" ]; then