trails-cool/trails
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #503 from trails-cool/fix/prometheus-hup-race

Browse source 
cd-infra: fix Prometheus 3.10 SIGHUP startup kill + add readiness gate
This commit is contained in:
Ullrich Schäfer 2026-06-09 16:03:37 +02:00 • committed by GitHub
commit 06c385f6ac
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 38 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

46
.github/workflows/cd-infra.yml vendored
View file

@ -84,6 +84,12 @@ jobs:
docker compose exec -T postgres psql -U trails -d trails -c "ALTER ROLE grafana_reader PASSWORD '$GRAFANA_DB_PW'" 2>/dev/null || true
fi
# Capture whether prometheus was recreated. A fresh container already
# reads the new config on startup; sending it SIGHUP immediately can
# kill Prometheus 3.10 during early boot (exit 2 observed on
# 2026-06-09).
PROMETHEUS_BEFORE_ID=$(docker inspect -f '{{.Id}}' trails-cool-prometheus-1 2>/dev/null || true)
# Full restart: gh workflow run cd-infra.yml -f restart_all=true
if [ "${{ github.event.inputs.restart_all }}" = "true" ]; then
docker compose --env-file .env up -d --remove-orphans
@ -95,24 +101,31 @@ jobs:
docker compose --env-file .env up -d --remove-orphans postgres prometheus loki promtail grafana postgres-exporter node-exporter cadvisor
fi
PROMETHEUS_AFTER_ID=$(docker inspect -f '{{.Id}}' trails-cool-prometheus-1 2>/dev/null || true)
# Apply config-only changes that `up -d` skips — it recreates a
# container only when its compose *definition* changes, not when a
# mounted config file's content changes. The configs are mounted as
# directories (not single files), so a reload/restart re-reads the
# freshly scp'd file; a single-file mount would have pinned the old
# inode. Prometheus hot-reloads on SIGHUP (zero downtime); Loki and
# Promtail reload their main config only on restart; Caddy reloads
# gracefully (validates, swaps live, no downtime).
docker compose --env-file .env kill -s SIGHUP prometheus
# inode. Prometheus hot-reloads on SIGHUP (zero downtime) only when
# the same container stayed up; a recreated container already loaded
# the new config on startup. Loki and Promtail reload their main
# config only on restart; Caddy reloads gracefully (validates, swaps
# live, no downtime).
if [ -n "$PROMETHEUS_BEFORE_ID" ] && [ "$PROMETHEUS_BEFORE_ID" = "$PROMETHEUS_AFTER_ID" ]; then
docker compose --env-file .env kill -s SIGHUP prometheus
fi
docker compose --env-file .env restart loki promtail
docker compose exec caddy caddy reload --config /etc/caddy/Caddyfile
docker compose ps
# Gate on the stack actually being up: postgres healthy and —
# since an infra restart bounces the app containers' database —
# journal back to healthy too. A deploy that leaves either down
# must fail loudly (see the 2026-06-06/07 outage).
# Gate on the stack actually being up: postgres healthy, journal
# back to healthy after the DB bounce, and Prometheus answering
# /-/ready. A deploy that leaves any of them down must fail loudly
# (see the 2026-06-06/07 outage, plus the 2026-06-09 Prometheus
# startup/HUP regression).
for ctr in trails-cool-postgres-1 trails-cool-journal-1; do
for i in $(seq 1 36); do
status=$(docker inspect -f '{{.State.Health.Status}}' "$ctr" 2>/dev/null || echo missing)
@ -125,6 +138,23 @@ jobs:
fi
done
PROMETHEUS_READY=
for i in $(seq 1 36); do
prom_status=$(docker inspect -f '{{.State.Status}}' trails-cool-prometheus-1 2>/dev/null || echo missing)
if [ "$prom_status" = "running" ]; then
prom_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' trails-cool-prometheus-1)
if curl -sf "http://$prom_ip:9090/-/ready" >/dev/null; then
PROMETHEUS_READY=1
break
fi
fi
sleep 5
done
if [ -z "$PROMETHEUS_READY" ]; then
echo "trails-cool-prometheus-1 did not become ready (last status: $prom_status)"
exit 1
fi
# Annotate deploy in Grafana
GRAFANA_TOKEN=$(grep GRAFANA_SERVICE_TOKEN .env | cut -d= -f2-)
if [ -n "$GRAFANA_TOKEN" ]; then