From f790da2ed35c94c450fa130624486a519ccb9173 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ullrich=20Sch=C3=A4fer?= Date: Sun, 7 Jun 2026 08:15:17 +0200 Subject: [PATCH] =?UTF-8?q?ci(deploy):=20fail=20loudly=20=E2=80=94=20set?= =?UTF-8?q?=20-e,=20drizzle=20output=20guards,=20health=20gates?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hardening from two incidents on 2026-06-06/07: Schema drift (morning): drizzle-kit push exits 0 even when it aborts on an interactive prompt it can't render in CI, so cd-staging's set -euo pipefail never fired and a month of staging schema drift accumulated silently until new code hit missing columns. All three drizzle push call sites now tee output and fail the deploy on any 'Error:' line. Production outage (overnight, ~9h): cd-infra and cd-apps had no failure handling at all. A network-option change stopped postgres for a network recreation that then deadlocked on containers from other compose projects holding trails-shared; the script carried on, the stack stayed down. Both scripts now run set -euo pipefail and gate on container health at the end (postgres+journal for cd-infra, journal+planner for cd-apps) so a deploy that leaves the stack down is a red X, not a shrug. docs/deployment.md gains the cross-project manual procedure for network-changing deploys — the CD workflows only manage their own compose project and cannot apply those safely. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/cd-apps.yml | 29 +++++++++++++++++++++++++- .github/workflows/cd-infra.yml | 21 +++++++++++++++++++ .github/workflows/cd-staging.yml | 17 ++++++++++++++-- docs/deployment.md | 35 ++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cd-apps.yml b/.github/workflows/cd-apps.yml index 36c18e4..c5cc328 100644 --- a/.github/workflows/cd-apps.yml +++ b/.github/workflows/cd-apps.yml @@ -114,6 +114,10 @@ jobs: username: root key: ${{ secrets.DEPLOY_SSH_KEY }} script: | + # Abort the deploy on the first failure. Without this, a failed + # schema push deploys new code against an old schema (the + # 2026-06-06 schema-drift incident). + set -euo pipefail cd /opt/trails-cool # Login to ghcr.io @@ -125,11 +129,34 @@ jobs: # Hand-written data migrations (idempotent) run BEFORE drizzle-kit # push so unique-key reshapes can collapse duplicate rows first. docker compose --env-file app.env run --rm journal node --experimental-strip-types /app/packages/db/src/migrate-data.ts - docker compose --env-file app.env run --rm journal npx drizzle-kit push --config /app/packages/db/drizzle.config.ts --force + # drizzle-kit exits 0 even when it aborts on an interactive + # prompt it can't show (no TTY in CI) — that exact lie hid a + # month of staging schema drift. Treat any Error in its output + # as a failed deploy. + docker compose --env-file app.env run --rm journal npx drizzle-kit push --config /app/packages/db/drizzle.config.ts --force 2>&1 | tee /tmp/drizzle-push.log + if grep -q "Error:" /tmp/drizzle-push.log; then + echo "drizzle-kit push reported an error — failing the deploy" + exit 1 + fi # --remove-orphans cleans up containers whose service was deleted # from the compose file, matching cd-infra's behaviour. docker compose --env-file app.env up -d --remove-orphans journal planner + # Gate on container health: a deploy that leaves journal or + # planner unhealthy must fail loudly, not report green. + for svc in journal planner; do + for i in $(seq 1 24); do + status=$(docker inspect -f '{{.State.Health.Status}}' "trails-cool-$svc-1" 2>/dev/null || echo missing) + [ "$status" = "healthy" ] && break + sleep 5 + done + if [ "$status" != "healthy" ]; then + echo "$svc did not become healthy (last status: $status)" + docker compose --env-file app.env logs "$svc" --tail 50 || true + exit 1 + fi + done + # Reload Caddy with the Caddyfile we just scp'd. cd-apps # ships infrastructure/Caddyfile alongside docker-compose.yml # (see scp step above), but containers don't auto-pick-up diff --git a/.github/workflows/cd-infra.yml b/.github/workflows/cd-infra.yml index d317ae1..89c11bf 100644 --- a/.github/workflows/cd-infra.yml +++ b/.github/workflows/cd-infra.yml @@ -64,6 +64,11 @@ jobs: username: root key: ${{ secrets.DEPLOY_SSH_KEY }} script: | + # Abort on first failure. The 2026-06-06/07 outage: a network + # recreation stopped postgres, a later step failed, and the + # deploy left production down for ~9h while the job's partial + # progress looked plausible. Fail fast, verify health at the end. + set -euo pipefail cd /opt/trails-cool # .env was placed by the SCP step (decrypted app + infra secrets) @@ -93,6 +98,22 @@ jobs: docker compose ps + # Gate on the stack actually being up: postgres healthy and — + # since an infra restart bounces the app containers' database — + # journal back to healthy too. A deploy that leaves either down + # must fail loudly (see the 2026-06-06/07 outage). + for ctr in trails-cool-postgres-1 trails-cool-journal-1; do + for i in $(seq 1 36); do + status=$(docker inspect -f '{{.State.Health.Status}}' "$ctr" 2>/dev/null || echo missing) + [ "$status" = "healthy" ] && break + sleep 5 + done + if [ "$status" != "healthy" ]; then + echo "$ctr did not become healthy (last status: $status)" + exit 1 + fi + done + # Annotate deploy in Grafana GRAFANA_TOKEN=$(grep GRAFANA_SERVICE_TOKEN .env | cut -d= -f2-) if [ -n "$GRAFANA_TOKEN" ]; then diff --git a/.github/workflows/cd-staging.yml b/.github/workflows/cd-staging.yml index 4ed9aba..7b87971 100644 --- a/.github/workflows/cd-staging.yml +++ b/.github/workflows/cd-staging.yml @@ -181,7 +181,14 @@ jobs: # Pull and deploy staging containers (journal + planner via "persistent" profile) docker compose -f docker-compose.staging.yml -p trails-staging --env-file staging.env --profile persistent pull - docker compose -f docker-compose.staging.yml -p trails-staging --env-file staging.env --profile persistent run --rm journal npx drizzle-kit push --config /app/packages/db/drizzle.config.ts --force + # drizzle-kit exits 0 even when it aborts on an interactive + # prompt (no TTY in CI) — set -e alone can't catch it. That lie + # hid a month of staging schema drift (2026-06-06 incident). + docker compose -f docker-compose.staging.yml -p trails-staging --env-file staging.env --profile persistent run --rm journal npx drizzle-kit push --config /app/packages/db/drizzle.config.ts --force 2>&1 | tee /tmp/drizzle-push.log + if grep -q "Error:" /tmp/drizzle-push.log; then + echo "drizzle-kit push reported an error — failing the deploy" + exit 1 + fi docker compose -f docker-compose.staging.yml -p trails-staging --env-file staging.env --profile persistent up -d --remove-orphans # Reload Caddy so new staging routes (or Caddyfile changes shipped @@ -337,7 +344,13 @@ jobs: # Pull, migrate, deploy (journal-only — no --profile means planner skipped) docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file "$ENV_FILE" pull journal - docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file "$ENV_FILE" run --rm journal npx drizzle-kit push --config /app/packages/db/drizzle.config.ts --force + # Same drizzle-kit exit-code-0-on-error guard as the persistent + # staging deploy above. + docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file "$ENV_FILE" run --rm journal npx drizzle-kit push --config /app/packages/db/drizzle.config.ts --force 2>&1 | tee /tmp/drizzle-push.log + if grep -q "Error:" /tmp/drizzle-push.log; then + echo "drizzle-kit push reported an error — failing the preview deploy" + exit 1 + fi docker compose -f docker-compose.staging.yml -p "$PROJECT" --env-file "$ENV_FILE" up -d --remove-orphans journal # Reload Caddy to pick up the per-PR snippet (writes/replaces it from the SCP step) diff --git a/docs/deployment.md b/docs/deployment.md index d48ec22..f4998e8 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -67,6 +67,41 @@ gh workflow run cd-infra.yml -f restart_all=true Restarts every flagship service. Does NOT touch the BRouter host. +## Network-changing deploys (flagship) + +Changing options on an existing Docker network (`enable_ipv6`, subnets, +drivers) requires Docker to **recreate** the network — and a network can +only be recreated when *no* container from *any* compose project is +attached. The flagship has three+ projects sharing `trails-shared` +(production, persistent staging, every PR preview), and the CD workflows +only manage their own project, so a network-option change shipped through +`cd-infra` alone WILL deadlock mid-deploy and can leave production down +(this is exactly the 2026-06-06/07 outage: postgres stopped for the +recreation, the deploy failed on the held network, nothing restarted it +for ~9 hours). + +The manual procedure, in order, on the flagship: + +```bash +cd /opt/trails-cool +# 1. Free trails-shared: down every preview + persistent staging +docker compose ls --filter name=trails-pr- --format json # enumerate previews +docker compose -f docker-compose.staging.yml -p trails-pr- --env-file staging-pr-.env down +docker compose -f docker-compose.staging.yml -p trails-staging --env-file staging.env --profile persistent down +# 2. Recreate networks via the production project +docker compose --env-file app.env down +docker compose --env-file app.env up -d +# 3. Bring staging + previews back +docker compose -f docker-compose.staging.yml -p trails-staging --env-file staging.env --profile persistent up -d +docker compose -f docker-compose.staging.yml -p trails-pr- --env-file staging-pr-.env up -d +# 4. Verify +docker network inspect trails-cool_default trails-shared --format '{{.Name}} ipv6={{.EnableIPv6}}' +curl -sf https://trails.cool/api/health && curl -sf https://staging.trails.cool/api/health +``` + +Plan it as a short maintenance window (~2–3 min downtime); don't ship +network-option changes expecting the workflows to apply them. + ## cd-brouter manual trigger ```bash