f790da2ed3
Hardening from two incidents on 2026-06-06/07: Schema drift (morning): drizzle-kit push exits 0 even when it aborts on an interactive prompt it can't render in CI, so cd-staging's set -euo pipefail never fired and a month of staging schema drift accumulated silently until new code hit missing columns. All three drizzle push call sites now tee output and fail the deploy on any 'Error:' line. Production outage (overnight, ~9h): cd-infra and cd-apps had no failure handling at all. A network-option change stopped postgres for a network recreation that then deadlocked on containers from other compose projects holding trails-shared; the script carried on, the stack stayed down. Both scripts now run set -euo pipefail and gate on container health at the end (postgres+journal for cd-infra, journal+planner for cd-apps) so a deploy that leaves the stack down is a red X, not a shrug. docs/deployment.md gains the cross-project manual procedure for network-changing deploys — the CD workflows only manage their own compose project and cannot apply those safely. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
193 lines
8.4 KiB
YAML
193 lines
8.4 KiB
YAML
name: CD Apps
|
|
|
|
on:
|
|
push:
|
|
branches: [main]
|
|
paths:
|
|
- "apps/**"
|
|
- "packages/**"
|
|
- "pnpm-lock.yaml"
|
|
workflow_dispatch: {}
|
|
|
|
concurrency:
|
|
group: deploy-apps
|
|
cancel-in-progress: true
|
|
|
|
# Public Sentry DSNs for the trails.cool flagship instance. Public by
|
|
# design — Sentry DSNs are transmitted unencrypted from the client JS
|
|
# bundle, embedding them in this workflow is no worse than embedding
|
|
# them in the runtime env. Self-hosted forks should either replace
|
|
# these with their own DSNs or remove the lines to ship without Sentry.
|
|
env:
|
|
SENTRY_DSN_JOURNAL: "https://a32ffcc575d34be072e91b20f247eeee@o4509530546634752.ingest.de.sentry.io/4509530555547728"
|
|
SENTRY_DSN_PLANNER: "https://5215134cd78d5e6c199e29300b8425af@o4509530546634752.ingest.de.sentry.io/4511102546608208"
|
|
|
|
jobs:
|
|
build-images:
|
|
name: Build & Push Docker Images
|
|
runs-on: ubuntu-latest
|
|
environment: production
|
|
permissions:
|
|
contents: read
|
|
packages: write
|
|
strategy:
|
|
matrix:
|
|
app: [journal, planner]
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
|
|
- uses: docker/login-action@v4
|
|
with:
|
|
registry: ghcr.io
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Decrypt Sentry auth token
|
|
run: |
|
|
curl -sLO https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64
|
|
chmod +x sops-v3.9.4.linux.amd64
|
|
SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.app.env > /tmp/secrets.env
|
|
grep SENTRY_AUTH_TOKEN /tmp/secrets.env | cut -d= -f2- | tr -d '\n' > /tmp/sentry_token
|
|
|
|
- uses: docker/build-push-action@v7
|
|
with:
|
|
context: .
|
|
file: apps/${{ matrix.app }}/Dockerfile
|
|
push: true
|
|
tags: |
|
|
ghcr.io/trails-cool/${{ matrix.app }}:latest
|
|
ghcr.io/trails-cool/${{ matrix.app }}:${{ github.sha }}
|
|
# VITE_SENTRY_DSN bakes the client-side DSN into the journal's
|
|
# built bundle. Only journal has a client Sentry init; planner
|
|
# ignores the build-arg if present.
|
|
build-args: |
|
|
SENTRY_RELEASE=${{ github.sha }}
|
|
VITE_SENTRY_DSN=${{ matrix.app == 'journal' && env.SENTRY_DSN_JOURNAL || '' }}
|
|
secrets: |
|
|
SENTRY_AUTH_TOKEN=/tmp/sentry_token
|
|
|
|
deploy:
|
|
name: Deploy Apps
|
|
needs: [build-images]
|
|
runs-on: ubuntu-latest
|
|
environment: production
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
|
|
- name: Decrypt secrets
|
|
run: |
|
|
curl -sLO https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64
|
|
chmod +x sops-v3.9.4.linux.amd64
|
|
SOPS_AGE_KEY="${{ secrets.AGE_SECRET_KEY }}" ./sops-v3.9.4.linux.amd64 -d infrastructure/secrets.app.env > infrastructure/app.env
|
|
echo "SENTRY_RELEASE=${{ github.sha }}" >> infrastructure/app.env
|
|
echo "DOMAIN=trails.cool" >> infrastructure/app.env
|
|
# Flagship marker — see cd-infra.yml for what this gates.
|
|
echo "IS_FLAGSHIP=true" >> infrastructure/app.env
|
|
# Sentry DSNs (public — see workflow top-level env for context).
|
|
echo "SENTRY_DSN_JOURNAL=$SENTRY_DSN_JOURNAL" >> infrastructure/app.env
|
|
echo "SENTRY_DSN_PLANNER=$SENTRY_DSN_PLANNER" >> infrastructure/app.env
|
|
|
|
- name: Copy files to server
|
|
uses: appleboy/scp-action@v1
|
|
with:
|
|
host: ${{ secrets.DEPLOY_HOST }}
|
|
username: root
|
|
key: ${{ secrets.DEPLOY_SSH_KEY }}
|
|
source: "infrastructure/docker-compose.yml,infrastructure/Caddyfile"
|
|
target: /opt/trails-cool
|
|
strip_components: 1
|
|
|
|
- name: Copy secrets to server
|
|
uses: appleboy/scp-action@v1
|
|
with:
|
|
host: ${{ secrets.DEPLOY_HOST }}
|
|
username: root
|
|
key: ${{ secrets.DEPLOY_SSH_KEY }}
|
|
source: "infrastructure/app.env"
|
|
target: /opt/trails-cool
|
|
strip_components: 1
|
|
|
|
- name: Deploy via SSH
|
|
uses: appleboy/ssh-action@v1
|
|
with:
|
|
host: ${{ secrets.DEPLOY_HOST }}
|
|
username: root
|
|
key: ${{ secrets.DEPLOY_SSH_KEY }}
|
|
script: |
|
|
# Abort the deploy on the first failure. Without this, a failed
|
|
# schema push deploys new code against an old schema (the
|
|
# 2026-06-06 schema-drift incident).
|
|
set -euo pipefail
|
|
cd /opt/trails-cool
|
|
|
|
# Login to ghcr.io
|
|
GHCR_TOKEN=$(grep DEPLOY_GHCR_TOKEN app.env | cut -d= -f2-)
|
|
echo "$GHCR_TOKEN" | docker login ghcr.io -u stigi --password-stdin
|
|
|
|
# Pull and deploy app containers
|
|
docker compose --env-file app.env pull journal planner
|
|
# Hand-written data migrations (idempotent) run BEFORE drizzle-kit
|
|
# push so unique-key reshapes can collapse duplicate rows first.
|
|
docker compose --env-file app.env run --rm journal node --experimental-strip-types /app/packages/db/src/migrate-data.ts
|
|
# drizzle-kit exits 0 even when it aborts on an interactive
|
|
# prompt it can't show (no TTY in CI) — that exact lie hid a
|
|
# month of staging schema drift. Treat any Error in its output
|
|
# as a failed deploy.
|
|
docker compose --env-file app.env run --rm journal npx drizzle-kit push --config /app/packages/db/drizzle.config.ts --force 2>&1 | tee /tmp/drizzle-push.log
|
|
if grep -q "Error:" /tmp/drizzle-push.log; then
|
|
echo "drizzle-kit push reported an error — failing the deploy"
|
|
exit 1
|
|
fi
|
|
# --remove-orphans cleans up containers whose service was deleted
|
|
# from the compose file, matching cd-infra's behaviour.
|
|
docker compose --env-file app.env up -d --remove-orphans journal planner
|
|
|
|
# Gate on container health: a deploy that leaves journal or
|
|
# planner unhealthy must fail loudly, not report green.
|
|
for svc in journal planner; do
|
|
for i in $(seq 1 24); do
|
|
status=$(docker inspect -f '{{.State.Health.Status}}' "trails-cool-$svc-1" 2>/dev/null || echo missing)
|
|
[ "$status" = "healthy" ] && break
|
|
sleep 5
|
|
done
|
|
if [ "$status" != "healthy" ]; then
|
|
echo "$svc did not become healthy (last status: $status)"
|
|
docker compose --env-file app.env logs "$svc" --tail 50 || true
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
# Reload Caddy with the Caddyfile we just scp'd. cd-apps
|
|
# ships infrastructure/Caddyfile alongside docker-compose.yml
|
|
# (see scp step above), but containers don't auto-pick-up
|
|
# config changes. cd-infra reloads Caddy as part of its
|
|
# deploy; cd-apps did NOT, which meant any Caddyfile change
|
|
# touching only `apps/`/`packages/` paths sat on disk
|
|
# unapplied until the next cd-infra run. The reload is
|
|
# idempotent (Caddy validates first, swaps live, no
|
|
# downtime) so doing it on every cd-apps deploy is safe
|
|
# even when Caddyfile is unchanged. `|| true` keeps the
|
|
# deploy from failing if Caddy itself is unhealthy.
|
|
docker compose exec -T caddy caddy reload --config /etc/caddy/Caddyfile || true
|
|
|
|
# Clean up
|
|
docker image prune -af
|
|
docker compose ps
|
|
|
|
# Annotate deploy in Grafana. GRAFANA_SERVICE_TOKEN lives
|
|
# in secrets.infra.env (decrypted by cd-infra.yml into the
|
|
# merged /opt/trails-cool/.env on the server). cd-apps's
|
|
# own app.env intentionally does NOT carry it — apps don't
|
|
# need it at runtime. So we read from the merged `.env`
|
|
# that cd-infra populated. If cd-infra has never run on
|
|
# this host, .env may not exist; the `2>/dev/null` and
|
|
# the `if -n` guard make the annotation a silent no-op
|
|
# rather than a deploy failure in that case.
|
|
GRAFANA_TOKEN=$(grep GRAFANA_SERVICE_TOKEN .env 2>/dev/null | cut -d= -f2-)
|
|
if [ -n "$GRAFANA_TOKEN" ]; then
|
|
docker compose exec -T grafana curl -sf -X POST \
|
|
-H "Authorization: Bearer $GRAFANA_TOKEN" \
|
|
-H "Content-Type: application/json" \
|
|
-d '{"text":"Deploy ${{ github.sha }}","tags":["deploy","apps"]}' \
|
|
http://localhost:3000/api/annotations || true
|
|
fi
|